From e8b92f7a40b9b64dc892c3958df229dba7b985b3 Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Thu, 18 Feb 2021 00:58:25 -0500 Subject: [PATCH 1/8] backup for hic --- CommandLines.cpp | 1 + CommandLines.h | 1 + Overlaps.cpp | 92 ++++++++++++++++++- hic.cpp | 231 ++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 309 insertions(+), 16 deletions(-) diff --git a/CommandLines.cpp b/CommandLines.cpp index deddc85..4685b0d 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -163,6 +163,7 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->b_low_cov = 0; asm_opt->b_high_cov = -1; asm_opt->m_rate = 0.75; + asm_opt->hap_occ = 2; } void destory_enzyme(enzyme* f) diff --git a/CommandLines.h b/CommandLines.h index ea9ec78..309fd8d 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -90,6 +90,7 @@ typedef struct { long long num_recorrected_bases; long long mem_buf; long long coverage; + int hap_occ; } hifiasm_opt_t; diff --git a/Overlaps.cpp b/Overlaps.cpp index 6c24438..3e320d8 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -11825,7 +11825,7 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov hc_links link; - ///if(load_hc_links(&link, output_file_name) == 0) + if(load_hc_links(&link, output_file_name) == 0) { init_hc_links(&link, ug->g->n_seq, R_INF.total_reads); asg_t *copy_sg = copy_read_graph(sg); @@ -11841,7 +11841,7 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp, asm_opt.hic_inconsist_rate, NULL, NULL, &link); - ///write_hc_links(&link, output_file_name); + write_hc_links(&link, output_file_name); } @@ -13779,9 +13779,11 @@ uint8_t is_final_check, float double_check_rate, uint8_t flag, float drop_rate) if(R_INF.trio_flag[rId] == flag) flag_occ++; if(R_INF.trio_flag[rId] != flag) non_flag_occ++; } + if(is_final_check == 0 && hap_label_occ == u->n) continue; ///if(is_double_check && non_flag_occ < u->n*DOUBLE_CHECK_THRES) continue; ///if(is_double_check && non_flag_occ < u->n*double_check_rate) continue; + if(is_final_check) { if(non_flag_occ < u->n*double_check_rate) continue; @@ -13813,10 +13815,12 @@ uint8_t is_final_check, float double_check_rate, uint8_t flag, float drop_rate) if(non_flag_occ > ((non_flag_occ+flag_occ)*drop_rate)) { + if(if_primary_unitig(u, read_g, coverage_cut, sources, ruIndex, primary_flag)) { continue; } + if(u->m != 0) { u->circ = u->end = u->len = u->m = u->n = u->start = 0; @@ -14429,7 +14433,8 @@ kvec_asg_arc_t_warp* new_rtg_edges) renew_utg(ug, read_g, new_rtg_edges); } - + ///if(flag == MOTHER) print_untig_by_read(*ug, "m64043_200627_000137/124716590/ccs", 2789716, NULL, NULL, "beg"); + update_unitig_graph((*ug), read_g, coverage_cut, sources, reverse_sources, ruIndex, 1, FINAL_DOUBLE_CHECK_THRES, flag, drop_rate); @@ -24046,6 +24051,87 @@ void minor_transitive_reduction_r_g(asg_t *r_g, asg_arc_t* rbub_edges, uint32_t } } + + +int if_recoverable(asg_t *sg, ma_ug_t *ug, bubble_type* bub, uint32_t bid, kvec_t_u32_warp* stack, uint8_t* vis_flag) +{ + uint32_t beg_utg, sink_utg, *a = NULL, n, begRid, sinkRid, i; + get_bubbles(bub, bid, &beg_utg, &sink_utg, &a, &n, NULL); + if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) return 0; + if(beg_utg&1) + { + begRid = ug->u.a[beg_utg>>1].start^1; + } + else + { + begRid = ug->u.a[beg_utg>>1].end^1; + } + + if(sink_utg&1) + { + sinkRid = ug->u.a[sink_utg>>1].start; + } + else + { + sinkRid = ug->u.a[sink_utg>>1].end; + } + + asg_arc_t *acur = NULL; + uint32_t cur, ncur, v, n_vx = sg->n_seq<<1; + stack->a.n = 0; + memset(vis_flag, 0, n_vx); + + kv_push(uint32_t, stack->a, begRid); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + ncur = asg_arc_n(sg, cur); + acur = asg_arc_a(sg, cur); + vis_flag[cur] |= 1; + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]&1) return 0; + if(vis_flag[acur[i].v^1]&1) return 0; + if(acur[i].v == sinkRid) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + vis_flag[sinkRid] |= 1; + + + + begRid ^= 1; sinkRid ^= 1; v = begRid; begRid = sinkRid; sinkRid = v; + + stack->a.n = 0; + kv_push(uint32_t, stack->a, begRid); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + ncur = asg_arc_n(sg, cur); + acur = asg_arc_a(sg, cur); + vis_flag[cur] |= 2; + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]&2) return 0; + if(vis_flag[acur[i].v]&1) return 0; + if(vis_flag[acur[i].v^1]&2) return 0; + if(acur[i].v == sinkRid) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + vis_flag[sinkRid] |= 2; + + + + begRid ^= 1; sinkRid ^= 1; v = begRid; begRid = sinkRid; sinkRid = v; + + return 1; +} + void rescue_bubbles_by_contained_reads(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t beg_idx, uint32_t occ, bubble_type* bub) { diff --git a/hic.cpp b/hic.cpp index 4e6081d..e6e0621 100644 --- a/hic.cpp +++ b/hic.cpp @@ -135,7 +135,7 @@ typedef struct { uint64_t pos_mode; uint64_t rev_mode; uint64_t k; - uint64_t max_cnt; + uint64_t hap_cnt; uint64_t pre; @@ -271,7 +271,8 @@ void init_ha_ug_index_opt(ha_ug_index* idx, ma_ug_t *ug, int k, pldat_t* p) idx->pre = HIC_COUNTER_BITS; idx->tot = 1 << idx->pre; idx->tot_pos = 0; - idx->up_bound = 1; + ///idx->up_bound = 1; + idx->up_bound = asm_opt.hap_occ; CALLOC(idx->idx_buf, idx->tot); for (i = 0; i < idx->tot; i++) { @@ -962,7 +963,7 @@ inline void print_pos_list(const ha_ug_index* idx, s_hit *l, uint64_t occ, uint6 } } -void get_alignment(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, +void get_alignment_back(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) { uint64_t i, j, l = 0, skip, *pos_list = NULL, cnt, rev, self_p, ref_p, u_len, uID; @@ -990,7 +991,7 @@ const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) // } /*******************************for debug************************************/ cnt = get_hc_pt1_count((ha_ug_index*)idx, hash, &pos_list); - if(cnt > idx->max_cnt) continue; + if(cnt > idx->hap_cnt) continue; if(cnt != 1) continue; ///might be able to be disabled in future @@ -1162,6 +1163,210 @@ const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) // fprintf(stderr, "\n"); /*******************************for debug************************************/ } + + +void get_alignment(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, +const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) +{ + uint64_t i, j, l = 0, skip, *pos_list = NULL, cnt, rev, self_p, ref_p, u_len, uID; + uint64_t x[4], mask = (1ULL<a.n = 0; + for (i = l = 0, x[0] = x[1] = x[2] = x[3] = 0; i < len; ++i) { + int c = seq_nt4_table[(uint8_t)r[i]]; + ///c = 00, 01, 10, 11 + if (c < 4) { // not an "N" base + ///x[0] & x[1] are the forward k-mer + ///x[2] & x[3] are the reverse complementary k-mer + x[0] = (x[0] << 1 | (c&1)) & mask; + x[1] = (x[1] << 1 | (c>>1)) & mask; + x[2] = x[2] >> 1 | (uint64_t)(1 - (c&1)) << shift; + x[3] = x[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift; + if (++l >= k_mer) + { + hash = hc_hash_long(x, &skip, k_mer); + if(skip == (uint64_t)-1) continue; + /*******************************for debug************************************/ + // if(debug_hash_value(r, i, k_mer) != hash) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + cnt = get_hc_pt1_count((ha_ug_index*)idx, hash, &pos_list); + if(cnt > idx->hap_cnt) continue; + if(cnt != 1) continue; ///might be able to be disabled in future + + ///rev:uID:pos + for (j = 0; j < cnt; j++) + { + kv_pushp(s_hit, buf->a, &p); + rev = (pos_list[j]>>63) != skip; + self_p = i; + ref_p = pos_list[j] & idx->pos_mode; + uID = (pos_list[j] << 1) >> (64 - idx->uID_bits); + u_len = idx->ug->u.a[uID].len; + if(rev) ref_p = u_len - 1 - (ref_p + 1 - k_mer); + p->off_cnt = self_p | ((uint64_t)k_mer << 32); ///high bits should be the legnth + + p->ref = ref_p >= self_p? (ref_p-self_p) + : (self_p-ref_p) + ((uint64_t)1 << (idx->pos_bits - 1)); + p->ref = (rev << 63)|(pos_list[j] & idx->uID_mode)|(p->ref&idx->pos_mode); + + + /*******************************for debug************************************/ + // if(check_exact_match(r, i + 1 - k_mer, len, + // idx->ug->u.a[uID].s, ref_p + 1 - k_mer, u_len, k_mer, rev, 0) != k_mer + // || + // check_exact_match(r, i, len, + // idx->ug->u.a[uID].s, ref_p, u_len, k_mer, rev, 1) != k_mer) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + } + + if(cnt == 1) + { + ///uint64_t debug_right = 0, debug_left = 0, debug_len; + + j = check_exact_match(r, self_p + 1, len, idx->ug->u.a[uID].s, ref_p + 1, u_len, len, rev, 0); + + ///debug_right = j; + ///if(j == 0) continue; + if((j + 1) >= k_mer) + { + l = 0, x[0] = x[1] = x[2] = x[3] = 0; + i = i + j - (k_mer - 1); + } + else + { + ///l = i - (i + j - (k_mer - 1)); + l = k_mer - j -1; + } + buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32) + j; + + if(self_p >= k_mer && ref_p >= k_mer) + { + j = check_exact_match(r, self_p - k_mer, len, idx->ug->u.a[uID].s, + ref_p - k_mer, u_len, len, rev, 1); + buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32); + ///debug_left = j; + } + + + // debug_len = check_exact_match(r, self_p + debug_right, len, idx->ug->u.a[uID].s, + // ref_p + debug_right, u_len, len, rev, 1); + // if(debug_len!= (debug_left + debug_right + k_mer)) + // { + // fprintf(stderr, "debug_len: %lu, debug_left: %lu, debug_right: %lu\n", + // debug_len, debug_left, debug_right); + // } + } + + } + + } else l = 0, x[0] = x[1] = x[2] = x[3] = 0; // if there is an "N", restart + } + + ///if(buf->a.n - buf_iter <= 1) return; + if(buf->a.n - buf_iter == 0) return; + if(buf->a.n - buf_iter > 1) radix_sort_hc_s_hit_an1(buf->a.a + buf_iter, buf->a.a + buf->a.n); + + + + /*******************************for debug************************************/ + // print_pos_list(idx, buf->a.a+buf_iter, buf->a.n - buf_iter, rid, (buf_iter != 0)); + // fprintf(stderr, "len0:%lu\n", buf->a.n - buf_iter); + // for (i = buf_iter; i < buf->a.n; i++) + // { + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); + // fprintf(stderr, "(%lu) rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu, len: %lu\n", + // i, rev, uID, ref_p, self_p, cnt); + // } + /*******************************for debug************************************/ + + + + + + uint64_t cur_ref_p, thres = (len * HIC_R_E_RATE) + 1, m, index_beg, ovlp, maxLen = 0, max_i = (uint64_t)-1; + i = m = buf_iter; + while (i < buf->a.n) + { + interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); + /*******************************for debug************************************/ + // if(check_exact_match(r, self_p, len, idx->ug->u.a[uID].s, + // ref_p, idx->ug->u.a[uID].len, cnt, rev, 1) != cnt) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + // if(self_p > ref_p) + // { + // i++; + // continue; ///fix this in future + // } + cur_ref_p = buf->a.a[i].ref; + index_beg = i; + while ((i < buf->a.n) && ((buf->a.a[i].ref>>idx->pos_bits) == (cur_ref_p>>idx->pos_bits)) && + (buf->a.a[i].ref - cur_ref_p <= thres)) + { + i++; + } + if(i - index_beg > 1) + { + radix_sort_hc_s_hit_an2(buf->a.a + index_beg, buf->a.a + i); + } + ovlp = collect_votes(buf->a.a + index_beg, i - index_beg); + buf->a.a[m] = buf->a.a[i - 1]; + buf->a.a[m].off_cnt = (buf->a.a[m].off_cnt << 32)>>32; + buf->a.a[m].off_cnt += ((uint64_t)ovlp<<32); + + if(maxLen < (ovlp&((uint64_t)65535))) maxLen = (ovlp&((uint64_t)65535)), max_i = m; + + m++; + } + buf->a.n = m; + + /*******************************for debug************************************/ + // for (i = buf_iter; i < buf->a.n; i++) + // { + // uint64_t eLen, tLen; + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // if(maxLen < eLen) fprintf(stderr, "ERROR1\n"); + // if(i == max_i && maxLen != eLen) fprintf(stderr, "ERROR2\n"); + // } + /*******************************for debug************************************/ + ///select the best alignment at [buf_iter, m) + + /*******************************for debug************************************/ + // fprintf(stderr, "len1:%lu, max_i: %lu\n", buf->a.n - buf_iter, max_i); + // for (i = buf_iter; i < buf->a.n; i++) + // { + // uint64_t eLen, tLen; + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "(%lu) rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu, eLen: %lu, tLen: %lu\n", + // i, rev, uID, ref_p, self_p, eLen, tLen); + // } + /*******************************for debug************************************/ + + compress_mapped_pos(idx, buf, buf_iter, max_i, thres); + + /*******************************for debug************************************/ + // fprintf(stderr, "len2:%lu, max_i: %lu\n", buf->a.n - buf_iter, max_i); + // for (i = buf_iter; i < buf->a.n; i++) + // { + // uint64_t eLen, tLen; + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "(%lu) rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu, eLen: %lu, tLen: %lu\n", + // i, rev, uID, ref_p, self_p, eLen, tLen); + // } + // if(buf->a.n != m) fprintf(stderr, "Changed\n"); + // fprintf(stderr, "\n"); + /*******************************for debug************************************/ +} + + inline int is_unreliable_hits(long long rev, long long ref_p, long long tLen, uint64_t uID, hc_links* link) { uint64_t i; @@ -2797,7 +3002,7 @@ void write_hc_links(hc_links* link, const char *fn) { uint64_t k; char *buf = (char*)calloc(strlen(fn) + 25, 1); - sprintf(buf, "%s.hic.link", fn); + sprintf(buf, "%s.hic.link.bin", fn); FILE* fp = fopen(buf, "w"); fwrite(&link->a.n, sizeof(link->a.n), 1, fp); @@ -2833,7 +3038,7 @@ int load_hc_links(hc_links* link, const char *fn) { uint64_t k, flag = 0; char *buf = (char*)calloc(strlen(fn) + 25, 1); - sprintf(buf, "%s.hic.link", fn); + sprintf(buf, "%s.hic.link.bin", fn); FILE* fp = NULL; fp = fopen(buf, "r"); @@ -2941,8 +3146,9 @@ void print_hc_links(hc_links* link, int dir, H_partition* hap) if(dir == 0) { double f_w, r_w; - for (i = 0, f_w = r_w = 0; i < link->a.n; ++i) + for (i = 0; i < link->a.n; ++i) { + f_w = r_w = 0; for (k = 0; k < link->a.a[i].e.n; k++) { if(link->a.a[i].e.a[k].del) continue; @@ -2964,7 +3170,7 @@ void print_hc_links(hc_links* link, int dir, H_partition* hap) } fprintf(stderr, "self-utg%.6dl\tFW:%f\tRW:%f\tRT:%f\n**************************************************\n", - (int)(i+1), f_w, r_w, r_w/f_w); + (int)(i+1), f_w, r_w, (f_w+r_w) != 0? r_w/(f_w+r_w):0); } } @@ -11272,8 +11478,6 @@ int alignment_worker_pipeline(sldat_t* sl, const enzyme *fn1, const enzyme *fn2) sl->ks2 = kseq_init(fp2); kt_pipeline(3, worker_pipeline, sl, 3); - - ///fprintf(stderr, "fn1->a[i]: %s, fn2->a[i]: %s, sl->hits.a.n: %u\n", fn1->a[i], fn2->a[i], (uint32_t)sl->hits.a.n); kseq_destroy(sl->ks1); @@ -11282,7 +11486,6 @@ int alignment_worker_pipeline(sldat_t* sl, const enzyme *fn1, const enzyme *fn2) gzclose(fp2); } - ///fprintf(stderr, "+sl->hits.a.n: %u\n", (uint32_t)sl->hits.a.n); dedup_hits(&(sl->hits)); @@ -11302,7 +11505,7 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) sl.chunk_size = 20000000; sl.n_thread = asm_opt.thread_num; sl.total_base = sl.total_pair = 0; - idx->max_cnt = 5; + idx->hap_cnt = asm_opt.hap_occ; kv_init(sl.hits.a); if(!load_hc_hits(&sl.hits, asm_opt.output_file_name)) @@ -11351,6 +11554,8 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) ///print_hc_links(idx->link, 0, &hap); } + ///print_hc_links(idx->link, 0, &hap); + cluster_contigs(&bub, idx, &sl.hits, &M, &hap); destory_MT(&M); @@ -11698,7 +11903,7 @@ int hic_short_align_bench(const enzyme *fn1, const enzyme *fn2, const char *outp sl.chunk_size = 20000000; sl.n_thread = asm_opt.thread_num; sl.total_base = sl.total_pair = 0; - idx->max_cnt = 5; + idx->hap_cnt = asm_opt.hap_occ; kv_init(sl.hits.a); fprintf(stderr, "u.n: %d, uID_bits: %lu, pos_bits: %lu\n", (uint32_t)idx->ug->u.n, idx->uID_bits, idx->pos_bits); From 8aa87fdce8380991bca534f438f5f82218a13177 Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Mon, 8 Mar 2021 20:52:11 -0500 Subject: [PATCH 2/8] purge_dups for high het --- CommandLines.cpp | 6 +- CommandLines.h | 2 +- Overlaps.cpp | 724 +++++-------- Overlaps.h | 43 +- Purge_Dups.cpp | 2534 +++++++++++++++++++++++++++------------------- Purge_Dups.h | 8 +- hic.cpp | 1211 +++++++++++++++++----- hifiasm.1 | 3 +- 8 files changed, 2766 insertions(+), 1765 deletions(-) diff --git a/CommandLines.cpp b/CommandLines.cpp index 4685b0d..caea7e4 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -89,7 +89,7 @@ void Print_H(hifiasm_opt_t* asm_opt) fprintf(stderr, " -4 FILE list of hap2/maternal read names []\n"); fprintf(stderr, " Purge-dups:\n"); - fprintf(stderr, " -l INT purge level. 0: no purging; 1: light; 2: aggressive [0 for trio; 2 for unzip]\n"); + fprintf(stderr, " -l INT purge level. 0: no purging; 1: light; 2/3: aggressive [0 for trio; 2 for unzip]\n"); fprintf(stderr, " -s FLOAT similarity threshold for duplicate haplotigs [%g]\n", asm_opt->purge_simi_rate); fprintf(stderr, " -O INT min number of overlapped reads for duplicate haplotigs [%d]\n", @@ -360,9 +360,9 @@ int check_option(hifiasm_opt_t* asm_opt) return 0; } - if(asm_opt->purge_level_primary < 0 || asm_opt->purge_level_primary > 2) + if(asm_opt->purge_level_primary < 0 || asm_opt->purge_level_primary > 3) { - fprintf(stderr, "[ERROR] the level of purge-dup should be [0, 2] (-l)\n"); + fprintf(stderr, "[ERROR] the level of purge-dup should be [0, 3] (-l)\n"); return 0; } diff --git a/CommandLines.h b/CommandLines.h index 309fd8d..253e6a8 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -3,7 +3,7 @@ #include -#define HA_VERSION "0.14-r312" +#define HA_VERSION "0.14-r313" #define VERBOSE 0 diff --git a/Overlaps.cpp b/Overlaps.cpp index 3e320d8..6f3ba54 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -11062,7 +11062,6 @@ inline uint64_t get_utg_len(buf_t* b, ma_ug_t *ug, asg_t *read_sg, uint64_t igno for (r_i = 0; r_i < u->n; r_i++, idx++) { v = ((uint64_t)((u->a[u->n - r_i - 1])^(uint64_t)(0x100000000)))>>32; - ///w = ((uint64_t)((u->a[u->n - x->r_i - 2])^(uint64_t)(0x100000000)))>>32; if(p_v == (uint32_t)-1) { p_v = v; @@ -11085,7 +11084,7 @@ inline uint64_t get_utg_len(buf_t* b, ma_ug_t *ug, asg_t *read_sg, uint64_t igno len += l; if(len_thre && occ && len >= (*len_thre)) { - (*occ) = idx - 1; + (*occ) = idx; return len; } p_v = v; @@ -11120,7 +11119,7 @@ inline uint64_t get_utg_len(buf_t* b, ma_ug_t *ug, asg_t *read_sg, uint64_t igno len += l; if(len_thre && occ && len >= (*len_thre)) { - (*occ) = idx - 1; + (*occ) = idx; return len; } p_v = v; @@ -11133,7 +11132,7 @@ inline uint64_t get_utg_len(buf_t* b, ma_ug_t *ug, asg_t *read_sg, uint64_t igno len += read_sg->seq[p_v>>1].len; if(len_thre && occ && len >= (*len_thre)) { - (*occ) = idx - 1; + (*occ) = idx; return len; } } @@ -11208,22 +11207,6 @@ void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug push_hc_edge(&(link->a.a[pre_0]), pre_1, 1, 1, &d); push_hc_edge(&(link->a.a[pre_1]), pre_0, 1, 1, &d); - - // if(pre_0 == 5 || pre_1 == 5) - // { - // fprintf(stderr, "\npre_0: utg%.6ul, len_0: %lu, thre_0: %lu, pre_1: utg%.6ul, len_1: %lu, thre_1: %lu\n", - // (int)(pre_0+1), len_0, thre_0, (int)(pre_1+1), len_1, thre_1); - // uint32_t xxx_i; - // for (xxx_i = 0; xxx_i < b_0->b.n; xxx_i++) - // { - // fprintf(stderr,"+: utg%.6ul\n", (int)((b_0->b.a[xxx_i]>>1)+1)); - // } - - // for (xxx_i = 0; xxx_i < b_1->b.n; xxx_i++) - // { - // fprintf(stderr,"-: utg%.6ul\n", (int)((b_1->b.a[xxx_i]>>1)+1)); - // } - // } } } @@ -11233,9 +11216,119 @@ void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug } +uint32_t set_utg_offset(buf_t* b, ma_ug_t *ug, asg_t *read_sg, hap_cov_t *cov, uint32_t is_clear) +{ + uint32_t ori, uid, v, nv, l, k; + uint32_t *a = b->b.a, a_n = b->b.n; + uint32_t u_i, r_i, len, p_v; + asg_arc_t *av = NULL; + ma_utg_t* u = NULL; + for (u_i = r_i = len = 0, p_v = (uint32_t)-1; u_i < a_n; u_i++) + { + uid = a[u_i] >> 1; + ori = a[u_i] & 1; + u = &(ug->u.a[uid]); + if(u->n == 0) continue; + + for (r_i = 0; r_i < u->n; r_i++) + { + l = 0; + v = (ori == 1?((uint64_t)((u->a[u->n - r_i - 1])^(uint64_t)(0x100000000)))>>32:((uint64_t)(u->a[r_i]))>>32); + + if(p_v != (uint32_t)-1 && is_clear == 0) + { + av = asg_arc_a(read_sg, p_v); + nv = asg_arc_n(read_sg, p_v); + + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + if(av[k].v == v) + { + l = asg_arc_len(av[k]); + break; + } + } + if(k == nv) fprintf(stderr, "ERROR\n"); + } + + p_v = v; len += l; + + if(is_clear == 1) + { + cov->pos_idx[v>>1] = (uint64_t)-1; + } + else + { + cov->pos_idx[v>>1] = len; + cov->pos_idx[v>>1] <<= 32; + cov->pos_idx[v>>1] |= (uint64_t)v; + } + } + } + + if(p_v != (uint32_t)-1) len += read_sg->seq[p_v>>1].len; + + return len; +} + + +void collect_trans_cov(buf_t* pri, buf_t* aux, ma_ug_t *ug, asg_t *read_sg, hap_cov_t *cov) +{ + uint32_t i, k, rid, occ, thre_pri; + uint64_t len_aux, uLen, uCov; + ma_utg_t* u = NULL; + if(pri->b.n == 0 || aux->b.n == 0) return; + + len_aux = set_utg_offset(aux, ug, read_sg, cov, 0); + chain_trans_ovlp(cov, ug, read_sg, pri, len_aux, &thre_pri); + if(thre_pri > 0) + { + for (i = uCov = 0; i < aux->b.n; i++) + { + u = &(ug->u.a[aux->b.a[i]>>1]); + if(u->n == 0) continue; + for (k = 0; k < u->n; k++) + { + rid = u->a[k]>>33; + uCov += cov->cov[rid]; + } + } + + + for (i = uLen = occ = 0; i < pri->b.n; i++) + { + u = &(ug->u.a[pri->b.a[i]>>1]); + if(u->n == 0) continue; + for (k = 0; k < u->n; k++, occ++) + { + if(occ >= thre_pri) break; + rid = u->a[k]>>33; + uLen += read_sg->seq[rid].len; + } + if(occ >= thre_pri) break; + } + + uCov = (uLen == 0? 0 : uCov / uLen); + + for (i = occ = 0; i < pri->b.n; i++) + { + u = &(ug->u.a[pri->b.a[i]>>1]); + if(u->n == 0) continue; + for (k = 0; k < u->n; k++, occ++) + { + if(occ >= thre_pri) break; + rid = u->a[k]>>33; + cov->cov[rid] += (uCov * read_sg->seq[rid].len); + } + if(occ >= thre_pri) break; + } + } + set_utg_offset(aux, ug, read_sg, cov, 1); +} int untig_asg_arc_simple_large_bubbles_trio(ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negative_flag, hc_links* link) +long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negative_flag, hc_links* link, hap_cov_t *cov) { asg_t *g = ug->g; double startTime = Get_T(); @@ -11360,6 +11453,7 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ } if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); is_hap++; } @@ -11551,11 +11645,13 @@ kvec_asg_arc_t_warp* new_rtg_edges, int max_hang, int min_ovlp) { uint64_t i, dip_thre_max, dip_thres, n_utg; uint8_t* primary_flag = (uint8_t*)calloc(sg->n_seq, sizeof(uint8_t)); + hap_cov_t *cov = init_hap_cov_t(ug, sg, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp); + int tmp_cov = asm_opt.hom_global_coverage; asm_opt.hom_global_coverage = -1; purge_dups(ug, sg, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 0, 1, NULL); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 0, 1, NULL, cov); dip_thre_max = ((double)asm_opt.hom_global_coverage)/((double)HOM_PEAK_RATE)*0.70; asm_opt.hom_global_coverage = tmp_cov; ///fprintf(stderr, "dip_thre_max: %lu\n", dip_thre_max); @@ -11579,7 +11675,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, int max_hang, int min_ovlp) } } free(primary_flag); - + destory_hap_cov_t(&cov); ///fprintf(stderr, "[M::%s] diploid coverage threshold: %lu\n", __func__, dip_thres); } @@ -11622,92 +11718,6 @@ void destory_hc_links(hc_links* link) kv_destroy(link->enzymes); } -void pop_small_bub(ma_ug_t *ug) -{ - bubble_type bub; - uint64_t n_vtx = ug->g->n_seq*2, tLen, pathLen, nodeLen; - uint32_t i, k, v, mode = (((uint32_t)-1)<<2); - asg_cleanup(ug->g); if (!ug->g->is_symm) asg_symm(ug->g); - memset(&bub, 0, sizeof(bubble_type)); - CALLOC(bub.index, n_vtx); - kv_init(bub.list); kv_init(bub.num); kv_init(bub.pathLen); - buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); - for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; - for (v = 0; v < n_vtx; ++v) - { - if(ug->g->seq[v>>1].del) continue; - if(asg_arc_n(ug->g, v) < 2) continue; - if((bub.index[v]&(uint32_t)3) != 0) continue; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) - { - //beg is v, end is b.S.a[0] - //note b.b include end, does not include beg - for (i = 0; i < b.b.n; i++) - { - if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; - bub.index[b.b.a[i]] &= mode; bub.index[b.b.a[i]] += 1; - bub.index[b.b.a[i]^1] &= mode; bub.index[b.b.a[i]^1] += 1; - } - bub.index[v] &= mode; bub.index[v] += 2; - bub.index[b.S.a[0]^1] &= mode; bub.index[b.S.a[0]^1] += 3; - } - } - - for (v = 0; v < n_vtx; ++v) - { - if((bub.index[v]&(uint32_t)3) !=2) continue; - kv_push(uint32_t, bub.num, bub.list.n); - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL)) - { - kv_push(uint64_t, bub.pathLen, pathLen); - //beg is v, end is b.S.a[0] - kv_push(uint32_t, bub.list, v); - kv_push(uint32_t, bub.list, b.S.a[0]^1); - - //note b.b include end, does not include beg - for (i = 0; i < b.b.n; i++) - { - if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; - kv_push(uint32_t, bub.list, b.b.a[i]); - } - } - } - - kv_push(uint32_t, bub.num, bub.list.n); - ///free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); - bub.f_bub = bub.num.n - 1; bub.b_bub = 0; - memset(bub.index, 0, n_vtx*sizeof(uint32_t)); - - uint32_t beg, sink, n, *a, total_nodes; - for (i = 0; i < bub.f_bub; i++) - { - get_bubbles(&bub, i, &beg, &sink, &a, &n, &pathLen); - for (k = total_nodes = 0; k < n; k++) - { - total_nodes += ug->u.a[a[k]>>1].n; - } - - for (k = 0; k < n; k++) - { - v = a[k]; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, &nodeLen)) - { - - } - - - v = a[k]^1; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, &nodeLen)) - { - - } - - } - - } - -} - void hic_clean(asg_t* read_g) { uint32_t n_vtx, v, u; @@ -11734,7 +11744,7 @@ void hic_clean(asg_t* read_g) if(ug->g->seq[v>>1].del) continue; if(asg_arc_n(ug->g, v) < 2) continue; if(bs_flag[v] != 0) continue; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL)) { //beg is v, end is b.S.a[0] //note b.b include end, does not include beg @@ -11751,7 +11761,7 @@ void hic_clean(asg_t* read_g) for (v = 0; v < n_vtx; ++v) { if(bs_flag[v] !=2) continue; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL)) { //note b.b include end, does not include beg for (i = v_occ = ax.n = 0; i < b.b.n; i++) @@ -11767,7 +11777,7 @@ void hic_clean(asg_t* read_g) { u = (ax.a[i]<<1) + k; if(asg_arc_n(ug->g, u) < 2) continue; - if(asg_bub_pop1_primary_trio(ug->g, NULL, u, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + if(asg_bub_pop1_primary_trio(ug->g, NULL, u, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL)) { for (k_i = u_occ = utg_occ = 0; k_i < b.b.n; k_i++) { @@ -11779,7 +11789,7 @@ void hic_clean(asg_t* read_g) if(u_occ >= v_occ*bub_rate) continue; if(u_occ > 3) continue; if(utg_occ > 2) continue; - asg_bub_pop1_primary_trio(ug->g, NULL, u, tLen, &b, (uint32_t)-1, (uint32_t)-1, 1, NULL, NULL); + asg_bub_pop1_primary_trio(ug->g, NULL, u, tLen, &b, (uint32_t)-1, (uint32_t)-1, 1, NULL, NULL, NULL); } } } @@ -11824,7 +11834,7 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov hc_links link; - + if(load_hc_links(&link, output_file_name) == 0) { init_hc_links(&link, ug->g->n_seq, R_INF.total_reads); @@ -12513,7 +12523,7 @@ asg_t *read_sg, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, uint32_t min_e int asg_arc_cut_trio_long_tip_primary(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link) +R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link, hap_cov_t *cov) { double startTime = Get_T(); ///the reason is that each read has two direction (query->target, target->query) @@ -12613,6 +12623,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link) } if(link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov && operation != CUT) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); } } } @@ -12635,7 +12646,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link) } int asg_arc_cut_trio_long_tip_primary_complex(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_threshold, hc_links* link) +R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_threshold, hc_links* link, hap_cov_t *cov) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, in, flag, operation; @@ -12711,6 +12722,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_thre } if(link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov && operation != CUT) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); break; } @@ -12810,7 +12822,7 @@ long long* base_maxLen, long long* base_maxLen_i, uint32_t stops_threshold, buf_ int asg_arc_cut_trio_long_equal_tips_assembly(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t trio_flag, -hc_links* link) +hc_links* link, hap_cov_t *cov) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag, is_hap, n_tips, return_flag, k; @@ -12898,6 +12910,7 @@ hc_links* link) } if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); is_hap++; } @@ -13130,7 +13143,7 @@ R_to_U* ruIndex, uint32_t positive_flag, float drop_rate) } int asg_arc_cut_trio_long_equal_tips_assembly_complex(asg_t *g, ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t stops_threshold, hc_links* link) +ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t stops_threshold, hc_links* link, hap_cov_t *cov) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, in, flag; @@ -13199,6 +13212,7 @@ ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_ } if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); ///lable the primary one b_0.b.n = 0; @@ -13600,8 +13614,8 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) redo: ///print_untig((ug), 61955, "i-0:", 0); - asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP); - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL); + asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, NULL); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL, NULL); magic_trio_phasing(g, ug, read_g, coverage_cut, sources, reverse_sources, 2, ruIndex, trio_flag, trio_drop_rate); ///drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); /**********debug**********/ @@ -13617,16 +13631,15 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) { pre_cons = get_graph_statistic(g); ///need consider tangles - ///asg_pop_bubble_primary(g, bubble_dist); - asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP); + asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, NULL); /**********debug**********/ if(just_bubble_pop == 0) { ///need consider tangles - asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, NULL); - asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, trio_flag, NULL); - asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, NULL); - asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, NULL); + asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, NULL, NULL); + asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, trio_flag, NULL, NULL); + asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, NULL, NULL); + asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, NULL, NULL); ///print_debug_gfa(read_g, ug, coverage_cut, "debug_chimeric", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); @@ -13636,7 +13649,7 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) /**********debug**********/ cur_cons = get_graph_statistic(g); } - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL, NULL); if(just_bubble_pop == 0) { @@ -13661,7 +13674,7 @@ void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* rever long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, -float drop_ratio, hc_links* link) +float drop_ratio, hc_links* link, hap_cov_t *cov) { #define T_ROUND 2 asg_t *g = ug->g; @@ -13669,9 +13682,8 @@ float drop_ratio, hc_links* link) redo: - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP); - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link); - + asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, cov); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link, cov); if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, @@ -13683,17 +13695,16 @@ float drop_ratio, hc_links* link) while(pre_cons != cur_cons) { pre_cons = get_graph_statistic(g); - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP); + asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, cov); if(just_bubble_pop == 0) { ///need consider tangles - asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, link); - asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, link); - asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, link); - asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, link); + asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, link, cov); + asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, link, cov); + asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, link, cov); + asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, link, cov); detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); - if(round != T_ROUND) { unitig_arc_del_short_diploid_by_length_topo(g, ug, drop_ratio, asm_opt.max_short_tip, @@ -13702,14 +13713,12 @@ float drop_ratio, hc_links* link) } cur_cons = get_graph_statistic(g); } - - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link, cov); if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, 2); } - resolve_tangles(ug, read_g, reverse_sources, 20, 100, 0.05, 0.2, ruIndex, (uint32_t)-1, drop_ratio); drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); unitig_arc_del_short_diploid_by_length_topo(g, ug, drop_ratio, asm_opt.max_short_tip, reverse_sources, 0, 1); @@ -14366,14 +14375,15 @@ kvec_asg_arc_t_warp* new_rtg_edges) { asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq; - + hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp); + purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 1, NULL); + drop_ratio, 1, 1, NULL, cov); if(asm_opt.recover_atg_cov_min == -1024) { asm_opt.recover_atg_cov_max = asm_opt.hom_global_coverage/HOM_PEAK_RATE; - asm_opt.recover_atg_cov_min = asm_opt.recover_atg_cov_max * 0.8; + asm_opt.recover_atg_cov_min = asm_opt.recover_atg_cov_max * 0.85; ///asm_opt.recover_atg_cov_max = asm_opt.recover_atg_cov_max * 1.2; asm_opt.recover_atg_cov_max = INT32_MAX; } @@ -14450,13 +14460,14 @@ kvec_asg_arc_t_warp* new_rtg_edges) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 0, NULL); + drop_ratio, 1, 0, NULL, cov); ///delete_useless_nodes(ug); delete_useless_trio_nodes(ug, read_g, coverage_cut, sources, ruIndex); } set_drop_trio_flag(*ug); + destory_hap_cov_t(&cov); } @@ -14726,10 +14737,73 @@ static inline int count_out(const asg_t *g, uint32_t v) } +// in a resolved bubble, mark unused vertices and arcs as "reduced" +static void asg_bub_backtrack_primary_cov(ma_ug_t *ug, uint32_t v0, buf_t *b, hap_cov_t *cov) +{ + uint32_t i, k, v, u, uLen = 0, uCov = 0, uId, rId; + ma_utg_t* p = NULL; + ///b->S.a[0] is the sink of this bubble + + ///assert(b->S.n == 1); + ///first remove all nodes in this bubble + for (i = 0; i < b->b.n; ++i) + { + uId = b->b.a[i]>>1; + if(uId == (b->S.a[0]>>1)) continue; + p = &(ug->u.a[uId]); + if(p->n == 0) continue; + for (k = 0; k < p->n; k++) + { + rId = p->a[k]>>33; + uCov += cov->cov[rId]; + } + } + + ///v is the sink of this bubble + v = b->S.a[0]; + ///recover node + do { + u = b->a[v].p; // u->v + if(v != b->S.a[0]) + { + uId = v>>1; + p = &(ug->u.a[uId]); + if(p->n == 0) continue; + for (k = 0; k < p->n; k++) + { + rId = p->a[k]>>33; + uCov -= cov->cov[rId]; + uLen += cov->read_g->seq[rId].len; + } + } + v = u; + } while (v != v0); + + uCov = (uLen == 0? 0 : uCov / uLen); + + ///v is the sink of this bubble + v = b->S.a[0]; + ///recover node + do { + u = b->a[v].p; // u->v + if(v != b->S.a[0]) + { + uId = v>>1; + p = &(ug->u.a[uId]); + if(p->n == 0) continue; + for (k = 0; k < p->n; k++) + { + rId = p->a[k]>>33; + cov->cov[rId] += (uCov * cov->read_g->seq[rId].len); + } + } + v = u; + } while (v != v0); +} // in a resolved bubble, mark unused vertices and arcs as "reduced" -static void asg_bub_backtrack_primary(asg_t *g, uint32_t v0, buf_t *b) +void asg_bub_backtrack_primary(asg_t *g, uint32_t v0, buf_t *b) { uint32_t i, v, qn, tn; ///b->S.a[0] is the sink of this bubble @@ -15108,7 +15182,8 @@ pop_reset: uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, -uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes) +uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, +hap_cov_t *cov) { uint32_t i, n_pending = 0, is_first = 1, cur_m, cur_c, cur_np, cur_nc, to_replace, n_tips, tip_end; uint64_t n_pop = 0; @@ -15330,9 +15405,9 @@ uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_ if (i < nv || b->S.n == 0) goto pop_reset; } while (b->S.n > 1 || n_pending); + if(cov && utg) asg_bub_backtrack_primary_cov(utg, v0, b, cov); if(is_pop) asg_bub_backtrack_primary(g, v0, b); if(path_base_len || path_nodes) asg_bub_backtrack_primary_length(g, utg, v0, b, path_base_len, path_nodes); - n_pop = 1; pop_reset: @@ -15345,7 +15420,7 @@ pop_reset: // pop bubbles -int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag) +int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov) { asg_t *g = ug->g; uint32_t v, n_vtx = g->n_seq * 2; @@ -15365,7 +15440,7 @@ int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_fla for (i = 0; i < nv; ++i) // asg_bub_pop1() may delete some edges/arcs if (!av[i].del) ++n_arc; if (n_arc > 1) - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL, cov); } free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); if (n_pop) asg_cleanup(g); @@ -15378,303 +15453,6 @@ int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_fla } - -// pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well -uint64_t asg_bub_pop1_primary_trio_debug(ma_ug_t *ug, uint32_t v0, int max_dist, buf_t *b, -uint32_t positive_flag, uint32_t negative_flag) -{ - asg_t *g = ug->g; - uint32_t i, n_pending = 0, is_first = 1, cur_m, cur_c, to_replace, n_tips, tip_end; - uint64_t n_pop = 0; - ///if this node has been deleted - if (g->seq[v0>>1].del || g->seq[v0>>1].c == ALTER_LABLE) return 0; // already deleted - ///asg_arc_n(n0) - if ((uint32_t)g->idx[v0] < 2) return 0; // no bubbles - ///S saves nodes with all incoming edges visited - b->S.n = b->T.n = b->b.n = b->e.n = 0; - ///for each node, b->a saves all related information - b->a[v0].c = b->a[v0].d = b->a[v0].m = 0; - ///b->S is the nodes with all incoming edges visited - kv_push(uint32_t, b->S, v0); - n_tips = 0; - tip_end = (uint32_t)-1; - - do { - ///v is a node that all incoming edges have been visited - ///d is the distance from v0 to v - uint32_t v = kv_pop(b->S), d = b->a[v].d, c = b->a[v].c, m = b->a[v].m; - uint32_t nv = asg_arc_n(g, v); - asg_arc_t *av = asg_arc_a(g, v); - ///why we have this assert? - ///assert(nv > 0); - ///all out-edges of v - for (i = 0; i < nv; ++i) { // loop through v's neighbors - /** - p->ul: |____________31__________|__________1___________|______________32_____________| - qn direction of overlap length of this node (not overlap length) - (in the view of query) - p->v : |___________31___________|__________1___________| - tn reverse direction of overlap - (in the view of target) - p->ol: overlap length - **/ - - uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l - binfo_t *t = &b->a[w]; - ///that means there is a circle, directly terminate the whole bubble poping - ///if (w == v0) goto pop_reset; - if ((w>>1) == (v0>>1)) goto pop_reset; - /****************************may have bugs********************************/ - ///important when poping at long untig graph - if(is_first) l = 0; - /****************************may have bugs********************************/ - - ///if this edge has been deleted - if (av[i].del) continue; - - ///push the edge - ///high 32-bit of g->idx[v] is the start point of v's edges - //so here is the point of this specfic edge - kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); - ///find a too far path? directly terminate the whole bubble poping - if (d + l > (uint32_t)max_dist) break; // too far - - ///if this node - if (t->s == 0) { // this vertex has never been visited - kv_push(uint32_t, b->b, w); // save it for revert - ///t->p is the parent node of - ///t->s = 1 means w has been visited - ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) - t->p = v, t->s = 1, t->d = d + l; - /****************************may have bugs********************************/ - cur_c = get_num_trio_flag(ug, w>>1, positive_flag); - cur_m = get_num_trio_flag(ug, w>>1, negative_flag); - t->c = c + cur_c; - t->m = m + cur_m; - /****************************may have bugs********************************/ - ///incoming edges of w - t->r = count_out(g, w^1); - ++n_pending; - } else { // visited before - /****************************may have bugs********************************/ - cur_c = get_num_trio_flag(ug, w>>1, positive_flag); - cur_m = get_num_trio_flag(ug, w>>1, negative_flag); - ///select the way with less negative_flag, more positive_flag, more distance - to_replace = 0; - if(m + cur_m < t->m) to_replace = 1; - if(to_replace == 0 && m + cur_m == t->m && c + cur_c > t->c) to_replace = 1; - if(to_replace == 0 && m + cur_m == t->m && c + cur_c == t->c && d + l > t->d) to_replace = 1; - if(to_replace) - { - t->p = v; - t->m = m + cur_m; - t->c = c + cur_c; - } - ///c is the weight (is very likely the number of node in this edge) of the parent node - ///select the longest edge (longest meams most reads/longest edge) - // if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; - // if (c + 1 > t->c) t->c = c + 1; - /****************************may have bugs********************************/ - ///update len(v0->w) - ///node: t->d is not the length from this node's parent - ///it is the shortest edge - if (d + l < t->d) t->d = d + l; // update dist - } - ///assert(t->r > 0); - //if all incoming edges of w have visited - //push it to b->S - if (--(t->r) == 0) { - uint32_t x = asg_arc_n(g, w); - /****************************may have bugs for bubble********************************/ - /** - if (x) kv_push(uint32_t, b->S, w); - ///else kv_push(uint32_t, b->T, w); // a tip - else goto pop_reset; - **/ - if(x > 0) - { - kv_push(uint32_t, b->S, w); - } - else - { - ///at most one tip - if(n_tips != 0) goto pop_reset; - n_tips++; - tip_end = w; - } - /****************************may have bugs for bubble********************************/ - --n_pending; - } - } - is_first = 0; - //if found a tip - /****************************may have bugs for bubble********************************/ - if(n_tips == 1) - { - if(tip_end != (uint32_t)-1 && n_pending == 0 && b->S.n == 0) - { - kv_push(uint32_t, b->S, tip_end); - break; - } - else - { - goto pop_reset; - } - } - /****************************may have bugs for bubble********************************/ - ///if i < nv, that means (d + l > max_dist) - if (i < nv || b->S.n == 0) goto pop_reset; - } while (b->S.n > 1 || n_pending); - - n_pop = 1; -pop_reset: - for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices - binfo_t *t = &b->a[b->b.a[i]]; - t->s = t->c = t->d = t->m = 0; - } - return n_pop; -} - - - -// pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well -static uint64_t asg_bub_pop1_primary(asg_t *g, uint32_t v0, int max_dist, buf_t *b) -{ - uint32_t i, n_pending = 0, is_first = 1; - uint64_t n_pop = 0; - ///if this node has been deleted - if (g->seq[v0>>1].del || g->seq[v0>>1].c == ALTER_LABLE) return 0; // already deleted - ///asg_arc_n(n0) - if ((uint32_t)g->idx[v0] < 2) return 0; // no bubbles - ///S saves nodes with all incoming edges visited - b->S.n = b->T.n = b->b.n = b->e.n = 0; - ///for each node, b->a saves all related information - b->a[v0].c = b->a[v0].d = 0; - ///b->S is the nodes with all incoming edges visited - kv_push(uint32_t, b->S, v0); - - do { - ///v is a node that all incoming edges have been visited - ///d is the distance from v0 to v - uint32_t v = kv_pop(b->S), d = b->a[v].d, c = b->a[v].c; - uint32_t nv = asg_arc_n(g, v); - asg_arc_t *av = asg_arc_a(g, v); - ///why we have this assert? - ///assert(nv > 0); - ///all out-edges of v - for (i = 0; i < nv; ++i) { // loop through v's neighbors - /** - p->ul: |____________31__________|__________1___________|______________32_____________| - qn direction of overlap length of this node (not overlap length) - (in the view of query) - p->v : |___________31___________|__________1___________| - tn reverse direction of overlap - (in the view of target) - p->ol: overlap length - **/ - - uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l - binfo_t *t = &b->a[w]; - ///that means there is a circle, directly terminate the whole bubble poping - ///if (w == v0) goto pop_reset; - if ((w>>1) == (v0>>1)) goto pop_reset; - /****************************may have bugs********************************/ - ///important when poping at long untig graph - if(is_first) l = 0; - /****************************may have bugs********************************/ - - ///if this edge has been deleted - if (av[i].del) continue; - - ///push the edge - ///high 32-bit of g->idx[v] is the start point of v's edges - //so here is the point of this specfic edge - kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); - ///find a too far path? directly terminate the whole bubble poping - if (d + l > (uint32_t)max_dist) break; // too far - - ///if this node - if (t->s == 0) { // this vertex has never been visited - kv_push(uint32_t, b->b, w); // save it for revert - ///t->p is the parent node of - ///t->s = 1 means w has been visited - ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) - t->p = v, t->s = 1, t->d = d + l, t->c = c + 1; - ///incoming edges of w - t->r = count_out(g, w^1); - ++n_pending; - } else { // visited before - ///c is the weight (is very likely the number of node in this edge) of the parent node - ///select the longest edge (longest meams most reads/longest edge) - if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; - if (c + 1 > t->c) t->c = c + 1; - ///update len(v0->w) - ///node: t->d is not the length from this node's parent - ///it is the shortest edge - if (d + l < t->d) t->d = d + l; // update dist - } - ///assert(t->r > 0); - //if all incoming edges of w have visited - //push it to b->S - if (--(t->r) == 0) { - uint32_t x = asg_arc_n(g, w); - if (x) kv_push(uint32_t, b->S, w); - ///else kv_push(uint32_t, b->T, w); // a tip - else goto pop_reset; - --n_pending; - } - } - is_first = 0; - ///if i < nv, that means (d + l > max_dist) - if (i < nv || b->S.n == 0) goto pop_reset; - } while (b->S.n > 1 || n_pending); - asg_bub_backtrack_primary(g, v0, b); - n_pop = 1; -pop_reset: - for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices - binfo_t *t = &b->a[b->b.a[i]]; - t->s = t->c = t->d = 0; - } - return n_pop; -} - - -// pop bubbles -int asg_pop_bubble_primary(asg_t *g, int max_dist) -{ - uint32_t v, n_vtx = g->n_seq * 2; - uint64_t n_pop = 0; - buf_t b; - if (!g->is_symm) asg_symm(g); - memset(&b, 0, sizeof(buf_t)); - ///set information for each node - b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); - //traverse all node with two directions - for (v = 0; v < n_vtx; ++v) { - uint32_t i, n_arc = 0, nv = asg_arc_n(g, v); - asg_arc_t *av = asg_arc_a(g, v); - ///some node could be deleted - if (nv < 2 || g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - ///some edges could be deleted - for (i = 0; i < nv; ++i) // asg_bub_pop1() may delete some edges/arcs - if (!av[i].del) ++n_arc; - if (n_arc > 1) - n_pop += asg_bub_pop1_primary(g, v, max_dist, &b); - } - free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); - if (n_pop) asg_cleanup(g); - - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] popped %lu bubbles\n", __func__, (unsigned long)n_pop); - } - return n_pop; -} - - - - - int test_triangular_directly(asg_t *g, uint32_t v, long long min_edge_length, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex) { @@ -18767,13 +18545,13 @@ uint32_t positive_flag, uint32_t negative_flag) v = beg; if((!g->seq[v>>1].del)&&(g->seq[v>>1].c!=ALTER_LABLE)&&get_real_length(g, v, NULL)>=2) { - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL, NULL); } v = end^1; if((!g->seq[v>>1].del)&&(g->seq[v>>1].c!=ALTER_LABLE)&&get_real_length(g, v, NULL)>=2) { - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL, NULL); } @@ -18788,7 +18566,7 @@ uint32_t positive_flag, uint32_t negative_flag) { v = v|k; if(get_real_length(g, v, NULL)<=1) continue; - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL, NULL); } } @@ -21323,6 +21101,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq, k, rId, just_contain; ma_utg_t* u = NULL; + hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp); ///print_utg_coverage(*ug, coverage_cut, 440, sources); ///exit(0); @@ -21344,12 +21123,10 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) } } } - drop_semi_circle((*ug), nsg, read_g, reverse_sources, ruIndex); asg_cleanup(nsg); adjust_utg_advance(read_g, (*ug), reverse_sources, ruIndex); - nsg = (*ug)->g; n_vtx = nsg->n_seq; for (v = 0; v < n_vtx; ++v) @@ -21358,22 +21135,17 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) nsg->seq[v].c = PRIMARY_LABLE; EvaluateLen((*ug)->u, v) = (*ug)->u.a[v].n; } - - clean_primary_untig_graph(*ug, read_g, reverse_sources, bubble_dist, tipsLen, - tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, - chimeric_rate, 0, 0, drop_ratio, link); - + clean_primary_untig_graph(*ug, read_g, reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, + stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, link, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); - if(asm_opt.purge_level_primary > 0) { just_contain = 0; if(asm_opt.purge_level_primary == 1) just_contain = 1; - purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0, link); + drop_ratio, just_contain, 0, link, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } @@ -21391,11 +21163,9 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) { just_contain = 0; if(asm_opt.purge_level_primary == 1) just_contain = 1; - purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0, link); - + drop_ratio, just_contain, 0, link, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } @@ -21405,7 +21175,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 0, 1, link); + drop_ratio, 0, 1, link, cov); } n_vtx = read_g->n_seq; @@ -21442,7 +21212,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) if(asm_opt.recover_atg_cov_min == -1024) { asm_opt.recover_atg_cov_max = asm_opt.hom_global_coverage/HOM_PEAK_RATE; - asm_opt.recover_atg_cov_min = asm_opt.recover_atg_cov_max * 0.8; + asm_opt.recover_atg_cov_min = asm_opt.recover_atg_cov_max * 0.85; ///asm_opt.recover_atg_cov_max = asm_opt.recover_atg_cov_max * 1.2; asm_opt.recover_atg_cov_max = INT32_MAX; } @@ -21474,6 +21244,8 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) link->a.a[v].f.n = m; } } + + destory_hap_cov_t(&cov); } @@ -21495,9 +21267,9 @@ long long tipsLen, R_to_U* ruIndex, int max_hang, int min_ovlp) nsg->seq[v].c = PRIMARY_LABLE; EvaluateLen(ug->u, v) = ug->u.a[v].n; } - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP); + asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, NULL); cut_trio_tip_primary(ug->g, ug, tipsLen, (uint32_t)-1, 0, sg, reverse_sources, ruIndex, 2); - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP); + asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, NULL); cut_trio_tip_primary(ug->g, ug, tipsLen, (uint32_t)-1, 0, sg, reverse_sources, ruIndex, 2); delete_useless_nodes(&ug); renew_utg(&ug, sg, &new_rtg_edges); @@ -22724,7 +22496,7 @@ void lable_all_bubbles(asg_t *r_g, long long bubble_dist) ///if this is a bubble ///if(asg_bub_finder_with_del_advance(r_g, v, bubble_dist, &b) == 1) - if(asg_bub_pop1_primary_trio(r_g, NULL, v, bubble_dist, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + if(asg_bub_pop1_primary_trio(r_g, NULL, v, bubble_dist, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL)) { //beg is v, end is b.S.a[0] //note b.b include end, does not include beg @@ -27358,6 +27130,8 @@ ma_sub_t **coverage_cut_ptr, int debug_g) // rescue_no_coverage_aggressive(sg, sources, reverse_sources, &coverage_cut, ruIndex, max_hang_length, // mini_overlap_length, bubble_dist, 10); + rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); if (asm_opt.flag & HA_F_VERBOSE_GFA) { @@ -27370,8 +27144,8 @@ ma_sub_t **coverage_cut_ptr, int debug_g) if (ha_opt_triobin(&asm_opt) && ha_opt_hic(&asm_opt)) { - rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.hic.bench", output_file_name); @@ -27381,8 +27155,8 @@ ma_sub_t **coverage_cut_ptr, int debug_g) } else if (ha_opt_triobin(&asm_opt)) { - rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.dip", output_file_name); @@ -27398,8 +27172,8 @@ ma_sub_t **coverage_cut_ptr, int debug_g) } else if(ha_opt_hic(&asm_opt)) { - rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.hic", output_file_name); @@ -27419,8 +27193,8 @@ ma_sub_t **coverage_cut_ptr, int debug_g) output_contig_graph_primary_pre(sg, coverage_cut, output_file_name, sources, reverse_sources, asm_opt.small_pop_bubble_size, asm_opt.max_short_tip, ruIndex, max_hang_length, mini_overlap_length); - rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); output_contig_graph_primary(sg, coverage_cut, output_file_name, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, diff --git a/Overlaps.h b/Overlaps.h index ca18270..3e3b0c2 100644 --- a/Overlaps.h +++ b/Overlaps.h @@ -478,7 +478,6 @@ void set_R_to_U(R_to_U* x, uint32_t rID, uint32_t uID, uint32_t is_Unitig, uint8 void get_R_to_U(R_to_U* x, uint32_t rID, uint32_t* uID, uint32_t* is_Unitig); void transfor_R_to_U(R_to_U* x); void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, kvec_asg_arc_t_warp* edge, int require_equal_nv, int test_tangle); -int asg_pop_bubble_primary(asg_t *g, int max_dist); long long asg_arc_del_simple_circle_untig(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *g, long long circleLen, int is_drop); typedef struct { @@ -493,9 +492,35 @@ typedef struct { uint32_t new_edges_i; } Edge_iter; +typedef struct { + asg_arc_t x; + uint64_t Off; + uint64_t weight; +}asg_arc_t_offset; + +typedef struct { + kvec_t(asg_arc_t_offset) a; + uint64_t i; +}kvec_asg_arc_t_offset; + +typedef struct { + uint32_t n; + uint32_t* cov; + uint64_t* pos_idx; + ma_hit_t_alloc* reverse_sources; + ma_sub_t *coverage_cut; + R_to_U* ruIndex; + asg_t *read_g; + int max_hang; + int min_ovlp; + kvec_asg_arc_t_offset u_buffer; + kvec_t_i32_warp tailIndex; + kvec_t_i32_warp prevIndex; +}hap_cov_t; + void init_Edge_iter(asg_t* g, uint32_t v, asg_arc_t* new_edges, uint32_t new_edges_n, Edge_iter* x); int get_arc_t(Edge_iter* x, asg_arc_t* get); -int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag); +int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov); inline int get_real_length(asg_t *g, uint32_t v, uint32_t* v_s) @@ -1040,10 +1065,10 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t bac uint32_t is_bubble_check, uint32_t is_primary_check); uint32_t get_edge_from_source(ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t target, asg_arc_t* t); -uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, -uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes); +uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, uint32_t positive_flag, +uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, hap_cov_t *cov); int unitig_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio); - +void asg_bub_backtrack_primary(asg_t *g, uint32_t v0, buf_t *b); typedef struct{ double weight; @@ -1084,11 +1109,6 @@ typedef struct{ void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num); void destory_hc_links(hc_links* link); -void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, -R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, -uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, -float drop_ratio, hc_links* link); void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, @@ -1112,6 +1132,9 @@ inline int inter_interval(int a_s, int a_e, int b_s, int b_e, int* i_s, int* i_e return 1; } + + + #define JUNK_COV 5 #define DISCARD_RATE 0.8 diff --git a/Purge_Dups.cpp b/Purge_Dups.cpp index 7474a14..d4920be 100644 --- a/Purge_Dups.cpp +++ b/Purge_Dups.cpp @@ -25,18 +25,6 @@ KDQ_INIT(uint64_t) uint8_t debug_enable = 0; -typedef struct { - asg_arc_t x; - uint64_t Off; - uint64_t weight; -}asg_arc_t_offset; - -typedef struct { - kvec_t(asg_arc_t_offset) a; - uint64_t i; -}kvec_asg_arc_t_offset; - - typedef struct { uint64_t weight; uint32_t x_beg_pos; @@ -45,6 +33,7 @@ typedef struct { uint32_t y_end_pos; uint32_t index_beg; uint32_t index_end; + long long score; uint8_t rev; asg_arc_t t; }hap_candidates; @@ -73,6 +62,7 @@ typedef struct { uint32_t xUid; uint32_t yUid; uint32_t weight; + long long score; }hap_overlaps; typedef struct { @@ -115,6 +105,7 @@ typedef struct { float chain_rate; hap_overlaps_list* all_ovlp; long long cov_threshold; + hap_cov_t *cov; }hap_alignment_struct_pip; @@ -492,7 +483,7 @@ void destory_hap_alignment_struct(hap_alignment_struct* x) void init_hap_alignment_struct_pip(hap_alignment_struct_pip* x, uint32_t num_threads, uint32_t n_seq, ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, ma_sub_t *coverage_cut, -uint64_t* position_index, float Hap_rate, int max_hang, int min_ovlp, float chain_rate, hap_overlaps_list* all_ovlp) +uint64_t* position_index, float Hap_rate, int max_hang, int min_ovlp, float chain_rate, hap_overlaps_list* all_ovlp, hap_cov_t *cov) { uint32_t i; x->num_threads = num_threads; @@ -514,6 +505,7 @@ uint64_t* position_index, float Hap_rate, int max_hang, int min_ovlp, float chai x->min_ovlp = min_ovlp; x->chain_rate = chain_rate; x->all_ovlp = all_ovlp; + x->cov = cov; } @@ -857,10 +849,127 @@ uint64_t get_pair_hap_coverage(uint64_t* readIDs, uint32_t Len, ma_hit_t_alloc* return C_bases/R_bases; } + +uint64_t get_pair_purge_coverage(ma_utg_t *xReads, long long xPosBeg, long long xPosEnd, +ma_utg_t *yReads, long long yPosBeg, long long yPosEnd, uint32_t rev, asg_t *read_g, hap_cov_t *cov) +{ + long long offset, r_beg, r_end, i_beg, i_end, ovlp, IdxBeg, IdxEnd; + uint64_t i, rId, uCov, uLen; + ma_utg_t *x = NULL; + uCov = uLen = 0; + if(rev) + { + yPosBeg = yReads->len - yPosBeg - 1; + yPosEnd = yReads->len - yPosEnd - 1; + offset = yPosBeg; yPosBeg = yPosEnd; yPosEnd = offset; + } + + + IdxBeg = IdxEnd = -1; + x = xReads; i_beg = xPosBeg; i_end = xPosEnd; + for (i = 0, offset = 0; i < x->n; i++) + { + rId = x->a[i]>>33; + r_beg = offset; r_end = offset + (long long)(read_g->seq[rId].len) - 1; + offset += (uint32_t)x->a[i]; + ovlp = (long long)(MIN(r_end, i_end)) - (long long)(MAX(r_beg, i_beg)) + 1; + if(ovlp <= 0 || ovlp < read_g->seq[rId].len * 0.8) + { + if(IdxBeg != -1 && IdxEnd != -1) break; + continue; + } + + if(IdxBeg == -1) IdxBeg = i; + IdxEnd = i; + } + if(IdxBeg != -1 && IdxEnd != -1) + { + for (i = IdxBeg; (long long)i <= IdxEnd; i++) + { + rId = x->a[i]>>33; + uCov += cov->cov[rId]; + uLen += cov->read_g->seq[rId].len; + } + } + + + + IdxBeg = IdxEnd = -1; + x = yReads; i_beg = yPosBeg; i_end = yPosEnd; + for (i = 0, offset = 0; i < x->n; i++) + { + rId = x->a[i]>>33; + r_beg = offset; r_end = offset + (long long)(read_g->seq[rId].len) - 1; + offset += (uint32_t)x->a[i]; + ovlp = (long long)(MIN(r_end, i_end)) - (long long)(MAX(r_beg, i_beg)) + 1; + if(ovlp <= 0 || ovlp < read_g->seq[rId].len * 0.8) + { + if(IdxBeg != -1 && IdxEnd != -1) break; + continue; + } + + if(IdxBeg == -1) IdxBeg = i; + IdxEnd = i; + } + if(IdxBeg != -1 && IdxEnd != -1) + { + for (i = IdxBeg; (long long)i <= IdxEnd; i++) + { + rId = x->a[i]>>33; + uCov += cov->cov[rId]; + uLen += cov->read_g->seq[rId].len; + } + } + + return (uLen == 0? 0 : uCov / uLen); +} + + +void get_pair_hap_similarity_by_base(ma_utg_t *xReads, asg_t *read_g, uint32_t target_uId, +ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, long long xBegPos, long long xEndPos, +double* Match, double* Total) +{ + uint32_t i, j, qn, tn, is_Unitig, uId, min_count = 0, max_count = 0; + long long offset, r_beg, r_end, ovlp; + + for (i = 0, offset = 0; i < xReads->n; i++) + { + qn = xReads->a[i]>>33; + r_beg = offset; r_end = offset + (long long)(read_g->seq[qn].len) - 1; + offset += (uint32_t)xReads->a[i]; + + ovlp = (long long)(MIN(r_end, xEndPos)) - (long long)(MAX(r_beg, xBegPos)) + 1; + if(ovlp <= 0) continue; + + if(reverse_sources[qn].length > 0) min_count++; + if(reverse_sources[qn].length == 0) continue; + for (j = 0; j < reverse_sources[qn].length; j++) + { + tn = Get_tn(reverse_sources[qn].buffer[j]); + if(read_g->seq[tn].del == 1) + { + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + + + get_R_to_U(ruIndex, tn, &uId, &is_Unitig); + if(uId!=(uint32_t)-1 && is_Unitig == 1 && uId == target_uId) + { + max_count++; + break; + } + } + } + + (*Match) = max_count; + (*Total) = min_count; +} + void get_pair_hap_similarity(uint64_t* readIDs, uint32_t Len, uint32_t target_uId, ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, double* Match, double* Total) { - #define CUTOFF_THRES 100 + #define CUTOFF_THRES 1000 uint32_t i, j, qn, tn, is_Unitig, uId, min_count = 0, max_count = 0, cutoff = 0;; for (i = 0; i < Len; i++) { @@ -872,6 +981,7 @@ ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, double* Match, } qn = readIDs[i]>>33; if(reverse_sources[qn].length > 0) min_count++; + if(reverse_sources[qn].length == 0) continue; for (j = 0; j < reverse_sources[qn].length; j++) { tn = Get_tn(reverse_sources[qn].buffer[j]); @@ -1028,7 +1138,7 @@ uint64_t* position_index, ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U void determin_hap_alignment_boundary_single_side(uint64_t* readIDs, long long queryLen, long long targetBeg, long long targetEnd, long long targetID, long long eMatch, long long eTotal, long long dir, -float Hap_rate, uint64_t* position_index, ma_hit_t_alloc* reverse_sources, asg_t *read_g, +float H_rate, int is_local, uint64_t* position_index, ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, uint32_t* n_matchLen, uint32_t* n_max_count, uint32_t* n_min_count) { if(queryLen == 0) @@ -1037,19 +1147,27 @@ R_to_U* ruIndex, uint32_t* n_matchLen, uint32_t* n_max_count, uint32_t* n_min_co return; } long long i, maxId, min_count = eTotal, max_count = eMatch, matchLen = 0; + long long rLen, score = 0, max_score = 0; uint32_t is_found, is_match; if(dir == 0) { for (i = 0, maxId = 0; i < queryLen; i++) { + check_hap_match(readIDs[i]>>33, targetBeg, targetEnd, targetID, position_index, reverse_sources, read_g, ruIndex, &is_found, &is_match); min_count += is_found; max_count += is_match; - if(max_count > min_count*Hap_rate) maxId = i; - } + if(max_count > min_count*H_rate) maxId = i; + if(is_local && is_found) + { + rLen = read_g->seq[readIDs[i]>>33].len; + score += (is_match? rLen : (rLen*(-1))); + if(score >= max_score) max_score = score, maxId = i; + } + } for (i = maxId; i >= 0; i--) { @@ -1064,7 +1182,7 @@ R_to_U* ruIndex, uint32_t* n_matchLen, uint32_t* n_max_count, uint32_t* n_min_co min_count -= is_found; max_count -= is_match; } - + matchLen = i+1; } else @@ -1076,7 +1194,14 @@ R_to_U* ruIndex, uint32_t* n_matchLen, uint32_t* n_max_count, uint32_t* n_min_co min_count += is_found; max_count += is_match; - if(max_count > min_count*Hap_rate) maxId = i; + if(max_count > min_count*H_rate) maxId = i; + + if(is_local && is_found) + { + rLen = read_g->seq[readIDs[i]>>33].len; + score += (is_match? rLen : (rLen*(-1))); + if(score >= max_score) max_score = score, maxId = i; + } } for (i = maxId; i < queryLen; i++) @@ -1123,7 +1248,7 @@ long long* target_beg, long long* target_end) void bi_direction_hap_alignment_extention(ma_utg_t* xReads, uint32_t xLeftBeg, uint32_t xLeftLen, uint32_t xRightBeg, uint32_t xRightLen, uint32_t targetUid, uint32_t target_beg, uint32_t target_end, -float Hap_rate, uint64_t* position_index, ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, +float Hap_rate, int is_local, uint64_t* position_index, ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, uint32_t rev, long long* x_interval_beg, long long* x_interval_end) { if(rev) @@ -1136,27 +1261,27 @@ uint32_t rev, long long* x_interval_beg, long long* x_interval_end) uint32_t n_matchLenRight, x_max_countRight, x_min_countRight; n_matchLenLeft = x_max_countLeft = x_min_countLeft = 0; determin_hap_alignment_boundary_single_side(xReads->a+xLeftBeg, xLeftLen, - target_beg, target_end, targetUid, x_max_countLeft, x_min_countLeft, 1, Hap_rate, + target_beg, target_end, targetUid, x_max_countLeft, x_min_countLeft, 1, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLenLeft, &x_max_countLeft, &x_min_countLeft); n_matchLenRight = x_max_countRight = x_min_countRight = 0; determin_hap_alignment_boundary_single_side(xReads->a+xRightBeg, xRightLen, - target_beg, target_end, targetUid, x_max_countRight, x_min_countRight, 0, Hap_rate, + target_beg, target_end, targetUid, x_max_countRight, x_min_countRight, 0, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLenRight, &x_max_countRight, &x_min_countRight); if(x_max_countLeft >= x_max_countRight) { determin_hap_alignment_boundary_single_side(xReads->a+xRightBeg, xRightLen, - target_beg, target_end, targetUid, x_max_countLeft, x_min_countLeft, 0, Hap_rate, + target_beg, target_end, targetUid, x_max_countLeft, x_min_countLeft, 0, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLenRight, &x_max_countRight, &x_min_countRight); } else { determin_hap_alignment_boundary_single_side(xReads->a+xLeftBeg, xLeftLen, - target_beg, target_end, targetUid, x_max_countRight, x_min_countRight, 1, Hap_rate, + target_beg, target_end, targetUid, x_max_countRight, x_min_countRight, 1, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLenLeft, &x_max_countLeft, &x_min_countLeft); } @@ -1170,7 +1295,7 @@ uint32_t xLeftMatch, uint32_t xLeftTotal, uint32_t yLeftMatch, uint32_t yLeftTot uint32_t xRightMatch, uint32_t xRightTotal, uint32_t yRightMatch, uint32_t yRightTotal, uint32_t xLeftBeg, uint32_t xLeftLen, uint32_t yLeftBeg, uint32_t yLeftLen, uint32_t xRightBeg, uint32_t xRightLen, uint32_t yRightBeg, uint32_t yRightLen, -uint32_t xUid, uint32_t yUid, float Hap_rate, uint64_t* position_index, +uint32_t xUid, uint32_t yUid, float Hap_rate, int is_local, uint64_t* position_index, ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, uint32_t rev, long long* r_x_interval_beg, long long* r_x_interval_end, long long* r_y_interval_beg, long long* r_y_interval_end) @@ -1189,7 +1314,7 @@ long long* r_y_interval_beg, long long* r_y_interval_end) modify_target_interval(yLeftBeg, yLeftBeg+yLeftLen-1, yReads->n, &target_beg, &target_end); determin_hap_alignment_boundary_single_side(xReads->a+xLeftBeg, xLeftLen, /**yLeftBeg, yLeftBeg+yLeftLen-1,**/ target_beg, target_end, yUid, - x_max_count, x_min_count, 1, Hap_rate, position_index, reverse_sources, + x_max_count, x_min_count, 1, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLen, &x_max_count, &x_min_count); x_interval_beg = xLeftBeg + xLeftLen; x_interval_beg -= n_matchLen; @@ -1202,7 +1327,7 @@ long long* r_y_interval_beg, long long* r_y_interval_end) modify_target_interval(xRightBeg, xRightBeg+xRightLen-1, xReads->n, &target_beg, &target_end); determin_hap_alignment_boundary_single_side(yReads->a+yRightBeg, yRightLen, /**xRightBeg, xRightBeg+xRightLen-1,**/ target_beg, target_end, xUid, - y_max_count, y_min_count, rev, Hap_rate, position_index, reverse_sources, + y_max_count, y_min_count, rev, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLen, &y_max_count, &y_min_count); if(rev == 0) { @@ -1225,7 +1350,7 @@ long long* r_y_interval_beg, long long* r_y_interval_end) modify_target_interval(yRightBeg, yRightBeg+yRightLen-1, yReads->n, &target_beg, &target_end); determin_hap_alignment_boundary_single_side(xReads->a+xRightBeg, xRightLen, /**yRightBeg, yRightBeg+yRightLen-1,**/ target_beg, target_end, yUid, - x_max_count, x_min_count, 0, Hap_rate, position_index, reverse_sources, + x_max_count, x_min_count, 0, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLen, &x_max_count, &x_min_count); x_interval_beg = xLeftBeg; @@ -1238,7 +1363,7 @@ long long* r_y_interval_beg, long long* r_y_interval_end) modify_target_interval(xLeftBeg, xLeftBeg+xLeftLen-1, xReads->n, &target_beg, &target_end); determin_hap_alignment_boundary_single_side(yReads->a+yLeftBeg, yLeftLen, /**xLeftBeg, xLeftBeg+xLeftLen-1,**/ target_beg, target_end, xUid, - y_max_count, y_min_count, 1-rev, Hap_rate, position_index, reverse_sources, + y_max_count, y_min_count, 1-rev, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, &n_matchLen, &y_max_count, &y_min_count); if(rev == 0) { @@ -1256,7 +1381,7 @@ long long* r_y_interval_beg, long long* r_y_interval_end) { /********************x*********************/ bi_direction_hap_alignment_extention(xReads, xLeftBeg, xLeftLen, xRightBeg, xRightLen, - yUid, 0, yReads->n - 1, Hap_rate, position_index, reverse_sources, read_g, ruIndex, 0, + yUid, 0, yReads->n - 1, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, 0, &x_interval_beg, &x_interval_end); /********************x*********************/ @@ -1274,7 +1399,7 @@ long long* r_y_interval_beg, long long* r_y_interval_end) /********************y*********************/ bi_direction_hap_alignment_extention(yReads, yLeftBeg, yLeftLen, yRightBeg, yRightLen, - xUid, 0, xReads->n - 1, Hap_rate, position_index, reverse_sources, read_g, ruIndex, rev, + xUid, 0, xReads->n - 1, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, rev, &y_interval_beg, &y_interval_end); /********************y*********************/ } else abort(); @@ -1506,7 +1631,7 @@ void quick_LIS(asg_arc_t_offset* x, uint32_t n, kvec_t_i32_warp* tailIndex, kvec if(Get_yOff(x[i].Off) < Get_yOff(x[tailIndex->a.a[0]].Off)) { // new smallest value - tailIndex->a.a[0] = i; + tailIndex->a.a[0] = i; ///doesn't matter too much } else if(Get_yOff(x[i].Off) > Get_yOff(x[tailIndex->a.a[len - 1]].Off)) { @@ -1534,7 +1659,309 @@ void quick_LIS(asg_arc_t_offset* x, uint32_t n, kvec_t_i32_warp* tailIndex, kvec tailIndex->a.n = len; } -void get_base_boundary_advance(R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, +inline uint64_t get_xy_pos_by_pos(asg_t *read_g, asg_arc_t* t, uint32_t v_in_unitig, uint32_t w_in_unitig, +uint32_t v_in_pos, uint32_t w_in_pos, uint32_t xUnitigLen, uint32_t yUnitigLen, uint8_t* rev) +{ + uint32_t x_pos, y_pos, x_dir = 0, y_dir = 0; + uint64_t tmp; + x_pos = y_pos = (uint32_t)-1; + if((t->ul>>32)==v_in_unitig)///end pos + { + x_pos = v_in_pos + read_g->seq[v_in_unitig>>1].len - 1; + x_dir = 0; + } + else if((t->ul>>32)==(v_in_unitig^1))///start pos + { + x_pos = v_in_pos; + x_dir = 1; + } + else + { + fprintf(stderr, "ERROR\n"); + } + + if(t->v == w_in_unitig) + { + y_pos = w_in_pos + t->ol - 1; + y_dir = 0; + } + else if(t->v == (w_in_unitig^1)) + { + y_pos = w_in_pos + read_g->seq[w_in_unitig>>1].len - t->ol; + y_dir = 1; + } + else + { + fprintf(stderr, "ERROR\n"); + } + + (*rev) = x_dir^y_dir; + if((*rev)) + { + if(yUnitigLen <= y_pos) + { + y_pos = (uint32_t)-1; + } + else + { + y_pos = yUnitigLen - y_pos - 1; + } + } + + if(x_pos>=xUnitigLen) x_pos = (uint32_t)-1; + if(y_pos>=yUnitigLen) y_pos = (uint32_t)-1; + + tmp = x_pos; tmp = tmp << 32; tmp = tmp | y_pos; + return tmp; +} + +void chain_trans_ovlp(hap_cov_t *cov, ma_ug_t *ug, asg_t *read_sg, buf_t* xReads, uint32_t targetBaseLen, uint32_t* xEnd) +{ + ma_hit_t_alloc* reverse_sources = cov->reverse_sources; + ma_sub_t *coverage_cut = cov->coverage_cut; + int max_hang = cov->max_hang; + int min_ovlp = cov->min_ovlp; + kvec_asg_arc_t_offset* u_buffer = &(cov->u_buffer); + kvec_t_i32_warp* tailIndex = &(cov->tailIndex); + kvec_t_i32_warp* prevIndex = &(cov->prevIndex); + ma_hit_t_alloc *xR = NULL; + ma_hit_t *h = NULL; + ma_sub_t *sq = NULL, *st = NULL; + int32_t r; + asg_arc_t t; + uint32_t rId, v, w; + uint64_t tmp; + asg_arc_t_offset t_offset; + u_buffer->a.n = 0; + (*xEnd) = (uint32_t)-1; + uint32_t u_i, r_i, k, j, m, len, p_v, *a = xReads->b.a, uid, ori, l, aOcc, nv, xOcc = (uint32_t)-1; + ma_utg_t* u = NULL; + asg_arc_t *av = NULL; + + + for (u_i = r_i = len = aOcc = 0, xOcc = (uint32_t)-1, p_v = (uint32_t)-1; u_i < xReads->b.n; u_i++) + { + uid = a[u_i] >> 1; + ori = a[u_i] & 1; + u = &(ug->u.a[uid]); + if(u->n == 0) continue; + + for (r_i = 0; r_i < u->n; r_i++, aOcc++) + { + l = 0; + v = (ori == 1?((uint64_t)((u->a[u->n - r_i - 1])^(uint64_t)(0x100000000)))>>32:((uint64_t)(u->a[r_i]))>>32); + + if(p_v != (uint32_t)-1) + { + av = asg_arc_a(read_sg, p_v); + nv = asg_arc_n(read_sg, p_v); + + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + if(av[k].v == v) + { + l = asg_arc_len(av[k]); + break; + } + } + if(k == nv) fprintf(stderr, "ERROR\n"); + } + + p_v = v; len += l; + if(len >= targetBaseLen) + { + xOcc = aOcc; + break; + } + } + + if(xOcc != (uint32_t)-1) break; + } + if(xOcc == (uint32_t)-1) xOcc = aOcc; + if(xOcc == 0) xOcc = 1; + + + + for (u_i = r_i = len = aOcc = 0, p_v = (uint32_t)-1; u_i < xReads->b.n; u_i++) + { + uid = a[u_i] >> 1; + ori = a[u_i] & 1; + u = &(ug->u.a[uid]); + if(u->n == 0) continue; + + for (r_i = 0; r_i < u->n; r_i++, aOcc++) + { + if(aOcc >= xOcc) break; + l = 0; + v = (ori == 1?((uint64_t)((u->a[u->n - r_i - 1])^(uint64_t)(0x100000000)))>>32:((uint64_t)(u->a[r_i]))>>32); + + if(p_v != (uint32_t)-1) + { + av = asg_arc_a(read_sg, p_v); + nv = asg_arc_n(read_sg, p_v); + + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + if(av[k].v == v) + { + l = asg_arc_len(av[k]); + break; + } + } + if(k == nv) fprintf(stderr, "ERROR\n"); + } + + p_v = v; len += l; + + xR = &(reverse_sources[v>>1]); + for (j = 0; j < xR->length; j++) + { + h = &(xR->buffer[j]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + if(st->del || read_sg->seq[Get_tn(*h)].del) continue; + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, &t); + ///if it is a contained overlap, skip + if(r < 0) continue; + + rId = t.v>>1; + if(read_sg->seq[rId].del == 1) continue; + if(cov->pos_idx[rId] == (uint64_t)-1) continue; + w = (uint32_t)(cov->pos_idx[rId]); + if(rId != (w>>1)) continue; + + tmp = get_xy_pos_by_pos(read_sg, &t, v, w, len, cov->pos_idx[w>>1]>>32, + (uint32_t)-1, targetBaseLen, &(t.el)); + if(((tmp>>32) == (uint32_t)-1) || (((uint32_t)tmp) == (uint32_t)-1)) continue; + if(t.el) continue; ///must + + t_offset.Off = tmp; + t_offset.x = t; + t_offset.weight = 1; + kv_push(asg_arc_t_offset, u_buffer->a, t_offset); + } + } + + if(aOcc >= xOcc) break; + } + + if(u_buffer->a.n == 0) return; + + qsort(u_buffer->a.a, u_buffer->a.n, sizeof(asg_arc_t_offset), cmp_hap_alignment_chaining); + + ///print_asg_arc_t_offset(u_buffer->a.a, u_buffer->a.n, "before"); + + + for (k = 1, l = 0, m = 0; k <= u_buffer->a.n; ++k) + { + if (k == u_buffer->a.n || u_buffer->a.a[k].x.el != u_buffer->a.a[l].x.el || + u_buffer->a.a[k].Off != u_buffer->a.a[l].Off) + { + u_buffer->a.a[m] = u_buffer->a.a[l]; + for (l += 1; l < k; l++) + { + u_buffer->a.a[m].weight += u_buffer->a.a[l].weight; + if(u_buffer->a.a[l].x.ol > u_buffer->a.a[m].x.ol) + { + u_buffer->a.a[m].x = u_buffer->a.a[l].x; + } + } + l = k; + m++; + } + } + u_buffer->a.n = m; + + ///print_asg_arc_t_offset(u_buffer->a.a, u_buffer->a.n, "after"); + quick_LIS(u_buffer->a.a, u_buffer->a.n, tailIndex, prevIndex); + if(tailIndex->a.n == 0) return; + + + uint32_t xLen_thres = (uint32_t)-1; + asg_arc_t_offset* best = &(u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]]); + for (u_i = r_i = len = aOcc = 0, p_v = (uint32_t)-1; u_i < xReads->b.n; u_i++) + { + uid = a[u_i] >> 1; + ori = a[u_i] & 1; + u = &(ug->u.a[uid]); + if(u->n == 0) continue; + + for (r_i = 0; r_i < u->n; r_i++, aOcc++) + { + l = 0; + v = (ori == 1?((uint64_t)((u->a[u->n - r_i - 1])^(uint64_t)(0x100000000)))>>32:((uint64_t)(u->a[r_i]))>>32); + + if(p_v != (uint32_t)-1) + { + av = asg_arc_a(read_sg, p_v); + nv = asg_arc_n(read_sg, p_v); + + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + if(av[k].v == v) + { + l = asg_arc_len(av[k]); + break; + } + } + if(k == nv) fprintf(stderr, "ERROR\n"); + } + + p_v = v; len += l; + + if((v>>1) == (best->x.ul>>33) && xLen_thres == (uint32_t)-1) + { + ///cov->pos_idx[v>>1] = len; + xR = &(reverse_sources[v>>1]); + for (j = 0; j < xR->length; j++) + { + h = &(xR->buffer[j]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + if(st->del || read_sg->seq[Get_tn(*h)].del) continue; + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, &t); + ///if it is a contained overlap, skip + if(r < 0) continue; + + rId = t.v>>1; + if(read_sg->seq[rId].del == 1) continue; + if(cov->pos_idx[rId] == (uint64_t)-1) continue; + w = (uint32_t)(cov->pos_idx[rId]); + if(rId != (w>>1)) continue; + + tmp = get_xy_pos_by_pos(read_sg, &t, v, w, len, cov->pos_idx[w>>1]>>32, + (uint32_t)-1, targetBaseLen, &(t.el)); + if(((tmp>>32) == (uint32_t)-1) || (((uint32_t)tmp) == (uint32_t)-1)) continue; + if(t.el) continue; ///must + + t_offset.Off = tmp; + t_offset.x = t; + t_offset.weight = 1; + if(t_offset.Off == best->Off && t_offset.x.v == best->x.v && t_offset.x.ul == best->x.ul) + { + xLen_thres = Get_xOff(best->Off) + targetBaseLen - Get_yOff(best->Off); + } + } + } + + if(len >= xLen_thres) + { + (*xEnd) = aOcc; + return; + } + } + } + + (*xEnd) = aOcc; +} + + +void get_base_boundary_advance_back(R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, int max_hang, int min_ovlp, ma_utg_t *xReads, ma_utg_t *yReads, uint32_t xUid, uint32_t yUid, long long xBegIndex, long long xEndIndex, long long yBegIndex, long long yEndIndex, uint32_t rev, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex, @@ -1635,15 +2062,14 @@ uint32_t* xBeg, uint32_t* xEnd, uint32_t* yBeg, uint32_t* yEnd) (*yEnd) = Get_yOff(u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].Off); } - -uint32_t determine_hap_overlap_type_advance(hap_candidates* hap_can, ma_utg_t *xReads, ma_utg_t *yReads, +uint32_t determine_hap_overlap_type_advance_back(hap_candidates* hap_can, ma_utg_t *xReads, ma_utg_t *yReads, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, int max_hang, int min_ovlp, uint32_t xUid, uint32_t yUid, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex, long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long long* r_y_pos_end) { uint32_t x_pos_beg, y_pos_beg, x_pos_end, y_pos_end; /*************************x***************************/ - get_base_boundary_advance(ruIndex, reverse_sources, coverage_cut, read_g, position_index, + get_base_boundary_advance_back(ruIndex, reverse_sources, coverage_cut, read_g, position_index, max_hang, min_ovlp, xReads, yReads, xUid, yUid, Get_x_beg(*hap_can), Get_x_end(*hap_can), Get_y_beg(*hap_can), Get_y_end(*hap_can), Get_rev(*hap_can), u_buffer, tailIndex, prevIndex, &x_pos_beg, &x_pos_end, &y_pos_beg, &y_pos_end); @@ -1666,6 +2092,321 @@ kvec_t_i32_warp* prevIndex, long long* r_x_pos_beg, long long* r_x_pos_end, long r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end); } +void get_idx_by_base(ma_utg_t *x, asg_t *read_g, long long beg_base, long long end_base, + long long* beg_idx, long long* end_idx) +{ + long long offset, r_beg, r_end; + uint64_t i, rId; + (*beg_idx) = (*end_idx) = -1; + for (i = 0, offset = 0; i < x->n; i++) + { + rId = x->a[i]>>33; + r_beg = offset; r_end = offset + (long long)(read_g->seq[rId].len) - 1; + offset += (uint32_t)x->a[i]; + if(beg_base > r_end || r_beg > end_base) + { + if((*beg_idx) != -1 && (*end_idx) != -1) break; + continue; + } + if((*beg_idx) == -1) (*beg_idx) = i; + (*end_idx) = i; + } +} + +int get_base_boundary_chain(R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, +asg_t *read_g, uint64_t* position_index, int max_hang, int min_ovlp, ma_utg_t *xReads, ma_utg_t *yReads, +uint32_t xUid, uint32_t yUid, long long xBegIndex, long long xEndIndex, long long yBegIndex, long long yEndIndex, +uint32_t rev, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex) +{ + long long k, j, l, offset, m; + ma_hit_t_alloc *xR = NULL; + ma_hit_t *h = NULL; + ma_sub_t *sq = NULL, *st = NULL; + int32_t r; + asg_arc_t t; + uint32_t rId, Hap_uId, is_Unitig, v, w, v_dir, w_dir; + uint64_t tmp; + asg_arc_t_offset t_offset; + u_buffer->a.n = 0; + for (k = xBegIndex; k <= xEndIndex; k++) + { + xR = &(reverse_sources[xReads->a[k]>>33]); + for (j = 0; j < xR->length; j++) + { + h = &(xR->buffer[j]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + if(st->del || read_g->seq[Get_tn(*h)].del) continue; + + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, &t); + ///if it is a contained overlap, skip + if(r < 0) continue; + + rId = t.v>>1; + if(read_g->seq[rId].del == 1) continue; + ///there are two cases: + ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id + ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 + get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); + if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; + if(Hap_uId != yUid) continue; + + v = xReads->a[k]>>32; + get_R_to_U(ruIndex, v>>1, &Hap_uId, &is_Unitig); + if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; + if(Hap_uId != xUid) continue; + if((uint32_t)(position_index[v>>1]) != k) continue; + + w = (yReads->a[(uint32_t)(position_index[rId])])>>32; + + v_dir = ((t.ul>>32)==v)?1:0; + w_dir = (t.v == w)?1:0; + if(rev == 0 && v_dir != w_dir) continue; + if(rev == 1 && v_dir == w_dir) continue; + + /****************************may have bugs********************************/ + offset = (uint32_t)(position_index[rId]); + if(offset < yBegIndex || offset > yEndIndex) continue; + /****************************may have bugs********************************/ + + tmp = get_xy_pos(read_g, &t, v, w, xReads->len, yReads->len, position_index, &(t.el)); + if(((tmp>>32) == (uint32_t)-1) || (((uint32_t)tmp) == (uint32_t)-1)) continue; + + t_offset.Off = tmp; + t_offset.x = t; + t_offset.weight = 1; + kv_push(asg_arc_t_offset, u_buffer->a, t_offset); + } + } + if(u_buffer->a.n == 0) return 0; + + qsort(u_buffer->a.a, u_buffer->a.n, sizeof(asg_arc_t_offset), cmp_hap_alignment_chaining); + + ///print_asg_arc_t_offset(u_buffer->a.a, u_buffer->a.n, "before"); + for (k = 1, l = 0, m = 0; k <= (long long)u_buffer->a.n; ++k) + { + if (k == (long long)u_buffer->a.n || u_buffer->a.a[k].Off != u_buffer->a.a[l].Off) + { + u_buffer->a.a[m] = u_buffer->a.a[l]; + for (l += 1; l < k; l++) + { + u_buffer->a.a[m].weight += u_buffer->a.a[l].weight; + if(u_buffer->a.a[l].x.ol > u_buffer->a.a[m].x.ol) + { + u_buffer->a.a[m].x = u_buffer->a.a[l].x; + } + } + l = k; + m++; + } + } + u_buffer->a.n = m; + + ///print_asg_arc_t_offset(u_buffer->a.a, u_buffer->a.n, "after"); + quick_LIS(u_buffer->a.a, u_buffer->a.n, tailIndex, prevIndex); + + if(tailIndex->a.n == 0) return 0; + return 1; +} + + +void get_base_boundary_advance(R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, +asg_t *read_g, uint64_t* position_index, int max_hang, int min_ovlp, ma_utg_t *xReads, ma_utg_t *yReads, +uint32_t xUid, uint32_t yUid, long long xBegIndex, long long xEndIndex, long long yBegIndex, long long yEndIndex, +uint32_t rev, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex, +uint32_t* xBeg, uint32_t* xEnd, uint32_t* yBeg, uint32_t* yEnd) +{ + long long offset; + long long new_xBeg, new_yBeg, new_xEnd, new_yEnd; + long long new_xIdxBeg, new_yIdxBeg, new_xIdxEnd, new_yIdxEnd; + + (*xBeg) = (*xEnd) = (*yBeg) = (*yEnd) = (uint32_t)-1; + if(!get_base_boundary_chain(ruIndex, reverse_sources, coverage_cut, read_g, position_index, + max_hang, min_ovlp, xReads, yReads, xUid, yUid, xBegIndex, xEndIndex, yBegIndex, yEndIndex, + rev, u_buffer, tailIndex, prevIndex)) + { + return; + } + ///base + new_xBeg = Get_xOff(u_buffer->a.a[tailIndex->a.a[0]].Off); + new_yBeg = Get_yOff(u_buffer->a.a[tailIndex->a.a[0]].Off); + new_xEnd = Get_xOff(u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].Off); + new_yEnd = Get_yOff(u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].Off); + + if(new_xBeg > new_xEnd || new_yBeg > new_yEnd) return; + + classify_hap_overlap(new_xBeg, new_xEnd, xReads->len, new_yBeg, new_yEnd, yReads->len, + &new_xBeg, &new_xEnd, &new_yBeg, &new_yEnd); + + if(rev) + { + new_yBeg = yReads->len - new_yBeg - 1; + new_yEnd = yReads->len - new_yEnd - 1; + offset = new_yBeg; new_yBeg = new_yEnd; new_yEnd = offset; + } + ///idx + get_idx_by_base(xReads, read_g, new_xBeg, new_xEnd, &new_xIdxBeg, &new_xIdxEnd); + get_idx_by_base(yReads, read_g, new_yBeg, new_yEnd, &new_yIdxBeg, &new_yIdxEnd); + if(new_xIdxBeg == -1 || new_xIdxEnd == -1 || new_yIdxBeg == -1 || new_yIdxEnd == -1) return; + + if(!get_base_boundary_chain(ruIndex, reverse_sources, coverage_cut, read_g, position_index, + max_hang, min_ovlp, xReads, yReads, xUid, yUid, new_xIdxBeg, new_xIdxEnd, new_yIdxBeg, + new_yIdxEnd, rev, u_buffer, tailIndex, prevIndex)) + { + return; + } + + (*xBeg) = Get_xOff(u_buffer->a.a[tailIndex->a.a[0]].Off); + (*yBeg) = Get_yOff(u_buffer->a.a[tailIndex->a.a[0]].Off); + (*xEnd) = Get_xOff(u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].Off); + (*yEnd) = Get_yOff(u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].Off); +} + +#define generic_key(x) (x) +KRADIX_SORT_INIT(i32, int32_t, generic_key, sizeof(int32_t)) + +long long get_chain_score(ma_utg_t *xReads, asg_t *read_g, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* idx, +ma_hit_t_alloc* reverse_sources, long long xBegPos, long long xEndPos) +{ + long long offset, r_beg, r_end, inp_beg, inp_end, hap_beg, hap_end, inp_match, hap_match, ovlp; + uint64_t i, k, rId; + idx->a.n = 0; + + for (i = k = 0; i < tailIndex->a.n; i++) + { + rId = u_buffer->a.a[tailIndex->a.a[i]].x.ul>>33; + + + for (; k < xReads->n; k++) + { + if(rId == (xReads->a[k]>>33)) break; + } + + if(k >= xReads->n) + { + for (k = 0; k < xReads->n; k++) + { + if(rId == (xReads->a[k]>>33)) break; + } + } + + if(k < xReads->n) kv_push(int32_t, idx->a, k); + else + { + fprintf(stderr, "\nERROR-get_chain_score: tailIndex->a.n: %lu, xReads->n: %lu\n", (uint64_t)tailIndex->a.n, (uint64_t)xReads->n); + } + } + + + radix_sort_i32(idx->a.a, idx->a.a + idx->a.n); + inp_beg = -1; inp_end = -2; + hap_beg = -1; hap_end = -2; + for (i = k = 0, offset = 0, inp_match = hap_match = 0; i < xReads->n; i++) + { + rId = xReads->a[i]>>33; + r_beg = offset; r_end = offset + (long long)(read_g->seq[rId].len) - 1; + offset += (uint32_t)xReads->a[i]; + + if(reverse_sources[rId].length > 0) + { + if(r_beg <= hap_end) + { + hap_end = MAX(hap_end, r_end); + } + else + { + ///match += (hap_end - hap_beg + 1); + ovlp = (long long)(MIN(hap_end, xEndPos)) - (long long)(MAX(hap_beg, xBegPos)) + 1; + hap_match += (ovlp >= 0? ovlp : 0); + hap_beg = r_beg; hap_end = r_end; + } + } + + for (; k < idx->a.n; k++) + { + if(i <= (uint64_t)idx->a.a[k]) break; + } + + if(k >= idx->a.n) continue; + + if(i == (uint64_t)idx->a.a[k]) + { + if(r_beg <= inp_end) + { + inp_end = MAX(inp_end, r_end); + } + else + { + ovlp = (long long)(MIN(inp_end, xEndPos)) - (long long)(MAX(inp_beg, xBegPos)) + 1; + inp_match += (ovlp >= 0? ovlp : 0); + inp_beg = r_beg; inp_end = r_end; + } + } + } + + ovlp = (long long)(MIN(inp_end, xEndPos)) - (long long)(MAX(inp_beg, xBegPos)) + 1; + inp_match += (ovlp >= 0? ovlp : 0); + + ovlp = (long long)(MIN(hap_end, xEndPos)) - (long long)(MAX(hap_beg, xBegPos)) + 1; + hap_match += (ovlp >= 0? ovlp : 0); + + // if(inp_match > (xEndPos - xBegPos + 1)) fprintf(stderr, "ERROR1\n"); + // if(hap_match > (xEndPos - xBegPos + 1)) fprintf(stderr, "ERRO2\n"); + // if(inp_match > hap_match) fprintf(stderr, "ERROR3\n"); + // fprintf(stderr, "tailIndex->a.n: %u, xReads->n: %u, total_match: %lld, hap_match: %lld, inp_match: %lld\n", + // tailIndex->a.n, xReads->n, (xEndPos - xBegPos + 1), hap_match, inp_match); + + return (inp_match*3) - ((hap_match-inp_match)*1); +} + + +uint32_t determine_hap_overlap_type_advance(hap_candidates* hap_can, ma_utg_t *xReads, ma_utg_t *yReads, +R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, +int max_hang, int min_ovlp, uint32_t xUid, uint32_t yUid, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, +kvec_t_i32_warp* prevIndex, long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long long* r_y_pos_end) +{ + uint32_t x_pos_beg, y_pos_beg, x_pos_end, y_pos_end; + /*************************x***************************/ + get_base_boundary_advance(ruIndex, reverse_sources, coverage_cut, read_g, position_index, + max_hang, min_ovlp, xReads, yReads, xUid, yUid, Get_x_beg(*hap_can), Get_x_end(*hap_can), + Get_y_beg(*hap_can), Get_y_end(*hap_can), Get_rev(*hap_can), u_buffer, tailIndex, prevIndex, + &x_pos_beg, &x_pos_end, &y_pos_beg, &y_pos_end); + /*************************x***************************/ + if(x_pos_beg == (uint32_t)-1 || y_pos_beg == (uint32_t)-1 + || x_pos_end == (uint32_t)-1 || y_pos_end == (uint32_t)-1) + { + return (uint32_t)-1; + } + if(x_pos_beg > x_pos_end || y_pos_beg > y_pos_end) return (uint32_t)-1; + + /** + #define X2Y 0 + #define Y2X 1 + #define XCY 2 + #define YCX 3 + **/ + hap_can->index_end = classify_hap_overlap(x_pos_beg, x_pos_end, xReads->len, y_pos_beg, y_pos_end, yReads->len, + r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end); + hap_can->x_beg_pos = MIN((uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[0]].x.ul>>33]), + (uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].x.ul>>33])); + hap_can->x_end_pos = MAX((uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[0]].x.ul>>33]), + (uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].x.ul>>33])); + hap_can->y_beg_pos = MIN((uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[0]].x.v>>1]), + (uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].x.v>>1])); + hap_can->y_end_pos = MAX((uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[0]].x.v>>1]), + (uint32_t)(position_index[u_buffer->a.a[tailIndex->a.a[tailIndex->a.n-1]].x.v>>1])); + double xLeftMatch, xLeftTotal; + get_pair_hap_similarity(xReads->a + hap_can->x_beg_pos, hap_can->x_end_pos + 1 - hap_can->x_beg_pos, + yUid, reverse_sources, read_g, ruIndex, &xLeftMatch, &xLeftTotal); + if(xLeftMatch == 0 || xLeftTotal == 0) return (uint32_t)-1; + hap_can->weight = xLeftMatch; + hap_can->index_beg = xLeftTotal; + hap_can->score = get_chain_score(xReads, read_g, u_buffer, tailIndex, prevIndex, reverse_sources, + (*r_x_pos_beg), (*r_x_pos_end)); + return hap_can->index_end; +} + uint32_t determine_hap_overlap_type(hap_candidates* hap_can, ma_utg_t *xReads, ma_utg_t *yReads, @@ -1702,133 +2443,12 @@ long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long lon } -uint32_t calculate_pair_hap_similarity(kvec_asg_arc_t_offset* u_buffer, hap_candidates* hap_can, -uint64_t* position_index, uint32_t xUid, uint32_t yUid, ma_utg_t* xReads, ma_utg_t* yReads, -ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, ma_sub_t *coverage_cut, -float Hap_rate, int max_hang, int min_ovlp, long long* r_x_pos_beg, long long* r_x_pos_end, -long long* r_y_pos_beg, long long* r_y_pos_end) -{ - uint32_t max_count = 0, min_count = 0, i, flag; - uint32_t xLen = xReads->n, xIndex/**, xBasePos**/; - uint32_t yLen = yReads->n, yIndex/**, yBasePos**/; - uint32_t xLeftBeg, xLeftLen, yLeftBeg, yLeftLen; - uint32_t xRightBeg, xRightLen, yRightBeg, yRightLen; - uint64_t totalWeigth; - double xLeftMatch = 0, xLeftTotal = 0, yLeftMatch = 0, yLeftTotal = 0; - double xRightMatch = 0, xRightTotal = 0, yRightMatch = 0, yRightTotal = 0; - asg_arc_t_offset* arch = NULL; - - for (i = hap_can->index_beg, totalWeigth = 0; i <= hap_can->index_end; i++) - { - totalWeigth += u_buffer->a.a[i].weight; - if(totalWeigth >= (hap_can->weight/2)) break; - } - if(i > hap_can->index_end) i = hap_can->index_end; - - arch = &(u_buffer->a.a[i]); - xIndex = (uint32_t)(position_index[arch->x.ul>>33]); - yIndex = (uint32_t)(position_index[arch->x.v>>1]); - ///xBasePos = (uint32_t)(arch->Off>>32); - ///yBasePos = (uint32_t)(arch->Off); - - if(hap_can->rev == 0) - { - xLeftBeg = 0; xLeftLen = xIndex; xRightBeg = xIndex; xRightLen = xLen - xRightBeg; - yLeftBeg = 0; yLeftLen = yIndex; yRightBeg = yIndex; yRightLen = yLen - yRightBeg; - } - else - { - xLeftBeg = 0; xLeftLen = xIndex; xRightBeg = xIndex; xRightLen = xLen - xRightBeg; - - yLeftBeg = yIndex + 1; yLeftLen = yLen - yLeftBeg; - yRightBeg = 0; yRightLen = yIndex + 1; - } - - ///flag = classify_hap_overlap(xBasePos, xBasePos, xReads->len, yBasePos, yBasePos, yReads->len); - flag = vote_overlap_type(u_buffer, hap_can, position_index, xReads, yReads); - - - if(flag == XCY) - { - get_pair_hap_similarity(yReads->a, yLen, xUid, reverse_sources, read_g, ruIndex, - &yLeftMatch, &yLeftTotal); - max_count = yLeftMatch; - min_count = yLeftTotal; - } - else if(flag == YCX) - { - get_pair_hap_similarity(xReads->a, xLen, yUid, reverse_sources, read_g, ruIndex, - &xLeftMatch, &xLeftTotal); - max_count = xLeftMatch; - min_count = xLeftTotal; - } - else if(flag == X2Y) - { - get_pair_hap_similarity(yReads->a+yLeftBeg, yLeftLen, xUid, reverse_sources, read_g, ruIndex, - &yLeftMatch, &yLeftTotal); - get_pair_hap_similarity(xReads->a+xRightBeg, xRightLen, yUid, reverse_sources, read_g, ruIndex, - &xRightMatch, &xRightTotal); - max_count = yLeftMatch + xRightMatch; - min_count = yLeftTotal + xRightTotal; - } - else if(flag == Y2X) - { - get_pair_hap_similarity(xReads->a+xLeftBeg, xLeftLen, yUid, reverse_sources, read_g, ruIndex, - &xLeftMatch, &xLeftTotal); - get_pair_hap_similarity(yReads->a+yRightBeg, yRightLen, xUid, reverse_sources, read_g, ruIndex, - &yRightMatch, &yRightTotal); - max_count = xLeftMatch + yRightMatch; - min_count = xLeftTotal + yRightTotal; - } else abort(); - - hap_can->weight = hap_can->index_beg = 0; - if(min_count == 0) return NON_PLOID; - if(max_count > min_count*Hap_rate) - { - long long r_x_interval_beg, r_x_interval_end, r_y_interval_beg, r_y_interval_end; - - ///for containment, don't need to do anything - get_hap_alignment_boundary(xReads, yReads, flag, xLeftMatch, xLeftTotal, - yLeftMatch, yLeftTotal, xRightMatch, xRightTotal, yRightMatch, yRightTotal, - xLeftBeg, xLeftLen, yLeftBeg, yLeftLen, xRightBeg, xRightLen, yRightBeg, yRightLen, - xUid, yUid, Hap_rate, position_index, reverse_sources, read_g, ruIndex, hap_can->rev, - &r_x_interval_beg, &r_x_interval_end, &r_y_interval_beg, &r_y_interval_end); - - if(r_x_interval_beg < 0 || r_x_interval_end < 0 || r_y_interval_beg < 0 || r_y_interval_end < 0) - { - return NON_PLOID; - } - - get_pair_hap_similarity(xReads->a + r_x_interval_beg, r_x_interval_end + 1 - r_x_interval_beg, - yUid, reverse_sources, read_g, ruIndex, &xLeftMatch, &xLeftTotal); - if(xLeftMatch == 0 || xLeftTotal == 0) return NON_PLOID; - - hap_can->weight = xLeftMatch; - hap_can->index_beg = xLeftTotal; - hap_can->index_end = flag; - hap_can->x_beg_pos = r_x_interval_beg; - hap_can->x_end_pos = r_x_interval_end; - hap_can->y_beg_pos = r_y_interval_beg; - hap_can->y_end_pos = r_y_interval_end; - - hap_can->index_end = determine_hap_overlap_type(hap_can, xReads, yReads, ruIndex, - reverse_sources, coverage_cut, read_g, position_index, max_hang, min_ovlp, xUid, - yUid, r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end); - if(hap_can->index_end == XCY && yReads->len > (xReads->len*2)) return NON_PLOID; - if(hap_can->index_end == YCX && xReads->len > (yReads->len*2)) return NON_PLOID; - if(hap_can->index_end == (uint32_t)-1) return NON_PLOID; - - return PLOID; - } - return NON_PLOID; -} - - uint32_t calculate_pair_hap_similarity_advance(hap_candidates* hap_can, uint64_t* position_index, uint32_t xUid, uint32_t yUid, ma_utg_t* xReads, ma_utg_t* yReads, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, asg_t *read_g, R_to_U* ruIndex, ma_sub_t *coverage_cut, -float Hap_rate, int max_hang, int min_ovlp, uint64_t cov_threshold, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex, -long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long long* r_y_pos_end) +float Hap_rate, int is_local, int max_hang, int min_ovlp, uint64_t cov_threshold, kvec_asg_arc_t_offset* u_buffer, +kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex, hap_cov_t *cov, long long* r_x_pos_beg, long long* r_x_pos_end, +long long* r_y_pos_beg, long long* r_y_pos_end) { uint32_t max_count = 0, min_count = 0, flag; uint32_t xLen = xReads->n, xIndex; @@ -1895,7 +2515,7 @@ long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long lon hap_can->weight = hap_can->index_beg = 0; if(min_count == 0) return NON_PLOID; - if(max_count > min_count*Hap_rate) + if((max_count > min_count*Hap_rate) || is_local) { long long r_x_interval_beg, r_x_interval_end, r_y_interval_beg, r_y_interval_end; uint64_t ploid_coverage = 0; @@ -1904,8 +2524,8 @@ long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long lon get_hap_alignment_boundary(xReads, yReads, flag, xLeftMatch, xLeftTotal, yLeftMatch, yLeftTotal, xRightMatch, xRightTotal, yRightMatch, yRightTotal, xLeftBeg, xLeftLen, yLeftBeg, yLeftLen, xRightBeg, xRightLen, yRightBeg, yRightLen, - xUid, yUid, Hap_rate, position_index, reverse_sources, read_g, ruIndex, hap_can->rev, - &r_x_interval_beg, &r_x_interval_end, &r_y_interval_beg, &r_y_interval_end); + xUid, yUid, Hap_rate, is_local, position_index, reverse_sources, read_g, ruIndex, + hap_can->rev, &r_x_interval_beg, &r_x_interval_end, &r_y_interval_beg, &r_y_interval_end); if(r_x_interval_beg < 0 || r_x_interval_end < 0 || r_y_interval_beg < 0 || r_y_interval_end < 0) { @@ -1914,7 +2534,11 @@ long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long lon get_pair_hap_similarity(xReads->a + r_x_interval_beg, r_x_interval_end + 1 - r_x_interval_beg, yUid, reverse_sources, read_g, ruIndex, &xLeftMatch, &xLeftTotal); - if(xLeftMatch == 0 || xLeftTotal == 0) return NON_PLOID; + if(xLeftMatch == 0 || xLeftTotal == 0 || (is_local == 0 && xLeftMatch <= xLeftTotal*Hap_rate)) + { + return NON_PLOID; + } + hap_can->weight = xLeftMatch; hap_can->index_beg = xLeftTotal; @@ -1923,25 +2547,26 @@ long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long lon hap_can->x_end_pos = r_x_interval_end; hap_can->y_beg_pos = r_y_interval_beg; hap_can->y_end_pos = r_y_interval_end; - hap_can->index_end = determine_hap_overlap_type_advance(hap_can, xReads, yReads, ruIndex, reverse_sources, coverage_cut, read_g, position_index, max_hang, min_ovlp, xUid, yUid, u_buffer, tailIndex, prevIndex, r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end); - if(hap_can->index_end == XCY && yReads->len > (xReads->len*2)) return NON_PLOID; if(hap_can->index_end == YCX && xReads->len > (yReads->len*2)) return NON_PLOID; if(hap_can->index_end == (uint32_t)-1) return NON_PLOID; - ploid_coverage = 0; - ploid_coverage += get_pair_hap_coverage(xReads->a+r_x_interval_beg, r_x_interval_end+1-r_x_interval_beg, - sources, coverage_cut); - ploid_coverage += get_pair_hap_coverage(yReads->a+r_y_interval_beg, r_y_interval_end+1-r_y_interval_beg, - sources, coverage_cut); - ///fprintf(stderr, "ploid_coverage: %lu, cov_threshold: %lu\n", ploid_coverage, cov_threshold); + ploid_coverage = get_pair_purge_coverage(xReads, *r_x_pos_beg, *r_x_pos_end, + yReads, *r_y_pos_beg, *r_y_pos_end, hap_can->rev, read_g, cov); if(cov_threshold > 0 && ploid_coverage >= cov_threshold) return NON_PLOID; + get_pair_hap_similarity_by_base(xReads, read_g, yUid, reverse_sources, ruIndex, + *r_x_pos_beg, *r_x_pos_end, &xLeftMatch, &xLeftTotal); + if(xLeftMatch == 0 || xLeftTotal == 0 || xLeftMatch <= xLeftTotal*Hap_rate) + { + return NON_PLOID; + } + return PLOID; } return NON_PLOID; @@ -1957,277 +2582,6 @@ void print_hap_paf(ma_ug_t *ug, hap_overlaps* ovlp) ovlp->y_beg_pos, ovlp->y_beg_id, ovlp->y_end_pos, ovlp->y_end_id, ovlp->type, (uint32_t)ovlp->weight); } -void hap_alignment(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, ma_sub_t *coverage_cut, uint64_t* position_index, uint64_t* vote_counting, -uint8_t* visit, kvec_t_u64_warp* u_vecs, kvec_asg_arc_t_offset* u_buffer, kvec_hap_candidates* u_can, -uint32_t Input_uId, float Hap_rate, int max_hang, int min_ovlp, float chain_rate, -hap_overlaps_list* all_ovlp) -{ - ma_utg_t *xReads = NULL, *yReads = NULL; - ma_hit_t_alloc *xR = NULL; - ma_hit_t *h = NULL; - ma_sub_t *sq = NULL, *st = NULL; - asg_t* nsg = ug->g; - uint32_t i, j, v, rId, k, is_Unitig, Hap_uId, xUid, yUid, seedOcc, xPos, yPos, is_update; - uint64_t tmp; - long long cur_offset, new_offset, interval_len; - long long r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end; - int32_t r; - asg_arc_t t; - asg_arc_t_offset t_offset; - hap_candidates hap_can; - memset(&hap_can, 0, sizeof(hap_candidates)); - hap_overlaps hap_align; - xUid = Input_uId; - if(nsg->seq[xUid].del || nsg->seq[xUid].c == ALTER_LABLE) return; - memset(vote_counting, 0, sizeof(uint64_t)*nsg->n_seq); - memset(visit, 0, nsg->n_seq); - u_vecs->a.n = 0; - u_can->a.n = 0; - - xReads = &(ug->u.a[xUid]); - for (i = 0; i < xReads->n; i++) - { - xR = &(reverse_sources[xReads->a[i]>>33]); - - for (k = 0; k < xR->length; k++) - { - rId = Get_tn(xR->buffer[k]); - - if(read_g->seq[rId].del == 1) - { - ///get the id of read that contains it - get_R_to_U(ruIndex, rId, &rId, &is_Unitig); - if(rId == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[rId].del == 1) continue; - } - - ///there are two cases: - ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id - ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 - get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - ///here rId is the id of the read coming from the different haplotype - ///Hap_cId is the id of the corresponding contig (note here is the contig, instead of untig) - if(visit[Hap_uId]!=0) continue; - visit[Hap_uId] = 1; - if(vote_counting[Hap_uId] < UINT64_MAX) vote_counting[Hap_uId]++; - } - - clean_visit_flag(visit, read_g, ruIndex, nsg->n_seq, xR); - } - - - - u_vecs->a.n = 0; - for (i = 0; i < nsg->n_seq; i++) - { - if(i == xUid) continue; - if(vote_counting[i] == 0) continue; - tmp = vote_counting[i]; tmp = tmp << 32; tmp = tmp | (uint64_t)i; - kv_push(uint64_t, u_vecs->a, tmp); - } - - if(u_vecs->a.n == 0) return; - sort_kvec_t_u64_warp(u_vecs, 1); - - - ///scan each candidate unitig - for (i = 0; i < u_vecs->a.n; i++) - { - yUid = (uint32_t)u_vecs->a.a[i]; - seedOcc = u_vecs->a.a[i]>>32; - xReads = &(ug->u.a[xUid]); - yReads = &(ug->u.a[yUid]); - u_buffer->a.n = 0; - - for (k = 0; k < xReads->n; k++) - { - xR = &(reverse_sources[xReads->a[k]>>33]); - for (j = 0; j < xR->length; j++) - { - h = &(xR->buffer[j]); - sq = &(coverage_cut[Get_qn(*h)]); - st = &(coverage_cut[Get_tn(*h)]); - if(st->del || read_g->seq[Get_tn(*h)].del) continue; - - r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, - asm_opt.max_hang_rate, min_ovlp, &t); - ///if it is a contained overlap, skip - if(r < 0) continue; - - rId = t.v>>1; - if(read_g->seq[rId].del == 1) continue; - ///there are two cases: - ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id - ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 - get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != yUid) continue; - - v = xReads->a[k]>>32; - get_R_to_U(ruIndex, v>>1, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != xUid) continue; - if((uint32_t)(position_index[v>>1]) != k) continue; - - - if((prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), - xReads->n, yReads->n, 0, Hap_rate, seedOcc)==NON_PLOID) && - (prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), - xReads->n, yReads->n, 1, Hap_rate, seedOcc)==NON_PLOID)) - { - continue; - } - - t_offset.Off = get_xy_pos(read_g, &t, v, (yReads->a[(uint32_t)(position_index[rId])])>>32, - xReads->len, yReads->len, position_index, &(t.el)); - if(((t_offset.Off>>32) == (uint32_t)-1) || (((uint32_t)t_offset.Off) == (uint32_t)-1)) continue; - - t_offset.x = t; - t_offset.weight = 1; - - kv_push(asg_arc_t_offset, u_buffer->a, t_offset); - } - - deduplicate_edge(u_buffer); - } - - if(u_buffer->a.n == 0) continue; - - qsort(u_buffer->a.a, u_buffer->a.n, sizeof(asg_arc_t_offset), cmp_hap_alignment); - k = 0; - u_can->a.n = 0; - while (k < u_buffer->a.n) - { - hap_can.rev = u_buffer->a.a[k].x.el; - hap_can.index_beg = k; - hap_can.index_end = k; - hap_can.weight = u_buffer->a.a[k].weight; - hap_can.x_beg_pos = hap_can.x_end_pos = (uint32_t)(u_buffer->a.a[k].Off>>32); - hap_can.y_beg_pos = hap_can.y_end_pos = (uint32_t)(u_buffer->a.a[k].Off); - cur_offset = Cal_Off(u_buffer->a.a[k].Off); - interval_len = get_hap_overlapLen(hap_can.x_beg_pos, hap_can.x_end_pos, xReads->len, - hap_can.y_beg_pos, hap_can.y_end_pos, yReads->len, NULL, NULL, NULL, NULL); - - - k++; - while (k < u_buffer->a.n) - { - new_offset = Cal_Off(u_buffer->a.a[k].Off); - if(u_buffer->a.a[k].x.el != hap_can.rev) break; - if((new_offset - cur_offset)>(interval_len*chain_rate)) break; - - - hap_can.index_end = k; - hap_can.weight += u_buffer->a.a[k].weight; - - is_update = 0; - xPos = (uint32_t)(u_buffer->a.a[k].Off>>32); - yPos = (uint32_t)(u_buffer->a.a[k].Off); - if(xPos < hap_can.x_beg_pos) - { - hap_can.x_beg_pos = xPos; - is_update = 1; - } - - if(xPos > hap_can.x_end_pos) - { - hap_can.x_end_pos = xPos; - is_update = 1; - } - - if(yPos < hap_can.y_beg_pos) - { - hap_can.y_beg_pos = yPos; - is_update = 1; - } - - if(yPos > hap_can.y_end_pos) - { - hap_can.y_end_pos = yPos; - is_update = 1; - } - - if(new_offset == cur_offset) is_update = 0; - - if(is_update) - { - interval_len = get_hap_overlapLen(hap_can.x_beg_pos, hap_can.x_end_pos, xReads->len, - hap_can.y_beg_pos, hap_can.y_end_pos, yReads->len, NULL, NULL, NULL, NULL); - } - - k++; - } - - kv_push(hap_candidates, u_can->a, hap_can); - } - - if(u_can->a.n == 0) continue; - - qsort(u_can->a.a, u_can->a.n, sizeof(hap_candidates), cmp_hap_candidates); - - Get_match(hap_can) = Get_total(hap_can) = 0; - memset(&hap_align, 0, sizeof(hap_overlaps)); - - for (k = 0; k < u_can->a.n; k++) - { - is_update = 0; - if(u_can->a.a[k].weight < Get_match(hap_can)*Hap_rate) continue; - - if(calculate_pair_hap_similarity(u_buffer, &(u_can->a.a[k]), position_index, xUid, yUid, - xReads, yReads, reverse_sources, read_g, ruIndex, coverage_cut, Hap_rate, max_hang, - min_ovlp, &r_x_pos_beg, &r_x_pos_end, &r_y_pos_beg, &r_y_pos_end)!=PLOID) - { - continue; - } - - if(Get_match(hap_can) < Get_match(u_can->a.a[k])) - { - is_update = 1; - } - else if(Get_match(hap_can) == Get_match(u_can->a.a[k]) && - Get_total(hap_can) > Get_total(u_can->a.a[k])) - { - is_update = 1; - } - - if(is_update) - { - hap_can = u_can->a.a[k]; - hap_align.rev = Get_rev(hap_can); - hap_align.type = Get_type(hap_can); - hap_align.x_beg_id = Get_x_beg(hap_can); - hap_align.x_end_id = Get_x_end(hap_can) + 1; - hap_align.y_beg_id = Get_y_beg(hap_can); - hap_align.y_end_id = Get_y_end(hap_can) + 1; - hap_align.weight = Get_match(hap_can); - hap_align.x_beg_pos = r_x_pos_beg; - hap_align.x_end_pos = r_x_pos_end + 1; - if(hap_align.rev == 0) - { - hap_align.y_beg_pos = r_y_pos_beg; - hap_align.y_end_pos = r_y_pos_end + 1; - } - else - { - hap_align.y_beg_pos = yReads->len - r_y_pos_end - 1; - hap_align.y_end_pos = yReads->len - r_y_pos_beg - 1 + 1; - } - hap_align.xUid = xUid; - hap_align.yUid = yUid; - hap_align.status = SELF_EXIST; - } - } - - if(Get_match(hap_can) == 0 || Get_total(hap_can) == 0) continue; - - kv_push(hap_overlaps, all_ovlp->x[hap_align.xUid].a, hap_align); - } - -} - - - inline long long get_max_index(asg_arc_t_offset* x, int32_t* Scores, uint8_t* Flag, long long n, long long x_readLen, long long y_readLen) { @@ -2474,7 +2828,7 @@ long long y_readLen) if(is_merge == 0 && (Get_xOff(u_buffer->a.a[m-1].Off)==Get_xOff(u_buffer->a.a[i].Off))) { if((Get_yOff(u_buffer->a.a[i].Off)-(Get_yOff(u_buffer->a.a[m-1].Off))) == - (i-anchor_i)) + (i-anchor_i))///not sure why, does it use for tolerate indels in overlaps? { is_merge = 1; } @@ -2503,300 +2857,15 @@ long long y_readLen) begIndex_vec, flag_vec, band_width_threshold, max_skip, x_readLen, y_readLen, u_can); } - -///static void hap_alignment_worker(void *_data, long eid, int tid) -void hap_alignment_worker(void *_data, long eid, int tid) +int filter_secondary_chain(long long max_score, long long cur_score, double rate) { - hap_alignment_struct_pip* hap_buf = (hap_alignment_struct_pip*)_data; - ma_ug_t *ug = hap_buf->ug; - asg_t *read_g = hap_buf->read_g; - ma_hit_t_alloc* reverse_sources = hap_buf->reverse_sources; - R_to_U* ruIndex = hap_buf->ruIndex; - ma_sub_t *coverage_cut = hap_buf->coverage_cut; - uint64_t* position_index = hap_buf->position_index; - float Hap_rate = hap_buf->Hap_rate; - int max_hang = hap_buf->max_hang; - int min_ovlp = hap_buf->min_ovlp; - float chain_rate = hap_buf->chain_rate; - hap_overlaps_list* all_ovlp = hap_buf->all_ovlp; - uint32_t Input_uId = eid; - uint64_t* vote_counting = hap_buf->buf[tid].vote_counting; - uint8_t* visit = hap_buf->buf[tid].visit; - kvec_t_u64_warp* u_vecs = &(hap_buf->buf[tid].u_vecs); - kvec_asg_arc_t_offset* u_buffer = &(hap_buf->buf[tid].u_buffer); - kvec_hap_candidates* u_can = &(hap_buf->buf[tid].u_can); - - - ma_utg_t *xReads = NULL, *yReads = NULL; - ma_hit_t_alloc *xR = NULL; - ma_hit_t *h = NULL; - ma_sub_t *sq = NULL, *st = NULL; - asg_t* nsg = ug->g; - uint32_t i, j, v, rId, k, is_Unitig, Hap_uId, xUid, yUid, seedOcc, xPos, yPos, is_update; - uint64_t tmp; - long long cur_offset, new_offset, interval_len; - long long r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end; - int32_t r; - asg_arc_t t; - asg_arc_t_offset t_offset; - hap_candidates hap_can; - memset(&hap_can, 0, sizeof(hap_candidates)); - hap_overlaps hap_align; - xUid = Input_uId; - if(nsg->seq[xUid].del || nsg->seq[xUid].c == ALTER_LABLE) return; - memset(vote_counting, 0, sizeof(uint64_t)*nsg->n_seq); - memset(visit, 0, nsg->n_seq); - u_vecs->a.n = 0; - u_can->a.n = 0; - - xReads = &(ug->u.a[xUid]); - for (i = 0; i < xReads->n; i++) - { - xR = &(reverse_sources[xReads->a[i]>>33]); - - for (k = 0; k < xR->length; k++) - { - rId = Get_tn(xR->buffer[k]); - - if(read_g->seq[rId].del == 1) - { - ///get the id of read that contains it - get_R_to_U(ruIndex, rId, &rId, &is_Unitig); - if(rId == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[rId].del == 1) continue; - } - - ///there are two cases: - ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id - ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 - get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - ///here rId is the id of the read coming from the different haplotype - ///Hap_cId is the id of the corresponding contig (note here is the contig, instead of untig) - if(visit[Hap_uId]!=0) continue; - visit[Hap_uId] = 1; - if(vote_counting[Hap_uId] < UINT64_MAX) vote_counting[Hap_uId]++; - } - - clean_visit_flag(visit, read_g, ruIndex, nsg->n_seq, xR); - } - - - - u_vecs->a.n = 0; - for (i = 0; i < nsg->n_seq; i++) - { - if(i == xUid) continue; - if(vote_counting[i] == 0) continue; - tmp = vote_counting[i]; tmp = tmp << 32; tmp = tmp | (uint64_t)i; - kv_push(uint64_t, u_vecs->a, tmp); - } - - if(u_vecs->a.n == 0) return; - sort_kvec_t_u64_warp(u_vecs, 1); - - - ///scan each candidate unitig - for (i = 0; i < u_vecs->a.n; i++) - { - yUid = (uint32_t)u_vecs->a.a[i]; - seedOcc = u_vecs->a.a[i]>>32; - xReads = &(ug->u.a[xUid]); - yReads = &(ug->u.a[yUid]); - u_buffer->a.n = 0; - - for (k = 0; k < xReads->n; k++) - { - xR = &(reverse_sources[xReads->a[k]>>33]); - for (j = 0; j < xR->length; j++) - { - h = &(xR->buffer[j]); - sq = &(coverage_cut[Get_qn(*h)]); - st = &(coverage_cut[Get_tn(*h)]); - if(st->del || read_g->seq[Get_tn(*h)].del) continue; - - r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, - asm_opt.max_hang_rate, min_ovlp, &t); - ///if it is a contained overlap, skip - if(r < 0) continue; - - rId = t.v>>1; - if(read_g->seq[rId].del == 1) continue; - ///there are two cases: - ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id - ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 - get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != yUid) continue; - - v = xReads->a[k]>>32; - get_R_to_U(ruIndex, v>>1, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != xUid) continue; - if((uint32_t)(position_index[v>>1]) != k) continue; - - - if((prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), - xReads->n, yReads->n, 0, Hap_rate, seedOcc)==NON_PLOID) && - (prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), - xReads->n, yReads->n, 1, Hap_rate, seedOcc)==NON_PLOID)) - { - continue; - } - - t_offset.Off = get_xy_pos(read_g, &t, v, (yReads->a[(uint32_t)(position_index[rId])])>>32, - xReads->len, yReads->len, position_index, &(t.el)); - if(((t_offset.Off>>32) == (uint32_t)-1) || (((uint32_t)t_offset.Off) == (uint32_t)-1)) continue; - - t_offset.x = t; - t_offset.weight = 1; - - kv_push(asg_arc_t_offset, u_buffer->a, t_offset); - } - - deduplicate_edge(u_buffer); - } - - if(u_buffer->a.n == 0) continue; - - // if(debug_enable) - // { - // print_debug_unitig(xReads, position_index, "xReads"); - // print_debug_unitig(yReads, position_index, "yReads"); - // } - - qsort(u_buffer->a.a, u_buffer->a.n, sizeof(asg_arc_t_offset), cmp_hap_alignment); - k = 0; - u_can->a.n = 0; - while (k < u_buffer->a.n) - { - hap_can.rev = u_buffer->a.a[k].x.el; - hap_can.index_beg = k; - hap_can.index_end = k; - hap_can.weight = u_buffer->a.a[k].weight; - hap_can.x_beg_pos = hap_can.x_end_pos = (uint32_t)(u_buffer->a.a[k].Off>>32); - hap_can.y_beg_pos = hap_can.y_end_pos = (uint32_t)(u_buffer->a.a[k].Off); - cur_offset = Cal_Off(u_buffer->a.a[k].Off); - interval_len = get_hap_overlapLen(hap_can.x_beg_pos, hap_can.x_end_pos, xReads->len, - hap_can.y_beg_pos, hap_can.y_end_pos, yReads->len, NULL, NULL, NULL, NULL); - - - k++; - while (k < u_buffer->a.n) - { - new_offset = Cal_Off(u_buffer->a.a[k].Off); - if(u_buffer->a.a[k].x.el != hap_can.rev) break; - if((new_offset - cur_offset)>(interval_len*chain_rate)) break; - - - hap_can.index_end = k; - hap_can.weight += u_buffer->a.a[k].weight; - - is_update = 0; - xPos = (uint32_t)(u_buffer->a.a[k].Off>>32); - yPos = (uint32_t)(u_buffer->a.a[k].Off); - if(xPos < hap_can.x_beg_pos) - { - hap_can.x_beg_pos = xPos; - is_update = 1; - } - - if(xPos > hap_can.x_end_pos) - { - hap_can.x_end_pos = xPos; - is_update = 1; - } - - if(yPos < hap_can.y_beg_pos) - { - hap_can.y_beg_pos = yPos; - is_update = 1; - } - - if(yPos > hap_can.y_end_pos) - { - hap_can.y_end_pos = yPos; - is_update = 1; - } - - if(new_offset == cur_offset) is_update = 0; - - if(is_update) - { - interval_len = get_hap_overlapLen(hap_can.x_beg_pos, hap_can.x_end_pos, xReads->len, - hap_can.y_beg_pos, hap_can.y_end_pos, yReads->len, NULL, NULL, NULL, NULL); - } - - k++; - } - - kv_push(hap_candidates, u_can->a, hap_can); - } - - if(u_can->a.n == 0) continue; - - qsort(u_can->a.a, u_can->a.n, sizeof(hap_candidates), cmp_hap_candidates); - - Get_match(hap_can) = Get_total(hap_can) = 0; - memset(&hap_align, 0, sizeof(hap_overlaps)); - - for (k = 0; k < u_can->a.n; k++) - { - is_update = 0; - if(u_can->a.a[k].weight < Get_match(hap_can)*Hap_rate) continue; - - if(calculate_pair_hap_similarity(u_buffer, &(u_can->a.a[k]), position_index, xUid, yUid, - xReads, yReads, reverse_sources, read_g, ruIndex, coverage_cut, Hap_rate, max_hang, - min_ovlp, &r_x_pos_beg, &r_x_pos_end, &r_y_pos_beg, &r_y_pos_end)!=PLOID) - { - continue; - } - - if(Get_match(hap_can) < Get_match(u_can->a.a[k])) - { - is_update = 1; - } - else if(Get_match(hap_can) == Get_match(u_can->a.a[k]) && - Get_total(hap_can) > Get_total(u_can->a.a[k])) - { - is_update = 1; - } - - if(is_update) - { - hap_can = u_can->a.a[k]; - hap_align.rev = Get_rev(hap_can); - hap_align.type = Get_type(hap_can); - hap_align.x_beg_id = Get_x_beg(hap_can); - hap_align.x_end_id = Get_x_end(hap_can) + 1; - hap_align.y_beg_id = Get_y_beg(hap_can); - hap_align.y_end_id = Get_y_end(hap_can) + 1; - hap_align.weight = Get_match(hap_can); - hap_align.x_beg_pos = r_x_pos_beg; - hap_align.x_end_pos = r_x_pos_end + 1; - if(hap_align.rev == 0) - { - hap_align.y_beg_pos = r_y_pos_beg; - hap_align.y_end_pos = r_y_pos_end + 1; - } - else - { - hap_align.y_beg_pos = yReads->len - r_y_pos_end - 1; - hap_align.y_end_pos = yReads->len - r_y_pos_beg - 1 + 1; - } - hap_align.xUid = xUid; - hap_align.yUid = yUid; - hap_align.status = SELF_EXIST; - } - } - - if(Get_match(hap_can) == 0 || Get_total(hap_can) == 0) continue; - - kv_push(hap_overlaps, all_ovlp->x[hap_align.xUid].a, hap_align); - } - + if(cur_score >= max_score) return 1; + long long diff = max_score - cur_score; + if(max_score < 0) max_score *= -1; + if(diff >= max_score*(1-rate)) return 0; + return 1; } - static void hap_alignment_advance_worker(void *_data, long eid, int tid) { hap_alignment_struct_pip* hap_buf = (hap_alignment_struct_pip*)_data; @@ -2823,21 +2892,21 @@ static void hap_alignment_advance_worker(void *_data, long eid, int tid) kvec_t_i32_warp* begIndex_vec = &(hap_buf->buf[tid].u_buffer_beg); kvec_t_u8_warp* flag_vec = &(hap_buf->buf[tid].u_buffer_flag); uint64_t cov_threshold = hap_buf->cov_threshold; + hap_cov_t *cov = hap_buf->cov; if(hap_buf->cov_threshold < 0) cov_threshold = (uint64_t)-1; - ma_utg_t *xReads = NULL, *yReads = NULL; ma_hit_t_alloc *xR = NULL; ma_hit_t *h = NULL; ma_sub_t *sq = NULL, *st = NULL; asg_t* nsg = ug->g; - uint32_t i, j, v, rId, k, is_Unitig, Hap_uId, xUid, yUid, seedOcc, is_update; - uint64_t tmp; - long long r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end; + uint32_t i, j, v, rId, k, is_Unitig, Hap_uId, xUid, yUid, seedOcc; + uint64_t tmp, max_weight, m; + long long r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end, max_score; int32_t r; asg_arc_t t; asg_arc_t_offset t_offset; - hap_candidates hap_can; hap_overlaps hap_align; + hap_overlaps *hap_align_x = NULL; xUid = Input_uId; if(nsg->seq[xUid].del || nsg->seq[xUid].c == ALTER_LABLE) return; memset(vote_counting, 0, sizeof(uint64_t)*nsg->n_seq); @@ -2868,7 +2937,7 @@ static void hap_alignment_advance_worker(void *_data, long eid, int tid) if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; ///here rId is the id of the read coming from the different haplotype ///Hap_cId is the id of the corresponding contig (note here is the contig, instead of untig) - if(visit[Hap_uId]!=0) continue; + if(visit[Hap_uId]!=0) continue; ///one read only has one vote for one hap unitig visit[Hap_uId] = 1; if(vote_counting[Hap_uId] < UINT64_MAX) vote_counting[Hap_uId]++; } @@ -2903,6 +2972,7 @@ static void hap_alignment_advance_worker(void *_data, long eid, int tid) for (k = 0; k < xReads->n; k++) { xR = &(reverse_sources[xReads->a[k]>>33]); + for (j = 0; j < xR->length; j++) { h = &(xR->buffer[j]); @@ -2930,8 +3000,8 @@ static void hap_alignment_advance_worker(void *_data, long eid, int tid) if(Hap_uId != xUid) continue; if((uint32_t)(position_index[v>>1]) != k) continue; - - if((prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), + if(asm_opt.purge_level_primary <= 2 && + (prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), xReads->n, yReads->n, 0, Hap_rate, seedOcc)==NON_PLOID) && (prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), xReads->n, yReads->n, 1, Hap_rate, seedOcc)==NON_PLOID)) @@ -2962,64 +3032,88 @@ static void hap_alignment_advance_worker(void *_data, long eid, int tid) qsort(u_can->a.a, u_can->a.n, sizeof(hap_candidates), cmp_hap_candidates); - Get_match(hap_can) = Get_total(hap_can) = 0; - memset(&hap_align, 0, sizeof(hap_overlaps)); - + memset(&hap_align, 0, sizeof(hap_overlaps)); + m = all_ovlp->x[xUid].a.n; + max_weight = 0; max_score = 0; for (k = 0; k < u_can->a.n; k++) { - is_update = 0; - if(u_can->a.a[k].weight < Get_match(hap_can)*Hap_rate) continue; - + if(u_can->a.a[k].weight < max_weight*0.33) continue; if(calculate_pair_hap_similarity_advance(&(u_can->a.a[k]), position_index, xUid, yUid, - xReads, yReads, sources, reverse_sources, read_g, ruIndex, coverage_cut, Hap_rate, max_hang, - min_ovlp, cov_threshold, u_buffer, score_vc, prevIndex_vec, &r_x_pos_beg, &r_x_pos_end, - &r_y_pos_beg, &r_y_pos_end)!=PLOID) + xReads, yReads, sources, reverse_sources, read_g, ruIndex, coverage_cut, Hap_rate, + (asm_opt.purge_level_primary<=2? 0:1), max_hang, min_ovlp, cov_threshold, u_buffer, + score_vc, prevIndex_vec, cov, &r_x_pos_beg, &r_x_pos_end, &r_y_pos_beg, &r_y_pos_end)!=PLOID) { continue; } - - - if(Get_match(hap_can) < Get_match(u_can->a.a[k])) + ///max_weight == 0 means the first matched chain + if(max_weight == 0 || max_score < u_can->a.a[k].score) max_score = u_can->a.a[k].score; + if(max_weight < u_can->a.a[k].weight) max_weight = u_can->a.a[k].weight; + ///if one is positive and another one is negative, it is wrong + if(!filter_secondary_chain(max_score, u_can->a.a[k].score, CHAIN_FILTER_RATE)) continue; + + hap_align.rev = Get_rev(u_can->a.a[k]); + hap_align.type = Get_type(u_can->a.a[k]); + hap_align.x_beg_id = Get_x_beg(u_can->a.a[k]); + hap_align.x_end_id = Get_x_end(u_can->a.a[k]) + 1; + hap_align.y_beg_id = Get_y_beg(u_can->a.a[k]); + hap_align.y_end_id = Get_y_end(u_can->a.a[k]) + 1; + hap_align.weight = Get_match(u_can->a.a[k]); + hap_align.score = u_can->a.a[k].score; + hap_align.x_beg_pos = r_x_pos_beg; + hap_align.x_end_pos = r_x_pos_end + 1; + if(hap_align.rev == 0) { - is_update = 1; + hap_align.y_beg_pos = r_y_pos_beg; + hap_align.y_end_pos = r_y_pos_end + 1; } - else if(Get_match(hap_can) == Get_match(u_can->a.a[k]) && - Get_total(hap_can) > Get_total(u_can->a.a[k])) + else { - is_update = 1; + hap_align.y_beg_pos = yReads->len - r_y_pos_end - 1; + hap_align.y_end_pos = yReads->len - r_y_pos_beg - 1 + 1; } - - if(is_update) + hap_align.xUid = xUid; + hap_align.yUid = yUid; + hap_align.status = SELF_EXIST; + kv_push(hap_overlaps, all_ovlp->x[hap_align.xUid].a, hap_align); + } + /** + ///chains with same xUid && yUid + for (k = m; k < all_ovlp->x[xUid].a.n; k++) + { + if(!filter_secondary_chain(max_score, + all_ovlp->x[xUid].a.a[k].score, CHAIN_FILTER_RATE)) { - hap_can = u_can->a.a[k]; - hap_align.rev = Get_rev(hap_can); - hap_align.type = Get_type(hap_can); - hap_align.x_beg_id = Get_x_beg(hap_can); - hap_align.x_end_id = Get_x_end(hap_can) + 1; - hap_align.y_beg_id = Get_y_beg(hap_can); - hap_align.y_end_id = Get_y_end(hap_can) + 1; - hap_align.weight = Get_match(hap_can); - hap_align.x_beg_pos = r_x_pos_beg; - hap_align.x_end_pos = r_x_pos_end + 1; - if(hap_align.rev == 0) + continue; + } + all_ovlp->x[xUid].a.a[m] = all_ovlp->x[xUid].a.a[k]; + m++; + } + all_ovlp->x[xUid].a.n = m; + **/ + + hap_align_x = NULL; + for (k = m; k < all_ovlp->x[xUid].a.n; k++) + { + if(all_ovlp->x[xUid].a.a[k].score != max_score) continue; + if(hap_align_x == NULL || all_ovlp->x[xUid].a.a[k].weight > hap_align_x->weight) + { + hap_align_x = &(all_ovlp->x[xUid].a.a[k]); + } + else if(all_ovlp->x[xUid].a.a[k].weight == hap_align_x->weight) + { + if((all_ovlp->x[xUid].a.a[k].x_end_pos + - all_ovlp->x[xUid].a.a[k].x_beg_pos) < + (hap_align_x->x_end_pos - hap_align_x->x_beg_pos)) { - hap_align.y_beg_pos = r_y_pos_beg; - hap_align.y_end_pos = r_y_pos_end + 1; + hap_align_x = &(all_ovlp->x[xUid].a.a[k]); } - else - { - hap_align.y_beg_pos = yReads->len - r_y_pos_end - 1; - hap_align.y_end_pos = yReads->len - r_y_pos_beg - 1 + 1; - } - hap_align.xUid = xUid; - hap_align.yUid = yUid; - hap_align.status = SELF_EXIST; } } - - if(Get_match(hap_can) == 0 || Get_total(hap_can) == 0) continue; - - kv_push(hap_overlaps, all_ovlp->x[hap_align.xUid].a, hap_align); + if(hap_align_x) + { + all_ovlp->x[xUid].a.a[m] = (*hap_align_x); + all_ovlp->x[xUid].a.n = m + 1; + } } } @@ -3137,8 +3231,9 @@ ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex) y = &(all_ovlp->x[tn].a.a[index]); if(x->rev == y->rev && types[x->type]==y->type) continue; ///if(x->weight >= y->weight) - if((calculate_bi_weight(x, ug, read_g, reverse_sources, ruIndex)) >= - (calculate_bi_weight(y, ug, read_g, reverse_sources, ruIndex))) + // if((calculate_bi_weight(x, ug, read_g, reverse_sources, ruIndex)) >= + // (calculate_bi_weight(y, ug, read_g, reverse_sources, ruIndex))) + if(x->score >= y->score) { kv_push(hap_overlaps, back_all_ovlp->x[tn].a, (*y)); set_reverse_hap_overlap(y, x, types); @@ -3160,6 +3255,7 @@ ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex) void filter_hap_overlaps_by_length(hap_overlaps_list* all_ovlp, uint32_t minLen) { + if(minLen == 0) return; hap_overlaps *x = NULL; uint32_t v, i, m, uId; @@ -3269,6 +3365,137 @@ void print_purge_gfa(ma_ug_t *ug, asg_t *purge_g) } +long long decode_score(uint32_t h_bits, uint32_t l_bits) +{ + uint64_t x; + x = h_bits; x <<= 32; x += l_bits; + long long score = ((uint64_t)((uint64_t)x<<1)>>1); + if((x>>63) == 0) score *= -1; + return score; +} + +void encode_score(long long i_s, uint32_t *h_bits, uint32_t *l_bits) +{ + uint64_t score = (i_s >= 0? (i_s) : (i_s*(-1))); + if(i_s >= 0) score += (((uint64_t)1)<<63); + (*l_bits) = (uint32_t)score; (*h_bits)= (score>>32); +} + +uint64_t asg_bub_pop1_purge_graph(asg_t *g, uint32_t v0, int max_dist, buf_t *b) +{ + uint32_t i, n_pending = 0, n_tips, tip_end; + uint64_t n_pop = 0; + ///if this node has been deleted + if (g->seq[v0>>1].del || g->seq[v0>>1].c == ALTER_LABLE) return 0; // already deleted + ///if ((uint32_t)g->idx[v0] < 2) return 0; // no bubbles + if(get_real_length(g, v0, NULL)<2) return 0; + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[v0].c = b->a[v0].d = b->a[v0].m = b->a[v0].nc = b->a[v0].np = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, v0); + n_tips = 0; + tip_end = (uint32_t)-1; + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), d = b->a[v].d; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + long long t_s = decode_score(b->a[v].c, b->a[v].m), c_s; + ///why we have this assert? + ///assert(nv > 0); + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + uint32_t w = av[i].v; // v->w with length l + binfo_t *t = &b->a[w]; + ///that means there is a circle, directly terminate the whole bubble poping + ///if (w == v0) goto pop_reset; + if ((w>>1) == (v0>>1)) goto pop_reset; + ///if this edge has been deleted + if (av[i].del) continue; + c_s = decode_score((uint32_t)av[i].ul, av[i].ol); + ///push the edge + ///high 32-bit of g->idx[v] is the start point of v's edges + //so here is the point of this specfic edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + ///find a too far path? directly terminate the whole bubble poping + if (d + 1 > (uint32_t)max_dist) break; // too far + + ///if this node + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p is the parent node of + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + 1; + encode_score(t_s + c_s, &(t->c), &(t->m)); + ///incoming edges of w + ///t->r = count_out(g, w^1); + t->r = get_real_length(g, w^1, NULL); + ++n_pending; + } else { // visited before + if((t_s + c_s)> decode_score(t->c, t->m)) + { + t->p = v; + encode_score(t_s + c_s, &(t->c), &(t->m)); + } + ///it is the shortest edge + if (d + 1 < t->d) t->d = d + 1; // update dist + } + ///assert(t->r > 0); + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = get_real_length(g, w, NULL); + /****************************may have bugs for bubble********************************/ + if(x > 0) + { + kv_push(uint32_t, b->S, w); + } + else + { + ///at most one tip + if(n_tips != 0) goto pop_reset; + n_tips++; + tip_end = w; + } + /****************************may have bugs for bubble********************************/ + --n_pending; + } + } + //if found a tip + /****************************may have bugs for bubble********************************/ + if(n_tips == 1) + { + if(tip_end != (uint32_t)-1 && n_pending == 0 && b->S.n == 0) + { + kv_push(uint32_t, b->S, tip_end); + break; + } + else + { + goto pop_reset; + } + } + /****************************may have bugs for bubble********************************/ + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) goto pop_reset; + } while (b->S.n > 1 || n_pending); + + asg_bub_backtrack_primary(g, v0, b); + + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_t *t = &b->a[b->b.a[i]]; + t->s = t->c = t->d = t->m = t->nc = t->np = 0; + } + return n_pop; +} + // pop bubbles @@ -3291,7 +3518,7 @@ int asg_pop_bubble_purge_graph(asg_t *purge_g, int max_dist) for (i = 0; i < nv; ++i) // asg_bub_pop1() may delete some edges/arcs if (!av[i].del) ++n_arc; if (n_arc > 1) - n_pop += asg_bub_pop1_primary_trio(purge_g, NULL, v, max_dist, &b, (uint32_t)-1, DROP, 1, NULL, NULL); + n_pop += asg_bub_pop1_purge_graph(purge_g, v, purge_g->n_seq, &b); } free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); if (n_pop) asg_cleanup(purge_g); @@ -3320,9 +3547,98 @@ int min_ovlp, asg_arc_t* t) h.bl = h.el = h.ml = h.no_l_indel = 0; r = ma_hit2arc(&h, qLen, tLen, max_hang, max_hang_rate, min_ovlp, t); + if(r < 0) return r; + uint64_t score = (hap->score >= 0? (hap->score) : (hap->score*(-1))); + if(hap->score >= 0) score += (((uint64_t)1)<<63); + t->ol = (uint32_t)score; + t->ul >>= 32; t->ul <<= 32; t->ul |= (score>>32); return r; } +typedef struct { + uint64_t eid; + uint64_t score; +}e_score; + +typedef struct { + size_t n, m; + e_score* a; +}e_score_warp; + +#define e_score_key(a) ((a).score) +KRADIX_SORT_INIT(e_score, e_score, e_score_key, member_size(e_score, score)) + +int purge_g_arc_del_short_diploid_by_score(asg_t *g, float drop_ratio) +{ + e_score_warp b; + kv_init(b); + e_score *p = NULL; + + uint32_t v, n_vtx = g->n_seq * 2; + long long n_cut = 0; + + for (v = 0; v < n_vtx; ++v) + { + if(g->seq[v>>1].c == ALTER_LABLE || g->seq[v>>1].del) continue; + asg_arc_t *av = asg_arc_a(g, v); + uint32_t nv = asg_arc_n(g, v); + if (nv < 2) continue; + uint64_t i; + for (i = 0; i < nv; ++i) + { + kv_pushp(e_score, b, &p); + p->eid = av - g->arc + i; + p->score = (uint32_t)av[i].ul; + p->score <<= 32; + p->score |= av[i].ol; + } + } + + radix_sort_e_score(b.a, b.a + b.n); + + uint64_t k; + for (k = 0; k < b.n; k++) + { + asg_arc_t *a = &g->arc[b.a[k].eid]; + ///v is self id, w is the id of another end + uint32_t i, v = (a->ul)>>32; + uint32_t nv = asg_arc_n(g, v), kv; + long long ovlp_max = 0, ovlp; + asg_arc_t *av = NULL; + ///nv must be >= 2 + if (nv <= 1) continue; + av = asg_arc_a(g, v); + + ///calculate the longest edge for v and w + for (i = 0, kv = 0; i < nv; ++i) { + if (av[i].del) continue; + ovlp = decode_score((uint32_t)av[i].ul, av[i].ol); + if (kv == 0 || ovlp_max < ovlp) ovlp_max = ovlp; + ++kv; + } + + if (kv <= 1) continue; + ovlp = decode_score((uint32_t)a->ul, a->ol); + if (kv >= 2) + { + if(ovlp >= 0 && ovlp_max >= 0 && ovlp > ovlp_max * drop_ratio) continue; + } + + a->del = 1; + asg_arc_del(g, a->v^1, av->ul>>32^1, 1); + ++n_cut; + } + + kv_destroy(b); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + + + return n_cut; +} void clean_purge_graph(asg_t *purge_g, int max_dist, float drop_ratio) @@ -3332,168 +3648,13 @@ void clean_purge_graph(asg_t *purge_g, int max_dist, float drop_ratio) { operation = 0; operation += asg_pop_bubble_purge_graph(purge_g, max_dist); - operation += unitig_arc_del_short_diploid_by_length(purge_g, drop_ratio); + operation += purge_g_arc_del_short_diploid_by_score(purge_g, drop_ratio); } - unitig_arc_del_short_diploid_by_length(purge_g, 1); + purge_g_arc_del_short_diploid_by_score(purge_g, 1); } -void get_node_boundary(R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, -asg_t *read_g, uint64_t* position_index, int max_hang, int min_ovlp, ma_utg_t *xReads, ma_utg_t *yReads, -uint32_t xUid, uint32_t yUid, long long xBegIndex, long long xEndIndex, long long yBegIndex, -long long yEndIndex, uint32_t dir, uint32_t rev, asg_arc_t* reture_t_f, asg_arc_t* reture_t_r) -{ - long long k, j, offset; - ma_hit_t_alloc *xR = NULL; - ma_hit_t *h = NULL; - ma_sub_t *sq = NULL, *st = NULL; - int r, index; - asg_arc_t t_f, t_r; - uint32_t rId, Hap_uId, is_Unitig, v, w, v_dir, w_dir, is_found = 0, oLen = 0; - reture_t_f->del = reture_t_r->del = 1; - if(dir == 1) - { - for (k = xEndIndex; k >= xBegIndex; k--) - { - xR = &(reverse_sources[xReads->a[k]>>33]); - is_found = 0; - for (j = 0; j < xR->length; j++) - { - h = &(xR->buffer[j]); - sq = &(coverage_cut[Get_qn(*h)]); - st = &(coverage_cut[Get_tn(*h)]); - if(st->del || read_g->seq[Get_tn(*h)].del) continue; - r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, - asm_opt.max_hang_rate, min_ovlp, &t_f); - ///if it is a contained overlap, skip - if(r < 0) continue; - - rId = t_f.v>>1; - if(read_g->seq[rId].del == 1) continue; - ///there are two cases: - ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id - ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 - get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != yUid) continue; - - v = xReads->a[k]>>32; - get_R_to_U(ruIndex, v>>1, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != xUid) continue; - if((uint32_t)(position_index[v>>1]) != k) continue; - - w = (yReads->a[(uint32_t)(position_index[rId])])>>32; - v_dir = ((t_f.ul>>32)==v)?1:0; - w_dir = (t_f.v == w)?1:0; - - if(rev == 0 && v_dir != w_dir) continue; - if(rev == 1 && v_dir == w_dir) continue; - if(v_dir == 1) continue; - - /****************************may have bugs********************************/ - offset = (uint32_t)(position_index[rId]); - if(offset < yBegIndex || offset > yEndIndex) continue; - /****************************may have bugs********************************/ - - /************************get reverse edge*************************/ - index = get_specific_overlap(&(reverse_sources[Get_tn(*h)]), Get_tn(*h), Get_qn(*h)); - if(index == -1) continue; - h = &(reverse_sources[Get_tn(*h)].buffer[index]); - sq = &(coverage_cut[Get_qn(*h)]); - st = &(coverage_cut[Get_tn(*h)]); - if(st->del || read_g->seq[Get_tn(*h)].del) continue; - r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, - asm_opt.max_hang_rate, min_ovlp, &t_r); - if(r < 0) continue; - /************************get reverse edge*************************/ - - if(is_found == 0 || t_f.ol > oLen) - { - (*reture_t_f) = t_f; - (*reture_t_r) = t_r; - oLen = t_f.ol; - } - - is_found = 1; - } - if(is_found) return; - } - } - else - { - for (k = xBegIndex; k <= xEndIndex; k++) - { - xR = &(reverse_sources[xReads->a[k]>>33]); - is_found = 0; oLen = 0; - for (j = 0; j < xR->length; j++) - { - h = &(xR->buffer[j]); - sq = &(coverage_cut[Get_qn(*h)]); - st = &(coverage_cut[Get_tn(*h)]); - if(st->del || read_g->seq[Get_tn(*h)].del) continue; - - r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, - asm_opt.max_hang_rate, min_ovlp, &t_f); - ///if it is a contained overlap, skip - if(r < 0) continue; - - rId = t_f.v>>1; - if(read_g->seq[rId].del == 1) continue; - ///there are two cases: - ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id - ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 - get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != yUid) continue; - - v = xReads->a[k]>>32; - get_R_to_U(ruIndex, v>>1, &Hap_uId, &is_Unitig); - if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; - if(Hap_uId != xUid) continue; - if((uint32_t)(position_index[v>>1]) != k) continue; - - w = (yReads->a[(uint32_t)(position_index[rId])])>>32; - - v_dir = ((t_f.ul>>32)==v)?1:0; - w_dir = (t_f.v == w)?1:0; - if(rev == 0 && v_dir != w_dir) continue; - if(rev == 1 && v_dir == w_dir) continue; - if(v_dir == 0) continue; - - /****************************may have bugs********************************/ - offset = (uint32_t)(position_index[rId]); - if(offset < yBegIndex || offset > yEndIndex) continue; - /****************************may have bugs********************************/ - - /************************get reverse edge*************************/ - index = get_specific_overlap(&(reverse_sources[Get_tn(*h)]), Get_tn(*h), Get_qn(*h)); - if(index == -1) continue; - h = &(reverse_sources[Get_tn(*h)].buffer[index]); - sq = &(coverage_cut[Get_qn(*h)]); - st = &(coverage_cut[Get_tn(*h)]); - if(st->del || read_g->seq[Get_tn(*h)].del) continue; - r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, - asm_opt.max_hang_rate, min_ovlp, &t_r); - if(r < 0) continue; - /************************get reverse edge*************************/ - - if(is_found == 0 || t_f.ol > oLen) - { - (*reture_t_f) = t_f; - (*reture_t_r) = t_r; - oLen = t_f.ol; - } - - is_found = 1; - } - if(is_found) return; - } - } - -} - void get_node_boundary_advance(R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, int max_hang, int min_ovlp, ma_utg_t *xReads, ma_utg_t *yReads, uint32_t xUid, uint32_t yUid, long long xBegIndex, long long xEndIndex, long long yBegIndex, @@ -3686,7 +3847,7 @@ uint32_t is_circle, uint64_t* rLen) if(k == edge->a.n) { - fprintf(stderr, "####ERROR1: i: %u, v>>1: %u, v&1: %u, w>>1: %u, w&1: %u\n", + fprintf(stderr, "####ERROR1-fill: i: %u, v>>1: %u, v&1: %u, w>>1: %u, w&1: %u\n", i, v>>1, v&1, w>>1, w&1); } } @@ -3731,7 +3892,7 @@ uint32_t is_circle, uint64_t* rLen) if(k == edge->a.n) { - fprintf(stderr, "####ERROR2: i: %u, v>>1: %u, v&1: %u, w>>1: %u, w&1: %u\n", + fprintf(stderr, "####ERROR2-fill: i: %u, v>>1: %u, v&1: %u, w>>1: %u, w&1: %u\n", i, v>>1, v&1, w>>1, w&1); } } @@ -3754,10 +3915,101 @@ uint32_t is_circle, uint64_t* rLen) } +void collect_trans_purge_cov(hap_cov_t *cov, ma_ug_t *ug, hap_overlaps* x, uint32_t is_keep_X) +{ + if(ug->u.a[x->xUid].n == 0 || ug->u.a[x->yUid].n == 0) return; + uint64_t *pri = NULL, pri_n, *aux = NULL, aux_n, i, rId, uCov = 0, uLen = 0; + + if(is_keep_X) + { + pri = ug->u.a[x->xUid].a + x->x_beg_id; + pri_n = x->x_end_id - x->x_beg_id; + + aux = ug->u.a[x->yUid].a + x->y_beg_id; + aux_n = x->y_end_id - x->y_beg_id; + } + else + { + pri = ug->u.a[x->yUid].a + x->y_beg_id; + pri_n = x->y_end_id - x->y_beg_id; + + aux = ug->u.a[x->xUid].a + x->x_beg_id; + aux_n = x->x_end_id - x->x_beg_id; + } + + + uCov = uLen = 0; + for (i = 0; i < aux_n; i++) + { + rId = aux[i]>>33; + uCov += cov->cov[rId]; + } + + for (i = 0; i < pri_n; i++) + { + rId = pri[i]>>33; + uLen += cov->read_g->seq[rId].len; + } + + uCov = (uLen == 0? 0 : uCov / uLen); + + + for (i = 0; i < pri_n; i++) + { + rId = pri[i]>>33; + cov->cov[rId] += (uCov * cov->read_g->seq[rId].len); + } +} + + +void collect_trans_purge_joint_cov(hap_cov_t *cov, ma_ug_t *ug, hap_overlaps* x) +{ + if(ug->u.a[x->xUid].n == 0 || ug->u.a[x->yUid].n == 0) return; + uint64_t *a[2], a_n[2], uCov[2], uLen[2], uDepth[2], i, rId; + + a[0] = ug->u.a[x->xUid].a + x->x_beg_id; + a_n[0] = x->x_end_id - x->x_beg_id; + uCov[0] = uLen[0] = 0; + for (i = 0; i < a_n[0]; i++) + { + rId = a[0][i]>>33; + uCov[0] += cov->cov[rId]; + uLen[0] += cov->read_g->seq[rId].len; + } + + a[1] = ug->u.a[x->yUid].a + x->y_beg_id; + a_n[1] = x->y_end_id - x->y_beg_id; + uCov[1] = uLen[1] = 0; + for (i = 0; i < a_n[1]; i++) + { + rId = a[1][i]>>33; + uCov[1] += cov->cov[rId]; + uLen[1] += cov->read_g->seq[rId].len; + } + + uDepth[0] = (uLen[0] == 0? 0 : uCov[1] / uLen[0]); + uDepth[1] = (uLen[1] == 0? 0 : uCov[0] / uLen[1]); + + for (i = 0; i < a_n[0]; i++) + { + rId = a[0][i]>>33; + cov->cov[rId] += (uDepth[0] * cov->read_g->seq[rId].len); + } + + for (i = 0; i < a_n[1]; i++) + { + rId = a[1][i]>>33; + cov->cov[rId] += (uDepth[1] * cov->read_g->seq[rId].len); + } +} + + + void purge_merge(asg_t *purge_g, ma_ug_t *ug, hap_overlaps_list* all_ovlp, buf_t* b_0, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, -kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edge, uint8_t* visit) +kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edge, uint8_t* visit, +hap_cov_t *cov) { uint32_t i, nv, k, v, w, x_beg_index, x_end_index, y_beg_index, y_end_index, cut_beg, cut_end, begIndex, endIndex, keepUid; hap_overlaps *x = NULL/**, *y = NULL**/; @@ -3780,7 +4032,6 @@ kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edg { for (k = 0; k < xReads->n; k++) { - ///aim[query->n - j - 1] = (query->a[j])^(uint64_t)(0x100000000); kv_push(uint64_t, buffer, (xReads->a[xReads->n - k - 1])^(uint64_t)(0x100000000)); } } @@ -3819,9 +4070,6 @@ kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edg endIndex = x->x_end_id-1; if(cut_end < endIndex) endIndex = cut_end; - // get_node_boundary(ruIndex, reverse_sources, coverage_cut, read_g, position_index, max_hang, - // min_ovlp, xReads, yReads, v>>1, w>>1, begIndex, endIndex, x->y_beg_id, x->y_end_id-1, v&1, - // x->rev, &t_forward, &t_backward); get_node_boundary_advance(ruIndex, reverse_sources, coverage_cut, read_g, position_index, max_hang, min_ovlp, xReads, yReads, v>>1, w>>1, begIndex, endIndex, x->y_beg_id, x->y_end_id-1, v&1, x->rev, u_buffer, tailIndex, prevIndex, &t_forward, &t_backward); @@ -3874,10 +4122,33 @@ kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edg } purge_g->seq[w>>1].c = ALTER_LABLE; + collect_trans_purge_joint_cov(cov, ug, x); + + // if(buffer.n > 1) + // { + // for (k = 0; k < buffer.n - 1; k++) + // { + // if((buffer.a[k]>>32) == 854769 && (buffer.a[k+1]>>32) == 64486) + // { + // fprintf(stderr, "+++++++v: %u, w: %u, xReads->n: %u, yReads->n: %u\n", + // v, w, xReads->n, yReads->n); + // fprintf(stderr, "x->rev: %u, x->x_beg_id: %u, x->x_end_id: %u, x->y_beg_id: %u, x->y_end_id: %u\n", + // x->rev, x->x_beg_id, x->x_end_id, x->y_beg_id, x->y_end_id); + // fprintf(stderr, "t_forward.ul>>32: %u, t_forward.v: %u, y_beg_index: %u, y_end_index: %u\n", + // t_forward.ul>>32, t_forward.v, y_beg_index, y_end_index); + // fprintf(stderr, "type: %u, x->x_beg_pos: %u, x->x_end_pos: %u, xReads->len: %u\n", + // x->type, x->x_beg_pos, x->x_end_pos, xReads->len); + // fprintf(stderr, "x->y_beg_pos: %u, x->y_end_pos: %u, yReads->len: %u\n", + // x->y_beg_pos, x->y_end_pos, yReads->len); + // } + // } + // } } - + // fprintf(stderr, "+keepUid: %u, i: %u, b_0->b.n: %u, buffer.n: %u\n", + // keepUid, i, (uint32_t)b_0->b.n, (uint32_t)buffer.n); fill_unitig(buffer.a, buffer.n, read_g, edge, 0, &totalLen); + ///fprintf(stderr, "-keepUid: %u\n", keepUid); xReads = &(ug->u.a[keepUid]); free(xReads->a); @@ -3922,11 +4193,8 @@ kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edg if(purge_g->seq[v>1].c != ALTER_LABLE) continue; asg_seq_drop(purge_g, v>1); } - } - - void collect_reverse_unitig_pair(hc_links* link, ma_ug_t *ug, hap_overlaps* t) { uint32_t i = 0, k = 0, rId_0, rId_1, pre_0, pre_1, b_0 = t->xUid, b_1 = t->yUid; @@ -3953,7 +4221,6 @@ void collect_reverse_unitig_pair(hc_links* link, ma_ug_t *ug, hap_overlaps* t) push_hc_edge(&(link->a.a[pre_1]), pre_0, 1, 1, &d); } } - } @@ -3975,7 +4242,7 @@ void link_unitigs(asg_t *purge_g, ma_ug_t *ug, hap_overlaps_list* all_ovlp, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edge, uint8_t* visit, -hc_links* link) +hc_links* link, hap_cov_t *cov) { uint32_t v, n_vtx = purge_g->n_seq * 2, beg, end; long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; @@ -3998,7 +4265,7 @@ hc_links* link) if(link) collect_reverse_unitigs_purge(&b_0, link, ug, all_ovlp); purge_merge(purge_g, ug, all_ovlp, &b_0, ruIndex, reverse_sources, coverage_cut, - read_g, position_index, u_buffer, tailIndex, prevIndex,max_hang, min_ovlp, edge, visit); + read_g, position_index, u_buffer, tailIndex, prevIndex,max_hang, min_ovlp, edge, visit, cov); } free(b_0.b.a); } @@ -4219,29 +4486,143 @@ uint32_t minLen, double purge_threshold) return 0; } +int cmp_chain_score(const void * a, const void * b) +{ + if((*(hap_overlaps*)a).score < (*(hap_overlaps*)b).score) return 1; + if((*(hap_overlaps*)a).score > (*(hap_overlaps*)b).score) return -1; + + return 0; +} +long long get_ovlp_len(long long a_beg, long long a_end, long long b_beg, long long b_end) +{ + long long ovlp = (long long)(MIN(a_end, b_end)) - (long long)(MAX(a_beg, b_beg)) + 1; + return ovlp <= 0? 0 : ovlp; +} +void sort_hap_chain(hap_overlaps_list* all_ovlp) +{ + hap_overlaps *x = NULL, *p = NULL; + uint32_t v, i, k, uId; + long long ovlp, xLen, pLen; + kvec_t(hap_overlaps) pri; kv_init(pri); + kvec_t(hap_overlaps) alt; kv_init(alt); + + for (v = 0; v < all_ovlp->num; v++) + { + uId = v; + qsort(all_ovlp->x[uId].a.a, all_ovlp->x[uId].a.n, sizeof(hap_overlaps), cmp_chain_score); + pri.n = alt.n = 0; + for (i = 0; i < all_ovlp->x[uId].a.n; i++) + { + x = &(all_ovlp->x[uId].a.a[i]); + xLen = x->x_end_pos - x->x_beg_pos; + for (k = 0; k < pri.n; k++) + { + p = &(pri.a[k]); + pLen = p->x_end_pos - p->x_beg_pos; + ovlp = get_ovlp_len(x->x_beg_pos, x->x_end_pos-1, p->x_beg_pos, p->x_end_pos-1); + if(ovlp == 0) continue; + if(ovlp >= (MIN(xLen, pLen))*0.5) break; + } + + if(k < pri.n) + { + x->xUid = k; + kv_push(hap_overlaps, alt, *x); + } + else + { + kv_push(hap_overlaps, pri, *x); + } + } + + } + + kv_destroy(pri); kv_destroy(alt); +} + +void remove_contained_haplotig(hap_overlaps_list* all_ovlp, ma_ug_t *ug, asg_t* nsg, asg_t *purge_g, hc_links* link, hap_cov_t *cov) +{ + uint32_t v, i, uId, xUid; + hap_overlaps *p = NULL; + for (v = 0; v < all_ovlp->num; v++) + { + uId = v; p = NULL; + for (i = 0; i < all_ovlp->x[uId].a.n; i++) + { + if(p == NULL || p->score < all_ovlp->x[uId].a.a[i].score) + { + p = &(all_ovlp->x[uId].a.a[i]); + } + } + + for (i = 0; i < all_ovlp->x[uId].a.n; i++) + { + if(all_ovlp->x[uId].a.a[i].type == YCX) + { + if(!filter_secondary_chain(p->score, all_ovlp->x[uId].a.a[i].score, 0.95)) + { + continue; + } + + xUid = all_ovlp->x[uId].a.a[i].xUid; + + nsg->seq[xUid].c = ALTER_LABLE; + purge_g->seq[xUid].c = ALTER_LABLE; + purge_g->seq[xUid].del = 1; + + all_ovlp->x[uId].a.a[i].status = DELETE; + if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp->x[uId].a.a[i])); + collect_trans_purge_cov(cov, ug, &(all_ovlp->x[uId].a.a[i]), 0); + } + + ///print_hap_paf(ug, &(all_ovlp.x[uId].a.a[i])); + } + } + + // for (v = 0; v < all_ovlp.num; v++) + // { + // uId = v; + // for (i = 0; i < all_ovlp.x[uId].a.n; i++) + // { + // if(all_ovlp.x[uId].a.a[i].type == YCX) + // { + // nsg->seq[all_ovlp.x[uId].a.a[i].xUid].c = ALTER_LABLE; + // purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].c = ALTER_LABLE; + // purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].del = 1; + // all_ovlp.x[uId].a.a[i].status = DELETE; + // if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp.x[uId].a.a[i])); + // collect_trans_purge_cov(cov, ug, &(all_ovlp.x[uId].a.a[i]), 0); + // } + + // if(all_ovlp.x[uId].a.a[i].type == XCY) + // { + // nsg->seq[all_ovlp.x[uId].a.a[i].yUid].c = ALTER_LABLE; + // purge_g->seq[all_ovlp.x[uId].a.a[i].yUid].c = ALTER_LABLE; + // purge_g->seq[all_ovlp.x[uId].a.a[i].yUid].del = 1; + // all_ovlp.x[uId].a.a[i].status = DELETE; + // if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp.x[uId].a.a[i])); + // collect_trans_purge_cov(cov, ug, &(all_ovlp.x[uId].a.a[i]), 1); + // } + // ///print_hap_paf(ug, &(all_ovlp.x[uId].a.a[i])); + // } + // } +} + + + void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage, hc_links* link) +uint32_t just_contain, uint32_t just_coverage, hc_links* link, hap_cov_t *cov) { asg_t *purge_g = NULL; purge_g = asg_init(); asg_t* nsg = ug->g; uint32_t v, rId, uId, i, offset; ma_utg_t* reads = NULL; - - // kvec_t_u64_warp u_vecs; - // kv_init(u_vecs.a); - // uint8_t* visit = NULL; - // visit = (uint8_t*)malloc(sizeof(uint8_t) * nsg->n_seq); - // memset(visit, 0, nsg->n_seq); - // uint64_t* vote_counting = (uint64_t*)malloc(sizeof(uint64_t)*nsg->n_seq); - // memset(vote_counting, 0, sizeof(uint64_t)*nsg->n_seq); - // kvec_asg_arc_t_offset u_buffer; - // kv_init(u_buffer.a); - // kvec_hap_candidates u_can; - // kv_init(u_can.a); - uint64_t* position_index = (uint64_t*)malloc(sizeof(uint64_t)*read_g->n_seq); + uint64_t* position_index = NULL; + if(cov) position_index = cov->pos_idx; + else position_index = (uint64_t*)malloc(sizeof(uint64_t)*read_g->n_seq); memset(position_index, -1, sizeof(uint64_t)*read_g->n_seq); hap_overlaps_list all_ovlp; @@ -4249,8 +4630,7 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link) hap_overlaps_list back_all_ovlp; init_hap_overlaps_list(&back_all_ovlp, nsg->n_seq); ///uint32_t junk_cov, hap_cov, dip_cov, junk_occ, repeat_occ, single_cov; - asg_arc_t t; - asg_arc_t* p = NULL; + asg_arc_t t, *p = NULL; int r; hap_alignment_struct_pip hap_buf; long long k_mer_only, coverage_only; @@ -4295,7 +4675,7 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link) init_hap_alignment_struct_pip(&hap_buf, asm_opt.thread_num, nsg->n_seq, ug, read_g, sources, reverse_sources, ruIndex, coverage_cut, position_index, density, max_hang, min_ovlp, - 0.05, &all_ovlp); + 0.1, &all_ovlp, cov); if(hap_buf.cov_threshold < 0) { @@ -4315,56 +4695,18 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link) fprintf(stderr, "[M::%s] purge duplication coverage threshold: %lld\n", __func__, hap_buf.cov_threshold); if(just_coverage) goto end_coverage; - ///kt_for(asm_opt.thread_num, hap_alignment_worker, &hap_buf, nsg->n_seq); kt_for(asm_opt.thread_num, hap_alignment_advance_worker, &hap_buf, nsg->n_seq); ///if(debug_enable) print_all_purge_ovlp(ug, &all_ovlp); - - - // for (v = 0; v < nsg->n_seq; v++) - // { - // uId = v; - // if(nsg->seq[uId].del || nsg->seq[uId].c == ALTER_LABLE) continue; - - // hap_alignment(ug, read_g, reverse_sources, ruIndex, coverage_cut, position_index, - // vote_counting, visit, &u_vecs, &u_buffer, &u_can, uId, density, max_hang, min_ovlp, - // 0.05, &all_ovlp); - // } - - filter_hap_overlaps_by_length(&all_ovlp, purege_minLen); ///normalize_hap_overlaps(&all_ovlp, &back_all_ovlp); normalize_hap_overlaps_advance(&all_ovlp, &back_all_ovlp, ug, read_g, reverse_sources, ruIndex); ///debug_hap_overlaps(&all_ovlp, &back_all_ovlp); + remove_contained_haplotig(&all_ovlp, ug, nsg, purge_g, link, cov); - for (v = 0; v < all_ovlp.num; v++) - { - uId = v; - for (i = 0; i < all_ovlp.x[uId].a.n; i++) - { - if(all_ovlp.x[uId].a.a[i].type == YCX) - { - nsg->seq[all_ovlp.x[uId].a.a[i].xUid].c = ALTER_LABLE; - purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].c = ALTER_LABLE; - purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].del = 1; - all_ovlp.x[uId].a.a[i].status = DELETE; - if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp.x[uId].a.a[i])); - } - - if(all_ovlp.x[uId].a.a[i].type == XCY) - { - nsg->seq[all_ovlp.x[uId].a.a[i].yUid].c = ALTER_LABLE; - purge_g->seq[all_ovlp.x[uId].a.a[i].yUid].c = ALTER_LABLE; - purge_g->seq[all_ovlp.x[uId].a.a[i].yUid].del = 1; - all_ovlp.x[uId].a.a[i].status = DELETE; - if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp.x[uId].a.a[i])); - } - ///print_hap_paf(ug, &(all_ovlp.x[uId].a.a[i])); - } - } if(just_contain == 0) { @@ -4375,6 +4717,7 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link) for (i = 0; i < all_ovlp.x[uId].a.n; i++) { if(all_ovlp.x[uId].a.a[i].status == DELETE) continue; + ///if(all_ovlp.x[uId].a.a[i].type == ) if(purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].c == ALTER_LABLE|| purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].del|| purge_g->seq[all_ovlp.x[uId].a.a[i].yUid].c == ALTER_LABLE|| @@ -4383,28 +4726,27 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link) continue; } + ///print_hap_paf(ug, &(all_ovlp.x[uId].a.a[i])); + r = get_hap_arch(&(all_ovlp.x[uId].a.a[i]), ug->u.a[all_ovlp.x[uId].a.a[i].xUid].len, ug->u.a[all_ovlp.x[uId].a.a[i].yUid].len, max_hang, asm_opt.max_hang_rate, min_ovlp, &t); - - if (r >= 0) - { - ///push node? - p = asg_arc_pushp(purge_g); - *p = t; - } - else - { - print_hap_paf(ug, &(all_ovlp.x[uId].a.a[i])); - fprintf(stderr, "error: uId: %u, i: %u, xUid: %u, yUid: %u\n", - uId, i, all_ovlp.x[uId].a.a[i].xUid, all_ovlp.x[uId].a.a[i].yUid); - } + + // if(all_ovlp.x[uId].a.a[i].xUid == 118 && all_ovlp.x[uId].a.a[i].yUid == 82) + // { + // fprintf(stderr, "r: %d\n", r); + // print_hap_paf(ug, &(all_ovlp.x[uId].a.a[i])); + // } + + if(r < 0) continue; + p = asg_arc_pushp(purge_g); + *p = t; } } asg_cleanup(purge_g); asg_symm(purge_g); - + ///may need to do transitive reduction clean_purge_graph(purge_g, bubble_dist, drop_ratio); // if(debug_enable) print_purge_gfa(ug, purge_g); @@ -4412,7 +4754,7 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link) link_unitigs(purge_g, ug, &all_ovlp, ruIndex, reverse_sources, coverage_cut, read_g, position_index, &(hap_buf.buf[0].u_buffer), &(hap_buf.buf[0].u_buffer_tailIndex), &(hap_buf.buf[0].u_buffer_prevIndex), - max_hang, min_ovlp, edge, hap_buf.buf[0].visit, link); + max_hang, min_ovlp, edge, hap_buf.buf[0].visit, link, cov); } for (v = 0; v < all_ovlp.num; v++) @@ -4437,12 +4779,130 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link) destory_hap_overlaps_list(&all_ovlp); destory_hap_overlaps_list(&back_all_ovlp); asg_destroy(purge_g); - free(position_index); - // kv_destroy(u_vecs.a); - // kv_destroy(u_buffer.a); - // kv_destroy(u_can.a); - // free(vote_counting); - // free(visit); + if(cov) memset(position_index, -1, sizeof(uint64_t)*read_g->n_seq); + else free(position_index); + destory_hap_alignment_struct_pip(&hap_buf); } +hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, +ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp) +{ + uint32_t n_ux = ug->g->n_seq, i, k, j, v, rId, tn, is_Unitig, r_i, nv, w, C_bases; + uint8_t *set = NULL; CALLOC(set, read_g->n_seq<<1); + hap_cov_t *x = NULL; CALLOC(x, 1); + x->n = read_g->n_seq; CALLOC(x->cov, x->n); + MALLOC(x->pos_idx, x->n); memset(x->pos_idx, -1, x->n*sizeof(uint64_t)); + x->reverse_sources = reverse_sources; + x->coverage_cut = coverage_cut; + x->ruIndex = ruIndex; + x->max_hang = max_hang; + x->min_ovlp = min_ovlp; + x->read_g = read_g; + kv_init(x->u_buffer.a); + kv_init(x->tailIndex.a); + kv_init(x->prevIndex.a); + ma_utg_t* u = NULL; + asg_arc_t *av = NULL; + ma_hit_t *h = NULL; + + for (i = 0; i < n_ux; i++) + { + if(ug->g->seq[i].del) continue; + u = &(ug->u.a[i]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; + + v = i<<1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; + } + + v = (i<<1)+1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; + } + + + + u = &(ug->u.a[i]); + for (k = 0; k < u->n; k++) + { + C_bases = 0; + rId = u->a[k]>>33; + for (j = 0; j < (uint64_t)(sources[rId].length); j++) + { + h = &(sources[rId].buffer[j]); + tn = Get_tn((*h)); + if(read_g->seq[tn].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + if(read_g->seq[tn].del == 1) continue; + if(!set[tn]) continue; + C_bases += (Get_qe((*h)) - Get_qs((*h))); + } + x->cov[rId] = MAX(C_bases, x->cov[rId]); + } + + + + u = &(ug->u.a[i]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; + + v = i<<1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; + } + + v = (i<<1)+1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; + } + } + + free(set); + return x; +} + +void destory_hap_cov_t(hap_cov_t **x) +{ + if(*x) + { + free((*x)->cov); + free((*x)->pos_idx); + kv_destroy((*x)->u_buffer.a); + kv_destroy((*x)->tailIndex.a); + kv_destroy((*x)->prevIndex.a); + free((*x)); + } +} \ No newline at end of file diff --git a/Purge_Dups.h b/Purge_Dups.h index 4635e22..9928624 100644 --- a/Purge_Dups.h +++ b/Purge_Dups.h @@ -11,14 +11,20 @@ #define HET_PEAK_RATE (HOM_PEAK_RATE*2) #define ALTER_COV_THRES 0.9 #define REAL_ALTER_THRES 0.1 +#define CHAIN_FILTER_RATE 0.7 void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage, hc_links* link); +uint32_t just_contain, uint32_t just_coverage, hc_links* link, hap_cov_t *cov); void fill_unitig(uint64_t* buffer, uint32_t bufferLen, asg_t* read_g, kvec_asg_arc_t_warp* edge, uint32_t is_circle, uint64_t* rLen); void get_contig_length(ma_ug_t *ug, asg_t *g, uint64_t* primaryLen, uint64_t* alterLen); void enable_debug_mode(uint32_t mode); +hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, +ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp); +void destory_hap_cov_t(hap_cov_t **x); +void chain_trans_ovlp(hap_cov_t *cov, ma_ug_t *ug, asg_t *read_sg, buf_t* xReads, uint32_t targetBaseLen, uint32_t* xEnd); + #endif \ No newline at end of file diff --git a/hic.cpp b/hic.cpp index e6e0621..f35724a 100644 --- a/hic.cpp +++ b/hic.cpp @@ -38,6 +38,7 @@ typedef struct{ kvec_t(uint64_t) name_Len; kvec_t(char) r; kvec_t(uint64_t) r_Len; + uint64_t idx; } reads_t; typedef struct{ @@ -167,6 +168,18 @@ typedef struct { // global data structure for kt_pipeline() uint64_t n_thread; } pldat_t; +typedef struct { + uint64_t *a, id; + uint16_t occ1, occ2; +} pe_hit_hap; + +typedef struct { + pe_hit_hap* a; + size_t n, m; + uint64_t n_u; +} kvec_pe_hit_hap; + + typedef struct { uint64_t s, e, id; } pe_hit; @@ -190,6 +203,18 @@ KRADIX_SORT_INIT(u32, uint32_t, generic_key, 4) #define g_partition_key(x) (((x)>>1)+((x)<<63)) KRADIX_SORT_INIT(g_partition, uint64_t, g_partition_key, 8) +#define get_pe_s(x) ((x).a[0]) +#define get_pe_e(x) ((x).a[(x).occ1]) +KRADIX_SORT_INIT(pe_an1, pe_hit_hap, get_pe_s, 8) +KRADIX_SORT_INIT(pe_an2, pe_hit_hap, get_pe_e, 8) + +#define pe_occ_key_1(x) ((x).occ1) +KRADIX_SORT_INIT(pe_occ1, pe_hit_hap, pe_occ_key_1, member_size(pe_hit_hap, occ1)) +#define pe_occ_key_2(x) ((x).occ2) +KRADIX_SORT_INIT(pe_occ2, pe_hit_hap, pe_occ_key_2, member_size(pe_hit_hap, occ2)) +#define pe_occ_key_t(x) (((uint64_t)((x).occ1))+((uint64_t)((x).occ2))) +KRADIX_SORT_INIT(pe_occ_t, pe_hit_hap, pe_occ_key_t, 8) + typedef struct { // global data structure for kt_pipeline() const ha_ug_index* idx; @@ -198,7 +223,8 @@ typedef struct { // global data structure for kt_pipeline() uint64_t n_thread; uint64_t total_base; uint64_t total_pair; - kvec_pe_hit hits; + ///kvec_pe_hit hits; + kvec_pe_hit_hap hits; hc_links* link; } sldat_t; @@ -218,7 +244,8 @@ typedef struct { // data structure for each step in kt_pipeline() char **seq; ch_buf_t *buf; kvec_vote* pos_buf; - pe_hit* pos; + ///pe_hit* pos; + pe_hit_hap* pos; hc_links* link; } stepdat_t; @@ -232,6 +259,8 @@ KRADIX_SORT_INIT(hc_pos, uint64_t, hc_pos_key, 8) KRADIX_SORT_INIT(hc_s_hit_an1, s_hit, hc_s_hit_an1_key, 8) #define hc_s_hit_an2_key(a) ((uint32_t)(a).off_cnt) KRADIX_SORT_INIT(hc_s_hit_an2, s_hit, hc_s_hit_an2_key, 8) +#define hc_s_hit_off_cnt_key(a) ((a).off_cnt) +KRADIX_SORT_INIT(hc_s_hit_off_cnt, s_hit, hc_s_hit_off_cnt_key, 8) #define hc_edge_key_u(a) ((a).uID) KRADIX_SORT_INIT(hc_edge_u, hc_edge, hc_edge_key_u, 4) #define hc_edge_key_d(a) ((a).dis) @@ -870,11 +899,12 @@ uint64_t debug_hash_value(char *r, uint64_t end, uint64_t k_mer) inline uint64_t collect_votes(s_hit* a, uint64_t n) { if(n == 0) return 0; - if(n == 1) return (a[0].off_cnt>>32); + if(n == 1) return (a[0].off_cnt>>32); //seed length long long i = 0; - uint64_t cur_beg, cur_end, beg, end, ovlp = 0, tLen = 0; + uint64_t cur_beg, cur_end, beg, end, ovlp = 0, tLen = 0; cur_end = (uint32_t)a[n-1].off_cnt; cur_beg = cur_end + 1 - (a[n-1].off_cnt>>32); + for (i = n - 2; i >= 0; i--) { @@ -1164,13 +1194,198 @@ const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) /*******************************for debug************************************/ } +uint64_t get_longest_hit(char *r, uint64_t len, uint64_t k_mer, uint64_t self_p, uint64_t self_rev, kvec_vote* buf, const ha_ug_index* idx, +uint64_t *pos_list, uint64_t cnt, uint64_t* c_sfx) +{ + uint64_t max_p, map_p_occ, i, j, m, rev, ref_p, u_len, uID, k_len; + s_hit *p = NULL; + ///each k-mer at different unitigs + ///rev:uID:pos + if(c_sfx) (*c_sfx) = (uint64_t)-1; + for (j = 0; j < cnt; j++) + { + ///get + kv_pushp(s_hit, buf->a, &p); + rev = (pos_list[j]>>63) != self_rev; + ref_p = pos_list[j] & idx->pos_mode; + uID = (pos_list[j] << 1) >> (64 - idx->uID_bits); + u_len = idx->ug->u.a[uID].len; + if(rev) ref_p = u_len - 1 - (ref_p + 1 - k_mer); + p->off_cnt = self_p | ((uint64_t)k_mer << 32); ///high bits should be the legnth + + p->ref = ref_p >= self_p? (ref_p-self_p) + : (self_p-ref_p) + ((uint64_t)1 << (idx->pos_bits - 1)); + p->ref = (rev << 63)|(pos_list[j] & idx->uID_mode)|(p->ref&idx->pos_mode); + + + ///extend + k_len = check_exact_match(r, self_p + 1, len, idx->ug->u.a[uID].s, ref_p + 1, u_len, len, rev, 0); + if(c_sfx && cnt == idx->hap_cnt && k_len < (*c_sfx)) (*c_sfx) = k_len; + + p->off_cnt += ((uint64_t)k_len << 32) + k_len; + + if(self_p >= k_mer && ref_p >= k_mer) + { + k_len = check_exact_match(r, self_p - k_mer, len, idx->ug->u.a[uID].s, + ref_p - k_mer, u_len, len, rev, 1); + p->off_cnt += ((uint64_t)k_len << 32); + } + // if(cnt > 0) fprintf(stderr, "inner j: %lu, rev: %lu, uID: %lu, ref_p: %lu, self_p: %u, len: %lu\n", j, rev, uID, ref_p, (uint32_t)p->off_cnt, p->off_cnt>>32); + } + + p = buf->a.a + buf->a.n - cnt; + if(cnt > 1) radix_sort_hc_s_hit_off_cnt(p, p + cnt); + max_p = map_p_occ = 0; + for (j = 1, i = 0; j <= cnt; ++j) + { + if(j == cnt || p[j].off_cnt != p[i].off_cnt) + { + if((max_p>>32) < (p[i].off_cnt>>32)) + { + max_p = p[i].off_cnt; + map_p_occ = j - i; + } + else if(((max_p>>32) == (p[i].off_cnt>>32)) && ((j - i) > map_p_occ)) + { + max_p = p[i].off_cnt; + map_p_occ = j - i; + } + i = j;///must + } + } + + buf->a.n -= cnt; + for (j = m = 0; j < cnt; j++) + { + if(p[j].off_cnt == max_p) + { + p[m] = p[j]; + m++; + } + } + cnt = m; + buf->a.n += cnt; + + // if(cnt > 0) fprintf(stderr, "max_p_offset: %u, max_p_len: %lu, map_p_occ: %lu\n", (uint32_t)max_p, max_p>>32, map_p_occ); + return max_p; +} + +#define is_update_hit(mL, mR, cL, cR) (((mL)<(cL))||((mL)==(cL)&&(mR)<(cR))) +inline void compress_mapped_pos_advance(const ha_ug_index* idx, kvec_vote* buf, uint64_t buf_iter, uint64_t ovlp_thre) +{ + if(buf_iter >= buf->a.n) + { + buf->a.n = buf_iter; + return; + } + s_hit *p = NULL; + uint64_t rev, uID, ref_p, self_p, eLen, tLen, i, j, cnt; + uint64_t max_beg = 0, max_end = 0, max_i, max_occ, cur_beg, cur_end, ovlp; + uint64_t second_i = (uint64_t)-1, second_occ; + uint64_t max_eLen, sec_eLen; + double max_eRate, sec_eRate, eRate; + p = buf->a.a + buf_iter; + cnt = buf->a.n - buf_iter; + radix_sort_hc_s_hit_off_cnt(p, p + cnt); ///buf save all hits, here sort by offset in reads + max_eLen = 0; max_i = (uint64_t)-1; max_occ = 0; max_eRate = 0; + for (j = 1, i = 0; j <= cnt; ++j) + { + if(j == cnt || p[j].off_cnt != p[i].off_cnt) + { + ///occ = j - i; + interpret_pos((ha_ug_index*)idx, &p[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + eRate = (double)(eLen)/(double)(tLen); + if(is_update_hit(max_eLen, max_eRate, eLen, eRate)) + { + max_eLen = eLen; max_eRate = eRate; + max_end = self_p; max_beg = self_p + 1 - tLen; + max_i = i; max_occ = j - i; + } + // fprintf(stderr, "\n++++++[%lu, %lu] uID: %lu, ref_p: %lu, self_p: %lu, eLen: %lu, tLen: %lu, max_i: %lu\n", + // i, j, uID, ref_p, self_p, eLen, tLen, max_i); + i = j;///must + } + } + + + sec_eLen = 0; second_i = (uint64_t)-1; second_occ = 0; sec_eRate = 0; + for (j = 1, i = 0; j <= cnt; ++j) + { + if(j == cnt || p[j].off_cnt != p[i].off_cnt) + { + if(i != max_i) + { + interpret_pos((ha_ug_index*)idx, &p[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + eRate = (double)(eLen)/(double)(tLen); + cur_end = self_p; + cur_beg = self_p + 1 - tLen; + // fprintf(stderr, "\n----[%lu, %lu] uID: %lu, ref_p: %lu, self_p: %lu, eLen: %lu, tLen: %lu, max_i: %lu\n", + // i, j, uID, ref_p, self_p, eLen, tLen, max_i); + // fprintf(stderr, "max_beg: %lu, max_end: %lu, cur_beg: %lu, cur_end: %lu\n", + // max_beg, max_end, cur_beg, cur_end); + ///overlap with max interval + if(MAX(cur_beg, max_beg) <= MIN(cur_end, max_end)) + { + ovlp = MIN(cur_end, max_end) - MAX(cur_beg, max_beg) + 1; + if(ovlp == MIN(max_end+1-max_end, tLen)) + { + i = j;///must + continue;///fully contain + } + + if(ovlp > ((max_end+1-max_end)*0.8) && eLen > (max_eLen*0.8))///best is not unique + { + buf->a.n = buf_iter; + return; + } + if(ovlp > ((max_end+1-max_end)*0.15)) + { + i = j;///must + continue;///fully contain + } + } + + if(is_update_hit(sec_eLen, sec_eRate, eLen, eRate)) + { + sec_eLen = eLen; sec_eRate = eRate; + second_i = i; second_occ = j - i; + } + } + i = j;///must + } + } + + // fprintf(stderr, "max_i: %lu, max_occ: %lu, second_i: %lu, second_occ: %lu\n", + // max_i, max_occ, second_i, second_occ); + + if(second_i == (uint64_t)-1) + { + i = 0; + for (j = max_i; j < max_i + max_occ; j++, i++) p[i] = p[j]; + } + else ///be carful about overwritten + { + i = 0; + if(max_i <= second_i) + { + for (j = max_i; j < max_i + max_occ; j++, i++) p[i] = p[j]; + for (j = second_i; j < second_i + second_occ; j++, i++) p[i] = p[j]; + } + else + { + for (j = second_i; j < second_i + second_occ; j++, i++) p[i] = p[j]; + for (j = max_i; j < max_i + max_occ; j++, i++) p[i] = p[j]; + } + } + + buf->a.n = buf_iter + max_occ + second_occ; +} void get_alignment(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) { - uint64_t i, j, l = 0, skip, *pos_list = NULL, cnt, rev, self_p, ref_p, u_len, uID; + uint64_t i, j, k, l = 0, k_len, c_sfx, m, skip, *pos_list = NULL, cnt, rev, self_p, ref_p, uID; uint64_t x[4], mask = (1ULL<a.n = 0; for (i = l = 0, x[0] = x[1] = x[2] = x[3] = 0; i < len; ++i) { int c = seq_nt4_table[(uint8_t)r[i]]; @@ -1186,144 +1401,75 @@ const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) { hash = hc_hash_long(x, &skip, k_mer); if(skip == (uint64_t)-1) continue; - /*******************************for debug************************************/ - // if(debug_hash_value(r, i, k_mer) != hash) - // { - // fprintf(stderr, "ERROR\n"); - // } - /*******************************for debug************************************/ cnt = get_hc_pt1_count((ha_ug_index*)idx, hash, &pos_list); - if(cnt > idx->hap_cnt) continue; - if(cnt != 1) continue; ///might be able to be disabled in future + if(cnt > idx->hap_cnt || cnt < 0) continue; - ///rev:uID:pos - for (j = 0; j < cnt; j++) + if(cnt > 1) { - kv_pushp(s_hit, buf->a, &p); - rev = (pos_list[j]>>63) != skip; - self_p = i; - ref_p = pos_list[j] & idx->pos_mode; - uID = (pos_list[j] << 1) >> (64 - idx->uID_bits); - u_len = idx->ug->u.a[uID].len; - if(rev) ref_p = u_len - 1 - (ref_p + 1 - k_mer); - p->off_cnt = self_p | ((uint64_t)k_mer << 32); ///high bits should be the legnth + for (j = 0; j < cnt; j++) + { + uID = (pos_list[j] << 1) >> (64 - idx->uID_bits); + for (k = j + 1; k < cnt; k++) + { + if(uID == ((pos_list[k] << 1) >> (64 - idx->uID_bits))) break; + } + if(k < cnt) break; + } - p->ref = ref_p >= self_p? (ref_p-self_p) - : (self_p-ref_p) + ((uint64_t)1 << (idx->pos_bits - 1)); - p->ref = (rev << 63)|(pos_list[j] & idx->uID_mode)|(p->ref&idx->pos_mode); - - - /*******************************for debug************************************/ - // if(check_exact_match(r, i + 1 - k_mer, len, - // idx->ug->u.a[uID].s, ref_p + 1 - k_mer, u_len, k_mer, rev, 0) != k_mer - // || - // check_exact_match(r, i, len, - // idx->ug->u.a[uID].s, ref_p, u_len, k_mer, rev, 1) != k_mer) - // { - // fprintf(stderr, "ERROR\n"); - // } - /*******************************for debug************************************/ + if(j < cnt) continue; } - - if(cnt == 1) + // if(cnt > 0) fprintf(stderr, "+i: %lu, l: %lu, cnt: %lu\n", i, l, cnt); + get_longest_hit(r, len, k_mer, i, skip, buf, idx, pos_list, cnt, &c_sfx); + // if(cnt > 0) fprintf(stderr, "c_sfx: %lu\n", c_sfx); + if(c_sfx != (uint64_t)-1) { - ///uint64_t debug_right = 0, debug_left = 0, debug_len; - - j = check_exact_match(r, self_p + 1, len, idx->ug->u.a[uID].s, ref_p + 1, u_len, len, rev, 0); - - ///debug_right = j; - ///if(j == 0) continue; - if((j + 1) >= k_mer) + k_len = c_sfx; + if((k_len + 1) >= k_mer) { l = 0, x[0] = x[1] = x[2] = x[3] = 0; - i = i + j - (k_mer - 1); + i = i + k_len - (k_mer - 1); } else { - ///l = i - (i + j - (k_mer - 1)); - l = k_mer - j -1; - } - buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32) + j; - - if(self_p >= k_mer && ref_p >= k_mer) - { - j = check_exact_match(r, self_p - k_mer, len, idx->ug->u.a[uID].s, - ref_p - k_mer, u_len, len, rev, 1); - buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32); - ///debug_left = j; - } - - - // debug_len = check_exact_match(r, self_p + debug_right, len, idx->ug->u.a[uID].s, - // ref_p + debug_right, u_len, len, rev, 1); - // if(debug_len!= (debug_left + debug_right + k_mer)) - // { - // fprintf(stderr, "debug_len: %lu, debug_left: %lu, debug_right: %lu\n", - // debug_len, debug_left, debug_right); - // } + ///l = i - (i + k_len - (k_mer - 1)); + l = k_mer - k_len - 1; + } } - + // if(cnt > 0) fprintf(stderr, "-i: %lu, l: %lu\n", i, l); } } else l = 0, x[0] = x[1] = x[2] = x[3] = 0; // if there is an "N", restart } - ///if(buf->a.n - buf_iter <= 1) return; if(buf->a.n - buf_iter == 0) return; if(buf->a.n - buf_iter > 1) radix_sort_hc_s_hit_an1(buf->a.a + buf_iter, buf->a.a + buf->a.n); - - /*******************************for debug************************************/ - // print_pos_list(idx, buf->a.a+buf_iter, buf->a.n - buf_iter, rid, (buf_iter != 0)); - // fprintf(stderr, "len0:%lu\n", buf->a.n - buf_iter); - // for (i = buf_iter; i < buf->a.n; i++) - // { - // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); - // fprintf(stderr, "(%lu) rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu, len: %lu\n", - // i, rev, uID, ref_p, self_p, cnt); - // } - /*******************************for debug************************************/ - - - - - - uint64_t cur_ref_p, thres = (len * HIC_R_E_RATE) + 1, m, index_beg, ovlp, maxLen = 0, max_i = (uint64_t)-1; + uint64_t cur_ref_p, thres = (len * HIC_R_E_RATE) + 1, index_beg, ovlp; i = m = buf_iter; while (i < buf->a.n) { interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); - /*******************************for debug************************************/ - // if(check_exact_match(r, self_p, len, idx->ug->u.a[uID].s, - // ref_p, idx->ug->u.a[uID].len, cnt, rev, 1) != cnt) - // { - // fprintf(stderr, "ERROR\n"); - // } - /*******************************for debug************************************/ - // if(self_p > ref_p) - // { - // i++; - // continue; ///fix this in future - // } + ///fprintf(stderr, "after-i: %lu, uID: %lu, ref_p: %lu, self_p: %lu\n", i, uID, ref_p, self_p); cur_ref_p = buf->a.a[i].ref; index_beg = i; - while ((i < buf->a.n) && ((buf->a.a[i].ref>>idx->pos_bits) == (cur_ref_p>>idx->pos_bits)) && + ///ref>>(idx->pos_bits-1) = (rev:1):(uID:uID-bits):(ref_pos>=self_pos:1) + while ((i < buf->a.n) && + ((buf->a.a[i].ref>>(idx->pos_bits-1)) == (cur_ref_p>>(idx->pos_bits-1))) && (buf->a.a[i].ref - cur_ref_p <= thres)) { i++; } if(i - index_beg > 1) { - radix_sort_hc_s_hit_an2(buf->a.a + index_beg, buf->a.a + i); + radix_sort_hc_s_hit_an2(buf->a.a + index_beg, buf->a.a + i);//sort by self_p } ovlp = collect_votes(buf->a.a + index_beg, i - index_beg); + ///fprintf(stderr, "i-1: %lu, self_p: %u\n", i-1, (uint32_t)buf->a.a[i - 1].off_cnt); buf->a.a[m] = buf->a.a[i - 1]; buf->a.a[m].off_cnt = (buf->a.a[m].off_cnt << 32)>>32; buf->a.a[m].off_cnt += ((uint64_t)ovlp<<32); - - if(maxLen < (ovlp&((uint64_t)65535))) maxLen = (ovlp&((uint64_t)65535)), max_i = m; - + ///fprintf(stderr, "m: %lu, self_p: %u\n", m, (uint32_t)buf->a.a[m].off_cnt); m++; } buf->a.n = m; @@ -1349,9 +1495,7 @@ const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) // i, rev, uID, ref_p, self_p, eLen, tLen); // } /*******************************for debug************************************/ - - compress_mapped_pos(idx, buf, buf_iter, max_i, thres); - + compress_mapped_pos_advance(idx, buf, buf_iter, (k_mer * 0.1) > 0? (k_mer * 0.1) : 1); /*******************************for debug************************************/ // fprintf(stderr, "len2:%lu, max_i: %lu\n", buf->a.n - buf_iter, max_i); // for (i = buf_iter; i < buf->a.n; i++) @@ -1394,6 +1538,7 @@ inline int is_unreliable_hits(long long rev, long long ref_p, long long tLen, ui return 0; } + inline void set_pe_pos(ha_ug_index* idx, s_hit *l1, uint64_t occ1, s_hit *l2, uint64_t occ2, pe_hit* x, uint64_t rid, hc_links* link) { @@ -1442,31 +1587,162 @@ pe_hit* x, uint64_t rid, hc_links* link) } } +void get_5_3_list(ha_ug_index* idx, s_hit* p, uint64_t cnt, s_hit** l5, uint64_t* l5_occ, +s_hit** l3, uint64_t* l3_occ) +{ + (*l5) = (*l3) = NULL; + (*l5_occ) = (*l3_occ) = 0; + uint64_t i, j, rev, uID, ref_p, self_p, eLen, tLen, cur_beg, num; + uint64_t beg_5 = (uint64_t)-1; + for (j = 1, i = 0, num = 0; j <= cnt; ++j) + { + if(j == cnt || p[j].off_cnt != p[i].off_cnt) + { + interpret_pos((ha_ug_index*)idx, &p[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + ///cur_end = self_p; + cur_beg = self_p + 1 - tLen; + num++; + if(cur_beg <= beg_5) + { + (*l3_occ) = (*l5_occ); (*l3) = (*l5); + beg_5 = cur_beg; (*l5_occ) = j - i; (*l5) = p + i; + } + else + { + (*l3_occ) = j - i; (*l3) = p + i; + } + i = j;///must + } + } + + ///if(num > 2) fprintf(stderr, "ERROR: get_5_3_list\n"); +} +inline void set_pe_pos_hap(ha_ug_index* idx, s_hit *l1, uint64_t occ1, s_hit *l2, uint64_t occ2, +pe_hit_hap* x, uint64_t rid, hc_links* link) +{ + if(occ1 == 0 || occ2 == 0) return; + uint64_t rev, uID, ref_p, self_p, eLen, tLen, i, is_unreliable = 0; + s_hit *l1_5 = NULL, *l1_3 = NULL, *l2_5 = NULL, *l2_3 = NULL; + uint64_t l1_5_occ = 0, l1_3_occ = 0, l2_5_occ = 0, l2_3_occ = 0; + + /***************************for debug******************************/ + // fprintf(stderr, "\nrid: %lu, occ1: %lu, occ2: %lu\n", rid, occ1, occ2); + // for (i = 0; i < occ1; i++) + // { + // interpret_pos(idx, &l1[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "***-1-rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu\n", + // rev, uID, ref_p, self_p); + // } + + // for (i = 0; i < occ2; i++) + // { + // interpret_pos(idx, &l2[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "***-2-rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu\n", + // rev, uID, ref_p, self_p); + // } + /***************************for debug******************************/ + + + get_5_3_list(idx, l1, occ1, &l1_5, &l1_5_occ, &l1_3, &l1_3_occ); + get_5_3_list(idx, l2, occ2, &l2_5, &l2_5_occ, &l2_3, &l2_3_occ); + if(l1_5_occ == 0 || l2_5_occ == 0) return; + x->id = rid; + MALLOC(x->a, l1_5_occ + l2_5_occ); + + x->occ1 = 0; + for (i = 0; i < l1_5_occ; i++) + { + interpret_pos(idx, &l1_5[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + if(ref_p < self_p) continue; + ref_p -= self_p; + if(rev) ref_p = idx->ug->u.a[uID].len - 1 - ref_p; + if(link && (is_unreliable_hits(rev, ref_p, tLen, uID, link))) + { + is_unreliable = 1; + continue; + } + x->a[x->occ1++] = (rev<<63) | ((uID << (64-idx->uID_bits))>>1) | (ref_p & idx->pos_mode); + } + + x->occ2 = x->occ1; + for (i = 0; i < l2_5_occ; i++) + { + interpret_pos(idx, &l2_5[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + if(ref_p < self_p) continue; + ref_p -= self_p; + if(rev) ref_p = idx->ug->u.a[uID].len - 1 - ref_p; + if(link && (is_unreliable_hits(rev, ref_p, tLen, uID, link))) + { + is_unreliable = 1; + continue; + } + x->a[x->occ2++] = (rev<<63) | ((uID << (64-idx->uID_bits))>>1) | (ref_p & idx->pos_mode); + } + x->occ2 -= x->occ1; + + if(x->occ1 == 0 || x->occ2 == 0 || is_unreliable) + { + free(x->a); x->occ1 = x->occ2 = 0; x->a = NULL; x->id = (uint64_t)-1; + return; + } + + + if(x->occ1 > 1) radix_sort_hc64(x->a, x->a + x->occ1); + if(x->occ2 > 1) radix_sort_hc64(x->a + x->occ1, x->a + x->occ1 + x->occ2); + + + /***************************for debug******************************/ + // fprintf(stderr, "-------------saved: x->occ1: %u, x->occ2: %u-------------\n", x->occ1, x->occ2); + // for (i = 0; i < x->occ1; i++) + // { + // fprintf(stderr, "###-1-rev: %lu, uID: %lu, ref_p: %lu\n", + // x->a[i]>>63, (x->a[i]<<1)>>(64-idx->uID_bits), x->a[i] & idx->pos_mode); + // } + + // for (i = 0; i < x->occ2; i++) + // { + // fprintf(stderr, "###-2-rev: %lu, uID: %lu, ref_p: %lu\n", + // x->a[i+x->occ1]>>63, (x->a[i+x->occ1]<<1)>>(64-idx->uID_bits), x->a[i+x->occ1] & idx->pos_mode); + // } + // fprintf(stderr, "-------------get_pe_s-rev: %lu, uID: %lu, ref_p: %lu-------------\n", + // get_pe_s(*x)>>63, (get_pe_s(*x)<<1)>>(64-idx->uID_bits), get_pe_s(*x) & idx->pos_mode); + // fprintf(stderr, "-------------get_pe_e-rev: %lu, uID: %lu, ref_p: %lu-------------\n", + // get_pe_e(*x)>>63, (get_pe_e(*x)<<1)>>(64-idx->uID_bits), get_pe_e(*x) & idx->pos_mode); + /***************************for debug******************************/ +} + +uint64_t if_debug_read(uint64_t rid) +{ + if(rid == 177 || rid == 439 || rid == 97 || rid == 114) + { + return 1; + } + return 0; +} + static void worker_for_alignment(void *data, long i, int tid) // callback for kt_for() { stepdat_t *s = (stepdat_t*)data; - s->pos[i].id = s->pos[i].s = s->pos[i].e = (uint64_t)-1; + s->pos[i].id = (uint64_t)-1; s->pos[i].occ1 = s->pos[i].occ2 = 0; s->pos[i].a = NULL; + + /*******************************for debug************************************/ + // if(!if_debug_read(s->id+i)) return; + // fprintf(stderr, "work-rid: %lu\n", (uint64_t)(s->id+i)); + /*******************************for debug************************************/ + uint64_t len1 = s->len[i]>>32, len2 = (uint32_t)s->len[i], occ1, occ2; char *r1 = s->seq[i], *r2 = s->seq[i] + len1; - /*******************************for debug************************************/ - // if(memcmp(r1, R1.r.a + R1.r_Len.a[s->id+i], len1) != 0) - // { - // fprintf(stderr, "haha1\n"); - // } - // if(memcmp(r2, R2.r.a + R2.r_Len.a[s->id+i], len2) != 0) - // { - // fprintf(stderr, "haha2\n"); - // } - /*******************************for debug************************************/ + // fprintf(stderr, "**********R1**********\n"); s->pos_buf[tid].a.n = 0; get_alignment(r1, len1, s->idx->k, &s->pos_buf[tid], s->idx, 0, s->id+i); occ1 = s->pos_buf[tid].a.n; if(occ1 == 0) return; + // fprintf(stderr, "**********R2**********\n"); get_alignment(r2, len2, s->idx->k, &s->pos_buf[tid], s->idx, occ1, s->id+i); occ2 = s->pos_buf[tid].a.n - occ1; if(occ2 == 0) return; - set_pe_pos((ha_ug_index*)s->idx, s->pos_buf[tid].a.a, occ1, s->pos_buf[tid].a.a + occ1, occ2, &(s->pos[i]), s->id+i, s->link); + set_pe_pos_hap((ha_ug_index*)s->idx, s->pos_buf[tid].a.a, occ1, s->pos_buf[tid].a.a + occ1, occ2, &(s->pos[i]), s->id+i, s->link); /*******************************for debug************************************/ // if(memcmp(r1, R1.r.a + R1.r_Len.a[s->id+i], len1) != 0) @@ -1532,7 +1808,7 @@ static void *worker_pipeline(void *data, int step, void *in) // callback for kt_ else if (step == 1) { // step 2: alignment stepdat_t *s = (stepdat_t*)in; CALLOC(s->pos_buf, p->n_thread); - MALLOC(s->pos, s->n); + CALLOC(s->pos, s->n); int i; kt_for(p->n_thread, worker_for_alignment, s, s->n); for (i = 0; i < s->n; ++i) { @@ -1551,8 +1827,8 @@ static void *worker_pipeline(void *data, int step, void *in) // callback for kt_ stepdat_t *s = (stepdat_t*)in; int i; for (i = 0; i < s->n; ++i) { - if(s->pos[i].s == (uint64_t)-1) continue; - kv_push(pe_hit, p->hits.a, s->pos[i]); + if(s->pos[i].a == NULL) continue; + kv_push(pe_hit_hap, p->hits, s->pos[i]); } free(s->pos); free(s); @@ -1561,38 +1837,53 @@ static void *worker_pipeline(void *data, int step, void *in) // callback for kt_ } -void load_reads(reads_t* x, const char *fn) +int load_reads(reads_t* x, const enzyme *fn1, const enzyme *fn2) { kv_init(x->name); kv_init(x->name_Len); kv_init(x->r); kv_init(x->r_Len); - gzFile fp; - kseq_t *ks; + int ret; uint64_t name_tot, base_total; - + int i; name_tot = base_total = 0; - if ((fp = gzopen(fn, "r")) == 0) return; - ks = kseq_init(fp); - while (((ret = kseq_read(ks)) >= 0)) + + for (i = 0; i < fn1->n && i < fn2->n; i++) { - kv_push(uint64_t, x->name_Len, name_tot); - kv_resize(char, x->name, name_tot + ks->name.l); - memcpy(x->name.a + name_tot, ks->name.s, ks->name.l); - name_tot += ks->name.l; + gzFile fp; + if ((fp = gzopen(fn1->a[i], "r")) == 0) + { + kv_destroy(x->name); + kv_destroy(x->name_Len); + kv_destroy(x->r); + kv_destroy(x->r_Len); + return 0; + } + + kseq_t *ks; + ks = kseq_init(fp); - kv_push(uint64_t, x->r_Len, base_total); - kv_resize(char, x->r, base_total + ks->seq.l); - memcpy(x->r.a + base_total, ks->seq.s, ks->seq.l); - base_total += ks->seq.l; + while (((ret = kseq_read(ks)) >= 0)) + { + kv_push(uint64_t, x->name_Len, name_tot); + kv_resize(char, x->name, name_tot + ks->name.l); + memcpy(x->name.a + name_tot, ks->name.s, ks->name.l); + name_tot += ks->name.l; + + kv_push(uint64_t, x->r_Len, base_total); + kv_resize(char, x->r, base_total + ks->seq.l); + memcpy(x->r.a + base_total, ks->seq.s, ks->seq.l); + base_total += ks->seq.l; + } + + kseq_destroy(ks); + gzclose(fp); } - kv_push(uint64_t, x->name_Len, name_tot); kv_push(uint64_t, x->r_Len, base_total); - - kseq_destroy(ks); - gzclose(fp); + x->idx = 0; + return 1; } @@ -1625,11 +1916,11 @@ void destory_reads(reads_t* x) kv_destroy(x->r_Len); } -void print_hits(ha_ug_index* idx, kvec_pe_hit* hits, const char *fn) +void print_hits(ha_ug_index* idx, kvec_pe_hit* hits, const enzyme *fn1, const enzyme *fn2) { uint64_t k, shif = 64 - idx->uID_bits; reads_t r1; - load_reads(&r1, fn); + load_reads(&r1, fn1, fn2); char dir[2] = {'+', '-'}; for (k = 0; k < hits->a.n; ++k) { @@ -1643,33 +1934,115 @@ void print_hits(ha_ug_index* idx, kvec_pe_hit* hits, const char *fn) destory_reads(&r1); } -void dedup_hits(kvec_pe_hit* hits) +inline void swap_pe_hit_hap(pe_hit_hap* x, pe_hit_hap* y) +{ + pe_hit_hap tmp; + tmp = (*x); (*x) = (*y); (*y) = tmp; +} + +void dedup_hits(kvec_pe_hit_hap* hits, const ha_ug_index* idx) { double index_time = yak_realtime(); - uint64_t k, l, m = 0, cur; - radix_sort_pe_hit_an1(hits->a.a, hits->a.a + hits->a.n); - for (k = 1, l = 0; k <= hits->a.n; ++k) - { - if (k == hits->a.n || hits->a.a[k].s != hits->a.a[l].s) + uint64_t k, l, m = 0, cur = (uint64_t)-1; + radix_sort_pe_an1(hits->a, hits->a + hits->n); + /***************************for debug******************************/ + // for (k = 0; k < hits->n; ++k) + // { + // for (l = k + 1; l < hits->n; l++) + // { + // if(get_pe_s(hits->a[k]) == get_pe_s(hits->a[l]) && + // get_pe_e(hits->a[k]) == get_pe_e(hits->a[l])) + // { + // fprintf(stderr, "DUP: k_id=%lu, l_id=%lu\n", hits->a[k].id, hits->a[l].id); + // } + // } + // } + + /** + fprintf(stderr, "\n\n\n\n\n\n\n\n\n\n*********************dedup_hits*********************\n"); + for (k = 0; k < hits->n; ++k) + { + pe_hit_hap *x = &(hits->a[k]); + fprintf(stderr, "\nsorted-rid: %lu, occ1: %u, occ2: %u\n", x->id, x->occ1, x->occ2); + fprintf(stderr, "---get_pe_s-rev: %lu, uID: %lu, ref_p: %lu---\n", + get_pe_s(*x)>>63, (get_pe_s(*x)<<1)>>(64-idx->uID_bits), get_pe_s(*x) & idx->pos_mode); + fprintf(stderr, "---get_pe_e-rev: %lu, uID: %lu, ref_p: %lu---\n", + get_pe_e(*x)>>63, (get_pe_e(*x)<<1)>>(64-idx->uID_bits), get_pe_e(*x) & idx->pos_mode); + uint64_t i; + for (i = 0; i < x->occ1; i++) { - if (k - l > 1) radix_sort_pe_hit_an2(hits->a.a + l, hits->a.a + k); - cur = (uint64_t)-1; + fprintf(stderr, "###-1-rev: %lu, uID: %lu, ref_p: %lu\n", + x->a[i]>>63, (x->a[i]<<1)>>(64-idx->uID_bits), x->a[i] & idx->pos_mode); + } + + for (i = 0; i < x->occ2; i++) + { + fprintf(stderr, "###-2-rev: %lu, uID: %lu, ref_p: %lu\n", + x->a[i+x->occ1]>>63, (x->a[i+x->occ1]<<1)>>(64-idx->uID_bits), x->a[i+x->occ1] & idx->pos_mode); + } + } + **/ + /***************************for debug******************************/ + for (k = 1, l = 0; k <= hits->n; ++k) + { + if (k == hits->n || get_pe_s(hits->a[k]) != get_pe_s(hits->a[l])) + { + if (k - l > 1) radix_sort_pe_an2(hits->a + l, hits->a + k); + ////fprintf(stderr, "\nl: %lu, k: %lu, %s\n", l, k, k - l > 1? "Found":"NONE"); + + cur = (uint64_t)-1; while (l < k) { - if(hits->a.a[l].e != cur) + if(get_pe_e(hits->a[l]) != cur) { - cur = hits->a.a[l].e; - hits->a.a[m++] = hits->a.a[l]; + cur = get_pe_e(hits->a[l]); + if(m != l) swap_pe_hit_hap(&hits->a[m], &hits->a[l]); + m++; } l++; } l = k; } } - hits->a.n = m; - fprintf(stderr, "[M::%s::%.3f] ==> Dedup\n", __func__, yak_realtime()-index_time); + + for (k = m; k < hits->n; k++) + { + hits->a[k].id = (uint64_t)-1; + hits->a[k].occ1 = hits->a[k].occ2 = 0; + free(hits->a[k].a); hits->a[k].a = NULL; + } + + radix_sort_pe_occ_t(hits->a, hits->a + m); + for (k = 0, hits->n_u = 0; k < m; k++) + { + if(hits->a[k].occ1 == 1 && hits->a[k].occ2 == 1) hits->n_u++; + } + + + fprintf(stderr, "[M::%s::%.3f] ==> Dedup (# dup: %lu, # non-dup: %lu, # non-dup-unique: %lu)\n", + __func__, yak_realtime()-index_time, (uint64_t)(hits->n - m), m, hits->n_u); + hits->n = m; } +void int_kvec_pe_hit_hap(kvec_pe_hit_hap* x) +{ + x->m = x->n = x->n_u = 0; + x->a = NULL; +} + +void destory_kvec_pe_hit_hap(kvec_pe_hit_hap* x) +{ + uint64_t k; + for (k = 0; k < x->n; k++) + { + x->a[k].id = (uint64_t)-1; + x->a[k].occ1 = x->a[k].occ2 = 0; + free(x->a[k].a); x->a[k].a = NULL; + } + free(x->a); +} + + void sort_hits(kvec_pe_hit* hits) { double index_time = yak_realtime(); @@ -1849,7 +2222,7 @@ void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link) if(ug->g->seq[v>>1].del) continue; if(asg_arc_n(ug->g, v) < 2) continue; if((bub->index[v]&(uint32_t)3) != 0) continue; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL)) { //beg is v, end is b.S.a[0] //note b.b include end, does not include beg @@ -1870,7 +2243,7 @@ void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link) for (v = 0; v < n_vtx; ++v) { if((bub->index[v]&(uint32_t)3) !=2) continue; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL)) + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL, NULL)) { //note b.b include end, does not include beg i = b.b.n + 1; @@ -1903,7 +2276,7 @@ void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link) if((bub->num.a[k]>>31) == 0) bub->s_bub++; v = (bub->num.a[k]<<1)>>1; bub->num.a[k] = bub->list.n; - if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL)) + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL, NULL)) { kv_push(uint64_t, bub->pathLen, pathLen); //beg is v, end is b.S.a[0] @@ -2026,7 +2399,7 @@ void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link) -void print_bubbles(ma_ug_t* ug, bubble_type* bub, kvec_pe_hit* hits, hc_links* link, ha_ug_index* idx) +void print_bubbles(ma_ug_t* ug, bubble_type* bub, kvec_pe_hit_hap* hits, hc_links* link, ha_ug_index* idx) { uint64_t tLen, t_utg, i, k; uint32_t beg, sink, n, *a; @@ -2070,10 +2443,10 @@ void print_bubbles(ma_ug_t* ug, bubble_type* bub, kvec_pe_hit* hits, hc_links* l uint64_t s_uid, e_uid, shif = 64 - idx->uID_bits; if(hits) { - for (k = 0; k < hits->a.n; ++k) + for (k = 0; k < hits->n_u; ++k) { - s_uid = ((hits->a.a[k].s<<1)>>shif); - e_uid = ((hits->a.a[k].e<<1)>>shif); + s_uid = ((get_pe_s(hits->a[k])<<1)>>shif); + e_uid = ((get_pe_e(hits->a[k])<<1)>>shif); if(bub->index[s_uid] == (uint32_t)-1 || bub->index[e_uid] == (uint32_t)-1) continue; if(IF_BUB(s_uid, *bub) && IF_BUB(e_uid, *bub)) { @@ -2803,14 +3176,14 @@ void destory_MT(MT* M) kv_destroy(M->matrix); } -void collect_hc_links(const ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, MT* M) +void collect_hc_links(const ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, MT* M) { double index_time = yak_realtime(); uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; - for (k = 0; k < hits->a.n; ++k) + for (k = 0; k < hits->n_u; ++k) { - beg = ((hits->a.a[k].s<<1)>>shif); - end = ((hits->a.a[k].e<<1)>>shif); + beg = ((get_pe_s(hits->a[k])<<1)>>shif); + end = ((get_pe_e(hits->a[k])<<1)>>shif); if(beg == end) continue; if(IF_HOM(beg, *bub)) continue; @@ -3092,22 +3465,179 @@ int load_hc_links(hc_links* link, const char *fn) } -void write_hc_hits(kvec_pe_hit* hits, const char *fn) +void write_hc_hits(kvec_pe_hit_hap* hits, const char *fn) { char *buf = (char*)calloc(strlen(fn) + 25, 1); sprintf(buf, "%s.hic.lk.bin", fn); FILE* fp = fopen(buf, "w"); - fwrite(&hits->a.n, sizeof(hits->a.n), 1, fp); - fwrite(hits->a.a, sizeof(pe_hit), hits->a.n, fp); - + uint64_t k; + fwrite(&hits->n_u, sizeof(hits->n_u), 1, fp); + fwrite(&hits->n, sizeof(hits->n), 1, fp); + for (k = 0; k < hits->n; k++) + { + fwrite(&hits->a[k].id, sizeof(hits->a[k].id), 1, fp); + fwrite(&hits->a[k].occ1, sizeof(hits->a[k].occ1), 1, fp); + fwrite(&hits->a[k].occ2, sizeof(hits->a[k].occ2), 1, fp); + fwrite(hits->a[k].a, sizeof(uint64_t), hits->a[k].occ1 + hits->a[k].occ2, fp); + } + fclose(fp); free(buf); } -int load_hc_hits(kvec_pe_hit* hits, const char *fn) +void write_hc_hits_v14(kvec_pe_hit_hap* i_hits, const char *fn) +{ + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.v14.hic.lk.bin", fn); + FILE* fp = fopen(buf, "w"); + kvec_pe_hit hits; + kv_init(hits.a); + uint64_t i, m_u = (uint64_t)-1, m_m = (uint64_t)-1; + pe_hit* p = NULL; + for (i = 0; i < i_hits->n; i++) + { + if(i_hits->a[i].occ1 == 1 && i_hits->a[i].occ2 == 1) + { + kv_pushp(pe_hit, hits.a, &p); + p->id = i_hits->a[i].id; + p->s = i_hits->a[i].a[0]; + p->e = i_hits->a[i].a[1]; + m_u = i; + } + else + { + if(m_m == (uint64_t)-1) m_m = i; + } + } + fprintf(stderr, "m_u: %lu, m_m: %lu, n_u: %lu\n", m_u, m_m, i_hits->n_u); + + fwrite(&hits.a.n, sizeof(hits.a.n), 1, fp); + fwrite(hits.a.a, sizeof(pe_hit), hits.a.n, fp); + + kv_destroy(hits.a); + fclose(fp); + free(buf); + exit(1); +} + +#define pe_hit_hap_id_key(x) ((x).id) +KRADIX_SORT_INIT(pe_hit_hap_id, pe_hit_hap, pe_hit_hap_id_key, member_size(pe_hit_hap, id)) + +#define pe_hit_id_key(x) ((x).id) +KRADIX_SORT_INIT(pe_hit_id, pe_hit, pe_hit_id_key, member_size(pe_hit, id)) + +void debug_hc_hits_v14(kvec_pe_hit_hap* i_hits, const char *fn, const ha_ug_index* idx) { uint64_t flag = 0; + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.v14.hic.lk.bin", fn); + kvec_pe_hit hits; + kv_init(hits.a); + FILE* fp = NULL; + fp = fopen(buf, "r"); + + kv_init(hits.a); + flag += fread(&hits.a.n, sizeof(hits.a.n), 1, fp); + hits.a.m = hits.a.n; MALLOC(hits.a.a, hits.a.n); + flag += fread(hits.a.a, sizeof(pe_hit), hits.a.n, fp); + + radix_sort_pe_hit_id(hits.a.a, hits.a.a + hits.a.n); + radix_sort_pe_hit_hap_id(i_hits->a, i_hits->a + i_hits->n_u); + + fprintf(stderr, "i_hits->n_u: %lu, hits.a.n: %lu\n", (uint64_t)i_hits->n_u, (uint64_t)hits.a.n); + + uint64_t i, k; + uint64_t i_beg_utg, i_beg_pos, i_beg_rev; + uint64_t i_end_utg, i_end_pos, i_end_rev; + uint64_t k_beg_utg, k_beg_pos, k_beg_rev; + uint64_t k_end_utg, k_end_pos, k_end_rev; + uint64_t i_id, k_id; + uint64_t same_occ = 0, diff_occ = 0, miss_occ = 0; + for (i = 0, k = 0; i < i_hits->n_u; i++) + { + i_beg_rev = get_pe_s(i_hits->a[i])>>63; + i_beg_utg = ((get_pe_s(i_hits->a[i])<<1)>>(64 - idx->uID_bits)); + i_beg_pos = get_pe_s(i_hits->a[i]) & idx->pos_mode; + + i_end_rev = get_pe_e(i_hits->a[i])>>63; + i_end_utg = ((get_pe_e(i_hits->a[i])<<1)>>(64 - idx->uID_bits)); + i_end_pos = get_pe_e(i_hits->a[i]) & idx->pos_mode; + + i_id = i_hits->a[i].id; + for (; k < hits.a.n; k++) + { + k_beg_rev = hits.a.a[k].s>>63; + k_beg_utg = ((hits.a.a[k].s<<1)>>(64 - idx->uID_bits)); + k_beg_pos = hits.a.a[k].s & idx->pos_mode; + + k_end_rev = hits.a.a[k].e>>63; + k_end_utg = ((hits.a.a[k].e<<1)>>(64 - idx->uID_bits)); + k_end_pos = hits.a.a[k].e & idx->pos_mode; + + k_id = hits.a.a[k].id; + + if(k_id > i_id) + { + miss_occ++; + fprintf(stderr, "\n[MISS]rid=%lu\n", i_id); + fprintf(stderr, "********v0.15********\n"); + fprintf(stderr, "beg_rev: %lu, beg_utg: %lu, beg_pos: %lu\n", + i_beg_rev, i_beg_utg, i_beg_pos); + fprintf(stderr, "end_rev: %lu, end_utg: %lu, end_pos: %lu\n", + i_end_rev, i_end_utg, i_end_pos); + break; + } + + if(k_id == i_id) + { + if(get_pe_s(i_hits->a[i]) == hits.a.a[k].s && get_pe_e(i_hits->a[i]) == hits.a.a[k].e) + { + same_occ++; + // fprintf(stderr, "\n[SAME]rid=%lu\n", i_id); + // fprintf(stderr, "********v0.15********\n"); + // fprintf(stderr, "beg_rev: %lu, beg_utg: %lu, beg_pos: %lu\n", + // i_beg_rev, i_beg_utg, i_beg_pos); + // fprintf(stderr, "end_rev: %lu, end_utg: %lu, end_pos: %lu\n", + // i_end_rev, i_end_utg, i_end_pos); + // fprintf(stderr, "********v0.14********\n"); + // fprintf(stderr, "beg_rev: %lu, beg_utg: %lu, beg_pos: %lu\n", + // k_beg_rev, k_beg_utg, k_beg_pos); + // fprintf(stderr, "end_rev: %lu, end_utg: %lu, end_pos: %lu\n", + // k_end_rev, k_end_utg, k_end_pos); + } + else + { + diff_occ++; + fprintf(stderr, "\n[DIFF]rid=%lu\n", i_id); + fprintf(stderr, "********v0.15********\n"); + fprintf(stderr, "beg_rev: %lu, beg_utg: %lu, beg_pos: %lu\n", + i_beg_rev, i_beg_utg, i_beg_pos); + fprintf(stderr, "end_rev: %lu, end_utg: %lu, end_pos: %lu\n", + i_end_rev, i_end_utg, i_end_pos); + fprintf(stderr, "********v0.14********\n"); + fprintf(stderr, "beg_rev: %lu, beg_utg: %lu, beg_pos: %lu\n", + k_beg_rev, k_beg_utg, k_beg_pos); + fprintf(stderr, "end_rev: %lu, end_utg: %lu, end_pos: %lu\n", + k_end_rev, k_end_utg, k_end_pos); + } + break; + } + } + } + + fprintf(stderr, "same_occ: %lu, diff_occ: %lu, miss_occ: %lu", same_occ, diff_occ, miss_occ); + + + kv_destroy(hits.a); + fclose(fp); + free(buf); + exit(1); +} + +int load_hc_hits(kvec_pe_hit_hap* hits, const char *fn) +{ + uint64_t flag = 0, k; char *buf = (char*)calloc(strlen(fn) + 25, 1); sprintf(buf, "%s.hic.lk.bin", fn); @@ -3115,10 +3645,19 @@ int load_hc_hits(kvec_pe_hit* hits, const char *fn) fp = fopen(buf, "r"); if(!fp) return 0; - kv_init(hits->a); - flag += fread(&hits->a.n, sizeof(hits->a.n), 1, fp); - hits->a.m = hits->a.n; MALLOC(hits->a.a, hits->a.n); - flag += fread(hits->a.a, sizeof(pe_hit), hits->a.n, fp); + kv_init(*hits); + flag += fread(&hits->n_u, sizeof(hits->n_u), 1, fp); + flag += fread(&hits->n, sizeof(hits->n), 1, fp); + hits->m = hits->n; MALLOC(hits->a, hits->n); + + for (k = 0; k < hits->n; k++) + { + flag += fread(&hits->a[k].id, sizeof(hits->a[k].id), 1, fp); + flag += fread(&hits->a[k].occ1, sizeof(hits->a[k].occ1), 1, fp); + flag += fread(&hits->a[k].occ2, sizeof(hits->a[k].occ2), 1, fp); + MALLOC(hits->a[k].a, hits->a[k].occ1 + hits->a[k].occ2); + flag += fread(hits->a[k].a, sizeof(uint64_t), hits->a[k].occ1 + hits->a[k].occ2, fp); + } fclose(fp); free(buf); @@ -4414,12 +4953,12 @@ G_partition* clean_bubbles(hc_links* link, bubble_type* bub, min_cut_t* m, const -uint64_t get_hic_distance(pe_hit* hit, hc_links* link, const ha_ug_index* idx) +uint64_t get_hic_distance(pe_hit_hap* hit, hc_links* link, const ha_ug_index* idx) { uint64_t s_uid, s_dir, e_uid, e_dir, u_dis, k; long long s_pos, e_pos; - s_uid = ((hit->s<<1)>>(64 - idx->uID_bits)); s_pos = hit->s & idx->pos_mode; - e_uid = ((hit->e<<1)>>(64 - idx->uID_bits)); e_pos = hit->e & idx->pos_mode; + s_uid = ((get_pe_s(*hit)<<1)>>(64 - idx->uID_bits)); s_pos = get_pe_s(*hit) & idx->pos_mode; + e_uid = ((get_pe_e(*hit)<<1)>>(64 - idx->uID_bits)); e_pos = get_pe_e(*hit) & idx->pos_mode; if(s_uid == e_uid) return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); hc_linkeage* t = &(link->a.a[s_uid]); for (k = 0; k < t->e.n; k++) @@ -4677,7 +5216,7 @@ void LeastSquare_advance(trans_idx* dis, ha_ug_index* idx, uint64_t med) } -void weight_edges(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub) +void weight_edges(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub) { uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; hc_edge *e1 = NULL, *e2 = NULL; @@ -4692,16 +5231,16 @@ void weight_edges(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_ty } } - for (k = 0; k < hits->a.n; ++k) + for (k = 0; k < hits->n_u; ++k) { - beg = ((hits->a.a[k].s<<1)>>shif); - end = ((hits->a.a[k].e<<1)>>shif); + beg = ((get_pe_s(hits->a[k])<<1)>>shif); + end = ((get_pe_e(hits->a[k])<<1)>>shif); if(beg == end) continue; if(IF_HOM(beg, *bub)) continue; if(IF_HOM(end, *bub)) continue; - t_d = get_hic_distance(&(hits->a.a[k]), link, idx); + t_d = get_hic_distance(&(hits->a[k]), link, idx); if(t_d == (uint64_t)-1) continue; e1 = get_hc_edge(link, beg, end, 0); @@ -4718,7 +5257,7 @@ void weight_edges(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_ty } -void weight_edges_advance(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, trans_idx* dis) +void weight_edges_advance(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, trans_idx* dis) { uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; hc_edge *e1 = NULL, *e2 = NULL; @@ -4733,16 +5272,16 @@ void weight_edges_advance(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, b } } - for (k = 0; k < hits->a.n; ++k) + for (k = 0; k < hits->n_u; ++k) { - beg = ((hits->a.a[k].s<<1)>>shif); - end = ((hits->a.a[k].e<<1)>>shif); + beg = ((get_pe_s(hits->a[k])<<1)>>shif); + end = ((get_pe_e(hits->a[k])<<1)>>shif); if(beg == end) continue; if(IF_HOM(beg, *bub)) continue; if(IF_HOM(end, *bub)) continue; - t_d = get_hic_distance(&(hits->a.a[k]), link, idx); + t_d = get_hic_distance(&(hits->a[k]), link, idx); if(t_d == (uint64_t)-1) continue; e1 = get_hc_edge(link, beg, end, 0); @@ -7747,7 +8286,7 @@ void get_forward_distance(uint32_t src, uint32_t dest, asg_t *sg, hc_links* link -int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, MT* M, H_partition* hap, trans_idx* dis) +int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, MT* M, H_partition* hap, trans_idx* dis) { kvec_t(uint64_t) buf, buf_idx; kv_init(buf); @@ -7766,16 +8305,16 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, buf.n = 0; - for (k = 0; k < hits->a.n; ++k) + for (k = 0; k < hits->n_u; ++k) { - beg = ((hits->a.a[k].s<<1)>>(64 - idx->uID_bits)); - end = ((hits->a.a[k].e<<1)>>(64 - idx->uID_bits)); + beg = ((get_pe_s(hits->a[k])<<1)>>(64 - idx->uID_bits)); + end = ((get_pe_e(hits->a[k])<<1)>>(64 - idx->uID_bits)); if(IF_HOM(beg, *bub)) continue; if(IF_HOM(end, *bub)) continue; - t_d = get_hic_distance(&(hits->a.a[k]), link, idx); + t_d = get_hic_distance(&(hits->a[k]), link, idx); if(t_d == (uint64_t)-1) continue; if(beg == end) { @@ -7809,7 +8348,6 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, if((buf.a[k]&1) == 1) f_idx = k; } buf.n = MIN(r_idx, f_idx); - for (k = 0; k < buf.n; k++) { if((buf.a[k]&1) == 1) @@ -7817,7 +8355,6 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, kv_push(uint64_t, buf_idx, buf.a[k]>>1); } } - uint64_t cutoff = buf_idx.n * 0.9, t = buf_idx.n * 0.005, pre, step; k = 0; if(cutoff >= t) k = cutoff - t; @@ -7829,14 +8366,12 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, pre = buf_idx.a[k]; i++; } - if(t_d == 0 || i == 0 || t == 0) { kv_destroy(buf); kv_destroy(buf_idx); return 0; } - step = (t_d/i)*20; if(step == 0) @@ -7845,7 +8380,6 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, kv_destroy(buf_idx); return 0; } - trans_p_t* p = NULL; dis->n = 0; uint64_t step_s = 0, step_e = step; @@ -7871,8 +8405,9 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, cnt[0] = cnt[1] = 0; } } + // fprintf(stderr, "-k: %lu, buf.n: %lu, buf.a[k]: %lu, step_s: %lu, step_e: %lu\n", + // k, (uint64_t)buf.n, (buf.a[k]>>1), step_s, step_e); } - if(cnt[0] > 0 || cnt[1] > 0) { kv_pushp(trans_p_t, *dis, &p); @@ -7882,7 +8417,6 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, p->cnt_1 = cnt[1]; } - uint64_t smooth_step = 20, k_i, cnt_0; if(dis->n > 0) med = dis->a[dis->n-1].end; for (k = 0; k+smooth_step < dis->n; k++) @@ -7898,7 +8432,6 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, break; } } - long long b_k = 0, b_i = 0, b_j, pass = 0; ///for (b_k = b_i = 0; b_k < (long long)dis->n; b_k++) @@ -7964,7 +8497,6 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, pass = 0; break; } - dis->n = b_i; if(dis->n == 0 || pass == 0) { @@ -7980,9 +8512,7 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, // dis->a[i].beg, dis->a[i].end, dis->a[i].cnt_0, dis->a[i].cnt_1, (double)(dis->a[i].cnt_1)/(double)(dis->a[i].cnt_1 + dis->a[i].cnt_0)); // } - LeastSquare_advance(dis, idx, med); - // fprintf(stderr, "idx->a: %f, idx->b: %f, idx->frac: %f, med: %lu\n", // (double)idx->a, (double)idx->b, (double)idx->frac, med); @@ -7991,7 +8521,6 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, kv_destroy(buf); kv_destroy(buf_idx); - if(idx->a < 0) idx->a = 0; if(idx->a == 0) { @@ -8002,25 +8531,24 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, idx->b = ((double)(dis->a[dis->n-1].cnt_1))/((double)(dis->a[dis->n-1].cnt_0 + dis->a[dis->n-1].cnt_1)); } - // fprintf(stderr, "idx->a: %f, idx->b: %f, idx->frac: %f, med: %lu\n", // (double)idx->a, (double)idx->b, (double)idx->frac, med); return 1; } -void init_hic_p(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, +void init_hic_p(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, kvec_hc_edge* back_hc_edge, MT* M, H_partition* hap, uint32_t ignore_dis) { uint64_t k, i, m, uID, is_comples_weight = 0; trans_idx dis; kv_init(dis); - + if(bub->round_id > 0 && ignore_dis == 0) { is_comples_weight = get_trans_rate_function(idx, hits, link, bub, M, hap, &dis); } - + hc_edge *e = NULL; for (i = 0; i < link->a.n; i++) @@ -8082,10 +8610,8 @@ kvec_hc_edge* back_hc_edge, MT* M, H_partition* hap, uint32_t ignore_dis) } link->a.a[i].e.n = m; } - weight_edges_advance(idx, hits, link, bub, is_comples_weight == 1? &dis : NULL); - for (i = 0; i < link->a.n; i++) { for (k = 0; k < link->a.a[i].e.n; k++) @@ -11340,7 +11866,7 @@ void print_bubble_chain(bubble_type* bub) } } -void init_contig_H_partition(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit* hits, H_partition* hap) +void init_contig_H_partition(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap* hits, H_partition* hap) { uint32_t i, k_i, k_j, uID, *a = NULL, n, *h0, h0_n, *h1, h1_n; destory_G_partition(&(hap->group_g_p)); memset(&(hap->group_g_p), 0, sizeof(G_partition)); @@ -11416,15 +11942,15 @@ void init_contig_H_partition(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit* hi label_unitigs(&(hap->group_g_p), idx->ug); } -void cluster_contigs(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit* hits, MT* M, H_partition* hap) +void cluster_contigs(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap* hits, MT* M, H_partition* hap) { uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; hc_links* link = idx->link; for (i = 0; i < link->a.n; i++) link->a.a[i].e.n = 0; - for (k = 0; k < hits->a.n; ++k) + for (k = 0; k < hits->n_u; ++k) { - beg = ((hits->a.a[k].s<<1)>>shif); - end = ((hits->a.a[k].e<<1)>>shif); + beg = ((get_pe_s(hits->a[k])<<1)>>shif); + end = ((get_pe_e(hits->a[k])<<1)>>shif); if(beg == end) continue; if(IF_HOM(beg, *bub)) continue; @@ -11468,6 +11994,7 @@ void reset_H_partition(H_partition* hap, uint32_t is_init) int alignment_worker_pipeline(sldat_t* sl, const enzyme *fn1, const enzyme *fn2) { + double index_time = yak_realtime(); int i; for (i = 0; i < fn1->n && i < fn2->n; i++) { @@ -11479,21 +12006,228 @@ int alignment_worker_pipeline(sldat_t* sl, const enzyme *fn1, const enzyme *fn2) kt_pipeline(3, worker_pipeline, sl, 3); - kseq_destroy(sl->ks1); kseq_destroy(sl->ks2); gzclose(fp1); gzclose(fp2); } + fprintf(stderr, "[M::%s::%.3f] ==> Qualification\n", __func__, yak_realtime()-index_time); - - dedup_hits(&(sl->hits)); - - ///fprintf(stderr, "-sl->hits.a.n: %u\n", (uint32_t)sl->hits.a.n); - + dedup_hits(&(sl->hits), sl->idx); return 1; } +/** +typedef struct{ + FILE* fp; + kvec_t(char) buf; + kvec_t(char) name; + kvec_t_u64_warp pos; +}pe_aln_t; + +int init_pe_aln_t(pe_aln_t* x, const char* aln) +{ + memset(x, 0, sizeof(*x)); + if(!strcmp(aln,"-")) x->fp = stdin; + else if ((x->fp = fopen(aln, "r")) == 0) return 0; + kv_malloc(x->buf, 10); x->buf.n = 0; + kv_malloc(x->name, 10); x->name.n = 0; + kv_init(x->pos.a); + return 1; +} + +void destory_pe_aln_t(pe_aln_t* x) +{ + fclose(x->fp); + kv_destroy(x->buf); + kv_destroy(x->name); + kv_destroy(x->pos.a); +} + +char* get_alnLine(pe_aln_t* x) +{ + uint64_t len; + uint64_t b_size = x->buf.m; + char* b = x->buf.a; + while (fgets(b, b_size, x->fp) != NULL) + { + len = strlen(x->buf.a); + if(x->buf.a[len - 1] == '\n') + { + x->buf.a[len - 1] = '\0'; + return x->buf.a; + } + kv_resize(char, x->buf, x->buf.m<<1); + b = x->buf.a + len; b_size = x->buf.m - len; + } + return NULL; +} + +uint64_t get_read_id_by_name(char* name, uint64_t name_len, reads_t* r1) +{ + uint64_t size = r1->r_Len.n - 1, r_len; + uint64_t end_idx = r1->idx; + char* r_char = NULL; + while(1) + { + r_len = r1->name_Len.a[r1->idx + 1] - r1->name_Len.a[r1->idx]; + r_char = r1->name.a + r1->name_Len.a[r1->idx]; + if(name_len == r_len && memcmp(r_char, name, r_len) == 0) return r1->idx; + r1->idx++; + if(r1->idx >= size) r1->idx = 0; + if(r1->idx == end_idx) break; + } + return (uint64_t)-1; +} +uint64_t get_utg_id_by_name(char* u_name) +{ + uint64_t i, len = strlen(u_name), id; + char c = u_name[len - 1]; + u_name[len - 1] = '\0'; + for (i = 3; i < len; i++) + { + if(u_name[i] != '0') break; + } + id = atoi(u_name + i); + u_name[len - 1] = c; + return id; +} + +uint64_t adjust_pos(uint64_t pos, uint64_t rev, char* cigar) +{ + long long i, occ = strlen(cigar); + + if(rev == 0) + { + for (i = 0; i < occ; i++) + { + if(cigar[i] < '0' || cigar[i] > '9') + { + break; + } + } + + if(cigar[i] == 'S') + { + cigar[i] = '\0'; + pos = pos + atoll(cigar); + cigar[i] = 'S'; + } + } + else + { + if(cigar[occ-1] == 'S') + { + cigar[occ-1] = '\0'; + for (i = occ-2; i >= 0; i--) + { + if(cigar[i] < '0' || cigar[i] > '9') + { + break; + } + } + pos = pos - atoll(cigar+i+1); + cigar[occ-1] = 'S'; + } + } + + return pos; +} + +uint64_t parse_sam(char *x, char** name, uint64_t* flag, uint64_t* uid, kvec_t_u64_warp* pos) +{ + uint64_t p_pos, p_err, n_len; + p->a.n = 0; + char *t = NULL; + + t = strtok (a, "\t\0");///name + n_len = strlen(t); + kv_resize(char, x->name, n_len+1); + memcpy(x->name, t, n_len+1); + (*name) = x->name; + + (*flag) = atoll(strtok (NULL, "\t\0"));//flag + if(!((*flag)&1) || ((*flag)&4) || ((*flag)&256) || ((*flag)&2048)) return 0; + + (*uid) = get_utg_id_by_name(strtok(NULL, "\t\0")); ///utg name + + p_pos = atoll(strtok(NULL, "\t\0")) - 1;//primary pos + + strtok(NULL, "\t\0");///MAPQ + + p_pos = adjust_pos(p_pos, !!((*flag)&16), strtok(NULL, "\t\0")); ///cigar + + strtok(NULL, "\t\0"); + strtok(NULL, "\t\0"); + strtok(NULL, "\t\0"); + strtok(NULL, "\t\0"); + strtok(NULL, "\t\0"); + + p_err = atoll(strtok(NULL, "\t\0") + 5); //NM:i: + + t = strtok(NULL, "\t\0"); + while (t != NULL) + { + n_len = strlen(t); + if(n_len > 5 && t[0] == 'X' && t[1] == 'A' && t[2] == ':' && t[3] == 'Z' && t[4] == ':') + { + break; + } + t = strtok(NULL, "\t\0"); + } +} + +uint64_t get_sam(pe_aln_t* x) +{ + char *a = x->buf.a, *t = NULL; + uint64_t n_len; + while (1) + { + if(x->buf.n == 0) + { + a = get_alnLine(x); + if(a == NULL) break; + x->buf.n = 1; + } + + ///parse_sam(char *x, char** name, uint64_t* flag, uint64_t* uid, kvec_t_u64_warp* pos) + + + } + + return 0; +} + + + +int debug_hits_sam(ha_ug_index* idx, kvec_pe_hit_hap* hits, const enzyme *fn1, const enzyme *fn2, +const char* aln) +{ + uint64_t k, id, uid, shif = 64 - idx->uID_bits, b_size = 100000, flag; + reads_t r1; + load_reads(&r1, fn1, fn2); r1.idx = 0; + pe_aln_t p; + init_pe_aln_t(&p, aln); + + + + + while (get_alnLine(p) != NULL) + { + str = strtok (buffer, "\t");///name + id = get_read_id_by_name(str, strlen(str), &r1); + if(id == (uint64_t)-1) fprintf(stderr, "ERROR\n"); + flag = atoi(strtok (NULL, "\t"));//flag + if(!(flag&1) || (flag&4) || (flag&256) || (flag&2048)) continue; + uid = get_utg_id_by_name(strtok (NULL, "\t")); ///utg name + + } + + destory_reads(&r1); + destory_pe_aln_t(&p); +} +**/ + int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) { double index_time = yak_realtime(); @@ -11506,7 +12240,8 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) sl.n_thread = asm_opt.thread_num; sl.total_base = sl.total_pair = 0; idx->hap_cnt = asm_opt.hap_occ; - kv_init(sl.hits.a); + int_kvec_pe_hit_hap(&sl.hits); + if(!load_hc_hits(&sl.hits, asm_opt.output_file_name)) { @@ -11528,6 +12263,9 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) write_hc_hits(&sl.hits, asm_opt.output_file_name); } + ///debug_hc_hits_v14(&sl.hits, asm_opt.output_file_name, sl.idx); + ////dedup_hits(&(sl.hits), sl.idx); + ///write_hc_hits_v14(&sl.hits, asm_opt.output_file_name); ///fprintf(stderr, "u.n: %d, uID_bits: %lu, pos_bits: %lu, sl.hits.a.n: %u\n", (uint32_t)idx->ug->u.n, idx->uID_bits, idx->pos_bits, (uint32_t)sl.hits.a.n); H_partition hap; @@ -11550,7 +12288,6 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) init_contig_partition(&hap, idx, &bub); phasing_improvement(&hap, &(hap.g_p), idx, &bub); label_unitigs(&(hap.g_p), idx->ug); - ///print_hc_links(idx->link, 0, &hap); } @@ -11581,13 +12318,14 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) destory_contig_partition(&hap); kv_destroy(back_hc_edge.a); + destory_kvec_pe_hit_hap(&sl.hits); return 1; /*******************************for debug************************************/ // destory_reads(&R1); // destory_reads(&R2); /*******************************for debug************************************/ - print_bubbles(idx->ug, &bub, sl.hits.a.n?&sl.hits:NULL, idx->link, idx); + print_bubbles(idx->ug, &bub, sl.hits.n?&sl.hits:NULL, idx->link, idx); collect_hc_reverse_links(idx->link, idx->ug, &bub); normalize_hc_links(idx->link); /*******************************for debug************************************/ @@ -11600,7 +12338,6 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) destory_min_cut_t(cut); free(cut); destory_G_partition(gp); free(gp); - kv_destroy(sl.hits.a); destory_bubbles(&bub); fprintf(stderr, "[M::%s::%.3f] processed %lu pairs; %lu bases\n", __func__, yak_realtime()-index_time, sl.total_pair, sl.total_base); @@ -11755,13 +12492,13 @@ void print_bench_idx(bench_idx* idx, ma_ug_t *ug) } -uint64_t get_hic_distance_bench(pe_hit* hit, hc_links* link, bench_idx* idx, ma_ug_t *ug, uint64_t* is_trans) +uint64_t get_hic_distance_bench(pe_hit_hap* hit, hc_links* link, bench_idx* idx, ma_ug_t *ug, uint64_t* is_trans) { (*is_trans) = (uint64_t)-1; uint64_t s_uid, e_uid; long long s_pos, e_pos; - s_uid = ((hit->s<<1)>>(64 - idx->uID_bits)); s_pos = hit->s & idx->pos_mode; - e_uid = ((hit->e<<1)>>(64 - idx->uID_bits)); e_pos = hit->e & idx->pos_mode; + s_uid = ((get_pe_s(*hit)<<1)>>(64 - idx->uID_bits)); s_pos = get_pe_s(*hit) & idx->pos_mode; + e_uid = ((get_pe_e(*hit)<<1)>>(64 - idx->uID_bits)); e_pos = get_pe_e(*hit) & idx->pos_mode; if(s_uid == e_uid) { (*is_trans) = 0; @@ -11829,14 +12566,14 @@ void init_bench_idx(bench_idx* idx, asg_t* read_g, ma_ug_t *ug) } } -void evaluate_bench_idx(bench_idx* idx, kvec_pe_hit* hits, ma_ug_t *ug) +void evaluate_bench_idx(bench_idx* idx, kvec_pe_hit_hap* hits, ma_ug_t *ug) { uint64_t k, distance, is_trans, trans[2]; kvec_t(uint64_t) buf; kv_init(buf); - for (k = trans[0] = trans[1] = 0; k < hits->a.n; ++k) + for (k = trans[0] = trans[1] = 0; k < hits->n_u; ++k) { - distance = get_hic_distance_bench(&(hits->a.a[k]), &(idx->link), idx, ug, &is_trans); + distance = get_hic_distance_bench(&(hits->a[k]), &(idx->link), idx, ug, &is_trans); if(is_trans != (uint64_t)-1) trans[is_trans]++; if(distance == (uint64_t)-1 || is_trans == (uint64_t)-1) continue; distance = (distance << 1) + is_trans; @@ -11904,7 +12641,7 @@ int hic_short_align_bench(const enzyme *fn1, const enzyme *fn2, const char *outp sl.n_thread = asm_opt.thread_num; sl.total_base = sl.total_pair = 0; idx->hap_cnt = asm_opt.hap_occ; - kv_init(sl.hits.a); + int_kvec_pe_hit_hap(&sl.hits); fprintf(stderr, "u.n: %d, uID_bits: %lu, pos_bits: %lu\n", (uint32_t)idx->ug->u.n, idx->uID_bits, idx->pos_bits); if(!load_hc_hits(&sl.hits, output_file_name)) @@ -11920,7 +12657,7 @@ int hic_short_align_bench(const enzyme *fn1, const enzyme *fn2, const char *outp evaluate_bench_idx(&bench, &sl.hits, idx->ug); destory_bench_idx(&bench); - kv_destroy(sl.hits.a); + destory_kvec_pe_hit_hap(&sl.hits); fprintf(stderr, "[M::%s::%.3f] processed %lu pairs; %lu bases\n", __func__, yak_realtime()-index_time, sl.total_pair, sl.total_base); return 1; } diff --git a/hifiasm.1 b/hifiasm.1 index 42bb851..245a7b1 100644 --- a/hifiasm.1 +++ b/hifiasm.1 @@ -286,7 +286,8 @@ times in the other sample. .TP 10 .BI -l \ INT Level of purge-dup. 0 to disable purge-dup, 1 to only purge contained haplotigs, -2 to purge all types of haplotigs. In default, [2] for non-trio assembly, [0] for trio assembly. +2 to purge all types of haplotigs, 3 to purge all types of haplotigs in most aggressive way. +In default, [2] for non-trio assembly, [0] for trio assembly. For trio assembly, only level 0 and level 1 are allowed. .TP From 4c2ce6fc6b039e84f75d13e7965d9fe2dcba2efb Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Wed, 10 Mar 2021 22:46:50 -0500 Subject: [PATCH 3/8] update trans chain --- CommandLines.cpp | 2 +- Overlaps.cpp | 270 +++++++--- Overlaps.h | 50 +- Purge_Dups.cpp | 135 +---- Purge_Dups.h | 6 +- hic.cpp | 1234 +++++++++++++++++++++++++++++++++++++++++----- 6 files changed, 1363 insertions(+), 334 deletions(-) diff --git a/CommandLines.cpp b/CommandLines.cpp index caea7e4..f69f93e 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -163,7 +163,7 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->b_low_cov = 0; asm_opt->b_high_cov = -1; asm_opt->m_rate = 0.75; - asm_opt->hap_occ = 2; + asm_opt->hap_occ = 1; } void destory_enzyme(enzyme* f) diff --git a/Overlaps.cpp b/Overlaps.cpp index 6f3ba54..a1a92dd 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -11328,7 +11328,7 @@ void collect_trans_cov(buf_t* pri, buf_t* aux, ma_ug_t *ug, asg_t *read_sg, hap_ } int untig_asg_arc_simple_large_bubbles_trio(ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negative_flag, hc_links* link, hap_cov_t *cov) +long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov) { asg_t *g = ug->g; double startTime = Get_T(); @@ -11452,7 +11452,7 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ asg_seq_drop(g, buffer.b.a[k]>>1); } - if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov->link) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); is_hap++; @@ -11645,13 +11645,13 @@ kvec_asg_arc_t_warp* new_rtg_edges, int max_hang, int min_ovlp) { uint64_t i, dip_thre_max, dip_thres, n_utg; uint8_t* primary_flag = (uint8_t*)calloc(sg->n_seq, sizeof(uint8_t)); - hap_cov_t *cov = init_hap_cov_t(ug, sg, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp); + hap_cov_t *cov = init_hap_cov_t(ug, sg, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, NULL); int tmp_cov = asm_opt.hom_global_coverage; asm_opt.hom_global_coverage = -1; purge_dups(ug, sg, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 0, 1, NULL, cov); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 0, 1, cov); dip_thre_max = ((double)asm_opt.hom_global_coverage)/((double)HOM_PEAK_RATE)*0.70; asm_opt.hom_global_coverage = tmp_cov; ///fprintf(stderr, "dip_thre_max: %lu\n", dip_thre_max); @@ -12523,7 +12523,7 @@ asg_t *read_sg, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, uint32_t min_e int asg_arc_cut_trio_long_tip_primary(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link, hap_cov_t *cov) +R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hap_cov_t *cov) { double startTime = Get_T(); ///the reason is that each read has two direction (query->target, target->query) @@ -12622,7 +12622,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link, hap } } - if(link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov->link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); if(cov && operation != CUT) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); } } @@ -12646,7 +12646,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link, hap } int asg_arc_cut_trio_long_tip_primary_complex(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_threshold, hc_links* link, hap_cov_t *cov) +R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_threshold, hap_cov_t *cov) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, in, flag, operation; @@ -12721,7 +12721,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_thre } } - if(link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov->link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); if(cov && operation != CUT) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); break; @@ -12822,7 +12822,7 @@ long long* base_maxLen, long long* base_maxLen_i, uint32_t stops_threshold, buf_ int asg_arc_cut_trio_long_equal_tips_assembly(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t trio_flag, -hc_links* link, hap_cov_t *cov) +hap_cov_t *cov) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag, is_hap, n_tips, return_flag, k; @@ -12909,7 +12909,7 @@ hc_links* link, hap_cov_t *cov) asg_seq_drop(g, b.b.a[k]>>1); } - if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov->link) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); is_hap++; @@ -13143,7 +13143,7 @@ R_to_U* ruIndex, uint32_t positive_flag, float drop_rate) } int asg_arc_cut_trio_long_equal_tips_assembly_complex(asg_t *g, ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t stops_threshold, hc_links* link, hap_cov_t *cov) +ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t stops_threshold, hap_cov_t *cov) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, in, flag; @@ -13211,7 +13211,7 @@ ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_ asg_seq_drop(g, b.b.a[k]>>1); } - if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + if(cov->link) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); ///lable the primary one @@ -13606,7 +13606,7 @@ ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, -float drop_ratio, uint32_t trio_flag, float trio_drop_rate) +float drop_ratio, uint32_t trio_flag, float trio_drop_rate, hap_cov_t *cov) { asg_t *g = ug->g; uint32_t is_first = 1; @@ -13614,15 +13614,14 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) redo: ///print_untig((ug), 61955, "i-0:", 0); - asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, NULL); - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL, NULL); + asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, cov); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, cov); magic_trio_phasing(g, ug, read_g, coverage_cut, sources, reverse_sources, 2, ruIndex, trio_flag, trio_drop_rate); ///drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); /**********debug**********/ if(just_bubble_pop == 0) { - cut_trio_tip_primary(g, ug, tipsLen, trio_flag, 0, read_g, reverse_sources, ruIndex, - 2); + cut_trio_tip_primary(g, ug, tipsLen, trio_flag, 0, read_g, reverse_sources, ruIndex, 2); } /**********debug**********/ long long pre_cons = get_graph_statistic(g); @@ -13631,30 +13630,28 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) { pre_cons = get_graph_statistic(g); ///need consider tangles - asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, NULL); + asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, cov); /**********debug**********/ if(just_bubble_pop == 0) { ///need consider tangles - asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, NULL, NULL); - asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, trio_flag, NULL, NULL); - asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, NULL, NULL); - asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, NULL, NULL); + asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, cov); + asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, trio_flag, cov); + asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, cov); + asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, cov); ///print_debug_gfa(read_g, ug, coverage_cut, "debug_chimeric", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); - detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, - ruIndex); + detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); ///need consider tangles ///note we need both the read graph and the untig graph } /**********debug**********/ cur_cons = get_graph_statistic(g); } - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL, NULL); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, cov); if(just_bubble_pop == 0) { - cut_trio_tip_primary(g, ug, tipsLen, trio_flag, 0, read_g, reverse_sources, ruIndex, - 2); + cut_trio_tip_primary(g, ug, tipsLen, trio_flag, 0, read_g, reverse_sources, ruIndex, 2); } resolve_tangles(ug, read_g, reverse_sources, 20, 100, 0.05, 0.2, ruIndex, trio_flag, drop_ratio); @@ -13674,7 +13671,7 @@ void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* rever long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, -float drop_ratio, hc_links* link, hap_cov_t *cov) +float drop_ratio, hap_cov_t *cov) { #define T_ROUND 2 asg_t *g = ug->g; @@ -13683,7 +13680,7 @@ float drop_ratio, hc_links* link, hap_cov_t *cov) redo: asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, cov); - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link, cov); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, cov); if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, @@ -13700,10 +13697,10 @@ float drop_ratio, hc_links* link, hap_cov_t *cov) if(just_bubble_pop == 0) { ///need consider tangles - asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, link, cov); - asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, link, cov); - asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, link, cov); - asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, link, cov); + asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, cov); + asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, cov); + asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, cov); + asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, cov); detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); if(round != T_ROUND) { @@ -13713,7 +13710,7 @@ float drop_ratio, hc_links* link, hap_cov_t *cov) } cur_cons = get_graph_statistic(g); } - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link, cov); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, cov); if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, @@ -14375,11 +14372,11 @@ kvec_asg_arc_t_warp* new_rtg_edges) { asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq; - hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp); + hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, NULL); purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 1, NULL, cov); + drop_ratio, 1, 1, cov); if(asm_opt.recover_atg_cov_min == -1024) { asm_opt.recover_atg_cov_max = asm_opt.hom_global_coverage/HOM_PEAK_RATE; @@ -14415,8 +14412,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) EvaluateLen((*ug)->u, v) = (*ug)->u.a[v].n; } clean_trio_untig_graph(*ug, read_g, coverage_cut, sources, reverse_sources, bubble_dist, - tipsLen, tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, - chimeric_rate, 0, 0, drop_ratio, flag, drop_rate); + tipsLen, tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, flag, drop_rate, cov); ///if(flag == MOTHER) fprintf(stderr, "(o.1) c: %u, del: %u, n: %u\n", (*ug)->g->seq[28141].c, (*ug)->g->seq[28141].del, (*ug)->u.a[28141].n); @@ -14460,7 +14456,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 0, NULL, cov); + drop_ratio, 1, 0, cov); ///delete_useless_nodes(ug); delete_useless_trio_nodes(ug, read_g, coverage_cut, sources, ruIndex); } @@ -20825,6 +20821,150 @@ char* output_file_name, ma_hit_t_alloc** reverse_sources, R_to_U* ruIndex) } +hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, +ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp, hc_links* link) +{ + uint32_t n_ux = ug->g->n_seq, i, k, j, v, rId, tn, is_Unitig, r_i, nv, w, C_bases; + uint8_t *set = NULL; + hap_cov_t *x = NULL; CALLOC(x, 1); + x->n = read_g->n_seq; + x->reverse_sources = reverse_sources; + x->coverage_cut = coverage_cut; + x->ruIndex = ruIndex; + x->max_hang = max_hang; + x->min_ovlp = min_ovlp; + x->read_g = read_g; + x->link = link; + kv_init(x->u_buffer.a); + kv_init(x->tailIndex.a); + kv_init(x->prevIndex.a); + ma_utg_t* u = NULL; + asg_arc_t *av = NULL; + ma_hit_t *h = NULL; + MALLOC(x->pos_idx, x->n); memset(x->pos_idx, -1, x->n*sizeof(uint64_t)); + CALLOC(set, read_g->n_seq<<1); + CALLOC(x->cov, x->n); + for (i = 0; i < n_ux; i++) + { + if(ug->g->seq[i].del) continue; + u = &(ug->u.a[i]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; + + v = i<<1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; + } + + v = (i<<1)+1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; + } + + + + u = &(ug->u.a[i]); + for (k = 0; k < u->n; k++) + { + C_bases = 0; + rId = u->a[k]>>33; + for (j = 0; j < (uint64_t)(sources[rId].length); j++) + { + h = &(sources[rId].buffer[j]); + tn = Get_tn((*h)); + if(read_g->seq[tn].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + if(read_g->seq[tn].del == 1) continue; + if(!set[tn]) continue; + C_bases += (Get_qe((*h)) - Get_qs((*h))); + } + x->cov[rId] = MAX(C_bases, x->cov[rId]); + } + + + + u = &(ug->u.a[i]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; + + v = i<<1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; + } + + v = (i<<1)+1; + nv = asg_arc_n(ug->g, v); + av = asg_arc_a(ug->g, v); + for (k = 0; k < nv; k++) + { + w = av[k].v; + if(av[k].del) continue; + if(ug->g->seq[w>>1].del) continue; + u = &(ug->u.a[w>>1]); + for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; + } + } + + if(set) free(set); + + + if(x->link) + { + memset(x->link->u_idx, -1, read_g->n_seq*sizeof(uint32_t)); + asg_t* nsg = ug->g; + uint32_t n_vtx = nsg->n_seq; + for (v = 0; v < n_vtx; ++v) + { + if(nsg->seq[v].del) continue; + u = &(ug->u.a[v]); + if(u->m == 0) continue; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + ///if(read_g->seq[rId].c == FAKE_LABLE) continue; + x->link->u_idx[rId] = v; + } + } + } + + return x; +} + +void destory_hap_cov_t(hap_cov_t **x) +{ + if(*x) + { + free((*x)->cov); + free((*x)->pos_idx); + kv_destroy((*x)->u_buffer.a); + kv_destroy((*x)->tailIndex.a); + kv_destroy((*x)->prevIndex.a); + free((*x)); + } +} void print_utg_hap(ma_ug_t *ug, asg_t* read_g, uint32_t uid, ma_hit_t_alloc* reverse_sources, @@ -21096,33 +21236,15 @@ void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, -kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) +kvec_asg_arc_t_warp* new_rtg_edges, hc_links* lk) { asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq, k, rId, just_contain; ma_utg_t* u = NULL; - hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp); + hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, lk); ///print_utg_coverage(*ug, coverage_cut, 440, sources); ///exit(0); - if(link) - { - memset(link->u_idx, -1, R_INF.total_reads*sizeof(uint32_t)); - nsg = (*ug)->g; - n_vtx = nsg->n_seq; - for (v = 0; v < n_vtx; ++v) - { - if(nsg->seq[v].del) continue; - u = &((*ug)->u.a[v]); - if(u->m == 0) continue; - for (k = 0; k < u->n; k++) - { - rId = u->a[k]>>33; - ///if(read_g->seq[rId].c == FAKE_LABLE) continue; - link->u_idx[rId] = v; - } - } - } drop_semi_circle((*ug), nsg, read_g, reverse_sources, ruIndex); asg_cleanup(nsg); @@ -21136,16 +21258,18 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) EvaluateLen((*ug)->u, v) = (*ug)->u.a[v].n; } clean_primary_untig_graph(*ug, read_g, reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, - stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, link, cov); + stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); + if(cov->link) goto skip_purge; + if(asm_opt.purge_level_primary > 0) { just_contain = 0; if(asm_opt.purge_level_primary == 1) just_contain = 1; purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0, link, cov); + drop_ratio, just_contain, 0, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } @@ -21165,7 +21289,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) if(asm_opt.purge_level_primary == 1) just_contain = 1; purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0, link, cov); + drop_ratio, just_contain, 0, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } @@ -21175,7 +21299,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 0, 1, link, cov); + drop_ratio, 0, 1, cov); } n_vtx = read_g->n_seq; @@ -21227,21 +21351,23 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) fprintf(stderr, "[M::%s] primary contig coverage range: [%d, infinity]\n", __func__, asm_opt.recover_atg_cov_min); } - - recover_utg_by_coverage(ug, read_g, coverage_cut, sources, ruIndex, link); - - if(link) + + skip_purge: + recover_utg_by_coverage(ug, read_g, coverage_cut, sources, ruIndex, lk); + + + if(lk) { uint32_t m; - for (v = 0; v < link->a.n; v++) + for (v = 0; v < lk->a.n; v++) { - for (k = m = 0; k < link->a.a[v].f.n; k++) + for (k = m = 0; k < lk->a.a[v].f.n; k++) { - if(link->a.a[v].f.a[k].del) continue; - link->a.a[v].f.a[m] = link->a.a[v].f.a[k]; + if(lk->a.a[v].f.a[k].del) continue; + lk->a.a[v].f.a[m] = lk->a.a[v].f.a[k]; m++; } - link->a.a[v].f.n = m; + lk->a.a[v].f.n = m; } } diff --git a/Overlaps.h b/Overlaps.h index 3e3b0c2..501da23 100644 --- a/Overlaps.h +++ b/Overlaps.h @@ -503,24 +503,10 @@ typedef struct { uint64_t i; }kvec_asg_arc_t_offset; -typedef struct { - uint32_t n; - uint32_t* cov; - uint64_t* pos_idx; - ma_hit_t_alloc* reverse_sources; - ma_sub_t *coverage_cut; - R_to_U* ruIndex; - asg_t *read_g; - int max_hang; - int min_ovlp; - kvec_asg_arc_t_offset u_buffer; - kvec_t_i32_warp tailIndex; - kvec_t_i32_warp prevIndex; -}hap_cov_t; + void init_Edge_iter(asg_t* g, uint32_t v, asg_arc_t* new_edges, uint32_t new_edges_n, Edge_iter* x); int get_arc_t(Edge_iter* x, asg_arc_t* get); -int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov); inline int get_real_length(asg_t *g, uint32_t v, uint32_t* v_s) @@ -1065,8 +1051,6 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t bac uint32_t is_bubble_check, uint32_t is_primary_check); uint32_t get_edge_from_source(ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t target, asg_arc_t* t); -uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, uint32_t positive_flag, -uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, hap_cov_t *cov); int unitig_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio); void asg_bub_backtrack_primary(asg_t *g, uint32_t v0, buf_t *b); @@ -1101,6 +1085,31 @@ typedef struct{ uint64_t r_num; } hc_links; +typedef struct{ + kvec_t(uint32_t) uIDs; + kvec_t(uint32_t) idx; + uint32_t chain_num; + uint32_t* u_idx; + uint64_t r_num; +}trans_chain; + +typedef struct { + uint32_t n; + uint32_t* cov; + uint64_t* pos_idx; + ma_hit_t_alloc* reverse_sources; + ma_sub_t *coverage_cut; + R_to_U* ruIndex; + asg_t *read_g; + int max_hang; + int min_ovlp; + kvec_asg_arc_t_offset u_buffer; + kvec_t_i32_warp tailIndex; + kvec_t_i32_warp prevIndex; + hc_links* link; + ///trans_chain t_ch; +}hap_cov_t; + typedef struct{ ///kvec_t(hc_edge) a; size_t n, m; @@ -1109,6 +1118,10 @@ typedef struct{ void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num); void destory_hc_links(hc_links* link); +int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov); +uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, uint32_t positive_flag, +uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, hap_cov_t *cov); + void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, @@ -1124,6 +1137,9 @@ asg_t* copy_read_graph(asg_t *src); ma_ug_t *ma_ug_gen(asg_t *g); void ma_ug_destroy(ma_ug_t *ug); + + + inline int inter_interval(int a_s, int a_e, int b_s, int b_e, int* i_s, int* i_e) { if(a_s > b_e || b_s > a_e) return 0; diff --git a/Purge_Dups.cpp b/Purge_Dups.cpp index d4920be..46c7cc7 100644 --- a/Purge_Dups.cpp +++ b/Purge_Dups.cpp @@ -4242,7 +4242,7 @@ void link_unitigs(asg_t *purge_g, ma_ug_t *ug, hap_overlaps_list* all_ovlp, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edge, uint8_t* visit, -hc_links* link, hap_cov_t *cov) +hap_cov_t *cov) { uint32_t v, n_vtx = purge_g->n_seq * 2, beg, end; long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; @@ -4263,7 +4263,7 @@ hc_links* link, hap_cov_t *cov) continue; } - if(link) collect_reverse_unitigs_purge(&b_0, link, ug, all_ovlp); + if(cov->link) collect_reverse_unitigs_purge(&b_0, cov->link, ug, all_ovlp); purge_merge(purge_g, ug, all_ovlp, &b_0, ruIndex, reverse_sources, coverage_cut, read_g, position_index, u_buffer, tailIndex, prevIndex,max_hang, min_ovlp, edge, visit, cov); } @@ -4540,7 +4540,7 @@ void sort_hap_chain(hap_overlaps_list* all_ovlp) kv_destroy(pri); kv_destroy(alt); } -void remove_contained_haplotig(hap_overlaps_list* all_ovlp, ma_ug_t *ug, asg_t* nsg, asg_t *purge_g, hc_links* link, hap_cov_t *cov) +void remove_contained_haplotig(hap_overlaps_list* all_ovlp, ma_ug_t *ug, asg_t* nsg, asg_t *purge_g, hap_cov_t *cov) { uint32_t v, i, uId, xUid; hap_overlaps *p = NULL; @@ -4571,7 +4571,7 @@ void remove_contained_haplotig(hap_overlaps_list* all_ovlp, ma_ug_t *ug, asg_t* purge_g->seq[xUid].del = 1; all_ovlp->x[uId].a.a[i].status = DELETE; - if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp->x[uId].a.a[i])); + if(cov->link) collect_reverse_unitig_pair(cov->link, ug, &(all_ovlp->x[uId].a.a[i])); collect_trans_purge_cov(cov, ug, &(all_ovlp->x[uId].a.a[i]), 0); } @@ -4613,7 +4613,7 @@ void remove_contained_haplotig(hap_overlaps_list* all_ovlp, ma_ug_t *ug, asg_t* void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage, hc_links* link, hap_cov_t *cov) +uint32_t just_contain, uint32_t just_coverage, hap_cov_t *cov) { asg_t *purge_g = NULL; purge_g = asg_init(); @@ -4705,7 +4705,7 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link, hap_cov_t *cov) normalize_hap_overlaps_advance(&all_ovlp, &back_all_ovlp, ug, read_g, reverse_sources, ruIndex); ///debug_hap_overlaps(&all_ovlp, &back_all_ovlp); - remove_contained_haplotig(&all_ovlp, ug, nsg, purge_g, link, cov); + remove_contained_haplotig(&all_ovlp, ug, nsg, purge_g, cov); if(just_contain == 0) @@ -4754,7 +4754,7 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link, hap_cov_t *cov) link_unitigs(purge_g, ug, &all_ovlp, ruIndex, reverse_sources, coverage_cut, read_g, position_index, &(hap_buf.buf[0].u_buffer), &(hap_buf.buf[0].u_buffer_tailIndex), &(hap_buf.buf[0].u_buffer_prevIndex), - max_hang, min_ovlp, edge, hap_buf.buf[0].visit, link, cov); + max_hang, min_ovlp, edge, hap_buf.buf[0].visit, cov); } for (v = 0; v < all_ovlp.num; v++) @@ -4785,124 +4785,3 @@ uint32_t just_contain, uint32_t just_coverage, hc_links* link, hap_cov_t *cov) destory_hap_alignment_struct_pip(&hap_buf); } -hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, -ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp) -{ - uint32_t n_ux = ug->g->n_seq, i, k, j, v, rId, tn, is_Unitig, r_i, nv, w, C_bases; - uint8_t *set = NULL; CALLOC(set, read_g->n_seq<<1); - hap_cov_t *x = NULL; CALLOC(x, 1); - x->n = read_g->n_seq; CALLOC(x->cov, x->n); - MALLOC(x->pos_idx, x->n); memset(x->pos_idx, -1, x->n*sizeof(uint64_t)); - x->reverse_sources = reverse_sources; - x->coverage_cut = coverage_cut; - x->ruIndex = ruIndex; - x->max_hang = max_hang; - x->min_ovlp = min_ovlp; - x->read_g = read_g; - kv_init(x->u_buffer.a); - kv_init(x->tailIndex.a); - kv_init(x->prevIndex.a); - ma_utg_t* u = NULL; - asg_arc_t *av = NULL; - ma_hit_t *h = NULL; - - for (i = 0; i < n_ux; i++) - { - if(ug->g->seq[i].del) continue; - u = &(ug->u.a[i]); - for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; - - v = i<<1; - nv = asg_arc_n(ug->g, v); - av = asg_arc_a(ug->g, v); - for (k = 0; k < nv; k++) - { - w = av[k].v; - if(av[k].del) continue; - if(ug->g->seq[w>>1].del) continue; - u = &(ug->u.a[w>>1]); - for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; - } - - v = (i<<1)+1; - nv = asg_arc_n(ug->g, v); - av = asg_arc_a(ug->g, v); - for (k = 0; k < nv; k++) - { - w = av[k].v; - if(av[k].del) continue; - if(ug->g->seq[w>>1].del) continue; - u = &(ug->u.a[w>>1]); - for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 1; - } - - - - u = &(ug->u.a[i]); - for (k = 0; k < u->n; k++) - { - C_bases = 0; - rId = u->a[k]>>33; - for (j = 0; j < (uint64_t)(sources[rId].length); j++) - { - h = &(sources[rId].buffer[j]); - tn = Get_tn((*h)); - if(read_g->seq[tn].del == 1) - { - ///get the id of read that contains it - get_R_to_U(ruIndex, tn, &tn, &is_Unitig); - if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; - } - if(read_g->seq[tn].del == 1) continue; - if(!set[tn]) continue; - C_bases += (Get_qe((*h)) - Get_qs((*h))); - } - x->cov[rId] = MAX(C_bases, x->cov[rId]); - } - - - - u = &(ug->u.a[i]); - for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; - - v = i<<1; - nv = asg_arc_n(ug->g, v); - av = asg_arc_a(ug->g, v); - for (k = 0; k < nv; k++) - { - w = av[k].v; - if(av[k].del) continue; - if(ug->g->seq[w>>1].del) continue; - u = &(ug->u.a[w>>1]); - for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; - } - - v = (i<<1)+1; - nv = asg_arc_n(ug->g, v); - av = asg_arc_a(ug->g, v); - for (k = 0; k < nv; k++) - { - w = av[k].v; - if(av[k].del) continue; - if(ug->g->seq[w>>1].del) continue; - u = &(ug->u.a[w>>1]); - for (r_i = 0; r_i < u->n; r_i++) set[u->a[r_i]>>33] = 0; - } - } - - free(set); - return x; -} - -void destory_hap_cov_t(hap_cov_t **x) -{ - if(*x) - { - free((*x)->cov); - free((*x)->pos_idx); - kv_destroy((*x)->u_buffer.a); - kv_destroy((*x)->tailIndex.a); - kv_destroy((*x)->prevIndex.a); - free((*x)); - } -} \ No newline at end of file diff --git a/Purge_Dups.h b/Purge_Dups.h index 9928624..4e3da12 100644 --- a/Purge_Dups.h +++ b/Purge_Dups.h @@ -16,13 +16,13 @@ void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage, hc_links* link, hap_cov_t *cov); +uint32_t just_contain, uint32_t just_coverage, hap_cov_t *cov); void fill_unitig(uint64_t* buffer, uint32_t bufferLen, asg_t* read_g, kvec_asg_arc_t_warp* edge, uint32_t is_circle, uint64_t* rLen); void get_contig_length(ma_ug_t *ug, asg_t *g, uint64_t* primaryLen, uint64_t* alterLen); void enable_debug_mode(uint32_t mode); -hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, -ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp); +hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, +ma_sub_t *coverage_cut, int max_hang, int min_ovlp, hc_links* link); void destory_hap_cov_t(hap_cov_t **x); void chain_trans_ovlp(hap_cov_t *cov, ma_ug_t *ug, asg_t *read_sg, buf_t* xReads, uint32_t targetBaseLen, uint32_t* xEnd); diff --git a/hic.cpp b/hic.cpp index f35724a..42f986b 100644 --- a/hic.cpp +++ b/hic.cpp @@ -181,7 +181,7 @@ typedef struct { typedef struct { - uint64_t s, e, id; + uint64_t s, e, id, len; } pe_hit; typedef struct { @@ -194,9 +194,9 @@ typedef struct { #define pe_hit_an1_key(x) ((x).s) -KRADIX_SORT_INIT(pe_hit_an1, pe_hit, pe_hit_an1_key, 8) +KRADIX_SORT_INIT(pe_hit_an1, pe_hit, pe_hit_an1_key, member_size(pe_hit, s)) #define pe_hit_an2_key(x) ((x).e) -KRADIX_SORT_INIT(pe_hit_an2, pe_hit, pe_hit_an2_key, 8) +KRADIX_SORT_INIT(pe_hit_an2, pe_hit, pe_hit_an2_key, member_size(pe_hit, e)) #define generic_key(x) (x) KRADIX_SORT_INIT(hc64, uint64_t, generic_key, 8) KRADIX_SORT_INIT(u32, uint32_t, generic_key, 4) @@ -223,8 +223,8 @@ typedef struct { // global data structure for kt_pipeline() uint64_t n_thread; uint64_t total_base; uint64_t total_pair; - ///kvec_pe_hit hits; - kvec_pe_hit_hap hits; + kvec_pe_hit hits; + ///kvec_pe_hit_hap hits; hc_links* link; } sldat_t; @@ -244,8 +244,8 @@ typedef struct { // data structure for each step in kt_pipeline() char **seq; ch_buf_t *buf; kvec_vote* pos_buf; - ///pe_hit* pos; - pe_hit_hap* pos; + pe_hit* pos; + ///pe_hit_hap* pos; hc_links* link; } stepdat_t; @@ -812,10 +812,11 @@ uint64_t* ref_p, uint64_t* self_p, uint64_t* exact_len, uint64_t* total_len) (*rev) = p->ref>>63; (*uID) = (p->ref << 1) >> (64 - idx->uID_bits); (*self_p) = (uint32_t)p->off_cnt; - (*exact_len) = p->off_cnt >> 32; + ///(*exact_len) = p->off_cnt >> 32; + (*exact_len) = (p->off_cnt>>32) & ((uint64_t)65535); if(total_len != NULL) { - (*exact_len) = (p->off_cnt>>32) & ((uint64_t)65535); + ///(*exact_len) = (p->off_cnt>>32) & ((uint64_t)65535); (*total_len) = (p->off_cnt>>48) + (*exact_len); } if((p->ref & idx->pos_mode)>>(idx->pos_bits - 1)) @@ -899,29 +900,32 @@ uint64_t debug_hash_value(char *r, uint64_t end, uint64_t k_mer) inline uint64_t collect_votes(s_hit* a, uint64_t n) { if(n == 0) return 0; - if(n == 1) return (a[0].off_cnt>>32); //seed length + if(n == 1) return (a[0].off_cnt>>32); //seed length, is right long long i = 0; uint64_t cur_beg, cur_end, beg, end, ovlp = 0, tLen = 0; cur_end = (uint32_t)a[n-1].off_cnt; cur_beg = cur_end + 1 - (a[n-1].off_cnt>>32); - - for (i = n - 2; i >= 0; i--) + if(n >= 2) { - end = (uint32_t)a[i].off_cnt; - beg = end + 1 - (a[i].off_cnt>>32); - if(MAX(cur_beg, beg) <= MIN(cur_end, end)) + for (i = n - 2; i >= 0; i--) { - cur_beg = MIN(cur_beg, beg); - ///cur_end = MAX(cur_end, end); - } - else - { - ovlp += (cur_end + 1 - cur_beg); - cur_beg = beg; - cur_end = end; + end = (uint32_t)a[i].off_cnt; + beg = end + 1 - (a[i].off_cnt>>32); + if(MAX(cur_beg, beg) <= MIN(cur_end, end)) + { + cur_beg = MIN(cur_beg, beg); + ///cur_end = MAX(cur_end, end); + } + else + { + ovlp += (cur_end + 1 - cur_beg); + cur_beg = beg; + cur_end = end; + } } } + ovlp += (cur_end + 1 - cur_beg); tLen = (uint32_t)a[n-1].off_cnt + 1 - cur_beg; tLen = tLen - ovlp; @@ -1286,8 +1290,8 @@ inline void compress_mapped_pos_advance(const ha_ug_index* idx, kvec_vote* buf, double max_eRate, sec_eRate, eRate; p = buf->a.a + buf_iter; cnt = buf->a.n - buf_iter; - radix_sort_hc_s_hit_off_cnt(p, p + cnt); ///buf save all hits, here sort by offset in reads - max_eLen = 0; max_i = (uint64_t)-1; max_occ = 0; max_eRate = 0; + if(cnt > 1) radix_sort_hc_s_hit_off_cnt(p, p + cnt); ///buf save all hits, here sort by offset in reads + max_eLen = 0; max_i = (uint64_t)-1; max_occ = 0; max_eRate = -1; for (j = 1, i = 0; j <= cnt; ++j) { if(j == cnt || p[j].off_cnt != p[i].off_cnt) @@ -1308,7 +1312,7 @@ inline void compress_mapped_pos_advance(const ha_ug_index* idx, kvec_vote* buf, } - sec_eLen = 0; second_i = (uint64_t)-1; second_occ = 0; sec_eRate = 0; + sec_eLen = 0; second_i = (uint64_t)-1; second_occ = 0; sec_eRate = -1; for (j = 1, i = 0; j <= cnt; ++j) { if(j == cnt || p[j].off_cnt != p[i].off_cnt) @@ -1327,22 +1331,34 @@ inline void compress_mapped_pos_advance(const ha_ug_index* idx, kvec_vote* buf, if(MAX(cur_beg, max_beg) <= MIN(cur_end, max_end)) { ovlp = MIN(cur_end, max_end) - MAX(cur_beg, max_beg) + 1; - if(ovlp == MIN(max_end+1-max_end, tLen)) + /*******************************for debug************************************/ + if(ovlp == MIN(max_end+1-max_end, tLen))///for non-unique k-mer { i = j;///must continue;///fully contain } - if(ovlp > ((max_end+1-max_end)*0.8) && eLen > (max_eLen*0.8))///best is not unique { buf->a.n = buf_iter; return; } - if(ovlp > ((max_end+1-max_end)*0.15)) + if(ovlp > ((max_end+1-max_end)*0.15) + 1) { i = j;///must continue;///fully contain } + + // if(ovlp > (MIN((max_end+1-max_end), (cur_end+1-cur_end))*0.15) + 1) + // { + // if(eLen > (max_eLen*0.8))///best is not unique + // { + // buf->a.n = buf_iter; + // return; + // } + // i = j;///must + // continue; + // } + /*******************************for debug************************************/ } if(is_update_hit(sec_eLen, sec_eRate, eLen, eRate)) @@ -1381,8 +1397,7 @@ inline void compress_mapped_pos_advance(const ha_ug_index* idx, kvec_vote* buf, buf->a.n = buf_iter + max_occ + second_occ; } -void get_alignment(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, -const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) +void get_alignment(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) { uint64_t i, j, k, l = 0, k_len, c_sfx, m, skip, *pos_list = NULL, cnt, rev, self_p, ref_p, uID; uint64_t x[4], mask = (1ULL< idx->hap_cnt || cnt < 0) continue; + if(cnt > idx->hap_cnt || cnt <= 0) continue; if(cnt > 1) { @@ -1495,7 +1510,7 @@ const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) // i, rev, uID, ref_p, self_p, eLen, tLen); // } /*******************************for debug************************************/ - compress_mapped_pos_advance(idx, buf, buf_iter, (k_mer * 0.1) > 0? (k_mer * 0.1) : 1); + compress_mapped_pos_advance(idx, buf, buf_iter, (k_mer * 0.1) > 0? (k_mer * 0.1) : 1); /*******************************for debug************************************/ // fprintf(stderr, "len2:%lu, max_i: %lu\n", buf->a.n - buf_iter, max_i); // for (i = buf_iter; i < buf->a.n; i++) @@ -1510,6 +1525,257 @@ const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) /*******************************for debug************************************/ } +inline void compress_mapped_pos_debug(const ha_ug_index* idx, kvec_vote* buf, uint64_t buf_iter, uint64_t max_i, uint64_t thres) +{ + if(buf_iter >= buf->a.n) + { + buf->a.n = buf_iter; + return; + } + uint64_t rev, uID, ref_p, self_p, eLen, tLen, i, max_beg, max_end, cur_beg, cur_end, ovlp, max_eLen; + uint64_t secondLen = 0, second_i = (uint64_t)-1; + interpret_pos((ha_ug_index*)idx, &buf->a.a[max_i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + max_end = self_p; + max_beg = self_p + 1 - tLen; + max_eLen = eLen; + for (i = buf_iter; i < buf->a.n; i++) + { + if(i == max_i) continue; + interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + cur_end = self_p; + cur_beg = self_p + 1 - tLen; + if(MAX(cur_beg, max_beg) <= MIN(cur_end, max_end)) + { + ovlp = MIN(cur_end, max_end) - MAX(cur_beg, max_beg) + 1; + if(ovlp > thres) + { + if(eLen >= max_eLen * 0.8) + { + buf->a.n = buf_iter; + return; + } + continue; + } + } + if(secondLen < eLen) secondLen = eLen, second_i = i; + } + + if(second_i == (uint64_t)-1) + { + buf->a.a[buf_iter] = buf->a.a[max_i]; + buf->a.n = buf_iter + 1; + } + else + { + buf->a.a[buf_iter] = buf->a.a[MIN(max_i, second_i)]; + buf->a.a[buf_iter+1] = buf->a.a[MAX(max_i, second_i)]; + buf->a.n = buf_iter + 2; + } +} + +void get_alignment_debug(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) +{ + uint64_t i, j, l = 0, m, skip, *pos_list = NULL, cnt, rev, self_p, ref_p, uID; + uint64_t x[4], mask = (1ULL<a.n = 0; + uint64_t k_len, c_sfx, k; + for (i = l = 0, x[0] = x[1] = x[2] = x[3] = 0; i < len; ++i) { + int c = seq_nt4_table[(uint8_t)r[i]]; + ///c = 00, 01, 10, 11 + if (c < 4) { // not an "N" base + ///x[0] & x[1] are the forward k-mer + ///x[2] & x[3] are the reverse complementary k-mer + x[0] = (x[0] << 1 | (c&1)) & mask; + x[1] = (x[1] << 1 | (c>>1)) & mask; + x[2] = x[2] >> 1 | (uint64_t)(1 - (c&1)) << shift; + x[3] = x[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift; + if (++l >= k_mer) + { + hash = hc_hash_long(x, &skip, k_mer); + if(skip == (uint64_t)-1) continue; + cnt = get_hc_pt1_count((ha_ug_index*)idx, hash, &pos_list); + if(cnt > idx->hap_cnt || cnt <= 0) continue; + + if(cnt > 1) + { + for (j = 0; j < cnt; j++) + { + uID = (pos_list[j] << 1) >> (64 - idx->uID_bits); + for (k = j + 1; k < cnt; k++) + { + if(uID == ((pos_list[k] << 1) >> (64 - idx->uID_bits))) break; + } + if(k < cnt) break; + } + + if(j < cnt) continue; + } + // if(cnt > 0) fprintf(stderr, "+i: %lu, l: %lu, cnt: %lu\n", i, l, cnt); + get_longest_hit(r, len, k_mer, i, skip, buf, idx, pos_list, cnt, &c_sfx); + // if(cnt > 0) fprintf(stderr, "c_sfx: %lu\n", c_sfx); + if(c_sfx != (uint64_t)-1) + { + k_len = c_sfx; + if((k_len + 1) >= k_mer) + { + l = 0, x[0] = x[1] = x[2] = x[3] = 0; + i = i + k_len - (k_mer - 1); + } + else + { + ///l = i - (i + k_len - (k_mer - 1)); + l = k_mer - k_len - 1; + } + } + // if(cnt > 0) fprintf(stderr, "-i: %lu, l: %lu\n", i, l); + } + + } else l = 0, x[0] = x[1] = x[2] = x[3] = 0; // if there is an "N", restart + } + **/ + + s_hit *p = NULL; uint64_t u_len; + ///buf->a.n = 0; + for (i = l = 0, x[0] = x[1] = x[2] = x[3] = 0; i < len; ++i) { + int c = seq_nt4_table[(uint8_t)r[i]]; + ///c = 00, 01, 10, 11 + if (c < 4) { // not an "N" base + ///x[0] & x[1] are the forward k-mer + ///x[2] & x[3] are the reverse complementary k-mer + x[0] = (x[0] << 1 | (c&1)) & mask; + x[1] = (x[1] << 1 | (c>>1)) & mask; + x[2] = x[2] >> 1 | (uint64_t)(1 - (c&1)) << shift; + x[3] = x[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift; + if (++l >= k_mer) + { + hash = hc_hash_long(x, &skip, k_mer); + if(skip == (uint64_t)-1) continue; + /*******************************for debug************************************/ + // if(debug_hash_value(r, i, k_mer) != hash) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + cnt = get_hc_pt1_count((ha_ug_index*)idx, hash, &pos_list); + if(cnt > idx->hap_cnt) continue; + if(cnt != 1) continue; ///might be able to be disabled in future + + + for (j = 0; j < cnt; j++) + { + kv_pushp(s_hit, buf->a, &p); + rev = (pos_list[j]>>63) != skip; + self_p = i; + ref_p = pos_list[j] & idx->pos_mode; + uID = (pos_list[j] << 1) >> (64 - idx->uID_bits); + u_len = idx->ug->u.a[uID].len; + if(rev) ref_p = u_len - 1 - (ref_p + 1 - k_mer); + p->off_cnt = self_p | ((uint64_t)k_mer << 32); ///high bits should be the legnth + + p->ref = ref_p >= self_p? (ref_p-self_p) + : (self_p-ref_p) + ((uint64_t)1 << (idx->pos_bits - 1)); + p->ref = (rev << 63)|(pos_list[j] & idx->uID_mode)|(p->ref&idx->pos_mode); + + + /*******************************for debug************************************/ + // if(check_exact_match(r, i + 1 - k_mer, len, + // idx->ug->u.a[uID].s, ref_p + 1 - k_mer, u_len, k_mer, rev, 0) != k_mer + // || + // check_exact_match(r, i, len, + // idx->ug->u.a[uID].s, ref_p, u_len, k_mer, rev, 1) != k_mer) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + } + + if(cnt == 1) + { + ///uint64_t debug_right = 0, debug_left = 0, debug_len; + + j = check_exact_match(r, self_p + 1, len, idx->ug->u.a[uID].s, ref_p + 1, u_len, len, rev, 0); + + ///debug_right = j; + ///if(j == 0) continue; + if((j + 1) >= k_mer) + { + l = 0, x[0] = x[1] = x[2] = x[3] = 0; + i = i + j - (k_mer - 1); + } + else + { + ///l = i - (i + j - (k_mer - 1)); + l = k_mer - j -1; + } + buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32) + j; + + if(self_p >= k_mer && ref_p >= k_mer) + { + j = check_exact_match(r, self_p - k_mer, len, idx->ug->u.a[uID].s, + ref_p - k_mer, u_len, len, rev, 1); + buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32); + ///debug_left = j; + } + + + // debug_len = check_exact_match(r, self_p + debug_right, len, idx->ug->u.a[uID].s, + // ref_p + debug_right, u_len, len, rev, 1); + // if(debug_len!= (debug_left + debug_right + k_mer)) + // { + // fprintf(stderr, "debug_len: %lu, debug_left: %lu, debug_right: %lu\n", + // debug_len, debug_left, debug_right); + // } + } + + } + + } else l = 0, x[0] = x[1] = x[2] = x[3] = 0; // if there is an "N", restart + } + /****************************may have bugs********************************/ + + if(buf->a.n - buf_iter == 0) return; + if(buf->a.n - buf_iter > 1) radix_sort_hc_s_hit_an1(buf->a.a + buf_iter, buf->a.a + buf->a.n); + + + uint64_t cur_ref_p, thres = (len * HIC_R_E_RATE) + 1, index_beg, ovlp; + /****************************may have bugs********************************/ + uint64_t maxLen = 0, max_i = (uint64_t)-1; + /****************************may have bugs********************************/ + i = m = buf_iter; + while (i < buf->a.n) + { + interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); + cur_ref_p = buf->a.a[i].ref; + index_beg = i; + while ((i < buf->a.n) && + ((buf->a.a[i].ref>>(idx->pos_bits-1)) == (cur_ref_p>>(idx->pos_bits-1))) && + (buf->a.a[i].ref - cur_ref_p <= thres)) + { + i++; + } + if(i - index_beg > 1) + { + radix_sort_hc_s_hit_an2(buf->a.a + index_beg, buf->a.a + i);//sort by self_p + } + ovlp = collect_votes(buf->a.a + index_beg, i - index_beg); + buf->a.a[m] = buf->a.a[i - 1]; + buf->a.a[m].off_cnt = (buf->a.a[m].off_cnt << 32)>>32; + buf->a.a[m].off_cnt += ((uint64_t)ovlp<<32); + m++; + /****************************may have bugs********************************/ + if(maxLen < (ovlp&((uint64_t)65535))) maxLen = (ovlp&((uint64_t)65535)), max_i = m; + /****************************may have bugs********************************/ + } + buf->a.n = m; + /****************************may have bugs********************************/ + ///compress_mapped_pos_advance(idx, buf, buf_iter, (k_mer * 0.1) > 0? (k_mer * 0.1) : 1); + compress_mapped_pos_debug(idx, buf, buf_iter, max_i, thres); + /****************************may have bugs********************************/ +} + + inline int is_unreliable_hits(long long rev, long long ref_p, long long tLen, uint64_t uID, hc_links* link) { @@ -1539,53 +1805,6 @@ inline int is_unreliable_hits(long long rev, long long ref_p, long long tLen, ui return 0; } -inline void set_pe_pos(ha_ug_index* idx, s_hit *l1, uint64_t occ1, s_hit *l2, uint64_t occ2, -pe_hit* x, uint64_t rid, hc_links* link) -{ - if(occ1 == 0 || occ2 == 0) return; - uint64_t rev1, rev2, uID1, uID2, ref_p1, ref_p2, self_p1, self_p2, eLen1, eLen2, tLen1, tLen2; - uint64_t rev_t, uID_t, ref_p_t, self_p_t, eLen_t, tLen_t; - ///5' end of r1 and r2 - interpret_pos(idx, &l1[0], &rev1, &uID1, &ref_p1, &self_p1, &eLen1, &tLen1); - interpret_pos(idx, &l2[0], &rev2, &uID2, &ref_p2, &self_p2, &eLen2, &tLen2); - /*******************************for debug************************************/ - // if(rid == 33045391 || rid == 4239289 || rid == 5267597 || rid == 34474764 || rid == 35016489 - // || rid == 36002255 || rid == 37811694 || rid == 46805824) - // { - // fprintf(stderr, "rid: %lu, rev1: %lu, uID1: %lu, ref_p1: %lu, self_p1: %lu, rev2: %lu, uID2: %lu, ref_p2: %lu, self_p2: %lu\n", - // rid, rev1, uID1, self_p1, ref_p1, rev2, uID2, ref_p2, self_p2); - // } - /*******************************for debug************************************/ - ///if(uID1 == uID2) return; - if(ref_p1 < self_p1 || ref_p2 < self_p2) return; - if(occ1 > 1) - { - interpret_pos(idx, &l1[1], &rev_t, &uID_t, &ref_p_t, &self_p_t, &eLen_t, &tLen_t); - if(uID_t != uID1 && uID_t != uID2) return; - } - - if(occ2 > 1) - { - interpret_pos(idx, &l2[1], &rev_t, &uID_t, &ref_p_t, &self_p_t, &eLen_t, &tLen_t); - if(uID_t != uID1 && uID_t != uID2) return; - } - - x->id = rid; - ref_p1 -= self_p1; - if(rev1) ref_p1 = idx->ug->u.a[uID1].len - 1 - ref_p1; - x->s = (rev1<<63) | ((uID1 << (64-idx->uID_bits))>>1) | (ref_p1 & idx->pos_mode); - - - ref_p2 -= self_p2; - if(rev2) ref_p2 = idx->ug->u.a[uID2].len - 1 - ref_p2; - x->e = (rev2<<63) | ((uID2 << (64-idx->uID_bits))>>1) | (ref_p2 & idx->pos_mode); - - if(link && (is_unreliable_hits(rev1, ref_p1, tLen1, uID1, link) || - is_unreliable_hits(rev2, ref_p2, tLen2, uID2, link))) - { - x->id = x->s = x->e = (uint64_t)-1; - } -} void get_5_3_list(ha_ug_index* idx, s_hit* p, uint64_t cnt, s_hit** l5, uint64_t* l5_occ, s_hit** l3, uint64_t* l3_occ) @@ -1617,6 +1836,7 @@ s_hit** l3, uint64_t* l3_occ) ///if(num > 2) fprintf(stderr, "ERROR: get_5_3_list\n"); } + inline void set_pe_pos_hap(ha_ug_index* idx, s_hit *l1, uint64_t occ1, s_hit *l2, uint64_t occ2, pe_hit_hap* x, uint64_t rid, hc_links* link) { @@ -1711,9 +1931,129 @@ pe_hit_hap* x, uint64_t rid, hc_links* link) /***************************for debug******************************/ } +inline void set_pe_pos(ha_ug_index* idx, s_hit *l1, uint64_t occ1, s_hit *l2, uint64_t occ2, +pe_hit* x, uint64_t rid, hc_links* link) +{ + if(occ1 == 0 || occ2 == 0) return; + uint64_t rev, uID, ref_p, self_p, eLen, tLen, i, is_unreliable = 0; + s_hit *l1_5 = NULL, *l1_3 = NULL, *l2_5 = NULL, *l2_3 = NULL; + uint64_t l1_5_occ = 0, l1_3_occ = 0, l2_5_occ = 0, l2_3_occ = 0; + + /***************************for debug******************************/ + // fprintf(stderr, "\nrid: %lu, occ1: %lu, occ2: %lu\n", rid, occ1, occ2); + // for (i = 0; i < occ1; i++) + // { + // interpret_pos(idx, &l1[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "***-1-rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu\n", + // rev, uID, ref_p, self_p); + // } + + // for (i = 0; i < occ2; i++) + // { + // interpret_pos(idx, &l2[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "***-2-rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu\n", + // rev, uID, ref_p, self_p); + // } + /***************************for debug******************************/ + + + get_5_3_list(idx, l1, occ1, &l1_5, &l1_5_occ, &l1_3, &l1_3_occ); + get_5_3_list(idx, l2, occ2, &l2_5, &l2_5_occ, &l2_3, &l2_3_occ); + if(l1_5_occ == 0 || l2_5_occ == 0) return; + x->id = rid; x->len = 0; + + ///if(l1_5_occ != 1 || l2_5_occ != 1) fprintf(stderr, "ERROR\n"); + + for (i = 0; i < l1_5_occ; i++) + { + interpret_pos(idx, &l1_5[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + if((ref_p + 1) < tLen) continue; + ref_p = ref_p + 1 - tLen; + if(rev) ref_p = idx->ug->u.a[uID].len - 1 - ref_p; + + if(link && (is_unreliable_hits(rev, ref_p, tLen, uID, link))) + { + is_unreliable = 1; + continue; + } + x->s = (rev<<63) | ((uID << (64-idx->uID_bits))>>1) | (ref_p & idx->pos_mode); + x->len = tLen; x->len <<= 32; + } + + for (i = 0; i < l2_5_occ; i++) + { + interpret_pos(idx, &l2_5[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + if((ref_p + 1) < tLen) continue; + ref_p = ref_p + 1 - tLen; + if(rev) ref_p = idx->ug->u.a[uID].len - 1 - ref_p; + + if(link && (is_unreliable_hits(rev, ref_p, tLen, uID, link))) + { + is_unreliable = 1; + continue; + } + x->e = (rev<<63) | ((uID << (64-idx->uID_bits))>>1) | (ref_p & idx->pos_mode); + x->len |= tLen; + } + + if(is_unreliable || x->s == (uint64_t)-1 || x->e == (uint64_t)-1) + { + x->id = x->s = x->e = x->len = (uint64_t)-1; + return; + } + + + /****************************may have bugs********************************/ + // for (i = 0; i < l1_3_occ; i++) + // { + // interpret_pos(idx, &l1_3[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // if(uID != ((x->s << 1) >> (64 - idx->uID_bits)) && + // uID != ((x->e << 1) >> (64 - idx->uID_bits))) + // { + // x->id = x->s = x->e = x->len = (uint64_t)-1; + // return; + // } + // } + + // for (i = 0; i < l2_3_occ; i++) + // { + // interpret_pos(idx, &l2_3[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // if(uID != ((x->s << 1) >> (64 - idx->uID_bits)) && + // uID != ((x->e << 1) >> (64 - idx->uID_bits))) + // { + // x->id = x->s = x->e = x->len = (uint64_t)-1; + // return; + // } + // } + /****************************may have bugs********************************/ + + + + + /***************************for debug******************************/ + // fprintf(stderr, "-------------saved: x->occ1: %u, x->occ2: %u-------------\n", x->occ1, x->occ2); + // for (i = 0; i < x->occ1; i++) + // { + // fprintf(stderr, "###-1-rev: %lu, uID: %lu, ref_p: %lu\n", + // x->a[i]>>63, (x->a[i]<<1)>>(64-idx->uID_bits), x->a[i] & idx->pos_mode); + // } + + // for (i = 0; i < x->occ2; i++) + // { + // fprintf(stderr, "###-2-rev: %lu, uID: %lu, ref_p: %lu\n", + // x->a[i+x->occ1]>>63, (x->a[i+x->occ1]<<1)>>(64-idx->uID_bits), x->a[i+x->occ1] & idx->pos_mode); + // } + // fprintf(stderr, "-------------get_pe_s-rev: %lu, uID: %lu, ref_p: %lu-------------\n", + // get_pe_s(*x)>>63, (get_pe_s(*x)<<1)>>(64-idx->uID_bits), get_pe_s(*x) & idx->pos_mode); + // fprintf(stderr, "-------------get_pe_e-rev: %lu, uID: %lu, ref_p: %lu-------------\n", + // get_pe_e(*x)>>63, (get_pe_e(*x)<<1)>>(64-idx->uID_bits), get_pe_e(*x) & idx->pos_mode); + /***************************for debug******************************/ +} + + uint64_t if_debug_read(uint64_t rid) { - if(rid == 177 || rid == 439 || rid == 97 || rid == 114) + if(rid == 1169718 || rid == 2665829 || rid == 4239289) { return 1; } @@ -1723,7 +2063,8 @@ uint64_t if_debug_read(uint64_t rid) static void worker_for_alignment(void *data, long i, int tid) // callback for kt_for() { stepdat_t *s = (stepdat_t*)data; - s->pos[i].id = (uint64_t)-1; s->pos[i].occ1 = s->pos[i].occ2 = 0; s->pos[i].a = NULL; + ///s->pos[i].id = (uint64_t)-1; s->pos[i].occ1 = s->pos[i].occ2 = 0; s->pos[i].a = NULL; + s->pos[i].id = s->pos[i].s = s->pos[i].e = s->pos[i].len = (uint64_t)-1; /*******************************for debug************************************/ // if(!if_debug_read(s->id+i)) return; @@ -1732,17 +2073,21 @@ static void worker_for_alignment(void *data, long i, int tid) // callback for kt uint64_t len1 = s->len[i]>>32, len2 = (uint32_t)s->len[i], occ1, occ2; char *r1 = s->seq[i], *r2 = s->seq[i] + len1; + + // fprintf(stderr, "**********R1**********\n"); s->pos_buf[tid].a.n = 0; get_alignment(r1, len1, s->idx->k, &s->pos_buf[tid], s->idx, 0, s->id+i); occ1 = s->pos_buf[tid].a.n; if(occ1 == 0) return; + + // fprintf(stderr, "**********R2**********\n"); get_alignment(r2, len2, s->idx->k, &s->pos_buf[tid], s->idx, occ1, s->id+i); occ2 = s->pos_buf[tid].a.n - occ1; if(occ2 == 0) return; - set_pe_pos_hap((ha_ug_index*)s->idx, s->pos_buf[tid].a.a, occ1, s->pos_buf[tid].a.a + occ1, occ2, &(s->pos[i]), s->id+i, s->link); + set_pe_pos((ha_ug_index*)s->idx, s->pos_buf[tid].a.a, occ1, s->pos_buf[tid].a.a + occ1, occ2, &(s->pos[i]), s->id+i, s->link); /*******************************for debug************************************/ // if(memcmp(r1, R1.r.a + R1.r_Len.a[s->id+i], len1) != 0) @@ -1827,8 +2172,10 @@ static void *worker_pipeline(void *data, int step, void *in) // callback for kt_ stepdat_t *s = (stepdat_t*)in; int i; for (i = 0; i < s->n; ++i) { - if(s->pos[i].a == NULL) continue; - kv_push(pe_hit_hap, p->hits, s->pos[i]); + // if(s->pos[i].a == NULL) continue; + // kv_push(pe_hit_hap, p->hits, s->pos[i]); + if(s->pos[i].s == (uint64_t)-1) continue; + kv_push(pe_hit, p->hits.a, s->pos[i]); } free(s->pos); free(s); @@ -1940,7 +2287,7 @@ inline void swap_pe_hit_hap(pe_hit_hap* x, pe_hit_hap* y) tmp = (*x); (*x) = (*y); (*y) = tmp; } -void dedup_hits(kvec_pe_hit_hap* hits, const ha_ug_index* idx) +void dedup_hits_hap(kvec_pe_hit_hap* hits, const ha_ug_index* idx) { double index_time = yak_realtime(); uint64_t k, l, m = 0, cur = (uint64_t)-1; @@ -2024,6 +2371,34 @@ void dedup_hits(kvec_pe_hit_hap* hits, const ha_ug_index* idx) hits->n = m; } + +void dedup_hits(kvec_pe_hit* hits) +{ + double index_time = yak_realtime(); + uint64_t k, l, m = 0, cur; + radix_sort_pe_hit_an1(hits->a.a, hits->a.a + hits->a.n); + for (k = 1, l = 0; k <= hits->a.n; ++k) + { + if (k == hits->a.n || hits->a.a[k].s != hits->a.a[l].s) + { + if (k - l > 1) radix_sort_pe_hit_an2(hits->a.a + l, hits->a.a + k); + cur = (uint64_t)-1; + while (l < k) + { + if(hits->a.a[l].e != cur) + { + cur = hits->a.a[l].e; + hits->a.a[m++] = hits->a.a[l]; + } + l++; + } + l = k; + } + } + hits->a.n = m; + fprintf(stderr, "[M::%s::%.3f] ==> Dedup\n", __func__, yak_realtime()-index_time); +} + void int_kvec_pe_hit_hap(kvec_pe_hit_hap* x) { x->m = x->n = x->n_u = 0; @@ -2399,7 +2774,7 @@ void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link) -void print_bubbles(ma_ug_t* ug, bubble_type* bub, kvec_pe_hit_hap* hits, hc_links* link, ha_ug_index* idx) +void print_bubbles(ma_ug_t* ug, bubble_type* bub, kvec_pe_hit* hits, hc_links* link, ha_ug_index* idx) { uint64_t tLen, t_utg, i, k; uint32_t beg, sink, n, *a; @@ -2443,10 +2818,10 @@ void print_bubbles(ma_ug_t* ug, bubble_type* bub, kvec_pe_hit_hap* hits, hc_link uint64_t s_uid, e_uid, shif = 64 - idx->uID_bits; if(hits) { - for (k = 0; k < hits->n_u; ++k) + for (k = 0; k < hits->a.n; ++k) { - s_uid = ((get_pe_s(hits->a[k])<<1)>>shif); - e_uid = ((get_pe_e(hits->a[k])<<1)>>shif); + s_uid = ((hits->a.a[k].s<<1)>>shif); + e_uid = ((hits->a.a[k].e<<1)>>shif); if(bub->index[s_uid] == (uint32_t)-1 || bub->index[e_uid] == (uint32_t)-1) continue; if(IF_BUB(s_uid, *bub) && IF_BUB(e_uid, *bub)) { @@ -3176,7 +3551,7 @@ void destory_MT(MT* M) kv_destroy(M->matrix); } -void collect_hc_links(const ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, MT* M) +void collect_hc_links_hap(const ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, MT* M) { double index_time = yak_realtime(); uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; @@ -3217,6 +3592,45 @@ void collect_hc_links(const ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* l fprintf(stderr, "[M::%s::%.3f] ==> Enzymes have been counted\n", __func__, yak_realtime()-index_time); } +void collect_hc_links(const ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, MT* M) +{ + double index_time = yak_realtime(); + uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>shif); + end = ((hits->a.a[k].e<<1)>>shif); + + if(beg == end) continue; + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + t_d = (uint64_t)-1; + push_hc_edge(&(link->a.a[beg]), end, 0, 0, &t_d); + push_hc_edge(&(link->a.a[end]), beg, 0, 0, &t_d); + } + all_pair_shortest_path(idx, link, M); + fill_utg_distance_multi(idx, link, M, bub); + + fprintf(stderr, "[M::%s::%.3f] ==> Hi-C linkages have been counted\n", __func__, yak_realtime()-index_time); + return; + + + + + + index_time = yak_realtime(); + for (k = 0; k < link->enzymes.n; k++) + { + link->enzymes.a[k] = 0; + for (i = 0; i < (uint64_t)asm_opt.hic_enzymes->n; i++) + { + link->enzymes.a[k] += get_enzyme_occ(idx->ug->u.a[k].s, idx->ug->u.a[k].len, + asm_opt.hic_enzymes->a[i], asm_opt.hic_enzymes->l[i]); + } + } + fprintf(stderr, "[M::%s::%.3f] ==> Enzymes have been counted\n", __func__, yak_realtime()-index_time); +} void set_reverse_links(uint32_t* bub, uint32_t n, kvec_t_u32_warp* reach, uint32_t root, hc_links* link) { @@ -3464,8 +3878,20 @@ int load_hc_links(hc_links* link, const char *fn) return 1; } +void write_hc_hits(kvec_pe_hit* hits, const char *fn) +{ + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.hic.lk.bin", fn); + FILE* fp = fopen(buf, "w"); -void write_hc_hits(kvec_pe_hit_hap* hits, const char *fn) + fwrite(&hits->a.n, sizeof(hits->a.n), 1, fp); + fwrite(hits->a.a, sizeof(pe_hit), hits->a.n, fp); + + fclose(fp); + free(buf); +} + +void write_hc_hits_hap(kvec_pe_hit_hap* hits, const char *fn) { char *buf = (char*)calloc(strlen(fn) + 25, 1); sprintf(buf, "%s.hic.lk.bin", fn); @@ -3635,7 +4061,7 @@ void debug_hc_hits_v14(kvec_pe_hit_hap* i_hits, const char *fn, const ha_ug_inde exit(1); } -int load_hc_hits(kvec_pe_hit_hap* hits, const char *fn) +int load_hc_hits_hap(kvec_pe_hit_hap* hits, const char *fn) { uint64_t flag = 0, k; char *buf = (char*)calloc(strlen(fn) + 25, 1); @@ -3665,6 +4091,27 @@ int load_hc_hits(kvec_pe_hit_hap* hits, const char *fn) return 1; } +int load_hc_hits(kvec_pe_hit* hits, const char *fn) +{ + uint64_t flag = 0; + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.hic.lk.bin", fn); + + FILE* fp = NULL; + fp = fopen(buf, "r"); + if(!fp) return 0; + + kv_init(hits->a); + flag += fread(&hits->a.n, sizeof(hits->a.n), 1, fp); + hits->a.m = hits->a.n; MALLOC(hits->a.a, hits->a.n); + flag += fread(hits->a.a, sizeof(pe_hit), hits->a.n, fp); + + fclose(fp); + free(buf); + fprintf(stderr, "[M::%s::] ==> Hi-C linkages have been loaded\n", __func__); + return 1; +} + inline int get_phase_status(H_partition* hap, uint32_t uID) { int d = -2; @@ -4953,7 +5400,7 @@ G_partition* clean_bubbles(hc_links* link, bubble_type* bub, min_cut_t* m, const -uint64_t get_hic_distance(pe_hit_hap* hit, hc_links* link, const ha_ug_index* idx) +uint64_t get_hic_distance_hap(pe_hit_hap* hit, hc_links* link, const ha_ug_index* idx) { uint64_t s_uid, s_dir, e_uid, e_dir, u_dis, k; long long s_pos, e_pos; @@ -4981,6 +5428,34 @@ uint64_t get_hic_distance(pe_hit_hap* hit, hc_links* link, const ha_ug_index* id return (uint64_t)-1; } +uint64_t get_hic_distance(pe_hit* hit, hc_links* link, const ha_ug_index* idx) +{ + uint64_t s_uid, s_dir, e_uid, e_dir, u_dis, k; + long long s_pos, e_pos; + s_uid = ((hit->s<<1)>>(64 - idx->uID_bits)); s_pos = hit->s & idx->pos_mode; + e_uid = ((hit->e<<1)>>(64 - idx->uID_bits)); e_pos = hit->e & idx->pos_mode; + if(s_uid == e_uid) return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + hc_linkeage* t = &(link->a.a[s_uid]); + for (k = 0; k < t->e.n; k++) + { + if(t->e.a[k].del || t->e.a[k].uID != e_uid) continue; + s_dir = (!!(t->e.a[k].dis&(uint64_t)2)); + e_dir = (!!(t->e.a[k].dis&(uint64_t)1)); + u_dis = (t->e.a[k].dis ==(uint64_t)-1? (uint64_t)-1 : t->e.a[k].dis>>3); + // if(s_uid == 24684 && s_pos == 124953 && e_uid == 16950 && e_pos == 93039) + // { + // fprintf(stderr, "*****************s_dir: %lu, e_dir: %lu, u_dis: %lu\n", s_dir, e_dir, u_dis); + // } + if(u_dis == (uint64_t)-1) return (uint64_t)-1; + if(s_dir == 1) s_pos = (long long)idx->ug->g->seq[s_uid].len - s_pos - 1; + if(e_dir == 1) e_pos = (long long)idx->ug->g->seq[e_uid].len - e_pos - 1; + e_pos = e_pos + u_dis - (long long)idx->ug->g->seq[e_uid].len; + return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + } + + return (uint64_t)-1; +} + hc_edge* get_hc_edge(hc_links* link, uint64_t src, uint64_t dest, uint64_t dir) { if(src >= link->a.n) return NULL; @@ -5240,7 +5715,7 @@ void weight_edges(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubbl if(IF_HOM(beg, *bub)) continue; if(IF_HOM(end, *bub)) continue; - t_d = get_hic_distance(&(hits->a[k]), link, idx); + t_d = get_hic_distance_hap(&(hits->a[k]), link, idx); if(t_d == (uint64_t)-1) continue; e1 = get_hc_edge(link, beg, end, 0); @@ -5256,8 +5731,7 @@ void weight_edges(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubbl } } - -void weight_edges_advance(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, trans_idx* dis) +void weight_edges_advance_hap(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, trans_idx* dis) { uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; hc_edge *e1 = NULL, *e2 = NULL; @@ -5281,7 +5755,49 @@ void weight_edges_advance(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* lin if(IF_HOM(beg, *bub)) continue; if(IF_HOM(end, *bub)) continue; - t_d = get_hic_distance(&(hits->a[k]), link, idx); + t_d = get_hic_distance_hap(&(hits->a[k]), link, idx); + if(t_d == (uint64_t)-1) continue; + + e1 = get_hc_edge(link, beg, end, 0); + e2 = get_hc_edge(link, end, beg, 0); + if(e1 == NULL || e2 == NULL) continue; + weight = 1; + if(dis) + { + weight = get_trans_weight_advance(idx, t_d, dis); + } + + e1->weight += weight; e1->occ++; + e2->weight += weight; e2->occ++; + } +} + + +void weight_edges_advance(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, trans_idx* dis) +{ + uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; + hc_edge *e1 = NULL, *e2 = NULL; + long double weight; + + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[k].weight = 0; + } + } + + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>shif); + end = ((hits->a.a[k].e<<1)>>shif); + + if(beg == end) continue; + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + t_d = get_hic_distance(&(hits->a.a[k]), link, idx); if(t_d == (uint64_t)-1) continue; e1 = get_hc_edge(link, beg, end, 0); @@ -8286,7 +8802,7 @@ void get_forward_distance(uint32_t src, uint32_t dest, asg_t *sg, hc_links* link -int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, MT* M, H_partition* hap, trans_idx* dis) +int get_trans_rate_function_hap(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, MT* M, H_partition* hap, trans_idx* dis) { kvec_t(uint64_t) buf, buf_idx; kv_init(buf); @@ -8314,7 +8830,7 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* l if(IF_HOM(end, *bub)) continue; - t_d = get_hic_distance(&(hits->a[k]), link, idx); + t_d = get_hic_distance_hap(&(hits->a[k]), link, idx); if(t_d == (uint64_t)-1) continue; if(beg == end) { @@ -8537,7 +9053,363 @@ int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* l return 1; } -void init_hic_p(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, +void init_hic_p_hap(ha_ug_index* idx, kvec_pe_hit_hap* hits, hc_links* link, bubble_type* bub, +kvec_hc_edge* back_hc_edge, MT* M, H_partition* hap, uint32_t ignore_dis) +{ + uint64_t k, i, m, uID, is_comples_weight = 0; + trans_idx dis; + kv_init(dis); + + if(bub->round_id > 0 && ignore_dis == 0) + { + is_comples_weight = get_trans_rate_function_hap(idx, hits, link, bub, M, hap, &dis); + } + + + hc_edge *e = NULL; + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].f.n; k++) + { + if(link->a.a[i].f.a[k].del) continue; + if(link->a.a[i].f.a[k].dis == RC_0) + { + uID = link->a.a[i].f.a[k].uID; + e = get_hc_edge(link, i, uID, 0); + if(e) + { + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = 1; + } + + e = get_hc_edge(link, uID, i, 0); + if(e) + { + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = 1; + } + } + else if(link->a.a[i].f.a[k].dis == RC_1) + { + uID = link->a.a[i].f.a[k].uID; + get_forward_distance(i, uID, idx->ug->g, link, M); + get_forward_distance(uID, i, idx->ug->g, link, M); + } + } + } + + + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + if(link->a.a[i].e.a[k].dis == (uint64_t)-1) + { + e = get_hc_edge(link, link->a.a[i].e.a[k].uID, i, 0); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, link->a.a[i].e.a[k]); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = link->a.a[i].e.a[k].del = 1; + } + } + } + + for (i = 0; i < link->a.n; i++) + { + for (k = m = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[m] = link->a.a[i].e.a[k]; + link->a.a[i].e.a[m].weight = 0; + link->a.a[i].e.a[m].occ = 0; + m++; + } + link->a.a[i].e.n = m; + } + weight_edges_advance_hap(idx, hits, link, bub, is_comples_weight == 1? &dis : NULL); + + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + if(link->a.a[i].e.a[k].weight <= 0) + { + e = get_hc_edge(link, link->a.a[i].e.a[k].uID, i, 0); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, link->a.a[i].e.a[k]); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = link->a.a[i].e.a[k].del = 1; + } + } + } + + for (i = 0; i < link->a.n; i++) + { + for (k = m = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[m] = link->a.a[i].e.a[k]; + m++; + } + link->a.a[i].e.n = m; + } + + kv_destroy(dis); +} + +int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, MT* M, H_partition* hap, trans_idx* dis) +{ + kvec_t(uint64_t) buf, buf_idx; + kv_init(buf); + kv_init(buf_idx); + uint64_t beg, end, cnt[2]; + uint64_t k, i, t_d, r_idx, f_idx, med = (uint64_t)-1; + int beg_status, end_status; + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + link->a.a[i].e.a[k].dis = (uint64_t)-1; + } + } + fill_utg_distance_multi(idx, link, M, bub); + + + buf.n = 0; + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>(64 - idx->uID_bits)); + end = ((hits->a.a[k].e<<1)>>(64 - idx->uID_bits)); + + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + + t_d = get_hic_distance(&(hits->a.a[k]), link, idx); + if(t_d == (uint64_t)-1) continue; + if(beg == end) + { + t_d = (t_d << 1); + } + else + { + beg_status = get_phase_status(hap, beg); + if(beg_status != 1 && beg_status != -1) continue; + end_status = get_phase_status(hap, end); + if(end_status != 1 && end_status != -1) continue; + if(beg_status != end_status) + { + t_d = (t_d << 1) + 1; + } + else + { + t_d = (t_d << 1); + } + } + + kv_push(uint64_t, buf, t_d); + } + + ///might have bias, we may not use right linkage larger than trans rc linkage + radix_sort_hc64(buf.a, buf.a+buf.n); + + for (k = 0, r_idx = f_idx = (uint64_t)-1; k < buf.n; k++) + { + if((buf.a[k]&1) == 0) r_idx = k; + if((buf.a[k]&1) == 1) f_idx = k; + } + buf.n = MIN(r_idx, f_idx); + for (k = 0; k < buf.n; k++) + { + if((buf.a[k]&1) == 1) + { + kv_push(uint64_t, buf_idx, buf.a[k]>>1); + } + } + uint64_t cutoff = buf_idx.n * 0.9, t = buf_idx.n * 0.005, pre, step; + k = 0; + if(cutoff >= t) k = cutoff - t; + pre = 0; + if(cutoff >= t + 1) pre = buf_idx.a[cutoff - t - 1]; + for (t_d = i = 0; k < cutoff + t; k++) + { + t_d += (buf_idx.a[k] - pre); + pre = buf_idx.a[k]; + i++; + } + if(t_d == 0 || i == 0 || t == 0) + { + kv_destroy(buf); + kv_destroy(buf_idx); + return 0; + } + step = (t_d/i)*20; + + if(step == 0) + { + kv_destroy(buf); + kv_destroy(buf_idx); + return 0; + } + trans_p_t* p = NULL; + dis->n = 0; + uint64_t step_s = 0, step_e = step; + if(buf.n>0) step_s = buf.a[0]>>1, step_e = (buf.a[0]>>1) + step; + for (k = cnt[0] = cnt[1] = 0; k < buf.n; k++) + { + if((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s) + { + cnt[buf.a[k]&1]++; + } + + if((buf.a[k]>>1) >= step_e) + { + while (!((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s)) + { + kv_pushp(trans_p_t, *dis, &p); + p->beg = step_s; + p->end = step_e; + p->cnt_0 = cnt[0]; + p->cnt_1 = cnt[1]; + step_s += step; + step_e += step; + cnt[0] = cnt[1] = 0; + } + } + // fprintf(stderr, "-k: %lu, buf.n: %lu, buf.a[k]: %lu, step_s: %lu, step_e: %lu\n", + // k, (uint64_t)buf.n, (buf.a[k]>>1), step_s, step_e); + } + if(cnt[0] > 0 || cnt[1] > 0) + { + kv_pushp(trans_p_t, *dis, &p); + p->beg = step_s; + p->end = step_e; + p->cnt_0 = cnt[0]; + p->cnt_1 = cnt[1]; + } + + uint64_t smooth_step = 20, k_i, cnt_0; + if(dis->n > 0) med = dis->a[dis->n-1].end; + for (k = 0; k+smooth_step < dis->n; k++) + { + for (k_i = cnt_0 = 0; k_i < smooth_step; k_i++) + { + if(dis->a[k+k_i].cnt_0 == 0 || dis->a[k+k_i].cnt_1 == 0) cnt_0++; + } + + if(cnt_0 >= smooth_step * 0.2) + { + med = dis->a[k].beg; + break; + } + } + + long long b_k = 0, b_i = 0, b_j, pass = 0; + ///for (b_k = b_i = 0; b_k < (long long)dis->n; b_k++) + while(b_k < (long long)dis->n) + { + pass = 1; + beg = dis->a[b_k].beg; + end = dis->a[b_k].end; + cnt[0] = dis->a[b_k].cnt_0; + cnt[1] = dis->a[b_k].cnt_1; + if(cnt[0] > 0 && cnt[1] > 0) + { + dis->a[b_i].beg = beg; + dis->a[b_i].end = end; + dis->a[b_i].cnt_0 = cnt[0]; + dis->a[b_i].cnt_1 = cnt[1]; + b_i++; + b_k++; + continue; + } + + b_k++; + for (b_j = b_k; b_j < (long long)dis->n; b_j++, b_k++) + { + end = dis->a[b_j].end; + cnt[0] += dis->a[b_j].cnt_0; + cnt[1] += dis->a[b_j].cnt_1; + if(cnt[0] > 0 && cnt[1] > 0) break; + } + + + if(b_j < (long long)dis->n) + { + dis->a[b_i].beg = beg; + dis->a[b_i].end = end; + dis->a[b_i].cnt_0 = cnt[0]; + dis->a[b_i].cnt_1 = cnt[1]; + b_i++; + b_k++; + continue; + } + + for(b_j = b_i-1; b_j >= 0; b_j--) + { + beg = dis->a[b_j].beg; + cnt[0] += dis->a[b_j].cnt_0; + cnt[1] += dis->a[b_j].cnt_1; + if(cnt[0] > 0 && cnt[1] > 0) break; + } + + if(b_j >= 0) + { + b_i = b_j; + dis->a[b_i].beg = beg; + dis->a[b_i].end = end; + dis->a[b_i].cnt_0 = cnt[0]; + dis->a[b_i].cnt_1 = cnt[1]; + b_i++; + b_k++; + continue; + } + + pass = 0; + break; + } + dis->n = b_i; + if(dis->n == 0 || pass == 0) + { + kv_destroy(buf); + kv_destroy(buf_idx); + return 0; + } + + // for (i = 0; i < dis->n; i++) + // { + // if(i > 0 && dis->a[i].beg != dis->a[i-1].end) fprintf(stderr, "ERROR: dis->a[i].beg: %lu, dis->a[i-1].end: %lu\n", dis->a[i].beg, dis->a[i-1].end); + // fprintf(stderr, "beg: %lu, end: %lu, cnt_0: %lu, cnt_1: %lu, error_rate: %f\n", + // dis->a[i].beg, dis->a[i].end, dis->a[i].cnt_0, dis->a[i].cnt_1, (double)(dis->a[i].cnt_1)/(double)(dis->a[i].cnt_1 + dis->a[i].cnt_0)); + // } + + LeastSquare_advance(dis, idx, med); + // fprintf(stderr, "idx->a: %f, idx->b: %f, idx->frac: %f, med: %lu\n", + // (double)idx->a, (double)idx->b, (double)idx->frac, med); + + dis->max = dis->a[dis->n-1].end; + + + kv_destroy(buf); + kv_destroy(buf_idx); + if(idx->a < 0) idx->a = 0; + if(idx->a == 0) + { + idx->b = MAX((((double)(dis->a[dis->n-1].cnt_1))/((double)(dis->a[dis->n-1].cnt_0 + dis->a[dis->n-1].cnt_1))), idx->b); + } + if(idx->b < 0 && get_trans(idx, dis->max) < 0) + { + idx->b = ((double)(dis->a[dis->n-1].cnt_1))/((double)(dis->a[dis->n-1].cnt_0 + dis->a[dis->n-1].cnt_1)); + } + + // fprintf(stderr, "idx->a: %f, idx->b: %f, idx->frac: %f, med: %lu\n", + // (double)idx->a, (double)idx->b, (double)idx->frac, med); + + return 1; +} + + +void init_hic_p(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, kvec_hc_edge* back_hc_edge, MT* M, H_partition* hap, uint32_t ignore_dis) { uint64_t k, i, m, uID, is_comples_weight = 0; @@ -11866,7 +12738,7 @@ void print_bubble_chain(bubble_type* bub) } } -void init_contig_H_partition(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap* hits, H_partition* hap) +void init_contig_H_partition(bubble_type* bub, ha_ug_index* idx, H_partition* hap) { uint32_t i, k_i, k_j, uID, *a = NULL, n, *h0, h0_n, *h1, h1_n; destory_G_partition(&(hap->group_g_p)); memset(&(hap->group_g_p), 0, sizeof(G_partition)); @@ -11942,7 +12814,7 @@ void init_contig_H_partition(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap label_unitigs(&(hap->group_g_p), idx->ug); } -void cluster_contigs(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap* hits, MT* M, H_partition* hap) +void cluster_contigs_hap(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap* hits, MT* M, H_partition* hap) { uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; hc_links* link = idx->link; @@ -11961,6 +12833,38 @@ void cluster_contigs(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap* hits, push_hc_edge(&(link->a.a[end]), beg, 0, 0, &t_d); } + init_hic_p_hap((ha_ug_index*)idx, hits, link, bub, NULL, M, NULL, 1); + + init_chain_hic_warp(idx->ug, link, bub, &bub->c_w); + + hap->link = link; + hap->n = idx->ug->u.n; + + init_contig_H_partition(bub, idx, hap); + + destory_chain_hic_warp(&bub->c_w); +} + + +void cluster_contigs(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit* hits, MT* M, H_partition* hap) +{ + uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; + hc_links* link = idx->link; + for (i = 0; i < link->a.n; i++) link->a.a[i].e.n = 0; + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>shif); + end = ((hits->a.a[k].e<<1)>>shif); + + if(beg == end) continue; + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + t_d = 1; + push_hc_edge(&(link->a.a[beg]), end, 0, 0, &t_d); + push_hc_edge(&(link->a.a[end]), beg, 0, 0, &t_d); + } + init_hic_p((ha_ug_index*)idx, hits, link, bub, NULL, M, NULL, 1); init_chain_hic_warp(idx->ug, link, bub, &bub->c_w); @@ -11968,7 +12872,7 @@ void cluster_contigs(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit_hap* hits, hap->link = link; hap->n = idx->ug->u.n; - init_contig_H_partition(bub, idx, hits, hap); + init_contig_H_partition(bub, idx, hap); destory_chain_hic_warp(&bub->c_w); } @@ -12013,7 +12917,7 @@ int alignment_worker_pipeline(sldat_t* sl, const enzyme *fn1, const enzyme *fn2) } fprintf(stderr, "[M::%s::%.3f] ==> Qualification\n", __func__, yak_realtime()-index_time); - dedup_hits(&(sl->hits), sl->idx); + dedup_hits(&(sl->hits)); return 1; } @@ -12240,7 +13144,9 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) sl.n_thread = asm_opt.thread_num; sl.total_base = sl.total_pair = 0; idx->hap_cnt = asm_opt.hap_occ; - int_kvec_pe_hit_hap(&sl.hits); + ///int_kvec_pe_hit_hap(&sl.hits); + ///int_kvec_pe_hit(&sl.hits); + kv_init(sl.hits.a); if(!load_hc_hits(&sl.hits, asm_opt.output_file_name)) @@ -12318,14 +13224,15 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) destory_contig_partition(&hap); kv_destroy(back_hc_edge.a); - destory_kvec_pe_hit_hap(&sl.hits); + ///destory_kvec_pe_hit_hap(&sl.hits); + kv_destroy(sl.hits.a); return 1; /*******************************for debug************************************/ // destory_reads(&R1); // destory_reads(&R2); /*******************************for debug************************************/ - print_bubbles(idx->ug, &bub, sl.hits.n?&sl.hits:NULL, idx->link, idx); + print_bubbles(idx->ug, &bub, sl.hits.a.n?&sl.hits:NULL, idx->link, idx); collect_hc_reverse_links(idx->link, idx->ug, &bub); normalize_hc_links(idx->link); /*******************************for debug************************************/ @@ -12492,7 +13399,7 @@ void print_bench_idx(bench_idx* idx, ma_ug_t *ug) } -uint64_t get_hic_distance_bench(pe_hit_hap* hit, hc_links* link, bench_idx* idx, ma_ug_t *ug, uint64_t* is_trans) +uint64_t get_hic_distance_bench_hap(pe_hit_hap* hit, hc_links* link, bench_idx* idx, ma_ug_t *ug, uint64_t* is_trans) { (*is_trans) = (uint64_t)-1; uint64_t s_uid, e_uid; @@ -12537,6 +13444,52 @@ uint64_t get_hic_distance_bench(pe_hit_hap* hit, hc_links* link, bench_idx* idx, return (uint64_t)-1; } +uint64_t get_hic_distance_bench(pe_hit* hit, hc_links* link, bench_idx* idx, ma_ug_t *ug, uint64_t* is_trans) +{ + (*is_trans) = (uint64_t)-1; + uint64_t s_uid, e_uid; + long long s_pos, e_pos; + s_uid = ((hit->s<<1)>>(64 - idx->uID_bits)); s_pos = hit->s & idx->pos_mode; + e_uid = ((hit->e<<1)>>(64 - idx->uID_bits)); e_pos = hit->e & idx->pos_mode; + if(s_uid == e_uid) + { + (*is_trans) = 0; + return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + } + + uint64_t s_i, e_i, k, ori; + for (s_i = 0; s_i < idx->ug_idx.n; s_i++) + { + if(s_uid >= idx->ug_idx.a[s_i].uID_start && s_uid < idx->ug_idx.a[s_i].uID_end) break; + } + for (e_i = 0; e_i < idx->ug_idx.n; e_i++) + { + if(e_uid >= idx->ug_idx.a[e_i].uID_start && e_uid < idx->ug_idx.a[e_i].uID_end) break; + } + if(s_i == idx->ug_idx.n || e_i == idx->ug_idx.n) return (uint64_t)-1; + if(s_i == e_i) + { + (*is_trans) = 0; + return (uint64_t)-1; + } + + (*is_trans) = 1; + hc_linkeage* t = &(link->a.a[s_uid]); + long long m_x[2] = {1, -1}, dis; + for (k = 0; k < t->e.n; k++) + { + if(t->e.a[k].del || t->e.a[k].uID != e_uid) continue; + ori = !!(t->e.a[k].dis & (uint64_t)2); + dis = (long long)(t->e.a[k].dis>>2) * m_x[t->e.a[k].dis & (uint64_t)1]; + if(ori) e_pos = ug->u.a[e_uid].len - e_pos - 1; + e_pos = e_pos + dis; + return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + } + + return (uint64_t)-1; +} + + void init_bench_idx(bench_idx* idx, asg_t* read_g, ma_ug_t *ug) { uint64_t i, occ; @@ -12566,14 +13519,67 @@ void init_bench_idx(bench_idx* idx, asg_t* read_g, ma_ug_t *ug) } } -void evaluate_bench_idx(bench_idx* idx, kvec_pe_hit_hap* hits, ma_ug_t *ug) +void evaluate_bench_idx_hap(bench_idx* idx, kvec_pe_hit_hap* hits, ma_ug_t *ug) { uint64_t k, distance, is_trans, trans[2]; kvec_t(uint64_t) buf; kv_init(buf); for (k = trans[0] = trans[1] = 0; k < hits->n_u; ++k) { - distance = get_hic_distance_bench(&(hits->a[k]), &(idx->link), idx, ug, &is_trans); + distance = get_hic_distance_bench_hap(&(hits->a[k]), &(idx->link), idx, ug, &is_trans); + if(is_trans != (uint64_t)-1) trans[is_trans]++; + if(distance == (uint64_t)-1 || is_trans == (uint64_t)-1) continue; + distance = (distance << 1) + is_trans; + kv_push(uint64_t, buf, distance); + } + + radix_sort_hc64(buf.a, buf.a+buf.n); + + for (k = 0; k < buf.n; k++) + { + fprintf(stderr, "%lu\t%lu\n", buf.a[k]>>1, buf.a[k]&1); + } + /** + uint64_t up_dis = buf.a[(uint64_t)(buf.n*0.99)]>>1, step = 7240; + uint64_t step_s = 0, step_e = step, cnt[2]; + for (k = cnt[0] = cnt[1] = 0; k < buf.n; k++) + { + if(step_s > up_dis) step_e = (buf.a[buf.n-1]>>1) + 1; + if((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s) + { + cnt[buf.a[k]&1]++; + } + if((buf.a[k]>>1) >= step_e) + { + while (!((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s)) + { + fprintf(stderr, "i: %lu, step_s: %lu, step_e: %lu, cnt[0]: %lu, cnt[1]: %lu, rate: %f\n", + step_s/step, step_s, step_e, cnt[0], cnt[1], ((double)cnt[1])/(double)(cnt[0] + cnt[1])); + step_s += step; + step_e += step; + cnt[0] = cnt[1] = 0; + } + } + } + + if(cnt[0] > 0 || cnt[1] > 0) + { + fprintf(stderr, "i: %lu, step_s: %lu, step_e: %lu, cnt[0]: %lu, cnt[1]: %lu, rate: %f\n", + step_s/step, step_s, step_e, cnt[0], cnt[1], ((double)cnt[1])/(double)(cnt[0] + cnt[1])); + } + **/ + kv_destroy(buf); +} + + +void evaluate_bench_idx(bench_idx* idx, kvec_pe_hit* hits, ma_ug_t *ug) +{ + uint64_t k, distance, is_trans, trans[2]; + kvec_t(uint64_t) buf; + kv_init(buf); + for (k = trans[0] = trans[1] = 0; k < hits->a.n; ++k) + { + distance = get_hic_distance_bench(&(hits->a.a[k]), &(idx->link), idx, ug, &is_trans); if(is_trans != (uint64_t)-1) trans[is_trans]++; if(distance == (uint64_t)-1 || is_trans == (uint64_t)-1) continue; distance = (distance << 1) + is_trans; @@ -12641,7 +13647,8 @@ int hic_short_align_bench(const enzyme *fn1, const enzyme *fn2, const char *outp sl.n_thread = asm_opt.thread_num; sl.total_base = sl.total_pair = 0; idx->hap_cnt = asm_opt.hap_occ; - int_kvec_pe_hit_hap(&sl.hits); + ///int_kvec_pe_hit_hap(&sl.hits); + kv_init(sl.hits.a); fprintf(stderr, "u.n: %d, uID_bits: %lu, pos_bits: %lu\n", (uint32_t)idx->ug->u.n, idx->uID_bits, idx->pos_bits); if(!load_hc_hits(&sl.hits, output_file_name)) @@ -12657,7 +13664,8 @@ int hic_short_align_bench(const enzyme *fn1, const enzyme *fn2, const char *outp evaluate_bench_idx(&bench, &sl.hits, idx->ug); destory_bench_idx(&bench); - destory_kvec_pe_hit_hap(&sl.hits); + ///destory_kvec_pe_hit_hap(&sl.hits); + kv_destroy(sl.hits.a); fprintf(stderr, "[M::%s::%.3f] processed %lu pairs; %lu bases\n", __func__, yak_realtime()-index_time, sl.total_pair, sl.total_base); return 1; } From 8e4b98f0a6c4115e60de401479d9ad39b1d59e57 Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Sun, 14 Mar 2021 03:25:19 -0400 Subject: [PATCH 4/8] bubble label --- Overlaps.cpp | 1415 +++++++++++++++++++++++++++++++++++++----------- Overlaps.h | 105 ++-- Purge_Dups.cpp | 37 +- Purge_Dups.h | 6 +- hic.cpp | 4 +- 5 files changed, 1161 insertions(+), 406 deletions(-) diff --git a/Overlaps.cpp b/Overlaps.cpp index a1a92dd..4e30e36 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -10,6 +10,8 @@ #include "Correct.h" #include "Purge_Dups.h" #include "hic.h" +#include "kthread.h" + uint32_t debug_purge_dup = 0; @@ -43,6 +45,58 @@ KSORT_INIT_GENERIC(uint32_t) ///this value has been updated at the first line of build_string_graph_without_clean long long min_thres; + + +void init_bub_label_t(bub_label_t* x, uint32_t n_thres, uint32_t n_reads) +{ + uint32_t i; + x->check_cross = 0; + x->bub_dist = 0; + x->n_thres = n_thres; + x->n_reads = n_reads; + x->g = NULL; + CALLOC(x->b, x->n_thres); + for (i = 0; i < x->n_thres; i++) + { + CALLOC(x->b[i].a, x->n_reads<<1); + } +} + +void reset_bub_label_t(bub_label_t* x, asg_t *g, uint64_t bub_dist, uint32_t check_cross) +{ + uint32_t i; + x->bub_dist = bub_dist; + x->check_cross = check_cross; + x->g = g; + if(x->n_reads < x->g->n_seq) + { + x->n_reads = x->g->n_seq; + for (i = 0; i < x->n_thres; i++) + { + REALLOC(x->b[i].a, x->n_reads<<1); + } + } + + for (i = 0; i < x->n_thres; i++) + { + x->b[i].S.n = x->b[i].b.n = x->b[i].e.n = 0; + memset(x->b[i].a, 0, (x->n_reads<<1)*sizeof(binfo_s_t)); + } +} + +void destory_bub_label_t(bub_label_t* x) +{ + uint32_t i; + for (i = 0; i < x->n_thres; i++) + { + free(x->b[i].a); + free(x->b[i].S.a); + free(x->b[i].b.a); + free(x->b[i].e.a); + } + free(x->b); +} + void ma_hit_sort_tn(ma_hit_t *a, long long n) { radix_sort_hit_tn(a, a + n); @@ -2377,6 +2431,12 @@ buf_t *b) **/ uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l binfo_t *t = &b->a[w]; + + ///if this edge has been deleted + /****************************may have bugs********************************/ + if (av[i].del) continue; + /****************************may have bugs********************************/ + ///that means there is a circle, directly terminate the whole bubble poping if (w == v0) { @@ -2384,10 +2444,6 @@ buf_t *b) goto pop_reset; } - ///if this edge has been deleted - /****************************may have bugs********************************/ - if (av[i].del) continue; - /****************************may have bugs********************************/ ///push the edge kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); @@ -3798,7 +3854,7 @@ void* asg_arc_identify_simple_bubbles_pthread(void* arg) return NULL; } -int asg_arc_identify_simple_bubbles_multi(asg_t *g, int check_cross) +int asg_arc_identify_simple_bubbles_multi_back(asg_t *g, int check_cross) { double startTime = Get_T(); memset(g->seq_vis, 0, g->n_seq*2*sizeof(uint8_t)); @@ -3846,7 +3902,66 @@ int asg_arc_identify_simple_bubbles_multi(asg_t *g, int check_cross) return bub_nodes+cross_nodes; } +uint64_t asg_bub_pop1_label(asg_t *g, uint32_t v0, uint64_t max_dist, buf_s_t *b); +static void bubble_identify_worker(void *_data, long eid, int tid) +{ + bub_label_t *buf = (bub_label_t*)_data; + buf_s_t *b = &(buf->b[tid]); + uint32_t v = eid, i; + asg_t *g = buf->g; + if(g->seq[v>>1].del) return; + if(asg_arc_n(g, v) < 2 || get_real_length(g, v, NULL) < 2) return; + if(g->seq_vis[v] != 1 && asg_bub_pop1_label(g, v, buf->bub_dist, b)) + { + //beg is v, end is b.S.a[0] + //note b.b include end, does not include beg + for (i = 0; i < b->b.n; i++) + { + if(b->b.a[i]==v || b->b.a[i]==b->S.a[0]) continue; + g->seq_vis[b->b.a[i]] = 1; + g->seq_vis[b->b.a[i]^1] = 1; + } + g->seq_vis[v] = 1; + g->seq_vis[b->S.a[0]^1] = 1; + } + + if(buf->check_cross == 1 && g->seq_vis[v] == 0 && check_if_cross(g, v)) + { + g->seq_vis[v] = 2; + } +} + +uint64_t get_s_bub_pop_max_dist_advance(asg_t *g, buf_s_t *b); +int asg_arc_identify_simple_bubbles_multi(asg_t *g, bub_label_t* x, int check_cross) +{ + double startTime = Get_T(); + memset(g->seq_vis, 0, g->n_seq*2*sizeof(uint8_t)); + uint64_t bub_dist = get_s_bub_pop_max_dist_advance(g, &(x->b[0])); + + // fprintf(stderr, "+++[M::%s] takes %0.2f s, bub_dist: %lu\n\n", __func__, Get_T()-startTime, bub_dist); + // startTime = Get_T(); + + reset_bub_label_t(x, g, bub_dist, check_cross); + kt_for(x->n_thres, bubble_identify_worker, x, g->n_seq<<1); + uint32_t v, n_vtx = g->n_seq<<1; + long long nodes, bub_nodes, cross_nodes; + bub_nodes = nodes = cross_nodes = 0; + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v>>1].del) continue; + nodes++; + if(g->seq_vis[v] == 1) bub_nodes++; + if(g->seq_vis[v] == 2) cross_nodes++; + } + ///fprintf(stderr, "---[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + if(VERBOSE >= 1) + { + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + } + return bub_nodes+cross_nodes; +} int check_small_bubble(asg_t *g, uint32_t begNode, uint32_t v, uint32_t w, long long* vLen, long long* wLen, uint32_t* endNode) @@ -4233,48 +4348,6 @@ int test_single_node_bubble_directly(asg_t *g, uint32_t v, long long longLen_thr } -int asg_arc_del_single_node_bubble(asg_t *g, long long max_dist) -{ - ///the reason is that each read has two direction (query->target, target->query) - uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0; - buf_t b; - if (!g->is_symm) asg_symm(g); - memset(&b, 0, sizeof(buf_t)); - ///set information for each node - b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); - for (v = 0; v < n_vtx; ++v) - { - uint32_t nv = asg_arc_n(g, v); - if (g->seq[v>>1].del) - { - continue; - } - - if(nv < 2) - { - continue; - } - - ///if this is a bubble - if(asg_bub_finder_with_del_advance(g, v, max_dist, &b) == 1) - { - n_reduced += test_single_node_bubble(g, b.b.a, b.b.n, v, b.S.a[0]); - } - - } - - free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); - - if (n_reduced) { - asg_cleanup(g); - asg_symm(g); - } - - fprintf(stderr, "[M::%s] removed %d short bubbles\n\n", __func__, n_reduced); - - return n_reduced; -} - int asg_arc_del_single_node_directly(asg_t *g, long long longLen_thres, ma_hit_t_alloc* sources) { double startTime = Get_T(); @@ -11272,53 +11345,144 @@ uint32_t set_utg_offset(buf_t* b, ma_ug_t *ug, asg_t *read_sg, hap_cov_t *cov, u return len; } - -void collect_trans_cov(buf_t* pri, buf_t* aux, ma_ug_t *ug, asg_t *read_sg, hap_cov_t *cov) +void dedup_push_trans_chain(trans_chain* t_ch) { - uint32_t i, k, rid, occ, thre_pri; + uint32_t beg = t_ch->iDXs.a[t_ch->iDXs.n -1], k, m = 0, p; + uint32_t end = t_ch->uIDs.n; + radix_sort_arch32(t_ch->uIDs.a + beg, t_ch->uIDs.a + end); + for (k = m = beg, p = (uint32_t)-1; k < end; k++) + { + if(p == t_ch->uIDs.a[k]) continue; + p = t_ch->uIDs.a[k]; + t_ch->uIDs.a[m] = p; + m++; + } + t_ch->uIDs.n = m; + kv_push(uint32_t, t_ch->iDXs, t_ch->uIDs.n); +} + +void get_chain_trans(trans_chain* t_ch, uint32_t id, uint32_t** x, uint32_t* x_occ, uint32_t** y, uint32_t* y_occ) +{ + if(x) (*x) = t_ch->uIDs.a + t_ch->iDXs.a[id<<1]; + if(x_occ) (*x_occ) = t_ch->iDXs.a[(id<<1)+1] - t_ch->iDXs.a[id<<1]; + if(y) (*y) = t_ch->uIDs.a + t_ch->iDXs.a[(id<<1)+1]; + if(y_occ) (*x_occ) = t_ch->iDXs.a[(id<<1)+2] - t_ch->iDXs.a[(id<<1)+1]; +} + +inline uint32_t get_origin_uid(uint32_t v, trans_chain* t_ch) +{ + if(t_ch->u_idx[v>>1] == (uint32_t)-1) return (uint32_t)-1; + return ((t_ch->u_idx[v>>1]>>1)<<1) + ((t_ch->u_idx[v>>1]^v)&1); +} + +void print_buf_t(ma_ug_t *ug, buf_t* x, const char* command) +{ + fprintf(stderr, "%s\n", command); + uint32_t i, ori; + ma_utg_t* u = NULL; + for (i = 0; i < x->b.n; i++) + { + u = &(ug->u.a[x->b.a[i]>>1]); + if(u->n == 0) continue; + ori = x->b.a[i] & 1; + fprintf(stderr, "utg%.6ul\tori:%u\tocc:%u\n", (x->b.a[i]>>1)+1, ori, (uint32_t)u->n); + } +} + +void collect_trans_cov(const char* cmd, buf_t* pri, buf_t* aux, ma_ug_t *ug, asg_t *read_sg, hap_cov_t *cov) +{ + uint32_t i, k, rid, occ, ori, thre_pri, p_uId, c_uId, x_occ, y_occ; uint64_t len_aux, uLen, uCov; ma_utg_t* u = NULL; + trans_chain* t_ch = cov->t_ch; if(pri->b.n == 0 || aux->b.n == 0) return; len_aux = set_utg_offset(aux, ug, read_sg, cov, 0); chain_trans_ovlp(cov, ug, read_sg, pri, len_aux, &thre_pri); if(thre_pri > 0) { - for (i = uCov = 0; i < aux->b.n; i++) + // fprintf(stderr, "\n%s, thre_pri: %u\n", cmd, thre_pri); + // print_buf_t(ug, pri, "pri"); + // print_buf_t(ug, aux, "aux"); + + for (i = uCov = 0, p_uId = (uint32_t)-1; i < aux->b.n; i++) { u = &(ug->u.a[aux->b.a[i]>>1]); if(u->n == 0) continue; + ori = aux->b.a[i] & 1; for (k = 0; k < u->n; k++) { - rid = u->a[k]>>33; + rid = (ori == 1?(u->a[u->n-k-1]>>33):(u->a[k]>>33)); uCov += cov->cov[rid]; + + if(t_ch) + { + c_uId = get_origin_uid((ori == 1?((u->a[u->n-k-1]^(uint64_t)(0x100000000))>>32):(u->a[k]>>32)), + t_ch); + if(c_uId == (uint32_t)-1 || p_uId == c_uId) continue; + p_uId = c_uId; + kv_push(uint32_t, t_ch->uIDs, c_uId); + } } } + if(t_ch) kv_push(uint32_t, t_ch->iDXs, t_ch->uIDs.n);///dedup_push_trans_chain(t_ch); + - for (i = uLen = occ = 0; i < pri->b.n; i++) + for (i = uLen = occ = 0, p_uId = (uint32_t)-1; i < pri->b.n; i++) { u = &(ug->u.a[pri->b.a[i]>>1]); if(u->n == 0) continue; + ori = pri->b.a[i] & 1; for (k = 0; k < u->n; k++, occ++) { if(occ >= thre_pri) break; - rid = u->a[k]>>33; + ///rid = u->a[k]>>33; + rid = (ori == 1?(u->a[u->n-k-1]>>33):(u->a[k]>>33)); uLen += read_sg->seq[rid].len; + + if(t_ch) + { + c_uId = get_origin_uid((ori == 1?((u->a[u->n-k-1]^(uint64_t)(0x100000000))>>32):(u->a[k]>>32)), + t_ch); + if(c_uId == (uint32_t)-1 || p_uId == c_uId) continue; + p_uId = c_uId; + kv_push(uint32_t, t_ch->uIDs, c_uId); + } } if(occ >= thre_pri) break; } + if(t_ch) kv_push(uint32_t, t_ch->iDXs, t_ch->uIDs.n);///dedup_push_trans_chain(t_ch); + + if(t_ch) + { + x_occ = y_occ = 0; + get_chain_trans(t_ch, t_ch->chain_num, NULL, &x_occ, NULL, &y_occ); + if(x_occ == 0 || y_occ == 0) + { + t_ch->uIDs.n -= (x_occ + y_occ); + t_ch->iDXs.n -= 2; + } + else + { + t_ch->chain_num++; + t_ch->l0_chain++; + } + } + uCov = (uLen == 0? 0 : uCov / uLen); for (i = occ = 0; i < pri->b.n; i++) { u = &(ug->u.a[pri->b.a[i]>>1]); if(u->n == 0) continue; + ori = pri->b.a[i] & 1; for (k = 0; k < u->n; k++, occ++) { if(occ >= thre_pri) break; - rid = u->a[k]>>33; + ///rid = u->a[k]>>33; + rid = (ori == 1?(u->a[u->n-k-1]>>33):(u->a[k]>>33)); cov->cov[rid] += (uCov * read_sg->seq[rid].len); } if(occ >= thre_pri) break; @@ -11367,14 +11531,12 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ { if (!av[i].del) { + if(get_real_length(g, av[i].v^1, NULL) != 1) break; + buffer.b.n = 0; flag = get_unitig(g, ug, av[i].v, &convex, &tmp, &ll, &max_stop_nodeLen, &max_stop_baseLen, 1, &buffer); if(flag != MUL_INPUT) break; - // if(flag != TWO_INPUT && flag != MUL_INPUT) - // { - // break; - // } - + get_real_length(g, convex, &convex); if(all_covex != -1 && (uint32_t)all_covex != convex) @@ -11452,8 +11614,7 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ asg_seq_drop(g, buffer.b.a[k]>>1); } - if(cov->link) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); - if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); + if(cov) collect_trans_cov(__func__, &b_0, &b_1, ug, read_sg, cov); is_hap++; } @@ -11645,13 +11806,13 @@ kvec_asg_arc_t_warp* new_rtg_edges, int max_hang, int min_ovlp) { uint64_t i, dip_thre_max, dip_thres, n_utg; uint8_t* primary_flag = (uint8_t*)calloc(sg->n_seq, sizeof(uint8_t)); - hap_cov_t *cov = init_hap_cov_t(ug, sg, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, NULL); + hap_cov_t *cov = init_hap_cov_t(ug, sg, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, 0); int tmp_cov = asm_opt.hom_global_coverage; asm_opt.hom_global_coverage = -1; purge_dups(ug, sg, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 0, 1, cov); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 1, cov); dip_thre_max = ((double)asm_opt.hom_global_coverage)/((double)HOM_PEAK_RATE)*0.70; asm_opt.hom_global_coverage = tmp_cov; ///fprintf(stderr, "dip_thre_max: %lu\n", dip_thre_max); @@ -11680,6 +11841,66 @@ kvec_asg_arc_t_warp* new_rtg_edges, int max_hang, int min_ovlp) } +trans_chain* init_trans_chain(ma_ug_t *ug, uint64_t r_num) +{ + trans_chain *x = NULL; CALLOC(x, 1); + x->r_num = r_num; + kv_init(x->uIDs); + kv_init(x->iDXs); kv_push(uint32_t, x->iDXs, 0); + kv_init(x->rescue_hom); + MALLOC(x->u_idx, r_num); + memset(x->u_idx, -1, x->r_num*sizeof(uint32_t)); + + ma_utg_t *u = NULL; + asg_t* nsg = ug->g; + uint64_t n_vtx = nsg->n_seq, v, k, rId, is_dup = 0, vid; + + + for (v = 0; v < n_vtx; ++v) + { + if(nsg->seq[v].del) continue; + u = &(ug->u.a[v]); + if(u->m == 0) continue; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + vid = v<<1; vid |= ((u->a[k]>>32) & 1); + if(x->u_idx[rId] != (uint32_t)-1 && x->u_idx[rId] != vid) is_dup = 1; + x->u_idx[rId] = vid; + } + } + + if(is_dup) + { + for (v = 0; v < n_vtx; ++v) + { + if(nsg->seq[v].del) continue; + u = &(ug->u.a[v]); + if(u->m == 0) continue; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + vid = v<<1; vid |= ((u->a[k]>>32) & 1); + if(x->u_idx[rId] != vid) x->u_idx[rId] = (uint32_t)-1; + } + } + } + + return x; +} + +void destory_trans_chain(trans_chain **x) +{ + if(x) + { + kv_destroy((*x)->uIDs); + kv_destroy((*x)->iDXs); + kv_destroy((*x)->rescue_hom); + free((*x)->u_idx); + free((*x)); + } +} + void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num) { kv_malloc(link->a, ug_num); link->a.n = ug_num; @@ -11727,7 +11948,8 @@ void hic_clean(asg_t* read_g) ug = ma_ug_gen_primary(read_g, PRIMARY_LABLE); n_vtx = ug->g->n_seq * 2; buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); - for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; + ///for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; + tLen = get_bub_pop_max_dist_advance(ug->g, &b); uint8_t* bs_flag = (uint8_t*)calloc(n_vtx, 1); kvec_t(uint32_t) ax; kv_init(ax); @@ -11819,7 +12041,8 @@ void hic_clean(asg_t* read_g) void output_hic_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, -R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) +R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, +bub_label_t* b_mask_t) { hic_clean(sg); @@ -11844,7 +12067,7 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov asm_opt.purge_simi_rate = asm_opt.purge_simi_rate_hic; adjust_utg_by_primary(©_ug, copy_sg, TRIO_THRES, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, - max_hang, min_ovlp, &new_rtg_edges, &link); + max_hang, min_ovlp, &new_rtg_edges, &link, b_mask_t); ma_ug_destroy(copy_ug); asg_destroy(copy_sg); @@ -11869,10 +12092,10 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov output_unitig_graph(sg, coverage_cut, output_file_name, sources, ruIndex, max_hang, min_ovlp); output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang, min_ovlp, 0); + 0.05, 0.9, max_hang, min_ovlp, 0, b_mask_t); output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang, min_ovlp, 0); + 0.05, 0.9, max_hang, min_ovlp, 0, b_mask_t); } ma_ug_t* merge_utg(ma_ug_t **dest, ma_ug_t **src) @@ -11935,15 +12158,15 @@ ma_ug_t* merge_utg(ma_ug_t **dest, ma_ug_t **src) void benchmark_hic_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, -float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, bub_label_t* b_mask_t) { ma_ug_t *ug_1 = output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, - chimeric_rate, drop_ratio, max_hang, min_ovlp, 1); + chimeric_rate, drop_ratio, max_hang, min_ovlp, 1, b_mask_t); ma_ug_t *ug_2 = output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, - chimeric_rate, drop_ratio, max_hang, min_ovlp, 1); + chimeric_rate, drop_ratio, max_hang, min_ovlp, 1, b_mask_t); fprintf(stderr, "ug_1->u.n: %u, ug_2->u.n: %u\n", (uint32_t)ug_1->u.n, (uint32_t)ug_2->u.n); ma_ug_t *ug = merge_utg(&ug_1, &ug_2); fprintf(stderr, "ug->u.n: %u\n", (uint32_t)ug->u.n); @@ -12556,16 +12779,6 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hap_cov_t *cov) return_flag = get_unitig(g, ug, av[i].v, &convex, &ll, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 1, NULL); - /**********************for debug************************/ - // uint32_t debug_return_flag; - // long long debug_ll; - // debug_return_flag = get_unitig_back(g, ug, av[i].v, &convex, &debug_ll, &tmp, NULL); - // if(debug_return_flag != return_flag || debug_ll != ll) - // { - // fprintf(stderr, "ERROR\n"); - // } - /**********************for debug************************/ - if(return_flag==LOOP) continue; if(return_flag==END_TIPS) n_tips++; @@ -12622,8 +12835,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hap_cov_t *cov) } } - if(cov->link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); - if(cov && operation != CUT) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); + if(cov && operation != CUT) collect_trans_cov(__func__, &b_0, &b_1, ug, read_sg, cov); } } } @@ -12721,8 +12933,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_thre } } - if(cov->link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); - if(cov && operation != CUT) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); + if(cov && operation != CUT) collect_trans_cov(__func__, &b_0, &b_1, ug, read_sg, cov); break; } @@ -12851,16 +13062,6 @@ hap_cov_t *cov) return_flag = get_unitig(g, ug, av[i].v, &convex, &tmp, &ll, &max_stop_nodeLen, &max_stop_baseLen, 1, NULL); - /**********************for debug************************/ - // uint32_t debug_return_flag; - // long long debug_ll; - // debug_return_flag = get_unitig_back(g, ug, av[i].v, &convex, &tmp, &debug_ll, NULL); - // if(debug_return_flag != return_flag || debug_ll != ll) - // { - // fprintf(stderr, "* ERROR\n"); - // } - /**********************for debug************************/ - if(return_flag==LOOP) continue; if(return_flag==END_TIPS) n_tips++; @@ -12909,8 +13110,7 @@ hap_cov_t *cov) asg_seq_drop(g, b.b.a[k]>>1); } - if(cov->link) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); - if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); + if(cov) collect_trans_cov(__func__, &b_0, &b_1, ug, read_sg, cov); is_hap++; } @@ -13211,8 +13411,7 @@ ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_ asg_seq_drop(g, b.b.a[k]>>1); } - if(cov->link) collect_reverse_unitigs(&b_0, &b_1, cov->link, ug, read_sg); - if(cov) collect_trans_cov(&b_0, &b_1, ug, read_sg, cov); + if(cov) collect_trans_cov(__func__, &b_0, &b_1, ug, read_sg, cov); ///lable the primary one b_0.b.n = 0; @@ -13614,7 +13813,7 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate, hap_cov_t *cov) redo: ///print_untig((ug), 61955, "i-0:", 0); - asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, cov); + asg_pop_bubble_primary_trio(ug, NULL, trio_flag, DROP, cov); untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, cov); magic_trio_phasing(g, ug, read_g, coverage_cut, sources, reverse_sources, 2, ruIndex, trio_flag, trio_drop_rate); ///drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); @@ -13630,7 +13829,7 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate, hap_cov_t *cov) { pre_cons = get_graph_statistic(g); ///need consider tangles - asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP, cov); + asg_pop_bubble_primary_trio(ug, NULL, trio_flag, DROP, cov); /**********debug**********/ if(just_bubble_pop == 0) { @@ -13667,8 +13866,25 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate, hap_cov_t *cov) } -void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, +void print_graph_statistic(asg_t *g, const char* cmd) +{ + uint64_t n_arc = 0, n_node = 0, size = 0; + uint32_t n_vtx = g->n_seq, v; + + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v].del || g->seq[v].c == ALTER_LABLE) continue; + n_arc += get_real_length(g, v<<1, NULL) + get_real_length(g, (v<<1)+1, NULL); + n_node++; + size += g->seq[v].len; + } + + fprintf(stderr, "%s->n_node: %lu, n_arc: %lu, size: %lu\n", cmd, n_node, n_arc, size); +} + +void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* sources, +ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, float drop_ratio, hap_cov_t *cov) @@ -13677,10 +13893,11 @@ float drop_ratio, hap_cov_t *cov) asg_t *g = ug->g; int round = T_ROUND; - - redo: - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, cov); + redo: + ///print_graph_statistic(g, "beg"); + asg_pop_bubble_primary_trio(ug, NULL, (uint32_t)-1, DROP, cov); untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, cov); + if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, @@ -13690,18 +13907,18 @@ float drop_ratio, hap_cov_t *cov) long long pre_cons = get_graph_statistic(g); long long cur_cons = 0; while(pre_cons != cur_cons) - { + { pre_cons = get_graph_statistic(g); - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, cov); + asg_pop_bubble_primary_trio(ug, NULL, (uint32_t)-1, DROP, cov); if(just_bubble_pop == 0) { - ///need consider tangles - asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, cov); - asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, cov); - asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, cov); + ///need consider tangles + asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, cov); + asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, cov); + asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, cov); asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, cov); - detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); + detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); if(round != T_ROUND) { unitig_arc_del_short_diploid_by_length_topo(g, ug, drop_ratio, asm_opt.max_short_tip, @@ -13709,16 +13926,18 @@ float drop_ratio, hap_cov_t *cov) } } cur_cons = get_graph_statistic(g); - } + } untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, cov); if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, 2); } - resolve_tangles(ug, read_g, reverse_sources, 20, 100, 0.05, 0.2, ruIndex, (uint32_t)-1, drop_ratio); - drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); - unitig_arc_del_short_diploid_by_length_topo(g, ug, drop_ratio, asm_opt.max_short_tip, reverse_sources, 0, 1); + + resolve_tangles(ug, read_g, reverse_sources, 20, 100, 0.05, 0.2, ruIndex, (uint32_t)-1, drop_ratio); + drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); + unitig_arc_del_short_diploid_by_length_topo(g, ug, drop_ratio, asm_opt.max_short_tip, reverse_sources, 0, 1); + ///print_graph_statistic(g, "end"); if(round > 0) { if(round != T_ROUND) @@ -14368,15 +14587,15 @@ void adjust_utg_by_trio(ma_ug_t **ug, asg_t* read_g, uint8_t flag, float drop_ra ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, -kvec_asg_arc_t_warp* new_rtg_edges) +kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) { asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq; - hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, NULL); + hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, 0); purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 1, cov); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, + 1, 1, cov); if(asm_opt.recover_atg_cov_min == -1024) { asm_opt.recover_atg_cov_max = asm_opt.hom_global_coverage/HOM_PEAK_RATE; @@ -14429,12 +14648,12 @@ kvec_asg_arc_t_warp* new_rtg_edges) if (!(asm_opt.flag & HA_F_BAN_POST_JOIN)) { rescue_missing_overlaps_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 0, 1, NULL); + min_ovlp, 0, 0, 1, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); rescue_contained_reads_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 10, 0, 1, NULL, NULL); + min_ovlp, 0, 10, 0, 1, NULL, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); } @@ -14455,8 +14674,8 @@ kvec_asg_arc_t_warp* new_rtg_edges) if(asm_opt.purge_level_trio == 1) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 0, cov); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, 1, 0, + cov); ///delete_useless_nodes(ug); delete_useless_trio_nodes(ug, read_g, coverage_cut, sources, ruIndex); } @@ -14488,7 +14707,7 @@ int debug_untig_length(ma_ug_t *g, uint32_t tipsLen, const char* name) ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, -float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench) +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, bub_label_t* b_mask_t) { char* gfa_name = (char*)malloc(strlen(output_file_name)+100); sprintf(gfa_name, "%s.%s.p_ctg.gfa", output_file_name, (flag==FATHER?"hap1":"hap2")); @@ -14505,7 +14724,7 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench) ///print_untig_by_read(ug, "m64011_190830_220126/117834372/ccs", 865264, sources, reverse_sources, "beg"); adjust_utg_by_trio(&ug, sg, flag, TRIO_THRES, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, - max_hang, min_ovlp, &new_rtg_edges); + max_hang, min_ovlp, &new_rtg_edges, b_mask_t); if(asm_opt.b_low_cov > 0) { @@ -14937,7 +15156,7 @@ int asg_bub_backtrack_check_switch(asg_t *g, ma_ug_t *utg, uint32_t v0, buf_t *b } // pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well -uint64_t asg_bub_pop1_primary_trio_switch_check(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, +uint64_t asg_bub_pop1_primary_trio_switch_check(asg_t *g, ma_ug_t *utg, uint32_t v0, uint64_t max_dist, buf_t *b, uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, int* is_switch) { @@ -14979,7 +15198,9 @@ int* is_switch) (in the view of target) p->ol: overlap length **/ - + ///if this edge has been deleted + if (av[i].del) continue; + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l binfo_t *t = &b->a[w]; ///that means there is a circle, directly terminate the whole bubble poping @@ -14990,15 +15211,14 @@ int* is_switch) if(is_first) l = 0; /****************************may have bugs********************************/ - ///if this edge has been deleted - if (av[i].del) continue; + ///push the edge ///high 32-bit of g->idx[v] is the start point of v's edges //so here is the point of this specfic edge kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); ///find a too far path? directly terminate the whole bubble poping - if (d + l > (uint32_t)max_dist) break; // too far + if (d + l > max_dist) break; // too far ///if this node if (t->s == 0) { // this vertex has never been visited @@ -15176,8 +15396,113 @@ pop_reset: } +uint64_t asg_bub_pop1_label(asg_t *g, uint32_t v0, uint64_t max_dist, buf_s_t *b) +{ + uint32_t i, n_pending = 0, is_first = 1, n_tips, tip_end; + uint64_t n_pop = 0; + if (g->seq[v0>>1].del) return 0; // already deleted + if(get_real_length(g, v0, NULL)<2) return 0; -uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, + ///S saves nodes with all incoming edges visited + b->S.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[v0].d = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, v0); n_tips = 0; tip_end = (uint32_t)-1; + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), d = b->a[v].d; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + ///why we have this assert? + ///assert(nv > 0); + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + ///if this edge has been deleted + if (av[i].del) continue; + + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l + binfo_s_t *t = &b->a[w]; + ///that means there is a circle, directly terminate the whole bubble poping + ///if (w == v0) goto pop_reset; + if ((w>>1) == (v0>>1)) goto pop_reset; + /****************************may have bugs********************************/ + ///important when poping at long untig graph + if(is_first) l = 0; + /****************************may have bugs********************************/ + ///find a too far path? directly terminate the whole bubble poping + if ((uint64_t)d + (uint64_t)l > max_dist) break; // too far + + ///if this node + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p is the parent node of + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l; + ///incoming edges of w + ///t->r = count_out(g, w^1); + t->r = get_real_length(g, w^1, NULL); + ++n_pending; + } else { // visited before + ///it is the shortest edge + if (d + l < t->d) t->d = d + l, t->p = v; // update dist + } + ///assert(t->r > 0); + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = get_real_length(g, w, NULL); + /****************************may have bugs for bubble********************************/ + if(x > 0) + { + kv_push(uint32_t, b->S, w); + } + else + { + ///at most one tip + if(n_tips != 0) goto pop_reset; + n_tips++; + tip_end = w; + } + /****************************may have bugs for bubble********************************/ + --n_pending; + } + } + is_first = 0; + //if found a tip + /****************************may have bugs for bubble********************************/ + if(n_tips == 1) + { + if(tip_end != (uint32_t)-1 && n_pending == 0 && b->S.n == 0) + { + kv_push(uint32_t, b->S, tip_end); + break; + } + else + { + goto pop_reset; + } + } + /****************************may have bugs for bubble********************************/ + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) goto pop_reset; + } while (b->S.n > 1 || n_pending); + + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_s_t *t = &b->a[b->b.a[i]]; + t->s = t->d = 0; + } + return n_pop; +} + + + +uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, uint64_t max_dist, buf_t *b, uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, hap_cov_t *cov) { @@ -15219,6 +15544,8 @@ hap_cov_t *cov) (in the view of target) p->ol: overlap length **/ + ///if this edge has been deleted + if (av[i].del) continue; uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l binfo_t *t = &b->a[w]; @@ -15230,15 +15557,14 @@ hap_cov_t *cov) if(is_first) l = 0; /****************************may have bugs********************************/ - ///if this edge has been deleted - if (av[i].del) continue; + ///push the edge ///high 32-bit of g->idx[v] is the start point of v's edges //so here is the point of this specfic edge kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); ///find a too far path? directly terminate the whole bubble poping - if (d + l > (uint32_t)max_dist) break; // too far + if (d + l > max_dist) break; // too far ///if this node if (t->s == 0) { // this vertex has never been visited @@ -15414,41 +15740,493 @@ pop_reset: return n_pop; } +uint64_t dfs_subgraph(asg_t *g, buf_t *b, uint32_t id, uint32_t *p_bub) +{ + uint64_t len = 0; + uint32_t cur, nv, v, w, i, kv_0, kv_1, flag_0 = 0, flag_1 = 0; + asg_arc_t *av = NULL; + (*p_bub) = 0; + if(b->a[id].s) return 0; + b->S.n = 0; + kv_push(uint32_t, b->S, id); + + while (b->S.n > 0) + { + b->S.n--; + cur = b->S.a[b->S.n]; + if(b->a[cur].s) continue; + b->a[cur].s = 1; + len += g->seq[cur].len; + + v = cur<<1; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + for (i = kv_0 = 0; i < nv; i++) + { + w = av[i].v>>1; + if(av[i].del) continue; + kv_0++; + if(b->a[w].s) continue; + kv_push(uint32_t, b->S, w); + } + + v = (cur<<1)+1; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + for (i = kv_1 = 0; i < nv; i++) + { + w = av[i].v>>1; + if(av[i].del) continue; + kv_1++; + if(b->a[w].s) continue; + kv_push(uint32_t, b->S, w); + } + + if(kv_0 > 0 && kv_1 > 0) flag_0++; + if(kv_0 > 1) flag_1++; + if(kv_1 > 1) flag_1++; + } + + if(flag_0 > 0 && flag_1 > 1) (*p_bub) = 1; + return len; +} +uint64_t get_bub_pop_max_dist(asg_t *g, buf_t *b) +{ + uint32_t n_vtx = g->n_seq, i, p_bub; + uint64_t cLen = 0, mLen = 0, tLen = 0; + + + for (i = 0; i < n_vtx; ++i) + { + if(b->a[i].s) continue; + cLen = dfs_subgraph(g, b, i, &p_bub); + tLen += cLen; + if(p_bub == 0) continue;///no bubble + if(cLen > mLen) mLen = cLen; + } + + for (i = 0; i < n_vtx; ++i) + { + ///if(b->a[i].s == 0) fprintf(stderr, "ERROR\n"); + b->a[i].s = 0; + ///debug_tLen += g->seq[i].len; + } + ///if(debug_tLen != tLen) fprintf(stderr, "ERROR\n"); + + ///fprintf(stderr, "mLen: %lu, tLen: %lu\n", mLen, tLen); + b->S.n = 0; + return mLen; +} + +uint64_t dfs_subgraph_advance(asg_t *g, buf_t *b, uint32_t x, uint32_t *p_bub) +{ + uint64_t len = 0; + uint32_t c_v, e_v, nv, convex, v, i, kv_0, kv_1, flag_0 = 0, flag_1 = 0, op; + asg_arc_t *av = NULL; + long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen, uLen; + (*p_bub) = 0; + if(b->a[x>>1].s || g->seq[x>>1].del) return 0; + b->S.n = 0; + kv_push(uint32_t, b->S, x); + + while (b->S.n > 0) + { + b->S.n--; + c_v = b->S.a[b->S.n]; + if(b->a[c_v>>1].s) continue; + + b->b.n = 0; + op = get_unitig(g, NULL, c_v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, b); + uLen = baseLen; + for(i = 0; i < b->b.n; i++) + { + ///if(b->a[b->b.a[i]>>1].s == 1) fprintf(stderr, "ERROR 3\n"); + b->a[b->b.a[i]>>1].s = 1; + } + + if(op == LOOP) return 0; + + + e_v = convex^1; + b->b.n = 0; + op = get_unitig(g, NULL, e_v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, b); + uLen = MAX(uLen, baseLen); + + len += uLen; + + + v = c_v^1; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + for (i = kv_0 = 0; i < nv; i++) + { + if(av[i].del) continue; + kv_0++; + if(b->a[av[i].v>>1].s) continue; + kv_push(uint32_t, b->S, av[i].v); + } + + v = e_v^1; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + for (i = kv_1 = 0; i < nv; i++) + { + if(av[i].del) continue; + kv_1++; + if(b->a[av[i].v>>1].s) continue; + kv_push(uint32_t, b->S, av[i].v); + } + + if(kv_0 > 0 && kv_1 > 0) flag_0++; + if(kv_0 > 1) flag_1++; + if(kv_1 > 1) flag_1++; + } + + if(flag_0 > 0 && flag_1 > 1) (*p_bub) = 1; + return len; +} + +uint64_t get_bub_pop_max_dist_advance(asg_t *g, buf_t *b) +{ + asg_arc_t *av = NULL; + uint32_t n_vtx = g->n_seq<<1, k, v, w, kv, nv, p_bub; + uint64_t cLen = 0, mLen = 0; + + + for (v = 0; v < n_vtx; ++v) + { + if(b->a[v>>1].s) continue; + if(g->seq[v>>1].del) continue; + + + av = asg_arc_a(g, v); + nv = asg_arc_n(g, v); + for (k = kv = 0; k < nv; k++) + { + if(av[k].del) continue; + w = av[k].v^1; + kv++; + } + if(kv == 1 && get_real_length(g, w, NULL) == 1) continue; + + cLen = dfs_subgraph_advance(g, b, v^1, &p_bub); + if(p_bub == 0) continue;///no bubble + if(cLen > mLen) mLen = cLen; + } + + for (k = 0; k < g->n_seq; ++k) + { + // if(b->a[k].s == 0 && !g->seq[k].del) + // { + // uint32_t convex; + // long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; + // if(get_unitig(g, NULL, k<<1, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + // &max_stop_baseLen, 1, NULL) != LOOP) + // { + // fprintf(stderr, "ERROR 1\n"); + // } + // } + b->a[k].s = 0; + } + + // for (; k < n_vtx; ++k) + // { + // if(b->a[k].s == 1) fprintf(stderr, "ERROR 2\n"); + // } + + ///fprintf(stderr, "mLen: %lu, tLen: %lu\n", mLen, tLen); + b->S.n = b->b.n = 0; + return mLen; +} + +inline uint32_t get_unitig_s(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* endNode, +long long* nodeLen, long long* baseLen, long long* max_stop_nodeLen, long long* max_stop_baseLen, +uint32_t stops_threshold, buf_s_t* b) +{ + ma_utg_v* u = NULL; + uint32_t v = begNode, w, k; + uint32_t kv, return_flag, n_stops = 0; + long long pre_baseLen = 0, pre_nodeLen = 0; + long long cur_baseLen = 0, cur_nodeLen = 0; + (*max_stop_nodeLen) = (*max_stop_baseLen) = (*nodeLen) = (*baseLen) = 0; + (*endNode) = (uint32_t)-1; + if(ug!=NULL) u = &(ug->u); + + while (1) + { + kv = get_real_length(sg, v, NULL); + (*endNode) = v; + if(u == NULL) + { + (*nodeLen)++; + } + else + { + (*nodeLen) += EvaluateLen((*u), v>>1); + } + if(b) kv_push(uint32_t, b->b, v); + ///means reach the end of a unitig + if(kv!=1) (*baseLen) += sg->seq[v>>1].len; + if(kv==0) + { + return_flag = END_TIPS; + break; + ///return END_TIPS; + } + if(kv>1) + { + return_flag = MUL_OUTPUT; + break; + ///return MUL_OUTPUT; + } + ///kv must be 1 here + kv = get_real_length(sg, v, &w); + ///means reach the end of a unitig + if(get_real_length(sg, w^1, NULL)!=1) + { + + n_stops++; + if(n_stops >= stops_threshold) + { + (*baseLen) += sg->seq[v>>1].len; + return_flag = MUL_INPUT; + break; + ///return MUL_INPUT; + } + else + { + for (k = 0; k < asg_arc_n(sg, v); k++) + { + if(asg_arc_a(sg, v)[k].del) continue; + ///here is just one undeleted edge + (*baseLen) += asg_arc_len(asg_arc_a(sg, v)[k]); + break; + } + } + + cur_baseLen = (*baseLen) - pre_baseLen; + pre_baseLen = (*baseLen); + if(cur_baseLen > (*max_stop_baseLen)) + { + (*max_stop_baseLen) = cur_baseLen; + } + + + cur_nodeLen = (*nodeLen) - pre_nodeLen; + pre_nodeLen = (*nodeLen); + if(cur_nodeLen > (*max_stop_nodeLen)) + { + (*max_stop_nodeLen) = cur_nodeLen; + } + } + else + { + for (k = 0; k < asg_arc_n(sg, v); k++) + { + if(asg_arc_a(sg, v)[k].del) continue; + ///here is just one undeleted edge + (*baseLen) += asg_arc_len(asg_arc_a(sg, v)[k]); + break; + } + } + + + v = w; + if(v == begNode) + { + return_flag = LOOP; + break; + ///return LOOP; + } + } + + + + + cur_baseLen = (*baseLen) - pre_baseLen; + pre_baseLen = (*baseLen); + if(cur_baseLen > (*max_stop_baseLen)) + { + (*max_stop_baseLen) = cur_baseLen; + } + + + cur_nodeLen = (*nodeLen) - pre_nodeLen; + pre_nodeLen = (*nodeLen); + if(cur_nodeLen > (*max_stop_nodeLen)) + { + (*max_stop_nodeLen) = cur_nodeLen; + } + + return return_flag; +} + +uint64_t dfs_subgraph_s_advance(asg_t *g, buf_s_t *b, uint32_t x, uint32_t *p_bub) +{ + uint64_t len = 0; + uint32_t c_v, e_v, nv, convex, v, i, kv_0, kv_1, flag_0 = 0, flag_1 = 0, op; + asg_arc_t *av = NULL; + long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen, uLen; + (*p_bub) = 0; + if(b->a[x>>1].s || g->seq[x>>1].del) return 0; + b->S.n = 0; + kv_push(uint32_t, b->S, x); + + while (b->S.n > 0) + { + b->S.n--; + c_v = b->S.a[b->S.n]; + if(b->a[c_v>>1].s) continue; + + b->b.n = 0; + op = get_unitig_s(g, NULL, c_v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, b); + + uLen = baseLen; + for(i = 0; i < b->b.n; i++) + { + ///if(b->a[b->b.a[i]>>1].s == 1) fprintf(stderr, "ERROR 3\n"); + b->a[b->b.a[i]>>1].s = 1; + } + + if(op == LOOP) return 0; + + + e_v = convex^1; + b->b.n = 0; + op = get_unitig_s(g, NULL, e_v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, b); + uLen = MAX(uLen, baseLen); + + len += uLen; + + + v = c_v^1; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + for (i = kv_0 = 0; i < nv; i++) + { + if(av[i].del) continue; + kv_0++; + if(b->a[av[i].v>>1].s) continue; + kv_push(uint32_t, b->S, av[i].v); + } + + v = e_v^1; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + for (i = kv_1 = 0; i < nv; i++) + { + if(av[i].del) continue; + kv_1++; + if(b->a[av[i].v>>1].s) continue; + kv_push(uint32_t, b->S, av[i].v); + } + + if(kv_0 > 0 && kv_1 > 0) flag_0++; + if(kv_0 > 1) flag_1++; + if(kv_1 > 1) flag_1++; + } + + if(flag_0 > 0 && flag_1 > 1) (*p_bub) = 1; + return len; +} + +uint64_t get_s_bub_pop_max_dist_advance(asg_t *g, buf_s_t *b) +{ + asg_arc_t *av = NULL; + uint32_t n_vtx = g->n_seq<<1, k, v, w, kv, nv, p_bub; + uint64_t cLen = 0, mLen = 0; + + + for (v = 0; v < n_vtx; ++v) + { + if(b->a[v>>1].s) continue; + if(g->seq[v>>1].del) continue; + + + av = asg_arc_a(g, v); + nv = asg_arc_n(g, v); + for (k = kv = 0; k < nv; k++) + { + if(av[k].del) continue; + w = av[k].v^1; + kv++; + } + if(kv == 1 && get_real_length(g, w, NULL) == 1) continue; + + cLen = dfs_subgraph_s_advance(g, b, v^1, &p_bub); + if(p_bub == 0) continue;///no bubble + if(cLen > mLen) mLen = cLen; + } + + for (k = 0; k < g->n_seq; ++k) + { + // if(b->a[k].s == 0 && !g->seq[k].del) + // { + // uint32_t convex; + // long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; + // if(get_unitig(g, NULL, k<<1, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + // &max_stop_baseLen, 1, NULL) != LOOP) + // { + // fprintf(stderr, "ERROR 1\n"); + // } + // } + b->a[k].s = 0; + } + + // for (; k < n_vtx; ++k) + // { + // if(b->a[k].s == 1) fprintf(stderr, "ERROR 2\n"); + // } + + ///fprintf(stderr, "mLen: %lu, tLen: %lu\n", mLen, tLen); + b->S.n = b->b.n = 0; + return mLen; +} + // pop bubbles -int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov) +int asg_pop_bubble_primary_trio(ma_ug_t *ug, uint64_t* i_max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov) { asg_t *g = ug->g; uint32_t v, n_vtx = g->n_seq * 2; - uint64_t n_pop = 0; + uint64_t n_pop = 0, max_dist; buf_t b; if (!g->is_symm) asg_symm(g); memset(&b, 0, sizeof(buf_t)); ///set information for each node b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); - //traverse all node with two directions - for (v = 0; v < n_vtx; ++v) { - uint32_t i, n_arc = 0, nv = asg_arc_n(g, v); - asg_arc_t *av = asg_arc_a(g, v); - ///some node could be deleted - if (nv < 2 || g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - ///some edges could be deleted - for (i = 0; i < nv; ++i) // asg_bub_pop1() may delete some edges/arcs - if (!av[i].del) ++n_arc; - if (n_arc > 1) - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL, cov); - } - free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); - if (n_pop) asg_cleanup(g); - - if(VERBOSE >= 1) + if(i_max_dist) max_dist = (*i_max_dist); + else max_dist = get_bub_pop_max_dist_advance(g, &b); + if(max_dist > 0) { - fprintf(stderr, "[M::%s] popped %lu bubbles\n", __func__, (unsigned long)n_pop); + //traverse all node with two directions + for (v = 0; v < n_vtx; ++v) { + uint32_t i, n_arc = 0, nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + ///some node could be deleted + if (nv < 2 || g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; + ///some edges could be deleted + for (i = 0; i < nv; ++i) // asg_bub_pop1() may delete some edges/arcs + if (!av[i].del) ++n_arc; + if (n_arc > 1) + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL, cov); + } + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + if (n_pop) asg_cleanup(g); + if(VERBOSE >= 1) + { + fprintf(stderr, "[M::%s] popped %lu bubbles\n", __func__, (unsigned long)n_pop); + } } return n_pop; } + int test_triangular_directly(asg_t *g, uint32_t v, long long min_edge_length, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex) { @@ -15893,7 +16671,7 @@ void destory_C_graph(C_graph* g) long long asg_arc_del_simple_circle_untig(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *g, long long circleLen, int is_drop) { uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag; - long long ll/**, coverage**/; + long long ll; asg_arc_t *aw; uint32_t nw, k; buf_t b; @@ -18526,8 +19304,8 @@ buf_t* bb) return reduce; } -int pop_bubble_at_tangle(ma_ug_t *ug, int max_dist, -uint64_t* nodes, uint64_t n, uint32_t beg, uint32_t end, +///max_dist is ok, since tangle shouldn't be too large +int pop_bubble_at_tangle(ma_ug_t *ug, uint64_t max_dist, uint64_t* nodes, uint64_t n, uint32_t beg, uint32_t end, uint32_t positive_flag, uint32_t negative_flag) { asg_t *g = ug->g; @@ -20822,7 +21600,8 @@ char* output_file_name, ma_hit_t_alloc** reverse_sources, R_to_U* ruIndex) hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, -ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp, hc_links* link) +ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp, +uint32_t is_collect_trans) { uint32_t n_ux = ug->g->n_seq, i, k, j, v, rId, tn, is_Unitig, r_i, nv, w, C_bases; uint8_t *set = NULL; @@ -20834,7 +21613,7 @@ ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_o x->max_hang = max_hang; x->min_ovlp = min_ovlp; x->read_g = read_g; - x->link = link; + x->t_ch = NULL; kv_init(x->u_buffer.a); kv_init(x->tailIndex.a); kv_init(x->prevIndex.a); @@ -20930,25 +21709,8 @@ ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, int max_hang, int min_o if(set) free(set); - - if(x->link) - { - memset(x->link->u_idx, -1, read_g->n_seq*sizeof(uint32_t)); - asg_t* nsg = ug->g; - uint32_t n_vtx = nsg->n_seq; - for (v = 0; v < n_vtx; ++v) - { - if(nsg->seq[v].del) continue; - u = &(ug->u.a[v]); - if(u->m == 0) continue; - for (k = 0; k < u->n; k++) - { - rId = u->a[k]>>33; - ///if(read_g->seq[rId].c == FAKE_LABLE) continue; - x->link->u_idx[rId] = v; - } - } - } + x->t_ch = NULL; + if(is_collect_trans) x->t_ch = init_trans_chain(ug, read_g->n_seq); return x; } @@ -20962,6 +21724,7 @@ void destory_hap_cov_t(hap_cov_t **x) kv_destroy((*x)->u_buffer.a); kv_destroy((*x)->tailIndex.a); kv_destroy((*x)->prevIndex.a); + if((*x)->t_ch) destory_trans_chain(&((*x)->t_ch)); free((*x)); } } @@ -21025,7 +21788,32 @@ void reset_reverse_unitigs(hc_links* link, ma_utg_t *u) } -void append_utg(ma_ug_t* ptg, ma_ug_t* atg, hc_links* link) +void reset_trans_chain(trans_chain* t_ch, ma_utg_t *u) +{ + uint32_t k = 0, i = 0, p_uId = (uint32_t)-1, c_uId; + if(u->n == 0 || u->m == 0) return; + for (k = 0; k < u->n; k++) + { + c_uId = get_origin_uid(u->a[k]>>32, t_ch); + if(c_uId == (uint32_t)-1) continue; + c_uId >>= 1; + if(p_uId == c_uId) continue; + p_uId = c_uId; + + for (i = 0; i < t_ch->uIDs.n; i++) + { + if((t_ch->uIDs.a[i]>>1) == p_uId) t_ch->uIDs.a[i] = (uint32_t)-1; + } + + + // for (i = 0; i < t_ch->uIDs.n; i++) + // { + // ///if((t_ch->uIDs.a[i]>>1) == p_uId) t_ch->uIDs.a[i] = (uint32_t)-1; + // } t_ch); + } +} + +void append_utg(ma_ug_t* ptg, ma_ug_t* atg, trans_chain* t_ch) { uint64_t num_nodes = 0; asg_t* nsg = atg->g; @@ -21050,7 +21838,8 @@ void append_utg(ma_ug_t* ptg, ma_ug_t* atg, hc_links* link) for (v = 0; v < atg->g->n_seq; ++v) { if(atg->g->seq[v].del || atg->u.a[v].m == 0) continue; - if(link) reset_reverse_unitigs(link, &(atg->u.a[v])); + ///if(link) reset_reverse_unitigs(link, &(atg->u.a[v])); + if(t_ch) reset_trans_chain(t_ch, &(atg->u.a[v])); p = &(ptg->u.a[ptg->u.n]); p->len = atg->u.a[v].len; @@ -21105,7 +21894,7 @@ void print_utg_coverage(ma_ug_t *ug, ma_sub_t* coverage_cut, uint32_t v, ma_hit_ } void recover_utg_by_coverage(ma_ug_t **ptg, asg_t* read_g, ma_sub_t* coverage_cut, -ma_hit_t_alloc* sources, R_to_U* ruIndex, hc_links* link) +ma_hit_t_alloc* sources, R_to_U* ruIndex, trans_chain* t_ch) { if(asm_opt.recover_atg_cov_min == -1) return; if(asm_opt.recover_atg_cov_max == -1) return; @@ -21192,7 +21981,7 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, hc_links* link) { asg_cleanup(nsg); asg_symm(nsg); - append_utg(*ptg, atg, link); + append_utg(*ptg, atg, t_ch); n_vtx = read_g->n_seq; for (v = 0; v < n_vtx; v++) @@ -21229,26 +22018,67 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, hc_links* link) ma_ug_destroy(atg); } +void update_hc_links_by_trans_chain(hc_links* link, trans_chain* t_ch) +{ + ///fprintf(stderr, "sbsbsbsbsbsb1sbsbsbsbsbsb, l0_chain: %u, chain_num: %u\n", t_ch->l0_chain, t_ch->chain_num); + uint64_t d = RC_1; + uint32_t i, k, m, *x = NULL, x_occ, *y = NULL, y_occ, v_x, v_y; + for (i = 0; i < t_ch->l0_chain; i++) + { + x_occ = y_occ = 0; + get_chain_trans(t_ch, i, &x, &x_occ, &y, &y_occ); + for (k = 0; k < x_occ; k++) + { + if(x[k] == (uint32_t)-1) continue; + v_x = x[k]>>1; + for (m = 0; m < y_occ; m++) + { + if(y[m] == (uint32_t)-1) continue; + v_y = y[m]>>1; + push_hc_edge(&(link->a.a[v_x]), v_y, 1, 1, &d); + push_hc_edge(&(link->a.a[v_y]), v_x, 1, 1, &d); + } + } + } + /*******************************for debug************************************/ + for (i = 0; i < t_ch->l0_chain; i++) + { + x_occ = y_occ = 0; + get_chain_trans(t_ch, i, &x, &x_occ, &y, &y_occ); + fprintf(stderr, "\n******x******"); + for (k = 0; k < x_occ; k++) + { + fprintf(stderr, "utg%.6ul\t%u\n", (x[k]>>1)+1, x[k]&1); + } + fprintf(stderr, "******y******"); + for (k = 0; k < y_occ; k++) + { + fprintf(stderr, "utg%.6ul\t%u\n", (y[k]>>1)+1, y[k]&1); + } + } + /*******************************for debug************************************/ +} void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, -kvec_asg_arc_t_warp* new_rtg_edges, hc_links* lk) +kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) { asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq, k, rId, just_contain; ma_utg_t* u = NULL; - hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, lk); - - ///print_utg_coverage(*ug, coverage_cut, 440, sources); - ///exit(0); - drop_semi_circle((*ug), nsg, read_g, reverse_sources, ruIndex); + hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, link? 1:0); + ///print_utg_coverage(*ug, coverage_cut, 440, sources); + ///exit(0); + drop_semi_circle((*ug), nsg, read_g, reverse_sources, ruIndex); + asg_cleanup(nsg); adjust_utg_advance(read_g, (*ug), reverse_sources, ruIndex); + nsg = (*ug)->g; n_vtx = nsg->n_seq; for (v = 0; v < n_vtx; ++v) @@ -21257,19 +22087,19 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* lk) nsg->seq[v].c = PRIMARY_LABLE; EvaluateLen((*ug)->u, v) = (*ug)->u.a[v].n; } - clean_primary_untig_graph(*ug, read_g, reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, + clean_primary_untig_graph(*ug, read_g, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); - if(cov->link) goto skip_purge; + if(link) goto skip_purge; if(asm_opt.purge_level_primary > 0) { just_contain = 0; if(asm_opt.purge_level_primary == 1) just_contain = 1; purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0, cov); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, + just_contain, 0, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } @@ -21277,10 +22107,10 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* lk) if (!(asm_opt.flag & HA_F_BAN_POST_JOIN)) { rescue_missing_overlaps_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 0, 1, NULL); + min_ovlp, 0, 0, 1, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); rescue_contained_reads_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 10, 0, 1, NULL, NULL); + min_ovlp, 0, 10, 0, 1, NULL, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); if(asm_opt.purge_level_primary > 0) @@ -21288,8 +22118,8 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* lk) just_contain = 0; if(asm_opt.purge_level_primary == 1) just_contain = 1; purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0, cov); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, + just_contain, 0, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } @@ -21298,8 +22128,8 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* lk) if(asm_opt.purge_level_primary == 0) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 0, 1, cov); + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, 0, + 1, cov); } n_vtx = read_g->n_seq; @@ -21353,31 +22183,15 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* lk) } skip_purge: - recover_utg_by_coverage(ug, read_g, coverage_cut, sources, ruIndex, lk); - - - if(lk) - { - uint32_t m; - for (v = 0; v < lk->a.n; v++) - { - for (k = m = 0; k < lk->a.a[v].f.n; k++) - { - if(lk->a.a[v].f.a[k].del) continue; - lk->a.a[v].f.a[m] = lk->a.a[v].f.a[k]; - m++; - } - lk->a.a[v].f.n = m; - } - } - + recover_utg_by_coverage(ug, read_g, coverage_cut, sources, ruIndex, cov->t_ch); + if(link) update_hc_links_by_trans_chain(link, cov->t_ch); destory_hap_cov_t(&cov); } void output_contig_graph_primary_pre(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, -ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, -long long tipsLen, R_to_U* ruIndex, int max_hang, int min_ovlp) +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, uint64_t bubble_dist, long long tipsLen, +R_to_U* ruIndex, int max_hang, int min_ovlp) { kvec_asg_arc_t_warp new_rtg_edges; kv_init(new_rtg_edges.a); @@ -21393,9 +22207,9 @@ long long tipsLen, R_to_U* ruIndex, int max_hang, int min_ovlp) nsg->seq[v].c = PRIMARY_LABLE; EvaluateLen(ug->u, v) = ug->u.a[v].n; } - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, NULL); + asg_pop_bubble_primary_trio(ug, &bubble_dist, (uint32_t)-1, DROP, NULL); cut_trio_tip_primary(ug->g, ug, tipsLen, (uint32_t)-1, 0, sg, reverse_sources, ruIndex, 2); - asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP, NULL); + asg_pop_bubble_primary_trio(ug, &bubble_dist, (uint32_t)-1, DROP, NULL); cut_trio_tip_primary(ug->g, ug, tipsLen, (uint32_t)-1, 0, sg, reverse_sources, ruIndex, 2); delete_useless_nodes(&ug); renew_utg(&ug, sg, &new_rtg_edges); @@ -21430,7 +22244,7 @@ long long tipsLen, R_to_U* ruIndex, int max_hang, int min_ovlp) void output_contig_graph_primary(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, -R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) +R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, bub_label_t* b_mask_t) { ma_ug_t *ug = NULL; ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); @@ -21441,7 +22255,7 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov adjust_utg_by_primary(&ug, sg, TRIO_THRES, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, - max_hang, min_ovlp, &new_rtg_edges, NULL); + max_hang, min_ovlp, &new_rtg_edges, NULL, b_mask_t); if(asm_opt.b_low_cov > 0) @@ -22599,53 +23413,17 @@ ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, -void lable_all_bubbles(asg_t *r_g, long long bubble_dist) +void lable_all_bubbles(asg_t *r_g, bub_label_t* b_mask_t) { ///must have this line, otherwise asg_arc_identify_simple_bubbles_multi will be wrong - asg_cleanup(r_g); - asg_arc_identify_simple_bubbles_multi(r_g, 0); - - uint32_t v, i, n_vtx = r_g->n_seq * 2; - buf_t b; - if (!r_g->is_symm) asg_symm(r_g); - memset(&b, 0, sizeof(buf_t)); - - b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); - for (v = 0; v < n_vtx; ++v) - { - uint32_t nv = asg_arc_n(r_g, v); - if(r_g->seq[v>>1].del) continue; - if(r_g->seq_vis[v] != 0) continue; - if(nv < 2) continue; - - - ///if this is a bubble - ///if(asg_bub_finder_with_del_advance(r_g, v, bubble_dist, &b) == 1) - if(asg_bub_pop1_primary_trio(r_g, NULL, v, bubble_dist, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL)) - { - //beg is v, end is b.S.a[0] - //note b.b include end, does not include beg - for (i = 0; i < b.b.n; i++) - { - if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; - - r_g->seq_vis[b.b.a[i]] = 1; - r_g->seq_vis[b.b.a[i]^1] = 1; - } - - r_g->seq_vis[v] = 1; - r_g->seq_vis[b.S.a[0]^1] = 1; - } - } - - free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + asg_arc_identify_simple_bubbles_multi(r_g, b_mask_t, 0); } -void drop_inexact_edegs_at_bubbles(asg_t *r_g, long long bubble_dist) +void drop_inexact_edegs_at_bubbles(asg_t *r_g, bub_label_t* b_mask_t, uint64_t bubble_dist) { - asg_arc_identify_simple_bubbles_multi(r_g, 0); + asg_arc_identify_simple_bubbles_multi(r_g, b_mask_t, 0); uint32_t v, k, i, n_vtx = r_g->n_seq * 2, nv, flag, n_reduce = 0; uint64_t oLen; @@ -23134,7 +23912,7 @@ uint8_t* expect_vis, uint8_t* circle_vis, uint8_t* utg_vis, uint32_t thresLen, u void rescue_contained_reads_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t chainLenThres, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, -kvec_t_u32_warp* new_rtg_nodes) +kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t) { uint32_t n_vtx, v, k, contain_rId, is_Unitig, uId, rId, endRid, oLen, w, max_oLen, max_oLen_i = (uint32_t)-1; uint32_t ava_max, ava_ol_max, ava_min_chain, ava_cur, ava_chainLen, test_oLen, is_update; @@ -23406,7 +24184,7 @@ kvec_t_u32_warp* new_rtg_nodes) } } - lable_all_bubbles(r_g, bubble_dist); + lable_all_bubbles(r_g, b_mask_t); for (k = 0; k < new_edges.n; k++) @@ -23501,7 +24279,7 @@ kvec_t_u32_warp* new_rtg_nodes) void rescue_missing_overlaps_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t is_bubble_check, uint32_t is_primary_check, -kvec_asg_arc_t_warp* new_rtg_edges) +kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) { uint32_t n_vtx, v, k, is_Unitig, uId, rId, endRid, oLen, w, max_oLen, max_oLen_i; asg_t* nsg = NULL; @@ -23677,7 +24455,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) } } - lable_all_bubbles(r_g, bubble_dist); + lable_all_bubbles(r_g, b_mask_t); for (k = 0; k < new_edges.n; k++) @@ -24031,7 +24809,8 @@ int if_recoverable(asg_t *sg, ma_ug_t *ug, bubble_type* bub, uint32_t bid, kvec_ } void rescue_bubbles_by_contained_reads(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t beg_idx, uint32_t occ, bubble_type* bub) +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t beg_idx, uint32_t occ, bubble_type* bub, +bub_label_t* b_mask_t) { asg_t* nsg = NULL; uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, contain_rId; @@ -24259,7 +25038,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t be } } - lable_all_bubbles(r_g, bubble_dist); + lable_all_bubbles(r_g, b_mask_t); for (k = 0; k < new_edges.n; k++) @@ -24317,7 +25096,8 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t be void rescue_bubbles_by_missing_ovlp(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t beg_idx, uint32_t occ, bubble_type* bub) +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t beg_idx, uint32_t occ, bubble_type* bub, +bub_label_t* b_mask_t) { asg_t* nsg = NULL; uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, v, w; @@ -24457,7 +25237,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t be } } - lable_all_bubbles(r_g, bubble_dist); + lable_all_bubbles(r_g, b_mask_t); for (k = 0; k < new_edges.n; k++) @@ -24492,7 +25272,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t be void update_unitig(long long step, long long init, ma_utg_t* nsu, asg_t *r_g, kvec_asg_arc_t_warp* recover_edges, uint32_t update_mode); void rescue_bubbles_by_missing_ovlp_backward(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t beg_idx, uint32_t occ, bubble_type* bub) +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t beg_idx, uint32_t occ, bubble_type* bub, bub_label_t* b_mask_t) { asg_t* nsg = NULL; uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, round, cur_backward_steps; @@ -24715,7 +25495,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t b } } - lable_all_bubbles(r_g, bubble_dist); + lable_all_bubbles(r_g, b_mask_t); for (k = 0; k < new_edges.n; k++) @@ -24805,7 +25585,7 @@ R_to_U* ruIndex, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges) { uint32_t v, k, uId, is_Unitig, occ_het, pre_het = 0, cur_het = 0; - uint64_t d = RC_0; + uint64_t d = RC_1; asg_t* nsg = NULL; ma_utg_t *nsu = NULL; ///ma_ug_t *ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); @@ -24818,8 +25598,6 @@ int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges) uint8_t* back_ug_flag = NULL; CALLOC(back_ug_flag, back_ug->g->n_seq); - ///fprintf(stderr, "ug->g->n_seq: %u, back_ug->g->n_seq: %u\n", ug->g->n_seq, back_ug->g->n_seq); - nsg = back_ug->g; for (v = 0; v < nsg->n_seq; v++) { @@ -24877,10 +25655,10 @@ int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges) int bub_complex(asg_t *sg, ma_ug_t *ug, bubble_type* bub, uint32_t bid, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, kvec_t_u32_warp* stack, -int max_hang, int min_ovlp, uint8_t* trio_flag, uint8_t* vis_flag, kv_asg_arc_t* e) +int max_hang, int min_ovlp, uint8_t* trio_flag, uint8_t* vis_flag, kv_asg_arc_t* e, buf_t *b, uint64_t tLen) { if(bid >= bub->f_bub) return 0; - uint32_t beg_utg, sink_utg, *a = NULL, n, begRid, sinkRid, tLen, i, k_i, k_j, k_v, rID/**, cur_flag, pre_flag, after_flag**/; + uint32_t beg_utg, sink_utg, *a = NULL, n, begRid, sinkRid, i, k_i, k_j, k_v, rID/**, cur_flag, pre_flag, after_flag**/; int is_switch_0, is_switch_1; ma_utg_t* nsu = NULL; get_bubbles(bub, bid, &beg_utg, &sink_utg, &a, &n, NULL); @@ -24904,17 +25682,15 @@ int max_hang, int min_ovlp, uint8_t* trio_flag, uint8_t* vis_flag, kv_asg_arc_t* } - buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(ug->g->n_seq * 2, sizeof(binfo_t)); - for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; + is_switch_0 = is_switch_1 = 1; - - asg_bub_pop1_primary_trio_switch_check(ug->g, ug, beg_utg, tLen, &b, FATHER, DROP, 0, NULL, NULL, &is_switch_0); + asg_bub_pop1_primary_trio_switch_check(ug->g, ug, beg_utg, tLen, b, FATHER, DROP, 0, NULL, NULL, &is_switch_0); if(is_switch_0 == 0) { - asg_bub_pop1_primary_trio_switch_check(ug->g, ug, beg_utg, tLen, &b, MOTHER, DROP, 0, NULL, NULL, &is_switch_1); + asg_bub_pop1_primary_trio_switch_check(ug->g, ug, beg_utg, tLen, b, MOTHER, DROP, 0, NULL, NULL, &is_switch_1); } - free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + for (k_i = 0; k_i < n; k_i++) { @@ -25123,11 +25899,14 @@ int max_hang, int min_ovlp, bubble_type* bub, long long gap_fuzz) kv_asg_arc_t e; kv_init(e); double index_time = yak_realtime(); - + buf_t b; memset(&b, 0, sizeof(buf_t)); + b.a = (binfo_t*)calloc(u_g->g->n_seq * 2, sizeof(binfo_t)); + uint64_t tLen = get_bub_pop_max_dist_advance(u_g->g, &b); for (i = 0; i < bub->f_bub; i++) { - fix_bub += bub_complex(r_g, u_g, bub, i, sources, coverage_cut, &stack, max_hang, min_ovlp, R_INF.trio_flag, vis_flag, &e); + fix_bub += bub_complex(r_g, u_g, bub, i, sources, coverage_cut, &stack, max_hang, min_ovlp, R_INF.trio_flag, vis_flag, &e, &b, tLen); } + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); asg_arc_t* p = NULL; for (i = 0; i < e.n; i++) @@ -25160,13 +25939,13 @@ int max_hang, int min_ovlp, bubble_type* bub, long long gap_fuzz) void rescue_bubble_by_chain(asg_t *sg, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, -float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chainLenThres, long long gap_fuzz) +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chainLenThres, long long gap_fuzz, +bub_label_t* b_mask_t) { kvec_asg_arc_t_warp new_rtg_edges; kv_init(new_rtg_edges.a); ma_ug_t *ug = NULL; ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); - hc_links copy_link, link; memset(©_link, 0, sizeof(hc_links)); memset(&link, 0, sizeof(hc_links)); @@ -25175,7 +25954,7 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chai ma_ug_t *copy_ug = copy_untig_graph(ug); adjust_utg_by_primary(©_ug, copy_sg, TRIO_THRES, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, - max_hang, min_ovlp, &new_rtg_edges, ©_link); + max_hang, min_ovlp, &new_rtg_edges, &link, b_mask_t); ma_ug_destroy(copy_ug); copy_ug = NULL; asg_destroy(copy_sg); copy_sg = NULL; @@ -25187,7 +25966,7 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chai reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, reverse_sources, max_hang, min_ovlp, &new_rtg_edges); beg_idx = bub.f_bub; occ = bub.b_bub + bub.b_end_bub + bub.tangle_bub; - rescue_bubbles_by_contained_reads(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub); + rescue_bubbles_by_contained_reads(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub, b_mask_t); ///output_unitig_graph(sg, coverage_cut, (char*)"debug_1.hic", sources, ruIndex, max_hang, min_ovlp); @@ -25196,7 +25975,7 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chai reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, reverse_sources, max_hang, min_ovlp, &new_rtg_edges); beg_idx = bub.f_bub; occ = bub.b_bub + bub.b_end_bub + bub.tangle_bub; - rescue_bubbles_by_missing_ovlp(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub); + rescue_bubbles_by_missing_ovlp(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub, b_mask_t); ///output_unitig_graph(sg, coverage_cut, (char*)"debug_2.hic", sources, ruIndex, max_hang, min_ovlp); @@ -25204,7 +25983,7 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chai reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, reverse_sources, max_hang, min_ovlp, &new_rtg_edges); beg_idx = bub.f_bub; occ = bub.b_bub + bub.b_end_bub + bub.tangle_bub; - rescue_bubbles_by_missing_ovlp_backward(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub); + rescue_bubbles_by_missing_ovlp_backward(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub, b_mask_t); ///output_unitig_graph(sg, coverage_cut, (char*)"debug_3.hic", sources, ruIndex, max_hang, min_ovlp); if(ha_opt_triobin(&asm_opt)) @@ -25284,7 +26063,7 @@ kvec_asg_arc_t_warp* recover_edges, uint32_t update_mode) } void rescue_missing_overlaps_backward(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t backward_steps, -uint32_t is_bubble_check, uint32_t is_primary_check) +uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t) { uint32_t v, vId, dir, k, cur_backward_steps, round, is_Unitig, uId, rId, endRid, oLen, w, max_oLen, max_oLen_i, mode, nv; uint64_t tmp; @@ -25550,7 +26329,7 @@ uint32_t is_bubble_check, uint32_t is_primary_check) } } - lable_all_bubbles(r_g, bubble_dist); + lable_all_bubbles(r_g, b_mask_t); for (k = 0; k < new_edges.n; k++) @@ -25664,7 +26443,7 @@ uint32_t is_bubble_check, uint32_t is_primary_check) void rescue_wrong_overlaps_to_unitigs(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, -kvec_asg_arc_t_warp* keep_edges) +kvec_asg_arc_t_warp* keep_edges, bub_label_t* b_mask_t) { uint32_t n_vtx, v, k, is_Unitig, uId, rId, endRid, oLen, w; asg_t* nsg = NULL; @@ -25811,7 +26590,7 @@ kvec_asg_arc_t_warp* keep_edges) nsg->seq_vis = (uint8_t*)calloc(nsg->n_seq*2, sizeof(uint8_t)); - lable_all_bubbles(nsg, bubble_dist); + lable_all_bubbles(nsg, b_mask_t); /*********************************for debug**************************************/ // uint32_t v_uId, w_uId; @@ -26545,7 +27324,7 @@ uint32_t strong, uint32_t el, uint32_t no_l_indel, asg_arc_t* t) ///chainLenThres is used to avoid circle void rescue_no_coverage_aggressive(asg_t *r_g, ma_hit_t_alloc* sources_count, ma_hit_t_alloc* reverse_source, ma_sub_t **coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, -long long bubble_dist, uint32_t chainLenThres) +long long bubble_dist, uint32_t chainLenThres, bub_label_t* b_mask_t) { uint32_t n_vtx, v, k, kv, is_Unitig, uId, rId, endRid, w; uint32_t ava_max, ava_ol_max, ava_min_chain, ava_cur, ava_chainLen, test_oLen, is_update; @@ -26783,7 +27562,7 @@ long long bubble_dist, uint32_t chainLenThres) } nsg->seq_vis = (uint8_t*)calloc(nsg->n_seq*2, sizeof(uint8_t)); - lable_all_bubbles(nsg, bubble_dist); + lable_all_bubbles(nsg, b_mask_t); for (k = 0; k < new_utg_edges.n; k++) @@ -27078,7 +27857,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) { ma_sub_t *coverage_cut = *coverage_cut_ptr; asg_t *sg = *sg_ptr; - + bub_label_t b_mask_t; if(debug_g) goto debug_gfa; ///just for debug @@ -27116,7 +27895,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) sg = ma_sg_gen(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); ///debug_info_of_specfic_node((char*)"m64043_200504_050026/93784180/ccs", sg, ruIndex, (char*)"sbsbsb"); - + init_bub_label_t(&b_mask_t, MIN(10, asm_opt.thread_num), sg->n_seq); asg_arc_del_trans(sg, gap_fuzz); @@ -27169,7 +27948,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) // asg_cut_tip(sg, asm_opt.max_short_tip); /****************************may have bugs********************************/ - asg_arc_identify_simple_bubbles_multi(sg, 1); + asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 1); //reomve edge between two chromesomes //this node must be a single read asg_arc_del_false_node(sg, sources, asm_opt.max_short_tip); @@ -27178,7 +27957,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) /****************************may have bugs********************************/ ///asg_arc_identify_simple_bubbles_multi(sg, 1); - asg_arc_identify_simple_bubbles_multi(sg, 0); + asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 0); ///asg_arc_del_short_diploid_unclean_exact(sg, drop_ratio, sources); if (ha_opt_triobin(&asm_opt)) { @@ -27191,7 +27970,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) asg_cut_tip(sg, asm_opt.max_short_tip); /****************************may have bugs********************************/ - asg_arc_identify_simple_bubbles_multi(sg, 1); + asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 1); if (ha_opt_triobin(&asm_opt)) { asg_arc_del_short_diploid_by_length_trio(sg, drop_ratio, asm_opt.max_short_tip, reverse_sources, @@ -27204,11 +27983,11 @@ ma_sub_t **coverage_cut_ptr, int debug_g) } asg_cut_tip(sg, asm_opt.max_short_tip); - asg_arc_identify_simple_bubbles_multi(sg, 1); + asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 1); asg_arc_del_short_false_link(sg, 0.6, 0.85, bubble_dist, reverse_sources, asm_opt.max_short_tip, ruIndex); - asg_arc_identify_simple_bubbles_multi(sg, 1); + asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 1); asg_arc_del_complex_false_link(sg, 0.6, 0.85, bubble_dist, reverse_sources, asm_opt.max_short_tip); asg_cut_tip(sg, asm_opt.max_short_tip); @@ -27229,7 +28008,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) asg_arc_del_triangular_directly(sg, asm_opt.max_short_tip, reverse_sources, ruIndex); - asg_arc_identify_simple_bubbles_multi(sg, 0); + asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 0); asg_arc_del_orthology_multiple_way(sg, reverse_sources, 0.4, asm_opt.max_short_tip, ruIndex); asg_cut_tip(sg, asm_opt.max_short_tip); @@ -27237,7 +28016,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) - asg_arc_identify_simple_bubbles_multi(sg, 0); + asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 0); asg_arc_del_too_short_overlaps(sg, 2000, min_ovlp_drop_ratio, reverse_sources, asm_opt.max_short_tip, ruIndex); asg_cut_tip(sg, asm_opt.max_short_tip); @@ -27246,18 +28025,18 @@ ma_sub_t **coverage_cut_ptr, int debug_g) ///note: don't apply asg_arc_del_too_short_overlaps() after this function!!!! rescue_contained_reads_aggressive(NULL, sg, sources, coverage_cut, ruIndex, max_hang_length, - mini_overlap_length, bubble_dist, 10, 1, 0, NULL, NULL); + mini_overlap_length, bubble_dist, 10, 1, 0, NULL, NULL, &b_mask_t); rescue_missing_overlaps_aggressive(NULL, sg, sources, coverage_cut, ruIndex, max_hang_length, - mini_overlap_length, bubble_dist, 1, 0, NULL); + mini_overlap_length, bubble_dist, 1, 0, NULL, &b_mask_t); rescue_missing_overlaps_backward(NULL, sg, sources, coverage_cut, ruIndex, max_hang_length, - mini_overlap_length, bubble_dist, 10, 1, 0); + mini_overlap_length, bubble_dist, 10, 1, 0, &b_mask_t); // rescue_wrong_overlaps_to_unitigs(NULL, sg, sources, reverse_sources, coverage_cut, ruIndex, // max_hang_length, mini_overlap_length, bubble_dist, NULL); // rescue_no_coverage_aggressive(sg, sources, reverse_sources, &coverage_cut, ruIndex, max_hang_length, // mini_overlap_length, bubble_dist, 10); rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz, &b_mask_t); if (asm_opt.flag & HA_F_VERBOSE_GFA) { @@ -27276,7 +28055,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.hic.bench", output_file_name); benchmark_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length); + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, &b_mask_t); free(buf); } else if (ha_opt_triobin(&asm_opt)) @@ -27291,10 +28070,10 @@ ma_sub_t **coverage_cut_ptr, int debug_g) output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang_length, mini_overlap_length, 0); + 0.05, 0.9, max_hang_length, mini_overlap_length, 0, &b_mask_t); output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang_length, mini_overlap_length, 0); + 0.05, 0.9, max_hang_length, mini_overlap_length, 0, &b_mask_t); } else if(ha_opt_hic(&asm_opt)) { @@ -27304,7 +28083,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.hic", output_file_name); output_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length); + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, &b_mask_t); free(buf); } else @@ -27324,14 +28103,14 @@ ma_sub_t **coverage_cut_ptr, int debug_g) output_contig_graph_primary(sg, coverage_cut, output_file_name, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, - mini_overlap_length); + mini_overlap_length, &b_mask_t); output_contig_graph_alternative(sg, coverage_cut, output_file_name, sources, ruIndex, max_hang_length, mini_overlap_length); } *coverage_cut_ptr = coverage_cut; *sg_ptr = sg; - + destory_bub_label_t(&b_mask_t); fprintf(stderr, "Inconsistency threshold for low-quality regions in BED files: %u%%\n", asm_opt.bed_inconsist_rate); } diff --git a/Overlaps.h b/Overlaps.h index 501da23..9c1cb88 100644 --- a/Overlaps.h +++ b/Overlaps.h @@ -551,55 +551,6 @@ inline uint32_t check_tip(asg_t *sg, uint32_t begNode, uint32_t* endNode, buf_t* } } -inline uint32_t get_unitig_back(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* endNode, -long long* nodeLen, long long* baseLen, buf_t* b) -{ - ma_utg_v* u = NULL; - uint32_t v = begNode, w, k; - uint32_t kv; - (*nodeLen) = (*baseLen) = 0; - (*endNode) = (uint32_t)-1; - if(ug!=NULL) u = &(ug->u); - - while (1) - { - kv = get_real_length(sg, v, NULL); - (*endNode) = v; - if(u == NULL) - { - (*nodeLen)++; - } - else - { - (*nodeLen) += EvaluateLen((*u), v>>1); - } - if(b) kv_push(uint32_t, b->b, v); - ///means reach the end of a unitig - if(kv!=1) (*baseLen) += sg->seq[v>>1].len; - if(kv==0) return END_TIPS; - if(kv>1) return MUL_OUTPUT; - ///kv must be 1 here - kv = get_real_length(sg, v, &w); - ///means reach the end of a unitig - if(get_real_length(sg, w^1, NULL)!=1) - { - (*baseLen) += sg->seq[v>>1].len; - return MUL_INPUT; - } - - for (k = 0; k < asg_arc_n(sg, v); k++) - { - if(asg_arc_a(sg, v)[k].del) continue; - ///here is just one undeleted edge - (*baseLen) += asg_arc_len(asg_arc_a(sg, v)[k]); - break; - } - - v = w; - if(v == begNode) return LOOP; - } -} - inline uint32_t get_unitig(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* endNode, long long* nodeLen, long long* baseLen, long long* max_stop_nodeLen, long long* max_stop_baseLen, uint32_t stops_threshold, buf_t* b) @@ -1030,25 +981,47 @@ typedef struct { uint32_t total; } Trio_counter; +typedef struct { + uint32_t p; // the optimal parent vertex + uint32_t d; // the shortest distance from the initial vertex + uint32_t r:31, s:1; // r: the number of remaining incoming arc; s: state +} binfo_s_t; + +typedef struct { + ///all information for each node + binfo_s_t *a; + kvec_t(uint32_t) S; // set of vertices without parents, nodes with all incoming edges visited + kvec_t(uint32_t) b; // visited vertices + kvec_t(uint32_t) e; // visited edges/arcs +} buf_s_t; + +typedef struct{ + buf_s_t *b; + uint32_t n_thres, n_reads; + asg_t *g; + uint32_t check_cross; + uint64_t bub_dist; +} bub_label_t; + void resolve_tangles(ma_ug_t *src, asg_t *read_g, ma_hit_t_alloc* reverse_sources, long long minLongUntig, long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, uint32_t trio_flag, float drop_ratio); void adjust_utg_advance(asg_t *sg, ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex); void rescue_contained_reads_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t chainLenThres, uint32_t is_bubble_check, -uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes); +uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t); void rescue_missing_overlaps_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t is_bubble_check, -uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges); +uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t); void all_to_all_deduplicate(ma_ug_t* ug, asg_t* read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, uint8_t postive_flag, float drop_rate, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, float double_check_rate); void drop_semi_circle(ma_ug_t *ug, asg_t* nsg, asg_t* read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex); void rescue_wrong_overlaps_to_unitigs(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, -ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, kvec_asg_arc_t_warp* keep_edges); +ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, kvec_asg_arc_t_warp* keep_edges, bub_label_t* b_mask_t); void get_unitig_trio_flag(ma_utg_t* nsu, uint32_t flag, uint32_t* require, uint32_t* non_require, uint32_t* ambigious); void rescue_missing_overlaps_backward(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t backward_steps, -uint32_t is_bubble_check, uint32_t is_primary_check); +uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t); uint32_t get_edge_from_source(ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t target, asg_arc_t* t); int unitig_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio); @@ -1085,12 +1058,15 @@ typedef struct{ uint64_t r_num; } hc_links; + typedef struct{ kvec_t(uint32_t) uIDs; - kvec_t(uint32_t) idx; - uint32_t chain_num; + kvec_t(uint32_t) iDXs; + kvec_t(uint32_t) rescue_hom; uint32_t* u_idx; - uint64_t r_num; + uint32_t r_num; + uint32_t chain_num; + uint32_t l0_chain, l1_chain; }trans_chain; typedef struct { @@ -1106,8 +1082,8 @@ typedef struct { kvec_asg_arc_t_offset u_buffer; kvec_t_i32_warp tailIndex; kvec_t_i32_warp prevIndex; - hc_links* link; - ///trans_chain t_ch; + ///hc_links* link; + trans_chain* t_ch; }hap_cov_t; typedef struct{ @@ -1118,28 +1094,27 @@ typedef struct{ void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num); void destory_hc_links(hc_links* link); -int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov); -uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, uint32_t positive_flag, +uint64_t get_bub_pop_max_dist(asg_t *g, buf_t *b); +uint64_t get_bub_pop_max_dist_advance(asg_t *g, buf_t *b); +int asg_pop_bubble_primary_trio(ma_ug_t *ug, uint64_t* i_max_dist, uint32_t positive_flag, uint32_t negative_flag, hap_cov_t *cov); +uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, uint64_t max_dist, buf_t *b, uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, hap_cov_t *cov); void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, -kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link); +kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t); void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg); ma_ug_t* copy_untig_graph(ma_ug_t *src); ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, -float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench); +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, bub_label_t* b_mask_t); asg_t* copy_read_graph(asg_t *src); ma_ug_t *ma_ug_gen(asg_t *g); void ma_ug_destroy(ma_ug_t *ug); - - - inline int inter_interval(int a_s, int a_e, int b_s, int b_e, int* i_s, int* i_e) { if(a_s > b_e || b_s > a_e) return 0; diff --git a/Purge_Dups.cpp b/Purge_Dups.cpp index 46c7cc7..e6fe0a6 100644 --- a/Purge_Dups.cpp +++ b/Purge_Dups.cpp @@ -2575,11 +2575,12 @@ long long* r_y_pos_beg, long long* r_y_pos_end) void print_hap_paf(ma_ug_t *ug, hap_overlaps* ovlp) { - fprintf(stderr, "utg%.6d%c\t%u(%u)\t%u(%u)\t%u(%u)\t%c\tutg%.6d%c\t%u(%u)\t%u(%u)\t%u(%u)\t%u\t%u\n", + fprintf(stderr, "utg%.6d%c\t%u(%u)\t%u(%u)\t%u(%u)\t%c\tutg%.6d%c\t%u(%u)\t%u(%u)\t%u(%u)\t%u\t%u\t%lld(%u)\n", ovlp->xUid+1, "lc"[ug->u.a[ovlp->xUid].circ], ug->u.a[ovlp->xUid].len, ug->u.a[ovlp->xUid].n, ovlp->x_beg_pos, ovlp->x_beg_id, ovlp->x_end_pos, ovlp->x_end_id, "+-"[ovlp->rev], ovlp->yUid+1, "lc"[ug->u.a[ovlp->yUid].circ], ug->u.a[ovlp->yUid].len, ug->u.a[ovlp->yUid].n, - ovlp->y_beg_pos, ovlp->y_beg_id, ovlp->y_end_pos, ovlp->y_end_id, ovlp->type, (uint32_t)ovlp->weight); + ovlp->y_beg_pos, ovlp->y_beg_id, ovlp->y_end_pos, ovlp->y_end_id, ovlp->type, ovlp->weight, + ovlp->score, ovlp->status); } inline long long get_max_index(asg_arc_t_offset* x, int32_t* Scores, uint8_t* Flag, long long n, @@ -3148,6 +3149,7 @@ void set_reverse_hap_overlap(hap_overlaps* dest, hap_overlaps* source, uint32_t* dest->x_end_id = source->y_end_id; dest->y_beg_id = source->x_beg_id; dest->y_end_id = source->x_end_id; + dest->score = source->score; } /** @@ -3409,13 +3411,13 @@ uint64_t asg_bub_pop1_purge_graph(asg_t *g, uint32_t v0, int max_dist, buf_t *b) ///assert(nv > 0); ///all out-edges of v for (i = 0; i < nv; ++i) { // loop through v's neighbors + ///if this edge has been deleted + if (av[i].del) continue; uint32_t w = av[i].v; // v->w with length l binfo_t *t = &b->a[w]; ///that means there is a circle, directly terminate the whole bubble poping ///if (w == v0) goto pop_reset; if ((w>>1) == (v0>>1)) goto pop_reset; - ///if this edge has been deleted - if (av[i].del) continue; c_s = decode_score((uint32_t)av[i].ul, av[i].ol); ///push the edge ///high 32-bit of g->idx[v] is the start point of v's edges @@ -3499,7 +3501,7 @@ pop_reset: // pop bubbles -int asg_pop_bubble_purge_graph(asg_t *purge_g, int max_dist) +int asg_pop_bubble_purge_graph(asg_t *purge_g) { uint32_t v, n_vtx = purge_g->n_seq * 2; uint64_t n_pop = 0; @@ -3641,13 +3643,13 @@ int purge_g_arc_del_short_diploid_by_score(asg_t *g, float drop_ratio) } -void clean_purge_graph(asg_t *purge_g, int max_dist, float drop_ratio) +void clean_purge_graph(asg_t *purge_g, float drop_ratio) { uint64_t operation = 1; while (operation > 0) { operation = 0; - operation += asg_pop_bubble_purge_graph(purge_g, max_dist); + operation += asg_pop_bubble_purge_graph(purge_g); operation += purge_g_arc_del_short_diploid_by_score(purge_g, drop_ratio); } @@ -4122,7 +4124,7 @@ hap_cov_t *cov) } purge_g->seq[w>>1].c = ALTER_LABLE; - collect_trans_purge_joint_cov(cov, ug, x); + if(cov) collect_trans_purge_joint_cov(cov, ug, x); // if(buffer.n > 1) // { @@ -4263,20 +4265,21 @@ hap_cov_t *cov) continue; } - if(cov->link) collect_reverse_unitigs_purge(&b_0, cov->link, ug, all_ovlp); + ///if(cov->link) collect_reverse_unitigs_purge(&b_0, cov->link, ug, all_ovlp); purge_merge(purge_g, ug, all_ovlp, &b_0, ruIndex, reverse_sources, coverage_cut, read_g, position_index, u_buffer, tailIndex, prevIndex,max_hang, min_ovlp, edge, visit, cov); } free(b_0.b.a); } -void print_all_purge_ovlp(ma_ug_t *ug, hap_overlaps_list* all_ovlp) +void print_all_purge_ovlp(ma_ug_t *ug, hap_overlaps_list* all_ovlp, const char* cmd) { + fprintf(stderr, "\n%s--->ug->u.n: %u\n", cmd, (uint32_t)ug->u.n); uint32_t v, uId, i; for (v = 0; v < all_ovlp->num; v++) { uId = v; - if(uId != 96 && uId != 272) continue; + ///if(uId != 96 && uId != 272) continue; for (i = 0; i < all_ovlp->x[uId].a.n; i++) { print_hap_paf(ug, &(all_ovlp->x[uId].a.a[i])); @@ -4571,7 +4574,7 @@ void remove_contained_haplotig(hap_overlaps_list* all_ovlp, ma_ug_t *ug, asg_t* purge_g->seq[xUid].del = 1; all_ovlp->x[uId].a.a[i].status = DELETE; - if(cov->link) collect_reverse_unitig_pair(cov->link, ug, &(all_ovlp->x[uId].a.a[i])); + ///if(cov->link) collect_reverse_unitig_pair(cov->link, ug, &(all_ovlp->x[uId].a.a[i])); collect_trans_purge_cov(cov, ug, &(all_ovlp->x[uId].a.a[i]), 0); } @@ -4612,8 +4615,8 @@ void remove_contained_haplotig(hap_overlaps_list* all_ovlp, ma_ug_t *ug, asg_t* void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, -uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage, hap_cov_t *cov) +uint32_t purege_minLen, int max_hang, int min_ovlp, float drop_ratio, uint32_t just_contain, +uint32_t just_coverage, hap_cov_t *cov) { asg_t *purge_g = NULL; purge_g = asg_init(); @@ -4697,17 +4700,15 @@ uint32_t just_contain, uint32_t just_coverage, hap_cov_t *cov) kt_for(asm_opt.thread_num, hap_alignment_advance_worker, &hap_buf, nsg->n_seq); - ///if(debug_enable) print_all_purge_ovlp(ug, &all_ovlp); filter_hap_overlaps_by_length(&all_ovlp, purege_minLen); ///normalize_hap_overlaps(&all_ovlp, &back_all_ovlp); normalize_hap_overlaps_advance(&all_ovlp, &back_all_ovlp, ug, read_g, reverse_sources, ruIndex); ///debug_hap_overlaps(&all_ovlp, &back_all_ovlp); - + remove_contained_haplotig(&all_ovlp, ug, nsg, purge_g, cov); - if(just_contain == 0) { for (v = 0; v < all_ovlp.num; v++) @@ -4747,7 +4748,7 @@ uint32_t just_contain, uint32_t just_coverage, hap_cov_t *cov) asg_cleanup(purge_g); asg_symm(purge_g); ///may need to do transitive reduction - clean_purge_graph(purge_g, bubble_dist, drop_ratio); + clean_purge_graph(purge_g, drop_ratio); // if(debug_enable) print_purge_gfa(ug, purge_g); // if(debug_enable) print_all_purge_ovlp(ug, &all_ovlp); diff --git a/Purge_Dups.h b/Purge_Dups.h index 4e3da12..f44a315 100644 --- a/Purge_Dups.h +++ b/Purge_Dups.h @@ -15,14 +15,14 @@ void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, -uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage, hap_cov_t *cov); +uint32_t purege_minLen, int max_hang, int min_ovlp, float drop_ratio, uint32_t just_contain, +uint32_t just_coverage, hap_cov_t *cov); void fill_unitig(uint64_t* buffer, uint32_t bufferLen, asg_t* read_g, kvec_asg_arc_t_warp* edge, uint32_t is_circle, uint64_t* rLen); void get_contig_length(ma_ug_t *ug, asg_t *g, uint64_t* primaryLen, uint64_t* alterLen); void enable_debug_mode(uint32_t mode); hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, -ma_sub_t *coverage_cut, int max_hang, int min_ovlp, hc_links* link); +ma_sub_t *coverage_cut, int max_hang, int min_ovlp, uint32_t is_collect_trans); void destory_hap_cov_t(hap_cov_t **x); void chain_trans_ovlp(hap_cov_t *cov, ma_ug_t *ug, asg_t *read_sg, buf_t* xReads, uint32_t targetBaseLen, uint32_t* xEnd); diff --git a/hic.cpp b/hic.cpp index 42f986b..9d12a06 100644 --- a/hic.cpp +++ b/hic.cpp @@ -2566,14 +2566,14 @@ void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link) if (!ug->g->is_symm) asg_symm(ug->g); uint32_t v, n_vtx = ug->g->n_seq * 2, i, k, mode = (((uint32_t)-1)<<2); uint32_t beg, sink, n, *a, n_occ; - uint64_t pathLen, tLen; + uint64_t pathLen; bub->ug = ug; - for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; bub->b_bub = bub->b_end_bub = bub->tangle_bub = bub->cross_bub = bub->mess_bub = 0; if(bub->round_id == 0) { buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + uint64_t tLen = get_bub_pop_max_dist_advance(ug->g, &b); kv_init(bub->list); kv_init(bub->num); kv_init(bub->pathLen); kv_init(bub->b_s_idx); kv_malloc(bub->b_s_idx, ug->g->n_seq); bub->b_ug = NULL; kv_init(bub->chain_weight); From 3b4953521fe2f3f38c48e9bf32b85e5b74febbe1 Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Tue, 16 Mar 2021 02:36:19 -0400 Subject: [PATCH 5/8] trio bug fixed --- Overlaps.cpp | 694 ++++++++++++++++++++++++++++++++++----------------- Overlaps.h | 12 +- hic.cpp | 166 +++++++----- 3 files changed, 582 insertions(+), 290 deletions(-) diff --git a/Overlaps.cpp b/Overlaps.cpp index 4e30e36..5be775b 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -3938,8 +3938,7 @@ int asg_arc_identify_simple_bubbles_multi(asg_t *g, bub_label_t* x, int check_cr double startTime = Get_T(); memset(g->seq_vis, 0, g->n_seq*2*sizeof(uint8_t)); uint64_t bub_dist = get_s_bub_pop_max_dist_advance(g, &(x->b[0])); - - // fprintf(stderr, "+++[M::%s] takes %0.2f s, bub_dist: %lu\n\n", __func__, Get_T()-startTime, bub_dist); + ///fprintf(stderr, "+++[M::%s] takes %0.2f s, bub_dist: %lu\n\n", __func__, Get_T()-startTime, bub_dist); // startTime = Get_T(); reset_bub_label_t(x, g, bub_dist, check_cross); @@ -11366,7 +11365,7 @@ void get_chain_trans(trans_chain* t_ch, uint32_t id, uint32_t** x, uint32_t* x_o if(x) (*x) = t_ch->uIDs.a + t_ch->iDXs.a[id<<1]; if(x_occ) (*x_occ) = t_ch->iDXs.a[(id<<1)+1] - t_ch->iDXs.a[id<<1]; if(y) (*y) = t_ch->uIDs.a + t_ch->iDXs.a[(id<<1)+1]; - if(y_occ) (*x_occ) = t_ch->iDXs.a[(id<<1)+2] - t_ch->iDXs.a[(id<<1)+1]; + if(y_occ) (*y_occ) = t_ch->iDXs.a[(id<<1)+2] - t_ch->iDXs.a[(id<<1)+1]; } inline uint32_t get_origin_uid(uint32_t v, trans_chain* t_ch) @@ -11427,7 +11426,6 @@ void collect_trans_cov(const char* cmd, buf_t* pri, buf_t* aux, ma_ug_t *ug, asg } if(t_ch) kv_push(uint32_t, t_ch->iDXs, t_ch->uIDs.n);///dedup_push_trans_chain(t_ch); - for (i = uLen = occ = 0, p_uId = (uint32_t)-1; i < pri->b.n; i++) { @@ -12039,7 +12037,7 @@ void hic_clean(asg_t* read_g) void output_hic_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, -ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, bub_label_t* b_mask_t) @@ -12066,7 +12064,7 @@ bub_label_t* b_mask_t) asm_opt.purge_overlap_len = asm_opt.purge_overlap_len_hic; asm_opt.purge_simi_rate = asm_opt.purge_simi_rate_hic; adjust_utg_by_primary(©_ug, copy_sg, TRIO_THRES, sources, reverse_sources, coverage_cut, - bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, + tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, min_ovlp, &new_rtg_edges, &link, b_mask_t); ma_ug_destroy(copy_ug); asg_destroy(copy_sg); @@ -12091,10 +12089,10 @@ bub_label_t* b_mask_t) output_unitig_graph(sg, coverage_cut, output_file_name, sources, ruIndex, max_hang, min_ovlp); output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, - reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang, min_ovlp, 0, b_mask_t); output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, - reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang, min_ovlp, 0, b_mask_t); } @@ -12156,16 +12154,16 @@ ma_ug_t* merge_utg(ma_ug_t **dest, ma_ug_t **src) } void benchmark_hic_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, -ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, bub_label_t* b_mask_t) { ma_ug_t *ug_1 = output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, - reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, + reverse_sources, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, min_ovlp, 1, b_mask_t); ma_ug_t *ug_2 = output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, - reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, + reverse_sources, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, min_ovlp, 1, b_mask_t); fprintf(stderr, "ug_1->u.n: %u, ug_2->u.n: %u\n", (uint32_t)ug_1->u.n, (uint32_t)ug_2->u.n); ma_ug_t *ug = merge_utg(&ug_1, &ug_2); @@ -13801,7 +13799,7 @@ asg_t* copy_read_graph(asg_t *src) } void clean_trio_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, -ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, @@ -13883,11 +13881,10 @@ void print_graph_statistic(asg_t *g, const char* cmd) } void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* sources, -ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, -long long tipsLen, float tip_drop_ratio, long long stops_threshold, -R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, -uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, -float drop_ratio, hap_cov_t *cov) +ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long tipsLen, float tip_drop_ratio, +long long stops_threshold, R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, +uint32_t miniHapLen, uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, +int just_bubble_pop, float drop_ratio, hap_cov_t *cov) { #define T_ROUND 2 asg_t *g = ug->g; @@ -13895,13 +13892,13 @@ float drop_ratio, hap_cov_t *cov) redo: ///print_graph_statistic(g, "beg"); + ///print_debug_gfa(read_g, ug, coverage_cut, "debug_chimeric", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); asg_pop_bubble_primary_trio(ug, NULL, (uint32_t)-1, DROP, cov); untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, cov); - + if(just_bubble_pop == 0) { - cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, - 2); + cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, 2); } long long pre_cons = get_graph_statistic(g); @@ -14585,7 +14582,7 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex) void adjust_utg_by_trio(ma_ug_t **ug, asg_t* read_g, uint8_t flag, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) { @@ -14630,8 +14627,8 @@ kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) nsg->seq[v].c = PRIMARY_LABLE; EvaluateLen((*ug)->u, v) = (*ug)->u.a[v].n; } - clean_trio_untig_graph(*ug, read_g, coverage_cut, sources, reverse_sources, bubble_dist, - tipsLen, tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, flag, drop_rate, cov); + clean_trio_untig_graph(*ug, read_g, coverage_cut, sources, reverse_sources, tipsLen, + tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, flag, drop_rate, cov); ///if(flag == MOTHER) fprintf(stderr, "(o.1) c: %u, del: %u, n: %u\n", (*ug)->g->seq[28141].c, (*ug)->g->seq[28141].del, (*ug)->u.a[28141].n); @@ -14648,12 +14645,12 @@ kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) if (!(asm_opt.flag & HA_F_BAN_POST_JOIN)) { rescue_missing_overlaps_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 0, 1, NULL, b_mask_t); + min_ovlp, 0, 1, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); rescue_contained_reads_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 10, 0, 1, NULL, NULL, b_mask_t); + min_ovlp, 10, 0, 1, NULL, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); } @@ -14705,7 +14702,7 @@ int debug_untig_length(ma_ug_t *g, uint32_t tipsLen, const char* name) ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, -uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, bub_label_t* b_mask_t) { @@ -14723,8 +14720,8 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, ///print_untig_by_read(ug, "m64011_190830_220126/117834372/ccs", 865264, sources, reverse_sources, "beg"); adjust_utg_by_trio(&ug, sg, flag, TRIO_THRES, sources, reverse_sources, coverage_cut, - bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, - max_hang, min_ovlp, &new_rtg_edges, b_mask_t); + tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, + min_ovlp, &new_rtg_edges, b_mask_t); if(asm_opt.b_low_cov > 0) { @@ -16156,7 +16153,6 @@ uint64_t get_s_bub_pop_max_dist_advance(asg_t *g, buf_s_t *b) kv++; } if(kv == 1 && get_real_length(g, w, NULL) == 1) continue; - cLen = dfs_subgraph_s_advance(g, b, v^1, &p_bub); if(p_bub == 0) continue;///no bubble if(cLen > mLen) mLen = cLen; @@ -22042,28 +22038,28 @@ void update_hc_links_by_trans_chain(hc_links* link, trans_chain* t_ch) } /*******************************for debug************************************/ - for (i = 0; i < t_ch->l0_chain; i++) - { - x_occ = y_occ = 0; - get_chain_trans(t_ch, i, &x, &x_occ, &y, &y_occ); - fprintf(stderr, "\n******x******"); - for (k = 0; k < x_occ; k++) - { - fprintf(stderr, "utg%.6ul\t%u\n", (x[k]>>1)+1, x[k]&1); - } - fprintf(stderr, "******y******"); - for (k = 0; k < y_occ; k++) - { - fprintf(stderr, "utg%.6ul\t%u\n", (y[k]>>1)+1, y[k]&1); - } - } + // for (i = 0; i < t_ch->l0_chain; i++) + // { + // x_occ = y_occ = 0; + // get_chain_trans(t_ch, i, &x, &x_occ, &y, &y_occ); + // fprintf(stderr, "\n******x******"); + // for (k = 0; k < x_occ; k++) + // { + // fprintf(stderr, "utg%.6ul\t%u\n", (x[k]>>1)+1, x[k]&1); + // } + // fprintf(stderr, "******y******"); + // for (k = 0; k < y_occ; k++) + // { + // fprintf(stderr, "utg%.6ul\t%u\n", (y[k]>>1)+1, y[k]&1); + // } + // } /*******************************for debug************************************/ } void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) { @@ -22087,7 +22083,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) nsg->seq[v].c = PRIMARY_LABLE; EvaluateLen((*ug)->u, v) = (*ug)->u.a[v].n; } - clean_primary_untig_graph(*ug, read_g, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, + clean_primary_untig_graph(*ug, read_g, sources, reverse_sources, coverage_cut, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, chimeric_rate, 0, 0, drop_ratio, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); @@ -22107,10 +22103,10 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) if (!(asm_opt.flag & HA_F_BAN_POST_JOIN)) { rescue_missing_overlaps_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 0, 1, NULL, b_mask_t); + min_ovlp, 0, 1, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); rescue_contained_reads_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, - min_ovlp, 0, 10, 0, 1, NULL, NULL, b_mask_t); + min_ovlp, 10, 0, 1, NULL, NULL, b_mask_t); renew_utg(ug, read_g, new_rtg_edges); if(asm_opt.purge_level_primary > 0) @@ -22242,8 +22238,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp) } void output_contig_graph_primary(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, -ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, -long long tipsLen, float tip_drop_ratio, long long stops_threshold, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, bub_label_t* b_mask_t) { ma_ug_t *ug = NULL; @@ -22254,7 +22249,7 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov adjust_utg_by_primary(&ug, sg, TRIO_THRES, sources, reverse_sources, coverage_cut, - bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, + tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, min_ovlp, &new_rtg_edges, NULL, b_mask_t); @@ -22662,24 +22657,29 @@ long long get_coverage(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, uint64_t } -void pre_clean(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *sg, long long bubble_dist) +void pre_clean(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *sg, uint32_t pop_s_node) { + int tri_flag = 0; while(1) { - int tri_flag = 0; + tri_flag = 0; ///remove very simple circle tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, sg, 100, 0); - ///remove isoloated single read - tri_flag += asg_arc_del_single_node_directly(sg, asm_opt.max_short_tip, sources); - if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) + ///remove isoloated single read + if(pop_s_node) { - tri_flag += asg_arc_del_triangular_advance(sg, bubble_dist); - ///remove the cross at the bubble carefully, just remove inexact cross - tri_flag += asg_arc_del_cross_bubble(sg, bubble_dist); - } + tri_flag += asg_arc_del_single_node_directly(sg, asm_opt.max_short_tip, sources); + } - tri_flag += asg_arc_del_single_node_directly(sg, asm_opt.max_short_tip, sources); + // if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) + // { + // tri_flag += asg_arc_del_triangular_advance(sg, bubble_dist); + // ///remove the cross at the bubble carefully, just remove inexact cross + // tri_flag += asg_arc_del_cross_bubble(sg, bubble_dist); + // } + // tri_flag += asg_arc_del_single_node_directly(sg, asm_opt.max_short_tip, sources); + if(tri_flag == 0) { break; @@ -23910,8 +23910,7 @@ uint8_t* expect_vis, uint8_t* circle_vis, uint8_t* utg_vis, uint32_t thresLen, u ///chainLenThres is used to avoid circle void rescue_contained_reads_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t chainLenThres, -uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t) { uint32_t n_vtx, v, k, contain_rId, is_Unitig, uId, rId, endRid, oLen, w, max_oLen, max_oLen_i = (uint32_t)-1; @@ -24166,27 +24165,10 @@ kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t) asg_cleanup(r_g); } - while(1) - { - int tri_flag = 0; - ///remove very simple circle - tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) - { - ///remove isoloated single read - tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); - ///remove the cross at the bubble carefully, just remove inexact cross - tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); - } - if(tri_flag == 0) - { - break; - } - } + pre_clean(sources, coverage_cut, r_g, 0); lable_all_bubbles(r_g, b_mask_t); - for (k = 0; k < new_edges.n; k++) { v = new_edges.a[k].ul>>32; @@ -24278,7 +24260,7 @@ kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t) void rescue_missing_overlaps_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t is_bubble_check, uint32_t is_primary_check, +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) { uint32_t n_vtx, v, k, is_Unitig, uId, rId, endRid, oLen, w, max_oLen, max_oLen_i; @@ -24437,23 +24419,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) } - while(1) - { - int tri_flag = 0; - ///remove very simple circle - tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) - { - ///remove isoloated single read - tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); - ///remove the cross at the bubble carefully, just remove inexact cross - tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); - } - if(tri_flag == 0) - { - break; - } - } + pre_clean(sources, coverage_cut, r_g, 0); lable_all_bubbles(r_g, b_mask_t); @@ -24813,21 +24779,19 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t be bub_label_t* b_mask_t) { asg_t* nsg = NULL; - uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, contain_rId; + uint32_t beg_utg, sink_utg, *a = NULL, n, i, bub_i, k_i, k_v, k, uId, endRid, is_Unitig, contain_rId; uint32_t ava_max, ava_ol_max, ava_min_chain, ava_cur, test_oLen, ava_chainLen, is_update, v, w; - uint64_t l_bub, m_bub, r_bub, bub_0, bub_1, a_nodes; + uint64_t l_bub, m_bub, r_bub, /**bub_0, bub_1,**/ a_nodes; ma_ug_t* ug = i_u_g; - ma_utg_t* nsu = NULL; + ma_utg_t *nsu = NULL, *u = NULL; ma_hit_t_alloc* x = NULL; ma_hit_t *h = NULL, *h_max = NULL; uint8_t* expect_vis = NULL; CALLOC(expect_vis, r_g->n_seq); uint8_t* circle_vis = NULL; CALLOC(circle_vis, r_g->n_seq); uint8_t* utg_vis = NULL; CALLOC(utg_vis, ug->g->n_seq); - long long bubble_dist; asg_arc_t t, t_max, r_edge; for (v = 0; v < ug->u.n; v++) ug->g->seq[v].c = PRIMARY_LABLE; - for (k = 0, bubble_dist = 1; k < ug->u.n; k++) bubble_dist += ug->u.a[k].len; kvec_t(asg_arc_t) new_edges; kv_init(new_edges); @@ -24855,10 +24819,156 @@ bub_label_t* b_mask_t) } + for (i = 0; i < bub->b_ug->u.n; i++) + { + u = &(bub->b_ug->u.a[i]); + if(u->n < 3) continue; ///should be at least 3 + for (bub_i = 1; bub_i+1 < u->n; bub_i++) + { + m_bub = u->a[bub_i]>>33; l_bub = r_bub = (uint64_t)-1; + if(m_bub < beg_idx || m_bub >= beg_idx + occ) continue; + l_bub = u->a[bub_i-1]>>33; + r_bub = u->a[bub_i+1]>>33; + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 1); + + get_bubbles(bub, m_bub, &beg_utg, &sink_utg, &a, &n, NULL); + for (k_i = 0; k_i < n; k_i++) + { + uId = a[k_i]>>1; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + for (k_v = 0; k_v < 2; k_v++) + { + v = (uId<<1) + k_v; + if(get_real_length(nsg, v, NULL) != 0) continue; + + if(v&1) + { + endRid = nsu->start^1; + } + else + { + endRid = nsu->end^1; + } + + //x is the end read of a tip + ///find all overlap of x + x = &(sources[(endRid>>1)]); + ava_ol_max = ava_max = 0; ava_min_chain = (uint32_t)-1; + h_max = NULL; + for (k = 0; k < x->length; k++) + { + ///fprintf(stderr, "k: %u\n", k); + ///h is the edge of endRid + h = &(x->buffer[k]); + ///means we found a contained read + if(get_contained_reads_chain_by_broken_bub(h, sources, coverage_cut, ruIndex, ug, r_g, + max_hang, min_ovlp, endRid, uId, &chain_edges, &ava_cur, &test_oLen, &ava_chainLen, + expect_vis, circle_vis, utg_vis, chainLenThres, 1)) + { + is_update = 0; + + if(ava_cur > ava_max) + { + is_update = 1; + } + else if(ava_cur == ava_max) + { + if(ava_chainLen < ava_min_chain) + { + is_update = 1; + } + else if(ava_chainLen == ava_min_chain && test_oLen > ava_ol_max) + { + is_update = 1; + } + } + + if(is_update) + { + ava_min_chain = ava_chainLen; + ava_max = ava_cur; + ava_ol_max = test_oLen; + h_max = h; + } + } + } + + if(ava_max > 0) + { + ///fprintf(stderr, "ava_max: %u\n", ava_max); + get_contained_reads_chain_by_broken_bub(h_max, sources, coverage_cut, ruIndex, ug, r_g, + max_hang, min_ovlp, endRid, uId, &chain_edges, &ava_cur, &test_oLen, &ava_chainLen, + expect_vis, circle_vis, NULL, chainLenThres, 1); + if(chain_edges.a.n < 1) continue; + ///the last cantained read + t_max = chain_edges.a.a[chain_edges.a.n-1]; + + k = 0; rbub_edges.n = 0; + ///edges from the last contained read to other unitigs + + while(get_edge2existing_node_advance_by_broken_bub(ug, r_g, sources, coverage_cut, + ruIndex, expect_vis, max_hang, min_ovlp, t_max.v, &k, &r_edge, 1)) + { + kv_push(asg_arc_t, rbub_edges, r_edge); + } + + ///need to do transitive reduction + ///note here is different to standard transitive reduction + minor_transitive_reduction_r_g(r_g, rbub_edges.a, rbub_edges.n); + + for (k = 0; k < chain_edges.a.n; k++) + { + t_max = chain_edges.a.a[k]; + ///save all infor for reverting + get_R_to_U(ruIndex, t_max.v>>1, &contain_rId, &is_Unitig); + a_nodes=contain_rId; + a_nodes=a_nodes<<32; + a_nodes=a_nodes|((uint64_t)(t_max.v>>1)); + kv_push(uint64_t, u_vecs.a, a_nodes); + + r_g->seq[t_max.v>>1].del = 0; + coverage_cut[t_max.v>>1].del = 0; + coverage_cut[t_max.v>>1].c = PRIMARY_LABLE; + + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t_max.ul>>32), t_max.v, &t); + kv_push(asg_arc_t, new_edges, t); + + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t_max.v^1), ((t_max.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + + + for (k = 0; k < rbub_edges.n; k++) + { + t = rbub_edges.a[k]; + if(t.del) continue; + + kv_push(asg_arc_t, new_edges, t); + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + + } + } + } + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); + } + } + /** for (i = beg_idx; i < beg_idx + occ; i++) { get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); - ///fprintf(stderr, "\n%u-th: bub-utg%.6ul, bub-utg%.6ul\n", i - beg_idx, (beg_utg>>1)+1, (sink_utg>>1)+1); if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) continue; l_bub = r_bub = (uint64_t)-1; m_bub = i; @@ -25004,6 +25114,7 @@ bub_label_t* b_mask_t) set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); } + **/ asg_arc_t* p = NULL; for (k = 0; k < new_edges.n; k++) @@ -25020,27 +25131,10 @@ bub_label_t* b_mask_t) asg_symm(r_g); } - while(1) - { - int tri_flag = 0; - ///remove very simple circle - tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) - { - ///remove isoloated single read - tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); - ///remove the cross at the bubble carefully, just remove inexact cross - tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); - } - if(tri_flag == 0) - { - break; - } - } + pre_clean(sources, coverage_cut, r_g, 0); lable_all_bubbles(r_g, b_mask_t); - for (k = 0; k < new_edges.n; k++) { v = new_edges.a[k].ul>>32; @@ -25074,7 +25168,6 @@ bub_label_t* b_mask_t) coverage_cut[w].del = 1; } } - for (v = 0; v < ruIndex->len; v++) { @@ -25100,16 +25193,14 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t be bub_label_t* b_mask_t) { asg_t* nsg = NULL; - uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, v, w; - uint64_t l_bub, m_bub, r_bub, bub_0, bub_1; + uint32_t beg_utg, sink_utg, *a = NULL, n, i, bub_i, k_i, k_v, k, uId, endRid, is_Unitig, v, w; + uint64_t l_bub, m_bub, r_bub/**, bub_0, bub_1**/; ma_ug_t* ug = i_u_g; - ma_utg_t* nsu = NULL; + ma_utg_t *nsu = NULL, *u = NULL; uint8_t* expect_vis = NULL; CALLOC(expect_vis, r_g->n_seq); - long long bubble_dist; asg_arc_t t, r_edge; for (v = 0; v < ug->u.n; v++) ug->g->seq[v].c = PRIMARY_LABLE; - for (k = 0, bubble_dist = 1; k < ug->u.n; k++) bubble_dist += ug->u.a[k].len; kvec_t(asg_arc_t) new_edges; kv_init(new_edges); @@ -25131,7 +25222,77 @@ bub_label_t* b_mask_t) } } + for (i = 0; i < bub->b_ug->u.n; i++) + { + u = &(bub->b_ug->u.a[i]); + if(u->n < 3) continue; ///should be at least 3 + for (bub_i = 1; bub_i+1 < u->n; bub_i++) + { + m_bub = u->a[bub_i]>>33; l_bub = r_bub = (uint64_t)-1; + if(m_bub < beg_idx || m_bub >= beg_idx + occ) continue; + l_bub = u->a[bub_i-1]>>33; + r_bub = u->a[bub_i+1]>>33; + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 1); + + get_bubbles(bub, m_bub, &beg_utg, &sink_utg, &a, &n, NULL); + + for (k_i = 0; k_i < n; k_i++) + { + uId = a[k_i]>>1; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + for (k_v = 0; k_v < 2; k_v++) + { + v = (uId<<1) + k_v; + if(get_real_length(nsg, v, NULL) != 0) continue; + + if(v&1) + { + endRid = nsu->start^1; + } + else + { + endRid = nsu->end^1; + } + + + k = 0; rbub_edges.n = 0; + while(get_edge2existing_node_advance_by_broken_bub(ug, r_g, sources, coverage_cut, + ruIndex, expect_vis, max_hang, min_ovlp, endRid, &k, &r_edge, 1)) + { + kv_push(asg_arc_t, rbub_edges, r_edge); + } + + if(rbub_edges.n > 0) + { + ///need to do transitive reduction + ///note here is different to standard transitive reduction + minor_transitive_reduction_r_g(r_g, rbub_edges.a, rbub_edges.n); + for (k = 0; k < rbub_edges.n; k++) + { + t = rbub_edges.a[k]; + if(t.del) continue; + + kv_push(asg_arc_t, new_edges, t); + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + } + } + } + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); + } + } + + /** for (i = beg_idx; i < beg_idx + occ; i++) { get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); @@ -25203,6 +25364,7 @@ bub_label_t* b_mask_t) set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); } + **/ asg_arc_t* p = NULL; for (k = 0; k < new_edges.n; k++) @@ -25219,23 +25381,7 @@ bub_label_t* b_mask_t) asg_symm(r_g); } - while(1) - { - int tri_flag = 0; - ///remove very simple circle - tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) - { - ///remove isoloated single read - tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); - ///remove the cross at the bubble carefully, just remove inexact cross - tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); - } - if(tri_flag == 0) - { - break; - } - } + pre_clean(sources, coverage_cut, r_g, 0); lable_all_bubbles(r_g, b_mask_t); @@ -25275,18 +25421,16 @@ void rescue_bubbles_by_missing_ovlp_backward(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t beg_idx, uint32_t occ, bubble_type* bub, bub_label_t* b_mask_t) { asg_t* nsg = NULL; - uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, round, cur_backward_steps; + uint32_t beg_utg, sink_utg, *a = NULL, n, i, bub_i, k_i, k_v, k, uId, endRid, is_Unitig, round, cur_backward_steps; uint32_t v, w, mode, nv; - uint64_t l_bub, m_bub, r_bub, bub_0, bub_1, tmp; + uint64_t l_bub, m_bub, r_bub, /**bub_0, bub_1,**/ tmp; ma_ug_t* ug = i_u_g; - ma_utg_t* nsu = NULL; + ma_utg_t *nsu = NULL, *u = NULL; uint8_t* expect_vis = NULL; CALLOC(expect_vis, r_g->n_seq); - long long bubble_dist; long long init, step = 0; asg_arc_t t, r_edge, *av = NULL; for (v = 0; v < ug->u.n; v++) ug->g->seq[v].c = PRIMARY_LABLE; - for (k = 0, bubble_dist = 1; k < ug->u.n; k++) bubble_dist += ug->u.a[k].len; kvec_t(asg_arc_t) new_edges; kv_init(new_edges); @@ -25315,10 +25459,154 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t b } + for (i = 0; i < bub->b_ug->u.n; i++) + { + u = &(bub->b_ug->u.a[i]); + if(u->n < 3) continue; ///should be at least 3 + for (bub_i = 1; bub_i+1 < u->n; bub_i++) + { + m_bub = u->a[bub_i]>>33; l_bub = r_bub = (uint64_t)-1; + if(m_bub < beg_idx || m_bub >= beg_idx + occ) continue; + l_bub = u->a[bub_i-1]>>33; + r_bub = u->a[bub_i+1]>>33; + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 1); + get_bubbles(bub, m_bub, &beg_utg, &sink_utg, &a, &n, NULL); + for (k_i = 0; k_i < n; k_i++) + { + uId = a[k_i]>>1; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + + rbub_edges.n = round = 0; + for (k_v = 0; k_v < 2; k_v++) + { + if(rbub_edges.n > 0) + { + cur_backward_steps = nsu->n - round - 1; + if(cur_backward_steps > backward_steps) + { + cur_backward_steps = backward_steps; + } + } + else + { + cur_backward_steps = backward_steps; + } + + + v = (uId<<1) + k_v; + if(get_real_length(nsg, v, NULL) != 0) continue; + ///fprintf(stderr, "++++++tig-utg%.6ul\n", uId+1); + ///that means this unitig has been changed + // if(nsu->start!=((uint64_t)(nsu->a[0])>>32)) continue; + // if((nsu->end^1)!=((uint64_t)(nsu->a[nsu->n-1])>>32)) continue; + + if(v&1) + { + init = 0; + step = 1; + mode = 1; + } + else + { + init = nsu->n - 1; + step = -1; + mode = 0; + } + + rbub_edges.n = 0; + for (round = 0; round < cur_backward_steps && init >= 0 && init < (long long)nsu->n; + init = init + step, round++) + { + endRid = ((uint64_t)(nsu->a[init]))>>32; + endRid = endRid^mode; + + k = 0; rbub_edges.n = 0; + while(get_edge2existing_node_advance_by_broken_bub(ug, r_g, sources, coverage_cut, + ruIndex, expect_vis, max_hang, min_ovlp, endRid, &k, &r_edge, 1)) + { + kv_push(asg_arc_t, rbub_edges, r_edge); + } + + if(rbub_edges.n > 0) break; + } + + if(rbub_edges.n > 0) + { + //save for revert + tmp = mode; tmp = tmp <<31; tmp = tmp | (uint64_t)(init); tmp = tmp << 32; tmp = tmp | uId; + kv_push(uint64_t, u_vecs.a, tmp); + ///need to do transitive reduction + ///note here is different to standard transitive reduction + minor_transitive_reduction_r_g(r_g, rbub_edges.a, rbub_edges.n); + + + ///modify read graph + for (init = init - step; init >= 0 && init < (long long)nsu->n; init = init - step) + { + w = ((uint64_t)(nsu->a[init]))>>32; + nv = asg_arc_n(r_g, w); + av = asg_arc_a(r_g, w); + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + kv_push(asg_arc_t, recover_edges.a, av[k]); + if(asg_get_arc(r_g, av[k].v^1, av[k].ul>>32^1, &t)==0) + { + fprintf(stderr, "error\n"); + } + kv_push(asg_arc_t, recover_edges.a, t); + } + + + nv = asg_arc_n(r_g, w^1); + av = asg_arc_a(r_g, w^1); + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + kv_push(asg_arc_t, recover_edges.a, av[k]); + if(asg_get_arc(r_g, av[k].v^1, av[k].ul>>32^1, &t)==0) + { + fprintf(stderr, "error\n"); + } + kv_push(asg_arc_t, recover_edges.a, t); + } + + ///w = ((uint64_t)(nsu->a[init]))>>32; + asg_seq_del(r_g, w>>1); + expect_vis[w>>1] = 0; + } + + + for (k = 0; k < rbub_edges.n; k++) + { + t = rbub_edges.a[k]; + if(t.del) continue; + + kv_push(asg_arc_t, new_edges, t); + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + } + + } + } + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); + } + } + + /** for (i = beg_idx; i < beg_idx + occ; i++) { get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); - ///fprintf(stderr, "\n%u-th: bub-utg%.6ul, bub-utg%.6ul\n", i - beg_idx, (beg_utg>>1)+1, (sink_utg>>1)+1); if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) continue; l_bub = r_bub = (uint64_t)-1; m_bub = i; @@ -25461,6 +25749,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t b set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); } + **/ asg_arc_t* p = NULL; for (k = 0; k < new_edges.n; k++) @@ -25477,27 +25766,10 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t b asg_symm(r_g); } - while(1) - { - int tri_flag = 0; - ///remove very simple circle - tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) - { - ///remove isoloated single read - tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); - ///remove the cross at the bubble carefully, just remove inexact cross - tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); - } - if(tri_flag == 0) - { - break; - } - } + pre_clean(sources, coverage_cut, r_g, 0); lable_all_bubbles(r_g, b_mask_t); - for (k = 0; k < new_edges.n; k++) { v = new_edges.a[k].ul>>32; @@ -25938,7 +26210,7 @@ int max_hang, int min_ovlp, bubble_type* bub, long long gap_fuzz) void rescue_bubble_by_chain(asg_t *sg, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chainLenThres, long long gap_fuzz, bub_label_t* b_mask_t) { @@ -25953,8 +26225,8 @@ bub_label_t* b_mask_t) asg_t *copy_sg = copy_read_graph(sg); ma_ug_t *copy_ug = copy_untig_graph(ug); adjust_utg_by_primary(©_ug, copy_sg, TRIO_THRES, sources, reverse_sources, coverage_cut, - bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, - max_hang, min_ovlp, &new_rtg_edges, &link, b_mask_t); + tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, + max_hang, min_ovlp, &new_rtg_edges, ©_link, b_mask_t); ma_ug_destroy(copy_ug); copy_ug = NULL; asg_destroy(copy_sg); copy_sg = NULL; @@ -25962,12 +26234,11 @@ bub_label_t* b_mask_t) bubble_type bub; memset(&bub, 0, sizeof(bubble_type)); copy_ug = copy_untig_graph(ug); - reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, reverse_sources, max_hang, min_ovlp, &new_rtg_edges); beg_idx = bub.f_bub; occ = bub.b_bub + bub.b_end_bub + bub.tangle_bub; rescue_bubbles_by_contained_reads(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub, b_mask_t); - ///output_unitig_graph(sg, coverage_cut, (char*)"debug_1.hic", sources, ruIndex, max_hang, min_ovlp); + ///output_unitig_graph(sg, coverage_cut, (char*)"debug_1.rescue", sources, ruIndex, max_hang, min_ovlp); @@ -26062,8 +26333,7 @@ kvec_asg_arc_t_warp* recover_edges, uint32_t update_mode) } } void rescue_missing_overlaps_backward(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t backward_steps, -uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t) +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t) { uint32_t v, vId, dir, k, cur_backward_steps, round, is_Unitig, uId, rId, endRid, oLen, w, max_oLen, max_oLen_i, mode, nv; uint64_t tmp; @@ -26311,23 +26581,7 @@ uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t) asg_cleanup(r_g); } - while(1) - { - int tri_flag = 0; - ///remove very simple circle - tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) - { - ///remove isoloated single read - tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); - ///remove the cross at the bubble carefully, just remove inexact cross - tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); - } - if(tri_flag == 0) - { - break; - } - } + pre_clean(sources, coverage_cut, r_g, 0); lable_all_bubbles(r_g, b_mask_t); @@ -27858,7 +28112,12 @@ ma_sub_t **coverage_cut_ptr, int debug_g) ma_sub_t *coverage_cut = *coverage_cut_ptr; asg_t *sg = *sg_ptr; bub_label_t b_mask_t; - if(debug_g) goto debug_gfa; + if(debug_g) + { + init_bub_label_t(&b_mask_t, MIN(10, asm_opt.thread_num), sg->n_seq); + goto debug_gfa; + } + ///just for debug renew_graph_init(sources, reverse_sources, sg, coverage_cut, ruIndex, n_read); @@ -27942,7 +28201,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) } ///just topological clean - pre_clean(sources, coverage_cut, sg, bubble_dist); + pre_clean(sources, coverage_cut, sg, 1); ///asg_arc_del_orthology(sg, reverse_sources, drop_ratio, asm_opt.max_short_tip); // asg_arc_del_orthology_multiple_way(sg, reverse_sources, drop_ratio, asm_opt.max_short_tip); // asg_cut_tip(sg, asm_opt.max_short_tip); @@ -27984,8 +28243,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) asg_cut_tip(sg, asm_opt.max_short_tip); asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 1); - asg_arc_del_short_false_link(sg, 0.6, 0.85, bubble_dist, reverse_sources, - asm_opt.max_short_tip, ruIndex); + asg_arc_del_short_false_link(sg, 0.6, 0.85, bubble_dist, reverse_sources, asm_opt.max_short_tip, ruIndex); asg_arc_identify_simple_bubbles_multi(sg, &b_mask_t, 1); asg_arc_del_complex_false_link(sg, 0.6, 0.85, bubble_dist, reverse_sources, asm_opt.max_short_tip); @@ -28000,7 +28258,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) } - pre_clean(sources, coverage_cut, sg, bubble_dist); + pre_clean(sources, coverage_cut, sg, 1); asg_arc_del_short_diploi_by_suspect_edge(sg, asm_opt.max_short_tip); @@ -28025,18 +28283,16 @@ ma_sub_t **coverage_cut_ptr, int debug_g) ///note: don't apply asg_arc_del_too_short_overlaps() after this function!!!! rescue_contained_reads_aggressive(NULL, sg, sources, coverage_cut, ruIndex, max_hang_length, - mini_overlap_length, bubble_dist, 10, 1, 0, NULL, NULL, &b_mask_t); + mini_overlap_length, 10, 1, 0, NULL, NULL, &b_mask_t); rescue_missing_overlaps_aggressive(NULL, sg, sources, coverage_cut, ruIndex, max_hang_length, - mini_overlap_length, bubble_dist, 1, 0, NULL, &b_mask_t); + mini_overlap_length, 1, 0, NULL, &b_mask_t); rescue_missing_overlaps_backward(NULL, sg, sources, coverage_cut, ruIndex, max_hang_length, - mini_overlap_length, bubble_dist, 10, 1, 0, &b_mask_t); + mini_overlap_length, 10, 1, 0, &b_mask_t); // rescue_wrong_overlaps_to_unitigs(NULL, sg, sources, reverse_sources, coverage_cut, ruIndex, // max_hang_length, mini_overlap_length, bubble_dist, NULL); // rescue_no_coverage_aggressive(sg, sources, reverse_sources, &coverage_cut, ruIndex, max_hang_length, // mini_overlap_length, bubble_dist, 10); - rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz, &b_mask_t); if (asm_opt.flag & HA_F_VERBOSE_GFA) { @@ -28046,44 +28302,38 @@ ma_sub_t **coverage_cut_ptr, int debug_g) /*******************************for debug***************************************/ } + rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, + ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz, &b_mask_t); + if (ha_opt_triobin(&asm_opt) && ha_opt_hic(&asm_opt)) { - // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); - char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.hic.bench", output_file_name); - benchmark_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, &b_mask_t); + benchmark_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, + ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, &b_mask_t); free(buf); } else if (ha_opt_triobin(&asm_opt)) { - // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); - char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.dip", output_file_name); output_unitig_graph(sg, coverage_cut, buf, sources, ruIndex, max_hang_length, mini_overlap_length); free(buf); output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, - reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 0, &b_mask_t); output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, - reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 0, &b_mask_t); } else if(ha_opt_hic(&asm_opt)) { - // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); - char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.hic", output_file_name); - output_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, bubble_dist, - (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, &b_mask_t); + output_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, (asm_opt.max_short_tip*2), + 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, &b_mask_t); free(buf); } else @@ -28098,12 +28348,8 @@ ma_sub_t **coverage_cut_ptr, int debug_g) output_contig_graph_primary_pre(sg, coverage_cut, output_file_name, sources, reverse_sources, asm_opt.small_pop_bubble_size, asm_opt.max_short_tip, ruIndex, max_hang_length, mini_overlap_length); - // rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, - // (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); - output_contig_graph_primary(sg, coverage_cut, output_file_name, sources, reverse_sources, - bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, - mini_overlap_length, &b_mask_t); + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, &b_mask_t); output_contig_graph_alternative(sg, coverage_cut, output_file_name, sources, ruIndex, max_hang_length, mini_overlap_length); } diff --git a/Overlaps.h b/Overlaps.h index 9c1cb88..d49c203 100644 --- a/Overlaps.h +++ b/Overlaps.h @@ -1008,11 +1008,10 @@ long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* r float drop_ratio); void adjust_utg_advance(asg_t *sg, ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex); void rescue_contained_reads_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t chainLenThres, uint32_t is_bubble_check, +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t); void rescue_missing_overlaps_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t is_bubble_check, -uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t); +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t); void all_to_all_deduplicate(ma_ug_t* ug, asg_t* read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, uint8_t postive_flag, float drop_rate, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, float double_check_rate); void drop_semi_circle(ma_ug_t *ug, asg_t* nsg, asg_t* read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex); @@ -1020,8 +1019,7 @@ void rescue_wrong_overlaps_to_unitigs(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, kvec_asg_arc_t_warp* keep_edges, bub_label_t* b_mask_t); void get_unitig_trio_flag(ma_utg_t* nsu, uint32_t flag, uint32_t* require, uint32_t* non_require, uint32_t* ambigious); void rescue_missing_overlaps_backward(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, -R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t backward_steps, -uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t); +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t); uint32_t get_edge_from_source(ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t target, asg_arc_t* t); int unitig_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio); @@ -1102,13 +1100,13 @@ uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t); void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg); ma_ug_t* copy_untig_graph(ma_ug_t *src); ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, -uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, bub_label_t* b_mask_t); asg_t* copy_read_graph(asg_t *src); diff --git a/hic.cpp b/hic.cpp index 9d12a06..61c8304 100644 --- a/hic.cpp +++ b/hic.cpp @@ -215,6 +215,8 @@ KRADIX_SORT_INIT(pe_occ2, pe_hit_hap, pe_occ_key_2, member_size(pe_hit_hap, occ2 #define pe_occ_key_t(x) (((uint64_t)((x).occ1))+((uint64_t)((x).occ2))) KRADIX_SORT_INIT(pe_occ_t, pe_hit_hap, pe_occ_key_t, 8) +#define asg_arc_key(a) ((a).ul) +KRADIX_SORT_INIT(asg_e, asg_arc_t, asg_arc_key, 8) typedef struct { // global data structure for kt_pipeline() const ha_ug_index* idx; @@ -285,6 +287,8 @@ typedef struct{ reads_t R1, R2; ha_ug_index* ug_index; +void print_debug_bubble_graph(bubble_type* bub, ma_ug_t* ug, const char *fn); + void build_bub_graph(ma_ug_t* ug, bubble_type* bub); void init_ha_ug_index_opt(ha_ug_index* idx, ma_ug_t *ug, int k, pldat_t* p) @@ -5884,6 +5888,7 @@ asg_arc_t *p, uint32_t check_het) get_bubbles(bub, id0, &beg_0, &sink_0, &a, &n, NULL); get_bubbles(bub, id1, &beg_1, &sink_1, &a, &n, NULL); + ori_0 = (uint64_t)-1; if(x_0 == (beg_0^1)) { @@ -6549,7 +6554,8 @@ int is_local_simple_circle(asg_t *g, uint32_t v) return 0; } -void update_bub_b_s_idx(bubble_type* bub) +///actually not useful, and may have bug when one bubble at multipe chains +void update_bub_b_s_idx(bubble_type* bub) { memset(bub->b_s_idx.a, -1, bub->b_s_idx.n * sizeof(uint64_t)); uint32_t i, v, beg, sink, n_bub = bub->num.n - 1; @@ -6598,6 +6604,7 @@ asg_arc_t *p_t, uint8_t *bsg_idx, ma_ug_t *unitig_ug, uint64_t* occ_thres, uint6 uint32_t i, pre, n, bub_id, v; uint64_t occ; asg_arc_t t_f, t_r; + radix_sort_u32(broken->a.a, broken->a.a + broken->a.n); for (i = n = occ = 0, pre = (uint32_t)-1; i < broken->a.n; i++) { @@ -6993,13 +7000,7 @@ uint32_t* extra_check) { stack->a.n--; cur = stack->a.a[stack->a.n]; - // if((beg>>1) == 162) - // { - // fprintf(stderr, ">>>###0###>>>beg=btg%.6ul, beg&1: %u, cur=btg%.6ul, cur&1: %u, sink=btg%.6ul, sink&1: %u, tan_occ: %u\n", - // (beg>>1)+1, beg&1, (cur>>1)+1, cur&1, (sink>>1)+1, sink&1, tan_occ); - // } - - + if(no_first && cur == beg) return 0; if(sink != (uint32_t)-1 && cur == sink) return 0; @@ -7102,6 +7103,7 @@ uint32_t* r_utg_beg, uint32_t* r_utg_sink) if(no_root && is_vis[cur] == 0 && is_vis[cur^1] == 0) { c_occ = get_btg_occ(bub, cur>>1); + ///assume v_occ is the beg node, c_occ is the end node, which means tan_occ cannot be too large if((tan_occ*side_rate) < c_occ && (tan_occ*total_rate) < (c_occ + v_occ)) { if(check_bubble_tangle(bub, bub->b_ug, v, cur^1, side_rate, total_rate, v_occ, c_occ, @@ -7114,7 +7116,7 @@ uint32_t* r_utg_beg, uint32_t* r_utg_sink) { set_b_utg_weight_flag(bub, NULL, res_btg->a.a[k], chain_flag, 0, NULL); } - btg_beg = v; btg_sink = cur^1; + btg_beg = v; btg_sink = cur^1;///b_utg id u = &(bub_ug->u.a[btg_beg>>1]); if((btg_beg&1)==1) @@ -7129,7 +7131,7 @@ uint32_t* r_utg_beg, uint32_t* r_utg_sink) ((u->a[u->n-1]>>32)&1) == 0?&w:NULL, NULL, NULL, NULL); (*r_b_tg_beg) = u->a[u->n-1]>>32; } - utg_beg = w^1; + utg_beg = w^1; ///ug id u = &(bub_ug->u.a[btg_sink>>1]); @@ -7145,12 +7147,7 @@ uint32_t* r_utg_beg, uint32_t* r_utg_sink) ((u->a[u->n-1]>>32)&1) == 0?&w:NULL, NULL, NULL, NULL); (*r_b_tg_sink) = u->a[u->n-1]>>32; } - utg_sink = w^1; - - // fprintf(stderr, "\nbtg_beg=btg%.6ul, utg_beg=utg%.6ul\n", - // (btg_beg>>1)+1, (utg_beg>>1)+1); - // fprintf(stderr, "btg_sink=btg%.6ul, utg_sink=utg%.6ul\n", - // (btg_sink>>1)+1, (utg_sink>>1)+1); + utg_sink = w^1; ///ug id is_t = check_bubble_tangle(NULL, ug, utg_beg, utg_sink, side_rate, total_rate, (uint32_t)-1, (uint32_t)-1, is_vis2, stack2, res_utg, chain_flag, &extra_check); @@ -7205,7 +7202,7 @@ uint32_t* r_utg_beg, uint32_t* r_utg_sink) if(tan_occ*side_rate >= v_occ) return 0; if(tan_occ*total_rate >= v_occ) return 0; - + //let one end as a tangle if(check_bubble_tangle(bub, bub->b_ug, v, (uint32_t)-1, side_rate, total_rate, v_occ, (uint32_t)-1, is_vis2, stack2, res_btg, NULL, NULL) == 0) { @@ -7382,7 +7379,78 @@ void debug_tangle_bubble(bubble_type* bub, long long beg_idx, long long end_idx, } } +uint32_t print_b_utg_occ(bubble_type* bub, uint32_t v) +{ + ma_ug_t *bub_ug = bub->b_ug; + ma_utg_t *u = NULL; + uint32_t k_i, k_j, *a = NULL, n, tan_occ = 0, beg, sink; + + u = &(bub_ug->u.a[v]); + fprintf(stderr, "\nstart: %u-th bubble-utg-start (# bubbles: %u)\n", v, (uint32_t)u->n); + for (k_i = 0; k_i < u->n; k_i++) + { + get_bubbles(bub, u->a[k_i]>>33, &beg, &sink, &a, &n, NULL); + for (k_j = 0; k_j < n; k_j++) + { + tan_occ += bub->ug->u.a[a[k_j]>>1].n; + } + + fprintf(stderr, "bid: %lu, n: %u, beg-utg%.6dl(%u), sink-utg%.6dl(%u)\n", u->a[k_i]>>33, n, (beg>>1)+1, beg&1, (sink>>1)+1, sink&1); + } + + fprintf(stderr, "end: %u-th bubble-utg-end\n\n", v); + return tan_occ; +} + +void update_bsg(asg_t *bsg, kvec_asg_arc_t_warp* edges) +{ + asg_arc_t *t = NULL; + uint32_t k, l, i, convex, max_i; + long long max, nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; + + for (k = 0; k < edges->a.n; k++) + { + t = asg_arc_pushp(bsg); + *t = edges->a.a[k]; + } + bsg->is_srt = 0; free(bsg->idx); bsg->idx = 0; + asg_cleanup(bsg); + + + + radix_sort_asg_e(edges->a.a, edges->a.a + edges->a.n); + for (k = 1, l = 0; k <= edges->a.n; ++k) + { + if (k == edges->a.n || (edges->a.a[k].ul>>32) != (edges->a.a[l].ul>>32)) + { + if(k - l > 1) + { + for (i = l, max = -1, max_i = (uint32_t)-1; i < k; i++) + { + get_unitig(bsg, NULL, edges->a.a[i].v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, NULL); + if(max < nodeLen) max = nodeLen, max_i = i; + } + + ///fprintf(stderr, "k - l: %u, max_i: %u\n", k - l, max_i); + for (i = l; i < k; i++) + { + // fprintf(stderr, "i: %u, +t->ul>>32: %lu, t->v: %u\n", + // i, edges->a.a[i].ul>>32, edges->a.a[i].v); + if(i == max_i) continue; + asg_arc_del(bsg, (edges->a.a[i].ul>>32), (edges->a.a[i].v), 1); + asg_arc_del(bsg, (edges->a.a[i].v)^1, (edges->a.a[i].ul>>32)^1, 1); + ///edges->a.a[i].del = 1; + } + } + l = k; + } + } + + asg_cleanup(bsg); + +} void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) { ma_ug_t *bub_ug = bub->b_ug; @@ -7402,10 +7470,6 @@ void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) uint32_t b_utg_beg, b_utg_sink, b_tg_beg, b_tg_sink, utg_beg, utg_sink; - - - - while(1) { occ_idx.n = 0; edges.a.n = 0; @@ -7423,7 +7487,7 @@ void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) memset(chain_flag, 0, n_vx); if(bub->num.n > 0) bub->num.n--; new_bub = bub->b_g->n_seq; - + //label all unitigs in bubble chain for (k = 0; k < bub_ug->g->n_seq; k++) { kv_pushp(uint64_t, occ_idx, &p); @@ -7465,12 +7529,10 @@ void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) if(b_utg_beg != (uint32_t)-1) is_used[b_utg_beg] = 1; if(b_utg_sink != (uint32_t)-1) is_used[b_utg_sink] = 1; if(b_tg_beg != (uint32_t)-1) b_tg_beg>>=1; - if(b_tg_sink != (uint32_t)-1) b_tg_sink>>=1; + if(b_tg_sink != (uint32_t)-1) b_tg_sink>>=1; update_bubble_graph(&res_utg, utg_beg, b_tg_beg, utg_sink, b_tg_sink, bub, &edges, bsg, NULL, NULL, ug, NULL, 0); - drop_g_edges_by_utg(bub, bsg, bub_ug, &res_btg, b_utg_beg, b_utg_sink); - ///fprintf(stderr, "+>>>>>>beg=btg%.6ul, sink=btg%.6ul\n", (b_utg_beg>>1)+1, (b_utg_sink>>1)+1); } } @@ -7502,9 +7564,16 @@ void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) if(b_utg_sink != (uint32_t)-1) is_used[b_utg_sink] = 1; if(b_tg_beg != (uint32_t)-1) b_tg_beg>>=1; if(b_tg_sink != (uint32_t)-1) b_tg_sink>>=1; - - update_bubble_graph(&res_utg, utg_beg, b_tg_beg, utg_sink, b_tg_sink, - bub, &edges, bsg, NULL, NULL, ug, NULL, 0); + /*******************************for debug************************************/ + // if(utg_beg == (utg_sink^1)) + // { + // print_b_utg_occ(bub, b_utg_beg>>1); + // print_b_utg_occ(bub, b_utg_sink>>1); + // print_b_utg_occ(bub, 42); + // ///print_debug_bubble_graph(bub, ug, asm_opt.output_file_name); + // } + /*******************************for debug************************************/ + update_bubble_graph(&res_utg, utg_beg, b_tg_beg, utg_sink, b_tg_sink, bub, &edges, bsg, NULL, NULL, ug, NULL, 0); drop_g_edges_by_utg(bub, bsg, bub_ug, &res_btg, b_utg_beg, b_utg_sink); ///fprintf(stderr, "->>>>>>beg=btg%.6ul, sink=btg%.6ul\n", (b_utg_beg>>1)+1, (b_utg_sink>>1)+1); } @@ -7513,16 +7582,10 @@ void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) kv_push(uint32_t, bub->num, bub->list.n); new_bub = bub->b_g->n_seq - new_bub; bub->tangle_bub += new_bub; + ///actually not useful, and may have bug when one bubble at multipe chains if(new_bub) update_bub_b_s_idx(bub); - asg_arc_t *t = NULL; - for (k = 0; k < edges.a.n; k++) - { - t = asg_arc_pushp(bsg); - *t = edges.a.a[k]; - } - - bsg->is_srt = 0; free(bsg->idx); bsg->idx = 0; - asg_cleanup(bsg); + update_bsg(bsg, &edges); + ma_ug_destroy(bub_ug); bub_ug = ma_ug_gen(bub->b_g); bub->b_ug = bub_ug; @@ -7556,6 +7619,8 @@ void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) free(is_vis); free(is_vis2); free(is_used); free(chain_flag); free(b.b.a); kv_destroy(occ_idx); kv_destroy(stack.a); kv_destroy(stack2.a); kv_destroy(res_btg.a); kv_destroy(res_utg.a); kv_destroy(edges.a); + + ///print_debug_bubble_graph(bub, ug, asm_opt.output_file_name); } @@ -7691,6 +7756,7 @@ void update_bubble_chain(ma_ug_t* ug, bubble_type* bub, uint32_t is_middle, uint kv_push(uint32_t, bub->num, bub->list.n); new_bub = bub->b_g->n_seq - new_bub; if(is_end) bub->b_end_bub += new_bub; + ///actually not useful, and may have bug when one bubble at multipe chains if(new_bub) update_bub_b_s_idx(bub); @@ -8298,21 +8364,12 @@ void clean_bubble_chain_by_HiC(ma_ug_t* ug, hc_links* link, bubble_type* bub) kv_push(uint32_t, bub->num, bub->list.n); new_bub = bub->b_g->n_seq - new_bub; bub->cross_bub += new_bub; + ///actually not useful, and may have bug when one bubble at multipe chains if(new_bub) update_bub_b_s_idx(bub); - ///fprintf(stderr, "bub->cross_bub: %u\n", (uint32_t)bub->cross_bub); ///debug_tangle_bubble(bub, bub->b_g->n_seq - bub->cross_bub, bub->b_g->n_seq - 1, "Cross-tangle"); + update_bsg(bub->b_g, &edges); - asg_arc_t *t = NULL; - for (i = 0; i < edges.a.n; i++) - { - t = asg_arc_pushp(bub->b_g); - *t = edges.a.a[i]; - } - bub->b_g->is_srt = 0; - free(bub->b_g->idx); - bub->b_g->idx = 0; - asg_cleanup(bub->b_g); ma_ug_destroy(bs_ug); bs_ug = ma_ug_gen(bub->b_g); bub->b_ug = bs_ug; @@ -8437,9 +8494,9 @@ void append_boundary_chain(ma_ug_t* ug, hc_links* link, bubble_type* bub) kv_push(uint32_t, bub->num, bub->list.n); new_bub = bub->b_g->n_seq - new_bub; bub->mess_bub += new_bub; + ///actually not useful, and may have bug when one bubble at multipe chains if(new_bub) update_bub_b_s_idx(bub); - ///fprintf(stderr, "bub->mess_bub: %lu\n", bub->mess_bub); for (v = 0; v < n_vx; v++) { @@ -8449,17 +8506,8 @@ void append_boundary_chain(ma_ug_t* ug, hc_links* link, bubble_type* bub) drop_g_edges_by_utg(bub, bub->b_g, bs_ug, NULL, v, (uint32_t)-1); } + update_bsg(bub->b_g, &edges); - asg_arc_t *t = NULL; - for (i = 0; i < edges.a.n; i++) - { - t = asg_arc_pushp(bub->b_g); - *t = edges.a.a[i]; - } - bub->b_g->is_srt = 0; - free(bub->b_g->idx); - bub->b_g->idx = 0; - asg_cleanup(bub->b_g); ma_ug_destroy(bs_ug); bs_ug = ma_ug_gen(bub->b_g); bub->b_ug = bs_ug; From 98a04c168ac550200102bc0aa269c1926184934a Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Thu, 18 Mar 2021 05:26:33 -0400 Subject: [PATCH 6/8] update r313 --- CommandLines.cpp | 30 +++++--- CommandLines.h | 8 ++- Overlaps.cpp | 174 +++++++---------------------------------------- Overlaps.h | 5 +- Purge_Dups.cpp | 2 + hic.cpp | 10 +-- hifiasm.1 | 10 +-- 7 files changed, 62 insertions(+), 177 deletions(-) diff --git a/CommandLines.cpp b/CommandLines.cpp index f69f93e..5586fa8 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -23,7 +23,7 @@ static ko_longopt_t long_options[] = { { "ex-iter", ko_required_argument, 308 }, { "purge-cov", ko_required_argument, 309 }, { "pri-range", ko_required_argument, 310 }, - { "high-het", ko_no_argument, 311 }, + ///{ "high-het", ko_no_argument, 311 }, { "lowQ", ko_required_argument, 312 }, { "min-hist-cnt", ko_required_argument, 313 }, { "h1", ko_required_argument, 314 }, @@ -90,13 +90,13 @@ void Print_H(hifiasm_opt_t* asm_opt) fprintf(stderr, " Purge-dups:\n"); fprintf(stderr, " -l INT purge level. 0: no purging; 1: light; 2/3: aggressive [0 for trio; 2 for unzip]\n"); - fprintf(stderr, " -s FLOAT similarity threshold for duplicate haplotigs [%g]\n", - asm_opt->purge_simi_rate); + fprintf(stderr, " -s FLOAT similarity threshold for duplicate haplotigs [%g for -l1/-l2, %g for -l3]\n", + asm_opt->purge_simi_rate_l2, asm_opt->purge_simi_rate_l3); fprintf(stderr, " -O INT min number of overlapped reads for duplicate haplotigs [%d]\n", asm_opt->purge_overlap_len); fprintf(stderr, " --purge-cov INT\n"); fprintf(stderr, " coverage upper bound of Purge-dups [auto]\n"); - fprintf(stderr, " --high-het enable this mode for high heterozygosity sample [experimental, not stable]\n"); + ///fprintf(stderr, " --high-het enable this mode for high heterozygosity sample [experimental, not stable]\n"); fprintf(stderr, " Hi-C-partition [experimental, not stable]:\n"); fprintf(stderr, " --h1 FILEs file names of Hi-C R1 [r1_1.fq,r1_2.fq,...]\n"); @@ -149,10 +149,11 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->mid_cnt = 5; asm_opt->purge_level_primary = 2; asm_opt->purge_level_trio = 0; - asm_opt->purge_simi_rate = 0.75; - asm_opt->purge_simi_rate_hic = 0.85; + asm_opt->purge_simi_rate_l2 = 0.75; + asm_opt->purge_simi_rate_l3 = 0.55; + ///asm_opt->purge_simi_rate_hic = 0.85; asm_opt->purge_overlap_len = 1; - asm_opt->purge_overlap_len_hic = 50; + ///asm_opt->purge_overlap_len_hic = 50; asm_opt->recover_atg_cov_min = -1024; asm_opt->recover_atg_cov_max = INT_MAX; asm_opt->hom_global_coverage = -1; @@ -453,6 +454,12 @@ int check_option(hifiasm_opt_t* asm_opt) return 0; } + if(asm_opt->purge_simi_thres < 0) + { + fprintf(stderr, "[ERROR] [-s] must >= 0\n"); + return 0; + } + // fprintf(stderr, "input file num: %d\n", asm_opt->num_reads); // fprintf(stderr, "output file: %s\n", asm_opt->output_file_name); // fprintf(stderr, "number of threads: %d\n", asm_opt->thread_num); @@ -470,7 +477,7 @@ int check_option(hifiasm_opt_t* asm_opt) // fprintf(stderr, "mid_cnt: %d\n", asm_opt->mid_cnt); // fprintf(stderr, "purge_level_primary: %d\n", asm_opt->purge_level_primary); // fprintf(stderr, "purge_level_trio: %d\n", asm_opt->purge_level_trio); - // fprintf(stderr, "purge_simi_rate: %f\n", asm_opt->purge_simi_rate); + // fprintf(stderr, "purge_simi_thres: %f\n", asm_opt->purge_simi_thres); // fprintf(stderr, "purge_overlap_len: %d\n", asm_opt->purge_overlap_len); return 1; @@ -620,7 +627,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) asm_opt->recover_atg_cov_min = asm_opt->recover_atg_cov_max = -1; } } - else if (c == 311) asm_opt->flag |= HA_F_HIGH_HET; + ///else if (c == 311) asm_opt->flag |= HA_F_HIGH_HET; else if (c == 312) asm_opt->bed_inconsist_rate = atoi(opt.arg); else if (c == 313) asm_opt->min_hist_kmer_cnt = atoi(opt.arg); else if (c == 314) get_hic_enzymes(opt.arg, &(asm_opt->hic_reads[0]), 0); @@ -633,7 +640,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) { ///0: disable purge_dup; 1: purge containment; 2: purge overlap asm_opt->purge_level_primary = asm_opt->purge_level_trio = atoi(opt.arg); } - else if (c == 's') asm_opt->purge_simi_rate = atof(opt.arg); + else if (c == 's') asm_opt->purge_simi_rate_l2 = asm_opt->purge_simi_rate_l3 = atof(opt.arg); else if (c == 'O') asm_opt->purge_overlap_len = atoll(opt.arg); else if (c == ':') { @@ -647,6 +654,9 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) } } + if(asm_opt->purge_level_primary > 2) asm_opt->purge_simi_thres = asm_opt->purge_simi_rate_l3; + else asm_opt->purge_simi_thres = asm_opt->purge_simi_rate_l2; + if (argc == opt.ind) { diff --git a/CommandLines.h b/CommandLines.h index 253e6a8..4f263f5 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -70,7 +70,7 @@ typedef struct { int purge_level_primary; int purge_level_trio; int purge_overlap_len; - int purge_overlap_len_hic; + ///int purge_overlap_len_hic; int recover_atg_cov_min; int recover_atg_cov_max; int hom_global_coverage; @@ -80,8 +80,10 @@ typedef struct { float max_hang_rate; float min_drop_rate; float max_drop_rate; - float purge_simi_rate; - float purge_simi_rate_hic; + float purge_simi_rate_l2; + float purge_simi_rate_l3; + float purge_simi_thres; + ///float purge_simi_rate_hic; long long small_pop_bubble_size; long long large_pop_bubble_size; diff --git a/Overlaps.cpp b/Overlaps.cpp index 5be775b..2d9782c 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -11093,27 +11093,6 @@ uint32_t get_num_trio_flag(ma_ug_t *ug, uint32_t v, uint32_t flag) } -void set_pre_uid(buf_t* b, hc_links* link, ma_ug_t *ug) -{ - uint32_t k = 0, i = 0, m = 0, rId, pre = (uint32_t)-1; - ma_utg_t* u = NULL; - for (i = 0; i < b->b.n; i++) - { - u = &(ug->u.a[b->b.a[i]>>1]); - if(u->m == 0) continue; - for (k = 0; k < u->n; k++) - { - rId = u->a[k]>>33; - if(link->u_idx[rId] == (uint32_t)-1) continue; - if(pre == link->u_idx[rId]) continue; - pre = link->u_idx[rId]; - b->b.a[m] = link->u_idx[rId]; - m++; - } - } - b->b.n = m; -} - inline uint64_t get_utg_len(buf_t* b, ma_ug_t *ug, asg_t *read_sg, uint64_t ignore_end, uint64_t* len_thre, uint64_t* occ) { @@ -11217,77 +11196,6 @@ inline uint64_t get_utg_len(buf_t* b, ma_ug_t *ug, asg_t *read_sg, uint64_t igno return len; } -void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg) -{ - uint32_t b_0_i, b_0_k, b_1_i, b_1_k, pre_0, pre_1, rId_0, rId_1, ori_0, ori_1; - uint64_t d = RC_1, len_0, len_1, thre_0, thre_1; - ma_utg_t* u_b_0 = NULL; - ma_utg_t* u_b_1 = NULL; - if(b_0->b.n == 0 || b_1->b.n == 0) return; - - len_0 = get_utg_len(b_0, ug, read_sg, 1, NULL, NULL); - len_1 = get_utg_len(b_1, ug, read_sg, 1, NULL, NULL); - - len_0 = MIN(len_0, len_1); - get_utg_len(b_0, ug, read_sg, 0, &len_0, &thre_0); - get_utg_len(b_1, ug, read_sg, 0, &len_0, &thre_1); - - - for (b_0_i = len_0 = 0, pre_0 = (uint32_t)-1; b_0_i < b_0->b.n; b_0_i++) - { - ori_0 = b_0->b.a[b_0_i]&1; - u_b_0 = &(ug->u.a[b_0->b.a[b_0_i]>>1]); - if(u_b_0->n == 0) continue; - for (b_0_k = 0; b_0_k < u_b_0->n; b_0_k++) - { - len_0++; - if(len_0 > thre_0) return; - if(ori_0 == 1) - { - rId_0 = u_b_0->a[u_b_0->n - b_0_k - 1]>>33; - } - else - { - rId_0 = u_b_0->a[b_0_k]>>33; - } - - if(link->u_idx[rId_0] == (uint32_t)-1) continue; - if(pre_0 == link->u_idx[rId_0]) continue; - pre_0 = link->u_idx[rId_0]; - - for (b_1_i = len_1 = 0, pre_1 = (uint32_t)-1; b_1_i < b_1->b.n; b_1_i++) - { - ori_1 = b_1->b.a[b_1_i]&1; - u_b_1 = &(ug->u.a[b_1->b.a[b_1_i]>>1]); - if(u_b_1->n == 0) continue; - for (b_1_k = 0; b_1_k < u_b_1->n; b_1_k++) - { - len_1++; - if(len_1 > thre_1) goto b_1_i_end; - if(ori_1 == 1) - { - rId_1 = u_b_1->a[u_b_1->n - b_1_k - 1]>>33; - } - else - { - rId_1 = u_b_1->a[b_1_k]>>33; - } - if(link->u_idx[rId_1] == (uint32_t)-1) continue; - if(pre_1 == link->u_idx[rId_1]) continue; - pre_1 = link->u_idx[rId_1]; - - - push_hc_edge(&(link->a.a[pre_0]), pre_1, 1, 1, &d); - push_hc_edge(&(link->a.a[pre_1]), pre_0, 1, 1, &d); - } - } - - b_1_i_end:; - } - } -} - - uint32_t set_utg_offset(buf_t* b, ma_ug_t *ug, asg_t *read_sg, hap_cov_t *cov, uint32_t is_clear) { uint32_t ori, uid, v, nv, l, k; @@ -11810,7 +11718,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, int max_hang, int min_ovlp) asm_opt.hom_global_coverage = -1; purge_dups(ug, sg, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 1, cov); + asm_opt.purge_simi_thres, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 1, cov); dip_thre_max = ((double)asm_opt.hom_global_coverage)/((double)HOM_PEAK_RATE)*0.70; asm_opt.hom_global_coverage = tmp_cov; ///fprintf(stderr, "dip_thre_max: %lu\n", dip_thre_max); @@ -11909,9 +11817,9 @@ void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num) kv_init(link->a.a[i].e); kv_init(link->a.a[i].f); } - MALLOC(link->u_idx, r_num); - memset(link->u_idx, -1, r_num*sizeof(uint32_t)); - link->r_num = r_num; + // MALLOC(link->u_idx, r_num); + // memset(link->u_idx, -1, r_num*sizeof(uint32_t)); + // link->r_num = r_num; kv_malloc(link->bed, ug_num); link->bed.n = ug_num; for (i = 0; i < link->bed.n; i++) { @@ -11928,7 +11836,7 @@ void destory_hc_links(hc_links* link) kv_destroy(link->a.a[i].f); } kv_destroy(link->a); - free(link->u_idx); + ///free(link->u_idx); for (i = 0; i < link->bed.n; i++) { kv_destroy(link->bed.a[i]); @@ -12061,8 +11969,8 @@ bub_label_t* b_mask_t) init_hc_links(&link, ug->g->n_seq, R_INF.total_reads); asg_t *copy_sg = copy_read_graph(sg); ma_ug_t *copy_ug = copy_untig_graph(ug); - asm_opt.purge_overlap_len = asm_opt.purge_overlap_len_hic; - asm_opt.purge_simi_rate = asm_opt.purge_simi_rate_hic; + ///asm_opt.purge_overlap_len = asm_opt.purge_overlap_len_hic; + ///asm_opt.purge_simi_thres = asm_opt.purge_simi_rate_hic; adjust_utg_by_primary(©_ug, copy_sg, TRIO_THRES, sources, reverse_sources, coverage_cut, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, min_ovlp, &new_rtg_edges, &link, b_mask_t); @@ -12363,7 +12271,7 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp) uint32_t print_untig_by_read(ma_ug_t *g, const char* name, uint32_t in, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, const char* info) { - uint32_t i, k, rId = (uint32_t)-1; + uint32_t i, k, rId = (uint32_t)-1, flag = 0; if(in != (uint32_t)-1) { rId = in; @@ -12405,7 +12313,8 @@ ma_hit_t_alloc* reverse_sources, const char* info) { fprintf(stderr, "%s: %s is the %u-th read at %u-th unitig (label: %u, occ: %u)\n", info, name, k, i, g->g->seq[i].c, u->n); - return i; + flag = 1; + ///return i; } } } @@ -12415,7 +12324,7 @@ ma_hit_t_alloc* reverse_sources, const char* info) - fprintf(stderr, "%s: %s is not at any unitig\n", info, name); + if(flag == 0) fprintf(stderr, "%s: %s is not at any unitig\n", info, name); return (uint32_t)-1; } @@ -13810,10 +13719,9 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate, hap_cov_t *cov) redo: ///print_untig((ug), 61955, "i-0:", 0); - asg_pop_bubble_primary_trio(ug, NULL, trio_flag, DROP, cov); untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, cov); - magic_trio_phasing(g, ug, read_g, coverage_cut, sources, reverse_sources, 2, ruIndex, trio_flag, trio_drop_rate); + magic_trio_phasing(g, ug, read_g, coverage_cut, sources, reverse_sources, 2, ruIndex, trio_flag, trio_drop_rate); ///drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); /**********debug**********/ if(just_bubble_pop == 0) @@ -13836,7 +13744,6 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate, hap_cov_t *cov) asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, trio_flag, cov); asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, cov); asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, cov); - ///print_debug_gfa(read_g, ug, coverage_cut, "debug_chimeric", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); ///need consider tangles ///note we need both the read graph and the untig graph @@ -13845,16 +13752,16 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate, hap_cov_t *cov) cur_cons = get_graph_statistic(g); } untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, cov); - if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, trio_flag, 0, read_g, reverse_sources, ruIndex, 2); } + ///print_debug_gfa(read_g, ug, coverage_cut, "debug_dups", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); + resolve_tangles(ug, read_g, reverse_sources, 20, 100, 0.05, 0.2, ruIndex, trio_flag, drop_ratio); drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); all_to_all_deduplicate(ug, read_g, coverage_cut, sources, trio_flag, trio_drop_rate, reverse_sources, ruIndex, DOUBLE_CHECK_THRES); - if(is_first) { is_first = 0; @@ -14591,7 +14498,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) hap_cov_t *cov = init_hap_cov_t(*ug, read_g, sources, ruIndex, reverse_sources, coverage_cut, max_hang, min_ovlp, 0); purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, + asm_opt.purge_simi_thres, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, 1, 1, cov); if(asm_opt.recover_atg_cov_min == -1024) { @@ -14612,13 +14519,11 @@ kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) } - + adjust_utg_advance(read_g, (*ug), reverse_sources, ruIndex); ///primary_flag = get_utg_attributes(*ug, read_g, coverage_cut, sources, ruIndex); update_unitig_graph((*ug), read_g, coverage_cut, sources, reverse_sources, ruIndex, 0, DOUBLE_CHECK_THRES, flag, drop_rate); - adjust_utg_advance(read_g, (*ug), reverse_sources, ruIndex); - nsg = (*ug)->g; n_vtx = nsg->n_seq; for (v = 0; v < n_vtx; ++v) @@ -14671,7 +14576,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t) if(asm_opt.purge_level_trio == 1) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, 1, 0, + asm_opt.purge_simi_thres, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, 1, 0, cov); ///delete_useless_nodes(ug); delete_useless_trio_nodes(ug, read_g, coverage_cut, sources, ruIndex); @@ -14718,7 +14623,6 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, kvec_asg_arc_t_warp new_rtg_edges; kv_init(new_rtg_edges.a); - ///print_untig_by_read(ug, "m64011_190830_220126/117834372/ccs", 865264, sources, reverse_sources, "beg"); adjust_utg_by_trio(&ug, sg, flag, TRIO_THRES, sources, reverse_sources, coverage_cut, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, min_ovlp, &new_rtg_edges, b_mask_t); @@ -19879,7 +19783,7 @@ float drop_ratio) } } - adjust_utg_advance(read_g, src, reverse_sources, ruIndex); + ///adjust_utg_advance(read_g, src, reverse_sources, ruIndex); ///note: we must reset start for each unitig n_vtx = src->g->n_seq; for (v = 0; v < n_vtx; ++v) @@ -19974,7 +19878,6 @@ float drop_ratio) nsu->n = m; } - n_vtx = nsg->n_seq; for (i = 0; i < n_vtx; ++i) { @@ -20108,7 +20011,6 @@ float drop_ratio) free(b_0.b.a); free(b_1.b.a); - ///note: we must reset start for each unitig n_vtx = src->g->n_seq; for (v = 0; v < n_vtx; ++v) @@ -20124,7 +20026,7 @@ float drop_ratio) src->u.a[v].start = UINT32_MAX; } } - adjust_utg_advance(read_g, src, reverse_sources, ruIndex); + ///adjust_utg_advance(read_g, src, reverse_sources, ruIndex); ///note: we must reset start for each unitig n_vtx = src->g->n_seq; for (v = 0; v < n_vtx; ++v) @@ -20133,6 +20035,7 @@ float drop_ratio) if(src->u.a[v].m==0) continue; EvaluateLen(src->u, v) = src->u.a[v].n; } + ///print_untig_by_read(src, "m64076_200203_181219/82511682/ccs", 2429597, NULL, NULL, "end-1"); } @@ -21759,31 +21662,6 @@ R_to_U* ruIndex) } -void reset_reverse_unitigs(hc_links* link, ma_utg_t *u) -{ - uint32_t k = 0, i = 0, rId, pre = (uint32_t)-1; - hc_edge *e = NULL; - if(u->n == 0 || u->m == 0) return; - for (k = 0; k < u->n; k++) - { - rId = u->a[k]>>33; - if(link->u_idx[rId] == (uint32_t)-1) continue; - if(pre == link->u_idx[rId]) continue; - pre = link->u_idx[rId]; - if(link->a.a[pre].f.n == 0) continue; - - for (i = 0; i < link->a.a[pre].f.n; i++) - { - if(link->a.a[pre].f.a[i].del) continue; - link->a.a[pre].f.a[i].del = 1; - e = get_hc_edge(link, link->a.a[pre].f.a[i].uID, pre, 1); - if(e == NULL) continue; - e->del = 1; - } - } -} - - void reset_trans_chain(trans_chain* t_ch, ma_utg_t *u) { uint32_t k = 0, i = 0, p_uId = (uint32_t)-1, c_uId; @@ -21834,7 +21712,6 @@ void append_utg(ma_ug_t* ptg, ma_ug_t* atg, trans_chain* t_ch) for (v = 0; v < atg->g->n_seq; ++v) { if(atg->g->seq[v].del || atg->u.a[v].m == 0) continue; - ///if(link) reset_reverse_unitigs(link, &(atg->u.a[v])); if(t_ch) reset_trans_chain(t_ch, &(atg->u.a[v])); p = &(ptg->u.a[ptg->u.n]); @@ -22070,9 +21947,8 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) ///print_utg_coverage(*ug, coverage_cut, 440, sources); ///exit(0); - drop_semi_circle((*ug), nsg, read_g, reverse_sources, ruIndex); - - asg_cleanup(nsg); + // drop_semi_circle((*ug), nsg, read_g, reverse_sources, ruIndex); + // asg_cleanup(nsg); adjust_utg_advance(read_g, (*ug), reverse_sources, ruIndex); nsg = (*ug)->g; @@ -22094,7 +21970,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) just_contain = 0; if(asm_opt.purge_level_primary == 1) just_contain = 1; purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, + asm_opt.purge_simi_thres, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, just_contain, 0, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); @@ -22114,7 +21990,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) just_contain = 0; if(asm_opt.purge_level_primary == 1) just_contain = 1; purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, + asm_opt.purge_simi_thres, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, just_contain, 0, cov); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); @@ -22124,7 +22000,7 @@ kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t) if(asm_opt.purge_level_primary == 0) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, - asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, 0, + asm_opt.purge_simi_thres, asm_opt.purge_overlap_len, max_hang, min_ovlp, drop_ratio, 0, 1, cov); } diff --git a/Overlaps.h b/Overlaps.h index d49c203..606b518 100644 --- a/Overlaps.h +++ b/Overlaps.h @@ -1052,8 +1052,8 @@ typedef struct{ kvec_t(hc_linkeage) a; kvec_t(uint64_t) enzymes; kvec_t(bed_in) bed; - uint32_t* u_idx; - uint64_t r_num; + ///uint32_t* u_idx; + ///uint64_t r_num; } hc_links; @@ -1103,7 +1103,6 @@ ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link, bub_label_t* b_mask_t); -void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg); ma_ug_t* copy_untig_graph(ma_ug_t *src); ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, diff --git a/Purge_Dups.cpp b/Purge_Dups.cpp index e6fe0a6..b8f5663 100644 --- a/Purge_Dups.cpp +++ b/Purge_Dups.cpp @@ -4197,6 +4197,7 @@ hap_cov_t *cov) } } +/** void collect_reverse_unitig_pair(hc_links* link, ma_ug_t *ug, hap_overlaps* t) { uint32_t i = 0, k = 0, rId_0, rId_1, pre_0, pre_1, b_0 = t->xUid, b_1 = t->yUid; @@ -4238,6 +4239,7 @@ void collect_reverse_unitigs_purge(buf_t* b_0, hc_links* link, ma_ug_t *ug, hap_ collect_reverse_unitig_pair(link, ug, &(all_ovlp->x[b_0->b.a[k]>>1].a.a[index])); } } +**/ void link_unitigs(asg_t *purge_g, ma_ug_t *ug, hap_overlaps_list* all_ovlp, diff --git a/hic.cpp b/hic.cpp index 61c8304..ad4e13e 100644 --- a/hic.cpp +++ b/hic.cpp @@ -3808,8 +3808,8 @@ void write_hc_links(hc_links* link, const char *fn) fwrite(&link->enzymes.n, sizeof(link->enzymes.n), 1, fp); fwrite(link->enzymes.a, sizeof(uint64_t), link->enzymes.n, fp); - fwrite(&link->r_num, sizeof(link->r_num), 1, fp); - fwrite(link->u_idx, sizeof(uint32_t), 1, fp); + // fwrite(&link->r_num, sizeof(link->r_num), 1, fp); + // fwrite(link->u_idx, sizeof(uint32_t), 1, fp); fwrite(&(link->bed.n), sizeof(link->bed.n), 1, fp); for (k = 0; k < link->bed.n; k++) @@ -3858,9 +3858,9 @@ int load_hc_links(hc_links* link, const char *fn) flag += fread(&link->enzymes.n, sizeof(link->enzymes.n), 1, fp); link->enzymes.m = link->enzymes.n; MALLOC(link->enzymes.a, link->enzymes.n); flag += fread(link->enzymes.a, sizeof(uint64_t), link->enzymes.n, fp); - fread(&link->r_num, sizeof(link->r_num), 1, fp); - MALLOC(link->u_idx, link->r_num); - fread(link->u_idx, sizeof(uint32_t), 1, fp); + // fread(&link->r_num, sizeof(link->r_num), 1, fp); + // MALLOC(link->u_idx, link->r_num); + // fread(link->u_idx, sizeof(uint32_t), 1, fp); diff --git a/hifiasm.1 b/hifiasm.1 index 245a7b1..c4bcb54 100644 --- a/hifiasm.1 +++ b/hifiasm.1 @@ -286,13 +286,14 @@ times in the other sample. .TP 10 .BI -l \ INT Level of purge-dup. 0 to disable purge-dup, 1 to only purge contained haplotigs, -2 to purge all types of haplotigs, 3 to purge all types of haplotigs in most aggressive way. +2 to purge all types of haplotigs, 3 to purge all types of haplotigs in most aggressive way +for high heterozygosity sample. In default, [2] for non-trio assembly, [0] for trio assembly. For trio assembly, only level 0 and level 1 are allowed. .TP .BI -s \ FLOAT -Similarity threshold for duplicate haplotigs that should be purged [0.75]. +Similarity threshold for duplicate haplotigs that should be purged [0.75 for -l1/-l2, 0.55 for -l3]. .TP .BI -O \ FLOAT @@ -303,11 +304,6 @@ Min number of overlapped reads for duplicate haplotigs that should be purged [1] Coverage upper bound of Purge-dups, which is inferred automatically in default. If the coverage of a contig is higher than this bound, don't apply Purge-dups. -.TP -.BI --high-het \ INT -Enable this mode for high heterozygosity sample, which will increase running time. -For ordinary samples, no need to enable this mode [experimental, not stable]. - .SS Debugging options From 2db42c8c0073d72b0a7b2815b1d69c12e59d3e95 Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Thu, 18 Mar 2021 13:18:46 -0400 Subject: [PATCH 7/8] update version number --- CommandLines.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CommandLines.h b/CommandLines.h index 4f263f5..8616112 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -3,7 +3,7 @@ #include -#define HA_VERSION "0.14-r313" +#define HA_VERSION "0.14.1-r313" #define VERBOSE 0 From e6e6dbf7b3cbec0b0165800d4c94bfc9a72ab2b6 Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Thu, 18 Mar 2021 13:44:30 -0400 Subject: [PATCH 8/8] remove unnecessary bin files of Hi-C --- CommandLines.h | 2 +- Overlaps.cpp | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CommandLines.h b/CommandLines.h index 8616112..a6b0924 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -3,7 +3,7 @@ #include -#define HA_VERSION "0.14.1-r313" +#define HA_VERSION "0.14.1-r314" #define VERBOSE 0 diff --git a/Overlaps.cpp b/Overlaps.cpp index 2d9782c..f55819d 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -11964,7 +11964,7 @@ bub_label_t* b_mask_t) hc_links link; - if(load_hc_links(&link, output_file_name) == 0) + ///if(load_hc_links(&link, output_file_name) == 0) { init_hc_links(&link, ug->g->n_seq, R_INF.total_reads); asg_t *copy_sg = copy_read_graph(sg); @@ -11980,7 +11980,7 @@ bub_label_t* b_mask_t) ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp, asm_opt.hic_inconsist_rate, NULL, NULL, &link); - write_hc_links(&link, output_file_name); + ///write_hc_links(&link, output_file_name); } @@ -28169,6 +28169,8 @@ ma_sub_t **coverage_cut_ptr, int debug_g) // rescue_no_coverage_aggressive(sg, sources, reverse_sources, &coverage_cut, ruIndex, max_hang_length, // mini_overlap_length, bubble_dist, 10); + rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, + ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz, &b_mask_t); if (asm_opt.flag & HA_F_VERBOSE_GFA) { @@ -28177,11 +28179,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) debug_gfa:; /*******************************for debug***************************************/ } - - rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, - ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz, &b_mask_t); - - + if (ha_opt_triobin(&asm_opt) && ha_opt_hic(&asm_opt)) { char *buf = (char*)calloc(strlen(output_file_name) + 25, 1);