diff --git a/CommandLines.cpp b/CommandLines.cpp index 37978f6..415f3c8 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -53,6 +53,7 @@ static ko_longopt_t long_options[] = { { "low-het", ko_no_argument, 339}, { "s-base", ko_required_argument, 340}, { "bin-only", ko_no_argument, 341}, + { "ul-round", ko_required_argument, 342}, { 0, 0, 0 } }; @@ -262,7 +263,7 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->is_topo_trans = 1; asm_opt->is_bub_trans = 1; asm_opt->bin_only = 0; - asm_opt->ul_clean_round = 2; + asm_opt->ul_clean_round = 1; } void destory_enzyme(enzyme* f) @@ -795,6 +796,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) if(asm_opt->trans_base_rate_sec < 0) asm_opt->is_base_trans = 0; } else if (c == 341) asm_opt->bin_only = 1; + else if (c == 342) asm_opt->ul_clean_round = atol(opt.arg); else if (c == 'l') { ///0: disable purge_dup; 1: purge containment; 2: purge overlap asm_opt->purge_level_primary = asm_opt->purge_level_trio = atoi(opt.arg); } diff --git a/CommandLines.h b/CommandLines.h index 5756596..eaef6ed 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -4,7 +4,7 @@ #include #include -#define HA_VERSION "0.18.6-r502" +#define HA_VERSION "0.18.6-r509" #define VERBOSE 0 diff --git a/Correct.cpp b/Correct.cpp index 9e07c44..49e3814 100644 --- a/Correct.cpp +++ b/Correct.cpp @@ -17,6 +17,7 @@ #define A_L 16 #define ext_w 6 #define r_simi_w 0.05 +#define rphase_thres 4 #define generic_key(x) (x) KRADIX_SORT_INIT(b32, uint32_t, generic_key, 4) @@ -15323,6 +15324,7 @@ UC_Read *tu, bit_extz_t *exz, overlap_region *aux_o, double e_rate, int64_t ql, #define ovlp_cur_wid(x) ((x).qn) #define ovlp_cur_xoff(x) ((x).qs) #define ovlp_cur_coff(x) ((x).qe) +#define ovlp_bd(x) ((x).sec) int64_t retrieve_cigar_err_debug(bit_extz_t *ez, int64_t s, int64_t e) { @@ -15788,6 +15790,159 @@ uint64_t gen_region_phase_robust(overlap_region* ol, uint64_t *id_a, uint64_t id return id_n; } +///[s, e) +int64_t extract_sub_cigar_err_rr(overlap_region *z, int64_t s, int64_t e, ul_ov_t *p) +{ + int64_t wk = ovlp_cur_wid(*p), xk = ovlp_cur_xoff(*p), ck = ovlp_cur_coff(*p); + int64_t min_w = ovlp_min_wid(*p), max_w = ovlp_max_wid(*p);//[min_w, max_w] + bit_extz_t ez; window_list *m; int64_t bd = ovlp_bd(*p), s0, e0; + s0 = z->w_list.a[min_w].x_start + bd; e0 = z->w_list.a[max_w].x_end+1-bd; + if(s < s0) s = s0; if(e > e0) e = e0;///exclude boundary + if(s >= e) return -1; + + int64_t ws, we, os, oe, ovlp, err = 0, xl, yl, werr, tot = e - s; + if(wk < min_w || wk > max_w) wk = min_w; + for (; wk >= min_w && z->w_list.a[wk].x_start > s; wk--); + if(wk < min_w || wk > max_w) return -1; + for (; wk <= max_w && z->w_list.a[wk].x_end < s; wk++); + if(wk < min_w || wk > max_w) return -1; + //s >= w_list.a[wk].x_start && s <= w_list.a[wk].x_end + if(wk != ovlp_cur_wid(*p)) {//xk is global, while ck is local + xk = z->w_list.a[wk].x_start; ck = 0; + } + + while(wk <= max_w && z->w_list.a[wk].x_start < e) {///[s, e) + m = &(z->w_list.a[wk]); + ws = m->x_start; we = m->x_end+1; + os = MAX(s, ws); oe = MIN(e, we); + ovlp = ((oe>os)? (oe-os):0); + + if(ovlp) { + xl = m->x_end+1-m->x_start; + yl = m->y_end+1-m->y_start; + if((is_ualn_win((*m))) || (is_est_aln((*m)))) { + if(is_ualn_win((*m))) { //unmapped + werr = gen_err_unaligned(xl, yl); + } else { + werr = m->error;//shared window + } + if(ovlp < xl) { + werr = (((double)ovlp)/((double)xl))*((double)werr); + } + //skip the whole window + err += werr; xk = m->x_end+1; ck = m->clen; + } else { + if(ovlp == xl) { + //skip the whole window + err += m->error; xk = m->x_end+1; ck = m->clen; + } else { + set_bit_extz_t(ez, (*z), wk); + err += retrieve_cigar_err(&ez, os, oe, &xk, &ck); + } + } + } + tot -= ovlp; + if(xk >= e) break;//[min_w, max_w] && [s, e) + wk++; if(wk > max_w) break; + xk = z->w_list.a[wk].x_start; ck = 0;//reset + } + assert(!tot); + ovlp_cur_wid(*p) = wk; ovlp_cur_xoff(*p) = xk; ovlp_cur_coff(*p) = ck; + return err; +} + +uint64_t gen_region_phase_robust_rr(overlap_region* ol, uint64_t *id_a, uint64_t id_n, uint64_t s, uint64_t e, uint64_t dp, ul_ov_t *c_idx, asg64_v *buf) +{ + if(!id_n) return id_n; + uint64_t k, m, mn, q[2], buf_n, rm_n, oid; int64_t err, msc, msc_k, msc_n; + overlap_region *z; ul_ov_t *p; buf->n = 0; kv_resize(uint64_t, *buf, dp); + for (k = buf_n = rm_n = 0; k < id_n; k++) { + p = &(c_idx[id_a[k]]); + q[0] = ol[ovlp_id(*p)].w_list.a[ovlp_min_wid(*p)].x_start; + q[1] = ol[ovlp_id(*p)].w_list.a[ovlp_max_wid(*p)].x_end+1; + if(q[0]<=s && q[1]>=e) { + kv_push(uint64_t, *buf, id_a[k]); + } + if(q[1] < e) rm_n++; + } + buf_n = buf->n; + assert(buf_n == dp);//not right + + if(buf_n > 0) { + for (k = 0, msc = INT32_MAX, msc_k = -1, msc_n = 0; k < buf_n; k++) { + p = &(c_idx[(uint32_t)buf->a[k]]); z = &(ol[ovlp_id(*p)]); + err = extract_sub_cigar_err_rr(z, s, e, p); + assert(err >= 0); + if(err < msc) { + msc = err; msc_k = k; msc_n = 1; + } else if(err == msc) { + msc_n++; + } + buf->a[k] |= (((uint64_t)err)<<32); + } + + if(msc_n == 1) { + p = &(c_idx[(uint32_t)buf->a[msc_k]]); + z = &(ol[ovlp_id(*p)]); mn = 1; + if(msc_k != 0) { + m = buf->a[msc_k]; + buf->a[msc_k] = buf->a[0]; + buf->a[0] = m; + } + } else { + for (k = mn = 0; k < buf_n && (int64_t)mn < msc_n; k++) { + p = &(c_idx[(uint32_t)buf->a[k]]); + z = &(ol[ovlp_id(*p)]); + if((buf->a[k]>>32) == (uint64_t)msc) { + if(mn != k) { + m = buf->a[k]; + buf->a[k] = buf->a[mn]; + buf->a[mn] = m; + } + mn++; + } + } + } + // fprintf(stderr, "[M::%s] buf_n::%ld, msc_n::%ld, mn::%lu\n", __func__, buf_n, msc_n, mn); + for (k = 0; k < buf_n; k++) { + oid = ovlp_id((c_idx[(uint32_t)buf->a[k]])); + buf->a[k] >>= 32; buf->a[k] <<= 32; buf->a[k] |= oid; + // ol[oid].overlapLen = k;//no need to set + // if(s == 158482) fprintf(stderr, "k->%ld::oid->%ld[M::%s::utg%.6dl] pos::[%lu, %lu)\n", k, oid, __func__, + // (int32_t)ol[oid].y_id+1, s, e); + } + + for (k = 0; k < mn; k++) {///best alignment + z = &(ol[id_get(buf->a[k])]); + // reassign_sec_err(ol, ovidx, buf, k); + // push_sec_aln(z, s, e, 0); + push_sec_aln_robust(z, s, e, 0); + } + + for (k = mn; k < buf_n; k++) { + z = &(ol[id_get(buf->a[k])]); + // push_sec_aln(z, s, e, ((buf->a[k]&id_set)?(0):(err_get(buf->a[k])-msc))); + push_sec_aln_robust(z, s, e, (err_get(buf->a[k])-msc)); + } + + // for (k = 0; k < buf_n; k++) { + // ol[id_get(buf->a[k])].overlapLen = (uint32_t)-1; + // } + } + + + if(rm_n) { + for (k = m = 0; k < id_n; k++) { + p = &(c_idx[id_a[k]]); + q[1] = ol[ovlp_id(*p)].w_list.a[ovlp_max_wid(*p)].x_end+1; + if(q[1] < e) continue; + id_a[m++] = id_a[k]; + } + id_n = m; + } + return id_n; +} + int64_t infer_rovlp(ul_ov_t *li, ul_ov_t *lj, uc_block_t *bi, uc_block_t *bj, All_reads *ridx, ma_ug_t *ug) { @@ -16023,11 +16178,11 @@ void region_phase(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t for (k = idx->n = c_idx->n = 0; k < on; k++) { z = &(ol->list[k]); zwn = z->w_list.n; // prt_overlap_region_stat(z, ulid); - // if(ulid == 35437 && z->y_id == 109111) { + // if(/**ulid == 35437 &&**/ z->y_id == 15199 || z->y_id == 31315) { // fprintf(stderr, "+0+[M::%s::utg%.6dl::%c] ulid::%ld, all::%u, non-best::%ld, best::%u\n", __func__, // (int32_t)z->y_id+1, "+-"[z->y_pos_strand], ulid, z->overlapLen, zwn, z->align_length); - // prt_overlap_region_stat(&(ol->list[k])); - // prt_overlap_region_phase_stat(&(ol->list[k])); + // prt_overlap_region_stat(&(ol->list[k]), ulid); + // // prt_overlap_region_phase_stat(&(ol->list[k]), ulid); // } // z->align_length = z->overlapLen = z->x_pos_e+1-z->x_pos_s; z->align_length = 0; z->overlapLen = (uint32_t)-1; @@ -16053,6 +16208,7 @@ void region_phase(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t ovlp_cur_wid(*cp) = w[0]; ///cur id of windows ovlp_cur_xoff(*cp) = z->w_list.a[w[0]].x_start; ///cur xpos ovlp_cur_coff(*cp) = 0; ///cur cigar off in cur window + ovlp_bd(*cp) = 0; } q[0] = z->w_list.a[i].x_start; q[1] = z->w_list.a[i].x_end; @@ -16073,6 +16229,7 @@ void region_phase(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t ovlp_cur_wid(*cp) = w[0]; ///cur id of windows ovlp_cur_xoff(*cp) = z->w_list.a[w[0]].x_start; ///cur xpos ovlp_cur_coff(*cp) = 0; ///cur cigar off in cur window + ovlp_bd(*cp) = 0; } // if(ulid == 35437 && z->y_id == 109111) { // fprintf(stderr, "+1+[M::%s::utg%.6dl::%c] ulid::%ld, all::%u, non-best::%ld, best::%u\n", __func__, @@ -16147,9 +16304,11 @@ void region_phase(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t // } assert(zwn <= z->overlapLen); z->align_length = z->overlapLen - zwn; - // prt_overlap_region_phase_stat(z, ulid); - // fprintf(stderr, "[M::%s::utg%.6dl::%c] all::%u, non-best::%ld, best::%u\n", __func__, - // (int32_t)z->y_id+1, "+-"[z->y_pos_strand], z->overlapLen, zwn, z->align_length); + // /**if(ulid == 35437 && z->y_id == 15199 || z->y_id == 31315)**/ { + // prt_overlap_region_phase_stat(z, ulid); + // fprintf(stderr, "[M::%s::utg%.6dl::%c] all::%u, non-best::%ld, best::%u\n\n", __func__, + // (int32_t)z->y_id+1, "+-"[z->y_pos_strand], z->overlapLen, zwn, z->align_length); + // } // prt_overlap_region_phase_stat(&(ol->list[k])); // z = &(ol->list[k]); // if(z->align_length == z->overlapLen) {///prefer alignments without any trans hit @@ -16158,6 +16317,1269 @@ void region_phase(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t } } + +void rphase_rr(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t *uopt, kv_ul_ov_t *c_idx, asg64_v* idx, asg64_v* buf, int64_t ulid, int64_t bd, int64_t cal_phase) +{ + int64_t on = ol->length, k, i, zwn, q[2], t[2], w[2]; + uint64_t m; overlap_region *z; ul_ov_t *cp; + kv_resize(uint64_t, *idx, (ol->length<<1)); + kv_resize(ul_ov_t, *c_idx, ol->length); + for (k = idx->n = c_idx->n = 0; k < on; k++) { + z = &(ol->list[k]); zwn = z->w_list.n; + z->align_length = 0; z->overlapLen = (uint32_t)-1; + z->non_homopolymer_errors = 0; + if(!zwn) continue; + q[0] = q[1] = t[0] = t[1] = w[0] = w[1] = INT32_MIN; + for (i = 0; i < zwn; i++) { + if((z->w_list.a[i].x_start==(q[1]+1)) && ((z->w_list.a[i].y_start==(t[1]+1)))) { + q[1] = z->w_list.a[i].x_end; + t[1] = z->w_list.a[i].y_end; + w[1] = i; + } else { + if(q[0] != INT32_MIN) { + q[0] += bd; q[1] -= bd; + if(q[1] >= q[0]) { + m = ((uint64_t)q[0])<<1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + m = (((uint64_t)q[1])<<1)+1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + + kv_pushp(ul_ov_t, *c_idx, &cp); + ovlp_id(*cp) = k; ///ovlp id + ovlp_min_wid(*cp) = w[0]; ///beg id of windows + ovlp_max_wid(*cp) = w[1]; ///end id of windows + ovlp_cur_wid(*cp) = w[0]; ///cur id of windows + ovlp_cur_xoff(*cp) = z->w_list.a[w[0]].x_start; ///cur xpos + ovlp_cur_coff(*cp) = 0; ///cur cigar off in cur window + ovlp_bd(*cp) = bd; + } + } + + q[0] = z->w_list.a[i].x_start; q[1] = z->w_list.a[i].x_end; + t[0] = z->w_list.a[i].y_start; t[1] = z->w_list.a[i].y_end; + w[0] = i; w[1] = i; + } + } + if(q[0] != INT32_MIN) { + q[0] += bd; q[1] -= bd; + if(q[1] >= q[0]) { + m = ((uint64_t)q[0])<<1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + m = (((uint64_t)q[1])<<1)+1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + + kv_pushp(ul_ov_t, *c_idx, &cp); + ovlp_id(*cp) = k; ///ovlp id + ovlp_min_wid(*cp) = w[0]; ///beg id of windows + ovlp_max_wid(*cp) = w[1]; ///end id of windows + ovlp_cur_wid(*cp) = w[0]; ///cur id of windows + ovlp_cur_xoff(*cp) = z->w_list.a[w[0]].x_start; ///cur xpos + ovlp_cur_coff(*cp) = 0; ///cur cigar off in cur window + ovlp_bd(*cp) = bd; + } + } + } + radix_sort_bc64(idx->a, idx->a+idx->n); + + int64_t srt_n = idx->n, dp, old_dp, beg, end; + for (i = k = 0, dp = old_dp = 0, beg = 0, end = -1; i < srt_n; ++i) {///[beg, end) but coordinates in idx is [, ] + ///if idx->a.a[] is qe + old_dp = dp; + if ((idx->a[i]>>32)&1) { + --dp; end = (idx->a[i]>>33)+1; + }else { + //meet a new overlap; the overlaps are pushed by the x_pos_s + ++dp; end = (idx->a[i]>>33); + kv_push(uint64_t, *idx, ((uint32_t)idx->a[i])); + } + if((end > beg) && (old_dp >= 2)) { + idx->n = srt_n + gen_region_phase_robust_rr(ol->list, idx->a+srt_n, idx->n-srt_n, beg, end, old_dp, c_idx->a, buf); + } + beg = end; + } + + if(cal_phase) { + for (k = 0; k < on; k++) { + z = &(ol->list[k]); + z->overlapLen = z->x_pos_e+1-z->x_pos_s; + z->non_homopolymer_errors = 0; zwn = 0; + for (i = m = 0; i < z->align_length; i++) { + z->w_list.a[m] = z->w_list.a[z->w_list.n+i]; + if(z->w_list.a[m].clen > 0) { + z->non_homopolymer_errors += z->w_list.a[m].clen; + zwn += z->w_list.a[m].x_end-z->w_list.a[m].x_start; + } + m++; + } + z->w_list.n = m; + assert(zwn <= z->overlapLen); + z->align_length = z->overlapLen - zwn; + } + } +} + +int64_t gen_aln_ul_ov_t(int64_t in_id, int64_t tl, overlap_region *in, ul_ov_t *ou) +{ + int64_t zwn; memset(ou, 0, sizeof((*ou))); + zwn = in->w_list.n; if(!zwn) return 0; + + ///in_id -> ovlp id + ou->qn = in_id; ou->tn = in->y_id; + ou->rev = in->y_pos_strand; + ou->qs = in->w_list.a[0].x_start; + ou->qe = in->w_list.a[zwn-1].x_end+1; + + if(!(ou->rev)) { + ou->ts = in->w_list.a[0].y_start; + ou->te = in->w_list.a[zwn-1].y_end+1; + } else { + ou->ts = tl-(in->w_list.a[zwn-1].y_end+1); + ou->te = tl-in->w_list.a[0].y_start; + } + return 1; +} + +uint32_t push_emask_flt(kv_emask_t *in, uint64_t *flt, uint64_t flt_n, uint64_t qn, kv_ul_ov_t *res) +{ + uint32_t k = 0, z = 0, tn, rn0 = res->n; ul_ov_t *p; + while (k < in->n && z < flt_n) { + if (in->a[k].tn < (flt[z]>>32)) { + k++; + } else if((flt[z]>>32) < in->a[k].tn) { + z++; + } else {///in->a[k].tn == (flt[z]>>32) + for (tn = in->a[k].tn; k < in->n && in->a[k].tn == tn; k++) { + kv_pushp(ul_ov_t, *res, &p); + p->qn = qn; p->qs = in->a[k].qs; p->qe = in->a[k].qe; + p->tn = in->a[k].tn; p->ts = in->a[k].ts; p->te = in->a[k].te; + p->rev = in->a[k].rev; p->el = in->a[k].full; p->sec = k; + } + for (; z < flt_n && (flt[z]>>32) == tn; z++); + } + } + return res->n-rn0; +} + +#define is_ul_ov_pe(z, zn, i, mm) ((((i)+1)<(zn))&&((z)[(i)+1].qn==(z)[(i)].qn)&&((z)[(i)+1].tn==(z)[(i)].tn)&&((z)[(i)+1].rev==(z)[(i)].rev)\ + &&((z)[(i)+1].sec==(z)[(i)].sec+1)&&(((z)[(i)].sec)<(mm).a[(z)[(i)].qn].n)&&(((z)[(i)+1].sec)<(mm).a[(z)[(i)+1].qn].n)\ + &&((mm).a[(z)[(i)].qn].a[((z)[(i)].sec)].pe)&&((mm).a[(z)[(i)+1].qn].a[((z)[(i)+1].sec)].pe)\ + &&((mm).a[(z)[(i)].qn].a[((z)[(i)].sec)].dir==0)&&((mm).a[(z)[(i)+1].qn].a[((z)[(i)+1].sec)].dir==1)) + +#define is_exact_ov(z, mm) (((((z).sec)<(mm).a[(z).qn].n))&&(!((mm).a[(z).qn].a[((z).sec)].pe))\ + &&((mm).a[(z).qn].a[((z).sec)].el==(uint32_t)-1)) + + +///ref_n <= 2 +uint64_t is_cover_ul_ov_t(ma_ug_t *ug, double diff, ul_ov_t *ref, uint64_t ref_n, ul_ov_t *in) +{ + if((!check_ul_ov_t_consist(in, &(ref[0]), ug->g->seq[in->qn].len, ug->g->seq[in->tn].len, diff))) return 0; + if((ref_n > 1) && (!check_ul_ov_t_consist(in, &(ref[ref_n-1]), ug->g->seq[in->qn].len, ug->g->seq[in->tn].len, 0.06))) return 0; + uint64_t i; int64_t sql, stl, os, oe, ovlp; + sql = in->qe - in->qs; stl = in->te - in->ts; + for (i = 0; i < ref_n && sql > 0 && stl > 0; i++) { + os = MAX(in->qs, ref[i].qs); oe = MIN(in->qe, ref[i].qe); + ovlp = ((oe>os)?(oe-os):(0)); sql -= ovlp; + + os = MAX(in->ts, ref[i].ts); oe = MIN(in->te, ref[i].te); + ovlp = ((oe>os)?(oe-os):(0)); stl -= ovlp; + } + if((sql > 256) && (sql > ((in->qe - in->qs)*0.06))) return 0; + if((sql <= 256) && (sql > ((in->qe - in->qs)*0.6))) return 0; + if((stl > 256) && (stl > ((in->te - in->ts)*0.06))) return 0; + if((stl <= 256) && (stl > ((in->te - in->ts)*0.6))) return 0; + return 1; +} + +uint64_t dedup_src_shared(ma_ug_t *ug, ul_ov_t *ta, uint64_t tn, ul_ov_t *qa, uint64_t qn, idx_emask_t *mm) +{ + uint32_t ti, qi, ts, te, id, k, l; + ti = qi = 0; + while (ti < tn && qi < qn) { + if (ta[ti].tn < qa[qi].tn) { + ti++; + } else if(qa[qi].tn < ta[ti].tn) { + qi++; + } else {///ta[ti].tn == qa[qi].tn + id = ta[ti].tn; + for (ts = ti; ti < tn && ta[ti].tn == id; ti++); te = ti; + for (; qi < qn && qa[qi].tn == id; qi++) { + for (k = ts; k < te; k++) { + l = 1; + if(is_ul_ov_pe(ta, te, k, (*mm))) { + l++; k++; + } + if(is_cover_ul_ov_t(ug, 0.06, ta+k+1-l, l, &(qa[qi]))) break; + } + if(k < te) qa[qi].qn = qa[qi].tn = (uint32_t)-1; + } + } + } + for (qi = l = 0; qi < qn; qi++) { + if(qa[qi].tn == (uint32_t)-1) continue; + qa[l++] = qa[qi]; + } + return l; +} + +void dedup_src_shared1(ma_ug_t *ug, kv_ul_ov_t *res, uint64_t tocc, uint64_t qocc, idx_emask_t *mm) +{ + if((!tocc) || (!qocc)) return; + uint32_t ti, qi, tn, qn, rn = res->n, id, ts, te, k, l; + ti = res->n - tocc - qocc; qi = res->n - qocc;//t first, and then q + tn = ti + tocc; qn = qi + qocc; kv_resize(ul_ov_t, *res, (rn+tn));///the buf size should be at least tn + ts = ti; te = tn; + while (ti < tn && qi < qn) { + if (res->a[ti].tn < res->a[qi].tn) { + ti++; kv_push(ul_ov_t, *res, res->a[ti]); + } else if(res->a[qi].tn < res->a[ti].tn) { + qi++; kv_push(ul_ov_t, *res, res->a[qi]); + } else {///ta[ti].tn == qa[qi].tn + id = res->a[ti].tn; + for (; ti < tn && res->a[ti].tn == id; ti++) { + kv_push(ul_ov_t, *res, res->a[ti]); + } + + for (; qi < qn && res->a[qi].tn == id; qi++) { + for (k = ts; k < te; k++) { + l = k; + if(is_ul_ov_pe(res->a, te, k, (*mm))) k++; + if(is_cover_ul_ov_t(ug, 0.06, res->a+l, k+1-l, &(res->a[qi]))) break; + } + if(k >= te) kv_push(ul_ov_t, *res, res->a[qi]); + } + } + } + if(res->n > rn) { + l = res->n; res->n = rn - tocc - qocc; + for (k = rn; k < l; k++) res->a[res->n++] = res->a[k]; + } +} + +#define aln2ov(in, ou, tl) \ + {(ou).qn=(in).x_id;(ou).tn=(in).y_id;(ou).rev=(in).y_pos_strand;\ + (ou).qs=(in).x_pos_s;(ou).qe=(in).x_pos_e+1;\ + if((in).y_pos_strand){(ou).ts=(tl)-(in).y_pos_e-1;(ou).te=(tl)-(in).y_pos_s;}\ + else {(ou).ts=(in).y_pos_s;(ou).te=(in).y_pos_e+1;}} + +uint64_t cal_x_ul_ovlp(ma_ug_t *ug, overlap_region *q, overlap_region *t, ul_ov_t *res) +{ + ul_ov_t qo, to; uint32_t ql, tl, os, oe, rqs, rqe, rts, rte; + ql = ug->g->seq[q->y_id].len; tl = ug->g->seq[t->y_id].len; + aln2ov((*q), qo, ql); aln2ov((*t), to, tl); memset(res, 0, sizeof(*res)); + os = MAX(qo.qs, to.qs); oe = MIN(qo.qe, to.qe); + if(oe <= os) return 0; + if(infer_se(qo.qs, qo.qe, qo.ts, qo.te, qo.rev, os, oe, &rqs, &rqe) && + infer_se(to.qs, to.qe, to.ts, to.te, to.rev, os, oe, &rts, &rte)) { + if(rqe > rqs && rte > rts) { + res->qn = qo.tn; res->tn = to.tn; res->rev = ((qo.rev==to.rev)?0:1); + res->qs = rqs; res->qe = rqe; res->ts = rts; res->te = rte; + return 1; + } + } + return 0; +} + +uint64_t check_mask_exist(ma_ug_t *ug, overlap_region *q, overlap_region *t, kv_ul_ov_t *ov_db, idx_emask_t *mm, double len_diff, uint64_t *is_exact, uint64_t *ovdb_idx) +{ + ul_ov_t rr, *a; uint64_t k, eid, an, ss; (*is_exact) = 0; (*ovdb_idx) = (uint64_t)-1; + if(!cal_x_ul_ovlp(ug, q, t, &rr)) return 0; + ss = q->overlapLen; + if(ss != (uint64_t)-1) {///if q does not have mask + eid = rr.tn; a = ov_db->a + ss; an = q->x_pos_strand; + for (k = 0; k < an && a[k].tn < eid; k++); + for (; k < an && a[k].tn == eid; k++) { + if((!check_ul_ov_t_consist(&rr, &(a[k]), ug->g->seq[rr.qn].len, ug->g->seq[rr.tn].len, len_diff))) continue; + if(is_ul_ov_pe(a, an, k, (*mm))) { + k++; if((!check_ul_ov_t_consist(&rr, &(a[k]), ug->g->seq[rr.qn].len, ug->g->seq[rr.tn].len, len_diff))) continue; + } + if(is_exact_ov(a[k],(*mm))) (*is_exact) = 1; + (*ovdb_idx) = ss + k; + return 1; + } + } + + k = rr.qn; rr.qn = rr.tn; rr.tn = k; + k = rr.qs; rr.qs = rr.ts; rr.ts = k; + k = rr.qe; rr.qe = rr.te; rr.te = k; + ss = t->overlapLen; + if(ss != (uint64_t)-1) {///if t does not have mask + eid = rr.tn; a = ov_db->a + ss; an = t->x_pos_strand; + for (k = 0; k < an && a[k].tn < eid; k++); + for (; k < an && a[k].tn == eid; k++) { + if((!check_ul_ov_t_consist(&rr, &(a[k]), ug->g->seq[rr.qn].len, ug->g->seq[rr.tn].len, len_diff))) continue; + if(is_ul_ov_pe(a, an, k, (*mm))) { + k++; if((!check_ul_ov_t_consist(&rr, &(a[k]), ug->g->seq[rr.qn].len, ug->g->seq[rr.tn].len, len_diff))) continue; + } + if(is_exact_ov(a[k],(*mm))) (*is_exact) = 1; + (*ovdb_idx) = ss + k; + return 1; + } + } + return 0; +} + +void push_consist_ul_ovlps(ma_ug_t *ug, kv_ul_ov_t *ov_db, uint64_t s, uint64_t e, double len_diff, ul_ov_t *ref, idx_emask_t *mm, kv_ul_ov_t *out, uint64_t flip_res, uint64_t *is_exact) +{ + (*is_exact) = 0; + if(s >= e) return; + uint64_t k, z, ql = ug->g->seq[ref->qn].len, tl = ug->g->seq[ref->tn].len; ul_ov_t *p; + for (k = s; k < e && ov_db->a[k].tn < ref->tn; k++); + for (; k < e && ov_db->a[k].tn == ref->tn; k++) { + if((!check_ul_ov_t_consist(ref, &(ov_db->a[k]), ql, tl, len_diff))) continue; + z = k; + if(is_ul_ov_pe(ov_db->a, ov_db->n, k, (*mm))) { + k++; if((!check_ul_ov_t_consist(ref, &(ov_db->a[k]), ql, tl, len_diff))) continue; + } + if(is_exact_ov(ov_db->a[k], (*mm))) { + (*is_exact) = 1; return; + } + + for (; z <= k; z++) { + kv_pushp(ul_ov_t, *out, &p); + *p = ov_db->a[k]; + if(flip_res) { + p->qn = ov_db->a[k].tn; p->tn = ov_db->a[k].qn; + p->qs = ov_db->a[k].ts; p->qe = ov_db->a[k].te; + p->ts = ov_db->a[k].qs; p->te = ov_db->a[k].qe; + } + } + } +} + +uint64_t cal_no_bd_coor(ul_ov_t *in, ul_ov_t *ou, idx_emask_t *mm, int64_t bd) +{ + int64_t qs, qe, ts, te; emask_t *z; + qs = in->qs; qe = in->qe; ts = in->ts; te = in->te; + if(in->sec != ((uint32_t)(0x3fffffff))) {///might be an end mask; no need shrink + if((in->sec < mm->a[in->qn].n) && (in->tn == mm->a[in->qn].a[in->sec].tn) && (in->rev == mm->a[in->qn].a[in->sec].rev) && + (in->qs == mm->a[in->qn].a[in->sec].qs) && (in->qe == mm->a[in->qn].a[in->sec].qe) && + (in->ts == mm->a[in->qn].a[in->sec].ts) && (in->te == mm->a[in->qn].a[in->sec].te)) { + z = &(mm->a[in->qn].a[in->sec]); assert(z->el != ((uint32_t)-1)); + if(z->dir == 0) { + qe -= (z->el>>1); te -= (z->el>>1); + } else { + qs += (z->el>>1); ts += (z->el>>1); + } + } else { + assert((in->sec < mm->a[in->tn].n) && (in->qn == mm->a[in->tn].a[in->sec].tn) && (in->rev == mm->a[in->tn].a[in->sec].rev) && + (in->qs == mm->a[in->tn].a[in->sec].ts) && (in->qe == mm->a[in->tn].a[in->sec].te) && + (in->ts == mm->a[in->tn].a[in->sec].qs) && (in->te == mm->a[in->tn].a[in->sec].qe)); + z = &(mm->a[in->tn].a[in->sec]); assert(z->el != ((uint32_t)-1)); + if(z->dir == 1) { + qe -= (z->el>>1); te -= (z->el>>1); + } else { + qs += (z->el>>1); ts += (z->el>>1); + } + } + return 1; + } else {///shrink anyway + qs += bd; qe -= bd; ts += bd; te -= bd; + } + if(qe > qs && te > ts) { + (*ou) = (*in); + ou->qs = qs; ou->qe = qe; ou->ts = ts; ou->te = te; + return 1; + } + return 0; +} + +typedef struct { + int64_t qi, qs, qe; + int64_t ti; + int64_t qis, qie, tis, tie; + int64_t wi, wn; + int64_t tot, lc, cql, ctl, ci, lerr; + overlap_region *z; +} pe_cigar_iter_t; + +#define citer_end(m) ((m).qi>=(m).qe) + +/** +inline void pop_citer(pe_cigar_iter_t *it) +{ + it->lc = it->lerr = -1; it->cql = it->ctl = 0; + it->qis = it->qie; it->tis = it->tie; + if(it->wi >= it->wn) return; + int64_t ws, we, os, oe, ovlp, ql, tl, werr; + window_list *m; bit_extz_t ez; + + while (it->wi < it->wn) { + m = &(it->z->w_list.a[it->wi]); + ws = m->x_start; we = m->x_end+1; + os = MAX(it->qs, ws); oe = MIN(it->qe, we); + ovlp = ((oe>os)? (oe-os):0); + + if(ovlp) { + ql = m->x_end+1-m->x_start; + tl = m->y_end+1-m->y_start; + if((is_ualn_win((*m))) || (is_est_aln((*m)))) { + it->lc = 4;//not an ordinary cigar + if(is_ualn_win((*m))) { //unmapped + werr = gen_err_unaligned(ql, tl); + } else { + werr = m->error;//shared window + if(!werr) it->lc = 0;///treat it as match + } + it->lerr = werr; + if(ovlp < ql) { + werr = (((double)ovlp)/((double)ql))*((double)werr); + } + //skip the whole window + it->tot += werr; + it->qis = it->qi; it->qi = it->qie = m->x_end+1; it->cql = it->qie - it->qis; + it->tis = it->ti; it->ti = it->tie = m->x_start; it->ctl = it->tie - it->tis; + it->ci = 0; it->wi++; + return; + } else { + if(it->qi < m->x_start || it->ti < m->y_start) { + it->qi = m->x_start; it->ti = m->y_start; it->ci = 0; + } + set_bit_extz_t(ez, (*(it->z)), it->wi); + // if(ovlp == ql) { + // //skip the whole window + // err += m->error; xk = m->x_end+1; ck = m->clen; + // } else { + // set_bit_extz_t(ez, (*z), wk); + // err += retrieve_cigar_err(&ez, os, oe, &xk, &ck); + // } + } + } + } +} +**/ + +int64_t get_pe_diff(overlap_region *q, uint64_t *qmask, uint64_t qmask_n, overlap_region *t, uint64_t *tmask, uint64_t tmask_n, int64_t bd) +{ + int64_t s, e; overlap_region *z; + pe_cigar_iter_t qi, ti; + z = q; if(!(z->w_list.n)) return 0; + s = z->w_list.a[0].x_start; e = z->w_list.a[z->w_list.n-1].x_end+1; + s += bd; e -= bd; if(s >= e) return 0; + memset(&qi, 0, sizeof(qi)); + qi.z = q; qi.qs = s; qi.qe = e; + qi.qi = q->w_list.a[0].x_start; + qi.ti = q->w_list.a[0].y_start; + qi.wi = 0; qi.wn = q->w_list.n; + qi.tot = 0; qi.ci = 0; qi.lc = -1; qi.cql = qi.ctl = 0; + qi.qis = qi.qie = qi.qi; + qi.tis = qi.tie = qi.ti; + qi.lerr = -1; + + z = t; if(!(z->w_list.n)) return 0; + s = z->w_list.a[0].x_start; e = z->w_list.a[z->w_list.n-1].x_end+1; + s += bd; e -= bd; if(s >= e) return 0; + memset(&ti, 0, sizeof(ti)); + ti.z = q; ti.qs = s; ti.qe = e; + ti.qi = t->w_list.a[0].x_start; + ti.ti = t->w_list.a[0].y_start; + ti.wi = 0; ti.wn = t->w_list.n; + ti.tot = 0; ti.ci = 0; ti.lc = -1; ti.cql = ti.ctl = 0; + ti.qis = ti.qie = ti.qi; + ti.tis = ti.tie = ti.ti; + ti.lerr = -1; + + while((!citer_end(qi)) && (!citer_end(ti))) { + if(qi.qi == ti.qi) {///check if it is been masked + + } else if((qi.lc == 0) && (ti.lc == 0)) {///check if it is been masked + + } + + if(qi.qi < ti.qi) { + + } if(ti.qi < qi.qi) { + + } else {//qi.qi == ti.qi + + } + } + return 0; +} + +void retrieve_cigar_xcoord(bit_extz_t *ez, int64_t is, int64_t ie, int64_t *yk, int64_t *xk, int64_t *ck, int64_t *rxs, int64_t *rxe) +{ + if(!ez->cigar.n) return; + int64_t cn = ez->cigar.n, op; int64_t ws, we, ws0, we0; + if(((*ck) < 0) || ((*ck) > cn)) {//(*ck) == cn is allowed + (*ck) = 0; (*yk) = ez->ps; (*xk) = ez->ts; + } + + while ((*ck) > 0 && (*yk) > is) { + --(*ck); + op = ez->cigar.a[(*ck)]>>14; + if(op!=3) (*yk) -= (ez->cigar.a[(*ck)]&(0x3fff)); + if(op!=2) (*xk) -= (ez->cigar.a[(*ck)]&(0x3fff)); + } + + //some cigar will span s or e + while ((*ck) < cn && (*yk) < ie) {//[s, e) + ws = (*yk); ws0 = (*xk); + op = ez->cigar.a[(*ck)]>>14; + if(op!=3) (*yk) += (ez->cigar.a[(*ck)]&(0x3fff)); + if(op!=2) (*xk) += (ez->cigar.a[(*ck)]&(0x3fff)); + we = (*yk); we0 = (*xk); + if(ws >= is && we <= ie) { + if((*rxs) > ws0) (*rxs) = ws0; + if((*rxe) < we0) (*rxe) = we0; + } + (*ck)++; + } +} + +uint64_t extract_xcoordates0(overlap_region *z, int64_t ys, int64_t ye, int64_t *rxs, int64_t *rxe, ul_ov_t *p) +{ + int64_t wk = ovlp_cur_wid(*p), yk = ovlp_cur_xoff(*p), xk = ovlp_id(*p), ck = ovlp_cur_coff(*p); + int64_t min_w = ovlp_min_wid(*p), max_w = ovlp_max_wid(*p);//[min_w, max_w] + bit_extz_t ez; window_list *m; (*rxs) = INT32_MAX; (*rxe) = -1; + if(ys >= ye) return 0; + + int64_t ws, we, os, oe, ovlp, yl, tot = ye - ys; + if(wk < min_w || wk > max_w) wk = min_w; + for (; wk >= min_w && z->w_list.a[wk].y_start > ys; wk--); + if(wk < min_w || wk > max_w) return -1; + for (; wk <= max_w && z->w_list.a[wk].y_end < ys; wk++); + if(wk < min_w || wk > max_w) return -1; + //s >= w_list.a[wk].x_start && s <= w_list.a[wk].x_end + if(wk != ovlp_cur_wid(*p)) {//xk is global, while ck is local + yk = z->w_list.a[wk].y_start; + xk = z->w_list.a[wk].x_start; + ck = 0; + } + + while(wk <= max_w && z->w_list.a[wk].y_start < ye) {///[s, e) + m = &(z->w_list.a[wk]); + ws = m->y_start; we = m->y_end+1; + os = MAX(ys, ws); oe = MIN(ye, we); + ovlp = ((oe>os)? (oe-os):0); + + if(ovlp) { + yl = m->y_end+1-m->y_start; + if((is_ualn_win((*m))) || (is_est_aln((*m)))) { + if(ovlp >= yl) { + if((*rxs) > m->x_start) (*rxs) = m->x_start; + if((*rxe) < (m->x_end+1)) (*rxe) = m->x_end+1; + } + //skip the whole window + yk = m->y_end+1; xk = m->x_end+1; ck = m->clen; + } else { + if(ovlp == yl) { + //skip the whole window + yk = m->y_end+1; xk = m->x_end+1; ck = m->clen; + if((*rxs) > m->x_start) (*rxs) = m->x_start; + if((*rxe) < (m->x_end+1)) (*rxe) = m->x_end+1; + } else { + set_bit_extz_t(ez, (*z), wk); + retrieve_cigar_xcoord(&ez, os, oe, &yk, &xk, &ck, rxs, rxe); + } + } + } + tot -= ovlp; + if(yk >= ye) break;//[min_w, max_w] && [s, e) + wk++; if(wk > max_w) break; + yk = z->w_list.a[wk].y_start; xk = z->w_list.a[wk].x_start; ck = 0;//reset + } + assert(!tot); + ovlp_cur_wid(*p) = wk; ovlp_cur_xoff(*p) = yk; ovlp_id(*p) = xk; ovlp_cur_coff(*p) = ck; + if((*rxe) > (*rxs)) return 1; + return 0; +} + +uint64_t extract_xcoordates(overlap_region *z, uint64_t *a, int64_t a_n, asg64_v *b) +{ + int64_t i, zwn, q[2], t[2], w[2], bn = b->n, a_i, a_z, as, ae, is, ie, os, oe, ovlp; + ul_ov_t m; int64_t qs, qe; + zwn = z->w_list.n; if(!zwn) return 0; + q[0] = q[1] = t[0] = t[1] = w[0] = w[1] = INT32_MIN; memset(&m, 0, sizeof(m)); + for (i = a_i = 0; i < zwn; i++) { + if((z->w_list.a[i].x_start==(q[1]+1)) && ((z->w_list.a[i].y_start==(t[1]+1)))) { + q[1] = z->w_list.a[i].x_end; + t[1] = z->w_list.a[i].y_end; + w[1] = i; + } else { + if(q[0] != INT32_MIN) { + ovlp_id(m) = z->w_list.a[w[0]].x_start; ///cur xpos + ovlp_min_wid(m) = w[0]; ///beg id of windows + ovlp_max_wid(m) = w[1]; ///end id of windows + ovlp_cur_wid(m) = w[0]; ///cur id of windows + ovlp_cur_xoff(m) = z->w_list.a[w[0]].y_start; ///cur ypos + ovlp_cur_coff(m) = 0; ///cur cigar off in cur window + + is = t[0]; ie = t[1]+1; + for (a_z = a_i; a_z >= 0; a_z--) { + as = a[a_z]>>32; ae = (uint32_t)a[a_z]; + if(ae <= is) break; + } + if(a_z < 0) a_z = 0; + for (; a_z < a_n; a_z++) { + as = a[a_z]>>32; ae = (uint32_t)a[a_z]; + os = MAX(is, as); oe = MIN(ie, ae); + ovlp = ((oe>os)? (oe-os):0); + if(ovlp) { + if(extract_xcoordates0(z, os, oe, &qs, &qe, &m)) { + kv_push(uint64_t, *b, (((uint64_t)qs)<<32)|((uint64_t)qe)); + } + } + if(as >= ie) break; + } + a_i = a_z; + } + q[0] = z->w_list.a[i].x_start; q[1] = z->w_list.a[i].x_end; + t[0] = z->w_list.a[i].y_start; t[1] = z->w_list.a[i].y_end; + w[0] = i; w[1] = i; + } + } + + if(q[0] != INT32_MIN) { + ovlp_id(m) = z->w_list.a[w[0]].x_start; ///cur xpos + ovlp_min_wid(m) = w[0]; ///beg id of windows + ovlp_max_wid(m) = w[1]; ///end id of windows + ovlp_cur_wid(m) = w[0]; ///cur id of windows + ovlp_cur_xoff(m) = z->w_list.a[w[0]].y_start; ///cur ypos + ovlp_cur_coff(m) = 0; ///cur cigar off in cur window + + is = t[0]; ie = t[1]+1; + for (a_z = a_i; a_z >= 0; a_z--) { + as = a[a_z]>>32; ae = (uint32_t)a[a_z]; + if(ae <= is) break; + } + if(a_z < 0) a_z = 0; + for (; a_z < a_n; a_z++) { + as = a[a_z]>>32; ae = (uint32_t)a[a_z]; + os = MAX(is, as); oe = MIN(ie, ae); + ovlp = ((oe>os)? (oe-os):0); + if(ovlp) { + if(extract_xcoordates0(z, os, oe, &qs, &qe, &m)) { + kv_push(uint64_t, *b, (((uint64_t)qs)<<32)|((uint64_t)qe)); + } + } + if(as >= ie) break; + } + a_i = a_z; + } + return b->n - bn; +} + +void update_masks(asg64_v *in, overlap_region *qi, overlap_region *ti, int64_t bd) +{ + uint64_t in_n0 = in->n, in_n1, k, s, e; + int64_t zwn, i, q[2], t[2]; overlap_region *z; + for (k = 0; k < in_n0; k++) { + s = in->a[k]>>32; e = (uint32_t)in->a[k]; + kv_push(uint64_t, *in, s<<1); + kv_push(uint64_t, *in, (((e-1)<<1)|1)); + } + in_n1 = in->n; + + uint64_t m, start, end; int64_t dp, old_dp, dp_mask, old_dp_mask; + radix_sort_bc64(in->a+in_n0, in->a+in_n1); + for (k = in_n0, dp = start = 0; k < in_n1; k++) { + old_dp = dp; + if (in->a[k]&1) --dp; + else ++dp; + + if (old_dp < 2 && dp >= 2) {///old_dp < dp, b.a[j] is qs + start = in->a[k]>>1; + } else if (old_dp >= 2 && dp < 2){ + end = in->a[k]>>1; + if(end >= start) kv_push(uint64_t, *in, ((start<<32)|end)); + } + } + + for (m = 0, k = in_n1; k < in->n; k++) in->a[m++] = in->a[k]; + in->n = in_n0 = m; + + z = qi; + zwn = z->w_list.n; + if(zwn > 0) { + q[0] = q[1] = t[0] = t[1] = INT32_MIN; + for (i = 0; i < zwn; i++) { + if((z->w_list.a[i].x_start==(q[1]+1)) && ((z->w_list.a[i].y_start==(t[1]+1)))) { + q[1] = z->w_list.a[i].x_end; + t[1] = z->w_list.a[i].y_end; + } else { + if(q[0] != INT32_MIN) { + q[0] += bd; q[1] -= bd; + if(q[1] >= q[0]) { + m = ((uint64_t)q[0])<<1; m <<= 32; + kv_push(uint64_t, *in, m); + m = (((uint64_t)(q[1]))<<1)+1; m <<= 32; + kv_push(uint64_t, *in, m); + } + } + q[0] = z->w_list.a[i].x_start; q[1] = z->w_list.a[i].x_end; + t[0] = z->w_list.a[i].y_start; t[1] = z->w_list.a[i].y_end; + } + } + if(q[0] != INT32_MIN) { + q[0] += bd; q[1] -= bd; + if(q[1] >= q[0]) { + m = ((uint64_t)q[0])<<1; m <<= 32; + kv_push(uint64_t, *in, m); + m = (((uint64_t)(q[1]))<<1)+1; m <<= 32; + kv_push(uint64_t, *in, m); + } + } + } + + z = ti; + zwn = z->w_list.n; + if(zwn > 0) { + q[0] = q[1] = t[0] = t[1] = INT32_MIN; + for (i = 0; i < zwn; i++) { + if((z->w_list.a[i].x_start==(q[1]+1)) && ((z->w_list.a[i].y_start==(t[1]+1)))) { + q[1] = z->w_list.a[i].x_end; + t[1] = z->w_list.a[i].y_end; + } else { + if(q[0] != INT32_MIN) { + q[0] += bd; q[1] -= bd; + if(q[1] >= q[0]) { + m = ((uint64_t)q[0])<<1; m <<= 32; + kv_push(uint64_t, *in, m); + m = (((uint64_t)(q[1]))<<1)+1; m <<= 32; + kv_push(uint64_t, *in, m); + } + } + q[0] = z->w_list.a[i].x_start; q[1] = z->w_list.a[i].x_end; + t[0] = z->w_list.a[i].y_start; t[1] = z->w_list.a[i].y_end; + } + } + if(q[0] != INT32_MIN) { + q[0] += bd; q[1] -= bd; + if(q[1] >= q[0]) { + m = ((uint64_t)q[0])<<1; m <<= 32; + kv_push(uint64_t, *in, m); + m = (((uint64_t)(q[1]))<<1)+1; m <<= 32; + kv_push(uint64_t, *in, m); + } + } + } + + + for (k = 0; k < in_n0; k++) { + start = in->a[k]>>32; end = (uint32_t)in->a[k]; + if(end >= start) { + q[0] = start; q[1] = end; + m = ((uint64_t)q[0])<<1; m <<= 32; m |= 1; + kv_push(uint64_t, *in, m); + m = (((uint64_t)(q[1]))<<1)+1; m <<= 32; m |= 1; + kv_push(uint64_t, *in, m); + } + } + in_n1 = in->n; + + radix_sort_bc64(in->a+in_n0, in->a+in_n1); + for (k = in_n0, dp = old_dp = dp_mask = old_dp_mask = 0, start = 0, end = -1; k < in_n1; ++k) {///[beg, end) but coordinates in idx is [, ] + ///if idx->a.a[] is qe + old_dp = dp; old_dp_mask = dp_mask; + if ((in->a[k]>>32)&1) { + --dp; end = (in->a[k]>>33)+1; if((uint32_t)in->a[k]) dp_mask--; + }else { + //meet a new overlap; the overlaps are pushed by the x_pos_s + ++dp; end = (in->a[k]>>33); if((uint32_t)in->a[k]) dp_mask++; + } + if((end > start) && (old_dp >= 2) && old_dp_mask <= 0) { + kv_push(uint64_t, *in, ((start<<32)|end)); + } + start = end; + } + + for (m = 0, k = in_n1; k < in->n; k++) in->a[m++] = in->a[k]; + in->n = in_n0 = m; +} + +int64_t cal_paired_distance(ma_ug_t *ug, overlap_region *q, overlap_region *t, ul_ov_t *a, uint32_t a_n, +asg64_v *srt, asg64_v* buf1, idx_emask_t *mm, int64_t bd) +{ + uint64_t k, cn, *qa, *ta, *ca, m, qn, tn, rev, rev_n, tl, mt, qocc, tocc; ul_ov_t ou; memset((&ou), 0, sizeof(ou)); + srt->n = 0; kv_resize(uint64_t, *srt, (a_n<<1)); qa = srt->a; ta = qa + a_n; + for (k = cn = 0; k < a_n; k++) { + if(!cal_no_bd_coor(&(a[k]), &ou, mm, bd)) continue; + qa[cn] = (((uint64_t)ou.qs)<<32)|((uint64_t)ou.qe); + ta[cn] = (((uint64_t)ou.ts)<<32)|((uint64_t)ou.te); + cn++; + } + radix_sort_bc64(qa, qa+cn); ca = qa; rev = q->y_pos_strand; tl = ug->g->seq[q->y_id].len; + for (k = m = 0; k < cn; k++) { + if(m <= 0 || (((uint32_t)ca[m-1])) < (ca[k]>>32)) { + ca[m++] = ca[k]; + } else if(((uint32_t)ca[m-1]) < ((uint32_t)ca[k])) { + ca[m-1] += ((uint32_t)ca[k])-((uint32_t)ca[m-1]); + } + } + if(rev) { + rev_n = m>>1; tl = (tl<<32) + tl; + for (k = 0; k < rev_n; k++) { + mt = ca[k]; ca[k] = ca[m-k-1]; ca[m-k-1] = mt; + ca[k] = tl - ca[k]; ca[m-k-1] = tl - ca[m-k-1]; + } + if(m&1) ca[k] = tl - ca[k]; + } + qn = m; + + radix_sort_bc64(ta, ta+cn); ca = ta; rev = t->y_pos_strand; tl = ug->g->seq[t->y_id].len; + for (k = m = 0; k < cn; k++) { + if(m <= 0 || (((uint32_t)ca[m-1])) < (ca[k]>>32)) { + ca[m++] = ca[k]; + } else if(((uint32_t)ca[m-1]) < ((uint32_t)ca[k])) { + ca[m-1] += ((uint32_t)ca[k])-((uint32_t)ca[m-1]); + } + } + if(rev) { + rev_n = m>>1; tl = (tl<<32) + tl; + for (k = 0; k < rev_n; k++) { + mt = ca[k]; ca[k] = ca[m-k-1]; ca[m-k-1] = mt; + ca[k] = tl - ca[k]; ca[m-k-1] = tl - ca[m-k-1]; + } + if(m&1) ca[k] = tl - ca[k]; + } + tn = m; + + buf1->n = 0; qocc = tocc = 0; srt->n = 0; + if(qn > 0 && tn > 0) { + qocc = extract_xcoordates(q, qa, qn, buf1); + tocc = extract_xcoordates(t, ta, tn, buf1); + if((!qocc) || (!tocc)) qocc = tocc = 0; + } + update_masks(buf1, q, t, bd); + + return get_pe_diff(q, qa, qn, t, ta, tn, bd); +} + +void gen_mask_ovlp0(ma_ug_t *ug, overlap_region *a, uint32_t qi, uint32_t ti, kv_ul_ov_t *ov_db, kv_ul_ov_t *buf, idx_emask_t *mm, double len_diff, int64_t rlen, uint64_t ovdb_idx, asg64_v *coor_srt, +asg64_v* buf1, int64_t bd, ul_ov_t *res) +{ + ul_ov_t *ref, r0, r1; uint32_t bn = buf->n, s, e; uint64_t is_exact = 0; + overlap_region *q = &(a[qi]), *t = &(a[ti]); int64_t dis; + assert(q->y_id < t->y_id); assert(ovdb_idx < ov_db->n); + cal_x_ul_ovlp(ug, q, t, &r0); + r1 = r0; + r1.qn = r0.tn; r1.tn = r0.qn; + r1.qs = r0.ts; r1.qe = r0.te; + r1.ts = r0.qs; r1.te = r0.qe; + + if((ov_db->a[ovdb_idx].qn==q->y_id) && (ov_db->a[ovdb_idx].tn==t->y_id)) { + ref = &(r0); s = ovdb_idx; e = q->overlapLen+q->x_pos_strand; + if((s!=((uint32_t)-1))&&(e > s)) push_consist_ul_ovlps(ug, ov_db, s, e, len_diff, ref, mm, buf, 0, &is_exact); + + if(!is_exact) { + ref = &(r1); s = t->overlapLen; e = t->overlapLen+t->x_pos_strand; + if((s!=((uint32_t)-1))&&(e > s)) push_consist_ul_ovlps(ug, ov_db, s, e, len_diff, ref, mm, buf, 1, &is_exact); + } + } else { + assert((ov_db->a[ovdb_idx].tn==q->y_id) && (ov_db->a[ovdb_idx].qn==t->y_id)); + ref = &(r1); s = ovdb_idx; e = t->overlapLen+t->x_pos_strand; + if((s!=((uint32_t)-1))&&(e > s)) push_consist_ul_ovlps(ug, ov_db, s, e, len_diff, ref, mm, buf, 1, &is_exact); + + if(!is_exact) { + ref = &(r0); s = q->overlapLen; e = q->overlapLen+q->x_pos_strand; + if((s!=((uint32_t)-1))&&(e > s)) push_consist_ul_ovlps(ug, ov_db, s, e, len_diff, ref, mm, buf, 0, &is_exact); + } + } + assert(buf->n > bn); + ref = &(r0); res->qn = qi; res->tn = ti; + if(is_exact) { + res->el = 1; res->qs = res->ts = 0; + } else { + dis = cal_paired_distance(ug, q, t, buf->a + bn, buf->n - bn, coor_srt, buf1, mm, bd); + assert(dis >= 0); + } + buf->n = bn; +} + +uint64_t mask_ovlps(ma_ug_t *ug, overlap_region_alloc* ol, kv_ul_ov_t *osrt, kv_ul_ov_t *ov_db, asg64_v *coor_srt, asg64_v* buf1, idx_emask_t *mm, double len_diff, uint64_t rlen, int64_t bd) +{ + int64_t k, m, i, osrt_n, osrt_n1, on = ol->length; uint64_t wt, w[2], is_exact, sid, tot, tol, qi, ti; + overlap_region *z; ul_ov_t ou; + for (k = osrt->n = 0; k < on; k++) { + z = &(ol->list[k]); if(!(z->x_pos_strand)) continue;///x_pos_strand: how many masks + if(gen_aln_ul_ov_t(k, ug->g->seq[z->y_id].len, z, &ou)) { + kv_push(ul_ov_t, *osrt, ou); + } + } + if(!(osrt->n)) return 0; + + osrt_n = osrt->n; memset(&ou, 0, sizeof(ou)); + radix_sort_uov_srt_qs(osrt->a, osrt->a+osrt->n); ///sort by qs; for quick filtering between overlaps + for (k = tot = tol = 0; k < osrt_n; k++) { + tol += osrt->a[k].qe-osrt->a[k].qs; + for (i = k+1; i < osrt_n && osrt->a[i].qs < osrt->a[k].qe; i++) { + ///for a pair of overlap, only need to calculate it in one side + if(ol->list[osrt->a[i].qn].y_id >= ol->list[osrt->a[k].qn].y_id) continue; + if(!check_mask_exist(ug, &(ol->list[osrt->a[i].qn]), &(ol->list[osrt->a[k].qn]), ov_db, mm, len_diff, &is_exact, &sid)) continue; + + if(is_exact) {///if two overlaps are exactly the same; no need to do anything + ou.qs = 0; ou.qe = sid; ou.ts = ou.te = 0; ou.el = 1; + } else { + w[0] = ol->list[osrt->a[k].qn].non_homopolymer_errors; + w[1] = ol->list[osrt->a[i].qn].non_homopolymer_errors; + wt = w[0] + w[1] + ((w[0]>=w[1])?(w[0]-w[1]):(w[1]-w[0])); + if(wt > (uint32_t)-1) wt = (uint32_t)-1; + //qs: weight; qe: idx with in ov_db; el: exact match? + ou.qs = wt; ou.qe = sid; ou.el = 0; + ou.ts = osrt->a[i].qe-osrt->a[i].qs; + ou.te = osrt->a[k].qe-osrt->a[k].qs; + } + ou.qn = osrt->a[i].qn; ou.tn = osrt->a[k].qn;///id of the ol->list, instead of real yid + tot += ou.ts + ou.te; + kv_push(ul_ov_t, *osrt, ou); + } + for (i = k-1; i >= 0; i--) { + ///for a pair of overlap, only need to calculate it in one side + if(ol->list[osrt->a[i].qn].y_id >= ol->list[osrt->a[k].qn].y_id) continue; + if(!check_mask_exist(ug, &(ol->list[osrt->a[i].qn]), &(ol->list[osrt->a[k].qn]), ov_db, mm, len_diff, &is_exact, &sid)) continue; + + if(is_exact) { + ou.qs = 0; ou.qe = sid; ou.ts = ou.te = 0; ou.el = 1; + } else { + w[0] = ol->list[osrt->a[k].qn].non_homopolymer_errors; + w[1] = ol->list[osrt->a[i].qn].non_homopolymer_errors; + wt = w[0] + w[1] + ((w[0]>=w[1])?(w[0]-w[1]):(w[1]-w[0])); + if(wt > (uint32_t)-1) wt = (uint32_t)-1; + ou.qs = wt; ou.qe = sid; ou.el = 0; + ou.ts = osrt->a[i].qe-osrt->a[i].qs; + ou.te = osrt->a[k].qe-osrt->a[k].qs; + } + ou.qn = osrt->a[i].qn; ou.tn = osrt->a[k].qn;///id of the ol->list, instead of real yid + tot += ou.ts + ou.te; + kv_push(ul_ov_t, *osrt, ou); + } + } + + if((tot > (rlen*256)) && (tot > (tol*32))) { + tot = MAX((rlen*256), (tol*32)); osrt_n1 = osrt->n; + radix_sort_uov_srt_qs(osrt->a+osrt_n, osrt->a+osrt->n); ///sort by weight (qs) + for (k = osrt_n, wt = 0; k < osrt_n1 && wt <= tot; k++) { + if(osrt->a[k].el) continue;///exact match, no additional work + wt += (uint64_t)osrt->a[k].ts + (uint64_t)osrt->a[k].te; + } + osrt->n = k; + } + + memset(&ou, 0, sizeof(ou)); osrt_n1 = osrt->n; + ///note: osrt will be updated here, so do not use the address within osrt + for (k = osrt_n, m = 0; k < osrt_n1; k++) { + qi = osrt->a[k].qn; ti = osrt->a[k].tn; + sid = osrt->a[k].qe; + ///p->qn/p->tn:: id within the ol->list + ///p->el:: if equally best + ///p->qs:: err(query)-err(target) + ///p->ts:: err(target)-err(query) + if(osrt->a[k].el) {///exact match; no need base-check + ou.qn = qi; ou.tn = ti; ou.el = 1; ou.qs = ou.ts = 0; + } else {///do base-check + gen_mask_ovlp0(ug, ol->list, qi, ti, ov_db, osrt, mm, len_diff, rlen, sid, coor_srt, buf1, bd, &(ou)); + } + osrt->a[m++] = ou; + } + return 1; +} + +void rphase_hl(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t *uopt, kv_ul_ov_t *c_idx, asg64_v* idx, asg64_v* buf, asg64_v* buf1, int64_t ulid, int64_t bd, +int64_t rlen, mask_ul_ov_t *mk, idx_emask_t *mm, double len_diff) +{ + uint64_t on0 = ol->length, k, i, l, m0, m1, sec; + overlap_region *z, h; ma_ug_t *ug = uref->ug; + + for (k = i = 0; k < ol->length; k++) { + z = &(ol->list[k]); + z->align_length = 0; + z->non_homopolymer_errors = 0; + z->overlapLen = (uint32_t)-1; + z->x_pos_strand = 0; + if(!(z->w_list.n)) continue;///no base-level; cis + if(k != i) { + h = ol->list[k]; + ol->list[k] = ol->list[i]; + ol->list[i] = h; + } + i++; + } + ol->length = i; ///[ol->length, on0) keeps cis overlaps without the base-level alignment + + rphase_rr(ol, uref, uopt, c_idx, idx, buf, ulid, bd, 0); + ///could use idx && buf here + for (k = idx->n = buf->n = 0; k < ol->length; k++) { + z = &(ol->list[k]); + ///sort overlaps by sec error + for (i = sec = 0; i < z->align_length; i++) { + sec += z->w_list.a[z->w_list.n+i].clen; + } + z->non_homopolymer_errors = sec; sec = sec - (sec&3);///normalize + kv_push(uint64_t, *buf, (((uint64_t)sec)<<32)|((uint64_t)k)); + + ///sort overlaps by yid for filtering + kv_push(uint64_t, *idx, (((uint64_t)z->y_id)<<32)|((uint64_t)k)); + } + radix_sort_bc64(idx->a, idx->a+idx->n); + + radix_sort_bc64(buf->a, buf->a+buf->n);///sort by error; smaller first + for (l = 0, k = 1; k <= buf->n; k++) { + if(k == buf->n || (buf->a[l]>>32) == (buf->a[k]>>32)) {///with equal number of normalized errors + if((k - l) >1) { + for (i = l; i < k; i++) { + z = &(ol->list[(uint32_t)buf->a[i]]); + m0 = z->x_pos_e+1-z->x_pos_s; + m1 = z->y_pos_e+1-z->y_pos_s; + sec = MIN(m0, m1); sec = ((uint32_t)-1)-((uint32_t)sec); + buf->a[i] = (((uint64_t)sec)<<32)|((uint32_t)buf->a[i]); + } + radix_sort_bc64(buf->a+l, buf->a+k);///sort by lenght; longer first + } + l = k; + } + } + + mk->idx.n = mk->srt.n = 0; + for (k = 0; k < buf->n && mk->srt.n <= 1000000; k++) {///need to add overlap that directly connected in the graph + z = &(ol->list[(uint32_t)buf->a[k]]); z->overlapLen = mk->srt.n; + m0 = push_emask_flt(&(mm->a[z->y_id]), idx->a, idx->n, z->y_id, &(mk->srt)); + m1 = gen_src_shared_interval_simple(z->y_id, ug, &(mk->srt)); + dedup_src_shared1(ug, &(mk->srt), m0, m1, mm); + z->x_pos_strand = mk->srt.n - z->overlapLen; + if(!(z->x_pos_strand)) z->overlapLen = (uint32_t)-1; + } + + ///if no mask overlaps, no need rephase + if(mk->srt.n && mask_ovlps(ug, ol, c_idx, &(mk->srt), idx, buf1, mm, len_diff, rlen, bd)) { + + } + assert(on0 <= ol->length); + // for (k = 0; k < on; k++) { + // z = &(ol->list[k]); + // push_emask_flt(kv_emask_t *in, uint64_t *flt, uint64_t flt_n, kv_ul_ov_t *res); + // ///if n > 1000, dedup + // } + + // for (k = 0; k < on; k++) { + // z = &(ol->list[k]); sec = 0; + // for (i = 0; i < z->align_length; i++) { + // sec += z->w_list.a[z->w_list.n+i].clen; + // } + // sec = sec - (sec&3);///normalize + // if(gen_aln_ul_ov_t(k, ug->g->seq[z->y_id].len, z, &ou)) { + // ; + // } + // } +} + +uint64_t rphase_detect0(overlap_region_alloc* ol, kv_ul_ov_t *c_idx, asg64_v* idx, asg64_v* buf, asg64_v* suf) +{ + uint64_t srt_n = idx->n, i, k, h, qs, qe, ts, te, rr = 0, ovlp; overlap_region *z, *p; + int64_t dp, old_dp, beg, end; + for (i = k = 0, dp = old_dp = 0, beg = 0, end = -1; i < srt_n; ++i) {///[beg, end) but coordinates in idx is [, ] + ///if idx->a.a[] is qe + old_dp = dp; + if ((idx->a[i]>>32)&1) { + --dp; end = (idx->a[i]>>33)+1; + }else { + //meet a new overlap; the overlaps are pushed by the x_pos_s + ++dp; end = (idx->a[i]>>33); + kv_push(uint64_t, *idx, ((uint32_t)idx->a[i])); + } + // fprintf(stderr, "\n[M::%s::] beg::%ld, end::%ld, old_dp::%ld\n", __func__, beg, end, old_dp); + if((end > beg) && (old_dp >= 2)) { + // fprintf(stderr, "\n[M::%s::] beg::%ld, end::%ld, old_dp::%ld\n", __func__, beg, end, old_dp); + // kv_resize(uint64_t, *buf, ((uint32_t)old_dp)<<1); + // idx->n = srt_n + gen_region_phase(ol->list, idx->a+srt_n, idx->n-srt_n, beg, end, old_dp, c_idx->a, buf, buf1); + idx->n = srt_n + gen_region_phase_robust(ol->list, idx->a+srt_n, idx->n-srt_n, beg, end, old_dp, c_idx->a, buf); + } + beg = end; + } + idx->n = srt_n; + + suf->n = 0; kv_resize(uint64_t, *suf, (ol->length<<1)); + uint64_t *sep = suf->a, *sid = suf->a + ol->length, sid_n; + for (k = h = sid_n = 0; k < ol->length; k++) { + z = &(ol->list[k]); + z->overlapLen = (uint32_t)-1; z->non_homopolymer_errors = 0; + for (i = 0; i < z->align_length; i++) { + z->non_homopolymer_errors += z->w_list.a[z->w_list.n+i].clen; + } + for (; h < c_idx->n && ovlp_id(c_idx->a[h]) < k; h++); + if(h < c_idx->n && ovlp_id(c_idx->a[h]) == k) { + qs = z->w_list.a[ovlp_min_wid(c_idx->a[h])].x_start; + qe = z->w_list.a[ovlp_max_wid(c_idx->a[h])].x_end+1; + for (; h < c_idx->n && ovlp_id(c_idx->a[h]) == k; h++) { + qe = z->w_list.a[ovlp_max_wid(c_idx->a[h])].x_end+1; + } + sid[sid_n] = ((qs<<32)|sid_n); + sep[sid_n] = ((k<<32)|qe); + sid_n++; + // kv_push(uint64_t, *suf, ((qs<<32)|h)); + } + } + radix_sort_bc64(sid, sid+sid_n); + for (k = 0; k < sid_n; k++) { + qs = sid[k]>>32; qe = ((uint32_t)sep[(uint32_t)sid[k]]); + z = &(ol->list[sep[(uint32_t)sid[k]]>>32]); + if(z->non_homopolymer_errors == 0) continue; + for (i = rr = 0; i < sid_n; i++) { + ts = sid[i]>>32; te = ((uint32_t)sep[(uint32_t)sid[i]]); + p = &(ol->list[sep[(uint32_t)sid[i]]>>32]); + if(ts >= qe) break; + if(te <= qs) continue; + ovlp = ((MIN(qe, te) > MAX(qs, ts))? (MIN(qe, te) - MAX(qs, ts)):0); + if(ovlp < 512) continue; + if(z->non_homopolymer_errors > p->non_homopolymer_errors + rphase_thres) continue; + rr = 1; break; + } + if(rr) z->overlapLen = 1; + } + return rr; +} + +void reacal_phase(overlap_region_alloc* ol, kv_ul_ov_t *c_idx, asg64_v* idx, asg64_v* buf, asg64_v* suf) +{ + uint64_t srt_n = idx->n, i, k, h, qs, qe, ts, te, rr = 0; overlap_region *z, *p; + int64_t dp, old_dp, beg, end; + for (i = k = 0, dp = old_dp = 0, beg = 0, end = -1; i < srt_n; ++i) {///[beg, end) but coordinates in idx is [, ] + ///if idx->a.a[] is qe + old_dp = dp; + if ((idx->a[i]>>32)&1) { + --dp; end = (idx->a[i]>>33)+1; + }else { + //meet a new overlap; the overlaps are pushed by the x_pos_s + ++dp; end = (idx->a[i]>>33); + kv_push(uint64_t, *idx, ((uint32_t)idx->a[i])); + } + // fprintf(stderr, "\n[M::%s::] beg::%ld, end::%ld, old_dp::%ld\n", __func__, beg, end, old_dp); + if((end > beg) && (old_dp >= 2)) { + // fprintf(stderr, "\n[M::%s::] beg::%ld, end::%ld, old_dp::%ld\n", __func__, beg, end, old_dp); + // kv_resize(uint64_t, *buf, ((uint32_t)old_dp)<<1); + // idx->n = srt_n + gen_region_phase(ol->list, idx->a+srt_n, idx->n-srt_n, beg, end, old_dp, c_idx->a, buf, buf1); + idx->n = srt_n + gen_region_phase_robust(ol->list, idx->a+srt_n, idx->n-srt_n, beg, end, old_dp, c_idx->a, buf); + } + beg = end; + } + idx->n = srt_n; + + suf->n = 0; kv_resize(uint64_t, *suf, (ol->length<<1)); + uint64_t *sep = suf->a, *sid = suf->a + ol->length, sid_n; + for (k = h = sid_n = 0; k < ol->length; k++) { + z = &(ol->list[k]); + z->overlapLen = (uint32_t)-1; z->non_homopolymer_errors = 0; + for (i = 0; i < z->align_length; i++) { + z->non_homopolymer_errors += z->w_list.a[z->w_list.n+i].clen; + } + for (; h < c_idx->n && ovlp_id(c_idx->a[h]) < k; h++); + if(h < c_idx->n && ovlp_id(c_idx->a[h]) == k) { + qs = z->w_list.a[ovlp_min_wid(c_idx->a[h])].x_start; + qe = z->w_list.a[ovlp_max_wid(c_idx->a[h])].x_end+1; + for (; h < c_idx->n && ovlp_id(c_idx->a[h]) == k; h++) { + qe = z->w_list.a[ovlp_max_wid(c_idx->a[h])].x_end+1; + } + sid[sid_n] = ((qs<<32)|sid_n); + sep[sid_n] = ((k<<32)|qe); + sid_n++; + // kv_push(uint64_t, *suf, ((qs<<32)|h)); + } + } + radix_sort_bc64(sid, sid+sid_n); + for (k = rr = 0; k < sid_n; k++) { + qs = sid[k]>>32; qe = ((uint32_t)sep[(uint32_t)sid[k]]); + z = &(ol->list[sep[(uint32_t)sid[k]]>>32]); + if(z->non_homopolymer_errors == 0) continue; + for (i = 0; i < sid_n; i++) { + ts = sid[i]>>32; te = ((uint32_t)sep[(uint32_t)sid[i]]); + p = &(ol->list[sep[(uint32_t)sid[i]]>>32]); + if(ts >= qe) break; + if(te <= qs) continue; + if(p->non_homopolymer_errors > z->non_homopolymer_errors) continue; + if(z->non_homopolymer_errors - p->non_homopolymer_errors <= rphase_thres) { + z->overlapLen = p->overlapLen = 1; rr++; + } + } + } +} + +void region_phase_adv(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t *uopt, kv_ul_ov_t *c_idx, asg64_v* idx, asg64_v* buf, asg64_v* buf1, int64_t ulid) +{ + int64_t on = ol->length, k, i, zwn, q[2], t[2], w[2]; + uint64_t m; overlap_region *z; ul_ov_t *cp; + kv_resize(uint64_t, *idx, (ol->length<<1)); + kv_resize(ul_ov_t, *c_idx, ol->length); + for (k = idx->n = c_idx->n = 0; k < on; k++) { + z = &(ol->list[k]); zwn = z->w_list.n; + z->align_length = 0; z->overlapLen = (uint32_t)-1; + z->non_homopolymer_errors = 0; + if(!zwn) continue; + q[0] = q[1] = t[0] = t[1] = w[0] = w[1] = INT32_MIN; + for (i = 0; i < zwn; i++) { + if((z->w_list.a[i].x_start==(q[1]+1)) && ((z->w_list.a[i].y_start==(t[1]+1)))) { + q[1] = z->w_list.a[i].x_end; + t[1] = z->w_list.a[i].y_end; + w[1] = i; + } else { + if(q[0] != INT32_MIN) { + m = ((uint64_t)q[0])<<1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + m = (((uint64_t)q[1])<<1)+1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + + kv_pushp(ul_ov_t, *c_idx, &cp); + ovlp_id(*cp) = k; ///ovlp id + ovlp_min_wid(*cp) = w[0]; ///beg id of windows + ovlp_max_wid(*cp) = w[1]; ///end id of windows + ovlp_cur_wid(*cp) = w[0]; ///cur id of windows + ovlp_cur_xoff(*cp) = z->w_list.a[w[0]].x_start; ///cur xpos + ovlp_cur_coff(*cp) = 0; ///cur cigar off in cur window + } + + q[0] = z->w_list.a[i].x_start; q[1] = z->w_list.a[i].x_end; + t[0] = z->w_list.a[i].y_start; t[1] = z->w_list.a[i].y_end; + w[0] = i; w[1] = i; + } + } + if(q[0] != INT32_MIN) { + m = ((uint64_t)q[0])<<1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + m = (((uint64_t)q[1])<<1)+1; m <<= 32; + m += c_idx->n; kv_push(uint64_t, *idx, m); + + kv_pushp(ul_ov_t, *c_idx, &cp); + ovlp_id(*cp) = k; ///ovlp id + ovlp_min_wid(*cp) = w[0]; ///beg id of windows + ovlp_max_wid(*cp) = w[1]; ///end id of windows + ovlp_cur_wid(*cp) = w[0]; ///cur id of windows + ovlp_cur_xoff(*cp) = z->w_list.a[w[0]].x_start; ///cur xpos + ovlp_cur_coff(*cp) = 0; ///cur cigar off in cur window + } + } + radix_sort_bc64(idx->a, idx->a+idx->n); + //this is used with gen_region_phase, now give up + //gen_gov_idx(ol, uref, uopt, G_CHAIN_BW, N_GCHAIN_RATE, buf1); + // for (m = 0; m < c_idx->n; m++) { + // fprintf(stderr, "+++[M::%s::utg%.6dl] q[%d, %d), t[%d, %d), wn::%d\n", __func__, + // (int32_t)ol->list[ovlp_id(c_idx->a[m])].y_id+1, + // ol->list[ovlp_id(c_idx->a[m])].w_list.a[ovlp_min_wid(c_idx->a[m])].x_start, + // ol->list[ovlp_id(c_idx->a[m])].w_list.a[ovlp_max_wid(c_idx->a[m])].x_end+1, + // ol->list[ovlp_id(c_idx->a[m])].w_list.a[ovlp_min_wid(c_idx->a[m])].y_start, + // ol->list[ovlp_id(c_idx->a[m])].w_list.a[ovlp_max_wid(c_idx->a[m])].y_end+1, + // ovlp_max_wid(c_idx->a[m])+1-ovlp_min_wid(c_idx->a[m])); + // } + + if(rphase_detect0(ol, c_idx, idx, buf, buf1)) {//refine phasing + reacal_phase(ol, c_idx, idx, buf, buf1); + } + + for (k = 0; k < on; k++) { + z = &(ol->list[k]); + z->overlapLen = z->x_pos_e+1-z->x_pos_s; + z->non_homopolymer_errors = 0; zwn = 0; + for (i = m = 0; i < z->align_length; i++) { + z->w_list.a[m] = z->w_list.a[z->w_list.n+i]; + if(z->w_list.a[m].clen > 0) { + z->non_homopolymer_errors += z->w_list.a[m].clen; + zwn += z->w_list.a[m].x_end-z->w_list.a[m].x_start; + } + m++; + } + z->w_list.n = m; + assert(zwn <= z->overlapLen); + z->align_length = z->overlapLen - zwn; + } +} + void ul_gap_filling_adv(overlap_region_alloc* ol, Candidates_list *cl, kv_ul_ov_t *aln, uint64_t wl, const ul_idx_t *uref, hpc_t *hpc_g, All_reads *rref, char* qstr, UC_Read *tu, bit_extz_t *exz, overlap_region *aux_o, asg64_v* buf, asg64_v* iidx, double e_rate, int64_t ql, uint64_t rid, int64_t khit, int64_t base_chekc_k_hit, @@ -16334,7 +17756,12 @@ int64_t max_lgap, double sgap_rate, int64_t sgap)///[s, e) if(!ovlp_win_check(z, z->align_length, wid, max_lgap, sgap_rate, sgap)) continue;///not co-linear rxl = z->w_list.a[wid].x_end+1-z->w_list.a[wid].x_start; ryl = r_y[1]-r_y[0]; + // fprintf(stderr, "+[M::%s::] buf_n::%lu, q::[%lu, %lu), w::[%lu, %lu), rxl::%ld, ryl::%ld, x::[%d, %d), y::[%ld, %ld)\n", + // __func__, buf_n, qs, qe, ws, we, rxl, ryl, z->w_list.a[wid].x_start, z->w_list.a[wid].x_end+1, r_y[0], r_y[1]); if(rxl > 1 && ryl > 1) continue;///length of window should be longer than 1 + // if(rxl <= 1 || ryl <= 1) continue;///length of window should be longer than 1 + // fprintf(stderr, "-[M::%s::] buf_n::%lu, q::[%lu, %lu), w::[%lu, %lu), rxl::%ld, ryl::%ld, x::[%d, %d), y::[%ld, %ld)\n", + // __func__, buf_n, qs, qe, ws, we, rxl, ryl, z->w_list.a[wid].x_start, z->w_list.a[wid].x_end+1, r_y[0], r_y[1]); qstring = tstring = NULL; for (i = 1; i < buf_n; i++) { z = &(ol->list[aln->a[(uint32_t)buf[i]].qn]); @@ -18923,8 +20350,12 @@ bit_extz_t *exz, double e_rate, int64_t w_l, uint64_t ql, uint64_t rid, uint64_t int64_t zwn = z->w_list.n, zerr = 0, zlen = z->x_pos_e+1-z->x_pos_s; for (i = 0; i < zwn; i++) { - if(is_ualn_win(z->w_list.a[i])) zerr += z->w_list.a[i].x_end+1-z->w_list.a[i].x_start; - else zerr += z->w_list.a[i].error; + if(is_ualn_win(z->w_list.a[i])) { + zerr += MAX((z->w_list.a[i].x_end+1-z->w_list.a[i].x_start), + (z->w_list.a[i].y_end+1-z->w_list.a[i].y_start)); + } else { + zerr += z->w_list.a[i].error; + } } z->non_homopolymer_errors = zerr; @@ -19094,7 +20525,7 @@ void ug_lalign(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *ur uint64_t chain_cut, void *km) { uint64_t i, bs, k, ovl, pid, kl, kn; Window_Pool w; double err; - overlap_region t; overlap_region *z; + overlap_region t; overlap_region *z; char *in = qstr; ol->mapped_overlaps_length = 0; if(ol->length <= 0) return; @@ -19105,7 +20536,9 @@ void ug_lalign(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *ur resize_UC_Read(tu, bs<<1); if(!aux_o) { - resize_UC_Read(qu, ql); qu->length = ql; memcpy(qu->seq, qstr, ql); + if(qu) { + resize_UC_Read(qu, ql); qu->length = ql; memcpy(qu->seq, qstr, ql); in = qu->seq; + } for (i = 0; i < ol->length; i++) { z = &(ol->list[i]); ovl = z->x_pos_e+1-z->x_pos_s; z->shared_seed = z->non_homopolymer_errors;///for index @@ -19113,7 +20546,7 @@ void ug_lalign(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *ur for (k = z->shared_seed; k < kl && cl->list[k].readID == pid && kn < chain_cut; k++) kn++; if(kn < chain_cut) continue; - if(!align_ul_ed_post_extz(z, uref, NULL, qu->seq, tu->seq, exz, err, w.window_length, -1, 0, km)) { + if(!align_ul_ed_post_extz(z, uref, NULL, in/**qu->seq**/, tu->seq, exz, err, w.window_length, -1, 0, km)) { continue; } if(uref && simi_pass(ovl, z->align_length, uref?1:0, -1, NULL)) { @@ -19128,7 +20561,7 @@ void ug_lalign(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *ur double e_max = err*1.5, rr; int64_t re; for (i = k = 0; i < ol->length; i++) { z = &(ol->list[i]); ovl = z->x_pos_e + 1 - z->x_pos_s; - rr = gen_extend_err_exz(z, uref, NULL, NULL, qu->seq, tu->seq, exz, NULL/**v_idx?v_idx->a.a:NULL**/, w.window_length, -1, err, (e_max+0.000001), &re); + rr = gen_extend_err_exz(z, uref, NULL, NULL, in/**qu->seq**/, tu->seq, exz, NULL/**v_idx?v_idx->a.a:NULL**/, w.window_length, -1, err, (e_max+0.000001), &re); z->is_match = 0;///must be here; // if(sid == 7 && z->y_id == 135) { @@ -19153,9 +20586,12 @@ void ug_lalign(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *ur if(ol->length <= 0) return; } else { if(ol->length <= 1) return; + if(qu) { + resize_UC_Read(qu, ql); qu->length = ql; memcpy(qu->seq, qstr, ql); in = qu->seq; + } for (i = k = 0; i < ol->length; i++) { z = &(ol->list[i]); - if(!ul_raw_aln(z, cl, uref, qu->seq, tu, exz, err, wl, ql, sid, khit, aux_o)) { + if(!ul_raw_aln(z, cl, uref, in/**qu->seq**/, tu, exz, err, wl, ql, sid, khit, aux_o)) { z->is_match = 0; continue; } diff --git a/Overlaps.cpp b/Overlaps.cpp index cb845e3..1b72381 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -64,6 +64,7 @@ KSORT_INIT_GENERIC(uint32_t) void reduce_hamming_error_adv(ma_ug_t *iug, asg_t *sg, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, int max_hang, int min_ovlp, long long gap_fuzz, R_to_U *ru, bubble_type* bub); +void print_vw_edge(asg_t *sg, uint32_t vid, uint32_t wid, const char *cmd); typedef struct { uint32_t d, tot, ma, p; @@ -10101,13 +10102,29 @@ uint64_t *n_utg) return C_bases/R_bases; } +uint32_t cal_circle_ov(const ma_ug_t *ug, uint32_t uid, uint32_t rev, asg_t *sg) +{ + uint32_t v, w, vx, wx, k; + v = w = (uid<<1)+(!!rev); + vx = (v&1?((ug->u.a[v>>1].a[0]>>32)^1):(ug->u.a[v>>1].a[ug->u.a[v>>1].n-1]>>32)); + wx = (w&1?((ug->u.a[w>>1].a[ug->u.a[w>>1].n-1]>>32)^1):(ug->u.a[w>>1].a[0]>>32)); + v = vx; w = wx; + if(sg) { + asg_arc_t *av; uint32_t nv; + av = asg_arc_a(sg, vx); nv = asg_arc_n(sg, vx); + for (k = 0; k < nv; k++) { + if(av[k].v == wx) return av[k].ol; + } + } + return 0; +} void ma_ug_print2(const ma_ug_t *ug, All_reads *RNF, asg_t* read_g, const ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, R_to_U* ruIndex, int print_seq, const char* prefix, FILE *fp) { uint8_t* primary_flag = read_g?(uint8_t*)calloc(read_g->n_seq, sizeof(uint8_t)):NULL; - uint32_t i, j, l, pc = read_g && coverage_cut && sources && ruIndex?1:0; + uint32_t i, j, l, pc = read_g && coverage_cut && sources && ruIndex?1:0, co; char name[32]; for (i = 0; i < ug->u.n; ++i) { // the Segment lines in GFA ma_utg_t *p = &ug->u.a[i]; @@ -10154,37 +10171,39 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, int print_seq, const char* prefix, FIL uint32_t nu, u, v; for (i = 0; i < ug->u.n; ++i) { if(ug->u.a[i].m == 0) continue; - if(ug->u.a[i].circ) - { - fprintf(fp, "L\t%s%.6dc\t+\t%s%.6dc\t+\t%dM\tL1:i:%d\n", - prefix, i+1, prefix, i+1, 0, ug->u.a[i].len); - fprintf(fp, "L\t%s%.6dc\t-\t%s%.6dc\t-\t%dM\tL1:i:%d\n", - prefix, i+1, prefix, i+1, 0, ug->u.a[i].len); + if(ug->u.a[i].circ) { + co = cal_circle_ov(ug, i, 0, read_g); + fprintf(fp, "L\t%s%.6dc\t+\t%s%.6dc\t+\t%dM\tL1:i:%d\tL2:i:%u\n", + prefix, i+1, prefix, i+1, co, ((ug->u.a[i].len>=co)?(ug->u.a[i].len-co):0), 0); + co = cal_circle_ov(ug, i, 1, read_g); + fprintf(fp, "L\t%s%.6dc\t-\t%s%.6dc\t-\t%dM\tL1:i:%d\tL2:i:%u\n", + prefix, i+1, prefix, i+1, co, ((ug->u.a[i].len>=co)?(ug->u.a[i].len-co):0), 0); + } else { + u = i<<1; + au = asg_arc_a(ug->g, u); + nu = asg_arc_n(ug->g, u); + for (j = 0; j < nu; j++) + { + if(au[j].del) continue; + v = au[j].v; + fprintf(fp, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\tL2:i:%u\n", + prefix, (u>>1)+1, "lc"[ug->u.a[u>>1].circ], "+-"[u&1], + prefix, (v>>1)+1, "lc"[ug->u.a[v>>1].circ], "+-"[v&1], au[j].ol, asg_arc_len(au[j]), 0/**au[j].ou**/); + } + + + u = (i<<1) + 1; + au = asg_arc_a(ug->g, u); + nu = asg_arc_n(ug->g, u); + for (j = 0; j < nu; j++) + { + if(au[j].del) continue; + v = au[j].v; + fprintf(fp, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\tL2:i:%u\n", + prefix, (u>>1)+1, "lc"[ug->u.a[u>>1].circ], "+-"[u&1], + prefix, (v>>1)+1, "lc"[ug->u.a[v>>1].circ], "+-"[v&1], au[j].ol, asg_arc_len(au[j]), 0/**au[j].ou**/); + } } - u = i<<1; - au = asg_arc_a(ug->g, u); - nu = asg_arc_n(ug->g, u); - for (j = 0; j < nu; j++) - { - if(au[j].del) continue; - v = au[j].v; - fprintf(fp, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\tL2:i:%u\n", - prefix, (u>>1)+1, "lc"[ug->u.a[u>>1].circ], "+-"[u&1], - prefix, (v>>1)+1, "lc"[ug->u.a[v>>1].circ], "+-"[v&1], au[j].ol, asg_arc_len(au[j]), 0/**au[j].ou**/); - } - - - u = (i<<1) + 1; - au = asg_arc_a(ug->g, u); - nu = asg_arc_n(ug->g, u); - for (j = 0; j < nu; j++) - { - if(au[j].del) continue; - v = au[j].v; - fprintf(fp, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\tL2:i:%u\n", - prefix, (u>>1)+1, "lc"[ug->u.a[u>>1].circ], "+-"[u&1], - prefix, (v>>1)+1, "lc"[ug->u.a[v>>1].circ], "+-"[v&1], au[j].ol, asg_arc_len(au[j]), 0/**au[j].ou**/); - } } } free(primary_flag); @@ -13494,12 +13513,16 @@ void hic_clean(asg_t* read_g) void hic_clean_adv(asg_t *sg, ug_opt_t *uopt) { - uint32_t i, k, m, z, v, w; ma_utg_t *u = NULL; uint32_t *ba, bn, n_vtx, beg, end, n0, n1; + uint32_t i, k, m, z, v, w, mk; ma_utg_t *u = NULL; uint32_t *ba, bn, n_vtx, beg, end, n0, n1; ma_utg_t *mz = NULL; ma_ug_t *ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); n_vtx = ug->g->n_seq<<1; double bub_rate = 0.1; uint8_t *bf = NULL; bubble_type *bub = gen_bubble_chain(sg, ug, uopt, &bf); uint64_t tLen, vocc, socc, pocc; buf_t b; memset(&b, 0, sizeof(buf_t)); CALLOC(b.a, n_vtx); REALLOC(bf, n_vtx); memset(bf, 0, sizeof((*bf))*n_vtx); kvec_t(uint64_t) buf; kv_init(buf); n0 = n1 = 0; + for (i = 0; i < ug->g->n_seq; ++i) { + if(ug->g->seq[i].del) continue; + ug->g->seq[i].c = PRIMARY_LABLE; + } for (i = 0; i < bub->b_ug->u.n; i++) { u = &(bub->b_ug->u.a[i]); @@ -13568,6 +13591,20 @@ void hic_clean_adv(asg_t *sg, ug_opt_t *uopt) if((pocc+socc) >= (vocc*bub_rate)) continue; pocc += socc; asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 1, NULL, NULL, NULL, 0, 0, NULL); + for (z = 0; z < b.b.n; z++) { + if(b.b.a[z]==v || b.b.a[z]==b.S.a[0]) continue; + // socc += ug->u.a[b.b.a[i]>>1].n; + if(ug->g->seq[b.b.a[z]>>1].del) continue; + if(ug->g->seq[b.b.a[z]>>1].c != ALTER_LABLE) continue; + mz = &(ug->u.a[b.b.a[z]>>1]); + if(mz->m == 0) continue; + for (mk = 0; mk < mz->n; mk++) asg_seq_del(sg, mz->a[mk]>>33); + asg_seq_del(ug->g, b.b.a[z]>>1); + if(ug->u.a[b.b.a[z]>>1].m) { + ug->u.a[b.b.a[z]>>1].m = ug->u.a[b.b.a[z]>>1].n = 0; + free(ug->u.a[b.b.a[z]>>1].a); ug->u.a[b.b.a[z]>>1].a = NULL; + } + } // fprintf(stderr, "+utg%.6dl\tutg%.6dl\n", (int32_t)(v>>1)+1, (int32_t)(b.S.a[0]>>1)+1); n0++; } @@ -13579,6 +13616,22 @@ void hic_clean_adv(asg_t *sg, ug_opt_t *uopt) } } + for (v = 0; v < ug->g->n_seq; ++v) { + if(ug->g->seq[v].del) continue; + if(ug->g->seq[v].c != ALTER_LABLE) continue; + mz = &(ug->u.a[v]); + if(mz->m == 0) continue; + for (k = 0; k < mz->n; k++) asg_seq_del(sg, mz->a[k]>>33); + asg_seq_del(ug->g, v); + if(ug->u.a[v].m) { + ug->u.a[v].m = ug->u.a[v].n = 0; + free(ug->u.a[v].a); ug->u.a[v].a = NULL; + } + } + asg_cleanup(ug->g); + + + tLen = get_bub_pop_max_dist_advance(ug->g, &b); for (v = buf.n = 0; v < n_vtx; ++v) { if(ug->g->seq[v>>1].del) continue; @@ -13624,14 +13677,43 @@ void hic_clean_adv(asg_t *sg, ug_opt_t *uopt) vocc += ug->u.a[b.b.a[k]>>1].n; } if((socc) >= (vocc*bub_rate)) continue; + asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 1, NULL, NULL, NULL, 0, 0, NULL); + for (i = 0; i < b.b.n; i++) { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + // socc += ug->u.a[b.b.a[i]>>1].n; + if(ug->g->seq[b.b.a[i]>>1].del) continue; + if(ug->g->seq[b.b.a[i]>>1].c != ALTER_LABLE) continue; + mz = &(ug->u.a[b.b.a[i]>>1]); + if(mz->m == 0) continue; + for (mk = 0; mk < mz->n; mk++) asg_seq_del(sg, mz->a[mk]>>33); + asg_seq_del(ug->g, b.b.a[i]>>1); + if(ug->u.a[b.b.a[i]>>1].m) { + ug->u.a[b.b.a[i]>>1].m = ug->u.a[b.b.a[i]>>1].n = 0; + free(ug->u.a[b.b.a[i]>>1].a); ug->u.a[b.b.a[i]>>1].a = NULL; + } + } // fprintf(stderr, "-utg%.6dl\tutg%.6dl\n", (int32_t)(v>>1)+1, (int32_t)(b.S.a[0]>>1)+1); n1++; } } } - filter_sg_by_ug(sg, ug, uopt); + // filter_sg_by_ug(sg, ug, uopt); + for (v = 0; v < ug->g->n_seq; ++v) { + if(ug->g->seq[v].del) continue; + if(ug->g->seq[v].c != ALTER_LABLE) continue; + mz = &(ug->u.a[v]); + if(mz->m == 0) continue; + for (k = 0; k < mz->n; k++) asg_seq_del(sg, mz->a[k]>>33); + asg_seq_del(ug->g, v); + if(ug->u.a[v].m) { + ug->u.a[v].m = ug->u.a[v].n = 0; + free(ug->u.a[v].a); ug->u.a[v].a = NULL; + } + } + asg_cleanup(ug->g); + asg_cleanup(sg); free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); ma_ug_destroy(ug); free(bf); kv_destroy(buf); destory_bubbles(bub); free(bub); @@ -15627,7 +15709,6 @@ bub_label_t* b_mask_t) ma_ug_t *ug = NULL; ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); - hap_cov_t *cov = NULL; asg_t *copy_sg = copy_read_graph(sg); ma_ug_t *copy_ug = copy_untig_graph(ug); @@ -27603,7 +27684,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges, void output_contig_graph_primary_pre(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, uint64_t bubble_dist, long long tipsLen, -R_to_U* ruIndex, int max_hang, int min_ovlp) +R_to_U* ruIndex, int max_hang, int min_ovlp, const ug_opt_t *uopt) { kvec_asg_arc_t_warp new_rtg_edges; kv_init(new_rtg_edges.a); @@ -27656,6 +27737,10 @@ R_to_U* ruIndex, int max_hang, int min_ovlp) fclose(output_file); } + ///for debug + // graph_ovlp_binning(ug, sg, uopt); + // gen_hpc_re_t(ug); + free(gfa_name); ma_ug_destroy(ug); kv_destroy(new_rtg_edges.a); @@ -33584,10 +33669,10 @@ ma_hit_t_alloc* src, uint64_t* readLen, R_to_U* ruIndex, bub_label_t *b_mask_t, } void renew_g(ma_hit_t_alloc **sources, ma_hit_t_alloc **reverse_sources, long long *n_read, -uint64_t **readLen, ma_sub_t **coverage_cut, R_to_U *ruIndex, asg_t **sg, -int64_t mini_overlap_length, int64_t max_hang_length, -ug_opt_t *uopt, int64_t clean_round, double min_ovlp_drop_ratio, double max_ovlp_drop_ratio, -int64_t max_tip, bub_label_t *b_mask_t, uint32_t is_trio, char *o_file, const char *bin_file, uint64_t free_uld) +uint64_t **readLen, ma_sub_t **coverage_cut, R_to_U *ruIndex, asg_t **sg, int64_t mini_overlap_length, +int64_t max_hang_length, ug_opt_t *uopt, int64_t clean_round, double min_ovlp_drop_ratio, +double max_ovlp_drop_ratio, int64_t max_tip, bub_label_t *b_mask_t, uint32_t is_trio, +char *o_file, const char *bin_file, uint64_t free_uld, uint64_t is_bridg, uint64_t deep_clean) { ul_renew_t nopt; memset(&nopt, 0, sizeof(nopt)); nopt.src = sources; nopt.r_src = reverse_sources; nopt.ruIndex = ruIndex; @@ -33595,7 +33680,7 @@ int64_t max_tip, bub_label_t *b_mask_t, uint32_t is_trio, char *o_file, const ch nopt.cov = coverage_cut; nopt.b_mask_t = b_mask_t; nopt.max_hang = max_hang_length; nopt.mini_ovlp = mini_overlap_length; ul_realignment_gfa(uopt, *sg, clean_round, min_ovlp_drop_ratio, max_ovlp_drop_ratio, - asm_opt.max_short_tip, asm_opt.max_short_ul_tip, b_mask_t, ha_opt_triobin(&asm_opt), o_file, &nopt, bin_file, free_uld); + asm_opt.max_short_tip, asm_opt.max_short_ul_tip, b_mask_t, ha_opt_triobin(&asm_opt), o_file, &nopt, bin_file, free_uld, is_bridg, deep_clean); // ma_ug_t *iug = ul_realignment_gfa(uopt, *sg, clean_round, min_ovlp_drop_ratio, max_ovlp_drop_ratio, // asm_opt.max_short_tip, b_mask_t, ha_opt_triobin(&asm_opt), o_file); // asg_t *ng = gen_ng(iug, *sg, uopt, coverage_cut, ruIndex, 256); @@ -33618,7 +33703,7 @@ bub_label_t *b_mask_t, uint32_t is_trio, int32_t ul_aln_round, char *o_file, con int32_t k, strl = strlen(bin_file)+1, kt, cl, sl; char *id = NULL; renew_g(src, rev_src, n_read, readLen, cov, ruIndex, sg, mini_overlap_length, max_hang_length, uopt, clean_round, min_ovlp_drop_ratio, max_ovlp_drop_ratio, asm_opt.max_short_tip, b_mask_t, - is_trio, o_file, bin_file, (ul_aln_round<=1)?1:0); + is_trio, o_file, bin_file, (ul_aln_round<=1)?1:0, 1, 1); gen_ug_opt_t(uopt, *src, *rev_src, max_hang_length, mini_overlap_length, gap_fuzz, min_dp, *readLen, *cov, ruIndex, (asm_opt.max_short_tip*2), 0.15, 3, 0.05, 0.9, b_mask_t); ug_ext_gfa(uopt, *sg, ug_ext_len); @@ -33634,7 +33719,7 @@ bub_label_t *b_mask_t, uint32_t is_trio, int32_t ul_aln_round, char *o_file, con sprintf(id, "%s%d", bin_file, k); renew_g(src, rev_src, n_read, readLen, cov, ruIndex, sg, mini_overlap_length, max_hang_length, uopt, clean_round, min_ovlp_drop_ratio, max_ovlp_drop_ratio, asm_opt.max_short_tip, b_mask_t, - is_trio, o_file, id, ((k+1)==ul_aln_round)?1:0); + is_trio, o_file, id, ((k+1)==ul_aln_round)?1:0, 0, 0); gen_ug_opt_t(uopt, *src, *rev_src, max_hang_length, mini_overlap_length, gap_fuzz, min_dp, *readLen, *cov, ruIndex, (asm_opt.max_short_tip*2), 0.15, 3, 0.05, 0.9, b_mask_t); ug_ext_gfa(uopt, *sg, ug_ext_len); @@ -33902,7 +33987,7 @@ ma_sub_t **coverage_cut_ptr, int debug_g) **/ output_contig_graph_primary_pre(sg, coverage_cut, o_file, sources, reverse_sources, - asm_opt.small_pop_bubble_size, asm_opt.max_short_tip, ruIndex, max_hang_length, mini_overlap_length); + asm_opt.small_pop_bubble_size, asm_opt.max_short_tip, ruIndex, max_hang_length, mini_overlap_length, &uopt); /** if (asm_opt.flag & HA_F_VERBOSE_GFA) { diff --git a/Overlaps.h b/Overlaps.h index 22b66c1..cef917e 100644 --- a/Overlaps.h +++ b/Overlaps.h @@ -67,6 +67,22 @@ typedef struct { size_t n, m; } kv_ul_ov_t; +typedef struct { + uint32_t tn, rn, el; + uint32_t qs, qe, ts, te; + uint8_t dir:5, pe:1, full:1, rev:1; +} emask_t; + +typedef struct { + emask_t *a; + size_t n, m; +} kv_emask_t; + +typedef struct { + kv_emask_t *a; + uint32_t n; +} idx_emask_t; + typedef struct { ///off: start idx in mg128_t * a[]; ///cnt: how many eles in this chain @@ -118,8 +134,6 @@ void ma_hit_sort_qns(ma_hit_t *a, long long n); int load_all_data_from_disk(ma_hit_t_alloc **sources, ma_hit_t_alloc **reverse_sources, char* output_file_name); - - typedef struct { uint32_t s:31, del:1, e; uint8_t c; @@ -239,6 +253,7 @@ void print_gfa(asg_t *g); typedef struct { size_t n, m; uint64_t *a; } asg64_v; typedef struct { size_t n, m; uint32_t *a; } asg32_v; typedef struct { size_t n, m; ma_utg_t *a;} ma_utg_v; +typedef struct { asg64_v idx; kv_ul_ov_t srt;} mask_ul_ov_t; typedef struct { ma_utg_v u; @@ -294,9 +309,33 @@ typedef struct { hmap_t *mm; } hpc_t; +typedef struct { + uint32_t s, e; + uint8_t k; +} hpc_ss_t; + +typedef struct { + uint32_t s, e; +} hpc_idx_t; + +typedef struct { + size_t n, m; + hpc_ss_t *a; + hpc_idx_t *idx; + uint64_t idx_n; +} hpc_re_t; + #define hpc_len(x, id) ((x).hg->u.a[(id)].len>>1) #define hpc_str(x, id, rev) (((x).hg->u.a[(id)].s)+((rev)?((x).hg->u.a[(id)].len>>1):(0))) +typedef struct { + uint32_t n; + uint8_t *a; +} bit_mask_t; + +#define set_bit_mask_t(x, i) ((x).a[(i)>>3]|=(((uint8_t)1)<<((i)&((uint32_t)7)))) +#define get_bit_mask_t(x, i) ((x).a[(i)>>3]&(((uint8_t)1)<<((i)&((uint32_t)7)))) + typedef struct { ma_ug_t *ug; hpc_t *hpc_g; @@ -1162,6 +1201,8 @@ void set_reverse_overlap(ma_hit_t* dest, ma_hit_t* source); // int* b_low_cov, int* b_high_cov, double m_rate); void ma_hit_contained_advance(ma_hit_t_alloc* sources, long long n_read, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp); +void hic_clean_adv(asg_t *sg, ug_opt_t *uopt); +void update_ug_ou(ma_ug_t *ug, asg_t *sg); #define JUNK_COV 5 #define DISCARD_RATE 0.8 diff --git a/Process_Read.cpp b/Process_Read.cpp index e11fcc0..b973a75 100644 --- a/Process_Read.cpp +++ b/Process_Read.cpp @@ -764,6 +764,11 @@ void destory_all_ul_t(all_ul_t *x) { for (i = 0; i < x->nid.n; i++) free(x->nid.a[i].a); free(x->nid.a); free(x->ridx.idx.a); free(x->ridx.occ.a); + + // if(x->mm) { + // for (i = 0; i < x->mm->n; i++) free(x->mm->a[i].a); + // free(x->mm->a); free(x->mm); x->mm = NULL; + // } } void ha_encode_base(uint8_t* dest, char* src, uint64_t src_l, N_t *nn, uint64_t nn_offset) diff --git a/Process_Read.h b/Process_Read.h index d7cd042..fb56d5d 100644 --- a/Process_Read.h +++ b/Process_Read.h @@ -198,6 +198,7 @@ typedef struct ul_vec_t *a; size_t n, m; All_reads *hR; + // idx_emask_t *mm; // uint32_t mm; } all_ul_t; diff --git a/anchor.cpp b/anchor.cpp index 6722baf..2e79b39 100644 --- a/anchor.cpp +++ b/anchor.cpp @@ -1806,3 +1806,23 @@ int64_t ug_map_lchain(ha_abufl_t *ab, uint32_t rid, char* rs, uint64_t rl, uint6 return 0; } } + +int64_t ug_map_lchain_simple(ha_abufl_t *ab, uint32_t rid, char* rs, uint64_t rl, uint64_t mz_w, uint64_t mz_k, const ul_idx_t *uref, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, + int max_n_chain, int apend_be, kvec_t_u8_warp* k_flag, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, st_mt_t *sp, uint32_t *high_occ, uint32_t *low_occ, uint32_t is_accurate, + uint32_t gen_off, double mcopy_rate, uint32_t mcopy_khit_cut, uint32_t is_hpc, ha_mzl_t *res, uint64_t res_n, ha_mzl_t *idx, uint64_t idx_n, uint64_t mzl_cutoff, uint64_t chain_cutoff) +{ + int64_t max_skip, max_iter, max_dis, quick_check; double chn_pen_gap, chn_pen_skip; + set_lchain_dp_op(is_accurate, mz_k, &max_skip, &max_iter, &max_dis, &chn_pen_gap, &chn_pen_skip, &quick_check); + if((!overlap_list) || (!cl)) { + ab->mz.n = 0; + mz2_ha_sketch(rs, rl, mz_w, mz_k, rid, is_hpc, &ab->mz, NULL, asm_opt.mz_sample_dist, k_flag, dbg_ct, NULL, -1, asm_opt.dp_min_len, -1, sp, asm_opt.mz_rewin, 0, NULL); + if(res) memcpy(res, ab->mz.a, ab->mz.n * (sizeof((*(ab->mz.a))))); + return ab->mz.n; + } else { + sp->n = 0; + minimizers_qgen_input(ab, rid, rs, rl, mz_w, mz_k, cl, k_flag, ha_flt_tab, ha_idx, NULL, uref, dbg_ct, sp, high_occ, low_occ, res, res_n, idx, idx_n, mzl_cutoff, chain_cutoff, NULL); + // lchain_qgen_mcopy_input(cl, overlap_list, rid, rl, NULL, uref, apend_be, max_n_chain, max_skip, max_iter, max_dis, chn_pen_gap, chn_pen_skip, bw_thres, bw_thres_sec, quick_check, gen_off, mcopy_rate, mcopy_khit_cut, sp); + lchain_qgen_mcopy(cl, overlap_list, rid, rl, NULL, uref, apend_be, max_n_chain, max_skip, max_iter, max_dis, chn_pen_gap, chn_pen_skip, bw_thres, quick_check, gen_off, mcopy_rate, chain_cutoff, mcopy_khit_cut, sp); + return 0; + } +} \ No newline at end of file diff --git a/gfa_ut.cpp b/gfa_ut.cpp index 641f3ee..f06235f 100644 --- a/gfa_ut.cpp +++ b/gfa_ut.cpp @@ -380,6 +380,25 @@ typedef struct{ // ul_path_srt_t psrt; }ul_resolve_t; +typedef struct{ + ug_opt_t *uopt; + asg_t *sg; + ma_ug_t *ug; + buf_t b; + uint32_t *idx, *bid; + uint32_t idx_n, bid_n; + uint8_t is_ou; + uint8_t is_trio; + int64_t max_ext; + uint64_t tlen; + double len_rat; + double ou_rat; + int64_t min_ou; +}ug_clean_t; + +void deep_graph_clean(ug_opt_t *uopt, asg_t *sg, uint8_t is_ou, uint8_t is_trio, int64_t max_ext, +double min_ovlp_drop_ratio, double max_ovlp_drop_ratio, double ou_rat, int64_t min_ou, int64_t clean_round, int64_t long_tip); + void init_integer_ml_t(integer_ml_t *x, ul_resolve_t *u, uint64_t n_thread) { memset(x, 0, sizeof((*x))); @@ -1368,6 +1387,44 @@ uint32_t is_topo, ma_hit_t_alloc *rev, R_to_U* rI, uint32_t *max_drop_len) if (cnt > 0) asg_cleanup(g); } +uint8_t get_ug_tip_trio_infor(ma_ug_t *ug, uint32_t begNode) +{ + uint32_t v = begNode, w, k, s; + uint32_t kv; + uint32_t eLen = 0, uLen = 0; + uint32_t father_occ = 0, mother_occ = 0, ambigious_occ = 0; + ma_utg_t *u; + + while (1) { + kv = get_real_length(ug->g, v, NULL); + u = &(ug->u.a[v>>1]); eLen += u->n; + + for (k = 0; k < u->n; k++) { + s = u->a[k]>>33; + if(R_INF.trio_flag[s]==FATHER) father_occ++; + else if(R_INF.trio_flag[s]==MOTHER) mother_occ++; + else if((R_INF.trio_flag[s]==AMBIGU) || (R_INF.trio_flag[s]==DROP)) ambigious_occ++; + } + if(kv!=1) break; + ///kv must be 1 here + kv = get_real_length(ug->g, v, &w); + if(get_real_length(ug->g, w^1, NULL)!=1) break; + v = w; + if(v == begNode) break; + } + + uLen = eLen;///how many nodes + eLen = father_occ + mother_occ;///haplotype-sepcific nodes + if(eLen == 0) return AMBIGU; + if(father_occ >= mother_occ) { + if((father_occ > TRIO_THRES*eLen) && (father_occ >= DOUBLE_CHECK_THRES*uLen)) return FATHER; + } else { + if((mother_occ > TRIO_THRES*eLen) && (mother_occ >= DOUBLE_CHECK_THRES*uLen)) return MOTHER; + } + return AMBIGU; +} + + asg_arc_t *iter_flex_asg(flex_asg_t *fg, flex_asg_e_retrive_t *rr, uint32_t v) { asg_arc_t *av; uint32_t nv; asg_arc_t *z; @@ -2255,31 +2312,65 @@ void print_node(asg_t *sg, uint32_t src) } } -void print_vw_edge(asg_t *sg, uint32_t v, uint32_t w, const char *cmd) +void print_vw_edge(asg_t *sg, uint32_t vid, uint32_t wid, const char *cmd) { - asg_arc_t *av; uint32_t nv, i; - av = asg_arc_a(sg, v); nv = asg_arc_n(sg, v); + asg_arc_t *av; uint32_t nv, i, sid, eid; + if(vid >= sg->n_seq || wid >= sg->n_seq) return; + + sid = vid; eid = wid; + av = asg_arc_a(sg, (sid<<1)); nv = asg_arc_n(sg, (sid<<1)); for (i = 0; i < nv; i++) { - if(av[i].v == w) { + if((av[i].v>>1) == eid) { fprintf(stderr, "[%s]\t%.*s(%c)\t%.*s(%c)\tol:%u\tou:%u\tdel:%u\n", cmd, (int32_t)Get_NAME_LENGTH(R_INF, (av[i].ul>>33)), Get_NAME(R_INF, (av[i].ul>>33)), "+-"[(av[i].ul>>32)&1], av[i].ul>>33, (int32_t)Get_NAME_LENGTH(R_INF, (av[i].v>>1)), Get_NAME(R_INF, (av[i].v>>1)), "+-"[av[i].v&1], av[i].v>>1, av[i].ol, av[i].ou, av[i].del); break; } } - if(i >= nv) fprintf(stderr, "[%s]\tno edges\n", cmd); + av = asg_arc_a(sg, ((sid<<1)+1)); nv = asg_arc_n(sg, ((sid<<1)+1)); + for (i = 0; i < nv; i++) { + if((av[i].v>>1) == eid) { + fprintf(stderr, "[%s]\t%.*s(%c)\t%.*s(%c)\tol:%u\tou:%u\tdel:%u\n", cmd, + (int32_t)Get_NAME_LENGTH(R_INF, (av[i].ul>>33)), Get_NAME(R_INF, (av[i].ul>>33)), "+-"[(av[i].ul>>32)&1], av[i].ul>>33, + (int32_t)Get_NAME_LENGTH(R_INF, (av[i].v>>1)), Get_NAME(R_INF, (av[i].v>>1)), "+-"[av[i].v&1], av[i].v>>1, av[i].ol, av[i].ou, av[i].del); + break; + } + } + + sid = wid; eid = vid; + av = asg_arc_a(sg, (sid<<1)); nv = asg_arc_n(sg, (sid<<1)); + for (i = 0; i < nv; i++) { + if((av[i].v>>1) == eid) { + fprintf(stderr, "[%s]\t%.*s(%c)\t%.*s(%c)\tol:%u\tou:%u\tdel:%u\n", cmd, + (int32_t)Get_NAME_LENGTH(R_INF, (av[i].ul>>33)), Get_NAME(R_INF, (av[i].ul>>33)), "+-"[(av[i].ul>>32)&1], av[i].ul>>33, + (int32_t)Get_NAME_LENGTH(R_INF, (av[i].v>>1)), Get_NAME(R_INF, (av[i].v>>1)), "+-"[av[i].v&1], av[i].v>>1, av[i].ol, av[i].ou, av[i].del); + break; + } + } + av = asg_arc_a(sg, ((sid<<1)+1)); nv = asg_arc_n(sg, ((sid<<1)+1)); + for (i = 0; i < nv; i++) { + if((av[i].v>>1) == eid) { + fprintf(stderr, "[%s]\t%.*s(%c)\t%.*s(%c)\tol:%u\tou:%u\tdel:%u\n", cmd, + (int32_t)Get_NAME_LENGTH(R_INF, (av[i].ul>>33)), Get_NAME(R_INF, (av[i].ul>>33)), "+-"[(av[i].ul>>32)&1], av[i].ul>>33, + (int32_t)Get_NAME_LENGTH(R_INF, (av[i].v>>1)), Get_NAME(R_INF, (av[i].v>>1)), "+-"[av[i].v&1], av[i].v>>1, av[i].ol, av[i].ou, av[i].del); + break; + } + } + // if(i >= nv) fprintf(stderr, "[%s]\tno edges\n", cmd); } -void prt_spec_edge(asg_t *rg, ma_hit_t_alloc *src, uint32_t tot_rid, uint32_t vid, uint32_t wid, const char *cmd) +void prt_spec_edge(asg_t *rg, ma_hit_t_alloc *src, uint32_t tot_rid, uint32_t vid, uint32_t wid, ug_opt_t *uopt, const char *cmd) { if(vid >= tot_rid) return; - uint32_t k, qn, tn; ma_hit_t_alloc *s = &(src[vid]); + uint32_t k, qn, tn; int32_t r; asg_arc_t p; ma_hit_t_alloc *s = &(src[vid]); for (k = 0; k < s->length; k++) { qn = Get_qn(s->buffer[k]); tn = Get_tn(s->buffer[k]); if(qn == vid && tn == wid) { - fprintf(stderr, "[M::%s]%s\tqn::%u\tq::[%u,\t%u)\t%c\ttn::%u\tt::[%u,\t%u)\n", - __func__, cmd, qn, Get_qs(s->buffer[k]), Get_qe(s->buffer[k]), - "+-"[s->buffer[k].rev], tn, Get_ts(s->buffer[k]), Get_te(s->buffer[k])); + r = ma_hit2arc(&(s->buffer[k]), rg->seq[qn].len, rg->seq[tn].len, uopt->max_hang, asm_opt.max_hang_rate, uopt->min_ovlp, &p); + fprintf(stderr, "[M::%s::]%s\tqn::%u(%c)\tq::[%u,\t%u)\t%c\ttn::%u(%c)\tt::[%u,\t%u)\n", + __func__, cmd, qn, (r>=0)?("+-"[(p.ul>>32)&1]):('*'), Get_qs(s->buffer[k]), Get_qe(s->buffer[k]), + "+-"[s->buffer[k].rev], + tn, (r>=0)?("+-"[p.v&1]):('*'), Get_ts(s->buffer[k]), Get_te(s->buffer[k])); } } } @@ -2348,12 +2439,27 @@ void filter_sg_by_ug(asg_t *rg, ma_ug_t *ug, ug_opt_t *uopt) // r = gen_spec_edge(rg, uopt, wx^1, vx^1, &t); // assert(r >= 0); p = asg_arc_pushp(rg); *p = t; } + + if(u->circ) { + v = w = i<<1; + vx = (v&1?((ug->u.a[v>>1].a[0]>>32)^1):(ug->u.a[v>>1].a[ug->u.a[v>>1].n-1]>>32)); + wx = (w&1?((ug->u.a[w>>1].a[ug->u.a[w>>1].n-1]>>32)^1):(ug->u.a[w>>1].a[0]>>32)); + r = gen_spec_edge(rg, uopt, vx, wx, &t); + assert(r >= 0); p = asg_arc_pushp(rg); *p = t; + + v = w = (i<<1)^1; + vx = (v&1?((ug->u.a[v>>1].a[0]>>32)^1):(ug->u.a[v>>1].a[ug->u.a[v>>1].n-1]>>32)); + wx = (w&1?((ug->u.a[w>>1].a[ug->u.a[w>>1].n-1]>>32)^1):(ug->u.a[w>>1].a[0]>>32)); + r = gen_spec_edge(rg, uopt, vx, wx, &t); + assert(r >= 0); p = asg_arc_pushp(rg); *p = t; + } } free(rg->idx); rg->idx = 0; rg->is_srt = 0; asg_cleanup(rg); + asg_symm(rg); /*******************************for debug************************************/ @@ -5823,10 +5929,6 @@ void update_raw_integer_seq(poa_g_t *pg, ma_ug_t *ug, uint32_t *cns_seq, uint32_ void integer_candidate(ul_resolve_t *uidx, integer_t *buf, uint32_t qid, uint32_t is_hom) { - // if(qid != 3165) return; - // if(qid != 17165) return; - // if(qid != 24100) return;///circle - // if(qid != 27512) return; uint64_t k, z, m_het, m_het_occ, ref_occ, b_n, m; uint32_t vk, vz, is_circle = 0; integer_aln_t *p; ul_chain_t sc; ul_str_idx_t *str_idx = &(uidx->pstr); ma_ug_t *ug = uidx->l1_ug; uint64_t *hid_a, hid_n; uc_block_t *xi; @@ -5839,7 +5941,7 @@ void integer_candidate(ul_resolve_t *uidx, integer_t *buf, uint32_t qid, uint32_ buf->u.a[k] = ug_occ_w(xi->ts, xi->te, &(ug->u.a[xi->hid])); assert(buf->u.a[k] > 0); if((!is_hom) && (!IF_HOM((((uint32_t)str->a[k])>>1), (*uidx->bub)))) { - m_het++; m_het_occ += buf->u.a[k]; + m_het++; m_het_occ += buf->u.a[k];///m_het_occ: how many het HiFi reads } ref_occ += buf->u.a[k]; // fprintf(stderr, "[M::%s::k->%lu] buf->u.a[k]->%lu, ts->%u, te->%u, pchain->%u\n", __func__, k, buf->u.a[k], xi->ts, xi->te, xi->pchain); @@ -14344,6 +14446,11 @@ asg_t *renew_ng(usg_t *eg, ma_ug_t *rug, asg_t *sg, ug_opt_t *uopt, ma_sub_t **c memset(ext.idx_a.a, -1, sizeof(*(ext.idx_a.a))*ext.idx_a.n);//map ug1 = convert_usg_t(eg, rug); + // fprintf(stderr, "-0-[M::%s]\n", __func__); + // print_vw_edge(sg, 681356, 73351, __func__); + // prt_spec_edge(sg, R_INF.paf, R_INF.total_reads, 681356, 73351, uopt, "src"); + // // prt_spec_edge(sg, R_INF.reverse_paf, R_INF.total_reads, 681356, 73351, uopt, "rsrc"); + for (i = 0; i < ug1->u.n; ++i) { if(ug1->g->seq[i].del) continue; u = &(ug1->u.a[i]); @@ -14375,6 +14482,7 @@ asg_t *renew_ng(usg_t *eg, ma_ug_t *rug, asg_t *sg, ug_opt_t *uopt, ma_sub_t **c ng->r_seq = ng->n_seq; assert(ng->n_seq == ext.idx_a.n); + for (i = 0, nocc = ng->r_seq, sa.n = 0; i < eg->n; ++i) { if(ug1->g->seq[i].del) continue; ///there shouldn't any scaffolding within the nodes @@ -14435,11 +14543,9 @@ asg_t *renew_ng(usg_t *eg, ma_ug_t *rug, asg_t *sg, ug_opt_t *uopt, ma_sub_t **c CALLOC(ng->seq_vis, (ng->n_seq<<1)); ///# scaffolding nodes if(sa.n > 0) fill_scaffolds(sa.a, sa.n, rug, uopt->min_ovlp, bin_file); - - realloc_rdb_adv(&(R_INF), &UL_INF, cov, ruI, ext.idx_a.a, ext.idx_a.n, scaffold_len, (char *)"scaf", ng, uopt, rug, sa.a); + realloc_rdb_adv(&(R_INF), &UL_INF, cov, ruI, ext.idx_a.a, ext.idx_a.n, scaffold_len, (char *)"scaf", ng, uopt, rug, sa.a); update_paf(R_INF.paf, R_INF.reverse_paf, ext.idx_a.a, ext.idx_a.n, sg->n_seq, ng); - for (i = 0, nocc = ng->r_seq; i < eg->n; ++i) { if(ug1->g->seq[i].del) continue; u = &(ug1->u.a[i]); @@ -14542,6 +14648,7 @@ asg_t *renew_ng(usg_t *eg, ma_ug_t *rug, asg_t *sg, ug_opt_t *uopt, ma_sub_t **c asg_cleanup(ng); ng->r_seq = ng->n_seq; free(ext.cnt.a); free(ext.idx_a.a); free(sa.a); ma_ug_destroy(ug1); + fprintf(stderr, "[M::%s] nocc::%lu, ng->n_seq::%u, sg->n_seq::%u\n", __func__, nocc, (uint32_t)ng->n_seq, (uint32_t)sg->n_seq); assert(nocc == ng->n_seq); @@ -15350,7 +15457,7 @@ ma_ug_t *gen_hybrid_ug(ul_resolve_t *uidx, usg_t *ng) void renew_ul2_utg(ul_resolve_t *uidx); -void u2g_threading(ul_resolve_t *uidx, ulg_opt_t *ulopt, uint64_t cov_cutoff, asg64_v *b, asg64_v *ub) +void u2g_threading(ul_resolve_t *uidx, ulg_opt_t *ulopt, uint64_t cov_cutoff, uint64_t is_bridg, asg64_v *b, asg64_v *ub) { renew_ul2_utg(uidx); ul2ul_idx_t *idx = &(uidx->uovl); ma_ug_t *iug = idx->i_ug; ma_ug_t *raw = uidx->l1_ug; @@ -15425,7 +15532,7 @@ void u2g_threading(ul_resolve_t *uidx, ulg_opt_t *ulopt, uint64_t cov_cutoff, as if(p->ou < (uint64_t)tm) p->ou = tm; p = get_usg_arc(ng, seq->a[i+1].v^1, seq->a[i].v^1); if(p->ou < (uint64_t)tm) p->ou = tm; - } else { + } else if(is_bridg) { v = seq->a[i].v; w = seq->a[i+1].v; p = NULL; vx = (v&1?((raw->u.a[v>>1].a[0]>>32)^1):(raw->u.a[v>>1].a[raw->u.a[v>>1].n-1]>>32)); wx = (w&1?((raw->u.a[w>>1].a[raw->u.a[w>>1].n-1]>>32)^1):(raw->u.a[w>>1].a[0]>>32)); @@ -15480,7 +15587,7 @@ void u2g_threading(ul_resolve_t *uidx, ulg_opt_t *ulopt, uint64_t cov_cutoff, as // fprintf(stderr, "-[M::%s::] idx->hybrid_ug->g->n_seq::%u\n", __func__, (uint32_t)idx->hybrid_ug->g->n_seq); } -void u2g_clean(ul_resolve_t *uidx, ulg_opt_t *ulopt, uint32_t keep_raw_utg) +void u2g_clean(ul_resolve_t *uidx, ulg_opt_t *ulopt, uint32_t keep_raw_utg, uint64_t is_bridg) { ul2ul_idx_t *idx = &(uidx->uovl); asg64_v bu = {0,0,0}, uu = {0,0,0}; int64_t max_tip_hifi0 = ulopt->max_tip_hifi; ma_ug_t *iug = idx->i_ug; int64_t i, mm_tip = ulopt->max_tip; uint64_t cnt = 1, topo_level, ss = 0; @@ -15527,7 +15634,7 @@ void u2g_clean(ul_resolve_t *uidx, ulg_opt_t *ulopt, uint32_t keep_raw_utg) } ulopt->max_tip_hifi = max_tip_hifi0; - u2g_threading(uidx, ulopt, 3, &bu, &uu); + u2g_threading(uidx, ulopt, 3, is_bridg, &bu, &uu); // fill_u2g(uidx, &bu, &uu); @@ -15995,7 +16102,7 @@ void renew_ul2_utg(ul_resolve_t *uidx) } -ul2ul_idx_t *gen_ul2ul(ul_resolve_t *uidx, ug_opt_t *uopt, ulg_opt_t *ulopt, uint32_t keep_raw_utg) +ul2ul_idx_t *gen_ul2ul(ul_resolve_t *uidx, ug_opt_t *uopt, ulg_opt_t *ulopt, uint32_t keep_raw_utg, uint64_t is_bridg) { uint64_t k, m; ma_ug_t *ug = uidx->l1_ug; all_ul_t *uls = uidx->idx; @@ -16039,7 +16146,7 @@ ul2ul_idx_t *gen_ul2ul(ul_resolve_t *uidx, ug_opt_t *uopt, ulg_opt_t *ulopt, uin renew_u2g_bg(uidx); // output_integer_graph(uidx, z->i_ug, asm_opt.output_file_name); - u2g_clean(uidx, ulopt, keep_raw_utg); + u2g_clean(uidx, ulopt, keep_raw_utg, is_bridg); // renew_ul2_utg(uidx); // output_integer_graph(uidx, z->i_ug, asm_opt.output_file_name, 0); @@ -16763,7 +16870,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp) void ul_realignment_gfa(ug_opt_t *uopt, asg_t *sg, int64_t clean_round, double min_ovlp_drop_ratio, double max_ovlp_drop_ratio, int64_t max_tip, int64_t max_ul_tip, bub_label_t *b_mask_t, uint32_t is_trio, char *o_file, -ul_renew_t *ropt, const char *bin_file, uint64_t free_uld) +ul_renew_t *ropt, const char *bin_file, uint64_t free_uld, uint64_t is_bridg, uint64_t deep_clean) { uint64_t i, bn = 0, idn = 0; uint32_t *bl = NULL; uint8_t *r_het = NULL; bubble_type *bub = NULL; ulg_opt_t uu; for (i = 0; i < sg->n_seq; ++i) { @@ -16772,10 +16879,25 @@ ul_renew_t *ropt, const char *bin_file, uint64_t free_uld) } // fprintf(stderr, "0[M::%s]\n", __func__); hic_clean(sg); + if(deep_clean) { + // print_debug_gfa(sg, NULL, uopt->coverage_cut, "bclean", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); + deep_graph_clean(uopt, sg, 1, is_trio, asm_opt.max_short_tip, asm_opt.min_drop_rate, + MIN(asm_opt.max_drop_rate, 0.7), 0.75, 1, asm_opt.clean_round, 20); + // print_debug_gfa(sg, NULL, uopt->coverage_cut, "aclean", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); + } + // fprintf(stderr, "1[M::%s]\n", __func__); ma_ug_t *init_ug = ul_realignment(uopt, sg, 0, bin_file); // fprintf(stderr, "2[M::%s]\n", __func__); // exit(1); + + // char* gfa_name = NULL; MALLOC(gfa_name, strlen(o_file)+strlen(bin_file)+50); + // sprintf(gfa_name, "%s.%s", o_file, bin_file); + // print_debug_gfa(sg, init_ug, uopt->coverage_cut, gfa_name, uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); + // // print_debug_gfa(sg, init_ug, uopt->coverage_cut, gfa_name, uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 1); + // free(gfa_name); + + filter_sg_by_ug(sg, init_ug, uopt); // fprintf(stderr, "-0-[M::%s]\tUL_INF.a[25].rlen::%u\n", __func__, UL_INF.a[25].rlen); // print_debug_gfa(sg, init_ug, uopt->coverage_cut, "UL.debug", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); @@ -16791,12 +16913,15 @@ ul_renew_t *ropt, const char *bin_file, uint64_t free_uld) // fprintf(stderr, "5[M::%s]\n", __func__); // print_ul_alignment(init_ug, &UL_INF, 47072, "after-2"); // exit(1); - // print_raw_uls_seq(uidx, asm_opt.output_file_name); - // print_raw_uls_aln(uidx, asm_opt.output_file_name); + // if(free_uld) { + // print_raw_uls_seq(uidx, asm_opt.output_file_name); + // print_raw_uls_aln(uidx, asm_opt.output_file_name); + // } + ul_re_correct(uidx, 3); init_ulg_opt_t(&uu, uopt, clean_round, min_ovlp_drop_ratio, max_ovlp_drop_ratio, 0.55, max_tip, max_ul_tip, b_mask_t, is_trio); // print_debug_gfa(sg, init_ug, uopt->coverage_cut, "UL.debug0", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 1); - /**ul2ul_idx_t *u2o = **/gen_ul2ul(uidx, uopt, &uu, 0); + /**ul2ul_idx_t *u2o = **/gen_ul2ul(uidx, uopt, &uu, 0, is_bridg); // print_ul_alignment(init_ug, &UL_INF, 47072, "after-3"); // print_debug_ul("UL.debug", init_ug, sg, uopt->coverage_cut, uopt->sources, uopt->ruIndex, bub, &UL_INF); @@ -16804,11 +16929,12 @@ ul_renew_t *ropt, const char *bin_file, uint64_t free_uld) // resolve_dip_bub_chains(uidx); // free(r_het); destory_bubbles(bub); free(bub); - // uidx->uovl.hybrid_ug = gen_hybrid_ug(uidx, uidx->uovl.h_usg); - // print_debug_gfa(sg, uidx->uovl.hybrid_ug, uopt->coverage_cut, "hybrid_ug", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 1); - // print_debug_gfa(sg, uidx->uovl.hybrid_ug, uopt->coverage_cut, "hybrid_ug", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); - // print_debug_gfa(sg, init_ug, uopt->coverage_cut, bin_file, uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); - + // if(free_uld) { + // uidx->uovl.hybrid_ug = gen_hybrid_ug(uidx, uidx->uovl.h_usg); + // // print_debug_gfa(sg, uidx->uovl.hybrid_ug, uopt->coverage_cut, "hybrid_ug", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 1); + // print_debug_gfa(sg, uidx->uovl.hybrid_ug, uopt->coverage_cut, "hybrid_ug", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); + // // print_debug_gfa(sg, init_ug, uopt->coverage_cut, bin_file, uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); + // } // if(is_trio) gen_ul_trio_graph(uopt, uidx, o_file); // exit(0); // return uidx->uovl.hybrid_ug; @@ -16839,4 +16965,504 @@ ul_renew_t *ropt, const char *bin_file, uint64_t free_uld) post_rescue(uopt, (*(ropt->sg)), (*(ropt->src)), (*(ropt->r_src)), ropt->ruIndex, ropt->b_mask_t, 0); // print_raw_uls_aln(uidx, asm_opt.output_file_name); // exit(0); -} \ No newline at end of file +} + +ug_clean_t *init_ug_clean_t(ug_opt_t *uopt, asg_t *sg, uint8_t is_ou, uint8_t is_trio, int64_t max_ext, double len_rat, double ou_rat, int64_t min_ou) +{ + uint64_t i; + ug_clean_t *sl; CALLOC(sl, 1); + ma_ug_t *ug = ma_ug_gen(sg); + for (i = 0; i < ug->g->n_seq; ++i) { + if(ug->g->seq[i].del) continue; + ug->g->seq[i].c = PRIMARY_LABLE; + } + MALLOC(sl->idx, (ug->g->n_seq<<1)); memset(sl->idx, -1, sizeof((*(sl->idx)))*(ug->g->n_seq<<1)); + MALLOC(sl->bid, (ug->g->n_seq<<1)); memset(sl->bid, -1, sizeof((*(sl->bid)))*(ug->g->n_seq<<1)); + sl->uopt = uopt; sl->sg = sg; sl->ug = ug; sl->idx_n = sl->bid_n = ug->g->n_seq<<1; + memset(&(sl->b), 0, sizeof(sl->b)); CALLOC(sl->b.a, (ug->g->n_seq<<1)); + sl->is_ou = is_ou; + sl->is_trio = is_trio; + sl->max_ext = max_ext; + sl->len_rat = len_rat; + sl->ou_rat = ou_rat; + sl->min_ou = min_ou; + if(is_ou) update_ug_ou(sl->ug, sl->sg); + return sl; +} + +void destroy_ug_clean_t(ug_clean_t *sl) +{ + free(sl->idx); free(sl->bid); ma_ug_destroy(sl->ug); + free(sl->b.a); free(sl->b.S.a); free(sl->b.T.a); free(sl->b.b.a); free(sl->b.e.a); +} + +void update_ug_clean_t(ug_clean_t *sl) +{ + uint32_t v, k, i, n_vtx = sl->ug->g->n_seq<<1; + sl->tlen = get_bub_pop_max_dist_advance(sl->ug->g, &(sl->b)); + for (k = 0; k < sl->ug->g->n_seq; ++k) { + sl->idx[k<<1] = sl->idx[(k<<1)+1] = sl->bid[k<<1] = sl->bid[(k<<1)+1] = (uint32_t)-1; + if(sl->ug->g->seq[k].del) continue; + sl->ug->g->seq[k].c = PRIMARY_LABLE; + } + + for (v = 0; v < n_vtx; ++v) { + if(sl->ug->g->seq[v>>1].del) continue; + if(asg_arc_n(sl->ug->g, v) < 2) continue; + if(sl->bid[v] != ((uint32_t)-1)) continue; + if(asg_bub_pop1_primary_trio(sl->ug->g, NULL, v, sl->tlen, &(sl->b), (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL, 0, 0, NULL)) { + //beg is v, end is b.S.a[0] + //note b.b include end, does not include beg + for (i = 0; i < sl->b.b.n; i++) { + if(sl->b.b.a[i]==v || sl->b.b.a[i]==sl->b.S.a[0]) continue; + sl->bid[sl->b.b.a[i]] = sl->bid[sl->b.b.a[i]^1] = 1; + } + sl->bid[v] = 2; sl->bid[sl->b.S.a[0]^1] = 3; + } + } + + for (v = 0; v < n_vtx; ++v) { + if(sl->bid[v] !=2) continue; + if(asg_bub_pop1_primary_trio(sl->ug->g, NULL, v, sl->tlen, &(sl->b), (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL, NULL, 0, 0, NULL)) { + //note b.b include end, does not include beg + for (i = 0; i < sl->b.b.n; i++) { + if(sl->b.b.a[i]==v || sl->b.b.a[i]==sl->b.S.a[0]) continue; + sl->idx[sl->b.b.a[i]] = v; + sl->idx[sl->b.b.a[i]^1] = sl->b.S.a[0]^1; + } + sl->idx[v] = v; sl->idx[sl->b.S.a[0]^1] = sl->b.S.a[0]^1; + } + } + memcpy(sl->bid, sl->idx, sizeof((*(sl->bid)))*n_vtx); + // memset(sl->idx, -1, sizeof((*(sl->idx)))*n_vtx); +} + +uint32_t get_best_het_node(ma_ug_t *ug, uint32_t v, uint32_t *bid, uint8_t is_ou, uint8_t is_trio, double len_rat, double ou_rat, uint32_t min_ou) +{ + uint32_t w = (uint32_t)-1, k, nv, kv, ol_max, ol_k; + asg_arc_t *av; uint32_t trioF = (uint32_t)-1, ntrioF = (uint32_t)-1; + if(ug->g->seq[v>>1].del) return (uint32_t)-1; + av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (k = kv = 0; k < nv; k++) { + if(av[k].del) continue; + ///could not connect to the beg/sink node + if((av[k].v>>1)==(bid[v]>>1) || (av[k].v>>1)==(bid[v^1]>>1)) return (uint32_t)-1; + kv++; w = av[k].v; + } + if(kv < 2) return w;///kv == 0, return (uint32_t)-1; kv == 1, return node; + // if(v == 351) fprintf(stderr, "-0-[M::%s]\tutg%.6ul(%c)\tv::%u\tkv::%u\n", __func__, (v>>1)+1, "+-"[v&1], v, kv); + + if(is_trio) { + trioF = get_ug_tip_trio_infor(ug, v^1); + ntrioF = (trioF==FATHER? MOTHER : (trioF==MOTHER? FATHER : (uint32_t)-1)); + } + + ol_max = 0; ol_k = (uint32_t)-1; + for (k = 0; k < nv; ++k) { + if(av[k].del) continue; + // if(is_trio && get_ug_tip_trio_infor(ug, av[k].v) == ntrioF) continue; + if(ol_max < av[k].ol) ol_max = av[k].ol, ol_k = k; + } + // if(v == 351) fprintf(stderr, "-1-[M::%s]\tutg%.6ul(%c)\tv::%u\tol_k::%u\n", __func__, (v>>1)+1, "+-"[v&1], v, ol_k); + if(ol_k == (uint32_t)-1) return (uint32_t)-1; + for (k = 0; k < nv; ++k) { + if(av[k].del || k == ol_k) continue; + // if(is_trio && get_ug_tip_trio_infor(ug, av[k].v) == ntrioF) continue; + if(av[k].ol > av[ol_k].ol*len_rat) return (uint32_t)-1; + if((is_ou) && (av[k].ou > min_ou) && ((av[k].ou) > (av[ol_k].ou*ou_rat))) return (uint32_t)-1; + } + if(is_trio && get_ug_tip_trio_infor(ug, av[ol_k].v) == ntrioF) return (uint32_t)-1; + // if(v == 351) fprintf(stderr, "-2-[M::%s]\tutg%.6ul(%c)\tv::%u\tav[ol_k].v::%u\n", __func__, (v>>1)+1, "+-"[v&1], v, av[ol_k].v); + return av[ol_k].v; +} + +static void cal_bub_best(void *data, long i, int tid) +{ + ug_clean_t *sl = (ug_clean_t *)data; + ma_ug_t *ug = sl->ug; uint32_t v, w; + sl->idx[i<<1] = sl->idx[(i<<1)+1] = (uint32_t)-1; + if(ug->g->seq[i].del) return; + if((sl->bid[i<<1] == (uint32_t)-1) || (sl->bid[(i<<1)+1] == (uint32_t)-1)) return;///not within a bubble + + v = i<<1; + if(sl->bid[v] != v) {///not the beg/sink node + w = get_best_het_node(ug, v, sl->bid, sl->is_ou, sl->is_trio, sl->len_rat, sl->ou_rat, sl->min_ou); + // if(v == 352) fprintf(stderr, "-v-[M::%s]\tutg%.6ul(%c)\tv::%u\tw::%u\n", __func__, (v>>1)+1, "+-"[v&1], v, w); + if((w != (uint32_t)-1) && ((v^1) == get_best_het_node(ug, w^1, sl->bid, sl->is_ou, sl->is_trio, sl->len_rat, sl->ou_rat, sl->min_ou))) { + sl->idx[v] = w; + } + } + + v = (i<<1)+1; + if(sl->bid[v] != v) {///not the beg/sink node + w = get_best_het_node(ug, v, sl->bid, sl->is_ou, sl->is_trio, sl->len_rat, sl->ou_rat, sl->min_ou); + if((w != (uint32_t)-1) && ((v^1) == get_best_het_node(ug, w^1, sl->bid, sl->is_ou, sl->is_trio, sl->len_rat, sl->ou_rat, sl->min_ou))) { + sl->idx[v] = w; + } + } +} + +uint32_t usg_topocut_aux_unambi1(asg_t *g, uint32_t v) +{ + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, nv = asg_arc_n(g, v); + uint32_t k = nv, kv; + for (i = 0, kv = 0; i < nv; ++i) + if (!av[i].del) ++kv, k = i; + if (kv != 1) return (uint32_t)-1; + return av[k].v; +} + +int usg_topocut_aux_del(ma_ug_t *ug, uint32_t v, int max_ext, asg64_v *b) +{ + int32_t n_ext; + for (n_ext = 1; n_ext < max_ext && v != (uint32_t)-1; ++n_ext) { + if (usg_topocut_aux_unambi1(ug->g, v^1) == (uint32_t)-1) { + --n_ext; + break; + } + kv_push(uint64_t, *b, v); + v = usg_topocut_aux_unambi1(ug->g, v); + } + return n_ext; +} + +#define is_best_arc(sl, v, w) (((sl).idx[(v)]==(w))&&((sl).idx[((w)^1)]==((v)^1))) + +inline void asg_arc_del_by_ug(asg_t *sg, ma_ug_t *ug, uint32_t uv, uint32_t uw, uint32_t del) +{ + uint32_t rv, rw, i, nv; asg_arc_t *av; + if(uv&1) rv = ug->u.a[uv>>1].start^1; + else rv = ug->u.a[uv>>1].end^1; + + if(uw&1) rw = ug->u.a[uw>>1].end; + else rw = ug->u.a[uw>>1].start; + + av = asg_arc_a(sg, rv); + nv = asg_arc_n(sg, rv); + for (i = 0; i < nv; i++) { + if(av[i].v == rw) break; + } + assert(i < nv); + av[i].del = del; +} + +void cal_bub_best_by_len(ug_clean_t *sl, asg64_v *in, uint32_t max_ext, uint32_t is_trio, uint32_t is_ou, double len_rat, double ou_rat, uint32_t min_ou) +{ + // fprintf(stderr, "[M::%s]\tStart\n", __func__); + ma_ug_t *ug = sl->ug; asg_t *g = sl->ug->g; uint32_t ol_max, ou_max; + uint32_t v, w, n_vtx = (g->n_seq<<1), nv, nw, i, k, z, kv, kw, bb, to_del, bn, tip; + asg64_v tx = {0,0,0}, *b = NULL; asg_arc_t *av, *aw, *ve, *we; ma_utg_t *u; + uint32_t trioF = (uint32_t)-1, ntrioF = (uint32_t)-1, mm_ol, mm_ou, cnt = 0, del_v, del_w; + + if(in) b = in; + else b = &tx; + b->n = 0; + + for (v = 0; v < n_vtx; ++v) { + if (g->seq[v>>1].del) continue; + av = asg_arc_a(g, v); + nv = asg_arc_n(g, v); + if (nv < 2) continue; + + for (i = kv = bb = 0; i < nv; ++i) { + if(av[i].del) continue; + if(is_best_arc((*sl), v, av[i].v)) bb++; + kv++; + } + if((kv < 2) || (!bb) || (kv<=bb)) continue;///it is impossible that kv <= bv + + for (i = 0; i < nv; ++i) { + if(av[i].del) continue; + if(is_best_arc((*sl), v, av[i].v)) continue; + kv_push(uint64_t, *b, (((uint64_t)av[i].ol)<<32) | ((uint64_t)(av-g->arc+i))); + } + } + + radix_sort_srt64(b->a, b->a + b->n); + for (k = 0; k < b->n; k++) { + if(g->arc[(uint32_t)b->a[k]].del) continue; + + v = g->arc[(uint32_t)b->a[k]].ul>>32; w = g->arc[(uint32_t)b->a[k]].v^1; + if(g->seq[v>>1].del || g->seq[w>>1].del) continue; + nv = asg_arc_n(g, v); nw = asg_arc_n(g, w); + av = asg_arc_a(g, v); aw = asg_arc_a(g, w); + if(nv<=1 && nw <= 1) continue; + + if(is_trio) { + if(get_arcs(g, v, NULL, 0)<=1 && get_arcs(g, w, NULL, 0)<=1) continue;///speedup + trioF = get_ug_tip_trio_infor(ug, v^1); + ntrioF = (trioF==FATHER? MOTHER : (trioF==MOTHER? FATHER : (uint32_t)-1)); + } + + ve = &(g->arc[(uint32_t)b->a[k]]); + for (i = 0; i < nw; ++i) { + if (aw[i].v == (v^1)) { + we = &(aw[i]); + break; + } + } + ///mm_ol and mm_ou are used to make edge with long indel more easy to be cutted + mm_ol = MIN(ve->ol, we->ol); mm_ou = MIN(ve->ou, we->ou); + + for (i = kv = ol_max = ou_max = 0; i < nv; ++i) { + if(av[i].del) continue; + kv++; + if(is_trio && get_ug_tip_trio_infor(ug, av[i].v) == ntrioF) continue; + if(ol_max < av[i].ol) ol_max = av[i].ol; + if(ou_max < av[i].ou) ou_max = av[i].ou; + } + if (kv < 1) continue; + if (kv >= 2) { + if (mm_ol > ol_max*len_rat) continue; + if ((is_ou) && (mm_ou > min_ou) && (mm_ou > (ou_max*ou_rat))) continue; + } + + + for (i = kw = ol_max = ou_max = 0; i < nw; ++i) { + if(aw[i].del) continue; + kw++; + if(is_trio && get_ug_tip_trio_infor(ug, aw[i].v) == ntrioF) continue; + if(ol_max < aw[i].ol) ol_max = aw[i].ol; + if(ou_max < aw[i].ou) ou_max = aw[i].ou; + } + if (kw < 1) continue; + if (kw >= 2) { + if (mm_ol > ol_max*len_rat) continue; + if ((is_ou) && (mm_ou > min_ou) && (mm_ou > (ou_max*ou_rat))) continue; + } + + if (kv <= 1 && kw <= 1) continue; + + to_del = 0; del_v = del_w = (uint32_t)-1; tip = 0; + if (kv > 1 && kw > 1) { + to_del = 1; + } else if (kw == 1) { + tip = asg_topocut_aux(g, w^1, max_ext); + if (tip < max_ext) { + to_del = 1; del_w = w^1; + } + } else if (kv == 1) { + tip = asg_topocut_aux(g, v^1, max_ext); + if (tip < max_ext) { + to_del = 1; del_v = v^1; + } + } + + if (to_del) { + bn = b->n; + if(del_v != ((uint32_t)-1)) usg_topocut_aux_del(ug, del_v, max_ext, b); + if(del_w != ((uint32_t)-1)) usg_topocut_aux_del(ug, del_w, max_ext, b); + for (i = bn; i < b->n; i++) { + u = &(ug->u.a[b->a[i]>>1]); + if(u->m == 0) continue; + for (z = 0; z < u->n; z++) asg_seq_del(sl->sg, u->a[z]>>33); + asg_seq_del(ug->g, (b->a[i]>>1)); + if(u->m) { + u->m = u->n = 0; free(u->a); u->a = NULL; + } + } + // assert(tip == (b->n-bn)); + b->n = bn; + + ve->del = we->del = 1, ++cnt; + asg_arc_del_by_ug(sl->sg, ug, ve->ul>>32, ve->v, 1); + asg_arc_del_by_ug(sl->sg, ug, we->ul>>32, we->v, 1); + + } + } + + if(!in) free(tx.a); + if (cnt > 0) { + asg_cleanup(g); asg_cleanup(sl->sg); + } + // fprintf(stderr, "[M::%s]\tEnd\n", __func__); +} + +/** +void cal_bub_best_by_topo(ug_clean_t *sl, asg64_v *in, uint32_t max_ext, uint32_t is_trio, uint32_t is_ou, double len_rat, double ou_rat, uint32_t min_ou, uint32_t long_tip) +{ + // fprintf(stderr, "[M::%s]\tStart\n", __func__); + ma_ug_t *ug = sl->ug; asg_t *g = sl->ug->g; uint32_t ol_max, ou_max; + uint32_t v, w, n_vtx = (g->n_seq<<1), nv, nw, i, k, z, kv, kw, to_del, bn, tip, avi, awi; + asg64_v tx = {0,0,0}, *b = NULL; asg_arc_t *av, *aw, *ve, *we; ma_utg_t *u; + uint32_t trioF = (uint32_t)-1, mm_ol, mm_ou, cnt = 0, del_v, del_w; + + if(in) b = in; + else b = &tx; + b->n = 0; + + for (v = 0; v < n_vtx; ++v) { + if (g->seq[v>>1].del) continue; + if((sl->bid[v] == (uint32_t)-1) || (sl->bid[v^1] == (uint32_t)-1)) continue;///not within a bubble + + av = asg_arc_a(g, v); + nv = asg_arc_n(g, v); + if (nv < 2) continue; + + for (i = kv = 0; i < nv; ++i) { + if(av[i].del) continue; + ///could not connect to the beg/sink node + if((av[k].v>>1)==(sl->bid[v]>>1) || (av[k].v>>1)==(sl->bid[v^1]>>1)) break; + kv++; + } + if(kv < 2 || i < nv) continue;///it is impossible that kv <= bv + + for (i = 0; i < nv; ++i) { + if(av[i].del) continue; + kv_push(uint64_t, *b, (((uint64_t)av[i].ol)<<32) | ((uint64_t)(av-g->arc+i))); + } + } + + radix_sort_srt64(b->a, b->a + b->n); + for (k = 0; k < b->n; k++) { + if(g->arc[(uint32_t)b->a[k]].del) continue; + + v = g->arc[(uint32_t)b->a[k]].ul>>32; w = g->arc[(uint32_t)b->a[k]].v^1; + if(g->seq[v>>1].del || g->seq[w>>1].del) continue; + nv = asg_arc_n(g, v); nw = asg_arc_n(g, w); + av = asg_arc_a(g, v); aw = asg_arc_a(g, w); + if(nv < 2 || nw < 2) continue; + kv = get_arcs(g, v, NULL, 0); kw = get_arcs(g, w, NULL, 0); + if(kv < 2 || kw < 2) continue; + + if(is_trio) { + trioF = get_ug_tip_trio_infor(ug, v^1); + if(trioF == FATHER || trioF == MOTHER) { + if(get_ug_tip_trio_infor(ug, w^1) == trioF) continue; + } + } + + avi = awi = (uint32_t)-1; + + avi = ((uint32_t)b->a[k]) - ((uint64_t)(av-g->arc)); + ve = &(g->arc[(uint32_t)b->a[k]]); + + for (i = 0; i < nw; ++i) { + if (aw[i].v == (v^1)) { + we = &(aw[i]); awi = i; + break; + } + } + + for (i = kv = ol_max = ou_max = 0; i < nv; ++i) { + if(av[i].del) continue; + } + + ///mm_ol and mm_ou are used to make edge with long indel more easy to be cutted + mm_ol = MIN(ve->ol, we->ol); mm_ou = MIN(ve->ou, we->ou); + + for (i = kv = ol_max = ou_max = 0; i < nv; ++i) { + if(av[i].del) continue; + kv++; + if(is_trio && get_ug_tip_trio_infor(ug, av[i].v) == ntrioF) continue; + if(ol_max < av[i].ol) ol_max = av[i].ol; + if(ou_max < av[i].ou) ou_max = av[i].ou; + } + if (kv < 1) continue; + if (kv >= 2) { + if (mm_ol > ol_max*len_rat) continue; + if ((is_ou) && (mm_ou > min_ou) && (mm_ou > (ou_max*ou_rat))) continue; + } + + + for (i = kw = ol_max = ou_max = 0; i < nw; ++i) { + if(aw[i].del) continue; + kw++; + if(is_trio && get_ug_tip_trio_infor(ug, aw[i].v) == ntrioF) continue; + if(ol_max < aw[i].ol) ol_max = aw[i].ol; + if(ou_max < aw[i].ou) ou_max = aw[i].ou; + } + if (kw < 1) continue; + if (kw >= 2) { + if (mm_ol > ol_max*len_rat) continue; + if ((is_ou) && (mm_ou > min_ou) && (mm_ou > (ou_max*ou_rat))) continue; + } + + if (kv <= 1 && kw <= 1) continue; + + to_del = 0; del_v = del_w = (uint32_t)-1; tip = 0; + if (kv > 1 && kw > 1) { + to_del = 1; + } else if (kw == 1) { + tip = asg_topocut_aux(g, w^1, max_ext); + if (tip < max_ext) { + to_del = 1; del_w = w^1; + } + } else if (kv == 1) { + tip = asg_topocut_aux(g, v^1, max_ext); + if (tip < max_ext) { + to_del = 1; del_v = v^1; + } + } + + if (to_del) { + bn = b->n; + if(del_v != ((uint32_t)-1)) usg_topocut_aux_del(ug, del_v, max_ext, b); + if(del_w != ((uint32_t)-1)) usg_topocut_aux_del(ug, del_w, max_ext, b); + for (i = bn; i < b->n; i++) { + u = &(ug->u.a[b->a[i]>>1]); + if(u->m == 0) continue; + for (z = 0; z < u->n; z++) asg_seq_del(sl->sg, u->a[z]>>33); + asg_seq_del(ug->g, (b->a[i]>>1)); + if(u->m) { + u->m = u->n = 0; free(u->a); u->a = NULL; + } + } + // assert(tip == (b->n-bn)); + b->n = bn; + + ve->del = we->del = 1, ++cnt; + asg_arc_del_by_ug(sl->sg, ug, ve->ul>>32, ve->v, 1); + asg_arc_del_by_ug(sl->sg, ug, we->ul>>32, we->v, 1); + + } + } + + if(!in) free(tx.a); + if (cnt > 0) { + asg_cleanup(g); asg_cleanup(sl->sg); + } + // fprintf(stderr, "[M::%s]\tEnd\n", __func__); +} +**/ + + + +void deep_graph_clean(ug_opt_t *uopt, asg_t *sg, uint8_t is_ou, uint8_t is_trio, int64_t max_ext, +double min_ovlp_drop_ratio, double max_ovlp_drop_ratio, double ou_rat, int64_t min_ou, int64_t clean_round, int64_t long_tip) +{ + ug_clean_t *sl; asg64_v b; double step; int64_t i; + kv_init(b); hic_clean_adv(sg, uopt); + step = (clean_round==1?max_ovlp_drop_ratio:((max_ovlp_drop_ratio-min_ovlp_drop_ratio)/(clean_round-1))); + sl = init_ug_clean_t(uopt, sg, is_ou, is_trio, max_ext, min_ovlp_drop_ratio, ou_rat, min_ou); + + // print_debug_gfa(sg, NULL, uopt->coverage_cut, "debug", uopt->sources, uopt->ruIndex, uopt->max_hang, uopt->min_ovlp, 0, 0, 0); + for (i = 0, sl->len_rat = min_ovlp_drop_ratio; i < clean_round; i++, sl->len_rat += step) { + if(sl->len_rat > max_ovlp_drop_ratio) sl->len_rat = max_ovlp_drop_ratio; + update_ug_clean_t(sl); + kt_for(asm_opt.thread_num, cal_bub_best, sl, sl->ug->g->n_seq); + cal_bub_best_by_len(sl, &b, max_ext, is_trio, sl->is_ou, sl->len_rat, sl->ou_rat, min_ou); + } + + if(is_ou) { + uint64_t k; sl->is_ou = 0; + for (k = 0; k < sl->ug->g->n_arc; k++) sl->ug->g->arc[k].ou = 0; + max_ovlp_drop_ratio = 0.6; + if(max_ovlp_drop_ratio > min_ovlp_drop_ratio) { + step = (clean_round==1?max_ovlp_drop_ratio:((max_ovlp_drop_ratio-min_ovlp_drop_ratio)/(clean_round-1))); + for (i = 0, sl->len_rat = min_ovlp_drop_ratio; i < clean_round; i++, sl->len_rat += step) { + if(sl->len_rat > max_ovlp_drop_ratio) sl->len_rat = max_ovlp_drop_ratio; + update_ug_clean_t(sl); + kt_for(asm_opt.thread_num, cal_bub_best, sl, sl->ug->g->n_seq); + cal_bub_best_by_len(sl, &b, max_ext, is_trio, sl->is_ou, sl->len_rat, sl->ou_rat, min_ou); + } + } + } + + // update_ug_clean_t(sl); + // cal_bub_best_by_topo(sl, &b, max_ext, is_trio, sl->is_ou, sl->len_rat, sl->ou_rat, min_ou, long_tip); + + kv_destroy(b); destroy_ug_clean_t(sl); free(sl); + hic_clean_adv(sg, uopt); +} diff --git a/gfa_ut.h b/gfa_ut.h index 87d513c..a34e621 100644 --- a/gfa_ut.h +++ b/gfa_ut.h @@ -27,7 +27,7 @@ void asg_arc_cut_complex_bub_links(asg_t *g, asg64_v *in, float len_rat, float o uint32_t asg_cut_large_indel(asg_t *g, asg64_v *in, int32_t max_ext, float ou_rat, uint32_t is_ou); uint32_t asg_cut_semi_circ(asg_t *g, uint32_t lim_len, uint32_t is_clean); void ul_realignment_gfa(ug_opt_t *uopt, asg_t *sg, int64_t clean_round, double min_ovlp_drop_ratio, -double max_ovlp_drop_ratio, int64_t max_tip, int64_t max_ul_tip, bub_label_t *b_mask_t, uint32_t is_trio, char *o_file, ul_renew_t *ropt, const char *bin_file, uint64_t free_uld); +double max_ovlp_drop_ratio, int64_t max_tip, int64_t max_ul_tip, bub_label_t *b_mask_t, uint32_t is_trio, char *o_file, ul_renew_t *ropt, const char *bin_file, uint64_t free_uld, uint64_t is_bridg, uint64_t deep_clean); void recover_contain_g(asg_t *g, ma_hit_t_alloc *src, R_to_U* ruIndex, int64_t max_hang, int64_t min_ovlp, int64_t ul_occ); void normalize_gou(asg_t *g); void prt_specfic_sge(asg_t *g, uint32_t src, uint32_t dst, const char* cmd); diff --git a/horder.cpp b/horder.cpp index a3d0af6..8c79952 100644 --- a/horder.cpp +++ b/horder.cpp @@ -838,6 +838,13 @@ void horder_clean_sg_by_utg(asg_t *sg, ma_ug_t *ug) if(av[k].del) continue; w = av[k].v; + vx = (v&1?((ug->u.a[v>>1].a[0]>>32)^1):(ug->u.a[v>>1].a[ug->u.a[v>>1].n-1]>>32)); + wx = (w&1?((ug->u.a[w>>1].a[ug->u.a[w>>1].n-1]>>32)^1):(ug->u.a[w>>1].a[0]>>32)); + asg_arc_unique_del(sg, vx, wx, 0); asg_arc_unique_del(sg, wx^1, vx^1, 0); + } + + if(u->circ) { + v = w = i<<1; vx = (v&1?((ug->u.a[v>>1].a[0]>>32)^1):(ug->u.a[v>>1].a[ug->u.a[v>>1].n-1]>>32)); wx = (w&1?((ug->u.a[w>>1].a[ug->u.a[w>>1].n-1]>>32)^1):(ug->u.a[w>>1].a[0]>>32)); asg_arc_unique_del(sg, vx, wx, 0); asg_arc_unique_del(sg, wx^1, vx^1, 0); diff --git a/inter.cpp b/inter.cpp index d49b41d..9317e60 100644 --- a/inter.cpp +++ b/inter.cpp @@ -30,6 +30,9 @@ void ul_map_lchain(ha_abufl_t *ab, uint32_t rid, char* rs, uint64_t rl, uint64_t int64_t ug_map_lchain(ha_abufl_t *ab, uint32_t rid, char* rs, uint64_t rl, uint64_t mz_w, uint64_t mz_k, const ul_idx_t *uref, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, double bw_thres_sec, int max_n_chain, int apend_be, kvec_t_u8_warp* k_flag, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, st_mt_t *sp, uint32_t *high_occ, uint32_t *low_occ, uint32_t is_accurate, uint32_t gen_off, double mcopy_rate, uint32_t mcopy_khit_cut, uint32_t is_hpc, ha_mzl_t *res, uint64_t res_n, ha_mzl_t *idx, uint64_t idx_n, uint64_t mzl_cutoff, uint64_t chain_cutoff, kv_u_trans_t *kov); +int64_t ug_map_lchain_simple(ha_abufl_t *ab, uint32_t rid, char* rs, uint64_t rl, uint64_t mz_w, uint64_t mz_k, const ul_idx_t *uref, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, + int max_n_chain, int apend_be, kvec_t_u8_warp* k_flag, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, st_mt_t *sp, uint32_t *high_occ, uint32_t *low_occ, uint32_t is_accurate, + uint32_t gen_off, double mcopy_rate, uint32_t mcopy_khit_cut, uint32_t is_hpc, ha_mzl_t *res, uint64_t res_n, ha_mzl_t *idx, uint64_t idx_n, uint64_t mzl_cutoff, uint64_t chain_cutoff); #define MG_SEED_IGNORE (1ULL<<41) #define MG_SEED_TANDEM (1ULL<<42) @@ -52,6 +55,15 @@ int64_t ug_map_lchain(ha_abufl_t *ab, uint32_t rid, char* rs, uint64_t rl, uint6 #define REA_ALIGN_CUTOFF 32 #define CHUNK_SIZE 1000000000 +#define GBIN_K 31 +#define GBIN_W 31 +#define GBIN_E 0.04 +#define GBIN_BE 0.005 +#define GBIN_ME 4 +///GBIN_L -> for the length of one HiFi read +// #define GBIN_L 15000 +#define GBIN_L 256 + #define generic_key(x) (x) KRADIX_SORT_INIT(gfa64, uint64_t, generic_key, 8) @@ -85,6 +97,9 @@ KRADIX_SORT_INIT(uc_block_t_qe_srt, uc_block_t, uc_block_t_qe_key, member_size(u #define uc_block_t_qs_key(x) ((x).qs) KRADIX_SORT_INIT(uc_block_t_qs_srt, uc_block_t, uc_block_t_qs_key, member_size(uc_block_t, qs)); +#define hpc_ss_t_s_key(p) ((p).s) +KRADIX_SORT_INIT(hpc_ss_t_s, hpc_ss_t, hpc_ss_t_s_key, member_size(hpc_ss_t, s)) + typedef struct { char *a; size_t n, m; @@ -199,6 +214,7 @@ typedef struct { // global data structure for kt_pipeline() const asg_t *rg; const ug_opt_t *uopt; const ul_idx_t *uu; + idx_emask_t *mm; ucr_file_t *ucr_s; kseq_t *ks; int64_t chunk_size; @@ -401,6 +417,28 @@ typedef struct { // global data structure for kt_pipeline() kv_u_trans_t *filter; } ug_trans_t; +typedef struct { // global data structure for kt_pipeline() + const ug_opt_t *uopt; + ma_ug_t *ug; + asg_t *rg; + ha_ovec_buf_t **hab; + glchain_t *ll; + int32_t w, k; + ///base-alignment + double bw_thres, diff_ov, diff_bin; + uint64_t min_bin_len, max_diff; + int32_t max_n_chain; + ha_mzl_v idx_a; + asg64_v idx_n; + ha_mzl_v srt_a; + int32_t is_HPC, bw, max_gap, chn_pen_gap, n_thread, is_cnt, is_ovlp, mini_cut, chain_cut, keep_unsymm_arc; + ul_idx_t udb; + // hpc_re_t *hre; + mask_ul_ov_t *mk; + idx_emask_t *mm; + // bit_mask_t *bm; +} ug_bin_t; + void hc_glchain_destroy(glchain_t *b) { if (!b) return; @@ -6980,18 +7018,25 @@ void filter_topN(overlap_region_alloc* ol, kv_ul_ov_t *aln, uint64_t ql, uint64_ } } - int64_t sc, m, sec_err; - for (i = m = 0; i < ol->length; i++) { + int64_t sc, m, sec_err, h0, h1; + for (i = m = h0 = h1 = 0; i < ol->length; i++) { sc = cal_sec_e_min(&(ol->list[i]), w_idx, wl, ql, NULL); if(sc >= 0) { - srt[m] = sc; srt[m] <<= 32; srt[m] |= i; srt[m] |= ((uint64_t)0x8000000000000000); + srt[m] = sc; srt[m] <<= 32; srt[m] |= i; srt[m] |= ((uint64_t)0x8000000000000000); h0++; } else { - srt[m] = -sc; srt[m] <<= 32; srt[m] |= i; + srt[m] = -sc; srt[m] <<= 32; srt[m] |= i; h1++; } m++; } radix_sort_gfa64(srt, srt + ol->length); + if(h1 > 0) {///have chains with negative weight + uint64_t zn = h1>>1, zt; + for (i = 0; i < zn; i++) { + zt = srt[i]; srt[i] = srt[h1-i-1]; srt[h1-i-1] = zt; + } + } + for (m = ((int64_t)ol->length)-1, cc = 0; m >= 0 && cc <= cc_max; m--) { cc += ol->list[(uint32_t)srt[m]].align_length; ol->list[(uint32_t)srt[m]].is_match = 1; @@ -9036,11 +9081,12 @@ static void worker_for_ul_rescall_alignment(void *data, long i, int tid) // call // uint64_t align = 0; // if(UL_INF.a[s->id+i].rlen != s->len[i]) { - // fprintf(stderr, "[M::%s] rid:%ld, s->len:%lu, UL_INF->rlen:%u\n", __func__, s->id+i, s->len[i], UL_INF.a[s->id+i].rlen); + // fprintf(stderr, "[M::%s] rid:%ld, s->len:%lu, UL_INF->rlen:%u\n", __func__, s->id+i, s->len[i], UL_INF.a[s->id+i].rlen); // } - assert(UL_INF.a[s->id+i].rlen == s->len[i]); + // assert(UL_INF.a[s->id+i].rlen == s->len[i]); // void *km = s->buf?(s->buf[tid]?s->buf[tid]->km:NULL):NULL; - // if(s->id+i!=3046/** && s->id+i!=3111**/) return; + // if(s->id+i!=0/** && s->id+i!=4 && s->id+i!=5**/) return; + // fprintf(stderr, "\n[M::%s] rid:%ld, s->len:%lu\n", __func__, s->id+i, s->len[i]); // if((s->id+i!=871) && (s->id+i!=963) && (s->id+i!=980)) return; // if(s->id+i!=944) return; // if(s->id+i != 35437) return; @@ -10040,6 +10086,1646 @@ static void worker_for_trans_ovlp_adv(void *data, long i, int tid) // callback f } } + +int64_t retrieve_cigar_err_dir(bit_extz_t *ez, int64_t s, int64_t e, int64_t *xk, int64_t *ck, int64_t is_back) +{ ///[ez->ts, ez->te]/[ez->qs, ez->qe]/[s, e) + if(!ez->cigar.n) return 0; + int64_t cn = ez->cigar.n, op, err = 0; int64_t ws, we, os, oe, ovlp; + if(!is_back) { + if(((*ck) < 0) || ((*ck) > cn)) {//(*ck) == cn is allowed + (*ck) = 0; (*xk) = ez->ts; + } + + while ((*ck) > 0 && (*xk) > s) { + --(*ck); + op = ez->cigar.a[(*ck)]>>14; + if(op!=2) (*xk) -= (ez->cigar.a[(*ck)]&(0x3fff)); + } + + //some cigar will span s or e + while ((*ck) < cn && (*xk) < e) {//[s, e) + ws = (*xk); + op = ez->cigar.a[(*ck)]>>14; + if(op!=2) (*xk) += (ez->cigar.a[(*ck)]&(0x3fff)); + we = (*xk); + os = MAX(s, ws); oe = MIN(e, we); + ovlp = ((oe>os)? (oe-os):0); + if((op==2) && (ws>=s) && (wscigar.a[(*ck)]&(0x3fff)); + } + (*ck)++; + if((!ovlp) || (!op)) continue; + err += ovlp; + } + } else { + if(((*ck) < 0) || ((*ck) >= cn)) { + (*ck) = cn-1; (*xk) = ez->te + 1; ///[ez->ts, ez->te] + } + while (((*ck) < cn) && ((*xk) < e)) { + ++(*ck); + op = ez->cigar.a[(*ck)]>>14; + if(op!=2) (*xk) += (ez->cigar.a[(*ck)]&(0x3fff)); + } + + //some cigar will span s or e + while ((*ck) >= 0 && (*xk) > s) {//[s, e) + we = (*xk); + op = ez->cigar.a[(*ck)]>>14; + if(op!=2) (*xk) -= (ez->cigar.a[(*ck)]&(0x3fff)); + ws = (*xk); + os = MAX(s, ws); oe = MIN(e, we); + ovlp = ((oe>os)? (oe-os):0); + if((op==2) && (ws>=s) && (wscigar.a[(*ck)]&(0x3fff)); + } + (*ck)--; + if((!ovlp) || (!op)) continue; + err += ovlp; + } + } + // int64_t debug_err = retrieve_cigar_err_debug(ez, s, e); + // if(!(err == debug_err)) { + // fprintf(stderr, "[M::%s::] err::%ld, debug_err::%ld, s::%ld, e::%ld, ez->ts::%u, ez->te::%u\n", + // __func__, err, debug_err, s, e, ez->ts, ez->te); + // } + // assert(err == debug_err); + return err; +} + +int64_t cal_cigar_err(overlap_region *z, int64_t s, int64_t e, int64_t is_back)///need debugging +{ + int64_t wn = z->w_list.n, wk, xk, ck; window_list *m; bit_extz_t ez; + int64_t ws, we, os, oe, ovlp, xl, yl, werr, err, tot; + err = 0; tot = e - s; wk = xk = ck = -1; + if(!wn) return INT32_MAX; + + if(!is_back) { + wk = 0; + if(wn) { + xk = z->w_list.a[wk].x_start; ck = 0; + } + while(wk < wn) { + m = &(z->w_list.a[wk]); + ws = m->x_start; we = m->x_end+1; + os = MAX(s, ws); oe = MIN(e, we); + ovlp = ((oe>os)? (oe-os):0); + if(ovlp) { + xl = m->x_end+1-m->x_start; + yl = m->y_end+1-m->y_start; + if((is_ualn_win((*m))) || (is_est_aln((*m)))) { + if(is_ualn_win((*m))) { //unmapped + werr = MAX(xl, yl);///gen_err_unaligned(xl, yl); + } else { + werr = m->error;//shared window + } + if(ovlp < xl) { + werr = (((double)ovlp)/((double)xl))*((double)werr); + } + //skip the whole window + err += werr; xk = m->x_end+1; ck = m->clen; + } else { + if(ovlp == xl) { //skip the whole window + err += m->error; xk = m->x_end+1; ck = m->clen; + } else { + set_bit_extz_t(ez, (*z), wk); + err += retrieve_cigar_err_dir(&ez, os, oe, &xk, &ck, is_back); + } + } + } + tot -= ovlp; + if(xk >= e) break;//[min_w, max_w] && [s, e) + wk++; if(wk >= wn) break; + xk = z->w_list.a[wk].x_start; ck = 0;//reset + } + } else { + wk = wn-1; + if(wn) { + xk = z->w_list.a[wk].x_end+1; ck = z->w_list.a[wk].clen-1; + } + while(wk >= 0) { + m = &(z->w_list.a[wk]); + ws = m->x_start; we = m->x_end+1; + os = MAX(s, ws); oe = MIN(e, we); + ovlp = ((oe>os)? (oe-os):0); + if(ovlp) { + xl = m->x_end+1-m->x_start; + yl = m->y_end+1-m->y_start; + if((is_ualn_win((*m))) || (is_est_aln((*m)))) { + if(is_ualn_win((*m))) { //unmapped + werr = MAX(xl, yl);///gen_err_unaligned(xl, yl); + } else { + werr = m->error;//shared window + } + if(ovlp < xl) { + werr = (((double)ovlp)/((double)xl))*((double)werr); + } + //skip the whole window + err += werr; xk = m->x_start; ck = -1; + } else { + if(ovlp == xl) { //skip the whole window + err += m->error; xk = m->x_start; ck = -1; + } else { + set_bit_extz_t(ez, (*z), wk); + err += retrieve_cigar_err_dir(&ez, os, oe, &xk, &ck, is_back); + } + } + } + tot -= ovlp; + if(xk <= s) break;//[s, e) + wk--; if(wk < 0) break; + xk = z->w_list.a[wk].x_end+1; ck = z->w_list.a[wk].clen-1;//reset + } + } + assert(!tot); + return err; +} + +uint64_t get_high_simi(overlap_region *z, double erate, uint64_t maxe, uint64_t minov, uint64_t *re0, uint64_t *re1) +{ + uint64_t ol, err = z->non_homopolymer_errors, e0, e1, f0 = 0, f1 = 0; + ol = z->x_pos_e+1-z->x_pos_s; (*re0) = (*re1) = (uint64_t)-1; + if(ol > (z->y_pos_e+1-z->y_pos_s)) ol = z->y_pos_e+1-z->y_pos_s; + if((err <= (ol*erate)) && (err <= maxe)) return 1; + if(ol <= minov) return 0;///too short + e0 = cal_cigar_err(z, 0, minov, 0); + e1 = cal_cigar_err(z, z->x_pos_e+1-minov, z->x_pos_e+1, 1); + if((e0 <= (minov*erate)) && (e0 <= maxe)) f0 = 1; + if((e1 <= (minov*erate)) && (e1 <= maxe)) f1 = 1; + if(f0 && f1 && (ol <= (minov<<1))) { + if(e0 >= e1) f0 = 0; + else f1 = 0; + } + if((!f0) && (!f1)) return 0; + if(f0) (*re0) = e0; + if(f1) (*re1) = e1; + return 2; +} + +int64_t hpc_check_naive(hpc_ss_t *a, int64_t n, int64_t s, int64_t e, int64_t bd) +{ + int64_t k, os, oe; + for (k = 0; k < n; k++) { + os = MAX(s, a[k].s); oe = MIN(e, a[k].e); + if(oe - os >= -bd) return 1; + if(a[k].s > (uint64_t)(e + bd)) break; + } + return 0; +} + +int64_t hpc_check(hpc_ss_t *a, int64_t n, int64_t s, int64_t e, int64_t *idx, int64_t bd) +{ + if(n<=0) return 0; + if((*idx) >= n) (*idx) = n - 1; + if((*idx) < 0) (*idx) = 0; + int64_t k = (*idx), os, oe; + for (; k < n; k++) { + os = MAX(s, a[k].s); oe = MIN(e, a[k].e); + if(oe - os >= -bd) { + (*idx) = k; return 1; + } + if(a[k].s > (uint64_t)(e + bd)) break; + } + os = (*idx); (*idx) = k; k = os - 1; + for (; k >= 0; k--) { + os = MAX(s, a[k].s); oe = MIN(e, a[k].e); + if(oe - os >= -bd) { + (*idx) = k; return 1; + } + } + // assert(!hpc_check_naive(a, n, s, e, bd)); + return 0; +} + +int64_t retrieve_npc_cigar_err(bit_extz_t *ez, int64_t s, int64_t e, int64_t *xk, int64_t *yk, int64_t *ck, +hpc_ss_t *fx, int64_t nx, hpc_ss_t *fy, int64_t ny, int64_t ylen, int64_t is_rev)///[s, e) for x +{ + if(!ez->cigar.n) return 0; + int64_t cn = ez->cigar.n, op, err = 0; int64_t ws, we, ws1, we1, os, oe, ovlp, xi, yi; + if(((*ck) < 0) || ((*ck) > cn)) {//(*ck) == cn is allowed + (*ck) = 0; (*xk) = ez->ts; (*yk) = ez->ps; + } + while ((*ck) > 0 && (*xk) > s) { + --(*ck); + op = ez->cigar.a[(*ck)]>>14; + if(op!=2) (*xk) -= (ez->cigar.a[(*ck)]&(0x3fff)); + if(op!=3) (*yk) -= (ez->cigar.a[(*ck)]&(0x3fff)); + } + xi = yi = 0; if(nx<=0) fx = NULL; if(ny<=0) fy = NULL; + + //some cigar will span s or e + while ((*ck) < cn && (*xk) < e) {//[s, e) + ws = (*xk); ws1 = (*yk); + op = ez->cigar.a[(*ck)]>>14; + if(op!=2) (*xk) += (ez->cigar.a[(*ck)]&(0x3fff)); + if(op!=3) (*yk) += (ez->cigar.a[(*ck)]&(0x3fff)); + we = (*xk); we1 = (*yk); + os = MAX(s, ws); oe = MIN(e, we); + ovlp = ((oe>os)? (oe-os):0); + if((op==2) && (ws>=s) && (wscigar.a[(*ck)]&(0x3fff)); + } + (*ck)++; + if((!ovlp) || (!op)) continue; + if(fx && hpc_check(fx, nx, ws, we, &xi, 1)) continue; + if(fy && hpc_check(fy, ny, ((is_rev)?(ylen-we1):(ws1)), ((is_rev)?(ylen-ws1):(we1)), &yi, 1)) continue; + err += ovlp; + } + return err; +} + +///[s, e) for x +int64_t cal_npc_err(overlap_region *z, int64_t xs, int64_t xe, hpc_re_t *hre, int64_t ylen) +{ + int64_t wn = z->w_list.n, wk, xk, yk, ck; window_list *m; bit_extz_t ez; + int64_t ws, we, os, oe, ovlp, xl, yl, werr, err, tot; + hpc_ss_t *fx = NULL, *fy = NULL; int64_t nx = 0, ny = 0; + err = 0; tot = xe - xs; wk = xk = yk = ck = -1; + if(!wn) return INT32_MAX; + wk = 0; + if(wn) { + xk = z->w_list.a[wk].x_start; yk = z->w_list.a[wk].y_start; ck = 0; + } + if(hre) { + fx = hre->a + hre->idx[z->x_id].s; nx = hre->idx[z->x_id].e - hre->idx[z->x_id].s; + fy = hre->a + hre->idx[z->y_id].s; ny = hre->idx[z->y_id].e - hre->idx[z->y_id].s; + } + // if(hre && ((z->x_id == 44 && z->y_id == 45) || (z->x_id == 45 && z->y_id == 44))) { + // fprintf(stderr, "[M::%s]\tutg%.6ul\tq::[%u,\t%u)\t%c\tutg%.6ul\tt::[%u,\t%u)\terr::%u\twn::%u\n", + // __func__, z->x_id+1, z->x_pos_s, z->x_pos_e+1, "+-"[z->y_pos_strand], + // z->y_id+1, z->y_pos_s, z->y_pos_e+1, z->non_homopolymer_errors, (uint32_t)z->w_list.n); + // } + + while(wk < wn) { + m = &(z->w_list.a[wk]); + // if(hre && ((z->x_id == 44 && z->y_id == 45) || (z->x_id == 45 && z->y_id == 44))) { + // fprintf(stderr, "[M::%s]k::%ld\tq::[%u,\t%u)\tt::[%u,\t%u)\terr::%u\tis_ualn::%u\tis_est::%u\n", + // __func__, wk, m->x_start, m->x_end+1, m->y_start, m->y_end+1, m->error, + // (is_ualn_win((*m))), is_est_aln((*m))); + // } + ws = m->x_start; we = m->x_end+1; + os = MAX(xs, ws); oe = MIN(xe, we); + ovlp = ((oe>os)? (oe-os):0); + if(ovlp) { + xl = m->x_end+1-m->x_start; + yl = m->y_end+1-m->y_start; + if((is_ualn_win((*m))) || (is_est_aln((*m)))) { + if(is_ualn_win((*m))) { //unmapped + werr = MAX(xl, yl);///gen_err_unaligned(xl, yl); + } else { + werr = m->error;//shared window + } + if(ovlp < xl) { + werr = (((double)ovlp)/((double)xl))*((double)werr); + } + //skip the whole window + err += werr; xk = m->x_end+1; yk = m->y_end+1; ck = m->clen; + } else { + if(ovlp == xl) { //skip the whole window + err += m->error; xk = m->x_end+1; yk = m->y_end+1; ck = m->clen; + } else { + set_bit_extz_t(ez, (*z), wk); + err += retrieve_npc_cigar_err(&ez, os, oe, &xk, &yk, &ck, fx, nx, fy, ny, ylen, z->y_pos_strand); + } + } + } + tot -= ovlp; + if(xk >= xe) break;//[min_w, max_w] && [s, e) + wk++; if(wk >= wn) break; + xk = z->w_list.a[wk].x_start; yk = z->w_list.a[wk].y_start; ck = 0;//reset + } + assert(!tot); + return err; +} + +void cal_s_interval(ma_ug_t *ug, asg_arc_t *p, ul_ov_t *res) +{ + memset(res, 0, sizeof((*res))); + res->rev = (((p->ul>>32)^(p->v))&((uint32_t)1)); + res->qn = p->ul>>33; res->tn = p->v>>1; + uint32_t ql = ug->g->seq[res->qn].len, tl = ug->g->seq[res->tn].len; + if((p->ul>>32)&1) { + res->qs = 0; res->qe = ((p->ol<=ql)?(p->ol):(ql)); + } else { + res->qe = ql; res->qs = ((p->ol<=res->qe)?(res->qe-p->ol):(0)); + } + + if(p->v&1) { + res->te = tl; res->ts = ((p->ol<=res->te)?(res->te-p->ol):(0)); + } else { + res->ts = 0; res->te = ((p->ol<=tl)?(p->ol):(tl)); + } +} + +uint32_t infer_se(uint32_t qs, uint32_t qe, uint32_t ts, uint32_t te, uint32_t rev, +uint32_t rqs, uint32_t rqe, uint32_t *rts, uint32_t *rte) +{ + if(rqs <= rqe && rqs >= qs && rqe <= qe) { + uint32_t s_shift, e_shift, m; + s_shift = get_offset_adjust(rqs-qs, qe-qs, te-ts); + e_shift = get_offset_adjust(qe-rqe, qe-qs, te-ts); + if(rev) { + m = s_shift; s_shift = e_shift; e_shift = m; + } + // if(rqs == 2623 && rqe == 17620) { + // fprintf(stderr, "[M::%s]\tqs::%u\tqe::%u\tts::%u\tte::%u\trqs::%u\trqe::%u\ts_shift::%u\te_shift::%u\n", __func__, + // qs, qe, ts, te, rqs, rqe, s_shift, e_shift); + // } + (*rts) = ts + s_shift; (*rte) = te - e_shift; + if((*rts) <= (*rte) && (*rts) >= ts && (*rte) <= te) return 1; + } + (*rts) = (*rte) = (uint32_t)-1; + return 0; +} + +uint32_t cal_x_interval(ma_ug_t *ug, ul_ov_t *v, ul_ov_t *w, ul_ov_t *res) +{ + uint32_t os, oe, rqs, rqe, rts, rte; memset(res, 0, sizeof((*res))); + // if(v->qn == 97) { + // fprintf(stderr, "-0-v-[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\n", __func__, + // v->qn+1, "lc"[ug->u.a[v->qn].circ], v->qs, v->qe, "+-"[v->rev], + // v->tn+1, "lc"[ug->u.a[v->tn].circ], v->ts, v->te); + // fprintf(stderr, "-0-w-[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\n", __func__, + // w->qn+1, "lc"[ug->u.a[w->qn].circ], w->qs, w->qe, "+-"[v->rev], + // w->tn+1, "lc"[ug->u.a[w->tn].circ], w->ts, w->te); + // } + if(v->tn == w->qn) { + os = MAX(v->ts, w->qs); oe = MIN(v->te, w->qe); + // if(v->qn == 97) { + // fprintf(stderr, "-1-[M::%s]\tutg%.6u%c->utg%.6u%c\tutg%.6u%c->utg%.6u%c\to::[%u,\t%u)\n", __func__, + // v->qn+1, "lc"[ug->u.a[v->qn].circ], + // v->tn+1, "lc"[ug->u.a[v->tn].circ], + // w->qn+1, "lc"[ug->u.a[w->qn].circ], + // w->tn+1, "lc"[ug->u.a[w->tn].circ], + // os, oe); + // } + if(oe <= os) return 0; + if(infer_se(v->ts, v->te, v->qs, v->qe, v->rev, os, oe, &rqs, &rqe) && + infer_se(w->qs, w->qe, w->ts, w->te, w->rev, os, oe, &rts, &rte)) { + // if(v->qn == 97) { + // fprintf(stderr, "-2-[M::%s]\tutg%.6u%c->utg%.6u%c\tutg%.6u%c->utg%.6u%c\trq::[%u,\t%u)\trt::[%u,\t%u)\n", __func__, + // v->qn+1, "lc"[ug->u.a[v->qn].circ], + // v->tn+1, "lc"[ug->u.a[v->tn].circ], + // w->qn+1, "lc"[ug->u.a[w->qn].circ], + // w->tn+1, "lc"[ug->u.a[w->tn].circ], + // rqs, rqe, rts, rte); + // } + if(rqe > rqs && rte > rts) { + res->qn = v->qn; res->tn = w->tn; res->rev = ((v->rev==w->rev)?0:1); + res->qs = rqs; res->qe = rqe; res->ts = rts; res->te = rte; + return 1; + } + } + } + return 0; +} + + +void gen_shared_interval(ul_ov_t *p, ma_ug_t *ug, kv_ul_ov_t *res) +{ + uint32_t st, et, v, w, i, k, nv, nw; asg_arc_t *av, *aw; + ul_ov_t s1, s2, rr; res->n = 0; + + st = p->qn<<1; et = (p->tn<<1) + (!!(p->rev)); + // if(p->qn == 101 && p->tn == 102) { + // if(p->qn == 97 && p->tn == 98) { + // fprintf(stderr, ">[M::%s]\tutg%.6u%c(%c)->utg%.6u%c(%c)\n", __func__, + // (st>>1)+1, "lc"[ug->u.a[st>>1].circ], "+-"[st&1], + // (et>>1)+1, "lc"[ug->u.a[et>>1].circ], "+-"[et&1]); + // } + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + // if(p->qn == 101 && p->tn == 102) { + // if(p->qn == 97 && p->tn == 98) { + // fprintf(stderr, "+[M::%s]\tutg%.6lu%c(%c)->utg%.6u%c(%c)\n", __func__, + // (av[i].ul>>33)+1, "lc"[ug->u.a[(av[i].ul>>33)].circ], "+-"[(av[i].ul>>32)&1], + // (av[i].v>>1)+1, "lc"[ug->u.a[av[i].v>>1].circ], "+-"[av[i].v&1]); + // } + if(w == et) { + kv_push(ul_ov_t, *res, s1); + } else { + aw = asg_arc_a(ug->g, (w^1)); nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + // if(p->qn == 101 && p->tn == 102) { + // if(p->qn == 97 && p->tn == 98) { + // fprintf(stderr, "-[M::%s]\tutg%.6lu%c(%c)->utg%.6u%c(%c)\n", __func__, + // (aw[k].ul>>33)+1, "lc"[ug->u.a[(aw[k].ul>>33)].circ], "+-"[(aw[k].ul>>32)&1], + // (aw[k].v>>1)+1, "lc"[ug->u.a[aw[k].v>>1].circ], "+-"[aw[k].v&1]); + // } + // cal_s_interval(ug, &(aw[k]), &s2); + // if(!cal_x_interval(ug, &s1, &s2, &rr)) continue; + + if(aw[k].v == (et^1)) { + // if(p->qn == 101 && p->tn == 102) { + // if(p->qn == 97 && p->tn == 98) { + // fprintf(stderr, "*[M::%s]\tutg%.6lu%c(%c)->utg%.6u%c(%c)\n", __func__, + // (aw[k].ul>>33)+1, "lc"[ug->u.a[(aw[k].ul>>33)].circ], "+-"[(aw[k].ul>>32)&1], + // (aw[k].v>>1)+1, "lc"[ug->u.a[aw[k].v>>1].circ], "+-"[aw[k].v&1]); + // } + cal_s_interval(ug, &(aw[k]), &s2); + if(cal_x_interval(ug, &s1, &s2, &rr)) { + kv_push(ul_ov_t, *res, rr); + } + } + } + } + } + + st ^= 1; et ^= 1; + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + // if(p->qn == 101 && p->tn == 102) { + // if(p->qn == 97 && p->tn == 98) { + // fprintf(stderr, "+[M::%s]\tutg%.6lu%c(%c)->utg%.6u%c(%c)\n", __func__, + // (av[i].ul>>33)+1, "lc"[ug->u.a[(av[i].ul>>33)].circ], "+-"[(av[i].ul>>32)&1], + // (av[i].v>>1)+1, "lc"[ug->u.a[av[i].v>>1].circ], "+-"[av[i].v&1]); + // } + if(w == et) { + kv_push(ul_ov_t, *res, s1); + } else { + aw = asg_arc_a(ug->g, (w^1)); nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + // if(p->qn == 101 && p->tn == 102) { + // if(p->qn == 97 && p->tn == 98) { + // fprintf(stderr, "-[M::%s]\tutg%.6lu%c(%c)->utg%.6u%c(%c)\n", __func__, + // (aw[k].ul>>33)+1, "lc"[ug->u.a[(aw[k].ul>>33)].circ], "+-"[(aw[k].ul>>32)&1], + // (aw[k].v>>1)+1, "lc"[ug->u.a[aw[k].v>>1].circ], "+-"[aw[k].v&1]); + // } + if(aw[k].v == (et^1)) { + // if(p->qn == 101 && p->tn == 102) { + // if(p->qn == 97 && p->tn == 98) { + // fprintf(stderr, "*[M::%s]\tutg%.6lu%c(%c)->utg%.6u%c(%c)\n", __func__, + // (aw[k].ul>>33)+1, "lc"[ug->u.a[(aw[k].ul>>33)].circ], "+-"[(aw[k].ul>>32)&1], + // (aw[k].v>>1)+1, "lc"[ug->u.a[aw[k].v>>1].circ], "+-"[aw[k].v&1]); + // } + cal_s_interval(ug, &(aw[k]), &s2); + if(cal_x_interval(ug, &s1, &s2, &rr)) { + kv_push(ul_ov_t, *res, rr); + } + } + } + } + } +} + +void gen_shared_interval_adv(ul_ov_t *p, ma_ug_t *ug, kv_ul_ov_t *res) +{ + uint32_t st, et, v, w, i, k, z, nv, nw, nz; asg_arc_t *av, *aw, *az; + ul_ov_t s1, s2, s3, s4, rr; res->n = 0; + + st = p->qn<<1; et = (p->tn<<1) + (!!(p->rev)); + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + if(w == et) { + kv_push(ul_ov_t, *res, s1); + } else { + aw = asg_arc_a(ug->g, (w^1)); nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + + cal_s_interval(ug, &(aw[k]), &s2); + if(!cal_x_interval(ug, &s1, &s2, &s3)) continue; + if(aw[k].v == (et^1)) { + kv_push(ul_ov_t, *res, s3); + } else { + az = asg_arc_a(ug->g, (aw[k].v^1)); nz = asg_arc_n(ug->g, (aw[k].v^1)); + for (z = 0; z < nz; z++) { + if(az[z].del) continue; + if(az[z].v == et) { + cal_s_interval(ug, &(az[z]), &s4); + if(cal_x_interval(ug, &s3, &s4, &rr)) { + kv_push(ul_ov_t, *res, rr); + } + } + } + } + } + } + } + + st ^= 1; et ^= 1; + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + if(w == et) { + kv_push(ul_ov_t, *res, s1); + } else { + aw = asg_arc_a(ug->g, (w^1)); nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + + cal_s_interval(ug, &(aw[k]), &s2); + if(!cal_x_interval(ug, &s1, &s2, &s3)) continue; + if(aw[k].v == (et^1)) { + kv_push(ul_ov_t, *res, s3); + } else { + az = asg_arc_a(ug->g, (aw[k].v^1)); nz = asg_arc_n(ug->g, (aw[k].v^1)); + for (z = 0; z < nz; z++) { + if(az[z].del) continue; + if(az[z].v == et) { + cal_s_interval(ug, &(az[z]), &s4); + if(cal_x_interval(ug, &s3, &s4, &rr)) { + kv_push(ul_ov_t, *res, rr); + } + } + } + } + } + } + } +} + + +void gen_src_shared_interval_adv(uint32_t src, ma_ug_t *ug, kv_ul_ov_t *res) +{ + uint32_t st, v, w, i, k, z, nv, nw, nz, rn; asg_arc_t *av, *aw, *az; + ul_ov_t s1, s2, s3, s4, s5; rn = res->n; ///res->n = 0; + + st = src<<1; + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + if((av[i].v>>1) == (st>>1)) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + if(s1.qn == src && s1.tn != src) kv_push(ul_ov_t, *res, s1); + + aw = asg_arc_a(ug->g, (w^1)); + nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + + cal_s_interval(ug, &(aw[k]), &s2); + if(!cal_x_interval(ug, &s1, &s2, &s3)) continue; + if(s3.qn == src && s3.tn != src) kv_push(ul_ov_t, *res, s3); + + az = asg_arc_a(ug->g, (aw[k].v^1)); + nz = asg_arc_n(ug->g, (aw[k].v^1)); + for (z = 0; z < nz; z++) { + if(az[z].del) continue; + cal_s_interval(ug, &(az[z]), &s4); + if(!cal_x_interval(ug, &s3, &s4, &s5)) continue; + if(s5.qn == src && s5.tn != src) kv_push(ul_ov_t, *res, s5); + } + } + } + + st ^= 1; + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + if((av[i].v>>1) == (st>>1)) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + if(s1.qn == src && s1.tn != src) kv_push(ul_ov_t, *res, s1); + + aw = asg_arc_a(ug->g, (w^1)); + nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + + cal_s_interval(ug, &(aw[k]), &s2); + if(!cal_x_interval(ug, &s1, &s2, &s3)) continue; + if(s3.qn == src && s3.tn != src) kv_push(ul_ov_t, *res, s3); + + az = asg_arc_a(ug->g, (aw[k].v^1)); + nz = asg_arc_n(ug->g, (aw[k].v^1)); + for (z = 0; z < nz; z++) { + if(az[z].del) continue; + cal_s_interval(ug, &(az[z]), &s4); + if(!cal_x_interval(ug, &s3, &s4, &s5)) continue; + if(s5.qn == src && s5.tn != src) kv_push(ul_ov_t, *res, s5); + } + } + } + + radix_sort_ul_ov_srt_tn(res->a + rn, res->a + res->n); + if(res->n > rn) { + uint64_t os, oe, ovlp, on; + for (k = rn + 1, i = rn; k <= res->n; k++) { + if(k == res->n || res->a[k].tn != res->a[i].tn) { + on = k - i; + if(k - i > 1) { + for (v = i; v < k; v++) { + if(res->a[v].tn == (uint32_t)-1) continue; + for (z = i; z < k; z++) { + if(v == z) continue; + if(res->a[z].tn == (uint32_t)-1) continue; + if(res->a[v].rev != res->a[z].rev) continue; + + os = MAX(res->a[v].qs, res->a[z].qs); + oe = MIN(res->a[v].qe, res->a[z].qe); + if(oe <= os) continue; + ovlp = oe - os; + if(ovlp <= ((res->a[v].qe-res->a[v].qs)*0.95)) continue; + if(ovlp <= ((res->a[z].qe-res->a[z].qs)*0.95)) continue; + + + os = MAX(res->a[v].ts, res->a[z].ts); + oe = MIN(res->a[v].te, res->a[z].te); + if(oe <= os) continue; + ovlp = oe - os; + if(ovlp <= ((res->a[v].te-res->a[v].ts)*0.95)) continue; + if(ovlp <= ((res->a[z].te-res->a[z].ts)*0.95)) continue; + + + if(res->a[v].qs > res->a[z].qs) res->a[v].qs = res->a[z].qs; + if(res->a[v].qe < res->a[z].qe) res->a[v].qe = res->a[z].qe; + if(res->a[v].ts > res->a[z].ts) res->a[v].ts = res->a[z].ts; + if(res->a[v].te < res->a[z].te) res->a[v].te = res->a[z].te; + res->a[z].tn = (uint32_t)-1; on--; + + // uint64_t vd, vz; + // vd = (uint64_t)(res->a[v].qe-res->a[v].qs) + (uint64_t)(res->a[v].te-res->a[v].ts); + // zd = (uint64_t)(res->a[z].qe-res->a[z].qs) + (uint64_t)(res->a[z].te-res->a[z].ts); + // ovlp = ((vd>=zd)?(vd-zd):(zd-vd)); + // if(ovlp > 64) { + // if(vd < zd) { + // res->a[v].tn = (uint32_t)-1; break; + // } else if(vd < zd) { + // res->a[z].tn = (uint32_t)-1; continue; + // } + // } + + // vd = ((res->a[v].qe-res->a[v].qs)>=(res->a[v].te-res->a[v].ts))? + // ((res->a[v].qe-res->a[v].qs)-(res->a[v].te-res->a[v].ts)): + // ((res->a[v].te-res->a[v].ts)-(res->a[v].qe-res->a[v].qs)); + // zd = ((res->a[z].qe-res->a[z].qs)>=(res->a[z].te-res->a[z].ts))? + // ((res->a[z].qe-res->a[z].qs)-(res->a[z].te-res->a[z].ts)): + // ((res->a[z].te-res->a[z].ts)-(res->a[z].qe-res->a[z].qs)); + // if(zd <= vd) { + // res->a[v].tn = (uint32_t)-1; break; + // } else { + // res->a[z].tn = (uint32_t)-1; + // } + } + } + if(on > 1) radix_sort_ul_ov_srt_qs(res->a + i, res->a + k); + } + i = k; + } + } + for (k = z = rn; k < res->n; k++) { + if(res->a[k].tn == (uint32_t)-1) continue; + res->a[z++] = res->a[k]; + } + res->n = z; + } +} + + +uint32_t gen_src_shared_interval_simple(uint32_t src, ma_ug_t *ug, kv_ul_ov_t *res) +{ + uint32_t st, v, w, i, k, z, nv, nw, nz, rn; asg_arc_t *av, *aw, *az; + ul_ov_t s1, s2, s3, s4, s5; rn = res->n; + + st = src<<1; + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + if((av[i].v>>1) == (st>>1)) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + if(s1.qn == src && s1.tn != src) kv_push(ul_ov_t, *res, s1); + + aw = asg_arc_a(ug->g, (w^1)); + nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + + cal_s_interval(ug, &(aw[k]), &s2); + if(!cal_x_interval(ug, &s1, &s2, &s3)) continue; + if(s3.qn == src && s3.tn != src) kv_push(ul_ov_t, *res, s3); + + az = asg_arc_a(ug->g, (aw[k].v^1)); + nz = asg_arc_n(ug->g, (aw[k].v^1)); + for (z = 0; z < nz; z++) { + if(az[z].del) continue; + cal_s_interval(ug, &(az[z]), &s4); + if(!cal_x_interval(ug, &s3, &s4, &s5)) continue; + if(s5.qn == src && s5.tn != src) kv_push(ul_ov_t, *res, s5); + } + } + } + + st ^= 1; + v = st; av = asg_arc_a(ug->g, v); nv = asg_arc_n(ug->g, v); + for (i = 0; i < nv; i++) { + if(av[i].del) continue; + if((av[i].v>>1) == (st>>1)) continue; + w = av[i].v; + cal_s_interval(ug, &(av[i]), &s1); + if(s1.qn == src && s1.tn != src) kv_push(ul_ov_t, *res, s1); + + aw = asg_arc_a(ug->g, (w^1)); + nw = asg_arc_n(ug->g, (w^1)); + for (k = 0; k < nw; k++) { + if(aw[k].del) continue; + if((aw[k].v>>1) == (st>>1)) continue; + + cal_s_interval(ug, &(aw[k]), &s2); + if(!cal_x_interval(ug, &s1, &s2, &s3)) continue; + if(s3.qn == src && s3.tn != src) kv_push(ul_ov_t, *res, s3); + + az = asg_arc_a(ug->g, (aw[k].v^1)); + nz = asg_arc_n(ug->g, (aw[k].v^1)); + for (z = 0; z < nz; z++) { + if(az[z].del) continue; + cal_s_interval(ug, &(az[z]), &s4); + if(!cal_x_interval(ug, &s3, &s4, &s5)) continue; + if(s5.qn == src && s5.tn != src) kv_push(ul_ov_t, *res, s5); + } + } + } + + radix_sort_ul_ov_srt_tn(res->a + rn, res->a + res->n); + if(res->n > rn) { + uint64_t os, oe, ovlp; + for (k = rn + 1, i = rn; k <= res->n; k++) { + if(k == res->n || res->a[k].tn != res->a[i].tn) { + // on = k - i; + if(k - i > 1) { + for (v = i; v < k; v++) { + if(res->a[v].tn == (uint32_t)-1) continue; + for (z = i; z < k; z++) { + if(v == z) continue; + if(res->a[z].tn == (uint32_t)-1) continue; + if(res->a[v].rev != res->a[z].rev) continue; + + os = MAX(res->a[v].qs, res->a[z].qs); + oe = MIN(res->a[v].qe, res->a[z].qe); + if(oe <= os) continue; + ovlp = oe - os; + if(ovlp <= ((res->a[v].qe-res->a[v].qs)*0.95)) continue; + if(ovlp <= ((res->a[z].qe-res->a[z].qs)*0.95)) continue; + + + os = MAX(res->a[v].ts, res->a[z].ts); + oe = MIN(res->a[v].te, res->a[z].te); + if(oe <= os) continue; + ovlp = oe - os; + if(ovlp <= ((res->a[v].te-res->a[v].ts)*0.95)) continue; + if(ovlp <= ((res->a[z].te-res->a[z].ts)*0.95)) continue; + + + if(res->a[v].qs > res->a[z].qs) res->a[v].qs = res->a[z].qs; + if(res->a[v].qe < res->a[z].qe) res->a[v].qe = res->a[z].qe; + if(res->a[v].ts > res->a[z].ts) res->a[v].ts = res->a[z].ts; + if(res->a[v].te < res->a[z].te) res->a[v].te = res->a[z].te; + res->a[z].tn = (uint32_t)-1; ///on--; + } + } + // if(on > 1) radix_sort_ul_ov_srt_qs(res->a + i, res->a + k); + } + i = k; + } + } + for (k = z = rn; k < res->n; k++) { + if(res->a[k].tn == (uint32_t)-1) continue; + res->a[z] = res->a[k]; res->a[z].sec = ((uint32_t)(0x3fffffff)); + z++; + } + res->n = z; + } + return res->n - rn; +} + +inline void overlap_region2ul_ov_t(overlap_region *in, ul_ov_t *ou, uint64_t tl) +{ + ou->qn = in->x_id; ou->tn = in->y_id; ou->el = 1; + ou->sec = in->non_homopolymer_errors; + ou->rev = in->y_pos_strand; + ou->qs = in->x_pos_s; ou->qe = in->x_pos_e+1; + if(ou->rev) { + ou->ts = tl - (in->y_pos_e+1); + ou->te = tl - in->y_pos_s; + } else { + ou->ts = in->y_pos_s; + ou->te = in->y_pos_e+1; + } +} + +void extend_batch(ul_ov_t *z, int64_t ql, int64_t tl, int64_t *rqs, int64_t *rqe, int64_t *rts, int64_t *rte) +{ + int64_t q0, t0, q1, t1; + (*rqs) = (*rqe) = (*rts) = (*rte) = -1; + if(!(z->rev)) { + q0 = z->qs; t0 = z->ts; + if(q0 <= t0) { + (*rqs) = z->qs - q0; (*rts) = z->ts - q0; + } else { + (*rqs) = z->qs - t0; (*rts) = z->ts - t0; + } + + q1 = ql - z->qe; t1 = tl - z->te; + if(q1 <= t1) { + (*rqe) = z->qe + q1; (*rte) = z->te + q1; + } else { + (*rqe) = z->qe + t1; (*rte) = z->te + t1; + } + } else { + q0 = z->qs; t0 = tl - z->te; + if(q0 <= t0) { + (*rqs) = z->qs - q0; (*rte) = z->te + q0; + } else { + (*rqs) = z->qs - t0; (*rte) = z->te + t0; + } + + q1 = ql - z->qe; t1 = z->ts; + if(q1 <= t1) { + (*rqe) = z->qe + q1; (*rts) = z->ts - q1; + } else { + (*rqe) = z->qe + t1; (*rts) = z->ts - t1; + } + } +} + +uint64_t check_ul_ov_t_consist(ul_ov_t *x, ul_ov_t *y, int64_t ql, int64_t tl, double diff) +{ + if(x->qn == y->qn && x->tn == y->tn && x->rev == y->rev) { + int64_t xqs, xqe, xts, xte, yqs, yqe, yts, yte, os, oe, ovlp; + extend_batch(x, ql, tl, &xqs, &xqe, &xts, &xte); + extend_batch(y, ql, tl, &yqs, &yqe, &yts, &yte); + + os = MAX(xqs, yqs); oe = MIN(xqe, yqe); + if(oe <= os) return 0; + ovlp = oe - os; + if(ovlp <= ((xqe-xqs)*(1-diff))) return 0; + if(ovlp <= ((yqe-yqs)*(1-diff))) return 0; + + os = MAX(xts, yts); oe = MIN(xte, yte); + if(oe <= os) return 0; + if(ovlp <= ((xte-xts)*(1-diff))) return 0; + if(ovlp <= ((yte-yts)*(1-diff))) return 0; + return 1; + } + return 0; +} + +ul_ov_t* get_mask_interval(ul_ov_t *in, kv_ul_ov_t *idx, uint64_t *ii, int64_t ql, int64_t tl, uint64_t *n, uint64_t *n_skip) +{ + int64_t idx_n = idx->n, i = (*ii), m; ul_ov_t *a, z; + (*n) = (*n_skip) = 0; + if(i >= idx_n) i = idx_n-1; + if(i < 0) i = 0; + // if(in->qn == 101 && in->tn == 102) { + // fprintf(stderr, "-0-[M::%s]\tutg%.6ul\t%c\tutg%.6ul\ti::%ld\tidx_n::%ld\n", __func__, + // in->qn+1, "+-"[in->rev], in->tn+1, i, idx_n); + // } + if(i >= 0 && i < idx_n) { + while (i >= 0 && in->tn <= idx->a[i].tn) i--; + // if(in->qn == 101 && in->tn == 102) { + // fprintf(stderr, "-1-[M::%s]\tutg%.6ul\t%c\tutg%.6ul\ti::%ld\tidx_n::%ld\n", __func__, + // in->qn+1, "+-"[in->rev], in->tn+1, i, idx_n); + // } + if(i >= 0) { + while (i < idx_n && in->tn > idx->a[i].tn) i++; + // if(in->qn == 101 && in->tn == 102) { + // fprintf(stderr, "-2-[M::%s]\tutg%.6ul\t%c\tutg%.6ul\ti::%ld\tidx_n::%ld\n", __func__, + // in->qn+1, "+-"[in->rev], in->tn+1, i, idx_n); + // } + if(i < idx_n && in->tn == idx->a[i].tn) { + for (m = i, a = idx->a + i; i < idx_n && in->tn == idx->a[i].tn; i++) { + if(!check_ul_ov_t_consist(in, &(idx->a[i]), ql, tl, 0.06)) { + (*n_skip)++; continue; + } + if(m != i) { + z = idx->a[i]; idx->a[i] = idx->a[m]; idx->a[m] = z; + } + m++; (*n)++; + } + // if(in->qn == 101 && in->tn == 102) { + // fprintf(stderr, "-3-[M::%s]\tutg%.6ul\t%c\tutg%.6ul\ti::%ld\tidx_n::%ld\n", __func__, + // in->qn+1, "+-"[in->rev], in->tn+1, i, idx_n); + // } + (*ii) = i; + if((*n)) return a; + } + } + } + (*ii) = i; + return NULL; +} + +inline uint64_t is_mask_err_full(ul_ov_t *sa, uint64_t sn, uint64_t qs, uint64_t qe, uint64_t ts, uint64_t te) +{ + uint64_t k; + for (k = 0; k < sn; k++) { + if(sa[k].qs <= qs && sa[k].qe >= qe && sa[k].ts <= ts && sa[k].te >= te) break; + } + if(k < sn) return 1; + return 0; +} + +int64_t cal_exact_len(bit_extz_t *ez, int64_t rev, ul_ov_t *sa, uint64_t sn, int64_t ylen, int64_t *xoff, int64_t *yoff, int64_t *exac, int64_t *err, int64_t backward) +{ + (*xoff) = (*yoff) = (*exac) = (*err) = 0; + if(!ez->cigar.n) return 0; + int64_t cn = ez->cigar.n, op, ck, xk, yk, xs, xe, ys, ye, se; + if(!backward) { + xk = ez->ts; yk = ez->ps; + for (ck = 0; ck < cn; ck++) { + xs = xk; ys = yk; + op = ez->cigar.a[ck]>>14; + se = (ez->cigar.a[ck]&(0x3fff)); + if(op!=2) xk += se; + if(op!=3) yk += se; + xe = xk; ye = yk; + if(!op) {///exact match + (*exac) += se; continue; + } + ///if there is an unmasked error + if((!sn) || (!is_mask_err_full(sa, sn, xs, xe, ((rev)?(ylen-ye):(ys)), ((rev)?(ylen-ys):(ye))))) { + (*xoff) = xs - ez->ts; (*yoff) = ys - ez->ps; + return 0; + } + (*exac) = 0; (*err) += se;///if the error has been masked + } + } else { + xk = ez->te+1; yk = ez->pe+1; + for (ck = cn-1; ck >= 0; ck--) { + xe = xk; ye = yk; + op = ez->cigar.a[ck]>>14; + se = (ez->cigar.a[ck]&(0x3fff)); + if(op!=2) xk -= se; + if(op!=3) yk -= se; + xs = xk; ys = yk; + if(!op) {///exact match + (*exac) += se; continue; + } + ///if there is an unmasked error + if((!sn) || (!is_mask_err_full(sa, sn, xs, xe, ((rev)?(ylen-ye):(ys)), ((rev)?(ylen-ys):(ye))))) { + (*xoff) = ez->te+1-xe; (*yoff) = ez->pe+1-ye; + return 0; + } + (*exac) = 0; (*err) += se;///if the error has been masked + } + + } + (*xoff) = ez->te+1-ez->ts; (*yoff) = ez->pe+1-ez->ps; + return 1; +} + +int64_t cal_exact_batch(overlap_region *z, ul_ov_t *sa, uint64_t sn, int64_t ylen, uint64_t *q0l, uint64_t *t0l, uint64_t *e0l, uint64_t *q1l, uint64_t *t1l, uint64_t *e1l) +{ + int64_t wn = z->w_list.n, wk, xk, yk, is_t, is_f, err0, err1, tot; window_list *m; bit_extz_t ez; + int64_t qs, qe, ts, te, rev = !!(z->y_pos_strand), qoff, toff, elen, sube; + tot = z->non_homopolymer_errors; + if(!tot) { + (*q0l) = (*q1l) = z->x_pos_e+1-z->x_pos_s; + (*t0l) = (*t1l) = z->y_pos_e+1-z->y_pos_s; + (*e0l) = (*e1l) = z->x_pos_e+1-z->x_pos_s; + return 0; + } + (*q0l) = (*t0l) = (*e0l) = (*q1l) = (*t1l) = (*e1l) = (uint64_t)-1; + wk = xk = yk = -1; if(!wn) return INT32_MAX; + + ///forward + (*q0l) = (*t0l) = (*e0l) = err0 = 0; + wk = 0; xk = z->w_list.a[wk].x_start; yk = z->w_list.a[wk].y_start; is_t = 0; + if((xk == z->x_pos_s) && (yk == z->y_pos_s)) is_t = 1; + if((!is_t) && (sn > 0)) {///if the flanking unmapped regions could be masked + qs = z->x_pos_s; qe = xk; + if(!rev) { + ts = z->y_pos_s; te = yk; + } else { + ts = ylen - yk; te = ylen - z->y_pos_s; + } + is_t = is_mask_err_full(sa, sn, qs, qe, ts, te); + if(is_t) { + (*q0l) += qe - qs; (*t0l) += te - ts; (*e0l) = 0; + } + } + if(is_t) { + for (; wk < wn; wk++) { + m = &(z->w_list.a[wk]); + qs = m->x_start; qe = m->x_end+1; + if(!rev) { + ts = m->y_start; te = m->y_end+1; + } else { + ts = ylen-(m->y_end+1); te = ylen-m->y_start; + } + + if((is_ualn_win((*m))) || ((is_est_aln((*m))) && (m->error > 0))) {///unmapped + if(is_mask_err_full(sa, sn, qs, qe, ts, te)) { + (*q0l) += qe - qs; (*t0l) += te - ts; (*e0l) = 0; + continue; + } + break; + } + if(m->error == 0) {///an exactly matched window + (*q0l) += qe - qs; (*t0l) += te - ts; (*e0l) += qe - qs; + continue; + } + set_bit_extz_t(ez, (*z), wk); + is_f = cal_exact_len(&ez, rev, sa, sn, ylen, &qoff, &toff, &elen, &sube, 0); + (*q0l) += qoff; (*t0l) += toff; err0 += sube; + ///exactly match + if(elen == qoff && elen == toff) (*e0l) += elen; + else (*e0l) = elen; + if(!is_f) break; + } + } + + + ///backward + (*q1l) = (*t1l) = (*e1l) = err1 = 0; + wk = wn-1; xk = z->w_list.a[wk].x_end+1; yk = z->w_list.a[wk].y_end+1; is_t = 0; + if((xk == (z->x_pos_e+1)) && (yk == (z->y_pos_e+1))) is_t = 1; + if((!is_t) && (sn > 0)) {///if the flanking unmapped regions could be masked + qs = xk; qe = z->x_pos_e+1; + if(!rev) { + ts = yk; te = z->y_pos_e+1; + } else { + ts = ylen-(z->y_pos_e+1); te = ylen-yk; + } + is_t = is_mask_err_full(sa, sn, qs, qe, ts, te); + if(is_t) { + (*q1l) += qe - qs; (*t1l) += te - ts; (*e1l) = 0; + } + } + if(is_t) { + for (; wk >= 0; wk--) { + m = &(z->w_list.a[wk]); + qs = m->x_start; qe = m->x_end+1; + if(!rev) { + ts = m->y_start; te = m->y_end+1; + } else { + ts = ylen-(m->y_end+1); te = ylen-m->y_start; + } + + if((is_ualn_win((*m))) || ((is_est_aln((*m))) && (m->error > 0))) {///unmapped + if(is_mask_err_full(sa, sn, qs, qe, ts, te)) { + (*q1l) += qe - qs; (*t1l) += te - ts; (*e1l) = 0; + continue; + } + break; + } + if(m->error == 0) {///an exactly matched window + (*q1l) += qe - qs; (*t1l) += te - ts; (*e1l) += qe - qs; + continue; + } + set_bit_extz_t(ez, (*z), wk); + is_f = cal_exact_len(&ez, rev, sa, sn, ylen, &qoff, &toff, &elen, &sube, 1); + (*q1l) += qoff; (*t1l) += toff; err1 += sube; + ///exactly match + if(elen == qoff && elen == toff) (*e1l) += elen; + else (*e1l) = elen; + if(!is_f) break; + } + } + + if((*q0l) >= (z->x_pos_e+1-z->x_pos_s) && (*t0l) >= (z->y_pos_e+1-z->y_pos_s)) { + assert(((*q0l) == (*q1l)) && ((*t0l) == (*t1l)) && (err0 == err1)); + assert(err0 == tot); + return 0; + } + + // if(!(err0+err1 < tot)) { + // fprintf(stderr, "[M::%s]\tutg%.6ul\tutg%.6ul\terr0::%ld\terr1::%ld\ttot::%ld\n", __func__, + // z->x_id+1, z->y_id+1, err0, err1, tot); + // fprintf(stderr, "[M::%s]\tutg%.6ul\tq0l::%lu\tt0l::%lu\te0l::%lu\tutg%.6ul\tq1l::%lu\tt1l::%lu\te1l::%lu\n", __func__, + // z->x_id+1, *q0l, *t0l, *e0l, z->y_id+1, *q1l, *t1l, *e1l); + // } + + assert(err0+err1 < tot); + return tot-(err0+err1); +} + +void push_graph_bin_back(overlap_region_alloc* ol, const ul_idx_t *udb, hpc_re_t *hre, +double erate, uint64_t maxe, uint64_t minov, uint64_t rid, asg64_v *b0, asg64_v *b1, mask_ul_ov_t *mk) +{ + uint64_t k, i, m, si, sn, skip_n, zt, re; overlap_region t, *z; ul_ov_t p, *sa; int64_t npc; + uint64_t q0l, t0l, e0l, q1l, t1l, e1l, mm, occ[3]; + b0->n = b1->n = 0; + + mk->srt.n = mk->idx.n = 0; + gen_src_shared_interval_adv(rid, udb->ug, &(mk->srt)); + kv_resize(uint64_t, mk->idx, ol->length); + for (k = 0; k < ol->length; k++) { + kv_push(uint64_t, mk->idx, (((uint64_t)ol->list[k].y_id)<<32)|((uint64_t)k)); + } + radix_sort_gfa64(mk->idx.a, mk->idx.a+mk->idx.n); + for (i = si = b0->n = 0, occ[0] = occ[1] = occ[2] = 0; i < mk->idx.n; i++) { + k = (uint32_t)mk->idx.a[i]; + z = &(ol->list[k]); + overlap_region2ul_ov_t(z, &p, udb->ug->u.a[z->y_id].len); + if(((p.qe-p.qs) >= minov) && ((p.te-p.ts) >= minov)) { + if(!trans_ovlp_connect(&p, udb->ug)) continue;//could be eaisly detected + if(p.sec == 0) {///no error + zt = ((uint32_t)-1); zt <<= 32; zt |= (i<<1); occ[2]++; + kv_push(uint64_t, *b0, zt); continue; + } + sa = get_mask_interval(&p, &(mk->srt), &si, udb->ug->u.a[p.qn].len, udb->ug->u.a[p.tn].len, &sn, &skip_n); + re = cal_exact_batch(z, sa, sn, udb->ug->u.a[p.tn].len, &q0l, &t0l, &e0l, &q1l, &t1l, &e1l); + if(!re) {//no unmask errors + zt = ((uint32_t)-1); zt <<= 32; zt |= (i<<1); occ[2]++; + kv_push(uint64_t, *b0, zt); continue; + } + + mm = ((re <= maxe)?((uint64_t)(0x8000000000000000)):(0)); + ///left end + if(q0l >= minov && t0l >= minov) { + zt = MIN(q0l, t0l); zt <<= 32; zt |= (i<<1); zt |= mm; occ[!!mm]++; + kv_push(uint64_t, *b0, zt); + } + + ///right end + if(q1l >= minov && t1l >= minov) { + zt = MIN(q1l, t1l); zt <<= 32; zt |= (i<<1); zt += 1; zt |= mm; occ[!!mm]++; + kv_push(uint64_t, *b0, zt); + } + } + } + + // radix_sort_gfa64(b0->a, b0->a+b0->n); + // uint64_t *a0, *a1, *a2, a0n, a1n, a2n; + // a2 = b0->a; a2n = occ[0]; + // a1 = a2 + a2n; a1n = occ[1]; + // a0 = a1 + a1n; a0n = occ[2]; + + + fprintf(stderr, "\n[M::%s]\tutg%.6lu%c\t#s::%u\n", __func__, rid+1, "lc"[udb->ug->u.a[rid].circ], (uint32_t)mk->srt.n); + for (k = 0; k < mk->srt.n; k++) { + fprintf(stderr, "[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\n", __func__, + mk->srt.a[k].qn+1, "lc"[udb->ug->u.a[mk->srt.a[k].qn].circ], mk->srt.a[k].qs, mk->srt.a[k].qe, "+-"[mk->srt.a[k].rev], + mk->srt.a[k].tn+1, "lc"[udb->ug->u.a[mk->srt.a[k].tn].circ], mk->srt.a[k].ts, mk->srt.a[k].te); + } + + + + for (k = m = 0; k < ol->length; k++) { + z = &(ol->list[k]); + if(z->non_homopolymer_errors == 0) continue; + p.qn = rid; p.tn = ol->list[k].y_id; p.el = 1; + p.sec = ol->list[k].non_homopolymer_errors; + p.rev = ol->list[k].y_pos_strand; + p.qs = ol->list[k].x_pos_s; p.qe = ol->list[k].x_pos_e+1; + if(p.rev) { + p.ts = udb->ug->u.a[p.tn].len - (ol->list[k].y_pos_e+1); + p.te = udb->ug->u.a[p.tn].len - ol->list[k].y_pos_s; + } else { + p.ts = ol->list[k].y_pos_s; + p.te = ol->list[k].y_pos_e+1; + } + // gen_shared_interval(&p, udb->ug, bl); + // gen_shared_interval_adv(&p, udb->ug, bl); + // if(!trans_ovlp_connect(&p, udb->ug)) continue; + // assert(z->x_id == rid); + // if(!get_high_simi(z, erate, maxe, minov, &e0, &e1)) continue; + npc = cal_npc_err(z, p.qs, p.qe, hre, udb->ug->u.a[p.tn].len); + if(npc == INT32_MAX || npc < 0) continue; + if(((uint64_t)npc <= maxe) && ((uint64_t)npc <= ((p.qe - p.qs)*erate)) && ((uint64_t)npc <= ((p.te - p.ts)*erate))) { + ; + } else if(((uint64_t)npc > maxe) && ((p.qe - p.qs) > minov) && ((p.te - p.ts) > minov)) { + ; + } + + // if((z->x_id == 44 && z->y_id == 45) || (z->x_id == 45 && z->y_id == 44)) + // if(npc <= 4) + { + // int64_t nnpc; + // nnpc = cal_npc_err(z, p.qs, p.qe, NULL, udb->ug->u.a[p.tn].len); + // fprintf(stderr, "[M::%s]\tutg%.6u%c->utg%.6u%c\terr::%u\tnpc::%ld\t#s::%u\n", __func__, p.qn+1, "lc"[udb->ug->u.a[p.qn].circ], + // p.tn+1, "lc"[udb->ug->u.a[p.tn].circ], ol->list[k].non_homopolymer_errors, npc, (uint32_t)bl->n); + // uint64_t t; + // for (t = 0; t < bl->n; t++) { + // fprintf(stderr, "[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\n", __func__, + // bl->a[t].qn+1, "lc"[udb->ug->u.a[bl->a[t].qn].circ], bl->a[t].qs, bl->a[t].qe, "+-"[bl->a[t].rev], + // bl->a[t].tn+1, "lc"[udb->ug->u.a[bl->a[t].tn].circ], bl->a[t].ts, bl->a[t].te); + // } + } + + if(k != m) { + t = ol->list[k]; + ol->list[k] = ol->list[m]; + ol->list[m] = t; + } + m++; + } + ol->length = m; +} + +void full_lst_set(emask_t *a, uint64_t a_n, ul_ov_t *sa, uint64_t sn, uint64_t skip_n) +{ + uint64_t ff = 0, k, i; int64_t sql, stl, os, oe, ovlp; + if(sn == 0 && skip_n == 0) { + ff = 1; + } else if(skip_n > 0) { + ff = 0; + } else {///sn > 0 && skip_n == 0 + for (k = 0; k < sn; k++) { + sql = sa[k].qe - sa[k].qs; + stl = sa[k].te - sa[k].ts; + for (i = 0; i < a_n && sql > 0 && stl > 0; i++) { + os = MAX(sa[k].qs, a[i].qs); oe = MIN(sa[k].qe, a[i].qe); + ovlp = ((oe>os)?(oe-os):(0)); sql -= ovlp; + + os = MAX(sa[k].ts, a[i].ts); oe = MIN(sa[k].te, a[i].te); + ovlp = ((oe>os)?(oe-os):(0)); stl -= ovlp; + } + if((sql > 256) && (sql > ((sa[k].qe - sa[k].qs)*0.06))) break; + if((sql <= 256) && (sql > ((sa[k].qe - sa[k].qs)*0.6))) break; + if((stl > 256) && (stl > ((sa[k].te - sa[k].ts)*0.06))) break; + if((stl <= 256) && (stl > ((sa[k].te - sa[k].ts)*0.6))) break; + } + if(k < sn) ff = 0; + else ff = 1; + } + for (k = 0; k < a_n; k++) a[k].full = ff; +} + +void push_emask0(kv_emask_t *res, uint64_t q0l, uint64_t t0l, uint64_t e0l, uint64_t q1l, uint64_t t1l, uint64_t e1l, +uint64_t end, uint64_t is_pe, uint64_t minov, ul_ov_t *p) +{ + emask_t *m; + m = &(res->a[res->n++]); memset(m, 0, sizeof((*m))); + m->dir = end; m->pe = is_pe; + m->tn = p->tn; m->rev = p->rev; m->rn = (uint32_t)-1; + if(!end) {///left end + assert(q0l >= minov && t0l >= minov); + m->el = e0l; + m->qs = p->qs; m->qe = p->qs + q0l; + if(!(m->rev)) { + m->ts = p->ts; m->te = p->ts + t0l; + } else { + m->te = p->te; m->ts = p->te - t0l; + } + } else {///right end + assert(q1l >= minov && t1l >= minov); + m->el = e1l; + m->qe = p->qe; m->qs = p->qe - q1l; + if(!(m->rev)) { + m->te = p->te; m->ts = p->te - t1l; + } else { + m->ts = p->ts; m->te = p->ts + t1l; + } + } +} + +void push_emask_lst(mask_ul_ov_t *mk, overlap_region_alloc* ol, uint64_t minov, ma_ug_t *ug, uint64_t *a, uint64_t an, kv_emask_t *res, uint64_t is_pe) +{ + uint64_t i, l, sn, si, re, cn, skip_n; overlap_region *z; ul_ov_t p, *sa; + uint64_t q0l, t0l, e0l, q1l, t1l, e1l; emask_t *m; + if(is_pe) { + for (i = 0; i < an; i++) { + a[i] <<= 32; a[i] >>= 32; + } + radix_sort_gfa64(a, a+an); + for (i = 1, l = si = 0; i <= an; i++) { + if(i == an || (a[i]>>1) != (a[l]>>1)) { + // k = (uint32_t)mk->idx.a[a[l]>>1]; + z = &(ol->list[(uint32_t)mk->idx.a[a[l]>>1]]); + overlap_region2ul_ov_t(z, &p, ug->u.a[z->y_id].len); + assert(i<=l+2); + assert((p.qe-p.qs) >= minov); + assert((p.te-p.ts) >= minov); + assert(p.sec > 0); + sa = get_mask_interval(&p, &(mk->srt), &si, ug->u.a[p.qn].len, ug->u.a[p.tn].len, &sn, &skip_n); + re = cal_exact_batch(z, sa, sn, ug->u.a[p.tn].len, &q0l, &t0l, &e0l, &q1l, &t1l, &e1l); + assert(re); + cn = i - l; + + push_emask0(res, q0l, t0l, e0l, q1l, t1l, e1l, a[l]&1, ((i==(l+2))?1:0), minov, &p); l++; + if(l < i) push_emask0(res, q0l, t0l, e0l, q1l, t1l, e1l, a[l]&1, ((i==(l+2))?1:0), minov, &p); + // fprintf(stderr, "[M::%s]\tres->n::%u\tcn::%lu\n", __func__, (uint32_t)res->n, cn); + full_lst_set(res->a+res->n-cn, cn, sa, sn, skip_n); + l = i; + } + } + } else { + for (i = 0; i < an; i++) { + a[i] <<= 32; a[i] >>= 32; + z = &(ol->list[(uint32_t)mk->idx.a[a[i]>>1]]); + overlap_region2ul_ov_t(z, &p, ug->u.a[z->y_id].len); + assert((p.qe-p.qs) >= minov); + assert((p.te-p.ts) >= minov); + if(p.sec > 0) { + sa = get_mask_interval(&p, &(mk->srt), &si, ug->u.a[p.qn].len, ug->u.a[p.tn].len, &sn, &skip_n); + re = cal_exact_batch(z, sa, sn, ug->u.a[p.tn].len, &q0l, &t0l, &e0l, &q1l, &t1l, &e1l); + assert(re == 0); + } + + m = &(res->a[res->n++]); memset(m, 0, sizeof((*m))); + m->dir = 2; m->pe = 0; m->tn = p.tn; m->rev = p.rev; m->full = 1; m->el = (uint32_t)-1; m->rn = (uint32_t)-1; + m->qs = p.qs; m->qe = p.qe; m->ts = p.ts; m->te = p.te; + } + } +} + +void srt_kv_emask_t(kv_emask_t *res, mask_ul_ov_t *mk, uint32_t rid, ma_ug_t *ug) +{ + uint64_t k, m, z; ul_ov_t *p; + mk->srt.n = mk->idx.n = 0; + kv_resize(uint64_t, mk->idx, res->n); + kv_resize(ul_ov_t, mk->srt, res->n); + + for (k = 0; k < res->n; k++) { + m = k + 1; z = (((uint64_t)res->a[k].tn)<<32)|(k<<1); + if((m < res->n) && (res->a[k].tn == res->a[m].tn) && (res->a[k].rev == res->a[m].rev) && + (res->a[k].pe) && (res->a[m].pe) && (res->a[k].dir == 0 && res->a[m].dir == 1)) { + z |= 1; + } + kv_push(uint64_t, mk->idx, z); + } + radix_sort_gfa64(mk->idx.a, mk->idx.a+mk->idx.n); + for (z = 0; z < mk->idx.n; z++) { + k = ((uint32_t)mk->idx.a[z])>>1; + kv_pushp(ul_ov_t, mk->srt, &p); + p->tn = res->a[k].tn; p->qn = res->a[k].el; + p->qs = res->a[k].qs; p->qe = res->a[k].qe; + p->ts = res->a[k].ts; p->te = res->a[k].te; + p->rev = res->a[k].rev; p->el = res->a[k].full; + m = res->a[k].dir; m <<= 1; m |= (uint32_t)res->a[k].pe; p->sec = m; + + if(mk->idx.a[z]&1) { + k++; + kv_pushp(ul_ov_t, mk->srt, &p); + p->tn = res->a[k].tn; p->qn = res->a[k].el; + p->qs = res->a[k].qs; p->qe = res->a[k].qe; + p->ts = res->a[k].ts; p->te = res->a[k].te; + p->rev = res->a[k].rev; p->el = res->a[k].full; + m = res->a[k].dir; m <<= 1; m |= (uint32_t)res->a[k].pe; p->sec = m; + } + } + + for (k = 0; k < res->n; k++) { + p = &(mk->srt.a[k]); + res->a[k].tn = p->tn; res->a[k].el = p->qn; + res->a[k].qs = p->qs; res->a[k].qe = p->qe; + res->a[k].ts = p->ts; res->a[k].te = p->te; + res->a[k].rev = p->rev; res->a[k].full = p->el; res->a[k].rn = (uint32_t)-1; + m = p->sec; res->a[k].pe = m&1; res->a[k].dir = m>>1; + } + + // for (k = 0; k < res->n; k++) { + // int64_t fqs, fqe, fts, fte, ql = ug->g->seq[rid].len, tl = ug->g->seq[res->a[k].tn].len; + // p = &(mk->srt.a[k]); extend_batch(p, ql, tl, &fqs, &fqe, &fts, &fte); + // ///if(res->a[k].full == 0) + // { + // // fprintf(stderr, "+[M::%s]\tutg%.6ul\tq::[%u,\t%u)[%ld,\t%ld)\t%c\tutg%.6ul\tt::[%u,\t%u)[%ld,\t%ld)\tel::%u\tfu::%u\n", __func__, + // // rid+1, res->a[k].qs, res->a[k].qe, fqs, fqe, "+-"[res->a[k].rev], res->a[k].tn+1, res->a[k].ts, res->a[k].te, fts, fte, res->a[k].el, res->a[k].full); + // // fprintf(stderr, "utg%.6ul\tq::[%u,\t%u)[%ld,\t%ld)\t%c\tutg%.6ul\tt::[%u,\t%u)[%ld,\t%ld)\tel::%u\tfu::%u\n", + // // rid+1, res->a[k].qs, res->a[k].qe, fqs, fqe, "+-"[res->a[k].rev], res->a[k].tn+1, res->a[k].ts, res->a[k].te, fts, fte, res->a[k].el, res->a[k].full); + // } + // } +} + +void push_graph_bin(overlap_region_alloc* ol, const ul_idx_t *udb, double erate, uint64_t maxe, uint64_t minov, uint64_t rid, asg64_v *b0, mask_ul_ov_t *mk, kv_emask_t *res) +{ + uint64_t k, i, si, sn, zt, re, skip_n; overlap_region *z; ul_ov_t p, *sa; + uint64_t q0l, t0l, e0l, q1l, t1l, e1l, mm, occ[3], mocc, mn[3]; + b0->n = 0; + mocc = (udb->ug->u.a[rid].len/sizeof((*(res->a))))/8; if(mocc < 16) mocc = 16; + + // fprintf(stderr, "\n[M::%s]\tutg%.6lu%c\n", __func__, rid+1, "lc"[udb->ug->u.a[rid].circ]); + + mk->srt.n = mk->idx.n = 0; + gen_src_shared_interval_adv(rid, udb->ug, &(mk->srt)); + kv_resize(uint64_t, mk->idx, ol->length); + for (k = 0; k < ol->length; k++) { + kv_push(uint64_t, mk->idx, (((uint64_t)ol->list[k].y_id)<<32)|((uint64_t)k)); + } + radix_sort_gfa64(mk->idx.a, mk->idx.a+mk->idx.n); + // if(rid == 4) { + // sa = mk->srt.a; sn = mk->srt.n; + // for (si = 0; si < sn; si++) { + // fprintf(stderr, "si::%lu[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\n", si, __func__, + // sa[si].qn+1, "lc"[udb->ug->u.a[sa[si].qn].circ], sa[si].qs, sa[si].qe, "+-"[sa[si].rev], + // sa[si].tn+1, "lc"[udb->ug->u.a[sa[si].tn].circ], sa[si].ts, sa[si].te); + // } + // } + + + + for (i = si = b0->n = 0, occ[0] = occ[1] = occ[2] = 0; i < mk->idx.n; i++) { + k = (uint32_t)mk->idx.a[i]; + z = &(ol->list[k]); + overlap_region2ul_ov_t(z, &p, udb->ug->u.a[z->y_id].len); + if(((p.qe-p.qs) >= minov) && ((p.te-p.ts) >= minov)) { + if(!trans_ovlp_connect(&p, udb->ug)) continue;//could be eaisly detected + if(p.sec == 0) {///no error + zt = ((uint32_t)-1); zt <<= 32; zt |= (i<<1); occ[2]++; + kv_push(uint64_t, *b0, zt); continue; + } + sa = get_mask_interval(&p, &(mk->srt), &si, udb->ug->u.a[p.qn].len, udb->ug->u.a[p.tn].len, &sn, &skip_n); + // if(p.qn == 4 && p.tn == 3) { + // fprintf(stderr, "[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\tsn::%lu\tskip_n::%lu\n", __func__, + // p.qn+1, "lc"[udb->ug->u.a[p.qn].circ], p.qs, p.qe, "+-"[p.rev], + // p.tn+1, "lc"[udb->ug->u.a[p.tn].circ], p.ts, p.te, sn, skip_n); + // uint32_t ci; + // for (ci = 0; ci < sn; ci++) { + // fprintf(stderr, "sa::[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\n", __func__, + // sa[ci].qn+1, "lc"[udb->ug->u.a[sa[ci].qn].circ], sa[ci].qs, sa[ci].qe, "+-"[sa[ci].rev], + // sa[ci].tn+1, "lc"[udb->ug->u.a[sa[ci].tn].circ], sa[ci].ts, sa[ci].te); + // } + // } + + + re = cal_exact_batch(z, sa, sn, udb->ug->u.a[p.tn].len, &q0l, &t0l, &e0l, &q1l, &t1l, &e1l); + if(!re) {//no unmask errors + zt = ((uint32_t)-1); zt <<= 32; zt |= (i<<1); occ[2]++; + kv_push(uint64_t, *b0, zt); continue; + } + // if(p.qn == 4 && p.tn == 3) { + // fprintf(stderr, "[M::%s]\tutg%.6u%c\tq0l::%lu\tt0l::%lu\te0l::%lu\tutg%.6u%c\tq1l::%lu\tt1l::%lu\te1l::%lu\n", __func__, + // p.qn+1, "lc"[udb->ug->u.a[p.qn].circ], q0l, t0l, e0l, + // p.tn+1, "lc"[udb->ug->u.a[p.tn].circ], q1l, t1l, e1l); + // } + mm = ((re <= maxe)?((uint64_t)(0x8000000000000000)):(0)); + ///left end + if(q0l >= minov && t0l >= minov) { + zt = MIN(q0l, t0l); zt <<= 32; zt |= (i<<1); zt |= mm; occ[!!mm]++; + kv_push(uint64_t, *b0, zt); + } + + ///right end + if(q1l >= minov && t1l >= minov) { + zt = MIN(q1l, t1l); zt <<= 32; zt |= (i<<1); zt += 1; zt |= mm; occ[!!mm]++; + kv_push(uint64_t, *b0, zt); + } + } + } + + radix_sort_gfa64(b0->a, b0->a+b0->n); + uint64_t *a0, *a1, *a2, a0n, a1n, a2n, *a, m; + a2 = b0->a; a2n = occ[0]; mn[2] = mocc>>2;//high error + a1 = a2 + a2n; a1n = occ[1]; mn[1] = mocc>>1;//low error + a0 = a1 + a1n; a0n = occ[2]; mn[0] = mocc>>2;//0 error + mn[1] = mocc - mn[0] - mn[2]; + + sn = a2n; a = a2;//high error + for (i = 0, si = sn>>1; i < si; i++) { + m = a[i]; a[i] = a[sn-i-1]; a[sn-i-1] = m; + } + + sn = a1n; a = a1;//low error + for (i = 0, si = sn>>1; i < si; i++) { + m = a[i]; a[i] = a[sn-i-1]; a[sn-i-1] = m; + } + + sn = a0n; a = a0;//0 error + for (i = 0; i < sn; i++) { + z = &(ol->list[(((uint32_t)a[i])>>1)]); + q0l = z->x_pos_e+1-z->x_pos_s; + t0l = z->y_pos_e+1-z->y_pos_s; + m = ((uint32_t)-1) - (MIN(q0l, t0l)); m <<= 32; m |= ((uint32_t)a[i]); + a[i] = m; + } + radix_sort_gfa64(a, a+sn);///sort by length + ///a1 -> a2 -> a0 + if(mn[1] > a1n) mn[1] = a1n; + if(mn[2] > a2n) mn[2] = a2n; + if(mn[0] > a0n) mn[0] = a0n; + + ///reassign a1 + mn[1] = mocc - mn[0] - mn[2]; + if(mn[1] > a1n) mn[1] = a1n; + + ///reassign a2 + mn[2] = mocc - mn[1] - mn[0]; + if(mn[2] > a2n) mn[2] = a2n; + + ///reassign a0 + mn[0] = mocc - mn[1] - mn[2]; + if(mn[0] > a0n) mn[0] = a0n; + + res->n = res->m = mn[0]+mn[1]+mn[2]; + MALLOC(res->a, res->n); + res->n = 0; + + // fprintf(stderr, "\n[M::%s]\tutg%.6lu%c\t#s::%u\tmn[0]::%lu\tmn[1]::%lu\tmn[2]::%lu\n", __func__, + // rid+1, "lc"[udb->ug->u.a[rid].circ], (uint32_t)mk->srt.n, mn[0], mn[1], mn[2]); + push_emask_lst(mk, ol, minov, udb->ug, a2, mn[2], res, 1); + // fprintf(stderr, "[M::%s]\tutg%.6lu%c\tres->n::%u\n", __func__, rid+1, "lc"[udb->ug->u.a[rid].circ], res->n); + push_emask_lst(mk, ol, minov, udb->ug, a1, mn[1], res, 1); + // fprintf(stderr, "[M::%s]\tutg%.6lu%c\tres->n::%u\n", __func__, rid+1, "lc"[udb->ug->u.a[rid].circ], res->n); + push_emask_lst(mk, ol, minov, udb->ug, a0, mn[0], res, 0); + // fprintf(stderr, "[M::%s]\tutg%.6lu%c\tres->n::%u\n", __func__, rid+1, "lc"[udb->ug->u.a[rid].circ], res->n); + srt_kv_emask_t(res, mk, rid, udb->ug); + + // for (k = 0; k < mk->srt.n; k++) { + // fprintf(stderr, "[M::%s]\tutg%.6u%c\tq::[%u,\t%u)\t%c\tutg%.6u%c\tt::[%u,\t%u)\n", __func__, + // mk->srt.a[k].qn+1, "lc"[udb->ug->u.a[mk->srt.a[k].qn].circ], mk->srt.a[k].qs, mk->srt.a[k].qe, "+-"[mk->srt.a[k].rev], + // mk->srt.a[k].tn+1, "lc"[udb->ug->u.a[mk->srt.a[k].tn].circ], mk->srt.a[k].ts, mk->srt.a[k].te); + // } +} + +static void worker_for_graph_bin(void *data, long i, int tid) // callback for kt_for() +{ + ug_bin_t *s = (ug_bin_t*)data; double erate; int64_t wl; overlap_region *aux_o; + ha_ovec_buf_t *b = s->hab[tid]; asg64_v b0; + uint32_t high_occ = asm_opt.polyploidy + 1; uint64_t cnt; + char *seq = s->ug->u.a[i].s; int64_t len = s->ug->u.a[i].len; + if((!s->is_ovlp) && (s->is_cnt)) s->idx_n.a[i] = 0; + if(s->ug->g->seq[i].del) return; + if(!seq) { + resize_UC_Read(&(b->self_read), len); + retrieve_u_seq(NULL, b->self_read.seq, &(s->ug->u.a[i]), 0, 0, len, NULL); + seq = b->self_read.seq; + } + + // asprintf(&as, "\n[M::%s] rid::%ld, len::%lu, name::%.*s\n", __func__, s->id+i, s->len[i], (int32_t)UL_INF.nid.a[s->id+i].n, UL_INF.nid.a[s->id+i].a); + // push_vlog(&(overall_zdbg->a[s->id+i]), as); free(as); as = NULL; + + if(!s->is_ovlp) { + if(s->is_cnt) { + // s->idx_n.a[i] = ug_map_lchain(b->abl, i, seq, len, s->w, s->k, &(s->udb), NULL, NULL, s->bw_thres, s->bw_thres_double, + // s->max_n_chain, 1, NULL, &(b->tmp_region), NULL, &(b->sp), &high_occ, NULL, 0, 1, 0.2, 3, s->is_HPC, NULL, 0, NULL, 0, s->mini_cut, s->chain_cut, NULL); + s->idx_n.a[i] = ug_map_lchain_simple(b->abl, i, seq, len, s->w, s->k, &(s->udb), NULL, NULL, s->bw_thres, + s->max_n_chain, 1, NULL, &(b->tmp_region), NULL, &(b->sp), &high_occ, NULL, 1, 1, 0.2, 3, s->is_HPC, NULL, 0, NULL, 0, s->mini_cut, s->chain_cut); + } else { + // cnt = ug_map_lchain(b->abl, i, seq, len, s->w, s->k, &(s->udb), NULL, NULL, s->bw_thres, s->bw_thres_double, + // s->max_n_chain, 1, NULL, &(b->tmp_region), NULL, &(b->sp), &high_occ, NULL, 0, 1, 0.2, 3, s->is_HPC, s->idx_a.a + s->idx_n.a[i], 0, NULL, 0, s->mini_cut, s->chain_cut, NULL); + cnt = ug_map_lchain_simple(b->abl, i, seq, len, s->w, s->k, &(s->udb), NULL, NULL, s->bw_thres, + s->max_n_chain, 1, NULL, &(b->tmp_region), NULL, &(b->sp), &high_occ, NULL, 1, 1, 0.2, 3, s->is_HPC, s->idx_a.a + s->idx_n.a[i], 0, NULL, 0, s->mini_cut, s->chain_cut); + assert(cnt == ((s->idx_n.a[i+1]-s->idx_n.a[i]))); + } + return; + } + + // if(i == 160) { + // fprintf(stderr, "\n-1-[M::%s] utg%.6u%c, rid::%ld, is_ovlp::%d, is_cnt::%d, len::%ld, str::%u\n", + // __func__, (uint32_t)i+1, "lc"[s->ug->u.a[i].circ], i, s->is_ovlp, s->is_cnt, len, (uint32_t)(!!seq)); + // } + + + // ug_map_lchain(b->abl, rid, seq, len, s->w, s->k, &(s->udb), &b->olist, &b->clist, bw_low, bw_high, + // s->max_n_chain, 1, NULL, &(b->tmp_region), NULL, &(b->sp), &high_occ, NULL, 0, 1, 0.2, 3, + // s->is_HPC, s->idx_a.a + s->idx_n.a[rid], cnt, s->srt_a.a, s->srt_a.n, s->mini_cut, s->chain_cut, fi); + cnt = ((s->idx_n.a[i+1]-s->idx_n.a[i])); + ug_map_lchain_simple(b->abl, i, seq, len, s->w, s->k, &(s->udb), &b->olist, &b->clist, s->bw_thres, + s->max_n_chain, 1, NULL, &(b->tmp_region), NULL, &(b->sp), &high_occ, NULL, 1, 1, 0.2, 3, s->is_HPC, s->idx_a.a + s->idx_n.a[i], cnt, s->srt_a.a, s->srt_a.n, s->mini_cut, s->chain_cut); + clear_Cigar_record(&b->cigar1); clear_Round2_alignment(&b->round2); + + erate = s->diff_ov; wl = MIN((((double)THRESHOLD_MAX_SIZE)/erate), WINDOW); aux_o = NULL; + ug_lalign(&b->olist, &b->clist, &(s->udb), s->uopt, seq, len, NULL/**&b->self_read**/, &b->ovlp_read, + &b->correct, &b->exz, aux_o, erate, wl, i, s->k, s->chain_cut, NULL); + aux_o = gen_aux_ovlp(&b->olist);///must be here + ug_lalign(&b->olist, &b->clist, &(s->udb), s->uopt, seq, len, NULL/**&b->self_read**/, &b->ovlp_read, + &b->correct, &b->exz, aux_o, erate, wl, i, s->k, s->chain_cut, NULL); + + copy_asg_arr(b0, s->ll[tid].srt.a); + push_graph_bin(&b->olist, &(s->udb), s->diff_bin, s->max_diff, s->min_bin_len, i, &b0, &(s->mk[tid]), &(s->mm->a[i])); + copy_asg_arr(s->ll[tid].srt.a, b0); + // if(!gen_trans_adaptive_aln(s, i, b, bl, seq, len, s->filter, s->diff_ec_ul, s->diff_ec_ul_double, s->bw_thres, s->bw_thres_double)) { + // gen_trans_adaptive_aln(s, i, b, bl, seq, len, NULL, s->diff_ec_ul_double, s->diff_ec_ul_double, s->bw_thres_double, s->bw_thres_double); + // } +} + +emask_t* get_emask_ovlp(idx_emask_t *idx, uint32_t qn, uint32_t tn, uint32_t *occ, uint32_t *st) +{ + emask_t *a = NULL; (*occ) = 0; if(st) (*st) = (uint32_t)-1; + if(qn >= idx->n) return a; + uint32_t k; + for (k = 0; k < idx->a[qn].n && idx->a[qn].a[k].tn < tn; k++); + if(k < idx->a[qn].n && idx->a[qn].a[k].tn == tn) { + a = idx->a[qn].a + k; if(st) (*st) = k; + for ((*occ) = k; k < idx->a[qn].n && idx->a[qn].a[k].tn == tn; k++); + (*occ) = k - (*occ); + return a; + } + return a; +} + +void detect_sysm_graph_bin(idx_emask_t *idx, uint32_t rid) +{ + kv_emask_t *r = &(idx->a[rid]); uint32_t k, an, st; + for (k = 0; k < r->n; k++) { + get_emask_ovlp(idx, r->a[k].tn, rid, &an, &st); + r->a[k].rn = (uint32_t)-1; + if(!an) r->a[k].rn = st; + } +} + +static void worker_for_graph_bin_sysm(void *data, long i, int tid) // callback for kt_for() +{ + ug_bin_t *s = (ug_bin_t*)data; + if(s->ug->g->seq[i].del) return; + if((s->is_cnt) && (!s->is_ovlp)) detect_sysm_graph_bin(s->mm, i); + if((!s->is_cnt) && (s->is_ovlp)) srt_kv_emask_t(&(s->mm->a[i]), &(s->mk[tid]), i, s->udb.ug); + if((s->is_cnt) && (s->is_ovlp)) detect_sysm_graph_bin(s->mm, i); +} + ul_ov_t* get_trans_ul_ov_t(ug_trans_t *s, uint32_t qn, uint32_t tn) { ha_mzl_t *f = &(s->srt_a.a[qn]); @@ -10557,9 +12243,8 @@ static void *worker_ul_rescall_pipeline(void *data, int step, void *in) // callb s->sum_len += l; memcpy(s->seq[s->n], p->ks->seq.s, l); // fprintf(stderr, "s->n->%d, l->%lu\n", s->n, l); - // if(s->id + s->n == 13706) { - // fprintf(stderr, "+++++rid->%lu, l->%lu, %.*s\n", - // s->id + s->n, l, (int32_t)p->ks->name.l, p->ks->name.s);} + // fprintf(stderr, "+++++rid->%lu, l->%lu, %.*s\n", + // s->id + s->n, l, (int32_t)p->ks->name.l, p->ks->name.s); s->len[s->n++] = l; if (s->sum_len >= p->chunk_size) break; } @@ -15457,7 +17142,7 @@ ul_idx_t *gen_ul_idx(const ug_opt_t *uopt, ma_ug_t *ug, asg_t *sg) uu->ct = ul_contain_gen(ug, sg, src, min_ovlp, max_hang, 0, 1); uu->cc = gen_cov_track(ug, sg, uu->ct, src, min_ovlp, max_hang, 0, 1); uu->cr = gen_r_contain(ug, sg, src, R_INF.total_reads, min_ovlp, max_hang, asm_opt.thread_num, 0, 1); - uu->hpc_g = hpc_g_gen(ug); + // uu->hpc_g = hpc_g_gen(ug); return uu; } @@ -15588,14 +17273,25 @@ void destroy_ul_idx_t(ul_idx_t *uu) free(uu); } +void destroy_idx_emask_t(idx_emask_t *mm) +{ + if(mm) { + uint64_t k; + for (k = 0; k < mm->n; k++) { + free(mm->a[k].a); memset(&(mm->a[k]), 0, sizeof(mm->a[k])); + } + free(mm->a); + } +} void gen_UL_ovlps(uldat_t *sl, int32_t cutoff) { ul_idx_t *uu = dedup_HiFis(sl->uopt, 1, 0); + ///for unitig index int exist = (asm_opt.load_index_from_disk? uidx_load(&ha_flt_tab, &ha_idx, asm_opt.output_file_name, NULL) : 0); if(exist == 0) uidx_l_build(uu->ug, (mg_idxopt_t *)sl->opt, cutoff); - // print_debug_gfa(sl->uopt, ug, coverage_cut, "debug_dups", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); if(exist == 0) uidx_write(ha_flt_tab, ha_idx, asm_opt.output_file_name, NULL); + // print_debug_gfa(sl->uopt, ug, coverage_cut, "debug_dups", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); sl->ha_flt_tab = ha_flt_tab; sl->ha_idx = (ha_pt_t *)ha_idx; sl->uu = uu; ul_v_call(sl, asm_opt.ar); // print_raw_u2rgfa_seq(&UL_INF, uu, 1); @@ -15603,14 +17299,68 @@ void gen_UL_ovlps(uldat_t *sl, int32_t cutoff) sl->ha_flt_tab = NULL; sl->ha_idx = NULL; sl->uu = NULL; } +int32_t load_emask_t(idx_emask_t **z, char* file_name, ma_ug_t *ug) +{ + char* gfa_name = NULL; MALLOC(gfa_name, strlen(file_name)+50); + sprintf(gfa_name, "%s.ul.msk.bin", file_name); + FILE* fp = fopen(gfa_name, "r"); free(gfa_name); + if (!fp) return 0; + if(ug && (!test_dbug(ug, fp))) { + fprintf(stderr, "[M::%s] Renew UL Index\n", __func__); + fclose(fp); + return 0; + } + uint64_t k; kv_emask_t *p; idx_emask_t *x; CALLOC(x, 1); -void gen_UL_reovlps(uldat_t *sl, ma_ug_t *ug, asg_t *sg, char* gfa_name, int32_t cutoff) + + fread(&x->n, sizeof(x->n), 1, fp); MALLOC(x->a, x->n); + for (k = 0; k < x->n; k++) { + p = &(x->a[k]); + fread(&p->n, sizeof(p->n), 1, fp); MALLOC(p->a, p->n); p->m = p->n; + fread(p->a, sizeof((*(p->a))), p->n, fp); + } + + fprintf(stderr, "[M::%s] Index has been written.\n", __func__); + fclose(fp); + *z = x; + return 1; +} + +int32_t write_emask_t(idx_emask_t *x, char* file_name, ma_ug_t *ug) +{ + char* gfa_name = NULL; MALLOC(gfa_name, strlen(file_name)+50); + sprintf(gfa_name, "%s.ul.msk.bin", file_name); + FILE* fp = fopen(gfa_name, "w"); free(gfa_name); + if (!fp) return 0; + uint64_t k; kv_emask_t *p = NULL; + + if(ug) write_dbug(ug, fp); + + fwrite(&x->n, sizeof(x->n), 1, fp); + for (k = 0; k < x->n; k++) { + p = &(x->a[k]); + fwrite(&p->n, sizeof(p->n), 1, fp); + fwrite(p->a, sizeof((*(p->a))), p->n, fp); + } + + fprintf(stderr, "[M::%s] Index has been written.\n", __func__); + fclose(fp); + return 1; +} + +void gen_UL_reovlps(uldat_t *sl, ma_ug_t *ug, asg_t *sg, char* gfa_name, int32_t cutoff, int32_t is_emask) { ul_idx_t *uu = gen_ul_idx(sl->uopt, ug, sg); int exist = (asm_opt.load_index_from_disk? uidx_load(&ha_flt_tab, &ha_idx, gfa_name, ug) : 0); if(exist == 0) uidx_l_build(uu->ug, (mg_idxopt_t *)sl->opt, cutoff); if(exist == 0) uidx_write(ha_flt_tab, ha_idx, gfa_name, ug); sl->ha_flt_tab = ha_flt_tab; sl->ha_idx = (ha_pt_t *)ha_idx; sl->uu = uu; + if(is_emask) { + if((!(asm_opt.load_index_from_disk)) || (!load_emask_t(&(sl->mm), gfa_name, ug))) { + sl->mm = graph_ovlp_binning(ug, sg, sl->uopt); + write_emask_t(sl->mm, gfa_name, ug); + } + } init_ucr_file_t(sl, gfa_name, 1); ul_v_recall(sl, asm_opt.ar); @@ -15618,7 +17368,7 @@ void gen_UL_reovlps(uldat_t *sl, ma_ug_t *ug, asg_t *sg, char* gfa_name, int32_t ///do not free ug uu->ug = NULL; destroy_ul_idx_t(uu); ha_ft_destroy(ha_flt_tab); ha_pt_destroy(ha_idx); sl->ha_flt_tab = NULL; sl->ha_idx = NULL; sl->uu = NULL; - + destroy_idx_emask_t(sl->mm); free(sl->mm); sl->mm = NULL; // exit(1); } @@ -15831,6 +17581,13 @@ void clear_all_ul_t(all_ul_t *x) memset(&(x->a[k]), 0, sizeof(x->a[k])); x->a[k].rlen = rlen; } free(x->ridx.idx.a); free(x->ridx.occ.a); memset(&(x->ridx), 0, sizeof((x->ridx))); + + // if(x->mm) { + // for (k = 0; k < x->mm->n; k++) { + // free(x->mm->a[k].a); memset(&(x->mm->a[k]), 0, sizeof(x->mm->a[k])); + // } + // free(x->mm->a); free(x->mm); x->mm = NULL; + // } } void init_ug_trans_t(ug_trans_t *opt, ug_opt_t *uopt, int32_t is_HPC, int32_t k, int32_t w, int32_t max_n_chain, @@ -16124,6 +17881,331 @@ void trans_base_infer(ma_ug_t *ug, asg_t *sg, ug_opt_t *uopt, kv_u_trans_t *res, } } +void init_ug_bin_t(ug_bin_t *sl, const ug_opt_t *uopt, int32_t is_HPC, int32_t k, int32_t w, int32_t max_n_chain, +double bw_thres, double diff_ov, double diff_bin, uint64_t max_diff, uint64_t min_bin_len, int32_t n_thread, +int32_t mini_cut, int32_t chain_cut, int32_t keep_unsymm_arc, ma_ug_t *ug, asg_t *sg) +{ + int64_t i; + memset(sl, 0, sizeof((*sl))); + sl->uopt = uopt; + sl->k = k; + sl->w = w; + sl->is_HPC = is_HPC; + sl->max_n_chain = max_n_chain; + sl->bw_thres = bw_thres; + sl->diff_ov = diff_ov; + sl->diff_bin = diff_bin; + sl->max_diff = max_diff; + sl->min_bin_len = min_bin_len; + sl->mini_cut = mini_cut; + sl->chain_cut = chain_cut; + sl->keep_unsymm_arc = keep_unsymm_arc; + + sl->bw = 10000;///2000 in minigraph + sl->max_gap = 500000;///5000 in minigraph + sl->ug = ug; + sl->rg = sg; + sl->n_thread = ((n_thread>=1)?n_thread:1); + + CALLOC(sl->mk, sl->n_thread); + CALLOC(sl->hab, sl->n_thread); + CALLOC(sl->ll, sl->n_thread); + for (i = 0; i < sl->n_thread; ++i) sl->hab[i] = ha_ovec_init(0, 0, 1); + + sl->idx_n.n = sl->idx_n.m = ug->u.n+1; + MALLOC(sl->idx_n.a, sl->idx_n.n); + + sl->udb.ug = ug; + // CALLOC(sl->bm, sl->n_thread); + // for (i = 0; i < sl->n_thread; ++i) { + // sl->bm[i].n = ((sl->udb.ug->g->n_seq<<1)>>3) + (!!((sl->udb.ug->g->n_seq<<1)&((uint32_t)7))); + // CALLOC(sl->bm[i].a, sl->bm[i].n); + // } +} + +void extract_microsatellite(const char *in, uint32_t in_len, hpc_re_t *res, asg64_v *buf, uint32_t mcs_len, +uint32_t min_klen_simple, uint32_t min_klen_complex, uint32_t ksimple_cut, hpc_idx_t *idx) +{ + uint32_t k, i, l, c, o, ls; char sk[256]; hpc_ss_t z, *za; + uint32_t m, cutoff, zn, bn, ovlp, os, oe, oks, oke; + idx->s = idx->e = res->n; + if(mcs_len > 255) mcs_len = 255; + for (k = 1; k <= mcs_len; k++) { + memset(sk, 'N', k); ///k->length of k-mer + o = ((k<=ksimple_cut)?(k*min_klen_simple):(k*min_klen_complex)); + for (i = l = ls = 0; i < in_len; i++) { + c = seq_nt4_table[(uint8_t)in[i]]; + if((c < 4) && (((l >= k) && (sk[(l+ls)%k] == in[i])) || (l < k))) { + if(l < k) sk[(l+ls)%k] = in[i]; + l++; + } else { + if(l >= o) { + z.s = i-l; z.e = i; z.k = k; + kv_push(hpc_ss_t, *res, z); + } + + if(c >= 4) { + l = ls = 0; + } else if(l >= 0) { + sk[(l+ls)%k] = in[i]; l++; + m = ((l<=k)?(l-1):(k)); + ls = (ls+(l-m))%k; l = m; + } + } + } + } + + uint32_t ks, ke, fs, fe; + radix_sort_hpc_ss_t_s(res->a+idx->s, res->a+res->n); + for (k = ks = ke = idx->e = idx->s; k < res->n; k++) { + fs = fe = (uint32_t)-1; + if(ke > idx->s) { + fs = res->a[ks].s; fe = res->a[ke-1].e; + } + ///new interval is not overlapped with existing one + if((fs == ((uint32_t)-1)) || (fe <= res->a[k].s)) { + res->a[idx->e++] = res->a[k]; ks = idx->e - 1; ke = idx->e; + } else { + assert(res->a[k].s >= fs); + ///new interval might be contained in an exist one + if(res->a[k].s >= fs && res->a[k].e <= fe) { + for (i = ks; i < ke; i++) { + if(res->a[k].s >= res->a[i].s && res->a[k].e <= res->a[i].e) {///new interval is contained in an exist one + if((res->a[k].s == res->a[i].s) && (res->a[k].e == res->a[i].e) + && (res->a[k].k < res->a[i].k)) { + res->a[i] = res->a[k]; + } + break; + } + } + if(i < ke) continue; + } + + for (i = m = ks; i < ke; i++) { + ///exist interval is contained in new interval + if(res->a[k].s <= res->a[i].s && res->a[k].e >= res->a[i].e) continue; + res->a[m++] = res->a[i]; + } + ke = idx->e = m; + res->a[idx->e++] = res->a[k]; ke = idx->e; + } + } + res->n = idx->e; + cutoff = (in_len/sizeof((*(res->a))))/4; if(cutoff < 10) cutoff = 10; + if(idx->e-idx->s > cutoff) {///compress if there are too many intervals + kv_resize(uint64_t, *buf, res->n-idx->s); + za = res->a + idx->s; zn = idx->e - idx->s; + for (k = buf->n = 0; k < zn; k++) { + kv_push(uint64_t, *buf, ((((uint64_t)za[k].k)<<32)|((uint64_t)(k)))); + } + radix_sort_gfa64(buf->a, buf->a+buf->n);///sorted by the k-mer length + for (i = 0, bn = buf->n; i < bn; i++) { + k = (uint32_t)buf->a[i]; oks = za[k].s; oke = za[k].e; + for (m = 0; m < zn && za[m].s < za[k].e; m++) { + if(!(buf->a[m]&((uint64_t)0x8000000000000000))) continue; + if(m == k) continue; + ///now has overlap + os = MAX(za[m].s, oks); oe = MIN(za[m].e, oke); + ovlp = ((oe > os)?(oe - os):(0)); + if(!ovlp) continue; + assert(os == oks || oe == oke); + if(os == oks) oks = oe; + if(oe == oke) oke = os; + if(oke < oks + za[k].k) break; + } + if(oke >= oks + za[k].k) { + buf->a[k] |= ((uint64_t)0x8000000000000000); + } + } + for (i = 0, idx->e = idx->s; i < bn; i++) { + if(!(buf->a[i]&((uint64_t)0x8000000000000000))) continue; + res->a[idx->e++] = za[i]; + } + res->n = idx->e; + } + // for (i = idx->s; i < idx->e; i++) { + // fprintf(stderr, "k::%u\tin::[%u,\t%u)\n", res->a[i].k, res->a[i].s, res->a[i].e); + // } +} + +hpc_re_t *gen_hpc_re_t(ma_ug_t *ug) +{ + uint64_t k; asg64_v buf; kv_init(buf); + hpc_re_t *p = NULL; CALLOC(p, 1); + kvec_t(char) cc; kv_init(cc); + p->idx_n = ug->u.n; CALLOC(p->idx, p->idx_n); + + for (k = 0; k < ug->u.n; k++) { + kv_resize(char, cc, ug->u.a[k].len); + retrieve_u_seq(NULL, cc.a, &(ug->u.a[k]), 0, 0, ug->u.a[k].len, NULL); + // fprintf(stderr, "\nutg%.6lu%c\tlen::%u\n", k + 1, "lc"[ug->u.a[k].circ], ug->u.a[k].len); + extract_microsatellite(cc.a, ug->u.a[k].len, p, &buf, 8, 8, 5, 1, &(p->idx[k])); + // fprintf(stderr, "utg%.6lu%c\t#::%u\tlen::%u\n", k + 1, "lc"[ug->u.a[k].circ], p->idx[k].e-p->idx[k].s, ug->u.a[k].len); + } + + kv_destroy(cc); kv_destroy(buf); + return p; +} + +void sysm_graph_bin(ug_bin_t *s) +{ + uint64_t k, i; kv_emask_t *r; + s->is_cnt = 1; s->is_ovlp = 0; + kt_for(s->n_thread, worker_for_graph_bin_sysm, s, s->ug->u.n); + for (i = 0; i < s->mm->n; i++) { + r = &(s->mm->a[i]); + for (k = 0; k < r->n; k++) { + if(r->a[k].rn == (uint32_t)-1) s->mm->a[r->a[k].tn].m++; + } + } + for (i = 0; i < s->mm->n; i++) { + r = &(s->mm->a[i]); + if(r->m > r->n) REALLOC(r->a, r->m); + } + + + s->is_cnt = 0; s->is_ovlp = 1; + kt_for(s->n_thread, worker_for_graph_bin_sysm, s, s->ug->u.n); + + s->is_cnt = 1; s->is_ovlp = 1; + kt_for(s->n_thread, worker_for_graph_bin_sysm, s, s->ug->u.n); +} + +void cal_graph_ovlp_binning(ug_bin_t *p) +{ + double index_time = yak_realtime(); + // ha_flt_tab = NULL; + uint64_t i, k, l, occ, m, cc; + // p->hre = gen_hpc_re_t(p->ug); + CALLOC(p->mm, 1); p->mm->n = p->udb.ug->g->n_seq; CALLOC(p->mm->a, p->mm->n); + + // fprintf(stderr, "[M::%s::] ==> 0\n", __func__); + p->is_cnt = 1; p->is_ovlp = 0; + kt_for(p->n_thread, worker_for_graph_bin, p, p->ug->u.n); + for (i = l = 0; i < p->ug->u.n; i++) { + occ = p->idx_n.a[i]; p->idx_n.a[i] = l; l += occ; + } + // fprintf(stderr, "[M::%s::] i::%lu, l::%lu\n", __func__, i, l); + p->idx_n.a[i] = l; + p->idx_a.n = p->idx_a.m = l; MALLOC(p->idx_a.a, p->idx_a.n); + // fprintf(stderr, "[M::%s::] ==> 1\n", __func__); + p->is_cnt = 0; p->is_ovlp = 0; + kt_for(p->n_thread, worker_for_graph_bin, p, p->ug->u.n); + p->srt_a.n = p->srt_a.m = p->idx_a.n; MALLOC(p->srt_a.a, p->srt_a.n); + // fprintf(stderr, "[M::%s::] p->idx_a.n::%lu \n", __func__, (uint64_t)p->idx_a.n); + // memcpy(p->srt_a.a, p->idx_a.a, p->srt_a.n*sizeof((*(p->srt_a.a)))); + for (i = 0; i < p->srt_a.n; i++) { + p->srt_a.a[i] = p->idx_a.a[i]; + p->srt_a.a[i].pos = (uint32_t)i; + p->srt_a.a[i].rid = i>>32; + } + radix_sort_ha_mzl_t_srt(p->srt_a.a, p->srt_a.a + p->srt_a.n); + kvec_t(uint64_t) cut; kv_init(cut); + for (k = 1, l = 0; k <= p->srt_a.n; k++) { + if(k == p->srt_a.n || p->srt_a.a[l].x != p->srt_a.a[k].x) { + for (i = l; i < k; i++) { + m = p->srt_a.a[i].rid; m <<= 32; m |= p->srt_a.a[i].pos; + assert(p->srt_a.a[i].x == p->idx_a.a[m].x); + p->srt_a.a[i] = p->idx_a.a[m]; p->idx_a.a[m].x = i; + } + kv_push(uint64_t, cut, (k - l)); + l = k; + } + } + + if(cut.n > 0) { + radix_sort_gfa64(cut.a, cut.a + cut.n); + m = cut.n * 0.0002; cc = cut.a[cut.n-1] + 1; + if(m > 0 && m <= cut.n) cc = cut.a[cut.n-m] + 1; + if(cc < (uint64_t)p->mini_cut) p->mini_cut = cc; + } + kv_destroy(cut); + // fprintf(stderr, "[M::%s::] p->mini_cut::%d \n", __func__, p->mini_cut); + + // fprintf(stderr, "[M::%s::] ==> 2\n", __func__); + p->is_cnt = 0; p->is_ovlp = 1; + kt_for(p->n_thread, worker_for_graph_bin, p, p->ug->u.n); + // fprintf(stderr, "[M::%s::] ==> 3\n", __func__); + // exit(1); + for (i = 0; (int64_t)i < p->n_thread; i++) { + ha_ovec_destroy(p->hab[i]); + free(p->ll[i].lo.a); free(p->ll[i].srt.a.a); free(p->ll[i].tc.a); free(p->ll[i].tk.a); + } + free(p->idx_a.a); free(p->idx_n.a); free(p->hab); free(p->srt_a.a); free(p->ll); + fprintf(stderr, "[M::%s::] ==> 4\n", __func__); + + // sysm_graph_bin(p); + for (i = 0; (int64_t)i < p->n_thread; i++) { + free(p->mk[i].idx.a); free(p->mk[i].srt.a); + } + free(p->mk); + fprintf(stderr, "[M::%s::%.3f] ==> Qualification\n", __func__, yak_realtime()-index_time); +} + +idx_emask_t* graph_ovlp_binning(ma_ug_t *ug, asg_t *sg, const ug_opt_t *uopt) +{ + ug_bin_t sl; + init_aux_table(); ha_opt_update_cov(&asm_opt, asm_opt.hom_cov); + init_ug_bin_t(&sl, uopt, 0, GBIN_K, GBIN_W, asm_opt.max_n_chain, GBIN_E, GBIN_E, GBIN_BE, GBIN_ME, GBIN_L, + asm_opt.thread_num, 512, 3, 1, ug, sg); + cal_graph_ovlp_binning(&sl); + return sl.mm; +} + + +ma_ug_t *ul_realignment_pending(const ug_opt_t *uopt, asg_t *sg, uint32_t double_check_cache, const char *bin_file) +{ + fprintf(stderr, "[M::%s::] ==> starting UL\n", __func__); + mg_idxopt_t opt; uldat_t sl; + int32_t cutoff; + char* gfa_name = NULL; MALLOC(gfa_name, strlen(asm_opt.output_file_name)+50); + sprintf(gfa_name, "%s.%s", asm_opt.output_file_name, bin_file); + + init_aux_table(); ha_opt_update_cov(&asm_opt, asm_opt.hom_cov); + cutoff = REA_ALIGN_CUTOFF; + init_mg_opt(&opt, !(asm_opt.flag&HA_F_NO_HPC), 19, 10, cutoff, asm_opt.max_n_chain, asm_opt.ul_error_rate, asm_opt.ul_error_rate, asm_opt.ul_error_rate_low, asm_opt.ul_error_rate_hpc, asm_opt.ul_ec_round); + init_uldat_t(&sl, NULL, NULL, &opt, CHUNK_SIZE, asm_opt.thread_num, uopt, NULL); + ma_ug_t *ug = gen_polished_ug(uopt, sg); + // dd_ug(sg, ug, uopt->coverage_cut, uopt->sources, uopt->ruIndex, "UL.sa"); + // debug_sl_compress_base_disk_0(&sl, asm_opt.ar); + // detect_outlier_len("ul_realignment"); + clear_all_ul_t(&UL_INF); + + ///for debug interval + if(1/**!load_all_ul_t(&UL_INF, gfa_name, &R_INF, ug)**/) { + gen_UL_reovlps(&sl, ug, sg, gfa_name, cutoff, 1); + exit(1); + write_all_ul_t(&UL_INF, gfa_name, ug); + } else{ + free(UL_INF.ridx.idx.a); free(UL_INF.ridx.occ.a); + memset(&(UL_INF.ridx), 0, sizeof((UL_INF.ridx))); + if(double_check_cache){ + if(drenew_UL_reovlps(&sl, ug, sg, gfa_name, cutoff)) { + write_all_ul_t(&UL_INF, gfa_name, ug); + } + } + } + + // print_ul_alignment(ug, &UL_INF, 41927, "init-0"); + filter_ul_ug(ug); + // print_ul_alignment(ug, &UL_INF, 41927, "init-1"); + gen_ul_vec_rid_t(&UL_INF, NULL, ug); + // print_ul_alignment(ug, &UL_INF, 41927, "init-2"); + update_ug_arch_ul_mul(ug); + // print_ul_alignment(ug, &UL_INF, 41927, "init-3"); + // kt_for(asm_opt.thread_num, update_ug_arch_ul, ug, ug->g->n_arc); + // print_all_ul_t_stat(&UL_INF); + // kt_for(sl.n_thread, update_ovlp_src, &sl, R_INF.total_reads); + // kt_for(sl.n_thread, update_ovlp_src_bl, &sl, R_INF.total_reads); + + // print_ovlp_src_bl_stat(&UL_INF, sl.uopt); + // print_ul_ovlps(&UL_INF, 0); print_ul_ovlps(&UL_INF, 1); + + // destory_all_ul_t(&UL_INF); + free(gfa_name); + return ug; +} + ma_ug_t *ul_realignment(const ug_opt_t *uopt, asg_t *sg, uint32_t double_check_cache, const char *bin_file) { @@ -16142,9 +18224,10 @@ ma_ug_t *ul_realignment(const ug_opt_t *uopt, asg_t *sg, uint32_t double_check_c // debug_sl_compress_base_disk_0(&sl, asm_opt.ar); // detect_outlier_len("ul_realignment"); clear_all_ul_t(&UL_INF); + ///for debug interval if(!load_all_ul_t(&UL_INF, gfa_name, &R_INF, ug)) { - gen_UL_reovlps(&sl, ug, sg, gfa_name, cutoff); + gen_UL_reovlps(&sl, ug, sg, gfa_name, cutoff, 0); // exit(1); write_all_ul_t(&UL_INF, gfa_name, ug); } else{ diff --git a/inter.h b/inter.h index c25729f..9414c63 100644 --- a/inter.h +++ b/inter.h @@ -118,5 +118,11 @@ void update_ug_arch_ul_mul(ma_ug_t *ug); void print_ul_alignment(ma_ug_t *ug, all_ul_t *aln, uint32_t id, const char* cmd); void clear_all_ul_t(all_ul_t *x); void trans_base_infer(ma_ug_t *ug, asg_t *sg, ug_opt_t *uopt, kv_u_trans_t *res, bubble_type *bub); +hpc_re_t *gen_hpc_re_t(ma_ug_t *ug); +idx_emask_t* graph_ovlp_binning(ma_ug_t *ug, asg_t *sg, const ug_opt_t *uopt); +uint32_t gen_src_shared_interval_simple(uint32_t src, ma_ug_t *ug, kv_ul_ov_t *res); +uint64_t check_ul_ov_t_consist(ul_ov_t *x, ul_ov_t *y, int64_t ql, int64_t tl, double diff); +uint32_t infer_se(uint32_t qs, uint32_t qe, uint32_t ts, uint32_t te, uint32_t rev, +uint32_t rqs, uint32_t rqe, uint32_t *rts, uint32_t *rte); #endif