diff --git a/Purge_Dups.cpp b/Purge_Dups.cpp index 9ce909d..6f5e78b 100644 --- a/Purge_Dups.cpp +++ b/Purge_Dups.cpp @@ -5,6 +5,7 @@ #include "Purge_Dups.h" #include "Overlaps.h" #include "Correct.h" +#include "kthread.h" #define Cal_Off(OFF) ((long long)((uint32_t)((OFF)>>32)) - (long long)((uint32_t)((OFF)))) #define Get_match(x) ((x).weight) @@ -96,9 +97,11 @@ void init_hap_alignment_struct(hap_alignment_struct* x, uint32_t size) kv_init(x->u_vecs.a); kv_init(x->u_buffer.a); kv_init(x->u_can.a); + + } -void destory_hap_alignment_struct(hap_alignment_struct* x, uint32_t size) +void destory_hap_alignment_struct(hap_alignment_struct* x) { free(x->vote_counting); free(x->visit); @@ -107,6 +110,61 @@ void destory_hap_alignment_struct(hap_alignment_struct* x, uint32_t size) kv_destroy(x->u_can.a); } + +typedef struct { + hap_alignment_struct* buf; + uint32_t num_threads; + + ma_ug_t *ug; + asg_t *read_g; + ma_hit_t_alloc* reverse_sources; + R_to_U* ruIndex; + ma_sub_t *coverage_cut; + uint64_t* position_index; + float Hap_rate; + int max_hang; + int min_ovlp; + float chain_rate; + hap_overlaps_list* all_ovlp; +}hap_alignment_struct_pip; + +void init_hap_alignment_struct_pip(hap_alignment_struct_pip* x, uint32_t num_threads, uint32_t n_seq, +ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, ma_sub_t *coverage_cut, +uint64_t* position_index, float Hap_rate, int max_hang, int min_ovlp, float chain_rate, hap_overlaps_list* all_ovlp) +{ + uint32_t i; + x->num_threads = num_threads; + x->buf = (hap_alignment_struct*)malloc(sizeof(hap_alignment_struct)*x->num_threads); + for (i = 0; i < x->num_threads; i++) + { + init_hap_alignment_struct(&(x->buf[i]), n_seq); + } + + x->ug = ug; + x->read_g = read_g; + x->reverse_sources = reverse_sources; + x->ruIndex = ruIndex; + x->coverage_cut = coverage_cut; + x->position_index = position_index; + x->Hap_rate = Hap_rate; + x->max_hang = max_hang; + x->min_ovlp = min_ovlp; + x->chain_rate = chain_rate; + x->all_ovlp = all_ovlp; +} + + +void destory_hap_alignment_struct_pip(hap_alignment_struct_pip* x) +{ + uint32_t i; + for (i = 0; i < x->num_threads; i++) + { + destory_hap_alignment_struct(&(x->buf[i])); + } + + free(x->buf); +} + void init_hap_overlaps_list(hap_overlaps_list* x, uint32_t num) { uint32_t i = 0; @@ -1345,6 +1403,308 @@ hap_overlaps_list* all_ovlp) } +static void hap_alignment_worker(void *_data, long eid, int tid) +{ + hap_alignment_struct_pip* hap_buf = (hap_alignment_struct_pip*)_data; + ma_ug_t *ug = hap_buf->ug; + asg_t *read_g = hap_buf->read_g; + ma_hit_t_alloc* reverse_sources = hap_buf->reverse_sources; + R_to_U* ruIndex = hap_buf->ruIndex; + ma_sub_t *coverage_cut = hap_buf->coverage_cut; + uint64_t* position_index = hap_buf->position_index; + float Hap_rate = hap_buf->Hap_rate; + int max_hang = hap_buf->max_hang; + int min_ovlp = hap_buf->min_ovlp; + float chain_rate = hap_buf->chain_rate; + hap_overlaps_list* all_ovlp = hap_buf->all_ovlp; + uint32_t Input_uId = eid; + uint64_t* vote_counting = hap_buf->buf[tid].vote_counting; + uint8_t* visit = hap_buf->buf[tid].visit; + kvec_t_u64_warp* u_vecs = &(hap_buf->buf[tid].u_vecs); + kvec_asg_arc_t_offset* u_buffer = &(hap_buf->buf[tid].u_buffer); + kvec_hap_candidates* u_can = &(hap_buf->buf[tid].u_can); + + + ma_utg_t *xReads = NULL, *yReads = NULL; + ma_hit_t_alloc *xR = NULL; + ma_hit_t *h = NULL; + ma_sub_t *sq = NULL, *st = NULL; + asg_t* nsg = ug->g; + uint32_t i, j, v, rId, k, is_Unitig, Hap_uId, xUid, yUid, seedOcc, xPos, yPos, is_update; + uint64_t tmp; + long long cur_offset, new_offset, interval_len; + long long r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end; + int32_t r; + asg_arc_t t; + asg_arc_t_offset t_offset; + hap_candidates hap_can; + hap_overlaps hap_align; + xUid = Input_uId; + if(nsg->seq[xUid].del || nsg->seq[xUid].c == ALTER_LABLE) return; + memset(vote_counting, 0, sizeof(uint64_t)*nsg->n_seq); + memset(visit, 0, nsg->n_seq); + u_vecs->a.n = 0; + u_can->a.n = 0; + + xReads = &(ug->u.a[xUid]); + for (i = 0; i < xReads->n; i++) + { + xR = &(reverse_sources[xReads->a[i]>>33]); + + for (k = 0; k < xR->length; k++) + { + rId = Get_tn(xR->buffer[k]); + + if(read_g->seq[rId].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, rId, &rId, &is_Unitig); + if(rId == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[rId].del == 1) continue; + } + + ///there are two cases: + ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id + ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 + get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); + if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; + ///here rId is the id of the read coming from the different haplotype + ///Hap_cId is the id of the corresponding contig (note here is the contig, instead of untig) + if(visit[Hap_uId]!=0) continue; + visit[Hap_uId] = 1; + if(vote_counting[Hap_uId] < UINT64_MAX) vote_counting[Hap_uId]++; + } + + clean_visit_flag(visit, read_g, ruIndex, nsg->n_seq, xR); + } + + + + u_vecs->a.n = 0; + for (i = 0; i < nsg->n_seq; i++) + { + if(i == xUid) continue; + if(vote_counting[i] == 0) continue; + tmp = vote_counting[i]; tmp = tmp << 32; tmp = tmp | (uint64_t)i; + kv_push(uint64_t, u_vecs->a, tmp); + } + + if(u_vecs->a.n == 0) return; + sort_kvec_t_u64_warp(u_vecs, 1); + + + ///scan each candidate unitig + for (i = 0; i < u_vecs->a.n; i++) + { + yUid = (uint32_t)u_vecs->a.a[i]; + seedOcc = u_vecs->a.a[i]>>32; + xReads = &(ug->u.a[xUid]); + yReads = &(ug->u.a[yUid]); + u_buffer->a.n = 0; + + + //if(xUid == 404 && yUid == 307) + // if(xReads->n <= 100 && yReads->n <= 100) + // { + // debug_enable = 1; + // } + // else + // { + // debug_enable = 0; + // } + // debug_enable = 1; + + for (k = 0; k < xReads->n; k++) + { + xR = &(reverse_sources[xReads->a[k]>>33]); + for (j = 0; j < xR->length; j++) + { + h = &(xR->buffer[j]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + if(st->del || read_g->seq[Get_tn(*h)].del) continue; + + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, &t); + ///if it is a contained overlap, skip + if(r < 0) continue; + + rId = t.v>>1; + if(read_g->seq[rId].del == 1) continue; + ///there are two cases: + ///1. read at primary contigs, get_R_to_U() return its corresponding contig Id + ///2. read at alternative contigs, get_R_to_U() return (uint32_t)-1 + get_R_to_U(ruIndex, rId, &Hap_uId, &is_Unitig); + if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; + if(Hap_uId != yUid) continue; + + v = xReads->a[k]>>32; + get_R_to_U(ruIndex, v>>1, &Hap_uId, &is_Unitig); + if(is_Unitig == 0 || Hap_uId == (uint32_t)-1) continue; + if(Hap_uId != xUid) continue; + if((uint32_t)(position_index[v>>1]) != k) continue; + + + if((prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), + xReads->n, yReads->n, 0, Hap_rate, seedOcc)==NON_PLOID) && + (prefilter((uint32_t)(position_index[v>>1]), (uint32_t)(position_index[rId]), + xReads->n, yReads->n, 1, Hap_rate, seedOcc)==NON_PLOID)) + { + continue; + } + + t_offset.Off = get_xy_pos(read_g, &t, v, (yReads->a[(uint32_t)(position_index[rId])])>>32, + xReads->len, yReads->len, position_index, &(t.el)); + if(((t_offset.Off>>32) == (uint32_t)-1) || (((uint32_t)t_offset.Off) == (uint32_t)-1)) continue; + + t_offset.x = t; + t_offset.weight = 1; + + kv_push(asg_arc_t_offset, u_buffer->a, t_offset); + } + + deduplicate_edge(u_buffer); + } + + if(u_buffer->a.n == 0) continue; + + // if(debug_enable) + // { + // print_debug_unitig(xReads, position_index, "xReads"); + // print_debug_unitig(yReads, position_index, "yReads"); + // } + + qsort(u_buffer->a.a, u_buffer->a.n, sizeof(asg_arc_t_offset), cmp_hap_alignment); + k = 0; + u_can->a.n = 0; + while (k < u_buffer->a.n) + { + hap_can.rev = u_buffer->a.a[k].x.el; + hap_can.index_beg = k; + hap_can.index_end = k; + hap_can.weight = u_buffer->a.a[k].weight; + hap_can.x_beg_pos = hap_can.x_end_pos = (uint32_t)(u_buffer->a.a[k].Off>>32); + hap_can.y_beg_pos = hap_can.y_end_pos = (uint32_t)(u_buffer->a.a[k].Off); + cur_offset = Cal_Off(u_buffer->a.a[k].Off); + interval_len = get_hap_overlapLen(hap_can.x_beg_pos, hap_can.x_end_pos, xReads->len, + hap_can.y_beg_pos, hap_can.y_end_pos, yReads->len, NULL, NULL, NULL, NULL); + + + k++; + while (k < u_buffer->a.n) + { + new_offset = Cal_Off(u_buffer->a.a[k].Off); + if(u_buffer->a.a[k].x.el != hap_can.rev) break; + if((new_offset - cur_offset)>(interval_len*chain_rate)) break; + + + hap_can.index_end = k; + hap_can.weight += u_buffer->a.a[k].weight; + + is_update = 0; + xPos = (uint32_t)(u_buffer->a.a[k].Off>>32); + yPos = (uint32_t)(u_buffer->a.a[k].Off); + if(xPos < hap_can.x_beg_pos) + { + hap_can.x_beg_pos = xPos; + is_update = 1; + } + + if(xPos > hap_can.x_end_pos) + { + hap_can.x_end_pos = xPos; + is_update = 1; + } + + if(yPos < hap_can.y_beg_pos) + { + hap_can.y_beg_pos = yPos; + is_update = 1; + } + + if(yPos > hap_can.y_end_pos) + { + hap_can.y_end_pos = yPos; + is_update = 1; + } + + if(new_offset == cur_offset) is_update = 0; + + if(is_update) + { + interval_len = get_hap_overlapLen(hap_can.x_beg_pos, hap_can.x_end_pos, xReads->len, + hap_can.y_beg_pos, hap_can.y_end_pos, yReads->len, NULL, NULL, NULL, NULL); + } + + k++; + } + + kv_push(hap_candidates, u_can->a, hap_can); + } + + if(u_can->a.n == 0) continue; + + qsort(u_can->a.a, u_can->a.n, sizeof(hap_candidates), cmp_hap_candidates); + + Get_match(hap_can) = Get_total(hap_can) = 0; + memset(&hap_align, 0, sizeof(hap_overlaps)); + + for (k = 0; k < u_can->a.n; k++) + { + is_update = 0; + if(u_can->a.a[k].weight < Get_match(hap_can)*Hap_rate) continue; + + if(calculate_pair_hap_similarity(u_buffer, &(u_can->a.a[k]), position_index, xUid, yUid, + xReads, yReads, reverse_sources, read_g, ruIndex, coverage_cut, Hap_rate, max_hang, + min_ovlp, &r_x_pos_beg, &r_x_pos_end, &r_y_pos_beg, &r_y_pos_end)!=PLOID) + { + continue; + } + + if(Get_match(hap_can) < Get_match(u_can->a.a[k])) + { + is_update = 1; + } + else if(Get_match(hap_can) == Get_match(u_can->a.a[k]) && + Get_total(hap_can) > Get_total(u_can->a.a[k])) + { + is_update = 1; + } + + if(is_update) + { + hap_can = u_can->a.a[k]; + hap_align.rev = Get_rev(hap_can); + hap_align.type = Get_type(hap_can); + hap_align.x_beg_id = Get_x_beg(hap_can); + hap_align.x_end_id = Get_x_end(hap_can) + 1; + hap_align.y_beg_id = Get_y_beg(hap_can); + hap_align.y_end_id = Get_y_end(hap_can) + 1; + hap_align.weight = Get_match(hap_can); + hap_align.x_beg_pos = r_x_pos_beg; + hap_align.x_end_pos = r_x_pos_end + 1; + if(hap_align.rev == 0) + { + hap_align.y_beg_pos = r_y_pos_beg; + hap_align.y_end_pos = r_y_pos_end + 1; + } + else + { + hap_align.y_beg_pos = yReads->len - r_y_pos_end - 1; + hap_align.y_end_pos = yReads->len - r_y_pos_beg - 1 + 1; + } + hap_align.xUid = xUid; + hap_align.yUid = yUid; + hap_align.status = SELF_EXIST; + } + } + + if(Get_match(hap_can) == 0 || Get_total(hap_can) == 0) continue; + + kv_push(hap_overlaps, all_ovlp->x[hap_align.xUid].a, hap_align); + } + +} + int inline get_specific_hap_overlap(kvec_hap_overlaps* x, uint32_t qn, uint32_t tn) { uint32_t i; @@ -2035,33 +2395,35 @@ uint32_t just_contain) fprintf(stderr, "*****************\n"); asg_t *purge_g = NULL; purge_g = asg_init(); - kvec_t_u64_warp u_vecs; - kv_init(u_vecs.a); asg_t* nsg = ug->g; uint32_t v, rId, uId, i, k, offset; ma_utg_t* reads = NULL; - uint8_t* visit = NULL; - visit = (uint8_t*)malloc(sizeof(uint8_t) * nsg->n_seq); - ///int flag; - memset(visit, 0, nsg->n_seq); + + // kvec_t_u64_warp u_vecs; + // kv_init(u_vecs.a); + // uint8_t* visit = NULL; + // visit = (uint8_t*)malloc(sizeof(uint8_t) * nsg->n_seq); + // memset(visit, 0, nsg->n_seq); + // uint64_t* vote_counting = (uint64_t*)malloc(sizeof(uint64_t)*nsg->n_seq); + // memset(vote_counting, 0, sizeof(uint64_t)*nsg->n_seq); + // kvec_asg_arc_t_offset u_buffer; + // kv_init(u_buffer.a); + // kvec_hap_candidates u_can; + // kv_init(u_can.a); uint64_t* position_index = (uint64_t*)malloc(sizeof(uint64_t)*read_g->n_seq); memset(position_index, -1, sizeof(uint64_t)*read_g->n_seq); - uint64_t* vote_counting = (uint64_t*)malloc(sizeof(uint64_t)*nsg->n_seq); - memset(vote_counting, 0, sizeof(uint64_t)*nsg->n_seq); - kvec_asg_arc_t_offset u_buffer; - kv_init(u_buffer.a); - kvec_hap_candidates u_can; - kv_init(u_can.a); + hap_overlaps_list all_ovlp; init_hap_overlaps_list(&all_ovlp, nsg->n_seq); hap_overlaps_list back_all_ovlp; init_hap_overlaps_list(&back_all_ovlp, nsg->n_seq); ///uint32_t junk_cov, hap_cov, dip_cov, junk_occ, repeat_occ, single_cov; - ma_sub_t* purge_cut = NULL; - purge_cut = (ma_sub_t*)malloc(sizeof(ma_sub_t)*nsg->n_seq); asg_arc_t t; asg_arc_t* p = NULL; int r; + hap_alignment_struct_pip hap_buf; + + /****************************may have bugs********************************/ for (v = 0; v < nsg->n_seq; ++v) @@ -2099,24 +2461,26 @@ uint32_t just_contain) offset += (uint32_t)reads->a[i]; } - purge_cut[uId].del = 0; - purge_cut[uId].s = 0; - purge_cut[uId].e = offset; - purge_cut[uId].c = PRIMARY_LABLE; asg_seq_set(purge_g, uId, offset, 0); purge_g->seq[uId].c = PRIMARY_LABLE; } - for (v = 0; v < nsg->n_seq; v++) - { - uId = v; - if(nsg->seq[uId].del || nsg->seq[uId].c == ALTER_LABLE) continue; + init_hap_alignment_struct_pip(&hap_buf, asm_opt.thread_num, nsg->n_seq, ug, read_g, + reverse_sources, ruIndex, coverage_cut, position_index, density, max_hang, min_ovlp, + 0.05, &all_ovlp); - hap_alignment(ug, read_g, reverse_sources, ruIndex, coverage_cut, position_index, - vote_counting, visit, &u_vecs, &u_buffer, &u_can, uId, density, max_hang, min_ovlp, - 0.05, &all_ovlp); - } + kt_for(asm_opt.thread_num, hap_alignment_worker, &hap_buf, nsg->n_seq); + + // for (v = 0; v < nsg->n_seq; v++) + // { + // uId = v; + // if(nsg->seq[uId].del || nsg->seq[uId].c == ALTER_LABLE) continue; + + // hap_alignment(ug, read_g, reverse_sources, ruIndex, coverage_cut, position_index, + // vote_counting, visit, &u_vecs, &u_buffer, &u_can, uId, density, max_hang, min_ovlp, + // 0.05, &all_ovlp); + // } normalize_hap_overlaps(&all_ovlp, &back_all_ovlp); ///debug_hap_overlaps(&all_ovlp, &back_all_ovlp); @@ -2188,7 +2552,7 @@ uint32_t just_contain) clean_purge_graph(purge_g, bubble_dist, drop_ratio); ///if(debug_enable) print_purge_gfa(ug, purge_g); link_unitigs(purge_g, ug, &all_ovlp, ruIndex, reverse_sources, coverage_cut, read_g, position_index, - max_hang, min_ovlp, edge, visit); + max_hang, min_ovlp, edge, hap_buf.buf[0].visit); } for (v = 0; v < all_ovlp.num; v++) @@ -2208,16 +2572,16 @@ uint32_t just_contain) } asg_cleanup(nsg); - kv_destroy(u_vecs.a); - kv_destroy(u_buffer.a); - kv_destroy(u_can.a); destory_hap_overlaps_list(&all_ovlp); destory_hap_overlaps_list(&back_all_ovlp); asg_destroy(purge_g); - free(purge_cut); free(position_index); - free(vote_counting); - free(visit); + // kv_destroy(u_vecs.a); + // kv_destroy(u_buffer.a); + // kv_destroy(u_can.a); + // free(vote_counting); + // free(visit); + destory_hap_alignment_struct_pip(&hap_buf); fprintf(stderr, "#################\n"); }