From dd8888aa90ba95cd2d1229c38a5ed2d69c31f2d2 Mon Sep 17 00:00:00 2001 From: chhylp123 Date: Thu, 5 May 2022 19:35:06 -0400 Subject: [PATCH] backup for memory issues --- Assembly.cpp | 56 +++++++++----- Assembly.h | 2 + CommandLines.h | 2 +- inter.cpp | 194 +++++++++++++++++++++++++++++++++---------------- 4 files changed, 173 insertions(+), 81 deletions(-) diff --git a/Assembly.cpp b/Assembly.cpp index e61427c..3abb398 100644 --- a/Assembly.cpp +++ b/Assembly.cpp @@ -517,11 +517,19 @@ static int64_t ha_Graph_mem(const Graph *g) return mem; } -int64_t ha_ovec_mem(const ha_ovec_buf_t *b) +int64_t ha_ovec_mem(const ha_ovec_buf_t *b, int64_t *mem_a) { - int64_t i, mem = 0, mem_clist, mem_olist; - mem_clist = b->clist.size * sizeof(k_mer_hit) + b->clist.chainDP.size * 7 * 4; - + int64_t i, mem_ab = 0, mem_clist, mem_olist, mem_hap = 0, mem_aux = 0; + // mem_clist = b->clist.size * sizeof(k_mer_hit) + b->clist.chainDP.size * 7 * 4; + mem_clist = (b->clist.size * sizeof(k_mer_hit)) + ((sizeof((*b->clist.chainDP.score)) + sizeof((*b->clist.chainDP.pre)) + + sizeof((*b->clist.chainDP.indels)) + sizeof((*b->clist.chainDP.self_length)) + + sizeof((*b->clist.chainDP.occ)) + sizeof((*b->clist.chainDP.tmp))) * b->clist.chainDP.size); + mem_clist += sizeof(*(b->b_buf.a.a)) * b->b_buf.a.m; + mem_clist += sizeof(*(b->r_buf.a.a)) * b->r_buf.a.m; + mem_clist += sizeof(*(b->k_flag.a.a)) * b->k_flag.a.m; + mem_clist += sizeof(*(b->sp.a)) * b->sp.m; + mem_clist += sizeof(*(b->tmp_region.f_cigar.buffer)) * b->tmp_region.f_cigar.size; + mem_olist = b->olist.size * sizeof(overlap_region); for (i = 0; i < (int64_t)b->olist.size; ++i) { const overlap_region *r = &b->olist.list[i]; @@ -529,6 +537,7 @@ int64_t ha_ovec_mem(const ha_ovec_buf_t *b) mem_olist += r->f_cigar.size * 8; mem_olist += r->boundary_cigars.size * sizeof(window_list); } + mem_olist += b->olist_hp.size * sizeof(overlap_region); for (i = 0; i < (int64_t)b->olist_hp.size; ++i) { const overlap_region *r = &b->olist_hp.list[i]; @@ -537,17 +546,28 @@ int64_t ha_ovec_mem(const ha_ovec_buf_t *b) mem_olist += r->boundary_cigars.size * sizeof(window_list); } - if(b->ab) mem = ha_abuf_mem(b->ab) + mem_clist + mem_olist; - if(b->abl) mem = ha_abufl_mem(b->abl) + mem_clist + mem_olist; + if(b->ab) mem_ab += ha_abuf_mem(b->ab); + if(b->abl) mem_ab += ha_abufl_mem(b->abl); + if (!b->is_final) { - mem += sizeof(Cigar_record) + b->cigar1.lost_base_size + b->cigar1.size * 4; - mem += sizeof(Correct_dumy) + b->correct.size * 8; - mem += sizeof(Round2_alignment) + b->round2.cigar.size * 4 + b->round2.tmp_cigar.size * 4; - mem += sizeof(haplotype_evdience_alloc) + b->hap.size * sizeof(haplotype_evdience) + b->hap.snp_matrix_size + b->hap.r_snp_size + b->hap.snp_stat.m * sizeof(SnpStats) + b->hap.snp_srt.m * sizeof(uint64_t); - mem += ha_Graph_mem(&b->POA_Graph); - mem += ha_Graph_mem(&b->DAGCon); + mem_hap += sizeof(Cigar_record) + b->cigar1.lost_base_size + b->cigar1.size * 4; + mem_hap += sizeof(Correct_dumy) + b->correct.size * 8; + mem_hap += sizeof(Round2_alignment) + b->round2.cigar.size * 4 + b->round2.tmp_cigar.size * 4; + mem_hap += sizeof(haplotype_evdience_alloc) + b->hap.size * sizeof(haplotype_evdience) + b->hap.snp_matrix_size + b->hap.r_snp_size + b->hap.snp_stat.m * sizeof(SnpStats) + b->hap.snp_srt.m * sizeof(uint64_t); + mem_hap += ha_Graph_mem(&b->POA_Graph); + mem_hap += ha_Graph_mem(&b->DAGCon); } - return mem; + + + mem_aux += sizeof(*(b->self_read.seq)) * b->self_read.size; + mem_aux += sizeof(*(b->ovlp_read.seq)) * b->ovlp_read.size; + + if(mem_a) { + mem_a[0] = mem_ab; mem_a[1] = mem_clist; mem_a[2] = mem_olist; + mem_a[3] = mem_hap; mem_a[4] = mem_aux; + } + + return mem_ab + mem_clist + mem_olist + mem_hap + mem_aux; } uint32_t get_het_cnt(haplotype_evdience_alloc *hap) @@ -567,7 +587,7 @@ static void worker_ovec(void *data, long i, int tid) int fully_cov, abnormal; ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, - 0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); + 0.02, asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); clear_Cigar_record(&b->cigar1); clear_Round2_alignment(&b->round2); @@ -627,7 +647,7 @@ static void worker_ovec_related_reads(void *data, long i, int tid) int fully_cov, abnormal, q_idx = k; ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, - 0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]), &(b->sp)); + 0.02, asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]), &(b->sp)); clear_Cigar_record(&b->cigar1); clear_Round2_alignment(&b->round2); @@ -917,7 +937,7 @@ void ha_overlap_and_correct(int round) asm_opt.num_bases += b[i]->num_read_base; asm_opt.num_corrected_bases += b[i]->num_correct_base; asm_opt.num_recorrected_bases += b[i]->num_recorrect_base; - asm_opt.mem_buf += ha_ovec_mem(b[i]); + asm_opt.mem_buf += ha_ovec_mem(b[i], NULL); ha_ovec_destroy(b[i]); } free(b); @@ -1339,7 +1359,7 @@ static void worker_ov_final(void *data, long i, int tid) //get_new_candidates(i, &g_read, &overlap_list, &array_list, &l, 0.001, 0); ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 0.001, - asm_opt.max_n_chain, 0, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); + asm_opt.max_n_chain, 0, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); overlap_region_sort_y_id(b->olist.list, b->olist.length); ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length); @@ -1405,7 +1425,7 @@ static void worker_ov_final_high_het(void *data, long i, int tid) ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid]; ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, HIGH_HET_ERROR_RATE, - asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); + asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); overlap_region_sort_y_id(b->olist.list, b->olist.length); ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length); diff --git a/Assembly.h b/Assembly.h index 11df5f2..40b98d6 100644 --- a/Assembly.h +++ b/Assembly.h @@ -47,5 +47,7 @@ void ug_idx_build(ma_ug_t *ug, int hap_n); ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov, int is_ug); ha_ovec_buf_t *ha_ovec_buf_init(void *km, int is_final, int save_ov, int is_ug); void ha_ovec_destroy(ha_ovec_buf_t *b); +int64_t ha_ovec_mem(const ha_ovec_buf_t *b, int64_t *mem_a) +; #endif diff --git a/CommandLines.h b/CommandLines.h index aa95dfd..e8166af 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -4,7 +4,7 @@ #include #include -#define HA_VERSION "0.16.4-r399" +#define HA_VERSION "0.16.4-r4000" #define VERBOSE 0 diff --git a/inter.cpp b/inter.cpp index bb34afc..8595865 100644 --- a/inter.cpp +++ b/inter.cpp @@ -56,7 +56,7 @@ void ha_get_ul_candidates_interface(ha_abufl_t *ab, int64_t rid, char* rs, uint6 #define SEC_LEN_DIF 0.03 #define REA_ALIGN_CUTOFF 32 -#define CHUNK_SIZE 500000000 +#define CHUNK_SIZE 250000000 #define generic_key(x) (x) KRADIX_SORT_INIT(gfa64, uint64_t, generic_key, 8) @@ -399,6 +399,7 @@ typedef struct { // data structure for each step in kt_pipeline() gdpchain_t *gdp; // glchain_t *sec_ll; uint64_t num_bases, num_corrected_bases, num_recorrected_bases; + int64_t n_thread; } utepdat_t; @@ -4941,10 +4942,12 @@ void update_ul_vec_t_ug(const ul_idx_t *uref, ul_vec_t *rch, vec_mg_lchain_t *uc if(rch->bb.a[k].pidx == (uint32_t)-1) k = -1; else k = rch->bb.a[k].pidx; } - // fprintf(stderr, "-ulid->%ld\n", ulid); + rch->dd = 0; if(sp != (uint32_t)-1) l += ep - sp; l = (int64_t)rch->rlen - l; + + // if(ulid == 1756) fprintf(stderr, "-ulid:%ld, l:%ld, rch->rlen:%u\n", ulid, l, rch->rlen); if(l == 0) { rch->dd = 1; } else if(l < ((int64_t)rch->rlen)*0.001) { @@ -5053,7 +5056,7 @@ int64_t debug_i, int64_t tid, void *km) f = check_trans_rate_gap(&(gdp->swap), &(ll->tk), olist, hap, uref, diff_ec_ul, winLen, G_CHAIN_TRANS_RATE); } } - // fprintf(stderr, "(beg2) [M::%s] debug_i:%ld, qlen:%ld\n", __func__, debug_i, qlen); + // if(debug_i == 1756) fprintf(stderr, "[M::%s] ulid:%ld, qlen:%ld, f:%ld\n", __func__, debug_i, qlen, f); if(f) update_ul_vec_t_ug(uref, rch, &(gdp->swap), debug_i); // debug_ul_vec_t_chain(km, uref->ug->g, rch, &(gdp->dst_done), &(gdp->out)); // fprintf(stderr, "(beg3) [M::%s::tid:%ld] debug_i:%ld, qlen:%ld\n", __func__, tid, debug_i, qlen); @@ -5080,6 +5083,35 @@ uint64_t kv_ul_ov_t_statistics(kv_ul_ov_t *olist, uint64_t qn, int64_t *occ) return l; } +#define kv_mem(v) ((v).m * sizeof(*((v).a))) + +int64_t get_utepdat_t_mem_tid(const utepdat_t *b, int64_t tid, int64_t *mem, int64_t *mem_hab) +{ + km_stat_t kmst; memset(mem, 0, sizeof(*(mem))*6); + if(b->hab) { + mem[0] += ha_ovec_mem(b->hab[tid], mem_hab); + } + if(b->ll) { + mem[1] += kv_mem(b->ll[tid].lo) + kv_mem(b->ll[tid].tk) + kv_mem(b->ll[tid].srt.a); + } + if(b->buf) { + km_stat(b->buf[tid]->km, &kmst); + mem[2] += kmst.capacity; + } + if(b->gdp) { + mem[3] += kv_mem(b->gdp[tid].l) + kv_mem(b->gdp[tid].swap) + kv_mem(b->gdp[tid].dst) + kv_mem(b->gdp[tid].out) + + kv_mem(b->gdp[tid].path) + kv_mem(b->gdp[tid].v) + kv_mem(b->gdp[tid].f) + kv_mem(b->gdp[tid].dst_done); + } + if(b->mzs) { + mem[4] += kv_mem(b->mzs[tid]); + } + if(b->sps) { + mem[5] += kv_mem(b->sps[tid]); + } + return mem[0] + mem[1] + mem[2] + mem[3] + mem[4] + mem[5]; +} + + static void worker_for_ul_scall_alignment(void *data, long i, int tid) // callback for kt_for() { utepdat_t *s = (utepdat_t*)data; @@ -5094,7 +5126,7 @@ static void worker_for_ul_scall_alignment(void *data, long i, int tid) // callba // if (memcmp(UL_INF.nid.a[s->id+i].a, "d0aab024-b3a7-40fb-83cc-22c3d6d951f8", UL_INF.nid.a[s->id+i].n-1)) return; // fprintf(stderr, "[M::%s::] ==> len: %lu\n", __func__, s->len[i]); ha_get_ul_candidates_interface(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres, - s->opt->max_n_chain, 1, &(b->k_flag), &b->r_buf, &(b->tmp_region), NULL, &(b->sp), asm_opt.hom_cov, km); + s->opt->max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(b->tmp_region), NULL, &(b->sp), asm_opt.hom_cov, km); clear_Cigar_record(&b->cigar1); clear_Round2_alignment(&b->round2); @@ -5146,74 +5178,85 @@ static void worker_for_ul_scall_alignment(void *data, long i, int tid) // callba } } + static void worker_for_ul_rescall_alignment(void *data, long i, int tid) // callback for kt_for() { utepdat_t *s = (utepdat_t*)data; - ha_ovec_buf_t *b = s->hab[tid]; - glchain_t *bl = &(s->ll[tid]); - int64_t /**rid = s->id+i,**/ winLen = MIN((((double)THRESHOLD_MAX_SIZE)/s->opt->diff_ec_ul), WINDOW); - // uint64_t align = 0; - int fully_cov, abnormal; - // void *km = s->buf?(s->buf[tid]?s->buf[tid]->km:NULL):NULL; - // if(s->id+i!=873) return; - // fprintf(stderr, "\n[M::%s] rid:%ld, len:%lu\n", __func__, s->id+i, s->len[i]); - // if (memcmp(UL_INF.nid.a[s->id+i].a, "d0aab024-b3a7-40fb-83cc-22c3d6d951f8", UL_INF.nid.a[s->id+i].n-1)) return; - // fprintf(stderr, "[M::%s::] ==> len: %lu\n", __func__, s->len[i]); - ha_get_ul_candidates_interface(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres, - s->opt->max_n_chain, 1, &(b->k_flag), &b->r_buf, &(b->tmp_region), NULL, &(b->sp), 1, NULL); - - clear_Cigar_record(&b->cigar1); - clear_Round2_alignment(&b->round2); - // return; - // b->num_correct_base += overlap_statistics(&b->olist, NULL, 0); + ha_ovec_buf_t *b = s->hab[tid]; + glchain_t *bl = &(s->ll[tid]); + int64_t /**rid = s->id+i,**/ winLen = MIN((((double)THRESHOLD_MAX_SIZE)/s->opt->diff_ec_ul), WINDOW); + // uint64_t align = 0; + int fully_cov, abnormal; + assert(UL_INF.a[s->id+i].rlen == s->len[i]); + // void *km = s->buf?(s->buf[tid]?s->buf[tid]->km:NULL):NULL; + // if(s->id+i!=873) return; + // fprintf(stderr, "\n[M::%s] rid:%ld, len:%lu\n", __func__, s->id+i, s->len[i]); + // if (memcmp(UL_INF.nid.a[s->id+i].a, "d0aab024-b3a7-40fb-83cc-22c3d6d951f8", UL_INF.nid.a[s->id+i].n-1)) return; + // fprintf(stderr, "[M::%s::] ==> len: %lu\n", __func__, s->len[i]); + ha_get_ul_candidates_interface(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres, + s->opt->max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(b->tmp_region), NULL, &(b->sp), 1, NULL); + + clear_Cigar_record(&b->cigar1); + clear_Round2_alignment(&b->round2); + // return; + // b->num_correct_base += overlap_statistics(&b->olist, NULL, 0); - b->self_read.seq = s->seq[i]; b->self_read.length = s->len[i]; b->self_read.size = 0; - correct_ul_overlap(&b->olist, s->uu, &b->self_read, &b->correct, &b->ovlp_read, &b->POA_Graph, &b->DAGCon, - &b->cigar1, &b->hap, &b->round2, 0, 1, &fully_cov, &abnormal, s->opt->diff_ec_ul, winLen, NULL); + b->self_read.seq = s->seq[i]; b->self_read.length = s->len[i]; b->self_read.size = 0; + correct_ul_overlap(&b->olist, s->uu, &b->self_read, &b->correct, &b->ovlp_read, &b->POA_Graph, &b->DAGCon, + &b->cigar1, &b->hap, &b->round2, 0, 1, &fully_cov, &abnormal, s->opt->diff_ec_ul, winLen, NULL); - // uint64_t k; - // for (k = 0; k < b->olist.length; k++) { - // if(b->olist.list[k].is_match == 1) b->num_correct_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s; - // if(b->olist.list[k].is_match == 2) b->num_recorrect_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s; - // } + // uint64_t k; + // for (k = 0; k < b->olist.length; k++) { + // if(b->olist.list[k].is_match == 1) b->num_correct_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s; + // if(b->olist.list[k].is_match == 2) b->num_recorrect_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s; + // } - // gl_chain_refine(&b->olist, &b->correct, &b->hap, bl, s->uu, s->opt->diff_ec_ul, winLen, s->len[i], km); - gl_chain_refine_advance_combine(s->buf[tid], &(UL_INF.a[i]), &b->olist, &b->correct, &b->hap, &(s->sps[tid]), bl, &(s->gdp[tid]), s->uu, s->opt->diff_ec_ul, winLen, s->len[i], s->uopt, s->id+i, tid, NULL); - // return; - // b->num_read_base += b->self_read.length; - // b->num_correct_base += b->correct.corrected_base; - // b->num_recorrect_base += b->round2.dumy.corrected_base; - memset(&b->self_read, 0, sizeof(b->self_read)); - if(UL_INF.a[i].dd) { - free(s->seq[i]); s->seq[i] = NULL; b->num_correct_base++; - } - s->hab[tid]->num_read_base++; + // gl_chain_refine(&b->olist, &b->correct, &b->hap, bl, s->uu, s->opt->diff_ec_ul, winLen, s->len[i], km); + gl_chain_refine_advance_combine(s->buf[tid], &(UL_INF.a[s->id+i]), &b->olist, &b->correct, &b->hap, &(s->sps[tid]), bl, &(s->gdp[tid]), s->uu, s->opt->diff_ec_ul, winLen, s->len[i], s->uopt, s->id+i, tid, NULL); + // return; + // b->num_read_base += b->self_read.length; + // b->num_correct_base += b->correct.corrected_base; + // b->num_recorrect_base += b->round2.dumy.corrected_base; + memset(&b->self_read, 0, sizeof(b->self_read)); + if(UL_INF.a[s->id+i].dd) { + free(s->seq[i]); s->seq[i] = NULL; b->num_correct_base++; + } + s->hab[tid]->num_read_base++; - // align = kv_ul_ov_t_statistics(&(bl->tk), i, &(b->num_recorrect_base)); - // if(align == s->len[i]) { - // free(s->seq[i]); s->seq[i] = NULL; - // } - // b->num_correct_base += align; + int64_t mem[6], mem_hab[6]; + if(get_utepdat_t_mem_tid(s, tid, mem, mem_hab)>((int64_t)5*(int64_t)1073741824)) { + fprintf(stderr, "[M::%s::tid->%d::rid->%ld] buffer[0]: %.3fGB(%.3fGB::%.3fGB::%.3fGB::%.3fGB::%.3fGB), buffer[1]: %.3fGB, buffer[2]: %.3fGB, buffer[3]: %.3fGB, buffer[4]: %.3fGB, buffer[5]: %.3fGB\n", + __func__, tid, i, mem[0]/1073741824.0, + mem_hab[0]/1073741824.0, mem_hab[1]/1073741824.0, mem_hab[2]/1073741824.0, + mem_hab[3]/1073741824.0, mem_hab[4]/1073741824.0, + mem[1]/1073741824.0, mem[2]/1073741824.0, + mem[3]/1073741824.0, mem[4]/1073741824.0, mem[5]/1073741824.0); + } - // uint64_t k; - // b->num_read_base += overlap_statistics(&b->olist, NULL, NULL, 1); - // for (k = 0; k < bl->tk.n; k++) { - // if(bl->tk.a[k].sec == 0) b->num_correct_base += bl->tk.a[k].qe - bl->tk.a[k].qs; - // if(bl->tk.a[k].sec > 0) b->num_recorrect_base += bl->tk.a[k].qe - bl->tk.a[k].qs; - // } - // for (k = 0; k < bl->lo.n; k++) { - // b->num_read_base += bl->lo.a[k].qe - bl->lo.a[k].qs; - // } + // align = kv_ul_ov_t_statistics(&(bl->tk), i, &(b->num_recorrect_base)); + // if(align == s->len[i]) { + // free(s->seq[i]); s->seq[i] = NULL; + // } + // b->num_correct_base += align; - // uint32_t l1 = overlap_statistics(&b->olist, s->uu->ug, 1), l2 = overlap_statistics(&b->olist, s->uu->ug, 2); + // uint64_t k; + // b->num_read_base += overlap_statistics(&b->olist, NULL, NULL, 1); + // for (k = 0; k < bl->tk.n; k++) { + // if(bl->tk.a[k].sec == 0) b->num_correct_base += bl->tk.a[k].qe - bl->tk.a[k].qs; + // if(bl->tk.a[k].sec > 0) b->num_recorrect_base += bl->tk.a[k].qe - bl->tk.a[k].qs; + // } + // for (k = 0; k < bl->lo.n; k++) { + // b->num_read_base += bl->lo.a[k].qe - bl->lo.a[k].qs; + // } + + // uint32_t l1 = overlap_statistics(&b->olist, s->uu->ug, 1), l2 = overlap_statistics(&b->olist, s->uu->ug, 2); // - // if(l1 == 0 && l2 > 0) fprintf(stderr, "[M::%s::%lu::no_match]\n", UL_INF.nid.a[s->id+i].a, s->len[i]); - // fprintf(stderr, "[M::%s::%lu::] l1->%u; l2->%u\n", UL_INF.nid.a[s->id+i].a, s->len[i], l1, l2); - // fprintf(stderr, "[M::%s::rid->%ld] done\n", __func__, s->id+i); + // if(l1 == 0 && l2 > 0) fprintf(stderr, "[M::%s::%lu::no_match]\n", UL_INF.nid.a[s->id+i].a, s->len[i]); + // fprintf(stderr, "[M::%s::%lu::] l1->%u; l2->%u\n", UL_INF.nid.a[s->id+i].a, s->len[i], l1, l2); + // fprintf(stderr, "[M::%s::rid->%ld] done\n", __func__, s->id+i); } - void dump_gaf(mg_gres_a *hits, const mg_gchains_t *gs, uint32_t only_p) { if (gs == NULL || gs->n_gc == 0 || gs->n_lc == 0) return; @@ -5262,6 +5305,27 @@ void dump_gaf(mg_gres_a *hits, const mg_gchains_t *gs, uint32_t only_p) } } + +int64_t get_utepdat_t_mem(const utepdat_t *b, int64_t is_print) +{ + int64_t i, mem[7] = {0}, tt[6] = {0}; + for (i = 0; i < b->n_thread; i++) { + get_utepdat_t_mem_tid(b, i, tt, NULL); + mem[0] += tt[0]; mem[1] += tt[1]; mem[2] += tt[2]; mem[3] += tt[3]; mem[4] += tt[4]; mem[5] += tt[5]; + } + + for (i = 0; i < b->n; ++i) mem[6] += b->len[i]; + mem[6] += (sizeof(*(b->len))*b->n) + (sizeof(*(b->seq))*b->n); + if(is_print) { + for (i = 0; i < 7; i++) { + fprintf(stderr, "[M::%s] size of buffer[%ld]: %.3fGB, %.3fKB, %ldB\n", + __func__, i, mem[i]/1073741824.0, mem[i]/1048576.0, mem[i]); + } + } + + return mem[0] + mem[1] + mem[2] + mem[3] + mem[4] + mem[5] + mem[6]; +} + static void *worker_ul_pipeline(void *data, int step, void *in) // callback for kt_pipeline() { uldat_t *p = (uldat_t*)data; @@ -5536,7 +5600,7 @@ static void *worker_ul_rescall_pipeline(void *data, int step, void *in) // callb else if (step == 1) { // step 2: alignment utepdat_t *s = (utepdat_t*)in; - uint64_t i; + uint64_t i; s->n_thread = p->n_thread; CALLOC(s->hab, p->n_thread); CALLOC(s->ll, p->n_thread); CALLOC(s->buf, p->n_thread); CALLOC(s->gdp, p->n_thread); CALLOC(s->mzs, p->n_thread); CALLOC(s->sps, p->n_thread); @@ -5547,9 +5611,12 @@ static void *worker_ul_rescall_pipeline(void *data, int step, void *in) // callb fprintf(stderr, "[M::%s::Start] ==> s->id: %lu, s->n:% d\n", __func__, s->id, s->n); kt_for(p->n_thread, worker_for_ul_rescall_alignment, s, s->n); fprintf(stderr, "[M::%s::Done] ==> s->id: %lu, s->n:% d\n", __func__, s->id, s->n); + get_utepdat_t_mem(s, 1); + for (i = 0; i < p->n_thread; ++i) { p->num_bases += s->hab[i]->num_read_base; p->num_corrected_bases += s->hab[i]->num_correct_base; + // s->num_recorrected_bases += s->hab[i]->num_recorrect_base; ha_ovec_destroy(s->hab[i]); hc_glchain_destroy(&(s->ll[i])); mg_tbuf_destroy(s->buf[i]); hc_gdpchain_destroy(&(s->gdp[i])); @@ -5564,6 +5631,8 @@ static void *worker_ul_rescall_pipeline(void *data, int step, void *in) // callb for (i = 0; i < s->n; ++i) { rid = s->id + i; if(UL_INF.a[rid].dd == 0 && p->ucr_s && p->ucr_s->flag == 1) { + assert(s->seq[i]); + // if(s->seq[i] == NULL) fprintf(stderr, "[M::%s::]rid->%ld, len->%lu\n", __func__, rid, s->len[i]); write_compress_base_disk(p->ucr_s->fp, rid, s->seq[i], s->len[i], &(p->ucr_s->u)); } free(s->seq[i]); @@ -8540,11 +8609,12 @@ int scall_ul_pipeline(uldat_t* sl, const enzyme *fn) int rescall_ul_pipeline(uldat_t* sl, const enzyme *fn) { double index_time = yak_realtime(); - int32_t i; uint32_t k; + int32_t i; uint32_t k, rlen; ///clean UL_INF for (k = 0; k < UL_INF.n; k++) { + rlen = UL_INF.a[k].rlen; free(UL_INF.a[k].bb.a); free(UL_INF.a[k].N_site.a); free(UL_INF.a[k].r_base.a); - memset(&(UL_INF.a[k]), 0, sizeof(UL_INF.a[k])); + memset(&(UL_INF.a[k]), 0, sizeof(UL_INF.a[k])); UL_INF.a[k].rlen = rlen; } for (i = 0; i < fn->n; i++){