xxay/hifiasm
1
0
Fork 0
mirror of https://github.com/chhylp123/hifiasm.git synced 2026-10-09 17:10:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

backup for memory issues

Browse Source
This commit is contained in:
chhylp123
2022-05-05 19:35:06 -04:00
parent 8a2bb0b112
commit dd8888aa90
4 changed files with 173 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Assembly.cpp
+38 -18
View File
@@ -517,11 +517,19 @@ static int64_t ha_Graph_mem(const Graph *g)
return mem; return mem;
} }
int64_t ha_ovec_mem(const ha_ovec_buf_t *b) int64_t ha_ovec_mem(const ha_ovec_buf_t *b, int64_t *mem_a)
{ {
int64_t i, mem = 0, mem_clist, mem_olist; int64_t i, mem_ab = 0, mem_clist, mem_olist, mem_hap = 0, mem_aux = 0;
mem_clist = b->clist.size * sizeof(k_mer_hit) + b->clist.chainDP.size * 7 * 4; // mem_clist = b->clist.size * sizeof(k_mer_hit) + b->clist.chainDP.size * 7 * 4;
mem_clist = (b->clist.size * sizeof(k_mer_hit)) + ((sizeof((*b->clist.chainDP.score)) + sizeof((*b->clist.chainDP.pre))
+ sizeof((*b->clist.chainDP.indels)) + sizeof((*b->clist.chainDP.self_length))
+ sizeof((*b->clist.chainDP.occ)) + sizeof((*b->clist.chainDP.tmp))) * b->clist.chainDP.size);
mem_clist += sizeof(*(b->b_buf.a.a)) * b->b_buf.a.m;
mem_clist += sizeof(*(b->r_buf.a.a)) * b->r_buf.a.m;
mem_clist += sizeof(*(b->k_flag.a.a)) * b->k_flag.a.m;
mem_clist += sizeof(*(b->sp.a)) * b->sp.m;
mem_clist += sizeof(*(b->tmp_region.f_cigar.buffer)) * b->tmp_region.f_cigar.size;
mem_olist = b->olist.size * sizeof(overlap_region); mem_olist = b->olist.size * sizeof(overlap_region);
for (i = 0; i < (int64_t)b->olist.size; ++i) { for (i = 0; i < (int64_t)b->olist.size; ++i) {
const overlap_region *r = &b->olist.list[i]; const overlap_region *r = &b->olist.list[i];
@@ -529,6 +537,7 @@ int64_t ha_ovec_mem(const ha_ovec_buf_t *b)
mem_olist += r->f_cigar.size * 8; mem_olist += r->f_cigar.size * 8;
mem_olist += r->boundary_cigars.size * sizeof(window_list); mem_olist += r->boundary_cigars.size * sizeof(window_list);
} }
mem_olist += b->olist_hp.size * sizeof(overlap_region); mem_olist += b->olist_hp.size * sizeof(overlap_region);
for (i = 0; i < (int64_t)b->olist_hp.size; ++i) { for (i = 0; i < (int64_t)b->olist_hp.size; ++i) {
const overlap_region *r = &b->olist_hp.list[i]; const overlap_region *r = &b->olist_hp.list[i];
@@ -537,17 +546,28 @@ int64_t ha_ovec_mem(const ha_ovec_buf_t *b)
mem_olist += r->boundary_cigars.size * sizeof(window_list); mem_olist += r->boundary_cigars.size * sizeof(window_list);
} }
if(b->ab) mem = ha_abuf_mem(b->ab) + mem_clist + mem_olist; if(b->ab) mem_ab += ha_abuf_mem(b->ab);
if(b->abl) mem = ha_abufl_mem(b->abl) + mem_clist + mem_olist; if(b->abl) mem_ab += ha_abufl_mem(b->abl);
if (!b->is_final) { if (!b->is_final) {
mem += sizeof(Cigar_record) + b->cigar1.lost_base_size + b->cigar1.size * 4; mem_hap += sizeof(Cigar_record) + b->cigar1.lost_base_size + b->cigar1.size * 4;
mem += sizeof(Correct_dumy) + b->correct.size * 8; mem_hap += sizeof(Correct_dumy) + b->correct.size * 8;
mem += sizeof(Round2_alignment) + b->round2.cigar.size * 4 + b->round2.tmp_cigar.size * 4; mem_hap += sizeof(Round2_alignment) + b->round2.cigar.size * 4 + b->round2.tmp_cigar.size * 4;
mem += sizeof(haplotype_evdience_alloc) + b->hap.size * sizeof(haplotype_evdience) + b->hap.snp_matrix_size + b->hap.r_snp_size + b->hap.snp_stat.m * sizeof(SnpStats) + b->hap.snp_srt.m * sizeof(uint64_t); mem_hap += sizeof(haplotype_evdience_alloc) + b->hap.size * sizeof(haplotype_evdience) + b->hap.snp_matrix_size + b->hap.r_snp_size + b->hap.snp_stat.m * sizeof(SnpStats) + b->hap.snp_srt.m * sizeof(uint64_t);
mem += ha_Graph_mem(&b->POA_Graph); mem_hap += ha_Graph_mem(&b->POA_Graph);
mem += ha_Graph_mem(&b->DAGCon); mem_hap += ha_Graph_mem(&b->DAGCon);
} }
return mem;
mem_aux += sizeof(*(b->self_read.seq)) * b->self_read.size;
mem_aux += sizeof(*(b->ovlp_read.seq)) * b->ovlp_read.size;
if(mem_a) {
mem_a[0] = mem_ab; mem_a[1] = mem_clist; mem_a[2] = mem_olist;
mem_a[3] = mem_hap; mem_a[4] = mem_aux;
}
return mem_ab + mem_clist + mem_olist + mem_hap + mem_aux;
} }
uint32_t get_het_cnt(haplotype_evdience_alloc *hap) uint32_t get_het_cnt(haplotype_evdience_alloc *hap)
@@ -567,7 +587,7 @@ static void worker_ovec(void *data, long i, int tid)
int fully_cov, abnormal; int fully_cov, abnormal;
ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist,
0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); 0.02, asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
clear_Cigar_record(&b->cigar1); clear_Cigar_record(&b->cigar1);
clear_Round2_alignment(&b->round2); clear_Round2_alignment(&b->round2);
@@ -627,7 +647,7 @@ static void worker_ovec_related_reads(void *data, long i, int tid)
int fully_cov, abnormal, q_idx = k; int fully_cov, abnormal, q_idx = k;
ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist,
0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]), &(b->sp)); 0.02, asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]), &(b->sp));
clear_Cigar_record(&b->cigar1); clear_Cigar_record(&b->cigar1);
clear_Round2_alignment(&b->round2); clear_Round2_alignment(&b->round2);
@@ -917,7 +937,7 @@ void ha_overlap_and_correct(int round)
asm_opt.num_bases += b[i]->num_read_base; asm_opt.num_bases += b[i]->num_read_base;
asm_opt.num_corrected_bases += b[i]->num_correct_base; asm_opt.num_corrected_bases += b[i]->num_correct_base;
asm_opt.num_recorrected_bases += b[i]->num_recorrect_base; asm_opt.num_recorrected_bases += b[i]->num_recorrect_base;
asm_opt.mem_buf += ha_ovec_mem(b[i]); asm_opt.mem_buf += ha_ovec_mem(b[i], NULL);
ha_ovec_destroy(b[i]); ha_ovec_destroy(b[i]);
} }
free(b); free(b);
@@ -1339,7 +1359,7 @@ static void worker_ov_final(void *data, long i, int tid)
//get_new_candidates(i, &g_read, &overlap_list, &array_list, &l, 0.001, 0); //get_new_candidates(i, &g_read, &overlap_list, &array_list, &l, 0.001, 0);
ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 0.001, ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 0.001,
asm_opt.max_n_chain, 0, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); asm_opt.max_n_chain, 0, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
overlap_region_sort_y_id(b->olist.list, b->olist.length); overlap_region_sort_y_id(b->olist.list, b->olist.length);
ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length); ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
@@ -1405,7 +1425,7 @@ static void worker_ov_final_high_het(void *data, long i, int tid)
ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid]; ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid];
ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, HIGH_HET_ERROR_RATE, ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, HIGH_HET_ERROR_RATE,
asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp)); asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
overlap_region_sort_y_id(b->olist.list, b->olist.length); overlap_region_sort_y_id(b->olist.list, b->olist.length);
ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length); ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
Assembly.h
+2
View File
@@ -47,5 +47,7 @@ void ug_idx_build(ma_ug_t *ug, int hap_n);
ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov, int is_ug); ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov, int is_ug);
ha_ovec_buf_t *ha_ovec_buf_init(void *km, int is_final, int save_ov, int is_ug); ha_ovec_buf_t *ha_ovec_buf_init(void *km, int is_final, int save_ov, int is_ug);
void ha_ovec_destroy(ha_ovec_buf_t *b); void ha_ovec_destroy(ha_ovec_buf_t *b);
int64_t ha_ovec_mem(const ha_ovec_buf_t *b, int64_t *mem_a)
;
#endif #endif
CommandLines.h
+1 -1
View File
@@ -4,7 +4,7 @@
#include <pthread.h> #include <pthread.h>
#include <stdint.h> #include <stdint.h>
#define HA_VERSION "0.16.4-r399" #define HA_VERSION "0.16.4-r4000"
#define VERBOSE 0 #define VERBOSE 0
inter.cpp
+132 -62
View File
@@ -56,7 +56,7 @@ void ha_get_ul_candidates_interface(ha_abufl_t *ab, int64_t rid, char* rs, uint6
#define SEC_LEN_DIF 0.03 #define SEC_LEN_DIF 0.03
#define REA_ALIGN_CUTOFF 32 #define REA_ALIGN_CUTOFF 32
#define CHUNK_SIZE 500000000 #define CHUNK_SIZE 250000000
#define generic_key(x) (x) #define generic_key(x) (x)
KRADIX_SORT_INIT(gfa64, uint64_t, generic_key, 8) KRADIX_SORT_INIT(gfa64, uint64_t, generic_key, 8)
@@ -399,6 +399,7 @@ typedef struct { // data structure for each step in kt_pipeline()
gdpchain_t *gdp; gdpchain_t *gdp;
// glchain_t *sec_ll; // glchain_t *sec_ll;
uint64_t num_bases, num_corrected_bases, num_recorrected_bases; uint64_t num_bases, num_corrected_bases, num_recorrected_bases;
int64_t n_thread;
} utepdat_t; } utepdat_t;
@@ -4941,10 +4942,12 @@ void update_ul_vec_t_ug(const ul_idx_t *uref, ul_vec_t *rch, vec_mg_lchain_t *uc
if(rch->bb.a[k].pidx == (uint32_t)-1) k = -1; if(rch->bb.a[k].pidx == (uint32_t)-1) k = -1;
else k = rch->bb.a[k].pidx; else k = rch->bb.a[k].pidx;
} }
// fprintf(stderr, "-ulid->%ld\n", ulid);
rch->dd = 0; rch->dd = 0;
if(sp != (uint32_t)-1) l += ep - sp; if(sp != (uint32_t)-1) l += ep - sp;
l = (int64_t)rch->rlen - l; l = (int64_t)rch->rlen - l;
// if(ulid == 1756) fprintf(stderr, "-ulid:%ld, l:%ld, rch->rlen:%u\n", ulid, l, rch->rlen);
if(l == 0) { if(l == 0) {
rch->dd = 1; rch->dd = 1;
} else if(l < ((int64_t)rch->rlen)*0.001) { } else if(l < ((int64_t)rch->rlen)*0.001) {
@@ -5053,7 +5056,7 @@ int64_t debug_i, int64_t tid, void *km)
f = check_trans_rate_gap(&(gdp->swap), &(ll->tk), olist, hap, uref, diff_ec_ul, winLen, G_CHAIN_TRANS_RATE); f = check_trans_rate_gap(&(gdp->swap), &(ll->tk), olist, hap, uref, diff_ec_ul, winLen, G_CHAIN_TRANS_RATE);
} }
} }
// fprintf(stderr, "(beg2) [M::%s] debug_i:%ld, qlen:%ld\n", __func__, debug_i, qlen); // if(debug_i == 1756) fprintf(stderr, "[M::%s] ulid:%ld, qlen:%ld, f:%ld\n", __func__, debug_i, qlen, f);
if(f) update_ul_vec_t_ug(uref, rch, &(gdp->swap), debug_i); if(f) update_ul_vec_t_ug(uref, rch, &(gdp->swap), debug_i);
// debug_ul_vec_t_chain(km, uref->ug->g, rch, &(gdp->dst_done), &(gdp->out)); // debug_ul_vec_t_chain(km, uref->ug->g, rch, &(gdp->dst_done), &(gdp->out));
// fprintf(stderr, "(beg3) [M::%s::tid:%ld] debug_i:%ld, qlen:%ld\n", __func__, tid, debug_i, qlen); // fprintf(stderr, "(beg3) [M::%s::tid:%ld] debug_i:%ld, qlen:%ld\n", __func__, tid, debug_i, qlen);
@@ -5080,6 +5083,35 @@ uint64_t kv_ul_ov_t_statistics(kv_ul_ov_t *olist, uint64_t qn, int64_t *occ)
return l; return l;
} }
#define kv_mem(v) ((v).m * sizeof(*((v).a)))
int64_t get_utepdat_t_mem_tid(const utepdat_t *b, int64_t tid, int64_t *mem, int64_t *mem_hab)
{
km_stat_t kmst; memset(mem, 0, sizeof(*(mem))*6);
if(b->hab) {
mem[0] += ha_ovec_mem(b->hab[tid], mem_hab);
}
if(b->ll) {
mem[1] += kv_mem(b->ll[tid].lo) + kv_mem(b->ll[tid].tk) + kv_mem(b->ll[tid].srt.a);
}
if(b->buf) {
km_stat(b->buf[tid]->km, &kmst);
mem[2] += kmst.capacity;
}
if(b->gdp) {
mem[3] += kv_mem(b->gdp[tid].l) + kv_mem(b->gdp[tid].swap) + kv_mem(b->gdp[tid].dst) + kv_mem(b->gdp[tid].out)
+ kv_mem(b->gdp[tid].path) + kv_mem(b->gdp[tid].v) + kv_mem(b->gdp[tid].f) + kv_mem(b->gdp[tid].dst_done);
}
if(b->mzs) {
mem[4] += kv_mem(b->mzs[tid]);
}
if(b->sps) {
mem[5] += kv_mem(b->sps[tid]);
}
return mem[0] + mem[1] + mem[2] + mem[3] + mem[4] + mem[5];
}
static void worker_for_ul_scall_alignment(void *data, long i, int tid) // callback for kt_for() static void worker_for_ul_scall_alignment(void *data, long i, int tid) // callback for kt_for()
{ {
utepdat_t *s = (utepdat_t*)data; utepdat_t *s = (utepdat_t*)data;
@@ -5094,7 +5126,7 @@ static void worker_for_ul_scall_alignment(void *data, long i, int tid) // callba
// if (memcmp(UL_INF.nid.a[s->id+i].a, "d0aab024-b3a7-40fb-83cc-22c3d6d951f8", UL_INF.nid.a[s->id+i].n-1)) return; // if (memcmp(UL_INF.nid.a[s->id+i].a, "d0aab024-b3a7-40fb-83cc-22c3d6d951f8", UL_INF.nid.a[s->id+i].n-1)) return;
// fprintf(stderr, "[M::%s::] ==> len: %lu\n", __func__, s->len[i]); // fprintf(stderr, "[M::%s::] ==> len: %lu\n", __func__, s->len[i]);
ha_get_ul_candidates_interface(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres, ha_get_ul_candidates_interface(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres,
s->opt->max_n_chain, 1, &(b->k_flag), &b->r_buf, &(b->tmp_region), NULL, &(b->sp), asm_opt.hom_cov, km); s->opt->max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(b->tmp_region), NULL, &(b->sp), asm_opt.hom_cov, km);
clear_Cigar_record(&b->cigar1); clear_Cigar_record(&b->cigar1);
clear_Round2_alignment(&b->round2); clear_Round2_alignment(&b->round2);
@@ -5146,74 +5178,85 @@ static void worker_for_ul_scall_alignment(void *data, long i, int tid) // callba
} }
} }
static void worker_for_ul_rescall_alignment(void *data, long i, int tid) // callback for kt_for() static void worker_for_ul_rescall_alignment(void *data, long i, int tid) // callback for kt_for()
{ {
utepdat_t *s = (utepdat_t*)data; utepdat_t *s = (utepdat_t*)data;
ha_ovec_buf_t *b = s->hab[tid]; ha_ovec_buf_t *b = s->hab[tid];
glchain_t *bl = &(s->ll[tid]); glchain_t *bl = &(s->ll[tid]);
int64_t /**rid = s->id+i,**/ winLen = MIN((((double)THRESHOLD_MAX_SIZE)/s->opt->diff_ec_ul), WINDOW); int64_t /**rid = s->id+i,**/ winLen = MIN((((double)THRESHOLD_MAX_SIZE)/s->opt->diff_ec_ul), WINDOW);
// uint64_t align = 0; // uint64_t align = 0;
int fully_cov, abnormal; int fully_cov, abnormal;
// void *km = s->buf?(s->buf[tid]?s->buf[tid]->km:NULL):NULL; assert(UL_INF.a[s->id+i].rlen == s->len[i]);
// if(s->id+i!=873) return; // void *km = s->buf?(s->buf[tid]?s->buf[tid]->km:NULL):NULL;
// fprintf(stderr, "\n[M::%s] rid:%ld, len:%lu\n", __func__, s->id+i, s->len[i]); // if(s->id+i!=873) return;
// if (memcmp(UL_INF.nid.a[s->id+i].a, "d0aab024-b3a7-40fb-83cc-22c3d6d951f8", UL_INF.nid.a[s->id+i].n-1)) return; // fprintf(stderr, "\n[M::%s] rid:%ld, len:%lu\n", __func__, s->id+i, s->len[i]);
// fprintf(stderr, "[M::%s::] ==> len: %lu\n", __func__, s->len[i]); // if (memcmp(UL_INF.nid.a[s->id+i].a, "d0aab024-b3a7-40fb-83cc-22c3d6d951f8", UL_INF.nid.a[s->id+i].n-1)) return;
ha_get_ul_candidates_interface(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres, // fprintf(stderr, "[M::%s::] ==> len: %lu\n", __func__, s->len[i]);
s->opt->max_n_chain, 1, &(b->k_flag), &b->r_buf, &(b->tmp_region), NULL, &(b->sp), 1, NULL); ha_get_ul_candidates_interface(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres,
s->opt->max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(b->tmp_region), NULL, &(b->sp), 1, NULL);
clear_Cigar_record(&b->cigar1);
clear_Round2_alignment(&b->round2); clear_Cigar_record(&b->cigar1);
// return; clear_Round2_alignment(&b->round2);
// b->num_correct_base += overlap_statistics(&b->olist, NULL, 0); // return;
// b->num_correct_base += overlap_statistics(&b->olist, NULL, 0);
b->self_read.seq = s->seq[i]; b->self_read.length = s->len[i]; b->self_read.size = 0; b->self_read.seq = s->seq[i]; b->self_read.length = s->len[i]; b->self_read.size = 0;
correct_ul_overlap(&b->olist, s->uu, &b->self_read, &b->correct, &b->ovlp_read, &b->POA_Graph, &b->DAGCon, correct_ul_overlap(&b->olist, s->uu, &b->self_read, &b->correct, &b->ovlp_read, &b->POA_Graph, &b->DAGCon,
&b->cigar1, &b->hap, &b->round2, 0, 1, &fully_cov, &abnormal, s->opt->diff_ec_ul, winLen, NULL); &b->cigar1, &b->hap, &b->round2, 0, 1, &fully_cov, &abnormal, s->opt->diff_ec_ul, winLen, NULL);
// uint64_t k; // uint64_t k;
// for (k = 0; k < b->olist.length; k++) { // for (k = 0; k < b->olist.length; k++) {
// if(b->olist.list[k].is_match == 1) b->num_correct_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s; // if(b->olist.list[k].is_match == 1) b->num_correct_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s;
// if(b->olist.list[k].is_match == 2) b->num_recorrect_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s; // if(b->olist.list[k].is_match == 2) b->num_recorrect_base += b->olist.list[k].x_pos_e+1-b->olist.list[k].x_pos_s;
// } // }
// gl_chain_refine(&b->olist, &b->correct, &b->hap, bl, s->uu, s->opt->diff_ec_ul, winLen, s->len[i], km); // gl_chain_refine(&b->olist, &b->correct, &b->hap, bl, s->uu, s->opt->diff_ec_ul, winLen, s->len[i], km);
gl_chain_refine_advance_combine(s->buf[tid], &(UL_INF.a[i]), &b->olist, &b->correct, &b->hap, &(s->sps[tid]), bl, &(s->gdp[tid]), s->uu, s->opt->diff_ec_ul, winLen, s->len[i], s->uopt, s->id+i, tid, NULL); gl_chain_refine_advance_combine(s->buf[tid], &(UL_INF.a[s->id+i]), &b->olist, &b->correct, &b->hap, &(s->sps[tid]), bl, &(s->gdp[tid]), s->uu, s->opt->diff_ec_ul, winLen, s->len[i], s->uopt, s->id+i, tid, NULL);
// return; // return;
// b->num_read_base += b->self_read.length; // b->num_read_base += b->self_read.length;
// b->num_correct_base += b->correct.corrected_base; // b->num_correct_base += b->correct.corrected_base;
// b->num_recorrect_base += b->round2.dumy.corrected_base; // b->num_recorrect_base += b->round2.dumy.corrected_base;
memset(&b->self_read, 0, sizeof(b->self_read)); memset(&b->self_read, 0, sizeof(b->self_read));
if(UL_INF.a[i].dd) { if(UL_INF.a[s->id+i].dd) {
free(s->seq[i]); s->seq[i] = NULL; b->num_correct_base++; free(s->seq[i]); s->seq[i] = NULL; b->num_correct_base++;
} }
s->hab[tid]->num_read_base++; s->hab[tid]->num_read_base++;
// align = kv_ul_ov_t_statistics(&(bl->tk), i, &(b->num_recorrect_base)); int64_t mem[6], mem_hab[6];
// if(align == s->len[i]) { if(get_utepdat_t_mem_tid(s, tid, mem, mem_hab)>((int64_t)5*(int64_t)1073741824)) {
// free(s->seq[i]); s->seq[i] = NULL; fprintf(stderr, "[M::%s::tid->%d::rid->%ld] buffer[0]: %.3fGB(%.3fGB::%.3fGB::%.3fGB::%.3fGB::%.3fGB), buffer[1]: %.3fGB, buffer[2]: %.3fGB, buffer[3]: %.3fGB, buffer[4]: %.3fGB, buffer[5]: %.3fGB\n",
// } __func__, tid, i, mem[0]/1073741824.0,
// b->num_correct_base += align; mem_hab[0]/1073741824.0, mem_hab[1]/1073741824.0, mem_hab[2]/1073741824.0,
mem_hab[3]/1073741824.0, mem_hab[4]/1073741824.0,
mem[1]/1073741824.0, mem[2]/1073741824.0,
mem[3]/1073741824.0, mem[4]/1073741824.0, mem[5]/1073741824.0);
}
// uint64_t k; // align = kv_ul_ov_t_statistics(&(bl->tk), i, &(b->num_recorrect_base));
// b->num_read_base += overlap_statistics(&b->olist, NULL, NULL, 1); // if(align == s->len[i]) {
// for (k = 0; k < bl->tk.n; k++) { // free(s->seq[i]); s->seq[i] = NULL;
// if(bl->tk.a[k].sec == 0) b->num_correct_base += bl->tk.a[k].qe - bl->tk.a[k].qs; // }
// if(bl->tk.a[k].sec > 0) b->num_recorrect_base += bl->tk.a[k].qe - bl->tk.a[k].qs; // b->num_correct_base += align;
// }
// for (k = 0; k < bl->lo.n; k++) {
// b->num_read_base += bl->lo.a[k].qe - bl->lo.a[k].qs;
// }
// uint32_t l1 = overlap_statistics(&b->olist, s->uu->ug, 1), l2 = overlap_statistics(&b->olist, s->uu->ug, 2); // uint64_t k;
// b->num_read_base += overlap_statistics(&b->olist, NULL, NULL, 1);
// for (k = 0; k < bl->tk.n; k++) {
// if(bl->tk.a[k].sec == 0) b->num_correct_base += bl->tk.a[k].qe - bl->tk.a[k].qs;
// if(bl->tk.a[k].sec > 0) b->num_recorrect_base += bl->tk.a[k].qe - bl->tk.a[k].qs;
// }
// for (k = 0; k < bl->lo.n; k++) {
// b->num_read_base += bl->lo.a[k].qe - bl->lo.a[k].qs;
// }
// uint32_t l1 = overlap_statistics(&b->olist, s->uu->ug, 1), l2 = overlap_statistics(&b->olist, s->uu->ug, 2);
// //
// if(l1 == 0 && l2 > 0) fprintf(stderr, "[M::%s::%lu::no_match]\n", UL_INF.nid.a[s->id+i].a, s->len[i]); // if(l1 == 0 && l2 > 0) fprintf(stderr, "[M::%s::%lu::no_match]\n", UL_INF.nid.a[s->id+i].a, s->len[i]);
// fprintf(stderr, "[M::%s::%lu::] l1->%u; l2->%u\n", UL_INF.nid.a[s->id+i].a, s->len[i], l1, l2); // fprintf(stderr, "[M::%s::%lu::] l1->%u; l2->%u\n", UL_INF.nid.a[s->id+i].a, s->len[i], l1, l2);
// fprintf(stderr, "[M::%s::rid->%ld] done\n", __func__, s->id+i); // fprintf(stderr, "[M::%s::rid->%ld] done\n", __func__, s->id+i);
} }
void dump_gaf(mg_gres_a *hits, const mg_gchains_t *gs, uint32_t only_p) void dump_gaf(mg_gres_a *hits, const mg_gchains_t *gs, uint32_t only_p)
{ {
if (gs == NULL || gs->n_gc == 0 || gs->n_lc == 0) return; if (gs == NULL || gs->n_gc == 0 || gs->n_lc == 0) return;
@@ -5262,6 +5305,27 @@ void dump_gaf(mg_gres_a *hits, const mg_gchains_t *gs, uint32_t only_p)
} }
} }
int64_t get_utepdat_t_mem(const utepdat_t *b, int64_t is_print)
{
int64_t i, mem[7] = {0}, tt[6] = {0};
for (i = 0; i < b->n_thread; i++) {
get_utepdat_t_mem_tid(b, i, tt, NULL);
mem[0] += tt[0]; mem[1] += tt[1]; mem[2] += tt[2]; mem[3] += tt[3]; mem[4] += tt[4]; mem[5] += tt[5];
}
for (i = 0; i < b->n; ++i) mem[6] += b->len[i];
mem[6] += (sizeof(*(b->len))*b->n) + (sizeof(*(b->seq))*b->n);
if(is_print) {
for (i = 0; i < 7; i++) {
fprintf(stderr, "[M::%s] size of buffer[%ld]: %.3fGB, %.3fKB, %ldB\n",
__func__, i, mem[i]/1073741824.0, mem[i]/1048576.0, mem[i]);
}
}
return mem[0] + mem[1] + mem[2] + mem[3] + mem[4] + mem[5] + mem[6];
}
static void *worker_ul_pipeline(void *data, int step, void *in) // callback for kt_pipeline() static void *worker_ul_pipeline(void *data, int step, void *in) // callback for kt_pipeline()
{ {
uldat_t *p = (uldat_t*)data; uldat_t *p = (uldat_t*)data;
@@ -5536,7 +5600,7 @@ static void *worker_ul_rescall_pipeline(void *data, int step, void *in) // callb
else if (step == 1) { // step 2: alignment else if (step == 1) { // step 2: alignment
utepdat_t *s = (utepdat_t*)in; utepdat_t *s = (utepdat_t*)in;
uint64_t i; uint64_t i; s->n_thread = p->n_thread;
CALLOC(s->hab, p->n_thread); CALLOC(s->ll, p->n_thread); CALLOC(s->buf, p->n_thread); CALLOC(s->hab, p->n_thread); CALLOC(s->ll, p->n_thread); CALLOC(s->buf, p->n_thread);
CALLOC(s->gdp, p->n_thread); CALLOC(s->mzs, p->n_thread); CALLOC(s->sps, p->n_thread); CALLOC(s->gdp, p->n_thread); CALLOC(s->mzs, p->n_thread); CALLOC(s->sps, p->n_thread);
@@ -5547,9 +5611,12 @@ static void *worker_ul_rescall_pipeline(void *data, int step, void *in) // callb
fprintf(stderr, "[M::%s::Start] ==> s->id: %lu, s->n:% d\n", __func__, s->id, s->n); fprintf(stderr, "[M::%s::Start] ==> s->id: %lu, s->n:% d\n", __func__, s->id, s->n);
kt_for(p->n_thread, worker_for_ul_rescall_alignment, s, s->n); kt_for(p->n_thread, worker_for_ul_rescall_alignment, s, s->n);
fprintf(stderr, "[M::%s::Done] ==> s->id: %lu, s->n:% d\n", __func__, s->id, s->n); fprintf(stderr, "[M::%s::Done] ==> s->id: %lu, s->n:% d\n", __func__, s->id, s->n);
get_utepdat_t_mem(s, 1);
for (i = 0; i < p->n_thread; ++i) { for (i = 0; i < p->n_thread; ++i) {
p->num_bases += s->hab[i]->num_read_base; p->num_bases += s->hab[i]->num_read_base;
p->num_corrected_bases += s->hab[i]->num_correct_base; p->num_corrected_bases += s->hab[i]->num_correct_base;
// s->num_recorrected_bases += s->hab[i]->num_recorrect_base; // s->num_recorrected_bases += s->hab[i]->num_recorrect_base;
ha_ovec_destroy(s->hab[i]); hc_glchain_destroy(&(s->ll[i])); ha_ovec_destroy(s->hab[i]); hc_glchain_destroy(&(s->ll[i]));
mg_tbuf_destroy(s->buf[i]); hc_gdpchain_destroy(&(s->gdp[i])); mg_tbuf_destroy(s->buf[i]); hc_gdpchain_destroy(&(s->gdp[i]));
@@ -5564,6 +5631,8 @@ static void *worker_ul_rescall_pipeline(void *data, int step, void *in) // callb
for (i = 0; i < s->n; ++i) { for (i = 0; i < s->n; ++i) {
rid = s->id + i; rid = s->id + i;
if(UL_INF.a[rid].dd == 0 && p->ucr_s && p->ucr_s->flag == 1) { if(UL_INF.a[rid].dd == 0 && p->ucr_s && p->ucr_s->flag == 1) {
assert(s->seq[i]);
// if(s->seq[i] == NULL) fprintf(stderr, "[M::%s::]rid->%ld, len->%lu\n", __func__, rid, s->len[i]);
write_compress_base_disk(p->ucr_s->fp, rid, s->seq[i], s->len[i], &(p->ucr_s->u)); write_compress_base_disk(p->ucr_s->fp, rid, s->seq[i], s->len[i], &(p->ucr_s->u));
} }
free(s->seq[i]); free(s->seq[i]);
@@ -8540,11 +8609,12 @@ int scall_ul_pipeline(uldat_t* sl, const enzyme *fn)
int rescall_ul_pipeline(uldat_t* sl, const enzyme *fn) int rescall_ul_pipeline(uldat_t* sl, const enzyme *fn)
{ {
double index_time = yak_realtime(); double index_time = yak_realtime();
int32_t i; uint32_t k; int32_t i; uint32_t k, rlen;
///clean UL_INF ///clean UL_INF
for (k = 0; k < UL_INF.n; k++) { for (k = 0; k < UL_INF.n; k++) {
rlen = UL_INF.a[k].rlen;
free(UL_INF.a[k].bb.a); free(UL_INF.a[k].N_site.a); free(UL_INF.a[k].r_base.a); free(UL_INF.a[k].bb.a); free(UL_INF.a[k].N_site.a); free(UL_INF.a[k].r_base.a);
memset(&(UL_INF.a[k]), 0, sizeof(UL_INF.a[k])); memset(&(UL_INF.a[k]), 0, sizeof(UL_INF.a[k])); UL_INF.a[k].rlen = rlen;
} }
for (i = 0; i < fn->n; i++){ for (i = 0; i < fn->n; i++){