diff --git a/Assembly.cpp b/Assembly.cpp index 75d706f..f7f65ab 100644 --- a/Assembly.cpp +++ b/Assembly.cpp @@ -848,7 +848,8 @@ void ha_overlap_and_correct(int round) b[i] = ha_ovec_init(0, (round == asm_opt.number_of_round - 1)); if(ha_idx) hom_cov = asm_opt.hom_cov; if(ha_idx == NULL) ha_idx = ha_pt_gen(&asm_opt, ha_flt_tab, round == 0? 0 : 1, 0, &R_INF, &hom_cov, &het_cov); // build the index - if (round == 0 && ha_flt_tab == 0) // then asm_opt.hom_cov hasn't been updated + ///debug_adapter(&asm_opt, &R_INF); + if (round == 0 && ha_flt_tab == 0) // then asm_opt.hom_cov hasn't been updated ha_opt_update_cov(&asm_opt, hom_cov); if (asm_opt.required_read_name) kt_for(asm_opt.thread_num, worker_ovec_related_reads, b, R_INF.total_reads); @@ -1600,6 +1601,8 @@ void ha_overlap_final(void) asm_opt.het_cov = het_cov; } + + int ha_assemble(void) { extern void ha_extract_print_list(const All_reads *rs, int n_rounds, const char *o); diff --git a/CommandLines.cpp b/CommandLines.cpp index 85a37e6..0919583 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -26,6 +26,12 @@ static ko_longopt_t long_options[] = { { "high-het", ko_no_argument, 311 }, { "lowQ", ko_required_argument, 312 }, { "min-hist-cnt", ko_required_argument, 313 }, + { "h1", ko_required_argument, 314 }, + { "h2", ko_required_argument, 315 }, + { "enzyme", ko_required_argument, 316 }, + { "b-cov", ko_required_argument, 317 }, + { "h-cov", ko_required_argument, 318 }, + { "m-rate", ko_required_argument, 319 }, { 0, 0, 0 } }; @@ -63,6 +69,14 @@ void Print_H(hifiasm_opt_t* asm_opt) fprintf(stderr, " -u disable post join contigs step which may improve N50\n"); fprintf(stderr, " --lowQ INT\n"); fprintf(stderr, " output contig regions with >=INT%% inconsistency in BED format; 0 to disable [%d]\n", asm_opt->bed_inconsist_rate); + fprintf(stderr, " --b-cov INT\n"); + fprintf(stderr, " break contigs at positions with b_low_cov); + fprintf(stderr, " --h-cov INT\n"); + fprintf(stderr, " break contigs at positions with >INT-fold coverage; work with '--m-rate'; -1 to disable [%d]\n", asm_opt->b_high_cov); + fprintf(stderr, " --m-rate FLOAT\n"); + fprintf(stderr, " break contigs at positions with <=FLOAT*coverage exact overlaps;\n"); + fprintf(stderr, " only work with '--b-cov' or '--h-cov'[%.2f]\n", asm_opt->m_rate); + // fprintf(stderr, " --pri-range INT1[,INT2]\n"); // fprintf(stderr, " keep contigs with coverage in this range in p_ctg.gfa; -1 to disable [auto,inf]\n"); @@ -84,6 +98,9 @@ void Print_H(hifiasm_opt_t* asm_opt) fprintf(stderr, " coverage upper bound of Purge-dups [auto]\n"); fprintf(stderr, " --high-het enable this mode for high heterozygosity sample [experimental, not stable]\n"); + fprintf(stderr, " Hi-C-partition [experimental, not stable]:\n"); + fprintf(stderr, " --h1 FILEs file names of Hi-C R1 [r1_1.fq,r1_2.fq,...]\n"); + fprintf(stderr, " --h2 FILEs file names of Hi-C R2 [r2_1.fq,r2_2.fq,...]\n"); fprintf(stderr, "Example: ./hifiasm -o NA12878.asm -t 32 NA12878.fq.gz\n"); fprintf(stderr, "See `man ./hifiasm.1' for detailed description of these command-line options.\n"); @@ -98,8 +115,12 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->read_file_names = NULL; asm_opt->output_file_name = (char*)(DEFAULT_OUTPUT); asm_opt->required_read_name = NULL; + asm_opt->hic_enzymes = NULL; + asm_opt->hic_reads[0] = NULL; + asm_opt->hic_reads[1] = NULL; asm_opt->thread_num = 1; asm_opt->k_mer_length = 51; + asm_opt->hic_mer_length = 31; asm_opt->mz_win = 51; asm_opt->mz_sample_dist = 500; asm_opt->bf_shift = 37; @@ -131,19 +152,42 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->purge_level_primary = 2; asm_opt->purge_level_trio = 0; asm_opt->purge_simi_rate = 0.75; + asm_opt->purge_simi_rate_hic = 0.85; asm_opt->purge_overlap_len = 1; + asm_opt->purge_overlap_len_hic = 50; asm_opt->recover_atg_cov_min = -1024; asm_opt->recover_atg_cov_max = INT_MAX; asm_opt->hom_global_coverage = -1; asm_opt->bed_inconsist_rate = 70; + asm_opt->hic_inconsist_rate = 30; + ///asm_opt->bub_mer_length = 3; + asm_opt->bub_mer_length = 1000000; + asm_opt->b_low_cov = 0; + asm_opt->b_high_cov = -1; + asm_opt->m_rate = 0.75; +} + +void destory_enzyme(enzyme* f) +{ + int i; + if(f != NULL) + { + for (i = 0; i < f->n; i++) + { + free(f->a[i]); + } + free(f->a); + free(f->l); + free(f); + } } void destory_opt(hifiasm_opt_t* asm_opt) { - if(asm_opt->read_file_names != NULL) - { - free(asm_opt->read_file_names); - } + if(asm_opt->read_file_names != NULL) free(asm_opt->read_file_names); + if(asm_opt->hic_enzymes != NULL) destory_enzyme(asm_opt->hic_enzymes); + if(asm_opt->hic_reads[0] != NULL) destory_enzyme(asm_opt->hic_reads[0]); + if(asm_opt->hic_reads[1] != NULL) destory_enzyme(asm_opt->hic_reads[1]); } void ha_opt_reset_to_round(hifiasm_opt_t* asm_opt, int round) @@ -183,6 +227,16 @@ static int check_file(char* name, const char* opt) return 1; } +static int check_hic_reads(enzyme* f, const char* opt) +{ + int i; + for (i = 0; i < f->n; i++) + { + if(check_file(f->a[i], opt) == 0) return 0; + } + return 1; +} + int check_option(hifiasm_opt_t* asm_opt) { if(asm_opt->read_file_names == NULL || asm_opt->num_reads == 0) @@ -327,7 +381,7 @@ int check_option(hifiasm_opt_t* asm_opt) if(asm_opt->bed_inconsist_rate < 0 || asm_opt->bed_inconsist_rate > 100) { - fprintf(stderr, "[ERROR] inconsistency rate should be [0, 100] (--pb-range)\n"); + fprintf(stderr, "[ERROR] inconsistency rate should be [0, 100] (--lowQ)\n"); return 0; } @@ -337,6 +391,69 @@ int check_option(hifiasm_opt_t* asm_opt) if(asm_opt->fn_bin_list[0] != NULL && check_file(asm_opt->fn_bin_list[0], "LIST1") == 0) return 0; if(asm_opt->fn_bin_list[1] != NULL && check_file(asm_opt->fn_bin_list[1], "LIST2") == 0) return 0; if(asm_opt->required_read_name != NULL && check_file(asm_opt->required_read_name, "b") == 0) return 0; + + if(asm_opt->hic_reads[0] != NULL && check_hic_reads(asm_opt->hic_reads[0], "HIC1") == 0) return 0; + if(asm_opt->hic_reads[1] != NULL && check_hic_reads(asm_opt->hic_reads[1], "HIC2") == 0) return 0; + if(asm_opt->hic_reads[0] != NULL && asm_opt->hic_reads[1] == NULL) + { + fprintf(stderr, "[ERROR] lack r2 of HiC reads (--h2)\n"); + return 0; + } + if(asm_opt->hic_reads[1] != NULL && asm_opt->hic_reads[0] == NULL) + { + fprintf(stderr, "[ERROR] lack r1 of HiC reads (--h1)\n"); + return 0; + } + + if(asm_opt->hic_reads[0] != NULL && asm_opt->hic_reads[1] != NULL && + asm_opt->hic_reads[0]->n != asm_opt->hic_reads[1]->n) + { + fprintf(stderr, "[ERROR] wrong r1 and r2 of HiC reads (--h1 && --h2)\n"); + return 0; + } + + if(asm_opt->hic_enzymes != NULL && asm_opt->hic_enzymes->n == 0) + { + fprintf(stderr, "[ERROR] wrong HiC enzymes (--enzyme)\n"); + return 0; + } + + if(asm_opt->hic_reads[0] != NULL && asm_opt->hic_reads[0]->n == 0) + { + fprintf(stderr, "[ERROR] wrong r1 of HiC reads (--h1)\n"); + return 0; + } + + if(asm_opt->hic_reads[1] != NULL && asm_opt->hic_reads[1]->n == 0) + { + fprintf(stderr, "[ERROR] wrong r2 of HiC reads (--h2)\n"); + return 0; + } + + if(asm_opt->b_low_cov < 0) + { + fprintf(stderr, "[ERROR] must >= 0 (--b-cov)\n"); + return 0; + } + + if(asm_opt->b_high_cov != -1 && asm_opt->b_high_cov < 0) + { + fprintf(stderr, "[ERROR] must >= 0 (--h-cov)\n"); + return 0; + } + + if(asm_opt->m_rate < 0) + { + fprintf(stderr, "[ERROR] must >= 0 (--m-rate)\n"); + return 0; + } + + if(asm_opt->b_high_cov != -1 && asm_opt->b_high_cov <= asm_opt->b_low_cov) + { + fprintf(stderr, "[ERROR] [--h-cov] must >= [--b-cov]\n"); + return 0; + } + // fprintf(stderr, "input file num: %d\n", asm_opt->num_reads); // fprintf(stderr, "output file: %s\n", asm_opt->output_file_name); // fprintf(stderr, "number of threads: %d\n", asm_opt->thread_num); @@ -386,6 +503,61 @@ void get_queries(int argc, char *argv[], ketopt_t* opt, hifiasm_opt_t* asm_opt) } } +void get_hic_enzymes(char *argv, enzyme** x, int check_name) +{ + int i, k, pre_i, len = strlen(argv); + (*x) = (enzyme*)calloc(1, sizeof(enzyme)); + if(len == 0) + { + (*x)->n = 0; (*x)->l = NULL; (*x)->a = NULL; + return; + } + + + (*x)->n = 1; + for (i = pre_i = 0; i < len; i++) + { + if(argv[i] == ',') + { + (*x)->n++; + continue; + } + + if(check_name) + { + if(argv[i] != 'A' && argv[i] != 'C' && argv[i] != 'G' && argv[i] != 'T' && + argv[i] != 'a' && argv[i] != 'c' && argv[i] != 'g' && argv[i] != 't' && + argv[i] != 'N' && argv[i] != 'n') + { + (*x)->n = 0; + (*x)->l = NULL; + (*x)->a = NULL; + return; + } + } + + } + (*x)->l = (int*)calloc((*x)->n, sizeof(int)); + (*x)->a = (char**)calloc((*x)->n, sizeof(char*)); + + for (i = pre_i = k = 0; i < len; i++) + { + if(argv[i] == ',') + { + (*x)->l[k] = i - pre_i; + (*x)->a[k] = (char*)malloc(sizeof(char)*((*x)->l[k]+1)); + memcpy((*x)->a[k], argv + pre_i, (*x)->l[k]); + (*x)->a[k][(*x)->l[k]] = '\0'; + pre_i = i + 1; + k++; + } + } + + (*x)->l[k] = i - pre_i; + (*x)->a[k] = (char*)malloc(sizeof(char)*((*x)->l[k]+1)); + memcpy((*x)->a[k], argv + pre_i, (*x)->l[k]); + (*x)->a[k][(*x)->l[k]] = '\0'; +} int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) { @@ -451,7 +623,13 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) } else if (c == 311) asm_opt->flag |= HA_F_HIGH_HET; else if (c == 312) asm_opt->bed_inconsist_rate = atoi(opt.arg); - else if (c == 313) asm_opt->min_hist_kmer_cnt = atoi(opt.arg); + else if (c == 313) asm_opt->min_hist_kmer_cnt = atoi(opt.arg); + else if (c == 314) get_hic_enzymes(opt.arg, &(asm_opt->hic_reads[0]), 0); + else if (c == 315) get_hic_enzymes(opt.arg, &(asm_opt->hic_reads[1]), 0); + else if (c == 316) get_hic_enzymes(opt.arg, &(asm_opt->hic_enzymes), 1); + else if (c == 317) asm_opt->b_low_cov = atoi(opt.arg); + else if (c == 318) asm_opt->b_high_cov = atoi(opt.arg); + else if (c == 319) asm_opt->m_rate = atof(opt.arg); else if (c == 'l') { ///0: disable purge_dup; 1: purge containment; 2: purge overlap asm_opt->purge_level_primary = asm_opt->purge_level_trio = atoi(opt.arg); @@ -470,7 +648,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) } } - + if (argc == opt.ind) { Print_H(asm_opt); @@ -479,5 +657,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt) get_queries(argc, argv, &opt, asm_opt); + + return check_option(asm_opt); } diff --git a/CommandLines.h b/CommandLines.h index d7e753b..a59fc70 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -3,7 +3,7 @@ #include -#define HA_VERSION "0.13-r334-dirty" +#define HA_VERSION "0.14-r312" #define VERBOSE 0 @@ -21,6 +21,11 @@ #define HA_MIN_OV_DIFF 0.02 // min sequence divergence in an overlap +typedef struct{ + int *l, n; + char **a; +}enzyme; + typedef struct { int flag; int num_reads; @@ -30,9 +35,13 @@ typedef struct { char *fn_bin_yak[2]; char *fn_bin_list[2]; char *extract_list; + enzyme *hic_reads[2]; + enzyme *hic_enzymes; int extract_iter; int thread_num; int k_mer_length; + int hic_mer_length; + int bub_mer_length; int mz_win; int mz_sample_dist; int bf_shift; @@ -42,6 +51,9 @@ typedef struct { double max_ov_diff_final; int hom_cov; int het_cov; + int b_low_cov; + int b_high_cov; + double m_rate; int max_n_chain; // fall-back max number of chains to consider int min_hist_kmer_cnt; int load_index_from_disk; @@ -60,15 +72,18 @@ typedef struct { int purge_level_primary; int purge_level_trio; int purge_overlap_len; + int purge_overlap_len_hic; int recover_atg_cov_min; int recover_atg_cov_max; int hom_global_coverage; int bed_inconsist_rate; + int hic_inconsist_rate; float max_hang_rate; float min_drop_rate; float max_drop_rate; float purge_simi_rate; + float purge_simi_rate_hic; long long small_pop_bubble_size; long long large_pop_bubble_size; @@ -94,4 +109,9 @@ static inline int ha_opt_triobin(const hifiasm_opt_t *opt) return ((opt->fn_bin_yak[0] && opt->fn_bin_yak[1]) || (opt->fn_bin_list[0] && opt->fn_bin_list[1])); } +static inline int ha_opt_hic(const hifiasm_opt_t *opt) +{ + return ((opt->hic_reads[0] && opt->hic_reads[1])); +} + #endif diff --git a/Correct.h b/Correct.h index 452cd29..9f09283 100644 --- a/Correct.h +++ b/Correct.h @@ -21,8 +21,8 @@ ///#define FLAG_THRE 0 -#define MAX(x, y) ((x >= y)?(x):(y)) -#define MIN(x, y) ((x <= y)?(x):(y)) +#define MAX(x, y) (((x) >= (y))?(x):(y)) +#define MIN(x, y) (((x) <= (y))?(x):(y)) #define DIFF(x, y) ((MAX((x), (y))) - (MIN((x), (y)))) #define OVERLAP(x_start, x_end, y_start, y_end) (MIN(x_end, y_end) - MAX(x_start, y_start) + 1) ///#define OVERLAP(x_start, x_end, y_start, y_end) MIN(x_end, y_end) - MAX(x_start, y_start) + 1 diff --git a/Makefile b/Makefile index 7da5789..2349e50 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ CPPFLAGS= INCLUDES= OBJS= CommandLines.o Process_Read.o Assembly.o Hash_Table.o \ POA.o Correct.o Levenshtein_distance.o Overlaps.o Trio.o kthread.o Purge_Dups.o \ - htab.o hist.o sketch.o anchor.o extract.o sys.o ksw2_extz2_sse.o + htab.o hist.o sketch.o anchor.o extract.o sys.o ksw2_extz2_sse.o hic.o EXE= hifiasm LIBS= -lz -lpthread -lm @@ -43,7 +43,7 @@ Assembly.o: kthread.h CommandLines.o: CommandLines.h ketopt.h Correct.o: Correct.h Hash_Table.h htab.h Process_Read.h Overlaps.h kvec.h Correct.o: kdq.h CommandLines.h Levenshtein_distance.h POA.h Assembly.h -Correct.o: ksw2.h +Correct.o: ksw2.h ksort.h Hash_Table.o: Hash_Table.h htab.h Process_Read.h Overlaps.h kvec.h kdq.h Hash_Table.o: CommandLines.h ksort.h Levenshtein_distance.o: Levenshtein_distance.h @@ -71,3 +71,4 @@ main.o: CommandLines.h Process_Read.h Overlaps.h kvec.h kdq.h Assembly.h main.o: Levenshtein_distance.h htab.h sketch.o: kvec.h htab.h Process_Read.h Overlaps.h kdq.h CommandLines.h sys.o: htab.h Process_Read.h Overlaps.h kvec.h kdq.h CommandLines.h +hic.o: hic.h diff --git a/Overlaps.cpp b/Overlaps.cpp index c026767..4f45904 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -9,6 +9,7 @@ #include "Hash_Table.h" #include "Correct.h" #include "Purge_Dups.h" +#include "hic.h" uint32_t debug_purge_dup = 0; @@ -177,6 +178,7 @@ void ma_ug_destroy(ma_ug_t *ug) } free(ug->u.a); asg_destroy(ug->g); + kv_destroy(ug->occ); free(ug); } @@ -284,8 +286,6 @@ void asg_cleanup(asg_t *g) - - // delete multi-arcs /** * remove edges like: v has two out-edges to w @@ -1063,7 +1063,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp) delete_single_edge(sources, coverage_cut, Get_tn(*h), Get_qn(*h)); delete_all_edges(sources, coverage_cut, Get_qn(*h)); - set_R_to_U(ruIndex, Get_qn(*h), Get_tn(*h), 0); + set_R_to_U(ruIndex, Get_qn(*h), Get_tn(*h), 0, NULL); // if(delete_all_edges_carefully(sources, coverage_cut, max_hang, min_ovlp, // Get_qn(*h))==0) @@ -1079,7 +1079,7 @@ R_to_U* ruIndex, int max_hang, int min_ovlp) delete_single_edge(sources, coverage_cut, Get_tn(*h), Get_qn(*h)); delete_all_edges(sources, coverage_cut, Get_tn(*h)); - set_R_to_U(ruIndex, Get_tn(*h), Get_qn(*h), 0); + set_R_to_U(ruIndex, Get_tn(*h), Get_qn(*h), 0, NULL); // if(delete_all_edges_carefully(sources, coverage_cut, max_hang, // min_ovlp, Get_tn(*h)) == 0) @@ -4715,72 +4715,6 @@ int asg_cut_tip(asg_t *g, int max_ext) } -///max_ext is 4 -int asg_cut_tip_primary(asg_t *g, ma_ug_t *ug, int max_ext) -{ - double startTime = Get_T(); - - asg64_v a = {0,0,0}; - uint32_t n_vtx = g->n_seq * 2, v, i, cnt = 0, tipEvaluateLen; - - for (v = 0; v < n_vtx; ++v) { - //if this seq has been deleted - if (g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - ///(v^1) is end node of a unitig - if (asg_is_utg_end(g, v, 0) != ASG_ET_TIP) continue; // not a tip - /** - the following second line is: - (v)--->()---->()---->()----->() - |--------max_ext-------| - **/ - ///that means here is a long tip, which is longer than max_ext - if (asg_extend(g, v, max_ext, &a) == ASG_ET_MERGEABLE) continue; // not a short unitig - - /** - * so combining the last two lines, they are designed to reomve(n(0), n(1), n(2)): - * ----->n(4) - * | - * n(0)--->n(1)---->n(2)---->n(3) - * | - * ----->n(5) - **/ - - if(ug!=NULL) - { - tipEvaluateLen = 0; - for (i = 0; i < a.n; ++i) - { - tipEvaluateLen += EvaluateLen(ug->u, ((uint32_t)a.a[i]>>1)); - } - if(tipEvaluateLen > (uint32_t)max_ext) continue; - } - - for (i = 0; i < a.n; ++i) - { - g->seq[((uint32_t)a.a[i]>>1)].c = ALTER_LABLE; - } - - for (i = 0; i < a.n; ++i) - { - asg_seq_drop(g, (uint32_t)a.a[i]>>1); - } - - ++cnt; - } - free(a.a); - if (cnt > 0) asg_cleanup(g); - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] cut %d tips\n", __func__, cnt); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); - } - return cnt; -} - - - - - // delete short arcs ///for best graph? int asg_arc_del_short(asg_t *g, float drop_ratio) @@ -5257,346 +5191,6 @@ long long* baseLen, long long* max_stop_base_Len, buf_t* b, uint32_t stops_thres /*****************************read graph*****************************/ - - -/*****************************untig graph*****************************/ - -uint32_t untig_detect_single_path_with_dels(asg_t *g, ma_ug_t *ug, uint32_t begNode, -uint32_t* endNode, long long* Len, buf_t* b) -{ - - uint32_t v = begNode, w; - uint32_t kv, kw; - (*Len) = 0; - - - while (1) - { - (*Len) = (*Len) + EvaluateLen((*ug).u, v>>1); - kv = get_real_length(g, v, NULL); - (*endNode) = v; - - if(b) kv_push(uint32_t, b->b, v>>1); - - if(kv == 0) - { - return END_TIPS; - } - - if(kv == 2) - { - return TWO_OUTPUT; - } - - if(kv > 2) - { - return MUL_OUTPUT; - } - - ///up to here, kv=1 - ///kw must >= 1 - get_real_length(g, v, &w); - kw = get_real_length(g, w^1, NULL); - - if(kw == 2) - { - return TWO_INPUT; - } - - if(kw > 2) - { - return MUL_INPUT; - } - - v = w; - - if((v>>1) == (begNode>>1)) - { - return LOOP; - } - } - - return LONG_TIPS; -} - -uint32_t untig_detect_single_path_with_dels_n_stops(asg_t *g, ma_ug_t *ug, uint32_t begNode, -uint32_t* endNode, long long* Len, long long* max_stop_Len, buf_t* b, uint32_t stops_threshold) -{ - - uint32_t v = begNode, w; - uint32_t kv, kw, n_stops = 0, flag = LONG_TIPS; - (*Len) = 0; - (*max_stop_Len) = 0; - long long preLen = 0, currentLen; - - while (1) - { - (*Len) = (*Len) + EvaluateLen((*ug).u, v>>1); - kv = get_real_length(g, v, NULL); - (*endNode) = v; - - if(b) kv_push(uint32_t, b->b, v>>1); - - if(kv == 0) - { - flag = END_TIPS; - break; - } - - if(kv == 2) - { - flag = TWO_OUTPUT; - break; - } - - if(kv > 2) - { - flag = MUL_OUTPUT; - break; - } - - ///up to here, kv=1 - ///kw must >= 1 - get_real_length(g, v, &w); - kw = get_real_length(g, w^1, NULL); - - ///just calculate the max_stop_Len - if(kw >= 2) - { - n_stops++; - currentLen = (*Len) - preLen; - preLen = (*Len); - if(currentLen > (*max_stop_Len)) - { - (*max_stop_Len) = currentLen; - } - } - if(kw >= 2 && n_stops >= stops_threshold) - { - if(kw == 2) flag = TWO_INPUT; - if(kw > 2) flag = MUL_INPUT; - break; - } - - v = w; - - if((v>>1) == (begNode>>1)) - { - flag = LOOP; - break; - } - } - - - currentLen = (*Len) - preLen; - preLen = (*Len); - if(currentLen > (*max_stop_Len)) - { - (*max_stop_Len) = currentLen; - } - return flag; -} - -uint32_t untig_detect_single_path_with_dels_contigLen(asg_t *g, uint32_t begNode, uint32_t* endNode, -long long* baseLen, buf_t* b) -{ - - uint32_t v = begNode, w = 0; - uint32_t kv, kw, k; - (*baseLen) = 0; - - - while (1) - { - kv = get_real_length(g, v, NULL); - (*endNode) = v; - - if(b) kv_push(uint32_t, b->b, v>>1); - - if(kv == 0) - { - (*baseLen) += g->seq[v>>1].len; - return END_TIPS; - } - - if(kv == 2) - { - (*baseLen) += g->seq[v>>1].len; - return TWO_OUTPUT; - } - - if(kv > 2) - { - (*baseLen) += g->seq[v>>1].len; - return MUL_OUTPUT; - } - - - ///up to here, kv=1 - ///kw must >= 1 - get_real_length(g, v, &w); - kw = get_real_length(g, w^1, NULL); - - if(kw == 2) - { - (*baseLen) += g->seq[v>>1].len; - return TWO_INPUT; - } - - if(kw > 2) - { - (*baseLen) += g->seq[v>>1].len; - return MUL_INPUT; - } - - - for (k = 0; k < asg_arc_n(g, v); k++) - { - if(!asg_arc_a(g, v)[k].del) - { - w = asg_arc_a(g, v)[k].v; - (*baseLen) += ((uint32_t)(asg_arc_a(g, v)[k].ul)); - break; - } - } - - - v = w; - - if((v>>1) == (begNode>>1)) - { - return LOOP; - } - } - - return LONG_TIPS; -} - -uint32_t untig_detect_single_path_with_dels_contigLen_complex(asg_t *g, uint32_t begNode, uint32_t* endNode, -long long* baseLen, long long* max_stop_base_Len, buf_t* b, uint32_t stops_threshold) -{ - - uint32_t v = begNode, w = 0; - uint32_t kv, kw, k, n_stops = 0, flag = LONG_TIPS; - (*baseLen) = 0; - (*max_stop_base_Len) = 0; - long long preBaseLen = 0, currentBaseLen; - - - while (1) - { - kv = get_real_length(g, v, NULL); - (*endNode) = v; - - if(b) kv_push(uint32_t, b->b, v>>1); - - if(kv == 0) - { - (*baseLen) += g->seq[v>>1].len; - flag = END_TIPS; - break; - } - - if(kv == 2) - { - (*baseLen) += g->seq[v>>1].len; - flag = TWO_OUTPUT; - break; - } - - if(kv > 2) - { - (*baseLen) += g->seq[v>>1].len; - flag = MUL_OUTPUT; - break; - } - - - - ///up to here, kv=1 - ///kw must >= 1 - get_real_length(g, v, &w); - kw = get_real_length(g, w^1, NULL); - - if(kw == 1) - { - ///kv must be 1 - for (k = 0; k < asg_arc_n(g, v); k++) - { - if(!asg_arc_a(g, v)[k].del) - { - w = asg_arc_a(g, v)[k].v; - (*baseLen) += ((uint32_t)(asg_arc_a(g, v)[k].ul)); - break; - } - } - } - - - ///just calculate the max_stop_Len - if(kw >= 2) - { - n_stops++; - if(n_stops >= stops_threshold) - { - (*baseLen) += g->seq[v>>1].len; - } - else - { - for (k = 0; k < asg_arc_n(g, v); k++) - { - if(!asg_arc_a(g, v)[k].del) - { - w = asg_arc_a(g, v)[k].v; - (*baseLen) += ((uint32_t)(asg_arc_a(g, v)[k].ul)); - break; - } - } - } - - currentBaseLen = (*baseLen) - preBaseLen; - preBaseLen = (*baseLen); - if(currentBaseLen > (*max_stop_base_Len)) - { - (*max_stop_base_Len) = currentBaseLen; - } - } - - if(kw >= 2 && n_stops >= stops_threshold) - { - if(kw == 2) flag = TWO_INPUT; - if(kw > 2) flag = MUL_INPUT; - break; - } - - - v = w; - - if((v>>1) == (begNode>>1)) - { - flag = LOOP; - break; - } - } - - currentBaseLen = (*baseLen) - preBaseLen; - preBaseLen = (*baseLen); - if(currentBaseLen > (*max_stop_base_Len)) - { - (*max_stop_base_Len) = currentBaseLen; - } - - return flag; -} - -/*****************************untig graph*****************************/ - - - - - - - long long check_if_diploid(uint32_t v1, uint32_t v2, asg_t *g, ma_hit_t_alloc* reverse_sources, long long min_edge_length, R_to_U* ruIndex) { @@ -5810,67 +5404,6 @@ int if_drop, R_to_U* ruIndex) } - -uint32_t get_long_tip_length_stops(asg_t *sg, ma_utg_v* u, uint32_t begNode, uint32_t* endNode, -buf_t* b, uint32_t stops_threshold) -{ - uint32_t v = begNode, w, n_stops = 0; - uint32_t kv, kw; - uint32_t eLen = 0; - (*endNode) = (uint32_t)-1; - if(u->a[v>>1].circ) return eLen; - while (1) - { - kv = get_real_length(sg, v, NULL); - (*endNode) = v; - eLen += EvaluateLen((*u), v>>1); - if(b) kv_push(uint32_t, b->b, v); - if(kv!=1) return eLen; - ///kw must be 1 here - kw = get_real_length(sg, v, &w); - kw = get_real_length(sg, w^1, NULL); - ///if(get_real_length(sg, w^1, NULL)!=1) return eLen; - if(kw >= 2) - { - n_stops++; - } - if(kw >= 2 && n_stops >= stops_threshold) - { - return eLen; - } - v = w; - if(v == begNode) return eLen; - } -} - - - -uint32_t get_long_tip_length(asg_t *sg, ma_utg_v* u, uint32_t begNode, uint32_t* endNode, buf_t* b) -{ - uint32_t v = begNode, w; - uint32_t kv; - uint32_t eLen = 0; - (*endNode) = (uint32_t)-1; - if(u->a[v>>1].circ) return eLen; - while (1) - { - kv = get_real_length(sg, v, NULL); - (*endNode) = v; - eLen += EvaluateLen((*u), v>>1); - if(b) kv_push(uint32_t, b->b, v); - if(kv!=1) return eLen; - ///kv must be 1 here - kv = get_real_length(sg, v, &w); - if(get_real_length(sg, w^1, NULL)!=1) return eLen; - v = w; - if(v == begNode) - { - u->a[begNode>>1].circ = 1; - return eLen; - } - } -} - uint32_t if_long_tip_length(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* untigLen, long long minLongUntig, long long maxShortUntig, float ShortUntigRate, long long mainLen) { @@ -5878,7 +5411,6 @@ long long minLongUntig, long long maxShortUntig, float ShortUntigRate, long long uint32_t Len, endNode; if(untigLen == NULL) { - ///Len = get_long_tip_length(sg, u, begNode, &endNode, NULL); if(get_unitig(sg, ug, begNode, &endNode, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, NULL) == LOOP) { @@ -5900,235 +5432,6 @@ long long minLongUntig, long long maxShortUntig, float ShortUntigRate, long long return 0; } - -long long check_if_diploid_untigs(asg_t *nsg, asg_t *read_sg, uint32_t v_0, uint32_t v_1, -ma_utg_v* ut_v, ma_hit_t_alloc* reverse_sources, long long min_edge_length, -float hap_rate, buf_t* b_0, buf_t* b_1, int if_drop, R_to_U* ruIndex) -{ - uint32_t vEnd; - b_0->b.n = b_1->b.n = 0; - if(get_long_tip_length(nsg, ut_v, v_0, &vEnd, b_0) == 0) return -1; - if(get_long_tip_length(nsg, ut_v, v_1, &vEnd, b_1) == 0) return -1; - uint32_t l_0 = 0, l_1 = 0, i; - for (i = 0; i < b_0->b.n; i++) - { - l_0 += ut_v->a[(b_0->b.a[i])>>1].n; - } - - for (i = 0; i < b_1->b.n; i++) - { - l_1 += ut_v->a[(b_1->b.a[i])>>1].n; - } - - if((long long)(l_0) <= min_edge_length || (long long)(l_1) <= min_edge_length) - { - return -1; - } - - buf_t* b_max; - buf_t* b_min; - - if(l_0 <= l_1) - { - b_min = b_0; - b_max = b_1; - } - else - { - b_min = b_1; - b_max = b_0; - } - - - - - uint32_t max_count = 0; - uint32_t min_count = 0; - ma_utg_t* node_a; - uint32_t i_a_1, i_a_2, untigID_a, qn, tn, j; - ma_utg_t* node_b; - uint32_t i_b_1, i_b_2, untigID_b; - uint32_t is_Unitig; - for (i_a_1 = 0; i_a_1 < b_min->b.n; i_a_1++) - { - untigID_a = b_min->b.a[i_a_1]>>1; - node_a = &(ut_v->a[untigID_a]); - for (i_a_2 = 0; i_a_2 < node_a->n; i_a_2++) - { - qn = node_a->a[i_a_2]>>33; - - for (j = 0; j < reverse_sources[qn].length; j++) - { - tn = Get_tn(reverse_sources[qn].buffer[j]); - /****************************may have bugs********************************/ - ///here is bug - // if(read_sg->seq[tn].del == 1 || (if_drop == 1 && read_sg->seq[tn].c == ALTER_LABLE)) - // { - // continue; - // } - if(read_sg->seq[tn].del == 1) - { - get_R_to_U(ruIndex, tn, &tn, &is_Unitig); - if(tn == (uint32_t)-1 || is_Unitig == 1 || read_sg->seq[tn].del == 1) continue; - } - /****************************may have bugs********************************/ - min_count++; - - - - for (i_b_1 = 0; i_b_1 < b_max->b.n; i_b_1++) - { - untigID_b = b_max->b.a[i_b_1]>>1; - node_b = &(ut_v->a[untigID_b]); - for (i_b_2 = 0; i_b_2 < node_b->n; i_b_2++) - { - if(tn == (node_b->a[i_b_2]>>33)) - { - max_count++; - goto end_reverse; - } - } - } - end_reverse:; - } - } - } - - - - if(min_count == 0) return -1; - if(max_count == 0) return 0; - if(max_count > min_count*hap_rate) return 1; - return 0; -} - -long long check_if_diploid_untigs_complex(asg_t *nsg, asg_t *read_sg, uint32_t v_0, uint32_t v_1, -ma_utg_v* ut_v, ma_hit_t_alloc* reverse_sources, long long min_edge_length, long long stops_threshold, -float hap_rate, buf_t* b_0, buf_t* b_1, int if_drop, R_to_U* ruIndex) -{ - uint32_t vEnd; - b_0->b.n = b_1->b.n = 0; - - - if(get_long_tip_length_stops(nsg, ut_v, v_0, &vEnd, b_0, stops_threshold) == 0) return -1; - if(get_long_tip_length_stops(nsg, ut_v, v_1, &vEnd, b_1, stops_threshold) == 0) return -1; - - - uint32_t l_0 = 0, l_1 = 0; - long long i, j; - i = b_0->b.n; i--; - j = b_1->b.n; j--; - while (i>=0 && j >=0) - { - if(b_0->b.a[i] == b_1->b.a[j]) - { - i--; - j--; - } - else - { - break; - } - } - b_0->b.n = i+1; - b_1->b.n = j+1; - - - l_0 = 0; l_1 = 0; - for (i = 0; i < (long long)b_0->b.n; i++) - { - l_0 += ut_v->a[(b_0->b.a[i])>>1].n; - } - - for (i = 0; i < (long long)b_1->b.n; i++) - { - l_1 += ut_v->a[(b_1->b.a[i])>>1].n; - } - - if((long long)(l_0) <= min_edge_length || (long long)(l_1) <= min_edge_length) - { - return -1; - } - - buf_t* b_max; - buf_t* b_min; - - if(l_0 <= l_1) - { - b_min = b_0; - b_max = b_1; - } - else - { - b_min = b_1; - b_max = b_0; - } - - - - - uint32_t max_count = 0; - uint32_t min_count = 0; - ma_utg_t* node_a; - uint32_t i_a_1, i_a_2, untigID_a, qn, tn; - ma_utg_t* node_b; - uint32_t i_b_1, i_b_2, untigID_b; - uint32_t is_Unitig; - for (i_a_1 = 0; i_a_1 < b_min->b.n; i_a_1++) - { - untigID_a = b_min->b.a[i_a_1]>>1; - node_a = &(ut_v->a[untigID_a]); - for (i_a_2 = 0; i_a_2 < node_a->n; i_a_2++) - { - qn = node_a->a[i_a_2]>>33; - - for (j = 0; j < (long long)reverse_sources[qn].length; j++) - { - tn = Get_tn(reverse_sources[qn].buffer[j]); - ///here is bug - /****************************may have bugs********************************/ - // if(read_sg->seq[tn].del == 1 || (if_drop == 1 && read_sg->seq[tn].c == ALTER_LABLE)) - // { - // continue; - // } - if(read_sg->seq[tn].del == 1) - { - get_R_to_U(ruIndex, tn, &tn, &is_Unitig); - if(tn == (uint32_t)-1 || is_Unitig == 1 || read_sg->seq[tn].del == 1) continue; - } - /****************************may have bugs********************************/ - min_count++; - - - - for (i_b_1 = 0; i_b_1 < b_max->b.n; i_b_1++) - { - untigID_b = b_max->b.a[i_b_1]>>1; - node_b = &(ut_v->a[untigID_b]); - for (i_b_2 = 0; i_b_2 < node_b->n; i_b_2++) - { - if(tn == (node_b->a[i_b_2]>>33)) - { - max_count++; - goto end_reverse; - } - } - } - end_reverse:; - } - } - } - - - - if(min_count == 0) return -1; - if(max_count == 0) return 0; - if(max_count > min_count*hap_rate) return 1; - return 0; -} - - - long long check_if_diploid_aggressive(uint32_t v1, uint32_t v2, asg_t *g, ma_hit_t_alloc* reverse_sources, long long min_edge_length) { @@ -8581,12 +7884,6 @@ add_unitig: return ug; } - - - - - - ma_ug_t *ma_ug_gen_primary(asg_t *g, uint8_t flag) { asg_cleanup(g); @@ -9263,24 +8560,17 @@ ma_sub_t *coverage_cut, kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp) return 1; } -inline int inter_interval(int a_s, int a_e, int b_s, int b_e, int* i_s, int* i_e) -{ - if(a_s > b_e || b_s > a_e) return 0; - (*i_s) = MAX(a_s, b_s); - (*i_e) = MIN(a_e, b_e); - return 1; -} - void print_rough_inconsistent_sites(ma_utg_t* collection, uint32_t cur_i, uint32_t next_i, asg_t* read_g, All_reads *RNF, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, kvec_asg_arc_t_warp* edge, UC_Read* r_read, UC_Read* q_read, int max_hang, int min_ovlp, uint32_t c_beg, uint32_t rate_thre, kvec_t_u32_warp* exact_count, kvec_t_u32_warp* total_count, -const char* prefix, int uID, FILE* fp) +const char* prefix, int uID, FILE* fp, bed_in* interval) { uint32_t v, w, i, rate; int v_beg, v_end, v_sub_beg, v_sub_end, w_beg, w_end, w_sub_beg, w_sub_end, i_beg, i_end, j; asg_arc_t t; + bed_interval* p = NULL; v = (uint64_t)(collection->a[cur_i])>>32; ///last element if(cur_i == collection->n-1 && next_i == collection->n) @@ -9405,7 +8695,7 @@ const char* prefix, int uID, FILE* fp) rate = ((total_count->a.a[i] - exact_count->a.a[i])*100)/total_count->a.a[i]; } - if(rate >= rate_thre) + if(rate >= rate_thre) ///inexact rate { ///start a new interval if(i_beg == -1 && i_end == -1) @@ -9435,20 +8725,30 @@ const char* prefix, int uID, FILE* fp) i_end++; rate = (total == 0)? 100 : (inexact*100)/total; - fprintf(fp,"%s%.6d%c\t%u\t%u\t%u\t", prefix, uID, "lc"[collection->circ], - (uint32_t)(i_beg + c_beg), (uint32_t)(i_end + c_beg), rate); - fprintf(fp,"%.*s", (int)Get_NAME_LENGTH((*RNF), (v>>1)), Get_NAME((*RNF), (v>>1))); - for (j = 0; j < (int)x->length; j++) + if(prefix != NULL && fp != NULL) { - h = &(x->buffer[j]); - if(inter_interval(v_sub_beg, v_sub_end, Get_qs((*h)), Get_qe((*h)) - 1, - &w_sub_beg, &w_sub_end) == 0) + fprintf(fp,"%s%.6d%c\t%u\t%u\t%u\t", prefix, uID, "lc"[collection->circ], + (uint32_t)(i_beg + c_beg), (uint32_t)(i_end + c_beg), rate); + fprintf(fp,"%.*s", (int)Get_NAME_LENGTH((*RNF), (v>>1)), Get_NAME((*RNF), (v>>1))); + for (j = 0; j < (int)x->length; j++) { - continue; + h = &(x->buffer[j]); + if(inter_interval(v_sub_beg, v_sub_end, Get_qs((*h)), Get_qe((*h)) - 1, + &w_sub_beg, &w_sub_end) == 0) + { + continue; + } + fprintf(fp,",%.*s", (int)Get_NAME_LENGTH((*RNF), Get_tn((*h))), Get_NAME((*RNF), Get_tn((*h)))); } - fprintf(fp,",%.*s", (int)Get_NAME_LENGTH((*RNF), Get_tn((*h))), Get_NAME((*RNF), Get_tn((*h)))); + fprintf(fp,"\n"); } - fprintf(fp,"\n"); + else if(interval != NULL) + { + kv_pushp(bed_interval, *interval, &p); + p->beg = (uint32_t)(i_beg + c_beg); + p->end = (uint32_t)(i_end + c_beg); + } + } i_beg = i_end = -1; @@ -9470,20 +8770,29 @@ const char* prefix, int uID, FILE* fp) i_end++; rate = (total == 0)? 100 : (inexact*100)/total; - fprintf(fp,"%s%.6d%c\t%u\t%u\t%u\t", prefix, uID, "lc"[collection->circ], - (uint32_t)(i_beg + c_beg), (uint32_t)(i_end + c_beg), rate); - fprintf(fp,"%.*s", (int)Get_NAME_LENGTH((*RNF), (v>>1)), Get_NAME((*RNF), (v>>1))); - for (j = 0; j < (int)x->length; j++) + if(prefix != NULL && fp != NULL) { - h = &(x->buffer[j]); - if(inter_interval(v_sub_beg, v_sub_end, Get_qs((*h)), Get_qe((*h)) - 1, - &w_sub_beg, &w_sub_end) == 0) + fprintf(fp,"%s%.6d%c\t%u\t%u\t%u\t", prefix, uID, "lc"[collection->circ], + (uint32_t)(i_beg + c_beg), (uint32_t)(i_end + c_beg), rate); + fprintf(fp,"%.*s", (int)Get_NAME_LENGTH((*RNF), (v>>1)), Get_NAME((*RNF), (v>>1))); + for (j = 0; j < (int)x->length; j++) { - continue; + h = &(x->buffer[j]); + if(inter_interval(v_sub_beg, v_sub_end, Get_qs((*h)), Get_qe((*h)) - 1, + &w_sub_beg, &w_sub_end) == 0) + { + continue; + } + fprintf(fp,",%.*s", (int)Get_NAME_LENGTH((*RNF), Get_tn((*h))), Get_NAME((*RNF), Get_tn((*h)))); } - fprintf(fp,",%.*s", (int)Get_NAME_LENGTH((*RNF), Get_tn((*h))), Get_NAME((*RNF), Get_tn((*h)))); + fprintf(fp,"\n"); + } + else if(interval != NULL) + { + kv_pushp(bed_interval, *interval, &p); + p->beg = (uint32_t)(i_beg + c_beg); + p->end = (uint32_t)(i_end + c_beg); } - fprintf(fp,"\n"); } } @@ -9794,6 +9103,100 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, uint8_t* r_flag) return C_bases/R_bases; } +uint32_t get_ug_coverage_aggressive(ma_ug_t *ug, uint32_t uID, asg_t* read_g, +const ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, R_to_U* ruIndex, uint8_t* r_flag, +uint64_t *n_utg) +{ + ma_utg_t* u = &(ug->u.a[uID]); + uint32_t k, j, rId, tn, is_Unitig; + long long R_bases = 0, C_bases = 0; + ma_hit_t *h; + if(u->m == 0) return 0; + + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 1; + } + + (*n_utg) = u->n; + uint32_t nv, i; + asg_arc_t *av = NULL; + for (i = 0; i < 2; i++) + { + nv = asg_arc_n(ug->g, (uID<<1)+i); + av = asg_arc_a(ug->g, (uID<<1)+i); + for (j = 0; j < nv; j++) + { + u = &(ug->u.a[av[j].v>>1]); + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 2; + } + } + } + + + u = &(ug->u.a[uID]); + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + R_bases += (coverage_cut[rId].e - coverage_cut[rId].s); + for (j = 0; j < (uint64_t)(sources[rId].length); j++) + { + h = &(sources[rId].buffer[j]); + if(h->el != 1) continue; + tn = Get_tn((*h)); + if(read_g->seq[tn].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + if(read_g->seq[tn].del == 1) continue; + if(r_flag[tn] == 0) continue; + if(r_flag[tn] == 2) + { + (*n_utg)++; + r_flag[tn] = 3; + } + C_bases += (Get_qe((*h)) - Get_qs((*h))); + ///if(uID == 35701) fprintf(stderr, "flag: %u, coverage: %d\n", r_flag[tn], (int)(Get_qe((*h)) - Get_qs((*h)))); + } + } + + + + + + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 0; + } + + for (i = 0; i < 2; i++) + { + nv = asg_arc_n(ug->g, (uID<<1)+i); + av = asg_arc_a(ug->g, (uID<<1)+i); + for (j = 0; j < nv; j++) + { + u = &(ug->u.a[av[j].v>>1]); + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 0; + } + } + } + + ///if(uID == 35701) fprintf(stderr, "C_bases: %lld, R_bases: %lld, (*n_utg): %lu\n", C_bases, R_bases, (*n_utg)); + return C_bases/R_bases; +} + + + void ma_ug_print2(const ma_ug_t *ug, All_reads *RNF, asg_t* read_g, const ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, R_to_U* ruIndex, int print_seq, const char* prefix, FILE *fp) { @@ -10101,7 +9504,7 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, const char* prefix, FILE *fp) void ma_ug_print_bed(const ma_ug_t *g, asg_t *read_g, All_reads *RNF, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, uint32_t rate_thres, -const char* prefix, FILE *fp) +const char* prefix, FILE *fp, hc_links* link) { UC_Read g_read; init_UC_Read(&g_read); @@ -10124,7 +9527,7 @@ const char* prefix, FILE *fp) print_rough_inconsistent_sites(u, j, j+1, read_g, RNF, sources, coverage_cut, edge, &g_read, &tmp, max_hang, min_ovlp, start, rate_thres, &exact_count, - &total_count, prefix, i+1, fp); + &total_count, prefix, i+1, fp, link? &(link->bed.a[i]): NULL); } } @@ -10134,106 +9537,1158 @@ const char* prefix, FILE *fp) kv_destroy(total_count.a); } - -int asg_arc_cut_long_tip_primary(asg_t *g, ma_ug_t *ug, float drop_ratio) +uint32_t get_break_point_cov(ma_utg_t* collection, uint32_t cur_i, uint32_t next_i, +asg_t* read_g, All_reads *RNF, ma_hit_t_alloc* sources, R_to_U* ruIndex, ma_sub_t *coverage_cut, +int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edge, uint8_t* r_flag) { - double startTime = Get_T(); - ///the reason is that each read has two direction (query->target, target->query) - uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, v_maxLen_i = (uint32_t)-1; - long long ll, v_maxLen; + uint32_t v, w, i, tn, is_Unitig; + int v_beg, v_end, v_sub_beg, v_sub_end; + asg_arc_t t; - buf_t b; - memset(&b, 0, sizeof(buf_t)); - - for (v = 0; v < n_vtx; ++v) + v = (uint64_t)(collection->a[cur_i])>>32; + ///last element + if(cur_i == collection->n-1 && next_i == collection->n) { - uint32_t i, n_arc = 0, nv = asg_arc_n(g, v), flag; - asg_arc_t *av = asg_arc_a(g, v); - ///some node could be deleted - if (nv < 2 || g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - n_arc = get_real_length(g, v, NULL); - if (n_arc < 2) continue; - - v_maxLen = -1; - v_maxLen_i = (uint32_t)-1; - - for (i = 0, n_arc = 0; i < nv; i++) + if(!collection->circ) { - if (!av[i].del) - { - if(ug == NULL) - { - detect_single_path_with_dels(g, av[i].v, &convex, &ll, NULL); - } - else - { - untig_detect_single_path_with_dels(g, ug, av[i].v, &convex, &ll, NULL); - } - - - if(v_maxLen < ll) - { - v_maxLen = ll; - v_maxLen_i = i; - } - } + next_i = (uint32_t)-1; } - - for (i = 0, n_arc = 0; i < nv; i++) + else { - if (!av[i].del) - { - if(v_maxLen_i == i) continue; - - b.b.n = 0; - - if(ug == NULL) - { - flag = detect_single_path_with_dels(g, av[i].v, &convex, &ll, &b); - } - else - { - flag = untig_detect_single_path_with_dels(g, ug, av[i].v, &convex, &ll, &b); - } - - if(flag == END_TIPS) - { - if(v_maxLen*drop_ratio > ll) - { - n_reduced++; - uint64_t k; - - for (k = 0; k < b.b.n; k++) - { - g->seq[b.b.a[k]].c = ALTER_LABLE; - } - - for (k = 0; k < b.b.n; k++) - { - asg_seq_drop(g, b.b.a[k]); - } - } - } - } + next_i = 0; } } - - asg_cleanup(g); - asg_symm(g); - free(b.b.a); - - if(VERBOSE >= 1) + if(next_i != ((uint32_t)-1)) { - fprintf(stderr, "[M::%s] removed %d long tips\n", - __func__, n_reduced); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + w = (uint64_t)(collection->a[next_i])>>32; + + get_specific_edge(sources, coverage_cut, NULL, edge, read_g, max_hang, min_ovlp, v, w, &t); + v_beg = 0; v_end = asg_arc_len(t) - 1; + if(v&1) + { + v_beg = Get_READ_LENGTH((*RNF), (v>>1)) - v_beg - 1; + v_end = Get_READ_LENGTH((*RNF), (v>>1)) - v_end - 1; + w = v_beg; v_beg = v_end; v_end = w; + } + } + else + { + v_beg = 0; v_end = Get_READ_LENGTH((*RNF), (v>>1)) - 1; } - return n_reduced; + ma_hit_t_alloc* x = &(sources[v>>1]); + ma_hit_t *h = NULL; + long long R_bases = v_end + 1 - v_beg, C_bases = 0; + ///[v_beg, v_end] must be the end of read, which means v_beg = 0 or v_end = Get_READ_LENGTH((*RNF), (v>>1)) - 1 + for (i = 0; i < x->length; i++) + { + h = &(x->buffer[i]); + tn = Get_tn((*h)); + if(read_g->seq[tn].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + if(read_g->seq[tn].del == 1) continue; + if(r_flag[tn] != 1) continue; + + + if(inter_interval(v_beg, v_end, Get_qs((*h)), Get_qe((*h)) - 1, + &v_sub_beg, &v_sub_end) == 0) + { + continue; + } + + C_bases += (v_sub_end + 1 - v_sub_beg); + } + if(R_bases <= 0 || C_bases <= 0) return 0; + + return C_bases/R_bases; } + +uint32_t push_cov_interval_direct(kvec_t_u64_warp* a, long long x_beg, long long x_end, long long utg_len, uint64_t is_circle) +{ + if(x_beg <= x_end && x_beg >= 0 && x_end >= 0 && x_beg < utg_len && x_end < utg_len) + { + uint64_t key; + key = x_beg; key <<= 1; key |= (!is_circle); key <<= 1; key|=1; + kv_push(uint64_t, a->a, key); + key = x_end + 1; key <<= 1; key |= (!is_circle); key <<= 1; + kv_push(uint64_t, a->a, key); + return 1; + } + return 0; +} + +uint32_t push_cov_interval_advance(kvec_t_u64_warp* a, long long x_beg, long long x_end, long long utg_len, uint32_t is_circle) +{ + if(x_beg > x_end) return 0; + + if(push_cov_interval_direct(a, x_beg, x_end, utg_len, 0)) return 1; + + + if(x_beg < 0 && x_end < 0) + { + x_beg = utg_len + x_beg; + x_end = utg_len + x_end; + return push_cov_interval_direct(a, x_beg, x_end, utg_len, 0); + } + + if(x_beg >= utg_len && x_end >= utg_len) + { + x_beg = x_beg - utg_len; + x_end = x_end - utg_len; + return push_cov_interval_direct(a, x_beg, x_end, utg_len, 0); + } + + if(x_beg < 0 && x_end >= 0) + { + x_beg = utg_len + x_beg; + if(push_cov_interval_direct(a, x_beg, utg_len - 1, utg_len, is_circle) || + push_cov_interval_direct(a, 0, x_end, utg_len, is_circle)) + { + return 1; + } + } + + return 0; +} + +void get_break_point_cov_advance(uint32_t v, long long c_beg, asg_t* read_g, All_reads *RNF, +ma_hit_t_alloc* sources, R_to_U* ruIndex, uint8_t* r_flag, kvec_t_u64_warp* depth, long long utg_len, +uint32_t is_circle, uint32_t* uID) +{ + uint32_t i, tn, is_Unitig; + long long v_beg, v_end, w_beg, w_end; + + v_beg = 0; v_end = Get_READ_LENGTH((*RNF), (v>>1)) - 1; + if(uID && (r_flag[v>>1]&1) && (!(r_flag[v>>1]&2))) + { + if(push_cov_interval_advance(depth, v_beg+c_beg, v_end+c_beg, utg_len, is_circle)) r_flag[v>>1] |= 2; + return; + } + + + ma_hit_t_alloc* x = &(sources[v>>1]); + ma_hit_t *h = NULL; + long long qs, qe, ts, te; + ///[v_beg, v_end] must be the end of read, which means v_beg = 0 or v_end = Get_READ_LENGTH((*RNF), (v>>1)) - 1 + for (i = 0; i < x->length; i++) + { + h = &(x->buffer[i]); + tn = Get_tn((*h)); + if(read_g->seq[tn].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + if(read_g->seq[tn].del == 1) continue; + if(!(r_flag[tn]&1)) continue; + tn = Get_tn((*h));///must!!!! + + if(r_flag[tn]&2) continue; + + qs = Get_qs((*h)); qe = Get_qe((*h)) - 1; + ts = Get_ts((*h)); te = Get_te((*h)) - 1; + if(h->rev) + { + ts = (long long)(Get_READ_LENGTH((*RNF), tn)) - ((long long)(Get_te((*h))) - 1) - 1; + te = (long long)(Get_READ_LENGTH((*RNF), tn)) - (long long)(Get_ts((*h))) - 1; + } + ts = qs - ts; + te = qe + ((long long)(Get_READ_LENGTH((*RNF), tn)) - te - 1); + w_beg = ts; w_end = te; + + if(v&1) + { + ts = (long long)(Get_READ_LENGTH((*RNF), v>>1)) - ts - 1; + te = (long long)(Get_READ_LENGTH((*RNF), v>>1)) - te - 1; + w_beg = te; w_end = ts; + } + + if(push_cov_interval_advance(depth, w_beg+c_beg, w_end+c_beg, utg_len, is_circle)) r_flag[tn] |= 2; + } +} +typedef struct { + uint32_t dp; + uint64_t k_beg, k_end; +} in_sub_t; + +typedef struct { + size_t n, m; + in_sub_t* a; +}kv_in_sub_t; + + +void debug_r_contig_pos(ma_utg_t *u, uint32_t tn, All_reads *RNF) +{ + uint32_t c_beg, c_end, l, k; + for (k = l = 0; k < u->n; k++) + { + c_beg = l; + c_end = c_beg + Get_READ_LENGTH((*RNF), (u->a[k]>>33)); + l += (uint32_t)u->a[k]; + if((u->a[k]>>33) == tn) + { + fprintf(stderr, "#####c_beg: %u, c_end: %u\n", c_beg, c_end); + break; + } + } +} + +uint32_t get_overlap_contig_dir(uint32_t v, uint32_t tn, ma_hit_t *h, All_reads *RNF, long long ctg_beg, +uint32_t p_beg, uint32_t p_end) +{ + long long qs, qe, ts, te, c_beg, c_end; + + qs = Get_qs((*h)); qe = Get_qe((*h)) - 1; + ts = Get_ts((*h)); te = Get_te((*h)) - 1; + if(h->rev) + { + ts = (long long)(Get_READ_LENGTH((*RNF), tn)) - ((long long)(Get_te((*h))) - 1) - 1; + te = (long long)(Get_READ_LENGTH((*RNF), tn)) - (long long)(Get_ts((*h))) - 1; + } + ts = qs - ts; + te = qe + ((long long)(Get_READ_LENGTH((*RNF), tn)) - te - 1); + c_beg = ts; c_end = te; + + if(v&1) + { + ts = (long long)(Get_READ_LENGTH((*RNF), v>>1)) - ts - 1; + te = (long long)(Get_READ_LENGTH((*RNF), v>>1)) - te - 1; + c_beg = te; c_end = ts; + } + + c_end++; + c_beg += ctg_beg; c_end += ctg_beg; + + if((p_beg != p_end && c_beg <= p_beg && c_end >= p_end) || + (p_beg == p_end && c_beg < p_beg && c_end > p_end)) + { + return 2; + } + + if(c_beg < p_beg) return 0; + if(c_end > p_end) return 1; + + return 2; + +} +///[beg, end) +uint32_t get_break_point_idx(ma_utg_t *u, All_reads *RNF, uint8_t* r_flag, ma_hit_t_alloc* sources, +asg_t* read_g, R_to_U* ruIndex, uint32_t p_beg, uint32_t p_end, double m_rate) +{ + uint32_t k, i, tn, is_Unitig, min_k = (uint32_t)-1, min_l = (uint32_t)-1, l, c_beg, c_end, index; + uint32_t e_occ = 0, ne_occ = 0; + double e_occ_dir[3], ne_occ_dir[3], rate[2]; + ma_hit_t_alloc* x = NULL; + ma_hit_t *h = NULL; + + e_occ = ne_occ = 0; + for (k = l = 0; k < u->n; k++) + { + c_beg = l; + c_end = c_beg + Get_READ_LENGTH((*RNF), (u->a[k]>>33)); + l += (uint32_t)u->a[k]; + + if((p_beg != p_end && c_beg <= p_beg && c_end >= p_end) || + (p_beg == p_end && c_beg < p_beg && c_end > p_end)) + { + + if(min_k == (uint32_t)-1) min_k = k, min_l = c_beg; + + x = &(sources[u->a[k]>>33]); + for (i = 0; i < x->length; i++) + { + h = &(x->buffer[i]); + tn = Get_tn((*h)); + if(read_g->seq[tn].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + if(read_g->seq[tn].del == 1) continue; + if(!(r_flag[tn]&1)) continue; + if(h->el) e_occ++; + else ne_occ++; + } + } + else if(min_k != (uint32_t)-1) + { + break; + } + } + + if(min_k == (uint32_t)-1) return (uint32_t)-1; + if(e_occ <= ((e_occ+ne_occ)*m_rate)) goto c_break; + + e_occ = ne_occ = 0; + for (k = min_k, l = min_l; k < u->n; k++) + { + c_beg = l; + c_end = c_beg + Get_READ_LENGTH((*RNF), (u->a[k]>>33)); + l += (uint32_t)u->a[k]; + + if((p_beg != p_end && c_beg <= p_beg && c_end >= p_end) || + (p_beg == p_end && c_beg < p_beg && c_end > p_end)) + { + e_occ_dir[0] = e_occ_dir[1] = e_occ_dir[2] = 0; + ne_occ_dir[0] = ne_occ_dir[1] = ne_occ_dir[2] = 0; + + x = &(sources[u->a[k]>>33]); + for (i = 0; i < x->length; i++) + { + h = &(x->buffer[i]); + tn = Get_tn((*h)); + if(read_g->seq[tn].del == 1) + { + ///get the id of read that contains it + get_R_to_U(ruIndex, tn, &tn, &is_Unitig); + if(tn == (uint32_t)-1 || is_Unitig == 1 || read_g->seq[tn].del == 1) continue; + } + if(read_g->seq[tn].del == 1) continue; + if(!(r_flag[tn]&1)) continue; + + tn = Get_tn((*h));///must!!!! + + index = get_overlap_contig_dir(u->a[k]>>32, tn, h, RNF, c_beg, p_beg, p_end); + + ///debug_r_contig_pos(u, tn, RNF); + if(h->el) e_occ_dir[index]++; + else ne_occ_dir[index]++; + } + + e_occ_dir[0] += e_occ_dir[2]; e_occ_dir[1] += e_occ_dir[2]; + ne_occ_dir[0] += ne_occ_dir[2]; ne_occ_dir[1] += ne_occ_dir[2]; + rate[0] = ne_occ_dir[0] / (ne_occ_dir[0] + e_occ_dir[0]); + rate[1] = ne_occ_dir[1] / (ne_occ_dir[1] + e_occ_dir[1]); + if(rate[0] >= rate[1]) + { + e_occ += e_occ_dir[0]; + ne_occ += ne_occ_dir[0]; + } + else + { + e_occ += e_occ_dir[1]; + ne_occ += ne_occ_dir[1]; + } + } + else if(min_k != (uint32_t)-1) + { + break; + } + } + + if(e_occ <= ((e_occ+ne_occ)*m_rate)) goto c_break; + return (uint32_t)-1; + + + c_break: + k = min_k; c_beg = min_l; c_end = c_beg + Get_READ_LENGTH((*RNF), (u->a[k]>>33)); + if((p_beg - c_beg) <= (c_end - p_end)) + { + if(k == 0 && !u->circ) return (uint32_t)-1; + return k; + } + else + { + if((k+1) < u->n) return k+1; + if((k+1) == u->n && u->circ) return 0; + return (uint32_t)-1; + } + + return (uint32_t)-1; +} + +void debug_break_point_advance(ma_utg_t *u, uint32_t uID, asg_t* read_g, All_reads *RNF, ma_hit_t_alloc* sources, +R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, uint8_t* r_flag) +{ + if(u->n < 2 || u->m == 0) return; + uint32_t k, l, c_beg; + memset(r_flag, 0, read_g->n_seq); + for (k = 0; k < u->n; k++) r_flag[u->a[k]>>33] = 1; + kvec_t_u64_warp d; kv_init(d.a); + kvec_t_u64_warp b_d; kv_init(b_d.a); + + d.a.n = 0; + for (k = l = 0; k < u->n; k++) + { + c_beg = l; + l += (uint32_t)u->a[k]; + get_break_point_cov_advance((uint64_t)(u->a[k])>>32, c_beg, read_g, RNF, sources, ruIndex, r_flag, &d, u->len, u->circ, &uID); + } + + for (k = l = 0; k < u->n; k++) + { + c_beg = l; + l += (uint32_t)u->a[k]; + get_break_point_cov_advance((uint64_t)(u->a[k])>>32, c_beg, read_g, RNF, sources, ruIndex, r_flag, &d, u->len, u->circ, NULL); + } + + kv_malloc(b_d.a, d.a.n); b_d.a.n = d.a.n; + memcpy(b_d.a.a, d.a.a, d.a.n*sizeof(uint64_t)); + + radix_sort_arch64(d.a.a, d.a.a + d.a.n); + + long long dp, o_dp; + uint32_t idx, o_idx, dir, o_dir, i; + uint64_t k_beg, k_end, k_dp, b_beg, b_end, occ; + dp = 0; o_idx = 0; o_dir = 1;///means it is a beg + for (k = 0; k < d.a.n; k++) + { + o_dp = dp; + ///if start = end, we should meet end first, otherwise it will have a bug + if(d.a.a[k]&1) ++dp; + else --dp; + + dir = d.a.a[k]&1; idx = d.a.a[k]>>2; + if((idx - o_idx > 0) || (idx == o_idx && o_dir != dir)) + { + k_beg = o_idx; k_end = idx; k_dp = o_dp; + + for (i = occ = 0; i < b_d.a.n; i += 2) + { + b_beg = b_d.a.a[i]>>2; + b_end = b_d.a.a[i+1]>>2; + if(b_beg <= k_beg && b_end >= k_end) occ++; + if((k_beg == k_end) && (b_beg == k_beg || b_end == k_beg)) occ--; + } + if(occ != k_dp) + { + fprintf(stderr, "k_beg: %lu, k_end: %lu, k_dp: %lu, occ: %lu\n", k_beg, k_end, k_dp, occ); + } + } + o_idx = idx; + o_dir = dir; + } + + if(o_idx != u->len) + { + k_beg = o_idx; k_end = u->len; k_dp = 0; + + for (i = occ = 0; i < b_d.a.n; i += 2) + { + b_beg = b_d.a.a[i]>>2; + b_end = b_d.a.a[i+1]>>2; + if(b_beg <= k_beg && b_end >= k_end) occ++; + } + if(occ != k_dp) + { + fprintf(stderr, "k_beg: %lu, k_end: %lu, k_dp: %lu, occ: %lu\n", k_beg, k_end, k_dp, occ); + } + } + + kv_destroy(d.a); kv_destroy(b_d.a); +} + +void detect_break_point_advance(ma_utg_t *u, uint32_t uID, asg_t* read_g, All_reads *RNF, ma_sub_t* coverage_cut, +ma_hit_t_alloc* sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, +uint8_t* r_flag, kvec_t_u64_warp* d, kv_in_sub_t* depth_i, kvec_t_u64_warp* res, int* b_low_cov, +int* b_high_cov, double m_rate) +{ + if(u->n < 2 || u->m == 0) return; + uint32_t k, l, c_beg; + memset(r_flag, 0, read_g->n_seq); + for (k = 0; k < u->n; k++) r_flag[u->a[k]>>33] = 1; + + d->a.n = 0; + for (k = l = 0; k < u->n; k++) + { + c_beg = l; + l += (uint32_t)u->a[k]; + get_break_point_cov_advance((uint64_t)(u->a[k])>>32, c_beg, read_g, RNF, sources, ruIndex, r_flag, d, u->len, u->circ, &uID); + } + + for (k = l = 0; k < u->n; k++) + { + c_beg = l; + l += (uint32_t)u->a[k]; + get_break_point_cov_advance((uint64_t)(u->a[k])>>32, c_beg, read_g, RNF, sources, ruIndex, r_flag, d, u->len, u->circ, NULL); + } + + radix_sort_arch64(d->a.a, d->a.a + d->a.n); + + long long dp, o_dp; + uint32_t idx, o_idx, dir, o_dir; + in_sub_t* p = NULL; + + + + /*******************************for debug************************************/ + ///debug_break_point_advance(u, uID, read_g, RNF, sources, ruIndex, edge, r_flag); + /*******************************for debug************************************/ + + + + + + + + + + depth_i->n = 0; + ///[start, end) + ///for circle + dp = 0; o_idx = 0; o_dir = 1;///means it is a beg + for (k = 0; k < d->a.n; k++) + { + o_dp = dp; + ///if start = end, we should meet end first, otherwise it will have a bug + if(d->a.a[k]&1) ++dp; + else --dp; + + dir = d->a.a[k]&1; idx = d->a.a[k]>>2; + if((idx - o_idx > 0) || (idx == o_idx && o_dir != dir)) + { + if(b_low_cov) + { + ///merge + if(o_dp >= (*b_low_cov) && depth_i->n > 0 && (int)depth_i->a[depth_i->n-1].dp >= (*b_low_cov)) + { + p = &(depth_i->a[depth_i->n-1]); + p->k_end = idx; + p->dp = o_dp; + } + else //insert new + { + kv_pushp(in_sub_t, *depth_i, &p); + p->k_beg = o_idx; + p->k_end = idx; + p->dp = o_dp; + } + } + + if(b_high_cov) + { + if(o_dp <= (*b_high_cov) && depth_i->n > 0 && (int)depth_i->a[depth_i->n-1].dp <= (*b_high_cov)) + { + p = &(depth_i->a[depth_i->n-1]); + p->k_end = idx; + p->dp = o_dp; + } + else //insert new + { + kv_pushp(in_sub_t, *depth_i, &p); + p->k_beg = o_idx; + p->k_end = idx; + p->dp = o_dp; + } + } + + } + o_idx = idx; + o_dir = dir; + } + + if(o_idx != u->len) + { + kv_pushp(in_sub_t, *depth_i, &p); + p->k_beg = o_idx; + p->k_end = u->len; + p->dp = 0; + } + + // if(b_high_cov) + // { + // fprintf(stderr, "\n\n\n\n\n\n\n\n\n\n"); + // fprintf(stderr, "uID: %u, u->n: %u, u->len: %u\n", uID, (uint32_t)u->n, (uint32_t)u->len); + // for (k = 0; k < depth_i->n; k++) + // { + // fprintf(stderr, "k: %u, k_beg: %lu, k_end: %lu, dp: %u\n", + // k, depth_i->a[k].k_beg, depth_i->a[k].k_end, depth_i->a[k].dp); + // } + // } + + + uint32_t beg_idx, end_idx, cir_beg_idx, cir_end_idx, min, min_idx, cur_idx, i; + beg_idx = end_idx = (uint32_t)-1; + cir_beg_idx = cir_end_idx = (uint32_t)-1; + uint64_t tmp; + + for (k = 0; k < depth_i->n; k++) + { + if(k > 0) + { + if((b_low_cov && (int)depth_i->a[k-1].dp >= (*b_low_cov) && (int)depth_i->a[k].dp < (*b_low_cov)) || + (b_high_cov && (int)depth_i->a[k-1].dp <= (*b_high_cov) && (int)depth_i->a[k].dp > (*b_high_cov))) + { + beg_idx = k; + } + } + + if(k < depth_i->n-1) + { + if((b_low_cov && (int)depth_i->a[k].dp < (*b_low_cov) && (int)depth_i->a[k+1].dp >= (*b_low_cov)) || + (b_high_cov && (int)depth_i->a[k].dp > (*b_high_cov) && (int)depth_i->a[k+1].dp <= (*b_high_cov))) + { + end_idx = k; + if(beg_idx == (uint32_t)-1) cir_end_idx = k; + + + if(beg_idx != (uint32_t)-1 && end_idx >= beg_idx) + { + min = min_idx = (uint32_t)-1; + for (i = beg_idx; i <= end_idx; i++) + { + // if(b_high_cov) + // { + // fprintf(stderr, "+k_end: %lu, k_end: %lu, dp: %u\n", depth_i->a[i].k_beg, depth_i->a[i].k_end, depth_i->a[i].dp); + // } + if(depth_i->a[i].dp < min) + { + cur_idx = get_break_point_idx(u, RNF, r_flag, sources, read_g, ruIndex, + depth_i->a[i].k_beg, depth_i->a[i].k_end, m_rate); + if(cur_idx == (uint32_t)-1) continue; + min = depth_i->a[i].dp; min_idx = cur_idx; + } + } + if(min_idx != (uint32_t)-1) + { + //fprintf(stderr, "+uID: %u, min_idx: %u, k_beg: %lu, k_end: %lu\n", uID, min_idx, depth_i->a[min_idx].k_beg, depth_i->a[min_idx].k_end); + if(min_idx != (uint32_t)-1) + { + tmp = uID; tmp <<=32; tmp += min_idx; + kv_push(uint64_t, res->a, tmp); + } + } + } + beg_idx = end_idx = (uint32_t)-1; + } + } + } + + if(beg_idx != (uint32_t)-1 && end_idx == (uint32_t)-1) cir_beg_idx = beg_idx; + + if(u->circ && (cir_beg_idx != (uint32_t)-1 || cir_end_idx != (uint32_t)-1)) + { + beg_idx = cir_beg_idx; + end_idx = cir_end_idx; + + min = min_idx = (uint32_t)-1; + if(beg_idx != (uint32_t)-1) + { + for (i = beg_idx; i < depth_i->n; i++) + { + // if(b_high_cov) + // { + // fprintf(stderr, "-0-k_end: %lu, k_end: %lu, dp: %u\n", depth_i->a[i].k_beg, depth_i->a[i].k_end, depth_i->a[i].dp); + // } + if(depth_i->a[i].dp < min) + { + // min = depth_i->a[i].dp; + // min_idx = i; + cur_idx = get_break_point_idx(u, RNF, r_flag, sources, read_g, ruIndex, + depth_i->a[i].k_beg, depth_i->a[i].k_end, m_rate); + if(cur_idx == (uint32_t)-1) continue; + min = depth_i->a[i].dp; min_idx = cur_idx; + } + } + } + + if(end_idx != (uint32_t)-1) + { + for (i = 0; i <= end_idx; i++) + { + // if(b_high_cov) + // { + // fprintf(stderr, "-1-k_end: %lu, k_end: %lu, dp: %u\n", depth_i->a[i].k_beg, depth_i->a[i].k_end, depth_i->a[i].dp); + // } + if(depth_i->a[i].dp < min) + { + cur_idx = get_break_point_idx(u, RNF, r_flag, sources, read_g, ruIndex, + depth_i->a[i].k_beg, depth_i->a[i].k_end, m_rate); + if(cur_idx == (uint32_t)-1) continue; + min = depth_i->a[i].dp; min_idx = cur_idx; + } + } + } + + if(min_idx != (uint32_t)-1) + { + ///fprintf(stderr, "-uID: %u, min_idx: %u, k_beg: %lu, k_end: %lu\n", uID, min_idx, depth_i->a[min_idx].k_beg, depth_i->a[min_idx].k_end); + if(min_idx != (uint32_t)-1) + { + tmp = uID; tmp <<=32; tmp += min_idx; + kv_push(uint64_t, res->a, tmp); + } + } + } +} + +void detect_break_point(ma_utg_t *u, uint32_t uID, asg_t* read_g, All_reads *RNF, ma_sub_t* coverage_cut, +ma_hit_t_alloc* sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, +uint8_t* r_flag, kvec_t_u32_warp* depth, kvec_t_u64_warp* res, uint32_t b_low_cov) +{ + depth->a.n = 0; + ///res->a.n = 0; + if(u->n < 2) return; + uint32_t k, i, min, min_idx, rId, *p = NULL, beg_idx, end_idx, cir_beg_idx, cir_end_idx; + uint64_t tmp; + if(u->m == 0) return; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 1; + } + + + for (k = 0; k < u->n; k++) + { + kv_pushp(uint32_t, depth->a, &p); + (*p) = get_break_point_cov(u, k, k+1, read_g, RNF, sources, ruIndex, coverage_cut, max_hang, min_ovlp, edge, r_flag); + } + + + beg_idx = end_idx = (uint32_t)-1; + cir_beg_idx = cir_end_idx = (uint32_t)-1; + for (k = 0; k < u->n; k++) + { + if(k > 0 && depth->a.a[k-1] >= b_low_cov && depth->a.a[k] < b_low_cov) + { + beg_idx = k; + } + + if(k < u->n-1 && depth->a.a[k] < b_low_cov && depth->a.a[k+1] >= b_low_cov) + { + end_idx = k; + if(beg_idx == (uint32_t)-1) cir_end_idx = k; + if(beg_idx != (uint32_t)-1 && end_idx >= beg_idx) + { + min = min_idx = (uint32_t)-1; + for (i = beg_idx; i <= end_idx; i++) + { + if(depth->a.a[i] < min) + { + min = depth->a.a[i]; + min_idx = i; + } + } + if(min_idx != (uint32_t)-1) + { + tmp = uID; tmp <<=32; tmp += min_idx; + kv_push(uint64_t, res->a, tmp); + } + } + beg_idx = end_idx = (uint32_t)-1; + } + } + + if(beg_idx != (uint32_t)-1 && end_idx == (uint32_t)-1) cir_beg_idx = beg_idx; + + if(u->circ && (cir_beg_idx != (uint32_t)-1 || cir_end_idx != (uint32_t)-1)) + { + beg_idx = cir_beg_idx; + end_idx = cir_end_idx; + + min = min_idx = (uint32_t)-1; + + if(beg_idx != (uint32_t)-1) + { + for (i = beg_idx; i < u->n; i++) + { + if(depth->a.a[i] < min) + { + min = depth->a.a[i]; + min_idx = i; + } + } + } + + if(end_idx != (uint32_t)-1) + { + for (i = 0; i <= end_idx; i++) + { + if(depth->a.a[i] < min) + { + min = depth->a.a[i]; + min_idx = i; + } + } + } + + if(min_idx != (uint32_t)-1) + { + tmp = uID; tmp <<=32; tmp += min_idx; + kv_push(uint64_t, res->a, tmp); + } + } + + + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 0; + } +} + +void debug_break_point(ma_utg_t *u, uint32_t uID, asg_t* read_g, All_reads *RNF, ma_sub_t* coverage_cut, +ma_hit_t_alloc* sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, +uint8_t* r_flag, kvec_t_u32_warp* depth, kvec_t_u64_warp* res, uint32_t b_low_cov) +{ + depth->a.n = 0; + ///res->a.n = 0; + if(u->n < 2) return; + uint32_t k, min, min_idx, rId, *p = NULL; + if(u->m == 0) return; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 1; + } + + + for (k = 0; k < u->n; k++) + { + kv_pushp(uint32_t, depth->a, &p); + (*p) = get_break_point_cov(u, k, k+1, read_g, RNF, sources, ruIndex, coverage_cut, max_hang, min_ovlp, edge, r_flag); + } + + int k_i, is_end; + min_idx = min = (uint32_t)-1; + for (k = 0; k < res->a.n; k++) + { + if((res->a.a[k]>>32) != uID) continue; + min_idx = (uint32_t)res->a.a[k]; + min = depth->a.a[min_idx]; + + is_end = 0; + k_i = (int)(min_idx) - 1; + while (k_i >= 0) + { + if(depth->a.a[k_i] >= b_low_cov) break; + if(depth->a.a[k_i] < min) fprintf(stderr, "ERROR1\n"); + k_i--; + if(k_i < 0 && u->circ && is_end == 0) + { + k_i = u->n - 1; + is_end = 1; + } + } + + is_end = 0; + k_i = (int)(min_idx) + 1; + while(k_i < (int)u->n) + { + if(depth->a.a[k_i] >= b_low_cov) break; + if(depth->a.a[k_i] < min) fprintf(stderr, "ERROR2\n"); + k_i++; + if(k_i >= (int)u->n && u->circ && is_end == 0) + { + k_i = 0; + is_end = 1; + } + } + } + + + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + r_flag[rId] = 0; + } +} + +void debug_contig_end(ma_ug_t *ug, asg_t* read_g, kvec_asg_arc_t_warp* edge) +{ + asg_t* nsg = ug->g; + uint32_t n_vtx = nsg->n_seq<<1, v, w, nv, v_occ, rv, rw, i; + asg_arc_t *av = NULL; + ma_utg_t* u = NULL; + + for (v = 0; v < n_vtx; v++) + { + if(ug->g->seq[v>>1].del) continue; + u = &(ug->u.a[v>>1]); + if(u->n == 0) continue; + av = asg_arc_a(ug->g, v); + nv = asg_arc_n(ug->g, v); + if(nv == 0) continue; + + if(v&1) rv = ug->u.a[v>>1].start^1; + else rv = ug->u.a[v>>1].end^1; + + for (i = v_occ = 0; i < nv; i++) + { + if(av[i].del) continue; + v_occ++; + } + if(v_occ == 0) continue; + + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + w = av[i].v; + if(w&1) rw = ug->u.a[w>>1].end; + else rw = ug->u.a[w>>1].start; + + fprintf(stderr, "utg: (v>>1: %u)[v&1: %u]->(w>>1: %u)[w&1: %u]\n", v>>1, v&1, w>>1, w&1); + fprintf(stderr, "rtg: (r_v>>1: %u)[r_v&1: %u]->(r_w>>1: %u)[r_w&1: %u]\n\n", + rv>>1, rv&1, rw>>1, rw&1); + } + } +} + +void renew_utg(ma_ug_t **ug, asg_t* read_g, kvec_asg_arc_t_warp* edge); +void push_sub_unitig(ma_ug_t *n_ug, ma_utg_t *src_u, asg_t *read_g, kvec_asg_arc_t_warp* edge, +uint32_t beg_idx, uint32_t occ) +{ + uint32_t i; + uint64_t totalLen; + ma_utg_t* p = NULL; + kv_pushp(ma_utg_t, n_ug->u, &p); + p->s = NULL; + p->n = occ; + p->circ = 0; + if(beg_idx == 0 && occ == src_u->n) p->circ = src_u->circ; + p->m = p->n; + p->a = (uint64_t*)malloc(8 * p->m); + for (i = 0; i < occ; i++) p->a[i] = src_u->a[beg_idx+i]; + fill_unitig(p->a, occ, read_g, edge, p->circ, &totalLen); + p->len = totalLen; + if(!p->circ) + { + p->start = p->a[0]>>32; + p->end = (p->a[p->n-1]>>32)^1; + } + else + { + p->start = p->end = UINT32_MAX; + } +} + +void break_all_contigs(ma_ug_t **ug, asg_t *read_g, kvec_asg_arc_t_warp* edge, kvec_t_u64_warp* break_points) +{ + asg_cleanup((*ug)->g); + uint32_t k, l, m, occ, uID, idx; + uint64_t *a = NULL, w; + ma_utg_t *u = NULL; + radix_sort_arch64(break_points->a.a, break_points->a.a + break_points->a.n); + uint32_t *utg_idx = NULL; MALLOC(utg_idx, (*ug)->u.n<<1); + memset(utg_idx, -1, sizeof(uint32_t)*((*ug)->u.n<<1)); + asg_arc_t *av = NULL; + uint32_t p_u_idx, nv, v, s_i, p_i; + ma_ug_t *n_ug = NULL; + ma_utg_t *p = NULL, *z = NULL; + n_ug = (ma_ug_t*)calloc(1, sizeof(ma_ug_t)); + n_ug->g = asg_init(); + + for (k = m = 0; k < break_points->a.n; k++) + { + if(k == 0 || (m > 0 && break_points->a.a[m-1] != break_points->a.a[k])) + { + break_points->a.a[m] = break_points->a.a[k]; + m++; + } + } + ///fprintf(stderr, "break_points->a.n: %u, m: %u\n", (uint32_t)break_points->a.n, m); + break_points->a.n = m; + + for (k = 1, l = 0, p_i = 0; k <= break_points->a.n; ++k) + { + if (k == break_points->a.n || (break_points->a.a[k]>>32) != (break_points->a.a[l]>>32)) + { + occ = k - l; + a = break_points->a.a + l; + l = k; + if(occ == 0) continue; + uID = a[0]>>32; + u = &((*ug)->u.a[uID]); + if(u->n < 2) continue; + + for (s_i = p_i; s_i < uID; s_i++) + { + kv_pushp(ma_utg_t, n_ug->u, &p); + z = &((*ug)->u.a[s_i]); + (*p) = (*z); + z->len = z->circ = /**z->start = z->end =**/ z->m = z->n = 0; + z->a = NULL; z->s = NULL; + + utg_idx[s_i<<1] = ((uint32_t)(n_ug->u.n-1))<<1; + utg_idx[(s_i<<1)+1] = (((uint32_t)(n_ug->u.n-1))<<1)+1; + } + p_i = uID + 1; + + + utg_idx[(uID<<1)+1] = (((uint32_t)(n_ug->u.n))<<1)+1; + for (m = p_u_idx = 0; m < occ; m++) + { + idx = (uint32_t)(a[m]); + if(!u->circ && idx == 0) + { + fprintf(stderr, "ERROR 1\n"); + continue; + } + + + if(u->circ && idx == 0) + { + av = asg_arc_a((*ug)->g, (uID<<1)+1); + nv = asg_arc_n((*ug)->g, (uID<<1)+1); + for (v = 0; v < nv; v++) + { + if(av[v].del) continue; + av[v].del = 1; + asg_arc_del((*ug)->g, (av[v].v)^1, (av[v].ul>>32)^1, 1); + } + u->circ = 0; + u->start = u->a[0]>>32; + u->end = (u->a[u->n-1]>>32)^1; + continue; + } + + if(idx - p_u_idx <= 0) + { + fprintf(stderr, "ERROR2: uID: %u, u->circ: %u, idx: %u, p_u_idx: %u\n", + uID, u->circ, idx, p_u_idx); + continue; + } + + + push_sub_unitig(n_ug, u, read_g, edge, p_u_idx, idx - p_u_idx); + p_u_idx = idx; + } + push_sub_unitig(n_ug, u, read_g, edge, p_u_idx, u->n - p_u_idx); + utg_idx[(uID<<1)] = ((uint32_t)(n_ug->u.n-1))<<1; + + } + } + + for (s_i = p_i; s_i < (*ug)->u.n; s_i++) + { + kv_pushp(ma_utg_t, n_ug->u, &p); + z = &((*ug)->u.a[s_i]); + (*p) = (*z); + z->len = z->circ = /**z->start = z->end =**/ z->m = z->n = 0; + z->a = NULL; z->s = NULL; + + utg_idx[s_i<<1] = ((uint32_t)(n_ug->u.n-1))<<1; + utg_idx[(s_i<<1)+1] = (((uint32_t)(n_ug->u.n-1))<<1)+1; + } + + // for (k = 0; k < (*ug)->u.n; k++) + // { + // if(utg_idx[(k<<1)] == (uint32_t)-1) fprintf(stderr, "ERROR 3\n"); + // if(utg_idx[(k<<1)+1] == (uint32_t)-1) fprintf(stderr, "ERROR 4\n"); + // } + + asg_arc_t *q = NULL; + for (k = 0; k < (*ug)->g->n_arc; k++) + { + if((*ug)->g->arc[k].del) continue; + q = asg_arc_pushp(n_ug->g); + (*q) = (*ug)->g->arc[k]; + + q->v = utg_idx[q->v^1]^1; + + w = q->ul>>32; w = utg_idx[w]; w <<= 32; + q->ul <<= 32; q->ul >>= 32; q->ul |= w; + } + + for (k = 0; k < n_ug->u.n; k++) + { + asg_seq_set(n_ug->g, k, n_ug->u.a[k].len, 0); + } + + asg_cleanup(n_ug->g); + + // fprintf(stderr, "n_ug->u.n: %u, n_ug->g->n_seq: %u, (*ug)->u.n: %u\n", (uint32_t)n_ug->u.n, (uint32_t)n_ug->g->n_seq, + // (uint32_t)(*ug)->u.n); + + // for (k = 0; k < ((*ug)->u.n<<1); k++) + // { + // uint32_t ug_rid, n_ug_rid; + // if(k&1) ug_rid = ((*ug)->u.a[k>>1]).start; + // else ug_rid = ((*ug)->u.a[k>>1]).end; + + // if(utg_idx[k]&1) n_ug_rid = n_ug->u.a[utg_idx[k]>>1].start; + // else n_ug_rid = n_ug->u.a[utg_idx[k]>>1].end; + + // if(ug_rid != n_ug_rid) + // { + // fprintf(stderr, "ERROR, uid: %u, dir: %u, circle: %u, ug_rid: %u, n_ug_rid: %u\n", + // k>>1, k&1, (*ug)->u.a[k>>1].circ, ug_rid, n_ug_rid); + // } + // } + + + + ma_ug_destroy((*ug)); + (*ug) = n_ug; + renew_utg(ug, read_g, edge); ///for circle + // fprintf(stderr, "***********(1)edge->a.n: %u***********\n", (uint32_t)edge->a.n); + // debug_utg_graph(*ug, read_g, edge, 0, 0); + // fprintf(stderr, "***********(1)edge->a.n: %u***********\n", (uint32_t)edge->a.n); + free(utg_idx); +} + +void print_utg_stats(ma_ug_t *ug, const char* command) +{ + uint32_t i; + uint64_t len, occ_n, occ_m, occ_n_0; + for (i = len = occ_n = occ_m = occ_n_0 = 0; i < ug->u.n; ++i) { + ma_utg_t *u = &ug->u.a[i]; + len += u->len; + occ_n += u->n; + occ_m += u->m; + if(u->n == 0) occ_n_0++; + } + + fprintf(stderr, "%s: len: %lu, occ_n: %lu, occ_m: %lu, occ_n_0: %lu\n", + command, len, occ_n, occ_m, occ_n_0); +} + +void break_ug_contig(ma_ug_t **ug, asg_t *read_g, All_reads *RNF, ma_sub_t *coverage_cut, +ma_hit_t_alloc* sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, +int* b_low_cov, int* b_high_cov, double m_rate) +{ + if(b_low_cov) + { + fprintf(stderr, "[M::%s] break potential misassemblies with <%d-fold coverage\n", + __func__, *b_low_cov); + } + + if(b_high_cov) + { + fprintf(stderr, "[M::%s] break potential misassemblies with >%d-fold coverage\n", + __func__, *b_high_cov); + } + + kvec_t_u64_warp depth; + kv_init(depth.a); + kvec_t_u64_warp break_points; + kv_init(break_points.a); + kv_in_sub_t depth_i; + kv_init(depth_i); + uint32_t i; + uint8_t* primary_flag = (uint8_t*)calloc(read_g->n_seq, sizeof(uint8_t)); + ma_utg_t *u = NULL; + + for (i = 0; i < (*ug)->u.n; ++i) + { + u = &((*ug)->u.a[i]); + if(u->m == 0) continue; + if(u->n < 2) continue; + // detect_break_point(u, i, read_g, RNF, coverage_cut, sources, ruIndex, edge, max_hang, min_ovlp, primary_flag, &depth, &break_points, b_low_cov); + // debug_break_point(u, i, read_g, RNF, coverage_cut, sources, ruIndex, edge, max_hang, min_ovlp, primary_flag, &depth, &break_points, b_low_cov); + detect_break_point_advance(u, i, read_g, RNF, coverage_cut, sources, ruIndex, edge, max_hang, min_ovlp, + primary_flag, &depth, &depth_i, &break_points, b_low_cov, b_high_cov, m_rate); + + } + + break_all_contigs(ug, read_g, edge, &break_points); + + kv_destroy(depth.a); + kv_destroy(break_points.a); + kv_destroy(depth_i); + free(primary_flag); +} + int asg_arc_cut_long_tip_primary_complex(asg_t *g, float drop_ratio, uint32_t stops_threshold) { double startTime = Get_T(); @@ -10312,203 +10767,6 @@ int asg_arc_cut_long_tip_primary_complex(asg_t *g, float drop_ratio, uint32_t st return n_reduced; } - -int untig_asg_arc_cut_long_tip_primary_complex(ma_ug_t *ug, float drop_ratio, -uint32_t stops_threshold) -{ - double startTime = Get_T(); - asg_t *g = ug->g; - - ///the reason is that each read has two direction (query->target, target->query) - uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag; - long long ll, max_stopLen; - - buf_t buf; - memset(&buf, 0, sizeof(buf_t)); - - for (v = 0; v < n_vtx; ++v) - { - uint32_t i; - ///some node could be deleted - if (g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - ///tip - if (get_real_length(g, v, NULL) != 0) continue; - if(get_real_length(g, v^1, NULL) != 1) continue; - /****************************may have bugs********************************/ - buf.b.n = 0; - flag = untig_detect_single_path_with_dels(g, ug, v^1, &convex, &ll, &buf); - if(flag != TWO_INPUT && flag != MUL_INPUT) continue; - get_real_length(g, convex, &convex); - /****************************may have bugs********************************/ - convex = convex^1; - ///note the convexLen here - uint32_t n_convex = asg_arc_n(g, convex), convexLen = ll; - asg_arc_t *a_convex = asg_arc_a(g, convex); - - - for (i = 0; i < n_convex; i++) - { - if (!a_convex[i].del) - { - ///if stops_threshold = 1, - ///untig_detect_single_path_with_dels_n_stops() is untig_detect_single_path_with_dels() - untig_detect_single_path_with_dels_n_stops(g, ug, a_convex[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - if(convex == v) continue; - - if(ll*drop_ratio > convexLen && max_stopLen*2>ll) - { - n_reduced++; - uint64_t k; - - for (k = 0; k < buf.b.n; k++) - { - g->seq[buf.b.a[k]].c = ALTER_LABLE; - } - - for (k = 0; k < buf.b.n; k++) - { - asg_seq_drop(g, buf.b.a[k]); - } - break; - } - } - } - } - - - asg_cleanup(g); - asg_symm(g); - free(buf.b.a); - - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] removed %d long tips\n", - __func__, n_reduced); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); - } - - return n_reduced; -} - - - -int untig_asg_arc_cut_long_equal_tips_assembly(ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex) -{ - asg_t *g = ug->g; - double startTime = Get_T(); - ///the reason is that each read has two direction (query->target, target->query) - uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag, is_hap; - long long ll, base_maxLen, base_maxLen_i; - buf_t b_0, b_1; - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - - buf_t b; - memset(&b, 0, sizeof(buf_t)); - - for (v = 0; v < n_vtx; ++v) - { - uint32_t i, n_arc = 0, nv = asg_arc_n(g, v), n_tips; - asg_arc_t *av = asg_arc_a(g, v); - ///some node could be deleted - if (nv < 2 || g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - n_arc = get_real_length(g, v, NULL); - if (n_arc < 2) continue; - - base_maxLen = -1; - base_maxLen_i = -1; - n_tips = 0; - is_hap = 0; - - ///there must be more than 1 out-edges - for (i = 0; i < nv; i++) - { - if (!av[i].del) - { - flag = untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, NULL); - - if(base_maxLen < ll) - { - base_maxLen = ll; - base_maxLen_i = i; - } - - if(flag == END_TIPS) - { - n_tips++; - } - } - } - - ///at least one tip - if(n_tips > 0) - { - for (i = 0; i < nv; i++) - { - if(i == base_maxLen_i) continue; - if (!av[i].del) - { - b.b.n = 0; - if(untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, &b) - != END_TIPS) - { - continue; - } - //we can only cut tips - if(check_if_diploid_untigs(g, read_sg, av[base_maxLen_i].v, av[i].v, &(ug->u), - reverse_sources, miniedgeLen, 0.3, &b_0, &b_1, 1, ruIndex) == 1) - { - n_reduced++; - uint64_t k; - - for (k = 0; k < b.b.n; k++) - { - g->seq[b.b.a[k]].c = ALTER_LABLE; - } - for (k = 0; k < b.b.n; k++) - { - asg_seq_drop(g, b.b.a[k]); - } - - is_hap++; - } - } - } - } - - if(is_hap > 0) - { - i = base_maxLen_i; - b.b.n = 0; - untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, &b); - uint64_t k; - for (k = 0; k < b.b.n; k++) - { - g->seq[b.b.a[k]].c = HAP_LABLE; - } - } - } - - - asg_cleanup(g); - asg_symm(g); - free(b.b.a); - free(b_0.b.a); - free(b_1.b.a); - - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] removed %d long tips\n", - __func__, n_reduced); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); - } - - - return n_reduced; -} - int asg_arc_cut_long_equal_tips_assembly(asg_t *g, ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex) { @@ -10743,132 +11001,6 @@ R_to_U* ruIndex) return n_reduced; } -int untig_asg_arc_simple_large_bubbles(ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -long long miniedgeLen, R_to_U* ruIndex) -{ - asg_t *g = ug->g; - double startTime = Get_T(); - ///the reason is that each read has two direction (query->target, target->query) - uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag, is_hap; - long long ll, base_maxLen, base_maxLen_i, all_covex; - - buf_t b; - memset(&b, 0, sizeof(buf_t)); - buf_t b_0, b_1; - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - - for (v = 0; v < n_vtx; ++v) - { - uint32_t i, n_arc = 0, nv = asg_arc_n(g, v); - asg_arc_t *av = asg_arc_a(g, v); - ///some node could be deleted - if (nv < 2 || g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - n_arc = get_real_length(g, v, NULL); - if (n_arc < 2) continue; - - base_maxLen = -1; - base_maxLen_i = -1; - all_covex = -1; - is_hap = 0; - - for (i = 0; i < nv; i++) - { - if (!av[i].del) - { - flag = untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, NULL); - if(flag != TWO_INPUT && flag != MUL_INPUT) - { - break; - } - - get_real_length(g, convex, &convex); - - if(all_covex != -1 && (uint32_t)all_covex != convex) - { - break; - } - - if(all_covex == -1) - { - all_covex = convex; - } - - if(base_maxLen < ll) - { - base_maxLen = ll; - base_maxLen_i = i; - } - - } - } - - - if(i == nv) - { - for (i = 0; i < nv; i++) - { - if(i == base_maxLen_i) continue; - if (!av[i].del) - { - b.b.n = 0; - untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, &b); - - - //we can only cut tips - if(check_if_diploid_untigs(g, read_sg, av[base_maxLen_i].v, av[i].v, &(ug->u), - reverse_sources, miniedgeLen, 0.3, &b_0, &b_1, 1, ruIndex) == 1) - { - n_reduced++; - uint64_t k; - - for (k = 0; k < b.b.n; k++) - { - g->seq[b.b.a[k]].c = ALTER_LABLE; - } - for (k = 0; k < b.b.n; k++) - { - asg_seq_drop(g, b.b.a[k]); - } - - is_hap++; - } - } - } - - if(is_hap > 0) - { - i = base_maxLen_i; - b.b.n = 0; - untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, &b); - uint64_t k; - for (k = 0; k < b.b.n; k++) - { - g->seq[b.b.a[k]].c = HAP_LABLE; - } - } - } - - } - - - asg_cleanup(g); - asg_symm(g); - free(b.b.a); - free(b_0.b.a); - free(b_1.b.a); - - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] removed %d long tips\n", - __func__, n_reduced); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); - } - - - return n_reduced; -} - uint32_t get_num_trio_flag(ma_ug_t *ug, uint32_t v, uint32_t flag) { if(flag == (uint32_t)-1) return 0; @@ -10887,8 +11019,222 @@ uint32_t get_num_trio_flag(ma_ug_t *ug, uint32_t v, uint32_t flag) return flag_occ; } + +void set_pre_uid(buf_t* b, hc_links* link, ma_ug_t *ug) +{ + uint32_t k = 0, i = 0, m = 0, rId, pre = (uint32_t)-1; + ma_utg_t* u = NULL; + for (i = 0; i < b->b.n; i++) + { + u = &(ug->u.a[b->b.a[i]>>1]); + if(u->m == 0) continue; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + if(link->u_idx[rId] == (uint32_t)-1) continue; + if(pre == link->u_idx[rId]) continue; + pre = link->u_idx[rId]; + b->b.a[m] = link->u_idx[rId]; + m++; + } + } + b->b.n = m; +} + + +inline uint64_t get_utg_len(buf_t* b, ma_ug_t *ug, asg_t *read_sg, uint64_t ignore_end, uint64_t* len_thre, uint64_t* occ) +{ + if(len_thre && occ)(*occ) = (uint64_t)-1; + uint32_t ori, uid, v, nv, l, k, idx; + uint32_t *a = b->b.a, a_n = b->b.n; + uint32_t u_i, r_i, len, p_v; + asg_arc_t *av = NULL; + ma_utg_t* u = NULL; + for (u_i = r_i = len = idx = 0, p_v = (uint32_t)-1; u_i < a_n; u_i++) + { + uid = a[u_i] >> 1; + ori = a[u_i] & 1; + u = &(ug->u.a[uid]); + if(u->n == 0) continue; + if(ori == 1) + { + for (r_i = 0; r_i < u->n; r_i++, idx++) + { + v = ((uint64_t)((u->a[u->n - r_i - 1])^(uint64_t)(0x100000000)))>>32; + ///w = ((uint64_t)((u->a[u->n - x->r_i - 2])^(uint64_t)(0x100000000)))>>32; + if(p_v == (uint32_t)-1) + { + p_v = v; + continue; + } + + av = asg_arc_a(read_sg, p_v); + nv = asg_arc_n(read_sg, p_v); + l = 0; + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + if(av[k].v == v) + { + l = asg_arc_len(av[k]); + break; + } + } + if(k == nv) fprintf(stderr, "ERROR\n"); + len += l; + if(len_thre && occ && len >= (*len_thre)) + { + (*occ) = idx - 1; + return len; + } + p_v = v; + } + } + else + { + for (r_i = 0; r_i < u->n; r_i++, idx++) + { + v = ((uint64_t)(u->a[r_i]))>>32; + ///w = ((uint64_t)(u->a[x->r_i + 1]))>>32; + if(p_v == (uint32_t)-1) + { + p_v = v; + continue; + } + + + av = asg_arc_a(read_sg, p_v); + nv = asg_arc_n(read_sg, p_v); + l = 0; + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + if(av[k].v == v) + { + l = asg_arc_len(av[k]); + break; + } + } + if(k == nv) fprintf(stderr, "ERROR\n"); + len += l; + if(len_thre && occ && len >= (*len_thre)) + { + (*occ) = idx - 1; + return len; + } + p_v = v; + } + } + } + + if(ignore_end == 0 && p_v != (uint32_t)-1) + { + len += read_sg->seq[p_v>>1].len; + if(len_thre && occ && len >= (*len_thre)) + { + (*occ) = idx - 1; + return len; + } + } + + if(len_thre && occ) + { + (*occ) = idx; + } + + return len; +} + +void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg) +{ + uint32_t b_0_i, b_0_k, b_1_i, b_1_k, pre_0, pre_1, rId_0, rId_1, ori_0, ori_1; + uint64_t d = RC_1, len_0, len_1, thre_0, thre_1; + ma_utg_t* u_b_0 = NULL; + ma_utg_t* u_b_1 = NULL; + if(b_0->b.n == 0 || b_1->b.n == 0) return; + + len_0 = get_utg_len(b_0, ug, read_sg, 1, NULL, NULL); + len_1 = get_utg_len(b_1, ug, read_sg, 1, NULL, NULL); + + len_0 = MIN(len_0, len_1); + get_utg_len(b_0, ug, read_sg, 0, &len_0, &thre_0); + get_utg_len(b_1, ug, read_sg, 0, &len_0, &thre_1); + + + for (b_0_i = len_0 = 0, pre_0 = (uint32_t)-1; b_0_i < b_0->b.n; b_0_i++) + { + ori_0 = b_0->b.a[b_0_i]&1; + u_b_0 = &(ug->u.a[b_0->b.a[b_0_i]>>1]); + if(u_b_0->n == 0) continue; + for (b_0_k = 0; b_0_k < u_b_0->n; b_0_k++) + { + len_0++; + if(len_0 > thre_0) return; + if(ori_0 == 1) + { + rId_0 = u_b_0->a[u_b_0->n - b_0_k - 1]>>33; + } + else + { + rId_0 = u_b_0->a[b_0_k]>>33; + } + + if(link->u_idx[rId_0] == (uint32_t)-1) continue; + if(pre_0 == link->u_idx[rId_0]) continue; + pre_0 = link->u_idx[rId_0]; + + for (b_1_i = len_1 = 0, pre_1 = (uint32_t)-1; b_1_i < b_1->b.n; b_1_i++) + { + ori_1 = b_1->b.a[b_1_i]&1; + u_b_1 = &(ug->u.a[b_1->b.a[b_1_i]>>1]); + if(u_b_1->n == 0) continue; + for (b_1_k = 0; b_1_k < u_b_1->n; b_1_k++) + { + len_1++; + if(len_1 > thre_1) goto b_1_i_end; + if(ori_1 == 1) + { + rId_1 = u_b_1->a[u_b_1->n - b_1_k - 1]>>33; + } + else + { + rId_1 = u_b_1->a[b_1_k]>>33; + } + if(link->u_idx[rId_1] == (uint32_t)-1) continue; + if(pre_1 == link->u_idx[rId_1]) continue; + pre_1 = link->u_idx[rId_1]; + + + push_hc_edge(&(link->a.a[pre_0]), pre_1, 1, 1, &d); + push_hc_edge(&(link->a.a[pre_1]), pre_0, 1, 1, &d); + + // if(pre_0 == 5 || pre_1 == 5) + // { + // fprintf(stderr, "\npre_0: utg%.6ul, len_0: %lu, thre_0: %lu, pre_1: utg%.6ul, len_1: %lu, thre_1: %lu\n", + // (int)(pre_0+1), len_0, thre_0, (int)(pre_1+1), len_1, thre_1); + // uint32_t xxx_i; + // for (xxx_i = 0; xxx_i < b_0->b.n; xxx_i++) + // { + // fprintf(stderr,"+: utg%.6ul\n", (int)((b_0->b.a[xxx_i]>>1)+1)); + // } + + // for (xxx_i = 0; xxx_i < b_1->b.n; xxx_i++) + // { + // fprintf(stderr,"-: utg%.6ul\n", (int)((b_1->b.a[xxx_i]>>1)+1)); + // } + // } + } + } + + b_1_i_end:; + } + } +} + + + int untig_asg_arc_simple_large_bubbles_trio(ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negative_flag) +long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negative_flag, hc_links* link) { asg_t *g = ug->g; double startTime = Get_T(); @@ -10928,7 +11274,6 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ if (!av[i].del) { buffer.b.n = 0; - ///flag = untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, &b); flag = get_unitig(g, ug, av[i].v, &convex, &tmp, &ll, &max_stop_nodeLen, &max_stop_baseLen, 1, &buffer); if(flag != MUL_INPUT) break; // if(flag != TWO_INPUT && flag != MUL_INPUT) @@ -10996,12 +11341,9 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ if (!av[i].del) { buffer.b.n = 0; - ///untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, &b); get_unitig(g, ug, av[i].v, &convex, &tmp, &ll, &max_stop_nodeLen, &max_stop_baseLen, 1, &buffer); //we can only cut tips - /**if(check_if_diploid_untigs(g, read_sg, av[base_best_i].v, av[i].v, &(ug->u), - reverse_sources, miniedgeLen, 0.3, &b_0, &b_1, 1, ruIndex) == 1)**/ if(check_different_haps(g, ug, read_sg, av[base_best_i].v, av[i].v, reverse_sources, &b_0, &b_1, ruIndex, miniedgeLen, 1)==PLOID) { @@ -11016,6 +11358,8 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ asg_seq_drop(g, buffer.b.a[k]>>1); } + if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + is_hap++; } } @@ -11025,7 +11369,6 @@ long long miniedgeLen, R_to_U* ruIndex, uint32_t positive_flag, uint32_t negativ { i = base_best_i; buffer.b.n = 0; - ///untig_detect_single_path_with_dels_contigLen(g, av[i].v, &convex, &ll, &b); get_unitig(g, ug, av[i].v, &convex, &tmp, &ll, &max_stop_nodeLen, &max_stop_baseLen, 1, &buffer); for (k = 0; k < buffer.b.n; k++) { @@ -11167,433 +11510,6 @@ long long miniedgeLen, uint32_t stops_threshold, R_to_U* ruIndex) } - -int untig_asg_arc_cut_long_equal_tips_assembly_complex(ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, uint32_t stops_threshold, R_to_U* ruIndex) -{ - asg_t *g = ug->g; - double startTime = Get_T(); - ///the reason is that each read has two direction (query->target, target->query) - uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag; - long long ll, max_stopLen; - - buf_t buf, b_0, b_1; - memset(&buf, 0, sizeof(buf_t)); - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - - - for (v = 0; v < n_vtx; ++v) - { - uint32_t i; - ///some node could be deleted - if (g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - ///tip - if (get_real_length(g, v, NULL) != 0) continue; - if(get_real_length(g, v^1, NULL) != 1) continue; - /****************************may have bugs********************************/ - buf.b.n = 0; - flag = untig_detect_single_path_with_dels_contigLen(g, v^1, &convex, &ll, &buf); - if(flag != TWO_INPUT && flag != MUL_INPUT) continue; - get_real_length(g, convex, &convex); - /****************************may have bugs********************************/ - convex = convex^1; - uint32_t n_convex = asg_arc_n(g, convex), convexLen = ll; - asg_arc_t *a_convex = asg_arc_a(g, convex); - - for (i = 0; i < n_convex; i++) - { - if (!a_convex[i].del) - { - untig_detect_single_path_with_dels_contigLen_complex(g, a_convex[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - - if(convex == v) continue; - - ///threshold = 0.8 - if(ll > convexLen && max_stopLen*1.25>ll) - { - - //we can only cut tips - if(check_if_diploid_untigs_complex(g, read_sg, v^1, a_convex[i].v, - &(ug->u), reverse_sources, miniedgeLen, stops_threshold, - 0.3, &b_0, &b_1, 1, ruIndex)==1) - { - n_reduced++; - uint64_t k; - for (k = 0; k < buf.b.n; k++) - { - g->seq[buf.b.a[k]].c = ALTER_LABLE; - } - for (k = 0; k < buf.b.n; k++) - { - asg_seq_drop(g, buf.b.a[k]); - } - - - - - ///lable the primary one - b_0.b.n = 0; - untig_detect_single_path_with_dels_contigLen_complex(g, a_convex[i].v, - &convex, &ll, &max_stopLen, &b_0, stops_threshold); - for (k = 0; k < b_0.b.n; k++) - { - g->seq[b_0.b.a[k]].c = HAP_LABLE; - } - - break; - } - - } - - } - } - } - - - asg_cleanup(g); - asg_symm(g); - free(buf.b.a); - free(b_0.b.a); - free(b_1.b.a); - - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] removed %d long tips\n", - __func__, n_reduced); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); - } - - - return n_reduced; -} - - - -int untig_asg_arc_cut_chimeric_back(ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, uint32_t stops_threshold, float drop_rate, -R_to_U* ruIndex) -{ - asg_t *g = ug->g; - double startTime = Get_T(); - ///the reason is that each read has two direction (query->target, target->query) - uint32_t v, w1, w2, wv, nw, n_vtx = g->n_seq * 2, n_reduced = 0, convex, convex_T; - asg_arc_t *aw; - long long ll, max_stopLen; - - buf_t b_0, b_1; - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - - - for (v = 0; v < n_vtx; ++v) - { - - uint32_t i; - ///some node could be deleted - if (g->seq[v>>1].del || g->seq[v>>1].c == ALTER_LABLE) continue; - if(get_real_length(g, v, NULL) != 1 || get_real_length(g, v^1, NULL) != 1 ) continue; - - get_real_length(g, v, &w1); - if(get_real_length(g, w1^1, NULL)<=1) continue; - - get_real_length(g, v^1, &w2); - if(get_real_length(g, w2^1, NULL)<=1) continue; - - - untig_detect_single_path_with_dels_n_stops(g, ug, w1, &convex, &ll, &max_stopLen, NULL, - stops_threshold); - if(ll*drop_rate < EvaluateLen((*ug).u, v>>1)) continue; - if(ll*0.4 > max_stopLen) continue; - - - untig_detect_single_path_with_dels_n_stops(g, ug, w2, &convex, &ll, &max_stopLen, NULL, - stops_threshold); - if(ll*drop_rate < EvaluateLen((*ug).u, v>>1)) continue; - if(ll*0.4 > max_stopLen) continue; - - stops_threshold++; - - - w1 = w1^1;wv=v^1;aw = asg_arc_a(g, w1); nw = asg_arc_n(g, w1);convex_T = (uint32_t)-1; - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - if(ll*drop_rate < EvaluateLen((*ug).u, v>>1)) break; - if(ll*0.4 > max_stopLen) continue; - - if((aw[i].v>>1)==(v>>1)) - { - if(convex_T != (uint32_t)-1) break; - convex_T = convex; - } - } - if(i!=nw) continue; - - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - if((aw[i].v>>1) == (v>>1)) continue; - - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - if(convex == convex_T) break; - - if(check_if_diploid_untigs_complex(g, read_sg, wv, aw[i].v, &(ug->u), reverse_sources, - miniedgeLen, stops_threshold, 0.3, &b_0, &b_1, 1, ruIndex)==1) - { - break; - } - } - if(i!=nw) continue; - - - - - - - w2 = w2^1;wv=v;aw = asg_arc_a(g, w2); nw = asg_arc_n(g, w2);convex_T = (uint32_t)-1; - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - if(ll*drop_rate < EvaluateLen((*ug).u, v>>1)) break; - if(ll*0.4 > max_stopLen) continue; - - if((aw[i].v>>1) == (v>>1)) - { - if(convex_T != (uint32_t)-1) break; - convex_T = convex; - } - } - if(i!=nw) continue; - - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - if((aw[i].v>>1) == (v>>1)) continue; - - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - if(convex == convex_T) break; - - if(check_if_diploid_untigs_complex(g, read_sg, wv, aw[i].v, &(ug->u), reverse_sources, - miniedgeLen, stops_threshold, 0.3, &b_0, &b_1, 1, ruIndex)==1) - { - break; - } - } - if(i!=nw) continue; - - - - - - - n_reduced++;g->seq[v>>1].c = ALTER_LABLE;asg_seq_drop(g, v>>1); - ///fprintf(stderr, "v>>1: %u\n", v>>1); - } - - - asg_cleanup(g); - free(b_0.b.a); - free(b_1.b.a); - - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] removed %d long tips\n", - __func__, n_reduced); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); - } - - - return n_reduced; -} - - -int untig_asg_arc_cut_chimeric(ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, uint32_t stops_threshold, float drop_rate, -R_to_U* ruIndex) -{ - asg_t *g = ug->g; - double startTime = Get_T(); - ///the reason is that each read has two direction (query->target, target->query) - uint32_t i, k, v_i, v_beg, v_end, selfLen, w1, w2, wv, nw, n_vtx = g->n_seq * 2, n_reduced = 0, convex, convex_T; - asg_arc_t *aw; - long long ll, max_stopLen; - - buf_t b_0, b_1; - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - - - for (v_i = 0; v_i < n_vtx; ++v_i) - { - - v_beg = v_i; - ///some node could be deleted - if (g->seq[v_beg>>1].del || g->seq[v_beg>>1].c == ALTER_LABLE) continue; - if(get_real_length(g, v_beg, NULL) != 1) continue; - - get_real_length(g, v_beg, &w1); - if(get_real_length(g, w1^1, NULL)<=1) continue; - - - selfLen = get_long_tip_length(g, &(ug->u), v_beg^1, &v_end, NULL); - if(ug->u.a[v_beg>>1].circ) continue; - if(selfLen == (uint32_t)-1) continue; - if(get_real_length(g, v_end, NULL) != 1) continue; - - - get_real_length(g, v_end, &w2); - if(get_real_length(g, w2^1, NULL)<=1) continue; - - - untig_detect_single_path_with_dels_n_stops(g, ug, w1, &convex, &ll, &max_stopLen, NULL, - stops_threshold); - if(ll*drop_rate < selfLen) continue; - if(ll*0.4 > max_stopLen) continue; - - - untig_detect_single_path_with_dels_n_stops(g, ug, w2, &convex, &ll, &max_stopLen, NULL, - stops_threshold); - if(ll*drop_rate < selfLen) continue; - if(ll*0.4 > max_stopLen) continue; - - stops_threshold++; - - - w1 = w1^1;wv=v_beg^1;aw = asg_arc_a(g, w1); nw = asg_arc_n(g, w1);convex_T = (uint32_t)-1; - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - if(ll*drop_rate < selfLen) break; - if(ll*0.4 > max_stopLen) continue; - - if((aw[i].v>>1)==(v_beg>>1)) - { - if(convex_T != (uint32_t)-1) break; - convex_T = convex; - } - } - if(i!=nw) continue; - - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - if((aw[i].v>>1) == (v_beg>>1)) continue; - - b_0.b.n = 0; - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, &b_0, stops_threshold); - ///if(convex == convex_T) break; - for (k = 0; k < b_0.b.n; k++) - { - if(b_0.b.a[k] == (convex_T>>1)) break; - } - if(k != b_0.b.n) break; - - - if(check_if_diploid_untigs_complex(g, read_sg, wv, aw[i].v, &(ug->u), reverse_sources, - miniedgeLen, stops_threshold, 0.3, &b_0, &b_1, 1, ruIndex)==1) - { - break; - } - } - if(i!=nw) continue; - - - - - - - w2 = w2^1;wv=v_end^1;aw = asg_arc_a(g, w2); nw = asg_arc_n(g, w2);convex_T = (uint32_t)-1; - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, NULL, stops_threshold); - if(ll*drop_rate < selfLen) break; - if(ll*0.4 > max_stopLen) continue; - - if((aw[i].v>>1) == (v_end>>1)) - { - if(convex_T != (uint32_t)-1) break; - convex_T = convex; - } - } - if(i!=nw) continue; - - for (i = 0; i < nw; i++) - { - if(aw[i].del) continue; - if((aw[i].v>>1) == (v_end>>1)) continue; - - b_0.b.n = 0; - untig_detect_single_path_with_dels_n_stops(g, ug, aw[i].v, &convex, &ll, - &max_stopLen, &b_0, stops_threshold); - ///if(convex == convex_T) break; - for (k = 0; k < b_0.b.n; k++) - { - if(b_0.b.a[k] == (convex_T>>1)) break; - } - if(k != b_0.b.n) break; - - - if(check_if_diploid_untigs_complex(g, read_sg, wv, aw[i].v, &(ug->u), reverse_sources, - miniedgeLen, stops_threshold, 0.3, &b_0, &b_1, 1, ruIndex)==1) - { - break; - } - } - if(i!=nw) continue; - - - - - - - n_reduced++; - b_0.b.n = 0; - get_long_tip_length(g, &(ug->u), v_beg^1, &v_end, &b_0); - for (k = 0; k < b_0.b.n; k++) - { - g->seq[b_0.b.a[k]>>1].c = ALTER_LABLE; - } - - for (k = 0; k < b_0.b.n; k++) - { - asg_seq_drop(g, b_0.b.a[k]>>1); - } - } - - - asg_cleanup(g); - free(b_0.b.a); - free(b_1.b.a); - - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] removed %d long tips\n", - __func__, n_reduced); - fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); - } - - - return n_reduced; -} - - void output_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp) { @@ -11619,7 +11535,7 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp) sprintf(gfa_name, "%s.r_utg.lowQ.bed", output_file_name); output_file = fopen(gfa_name, "w"); ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, - max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "utg", output_file); + max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "utg", output_file, NULL); fclose(output_file); } @@ -11628,6 +11544,403 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp) kv_destroy(new_rtg_edges.a); } +void classify_untigs(ma_ug_t *ug, asg_t *sg, ma_sub_t* coverage_cut, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, +kvec_asg_arc_t_warp* new_rtg_edges, int max_hang, int min_ovlp) +{ + uint64_t i, dip_thre_max, dip_thres, n_utg; + uint8_t* primary_flag = (uint8_t*)calloc(sg->n_seq, sizeof(uint8_t)); + int tmp_cov = asm_opt.hom_global_coverage; + asm_opt.hom_global_coverage = -1; + + purge_dups(ug, sg, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, + asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, 0, 0, 0, 1, NULL); + dip_thre_max = ((double)asm_opt.hom_global_coverage)/((double)HOM_PEAK_RATE)*0.70; + asm_opt.hom_global_coverage = tmp_cov; + ///fprintf(stderr, "dip_thre_max: %lu\n", dip_thre_max); + + for (i = 0; i < ug->g->n_seq; i++) + { + dip_thres = dip_thre_max; + if(ug->u.a[i].n <= dip_thre_max) dip_thres = dip_thre_max * 0.9; + + if(get_ug_coverage(&ug->u.a[i], sg, coverage_cut, sources, ruIndex, primary_flag)g->seq[i].c = 1; + if(get_ug_coverage_aggressive(ug, i, sg, coverage_cut, sources, ruIndex, primary_flag, &n_utg) dip_thres*2) ug->g->seq[i].c = 2; + } + } + else + { + ug->g->seq[i].c = 0; + } + } + free(primary_flag); + + ///fprintf(stderr, "[M::%s] diploid coverage threshold: %lu\n", __func__, dip_thres); +} + + +void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num) +{ + kv_malloc(link->a, ug_num); link->a.n = ug_num; + kv_malloc(link->enzymes, ug_num); link->enzymes.n = ug_num; + uint64_t i; + for (i = 0; i < link->a.n; i++) + { + kv_init(link->a.a[i].e); + kv_init(link->a.a[i].f); + } + MALLOC(link->u_idx, r_num); + memset(link->u_idx, -1, r_num*sizeof(uint32_t)); + link->r_num = r_num; + kv_malloc(link->bed, ug_num); link->bed.n = ug_num; + for (i = 0; i < link->bed.n; i++) + { + kv_init(link->bed.a[i]); + } +} + +void destory_hc_links(hc_links* link) +{ + uint64_t i; + for (i = 0; i < link->a.n; i++) + { + kv_destroy(link->a.a[i].e); + kv_destroy(link->a.a[i].f); + } + kv_destroy(link->a); + free(link->u_idx); + for (i = 0; i < link->bed.n; i++) + { + kv_destroy(link->bed.a[i]); + } + kv_destroy(link->bed); + kv_destroy(link->enzymes); +} + +void pop_small_bub(ma_ug_t *ug) +{ + bubble_type bub; + uint64_t n_vtx = ug->g->n_seq*2, tLen, pathLen, nodeLen; + uint32_t i, k, v, mode = (((uint32_t)-1)<<2); + asg_cleanup(ug->g); if (!ug->g->is_symm) asg_symm(ug->g); + memset(&bub, 0, sizeof(bubble_type)); + CALLOC(bub.index, n_vtx); + kv_init(bub.list); kv_init(bub.num); kv_init(bub.pathLen); + buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; + for (v = 0; v < n_vtx; ++v) + { + if(ug->g->seq[v>>1].del) continue; + if(asg_arc_n(ug->g, v) < 2) continue; + if((bub.index[v]&(uint32_t)3) != 0) continue; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + { + //beg is v, end is b.S.a[0] + //note b.b include end, does not include beg + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + bub.index[b.b.a[i]] &= mode; bub.index[b.b.a[i]] += 1; + bub.index[b.b.a[i]^1] &= mode; bub.index[b.b.a[i]^1] += 1; + } + bub.index[v] &= mode; bub.index[v] += 2; + bub.index[b.S.a[0]^1] &= mode; bub.index[b.S.a[0]^1] += 3; + } + } + + for (v = 0; v < n_vtx; ++v) + { + if((bub.index[v]&(uint32_t)3) !=2) continue; + kv_push(uint32_t, bub.num, bub.list.n); + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL)) + { + kv_push(uint64_t, bub.pathLen, pathLen); + //beg is v, end is b.S.a[0] + kv_push(uint32_t, bub.list, v); + kv_push(uint32_t, bub.list, b.S.a[0]^1); + + //note b.b include end, does not include beg + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + kv_push(uint32_t, bub.list, b.b.a[i]); + } + } + } + + kv_push(uint32_t, bub.num, bub.list.n); + ///free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + bub.f_bub = bub.num.n - 1; bub.b_bub = 0; + memset(bub.index, 0, n_vtx*sizeof(uint32_t)); + + uint32_t beg, sink, n, *a, total_nodes; + for (i = 0; i < bub.f_bub; i++) + { + get_bubbles(&bub, i, &beg, &sink, &a, &n, &pathLen); + for (k = total_nodes = 0; k < n; k++) + { + total_nodes += ug->u.a[a[k]>>1].n; + } + + for (k = 0; k < n; k++) + { + v = a[k]; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, &nodeLen)) + { + + } + + + v = a[k]^1; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, &nodeLen)) + { + + } + + } + + } + +} + +void hic_clean(asg_t* read_g) +{ + uint32_t n_vtx, v, u; + uint64_t i, k, k_i, tLen, v_occ, u_occ, utg_occ; + double bub_rate = 0.1; + ma_ug_t *ug = NULL; + ug = ma_ug_gen_primary(read_g, PRIMARY_LABLE); + n_vtx = ug->g->n_seq * 2; + buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; + uint8_t* bs_flag = (uint8_t*)calloc(n_vtx, 1); + kvec_t(uint32_t) ax; + kv_init(ax); + for (v = 0; v < ug->g->n_seq; ++v) + { + if(ug->g->seq[v].del) continue; + ug->g->seq[v].c = PRIMARY_LABLE; + EvaluateLen(ug->u, v) = ug->u.a[v].n; + } + + + for (v = 0; v < n_vtx; ++v) + { + if(ug->g->seq[v>>1].del) continue; + if(asg_arc_n(ug->g, v) < 2) continue; + if(bs_flag[v] != 0) continue; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + { + //beg is v, end is b.S.a[0] + //note b.b include end, does not include beg + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + bs_flag[b.b.a[i]] = bs_flag[b.b.a[i]^1] = 1; + } + bs_flag[v] = 2; bs_flag[b.S.a[0]^1] = 3; + } + } + + + for (v = 0; v < n_vtx; ++v) + { + if(bs_flag[v] !=2) continue; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + { + //note b.b include end, does not include beg + for (i = v_occ = ax.n = 0; i < b.b.n; i++) + { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + v_occ += ug->u.a[b.b.a[i]>>1].n; + kv_push(uint32_t, ax, b.b.a[i]>>1); + } + + for (i = 0; i < ax.n; i++) + { + for (k = 0; k < 2; k++) + { + u = (ax.a[i]<<1) + k; + if(asg_arc_n(ug->g, u) < 2) continue; + if(asg_bub_pop1_primary_trio(ug->g, NULL, u, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + { + for (k_i = u_occ = utg_occ = 0; k_i < b.b.n; k_i++) + { + if(b.b.a[k_i]==u || b.b.a[k_i]==b.S.a[0]) continue; + u_occ += ug->u.a[b.b.a[k_i]>>1].n; + utg_occ++; + } + + if(u_occ >= v_occ*bub_rate) continue; + if(u_occ > 3) continue; + if(utg_occ > 2) continue; + asg_bub_pop1_primary_trio(ug->g, NULL, u, tLen, &b, (uint32_t)-1, (uint32_t)-1, 1, NULL, NULL); + } + } + } + } + } + + ma_utg_t* m = NULL; + for (v = 0; v < ug->g->n_seq; ++v) + { + if(ug->g->seq[v].del) continue; + if(ug->g->seq[v].c != ALTER_LABLE) continue; + m = &(ug->u.a[v]); + if(m->m == 0) continue; + for (k = 0; k < m->n; k++) + { + asg_seq_del(read_g, m->a[k]>>33); + } + } + + asg_cleanup(read_g); + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); free(bs_flag); + ma_ug_destroy(ug); + kv_destroy(ax); +} + + +void output_hic_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, +R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) +{ + hic_clean(sg); + + kvec_asg_arc_t_warp new_rtg_edges; + kv_init(new_rtg_edges.a); + ma_ug_t *ug = NULL; + ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); + + + + + + + hc_links link; + + ///if(load_hc_links(&link, output_file_name) == 0) + { + init_hc_links(&link, ug->g->n_seq, R_INF.total_reads); + asg_t *copy_sg = copy_read_graph(sg); + ma_ug_t *copy_ug = copy_untig_graph(ug); + asm_opt.purge_overlap_len = asm_opt.purge_overlap_len_hic; + asm_opt.purge_simi_rate = asm_opt.purge_simi_rate_hic; + adjust_utg_by_primary(©_ug, copy_sg, TRIO_THRES, sources, reverse_sources, coverage_cut, + bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, + max_hang, min_ovlp, &new_rtg_edges, &link); + ma_ug_destroy(copy_ug); + asg_destroy(copy_sg); + + ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, + max_hang, min_ovlp, asm_opt.hic_inconsist_rate, NULL, NULL, &link); + + ///write_hc_links(&link, output_file_name); + } + + + + + new_rtg_edges.a.n = 0; + ma_ug_seq(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp); + classify_untigs(ug, sg, coverage_cut, sources, reverse_sources, ruIndex, &new_rtg_edges, + max_hang, min_ovlp); + hic_analysis(ug, sg, &link); + destory_hc_links(&link); + ma_ug_destroy(ug); + kv_destroy(new_rtg_edges.a); + + output_unitig_graph(sg, coverage_cut, output_file_name, sources, ruIndex, max_hang, min_ovlp); + output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, + reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + 0.05, 0.9, max_hang, min_ovlp, 0); + output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, + reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + 0.05, 0.9, max_hang, min_ovlp, 0); +} + +ma_ug_t* merge_utg(ma_ug_t **dest, ma_ug_t **src) +{ + asg_t *g_d = (*dest)->g, *g_s = (*src)->g; + uint64_t occ_d = g_d->n_seq, occ_s = g_s->n_seq, i; + asg_arc_t *p = NULL; + g_d->is_srt = g_d->is_symm = 0; + + for (i = 0; i < occ_s; i++) + { + asg_seq_set(g_d, i+occ_d, g_s->seq[i].len, g_s->seq[i].del); + g_d->seq[i+occ_d].c = g_s->seq[i].c; + } + + g_d->seq_vis = (uint8_t*)realloc(g_d->seq_vis, g_d->n_seq*2*sizeof(uint8_t)); + + + for (i = 0; i < g_s->n_arc; i++) + { + p = asg_arc_pushp(g_d); + (*p) = g_s->arc[i]; + p->ul += (occ_d<<33); + p->v += (occ_d<<1); + } + + asg_cleanup(g_d); + g_d->r_seq = g_d->n_seq; + + if(g_s->n_F_seq > 0 && g_s->F_seq) + { + uint64_t n_F_seq = g_d->n_F_seq + g_s->n_F_seq; + g_d->F_seq = (ma_utg_t*)realloc(g_d->F_seq, n_F_seq*sizeof(ma_utg_t)); + memcpy(g_d->F_seq + g_d->n_F_seq, g_s->F_seq, g_s->n_F_seq*sizeof(ma_utg_t)); + g_d->n_F_seq = n_F_seq; + free(g_s->F_seq); + g_s->F_seq = NULL; + g_s->n_F_seq = 0; + } + + ma_utg_v *u_d = &((*dest)->u), *u_s = &((*src)->u); + if(u_s->n > 0) + { + uint64_t n = u_d->n + u_s->n; + u_d->a = (ma_utg_t*)realloc(u_d->a, n*sizeof(ma_utg_t)); + memcpy(u_d->a + u_d->n, u_s->a, u_s->n*sizeof(ma_utg_t)); + u_d->n = u_d->m = n; + free(u_s->a); + u_s->a = NULL; + u_s->n = u_s->m = 0; + } + + kv_push(uint64_t, (*dest)->occ, occ_d); + kv_push(uint64_t, (*dest)->occ, occ_s); + + ma_ug_destroy(*src); + return (*dest); +} + +void benchmark_hic_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) +{ + ma_ug_t *ug_1 = output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, + reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, + chimeric_rate, drop_ratio, max_hang, min_ovlp, 1); + + ma_ug_t *ug_2 = output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, + reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, + chimeric_rate, drop_ratio, max_hang, min_ovlp, 1); + fprintf(stderr, "ug_1->u.n: %u, ug_2->u.n: %u\n", (uint32_t)ug_1->u.n, (uint32_t)ug_2->u.n); + ma_ug_t *ug = merge_utg(&ug_1, &ug_2); + fprintf(stderr, "ug->u.n: %u\n", (uint32_t)ug->u.n); + + hic_benchmark(ug, sg); + + ma_ug_destroy(ug); +} void merge_unitig_content(ma_utg_t* collection, ma_ug_t* ug, asg_t* read_g, kvec_asg_arc_t_warp* edge) { @@ -11672,7 +11985,9 @@ void merge_unitig_content(ma_utg_t* collection, ma_ug_t* ug, asg_t* read_g, kvec if(index == 0) return; - fill_unitig(buffer, index, read_g, edge, collection->circ, &totalLen); + ///fill_unitig(buffer, index, read_g, edge, collection->circ, &totalLen); + fill_unitig(buffer, index, read_g, edge, /**collection->circ**/ + (collection->n == 1 && ug->u.a[collection->a[0]>>33].circ), &totalLen); ///important. must be here if(collection->n == 1 && ug->u.a[collection->a[0]>>33].circ) collection->circ = 1; @@ -11856,8 +12171,8 @@ ma_hit_t_alloc* reverse_sources, const char* info) { if(rId == (u->a[k]>>33)) { - fprintf(stderr, "%s: %s is the %u-th read at %u-th unitig (label: %u)\n", - info, name, k, i, g->g->seq[i].c); + fprintf(stderr, "%s: %s is the %u-th read at %u-th unitig (label: %u, occ: %u)\n", + info, name, k, i, g->g->seq[i].c, u->n); return i; } } @@ -12195,8 +12510,9 @@ asg_t *read_sg, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, uint32_t min_e return cnt; } + int asg_arc_cut_trio_long_tip_primary(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio) +R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, hc_links* link) { double startTime = Get_T(); ///the reason is that each read has two direction (query->target, target->query) @@ -12277,7 +12593,6 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio) // #define NON_PLOID 1 if(flag == NON_PLOID) operation = CUT; - for (k = 0; k < b.b.n; k++) { g->seq[b.b.a[k]>>1].c = ALTER_LABLE; @@ -12296,7 +12611,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio) } } - + if(link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); } } } @@ -12319,7 +12634,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio) } int asg_arc_cut_trio_long_tip_primary_complex(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_threshold) +R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_threshold, hc_links* link) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, in, flag, operation; @@ -12364,32 +12679,6 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_thre return_flag = get_unitig(g, ug, a_convex[i].v, &convex, &ll, &tmp, &max_stop_nodeLen, &max_stop_baseLen, stops_threshold, NULL); - - /**********************for debug************************/ - // uint32_t debug_return_flag, debug_convex; - // long long debug_ll, debug_max_stop; - - // debug_return_flag = untig_detect_single_path_with_dels_n_stops(g, ug, - // a_convex[i].v, &debug_convex, &debug_ll, &debug_max_stop, NULL, stops_threshold); - - // if(debug_convex != convex || debug_ll != ll || debug_max_stop != max_stop_nodeLen) - // { - // fprintf(stderr, "ERROR\n"); - // } - // if(debug_return_flag != return_flag) - // { - // if( - // !((debug_return_flag == TWO_OUTPUT && return_flag == MUL_OUTPUT) - // || - // (debug_return_flag == TWO_INPUT && return_flag == MUL_INPUT)) - // ) - // { - // fprintf(stderr, "ERROR\n"); - // } - // } - /**********************for debug************************/ - - if(convexLen < ll*drop_ratio && max_stop_nodeLen >= ll*MAX_STOP_RATE) { @@ -12420,6 +12709,8 @@ R_to_U* ruIndex, uint32_t min_edge_length, float drop_ratio, uint32_t stops_thre } } + if(link && operation != CUT) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + break; } } @@ -12517,7 +12808,8 @@ long long* base_maxLen, long long* base_maxLen_i, uint32_t stops_threshold, buf_ } int asg_arc_cut_trio_long_equal_tips_assembly(asg_t *g, ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t trio_flag) +ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t trio_flag, +hc_links* link) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, flag, is_hap, n_tips, return_flag, k; @@ -12604,6 +12896,8 @@ ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_ asg_seq_drop(g, b.b.a[k]>>1); } + if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); + is_hap++; } @@ -12835,7 +13129,7 @@ R_to_U* ruIndex, uint32_t positive_flag, float drop_rate) } int asg_arc_cut_trio_long_equal_tips_assembly_complex(asg_t *g, ma_ug_t *ug, asg_t *read_sg, -ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t stops_threshold) +ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_t stops_threshold, hc_links* link) { double startTime = Get_T(); uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0, convex, in, flag; @@ -12882,31 +13176,6 @@ ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_ return_flag = get_unitig(g, ug, a_convex[i].v, &convex, &tmp, &ll, &max_stop_nodeLen, &max_stop_baseLen, stops_threshold, NULL); - - /**********************for debug************************/ - // uint32_t debug_return_flag, debug_convex; - // long long debug_ll, debug_max_stop; - - // debug_return_flag = untig_detect_single_path_with_dels_contigLen_complex(g, - // a_convex[i].v, &debug_convex, &debug_ll, &debug_max_stop, NULL, stops_threshold); - - // if(debug_convex != convex || debug_ll != ll || debug_max_stop != max_stop_baseLen) - // { - // fprintf(stderr, "ERROR2\n"); - // } - // if(debug_return_flag != return_flag) - // { - // if( - // !((debug_return_flag == TWO_OUTPUT && return_flag == MUL_OUTPUT) - // || - // (debug_return_flag == TWO_INPUT && return_flag == MUL_INPUT)) - // ) - // { - // fprintf(stderr, "ERROR3\n"); - // } - // } - /**********************for debug************************/ - if(ll>convexLen && max_stop_baseLen>=ll*MAX_STOP_RATE) { flag = check_different_haps(g, ug, read_sg, a_convex[convex_i].v, a_convex[i].v, @@ -12928,6 +13197,7 @@ ma_hit_t_alloc* reverse_sources, long long miniedgeLen, R_to_U* ruIndex, uint32_ asg_seq_drop(g, b.b.a[k]>>1); } + if(link) collect_reverse_unitigs(&b_0, &b_1, link, ug, read_sg); ///lable the primary one b_0.b.n = 0; @@ -13100,7 +13370,6 @@ R_to_U* ruIndex) n_reduced++; b_0.b.n = 0; read_num = 0; - ///get_long_tip_length(g, &(ug->u), v_beg^1, &v_end, &b_0); get_unitig(g, ug, v_beg^1, &v_end, &ll, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 1, &b_0); for (k = 0; k < b_0.b.n; k++) { @@ -13282,6 +13551,41 @@ ma_ug_t* copy_untig_graph(ma_ug_t *src) return ug; } +asg_t* copy_read_graph(asg_t *src) +{ + asg_t *dest = NULL; + dest = asg_init(); + + dest->r_seq = src->r_seq; + dest->m_seq = dest->n_seq = src->n_seq; + dest->seq = (asg_seq_t*)malloc(dest->n_seq * sizeof(asg_seq_t)); + memcpy(dest->seq, src->seq, sizeof(asg_seq_t)*dest->n_seq); + + dest->m_arc = dest->n_arc = src->n_arc; + dest->arc = (asg_arc_t*)malloc(dest->n_arc*sizeof(asg_arc_t)); + memcpy(dest->arc, src->arc, sizeof(asg_arc_t)*dest->n_arc); + + dest->is_srt = src->is_srt; + dest->is_symm = src->is_symm; + dest->idx = (uint64_t*)malloc(dest->n_seq*2*8); + memcpy(dest->idx, src->idx, dest->n_seq*2*8); + asg_cleanup(dest); + + if(src->seq_vis) + { + dest->seq_vis = (uint8_t*)malloc(dest->n_seq*2*sizeof(uint8_t)); + memcpy(dest->seq_vis, src->seq_vis, dest->n_seq*2*sizeof(uint8_t)); + } + + if(src->n_F_seq > 0 && src->F_seq) + { + dest->n_F_seq = src->n_F_seq; + dest->F_seq = (ma_utg_t*)malloc(dest->n_F_seq*sizeof(ma_utg_t)); + memcpy(dest->F_seq, src->F_seq, dest->n_F_seq*sizeof(ma_utg_t)); + } + return dest; +} + void clean_trio_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, @@ -13296,13 +13600,12 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) ///print_untig((ug), 61955, "i-0:", 0); asg_pop_bubble_primary_trio(ug, bubble_dist, trio_flag, DROP); - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL); magic_trio_phasing(g, ug, read_g, coverage_cut, sources, reverse_sources, 2, ruIndex, trio_flag, trio_drop_rate); ///drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); /**********debug**********/ if(just_bubble_pop == 0) { - ///asg_cut_tip_primary(g, ug, tipsLen); cut_trio_tip_primary(g, ug, tipsLen, trio_flag, 0, read_g, reverse_sources, ruIndex, 2); } @@ -13319,12 +13622,10 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) if(just_bubble_pop == 0) { ///need consider tangles - asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio); - asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, trio_flag); - asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, - 2, tip_drop_ratio, stops_threshold); - asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, - 2, ruIndex, stops_threshold); + asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, NULL); + asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, trio_flag, NULL); + asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, NULL); + asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, NULL); ///print_debug_gfa(read_g, ug, coverage_cut, "debug_chimeric", sources, ruIndex, asm_opt.max_hang_Len, asm_opt.min_overlap_Len); detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); @@ -13334,11 +13635,10 @@ float drop_ratio, uint32_t trio_flag, float trio_drop_rate) /**********debug**********/ cur_cons = get_graph_statistic(g); } - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, trio_flag, DROP, NULL); if(just_bubble_pop == 0) { - ///asg_cut_tip_primary(g, ug, tipsLen); cut_trio_tip_primary(g, ug, tipsLen, trio_flag, 0, read_g, reverse_sources, ruIndex, 2); } @@ -13360,7 +13660,7 @@ void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* rever long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, -float drop_ratio) +float drop_ratio, hc_links* link) { #define T_ROUND 2 asg_t *g = ug->g; @@ -13369,7 +13669,7 @@ float drop_ratio) redo: asg_pop_bubble_primary_trio(ug, bubble_dist, (uint32_t)-1, DROP); - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link); if(just_bubble_pop == 0) { @@ -13387,16 +13687,12 @@ float drop_ratio) if(just_bubble_pop == 0) { ///need consider tangles - asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, - 2, tip_drop_ratio); - asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1); - asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, - 2, tip_drop_ratio, stops_threshold); - asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, - 2, ruIndex, stops_threshold); - detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, - ruIndex); - + asg_arc_cut_trio_long_tip_primary(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, link); + asg_arc_cut_trio_long_equal_tips_assembly(g, ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, link); + asg_arc_cut_trio_long_tip_primary_complex(g, ug, read_g, reverse_sources, ruIndex, 2, tip_drop_ratio, stops_threshold, link); + asg_arc_cut_trio_long_equal_tips_assembly_complex(g, ug, read_g, reverse_sources, 2, ruIndex, stops_threshold, link); + detect_chimeric_by_topo(g, ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, ruIndex); + if(round != T_ROUND) { unitig_arc_del_short_diploid_by_length_topo(g, ug, drop_ratio, asm_opt.max_short_tip, @@ -13405,21 +13701,17 @@ float drop_ratio) } cur_cons = get_graph_statistic(g); } - untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP); + untig_asg_arc_simple_large_bubbles_trio(ug, read_g, reverse_sources, 2, ruIndex, (uint32_t)-1, DROP, link); if(just_bubble_pop == 0) { cut_trio_tip_primary(g, ug, tipsLen, (uint32_t)-1, 0, read_g, reverse_sources, ruIndex, 2); } - - resolve_tangles(ug, read_g, reverse_sources, 20, 100, 0.05, 0.2, ruIndex, (uint32_t)-1, drop_ratio); drop_semi_circle(ug, g, read_g, reverse_sources, ruIndex); - unitig_arc_del_short_diploid_by_length_topo(g, ug, drop_ratio, asm_opt.max_short_tip, reverse_sources, 0, 1); - if(round > 0) { if(round != T_ROUND) @@ -13475,8 +13767,7 @@ uint8_t is_final_check, float double_check_rate, uint8_t flag, float drop_rate) u = &((ug)->u.a[v]); if(u->m == 0) continue; if((get_real_length(nsg, v<<1, NULL)!=0) - && (get_real_length(nsg, ((v<<1)^1), NULL)!=0)) continue; - + && (get_real_length(nsg, ((v<<1)^1), NULL)!=0)) continue;///check tig flag_occ = non_flag_occ = hap_label_occ = 0; for (k = 0; k < u->n; k++) { @@ -13487,8 +13778,7 @@ uint8_t is_final_check, float double_check_rate, uint8_t flag, float drop_rate) if(R_INF.trio_flag[rId] == flag) flag_occ++; if(R_INF.trio_flag[rId] != flag) non_flag_occ++; } - - if(hap_label_occ == u->n) continue; + if(is_final_check == 0 && hap_label_occ == u->n) continue; ///if(is_double_check && non_flag_occ < u->n*DOUBLE_CHECK_THRES) continue; ///if(is_double_check && non_flag_occ < u->n*double_check_rate) continue; if(is_final_check) @@ -13520,13 +13810,12 @@ uint8_t is_final_check, float double_check_rate, uint8_t flag, float drop_rate) } } - if(non_flag_occ > ((non_flag_occ+flag_occ)*drop_rate)) { if(if_primary_unitig(u, read_g, coverage_cut, sources, ruIndex, primary_flag)) { continue; - } + } if(u->m != 0) { u->circ = u->end = u->len = u->m = u->n = u->start = 0; @@ -13546,7 +13835,6 @@ uint8_t is_final_check, float double_check_rate, uint8_t flag, float drop_rate) - void get_candidate_uids(asg_t* nsg, ma_utg_t* nsu, kvec_t_u64_warp* u_vecs, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex) { @@ -13692,7 +13980,7 @@ ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, float double_check_rate) for (k = 0; k < nsu->n; k++) { rId = nsu->a[k]>>33; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(read_g->seq[rId].c)); } } @@ -14073,13 +14361,10 @@ kvec_asg_arc_t_warp* new_rtg_edges) { asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq; - /** - kvec_t_u32_warp new_rtg_nodes; - kv_init(new_rtg_nodes.a); - **/ + purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 1); + drop_ratio, 1, 1, NULL); if(asm_opt.recover_atg_cov_min == -1024) { asm_opt.recover_atg_cov_max = asm_opt.hom_global_coverage/HOM_PEAK_RATE; @@ -14142,6 +14427,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) renew_utg(ug, read_g, new_rtg_edges); } + update_unitig_graph((*ug), read_g, coverage_cut, sources, reverse_sources, ruIndex, 1, FINAL_DOUBLE_CHECK_THRES, flag, drop_rate); @@ -14153,21 +14439,18 @@ kvec_asg_arc_t_warp* new_rtg_edges) ///delete_useless_nodes(ug); delete_useless_trio_nodes(ug, read_g, coverage_cut, sources, ruIndex); + if(asm_opt.purge_level_trio == 1) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 1, 0); + drop_ratio, 1, 0, NULL); ///delete_useless_nodes(ug); delete_useless_trio_nodes(ug, read_g, coverage_cut, sources, ruIndex); } set_drop_trio_flag(*ug); - - /** - kv_destroy(new_rtg_nodes.a); - **/ } @@ -14189,15 +14472,16 @@ int debug_untig_length(ma_ug_t *g, uint32_t tipsLen, const char* name) -void output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, +ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, -float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench) { char* gfa_name = (char*)malloc(strlen(output_file_name)+100); sprintf(gfa_name, "%s.%s.p_ctg.gfa", output_file_name, (flag==FATHER?"hap1":"hap2")); fprintf(stderr, "Writing %s to disk... \n", gfa_name); - FILE* output_file = fopen(gfa_name, "w"); + FILE* output_file = NULL; + if(is_bench == 0) output_file = fopen(gfa_name, "w"); ma_ug_t *ug = NULL; ug = ma_ug_gen(sg); @@ -14209,10 +14493,29 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) adjust_utg_by_trio(&ug, sg, flag, TRIO_THRES, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, max_hang, min_ovlp, &new_rtg_edges); + + if(asm_opt.b_low_cov > 0) + { + break_ug_contig(&ug, sg, &R_INF, coverage_cut, sources, ruIndex, &new_rtg_edges, max_hang, min_ovlp, + &asm_opt.b_low_cov, NULL, asm_opt.m_rate); + } + + if(asm_opt.b_high_cov > 0) + { + break_ug_contig(&ug, sg, &R_INF, coverage_cut, sources, ruIndex, &new_rtg_edges, max_hang, min_ovlp, + NULL, &asm_opt.b_high_cov, asm_opt.m_rate); + } + ///debug_utg_graph(ug, sg, 0, 0); ///debug_untig_length(ug, tipsLen, gfa_name); ///print_untig_by_read(ug, "m64011_190901_095311/125831121/ccs", 2310925, "end"); ma_ug_seq(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp); + if(is_bench) + { + free(gfa_name); + kv_destroy(new_rtg_edges.a); + return ug; + } ma_ug_print(ug, &R_INF, sg, coverage_cut, sources, ruIndex, (flag==FATHER?"h1tg":"h2tg"), output_file); fclose(output_file); @@ -14225,13 +14528,14 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp) sprintf(gfa_name, "%s.%s.p_ctg.lowQ.bed", output_file_name, (flag==FATHER?"hap1":"hap2")); output_file = fopen(gfa_name, "w"); ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, - max_hang, min_ovlp, asm_opt.bed_inconsist_rate, (flag==FATHER?"h1tg":"h2tg"), output_file); + max_hang, min_ovlp, asm_opt.bed_inconsist_rate, (flag==FATHER?"h1tg":"h2tg"), output_file, NULL); fclose(output_file); } free(gfa_name); ma_ug_destroy(ug); kv_destroy(new_rtg_edges.a); + return NULL; } @@ -14475,12 +14779,91 @@ static void asg_bub_backtrack_primary(asg_t *g, uint32_t v0, buf_t *b) } +// in a resolved bubble, mark unused vertices and arcs as "reduced" +void asg_bub_backtrack_primary_length(asg_t *g, ma_ug_t *utg, uint32_t v0, buf_t *b, uint64_t* path_base_len, uint64_t* path_nodes) +{ + uint32_t i, v, u, nv; + uint64_t len = 0, node = 0; + ///b->S.a[0] is the sink of this bubble + asg_arc_t *av = NULL; + + ///v is the sink of this bubble + v = b->S.a[0]; + len = 0; + ///recover node + while (1) + { + u = b->a[v].p; // u->v + if(u == v0) break; + if(v == b->S.a[0]) + { + len += g->seq[u>>1].len; + } + else + { + nv = asg_arc_n(g, u); + av = asg_arc_a(g, u); + for (i = 0; i < nv; ++i) + { + if(av[i].del) continue; + if(av[i].v == v) break; + } + ///if(i == nv) fprintf(stderr, "ERROR\n"); + len += (uint32_t)av[i].ul; + } + + if(utg) + { + node += utg->u.a[u>>1].n; + } + else + { + node++; + } + + v = u; + } + + if(path_base_len) (*path_base_len) = len; + if(path_nodes) (*path_nodes) = node; +} +// in a resolved bubble, mark unused vertices and arcs as "reduced" +int asg_bub_backtrack_check_switch(asg_t *g, ma_ug_t *utg, uint32_t v0, buf_t *b) +{ + uint32_t v, k, rId, father_occ = 0, mother_occ = 0; + ma_utg_t* p = NULL; + ///b->S.a[0] is the sink of this bubble + ///v is the sink of this bubble + v = b->S.a[0]; + ///recover node + do { + uint32_t u = b->a[v].p; // u->v + if(v != b->S.a[0]) + { + p = &(utg->u.a[v>>1]); + for (k = 0; k < p->n; k++) + { + rId = p->a[k]>>33; + if(R_INF.trio_flag[rId] == FATHER) father_occ++; + if(R_INF.trio_flag[rId] == MOTHER) mother_occ++; + if(R_INF.trio_flag[rId] != AMBIGU) continue; + R_INF.trio_flag[rId] = DROP; + } + } + v = u; + } while (v != v0); + + if(father_occ > 0 && mother_occ > 0) return 1; + + return 0; +} // pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well -uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, -uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop) +uint64_t asg_bub_pop1_primary_trio_switch_check(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, +uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, +int* is_switch) { uint32_t i, n_pending = 0, is_first = 1, cur_m, cur_c, cur_np, cur_nc, to_replace, n_tips, tip_end; uint64_t n_pop = 0; @@ -14702,8 +15085,248 @@ uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop) if (i < nv || b->S.n == 0) goto pop_reset; } while (b->S.n > 1 || n_pending); + if(is_switch) (*is_switch) = asg_bub_backtrack_check_switch(g, utg, v0, b); + if(is_pop) asg_bub_backtrack_primary(g, v0, b); + if(path_base_len || path_nodes) asg_bub_backtrack_primary_length(g, utg, v0, b, path_base_len, path_nodes); + + + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_t *t = &b->a[b->b.a[i]]; + t->s = t->c = t->d = t->m = t->nc = t->np = 0; + } + return n_pop; +} + + + +uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, +uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes) +{ + uint32_t i, n_pending = 0, is_first = 1, cur_m, cur_c, cur_np, cur_nc, to_replace, n_tips, tip_end; + uint64_t n_pop = 0; + long long cur_weight = -1, max_weight = -1; + ///if this node has been deleted + if (g->seq[v0>>1].del || g->seq[v0>>1].c == ALTER_LABLE) return 0; // already deleted + ///if ((uint32_t)g->idx[v0] < 2) return 0; // no bubbles + if(get_real_length(g, v0, NULL)<2) return 0; + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[v0].c = b->a[v0].d = b->a[v0].m = b->a[v0].nc = b->a[v0].np = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, v0); + n_tips = 0; + tip_end = (uint32_t)-1; + uint32_t non_positive_flag = (uint32_t)-1; + if(positive_flag == FATHER) non_positive_flag = MOTHER; + if(positive_flag == MOTHER) non_positive_flag = FATHER; + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), d = b->a[v].d, c = b->a[v].c, m = b->a[v].m, nc = b->a[v].nc, np = b->a[v].np; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + ///why we have this assert? + ///assert(nv > 0); + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l + binfo_t *t = &b->a[w]; + ///that means there is a circle, directly terminate the whole bubble poping + ///if (w == v0) goto pop_reset; + if ((w>>1) == (v0>>1)) goto pop_reset; + /****************************may have bugs********************************/ + ///important when poping at long untig graph + if(is_first) l = 0; + /****************************may have bugs********************************/ + + ///if this edge has been deleted + if (av[i].del) continue; + + ///push the edge + ///high 32-bit of g->idx[v] is the start point of v's edges + //so here is the point of this specfic edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + ///find a too far path? directly terminate the whole bubble poping + if (d + l > (uint32_t)max_dist) break; // too far + + ///if this node + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p is the parent node of + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l; + /****************************may have bugs********************************/ + cur_c = cur_m = cur_np = 0; cur_nc = 1; + if(utg) + { + cur_c = get_num_trio_flag(utg, w>>1, positive_flag); + cur_m = get_num_trio_flag(utg, w>>1, negative_flag); + cur_np = 0; + if(non_positive_flag != (uint32_t)-1) + { + cur_np = get_num_trio_flag(utg, w>>1, non_positive_flag); + } + cur_nc = utg->u.a[(w>>1)].n; + } + + + t->c = c + cur_c; + t->m = m + cur_m; + t->nc = nc + cur_nc; + t->np = np + cur_np; + /****************************may have bugs********************************/ + ///incoming edges of w + ///t->r = count_out(g, w^1); + t->r = get_real_length(g, w^1, NULL); + ++n_pending; + } else { // visited before + /****************************may have bugs********************************/ + cur_c = cur_m = cur_np = 0; cur_nc = 1; + if(utg) + { + cur_c = get_num_trio_flag(utg, w>>1, positive_flag); + cur_m = get_num_trio_flag(utg, w>>1, negative_flag); + cur_np = 0; + if(non_positive_flag != (uint32_t)-1) + { + cur_np = get_num_trio_flag(utg, w>>1, non_positive_flag); + } + cur_nc = utg->u.a[(w>>1)].n; + } + ///BUG: select the path with less negative_flag, less non_positive_flag, more positive_flag, more distance + ///FIXED: select the path with less (negative_flag+non_positive_flag), more positive_flag, more distance + to_replace = 0; + + /****************************may have bugs********************************/ + cur_weight = (long long)(c + cur_c) - ((long long)(m + cur_m) + (long long)(np + cur_np)); + max_weight = (long long)t->c - ((long long)t->m + (long long)t->np); + if(cur_weight > max_weight) + { + to_replace = 1; + } + else if(cur_weight == max_weight) + { + if(nc + cur_nc > t->nc) + { + to_replace = 1; + } + else if(nc + cur_nc == t->nc) + { + if(d + l > t->d) + { + to_replace = 1; + } + } + } + /****************************may have bugs********************************/ + + /** + if(((m + cur_m) + (np + cur_np)) < (t->m + t->np)) + { + to_replace = 1; + } + else if(((m + cur_m) + (np + cur_np)) == (t->m + t->np)) + { + if(c + cur_c > t->c) + { + to_replace = 1; + } + else if(c + cur_c == t->c) + { + if(nc + cur_nc > t->nc) + { + to_replace = 1; + } + else if(nc + cur_nc == t->nc) + { + if(d + l > t->d) + { + to_replace = 1; + } + } + + } + } + **/ + + + if(to_replace) + { + t->p = v; + t->m = m + cur_m; + t->c = c + cur_c; + t->nc = nc + cur_nc; + t->np = np + cur_np; + } + ///c is the weight (is very likely the number of node in this edge) of the parent node + ///select the longest edge (longest meams most reads/longest edge) + // if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; + // if (c + 1 > t->c) t->c = c + 1; + /****************************may have bugs********************************/ + ///update len(v0->w) + ///node: t->d is not the length from this node's parent + ///it is the shortest edge + if (d + l < t->d) t->d = d + l; // update dist + } + ///assert(t->r > 0); + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = get_real_length(g, w, NULL); + /****************************may have bugs for bubble********************************/ + if(x > 0) + { + kv_push(uint32_t, b->S, w); + } + else + { + ///at most one tip + if(n_tips != 0) goto pop_reset; + n_tips++; + tip_end = w; + } + /****************************may have bugs for bubble********************************/ + --n_pending; + } + } + is_first = 0; + //if found a tip + /****************************may have bugs for bubble********************************/ + if(n_tips == 1) + { + if(tip_end != (uint32_t)-1 && n_pending == 0 && b->S.n == 0) + { + kv_push(uint32_t, b->S, tip_end); + break; + } + else + { + goto pop_reset; + } + } + /****************************may have bugs for bubble********************************/ + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) goto pop_reset; + } while (b->S.n > 1 || n_pending); if(is_pop) asg_bub_backtrack_primary(g, v0, b); + if(path_base_len || path_nodes) asg_bub_backtrack_primary_length(g, utg, v0, b, path_base_len, path_nodes); + n_pop = 1; pop_reset: @@ -14736,7 +15359,7 @@ int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_fla for (i = 0; i < nv; ++i) // asg_bub_pop1() may delete some edges/arcs if (!av[i].del) ++n_arc; if (n_arc > 1) - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); } free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); if (n_pop) asg_cleanup(g); @@ -15688,10 +16311,7 @@ buf_t* bb, uint32_t** r, size_t* rm, size_t* rn, uint8_t* visit, ma_ug_t *ug, ui uint32_t vEnd, threshold, num_reads = 0, i, k, v, end = (uint32_t)-1, in = 0, vELen, tmp; long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; - /** - bb->b.n = 0; - threshold = get_long_tip_length(nsg, ut_v, vBeg, &vEnd, bb); - **/ + bb->b.n = 0; if(get_unitig(nsg, ug, vBeg, &vEnd, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, bb)==LOOP) @@ -15719,10 +16339,6 @@ buf_t* bb, uint32_t** r, size_t* rm, size_t* rn, uint8_t* visit, ma_ug_t *ug, ui if(in != 1) { ///get current untig length - /** - bb->b.n = 0; - vELen = get_long_tip_length(nsg, ut_v, v, &vEnd, bb); - **/ bb->b.n = 0; if(get_unitig(nsg, ug, v, &vEnd, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, bb)==LOOP) @@ -15788,10 +16404,6 @@ buf_t* bb, uint32_t** r, size_t* rm, size_t* rn, uint8_t* visit, ma_ug_t *ug, ui { kdq_push(uint32_t, buf, av[k].v); - /** - bb->b.n = 0; - get_long_tip_length(nsg, ut_v, av[k].v, &vEnd, bb); - **/ bb->b.n = 0; get_unitig(nsg, ug, av[k].v, &vEnd, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, bb); @@ -15823,10 +16435,6 @@ buf_t* bb, uint32_t** r, size_t* rm, size_t* rn, uint8_t* visit, ma_ug_t *ug, ui { kdq_push(uint32_t, buf, av[k].v); - /** - bb->b.n = 0; - get_long_tip_length(nsg, ut_v, av[k].v, &vEnd, bb); - **/ bb->b.n = 0; get_unitig(nsg, ug, av[k].v, &vEnd, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, bb); @@ -15866,148 +16474,6 @@ buf_t* bb, uint32_t** r, size_t* rm, size_t* rn, uint8_t* visit, ma_ug_t *ug, ui } } - -int explore_graph_back(asg_t *nsg, uint32_t start, uint32_t threshold, float single_threshold, -float l_untig_rate_threshold, long long minLongUntig, long long maxShortUntig, uint32_t ignore_d, -uint32_t** r, size_t* rm, size_t* rn, uint8_t* visit, ma_utg_v* ut_v, uint32_t* r_ID) -{ - (*r_ID) = (uint32_t)-1; - uint32_t nv; - asg_arc_t *av; - - kvec_t(uint32_t) u_vecs; - kv_init(u_vecs); - if(r && rm && rn) kv_reuse(u_vecs, 0, (*rm), (*r)); - - memset(visit, 0, nsg->n_seq); - kdq_t(uint32_t) *buf; - buf = kdq_init(uint32_t); - - uint32_t num_reads = 0; - uint32_t v = start, k; - uint32_t end = (uint32_t)-1; - uint32_t in = 0; - Set_vis(visit, v, ignore_d); - Set_vis(visit, v^1, ignore_d); - - if(nsg->seq[v>>1].del) - { - in = (uint32_t)-1; - goto termi; - } - - kdq_push(uint32_t, buf, v); - while (kdq_size(buf) != 0) - { - in++; - v = *(kdq_pop(uint32_t, buf)); - - ///in == 1 means the start node, it is useless - if(in != 1) - { - ///first long unitig except the start node - if(check_long_tip(*ut_v, v>>1, minLongUntig, - maxShortUntig, l_untig_rate_threshold, threshold)) - { - if(end == (uint32_t)-1) - { - end = v; - continue; - } - else - { - in = (uint32_t)-1; - break; - } - } - - if(EvaluateLen(*ut_v, v>>1) > (threshold * single_threshold)) - { - in = (uint32_t)-1; - break; - } - - num_reads = num_reads + EvaluateLen(*ut_v, v>>1); - if(num_reads > threshold) - { - in = (uint32_t)-1; - break; - } - - if(r && rm && rn) kv_push(uint32_t, u_vecs, v); - } - - - - nv = asg_arc_n(nsg, v); - av = asg_arc_a(nsg, v); - for(k = 0; k < nv; k++) - { - if(av[k].del) continue; - - if(av[k].v == start) - { - in = (uint32_t)-1; - goto termi; - } - - if(Get_vis(visit,av[k].v,ignore_d)==0) - { - Set_vis(visit,av[k].v,ignore_d); - kdq_push(uint32_t, buf, av[k].v); - } - } - - ///for start node, we just need one direction - if(in != 1 && ignore_d) - { - v = v^1; - nv = asg_arc_n(nsg, v); - av = asg_arc_a(nsg, v); - for(k = 0; k < nv; k++) - { - if(av[k].del) continue; - - if(av[k].v == start) - { - in = (uint32_t)-1; - goto termi; - } - - if(Get_vis(visit,av[k].v,ignore_d)==0) - { - Set_vis(visit,av[k].v,ignore_d); - kdq_push(uint32_t, buf, av[k].v); - } - } - } - } - - termi: - kdq_destroy(uint32_t, buf); - if(r && rm && rn) - { - (*rn) = u_vecs.n; - (*rm) = u_vecs.m; - (*r) = u_vecs.a; - } - - - (*r_ID) = end; - - if(in == (uint32_t)-1) - { - return 0; - } - else - { - return 1; - } - -} - - - void output_tangles(uint32_t startID, uint32_t endId, uint32_t* a, uint32_t n, const char* lable) { kvec_t(uint32_t) u_vecs; @@ -16375,13 +16841,6 @@ uint32_t* r_next_uID, R_to_U* ruIndex) } ///if the contig here is too small - /** - primaryLen = get_long_tip_length(nsg, &(ug->u), beg, &end, NULL); - if(primaryLen == 0 || primaryLen < minLongUntig || ug->u.a[beg>>1].circ) - { - return 0; - } - **/ primaryLen = get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, NULL); if(primaryLen == LOOP || nodeLen < minLongUntig) @@ -16455,9 +16914,6 @@ uint32_t* r_next_uID, R_to_U* ruIndex) // #define UNAVAILABLE (uint32_t)-1 // #define PLOID 0 // #define NON_PLOID 1 - /** - if(returnFlag == 1 && check_if_diploid_untigs(nsg, read_g, beg, next_uID, &(ug->u), - reverse_sources, minLongUntig-1, 0.3, b_0, b_1, 0, ruIndex) == 1)**/ if(returnFlag == 1 && check_different_haps(nsg, ug, read_g, beg, next_uID, reverse_sources, b_0, b_1, ruIndex, minLongUntig-1, 1) == PLOID) { @@ -16833,793 +17289,6 @@ R_to_U* ruIndex, buf_t* b_0, uint32_t beg, uint32_t end, uint32_t query_cId, flo return 0; } -#define fully_cover_rate 0.7 -#define extraord_rate 0.2 -///#define hap_seed 20 -#define hap_seed 5 -#define GetCid(x) ((uint64_t)(0x7fffffff)&(uint64_t)(x)) -#define GetOff(x) ((uint64_t)(x)>>33) -#define IfAlter(x) (GetCid((x))==(0x7fffffff)) -#define IfColor(x) (((uint64_t)(0x100000000)&(uint64_t)(x))!=0) -#define IfVisit(x) (((uint64_t)(0x80000000)&(uint64_t)(x))!=0) -#define SetVisit(x) ((x) = ((uint64_t)(0x80000000)|(uint64_t)(x))) -inline int get_useful_contig_advance(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, kvec_t_u64_warp* r_vecs, buf_t* b_0, uint64_t* contigBeg, asg_t *bi_g, -uint32_t currentId, float Hap_rate, uint32_t long_hap_overlap, float long_hap_overlap_rate, -uint32_t is_bi_edge) -{ - uint32_t i = 0, j, k, sLen, is_found, rId, beg, end, is_build; - uint32_t q_beg, q_end, t_beg, t_end, qLen, tLen; - uint32_t interval_q_beg, interval_q_end, interval_t_beg, interval_t_end; - uint64_t anchor_offset, offset; - uint64_t anchor_cId, cId; - - buf_t b_tid; - memset(&b_tid, 0, sizeof(buf_t)); - - kvec_t(Hap_Align) seed; - kv_init(seed); - - kvec_t(Hap_Align) alignment; - kv_init(alignment); - - Hap_Align x; - rIdContig iterator, iterator_tid; - iterator.b_0 = b_0; - iterator.offset = iterator.readI = iterator.untigI = 0; - if(r_vecs->a.n == 0) return -1; - ///number of reads in this contig - qLen = get_contig_len(ug, b_0); - - i = 0; - alignment.n = 0; - while (i < r_vecs->a.n) - { - ///there are three cases: - /** - * 1) u_vecs->a.a[i] has haplotype infor at the primary contigs - * 2) u_vecs->a.a[i] has haplotype infor at the alternative contigs, - * IfAlter(u_vecs->a.a[i])==1 - * 3) u_vecs->a.a[i] itself is labled with HAP_LABLE, - * IfColor(u_vecs->a.a[i])==1 - * 4) if u_vecs->a.a[i] has been labled - * IfVisit(u_vecs->a.a[i])==1 - * 4) u_vecs->a.a[i] does not have any haplotype infor. In this case, - * self_offsetLen is not consecutive - **/ - if(IfAlter(r_vecs->a.a[i]) || IfColor(r_vecs->a.a[i]) || IfVisit(r_vecs->a.a[i])) - { - i++; - continue; - } - - ///anchor_offset is the self offset, instead of the offset in anchor_cId - anchor_offset = GetOff(r_vecs->a.a[i]); anchor_cId = GetCid(r_vecs->a.a[i]); - sLen = 0; is_found = 0; seed.n = 0; - for (j = i; j < r_vecs->a.n; j++) - { - ///offset is the self offset at query - ///cId is the target Id (contig) - offset = GetOff(r_vecs->a.a[j]); - cId = GetCid(r_vecs->a.a[j]); - - ///means we go ahead one step at query - if(offset != anchor_offset) - { - ///sLen must be >=1 - if(is_found == 0) - { - break; - } - is_found = 0; - sLen++; - anchor_offset = offset; - } - - if(IfVisit(r_vecs->a.a[j]) || IfColor(r_vecs->a.a[j])) continue; - - if(cId == anchor_cId) - { - x.q_pos = GetOff(r_vecs->a.a[j]); - x.t_id = GetCid(r_vecs->a.a[j]); - x.is_color = 1; - x.t_pos = (uint32_t)-1; - SetVisit(r_vecs->a.a[j]); - is_found++; - kv_push(Hap_Align, seed, x); - } - } - - if(j == r_vecs->a.n && is_found > 0) - { - sLen++; - } - - - ///don't need to know how long of this seed - if(sLen >= hap_seed && seed.n > 0) - { - uint32_t inner_off = 0, pre_q_pos = (uint32_t)-1, RrId; - iterator.offset = iterator.readI = iterator.untigI = 0; - - ///for one vector, all t_id should be same; that is why we recover tid here - beg = (uint32_t)(contigBeg[seed.a[0].t_id]); - b_tid.b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(ug->g, &(ug->u), beg, &end, &b_tid); - iterator_tid.b_0 = &b_tid; - iterator_tid.offset = iterator_tid.readI = iterator_tid.untigI = 0; - ///we have already got qLen at the begining - tLen = get_contig_len(ug, &b_tid); - - ///just set t_pos - for (k = 0; k < seed.n; k++) - { - rId = get_rId_from_contig_by_offset(ug, &iterator, seed.a[k].q_pos); - ///actually we shouldn't have this case - if(rId == (uint32_t)-1) continue; - if(pre_q_pos != seed.a[k].q_pos) - { - inner_off = 0; - pre_q_pos = seed.a[k].q_pos; - } - else - { - inner_off++; - } - ///rId is the read ID at the - RrId = get_reverseId(read_g, reverse_sources, ruIndex, rId, seed.a[k].t_id, - inner_off); - ///if(RrId == (uint32_t)-1) fprintf(stderr, "ERROR1\n"); - ///for one vector, all t_id should be same - seed.a[k].t_pos = get_offset_from_contig_by_rId(ug, &iterator_tid, RrId); - // if(seed.a[k].t_pos == (uint32_t)-1) fprintf(stderr, "ERROR2\n"); - // uint32_t debug_uID, debug_is_Unitig; - // get_R_to_U(ruIndex, RrId, &debug_uID, &debug_is_Unitig); - // if(seed.a[k].t_id != debug_uID) fprintf(stderr, "ERROR1\n"); - } - - q_beg = t_beg = (uint32_t)-1; q_end = t_end = 0; - for (k = 0; k < seed.n; k++) - { - if(seed.a[k].t_pos < t_beg) t_beg = seed.a[k].t_pos; - if(seed.a[k].t_pos > t_end) t_end = seed.a[k].t_pos; - - if(seed.a[k].q_pos < q_beg) q_beg = seed.a[k].q_pos; - if(seed.a[k].q_pos > q_end) q_end = seed.a[k].q_pos; - } - - if(q_beg<=q_end && t_beg<=t_end && tLen > 0 && qLen > 0) - { - - ///exclude extraordinary points of t_pos - ///note here all q_pos are continual - ///here we should use base position, instead of the read offset - if((DIFF((q_end+1-q_beg), (t_end+1-t_beg))) > (q_end+1-q_beg)*extraord_rate) - { - uint32_t leftLen = 0, rightLen = 0, m = 0, median = 0, diff = (q_end+1-q_beg)*(1+extraord_rate)*0.5; - for (k = 0; k < seed.n; k++) - { - median += seed.a[k].t_pos; - } - median = median / seed.n; - - leftLen = median - diff; - if(median < diff) leftLen = 0; - - rightLen = median + diff; - if(rightLen >= tLen) rightLen = tLen - 1; - - t_beg = (uint32_t)-1; t_end = 0; m = 0; - for (k = 0; k < seed.n; k++) - { - if(seed.a[k].t_pos >= leftLen && seed.a[k].t_pos <= rightLen) - { - if(seed.a[k].t_pos < t_beg) t_beg = seed.a[k].t_pos; - if(seed.a[k].t_pos > t_end) t_end = seed.a[k].t_pos; - seed.a[m] = seed.a[k]; - m++; - } - } - seed.n = m; - - if(t_beg>t_end) goto direct_skip; - } - - ///here we know q_beg, q_end, qLen - ///and t_beg, t_end, tLen - ///there might be two directions: - ///a) query and target at the same direction - ///b) query and target at different direction - get_contig_overlap_interval(0, q_beg, q_end, qLen, - t_beg, t_end, tLen, &interval_q_beg, &interval_q_end, - &interval_t_beg, &interval_t_end); - - uint32_t flag_forward = get_haplotype_rate(ug, read_g, reverse_sources, ruIndex, b_0, - interval_q_beg, interval_q_end, anchor_cId, Hap_rate); - if(flag_forward) - { - x.t_id = anchor_cId; - x.q_pos = interval_q_beg; - x.t_pos = interval_q_end; - ///x.is_color = 0; - x.is_color = tLen; - kv_push(Hap_Align, alignment, x); - } - - get_contig_overlap_interval(1, q_beg, q_end, qLen, - t_beg, t_end, tLen, &interval_q_beg, &interval_q_end, - &interval_t_beg, &interval_t_end); - - - uint32_t flag_backward = get_haplotype_rate(ug, read_g, reverse_sources, ruIndex, b_0, - interval_q_beg, interval_q_end, anchor_cId, Hap_rate); - if(flag_backward) - { - x.t_id = anchor_cId; - x.q_pos = interval_q_beg; - x.t_pos = interval_q_end; - ///x.is_color = 1; - x.is_color = tLen; - kv_push(Hap_Align, alignment, x); - } - } - - } - - direct_skip: - i++; - } - - ///must sort here - radix_sort_Hap_Align_sort(alignment.a, alignment.a+alignment.n); - - uint32_t m = 0; - ///here is a bug - sLen = 0; anchor_cId = tLen = q_beg = q_end = (uint32_t)-1; - for (i = 0; i < alignment.n; i++) - { - if(anchor_cId != alignment.a[i].t_id) - { - if(anchor_cId != (uint32_t)-1) - { - alignment.a[m].t_id = anchor_cId; - alignment.a[m].q_pos = q_beg; - alignment.a[m].t_pos = q_end; - alignment.a[m].is_color = tLen; - m++; - } - anchor_cId = alignment.a[i].t_id; - sLen = 0; - } - if(alignment.a[i].t_pos < alignment.a[i].q_pos) continue; - if(alignment.a[i].t_pos - alignment.a[i].q_pos + 1 > sLen) - { - sLen = alignment.a[i].t_pos - alignment.a[i].q_pos + 1; - q_beg = alignment.a[i].q_pos; - q_end = alignment.a[i].t_pos; - tLen = alignment.a[i].is_color; - } - } - if(anchor_cId != (uint32_t)-1) - { - alignment.a[m].t_id = anchor_cId; - alignment.a[m].q_pos = q_beg; - alignment.a[m].t_pos = q_end; - alignment.a[m].is_color = tLen; - m++; - } - alignment.n = m; - - /**********************for debug****************************/ - // fprintf(stderr, "***alignment.m: %u\n", (uint32_t)alignment.n); - // for (i = 0; i < alignment.n; i++) - // { - // fprintf(stderr, "c_id: %u, beg: %u, end: %u, tLen: %u\n", - // alignment.a[i].t_id, alignment.a[i].q_pos, - // alignment.a[i].t_pos, alignment.a[i].is_color); - // } - /**********************for debug****************************/ - asg_arc_t *e; - for (i = 0; i < alignment.n; i++) - { - is_build = 0; - if(alignment.a[i].t_pos < alignment.a[i].q_pos) continue; - sLen = alignment.a[i].t_pos - alignment.a[i].q_pos + 1; - tLen = alignment.a[i].is_color; - ///if overlap is short, must fully cover one of the read - if(sLen < long_hap_overlap) - { - if(sLen >= (fully_cover_rate*tLen) || sLen >= (fully_cover_rate*qLen)) - { - is_build = 1; - } - } - else///if is long, can be partly cover one of the read - { - if(sLen >= (long_hap_overlap_rate*tLen) || sLen >= (long_hap_overlap_rate*qLen)) - { - is_build = 1; - } - } - - if(is_build) - { - e = asg_arc_pushp(bi_g); - e->del = 0; - e->ol = sLen; - e->ul = currentId; e->ul = e->ul << 32; e->ul = e->ul | (uint64_t)(qLen); - e->v = alignment.a[i].t_id; - - if(is_bi_edge) - { - e = asg_arc_pushp(bi_g); - e->del = 0; - e->ol = sLen; - e->ul = alignment.a[i].t_id; e->ul = e->ul << 32; e->ul = e->ul | (uint64_t)(qLen); - e->v = currentId; - } - - } - } - /** - is_build = 0; - ///uint32_t long_hap_overlap, float long_hap_overlap_rate - ///if overlap is short, must fully cover one of the read - if(sLen < long_hap_overlap) - { - if(sLen >= (fully_cover_rate*tLen) || sLen >= (fully_cover_rate*qLen)) - { - is_build = 1; - } - } - else///if is long, can be partly cover one of the read - { - if(sLen >= (long_hap_overlap_rate*tLen) || sLen >= (long_hap_overlap_rate*qLen)) - { - is_build = 1; - } - } - - if(is_build) - { - asg_arc_t *e; - e = asg_arc_pushp(bi_g); - e->del = 0; - e->ol = flag; - e->ul = i; e->ul = e->ul << 32; e->ul = e->ul | (uint64_t)(u_vecs.a.n); - e->v = cId; - } - **/ - - - /** - radix_sort_Hap_Align_sort(alignment.a, alignment.a+alignment.n); - fprintf(stderr, "alignment.n: %u, sLen: %u, q_beg: %u, q_end: %u\n", - (uint32_t)alignment.n, sLen, q_beg, q_end); - for (i = 0; i < alignment.n; i++) - { - fprintf(stderr, "dir: %u, c_id: %u, beg: %u, end: %u\n", - alignment.a[i].is_color, alignment.a[i].t_id, alignment.a[i].q_pos, - alignment.a[i].t_pos); - } - **/ - - - - - /** - uint32_t self_offset, uId; - ma_utg_t* reads; - iterator.offset = iterator.readI = iterator.untigI = 0; - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - rId = reads->a[k]>>33; - if(get_rId_from_contig_by_offset(ug, &iterator, self_offset)!=rId) - { - fprintf(stderr, "ERROR1\n"); - } - } - } - - iterator.offset = iterator.readI = iterator.untigI = 0; - for (long long debug_i = self_offset; debug_i >= 0; debug_i--) - { - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - if(debug_i != self_offset) continue; - rId = reads->a[k]>>33; - if(get_rId_from_contig_by_offset(ug, &iterator, self_offset)!=rId) - { - fprintf(stderr, "ERROR2: self_offset: %u\n", self_offset); - } - } - } - } - - - - - - - iterator.offset = iterator.readI = iterator.untigI = 0; - iterator.offset = iterator.readI = iterator.untigI = 0; - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - rId = reads->a[k]>>33; - if(get_offset_from_contig_by_rId(ug, &iterator, rId)!=self_offset) - { - fprintf(stderr, "ERROR3\n"); - } - } - } - - iterator.offset = iterator.readI = iterator.untigI = 0; - for (long long debug_i = self_offset; debug_i >= 0; debug_i--) - { - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - if(debug_i != self_offset) continue; - rId = reads->a[k]>>33; - if(get_offset_from_contig_by_rId(ug, &iterator, rId)!=self_offset) - { - fprintf(stderr, "ERROR33: self_offset: %u\n", self_offset); - } - } - } - } - **/ - - - kv_destroy(alignment); - kv_destroy(seed); - free(b_tid.b.a); - return 1; -} - -inline int get_useful_contig_advance_back(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, kvec_t_u64_warp* u_vecs, buf_t* b_0, uint64_t* contigBeg) -{ - uint32_t i = 0, j, k, sLen, is_found, rId, beg, end; - uint32_t q_beg, q_end, t_beg, t_end, qLen, tLen; - uint32_t interval_q_beg, interval_q_end, interval_t_beg, interval_t_end; - uint64_t anchor_offset, offset; - uint64_t anchor_cId, cId; - - buf_t b_tid; - memset(&b_tid, 0, sizeof(buf_t)); - - kvec_t(Hap_Align) seed; - kv_init(seed); - Hap_Align x; - rIdContig iterator, iterator_tid; - iterator.b_0 = b_0; - iterator.offset = iterator.readI = iterator.untigI = 0; - if(u_vecs->a.n == 0) return -1; - - qLen = get_contig_len(ug, b_0); - - /**********************for debug****************************/ - // if(u_vecs->a.n > 20 && u_vecs->a.n < 50) - // { - // i = 0; - // fprintf(stderr,"*******\n"); - // for (i = 0; i < u_vecs->a.n; i++) - // { - // offset = u_vecs->a.a[i]>>33; - // cId = (uint32_t)(u_vecs->a.a[i]); - // fprintf(stderr, "((%u) off: %u, cId: %d, HAP: %u)\n", i, offset, (int)cId, - // ((u_vecs->a.a[i]&(uint64_t)(0x100000000))!=0)); - // } - // } - /**********************for debug****************************/ - - - i = 0; - while (i < u_vecs->a.n) - { - ///there are three cases: - /** - * 1) u_vecs->a.a[i] has haplotype infor at the primary contigs - * 2) u_vecs->a.a[i] has haplotype infor at the alternative contigs, - * (uint32_t)u_vecs->a.a[i]==(uint32_t)-1 - * 3) u_vecs->a.a[i] itself is labled with HAP_LABLE, - * u_vecs->a.a[i]&(uint64_t)(0x100000000)>0 && (uint32_t)u_vecs->a.a[i]==(uint32_t)-1 - * 4) u_vecs->a.a[i] does not have any haplotype infor. In this case, - * self_offsetLen is not consecutive - * - **/ - if((u_vecs->a.a[i]&(uint64_t)(0x100000000)) || ((uint32_t)u_vecs->a.a[i]==((uint32_t)-1))) - { - i++; - continue; - } - - - anchor_offset = (u_vecs->a.a[i]>>33); anchor_cId = (uint32_t)(u_vecs->a.a[i]); - sLen = 0; is_found = 0; seed.n = 0; - for (j = i; j < u_vecs->a.n; j++) - { - ///offset is the self offset at query - ///cId is the target Id (contig) - offset = u_vecs->a.a[j]>>33; - cId = (uint32_t)(u_vecs->a.a[j]); - - ///means we go ahead one step at query - if(offset != anchor_offset) - { - ///sLen must be >=1 - if(is_found == 0) - { - break; - } - is_found = 0; - sLen++; - anchor_offset = offset; - } - - if(cId == anchor_cId) - { - x.q_pos = u_vecs->a.a[j]>>33; - x.t_id = (uint32_t)(u_vecs->a.a[j]); - x.is_color = 1; - x.t_pos = (uint32_t)-1; - u_vecs->a.a[j] = u_vecs->a.a[j]|0xffffffff; - is_found++; - // if(seed.n > 0 && seed.a[seed.n-1].t_id == x.t_id && - // seed.a[seed.n-1].q_pos == x.q_pos) - // { - // seed.a[seed.n-1].is_color++; - // continue; - // } - kv_push(Hap_Align, seed, x); - } - } - - if(j == u_vecs->a.n && is_found > 0) - { - sLen++; - } - - - // for (j = 1; j < seed.n; j++) - // { - // if(seed.a[j].t_id != seed.a[j-1].t_id) - // { - // fprintf(stderr, "hehe\n"); - // fprintf(stderr, "sLen: %u, seed.n: %u\n", sLen, seed.n); - // } - - // if(seed.a[j].q_pos < seed.a[j-1].q_pos) - // { - // fprintf(stderr, "haha\n"); - // fprintf(stderr, "sLen: %u, seed.n: %u\n", sLen, seed.n); - // } - // } - - ///don't need to know how long of this seed - if(sLen >= hap_seed && seed.n > 0) - { - /**********************for debug****************************/ - ///if(u_vecs->a.n > 20 && u_vecs->a.n < 50) - ///fprintf(stderr, "\nsLen: %u, seed.n: %u, anchor_cId: %u\n", sLen, seed.n, anchor_cId); - /**********************for debug****************************/ - - uint32_t inner_off = 0, pre_q_pos = (uint32_t)-1, RrId; - iterator.offset = iterator.readI = iterator.untigI = 0; - - ///for one vector, all t_id should be same; that is why we recover tid here - beg = (uint32_t)(contigBeg[seed.a[0].t_id]); - b_tid.b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(ug->g, &(ug->u), beg, &end, &b_tid); - iterator_tid.b_0 = &b_tid; - iterator_tid.offset = iterator_tid.readI = iterator_tid.untigI = 0; - ///we have already got qLen at the begining - tLen = get_contig_len(ug, &b_tid); - - for (k = 0; k < seed.n; k++) - { - - rId = get_rId_from_contig_by_offset(ug, &iterator, seed.a[k].q_pos); - ///actually we shouldn't have this case - if(rId == (uint32_t)-1) continue; - if(pre_q_pos != seed.a[k].q_pos) - { - inner_off = 0; - pre_q_pos = seed.a[k].q_pos; - } - else - { - inner_off++; - } - - RrId = get_reverseId(read_g, reverse_sources, ruIndex, rId, seed.a[k].t_id, - inner_off); - ///if(RrId == (uint32_t)-1) fprintf(stderr, "ERROR1\n"); - ///for one vector, all t_id should be same - seed.a[k].t_pos = get_offset_from_contig_by_rId(ug, &iterator_tid, RrId); - // if(seed.a[k].t_pos == (uint32_t)-1) fprintf(stderr, "ERROR2\n"); - // uint32_t debug_uID, debug_is_Unitig; - // get_R_to_U(ruIndex, RrId, &debug_uID, &debug_is_Unitig); - // if(seed.a[k].t_id != debug_uID) fprintf(stderr, "ERROR1\n"); - - /**********************for debug****************************/ - ///if(u_vecs->a.n > 20 && u_vecs->a.n < 50) - // fprintf(stderr, "%u, q_pos: %u, t_id: %u, t_pos: %u, inner_off: %u, rId: %u, RrId: %u\n", - // k, seed.a[k].q_pos, seed.a[k].t_id, seed.a[k].t_pos, inner_off, rId, RrId); - /**********************for debug****************************/ - } - - q_beg = t_beg = (uint32_t)-1; q_end = t_end = 0; - for (k = 0; k < seed.n; k++) - { - if(seed.a[k].t_pos < t_beg) t_beg = seed.a[k].t_pos; - if(seed.a[k].t_pos > t_end) t_end = seed.a[k].t_pos; - - if(seed.a[k].q_pos < q_beg) q_beg = seed.a[k].q_pos; - if(seed.a[k].q_pos > q_end) q_end = seed.a[k].q_pos; - } - - - - if(q_beg<=q_end && t_beg<=t_end) - { - ///here we know q_beg, q_end, qLen - ///and t_beg, t_end, tLen - ///there might be two directions: - ///a) query and target at the same direction - ///b) query and target at different direction - get_contig_overlap_interval(0, q_beg, q_end, qLen, - t_beg, t_end, tLen, &interval_q_beg, &interval_q_end, - &interval_t_beg, &interval_t_end); - - - /**********************for debug****************************/ - ///if(u_vecs->a.n > 20 && u_vecs->a.n < 50) - { - fprintf(stderr, "q_beg: %u, q_end: %u, qLen: %u, t_beg: %u, t_end: %u, tLen: %u\n", - q_beg, q_end, qLen, t_beg, t_end, tLen); - - fprintf(stderr, "interval_q_beg: %u, interval_q_end: %u, interval_t_beg: %u, interval_t_end: %u\n", - interval_q_beg, interval_q_end, interval_t_beg, interval_t_end); - } - /**********************for debug****************************/ - - } - - } - i++; - } - - - - - - - /** - uint32_t self_offset, uId; - ma_utg_t* reads; - iterator.offset = iterator.readI = iterator.untigI = 0; - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - rId = reads->a[k]>>33; - if(get_rId_from_contig_by_offset(ug, &iterator, self_offset)!=rId) - { - fprintf(stderr, "ERROR1\n"); - } - } - } - - iterator.offset = iterator.readI = iterator.untigI = 0; - for (long long debug_i = self_offset; debug_i >= 0; debug_i--) - { - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - if(debug_i != self_offset) continue; - rId = reads->a[k]>>33; - if(get_rId_from_contig_by_offset(ug, &iterator, self_offset)!=rId) - { - fprintf(stderr, "ERROR2: self_offset: %u\n", self_offset); - } - } - } - } - - - - - - - iterator.offset = iterator.readI = iterator.untigI = 0; - iterator.offset = iterator.readI = iterator.untigI = 0; - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - rId = reads->a[k]>>33; - if(get_offset_from_contig_by_rId(ug, &iterator, rId)!=self_offset) - { - fprintf(stderr, "ERROR3\n"); - } - } - } - - iterator.offset = iterator.readI = iterator.untigI = 0; - for (long long debug_i = self_offset; debug_i >= 0; debug_i--) - { - for (j = 0, self_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - if(debug_i != self_offset) continue; - rId = reads->a[k]>>33; - if(get_offset_from_contig_by_rId(ug, &iterator, rId)!=self_offset) - { - fprintf(stderr, "ERROR33: self_offset: %u\n", self_offset); - } - } - } - } - **/ - - - - kv_destroy(seed); - free(b_tid.b.a); - return 1; -} - #define contig_seed 20 inline int get_useful_contig(kvec_t_u64_warp* u_vecs, float density, uint32_t miniLen, uint32_t* r_cId) @@ -17921,290 +17590,6 @@ void bi_paration(asg_t *bi_g, uint64_t* array, uint32_t bi_graph_Len) kdq_destroy(uint32_t, buf); } -void further_clean_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniLen, uint32_t bi_graph_Len, -uint32_t long_hap_overlap, float long_hap_overlap_rate, float lable_match_rate) -{ - asg_t *bi_g = NULL; - bi_g = asg_init(); - kvec_t(uint64_t) a; - kv_init(a); - kvec_t_u64_warp u_vecs; - kv_init(u_vecs.a); - asg_t* nsg = ug->g; - uint32_t v, n_vtx = nsg->n_seq * 2, beg = 0, end, uId, cId, rId, i, j, k, self_offset, self_label_offset; - uint64_t uInfor = 0; - ma_utg_t* reads; - ///int flag; - memset(visit, 0, nsg->n_seq); - /******************************set ruIndex*********************************/ - for (v = 0; v < n_vtx; ++v) - { - beg = v; - if(nsg->seq[v>>1].c == ALTER_LABLE || nsg->seq[beg>>1].del || visit[beg>>1]) - { - continue; - } - - ///why I have this line? It might be wrong - ///for example, if num(beg) == 0, and num(beg^1) == 2, beg is a unitig - if(get_real_length(nsg, beg, NULL)<=0 && get_real_length(nsg, beg^1, NULL)>0) - { - continue; - } - - - if(get_real_length(nsg, beg^1, NULL) == 1) - { - get_real_length(nsg, beg^1, &end); - if(get_real_length(nsg, end^1, NULL) == 1) - { - continue; - } - } - - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - - uInfor = 0; - //scan all untigs - for (i = 0; i < b_0->b.n; i++) - { - uId = b_0->b.a[i]>>1; - visit[uId] = 1; - reads = &(ug->u.a[uId]); - uInfor += reads->n; - } - /****************************may have bugs********************************/ - ///uInfor = uInfor << 32; uInfor = uInfor | (uint64_t)beg; - uInfor = uInfor << 33; uInfor = uInfor | (uint64_t)beg; - /****************************may have bugs********************************/ - kv_push(uint64_t, a, uInfor); - } - - ///sort by number of reads in a contig - radix_sort_arch64(a.a, a.a + a.n); - for (i = 0; i < (a.n>>1); ++i) - { - uInfor = a.a[i]; - a.a[i] = a.a[a.n - i - 1]; - a.a[a.n - i - 1] = uInfor; - } - - - for (v = 0; v < a.n; v++) - { - ///all untig ID of this contig - beg = (uint32_t)(a.a[v]); - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - cId = v; - ///separated contig - if(get_real_length(nsg, beg^1, NULL) == 0 && get_real_length(nsg, end, NULL) == 0) - { - a.a[v] = a.a[v] | (uint64_t)(0x100000000); - } - ///set the contig Id for each read - for (i = 0; i < b_0->b.n; i++) - { - uId = b_0->b.a[i]>>1; - reads = &(ug->u.a[uId]); - for (j = 0; j < reads->n; j++) - { - rId = reads->a[j]>>33; - set_R_to_U(ruIndex, rId, cId, 1); - } - } - } - /******************************set ruIndex*********************************/ - - - ///scan all contigs from longest one to the shortest one - for (i = 0; i < a.n; i++) - { - ///all untig ID of this contig - beg = (uint32_t)(a.a[i]); - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - - - u_vecs.a.n = 0; - ///scan all untigs of this contig - for (j = 0, self_offset = 0, self_label_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - rId = reads->a[k]>>33; - ///if(read_g->seq[rId].c == HAP_LABLE) continue; - ///self_offset is the offset of this read in contig - query_reverse_sources(read_g, reverse_sources, ruIndex, rId, self_offset, &u_vecs); - - if(read_g->seq[rId].c == HAP_LABLE) self_label_offset++; - } - } - - if(self_offset >= long_hap_overlap && self_label_offset >= (self_offset*lable_match_rate)) - { - asg_seq_set(bi_g, i, RED, 0); - } - else - { - asg_seq_set(bi_g, i, UNVISIT, 0); - } - - ///fprintf(stderr, "self_label_offset: %u, self_offset: %u\n", self_label_offset, self_offset); - - get_useful_contig_advance(ug, read_g, reverse_sources, ruIndex, &u_vecs, b_0, a.a, - bi_g, i, density, long_hap_overlap, long_hap_overlap_rate, 1); - } - - asg_cleanup(bi_g); - - ///fprintf(stderr, "***********n_seq: %u, n_arc: %u\n", bi_g->n_seq, bi_g->n_arc); - for (v = 0; v < bi_g->n_seq; v++) - { - uint32_t nv = asg_arc_n(bi_g, v), nw, w; - asg_arc_t *av = asg_arc_a(bi_g, v), *aw; - for (i = 0; i < nv; ++i) - { - w = av[i].v; - if(w == v) - { - av[i].del = 1; - continue; - } - nw = asg_arc_n(bi_g, w); - aw = asg_arc_a(bi_g, w); - for (j = 0; j < nw; j++) - { - if(aw[j].v == v) break; - } - - if(j == nw) av[i].del = 1; - } - } - - asg_cleanup(bi_g); - ///fprintf(stderr, "***********n_seq: %u, n_arc: %u\n", bi_g->n_seq, bi_g->n_arc); - - bi_paration(bi_g, a.a, bi_graph_Len); - - for (v = 0; v < bi_g->n_seq; v++) - { - beg = (uint32_t)(a.a[v]); - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - - if(bi_g->seq[v].len == BLACK) - { - for (i = 0; i < b_0->b.n; i++) - { - nsg->seq[(b_0->b.a[i])>>1].c = ALTER_LABLE; - } - - for (i = 0; i < b_0->b.n; i++) - { - asg_seq_drop(nsg, ((b_0->b.a[i])>>1)); - } - } - - - - - - - - - // fprintf(stderr, "cId: %u, beg>>1: %u, end>>1: %u, b_0->b.n: %u, Len: %u, type: %u\n", - // v, beg>>1, end>>1, (uint32_t)b_0->b.n, (uint32_t)(a.a[v]>>33), bi_g->seq[v].len); - // uint32_t nv = asg_arc_n(bi_g, v), w; - // asg_arc_t *av = asg_arc_a(bi_g, v); - // for (i = 0; i < nv; ++i) - // { - // w = av[i].v; - // fprintf(stderr, "w: %u\n", w); - // } - } - - - /** - for (i = 1; i < a.n; i++) - { - if((a.a[i]>>33) > (a.a[i-1]>>33)) - { - fprintf(stderr, "hehe\n"); - } - } - - - for (i = 0; i < a.n; i++) - { - uint32_t k, get_cId, tLen = 0, is_Unitig; - beg = (uint32_t)(a.a[i]); - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - for (j = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - reads = &(ug->u.a[uId]); - tLen += reads->n; - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - get_R_to_U(ruIndex, rId, &get_cId, &is_Unitig); - if(is_Unitig != 1 || get_cId != i) - { - fprintf(stderr, "###is_Unitig: %u, get_cId: %u, i: %u\n", - is_Unitig, get_cId, i); - } - } - } - - uint32_t qLen = 0, m; - for (m = 0; m < ruIndex->len; m++) - { - get_R_to_U(ruIndex, m, &get_cId, &is_Unitig); - if(get_cId == (uint32_t)-1 || is_Unitig != 1) continue; - if(get_cId == i) - { - qLen = 0; - for (j = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - reads = &(ug->u.a[uId]); - tLen += reads->n; - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - if(m == rId) - { - qLen = 1; - goto found; - } - } - } - found:; - if(qLen == 0) - { - fprintf(stderr, "***is_Unitig: %u, get_cId: %u, m: %u\n", - is_Unitig, get_cId, m); - } - } - } - } - **/ - - asg_cleanup(nsg); - free(a.a); - kv_destroy(u_vecs.a); - asg_destroy(bi_g); -} - void process_bi_graph(asg_t *bi_g) { @@ -18240,283 +17625,6 @@ void process_bi_graph(asg_t *bi_g) ///asg_symm(bi_g); } -void further_clean_untig_graph_trio(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t bi_graph_Len, -uint32_t long_hap_overlap, float long_hap_overlap_rate, float lable_match_rate) -{ - asg_t *bi_g = NULL; - bi_g = asg_init(); - kvec_t(uint64_t) a; - kv_init(a); - kvec_t_u64_warp u_vecs; - kv_init(u_vecs.a); - asg_t* nsg = ug->g; - uint32_t v, n_vtx = nsg->n_seq * 2, beg = 0, end, uId, cId, rId, i, j, k, self_offset, self_label_offset; - long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; - uint64_t uInfor = 0; - ma_utg_t* reads; - ///int flag; - memset(visit, 0, nsg->n_seq); - - - /****************************may have bugs********************************/ - for (v = 0; v < nsg->n_seq; ++v) - { - uId = v; - if(nsg->seq[uId].c != HAP_LABLE) continue; - reads = &(ug->u.a[uId]); - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - read_g->seq[rId].c = HAP_LABLE; - } - } - /****************************may have bugs********************************/ - - - - /******************************set ruIndex*********************************/ - for (v = 0; v < n_vtx; ++v) - { - beg = v; - if(nsg->seq[v>>1].c == ALTER_LABLE || nsg->seq[beg>>1].del || visit[beg>>1]) - { - continue; - } - - if(get_real_length(nsg, beg^1, NULL) == 1) - { - get_real_length(nsg, beg^1, &end); - if(get_real_length(nsg, end^1, NULL) == 1) continue; - } - - b_0->b.n = 0; - get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, b_0); - - uInfor = 0; - //scan all untigs - for (i = 0; i < b_0->b.n; i++) - { - uId = b_0->b.a[i]>>1; - visit[uId] = 1; - reads = &(ug->u.a[uId]); - uInfor += reads->n; - } - /****************************may have bugs********************************/ - ///uInfor = uInfor << 32; uInfor = uInfor | (uint64_t)beg; - uInfor = uInfor << 33; uInfor = uInfor | (uint64_t)beg; - /****************************may have bugs********************************/ - kv_push(uint64_t, a, uInfor); - } - - ///sort by number of reads in a contig - radix_sort_arch64(a.a, a.a + a.n); - for (i = 0; i < (a.n>>1); ++i) - { - uInfor = a.a[i]; - a.a[i] = a.a[a.n - i - 1]; - a.a[a.n - i - 1] = uInfor; - } - - - for (v = 0; v < a.n; v++) - { - ///all untig ID of this contig - beg = (uint32_t)(a.a[v]); - - b_0->b.n = 0; - get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, b_0); - - cId = v; - ///individual contig - if(get_real_length(nsg, beg^1, NULL) == 0 && get_real_length(nsg, end, NULL) == 0) - { - a.a[v] = a.a[v] | (uint64_t)(0x100000000); - } - ///set the contig Id for each read - for (i = 0; i < b_0->b.n; i++) - { - uId = b_0->b.a[i]>>1; - reads = &(ug->u.a[uId]); - for (j = 0; j < reads->n; j++) - { - rId = reads->a[j]>>33; - set_R_to_U(ruIndex, rId, cId, 1); - } - } - } - /******************************set ruIndex*********************************/ - - - ///scan all contigs from the longest one to the shortest one - for (i = 0; i < a.n; i++) - { - ///all untig ID of this contig - beg = (uint32_t)(a.a[i]); - b_0->b.n = 0; - get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, b_0); - - u_vecs.a.n = 0; - ///scan all unitigs of this contig - for (j = 0, self_offset = 0, self_label_offset = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - ///if(IsMerge(ug->u, uId)>0) continue; - reads = &(ug->u.a[uId]); - ///scan all reads - ///self_offset will skip fake(merge) nodes, but not skip HAP_LABLE - for (k = 0; k < reads->n; k++, self_offset++) - { - rId = reads->a[k]>>33; - ///if(read_g->seq[rId].c == HAP_LABLE) continue; - ///self_offset is the offset of this read in contig - query_reverse_sources(read_g, reverse_sources, ruIndex, rId, self_offset, &u_vecs); - - if(read_g->seq[rId].c == HAP_LABLE) self_label_offset++; - } - } - - if(self_offset >= long_hap_overlap && self_label_offset >= (self_offset*lable_match_rate)) - { - asg_seq_set(bi_g, i, RED, 0); - } - else - { - asg_seq_set(bi_g, i, UNVISIT, 0); - } - - - ///a.a saves all offest and its corresponding contig ID - get_useful_contig_advance(ug, read_g, reverse_sources, ruIndex, &u_vecs, b_0, a.a, - bi_g, i, density, long_hap_overlap, long_hap_overlap_rate, 1); - } - - process_bi_graph(bi_g); - - bi_paration(bi_g, a.a, bi_graph_Len); - - for (v = 0; v < bi_g->n_seq; v++) - { - beg = (uint32_t)(a.a[v]); - /** - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - **/ - b_0->b.n = 0; - get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, b_0); - - if(bi_g->seq[v].len == BLACK) - { - for (i = 0; i < b_0->b.n; i++) - { - nsg->seq[(b_0->b.a[i])>>1].c = ALTER_LABLE; - } - - for (i = 0; i < b_0->b.n; i++) - { - asg_seq_drop(nsg, ((b_0->b.a[i])>>1)); - } - } - - - - - - - - - // fprintf(stderr, "cId: %u, beg>>1: %u, end>>1: %u, b_0->b.n: %u, Len: %u, type: %u\n", - // v, beg>>1, end>>1, (uint32_t)b_0->b.n, (uint32_t)(a.a[v]>>33), bi_g->seq[v].len); - // uint32_t nv = asg_arc_n(bi_g, v), w; - // asg_arc_t *av = asg_arc_a(bi_g, v); - // for (i = 0; i < nv; ++i) - // { - // w = av[i].v; - // fprintf(stderr, "w: %u\n", w); - // } - } - - - /** - for (i = 1; i < a.n; i++) - { - if((a.a[i]>>33) > (a.a[i-1]>>33)) - { - fprintf(stderr, "hehe\n"); - } - } - - - for (i = 0; i < a.n; i++) - { - uint32_t k, get_cId, tLen = 0, is_Unitig; - beg = (uint32_t)(a.a[i]); - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - for (j = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - reads = &(ug->u.a[uId]); - tLen += reads->n; - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - get_R_to_U(ruIndex, rId, &get_cId, &is_Unitig); - if(is_Unitig != 1 || get_cId != i) - { - fprintf(stderr, "###is_Unitig: %u, get_cId: %u, i: %u\n", - is_Unitig, get_cId, i); - } - } - } - - uint32_t qLen = 0, m; - for (m = 0; m < ruIndex->len; m++) - { - get_R_to_U(ruIndex, m, &get_cId, &is_Unitig); - if(get_cId == (uint32_t)-1 || is_Unitig != 1) continue; - if(get_cId == i) - { - qLen = 0; - for (j = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - reads = &(ug->u.a[uId]); - tLen += reads->n; - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - if(m == rId) - { - qLen = 1; - goto found; - } - } - } - found:; - if(qLen == 0) - { - fprintf(stderr, "***is_Unitig: %u, get_cId: %u, m: %u\n", - is_Unitig, get_cId, m); - } - } - } - } - **/ - - uint32_t is_Unitig; - for (v = 0; v < ruIndex->len; v++) - { - get_R_to_U(ruIndex, v, &uId, &is_Unitig); - if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; - } - - asg_cleanup(nsg); - free(a.a); - kv_destroy(u_vecs.a); - asg_destroy(bi_g); -} - inline void reset_visit_flag(uint8_t* visit, asg_t *read_g, R_to_U* ruIndex, uint32_t contigNum, ma_hit_t_alloc* x) { @@ -19390,498 +18498,6 @@ uint32_t get_single_coverage(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, ui return (C_bases/R_bases); } - -void further_clean_untig_graph_trio_advance(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, -ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, buf_t* b_0, -uint8_t* visit, float density, uint32_t bi_graph_Len, uint32_t long_hap_overlap, float lable_match_rate) -{ - asg_t *bi_g = NULL; - bi_g = asg_init(); - kvec_t(uint64_t) a; - kv_init(a); - kvec_t_u64_warp u_vecs; - kv_init(u_vecs.a); - asg_t* nsg = ug->g; - uint32_t v, n_vtx = nsg->n_seq * 2, beg = 0, end, uId, uOri, cId, rId, i, j, k, self_offset, self_label_offset; - long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; - uint64_t uInfor = 0; - ma_utg_t* reads; - ///int flag; - memset(visit, 0, nsg->n_seq); - uint32_t* position_index = (uint32_t*)malloc(sizeof(uint32_t)*read_g->n_seq); - memset(position_index, -1, sizeof(uint32_t)*read_g->n_seq); - uint64_t* vote_counting = NULL; - Hap_Align_warp u_buffer; - kv_init(u_buffer.x); - kvec_t_u32_warp x_vecs; - kv_init(x_vecs.a); - kvec_t_u32_warp y_vecs; - kv_init(y_vecs.a); - uint32_t junk_cov, hap_cov, dip_cov, junk_occ, repeat_occ, single_cov; - - - /****************************may have bugs********************************/ - for (v = 0; v < nsg->n_seq; ++v) - { - uId = v; - if(nsg->seq[uId].c != HAP_LABLE) continue; - reads = &(ug->u.a[uId]); - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - read_g->seq[rId].c = HAP_LABLE; - } - } - /****************************may have bugs********************************/ - - - get_purge_coverage(read_g, sources, coverage_cut, &junk_cov, &hap_cov, &dip_cov); - fprintf(stderr, "junk_cov: %u, hap_cov: %u, dip_cov: %u\n", junk_cov, hap_cov, dip_cov); - ///junk_cov = 50; dip_cov = 200; - - /******************************set ruIndex*********************************/ - for (v = 0; v < n_vtx; ++v) - { - beg = v; - if(nsg->seq[v>>1].c == ALTER_LABLE || nsg->seq[beg>>1].del || visit[beg>>1]) - { - continue; - } - - if(get_real_length(nsg, beg^1, NULL) == 1) - { - get_real_length(nsg, beg^1, &end); - if(get_real_length(nsg, end^1, NULL) == 1) continue; - } - - b_0->b.n = 0; - get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, b_0); - - uInfor = 0; - //scan all untigs - for (i = 0; i < b_0->b.n; i++) - { - uId = b_0->b.a[i]>>1; - visit[uId] = 1; - reads = &(ug->u.a[uId]); - uInfor += reads->n; - } - /****************************may have bugs********************************/ - ///uInfor = uInfor << 32; uInfor = uInfor | (uint64_t)beg; - uInfor = uInfor << 33; uInfor = uInfor | (uint64_t)beg; - /****************************may have bugs********************************/ - kv_push(uint64_t, a, uInfor); - } - - ///sort by number of reads in a contig - radix_sort_arch64(a.a, a.a + a.n); - for (i = 0; i < (a.n>>1); ++i) - { - uInfor = a.a[i]; - a.a[i] = a.a[a.n - i - 1]; - a.a[a.n - i - 1] = uInfor; - } - - - for (v = 0; v < a.n; v++) - { - ///all untig ID of this contig - beg = (uint32_t)(a.a[v]); - - b_0->b.n = 0; - get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, b_0); - - cId = v; - ///individual contig - if(get_real_length(nsg, beg^1, NULL) == 0 && get_real_length(nsg, end, NULL) == 0) - { - a.a[v] = a.a[v] | (uint64_t)(0x100000000); - } - ///set the contig Id for each read - for (i = 0, self_offset = 0, self_label_offset = 0, junk_occ = 0, repeat_occ = 0; - i < b_0->b.n; i++) - { - uId = b_0->b.a[i]>>1; - uOri = b_0->b.a[i]&(uint32_t)1; - reads = &(ug->u.a[uId]); - for (j = 0; j < reads->n; j++, self_offset++) - { - if(uOri == 1) - { - rId = reads->a[reads->n - j - 1]>>33; - } - else - { - rId = reads->a[j]>>33; - } - - set_R_to_U(ruIndex, rId, cId, 1); - position_index[rId] = self_offset; - - if(read_g->seq[rId].c == HAP_LABLE) self_label_offset++; - single_cov = get_single_coverage(sources, coverage_cut, rId); - if(single_cov <= junk_cov) junk_occ++; - if(single_cov > dip_cov) repeat_occ++; - - } - } - - - if(junk_occ >= self_offset*DISCARD_RATE || repeat_occ >= self_offset*DISCARD_RATE) - { - for(i = 0; i < b_0->b.n; i++) - { - uId = b_0->b.a[i]>>1; - uOri = b_0->b.a[i]&(uint32_t)1; - reads = &(ug->u.a[uId]); - for (j = 0; j < reads->n; j++, self_offset++) - { - if(uOri == 1) - { - rId = reads->a[reads->n - j - 1]>>33; - } - else - { - rId = reads->a[j]>>33; - } - - ruIndex->index[rId] = (uint32_t)-1; - position_index[rId] = (uint32_t)-1; - } - } - - asg_seq_set(bi_g, cId, BLACK, 0); - } - else if(self_offset >= long_hap_overlap && self_label_offset >= (self_offset*lable_match_rate)) - { - asg_seq_set(bi_g, cId, RED, 0); - } - else - { - asg_seq_set(bi_g, cId, UNVISIT, 0); - } - } - /******************************set ruIndex*********************************/ - - - - - - - vote_counting = (uint64_t*)malloc(sizeof(uint64_t)*a.n); - memset(vote_counting, 0, sizeof(uint64_t)*a.n); - ///scan all contigs from the longest one to the shortest one - for (i = 0; i < a.n; i++) - { - if(debug_purge_dup) - { - if(i == 53) - { - fprintf(stderr, "+i: %u, Len: %u, start>>1: %u, flag: %u\n", i, - (uint32_t)(a.a[i]>>33), ((uint32_t)a.a[i])>>1, bi_g->seq[i].len); - } - } - if(bi_g->seq[i].len == BLACK) continue; - get_hap_alignment(ug, read_g, reverse_sources, b_0, ruIndex, position_index, - vote_counting, visit, &u_vecs, &u_buffer, &x_vecs, &y_vecs, i, a.n, a.a, - density, bi_g, 1); - - if(debug_purge_dup) - { - if(i == 53) - { - fprintf(stderr, "+i: %u, Len: %u, start>>1: %u, flag: %u\n", v, - (uint32_t)(a.a[i]>>33), ((uint32_t)a.a[i])>>1, bi_g->seq[i].len); - } - } - - } - - fprintf(stderr, "\n*****0*****\n"); - fprintf(stderr, "a.n: %u\n", (uint32_t)(a.n)); - process_bi_graph(bi_g); - fprintf(stderr, "*****1*****\n"); - if(debug_purge_dup) print_purge_gfa(bi_g, a.a); - fprintf(stderr, "*****2*****\n"); - bi_paration(bi_g, a.a, bi_graph_Len); - fprintf(stderr, "*****3*****\n"); - - for (v = 0; v < bi_g->n_seq; v++) - { - beg = (uint32_t)(a.a[v]); - - b_0->b.n = 0; - get_unitig(nsg, ug, beg, &end, &nodeLen, &baseLen, &max_stop_nodeLen, &max_stop_baseLen, 1, b_0); - - if(bi_g->seq[v].len == BLACK) - { - ///if(debug_purge_dup) fprintf(stderr, "BLACK v: %u, beg>>1: %u\n", v, beg>>1); - for (i = 0; i < b_0->b.n; i++) - { - nsg->seq[(b_0->b.a[i])>>1].c = ALTER_LABLE; - } - - for (i = 0; i < b_0->b.n; i++) - { - asg_seq_drop(nsg, ((b_0->b.a[i])>>1)); - } - } - - - - - - - - - // fprintf(stderr, "cId: %u, beg>>1: %u, end>>1: %u, b_0->b.n: %u, Len: %u, type: %u\n", - // v, beg>>1, end>>1, (uint32_t)b_0->b.n, (uint32_t)(a.a[v]>>33), bi_g->seq[v].len); - // uint32_t nv = asg_arc_n(bi_g, v), w; - // asg_arc_t *av = asg_arc_a(bi_g, v); - // for (i = 0; i < nv; ++i) - // { - // w = av[i].v; - // fprintf(stderr, "w: %u\n", w); - // } - } - - - /** - for (i = 1; i < a.n; i++) - { - if((a.a[i]>>33) > (a.a[i-1]>>33)) - { - fprintf(stderr, "hehe\n"); - } - } - - - for (i = 0; i < a.n; i++) - { - uint32_t k, get_cId, tLen = 0, is_Unitig; - beg = (uint32_t)(a.a[i]); - b_0->b.n = 0;ug->u.a[beg>>1].circ = 0; - get_long_tip_length(nsg, &(ug->u), beg, &end, b_0); - for (j = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - reads = &(ug->u.a[uId]); - tLen += reads->n; - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - get_R_to_U(ruIndex, rId, &get_cId, &is_Unitig); - if(is_Unitig != 1 || get_cId != i) - { - fprintf(stderr, "###is_Unitig: %u, get_cId: %u, i: %u\n", - is_Unitig, get_cId, i); - } - } - } - - uint32_t qLen = 0, m; - for (m = 0; m < ruIndex->len; m++) - { - get_R_to_U(ruIndex, m, &get_cId, &is_Unitig); - if(get_cId == (uint32_t)-1 || is_Unitig != 1) continue; - if(get_cId == i) - { - qLen = 0; - for (j = 0; j < b_0->b.n; j++) - { - uId = b_0->b.a[j]>>1; - reads = &(ug->u.a[uId]); - tLen += reads->n; - for (k = 0; k < reads->n; k++) - { - rId = reads->a[k]>>33; - if(m == rId) - { - qLen = 1; - goto found; - } - } - } - found:; - if(qLen == 0) - { - fprintf(stderr, "***is_Unitig: %u, get_cId: %u, m: %u\n", - is_Unitig, get_cId, m); - } - } - } - } - **/ - - uint32_t is_Unitig; - for (v = 0; v < ruIndex->len; v++) - { - get_R_to_U(ruIndex, v, &uId, &is_Unitig); - if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; - } - - asg_cleanup(nsg); - free(a.a); - kv_destroy(u_vecs.a); - kv_destroy(u_buffer.x); - kv_destroy(x_vecs.a); - kv_destroy(y_vecs.a); - asg_destroy(bi_g); - free(position_index); - free(vote_counting); -} - - -void clean_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, -R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, -uint32_t miniBiGraph, float chimeric_rate, int is_final_clean) -{ - asg_t *g = ug->g; - asg_cut_tip_primary(g, ug, tipsLen); - long long pre_cons = get_graph_statistic(g); - long long cur_cons = 0; - while(pre_cons != cur_cons) - { - pre_cons = get_graph_statistic(g); - ///need consider tangles - asg_pop_bubble_primary(g, bubble_dist); - ///need consider tangles - asg_arc_cut_long_tip_primary(g, ug, tip_drop_ratio); - ///need consider tangles - ///note we need both the read graph and the untig graph - untig_asg_arc_cut_long_equal_tips_assembly(ug, read_g, reverse_sources, 2, ruIndex); - untig_asg_arc_cut_long_tip_primary_complex(ug, tip_drop_ratio, stops_threshold); - untig_asg_arc_cut_long_equal_tips_assembly_complex(ug, read_g, reverse_sources, 2, - stops_threshold, ruIndex); - if(is_final_clean) - { - untig_asg_arc_cut_chimeric(ug, read_g, reverse_sources, 2, stops_threshold, chimeric_rate, - ruIndex); - } - cur_cons = get_graph_statistic(g); - } - - asg_cut_tip_primary(g, ug, tipsLen); - untig_asg_arc_simple_large_bubbles(ug, read_g, reverse_sources, 2, ruIndex); - - if(is_final_clean) - { - lable_hap_asg_by_ug(ug, read_g); - further_clean_untig_graph(ug, read_g, reverse_sources, ruIndex, b_0, visit, density, miniHapLen, - miniBiGraph, 200, 0.4, 0.5); - adjust_asg_by_ug(ug, read_g); - } - -} - - -void clean_untig_graph_bubbles(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, -R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, -uint32_t miniBiGraph, float chimeric_rate, int is_final_clean) -{ - asg_t *g = ug->g; - asg_cut_tip_primary(g, ug, tipsLen); - long long pre_cons = get_graph_statistic(g); - long long cur_cons = 0; - while(pre_cons != cur_cons) - { - pre_cons = get_graph_statistic(g); - ///need consider tangles - asg_pop_bubble_primary(g, bubble_dist); - cur_cons = get_graph_statistic(g); - } - - asg_cut_tip_primary(g, ug, tipsLen); - untig_asg_arc_simple_large_bubbles(ug, read_g, reverse_sources, 2, ruIndex); - - if(is_final_clean) - { - lable_hap_asg_by_ug(ug, read_g); - adjust_asg_by_ug(ug, read_g); - } - -} - - - -void resolve_simple_case(ma_ug_t *ug, asg_t* nsg) -{ - uint32_t v, n_vtx = nsg->n_seq * 2, nw, nv, w1, w2, i; - asg_arc_t *av, *aw; - ma_utg_t *v_x = NULL, *v_y = NULL; - - for (v = 0; v < n_vtx; ++v) - { - if (nsg->seq[v>>1].del || nsg->seq[v>>1].c == ALTER_LABLE) continue; - if(asg_arc_n(nsg, v) < 1 || asg_arc_n(nsg, v^1) < 1) continue; - if(get_real_length(nsg, v, NULL) != 1 || get_real_length(nsg, v^1, NULL) != 1) continue; - get_real_length(nsg, v, &w1); get_real_length(nsg, v^1, &w2); - - ///for simple circle - if(w1 == (w2^1)) - { - ///fprintf(stderr, "* v>>1: %u\n", v>>1); - EvaluateLen(ug->u, w1>>1) = EvaluateLen(ug->u, w1>>1) + EvaluateLen(ug->u, v>>1); - ///nsg->seq[w1>>1].len = nsg->seq[w1>>1].len + nsg->seq[v>>1].len; - - v_x = &(ug->u.a[w1>>1]); - v_y = &(ug->u.a[v>>1]); - append_ma_utg_t(v_x, v_y); - asg_seq_del(nsg, v>>1); - } - else if(w1 == w2) - { - if(get_real_length(nsg, w1^1, NULL) != 2) continue; - if(get_real_length(nsg, w1, NULL) != 1 && get_real_length(nsg, w1, NULL) != 2) continue; - - ///fprintf(stderr, "# v>>1: %u\n", v>>1); - EvaluateLen(ug->u, w1>>1) = EvaluateLen(ug->u, w1>>1) + EvaluateLen(ug->u, v>>1); - ///nsg->seq[w1>>1].len = nsg->seq[w1>>1].len + nsg->seq[v>>1].len; - - v_x = &(ug->u.a[w1>>1]); - v_y = &(ug->u.a[v>>1]); - - asg_seq_del(nsg, v>>1); - if(get_real_length(nsg, w1, NULL) == 1) continue; - - aw = asg_arc_a(nsg, w1); - nw = asg_arc_n(nsg, w1); - av = asg_arc_a(nsg, w1^1); - for (i = 0; i < nw; i++) - { - if(!aw[i].del) - { - av[0].del = 0;aw[i].del = 1; - av[0].el = aw[i].el; - av[0].no_l_indel = aw[i].no_l_indel; - av[0].ol = aw[i].ol; - av[0].strong = aw[i].strong; - av[0].v = aw[i].v; - av[0].ul = (aw[i].ul)^(0x100000000); - - - av = asg_arc_a(nsg, aw[i].v^1); - nv = asg_arc_n(nsg, aw[i].v^1); - uint32_t k = 0; - for (k = 0; k < nv; k++) - { - if(av[k].v == (aw[i].ul>>32^1)) - { - av[k].v = av[k].v^1; - break; - } - } - - break; - } - } - - } - } -} - void print_node(asg_t* g, ma_ug_t *ug) { int input_iv; @@ -19972,111 +18588,6 @@ void print_node(asg_t* g, ma_ug_t *ug) } } -void label_tangles(asg_t *sg, ma_hit_t_alloc* reverse_sources, long long minLongUntig, -long long maxShortUntig, float l_untig_rate, float max_node_threshold, long long bubble_dist, -long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, int just_bubble) -{ - double startTime = Get_T(); - buf_t b_0, b_1; - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - uint32_t v, sv, n_vtx, beg, end, next_uID = (uint32_t)-1; - kvec_t_u32_warp u_vecs; - kv_init(u_vecs.a); - ma_ug_t *ug = NULL; - ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); - - ///for each untig, all node have the same direction - ///and all node except the last one just have one edge - ///the last one may have multiple edges - ///for the useful untig, the signal is (u->n >= LongUntigThreshold && !u->circ) - asg_t* nsg = ug->g; - n_vtx = nsg->n_seq; - for (v = 0; v < n_vtx; ++v) - { - nsg->seq[v].c = PRIMARY_LABLE; - EvaluateLen(ug->u, v) = ug->u.a[v].n; - IsMerge(ug->u, v) = 0; - } - - resolve_simple_case(ug, nsg); - asg_cleanup(nsg); - asg_symm(nsg); - - ///print_node(nsg); - - if(just_bubble) - { - clean_untig_graph_bubbles(ug, sg, reverse_sources, bubble_dist, tipsLen, - tip_drop_ratio, stops_threshold, ruIndex, &b_0, NULL, 0.8, 20, 200, 0.05, 0); - } - else - { - clean_untig_graph(ug, sg, reverse_sources, bubble_dist, tipsLen, - tip_drop_ratio, stops_threshold, ruIndex, &b_0, NULL, 0.8, 20, 200, 0.05, 0); - } - - - - uint8_t* visit = NULL; - visit = (uint8_t*)malloc(sizeof(uint8_t) * nsg->n_seq); - uint32_t n_reduce, flag; - n_vtx = nsg->n_seq * 2; - while (1) - { - n_reduce = 0; - for (v = 0; v < n_vtx; ++v) - { - //as for return value: 0: do nothing, 1: unroll, 2: convex - //we just need 1 - sv = v; - flag = 0; - while (1) - { - flag = walk_through(sg, ug, reverse_sources, minLongUntig, - maxShortUntig, l_untig_rate, max_node_threshold, &b_0, &b_1, - &u_vecs, visit, sv, &beg, &end, &next_uID, ruIndex); - n_reduce += flag; - if(flag != UNROLL_M) - { - break; - } - } - } - if(n_reduce == 0) break; - } - - asg_cleanup(nsg); - asg_symm(nsg); - - if(just_bubble) - { - clean_untig_graph_bubbles(ug, sg, reverse_sources, bubble_dist, tipsLen, - tip_drop_ratio, stops_threshold, ruIndex, &b_0, visit, 0.8, 20, 200, 0.05, 1); - } - else - { - clean_untig_graph(ug, sg, reverse_sources, bubble_dist, tipsLen, - tip_drop_ratio, stops_threshold, ruIndex, &b_0, visit, 0.8, 20, 200, 0.05, 1); - } - - - - - - kv_destroy(u_vecs.a); - ma_ug_destroy(ug); - - - free(visit); - free(b_0.b.a); - free(b_1.b.a); - if(VERBOSE >= 1) - { - fprintf(stderr, "[M::%s] takes %0.2f s\n", __func__, Get_T()-startTime); - } -} - /*************************************for tangle resolve*************************************/ void recover_edges(asg_t* nsg, kvec_t_u64_warp* edges, uint64_t* nodes, uint64_t n, @@ -20250,13 +18761,13 @@ uint32_t positive_flag, uint32_t negative_flag) v = beg; if((!g->seq[v>>1].del)&&(g->seq[v>>1].c!=ALTER_LABLE)&&get_real_length(g, v, NULL)>=2) { - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); } v = end^1; if((!g->seq[v>>1].del)&&(g->seq[v>>1].c!=ALTER_LABLE)&&get_real_length(g, v, NULL)>=2) { - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); } @@ -20271,7 +18782,7 @@ uint32_t positive_flag, uint32_t negative_flag) { v = v|k; if(get_real_length(g, v, NULL)<=1) continue; - n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1); + n_pop += asg_bub_pop1_primary_trio(ug->g, ug, v, max_dist, &b, positive_flag, negative_flag, 1, NULL, NULL); } } @@ -20884,7 +19395,7 @@ float drop_ratio) { rId = nsu->a[i]>>33; ///ori = nsu->a[k]>>32&1; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(read_g->seq[rId].c)); } } @@ -21072,228 +19583,6 @@ float drop_ratio) } - - - -void deduplicate(ma_ug_t *src, asg_t *read_g, ma_hit_t_alloc* reverse_sources, long long minLongUntig, -long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, uint32_t resolve_tangle) -{ - uint32_t i, v, sv, n_vtx, beg, end, next_uID = (uint32_t)-1, uId, is_Unitig, rId; - ma_utg_t* nsu = NULL; - ma_ug_t *ug = NULL; - kvec_t_u32_warp u_vecs; - kv_init(u_vecs.a); - buf_t b_0, b_1; - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - - ug = copy_untig_graph(src); - - asg_t* nsg = ug->g; - n_vtx = nsg->n_seq; - for (v = 0; v < n_vtx; ++v) - { - if(nsg->seq[v].del) continue; - EvaluateLen(ug->u, v) = ug->u.a[v].n; - IsMerge(ug->u, v) = 0; - } - - uint8_t* visit = NULL; - visit = (uint8_t*)malloc(sizeof(uint8_t) * nsg->n_seq); - uint32_t n_reduce, flag; - n_vtx = nsg->n_seq * 2; - - if(resolve_tangle) - { - while (1) - { - n_reduce = 0; - for (v = 0; v < n_vtx; ++v) - { - //as for return value: 0: do nothing, 1: unroll, 2: convex - //we just need 1 - sv = v; - flag = 0; - while (1) - { - flag = walk_through(read_g, ug, reverse_sources, minLongUntig, - maxShortUntig, l_untig_rate, max_node_threshold, &b_0, &b_1, - &u_vecs, visit, sv, &beg, &end, &next_uID, ruIndex); - n_reduce += flag; - if(flag != UNROLL_M) - { - break; - } - } - } - if(n_reduce == 0) break; - } - - asg_cleanup(nsg); - asg_symm(nsg); - } - - - further_clean_untig_graph_trio(ug, read_g, reverse_sources, ruIndex, &b_0, visit, - 0.8, 200, 200, 0.4, 0.5); - - for (v = 0; v < src->g->n_seq; v++) - { - uId = v; - nsu = &(src->u.a[v]); - if(nsu->m == 0) continue; - if(src->g->seq[v].del) continue; - if(src->g->seq[v].c==ALTER_LABLE) continue; - for (i = 0; i < nsu->n; i++) - { - rId = nsu->a[i]>>33; - set_R_to_U(ruIndex, rId, uId, 1); - } - } - - - - n_vtx = nsg->n_seq; - for (v = 0; v < n_vtx; ++v) - { - if(nsg->seq[v].del) continue; - if(nsg->seq[v].c!=ALTER_LABLE) continue; - nsu = &(ug->u.a[v]); - for (i = 0; i < nsu->n; i++) - { - rId = nsu->a[i]>>33; - get_R_to_U(ruIndex, rId, &uId, &is_Unitig); - if(is_Unitig != 1 || uId == ((uint32_t)(-1))) continue; - src->g->seq[uId].c = ALTER_LABLE; - } - } - - - for (v = 0; v < ruIndex->len; v++) - { - get_R_to_U(ruIndex, v, &uId, &is_Unitig); - if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; - } - - kv_destroy(u_vecs.a); - ma_ug_destroy(ug); - free(visit); - free(b_0.b.a); - free(b_1.b.a); -} - - -void deduplicate_advance(ma_ug_t *src, asg_t *read_g, ma_sub_t* coverage_cut, -ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long minLongUntig, -long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, -uint32_t resolve_tangle) -{ - uint32_t i, v, sv, n_vtx, beg, end, next_uID = (uint32_t)-1, uId, is_Unitig, rId; - ma_utg_t* nsu = NULL; - ma_ug_t *ug = NULL; - kvec_t_u32_warp u_vecs; - kv_init(u_vecs.a); - buf_t b_0, b_1; - memset(&b_0, 0, sizeof(buf_t)); - memset(&b_1, 0, sizeof(buf_t)); - - ug = copy_untig_graph(src); - - asg_t* nsg = ug->g; - n_vtx = nsg->n_seq; - for (v = 0; v < n_vtx; ++v) - { - if(nsg->seq[v].del) continue; - EvaluateLen(ug->u, v) = ug->u.a[v].n; - IsMerge(ug->u, v) = 0; - } - - uint8_t* visit = NULL; - visit = (uint8_t*)malloc(sizeof(uint8_t) * nsg->n_seq); - uint32_t n_reduce, flag; - n_vtx = nsg->n_seq * 2; - - if(resolve_tangle) - { - while (1) - { - n_reduce = 0; - for (v = 0; v < n_vtx; ++v) - { - //as for return value: 0: do nothing, 1: unroll, 2: convex - //we just need 1 - sv = v; - flag = 0; - while (1) - { - flag = walk_through(read_g, ug, reverse_sources, minLongUntig, - maxShortUntig, l_untig_rate, max_node_threshold, &b_0, &b_1, - &u_vecs, visit, sv, &beg, &end, &next_uID, ruIndex); - n_reduce += flag; - if(flag != UNROLL_M) - { - break; - } - } - } - if(n_reduce == 0) break; - } - - asg_cleanup(nsg); - asg_symm(nsg); - } - - - further_clean_untig_graph_trio_advance(ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, &b_0, visit, - 0.7, /**200, 200,**/50, 50, 0.5); - - for (v = 0; v < src->g->n_seq; v++) - { - uId = v; - nsu = &(src->u.a[v]); - if(nsu->m == 0) continue; - if(src->g->seq[v].del) continue; - if(src->g->seq[v].c==ALTER_LABLE) continue; - for (i = 0; i < nsu->n; i++) - { - rId = nsu->a[i]>>33; - set_R_to_U(ruIndex, rId, uId, 1); - } - } - - - - n_vtx = nsg->n_seq; - for (v = 0; v < n_vtx; ++v) - { - if(nsg->seq[v].del) continue; - if(nsg->seq[v].c!=ALTER_LABLE) continue; - nsu = &(ug->u.a[v]); - for (i = 0; i < nsu->n; i++) - { - rId = nsu->a[i]>>33; - get_R_to_U(ruIndex, rId, &uId, &is_Unitig); - if(is_Unitig != 1 || uId == ((uint32_t)(-1))) continue; - src->g->seq[uId].c = ALTER_LABLE; - } - } - - - for (v = 0; v < ruIndex->len; v++) - { - get_R_to_U(ruIndex, v, &uId, &is_Unitig); - if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; - } - - kv_destroy(u_vecs.a); - ma_ug_destroy(ug); - free(visit); - free(b_0.b.a); - free(b_1.b.a); -} - - - /*************************************for tangle resolve*************************************/ uint32_t copy_ug_node(ma_ug_t *ug, asg_t* nsg, uint32_t v) @@ -21359,11 +19648,11 @@ uint32_t collect_ma_utg_ts(ma_ug_t *ug, uint32_t v, uint32_t w, ma_utg_t* result } -void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_tangle) +void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, kvec_asg_arc_t_warp* edge, int require_equal_nv, int test_tangle) { asg_t* nsg = ug->g; uint32_t n_vtx = nsg->n_seq, i, j, k, l, totalLen, v, nv, nw, w, untig_v, rid_v; - asg_arc_t *aw = NULL, *av = NULL; + asg_arc_t *aw = NULL, *av = NULL, *t_v = NULL, *t_w = NULL; for (i = 0; i < n_vtx; i++) { if(ug->g->seq[i].del) continue; @@ -21393,7 +19682,7 @@ void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_ av = asg_arc_a(read_g, v); nv = asg_arc_n(read_g, v); - l = 0; + l = (uint32_t)-1; for (k = 0; k < nv; k++) { if(av[k].del) continue; @@ -21403,8 +19692,28 @@ void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_ break; } } + + if(edge && k == nv) + { + for (k = 0; k < edge->a.n; k++) + { + if(edge->a.a[k].del) continue; + if((edge->a.a[k].ul>>32) == v && edge->a.a[k].v == w) + { + l = asg_arc_len(edge->a.a[k]); + k = nv + 1; + break; + } + } + } + if(k == nv) fprintf(stderr ,"******error, j: %u, k: %u, nv: %u\n", j, k, nv); - if(l != (uint32_t)(result->a[j])) fprintf(stderr ,"ERROR Length\n"); + if(l != (uint32_t)(result->a[j])) + { + fprintf(stderr ,"(i: %u) ERROR Length, l: %u, result->a[j]: %u, j: %u, k: %u, nv: %u, circ: %u, result->n: %u\n", + i, l, (uint32_t)(result->a[j]), j, k, nv, result->circ, (uint32_t)result->n); + } + totalLen = totalLen + l; } @@ -21446,20 +19755,40 @@ void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_ untig_v = av[j].v; if(untig_v&1) rid_v = ug->u.a[untig_v>>1].end; else rid_v = ug->u.a[untig_v>>1].start; - + + t_v = t_w = NULL; for (k = 0; k < nw; k++) { if(aw[k].del) continue; - if(aw[k].v == rid_v) break; + if(aw[k].v == rid_v) + { + t_w = &(aw[k]); + break; + } + } - if(k == nw) fprintf(stderr, "#########ERROR: i: %u\n", i); - if((k != nw) && (av[j].ol != aw[k].ol)) + if(edge && t_w == NULL) + { + for (k = 0; k < edge->a.n; k++) + { + if(edge->a.a[k].del) continue; + if((edge->a.a[k].ul>>32) == w && edge->a.a[k].v == rid_v) + { + t_w = &(edge->a.a[k]); + break; + } + } + } + + if(t_w == NULL) fprintf(stderr, "#########ERROR: i: %u\n", i); + t_v = &av[j]; + if(t_w && (t_v->ol != t_w->ol)) { fprintf(stderr, "#########????????ERROR\n"); fprintf(stderr, "nv: %u, nw: %u\n", nv, nw); fprintf(stderr, "av[%u].ol: %u, aw[%u].ol: %u, untig_v>>1: %u, untig_v&1: %u\n", - j, av[j].ol, k, aw[k].ol, untig_v>>1, untig_v&1); + j, t_v->ol, k, t_w->ol, untig_v>>1, untig_v&1); } } @@ -21486,19 +19815,39 @@ void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_ if(untig_v&1) rid_v = ug->u.a[untig_v>>1].end; else rid_v = ug->u.a[untig_v>>1].start; + t_v = t_w = NULL; for (k = 0; k < nw; k++) { if(aw[k].del) continue; - if(aw[k].v == rid_v) break; + if(aw[k].v == rid_v) + { + t_w = &(aw[k]); + break; + } + } - if(k == nw) fprintf(stderr, "***********ERROR: i: %u\n", i); - if((k != nw) && (av[j].ol != aw[k].ol)) + if(edge && t_w == NULL) { - fprintf(stderr, "***********????????ERROR\n"); + for (k = 0; k < edge->a.n; k++) + { + if(edge->a.a[k].del) continue; + if((edge->a.a[k].ul>>32) == w && edge->a.a[k].v == rid_v) + { + t_w = &(edge->a.a[k]); + break; + } + } + } + + if(t_w == NULL) fprintf(stderr, "#########ERROR: i: %u\n", i); + t_v = &av[j]; + if(t_w && (t_v->ol != t_w->ol)) + { + fprintf(stderr, "#########????????ERROR\n"); fprintf(stderr, "nv: %u, nw: %u\n", nv, nw); fprintf(stderr, "av[%u].ol: %u, aw[%u].ol: %u, untig_v>>1: %u, untig_v&1: %u\n", - j, av[j].ol, k, aw[k].ol, untig_v>>1, untig_v&1); + j, t_v->ol, k, t_w->ol, untig_v>>1, untig_v&1); } } @@ -21814,209 +20163,6 @@ int get_arc_t(Edge_iter* x, asg_arc_t* get) return 0; } -///need to consider circle -void merge_ug_nodes_advance(ma_ug_t *ug, asg_t* read_g, R_to_U* ruIndex, kvec_t_u64_warp* array, -asg_arc_t* new_edges, uint32_t new_edges_n) -{ - uint32_t beg_uid = array->a.a[0]; - uint32_t end_uid = array->a.a[array->a.n-1]; - uint32_t realLen = array->a.n; - uint32_t new_uid = array->a.a[0]>>1; - uint64_t kmp; - - - - if(array->a.n == 0) return; - uint32_t i, v, w; - ma_utg_t result; - memset(&result, 0, sizeof(ma_utg_t)); - v = w = (uint32_t)-1; - for (i = 0; i < array->a.n; i++) - { - w = array->a.a[i]; - collect_ma_utg_ts(ug, v, w, &result); - v = w; - } - - - - if(result.n == 0) return; - uint32_t l; - result.len = 0; - Edge_iter iter_x, iter_y; - asg_arc_t get_x, get_y; - get_x.v=get_x.ul=get_x.ol=0; - get_y.v=get_y.ul=get_y.ol=0; - for (i = 0; i < result.n - 1; i++) - { - l = (uint32_t)-1; - v = (uint64_t)(result.a[i])>>32; - w = (uint64_t)(result.a[i + 1])>>32; - init_Edge_iter(read_g, v, new_edges, new_edges_n, &iter_x); - while(get_arc_t(&iter_x, &get_x)) - { - if(get_x.v == w) - { - l = asg_arc_len(get_x); - break; - } - } - - if(l==(uint32_t)-1) fprintf(stderr ,"******error, i: %u\n", i); - - result.a[i] = v; result.a[i] = result.a[i]<<32; result.a[i] = result.a[i] | (uint64_t)(l); - result.len += l; - - set_R_to_U(ruIndex, v>>1, new_uid, 1); - } - - - if(i < result.n) - { - v = (uint64_t)(result.a[i])>>32; - l = read_g->seq[v>>1].len; - result.a[i] = v; - result.a[i] = result.a[i]<<32; - result.a[i] = result.a[i] | (uint64_t)(l); - result.len += l; - - set_R_to_U(ruIndex, v>>1, new_uid, 1); - } - //has already set result.a, result.len, result.n, result.m - result.circ = 0; - result.start = result.a[0]>>32; - result.end = (result.a[result.n-1]>>32)^1; - - - - - - ///corresponding to direction 1 of new node - v = beg_uid^1; - init_Edge_iter(ug->g, v, new_edges, new_edges_n, &iter_x); - while(get_arc_t(&iter_x, &get_x)) - { - - ///if((av[k].v>>1) == (end_uid>>1)) continue; - w = get_x.v^1; - init_Edge_iter(ug->g, w, new_edges, new_edges_n, &iter_y); - while (get_arc_t(&iter_y, &get_y)) - { - if(get_y.v==(v^1)) break; - } - - if((get_y.ul>>32)!=w||get_y.v==(v^1)) fprintf(stderr ,"******error\n"); - - ///here kmp is ul - kmp = new_uid<<1; kmp = kmp^1; kmp = kmp << 32; kmp=kmp|(uint64_t)(get_y.ol); - kv_push(uint64_t, array->a, kmp); - - - ///here kmp is ol + v - kmp = get_x.ol; kmp = kmp<<32; kmp = kmp|(uint64_t)(get_x.v); - kv_push(uint64_t, array->a, kmp); - } - - - - ///corresponding to direction 0 of new node - v = end_uid; - init_Edge_iter(ug->g, v, new_edges, new_edges_n, &iter_x); - while(get_arc_t(&iter_x, &get_x)) - { - - ///if((av[k].v>>1) == (end_uid>>1)) continue; - w = get_x.v^1; - init_Edge_iter(ug->g, w, new_edges, new_edges_n, &iter_y); - while (get_arc_t(&iter_y, &get_y)) - { - if(get_y.v==(v^1)) break; - } - - if((get_y.ul>>32)!=w||get_y.v==(v^1)) fprintf(stderr ,"******error\n"); - - ///here kmp is ul - kmp = new_uid<<1; kmp = kmp << 32; kmp=kmp|(uint64_t)(get_y.ol); - kv_push(uint64_t, array->a, kmp); - - - ///here kmp is ol + v - kmp = get_x.ol; kmp = kmp<<32; kmp = kmp|(uint64_t)(get_x.v); - kv_push(uint64_t, array->a, kmp); - } - - - - - ma_utg_t* tmp; - for (i = 0; i < realLen; i++) - { - w = array->a.a[i]; - tmp = &(ug->u.a[w>>1]); - if(tmp->m != 0) - { - tmp->circ = tmp->end = tmp->len = tmp->m = tmp->n = tmp->start = 0; - free(tmp->a); - tmp->a = NULL; - } - asg_seq_del(ug->g, w>>1); - } - - ug->u.a[beg_uid>>1] = result; - ug->g->seq[beg_uid>>1].del = 0; - - uint32_t oLen = 0; - for (; i < array->a.n; i += 2) - { - v = array->a.a[i]>>32; - w = (uint32_t)array->a.a[i+1]; - /****************************may have bugs********************************/ - ///may have bug here, if there is an edge between beg_uid and end_uid - ///if(((w>>1) == (beg_uid>>1)) || ((w>>1) == (end_uid>>1))) continue; - if(((w>>1) == (beg_uid>>1)) || ((w>>1) == (end_uid>>1))) w = v; - /****************************may have bugs********************************/ - - oLen = array->a.a[i+1]>>32; - asg_append_edges_to_srt(ug->g, v, ug->u.a[v>>1].len, w, oLen, 0, 0, 0); - oLen = (uint32_t)array->a.a[i]; - asg_append_edges_to_srt(ug->g, w^1, ug->u.a[w>>1].len, v^1, oLen, 0, 0, 0); - } - - - ///check if it is a circle - if(get_real_length(ug->g, new_uid, NULL)!=1||get_real_length(ug->g, new_uid^1, NULL)!=1) return; - - get_real_length(ug->g, new_uid, &w); - if(w!=new_uid) return; - - new_uid = new_uid^1; - get_real_length(ug->g, new_uid, &w); - if(w!=new_uid) return; - - l = (uint32_t)-1; - v = (uint64_t)(result.a[result.n - 1])>>32; - w = (uint64_t)(result.a[0])>>32; - init_Edge_iter(read_g, v, new_edges, new_edges_n, &iter_x); - while(get_arc_t(&iter_x, &get_x)) - { - if(get_x.v == w) - { - l = asg_arc_len(get_x); - break; - } - } - - if(l==(uint32_t)-1) fprintf(stderr ,"******error, i: %u\n", i); - - result.circ = 1; result.start = result.end = UINT32_MAX; - result.len = result.len - (uint32_t)(result.a[result.n - 1]); result.len = result.len + l; - result.a[result.n - 1] = v; result.a[result.n - 1] = result.a[result.n - 1]<<32; - result.a[result.n - 1] = result.a[result.n - 1] | (uint64_t)(l); - - - ug->u.a[beg_uid>>1] = result; - ug->g->seq[beg_uid>>1].del = 0; -} void unroll_simple_case(ma_ug_t *ug, asg_t* read_g) { @@ -22792,7 +20938,6 @@ int load_asg_t(asg_t **sg, char* read_file_name) return 1; } - int write_debug_graph(asg_t *sg, ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, char* output_file_name, long long n_read, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex) { @@ -22931,7 +21076,32 @@ R_to_U* ruIndex) } -void append_utg(ma_ug_t* ptg, ma_ug_t* atg) +void reset_reverse_unitigs(hc_links* link, ma_utg_t *u) +{ + uint32_t k = 0, i = 0, rId, pre = (uint32_t)-1; + hc_edge *e = NULL; + if(u->n == 0 || u->m == 0) return; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + if(link->u_idx[rId] == (uint32_t)-1) continue; + if(pre == link->u_idx[rId]) continue; + pre = link->u_idx[rId]; + if(link->a.a[pre].f.n == 0) continue; + + for (i = 0; i < link->a.a[pre].f.n; i++) + { + if(link->a.a[pre].f.a[i].del) continue; + link->a.a[pre].f.a[i].del = 1; + e = get_hc_edge(link, link->a.a[pre].f.a[i].uID, pre, 1); + if(e == NULL) continue; + e->del = 1; + } + } +} + + +void append_utg(ma_ug_t* ptg, ma_ug_t* atg, hc_links* link) { uint64_t num_nodes = 0; asg_t* nsg = atg->g; @@ -22956,6 +21126,7 @@ void append_utg(ma_ug_t* ptg, ma_ug_t* atg) for (v = 0; v < atg->g->n_seq; ++v) { if(atg->g->seq[v].del || atg->u.a[v].m == 0) continue; + if(link) reset_reverse_unitigs(link, &(atg->u.a[v])); p = &(ptg->u.a[ptg->u.n]); p->len = atg->u.a[v].len; @@ -23010,7 +21181,7 @@ void print_utg_coverage(ma_ug_t *ug, ma_sub_t* coverage_cut, uint32_t v, ma_hit_ } void recover_utg_by_coverage(ma_ug_t **ptg, asg_t* read_g, ma_sub_t* coverage_cut, -ma_hit_t_alloc* sources, R_to_U* ruIndex) +ma_hit_t_alloc* sources, R_to_U* ruIndex, hc_links* link) { if(asm_opt.recover_atg_cov_min == -1) return; if(asm_opt.recover_atg_cov_max == -1) return; @@ -23097,7 +21268,7 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex) { asg_cleanup(nsg); asg_symm(nsg); - append_utg(*ptg, atg); + append_utg(*ptg, atg, link); n_vtx = read_g->n_seq; for (v = 0; v < n_vtx; v++) @@ -23141,7 +21312,7 @@ void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, -kvec_asg_arc_t_warp* new_rtg_edges) +kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link) { asg_t* nsg = (*ug)->g; uint32_t v, n_vtx = nsg->n_seq, k, rId, just_contain; @@ -23149,12 +21320,27 @@ kvec_asg_arc_t_warp* new_rtg_edges) ///print_utg_coverage(*ug, coverage_cut, 440, sources); ///exit(0); - /** - kvec_t_u32_warp new_rtg_nodes; - kv_init(new_rtg_nodes.a); - **/ + if(link) + { + memset(link->u_idx, -1, R_INF.total_reads*sizeof(uint32_t)); + nsg = (*ug)->g; + n_vtx = nsg->n_seq; + for (v = 0; v < n_vtx; ++v) + { + if(nsg->seq[v].del) continue; + u = &((*ug)->u.a[v]); + if(u->m == 0) continue; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + ///if(read_g->seq[rId].c == FAKE_LABLE) continue; + link->u_idx[rId] = v; + } + } + } drop_semi_circle((*ug), nsg, read_g, reverse_sources, ruIndex); + asg_cleanup(nsg); adjust_utg_advance(read_g, (*ug), reverse_sources, ruIndex); @@ -23169,7 +21355,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) clean_primary_untig_graph(*ug, read_g, reverse_sources, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, NULL, NULL, 0, 0, 0, - chimeric_rate, 0, 0, drop_ratio); + chimeric_rate, 0, 0, drop_ratio, link); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); @@ -23181,18 +21367,16 @@ kvec_asg_arc_t_warp* new_rtg_edges) purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0); + drop_ratio, just_contain, 0, link); delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } - if (!(asm_opt.flag & HA_F_BAN_POST_JOIN)) { rescue_missing_overlaps_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, min_ovlp, 0, 0, 1, NULL); renew_utg(ug, read_g, new_rtg_edges); - rescue_contained_reads_aggressive(*ug, read_g, sources, coverage_cut, ruIndex, max_hang, min_ovlp, 0, 10, 0, 1, NULL, NULL); renew_utg(ug, read_g, new_rtg_edges); @@ -23204,7 +21388,8 @@ kvec_asg_arc_t_warp* new_rtg_edges) purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, just_contain, 0); + drop_ratio, just_contain, 0, link); + delete_useless_nodes(ug); renew_utg(ug, read_g, new_rtg_edges); } @@ -23214,7 +21399,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) { purge_dups(*ug, read_g, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, asm_opt.purge_simi_rate, asm_opt.purge_overlap_len, max_hang, min_ovlp, bubble_dist, - drop_ratio, 0, 1); + drop_ratio, 0, 1, link); } n_vtx = read_g->n_seq; @@ -23267,11 +21452,22 @@ kvec_asg_arc_t_warp* new_rtg_edges) __func__, asm_opt.recover_atg_cov_min); } - recover_utg_by_coverage(ug, read_g, coverage_cut, sources, ruIndex); - - /** - kv_destroy(new_rtg_nodes.a); - **/ + recover_utg_by_coverage(ug, read_g, coverage_cut, sources, ruIndex, link); + + if(link) + { + uint32_t m; + for (v = 0; v < link->a.n; v++) + { + for (k = m = 0; k < link->a.a[v].f.n; k++) + { + if(link->a.a[v].f.a[k].del) continue; + link->a.a[v].f.a[m] = link->a.a[v].f.a[k]; + m++; + } + link->a.a[v].f.n = m; + } + } } @@ -23318,7 +21514,7 @@ long long tipsLen, R_to_U* ruIndex, int max_hang, int min_ovlp) sprintf(gfa_name, "%s.p_utg.lowQ.bed", output_file_name); output_file = fopen(gfa_name, "w"); ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, - max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "utg", output_file); + max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "utg", output_file, NULL); fclose(output_file); } @@ -23341,9 +21537,24 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov adjust_utg_by_primary(&ug, sg, TRIO_THRES, sources, reverse_sources, coverage_cut, bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, - max_hang, min_ovlp, &new_rtg_edges); + max_hang, min_ovlp, &new_rtg_edges, NULL); + + + if(asm_opt.b_low_cov > 0) + { + break_ug_contig(&ug, sg, &R_INF, coverage_cut, sources, ruIndex, &new_rtg_edges, max_hang, min_ovlp, + &asm_opt.b_low_cov, NULL, asm_opt.m_rate); + } + + if(asm_opt.b_high_cov > 0) + { + break_ug_contig(&ug, sg, &R_INF, coverage_cut, sources, ruIndex, &new_rtg_edges, max_hang, min_ovlp, + NULL, &asm_opt.b_high_cov, asm_opt.m_rate); + } ma_ug_seq(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp); + + fprintf(stderr, "Writing primary contig GFA to disk... \n"); char* gfa_name = (char*)malloc(strlen(output_file_name)+35); @@ -23361,7 +21572,7 @@ R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ov sprintf(gfa_name, "%s.p_ctg.lowQ.bed", output_file_name); output_file = fopen(gfa_name, "w"); ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, - max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "ptg", output_file); + max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "ptg", output_file, NULL); fclose(output_file); } @@ -23380,6 +21591,12 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp) kv_init(new_rtg_edges.a); ma_ug_t *ug = NULL; ug = ma_ug_gen_primary(sg, ALTER_LABLE); + + // if(asm_opt.b_low_cov > 0) + // { + // break_ug_contig(&ug, sg, &R_INF, coverage_cut, sources, ruIndex, &new_rtg_edges, max_hang, min_ovlp, asm_opt.b_low_cov); + // } + ma_ug_seq(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp); fprintf(stderr, "Writing alternate contig GFA to disk... \n"); @@ -23398,7 +21615,7 @@ ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp) sprintf(gfa_name, "%s.a_ctg.lowQ.bed", output_file_name); output_file = fopen(gfa_name, "w"); ma_ug_print_bed(ug, sg, &R_INF, coverage_cut, sources, &new_rtg_edges, - max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "atg", output_file); + max_hang, min_ovlp, asm_opt.bed_inconsist_rate, "atg", output_file, NULL); fclose(output_file); } @@ -23737,7 +21954,7 @@ void pre_clean(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *sg, long ///remove isoloated single read tri_flag += asg_arc_del_single_node_directly(sg, asm_opt.max_short_tip, sources); - if (!ha_opt_triobin(&asm_opt)) + if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) { tri_flag += asg_arc_del_triangular_advance(sg, bubble_dist); ///remove the cross at the bubble carefully, just remove inexact cross @@ -23765,8 +21982,10 @@ void destory_R_to_U(R_to_U* x) free(x->index); } -void set_R_to_U(R_to_U* x, uint32_t rID, uint32_t uID, uint32_t is_Unitig) +void set_R_to_U(R_to_U* x, uint32_t rID, uint32_t uID, uint32_t is_Unitig, uint8_t* flag) { + if(flag && (*flag) == FAKE_LABLE) return; + if(rID >= x->len) { x->index = (uint32_t*)realloc(x->index, (rID + 1)*sizeof(uint32_t)); @@ -23814,7 +22033,7 @@ void transfor_R_to_U(R_to_U* x) rID = uID; } - set_R_to_U(x, i, rID, 0); + set_R_to_U(x, i, rID, 0, NULL); } @@ -23976,6 +22195,66 @@ uint32_t* skip_uId, uint32_t skip_uId_n, uint32_t ignore_trio_flag) } +inline uint32_t get_num_edges2existing_nodes_advance_by_broken_bub(ma_ug_t *ug, asg_t *g, +ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, uint8_t* expect_vis, +int max_hang, int min_ovlp, uint32_t query, uint32_t* oLen, uint8_t* utg_vis, +uint32_t ignore_trio_flag) +{ + (*oLen) = 0; + uint32_t qn = query>>1, is_first = 1;; + int32_t r; + asg_arc_t t; + ma_hit_t *h = NULL; + ma_sub_t *sq = NULL; + ma_sub_t *st = NULL; + ma_hit_t_alloc* x = &(sources[qn]); + uint32_t i, occ = 0, uId, is_Unitig; + + uint32_t trio_flag = R_INF.trio_flag[qn], non_trio_flag = (uint32_t)-1; + if(ignore_trio_flag == 0) + { + if(trio_flag == FATHER) non_trio_flag = MOTHER; + if(trio_flag == MOTHER) non_trio_flag = FATHER; + } + + + + for (i = 0; i < x->length; i++) + { + h = &(x->buffer[i]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + ///sq has already been removed + ///st must not be removed + ///g-seq must not be removed + if(st->del || g->seq[Get_tn(*h)].del) continue; + if(R_INF.trio_flag[Get_tn(*h)] == non_trio_flag) continue; + + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, &t); + + ///if it is a contained read, skip + if(r < 0) continue; + + if((t.ul>>32) != query) continue; + get_R_to_U(ruIndex, t.v>>1, &uId, &is_Unitig); + if(uId == (uint32_t)-1 || is_Unitig == 0 || ug->g->seq[uId].del) continue; + if(expect_vis[t.v>>1] == 0) continue; + + if(is_first && utg_vis) memset(utg_vis, 0, ug->g->n_seq); + if(utg_vis == NULL || (utg_vis && utg_vis[uId] == 0)) occ++; + ////fprintf(stderr, "found-utg%.6ul, occ: %u\n", uId+1, occ); + if(utg_vis) utg_vis[uId] = 1; + + (*oLen) += t.ol; + is_first = 0; + } + + return occ; +} + + + inline uint32_t get_edge2existing_node_advance(ma_ug_t *ug, asg_t *g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t* skip_uId, uint32_t skip_uId_n, uint32_t* index, asg_arc_t* t, uint32_t ignore_trio_flag) @@ -24067,6 +22346,57 @@ uint32_t* index, asg_arc_t* t, uint32_t ignore_trio_flag) } +inline uint32_t get_edge2existing_node_advance_by_broken_bub(ma_ug_t *ug, asg_t *g, +ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, uint8_t* expect_vis, +int max_hang, int min_ovlp, uint32_t query, uint32_t* index, asg_arc_t* t, +uint32_t ignore_trio_flag) +{ + uint32_t qn = query>>1, uId, is_Unitig; + int32_t r; + ma_hit_t *h = NULL; + ma_sub_t *sq = NULL; + ma_sub_t *st = NULL; + ma_hit_t_alloc* x = &(sources[qn]); + + uint32_t trio_flag = R_INF.trio_flag[qn], non_trio_flag = (uint32_t)-1; + if(ignore_trio_flag == 0) + { + if(trio_flag == FATHER) non_trio_flag = MOTHER; + if(trio_flag == MOTHER) non_trio_flag = FATHER; + } + + + + for (; (*index) < x->length; (*index)++) + { + h = &(x->buffer[(*index)]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + ///sq has already been removed + ///st must not be removed + ///g-seq must not be removed + if(st->del || g->seq[Get_tn(*h)].del) continue; + if(R_INF.trio_flag[Get_tn(*h)] == non_trio_flag) continue; + + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, t); + + ///if it is a contained read, skip + if(r < 0) continue; + + if((t->ul>>32) != query) continue; + get_R_to_U(ruIndex, t->v>>1, &uId, &is_Unitig); + if(uId == (uint32_t)-1 || is_Unitig == 0 || ug->g->seq[uId].del) continue; + if(expect_vis[t->v>>1] == 0) continue; + + (*index)++; + return 1; + } + + return 0; +} + + inline uint32_t get_num_edges2existing_nodes(ma_ug_t *ug, asg_t *g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t* oLen, @@ -24337,7 +22667,7 @@ ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, } - set_R_to_U(ruIndex, rId>>1, uId>>1, 1); + set_R_to_U(ruIndex, rId>>1, uId>>1, 1, &(r_g->seq[rId>>1].c)); @@ -24388,7 +22718,7 @@ void lable_all_bubbles(asg_t *r_g, long long bubble_dist) ///if this is a bubble ///if(asg_bub_finder_with_del_advance(r_g, v, bubble_dist, &b) == 1) - if(asg_bub_pop1_primary_trio(r_g, NULL, v, bubble_dist, &b, (uint32_t)-1, (uint32_t)-1, 0)) + if(asg_bub_pop1_primary_trio(r_g, NULL, v, bubble_dist, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) { //beg is v, end is b.S.a[0] //note b.b include end, does not include beg @@ -24793,6 +23123,109 @@ uint32_t* return_ava_ol, uint32_t* return_chainLen, uint32_t thresLen, uint32_t return 0; } +int get_contained_reads_chain_by_broken_bub(ma_hit_t *h, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, +R_to_U* ruIndex, ma_ug_t *ug, asg_t *r_g, int max_hang, int min_ovlp, uint32_t endRid, uint32_t uId, +kvec_asg_arc_t_warp* chain_edges, uint32_t* return_ava_cur, uint32_t* return_ava_ol, uint32_t* return_chainLen, +uint8_t* expect_vis, uint8_t* circle_vis, uint8_t* utg_vis, uint32_t thresLen, uint32_t ignore_trio_flag) +{ + (*return_chainLen) = (*return_ava_cur) = (*return_ava_ol) = (uint32_t)-1; + uint32_t trio_flag, non_trio_flag = (uint32_t)-1, contain_rId, contain_uId, is_Unitig, i; + uint32_t chainLen = 0, ava_cur, test_oLen; + int ql, tl; + int32_t r; + ma_sub_t *sq = NULL; + ma_sub_t *st = NULL; + asg_arc_t t; + asg_t* nsg = ug->g; + chain_edges->a.n = 0; + + + ///continue + ///need to update h, endRid, chainLen, chain_buffer and chain_edges + while (h) + { + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + trio_flag = R_INF.trio_flag[Get_qn(*h)]; + + ///don't want to edges between different haps + non_trio_flag = (uint32_t)-1; + if(ignore_trio_flag == 0) + { + if(trio_flag == FATHER) non_trio_flag = MOTHER; + if(trio_flag == MOTHER) non_trio_flag = FATHER; + if(R_INF.trio_flag[Get_tn(*h)] == non_trio_flag) break; + } + + ///just need deleted edges + ///sq might be deleted or not + if(!st->del) break; + if(!h->del) break; + + ///tn must be contained in another existing read + get_R_to_U(ruIndex, Get_tn(*h), &contain_rId, &is_Unitig); + if(contain_rId == (uint32_t)-1 || is_Unitig == 1) break; + if(r_g->seq[contain_rId].del) break; + + get_R_to_U(ruIndex, contain_rId, &contain_uId, &is_Unitig); + if(contain_uId == (uint32_t)-1 || is_Unitig != 1) break; + if(nsg->seq[contain_uId].del) break; + ///contain_uId must be a unitig + + ql = sq->e - sq->s; tl = st->e - st->s; + r = ma_hit2arc(h, ql, tl, max_hang, asm_opt.max_hang_rate, min_ovlp, &t); + + ///if st is contained in sq, or vice verse, skip + if(r < 0) break; + + ///if sq and v are not in the same direction, skip + ///endRid is (t.ul>>32), and t.v is a contained read + if((t.ul>>32) != endRid) break; + + kv_push(asg_arc_t, chain_edges->a, t); + chainLen++; + + if(circle_vis[t.ul>>33] || circle_vis[t.v>>1]) break; + + circle_vis[t.ul>>33] = circle_vis[t.v>>1] = 1; + ///endRid is (t.ul>>32), and t.v is a contained read + ///find edges from t.v to existing unitigs + ava_cur = get_num_edges2existing_nodes_advance_by_broken_bub(ug, r_g, sources, + coverage_cut, ruIndex, expect_vis, max_hang, min_ovlp, t.v, &test_oLen, utg_vis, ignore_trio_flag); + //means find an aim + if(ava_cur > 0) + { + (*return_ava_cur) = ava_cur; + (*return_ava_ol) = test_oLen; + (*return_chainLen) = chainLen; + h = NULL; + for (i = 0; i < chain_edges->a.n; i++) + { + circle_vis[chain_edges->a.a[i].ul>>33] = 0; + circle_vis[chain_edges->a.a[i].v>>1] = 0; + } + return 1; + } + + ///if(chainLen >= thresLen) break; + + ///endRid is (t.ul>>32), and t.v is a contained read + ///haven't found a existing unitig from t.v + ///check if t.v can link to a new contained read + endRid = t.v; + h = get_best_contained_read(ug, r_g, sources, coverage_cut, ruIndex, + max_hang, min_ovlp, endRid, ignore_trio_flag); + } + + for (i = 0; i < chain_edges->a.n; i++) + { + circle_vis[chain_edges->a.a[i].ul>>33] = 0; + circle_vis[chain_edges->a.a[i].v>>1] = 0; + } + return 0; +} + + ///chainLenThres is used to avoid circle void rescue_contained_reads_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t chainLenThres, @@ -24849,7 +23282,7 @@ kvec_t_u32_warp* new_rtg_nodes) for (k = 0; k < nsu->n; k++) { rId = nsu->a[k]>>33; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(r_g->seq[rId].c)); } } @@ -25056,7 +23489,7 @@ kvec_t_u32_warp* new_rtg_nodes) int tri_flag = 0; ///remove very simple circle tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if (!ha_opt_triobin(&asm_opt)) + if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) { ///remove isoloated single read tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); @@ -25099,7 +23532,7 @@ kvec_t_u32_warp* new_rtg_nodes) w=w>>1; r_g->seq[w].del = 1; coverage_cut[w].del = 1; - set_R_to_U(ruIndex, ((uint32_t)(u_vecs.a.a[k])), (u_vecs.a.a[k]>>32), 0); + set_R_to_U(ruIndex, ((uint32_t)(u_vecs.a.a[k])), (u_vecs.a.a[k]>>32), 0, NULL); } } @@ -25202,7 +23635,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) for (k = 0; k < nsu->n; k++) { rId = nsu->a[k]>>33; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(r_g->seq[rId].c)); } } @@ -25327,7 +23760,7 @@ kvec_asg_arc_t_warp* new_rtg_edges) int tri_flag = 0; ///remove very simple circle tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if (!ha_opt_triobin(&asm_opt)) + if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) { ///remove isoloated single read tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); @@ -25393,6 +23826,1420 @@ kvec_asg_arc_t_warp* new_rtg_edges) /*************************just for debug**************************/ } +void set_rtg_flag_by_bubble(bubble_type* bub, ma_ug_t* ug, asg_t *r_g, uint32_t v, uint8_t* vis_flag, +uint32_t flag) +{ + uint32_t beg, sink, *a = NULL, n, i, k, uId, rId; + ma_utg_t* nsu = NULL; + get_bubbles(bub, v, &beg, &sink, &a, &n, NULL); + for (i = 0; i < n; i++) + { + uId = a[i]>>1; + + nsu = &(ug->u.a[uId]); + if(nsu->m > 0) + { + for (k = 0; k < nsu->n; k++) + { + rId = nsu->a[k]>>33; + if(r_g->seq[rId].del) continue; + vis_flag[rId] = flag; + } + } + } + + if(beg != (uint32_t)-1) + { + uId = beg>>1; + + nsu = &(ug->u.a[uId]); + if(nsu->m > 0) + { + for (k = 0; k < nsu->n; k++) + { + rId = nsu->a[k]>>33; + if(r_g->seq[rId].del) continue; + vis_flag[rId] = flag; + } + } + } + + if(sink != (uint32_t)-1) + { + uId = sink>>1; + + nsu = &(ug->u.a[uId]); + if(nsu->m > 0) + { + for (k = 0; k < nsu->n; k++) + { + rId = nsu->a[k]>>33; + if(r_g->seq[rId].del) continue; + vis_flag[rId] = flag; + } + } + } +} + +void print_bubble_filling_status(ma_ug_t *copy_ug, asg_t *r_g, R_to_U* ruIndex, bubble_type* bub, +uint32_t beg_idx, uint32_t occ, asg_arc_t* new_edges, uint32_t new_edges_len) +{ + ma_utg_t* nsu = NULL; + ma_ug_t* ug = copy_ug; + uint32_t i, k_i, k_v, v, k, beg_utg, sink_utg, *a = NULL, n, uId, endRid, is_broken, is_tangle; + uint32_t tangle_occ = 0, broken_occ = 0, recover_occ = 0, is_Unitig, contain_uId; + asg_t* nsg = ug->g; + for (i = beg_idx; i < beg_idx + occ; i++) + { + get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); + if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) continue; + for (k_i = 0, is_broken = 1, is_tangle = 1; k_i < n; k_i++) + { + uId = a[k_i]>>1; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + for (k_v = 0; k_v < 2; k_v++) + { + v = (uId<<1) + k_v; + if(get_real_length(nsg, v, NULL) != 0) continue; + + ///tig + is_tangle = 0; + if(v&1) + { + endRid = nsu->start^1; + } + else + { + endRid = nsu->end^1; + } + + if(r_g->seq[endRid>>1].del) + { + is_broken = 0; + continue; + } + + if(get_real_length(r_g, endRid, NULL) == 0) + { + is_broken = 1; + goto tig_end; + } + else + { + is_broken = 0; + } + } + } + + tig_end: + if(is_tangle) + { + tangle_occ++; + fprintf(stderr, "tangle: beg-utg%.6ul, end-utg%.6ul\n", (beg_utg>>1)+1, (sink_utg>>1)+1); + } + else if(is_broken) + { + broken_occ++; + fprintf(stderr, "broken: beg-utg%.6ul, end-utg%.6ul\n", (beg_utg>>1)+1, (sink_utg>>1)+1); + } + else + { + recover_occ++; + fprintf(stderr, "recover: beg-utg%.6ul, end-utg%.6ul\n", (beg_utg>>1)+1, (sink_utg>>1)+1); + } + } + + fprintf(stderr, "###########tangle_occ: %u, broken_occ: %u, recover_occ: %u\n", tangle_occ, broken_occ, recover_occ); + nsg = ug->g; + for (v = 0; v < nsg->n_seq; v++) + { + uId = v; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + if(nsg->seq[v].del) continue; + for (k = 0; k < nsu->n; k++) + { + set_R_to_U(ruIndex, nsu->a[k]>>33, uId, 1, &(r_g->seq[nsu->a[k]>>33].c)); + } + } + #define check_debug_edge(g, t) (((g).seq[(t).ul>>33].del == 1) && ((g).seq[(t).v>>1].del == 0)) + + + + for (k = 0; k < new_edges_len; k++) + { + v = new_edges[k].ul>>32; + + if(check_debug_edge(*r_g, new_edges[k])) + { + for (i = broken_occ = 0; i < new_edges_len; i++) + { + if((new_edges[i].ul>>32) == v) broken_occ++; + } + + if(broken_occ > 1) + { + fprintf(stderr, "*************tig_to_occ: %u\n", broken_occ); + for (i = 0; i < new_edges_len; i++) + { + if((new_edges[i].ul>>32) == v) + { + get_R_to_U(ruIndex, new_edges[i].v>>1, &contain_uId, &is_Unitig); + if(is_Unitig == 1) + { + nsu = &(ug->u.a[contain_uId]); + for (k_i = 0; k_i < nsu->n; k_i++) + { + if((nsu->a[k_i]>>33) == (new_edges[i].v>>1)) break; + } + + fprintf(stderr, "to-utg%.6ul, idx_of_u: %u, u_n: %u\n", + contain_uId+1, k_i, nsu->n); + } + + } + } + } + } + } + + + + for (v = 0; v < ruIndex->len; v++) + { + get_R_to_U(ruIndex, v, &uId, &is_Unitig); + if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; + } + +} + +void minor_transitive_reduction_r_g(asg_t *r_g, asg_arc_t* rbub_edges, uint32_t num) +{ + uint32_t i, j, k, rId, nv, w; + asg_arc_t* t = NULL; + asg_arc_t* p = NULL; + asg_arc_t *av = NULL; + ///here all edges from v are saved in rbub_edges + ///for edges already in graph, need to check del + ///but for edges in rbub_edges, don't check del + for (i = 0; i < num; i++) + { + t = &rbub_edges[i]; + rId = t->v; + + nv = asg_arc_n(r_g, rId); + av = asg_arc_a(r_g, rId); + for (j = 0; j < nv; j++) + { + if(av[j].del) continue; + w = av[j].v; + for (k = 0; k < num; k++) + { + p = &rbub_edges[k]; + ///this line is not necessary at all + if(k==i) continue; + if(p->v == w) p->del = 1; + } + } + } +} + +void rescue_bubbles_by_contained_reads(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t beg_idx, uint32_t occ, bubble_type* bub) +{ + asg_t* nsg = NULL; + uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, contain_rId; + uint32_t ava_max, ava_ol_max, ava_min_chain, ava_cur, test_oLen, ava_chainLen, is_update, v, w; + uint64_t l_bub, m_bub, r_bub, bub_0, bub_1, a_nodes; + ma_ug_t* ug = i_u_g; + ma_utg_t* nsu = NULL; + ma_hit_t_alloc* x = NULL; + ma_hit_t *h = NULL, *h_max = NULL; + uint8_t* expect_vis = NULL; CALLOC(expect_vis, r_g->n_seq); + uint8_t* circle_vis = NULL; CALLOC(circle_vis, r_g->n_seq); + uint8_t* utg_vis = NULL; CALLOC(utg_vis, ug->g->n_seq); + long long bubble_dist; + asg_arc_t t, t_max, r_edge; + + for (v = 0; v < ug->u.n; v++) ug->g->seq[v].c = PRIMARY_LABLE; + for (k = 0, bubble_dist = 1; k < ug->u.n; k++) bubble_dist += ug->u.a[k].len; + + kvec_t(asg_arc_t) new_edges; + kv_init(new_edges); + + kvec_t(asg_arc_t) rbub_edges; + kv_init(rbub_edges); + + kvec_t_u64_warp u_vecs; + kv_init(u_vecs.a); + + kvec_asg_arc_t_warp chain_edges; + kv_init(chain_edges.a); + + nsg = ug->g; + for (v = 0; v < nsg->n_seq; v++) + { + uId = v; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + if(nsg->seq[v].del) continue; + for (k = 0; k < nsu->n; k++) + { + set_R_to_U(ruIndex, nsu->a[k]>>33, uId, 1, &(r_g->seq[nsu->a[k]>>33].c)); + } + } + + + for (i = beg_idx; i < beg_idx + occ; i++) + { + get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); + ///fprintf(stderr, "\n%u-th: bub-utg%.6ul, bub-utg%.6ul\n", i - beg_idx, (beg_utg>>1)+1, (sink_utg>>1)+1); + if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) continue; + + l_bub = r_bub = (uint64_t)-1; m_bub = i; + get_bub_id(bub, beg_utg>>1, &bub_0, &bub_1, 0); + if(bub_0 != i) l_bub = bub_0; + if(bub_1 != i) l_bub = bub_1; + + get_bub_id(bub, sink_utg>>1, &bub_0, &bub_1, 0); + if(bub_0 != i) r_bub = bub_0; + if(bub_1 != i) r_bub = bub_1; + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 1); + for (k_i = 0; k_i < n; k_i++) + { + uId = a[k_i]>>1; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + for (k_v = 0; k_v < 2; k_v++) + { + v = (uId<<1) + k_v; + if(get_real_length(nsg, v, NULL) != 0) continue; + ///fprintf(stderr, "++++++tig-utg%.6ul\n", uId+1); + + if(v&1) + { + endRid = nsu->start^1; + } + else + { + endRid = nsu->end^1; + } + + //x is the end read of a tip + ///find all overlap of x + x = &(sources[(endRid>>1)]); + ava_ol_max = ava_max = 0; ava_min_chain = (uint32_t)-1; + h_max = NULL; + for (k = 0; k < x->length; k++) + { + ///fprintf(stderr, "k: %u\n", k); + ///h is the edge of endRid + h = &(x->buffer[k]); + ///means we found a contained read + if(get_contained_reads_chain_by_broken_bub(h, sources, coverage_cut, ruIndex, ug, r_g, + max_hang, min_ovlp, endRid, uId, &chain_edges, &ava_cur, &test_oLen, &ava_chainLen, + expect_vis, circle_vis, utg_vis, chainLenThres, 1)) + { + is_update = 0; + + if(ava_cur > ava_max) + { + is_update = 1; + } + else if(ava_cur == ava_max) + { + if(ava_chainLen < ava_min_chain) + { + is_update = 1; + } + else if(ava_chainLen == ava_min_chain && test_oLen > ava_ol_max) + { + is_update = 1; + } + } + + if(is_update) + { + ava_min_chain = ava_chainLen; + ava_max = ava_cur; + ava_ol_max = test_oLen; + h_max = h; + } + } + } + + if(ava_max > 0) + { + ///fprintf(stderr, "ava_max: %u\n", ava_max); + get_contained_reads_chain_by_broken_bub(h_max, sources, coverage_cut, ruIndex, ug, r_g, + max_hang, min_ovlp, endRid, uId, &chain_edges, &ava_cur, &test_oLen, &ava_chainLen, + expect_vis, circle_vis, NULL, chainLenThres, 1); + if(chain_edges.a.n < 1) continue; + ///the last cantained read + t_max = chain_edges.a.a[chain_edges.a.n-1]; + + k = 0; rbub_edges.n = 0; + ///edges from the last contained read to other unitigs + + while(get_edge2existing_node_advance_by_broken_bub(ug, r_g, sources, coverage_cut, + ruIndex, expect_vis, max_hang, min_ovlp, t_max.v, &k, &r_edge, 1)) + { + kv_push(asg_arc_t, rbub_edges, r_edge); + } + + ///need to do transitive reduction + ///note here is different to standard transitive reduction + minor_transitive_reduction_r_g(r_g, rbub_edges.a, rbub_edges.n); + + for (k = 0; k < chain_edges.a.n; k++) + { + t_max = chain_edges.a.a[k]; + ///save all infor for reverting + get_R_to_U(ruIndex, t_max.v>>1, &contain_rId, &is_Unitig); + a_nodes=contain_rId; + a_nodes=a_nodes<<32; + a_nodes=a_nodes|((uint64_t)(t_max.v>>1)); + kv_push(uint64_t, u_vecs.a, a_nodes); + + r_g->seq[t_max.v>>1].del = 0; + coverage_cut[t_max.v>>1].del = 0; + coverage_cut[t_max.v>>1].c = PRIMARY_LABLE; + + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t_max.ul>>32), t_max.v, &t); + kv_push(asg_arc_t, new_edges, t); + + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t_max.v^1), ((t_max.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + + + for (k = 0; k < rbub_edges.n; k++) + { + t = rbub_edges.a[k]; + if(t.del) continue; + + kv_push(asg_arc_t, new_edges, t); + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + + } + } + } + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); + } + + asg_arc_t* p = NULL; + for (k = 0; k < new_edges.n; k++) + { + p = asg_arc_pushp(r_g); + *p = new_edges.a[k]; + } + if(new_edges.n != 0) + { + free(r_g->idx); + r_g->idx = 0; + r_g->is_srt = 0; + asg_cleanup(r_g); + asg_symm(r_g); + } + + while(1) + { + int tri_flag = 0; + ///remove very simple circle + tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); + if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) + { + ///remove isoloated single read + tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); + ///remove the cross at the bubble carefully, just remove inexact cross + tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); + } + if(tri_flag == 0) + { + break; + } + } + + lable_all_bubbles(r_g, bubble_dist); + + + for (k = 0; k < new_edges.n; k++) + { + v = new_edges.a[k].ul>>32; + w = new_edges.a[k].v; + + if(r_g->seq[v>>1].del) continue; + if(r_g->seq[w>>1].del) continue; + ///if this edge is at a bubble + if(r_g->seq_vis[v]!=0 && r_g->seq_vis[w^1]!=0) continue; + + asg_arc_del(r_g, v, w, 1); + asg_arc_del(r_g, w^1, v^1, 1); + } + asg_cleanup(r_g); + asg_symm(r_g); + + for (k = 0; k < u_vecs.a.n; k++) + { + w = (uint32_t)u_vecs.a.a[k]; + w = w<<1; + if((!r_g->seq[w>>1].del) && + (get_real_length(r_g, w, NULL)!=0 || get_real_length(r_g, (w^1), NULL)!=0)) + { + w=w>>1; + ruIndex->index[w] = (uint32_t)-1; + } + else + { + w=w>>1; + r_g->seq[w].del = 1; + coverage_cut[w].del = 1; + } + } + + + for (v = 0; v < ruIndex->len; v++) + { + get_R_to_U(ruIndex, v, &uId, &is_Unitig); + if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; + } + + // fprintf(stderr, "M::%s has done!\n", __func__); + // print_bubble_filling_status(ug, r_g, ruIndex, bub, beg_idx, occ, new_edges.a, new_edges.n); + + kv_destroy(new_edges); + kv_destroy(rbub_edges); + kv_destroy(u_vecs.a); + kv_destroy(chain_edges.a); + free(expect_vis); + free(circle_vis); + free(utg_vis); +} + + +void rescue_bubbles_by_missing_ovlp(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t beg_idx, uint32_t occ, bubble_type* bub) +{ + asg_t* nsg = NULL; + uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, v, w; + uint64_t l_bub, m_bub, r_bub, bub_0, bub_1; + ma_ug_t* ug = i_u_g; + ma_utg_t* nsu = NULL; + uint8_t* expect_vis = NULL; CALLOC(expect_vis, r_g->n_seq); + long long bubble_dist; + asg_arc_t t, r_edge; + + for (v = 0; v < ug->u.n; v++) ug->g->seq[v].c = PRIMARY_LABLE; + for (k = 0, bubble_dist = 1; k < ug->u.n; k++) bubble_dist += ug->u.a[k].len; + + kvec_t(asg_arc_t) new_edges; + kv_init(new_edges); + + kvec_t(asg_arc_t) rbub_edges; + kv_init(rbub_edges); + + + nsg = ug->g; + for (v = 0; v < nsg->n_seq; v++) + { + uId = v; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + if(nsg->seq[v].del) continue; + for (k = 0; k < nsu->n; k++) + { + set_R_to_U(ruIndex, nsu->a[k]>>33, uId, 1, &(r_g->seq[nsu->a[k]>>33].c)); + } + } + + + for (i = beg_idx; i < beg_idx + occ; i++) + { + get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); + ///fprintf(stderr, "\n%u-th: bub-utg%.6ul, bub-utg%.6ul\n", i - beg_idx, (beg_utg>>1)+1, (sink_utg>>1)+1); + if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) continue; + + l_bub = r_bub = (uint64_t)-1; m_bub = i; + get_bub_id(bub, beg_utg>>1, &bub_0, &bub_1, 0); + if(bub_0 != i) l_bub = bub_0; + if(bub_1 != i) l_bub = bub_1; + + get_bub_id(bub, sink_utg>>1, &bub_0, &bub_1, 0); + if(bub_0 != i) r_bub = bub_0; + if(bub_1 != i) r_bub = bub_1; + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 1); + for (k_i = 0; k_i < n; k_i++) + { + uId = a[k_i]>>1; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + for (k_v = 0; k_v < 2; k_v++) + { + v = (uId<<1) + k_v; + if(get_real_length(nsg, v, NULL) != 0) continue; + ///fprintf(stderr, "++++++tig-utg%.6ul\n", uId+1); + + if(v&1) + { + endRid = nsu->start^1; + } + else + { + endRid = nsu->end^1; + } + + + k = 0; rbub_edges.n = 0; + while(get_edge2existing_node_advance_by_broken_bub(ug, r_g, sources, coverage_cut, + ruIndex, expect_vis, max_hang, min_ovlp, endRid, &k, &r_edge, 1)) + { + kv_push(asg_arc_t, rbub_edges, r_edge); + } + + if(rbub_edges.n > 0) + { + ///need to do transitive reduction + ///note here is different to standard transitive reduction + minor_transitive_reduction_r_g(r_g, rbub_edges.a, rbub_edges.n); + for (k = 0; k < rbub_edges.n; k++) + { + t = rbub_edges.a[k]; + if(t.del) continue; + + kv_push(asg_arc_t, new_edges, t); + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + } + } + } + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); + } + + asg_arc_t* p = NULL; + for (k = 0; k < new_edges.n; k++) + { + p = asg_arc_pushp(r_g); + *p = new_edges.a[k]; + } + if(new_edges.n != 0) + { + free(r_g->idx); + r_g->idx = 0; + r_g->is_srt = 0; + asg_cleanup(r_g); + asg_symm(r_g); + } + + while(1) + { + int tri_flag = 0; + ///remove very simple circle + tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); + if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) + { + ///remove isoloated single read + tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); + ///remove the cross at the bubble carefully, just remove inexact cross + tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); + } + if(tri_flag == 0) + { + break; + } + } + + lable_all_bubbles(r_g, bubble_dist); + + + for (k = 0; k < new_edges.n; k++) + { + v = new_edges.a[k].ul>>32; + w = new_edges.a[k].v; + + if(r_g->seq[v>>1].del) continue; + if(r_g->seq[w>>1].del) continue; + ///if this edge is at a bubble + if(r_g->seq_vis[v]!=0 && r_g->seq_vis[w^1]!=0) continue; + + asg_arc_del(r_g, v, w, 1); + asg_arc_del(r_g, w^1, v^1, 1); + } + asg_cleanup(r_g); + asg_symm(r_g); + + + for (v = 0; v < ruIndex->len; v++) + { + get_R_to_U(ruIndex, v, &uId, &is_Unitig); + if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; + } + // fprintf(stderr, "M::%s has done!\n", __func__); + // print_bubble_filling_status(ug, r_g, ruIndex, bub, beg_idx, occ, new_edges.a, new_edges.n); + + kv_destroy(new_edges); + kv_destroy(rbub_edges); + free(expect_vis); +} + +void update_unitig(long long step, long long init, ma_utg_t* nsu, asg_t *r_g, kvec_asg_arc_t_warp* recover_edges, uint32_t update_mode); +void rescue_bubbles_by_missing_ovlp_backward(ma_ug_t *i_u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, +R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t beg_idx, uint32_t occ, bubble_type* bub) +{ + asg_t* nsg = NULL; + uint32_t beg_utg, sink_utg, *a = NULL, n, i, k_i, k_v, k, uId, endRid, is_Unitig, round, cur_backward_steps; + uint32_t v, w, mode, nv; + uint64_t l_bub, m_bub, r_bub, bub_0, bub_1, tmp; + ma_ug_t* ug = i_u_g; + ma_utg_t* nsu = NULL; + uint8_t* expect_vis = NULL; CALLOC(expect_vis, r_g->n_seq); + long long bubble_dist; + long long init, step = 0; + asg_arc_t t, r_edge, *av = NULL; + + for (v = 0; v < ug->u.n; v++) ug->g->seq[v].c = PRIMARY_LABLE; + for (k = 0, bubble_dist = 1; k < ug->u.n; k++) bubble_dist += ug->u.a[k].len; + + kvec_t(asg_arc_t) new_edges; + kv_init(new_edges); + + kvec_asg_arc_t_warp recover_edges; + kv_init(recover_edges.a); + + kvec_t(asg_arc_t) rbub_edges; + kv_init(rbub_edges); + + kvec_t_u64_warp u_vecs; + kv_init(u_vecs.a); + + + nsg = ug->g; + for (v = 0; v < nsg->n_seq; v++) + { + uId = v; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + if(nsg->seq[v].del) continue; + for (k = 0; k < nsu->n; k++) + { + set_R_to_U(ruIndex, nsu->a[k]>>33, uId, 1, &(r_g->seq[nsu->a[k]>>33].c)); + } + } + + + for (i = beg_idx; i < beg_idx + occ; i++) + { + get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); + ///fprintf(stderr, "\n%u-th: bub-utg%.6ul, bub-utg%.6ul\n", i - beg_idx, (beg_utg>>1)+1, (sink_utg>>1)+1); + if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) continue; + + l_bub = r_bub = (uint64_t)-1; m_bub = i; + get_bub_id(bub, beg_utg>>1, &bub_0, &bub_1, 0); + if(bub_0 != i) l_bub = bub_0; + if(bub_1 != i) l_bub = bub_1; + + get_bub_id(bub, sink_utg>>1, &bub_0, &bub_1, 0); + if(bub_0 != i) r_bub = bub_0; + if(bub_1 != i) r_bub = bub_1; + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 1); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 1); + for (k_i = 0; k_i < n; k_i++) + { + uId = a[k_i]>>1; + nsu = &(ug->u.a[uId]); + if(nsu->m == 0) continue; + + rbub_edges.n = round = 0; + for (k_v = 0; k_v < 2; k_v++) + { + if(rbub_edges.n > 0) + { + cur_backward_steps = nsu->n - round - 1; + if(cur_backward_steps > backward_steps) + { + cur_backward_steps = backward_steps; + } + } + else + { + cur_backward_steps = backward_steps; + } + + + v = (uId<<1) + k_v; + if(get_real_length(nsg, v, NULL) != 0) continue; + ///fprintf(stderr, "++++++tig-utg%.6ul\n", uId+1); + ///that means this unitig has been changed + // if(nsu->start!=((uint64_t)(nsu->a[0])>>32)) continue; + // if((nsu->end^1)!=((uint64_t)(nsu->a[nsu->n-1])>>32)) continue; + + if(v&1) + { + init = 0; + step = 1; + mode = 1; + } + else + { + init = nsu->n - 1; + step = -1; + mode = 0; + } + + rbub_edges.n = 0; + for (round = 0; round < cur_backward_steps && init >= 0 && init < (long long)nsu->n; + init = init + step, round++) + { + endRid = ((uint64_t)(nsu->a[init]))>>32; + endRid = endRid^mode; + + k = 0; rbub_edges.n = 0; + while(get_edge2existing_node_advance_by_broken_bub(ug, r_g, sources, coverage_cut, + ruIndex, expect_vis, max_hang, min_ovlp, endRid, &k, &r_edge, 1)) + { + kv_push(asg_arc_t, rbub_edges, r_edge); + } + + if(rbub_edges.n > 0) break; + } + + if(rbub_edges.n > 0) + { + //save for revert + tmp = mode; tmp = tmp <<31; tmp = tmp | (uint64_t)(init); tmp = tmp << 32; tmp = tmp | uId; + kv_push(uint64_t, u_vecs.a, tmp); + ///need to do transitive reduction + ///note here is different to standard transitive reduction + minor_transitive_reduction_r_g(r_g, rbub_edges.a, rbub_edges.n); + + + ///modify read graph + for (init = init - step; init >= 0 && init < (long long)nsu->n; init = init - step) + { + w = ((uint64_t)(nsu->a[init]))>>32; + nv = asg_arc_n(r_g, w); + av = asg_arc_a(r_g, w); + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + kv_push(asg_arc_t, recover_edges.a, av[k]); + if(asg_get_arc(r_g, av[k].v^1, av[k].ul>>32^1, &t)==0) + { + fprintf(stderr, "error\n"); + } + kv_push(asg_arc_t, recover_edges.a, t); + } + + + nv = asg_arc_n(r_g, w^1); + av = asg_arc_a(r_g, w^1); + for (k = 0; k < nv; k++) + { + if(av[k].del) continue; + kv_push(asg_arc_t, recover_edges.a, av[k]); + if(asg_get_arc(r_g, av[k].v^1, av[k].ul>>32^1, &t)==0) + { + fprintf(stderr, "error\n"); + } + kv_push(asg_arc_t, recover_edges.a, t); + } + + ///w = ((uint64_t)(nsu->a[init]))>>32; + asg_seq_del(r_g, w>>1); + expect_vis[w>>1] = 0; + } + + + for (k = 0; k < rbub_edges.n; k++) + { + t = rbub_edges.a[k]; + if(t.del) continue; + + kv_push(asg_arc_t, new_edges, t); + get_edge_from_source(sources, coverage_cut, ruIndex, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, new_edges, t); + } + } + + } + } + + + set_rtg_flag_by_bubble(bub, ug, r_g, l_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, r_bub, expect_vis, 0); + set_rtg_flag_by_bubble(bub, ug, r_g, m_bub, expect_vis, 0); + } + + asg_arc_t* p = NULL; + for (k = 0; k < new_edges.n; k++) + { + p = asg_arc_pushp(r_g); + *p = new_edges.a[k]; + } + if(new_edges.n != 0) + { + free(r_g->idx); + r_g->idx = 0; + r_g->is_srt = 0; + asg_cleanup(r_g); + asg_symm(r_g); + } + + while(1) + { + int tri_flag = 0; + ///remove very simple circle + tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); + if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) + { + ///remove isoloated single read + tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); + ///remove the cross at the bubble carefully, just remove inexact cross + tri_flag += asg_arc_del_cross_bubble(r_g, bubble_dist); + } + if(tri_flag == 0) + { + break; + } + } + + lable_all_bubbles(r_g, bubble_dist); + + + for (k = 0; k < new_edges.n; k++) + { + v = new_edges.a[k].ul>>32; + w = new_edges.a[k].v; + + if(r_g->seq[v>>1].del) continue; + if(r_g->seq[w>>1].del) continue; + ///if this edge is at a bubble + if(r_g->seq_vis[v]!=0 && r_g->seq_vis[w^1]!=0) continue; + + asg_arc_del(r_g, v, w, 1); + asg_arc_del(r_g, w^1, v^1, 1); + } + asg_cleanup(r_g); + asg_symm(r_g); + + for (k = 0; k < recover_edges.a.n; k++) + { + recover_edges.a.a[k].del = 1; + } + + for (k = 0; k < u_vecs.a.n; k++) + { + mode = (uint64_t)u_vecs.a.a[k]>>63; + if(mode == 1) step = 1; + if(mode == 0) step = -1; + init = (uint64_t)((uint64_t)u_vecs.a.a[k]>>32)&((uint64_t)(0x7fffffff)); + uId = (uint32_t)u_vecs.a.a[k]; + nsu = &(ug->u.a[uId]); + + endRid = ((uint64_t)(nsu->a[init]))>>32; + endRid = endRid^mode; + if(get_real_length(r_g, endRid, NULL) > 0) + { + update_unitig(step, init, nsu, r_g, &recover_edges, 1); + } + else + { + update_unitig(step, init, nsu, r_g, &recover_edges, 0); + } + // if(get_real_length(r_g, endRid, NULL) <= 0) + // { + // update_unitig(step, init, nsu, r_g, &recover_edges, 0); + // } + } + + uint64_t recov_occ = 0; + for (k = 0; k < recover_edges.a.n; k++) + { + if(recover_edges.a.a[k].del) continue; + p = asg_arc_pushp(r_g); + *p = recover_edges.a.a[k]; + recov_occ++; + } + + if(recov_occ != 0) + { + free(r_g->idx); + r_g->idx = 0; + r_g->is_srt = 0; + asg_cleanup(r_g); + asg_symm(r_g); + } + + + for (v = 0; v < ruIndex->len; v++) + { + get_R_to_U(ruIndex, v, &uId, &is_Unitig); + if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; + } + + // fprintf(stderr, "M::%s has done!\n", __func__); + // print_bubble_filling_status(ug, r_g, ruIndex, bub, beg_idx, occ, new_edges.a, new_edges.n); + + kv_destroy(new_edges); + kv_destroy(rbub_edges); + kv_destroy(u_vecs.a); + kv_destroy(recover_edges.a); + free(expect_vis); +} + + +void reset_bub(bubble_type* bub, ma_ug_t *ug, asg_t *sg, ma_ug_t *back_ug, hc_links* link, hc_links* back_link, +R_to_U* ruIndex, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, +int max_hang, int min_ovlp, kvec_asg_arc_t_warp* new_rtg_edges) +{ + uint32_t v, k, uId, is_Unitig, occ_het, pre_het = 0, cur_het = 0; + uint64_t d = RC_0; + asg_t* nsg = NULL; + ma_utg_t *nsu = NULL; + ///ma_ug_t *ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); + + destory_hc_links(link); + memset(link, 0, sizeof(hc_links)); + init_hc_links(link, ug->g->n_seq, R_INF.total_reads); + destory_bubbles(bub); + memset(bub, 0, sizeof(bubble_type)); + + uint8_t* back_ug_flag = NULL; CALLOC(back_ug_flag, back_ug->g->n_seq); + + ///fprintf(stderr, "ug->g->n_seq: %u, back_ug->g->n_seq: %u\n", ug->g->n_seq, back_ug->g->n_seq); + + nsg = back_ug->g; + for (v = 0; v < nsg->n_seq; v++) + { + nsu = &(back_ug->u.a[v]); + if(nsu->m == 0) continue; + if(nsg->seq[v].del) continue; + for (k = 0; k < back_link->a.a[v].f.n; k++) + { + if(back_link->a.a[v].f.a[k].del || back_link->a.a[v].f.a[k].dis != RC_1) continue; + back_ug_flag[v] = 1; + pre_het++; + break; + } + for (k = 0; k < nsu->n; k++) + { + set_R_to_U(ruIndex, nsu->a[k]>>33, v, 1, &(sg->seq[nsu->a[k]>>33].c)); + } + } + + for (v = 0; v < ug->g->n_seq; v++) + { + nsu = &(ug->u.a[v]); + for (k = occ_het = 0; k < nsu->n; k++) + { + get_R_to_U(ruIndex, nsu->a[k]>>33, &uId, &is_Unitig); + if(uId == (uint32_t)-1 || is_Unitig != 1) continue; + if(back_ug_flag[uId]) occ_het++; + } + if(occ_het > (nsu->n*0.8)) + { + push_hc_edge(&(link->a.a[v]), v, 1, 1, &d); + cur_het++; + } + } + + ///fprintf(stderr, "pre_het: %u, cur_het: %u\n", pre_het, cur_het); + + for (v = 0; v < ruIndex->len; v++) + { + get_R_to_U(ruIndex, v, &uId, &is_Unitig); + if(is_Unitig == 1) ruIndex->index[v] = (uint32_t)-1; + } + + + new_rtg_edges->a.n = 0; + classify_untigs(ug, sg, coverage_cut, sources, reverse_sources, ruIndex, new_rtg_edges, + max_hang, min_ovlp); + identify_bubbles(ug, bub, link); + update_bubble_chain(ug, bub, 0, 1); + resolve_bubble_chain_tangle(ug, bub); + free(back_ug_flag); + // fprintf(stderr, "bub.f_bub: %lu, bub.b_bub: %lu, bub.b_end_bub: %lu, bub.tangle_bub: %lu, bub.cross_bub: %lu\n", + // bub->f_bub, bub->b_bub, bub->b_end_bub, bub->tangle_bub, bub->cross_bub); +} + + +int bub_complex(asg_t *sg, ma_ug_t *ug, bubble_type* bub, uint32_t bid, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, kvec_t_u32_warp* stack, +int max_hang, int min_ovlp, uint8_t* trio_flag, uint8_t* vis_flag, kv_asg_arc_t* e) +{ + if(bid >= bub->f_bub) return 0; + uint32_t beg_utg, sink_utg, *a = NULL, n, begRid, sinkRid, tLen, i, k_i, k_j, k_v, rID/**, cur_flag, pre_flag, after_flag**/; + int is_switch_0, is_switch_1; + ma_utg_t* nsu = NULL; + get_bubbles(bub, bid, &beg_utg, &sink_utg, &a, &n, NULL); + if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) return 0; + if(beg_utg&1) + { + begRid = ug->u.a[beg_utg>>1].start^1; + } + else + { + begRid = ug->u.a[beg_utg>>1].end^1; + } + + if(sink_utg&1) + { + sinkRid = ug->u.a[sink_utg>>1].start; + } + else + { + sinkRid = ug->u.a[sink_utg>>1].end; + } + + + buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(ug->g->n_seq * 2, sizeof(binfo_t)); + for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; + is_switch_0 = is_switch_1 = 1; + + asg_bub_pop1_primary_trio_switch_check(ug->g, ug, beg_utg, tLen, &b, FATHER, DROP, 0, NULL, NULL, &is_switch_0); + + if(is_switch_0 == 0) + { + asg_bub_pop1_primary_trio_switch_check(ug->g, ug, beg_utg, tLen, &b, MOTHER, DROP, 0, NULL, NULL, &is_switch_1); + } + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + + for (k_i = 0; k_i < n; k_i++) + { + nsu = &(ug->u.a[a[k_i]>>1]); + for (k_j = 0; k_j < nsu->n; k_j++) + { + rID = nsu->a[k_j]>>33; + if(R_INF.trio_flag[rID] == DROP) R_INF.trio_flag[rID] = AMBIGU; + } + } + /*******************************for debug************************************/ + // for (i = 0; i < sg->n_seq; i++) + // { + // if(R_INF.trio_flag[i] == DROP) fprintf(stderr, "ERROR-1\n"); + // } + /*******************************for debug************************************/ + + if(is_switch_0 == 0 && is_switch_1 == 0) return 0; + + asg_arc_t *acur = NULL; + uint32_t cur, ncur, v, n_vx = sg->n_seq<<1; + stack->a.n = 0; + memset(vis_flag, 0, n_vx); + + kv_push(uint32_t, stack->a, begRid); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + ncur = asg_arc_n(sg, cur); + acur = asg_arc_a(sg, cur); + vis_flag[cur] = 1; + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]) continue; + if(acur[i].v == sinkRid) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + vis_flag[sinkRid] = 1; + + + ma_hit_t_alloc* x = NULL; + ma_hit_t *h = NULL; + ma_sub_t *sq = NULL; + ma_sub_t *st = NULL; + int32_t r; + asg_arc_t t; + + + for (k_i = 0; k_i < n; k_i++) + { + nsu = &(ug->u.a[a[k_i]>>1]); + for (k_j = 0; k_j < nsu->n; k_j++) + { + rID = nsu->a[k_j]>>33; + for (k_v = 0; k_v < 2; k_v++) + { + v = (rID<<1) + k_v; + if(vis_flag[v] == 0) continue; + x = &(sources[v>>1]); + for (i = 0; i < x->length; i++) + { + h = &(x->buffer[i]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + if(st->del || sg->seq[Get_tn(*h)].del) continue; + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, &t); + + ///if it is a contained read, skip + if(r < 0) continue; + if((t.ul>>32) != v) continue; + if(vis_flag[t.ul>>32] == 0 || vis_flag[t.v] == 0) continue; + kv_push(asg_arc_t, *e, t); + get_edge_from_source(sources, coverage_cut, NULL, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, *e, t); + } + + } + } + } + + return 1; + + /** + for (v = 0; v < n_vx; v++) + { + if(vis_flag[v] == 0) continue; + fprintf(stderr, "v: %u, n_vx: %u\n", v, n_vx); + x = &(sources[v>>1]); + for (i = 0; i < x->length; i++) + { + fprintf(stderr, "i: %u, x->length: %u\n", i, x->length); + h = &(x->buffer[i]); + sq = &(coverage_cut[Get_qn(*h)]); + st = &(coverage_cut[Get_tn(*h)]); + if(st->del || sg->seq[Get_tn(*h)].del) continue; + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, + asm_opt.max_hang_rate, min_ovlp, &t); + + ///if it is a contained read, skip + if(r < 0) continue; + if((t.ul>>32) != v) continue; + if(vis_flag[t.ul>>32] == 0 || vis_flag[t.v] == 0) continue; + kv_push(asg_arc_t, *e, t); + get_edge_from_source(sources, coverage_cut, NULL, max_hang, min_ovlp, + (t.v^1), ((t.ul>>32)^1), &t); + kv_push(asg_arc_t, *e, t); + } + } + **/ + + + /*******************************for debug************************************/ + // uint32_t utg_occ = 0, rtg_occ = 0; + // for (i = 0; i < n; i++) + // { + // utg_occ += ug->u.a[a[i]>>1].n; + // } + + // for (i = 0; i < n_vx; i++) + // { + // if(vis_flag[i]) rtg_occ++; + // } + + // fprintf(stderr, "bid: %u, rtg_occ: %u, utg_occ: %u\n", bid, rtg_occ, utg_occ); + // if(rtg_occ != utg_occ + 2) fprintf(stderr, "ERROR\n"); + /*******************************for debug************************************/ +} + + +void debug_bubble_chain(asg_t *sg, ma_ug_t *ug, bubble_type* bub, uint32_t bid, kvec_t_u32_warp* stack, uint8_t* vis_flag) +{ + if(bid >= bub->f_bub) return; + uint32_t beg_utg, sink_utg, *a = NULL, n, begRid, sinkRid; + get_bubbles(bub, bid, &beg_utg, &sink_utg, &a, &n, NULL); + if(beg_utg == (uint32_t)-1 || sink_utg == (uint32_t)-1) return; + if(beg_utg&1) + { + begRid = ug->u.a[beg_utg>>1].start^1; + } + else + { + begRid = ug->u.a[beg_utg>>1].end^1; + } + + if(sink_utg&1) + { + sinkRid = ug->u.a[sink_utg>>1].start; + } + else + { + sinkRid = ug->u.a[sink_utg>>1].end; + } + + asg_arc_t *acur = NULL; + uint32_t cur, ncur, i, n_vx = sg->n_seq<<1; + stack->a.n = 0; + memset(vis_flag, 0, n_vx); + + kv_push(uint32_t, stack->a, begRid); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + ncur = asg_arc_n(sg, cur); + acur = asg_arc_a(sg, cur); + vis_flag[cur] = 1; + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]) continue; + if(acur[i].v == sinkRid) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + vis_flag[sinkRid] = 1; + + + /*******************************for debug************************************/ + uint32_t utg_occ = 0, rtg_occ = 0; + for (i = 0; i < n; i++) + { + utg_occ += ug->u.a[a[i]>>1].n; + } + + for (i = 0; i < n_vx; i++) + { + if(vis_flag[i]) rtg_occ++; + } + + fprintf(stderr, "bid: %u, rtg_occ: %u, utg_occ: %u\n", bid, rtg_occ, utg_occ); + if(rtg_occ != utg_occ + 2) fprintf(stderr, "ERROR\n"); + /*******************************for debug************************************/ +} + +void rescue_missing_hap_ovlp(ma_ug_t *u_g, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, +int max_hang, int min_ovlp, bubble_type* bub, long long gap_fuzz) +{ + uint32_t i, n_arc = r_g->n_arc, fix_bub = 0; + uint8_t* vis_flag = NULL; CALLOC(vis_flag, r_g->n_seq*2); + kvec_t_u32_warp stack; kv_init(stack.a); + kv_asg_arc_t e; kv_init(e); + double index_time = yak_realtime(); + + + for (i = 0; i < bub->f_bub; i++) + { + fix_bub += bub_complex(r_g, u_g, bub, i, sources, coverage_cut, &stack, max_hang, min_ovlp, R_INF.trio_flag, vis_flag, &e); + } + + asg_arc_t* p = NULL; + for (i = 0; i < e.n; i++) + { + p = asg_arc_pushp(r_g); + *p = e.a[i]; + } + if(e.n != 0) + { + free(r_g->idx); + r_g->idx = 0; + r_g->is_srt = 0; + asg_cleanup(r_g); + asg_symm(r_g); + asg_arc_del_trans(r_g, gap_fuzz); + // for (i = 0; i < bub->f_bub; i++) + // { + // debug_bubble_chain(r_g, u_g, bub, i, &stack, vis_flag); + // } + } + + fprintf(stderr, "[M::%s::%.3f] # inserted edges: %u, # fixed bubbles: %u\n", + __func__, yak_realtime() - index_time, r_g->n_arc - n_arc, fix_bub); + + free(vis_flag); + kv_destroy(stack.a); + kv_destroy(e); +} + + +void rescue_bubble_by_chain(asg_t *sg, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, +long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chainLenThres, long long gap_fuzz) +{ + kvec_asg_arc_t_warp new_rtg_edges; + kv_init(new_rtg_edges.a); + ma_ug_t *ug = NULL; + ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); + + hc_links copy_link, link; + memset(©_link, 0, sizeof(hc_links)); + memset(&link, 0, sizeof(hc_links)); + init_hc_links(©_link, ug->g->n_seq, R_INF.total_reads); + asg_t *copy_sg = copy_read_graph(sg); + ma_ug_t *copy_ug = copy_untig_graph(ug); + adjust_utg_by_primary(©_ug, copy_sg, TRIO_THRES, sources, reverse_sources, coverage_cut, + bubble_dist, tipsLen, tip_drop_ratio, stops_threshold, ruIndex, chimeric_rate, drop_ratio, + max_hang, min_ovlp, &new_rtg_edges, ©_link); + ma_ug_destroy(copy_ug); copy_ug = NULL; + asg_destroy(copy_sg); copy_sg = NULL; + + uint32_t beg_idx, occ; + bubble_type bub; + memset(&bub, 0, sizeof(bubble_type)); + copy_ug = copy_untig_graph(ug); + + reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, + reverse_sources, max_hang, min_ovlp, &new_rtg_edges); + beg_idx = bub.f_bub; occ = bub.b_bub + bub.b_end_bub + bub.tangle_bub; + rescue_bubbles_by_contained_reads(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub); + ///output_unitig_graph(sg, coverage_cut, (char*)"debug_1.hic", sources, ruIndex, max_hang, min_ovlp); + + + + ma_ug_destroy(ug); ug = NULL; ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); + reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, + reverse_sources, max_hang, min_ovlp, &new_rtg_edges); + beg_idx = bub.f_bub; occ = bub.b_bub + bub.b_end_bub + bub.tangle_bub; + rescue_bubbles_by_missing_ovlp(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub); + ///output_unitig_graph(sg, coverage_cut, (char*)"debug_2.hic", sources, ruIndex, max_hang, min_ovlp); + + + ma_ug_destroy(ug); ug = NULL; ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); + reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, + reverse_sources, max_hang, min_ovlp, &new_rtg_edges); + beg_idx = bub.f_bub; occ = bub.b_bub + bub.b_end_bub + bub.tangle_bub; + rescue_bubbles_by_missing_ovlp_backward(ug, sg, sources, coverage_cut, ruIndex, max_hang, min_ovlp, chainLenThres, beg_idx, occ, &bub); + ///output_unitig_graph(sg, coverage_cut, (char*)"debug_3.hic", sources, ruIndex, max_hang, min_ovlp); + + if(ha_opt_triobin(&asm_opt)) + { + ma_ug_destroy(ug); ug = NULL; ug = ma_ug_gen_primary(sg, PRIMARY_LABLE); + reset_bub(&bub, ug, sg, copy_ug, &link, ©_link, ruIndex, coverage_cut, sources, + reverse_sources, max_hang, min_ovlp, &new_rtg_edges); + rescue_missing_hap_ovlp(ug, sg, sources, coverage_cut, max_hang, min_ovlp, &bub, gap_fuzz); + } + + + + destory_bubbles(&bub); + destory_hc_links(©_link); + destory_hc_links(&link); + ma_ug_destroy(ug); + kv_destroy(new_rtg_edges.a); + ma_ug_destroy(copy_ug); copy_ug = NULL; +} + void update_unitig(long long step, long long init, ma_utg_t* nsu, asg_t *r_g, kvec_asg_arc_t_warp* recover_edges, uint32_t update_mode) { @@ -25499,7 +25346,7 @@ uint32_t is_bubble_check, uint32_t is_primary_check) for (k = 0; k < nsu->n; k++) { rId = nsu->a[k]>>33; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(r_g->seq[rId].c)); } } @@ -25705,7 +25552,7 @@ uint32_t is_bubble_check, uint32_t is_primary_check) int tri_flag = 0; ///remove very simple circle tri_flag += asg_arc_del_simple_circle_untig(sources, coverage_cut, r_g, 100, 0); - if (!ha_opt_triobin(&asm_opt)) + if ((!ha_opt_triobin(&asm_opt))&&(!ha_opt_hic(&asm_opt))) { ///remove isoloated single read tri_flag += asg_arc_del_triangular_advance(r_g, bubble_dist); @@ -25879,7 +25726,7 @@ kvec_asg_arc_t_warp* keep_edges) for (k = 0; k < nsu->n; k++) { rId = nsu->a[k]>>33; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(r_g->seq[rId].c)); } } @@ -26767,7 +26614,7 @@ long long bubble_dist, uint32_t chainLenThres) for (k = 0; k < nsu->n; k++) { rId = nsu->a[k]>>33; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(r_g->seq[rId].c)); } } @@ -27283,7 +27130,9 @@ ma_sub_t **coverage_cut_ptr, int debug_g) ma_hit_contained_advance(sources, n_read, coverage_cut, ruIndex, max_hang_length, mini_overlap_length); sg = ma_sg_gen(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); - ///debug_info_of_specfic_node((char*)"m64062_190804_172951/130483063/ccs", sg, ruIndex, (char*)"sbsbsb"); + ///debug_info_of_specfic_node((char*)"m64043_200504_050026/93784180/ccs", sg, ruIndex, (char*)"sbsbsb"); + + asg_arc_del_trans(sg, gap_fuzz); @@ -27298,8 +27147,9 @@ ma_sub_t **coverage_cut_ptr, int debug_g) } asg_cut_tip(sg, asm_opt.max_short_tip); - ///debug_info_of_specfic_node("m64062_190803_042216/15205346/ccs", sg, "inner_1"); + ///debug_info_of_specfic_node("m64043_200505_112554/8849050/ccs", sg, "inner_1"); ///drop_inexact_edegs_at_bubbles(sg, bubble_dist); + if(clean_round > 0) { @@ -27409,20 +27259,6 @@ ma_sub_t **coverage_cut_ptr, int debug_g) asg_arc_del_simple_circle_untig(sources, coverage_cut, sg, 100, 0); - if (asm_opt.flag & HA_F_VERBOSE_GFA) - { - /*******************************for debug***************************************/ - write_debug_graph(sg, sources, coverage_cut, output_file_name, n_read, reverse_sources, ruIndex); - debug_gfa:; - /*******************************for debug***************************************/ - } - /** - debug_ma_hit_t(sources, coverage_cut, n_read, max_hang_length, - mini_overlap_length); - debug_ma_hit_t(reverse_sources, coverage_cut, n_read, max_hang_length, - mini_overlap_length); - **/ - ///note: don't apply asg_arc_del_too_short_overlaps() after this function!!!! rescue_contained_reads_aggressive(NULL, sg, sources, coverage_cut, ruIndex, max_hang_length, mini_overlap_length, bubble_dist, 10, 1, 0, NULL, NULL); @@ -27435,8 +27271,32 @@ ma_sub_t **coverage_cut_ptr, int debug_g) // rescue_no_coverage_aggressive(sg, sources, reverse_sources, &coverage_cut, ruIndex, max_hang_length, // mini_overlap_length, bubble_dist, 10); - if (ha_opt_triobin(&asm_opt)) + + if (asm_opt.flag & HA_F_VERBOSE_GFA) { + /*******************************for debug***************************************/ + write_debug_graph(sg, sources, coverage_cut, output_file_name, n_read, reverse_sources, ruIndex); + debug_gfa:; + /*******************************for debug***************************************/ + } + + + if (ha_opt_triobin(&asm_opt) && ha_opt_hic(&asm_opt)) + { + rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + + char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); + sprintf(buf, "%s.hic.bench", output_file_name); + benchmark_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, bubble_dist, + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length); + free(buf); + } + else if (ha_opt_triobin(&asm_opt)) + { + rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); sprintf(buf, "%s.dip", output_file_name); output_unitig_graph(sg, coverage_cut, buf, sources, ruIndex, max_hang_length, mini_overlap_length); @@ -27444,14 +27304,24 @@ ma_sub_t **coverage_cut_ptr, int debug_g) output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang_length, mini_overlap_length); + 0.05, 0.9, max_hang_length, mini_overlap_length, 0); output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang_length, mini_overlap_length); + 0.05, 0.9, max_hang_length, mini_overlap_length, 0); + } + else if(ha_opt_hic(&asm_opt)) + { + rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + + char *buf = (char*)calloc(strlen(output_file_name) + 25, 1); + sprintf(buf, "%s.hic", output_file_name); + output_hic_graph(sg, coverage_cut, buf, sources, reverse_sources, bubble_dist, + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length); + free(buf); } else { - output_unitig_graph(sg, coverage_cut, output_file_name, sources, ruIndex, max_hang_length, mini_overlap_length); if(VERBOSE >= 1) @@ -27462,6 +27332,9 @@ ma_sub_t **coverage_cut_ptr, int debug_g) output_contig_graph_primary_pre(sg, coverage_cut, output_file_name, sources, reverse_sources, asm_opt.small_pop_bubble_size, asm_opt.max_short_tip, ruIndex, max_hang_length, mini_overlap_length); + rescue_bubble_by_chain(sg, coverage_cut, sources, reverse_sources, bubble_dist, + (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length, 10, gap_fuzz); + output_contig_graph_primary(sg, coverage_cut, output_file_name, sources, reverse_sources, bubble_dist, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, 0.05, 0.9, max_hang_length, mini_overlap_length); @@ -27512,8 +27385,7 @@ long long bubble_dist, int read_graph, int write) &R_INF, output_file_name); } - ///debug_info_of_specfic_read("m64062_190803_042216/177341795/ccs", sources, reverse_sources, -1, "beg"); - // debug_info_of_specfic_read("m64062_190807_194840/126682874/ccs", sources, reverse_sources, -1, "beg"); + ///debug_info_of_specfic_read("m64011_190830_220126/31720629/ccs", sources, reverse_sources, -1, "beg"); if (!(asm_opt.flag & HA_F_BAN_ASSEMBLY)) { diff --git a/Overlaps.h b/Overlaps.h index cc16f5e..ca18270 100644 --- a/Overlaps.h +++ b/Overlaps.h @@ -21,6 +21,7 @@ #define PRIMARY_LABLE 0 #define ALTER_LABLE 1 #define HAP_LABLE 2 +#define FAKE_LABLE 4 #define TRIO_THRES 0.9 #define DOUBLE_CHECK_THRES 0.1 #define FINAL_DOUBLE_CHECK_THRES 0.2 @@ -106,6 +107,11 @@ typedef struct { uint8_t no_l_indel; } asg_arc_t; +typedef struct { + size_t n, m; + asg_arc_t* a; +} kv_asg_arc_t; + typedef struct { uint32_t len:31, circ:1; // len: length of the unitig; circ: circular if non-zero @@ -155,6 +161,7 @@ typedef struct { size_t n, m; ma_utg_t *a; } ma_utg_v; typedef struct { ma_utg_v u; asg_t *g; + kvec_t(uint64_t) occ; } ma_ug_t; typedef struct { @@ -394,7 +401,8 @@ typedef struct { }kvec_asg_arc_t_warp; void sort_kvec_t_u64_warp(kvec_t_u64_warp* u_vecs, uint32_t is_descend); - +int asg_arc_del_multi(asg_t *g); +int asg_arc_del_asymm(asg_t *g); typedef struct { uint32_t q_pos; @@ -442,11 +450,8 @@ long long max_hang_length, long long clean_round, long long gap_fuzz, float min_ovlp_drop_ratio, float max_ovlp_drop_ratio, char* output_file_name, long long bubble_dist, int read_graph, int write); -void debug_info_of_specfic_read(char* name, ma_hit_t_alloc* sources, -ma_hit_t_alloc* reverse_sources, int id, char* command); - +void debug_info_of_specfic_read(char* name, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, int id, char* command); void collect_abnormal_edges(ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, long long readNum); - void add_overlaps(ma_hit_t_alloc* source_paf, ma_hit_t_alloc* dest_paf, uint64_t* source_index, long long listLen); void remove_overlaps(ma_hit_t_alloc* source_paf, uint64_t* source_index, long long listLen); void add_overlaps_from_different_sources(ma_hit_t_alloc* source_paf_list, ma_hit_t_alloc* dest_paf, @@ -455,18 +460,6 @@ uint64_t* source_index, long long listLen); #define EvaluateLen(U, id) ((U).a[(id)].start) #define IsMerge(U, id) ((U).a[(id)].end) #define kv_reuse(v, rn, rm, r) ((v).n = (rn), (v).m = (rm), (v).a = (r)) -#define long_tip(U, id, threshold) ((EvaluateLen((U), (id))>=(threshold))&&(!((U).a[(id)].circ))) -///there are threee cases: -///1. if this untig is too long (>maxShortUntig), it must be not short untig/must be a long untig -///2. if this untig is long (>minLongUntig && EvaluateLen(ug->u, av[i].v>>1) > (EvaluateLen(ug->u, v>>1)*l_untig_rate)), it might be a long tip -#define check_long_tip(U, id, minLongUntig, maxShortUntig, ShortUntigRate, mainLen) \ - ((!((U).a[(id)].circ)) \ - && \ - ((EvaluateLen((U), (id)) > (maxShortUntig))\ - ||\ - ((long_tip((U), (id), (minLongUntig)))\ - &&\ - (EvaluateLen((U), (id)) > (ShortUntigRate)*(mainLen))))) #define Get_vis(visit, v, d) (((visit)[(v)>>1])&(((((v)<<(d))&1)+1))) #define Set_vis(visit, v, d) (((visit)[(v)>>1])|=(((((v)<<(d))&1)+1))) @@ -481,14 +474,10 @@ typedef struct { void init_R_to_U(R_to_U* x, uint64_t len); void destory_R_to_U(R_to_U* x); -void set_R_to_U(R_to_U* x, uint32_t rID, uint32_t uID, uint32_t is_Unitig); +void set_R_to_U(R_to_U* x, uint32_t rID, uint32_t uID, uint32_t is_Unitig, uint8_t* flag); void get_R_to_U(R_to_U* x, uint32_t rID, uint32_t* uID, uint32_t* is_Unitig); void transfor_R_to_U(R_to_U* x); -void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_tangle); -void clean_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, -long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, -R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, -uint32_t miniBiGraph, float chimeric_rate, int is_final_clean); +void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, kvec_asg_arc_t_warp* edge, int require_equal_nv, int test_tangle); int asg_pop_bubble_primary(asg_t *g, int max_dist); long long asg_arc_del_simple_circle_untig(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *g, long long circleLen, int is_drop); @@ -743,13 +732,12 @@ R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) stops_threshold, b_0) == LOOP) { return UNAVAILABLE; - } + } if(get_unitig(nsg, ug, v_1, &vEnd, &ELen_1, &tmp, &max_stop_nodeLen, &max_stop_baseLen, stops_threshold, b_1) == LOOP) { return UNAVAILABLE; } - if(ELen_0<=min_edge_length || ELen_1<=min_edge_length) return UNAVAILABLE; rIdContig b_max, b_min; @@ -770,7 +758,6 @@ R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) uint32_t max_count = 0, min_count = 0; ma_utg_t *node_min = NULL, *node_max = NULL; - if(ug != NULL) { /*****************************label all unitigs****************************************/ @@ -781,12 +768,11 @@ R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) for (b_max.readI = 0; b_max.readI < node_max->n; b_max.readI++) { qn = (node_max->a[b_max.readI]>>33); - set_R_to_U(ruIndex, qn, (b_max.b_0->b.a[b_max.untigI]>>1), 1); + set_R_to_U(ruIndex, qn, (b_max.b_0->b.a[b_max.untigI]>>1), 1, &(read_sg->seq[qn].c)); } } /*****************************label all unitigs****************************************/ - ///each unitig for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++) { @@ -820,7 +806,6 @@ R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) } } } - /*****************************label all unitigs****************************************/ for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++) { @@ -833,7 +818,6 @@ R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) } } /*****************************label all unitigs****************************************/ - } else { @@ -841,7 +825,7 @@ R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++) { qn = (b_max.b_0->b.a[b_max.untigI]>>1); - set_R_to_U(ruIndex, qn, 1, 1); + set_R_to_U(ruIndex, qn, 1, 1, &(read_sg->seq[qn].c)); } /*****************************label all reads****************************************/ @@ -896,8 +880,6 @@ R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) return NON_PLOID; } - - inline uint32_t check_different_haps_naive(asg_t *nsg, ma_ug_t *ug, asg_t *read_sg, uint32_t v_0, uint32_t v_1, ma_hit_t_alloc* reverse_sources, buf_t* b_0, buf_t* b_1, R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold) @@ -1047,8 +1029,6 @@ uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* void rescue_missing_overlaps_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges); -void deduplicate(ma_ug_t *src, asg_t *read_g, ma_hit_t_alloc* reverse_sources, long long minLongUntig, -long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, uint32_t resolve_tangle); void all_to_all_deduplicate(ma_ug_t* ug, asg_t* read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, uint8_t postive_flag, float drop_rate, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, float double_check_rate); void drop_semi_circle(ma_ug_t *ug, asg_t* nsg, asg_t* read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex); @@ -1061,9 +1041,77 @@ uint32_t is_bubble_check, uint32_t is_primary_check); uint32_t get_edge_from_source(ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t target, asg_arc_t* t); uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, -uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop); +uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes); int unitig_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio); + +typedef struct{ + double weight; + uint32_t uID:31, del:1; + uint64_t dis; + uint64_t occ; + ///uint64_t occ:63, scaff:1; + ///uint32_t enzyme; +} hc_edge; + +typedef struct{ + kvec_t(hc_edge) e; + kvec_t(hc_edge) f;//forbiden +} hc_linkeage; + +typedef struct{ + uint64_t beg, end; +}bed_interval; + +typedef struct{ + size_t n, m; + bed_interval* a; +}bed_in; + +typedef struct{ + kvec_t(hc_linkeage) a; + kvec_t(uint64_t) enzymes; + kvec_t(bed_in) bed; + uint32_t* u_idx; + uint64_t r_num; +} hc_links; + +typedef struct{ + ///kvec_t(hc_edge) a; + size_t n, m; + hc_edge *a; +}hc_edge_warp; + +void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num); +void destory_hc_links(hc_links* link); +void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources, +long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, +R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, +uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, +float drop_ratio, hc_links* link); +void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate, +ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, +long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, +R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, +kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link); +void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg); +ma_ug_t* copy_untig_graph(ma_ug_t *src); +ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, +uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, +long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, +float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench); +asg_t* copy_read_graph(asg_t *src); +ma_ug_t *ma_ug_gen(asg_t *g); +void ma_ug_destroy(ma_ug_t *ug); + +inline int inter_interval(int a_s, int a_e, int b_s, int b_e, int* i_s, int* i_e) +{ + if(a_s > b_e || b_s > a_e) return 0; + if(i_s) (*i_s) = a_s >= b_s? a_s : b_s; ///MAX(a_s, b_s); + if(i_e) (*i_e) = a_e <= b_e? a_e : b_e; ///MIN(a_e, b_e); + return 1; +} + #define JUNK_COV 5 #define DISCARD_RATE 0.8 diff --git a/Purge_Dups.cpp b/Purge_Dups.cpp index f840f1f..7474a14 100644 --- a/Purge_Dups.cpp +++ b/Purge_Dups.cpp @@ -7,6 +7,7 @@ #include "Correct.h" #include "kthread.h" #include "kdq.h" +#include "hic.h" KDQ_INIT(uint64_t) @@ -1492,11 +1493,10 @@ void quick_LIS(asg_arc_t_offset* x, uint32_t n, kvec_t_i32_warp* tailIndex, kvec tailIndex->a.n = prevIndex->a.n = 0; if(n == 0) return; - kv_resize(int32_t, tailIndex->a, n); kv_resize(int32_t, prevIndex->a, n); - long long len = 1, i, pos, m; + long long len = 1, i, pos, m; ///the length of chain must be >=1 tailIndex->a.a[0] = 0; prevIndex->a.a[0] = -1; @@ -1520,8 +1520,7 @@ void quick_LIS(asg_arc_t_offset* x, uint32_t n, kvec_t_i32_warp* tailIndex, kvec // future subsequence // It will replace ceil value in tailIndices pos = GetCeilIndex(x, tailIndex, -1, len - 1, Get_yOff(x[i].Off)); - - prevIndex->a.a[i] = tailIndex->a.a[pos - 1]; + prevIndex->a.a[i] = pos > 0? tailIndex->a.a[pos - 1] : -1; tailIndex->a.a[pos] = i; } } @@ -1626,7 +1625,6 @@ uint32_t* xBeg, uint32_t* xEnd, uint32_t* yBeg, uint32_t* yEnd) u_buffer->a.n = m; ///print_asg_arc_t_offset(u_buffer->a.a, u_buffer->a.n, "after"); - quick_LIS(u_buffer->a.a, u_buffer->a.n, tailIndex, prevIndex); if(tailIndex->a.n == 0) return; @@ -1930,6 +1928,7 @@ long long* r_x_pos_beg, long long* r_x_pos_end, long long* r_y_pos_beg, long lon ruIndex, reverse_sources, coverage_cut, read_g, position_index, max_hang, min_ovlp, xUid, yUid, u_buffer, tailIndex, prevIndex, r_x_pos_beg, r_x_pos_end, r_y_pos_beg, r_y_pos_end); + if(hap_can->index_end == XCY && yReads->len > (xReads->len*2)) return NON_PLOID; if(hap_can->index_end == YCX && xReads->len > (yReads->len*2)) return NON_PLOID; if(hap_can->index_end == (uint32_t)-1) return NON_PLOID; @@ -2978,6 +2977,7 @@ static void hap_alignment_advance_worker(void *_data, long eid, int tid) { continue; } + if(Get_match(hap_can) < Get_match(u_can->a.a[k])) { @@ -3291,7 +3291,7 @@ int asg_pop_bubble_purge_graph(asg_t *purge_g, int max_dist) for (i = 0; i < nv; ++i) // asg_bub_pop1() may delete some edges/arcs if (!av[i].del) ++n_arc; if (n_arc > 1) - n_pop += asg_bub_pop1_primary_trio(purge_g, NULL, v, max_dist, &b, (uint32_t)-1, DROP, 1); + n_pop += asg_bub_pop1_primary_trio(purge_g, NULL, v, max_dist, &b, (uint32_t)-1, DROP, 1, NULL, NULL); } free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); if (n_pop) asg_cleanup(purge_g); @@ -3925,10 +3925,57 @@ kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edg } + + +void collect_reverse_unitig_pair(hc_links* link, ma_ug_t *ug, hap_overlaps* t) +{ + uint32_t i = 0, k = 0, rId_0, rId_1, pre_0, pre_1, b_0 = t->xUid, b_1 = t->yUid; + uint64_t d = RC_2; + ma_utg_t* u_b_0 = &(ug->u.a[b_0]); + ma_utg_t* u_b_1 = &(ug->u.a[b_1]); + if(u_b_0->n == 0) return; + if(u_b_1->n == 0) return; + + for (i = t->x_beg_id, pre_0 = (uint32_t)-1; i < t->x_end_id; i++) + { + rId_0 = u_b_0->a[i]>>33; + if(link->u_idx[rId_0] == (uint32_t)-1) continue; + if(pre_0 == link->u_idx[rId_0]) continue; + pre_0 = link->u_idx[rId_0]; + + for (k = t->y_beg_id, pre_1 = (uint32_t)-1; k < t->y_end_id; k++) + { + rId_1 = u_b_1->a[k]>>33; + if(link->u_idx[rId_1] == (uint32_t)-1) continue; + if(pre_1 == link->u_idx[rId_1]) continue; + pre_1 = link->u_idx[rId_1]; + push_hc_edge(&(link->a.a[pre_0]), pre_1, 1, 1, &d); + push_hc_edge(&(link->a.a[pre_1]), pre_0, 1, 1, &d); + } + } + +} + + +void collect_reverse_unitigs_purge(buf_t* b_0, hc_links* link, ma_ug_t *ug, hap_overlaps_list* all_ovlp) +{ + if(b_0->b.n <= 1) return; + uint32_t k; + int index = 0; + for (k = 0; k < b_0->b.n - 1; k++) + { + index = get_specific_hap_overlap(&(all_ovlp->x[b_0->b.a[k]>>1]), b_0->b.a[k]>>1, b_0->b.a[k+1]>>1); + if(index == -1) continue; + collect_reverse_unitig_pair(link, ug, &(all_ovlp->x[b_0->b.a[k]>>1].a.a[index])); + } +} + + void link_unitigs(asg_t *purge_g, ma_ug_t *ug, hap_overlaps_list* all_ovlp, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, asg_t *read_g, uint64_t* position_index, kvec_asg_arc_t_offset* u_buffer, kvec_t_i32_warp* tailIndex, -kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edge, uint8_t* visit) +kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edge, uint8_t* visit, +hc_links* link) { uint32_t v, n_vtx = purge_g->n_seq * 2, beg, end; long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; @@ -3948,6 +3995,8 @@ kvec_t_i32_warp* prevIndex, int max_hang, int min_ovlp, kvec_asg_arc_t_warp* edg { continue; } + + if(link) collect_reverse_unitigs_purge(&b_0, link, ug, all_ovlp); purge_merge(purge_g, ug, all_ovlp, &b_0, ruIndex, reverse_sources, coverage_cut, read_g, position_index, u_buffer, tailIndex, prevIndex,max_hang, min_ovlp, edge, visit); } @@ -4086,6 +4135,7 @@ long long get_specific_contig_length(asg_t *g, uint8_t *del) totalLen += len; } //kdq_destroy(uint64_t, q); + free(mark); return totalLen; } @@ -4131,6 +4181,7 @@ uint32_t minLen, double purge_threshold) asg_t* nsg = ug->g; uint64_t v, k, total_bases = 0, alter_bases = 0, primary_bases = 0, purge_bases = 0; kt_for(asm_opt.thread_num, hap_alignment_advance_worker, hap_buf, nsg->n_seq); + filter_hap_overlaps_by_length(all_ovlp, minLen); normalize_hap_overlaps_advance(all_ovlp, back_all_ovlp, ug, read_g, reverse_sources, ruIndex); @@ -4168,11 +4219,10 @@ uint32_t minLen, double purge_threshold) return 0; } - void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage) +uint32_t just_contain, uint32_t just_coverage, hc_links* link) { asg_t *purge_g = NULL; purge_g = asg_init(); @@ -4229,7 +4279,7 @@ uint32_t just_contain, uint32_t just_coverage) for (i = 0, offset = 0; i < reads->n; i++) { rId = reads->a[i]>>33; - set_R_to_U(ruIndex, rId, uId, 1); + set_R_to_U(ruIndex, rId, uId, 1, &(read_g->seq[rId].c)); position_index[rId] = offset; position_index[rId] = position_index[rId] << 32; @@ -4246,7 +4296,7 @@ uint32_t just_contain, uint32_t just_coverage) init_hap_alignment_struct_pip(&hap_buf, asm_opt.thread_num, nsg->n_seq, ug, read_g, sources, reverse_sources, ruIndex, coverage_cut, position_index, density, max_hang, min_ovlp, 0.05, &all_ovlp); - + if(hap_buf.cov_threshold < 0) { if(if_ploid_sample(ug, read_g, ruIndex, sources, reverse_sources, coverage_cut, @@ -4301,6 +4351,7 @@ uint32_t just_contain, uint32_t just_coverage) purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].c = ALTER_LABLE; purge_g->seq[all_ovlp.x[uId].a.a[i].xUid].del = 1; all_ovlp.x[uId].a.a[i].status = DELETE; + if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp.x[uId].a.a[i])); } if(all_ovlp.x[uId].a.a[i].type == XCY) @@ -4309,6 +4360,7 @@ uint32_t just_contain, uint32_t just_coverage) purge_g->seq[all_ovlp.x[uId].a.a[i].yUid].c = ALTER_LABLE; purge_g->seq[all_ovlp.x[uId].a.a[i].yUid].del = 1; all_ovlp.x[uId].a.a[i].status = DELETE; + if(link) collect_reverse_unitig_pair(link, ug, &(all_ovlp.x[uId].a.a[i])); } ///print_hap_paf(ug, &(all_ovlp.x[uId].a.a[i])); } @@ -4360,7 +4412,7 @@ uint32_t just_contain, uint32_t just_coverage) link_unitigs(purge_g, ug, &all_ovlp, ruIndex, reverse_sources, coverage_cut, read_g, position_index, &(hap_buf.buf[0].u_buffer), &(hap_buf.buf[0].u_buffer_tailIndex), &(hap_buf.buf[0].u_buffer_prevIndex), - max_hang, min_ovlp, edge, hap_buf.buf[0].visit); + max_hang, min_ovlp, edge, hap_buf.buf[0].visit, link); } for (v = 0; v < all_ovlp.num; v++) diff --git a/Purge_Dups.h b/Purge_Dups.h index 5824858..4635e22 100644 --- a/Purge_Dups.h +++ b/Purge_Dups.h @@ -15,7 +15,7 @@ void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, -uint32_t just_contain, uint32_t just_coverage); +uint32_t just_contain, uint32_t just_coverage, hc_links* link); void fill_unitig(uint64_t* buffer, uint32_t bufferLen, asg_t* read_g, kvec_asg_arc_t_warp* edge, uint32_t is_circle, uint64_t* rLen); void get_contig_length(ma_ug_t *ug, asg_t *g, uint64_t* primaryLen, uint64_t* alterLen); diff --git a/README.md b/README.md index ac76766..5dd7a5a 100644 --- a/README.md +++ b/README.md @@ -192,4 +192,8 @@ page](https://github.com/chhylp123/hifiasm/issues). ## Limitations -1. Purging haplotig duplications may introduce misassemblies. \ No newline at end of file +1. Purging haplotig duplications may introduce misassemblies. + +## Citation + +Cheng, H., Concepcion, G.T., Feng, X., Zhang, H., Li H. Haplotype-resolved de novo assembly using phased assembly graphs with hifiasm. Nat Methods 18, 170–175 (2021). https://doi.org/10.1038/s41592-020-01056-5 diff --git a/Trio.cpp b/Trio.cpp index b4934a9..75f20f4 100644 --- a/Trio.cpp +++ b/Trio.cpp @@ -67,7 +67,7 @@ static yak_ch_t *yak_ch_restore_core(yak_ch_t *ch0, const char *fn, int mode, .. { va_list ap; FILE *fp; - uint32_t t[3]; + uint32_t t[3], f_tmp = 0; char magic[4]; int i, j, absent, min_cnt = 0, mid_cnt = 0, mode_err = 0; uint64_t mask = (1ULL<k && (int)t[1] == ch->pre); for (i = 0; i < 1<pre; ++i) { yak_ht_t *h = ch->h[i].h; - fread(t, 4, 2, fp); + f_tmp += fread(t, 4, 2, fp); + ///t[0] = kh_capacity(h), t[1] = kh_size(h); if (ch0 == 0) yak_ht_resize(h, t[0]); for (j = 0; j < (int)t[1]; ++j) { uint64_t key; - fread(&key, 8, 1, fp); + f_tmp += fread(&key, 8, 1, fp); if (mode == YAK_LOAD_ALL) { ++n_ins; yak_ht_put(h, key, &absent); if (absent) ++n_new; } else if (mode == YAK_LOAD_TRIOBIN1 || mode == YAK_LOAD_TRIOBIN2) { int cnt = key & mask, x, shift = mode == YAK_LOAD_TRIOBIN1? 0 : 2; + //1. filter singleton k-mer; 2. label non-repeat and repeat if (cnt >= mid_cnt) x = 2<= min_cnt) x = 1<= 0) { khint_t k; + ///no need cnt at all key = (key & ~mask) | x; ++n_ins; k = yak_ht_put(h, key, &absent); diff --git a/hic.cpp b/hic.cpp new file mode 100644 index 0000000..4e6081d --- /dev/null +++ b/hic.cpp @@ -0,0 +1,11737 @@ +#define __STDC_LIMIT_MACROS +#include "float.h" +#include +#include "hic.h" +#include "htab.h" +#include "assert.h" +#include "Overlaps.h" +#include "Hash_Table.h" +#include "Correct.h" +#include "khashl.h" +#include "kthread.h" +#include "ksort.h" +#include "kseq.h" // FASTA/Q parser +#include "kdq.h" +KSEQ_INIT(gzFile, gzread) +KDQ_INIT(uint64_t) + + +#define OFFSET_RATE 0.000000001 +#define OFFSET_SECOND_RATE 0.0000000001 +#define SCALL 10000 +#define OFFSET_RATE_MAX_W 20.8286263517*SCALL +#define OFFSET_RATE_MIN_W 4.0000003e-10*SCALL + +#define HIC_COUNTER_BITS 12 +#define HIC_MAX_COUNT ((1<>HIC_COUNTER_BITS == (b)>>HIC_COUNTER_BITS) +#define hic_ct_hash(a) ((a)>>HIC_COUNTER_BITS) +KHASHL_MAP_INIT(static klib_unused, hc_pt_t, hc_pt, uint64_t, uint64_t, hic_ct_hash, hic_ct_eq) + +typedef struct{ + kvec_t(char) name; + kvec_t(uint64_t) name_Len; + kvec_t(char) r; + kvec_t(uint64_t) r_Len; +} reads_t; + +typedef struct{ + kvec_t(uint8_t) vis; + kvec_t(uint64_t) x; + kvec_t(uint64_t) dis; + uint64_t uID_mode, uID_shift, tmp_v, tmp_d; +}pdq; + +typedef struct{ + kvec_t(hc_edge_warp) rGraph; + kvec_t(uint64_t) order; + pdq pq; + kvec_t(uint8_t) rGraphSet; + kvec_t(uint8_t) rGraphVis; + kvec_t(uint8_t) utgVis; + kvec_t(uint8_t) bmerVis; + kdq_t(uint64_t) *q; + kvec_t(uint32_t) parent; + kvec_t(double) p_weight; + const uint64_t* enzymes; + uint64_t uID_mode, uID_shift, n, src, dest, n_e, c_e; + int p_mer, a_mer, b_mer; +} min_cut_t; + + +typedef struct{ + kvec_t(uint32_t) a; + uint32_t h[2]; + uint8_t full_bub; + int status[2]; + double weight[2], weight_convex; +}partition_warp; + +typedef struct{ + size_t n, m; + partition_warp* a; + uint32_t* index; +}G_partition; + +typedef struct{ + kvec_t(uint8_t) vis; + double weight; + long long bid, uid, chainID; +}block_phase_type; + +typedef struct{ + uint64_t n; + uint8_t* lock; + uint32_t* hap; + uint32_t m[3]; + uint32_t label, label_add, label_shift; + hc_links* link; + G_partition g_p; + G_partition group_g_p; + kvec_t(double) label_buffer; + block_phase_type b; +}H_partition; + +typedef struct { + uint32_t p; // the optimal parent vertex + uint32_t d; // the shortest distance from the initial vertex + uint32_t nc; // max count of reads, no matter positive or negative + double nh, w[2]; + uint32_t uc, ac; // used vertex/allowed vertex + uint32_t r:31, s:1; // r: the number of remaining incoming arc; s: state + //s: state, s=0, this edge has not been visited, otherwise, s=1 +} bub_p_t; + +typedef struct { + ///all information for each node + bub_p_t *a; + kvec_t(uint32_t) S; // set of vertices without parents, nodes with all incoming edges visited + kvec_t(uint32_t) T; // set of tips + kvec_t(uint32_t) b; // visited vertices + kvec_t(uint32_t) e; // visited edges/arcs + uint32_t exist_hap_label; +} bub_p_t_warp; + + +typedef struct { + hc_pt_t *h; + uint64_t n; + uint64_t *a; + khint_t end;///end of total idx +} hc_pt1_t; + +typedef struct { + ma_ug_t* ug; + asg_t* read_g; + hc_links* link; + uint64_t uID_bits; + uint64_t uID_mode; + uint64_t pos_bits; + uint64_t pos_mode; + uint64_t rev_mode; + uint64_t k; + uint64_t max_cnt; + + + uint64_t pre; + uint64_t tot; + uint64_t tot_pos; + uint64_t up_bound; + hc_pt1_t* idx_buf; + long double a, b, frac, max_d; +} ha_ug_index; + +typedef struct { // data structure for each step in kt_pipeline() + uint64_t key, pos; +} ch_buf_t; + +typedef struct { + kvec_t(uint64_t) a; +} kvec_cnt; + +typedef struct { + kvec_t(ch_buf_t) a; +} kvec_pos; + +typedef struct { // global data structure for kt_pipeline() + int is_cnt; + uint64_t buf_bytes; + ha_ug_index *h; + kvec_cnt* cnt; + kvec_pos* buf; + uint64_t n_thread; +} pldat_t; + +typedef struct { + uint64_t s, e, id; +} pe_hit; + +typedef struct { + kvec_t(pe_hit) a; +} kvec_pe_hit; + +typedef struct { + kvec_t(hc_edge) a; +}kvec_hc_edge; + + +#define pe_hit_an1_key(x) ((x).s) +KRADIX_SORT_INIT(pe_hit_an1, pe_hit, pe_hit_an1_key, 8) +#define pe_hit_an2_key(x) ((x).e) +KRADIX_SORT_INIT(pe_hit_an2, pe_hit, pe_hit_an2_key, 8) +#define generic_key(x) (x) +KRADIX_SORT_INIT(hc64, uint64_t, generic_key, 8) +KRADIX_SORT_INIT(u32, uint32_t, generic_key, 4) +#define g_partition_key(x) (((x)>>1)+((x)<<63)) +KRADIX_SORT_INIT(g_partition, uint64_t, g_partition_key, 8) + + +typedef struct { // global data structure for kt_pipeline() + const ha_ug_index* idx; + kseq_t *ks1, *ks2; + int64_t chunk_size; + uint64_t n_thread; + uint64_t total_base; + uint64_t total_pair; + kvec_pe_hit hits; + hc_links* link; +} sldat_t; + +typedef struct { + uint64_t ref; + uint64_t off_cnt; +} s_hit; + +typedef struct { + kvec_t(s_hit) a; +} kvec_vote; + +typedef struct { // data structure for each step in kt_pipeline() + const ha_ug_index* idx; + int n, m, sum_len; + uint64_t *len, id; + char **seq; + ch_buf_t *buf; + kvec_vote* pos_buf; + pe_hit* pos; + hc_links* link; +} stepdat_t; + +#define generic_key(x) (x) +KRADIX_SORT_INIT(b64, uint64_t, generic_key, 8) +#define ch_buf_t_key(a) ((a).key) +KRADIX_SORT_INIT(ch_buf, ch_buf_t, ch_buf_t_key, member_size(ch_buf_t, key)) +#define hc_pos_key(x) ((x)<<1) +KRADIX_SORT_INIT(hc_pos, uint64_t, hc_pos_key, 8) +#define hc_s_hit_an1_key(a) ((a).ref) +KRADIX_SORT_INIT(hc_s_hit_an1, s_hit, hc_s_hit_an1_key, 8) +#define hc_s_hit_an2_key(a) ((uint32_t)(a).off_cnt) +KRADIX_SORT_INIT(hc_s_hit_an2, s_hit, hc_s_hit_an2_key, 8) +#define hc_edge_key_u(a) ((a).uID) +KRADIX_SORT_INIT(hc_edge_u, hc_edge, hc_edge_key_u, 4) +#define hc_edge_key_d(a) ((a).dis) +KRADIX_SORT_INIT(hc_edge_d, hc_edge, hc_edge_key_d, member_size(hc_edge, dis)) + +typedef struct { + kvec_t(kvec_t_u64_warp) matrix; + uint64_t uID_shift, dis_mode; +} MT; + +typedef struct{ + uint64_t beg, end, dis, cnt_0, cnt_1; +} trans_p_t; + +typedef struct{ + trans_p_t* a; + size_t n, m; + uint64_t max; +} trans_idx; + + +reads_t R1, R2; +ha_ug_index* ug_index; + +void build_bub_graph(ma_ug_t* ug, bubble_type* bub); + +void init_ha_ug_index_opt(ha_ug_index* idx, ma_ug_t *ug, int k, pldat_t* p) +{ + uint64_t i, n; + for (idx->uID_bits=1; (uint64_t)(1<uID_bits)<(uint64_t)ug->u.n; idx->uID_bits++); + idx->pos_bits = 64 - idx->uID_bits - 1; + idx->uID_mode = (((uint64_t)-1) << (64-idx->uID_bits))>>1; + idx->pos_mode = ((uint64_t)-1) >> (64-idx->pos_bits); + idx->rev_mode = ((uint64_t)1) << 63; + idx->ug = ug; + idx->k = k; + idx->pre = HIC_COUNTER_BITS; + idx->tot = 1 << idx->pre; + idx->tot_pos = 0; + idx->up_bound = 1; + CALLOC(idx->idx_buf, idx->tot); + for (i = 0; i < idx->tot; i++) + { + idx->idx_buf[i].h = hc_pt_init(); + } + for (i = n = 0; i < ug->u.n; i++) + { + n += ug->u.a[i].len; + } + n = n << 3; + p->h = idx; + p->buf_bytes = n>>7; + CALLOC(p->cnt, idx->tot); + CALLOC(p->buf, idx->tot); + for (i = 0; i < idx->tot; i++) + { + kv_init(p->cnt[i].a); + kv_init(p->buf[i].a); + } + p->n_thread = asm_opt.thread_num; +} + +inline uint64_t get_k_direction(uint64_t x[4]) +{ + if(x[1] != x[3]) + { + return x[1] < x[3]? 0 : 1; + } + else if(x[0] != x[2]) + { + return x[0] < x[2]? 0 : 1; + } + else + { + return (uint64_t)-1; + } +} + +inline uint64_t hc_hash_long(uint64_t x[4], uint64_t* skip, uint64_t k) +{ + ///compare forward k-mer and reverse complementary strand + (*skip) = get_k_direction(x); + if((*skip) == (uint64_t)-1) return (*skip); + if (k <= 32) return ((x[(*skip)<<1|0]<<32)|(x[(*skip)<<1|1])); + return yak_hash64_64(x[(*skip)<<1|0]) + yak_hash64_64(x[(*skip)<<1|1]); +} + +inline uint64_t get_hc_pt1_count(ha_ug_index* index, uint64_t key, uint64_t** pos_list) +{ + uint64_t bucket_mask = (1ULL<pre) - 1; + hc_pt1_t* h = &(index->idx_buf[key & bucket_mask]); + uint64_t beg; + khint_t k; + k = hc_pt_get(h->h, key); + if (k == kh_end(h->h)) + { + return 0; + } + beg = kh_val(h->h, k); + if(pos_list) *pos_list = h->a + beg; + if((kh_key(h->h, k)&HIC_MAX_COUNT)h, k)&HIC_MAX_COUNT; + if(k == h->end) return h->n - beg; + for (k++; k != kh_end(h->h); ++k) + { + if (kh_exist(h->h, k)) + { + return kh_val(h->h, k) - beg; + } + } + return h->n - beg; +} + +void test_hc_pt1(char* seq, uint64_t len, uint64_t uID, ha_ug_index* idx) +{ + uint64_t i, l, k, pos, *pos_list = NULL, cnt; + uint64_t x[4], mask = (1ULL<k) - 1, shift = idx->k - 1, hash, skip; + for (i = l = 0, x[0] = x[1] = x[2] = x[3] = 0; i < len; ++i) { + int c = seq_nt4_table[(uint8_t)seq[i]]; + ///c = 00, 01, 10, 11 + if (c < 4) { // not an "N" base + ///x[0] & x[1] are the forward k-mer + ///x[2] & x[3] are the reverse complementary k-mer + x[0] = (x[0] << 1 | (c&1)) & mask; + x[1] = (x[1] << 1 | (c>>1)) & mask; + x[2] = x[2] >> 1 | (uint64_t)(1 - (c&1)) << shift; + x[3] = x[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift; + if (++l >= idx->k) + { + hash = hc_hash_long(x, &skip, idx->k); + if(skip == (uint64_t)-1) continue; + pos = (skip << 63) | ((uID << (64-idx->uID_bits))>>1) | (i & idx->pos_mode); + cnt = get_hc_pt1_count(idx, hash, &pos_list); + if(cnt == 0) fprintf(stderr, "ERROR cnt, uID: %lu\n", uID); + for (k = 0; k < cnt; k++) + { + if(pos_list[k]==pos) + { + pos_list[k] = (uint64_t)-1; + break; + } + } + if(k == cnt) fprintf(stderr, "ERROR k\n"); + + } + } else l = 0, x[0] = x[1] = x[2] = x[3] = 0; // if there is an "N", restart + } +} + +void test_unitig_index(ha_ug_index* idx, ma_ug_t *ug) +{ + double index_time = yak_realtime(); + uint32_t i, j; + ma_utg_t *u = NULL; + hc_pt1_t *h = NULL; + idx->ug = ug; + for (i = 0; i < idx->ug->u.n; i++) + { + u = &(idx->ug->u.a[i]); + if(u->m == 0) continue; + test_hc_pt1(u->s, u->len, i, idx); + } + + for (i = 0; i < idx->tot; i++) + { + h = &(idx->idx_buf[i]); + for (j = 0; j < h->n; j++) + { + if(h->a[j] != (uint64_t)-1) + { + fprintf(stderr, "ERROR j\n"); + } + } + + } + + fprintf(stderr, "[M::%s::%.3f] ==> Test has been passed\n", __func__, yak_realtime()-index_time); +} + +void hc_pt_t_gen_single(hc_pt1_t* pt, uint64_t* up_bound) +{ + khint_t k; + uint64_t c; + + if(up_bound) + { + for (k = 0; k != kh_end(pt->h); ++k) { + if (kh_exist(pt->h, k)) { + if(kh_val(pt->h, k) > (*up_bound)) + { + kh_val(pt->h, k) = 0; + kh_key(pt->h, k) = (kh_key(pt->h, k)&HIC_KEY_MODE)| + (kh_val(pt->h, k)h, k):HIC_MAX_COUNT); + } + } + } + } + + + for (k = 0, pt->n = 0; k != kh_end(pt->h); ++k) { + if (kh_exist(pt->h, k)) { + c = kh_val(pt->h, k); + kh_val(pt->h, k) = pt->n; + pt->n += c; + pt->end = k; + } + } + CALLOC(pt->a, pt->n); +} + +int write_hc_pt_index(ha_ug_index* idx, char* file_name) +{ + char* gfa_name = (char*)malloc(strlen(file_name)+25); + sprintf(gfa_name, "%s.hic.tlb.bin", file_name); + FILE* fp = fopen(gfa_name, "w"); + if (!fp) { + free(gfa_name); + return 0; + } + + fwrite(&idx->uID_bits, sizeof(idx->uID_bits), 1, fp); + fwrite(&idx->uID_mode, sizeof(idx->uID_mode), 1, fp); + fwrite(&idx->pos_bits, sizeof(idx->pos_bits), 1, fp); + fwrite(&idx->pos_mode, sizeof(idx->pos_mode), 1, fp); + fwrite(&idx->rev_mode, sizeof(idx->rev_mode), 1, fp); + fwrite(&idx->k, sizeof(idx->k), 1, fp); + fwrite(&idx->pre, sizeof(idx->pre), 1, fp); + fwrite(&idx->tot, sizeof(idx->tot), 1, fp); + fwrite(&idx->tot_pos, sizeof(idx->tot_pos), 1, fp); + + uint64_t i = 0; + for (i = 0; i < idx->tot; i++) + { + fwrite(&idx->idx_buf[i].n, sizeof(idx->idx_buf[i].n), 1, fp); + fwrite(&idx->idx_buf[i].end, sizeof(idx->idx_buf[i].end), 1, fp); + fwrite(idx->idx_buf[i].a, sizeof(uint64_t), idx->idx_buf[i].n, fp); + hc_pt_save(idx->idx_buf[i].h, fp); + } + + fprintf(stderr, "[M::%s] Index has been written.\n", __func__); + free(gfa_name); + fclose(fp); + return 1; +} + +int load_hc_pt_index(ha_ug_index** r_idx, char* file_name) +{ + uint64_t flag = 0; + double index_time = yak_realtime(); + char* gfa_name = (char*)malloc(strlen(file_name)+25); + sprintf(gfa_name, "%s.hic.tlb.bin", file_name); + FILE* fp = fopen(gfa_name, "r"); + if (!fp) { + free(gfa_name); + return 0; + } + ha_ug_index* idx = NULL; CALLOC(idx, 1); + + flag += fread(&idx->uID_bits, sizeof(idx->uID_bits), 1, fp); + flag += fread(&idx->uID_mode, sizeof(idx->uID_mode), 1, fp); + flag += fread(&idx->pos_bits, sizeof(idx->pos_bits), 1, fp); + flag += fread(&idx->pos_mode, sizeof(idx->pos_mode), 1, fp); + flag += fread(&idx->rev_mode, sizeof(idx->rev_mode), 1, fp); + flag += fread(&idx->k, sizeof(idx->k), 1, fp); + flag += fread(&idx->pre, sizeof(idx->pre), 1, fp); + flag += fread(&idx->tot, sizeof(idx->tot), 1, fp); + flag += fread(&idx->tot_pos, sizeof(idx->tot_pos), 1, fp); + MALLOC(idx->idx_buf, idx->tot); + uint64_t i = 0; + for (i = 0; i < idx->tot; i++) + { + flag += fread(&idx->idx_buf[i].n, sizeof(idx->idx_buf[i].n), 1, fp); + flag += fread(&idx->idx_buf[i].end, sizeof(idx->idx_buf[i].end), 1, fp); + MALLOC(idx->idx_buf[i].a, idx->idx_buf[i].n); + flag += fread(idx->idx_buf[i].a, sizeof(uint64_t), idx->idx_buf[i].n, fp); + hc_pt_load(&(idx->idx_buf[i].h), fp); + } + + + (*r_idx) = idx; + + free(gfa_name); + fclose(fp); + fprintf(stderr, "[M::%s::%.3f] ==> HiC index has been loaded\n", __func__, yak_realtime()-index_time); + return 1; +} + +static void worker_for_sort(void *data, long i, int tid) // callback for kt_for() +{ + pldat_t *pl = (pldat_t*)data; + hc_pt1_t *h = &(pl->h->idx_buf[i]); + khint_t k; + uint64_t beg, cnt = 0; + uint64_t* pos_list; + for (k = 0; k != kh_end(h->h); ++k) { + if (kh_exist(h->h, k)) { + beg = kh_val(h->h, k); + pos_list = h->a + beg; + if((kh_key(h->h, k)&HIC_MAX_COUNT)h, k)&HIC_MAX_COUNT; + } + else if(k == h->end) + { + cnt = h->n - beg; + } + else + { + for (k++; k != kh_end(h->h); ++k) + { + if (kh_exist(h->h, k)) + { + cnt = kh_val(h->h, k) - beg; + break; + } + } + } + if(cnt > 0) radix_sort_hc_pos(pos_list, pos_list+cnt); + } + } + +} + +void hc_pt_t_gen(ha_ug_index* idx, pldat_t* pl) +{ + if(pl == NULL) + { + uint64_t i; + for (i = 0; i < idx->tot; i++) + { + hc_pt_t_gen_single(&(idx->idx_buf[i]), &(idx->up_bound)); + } + } + else + { + kt_for(pl->n_thread, worker_for_sort, pl, pl->h->tot); + } +} + +static void worker_for(void *data, long i, int tid) // callback for kt_for() +{ + pldat_t *pl = (pldat_t*)data; + hc_pt1_t *h = &(pl->h->idx_buf[i]); + uint64_t m = 0, beg, end, occ; + khint_t key; + int absent; + + + if(pl->is_cnt) + { + uint64_t* cnt = NULL; + if(pl->cnt[i].a.n > 2) radix_sort_b64(pl->cnt[i].a.a, pl->cnt[i].a.a + pl->cnt[i].a.n); + cnt = pl->cnt[i].a.a; + occ = pl->cnt[i].a.n; + for (m = beg = end = 0; m < occ; m++) + { + if(cnt[beg] == cnt[m]) + { + end = m; + } + else + { + key = hc_pt_put(h->h, cnt[beg], &absent); + if(absent) kh_val(h->h, key) = 0; + kh_val(h->h, key) += (end - beg + 1); + kh_key(h->h, key) = (kh_key(h->h, key)&HIC_KEY_MODE)| + (kh_val(h->h, key)h, key):HIC_MAX_COUNT); + beg = end = m; + } + } + if(occ > 0) + { + key = hc_pt_put(h->h, cnt[beg], &absent); + if(absent) kh_val(h->h, key) = 0; + kh_val(h->h, key) += (end - beg + 1); + kh_key(h->h, key) = (kh_key(h->h, key)&HIC_KEY_MODE)| + (kh_val(h->h, key)h, key):HIC_MAX_COUNT); + } + pl->cnt[i].a.n = 0; + } + + if(!pl->is_cnt) + { + ch_buf_t* pos = NULL; + uint64_t num, *pos_list = NULL, k, k_n, pos_k; + if(pl->buf[i].a.n > 2) radix_sort_ch_buf(pl->buf[i].a.a, pl->buf[i].a.a + pl->buf[i].a.n); + pos = pl->buf[i].a.a; + occ = pl->buf[i].a.n; + for (m = beg = end = 0; m < occ; m++) + { + if(pos[beg].key == pos[m].key) + { + end = m; + } + else + { + num = get_hc_pt1_count(pl->h, pos[beg].key, &pos_list); + if(num > 0) + { + k_n=(end-beg+1);pos_k=pos_list[num-1];pos_list[num-1]+=k_n; + for (k = 0; k < k_n; k++) + { + pos_list[pos_k+k] = pos[beg+k].pos; + } + } + + beg = end = m; + } + + } + if(occ > 0) + { + num = get_hc_pt1_count(pl->h, pos[beg].key, &pos_list); + if(num > 0) + { + k_n=(end-beg+1);pos_k=pos_list[num-1];pos_list[num-1]+=k_n; + for (k = 0; k < k_n; k++) + { + pos_list[pos_k+k] = pos[beg+k].pos; + } + } + } + pl->buf[i].a.n = 0; + } +} + +void parallel_count_hc_pt1(pldat_t* pl) +{ + uint64_t i, l = 0, uID, num_pos = 0, pos_thre; + uint64_t x[4], mask = (1ULL<h->k) - 1, shift = pl->h->k - 1, hash, pos, skip, bucket_mask = (1ULL<h->pre) - 1; + ma_utg_t *u = NULL; + ch_buf_t k_pos; + if(pl->is_cnt) l = ((pl->buf_bytes>>3)/pl->h->tot) + 1, pos_thre = pl->buf_bytes>>3; + if(!pl->is_cnt) l = ((pl->buf_bytes>>4)/pl->h->tot) + 1, pos_thre = pl->buf_bytes>>4; + for (i = 0; i < pl->h->tot; i++) + { + if(pl->is_cnt) + { + kv_resize(uint64_t, pl->cnt[i].a, l); + pl->cnt[i].a.n = 0; + } + + if(!pl->is_cnt) + { + kv_resize(ch_buf_t, pl->buf[i].a, l); + pl->buf[i].a.n = 0; + } + } + + + for (uID = 0; uID < pl->h->ug->u.n; uID++) + { + u = &(pl->h->ug->u.a[uID]); + if(u->m == 0) continue; + + for (i = l = 0, x[0] = x[1] = x[2] = x[3] = 0; i < u->len; ++i) { + int c = seq_nt4_table[(uint8_t)u->s[i]]; + ///c = 00, 01, 10, 11 + if (c < 4) { // not an "N" base + ///x[0] & x[1] are the forward k-mer + ///x[2] & x[3] are the reverse complementary k-mer + x[0] = (x[0] << 1 | (c&1)) & mask; + x[1] = (x[1] << 1 | (c>>1)) & mask; + x[2] = x[2] >> 1 | (uint64_t)(1 - (c&1)) << shift; + x[3] = x[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift; + if (++l >= pl->h->k) + { + hash = hc_hash_long(x, &skip, pl->h->k); + if(skip == (uint64_t)-1) continue; + if(pl->is_cnt) + { + kv_push(uint64_t, pl->cnt[hash & bucket_mask].a, hash); + } + else + { + pos = (skip << 63) | ((uID << (64-pl->h->uID_bits))>>1) | (i & pl->h->pos_mode); + k_pos.key = hash; k_pos.pos = pos; + kv_push(ch_buf_t, pl->buf[hash & bucket_mask].a, k_pos); + } + num_pos++; + + if(num_pos >= pos_thre) + { + num_pos = 0; + kt_for(pl->n_thread, worker_for, pl, pl->h->tot); + } + } + + } else l = 0, x[0] = x[1] = x[2] = x[3] = 0; // if there is an "N", restart + } + } + + if(num_pos > 0) kt_for(pl->n_thread, worker_for, pl, pl->h->tot); + + for (i = 0; i < pl->h->tot; i++) + { + if(pl->cnt[i].a.m > 0) kv_destroy(pl->cnt[i].a), kv_init(pl->cnt[i].a); + if(pl->buf[i].a.m > 0) kv_destroy(pl->buf[i].a), kv_init(pl->buf[i].a); + } +} + +ha_ug_index* build_unitig_index(ma_ug_t *ug, int k) +{ + ha_ug_index* idx = NULL; CALLOC(idx, 1); + pldat_t pl; pl.h = idx; pl.is_cnt = 1; + double index_time = yak_realtime(), beg_time; + init_ha_ug_index_opt(idx, ug, k, &pl); + + beg_time = yak_realtime(); + pl.is_cnt = 1; + parallel_count_hc_pt1(&pl); + fprintf(stderr, "[M::%s::%.3f] ==> Counting\n", __func__, yak_realtime()-beg_time); + + beg_time = yak_realtime(); + hc_pt_t_gen(pl.h, NULL); + fprintf(stderr, "[M::%s::%.3f] ==> Memory allocating\n", __func__, yak_realtime()-beg_time); + + beg_time = yak_realtime(); + pl.is_cnt = 0; + parallel_count_hc_pt1(&pl); + fprintf(stderr, "[M::%s::%.3f] ==> Filling pos\n", __func__, yak_realtime()-beg_time); + + beg_time = yak_realtime(); + hc_pt_t_gen(pl.h, &pl); + fprintf(stderr, "[M::%s::%.3f] ==> Sorting pos\n", __func__, yak_realtime()-beg_time); + + fprintf(stderr, "[M::%s::%.3f] ==> HiC index has been built\n", __func__, yak_realtime()-index_time); + + return idx; +} + +void destory_hc_pt_index(ha_ug_index* idx) +{ + if(idx->idx_buf) + { + uint64_t i = 0; + for (i = 0; i < idx->tot; i++) + { + if(idx->idx_buf[i].a) free(idx->idx_buf[i].a); + if(idx->idx_buf[i].h) hc_pt_destroy(idx->idx_buf[i].h); + } + free(idx->idx_buf); + } +} + +inline void interpret_pos(const ha_ug_index* idx, s_hit *p, uint64_t* rev, uint64_t* uID, +uint64_t* ref_p, uint64_t* self_p, uint64_t* exact_len, uint64_t* total_len) +{ + (*rev) = p->ref>>63; + (*uID) = (p->ref << 1) >> (64 - idx->uID_bits); + (*self_p) = (uint32_t)p->off_cnt; + (*exact_len) = p->off_cnt >> 32; + if(total_len != NULL) + { + (*exact_len) = (p->off_cnt>>32) & ((uint64_t)65535); + (*total_len) = (p->off_cnt>>48) + (*exact_len); + } + if((p->ref & idx->pos_mode)>>(idx->pos_bits - 1)) + { + (*ref_p) = (*self_p) - (p->ref&(idx->pos_mode>>1)); + } + else + { + (*ref_p) = (*self_p) + (p->ref&(idx->pos_mode)); + } +} + + +inline uint64_t check_exact_match(char* a, long long a_beg, long long a_total, char* b, long long b_beg, +long long b_total, long long Len, uint64_t rev, uint64_t dir) +{ + long long i = 0; + if(rev == 0) + { + if(dir == 0) + { + for (i = 0; i < Len && a_beg < a_total && b_beg < b_total; i++) + { + if(a[a_beg++] != b[b_beg++]) return i; + } + } + else + { + for (i = 0; i < Len && a_beg >= 0 && b_beg >= 0; i++) + { + if(a[a_beg--] != b[b_beg--]) return i; + } + } + } + else + { + + if(dir == 0) + { + for (i = 0; i < Len && a_beg < a_total && b_beg < b_total; i++) + { + if(a[a_beg] != b2rc[seq_nt4_table[(uint8_t)b[b_total - b_beg - 1]]]) return i; + a_beg++; b_beg++; + } + } + else + { + for (i = 0; i < Len && a_beg >= 0 && b_beg >= 0; i++) + { + if(a[a_beg] != b2rc[seq_nt4_table[(uint8_t)b[b_total - b_beg - 1]]]) return i; + a_beg--; b_beg--; + } + } + } + + return i; +} + +uint64_t debug_hash_value(char *r, uint64_t end, uint64_t k_mer) +{ + uint64_t i; + uint64_t x[4], mask = (1ULL<>1)) & mask; + x[2] = x[2] >> 1 | (uint64_t)(1 - (c&1)) << shift; + x[3] = x[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift; + } + } + + return hc_hash_long(x, &skip, k_mer); + +} + +inline uint64_t collect_votes(s_hit* a, uint64_t n) +{ + if(n == 0) return 0; + if(n == 1) return (a[0].off_cnt>>32); + long long i = 0; + uint64_t cur_beg, cur_end, beg, end, ovlp = 0, tLen = 0; + cur_end = (uint32_t)a[n-1].off_cnt; + cur_beg = cur_end + 1 - (a[n-1].off_cnt>>32); + + for (i = n - 2; i >= 0; i--) + { + end = (uint32_t)a[i].off_cnt; + beg = end + 1 - (a[i].off_cnt>>32); + if(MAX(cur_beg, beg) <= MIN(cur_end, end)) + { + cur_beg = MIN(cur_beg, beg); + ///cur_end = MAX(cur_end, end); + } + else + { + ovlp += (cur_end + 1 - cur_beg); + cur_beg = beg; + cur_end = end; + } + } + ovlp += (cur_end + 1 - cur_beg); + tLen = (uint32_t)a[n-1].off_cnt + 1 - cur_beg; + tLen = tLen - ovlp; + tLen = tLen << 16; + return ovlp | tLen; +} + +inline void compress_mapped_pos(const ha_ug_index* idx, kvec_vote* buf, uint64_t buf_iter, uint64_t max_i, uint64_t thres) +{ + if(buf_iter >= buf->a.n) + { + buf->a.n = buf_iter; + return; + } + uint64_t rev, uID, ref_p, self_p, eLen, tLen, i, max_beg, max_end, cur_beg, cur_end, ovlp, max_eLen; + uint64_t secondLen = 0, second_i = (uint64_t)-1; + interpret_pos((ha_ug_index*)idx, &buf->a.a[max_i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + max_end = self_p; + max_beg = self_p + 1 - tLen; + max_eLen = eLen; + for (i = buf_iter; i < buf->a.n; i++) + { + if(i == max_i) continue; + interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + cur_end = self_p; + cur_beg = self_p + 1 - tLen; + if(MAX(cur_beg, max_beg) <= MIN(cur_end, max_end)) + { + ovlp = MIN(cur_end, max_end) - MAX(cur_beg, max_beg) + 1; + if(ovlp > thres) + { + if(eLen >= max_eLen * 0.8) + { + buf->a.n = buf_iter; + return; + } + continue; + } + } + if(secondLen < eLen) secondLen = eLen, second_i = i; + } + + if(second_i == (uint64_t)-1) + { + buf->a.a[buf_iter] = buf->a.a[max_i]; + buf->a.n = buf_iter + 1; + } + else + { + buf->a.a[buf_iter] = buf->a.a[MIN(max_i, second_i)]; + buf->a.a[buf_iter+1] = buf->a.a[MAX(max_i, second_i)]; + buf->a.n = buf_iter + 2; + } +} + +inline void print_pos_list(const ha_ug_index* idx, s_hit *l, uint64_t occ, uint64_t rid, uint64_t r1) +{ + if(rid == 33045391 || rid == 4239289 || rid == 5267597 || rid == 34474764 || rid == 35016489 + || rid == 36002255 || rid == 37811694 || rid == 46805824) + { + uint64_t i, rev, uID, ref_p, self_p, cnt; + for (i = 0; i < occ; i++) + { + interpret_pos(idx, &l[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); + fprintf(stderr, "(r%lu) rid: %lu, i: %lu, rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu\n", + r1, rid, i, rev, uID, ref_p, self_p); + } + + } +} + +void get_alignment(char *r, uint64_t len, uint64_t k_mer, kvec_vote* buf, +const ha_ug_index* idx, uint64_t buf_iter, uint64_t rid) +{ + uint64_t i, j, l = 0, skip, *pos_list = NULL, cnt, rev, self_p, ref_p, u_len, uID; + uint64_t x[4], mask = (1ULL<a.n = 0; + for (i = l = 0, x[0] = x[1] = x[2] = x[3] = 0; i < len; ++i) { + int c = seq_nt4_table[(uint8_t)r[i]]; + ///c = 00, 01, 10, 11 + if (c < 4) { // not an "N" base + ///x[0] & x[1] are the forward k-mer + ///x[2] & x[3] are the reverse complementary k-mer + x[0] = (x[0] << 1 | (c&1)) & mask; + x[1] = (x[1] << 1 | (c>>1)) & mask; + x[2] = x[2] >> 1 | (uint64_t)(1 - (c&1)) << shift; + x[3] = x[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift; + if (++l >= k_mer) + { + hash = hc_hash_long(x, &skip, k_mer); + if(skip == (uint64_t)-1) continue; + /*******************************for debug************************************/ + // if(debug_hash_value(r, i, k_mer) != hash) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + cnt = get_hc_pt1_count((ha_ug_index*)idx, hash, &pos_list); + if(cnt > idx->max_cnt) continue; + if(cnt != 1) continue; ///might be able to be disabled in future + + + for (j = 0; j < cnt; j++) + { + kv_pushp(s_hit, buf->a, &p); + rev = (pos_list[j]>>63) != skip; + self_p = i; + ref_p = pos_list[j] & idx->pos_mode; + uID = (pos_list[j] << 1) >> (64 - idx->uID_bits); + u_len = idx->ug->u.a[uID].len; + if(rev) ref_p = u_len - 1 - (ref_p + 1 - k_mer); + p->off_cnt = self_p | ((uint64_t)k_mer << 32); ///high bits should be the legnth + + p->ref = ref_p >= self_p? (ref_p-self_p) + : (self_p-ref_p) + ((uint64_t)1 << (idx->pos_bits - 1)); + p->ref = (rev << 63)|(pos_list[j] & idx->uID_mode)|(p->ref&idx->pos_mode); + + + /*******************************for debug************************************/ + // if(check_exact_match(r, i + 1 - k_mer, len, + // idx->ug->u.a[uID].s, ref_p + 1 - k_mer, u_len, k_mer, rev, 0) != k_mer + // || + // check_exact_match(r, i, len, + // idx->ug->u.a[uID].s, ref_p, u_len, k_mer, rev, 1) != k_mer) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + } + + if(cnt == 1) + { + ///uint64_t debug_right = 0, debug_left = 0, debug_len; + + j = check_exact_match(r, self_p + 1, len, idx->ug->u.a[uID].s, ref_p + 1, u_len, len, rev, 0); + + ///debug_right = j; + ///if(j == 0) continue; + if((j + 1) >= k_mer) + { + l = 0, x[0] = x[1] = x[2] = x[3] = 0; + i = i + j - (k_mer - 1); + } + else + { + ///l = i - (i + j - (k_mer - 1)); + l = k_mer - j -1; + } + buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32) + j; + + if(self_p >= k_mer && ref_p >= k_mer) + { + j = check_exact_match(r, self_p - k_mer, len, idx->ug->u.a[uID].s, + ref_p - k_mer, u_len, len, rev, 1); + buf->a.a[buf->a.n-1].off_cnt += ((uint64_t)j << 32); + ///debug_left = j; + } + + + // debug_len = check_exact_match(r, self_p + debug_right, len, idx->ug->u.a[uID].s, + // ref_p + debug_right, u_len, len, rev, 1); + // if(debug_len!= (debug_left + debug_right + k_mer)) + // { + // fprintf(stderr, "debug_len: %lu, debug_left: %lu, debug_right: %lu\n", + // debug_len, debug_left, debug_right); + // } + } + + } + + } else l = 0, x[0] = x[1] = x[2] = x[3] = 0; // if there is an "N", restart + } + + ///if(buf->a.n - buf_iter <= 1) return; + if(buf->a.n - buf_iter == 0) return; + if(buf->a.n - buf_iter > 1) radix_sort_hc_s_hit_an1(buf->a.a + buf_iter, buf->a.a + buf->a.n); + + + + /*******************************for debug************************************/ + // print_pos_list(idx, buf->a.a+buf_iter, buf->a.n - buf_iter, rid, (buf_iter != 0)); + // fprintf(stderr, "len0:%lu\n", buf->a.n - buf_iter); + // for (i = buf_iter; i < buf->a.n; i++) + // { + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); + // fprintf(stderr, "(%lu) rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu, len: %lu\n", + // i, rev, uID, ref_p, self_p, cnt); + // } + /*******************************for debug************************************/ + + + + + + uint64_t cur_ref_p, thres = (len * HIC_R_E_RATE) + 1, m, index_beg, ovlp, maxLen = 0, max_i = (uint64_t)-1; + i = m = buf_iter; + while (i < buf->a.n) + { + interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &cnt, NULL); + /*******************************for debug************************************/ + // if(check_exact_match(r, self_p, len, idx->ug->u.a[uID].s, + // ref_p, idx->ug->u.a[uID].len, cnt, rev, 1) != cnt) + // { + // fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + // if(self_p > ref_p) + // { + // i++; + // continue; ///fix this in future + // } + cur_ref_p = buf->a.a[i].ref; + index_beg = i; + while ((i < buf->a.n) && ((buf->a.a[i].ref>>idx->pos_bits) == (cur_ref_p>>idx->pos_bits)) && + (buf->a.a[i].ref - cur_ref_p <= thres)) + { + i++; + } + if(i - index_beg > 1) + { + radix_sort_hc_s_hit_an2(buf->a.a + index_beg, buf->a.a + i); + } + ovlp = collect_votes(buf->a.a + index_beg, i - index_beg); + buf->a.a[m] = buf->a.a[i - 1]; + buf->a.a[m].off_cnt = (buf->a.a[m].off_cnt << 32)>>32; + buf->a.a[m].off_cnt += ((uint64_t)ovlp<<32); + + if(maxLen < (ovlp&((uint64_t)65535))) maxLen = (ovlp&((uint64_t)65535)), max_i = m; + + m++; + } + buf->a.n = m; + + /*******************************for debug************************************/ + // for (i = buf_iter; i < buf->a.n; i++) + // { + // uint64_t eLen, tLen; + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // if(maxLen < eLen) fprintf(stderr, "ERROR1\n"); + // if(i == max_i && maxLen != eLen) fprintf(stderr, "ERROR2\n"); + // } + /*******************************for debug************************************/ + ///select the best alignment at [buf_iter, m) + + /*******************************for debug************************************/ + // fprintf(stderr, "len1:%lu, max_i: %lu\n", buf->a.n - buf_iter, max_i); + // for (i = buf_iter; i < buf->a.n; i++) + // { + // uint64_t eLen, tLen; + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "(%lu) rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu, eLen: %lu, tLen: %lu\n", + // i, rev, uID, ref_p, self_p, eLen, tLen); + // } + /*******************************for debug************************************/ + + compress_mapped_pos(idx, buf, buf_iter, max_i, thres); + + /*******************************for debug************************************/ + // fprintf(stderr, "len2:%lu, max_i: %lu\n", buf->a.n - buf_iter, max_i); + // for (i = buf_iter; i < buf->a.n; i++) + // { + // uint64_t eLen, tLen; + // interpret_pos(idx, &buf->a.a[i], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "(%lu) rev: %lu, uID: %lu, ref_p: %lu, self_p: %lu, eLen: %lu, tLen: %lu\n", + // i, rev, uID, ref_p, self_p, eLen, tLen); + // } + // if(buf->a.n != m) fprintf(stderr, "Changed\n"); + // fprintf(stderr, "\n"); + /*******************************for debug************************************/ +} +inline int is_unreliable_hits(long long rev, long long ref_p, long long tLen, uint64_t uID, hc_links* link) +{ + uint64_t i; + long long p_beg, p_end; + bed_in* p = NULL; + if(rev) + { + p_end = ref_p; + p_beg = p_end + 1 - tLen; + } + else + { + p_beg = ref_p; + p_end = p_beg + tLen - 1; + } + if(p_beg < 0) p_beg = 0; + if(p_end < 0) p_end = 0; + + p = &(link->bed.a[uID]); + for (i = 0; i < p->n; i++) + { + if(inter_interval(p_beg, p_end, p->a[i].beg, p->a[i].end, NULL, NULL)) break; + } + if(p->n > 0 && i < p->n) return 1; + + return 0; +} +inline void set_pe_pos(ha_ug_index* idx, s_hit *l1, uint64_t occ1, s_hit *l2, uint64_t occ2, +pe_hit* x, uint64_t rid, hc_links* link) +{ + if(occ1 == 0 || occ2 == 0) return; + uint64_t rev1, rev2, uID1, uID2, ref_p1, ref_p2, self_p1, self_p2, eLen1, eLen2, tLen1, tLen2; + uint64_t rev_t, uID_t, ref_p_t, self_p_t, eLen_t, tLen_t; + ///5' end of r1 and r2 + interpret_pos(idx, &l1[0], &rev1, &uID1, &ref_p1, &self_p1, &eLen1, &tLen1); + interpret_pos(idx, &l2[0], &rev2, &uID2, &ref_p2, &self_p2, &eLen2, &tLen2); + /*******************************for debug************************************/ + // if(rid == 33045391 || rid == 4239289 || rid == 5267597 || rid == 34474764 || rid == 35016489 + // || rid == 36002255 || rid == 37811694 || rid == 46805824) + // { + // fprintf(stderr, "rid: %lu, rev1: %lu, uID1: %lu, ref_p1: %lu, self_p1: %lu, rev2: %lu, uID2: %lu, ref_p2: %lu, self_p2: %lu\n", + // rid, rev1, uID1, self_p1, ref_p1, rev2, uID2, ref_p2, self_p2); + // } + /*******************************for debug************************************/ + ///if(uID1 == uID2) return; + if(ref_p1 < self_p1 || ref_p2 < self_p2) return; + if(occ1 > 1) + { + interpret_pos(idx, &l1[1], &rev_t, &uID_t, &ref_p_t, &self_p_t, &eLen_t, &tLen_t); + if(uID_t != uID1 && uID_t != uID2) return; + } + + if(occ2 > 1) + { + interpret_pos(idx, &l2[1], &rev_t, &uID_t, &ref_p_t, &self_p_t, &eLen_t, &tLen_t); + if(uID_t != uID1 && uID_t != uID2) return; + } + + x->id = rid; + ref_p1 -= self_p1; + if(rev1) ref_p1 = idx->ug->u.a[uID1].len - 1 - ref_p1; + x->s = (rev1<<63) | ((uID1 << (64-idx->uID_bits))>>1) | (ref_p1 & idx->pos_mode); + + + ref_p2 -= self_p2; + if(rev2) ref_p2 = idx->ug->u.a[uID2].len - 1 - ref_p2; + x->e = (rev2<<63) | ((uID2 << (64-idx->uID_bits))>>1) | (ref_p2 & idx->pos_mode); + + if(link && (is_unreliable_hits(rev1, ref_p1, tLen1, uID1, link) || + is_unreliable_hits(rev2, ref_p2, tLen2, uID2, link))) + { + x->id = x->s = x->e = (uint64_t)-1; + } +} + +static void worker_for_alignment(void *data, long i, int tid) // callback for kt_for() +{ + stepdat_t *s = (stepdat_t*)data; + s->pos[i].id = s->pos[i].s = s->pos[i].e = (uint64_t)-1; + uint64_t len1 = s->len[i]>>32, len2 = (uint32_t)s->len[i], occ1, occ2; + char *r1 = s->seq[i], *r2 = s->seq[i] + len1; + /*******************************for debug************************************/ + // if(memcmp(r1, R1.r.a + R1.r_Len.a[s->id+i], len1) != 0) + // { + // fprintf(stderr, "haha1\n"); + // } + // if(memcmp(r2, R2.r.a + R2.r_Len.a[s->id+i], len2) != 0) + // { + // fprintf(stderr, "haha2\n"); + // } + /*******************************for debug************************************/ + s->pos_buf[tid].a.n = 0; + get_alignment(r1, len1, s->idx->k, &s->pos_buf[tid], s->idx, 0, s->id+i); + occ1 = s->pos_buf[tid].a.n; + if(occ1 == 0) return; + get_alignment(r2, len2, s->idx->k, &s->pos_buf[tid], s->idx, occ1, s->id+i); + occ2 = s->pos_buf[tid].a.n - occ1; + if(occ2 == 0) return; + + set_pe_pos((ha_ug_index*)s->idx, s->pos_buf[tid].a.a, occ1, s->pos_buf[tid].a.a + occ1, occ2, &(s->pos[i]), s->id+i, s->link); + + /*******************************for debug************************************/ + // if(memcmp(r1, R1.r.a + R1.r_Len.a[s->id+i], len1) != 0) + // { + // fprintf(stderr, "haha1\n"); + // } + // if(memcmp(r2, R2.r.a + R2.r_Len.a[s->id+i], len2) != 0) + // { + // fprintf(stderr, "haha2\n"); + // } + // uint64_t j, rev, uID, ref_p, self_p, eLen, tLen; + // char dir[2] = {'+', '-'}; + // fprintf(stderr, "(R1) %.*s\n", (int)(R1.name_Len.a[s->id + i + 1] - R1.name_Len.a[s->id+i]), + // R1.name.a + R1.name_Len.a[s->id+i]); + // for (j = 0; j < occ1; j++) + // { + // interpret_pos(s->idx, &s->pos_buf[tid].a.a[j], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "utg%.6lu\t%c\t%lu\t%lu-%lu\n", uID+1, dir[rev], ref_p, self_p + 1 - tLen, self_p); + // } + // fprintf(stderr, "(R2) %.*s\n", (int)(R2.name_Len.a[s->id + i + 1] - R2.name_Len.a[s->id+i]), + // R2.name.a + R2.name_Len.a[s->id+i]); + // for (j = 0; j < occ2; j++) + // { + // interpret_pos(s->idx, &s->pos_buf[tid].a.a[j+occ1], &rev, &uID, &ref_p, &self_p, &eLen, &tLen); + // fprintf(stderr, "utg%.6lu\t%c\t%lu\t%lu-%lu\n", uID+1, dir[rev], ref_p, self_p + 1 - tLen, self_p); + // } + // fprintf(stderr, "\n"); + /*******************************for debug************************************/ +} + +static void *worker_pipeline(void *data, int step, void *in) // callback for kt_pipeline() +{ + sldat_t *p = (sldat_t*)data; + ///uint64_t total_base = 0, total_pair = 0; + if (step == 0) { // step 1: read a block of sequences + int ret1, ret2; + uint64_t l1, l2; + stepdat_t *s; + CALLOC(s, 1); + s->idx = p->idx; s->id = p->total_pair; s->link = p->link; + while (((ret1 = kseq_read(p->ks1)) >= 0)&&((ret2 = kseq_read(p->ks2)) >= 0)) + { + if (p->ks1->seq.l < p->idx->k || p->ks2->seq.l < p->idx->k) continue; + if (s->n == s->m) { + s->m = s->m < 16? 16 : s->m + (s->n>>1); + REALLOC(s->len, s->m); + REALLOC(s->seq, s->m); + } + + l1 = p->ks1->seq.l; l2 = p->ks2->seq.l; + MALLOC(s->seq[s->n], l1+l2); + s->sum_len += l1+l2; + memcpy(s->seq[s->n], p->ks1->seq.s, l1); + memcpy(s->seq[s->n]+l1, p->ks2->seq.s, l2); + s->len[s->n++] = (uint64_t)(l1<<32)|(uint64_t)l2; + + if (s->sum_len >= p->chunk_size) break; + } + p->total_pair += s->n; + if (s->sum_len == 0) free(s); + else return s; + } + else if (step == 1) { // step 2: alignment + stepdat_t *s = (stepdat_t*)in; + CALLOC(s->pos_buf, p->n_thread); + MALLOC(s->pos, s->n); + int i; + kt_for(p->n_thread, worker_for_alignment, s, s->n); + for (i = 0; i < s->n; ++i) { + free(s->seq[i]); + p->total_base += (s->len[i]>>32) + (uint32_t)s->len[i]; + } + + free(s->seq); free(s->len); + for (i = 0; i < (int)p->n_thread; ++i) { + free(s->pos_buf[i].a.a); + } + free(s->pos_buf); + return s; + } + else if (step == 2) { // step 3: dump + stepdat_t *s = (stepdat_t*)in; + int i; + for (i = 0; i < s->n; ++i) { + if(s->pos[i].s == (uint64_t)-1) continue; + kv_push(pe_hit, p->hits.a, s->pos[i]); + } + free(s->pos); + free(s); + } + return 0; +} + + +void load_reads(reads_t* x, const char *fn) +{ + kv_init(x->name); + kv_init(x->name_Len); + kv_init(x->r); + kv_init(x->r_Len); + gzFile fp; + kseq_t *ks; + int ret; + uint64_t name_tot, base_total; + + name_tot = base_total = 0; + if ((fp = gzopen(fn, "r")) == 0) return; + ks = kseq_init(fp); + while (((ret = kseq_read(ks)) >= 0)) + { + kv_push(uint64_t, x->name_Len, name_tot); + kv_resize(char, x->name, name_tot + ks->name.l); + memcpy(x->name.a + name_tot, ks->name.s, ks->name.l); + name_tot += ks->name.l; + + kv_push(uint64_t, x->r_Len, base_total); + kv_resize(char, x->r, base_total + ks->seq.l); + memcpy(x->r.a + base_total, ks->seq.s, ks->seq.l); + base_total += ks->seq.l; + } + + kv_push(uint64_t, x->name_Len, name_tot); + kv_push(uint64_t, x->r_Len, base_total); + + kseq_destroy(ks); + gzclose(fp); +} + + +void test_reads(reads_t* x, const char *fn) +{ + gzFile fp; + kseq_t *ks; + int ret, i = 0; + + if ((fp = gzopen(fn, "r")) == 0) return; + ks = kseq_init(fp); + while (((ret = kseq_read(ks)) >= 0)) + { + if(memcmp(ks->name.s, x->name.a + x->name_Len.a[i], ks->name.l) != 0) + { + fprintf(stderr, "ERROR222: i: %d, len: %lu\n", i, x->name_Len.a[i]); + } + i++; + } + + kseq_destroy(ks); + gzclose(fp); +} + +void destory_reads(reads_t* x) +{ + kv_destroy(x->name); + kv_destroy(x->name_Len); + kv_destroy(x->r); + kv_destroy(x->r_Len); +} + +void print_hits(ha_ug_index* idx, kvec_pe_hit* hits, const char *fn) +{ + uint64_t k, shif = 64 - idx->uID_bits; + reads_t r1; + load_reads(&r1, fn); + char dir[2] = {'+', '-'}; + for (k = 0; k < hits->a.n; ++k) + { + fprintf(stderr, "%.*s\t%c\ts-utg%.6dl\t%lu\t%c\te-utg%.6dl\t%lu\ti:%lu\n", + (int)(r1.name_Len.a[hits->a.a[k].id + 1] - r1.name_Len.a[hits->a.a[k].id]), + r1.name.a + r1.name_Len.a[hits->a.a[k].id], + dir[hits->a.a[k].s>>63], (int)((hits->a.a[k].s<<1)>>shif)+1, hits->a.a[k].s&idx->pos_mode, + dir[hits->a.a[k].e>>63], (int)((hits->a.a[k].e<<1)>>shif)+1, hits->a.a[k].e&idx->pos_mode, + hits->a.a[k].id); + } + destory_reads(&r1); +} + +void dedup_hits(kvec_pe_hit* hits) +{ + double index_time = yak_realtime(); + uint64_t k, l, m = 0, cur; + radix_sort_pe_hit_an1(hits->a.a, hits->a.a + hits->a.n); + for (k = 1, l = 0; k <= hits->a.n; ++k) + { + if (k == hits->a.n || hits->a.a[k].s != hits->a.a[l].s) + { + if (k - l > 1) radix_sort_pe_hit_an2(hits->a.a + l, hits->a.a + k); + cur = (uint64_t)-1; + while (l < k) + { + if(hits->a.a[l].e != cur) + { + cur = hits->a.a[l].e; + hits->a.a[m++] = hits->a.a[l]; + } + l++; + } + l = k; + } + } + hits->a.n = m; + fprintf(stderr, "[M::%s::%.3f] ==> Dedup\n", __func__, yak_realtime()-index_time); +} + +void sort_hits(kvec_pe_hit* hits) +{ + double index_time = yak_realtime(); + uint64_t k, l; + radix_sort_pe_hit_an1(hits->a.a, hits->a.a + hits->a.n); + for (k = 1, l = 0; k <= hits->a.n; ++k) + { + if (k == hits->a.n || (hits->a.a[k].s<<1) != (hits->a.a[l].s<<1)) + { + if (k - l > 1) radix_sort_pe_hit_an2(hits->a.a + l, hits->a.a + k); + l = k; + } + } + fprintf(stderr, "[M::%s::%.3f] ==> Sort\n", __func__, yak_realtime()-index_time); +} + +void destory_bubbles(bubble_type* bub) +{ + if(bub->index) free(bub->index); + kv_destroy(bub->list); + kv_destroy(bub->num); + kv_destroy(bub->pathLen); + kv_destroy(bub->b_s_idx); + kv_destroy(bub->chain_weight); + asg_destroy(bub->b_g); + ma_ug_destroy(bub->b_ug); +} + +void get_bubbles(bubble_type* bub, uint64_t id, uint32_t* beg, uint32_t* sink, uint32_t** a, uint32_t* n, uint64_t* pathBase) +{ + if(a) (*a) = bub->list.a + bub->num.a[id] + 2; + if(n) (*n) = bub->num.a[id+1] - bub->num.a[id] - 2; + if(beg) (*beg) = bub->list.a[bub->num.a[id]]; + if(sink) (*sink) = bub->list.a[bub->num.a[id] + 1]; + if(pathBase) (*pathBase) = bub->pathLen.a[id]; +} + + +void dfs_bubble_broken(asg_t *g, kvec_t_u32_warp* stack, kvec_t_u32_warp* result, uint8_t* vis_flag, +uint32_t vis_flag_n, uint32_t v_d, uint32_t beg_d, uint32_t sink_d) +{ + memset(vis_flag, 0, vis_flag_n); + asg_arc_t *acur = NULL; + uint32_t cur, ncur, i, p_beg = (uint32_t)-1, p_sink = (uint32_t)-1, v; + stack->a.n = result->a.n = 0; + v = v_d; + if(v != (beg_d^1) && v != (sink_d^1)) kv_push(uint32_t, stack->a, v); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + vis_flag[cur] = 1; + if((v>>1) != (cur>>1)) kv_push(uint32_t, result->a, cur>>1); + ncur = asg_arc_n(g, cur); + acur = asg_arc_a(g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]) continue; + if((acur[i].v>>1) == (beg_d>>1) || (acur[i].v>>1) == (sink_d>>1)) + { + if((acur[i].v>>1) == (beg_d>>1)) p_beg = acur[i].v; + if((acur[i].v>>1) == (sink_d>>1)) p_sink = acur[i].v; + continue; + } + kv_push(uint32_t, stack->a, acur[i].v); + } + } + + memset(vis_flag, 0, vis_flag_n); + v ^= 1; + if(v != (beg_d^1) && v != (sink_d^1)) kv_push(uint32_t, stack->a, v); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + vis_flag[cur] = 1; + if((v>>1) != (cur>>1)) kv_push(uint32_t, result->a, cur>>1); + ncur = asg_arc_n(g, cur); + acur = asg_arc_a(g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]) continue; + if((acur[i].v>>1) == (beg_d>>1) || (acur[i].v>>1) == (sink_d>>1)) + { + if((acur[i].v>>1) == (beg_d>>1)) p_beg = acur[i].v; + if((acur[i].v>>1) == (sink_d>>1)) p_sink = acur[i].v; + continue; + } + kv_push(uint32_t, stack->a, acur[i].v); + } + } + + if(p_beg != (uint32_t)-1) kv_push(uint32_t, result->a, beg_d>>1); + if(p_sink != (uint32_t)-1) kv_push(uint32_t, result->a, sink_d>>1); +} + + + +void dfs_bubble(asg_t *g, kvec_t_u32_warp* stack, kvec_t_u32_warp* result, uint32_t v, uint32_t beg, uint32_t sink) +{ + asg_arc_t *acur = NULL; + uint32_t cur, ncur, i; + stack->a.n = result->a.n = 0; + v = v << 1; + kv_push(uint32_t, stack->a, v); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if((v>>1) != (cur>>1)) kv_push(uint32_t, result->a, cur>>1); + ncur = asg_arc_n(g, cur); + acur = asg_arc_a(g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if((acur[i].v>>1) == beg || (acur[i].v>>1) == sink) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + + + v = v + 1; + kv_push(uint32_t, stack->a, v); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if((v>>1) != (cur>>1)) kv_push(uint32_t, result->a, cur>>1); + ncur = asg_arc_n(g, cur); + acur = asg_arc_a(g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if((acur[i].v>>1) == beg || (acur[i].v>>1) == sink) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } +} + +void update_bub_b_s_idx(bubble_type* bub); +void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link) +{ + asg_cleanup(ug->g); + if (!ug->g->is_symm) asg_symm(ug->g); + uint32_t v, n_vtx = ug->g->n_seq * 2, i, k, mode = (((uint32_t)-1)<<2); + uint32_t beg, sink, n, *a, n_occ; + uint64_t pathLen, tLen; + bub->ug = ug; + for (i = 0, tLen = 1; i < ug->u.n; i++) tLen += ug->u.a[i].len; + bub->b_bub = bub->b_end_bub = bub->tangle_bub = bub->cross_bub = bub->mess_bub = 0; + + if(bub->round_id == 0) + { + buf_t b; memset(&b, 0, sizeof(buf_t)); b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + kv_init(bub->list); kv_init(bub->num); kv_init(bub->pathLen); + kv_init(bub->b_s_idx); kv_malloc(bub->b_s_idx, ug->g->n_seq); + bub->b_ug = NULL; kv_init(bub->chain_weight); + bub->b_s_idx.n = ug->g->n_seq; + memset(bub->b_s_idx.a, -1, bub->b_s_idx.n * sizeof(uint64_t)); + + CALLOC(bub->index, n_vtx); + for (i = 0; i < ug->g->n_seq; i++) + { + if(ug->g->seq[i].c > 0) + { + bub->index[i] = (ug->g->seq[i].c << 2); + ug->g->seq[i].c = 0; + } + } + + + + for (v = 0; v < n_vtx; ++v) + { + if(ug->g->seq[v>>1].del) continue; + if(asg_arc_n(ug->g, v) < 2) continue; + if((bub->index[v]&(uint32_t)3) != 0) continue; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, NULL, NULL)) + { + //beg is v, end is b.S.a[0] + //note b.b include end, does not include beg + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + bub->index[b.b.a[i]] &= mode; bub->index[b.b.a[i]] += 1; + bub->index[b.b.a[i]^1] &= mode; bub->index[b.b.a[i]^1] += 1; + } + bub->index[v] &= mode; bub->index[v] += 2; + bub->index[b.S.a[0]^1] &= mode; bub->index[b.S.a[0]^1] += 3; + } + } + + + kvec_t_u32_warp stack, result; + kv_init(stack.a); kv_init(result.a); + for (v = 0; v < n_vtx; ++v) + { + if((bub->index[v]&(uint32_t)3) !=2) continue; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL)) + { + //note b.b include end, does not include beg + i = b.b.n + 1; + if(b.b.n == 2 || b.b.n == 3 || b.b.n == 5) + { + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + dfs_bubble(ug->g, &stack, &result, b.b.a[i]>>1, v>>1, b.S.a[0]>>1); + if((result.a.n + 3) != b.b.n && (result.a.n + 2) != b.b.n) break; + } + } + + + if(i == b.b.n) + { + kv_push(uint32_t, bub->num, v); + } + else + { + kv_push(uint32_t, bub->num, v + (1<<31)); + } + } + } + kv_destroy(stack.a); kv_destroy(result.a); + radix_sort_u32(bub->num.a, bub->num.a + bub->num.n); + bub->s_bub = 0; + for (k = 0; k < bub->num.n; k++) + { + if((bub->num.a[k]>>31) == 0) bub->s_bub++; + v = (bub->num.a[k]<<1)>>1; + bub->num.a[k] = bub->list.n; + if(asg_bub_pop1_primary_trio(ug->g, NULL, v, tLen, &b, (uint32_t)-1, (uint32_t)-1, 0, &pathLen, NULL)) + { + kv_push(uint64_t, bub->pathLen, pathLen); + //beg is v, end is b.S.a[0] + kv_push(uint32_t, bub->list, v); + kv_push(uint32_t, bub->list, b.S.a[0]^1); + + //note b.b include end, does not include beg + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i]==v || b.b.a[i]==b.S.a[0]) continue; + kv_push(uint32_t, bub->list, b.b.a[i]); + } + } + } + + kv_push(uint32_t, bub->num, bub->list.n); + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + bub->f_bub = bub->num.n - 1; ///bub->s_bub = bub->num.n - 1; + + for (i = 0; i < ug->g->n_seq; i++) + { + if((bub->index[i]>>2) == 0) + { + bub->index[i] = (uint32_t)-1; + } + else + { + if((bub->index[i]>>2) == 1) + { + bub->index[i] = P_het(*bub); ///potential het + } + else + { + bub->index[i] = M_het(*bub); ///must het + } + } + } + + + for (i = 0; i < bub->f_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, &pathLen); + for (v = n_occ = 0; v < n; v++) + { + bub->index[(a[v]>>1)] = i; + n_occ += ug->u.a[a[v]>>1].n; + } + + if((pathLen*2) >= ug->g->seq[beg>>1].len && (pathLen*2) >= ug->g->seq[sink>>1].len) + { + bub->index[(beg>>1)] = (uint32_t)-1; + bub->index[(sink>>1)] = (uint32_t)-1; + } + + if(n_occ > 3) + { + if(bub->index[(beg>>1)] != M_het(*bub)) bub->index[(beg>>1)] = (uint32_t)-1; + if(bub->index[(sink>>1)] != M_het(*bub)) bub->index[(sink>>1)] = (uint32_t)-1; + } + + + v = beg>>1; + if(bub->b_s_idx.a[v] == (uint64_t)-1) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + else if((bub->b_s_idx.a[v] & 0xffffffff00000000) == 0xffffffff00000000) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + + + v = sink>>1; + if(bub->b_s_idx.a[v] == (uint64_t)-1) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + else if((bub->b_s_idx.a[v] & 0xffffffff00000000) == 0xffffffff00000000) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + } + + for (i = 0; i < ug->g->n_seq; i++) + { + if(bub->index[i] == M_het(*bub)) bub->index[i] = P_het(*bub); + if(bub->index[i] > P_het(*bub)) + { + if(link) + { + for (k = 0; k < link->a.a[i].f.n; k++) + { + if(link->a.a[i].f.a[k].del || link->a.a[i].f.a[k].dis != RC_1) continue; + bub->index[i] = P_het(*bub); + break; + } + } + } + } + } + else + { + bub->num.n = bub->f_bub + 1; + bub->pathLen.n = bub->f_bub; + bub->list.n = bub->num.a[bub->num.n-1]; + update_bub_b_s_idx(bub); + bub->check_het = 0; + asg_destroy(bub->b_g); bub->b_g = NULL; + ma_ug_destroy(bub->b_ug); bub->b_ug = NULL; + kv_destroy(bub->chain_weight); kv_init(bub->chain_weight); + } + bub->b_g = NULL; + bub->b_ug = NULL; + build_bub_graph(ug, bub); +} + + + +void print_bubbles(ma_ug_t* ug, bubble_type* bub, kvec_pe_hit* hits, hc_links* link, ha_ug_index* idx) +{ + uint64_t tLen, t_utg, i, k; + uint32_t beg, sink, n, *a; + for (i = 0, tLen = 0; i < bub->ug->u.n; i++) tLen += bub->ug->u.a[i].len; + fprintf(stderr, "[M::%s] # unitigs: %lu, # bases: %lu\n", __func__, bub->ug->u.n, tLen); + for (i = 0, tLen = 0, t_utg = 0; i < bub->f_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, NULL); + t_utg += n; + for (k = 0; k < n; k++) + { + tLen +=bub->ug->u.a[(a[k]>>1)].len; + } + } + fprintf(stderr, "[M::%s] # bubbles: %lu, # unitigs: %lu, # bases: %lu\n", __func__, + Get_bub_num(*bub), t_utg, tLen); + + + for (i = 0, tLen = 0, t_utg = 0; i < ug->g->n_seq; i++) + { + if(IF_BUB(i, *bub)) + { + t_utg++; + tLen +=bub->ug->u.a[i].len; + } + } + fprintf(stderr, "[M::%s] # bubbles: %lu, # unitigs: %lu, # bases: %lu\n", __func__, + Get_bub_num(*bub), t_utg, tLen); + + for (i = 0, tLen = 0, t_utg = 0; i < ug->g->n_seq; i++) + { + if(IF_HET(i, *bub)) + { + t_utg++; + tLen +=bub->ug->u.a[i].len; + } + } + fprintf(stderr, "[M::%s] # het unitigs: %lu, # het bases: %lu\n", __func__, t_utg, tLen); + + uint8_t* flag; CALLOC(flag, ug->g->n_seq); + uint64_t s_uid, e_uid, shif = 64 - idx->uID_bits; + if(hits) + { + for (k = 0; k < hits->a.n; ++k) + { + s_uid = ((hits->a.a[k].s<<1)>>shif); + e_uid = ((hits->a.a[k].e<<1)>>shif); + if(bub->index[s_uid] == (uint32_t)-1 || bub->index[e_uid] == (uint32_t)-1) continue; + if(IF_BUB(s_uid, *bub) && IF_BUB(e_uid, *bub)) + { + flag[s_uid] |= 1; + flag[e_uid] |= 1; + continue; + } + if(IF_HET(s_uid, *bub) && IF_HET(e_uid, *bub)) + { + flag[s_uid] |= 4; + flag[e_uid] |= 4; + continue; + } + if(IF_BUB(s_uid, *bub)) flag[s_uid] |= 2, flag[e_uid] |= 2; + if(IF_BUB(e_uid, *bub)) flag[e_uid] |= 2, flag[s_uid] |= 2; + } + } + else if(link) + { + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; ++k) + { + if(link->a.a[i].e.a[k].del) continue; + s_uid = i; + e_uid = link->a.a[i].e.a[k].uID; + if(bub->index[s_uid] == (uint32_t)-1 || bub->index[e_uid] == (uint32_t)-1) continue; + if(IF_BUB(s_uid, *bub) && IF_BUB(e_uid, *bub)) + { + flag[s_uid] |= 1; + flag[e_uid] |= 1; + continue; + } + if(IF_HET(s_uid, *bub) && IF_HET(e_uid, *bub)) + { + flag[s_uid] |= 4; + flag[e_uid] |= 4; + continue; + } + if(IF_BUB(s_uid, *bub)) flag[s_uid] |= 2, flag[e_uid] |= 2; + if(IF_BUB(e_uid, *bub)) flag[e_uid] |= 2, flag[s_uid] |= 2; + } + } + } + + + + + for (i = 0, tLen = 0, t_utg = 0; i < ug->g->n_seq; i++) + { + if(flag[i] & (uint32_t)1) + { + t_utg++; + tLen +=bub->ug->u.a[i].len; + } + } + fprintf(stderr, "[M::%s] # bubble-chained unitigs: %lu, # bubble-chained bases: %lu\n", + __func__, t_utg, tLen); + + for (i = 0, tLen = 0, t_utg = 0; i < ug->g->n_seq; i++) + { + if((flag[i] & (uint32_t)1) || (flag[i] & (uint32_t)2)) + { + t_utg++; + tLen +=bub->ug->u.a[i].len; + } + } + fprintf(stderr, "[M::%s] # (bubble && het)-chained unitigs: %lu, # (bubble && het)-chained bases: %lu\n", + __func__, t_utg, tLen); + + + for (i = 0, tLen = 0, t_utg = 0; i < ug->g->n_seq; i++) + { + if((flag[i] & (uint32_t)1) || (flag[i] & (uint32_t)2) || (flag[i] & (uint32_t)4)) + { + t_utg++; + tLen +=bub->ug->u.a[i].len; + } + } + fprintf(stderr, "[M::%s] # (bubble || het)-chained unitigs: %lu, # (bubble || het)-chained bases: %lu\n", + __func__, t_utg, tLen); + free(flag); + + + fprintf(stderr, "************bubble utgs************\n"); + uint64_t pathLen; + for (i = 0, tLen = 0, t_utg = 0; i < bub->f_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, &pathLen); + t_utg += n; + fprintf(stderr, "(full-%lu)\tbeg:utg%.6u\tsink:utg%.6u\tpathLen:%lu\t%s\n", + i, (beg>>1)+1, (sink>>1)+1, pathLen, i < bub->s_bub? "s-bub":(if_bub?"f-bub":"b-bub")); + for (k = 0; k < n; k++) + { + tLen +=bub->ug->u.a[(a[k]>>1)].len; + fprintf(stderr, "utg%.6u,", (a[k]>>1)+1); + } + fprintf(stderr, "\n"); + ///if(i < bub->s_bub && (n != 4 && n != 2 && n != 1)) fprintf(stderr, "weird\n"); + } + + for (i = bub->f_bub, tLen = 0, t_utg = 0; i < bub->f_bub + bub->b_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, &pathLen); + t_utg += n; + fprintf(stderr, "(broken-%lu)\tbeg:utg%.6u\tsink:utg%.6u\tpathLen:%lu\t%s\n", + i, (beg>>1)+1, (sink>>1)+1, pathLen, i < bub->s_bub? "s-bub":(if_bub?"f-bub":"b-bub")); + for (k = 0; k < n; k++) + { + tLen +=bub->ug->u.a[(a[k]>>1)].len; + fprintf(stderr, "utg%.6u,", (a[k]>>1)+1); + } + fprintf(stderr, "\n"); + } + + // fprintf(stderr, "************het utgs************\n"); + // for (i = 0; i < ug->g->n_seq; i++) + // { + // if(IF_HET(i, *bub)) fprintf(stderr, "utg%.6lu\n", i+1); + // } + // fprintf(stderr, "************het utgs************\n"); +} + + + + + +void push_hc_edge(hc_linkeage* x, uint64_t uID, double weight, int dir, uint64_t* d) +{ + uint64_t k, n; + hc_edge* a = NULL; + hc_edge* p = NULL; + if(dir == 0) + { + a = x->e.a; + n = x->e.n; + } + else + { + a = x->f.a; + n = x->f.n; + } + + for (k = 0; k < n; k++) + { + if(a[k].del) continue; + if(a[k].uID == uID) + { + a[k].weight += weight; + if(d) a[k].dis = (*d); + return; + } + } + + if(dir == 0) + { + kv_pushp(hc_edge, x->e, &p); + } + else + { + kv_pushp(hc_edge, x->f, &p); + } + + ///p->del = p->enzyme = 0; + p->del = 0; + p->uID = uID; + p->weight = weight; + if(d) p->dis = (*d); +} + +long long get_enzyme_occ_debug(char* t, long long tlen, char* p, long long plen) +{ + long long s = 0, j, occ = 0; + while(s <= (tlen - plen)) + { + j = plen-1; + + while(j >= 0) + { + if(seq_nt4_table[(uint8_t)t[s+j]] >= 4) break; + if((p[j] != t[s+j]) && seq_nt4_table[(uint8_t)p[j]] < 4) break; + j--; + } + + if (j < 0) occ++; + s++; + } + + return occ; +} + +int check_exact_match(char* x, long long xlen, char* y, long long ylen) +{ + long long i; + if(xlen != ylen) return 0; + for (i = 0; i < xlen; i++) + { + if(seq_nt4_table[(uint8_t)x[i]] >= 4) return 0; + if((x[i] != y[i]) && seq_nt4_table[(uint8_t)y[i]] < 4) return 0; + } + + return 1; +} + +long long get_enzyme_occ(char* t, long long tlen, char* p, long long plen) +{ + long long i, c, s = 0, j, occ = 0; + int badchar[5]; badchar[0] = badchar[1] = badchar[2] = badchar[3] = badchar[4] = -1; + for (i = 0; i < plen; i++) + { + c = seq_nt4_table[(uint8_t)p[i]]; + badchar[c] = i; + if(c == 4) badchar[0] = badchar[1] = badchar[2] = badchar[3] = i; + } + badchar[4] = -1; + + while(s <= (tlen - plen)) + { + j = plen-1; + + while(j >= 0) + { + if(seq_nt4_table[(uint8_t)t[s+j]] >= 4) break; + if((p[j] != t[s+j]) && seq_nt4_table[(uint8_t)p[j]] < 4) break; + j--; + } + + + if (j < 0) + { + occ++; + ///s += (s+m < n)? m-badchar[txt[s+m]] : 1; + s++; + } + else + { + /*******************************for debug************************************/ + // long long f, end = s + MAX(1, j - badchar[seq_nt4_table[(uint8_t)t[s+j]]]); + // for (f = s+1; f < end; f++) + // { + // if(check_exact_match(t+f, plen, p, plen)) + // { + // fprintf(stderr, "s: %lld, end: %lld, s+j: %lld, t[s+j]: %c, badchar: %d, j: %lld\n", + // s, end, s+j, t[s+j], badchar[seq_nt4_table[(uint8_t)t[s+j]]], j); + // } + // } + /*******************************for debug************************************/ + + s += MAX(1, j - badchar[seq_nt4_table[(uint8_t)t[s+j]]]); + } + } + + return occ; +} + +#define pdq_cnt(q) ((q).x.a[0]) + +void init_pdq(pdq* q, uint64_t utg_num) +{ + kv_init(q->x); kv_push(uint64_t, q->x, 0); + kv_malloc(q->dis, utg_num); q->dis.n = utg_num; + kv_malloc(q->vis, utg_num); q->vis.n = utg_num; + + uint64_t i; + for (i = 1; (uint64_t)(1<uID_mode = ((uint64_t)-1) >> (64-i); + q->uID_shift = i; +} + +void destory_pdq(pdq* q) +{ + kv_destroy(q->x); + kv_destroy(q->dis); + kv_destroy(q->vis); +} + +void reset_pdq(pdq* q) +{ + q->x.n = 1; pdq_cnt(*q) = 0; + memset(q->dis.a, -1, sizeof(uint64_t)*q->dis.n); + memset(q->vis.a, 0, sizeof(uint8_t)*q->vis.n); +} + +void swap_pdq(uint64_t* i, uint64_t* j) +{ + uint64_t k; + k = (*i); + (*i) = (*j); + (*j) = k; +} + +#define weight(q, i) (get_dv_adv((q).x.a[i], (q).uID_mode, (q).uID_shift, &(q).tmp_v, &(q).tmp_d)) + +uint64_t inline set_dv_adv(uint64_t v, uint64_t dis, uint64_t v_mode, uint64_t v_shift) +{ + dis <<= v_shift; dis |= (v&v_mode); + return dis; +} + +uint64_t inline get_dv_adv(uint64_t x, uint64_t v_mode, uint64_t v_shift, uint64_t* v, uint64_t* dis) +{ + (*v) = x & v_mode; + (*dis) = x >> v_shift; + return (*dis); +} + + +void push_pdq(pdq* q, uint64_t v, uint64_t dis) +{ + kv_push(uint64_t, q->x, set_dv_adv(v, dis, q->uID_mode, q->uID_shift)); + pdq_cnt(*q)++; + int c_i = pdq_cnt(*q), p_i = c_i>>1; + + while ((p_i > 0) && (weight(*q, c_i) < weight(*q, p_i))) + { + swap_pdq(&(q->x.a[c_i]), &(q->x.a[p_i])); + c_i = p_i; + p_i = c_i >> 1; + } +} + +void pop_pdq(pdq* q, uint64_t* min_v, uint64_t* min_dis) +{ + (*min_v) = (*min_dis) = (uint64_t)-1; + if(pdq_cnt(*q) == 0) return; + get_dv_adv((*q).x.a[1], (*q).uID_mode, (*q).uID_shift, min_v, min_dis); + /*******************************for debug************************************/ + // uint64_t i; + // for (i = 1; i < q->x.n; i++) + // { + // if(weight(*q, i) < (*min_dis)) fprintf(stderr, "ERROR\n"); + // } + /*******************************for debug************************************/ + ///min = q->x.a[1]; + swap_pdq(&(q->x.a[1]), &(q->x.a[pdq_cnt(*q)])); + pdq_cnt(*q)--; + q->x.n--; + + int c_i = 1, left_i, right_i, min_i, flag = 1; + while(flag == 1) + { + flag = 0; + left_i = c_i << 1; + right_i = left_i + 1; + if(left_i > (int)(pdq_cnt(*q))) + { + break; // both children are null + } + else if(right_i > (int)(pdq_cnt(*q))) + { + min_i = left_i; // right children is null + } + else + { + min_i = (weight(*q, left_i) < weight(*q, right_i))? left_i : right_i; + } + + if(weight(*q, c_i) > weight(*q, min_i)) + { + swap_pdq(&(q->x.a[c_i]), &(q->x.a[min_i])); + c_i = min_i; + flag = 1; + } + } +} + + +void get_shortest_path(uint32_t src, pdq* pq, asg_t *sg, uint32_t* pre) +{ + uint64_t v, u, i, nv, w; + asg_arc_t *av = NULL; + reset_pdq(pq); + pq->dis.a[src] = 0; + if(pre) pre[src] = (uint32_t)-1; + push_pdq(pq, src, 0); + while (pdq_cnt(*pq) > 0) + { + pop_pdq(pq, &v, &w); + pq->vis.a[v] = 1; + + av = asg_arc_a(sg, v); + nv = asg_arc_n(sg, v); + + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + u = av[i].v; + w = (uint32_t)av[i].ul; + + if(pq->vis.a[u] == 0 && pq->dis.a[u] > pq->dis.a[v] + w) + { + pq->dis.a[u] = pq->dis.a[v] + w; + push_pdq(pq, u, pq->dis.a[u]); + if(pre) pre[u] = v; + } + } + } +} + + + +void all_pair_shortest_path(const ha_ug_index* idx, hc_links* link, MT* M) +{ + double index_time = yak_realtime(); + asg_t *sg = idx->ug->g; + hc_linkeage* t = NULL; + pdq pq; + init_pdq(&pq, sg->n_seq<<1); + uint32_t n_vtx = sg->n_seq<<1, v; + uint64_t k, *p = NULL; + + for (v = 0; v < n_vtx; ++v) + { + if (sg->seq[v>>1].del) continue; + t = &(link->a.a[v>>1]); + if (t->e.n == 0) continue; + get_shortest_path(v, &pq, sg, NULL); + for (k = 0; k < pq.dis.n; k++) + { + if(pq.dis.a[k] == (uint64_t)-1) continue; + kv_pushp(uint64_t, M->matrix.a[v].a, &p); + (*p) = k << M->uID_shift; + (*p) = (*p) | pq.dis.a[k]; + } + } + + destory_pdq(&pq); + fprintf(stderr, "[M::%s::%.3f]\n", __func__, yak_realtime()-index_time); +} + +uint64_t LCA_distance(long long d_x, long long d_y, long long xLen, long long yLen, uint8_t* rev) +{ + (*rev) = 0; + long long x_beg, x_end, y_beg, y_end, t_beg, t_end; + x_end = d_x; x_beg = x_end - xLen + 1; + y_end = d_y; y_beg = y_end - yLen + 1; + if(x_end >= y_end) + { + t_end = x_end; (*rev) = 0; + t_beg = y_beg; + } + else + { + t_end = y_end; (*rev) = 1; + t_beg = x_beg; + } + + return t_end + 1 - t_beg; +} + +uint64_t get_LCA_bubble(uint32_t x, uint64_t xLen, uint32_t y, uint64_t yLen, uint8_t* dis, uint64_t n, MT* M, bubble_type* bub, uint64_t* min_rev) +{ + uint32_t j, v, k; + uint64_t u, d = (uint64_t)-1, tmp; + uint8_t rev; + uint32_t root[2], a_n, *a; + get_bubbles(bub, bub->index[x>>1], &root[0], &root[1], &a, &a_n, NULL); + root[0] ^= 1; root[1] ^= 1; + if(root[0] > root[1]) + { + k = root[0]; + root[0] = root[1]; + root[1] = k; + } + + dis[root[0]] = (uint8_t)-1; + dis[root[1]] = (uint8_t)-1; + + v = x; + for (j = 0; j < M->matrix.a[v].a.n; j++) + { + u = M->matrix.a[v].a.a[j] >> M->uID_shift; + d = M->matrix.a[v].a.a[j] & M->dis_mode; + dis[u] = dis[u] >> 4; + } + + v = y; + for (j = 0; j < M->matrix.a[v].a.n; j++) + { + u = M->matrix.a[v].a.a[j] >> M->uID_shift; + d = M->matrix.a[v].a.a[j] & M->dis_mode; + dis[u] = dis[u] >> 4; + } + uint64_t x_i = 0, y_i = 0, d_x, d_y, min_d = (uint64_t)-1; + uint32_t min_j = (uint32_t)-1; + (*min_rev) = (uint64_t)-1; + + + for (k = 0; k < 2; k++) + { + j = root[k]; + if(dis[j] != 0) + { + dis[j] = (uint8_t)-1; + continue; + } + + for (; x_i < M->matrix.a[x].a.n; x_i++) + { + u = M->matrix.a[x].a.a[x_i] >> M->uID_shift; + d = M->matrix.a[x].a.a[x_i] & M->dis_mode; + if(u == j) break; + } + if(x_i == M->matrix.a[x].a.n && M->matrix.a[x].a.n != 0) fprintf(stderr, "ERROR X\n"); + d_x = d; + + for (; y_i < M->matrix.a[y].a.n; y_i++) + { + u = M->matrix.a[y].a.a[y_i] >> M->uID_shift; + d = M->matrix.a[y].a.a[y_i] & M->dis_mode; + if(u == j) break; + } + if(y_i == M->matrix.a[y].a.n && M->matrix.a[y].a.n != 0) fprintf(stderr, "ERROR Y\n"); + d_y = d; + + tmp = LCA_distance(d_x, d_y, xLen, yLen, &rev); + if(tmp < min_d) min_d = tmp, (*min_rev) = rev, min_j = j; + } + + if(min_j == x || min_j == y) return (uint64_t)-1; + + return min_d; + + +} + +uint64_t get_LCA(uint32_t x, uint64_t xLen, uint32_t y, uint64_t yLen, uint8_t* dis, uint64_t n, MT* M, bubble_type* bub, uint64_t* min_rev) +{ + if(IF_BUB(x>>1, *bub) && IF_BUB(y>>1, *bub) && bub->index[x>>1] == bub->index[y>>1]) + { + return get_LCA_bubble(x, xLen, y, yLen, dis, n, M, bub, min_rev); + } + else + { + memset(dis, -1, sizeof(uint8_t)*n); + } + + uint32_t j, v; + uint64_t u, d = (uint64_t)-1, tmp; + uint8_t rev; + + v = x; + for (j = 0; j < M->matrix.a[v].a.n; j++) + { + u = M->matrix.a[v].a.a[j] >> M->uID_shift; + d = M->matrix.a[v].a.a[j] & M->dis_mode; + dis[u] = dis[u] >> 4; + } + + v = y; + for (j = 0; j < M->matrix.a[v].a.n; j++) + { + u = M->matrix.a[v].a.a[j] >> M->uID_shift; + d = M->matrix.a[v].a.a[j] & M->dis_mode; + dis[u] = dis[u] >> 4; + } + + uint64_t x_i = 0, y_i = 0, d_x, d_y, min_d = (uint64_t)-1; + uint32_t min_j = (uint32_t)-1; + (*min_rev) = (uint64_t)-1; + for (j = 0; j < n; j++) + { + if(dis[j] != 0) + { + dis[j] = (uint8_t)-1; + continue; + } + + for (; x_i < M->matrix.a[x].a.n; x_i++) + { + u = M->matrix.a[x].a.a[x_i] >> M->uID_shift; + d = M->matrix.a[x].a.a[x_i] & M->dis_mode; + if(u == j) break; + } + if(x_i == M->matrix.a[x].a.n && M->matrix.a[x].a.n != 0) fprintf(stderr, "ERROR X\n"); + d_x = d; + + for (; y_i < M->matrix.a[y].a.n; y_i++) + { + u = M->matrix.a[y].a.a[y_i] >> M->uID_shift; + d = M->matrix.a[y].a.a[y_i] & M->dis_mode; + if(u == j) break; + } + if(y_i == M->matrix.a[y].a.n && M->matrix.a[y].a.n != 0) fprintf(stderr, "ERROR Y\n"); + d_y = d; + + + tmp = LCA_distance(d_x, d_y, xLen, yLen, &rev); + if(tmp < min_d) min_d = tmp, (*min_rev) = rev, min_j = j; + } + + if(min_j == x || min_j == y) return (uint64_t)-1; + + return min_d; +} + +typedef struct { // data structure for each step in kt_pipeline() + const ha_ug_index* idx; + hc_links* link; + MT* M; + bubble_type* bub; + uint8_t** dis_buf; +} utg_d_t; + + +static void worker_for_dis(void *data, long i, int tid) +{ + utg_d_t* s = (utg_d_t*)data; + const ha_ug_index* idx = s->idx; + hc_links* link = s->link; + MT* M = s->M; + bubble_type* bub = s->bub; + uint8_t* dis_buf = s->dis_buf[tid]; + asg_t *sg = idx->ug->g; + hc_linkeage* t = NULL; + uint32_t n_vtx = sg->n_seq<<1, v, u, k, j; + uint64_t d[2], db[2], q_u, min, min_i, min_b, rev[2], min_rev; + + if (sg->seq[i].del) return; + t = &(link->a.a[i]); + if (t->e.n == 0) return; + + for (k = 0; k < t->e.n; k++) + { + if(t->e.a[k].del) continue; + u = t->e.a[k].uID; + + for (v = ((uint64_t)(i)<<1); v < ((uint64_t)(i+1)<<1); v++) + { + d[0] = d[1] = db[0] = db[1] = (uint64_t)-1; + for (j = 0; j < M->matrix.a[v].a.n; j++) + { + q_u = M->matrix.a[v].a.a[j] >> M->uID_shift; + if((q_u>>1) == u) d[q_u&1] = (M->matrix.a[v].a.a[j] & M->dis_mode) + sg->seq[q_u>>1].len; + if((q_u>>1) > u) break;///just for speeding up, doesn't affect results + } + + min = min_i = min_b = (uint64_t)-1; + if(t->e.a[k].dis != (uint64_t)-1) min = t->e.a[k].dis >> 3; + + if(d[0] < min) min = d[0], min_i = 0, min_b = 0; + if(d[1] < min) min = d[1], min_i = 1, min_b = 0; + + if(min_i != (uint64_t)-1 && min != (uint64_t)-1) + { + t->e.a[k].dis = min<<1; + t->e.a[k].dis += min_b; + t->e.a[k].dis <<=1; + t->e.a[k].dis += v&1; + t->e.a[k].dis <<=1; + t->e.a[k].dis += min_i; + } + } + + + ///might be wrong + if(IF_BUB(i, *bub) && IF_BUB(u, *bub) + && bub->index[i] != bub->index[u] && t->e.a[k].dis != (uint64_t)-1) + { + continue; + } + + for (v = ((uint64_t)(i)<<1); v < ((uint64_t)(i+1)<<1); v++) + { + d[0] = d[1] = db[0] = db[1] = (uint64_t)-1; + db[0] = get_LCA(v, sg->seq[v>>1].len, u<<1, sg->seq[u].len, + dis_buf, n_vtx, M, bub, &rev[0]); + db[1] = get_LCA(v, sg->seq[v>>1].len, (u<<1) + 1, sg->seq[u].len, + dis_buf, n_vtx, M, bub, &rev[1]); + + min = min_i = min_b = min_rev = (uint64_t)-1; + if(t->e.a[k].dis != (uint64_t)-1) min = t->e.a[k].dis >> 3; + + if(db[0] < min) min = db[0], min_i = 0, min_b = 1, min_rev = rev[0]; + if(db[1] < min) min = db[1], min_i = 1, min_b = 1, min_rev = rev[1]; + + if(min_i != (uint64_t)-1 && min != (uint64_t)-1) + { + t->e.a[k].dis = min<<1; + t->e.a[k].dis += min_b; + t->e.a[k].dis <<=1; + t->e.a[k].dis += ((v&1)^min_rev); + t->e.a[k].dis <<=1; + t->e.a[k].dis += (min_i^min_rev); + } + } + } + +} + +void fill_utg_distance_multi(const ha_ug_index* idx, hc_links* link, MT* M, bubble_type* bub) +{ + double index_time = yak_realtime(); + uint32_t i; + utg_d_t s; + s.idx = idx; s.link = link; s.M = M; s.bub = bub; + s.dis_buf = (uint8_t**)malloc(sizeof(uint8_t*)*asm_opt.thread_num); + for (i = 0; i < (uint32_t)asm_opt.thread_num; i++) + { + s.dis_buf[i] = (uint8_t*)malloc(sizeof(uint8_t)*(s.idx->ug->g->n_seq<<1)); + } + + kt_for(asm_opt.thread_num, worker_for_dis, &s, s.idx->ug->g->n_seq); + + + for (i = 0; i < (uint32_t)asm_opt.thread_num; i++) + { + free(s.dis_buf[i]); + } + free(s.dis_buf); + fprintf(stderr, "[M::%s::%.3f]\n", __func__, yak_realtime()-index_time); +} + +void init_MT(MT* M, uint32_t n_vtx) +{ + uint32_t v; + kv_init(M->matrix); kv_malloc(M->matrix, n_vtx); M->matrix.n = n_vtx; + for (v = 0; v < n_vtx; ++v) kv_init(M->matrix.a[v].a); + for (v = 1; (uint64_t)(1<uID_shift = 64 - v; M->dis_mode = ((uint64_t)-1) >> v; +} + +void destory_MT(MT* M) +{ + uint32_t v; + for (v = 0; v < M->matrix.n; ++v) kv_destroy(M->matrix.a[v].a); + kv_destroy(M->matrix); +} + +void collect_hc_links(const ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, MT* M) +{ + double index_time = yak_realtime(); + uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>shif); + end = ((hits->a.a[k].e<<1)>>shif); + + if(beg == end) continue; + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + t_d = (uint64_t)-1; + push_hc_edge(&(link->a.a[beg]), end, 0, 0, &t_d); + push_hc_edge(&(link->a.a[end]), beg, 0, 0, &t_d); + } + + all_pair_shortest_path(idx, link, M); + fill_utg_distance_multi(idx, link, M, bub); + + fprintf(stderr, "[M::%s::%.3f] ==> Hi-C linkages have been counted\n", __func__, yak_realtime()-index_time); + return; + + + + + + index_time = yak_realtime(); + for (k = 0; k < link->enzymes.n; k++) + { + link->enzymes.a[k] = 0; + for (i = 0; i < (uint64_t)asm_opt.hic_enzymes->n; i++) + { + link->enzymes.a[k] += get_enzyme_occ(idx->ug->u.a[k].s, idx->ug->u.a[k].len, + asm_opt.hic_enzymes->a[i], asm_opt.hic_enzymes->l[i]); + } + } + fprintf(stderr, "[M::%s::%.3f] ==> Enzymes have been counted\n", __func__, yak_realtime()-index_time); +} + + +void set_reverse_links(uint32_t* bub, uint32_t n, kvec_t_u32_warp* reach, uint32_t root, hc_links* link) +{ + uint64_t i, k, d = RC_0; + uint32_t v; + for (i = 0; i < n; i++) + { + v = bub[i]>>1; + if(v == root) continue; + for (k = 0; k < reach->a.n; k++) + { + if(v == reach->a.a[k]) break; + } + + ///if(k == reach->a.n && reach->a.n > 0) + if(k == reach->a.n) + { + push_hc_edge(&(link->a.a[root]), v, 1, 1, &d); + push_hc_edge(&(link->a.a[v]), root, 1, 1, &d); + } + } + +} + +void collect_hc_reverse_links(hc_links* link, ma_ug_t* ug, bubble_type* bub) +{ + uint64_t i, j, k, d = RC_0, m, pre; + uint32_t beg, sink, n, v, *a = NULL; + kvec_t_u32_warp stack, result; + hc_edge *e = NULL; + kv_init(stack.a); kv_init(result.a); + ///clean all reverse overlaps within bubbles + ///might be wrong + for (i = 0; i < bub->f_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, NULL); + for (k = 0; k < n; k++) + { + v = a[k]>>1; + for (j = 0; j < link->a.a[v].f.n; j++) + { + if(link->a.a[v].f.a[j].del) continue; + e = get_hc_edge(link, link->a.a[v].f.a[j].uID, v, 1); + e->del = 1; + } + link->a.a[v].f.n = 0; + } + + v = beg>>1; + if(IF_HOM(v, *bub)) + { + for (j = 0; j < link->a.a[v].f.n; j++) + { + if(link->a.a[v].f.a[j].del) continue; + e = get_hc_edge(link, link->a.a[v].f.a[j].uID, v, 1); + e->del = 1; + } + link->a.a[v].f.n = 0; + } + + v = sink>>1; + if(IF_HOM(v, *bub)) + { + for (j = 0; j < link->a.a[v].f.n; j++) + { + if(link->a.a[v].f.a[j].del) continue; + e = get_hc_edge(link, link->a.a[v].f.a[j].uID, v, 1); + e->del = 1; + } + link->a.a[v].f.n = 0; + } + } + + for (i = 0; i < bub->f_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, NULL); + if(n == 2) + { + push_hc_edge(&(link->a.a[a[0]>>1]), a[1]>>1, 1, 1, &d); + push_hc_edge(&(link->a.a[a[1]>>1]), a[0]>>1, 1, 1, &d); + continue; + } + ///for complex bubbles, shouldn't have any assumption + ///if(i >= bub->s_bub) continue; + + beg = beg>>1; sink = sink>>1; + for (k = 0; k < n; k++) + { + v = a[k]>>1; + dfs_bubble(ug->g, &stack, &result, v, beg, sink); + set_reverse_links(a, n, &result, v, link); + } + } + + uint8_t* vis_flag = NULL; + MALLOC(vis_flag, ug->g->n_seq*2); + ///for broken bubbles + for (i = bub->f_bub; i < bub->f_bub + bub->b_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, NULL); + + for (k = 0; k < n; k++) + { + v = a[k]; + dfs_bubble_broken(ug->g, &stack, &result, vis_flag, ug->g->n_seq*2, v, beg, sink); + set_reverse_links(a, n, &result, v>>1, link); + } + } + + kv_destroy(stack.a); kv_destroy(result.a); free(vis_flag); + + + for (i = 0; i < link->a.n; i++) + { + for (k = m = 0; k < link->a.a[i].f.n; k++) + { + if(link->a.a[i].f.a[k].del) continue; + link->a.a[i].f.a[m] = link->a.a[i].f.a[k]; + m++; + } + link->a.a[i].f.n = m; + radix_sort_hc_edge_u(link->a.a[i].f.a, link->a.a[i].f.a + link->a.a[i].f.n); + + for (k = m = 0, pre = (uint64_t)-1; k < link->a.a[i].f.n; k++) + { + if(link->a.a[i].f.a[k].del) continue; + if(link->a.a[i].f.a[k].uID == pre) + { + if(link->a.a[i].f.a[k].dis == RC_0) link->a.a[i].f.a[m-1].dis = RC_0; + continue; + } + + pre = link->a.a[i].f.a[k].uID; + link->a.a[i].f.a[m] = link->a.a[i].f.a[k]; + m++; + } + link->a.a[i].f.n = m; + radix_sort_hc_edge_d(link->a.a[i].f.a, link->a.a[i].f.a + link->a.a[i].f.n); + } + + + // hc_edge *e = NULL; + // for (i = 0; i < link->a.n; i++) + // { + // for (k = 0; k < link->a.a[i].f.n; k++) + // { + // if(link->a.a[i].f.a[k].del) continue; + // e = get_hc_edge(link, link->a.a[i].f.a[k].uID, i, 1); + // if(e == NULL) fprintf(stderr, "ERROR\n"); + // } + // } + +} + +void write_hc_links(hc_links* link, const char *fn) +{ + uint64_t k; + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.hic.link", fn); + FILE* fp = fopen(buf, "w"); + + fwrite(&link->a.n, sizeof(link->a.n), 1, fp); + for (k = 0; k < link->a.n; k++) + { + fwrite(&link->a.a[k].e.n, sizeof(link->a.a[k].e.n), 1, fp); + fwrite(link->a.a[k].e.a, sizeof(hc_edge), link->a.a[k].e.n, fp); + + fwrite(&link->a.a[k].f.n, sizeof(link->a.a[k].f.n), 1, fp); + fwrite(link->a.a[k].f.a, sizeof(hc_edge), link->a.a[k].f.n, fp); + } + + fwrite(&link->enzymes.n, sizeof(link->enzymes.n), 1, fp); + fwrite(link->enzymes.a, sizeof(uint64_t), link->enzymes.n, fp); + fwrite(&link->r_num, sizeof(link->r_num), 1, fp); + fwrite(link->u_idx, sizeof(uint32_t), 1, fp); + + fwrite(&(link->bed.n), sizeof(link->bed.n), 1, fp); + for (k = 0; k < link->bed.n; k++) + { + fwrite(&(link->bed.a[k].n), sizeof(link->bed.a[k].n), 1, fp); + fwrite(link->bed.a[k].a, sizeof(uint64_t)*link->bed.a[k].n, 1, fp); + } + + + + fclose(fp); + free(buf); + fprintf(stderr, "[M::%s::] ==> Hi-C linkages have been written\n", __func__); +} + +int load_hc_links(hc_links* link, const char *fn) +{ + uint64_t k, flag = 0; + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.hic.link", fn); + + FILE* fp = NULL; + fp = fopen(buf, "r"); + if(!fp) + { + free(buf); + return 0; + } + + + kv_init(link->a); + flag += fread(&link->a.n, sizeof(link->a.n), 1, fp); + link->a.m = link->a.n; CALLOC(link->a.a, link->a.n); + for (k = 0; k < link->a.n; k++) + { + flag += fread(&link->a.a[k].e.n, sizeof(link->a.a[k].e.n), 1, fp); + link->a.a[k].e.m = link->a.a[k].e.n; MALLOC(link->a.a[k].e.a, link->a.a[k].e.n); + flag += fread(link->a.a[k].e.a, sizeof(hc_edge), link->a.a[k].e.n, fp); + + flag += fread(&link->a.a[k].f.n, sizeof(link->a.a[k].f.n), 1, fp); + link->a.a[k].f.m = link->a.a[k].f.n; MALLOC(link->a.a[k].f.a, link->a.a[k].f.n); + flag += fread(link->a.a[k].f.a, sizeof(hc_edge), link->a.a[k].f.n, fp); + } + + kv_init(link->enzymes); + flag += fread(&link->enzymes.n, sizeof(link->enzymes.n), 1, fp); + link->enzymes.m = link->enzymes.n; MALLOC(link->enzymes.a, link->enzymes.n); + flag += fread(link->enzymes.a, sizeof(uint64_t), link->enzymes.n, fp); + fread(&link->r_num, sizeof(link->r_num), 1, fp); + MALLOC(link->u_idx, link->r_num); + fread(link->u_idx, sizeof(uint32_t), 1, fp); + + + + kv_init(link->bed); + flag += fread(&(link->bed.n), sizeof(link->bed.n), 1, fp); + link->bed.m = link->bed.n; CALLOC(link->bed.a, link->bed.n); + for (k = 0; k < link->bed.n; k++) + { + flag += fread(&(link->bed.a[k].n), sizeof(link->bed.a[k].n), 1, fp); + link->bed.a[k].m = link->bed.a[k].n; MALLOC(link->bed.a[k].a, link->bed.a[k].n); + flag += fread(link->bed.a[k].a, sizeof(uint64_t)*link->bed.a[k].n, 1, fp); + } + + + + fclose(fp); + free(buf); + fprintf(stderr, "[M::%s::] ==> Hi-C linkages have been loaded\n", __func__); + return 1; +} + + +void write_hc_hits(kvec_pe_hit* hits, const char *fn) +{ + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.hic.lk.bin", fn); + FILE* fp = fopen(buf, "w"); + + fwrite(&hits->a.n, sizeof(hits->a.n), 1, fp); + fwrite(hits->a.a, sizeof(pe_hit), hits->a.n, fp); + + fclose(fp); + free(buf); +} + +int load_hc_hits(kvec_pe_hit* hits, const char *fn) +{ + uint64_t flag = 0; + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.hic.lk.bin", fn); + + FILE* fp = NULL; + fp = fopen(buf, "r"); + if(!fp) return 0; + + kv_init(hits->a); + flag += fread(&hits->a.n, sizeof(hits->a.n), 1, fp); + hits->a.m = hits->a.n; MALLOC(hits->a.a, hits->a.n); + flag += fread(hits->a.a, sizeof(pe_hit), hits->a.n, fp); + + fclose(fp); + free(buf); + fprintf(stderr, "[M::%s::] ==> Hi-C linkages have been loaded\n", __func__); + return 1; +} + +inline int get_phase_status(H_partition* hap, uint32_t uID) +{ + int d = -2; + if(hap->hap[uID] & hap->m[0]) d = 1; + if(hap->hap[uID] & hap->m[1]) d = -1; + if(hap->hap[uID] & hap->m[2]) d = 0; + return d; +} + +inline uint32_t get_phase_group(H_partition* hap, uint32_t uID) +{ + return hap->hap[uID]>>hap->label_shift; +} + +void print_hc_links(hc_links* link, int dir, H_partition* hap) +{ + uint64_t i, k; + if(dir == 0) + { + double f_w, r_w; + for (i = 0, f_w = r_w = 0; i < link->a.n; ++i) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + fprintf(stderr, "s-utg%.6dl(%c)\tCLU:%d:%u\td-utg%.6dl(%c)\tCLU:%d:%u\t%lu\t%c\t%f\te\n", + (int)(i+1), "01"[!!(link->a.a[i].e.a[k].dis&(uint64_t)2)], + get_phase_status(hap, i), hap->hap[i]>>3, + (int)(link->a.a[i].e.a[k].uID+1), "01"[!!(link->a.a[i].e.a[k].dis&(uint64_t)1)], + get_phase_status(hap, link->a.a[i].e.a[k].uID), hap->hap[link->a.a[i].e.a[k].uID]>>3, + link->a.a[i].e.a[k].dis == (uint64_t)-1? (uint64_t)-1 : link->a.a[i].e.a[k].dis>>3, + "fb"[!!(link->a.a[i].e.a[k].dis&(uint64_t)4)], link->a.a[i].e.a[k].weight); + if(get_phase_status(hap, i) == get_phase_status(hap, link->a.a[i].e.a[k].uID)) + { + f_w += link->a.a[i].e.a[k].weight; + } + else + { + r_w += link->a.a[i].e.a[k].weight; + } + } + + fprintf(stderr, "self-utg%.6dl\tFW:%f\tRW:%f\tRT:%f\n**************************************************\n", + (int)(i+1), f_w, r_w, r_w/f_w); + } + } + + + if(dir == 1) + { + for (i = 0; i < link->a.n; ++i) + { + for (k = 0; k < link->a.a[i].f.n; k++) + { + if(link->a.a[i].f.a[k].del) continue; + fprintf(stderr, "s-utg%.6d\td-utg%.6d\t%lu\te\n", + (int)(i+1), (int)(link->a.a[i].f.a[k].uID+1), link->a.a[i].f.a[k].dis); + } + } + } + +} + +void normalize_hc_links(hc_links* link) +{ + uint64_t i, k; + for (i = 0; i < link->a.n; ++i) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[k].weight *= 100; + link->a.a[i].e.a[k].weight /= (double)(MIN(link->enzymes.a[i], link->enzymes.a[link->a.a[i].e.a[k].uID])); + ///link->a.a[i].e.a[k].weight /= (double)(link->enzymes.a[i] + link->enzymes.a[link->a.a[i].e.a[k].uID]); + } + } +} + +hc_edge* get_rGraph_edge(min_cut_t* x, uint64_t src, uint64_t dest) +{ + if(src >= x->rGraph.n) return NULL; + uint64_t i; + for (i = 0; i < x->rGraph.a[src].n; i++) + { + if(x->rGraph.a[src].a[i].del) continue; + if(x->rGraph.a[src].a[i].uID == dest) return &(x->rGraph.a[src].a[i]); + } + return NULL; +} + + +void init_min_cut_t(min_cut_t* x, hc_links* link, const bubble_type* bub, const ma_ug_t *ug) +{ + uint64_t utg_num = link->a.n, i, k, u, v; + x->n = utg_num; + x->n_e = x->c_e = 0; + + kv_malloc(x->rGraphSet, utg_num); x->rGraphSet.n = utg_num; + ///must utg_num<<1) + kv_malloc(x->rGraphVis, utg_num); x->rGraphVis.n = utg_num; + kv_malloc(x->utgVis, utg_num); x->utgVis.n = utg_num; + kv_malloc(x->bmerVis, utg_num); x->bmerVis.n = utg_num; + kv_malloc(x->order, utg_num); x->order.n = utg_num; + kv_malloc(x->parent, utg_num); x->parent.n = utg_num; + kv_malloc(x->p_weight, utg_num); x->p_weight.n = utg_num; + ///uresolved BUGs, if use kv_resize segfault; if use kv_malloc, work????? + kv_malloc(x->rGraph, utg_num); x->rGraph.n = utg_num; + // kv_init(x->rGraph); kv_resize(hc_edge_warp, x->rGraph, utg_num); x->rGraph.n = utg_num; + x->enzymes = link->enzymes.a; + init_pdq(&(x->pq), utg_num<<1); + + //must be utg_num + 2 since we may need to add fake nodes + for (i = 1; (uint64_t)(1<uID_mode = ((uint64_t)-1) >> (64-i); + x->uID_shift = i; + for (i = 0; i < utg_num; i++) + { + ///x->order.a[i] = link->a.a[i].f.n; + ///x->order.a[i] = ug->u.a[i].len; + x->order.a[i] = x->enzymes[i]; + x->order.a[i] <<= x->uID_shift; + x->order.a[i] |= (uint64_t)(i & x->uID_mode); + + x->rGraphSet.a[i] = 0; + x->rGraphVis.a[i] = 0; + x->utgVis.a[i] = 0; + x->bmerVis.a[i] = 0; + x->parent.a[i] = (uint32_t)-1; + + ///uresolved BUGs, if use kv_resize segfault; if use kv_malloc, work????? + // kv_init(x->rGraph.a[i]); kv_resize(hc_edge, x->rGraph.a[i], link->a.a[i].e.n); + kv_malloc(x->rGraph.a[i], link->a.a[i].e.n); + x->rGraph.a[i].n = link->a.a[i].e.n; + + if(x->rGraph.a[i].n) + { + for (k = 0; k < x->rGraph.a[i].n; k++) + { + ///kv_push(hc_edge, x->rGraph.a[i], link->a.a[i].e.a[k]); + x->rGraph.a[i].a[k] = link->a.a[i].e.a[k]; + x->n_e++; + + if((x->rGraph.a[i].a[k].weight == 0) || IF_HOM(x->rGraph.a[i].a[k].uID, *bub) + || IF_HOM(i, *bub) || (x->rGraph.a[i].a[k].del)) + { + x->rGraph.a[i].a[k].del = 1; + x->n_e--; + } + } + } + } + + hc_edge *p = NULL; + for (i = 0; i < utg_num; i++) + { + v = i; + for (k = 0; k < link->a.a[v].f.n; k++) + { + if(link->a.a[v].f.a[k].del) continue; + u = link->a.a[v].f.a[k].uID; + + + p = get_rGraph_edge(x, v, u); + if(p) + { + p->del = 1; + x->n_e--; + } + + + p = get_rGraph_edge(x, u, v); + if(p) + { + p->del = 1; + x->n_e--; + } + } + } + + x->q = kdq_init(uint64_t); + radix_sort_hc64(x->order.a, x->order.a + x->order.n); + + x->b_mer = asm_opt.bub_mer_length; + ///fprintf(stderr, "[M::%s]\n", __func__); + ///exit(0); +} + +void destory_min_cut_t(min_cut_t* x) +{ + kv_destroy(x->order); + kv_destroy(x->parent); + kv_destroy(x->p_weight); + kv_destroy(x->rGraphSet); + kv_destroy(x->rGraphVis); + kv_destroy(x->utgVis); + kv_destroy(x->bmerVis); + destory_pdq(&(x->pq)); + uint64_t i; + for (i = 0; i < x->rGraph.m; i++) + { + kv_destroy(x->rGraph.a[i]); + } + kv_destroy(x->rGraph); + kdq_destroy(uint64_t, x->q); +} + +void reset_min_cut_t(min_cut_t* x, hc_links* link) +{ + ///no need to reset parent[] and q + uint64_t i, j; + ///important to have this line + x->bmerVis.n = x->parent.n = x->p_weight.n = x->order.n = x->rGraph.n = x->rGraphVis.n = x->rGraphSet.n = link->a.n; + kdq_clear(x->q); + + for (i = 0; i < x->rGraphSet.n; i++) + { + x->rGraphVis.a[i] = 0; + ///x->bmerVis.a[i] = 0; + ///important to have this line + x->rGraph.a[i].n = link->a.a[i].e.n; + + if(x->rGraphSet.a[i] == 0) continue; + for (j = 0; j < x->rGraph.a[i].n; j++) + { + x->rGraph.a[i].a[j].weight = link->a.a[i].e.a[j].weight; + } + x->rGraphSet.a[i] = 0; + } +} + +void update_link_by_min_cut_t(min_cut_t* x, hc_links* link) +{ + uint64_t i, j; + + for (i = 0; i < link->a.n; i++) + { + for (j = 0; j < link->a.a[i].e.n; j++) + { + link->a.a[i].e.a[j].del = x->rGraph.a[i].a[j].del; + } + } +} +uint64_t add_mul_convex(min_cut_t* x, uint64_t* a, uint64_t n) +{ + if(n == 0) return (uint64_t)-1; + if(n == 1) return a[0]; + kv_push(uint8_t, x->rGraphSet, 0); + kv_push(uint8_t, x->rGraphVis, 0); + kv_push(uint8_t, x->bmerVis, 0); + kv_push(uint32_t, x->parent, 0); + kv_push(double, x->p_weight, 0); + kv_resize(hc_edge_warp, x->rGraph, x->rGraph.n+1); + kv_init(x->rGraph.a[x->rGraph.n]); + uint64_t i, k; + hc_edge t; + for (i = 0; i < n; i++) + { + ///t.uID = a[i]; t.del = t.enzyme = t.weight = 0; + t.uID = a[i]; t.del = t.weight = 0; + for (k = 0; k < x->rGraph.a[a[i]].n; k++) + { + if(x->rGraph.a[a[i]].a[k].del) continue; + t.weight += x->rGraph.a[a[i]].a[k].weight; + } + kv_push(hc_edge, x->rGraph.a[x->rGraph.n], t); + t.uID = x->rGraph.n; + kv_push(hc_edge, x->rGraph.a[a[i]], t); + } + + x->rGraph.n++; + return x->rGraph.n - 1; +} + +void get_s_t(min_cut_t* x, hc_links* link, uint64_t uID, uint64_t* src, uint64_t* dest, kvec_t_u64_warp* buff) +{ + buff->a.n = 0; (*src) = (*dest) = (uint64_t)-1; + if(link->a.a[uID].f.n == 0) return; + (*src) = uID; + + uint64_t i, n; + for (i = 0, n = 0; i < link->a.a[uID].f.n; i++) + { + if(link->a.a[uID].f.a[i].del) continue; + kv_push(uint64_t, buff->a, link->a.a[uID].f.a[i].uID); + (*dest) = link->a.a[uID].f.a[i].uID; + n++; + } + if(n == 1 || n == 0) return; + (*dest) = add_mul_convex(x, buff->a.a, buff->a.n); +} + + +uint64_t bfs_flow(uint64_t src, uint64_t dest, min_cut_t* x, kvec_t_u64_warp* buff) +{ + uint64_t *p = NULL, v, u, i; + if(dest != (uint64_t)-1) memset(x->rGraphVis.a, 0, x->rGraphVis.n); + + kdq_push(uint64_t, x->q, src); + if(buff) kv_push(uint64_t, buff->a, src); + + x->rGraphVis.a[src] = 1; + x->parent.a[src] = (uint32_t)-1; + + while (1) + { + p = kdq_shift(uint64_t, x->q); + if(!p) break; + v = *p; + if(v == dest) return 1; + for (i = 0; i < x->rGraph.a[v].n; i++) + { + if(x->rGraph.a[v].a[i].del) continue; + if(x->rGraph.a[v].a[i].weight == 0) continue; + u = x->rGraph.a[v].a[i].uID; + if(x->rGraphVis.a[u]) continue; + if(!x->bmerVis.a[u]) continue; + + x->parent.a[u] = v; + x->p_weight.a[u] = x->rGraph.a[v].a[i].weight; + + kdq_push(uint64_t, x->q, u); + if(buff) kv_push(uint64_t, buff->a, u); + ///set u or v to be 1? doesn't matter + x->rGraphVis.a[u] = 1; + } + } + + return 0; +} + +uint64_t maxFlow(uint64_t src, uint64_t dest, min_cut_t* x) +{ + double flow = 0, max_flow = 0; + uint64_t v, u; + hc_edge *p; + + while (bfs_flow(src, dest, x, NULL)) + { + kdq_clear(x->q); + flow = DBL_MAX; + for (v = dest; v != src; v = x->parent.a[v]) + { + flow = MIN(flow, x->p_weight.a[v]); + } + + /*******************************for debug************************************/ + // if(src == 26818) fprintf(stderr, "***********flow: %f*********\n", flow); + /*******************************for debug************************************/ + + for (v = dest; v != src; v = x->parent.a[v]) + { + u = x->parent.a[v]; + p = get_rGraph_edge(x, u, v); + + /*******************************for debug************************************/ + // if(src == 26818) fprintf(stderr, "utg%.6lul (%f)\n", u+1, p->weight); + /*******************************for debug************************************/ + + p->weight -= flow; + p = get_rGraph_edge(x, v, u); + p->weight += flow; + x->rGraphSet.a[u] = x->rGraphSet.a[v] = 1; + } + + max_flow += flow; + } + + return (max_flow != 0); +} + + +uint64_t print_path(uint64_t src, uint64_t dest, min_cut_t* x) +{ + double flow = 0, max_flow = 0; + uint64_t v, u; + hc_edge *p; + + if(bfs_flow(src, dest, x, NULL)) + { + kdq_clear(x->q); + flow = DBL_MAX; + for (v = dest; v != src; v = x->parent.a[v]) + { + flow = MIN(flow, x->p_weight.a[v]); + } + + /*******************************for debug************************************/ + fprintf(stderr, "***********flow: %f*********\n", flow); + /*******************************for debug************************************/ + + for (v = dest; v != src; v = x->parent.a[v]) + { + u = x->parent.a[v]; + p = get_rGraph_edge(x, u, v); + + /*******************************for debug************************************/ + fprintf(stderr, "utg%.6lul (%f)\n", u+1, p->weight); + /*******************************for debug************************************/ + } + + max_flow += flow; + } + + return (max_flow != 0); +} + +void print_src_dest(uint64_t src, min_cut_t* x, const char* command) +{ + uint64_t i; + fprintf(stderr, "********************\n%s\n", command); + if(src >= x->n) + { + for (i = 0; i < x->rGraph.a[src].n; i++) + { + if(x->rGraph.a[src].a[i].del) continue; + fprintf(stderr, "utg%.6ul\n", x->rGraph.a[src].a[i].uID + 1); + } + + } + else + { + fprintf(stderr, "utg%.6lul\n", src+1); + } + fprintf(stderr, "!!!!!!!!!!!!!!!!!!!!\n"); + +} + +void print_debug_rGraph(min_cut_t* x) +{ + fprintf(stderr, "******rGraph******\n"); + uint64_t i, j, u; + for (i = 0; i < x->rGraphVis.n; i++) + { + if(!x->bmerVis.a[i]) continue; + for (j = 0; j < x->rGraph.a[i].n; j++) + { + if(x->rGraph.a[i].a[j].del) continue; + u = x->rGraph.a[i].a[j].uID; + if(!x->bmerVis.a[u]) continue; + fprintf(stderr, "***utg%.6lul\tutg%.6lul\t%f\n", i+1, u+1, x->rGraph.a[i].a[j].weight); + } + } + fprintf(stderr, "******rGraph******\n"); +} + +void graph_cut(uint64_t src, uint64_t dest, min_cut_t* x) +{ + /*******************************for debug************************************/ + ///if(src == 45179) print_debug_rGraph(x); + /*******************************for debug************************************/ + if(maxFlow(src, dest, x)) + { + ///in the last time bfs of maxFlow, rGraphVis has already been set + uint64_t i, j, v, u; + hc_edge *p; + /*******************************for debug************************************/ + if(src == 45179) + ///if(src == 26818) + { + ///print_debug_rGraph(x); + print_src_dest(src, x, "src utg:"); + print_src_dest(dest, x, "dest utg:"); + } + /*******************************for debug************************************/ + for (i = 0; i < x->rGraphVis.n; i++) + { + if(x->rGraphVis.a[i] == 0) continue; + if(!x->bmerVis.a[i]) continue; + v = i; + for (j = 0; j < x->rGraph.a[i].n; j++) + { + if(x->rGraph.a[i].a[j].del) continue; + u = x->rGraph.a[i].a[j].uID; + if(x->rGraphVis.a[u]) continue; + if(!x->bmerVis.a[u]) continue; + /*******************************for debug************************************/ + if(src == 45179) fprintf(stderr, "utg%.6lul\tutg%.6lul\t%f\n", v+1, u+1, x->rGraph.a[i].a[j].weight); + /*******************************for debug************************************/ + ///delete + x->rGraph.a[i].a[j].del = 1; + ///delete + p = get_rGraph_edge(x, u, v); + p->del = 1; + x->c_e += 2; + } + + } + + /*******************************for debug************************************/ + ///if(src == 45179 || src == 31635) + // if(src == 26818) + // { + // fprintf(stderr, "hahahaha\n"); + // print_path(26818, 1143, x); + // } + /*******************************for debug************************************/ + } + /*******************************for debug************************************/ + ///if(src == 45179 || src == 31635) + // { + // print_src_dest(src, x, "++++++src utg:"); + // uint64_t m; + // for (m = 0; m < x->rGraph.a[src].n; m++) + // { + // if(x->rGraph.a[src].a[m].del) continue; + // fprintf(stderr, "src(utg%.6dl, enz:%lu)\tdes(utg%.6dl, enz:%lu)\t%f\n", + // (int)(src+1), x->enzymes[src], + // (int)(x->rGraph.a[src].a[m].uID+1), x->enzymes[x->rGraph.a[src].a[m].uID], + // x->rGraph.a[src].a[m].weight); + // } + // } + /*******************************for debug************************************/ +} + +void check_connective(min_cut_t* x, hc_links* link) +{ + double index_time = yak_realtime(); + kvec_t_u64_warp buff; + kv_init(buff.a); + uint64_t i, k, uID; + for (i = 0; i < x->n; i++) + { + uID = x->order.a[i] & x->uID_mode; + if(link->a.a[uID].f.n == 0) continue; + for (k = 0; k < link->a.a[uID].f.n; k++) + { + if(link->a.a[uID].f.a[k].del) continue; + if(x->utgVis.a[link->a.a[uID].f.a[k].uID] == 0) break; + } + if(k == link->a.a[uID].f.n) continue; + reset_min_cut_t(x, link); + get_s_t(x, link, uID, &(x->src), &(x->dest), &buff); + + bfs_flow(x->src, x->dest, x, NULL); + + x->utgVis.a[uID] = 1; + } + + //reset x.utgVis + memset(x->utgVis.a, 0, x->utgVis.n); + kv_destroy(buff.a); + fprintf(stderr, "[M::%s::%.3f] \n", __func__, yak_realtime()-index_time); +} + +void get_Connected_Components(min_cut_t* x) +{ + double index_time = yak_realtime(); + uint64_t i, j, k = 0, uID, e; + kvec_t_u64_warp buff; + kv_init(buff.a); + while (1) + { + for (i = 0; i < x->n; i++) + { + uID = x->order.a[i] & x->uID_mode; + if(x->rGraphVis.a[uID] == 0) break; + } + if(i < x->n) + { + e = buff.a.n = 0; + bfs_flow(uID, (uint64_t)-1, x, &buff); + for (i = 0; i < buff.a.n; i++) + { + for (j = 0; j < x->rGraph.a[buff.a.a[i]].n; j++) + { + if(x->rGraph.a[buff.a.a[i]].a[j].del == 0) e++; + } + } + e >>= 1; + if(buff.a.n > 1) + { + fprintf(stderr, "(%lu) Component: # nodes: %lu, # edges: %lu\n", + k, (uint64_t)buff.a.n, e); + } + k++; + } + else + { + break; + } + } + + kv_destroy(buff.a); + fprintf(stderr, "[M::%s::%.3f] # Connected Components: %lu\n", + __func__, yak_realtime()-index_time, k); +} + +void print_rGraph(min_cut_t* x) +{ + uint64_t i, k; + for (i = 0; i < x->rGraph.n; ++i) + { + for (k = 0; k < x->rGraph.a[i].n; k++) + { + if(x->rGraph.a[i].a[k].del) continue; + fprintf(stderr, "src(utg%.6dl, enz:%lu)\tdes(utg%.6dl, enz:%lu)\t%f\n", + (int)(i+1), x->enzymes[i], + (int)(x->rGraph.a[i].a[k].uID+1), x->enzymes[x->rGraph.a[i].a[k].uID], + x->rGraph.a[i].a[k].weight); + } + } +} + + +int select_large_node(const ma_ug_t *ug, min_cut_t* x, +uint64_t src, uint64_t dest, uint64_t utg_thres, int weight_thres) +{ + if(src >= ug->u.n || dest >= ug->u.n) return 0; + if(ug->u.a[src].n < utg_thres || ug->u.a[dest].n < utg_thres) return 0; + + uint64_t k; + for (k = 0; k < x->rGraph.a[src].n; k++) + { + if(x->rGraph.a[src].a[k].del) continue; + if(x->rGraph.a[src].a[k].weight >= weight_thres) break; + } + if(k == x->rGraph.a[src].n) return 0; + + src = dest; + for (k = 0; k < x->rGraph.a[src].n; k++) + { + if(x->rGraph.a[src].a[k].del) continue; + if(x->rGraph.a[src].a[k].weight >= weight_thres) break; + } + if(k == x->rGraph.a[src].n) return 0; + + return 1; +} + +uint64_t inline set_dv(uint64_t v, uint64_t dis) +{ + dis <<= 32; dis |= v; + return dis; +} + + +uint64_t select_bmer(uint32_t src, uint64_t k, const bubble_type* bub, min_cut_t* x, uint32_t bub_only) +{ + uint32_t beg, sink, n, *a; + uint32_t v, d, u, i, nv, b_mer_d, j; + asg_t *sg = bub->ug->g; + uint64_t *p = NULL; + asg_arc_t *av = NULL; + + memset(x->rGraphVis.a, 0, x->rGraphVis.n); + kdq_push(uint64_t, x->q, set_dv(src , 0)); + b_mer_d = 0; + + x->rGraphVis.a[src] = 1; + x->bmerVis.a[src] = 1; + + while (1) + { + p = kdq_shift(uint64_t, x->q); + if(!p) break; + v = (uint32_t)(*p); d = ((uint64_t)(*p))>>32; + + v = v<<1; + av = asg_arc_a(sg, v); + nv = asg_arc_n(sg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + u = av[i].v>>1; + if(x->rGraphVis.a[u]) continue; + x->rGraphVis.a[u] = 1; + if(IF_HOM(u, *bub)) + { + if(d < k) kdq_push(uint64_t, x->q, set_dv(u, d+1)); + } + else + { + kdq_push(uint64_t, x->q, set_dv(u , d)); + b_mer_d = d; + if(IF_BUB(u, *bub) && x->bmerVis.a[u] == 0) + { + get_bubbles((bubble_type*)bub, bub->index[u], &beg, &sink, &a, &n, NULL); + for (j = 0; j < n; j++) x->bmerVis.a[(a[j]>>1)] = 1; + } + //must be here + if(bub_only == 0) x->bmerVis.a[u] = 1; + } + } + + + v = v + 1; + av = asg_arc_a(sg, v); + nv = asg_arc_n(sg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + u = av[i].v>>1; + if(x->rGraphVis.a[u]) continue; + x->rGraphVis.a[u] = 1; + if(IF_HOM(u, *bub)) + { + if(d < k) kdq_push(uint64_t, x->q, set_dv(u, d+1)); + } + else + { + kdq_push(uint64_t, x->q, set_dv(u , d)); + b_mer_d = d; + if(IF_BUB(u, *bub) && x->bmerVis.a[u] == 0) + { + get_bubbles((bubble_type*)bub, bub->index[u], &beg, &sink, &a, &n, NULL); + for (j = 0; j < n; j++) x->bmerVis.a[(a[j]>>1)] = 1; + } + //must be here + if(bub_only == 0) x->bmerVis.a[u] = 1; + } + } + } + + return b_mer_d; +} + + +void select_bmer_distance(uint32_t src, uint64_t k, const bubble_type* bub, min_cut_t* x, +uint32_t bub_only, uint32_t bub_extend) +{ + uint32_t beg, sink, n, *a; + asg_t *sg = bub->ug->g; + uint64_t v, u, i, j, nv, w, first = 1; + asg_arc_t *av = NULL; + reset_pdq(&(x->pq)); + + x->bmerVis.a[src>>1] = 1; + x->pq.dis.a[src] = 0; + push_pdq(&(x->pq), src, 0); + + while (pdq_cnt(x->pq) > 0) + { + pop_pdq(&(x->pq), &v, &w); + x->pq.vis.a[v] = 1; + if(x->pq.dis.a[v] > k) break; + + ///fprintf(stderr, "******utg%.6dl, dis: %lu\n", (int)((v>>1)+1), x->pq.dis.a[v]); + + if(IF_BUB(v>>1, *bub)) + { + if(bub_extend && x->bmerVis.a[v>>1] == 0) + { + get_bubbles((bubble_type*)bub, bub->index[v>>1], &beg, &sink, &a, &n, NULL); + for (j = 0; j < n; j++) x->bmerVis.a[(a[j]>>1)] = 1; + } + x->bmerVis.a[v>>1] = 1; + } + + if(IF_HET(v>>1, *bub) && bub_only == 0) x->bmerVis.a[v>>1] = 1; + + av = asg_arc_a(sg, v); + nv = asg_arc_n(sg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + u = av[i].v; + w = (uint32_t)av[i].ul; + if(first) w = 0; + + if(x->pq.vis.a[u] == 0 && x->pq.dis.a[u] > x->pq.dis.a[v] + w) + { + x->pq.dis.a[u] = x->pq.dis.a[v] + w; + push_pdq(&(x->pq), u, x->pq.dis.a[u]); + } + } + + first = 0; + } +} + +void get_bmer_unitgs(min_cut_t* x, const bubble_type* bub, uint64_t k, uint64_t src) +{ + uint32_t beg, sink, n, *a; + if(!IF_BUB(src, *bub)) return; + get_bubbles((bubble_type*)bub, bub->index[src], &beg, &sink, &a, &n, NULL); + memset(x->bmerVis.a, 0, x->bmerVis.n); + ///select_bmer(src, k, bub, x, 1); + select_bmer_distance(beg^1, k, bub, x, 1, 1); + select_bmer_distance(sink^1, k, bub, x, 1, 1); +} + + +min_cut_t* clean_hap(hc_links* link, bubble_type* bub, const ma_ug_t *ug) +{ + double index_time = yak_realtime(); + min_cut_t* x; CALLOC(x, 1); + kvec_t_u64_warp buff; + kv_init(buff.a); + init_min_cut_t(x, link, (const bubble_type*)bub, ug); + + // get_Connected_Components(&x); + // check_connective(&x, link); + // print_rGraph(&x); + + long long i; + uint64_t k, uID; + + ///for (i = 0; (uint64_t)i < x.n; i++) + for (i = x->n - 1; i >= 0; i--) + { + uID = x->order.a[i] & x->uID_mode; + ///fprintf(stderr, "uID: %lu, f.n: %lu\n", uID, (uint64_t)link->a.a[uID].f.n); + if(link->a.a[uID].f.n == 0) continue; + for (k = 0; k < link->a.a[uID].f.n; k++) + { + if(link->a.a[uID].f.a[k].del) continue; + if(x->utgVis.a[link->a.a[uID].f.a[k].uID] == 0) break; + } + ///fprintf(stderr, "k: %lu\n", k); + if(k == link->a.a[uID].f.n) continue; + reset_min_cut_t(x, link); + ///fprintf(stderr, "reset\n"); + get_s_t(x, link, uID, &(x->src), &(x->dest), &buff); + ///fprintf(stderr, "x.src: %lu, x.dest: %lu\n", x.src, x.dest); + ///Note: should only consider edges betweem bubbles, ignore edges to homo untigs + + /*******************************for debug************************************/ + ///if(!select_large_node(ug, &x, x.src, x.dest, 10, 0)) continue; + ///if(uID != 26818) continue; + //if(uID != 45179) continue; + ///memset(x.bmerVis.a, 1, x.bmerVis.n); + get_bmer_unitgs(x, bub, x->b_mer, x->src); + x->bmerVis.a[x->src] = x->bmerVis.a[x->dest] = 1; + /*******************************for debug************************************/ + + graph_cut(x->src, x->dest, x); + ///fprintf(stderr, "graph_cut\n"); + x->utgVis.a[uID] = 1; + ///exit(0); + } + + reset_min_cut_t(x, link); + + fprintf(stderr, "[M::%s::%.3f] # edges: %lu, # cutted edges: %lu\n", + __func__, yak_realtime()-index_time, x->n_e, x->c_e); + update_link_by_min_cut_t(x, link); + ///destory_min_cut_t(x); + kv_destroy(buff.a); + return x; +} + +void init_G_partition(G_partition* x, uint64_t n_utg) +{ + uint64_t i; + kv_init(*x); + MALLOC(x->index, n_utg); + for (i = 0; i < n_utg; i++) + { + x->index[i] = (uint32_t)-1; + } +} + +void reset_G_partition(G_partition* x, uint64_t n_utg) +{ + uint64_t i; + x->n = 0; + for (i = 0; i < n_utg; i++) + { + x->index[i] = (uint32_t)-1; + } +} + +void destory_G_partition(G_partition* x) +{ + uint64_t i; + for (i = 0; i < x->n; i++) + { + kv_destroy(x->a[i].a); + } + kv_destroy(*x); + free(x->index); +} + + +double get_hc_weight(uint32_t query, uint32_t v0, uint32_t root, bub_p_t_warp *b, min_cut_t* x) +{ + if(v0 == root) return 0; + uint32_t v, u; + hc_edge *p = NULL; + double weight = 0; + v = v0; + do { + u = b->a[v].p; // u->v + p = get_rGraph_edge(x, query>>1, v>>1); + if(p) weight += p->weight; + v = u; + } while (v != root); + + return weight; +} + +void set_path(bub_p_t_warp *b, uint32_t root, uint8_t* flag, uint8_t label) +{ + uint32_t v, u; + ///v is the sink of this bubble + v = b->S.a[0]; + do { + u = b->a[v].p; // u->v + flag[v>>1] |= label; + v = u; + } while (v != root); + flag[b->S.a[0]>>1] = 0; +} + +uint64_t trace_phase_path(ma_ug_t *ug, uint32_t s, uint32_t d, bub_p_t_warp *b, min_cut_t* x, uint8_t* flag, uint8_t label) +{ + asg_t *g = ug->g; + if(g->seq[s>>1].del) return 0; // already deleted + if(get_real_length(g, s, NULL)<2) return 0; + uint32_t i, n_pending, is_first, to_replace, cur_nc, cur_uc, cur_ac, n_tips, tip_end, n_pop; + double cur_nh, cur_rate, max_rate; + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[s].d = b->a[s].nc = b->a[s].ac = b->a[s].uc = 0; b->a[s].nh = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, s); + n_pop = n_tips = n_pending = 0; + tip_end = (uint32_t)-1; + is_first = 1; + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S); + uint32_t d = b->a[v].d, nc = b->a[v].nc, uc = b->a[v].uc, ac = b->a[v].ac; + double nh = b->a[v].nh; + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + for (i = 0; i < nv; ++i) { + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l, not overlap length + bub_p_t *t = &b->a[w]; + //got a circle + if ((w>>1) == (s>>1)) goto pop_reset; + //important when poping at long untig graph + if(is_first) l = 0; + if (av[i].del) continue; + ///push the edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + + if (t->s == 0) + { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p is the parent node of + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l, t->nc = nc + ug->u.a[(w>>1)].n; + t->r = get_real_length(g, w^1, NULL); + /**need fix**/ + t->nh = nh + get_hc_weight(w, v, s, b, x); + t->ac = ac + (flag[(w>>1)] == 0? ug->u.a[(w>>1)].n : 0); + t->uc = uc + (flag[(w>>1)] != 0? ug->u.a[(w>>1)].n : 0); + ++n_pending; + } + else { + to_replace = 0; + + cur_nc = nc + ug->u.a[(w>>1)].n; + /**need fix**/ + cur_nh = nh + get_hc_weight(w, v, s, b, x); + cur_ac = ac + (flag[(w>>1)] == 0? ug->u.a[(w>>1)].n : 0); + cur_uc = uc + (flag[(w>>1)] != 0? ug->u.a[(w>>1)].n : 0); + cur_rate = ((double)(cur_ac)/(double)(cur_ac+cur_uc)); + max_rate = ((double)(t->ac)/(double)(t->ac+t->uc)); + + if(cur_rate > max_rate) + { + to_replace = 1; + } + else if(cur_rate == max_rate) + { + if(cur_nh > t->nh) + { + to_replace = 1; + } + else if(cur_nh == t->nh) + { + if(cur_nc > t->nc) + { + to_replace = 1; + } + else if(cur_nc == t->nc) + { + if(d + l > t->d) + { + to_replace = 1; + } + } + } + } + + + if(to_replace) + { + t->p = v; + t->nc = cur_nc; + t->nh = cur_nh; + t->ac = cur_ac; + t->uc = cur_uc; + } + + + if (d + l < t->d) t->d = d + l; // update dist + } + + if (--(t->r) == 0) { + uint32_t x = get_real_length(g, w, NULL); + if(x > 0) + { + kv_push(uint32_t, b->S, w); + } + else + { + ///at most one tip + if(n_tips != 0) goto pop_reset; + n_tips++; + tip_end = w; + } + --n_pending; + } + } + is_first = 0; + + + if(n_tips == 1) + { + if(tip_end != (uint32_t)-1 && n_pending == 0 && b->S.n == 0) + { + ///sink is b.S.a[0] + kv_push(uint32_t, b->S, tip_end); + break; + } + else + { + goto pop_reset; + } + } + + if (i < nv || b->S.n == 0) goto pop_reset; + }while (b->S.n > 1 || n_pending); + + + n_pop = 1; + /**need fix**/ + set_path(b, s, flag, label); + pop_reset: + + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + bub_p_t *t = &b->a[b->b.a[i]]; + t->p = t->d = t->nc = t->ac = t->uc = t->r = t->s = 0; + t->nh = 0; + } + + return n_pop; +} + +inline void get_phased_block(G_partition* x, bubble_type* bub, uint64_t id, +uint32_t* beg, uint32_t* sink, uint32_t** h0, uint32_t* h0_n, uint32_t** h1, uint32_t* h1_n, +uint32_t* phased, uint32_t* bub_id) +{ + if(bub && beg && sink && bub_id) + { + (*bub_id) = (*beg) = (*sink) = (uint32_t)-1; + if(x->a[id].a.n > 0) + { + (*bub_id) = bub->index[x->a[id].a.a[0]]; + if(IF_BUB(x->a[id].a.a[0], *bub)) + { + (*beg) = bub->list.a[bub->num.a[(*bub_id)]]; + (*sink) = bub->list.a[bub->num.a[(*bub_id)] + 1]; + } + } + } + + (*h0) = x->a[id].a.a; + (*h0_n) = x->a[id].h[0]; + + (*h1) = x->a[id].a.a + x->a[id].h[0]; + (*h1_n) = x->a[id].h[1]; + if(phased) (*phased) = x->a[id].full_bub; + if((*h0_n) == 0) (*h0) = NULL; + if((*h1_n) == 0) (*h1) = NULL; +} + +double get_co_weight(uint32_t *query, uint32_t query_n, uint32_t *target, uint32_t target_n, min_cut_t* m) +{ + double weight = 0; + hc_edge *p = NULL; + uint32_t i, k; + for (i = 0; i < query_n; i++) + { + for (k = 0; k < target_n; k++) + { + p = get_rGraph_edge(m, query[i], target[k]); + if(p) weight += p->weight; + } + } + + return weight; +} + +void phase_bubble(uint64_t bid, bub_p_t_warp *b, bubble_type* bub, uint8_t* flag, const ma_ug_t *ug, +min_cut_t* m, hc_links* link, G_partition* x) +{ + #define HAP1_LAB 1 + #define HAP2_LAB 2 + + partition_warp* res = NULL; + kv_pushp(partition_warp, *x, &res); + memset(flag, 0, ug->g->n_seq); + uint32_t beg, sink, n, *a, i, k; + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + res->full_bub = 0; + trace_phase_path((ma_ug_t *)ug, beg, sink, b, m, flag, HAP1_LAB); + trace_phase_path((ma_ug_t *)ug, beg, sink, b, m, flag, HAP2_LAB); + kv_init(res->a); + for (i = 0; i < ug->g->n_seq; i++) + { + if(flag[i] & (uint8_t)HAP1_LAB) kv_push(uint32_t, res->a, i); + } + res->h[0] = res->a.n; + for (i = 0; i < ug->g->n_seq; i++) + { + if(flag[i] & (uint8_t)HAP2_LAB) kv_push(uint32_t, res->a, i); + } + res->h[1] = res->a.n - res->h[0]; + if(n == 2) res->full_bub = 1; + if(res->full_bub == 0) + { + double self = 0, intersec = 0; + uint32_t *h0 = NULL, *h1 = NULL; + h0 = res->a.a; h1 = res->a.a + res->h[0]; + self += get_co_weight(h0, res->h[0], h0, res->h[0], m); + self += get_co_weight(h1, res->h[1], h1, res->h[1], m); + + intersec += get_co_weight(h0, res->h[0], h1, res->h[1], m); + intersec = intersec * 2; + + if(self > intersec) res->full_bub = 1; + } + + if(res->full_bub == 0) + { + res->a.n = 0; + uint32_t v, u = 0, uv, k_n, pre_n = x->n; + hc_linkeage* t = NULL; + x->n--; + for (i = 0; i < n; i++) + { + v = a[i]>>1; + t = &(link->a.a[v]); + for (k = k_n = 0; k < t->f.n; k++) + { + if(t->f.a[k].del) continue; + k_n++; + u = t->f.a[k].uID; + } + if(k_n != 1) continue; + + t = &(link->a.a[u]); + for (k = k_n = 0; k < t->f.n; k++) + { + if(t->f.a[k].del) continue; + k_n++; + uv = t->f.a[k].uID; + } + if(k_n != 1) continue; + if(uv != v) continue; + + ///avoid dups + for (k = 0; k < i; k++) + { + if((a[k]>>1) == u) break; + } + if(k < i) continue; + + + kv_pushp(partition_warp, *x, &res); + if(x->n > pre_n) kv_init(res->a); + res->full_bub = 0; + res->h[0] = res->h[1] = 1; + kv_push(uint32_t, res->a, v); + kv_push(uint32_t, res->a, u); + + for (k = 0; k < res->a.n; k++) + { + x->index[res->a.a[k]] = x->n-1; + } + } + } + else + { + for (k = 0; k < res->a.n; k++) + { + x->index[res->a.a[k]] = x->n-1; + } + } + + /*******************************for debug************************************/ + // for (i = 0; i < ug->g->n_seq; i++) + // { + // if(flag[i] & (uint8_t)3) + // { + // uint32_t k; + // for (k = 0; k < n; k++) + // { + // if((a[k]>>1) == i) + // { + // break; + // } + // } + + // if(k == n) fprintf(stderr, "ERROR5\n"); + // } + // } + /*******************************for debug************************************/ +} + + +void print_phased_bubble(G_partition* x, bubble_type* bub, uint32_t utg_n) +{ + uint64_t i, k; + uint32_t beg = 0, sink = 0, h0_n, h1_n, *h0, *h1, full_bub = 0, bubID = 0; + + for (i = 0; i < x->n; i++) + { + get_phased_block(x, bub, i, &beg, &sink, &h0, &h0_n, &h1, &h1_n, &full_bub, &bubID); + + fprintf(stderr, "\n[%lu]\tbeg:utg%.6ul\tsink:utg%.6ul\tphased=%u\n", i, (beg>>1)+1, (sink>>1)+1, full_bub); + for (k = 0; k < h0_n; k++) + { + fprintf(stderr, "(0) utg%.6ul\n", h0[k] + 1); + } + + for (k = 0; k < h1_n; k++) + { + fprintf(stderr, "(1) utg%.6ul\n", h1[k] + 1); + } + + uint32_t n, *a; + get_bubbles(bub, bubID, &beg, &sink, &a, &n, NULL); + if(n > 2) fprintf(stderr, "complex\n"); + } + + /*******************************for debug************************************/ + for (i = 0; i < utg_n; i++) + { + if(x->index[i] == (uint32_t)-1) continue; + partition_warp* p = &(x->a[x->index[i]]); + for (k = 0; k < p->a.n; k++) + { + if(p->a.a[k] != i) break; + } + + if(k == p->a.n) fprintf(stderr, "ERROR\n"); + } + /*******************************for debug************************************/ +} + +G_partition* clean_bubbles(hc_links* link, bubble_type* bub, min_cut_t* m, const ma_ug_t *ug) +{ + double index_time = yak_realtime(); + uint64_t i; + bub_p_t_warp b; + memset(&b, 0, sizeof(bub_p_t_warp)); + CALLOC(b.a, ug->g->n_seq*2); + uint8_t* flag = NULL; + CALLOC(flag, ug->g->n_seq); + G_partition* x; CALLOC(x, 1); + init_G_partition(x, ug->g->n_seq); + + for (i = 0; i < bub->f_bub; i++) + { + phase_bubble(i, &b, bub, flag, ug, m, link, x); + } + + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + free(flag); + fprintf(stderr, "[M::%s::%.3f]\n", __func__, yak_realtime()-index_time); + ///print_phased_bubble(x, bub, ug->g->n_seq); + + return x; +} + + + +uint64_t get_hic_distance(pe_hit* hit, hc_links* link, const ha_ug_index* idx) +{ + uint64_t s_uid, s_dir, e_uid, e_dir, u_dis, k; + long long s_pos, e_pos; + s_uid = ((hit->s<<1)>>(64 - idx->uID_bits)); s_pos = hit->s & idx->pos_mode; + e_uid = ((hit->e<<1)>>(64 - idx->uID_bits)); e_pos = hit->e & idx->pos_mode; + if(s_uid == e_uid) return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + hc_linkeage* t = &(link->a.a[s_uid]); + for (k = 0; k < t->e.n; k++) + { + if(t->e.a[k].del || t->e.a[k].uID != e_uid) continue; + s_dir = (!!(t->e.a[k].dis&(uint64_t)2)); + e_dir = (!!(t->e.a[k].dis&(uint64_t)1)); + u_dis = (t->e.a[k].dis ==(uint64_t)-1? (uint64_t)-1 : t->e.a[k].dis>>3); + // if(s_uid == 24684 && s_pos == 124953 && e_uid == 16950 && e_pos == 93039) + // { + // fprintf(stderr, "*****************s_dir: %lu, e_dir: %lu, u_dis: %lu\n", s_dir, e_dir, u_dis); + // } + if(u_dis == (uint64_t)-1) return (uint64_t)-1; + if(s_dir == 1) s_pos = (long long)idx->ug->g->seq[s_uid].len - s_pos - 1; + if(e_dir == 1) e_pos = (long long)idx->ug->g->seq[e_uid].len - e_pos - 1; + e_pos = e_pos + u_dis - (long long)idx->ug->g->seq[e_uid].len; + return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + } + + return (uint64_t)-1; +} + +hc_edge* get_hc_edge(hc_links* link, uint64_t src, uint64_t dest, uint64_t dir) +{ + if(src >= link->a.n) return NULL; + uint64_t i, n; + hc_edge* a = NULL; + if(dir == 0) + { + n = link->a.a[src].e.n; + a = link->a.a[src].e.a; + } + else + { + n = link->a.a[src].f.n; + a = link->a.a[src].f.a; + } + + for (i = 0; i < n; i++) + { + if(a[i].del) continue; + if(a[i].uID == dest) return &(a[i]); + } + + return NULL; +} + +inline double get_trans(const ha_ug_index* idx, uint64_t x) +{ + return idx->a*(x/idx->frac) + idx->b; +} + +inline double get_trans_weight(const ha_ug_index* idx, uint64_t x) +{ + ///return 1.0; + long double rate = get_trans(idx, x); + if(rate < 0) rate = 0; + rate += OFFSET_RATE; + if(rate > 0.5) rate = 0.5; + rate -= OFFSET_SECOND_RATE; //[OFFSET_RATE - OFFSET_SECOND_RATE, 0.5 - OFFSET_SECOND_RATE] + + long double w = logl((1/rate)-1)*SCALL; + if(w < OFFSET_RATE_MIN_W) w = OFFSET_RATE_MIN_W; + if(w > OFFSET_RATE_MAX_W) w = OFFSET_RATE_MAX_W; + return w; +} + +inline double get_trans_weight_advance(const ha_ug_index* idx, uint64_t x, trans_idx* dis) +{ + long double rate = 0; + if(x < dis->max) + { + uint64_t i; + for (i = 0; i < dis->n; i++) + { + if(x < dis->a[i].end && x >= dis->a[i].beg) break; + } + if(i < dis->n) + { + rate = ((double)(dis->a[i].cnt_1))/((double)(dis->a[i].cnt_0 + dis->a[i].cnt_1)); + } + else + { + rate = get_trans(idx, x); + } + } + else + { + rate = get_trans(idx, x); + } + + if(rate < 0) rate = 0; + rate += OFFSET_RATE; + if(rate > 0.5) rate = 0.5; + rate -= OFFSET_SECOND_RATE; //[OFFSET_RATE - OFFSET_SECOND_RATE, 0.5 - OFFSET_SECOND_RATE] + + long double w = logl((1/rate)-1)*SCALL; + if(w < OFFSET_RATE_MIN_W) w = OFFSET_RATE_MIN_W; + if(w > OFFSET_RATE_MAX_W) w = OFFSET_RATE_MAX_W; + return w; +} + +void LeastSquare(uint64_t* vec, uint64_t len, ha_ug_index* idx, uint64_t med) +{ + #define SCAL_RATE 1000 + long double t1=0, t2=0, t3=0, t4=0, x, y, thres; + uint64_t i, len_convince, m; + + + for (i = 0; i < len; i += 4) + { + if(vec[i+1] > med) break; + x = ((double)(vec[i] + vec[i+1]))/2; + y = ((double)(vec[i+3]))/((double)(vec[i+2] + vec[i+3])); + + t1 += x*x; + t2 += x; + t3 += x*y; + t4 += y; + } + len_convince = i; + + if(t2 > t4) + { + idx->frac = t2/t4; + if(idx->frac > SCAL_RATE) idx->frac = idx->frac / SCAL_RATE; + } + + t1 /= (idx->frac*idx->frac); + t2 /= idx->frac; + t3 /= idx->frac; + idx->a = idx->b = 0; + if((t1*(len_convince>>2) - t2*t2) != 0) + { + idx->a = (t3*(len_convince>>2) - t2*t4) / (t1*(len_convince>>2) - t2*t2); + } + if((t1*(len_convince>>2) - t2*t2) != 0) + { + idx->b = (t1*t4 - t2*t3) / (t1*(len_convince>>2) - t2*t2); + } + + + + if(len > 0) + { + vec[len - 3] = vec[len - 4] + (vec[1] - vec[0]); + } + if(len_convince >= len) return; + + thres = get_trans(idx, vec[len_convince] + vec[len_convince+1]); + fprintf(stderr, "len_convince: %lu, len: %lu, t1: %f, t2: %f, t3: %f, t4: %f, idx->a: %f, idx->b: %f, thres: %f\n", + len_convince, len, (double)t1, (double)t2, (double)t3, (double)t4, (double)idx->a, (double)idx->b, (double)thres); + + + + for (i = m = 0; i < len; i += 4) + { + x = ((double)(vec[i] + vec[i+1]))/2; + y = ((double)(vec[i+3]))/((double)(vec[i+2] + vec[i+3])); + if(vec[i+1] > med && y < thres) continue; + + t1 += x*x; + t2 += x; + t3 += x*y; + t4 += y; + m++; + } + + if(t2 > t4) + { + idx->frac = t2/t4; + if(idx->frac > SCAL_RATE) idx->frac = idx->frac / SCAL_RATE; + } + + len = m; + t1 /= (idx->frac*idx->frac); + t2 /= idx->frac; + t3 /= idx->frac; + ///fprintf(stderr, "len: %lu, t1: %f, t2: %f, t3: %f, t4: %f\n", len, (double)t1, (double)t2, (double)t3, (double)t4); + if((t1*(len>>2) - t2*t2) != 0) + { + idx->a = (t3*(len>>2) - t2*t4) / (t1*(len>>2) - t2*t2); + } + if((t1*(len>>2) - t2*t2) != 0) + { + idx->b = (t1*t4 - t2*t3) / (t1*(len>>2) - t2*t2); + } + fprintf(stderr, "len: %lu, t1: %f, t2: %f, t3: %f, t4: %f, idx->a: %f, idx->b: %f\n", + len, (double)t1, (double)t2, (double)t3, (double)t4, (double)idx->a, (double)idx->b); + +} + + +void LeastSquare_advance(trans_idx* dis, ha_ug_index* idx, uint64_t med) +{ + #define SCAL_RATE 1000 + long double t1=0, t2=0, t3=0, t4=0, x, y; + uint64_t i, m, ava_size; + + for (i = m = 0; i < dis->n; i++) + { + x = ((double)(dis->a[i].beg + dis->a[i].end))/2; + y = ((double)(dis->a[i].cnt_1))/((double)(dis->a[i].cnt_0 + dis->a[i].cnt_1)); + if(dis->a[i].beg >= med) break; + + t1 += x*x; + t2 += x; + t3 += x*y; + t4 += y; + m++; + } + + if(i < dis->n) + { + uint64_t beg, end, cnt_0, cnt_1; + for (beg = dis->a[i].beg, end = dis->a[i].end, cnt_0 = cnt_1 = 0; i < dis->n; i++) + { + cnt_0 += dis->a[i].cnt_0; + cnt_1 += dis->a[i].cnt_1; + beg = MIN(beg, dis->a[i].beg); + end = MAX(end, dis->a[i].end); + } + + x = ((double)(beg + end))/2; + y = ((double)(cnt_1))/((double)(cnt_0 + cnt_1)); + + t1 += x*x; + t2 += x; + t3 += x*y; + t4 += y; + m++; + } + + + + if(t2 > t4) + { + idx->frac = t2/t4; + if(idx->frac > SCAL_RATE) idx->frac = idx->frac / SCAL_RATE; + if(idx->frac < 1) idx->frac = 1; + } + + ava_size = m; + t1 /= (idx->frac*idx->frac); + t2 /= idx->frac; + t3 /= idx->frac; + if((t1*ava_size - t2*t2) != 0) + { + idx->a = (t3*ava_size - t2*t4) / (t1*ava_size - t2*t2); + } + if((t1*ava_size - t2*t2) != 0) + { + idx->b = (t1*t4 - t2*t3) / (t1*ava_size - t2*t2); + } +} + + +void weight_edges(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub) +{ + uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; + hc_edge *e1 = NULL, *e2 = NULL; + long double weight; + + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[k].weight = 0; + } + } + + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>shif); + end = ((hits->a.a[k].e<<1)>>shif); + + if(beg == end) continue; + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + t_d = get_hic_distance(&(hits->a.a[k]), link, idx); + if(t_d == (uint64_t)-1) continue; + + e1 = get_hc_edge(link, beg, end, 0); + e2 = get_hc_edge(link, end, beg, 0); + if(e1 == NULL || e2 == NULL) continue; + weight = get_trans_weight(idx, t_d); + /*******************************for distance debug************************************/ + weight = 1; + /*******************************for distance debug************************************/ + + e1->weight += weight; e1->occ++; + e2->weight += weight; e2->occ++; + } +} + + +void weight_edges_advance(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, trans_idx* dis) +{ + uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; + hc_edge *e1 = NULL, *e2 = NULL; + long double weight; + + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[k].weight = 0; + } + } + + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>shif); + end = ((hits->a.a[k].e<<1)>>shif); + + if(beg == end) continue; + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + t_d = get_hic_distance(&(hits->a.a[k]), link, idx); + if(t_d == (uint64_t)-1) continue; + + e1 = get_hc_edge(link, beg, end, 0); + e2 = get_hc_edge(link, end, beg, 0); + if(e1 == NULL || e2 == NULL) continue; + weight = 1; + if(dis) + { + weight = get_trans_weight_advance(idx, t_d, dis); + } + + e1->weight += weight; e1->occ++; + e2->weight += weight; e2->occ++; + } +} + + +void get_bub_id(bubble_type* bub, uint32_t root, uint64_t* id0, uint64_t* id1, uint32_t check_het) +{ + if(id0) (*id0) = (uint64_t)-1; + if(id1) (*id1) = (uint64_t)-1; + uint64_t b_id0 = (uint64_t)-1, b_id1 = (uint64_t)-1; + uint32_t beg, sink; + + if((bub->b_s_idx.a[root]&0xffffffff) != 0xffffffff) + { + b_id0 = bub->b_s_idx.a[root]&0xffffffff; + if(check_het) + { + get_bubbles(bub, b_id0, &beg, &sink, NULL, NULL, NULL); + if(IF_HET(beg>>1, *bub) && IF_HET(sink>>1, *bub)) b_id0 = (uint64_t)-1; + } + } + + if((bub->b_s_idx.a[root]&0xffffffff00000000) != 0xffffffff00000000) + { + b_id1 = bub->b_s_idx.a[root]&0xffffffff00000000; b_id1 >>= 32; + if(check_het) + { + get_bubbles(bub, b_id1, &beg, &sink, NULL, NULL, NULL); + if(IF_HET(beg>>1, *bub) && IF_HET(sink>>1, *bub)) b_id1 = (uint64_t)-1; + } + } + + if(b_id0 == (uint64_t)-1 && b_id1 != (uint64_t)-1) + { + b_id0 = b_id1; + b_id1 = (uint64_t)-1; + } + + if(id0) (*id0) = b_id0; + if(id1) (*id1) = b_id1; +} + + +///return how many bubbles linked by this node +uint32_t connect_bub_occ(bubble_type* bub, uint32_t root_id, uint32_t check_het) +{ + uint64_t id0, id1, occ = 2; + get_bub_id(bub, root_id, &id0, &id1, check_het); + if(id0 == (uint64_t)-1) occ--; + if(id1 == (uint64_t)-1) occ--; + return occ; +} +///x_0 and x_1 are the ids of unitigs; +///x_0_b_id and x_1_b_id are the ids of bubble graph; +int ma_2_bub_arc(bubble_type* bub, uint32_t x_0, uint32_t* x_0_b_id, uint32_t x_1, uint32_t* x_1_b_id, +asg_arc_t *p, uint32_t check_het) +{ + uint64_t id0, ori_0, id1, ori_1, tmp_id; + uint32_t beg, sink, n, *a, x; + uint32_t beg_0, sink_0, beg_1, sink_1; + if(x_0_b_id) id0 = (*x_0_b_id); + if(x_1_b_id) id1 = (*x_1_b_id); + if((x_0 != (uint32_t)-1) && (x_1 != (uint32_t)-1)) + { + if(((x_0>>1) == (x_1>>1))) + { + if(x_0_b_id == NULL && x_1_b_id == NULL) + { + get_bub_id(bub, x_0>>1, &id0, &id1, check_het); + } + + get_bubbles(bub, id0, &beg_0, &sink_0, &a, &n, NULL); + get_bubbles(bub, id1, &beg_1, &sink_1, &a, &n, NULL); + + ori_0 = (uint64_t)-1; + if(x_0 == (beg_0^1)) + { + ori_0 = 1; + } + else if(x_0 == (sink_0^1)) + { + ori_0 = 0; + } + else if(x_0 == (beg_1^1)) + { + ori_0 = 1+2; + } + else if(x_0 == (sink_1^1)) + { + ori_0 = 0+2; + } + else + { + fprintf(stderr, "error 0\n"); + return 0; + } + + + ori_1 = (uint64_t)-1; + if(x_1 == (beg_0^1)) + { + ori_1 = 1; + } + else if(x_1 == (sink_0^1)) + { + ori_1 = 0; + } + else if(x_1 == (beg_1^1)) + { + ori_1 = 1 + 2; + } + else if(x_1 == (sink_1^1)) + { + ori_1 = 0 + 2; + } + else + { + fprintf(stderr, "error 1\n"); + return 0; + } + + + + if((((ori_0>>1)^(ori_1>>1))&1) != 1) + { + fprintf(stderr, "error 10\n"); + fprintf(stderr, "x_0: %u, id0: %lu, beg_0: %u, sink_0: %u, ori_0: %lu\n", + x_0, id0, beg_0, sink_0, ori_0); + fprintf(stderr, "x_1: %u, id1: %lu, beg_1: %u, sink_1: %u, ori_1: %lu\n", + x_1, id1, beg_1, sink_1, ori_1); + return 0; + } + + if(ori_0 & 2) + { + tmp_id = id0; id0 = id1; id1 = tmp_id; + } + + ori_0 &= 1; ori_1 &= 1; ori_1 ^= 1; + + + + p->ul = (id0<<1) | ori_0; p->ul <<= 32; p->ul += 0; + p->v = (id1<<1) | ori_1; + p->ol = 0; p->del = 0; p->el = p->no_l_indel = p->strong = 1; + } + else + { + if(x_0_b_id == NULL) get_bub_id(bub, x_0>>1, &id0, NULL, check_het); + if(x_1_b_id == NULL) get_bub_id(bub, x_1>>1, &id1, NULL, check_het); + + + get_bubbles(bub, id0, &beg, &sink, &a, &n, NULL); + ori_0 = (uint64_t)-1; + if(x_0 == (beg^1)) + { + ori_0 = 1; + } + else if(x_0 == (sink^1)) + { + ori_0 = 0; + } + else + { + fprintf(stderr, "error 0\n"); + return 0; + } + + + + get_bubbles(bub, id1, &beg, &sink, &a, &n, NULL); + ori_1 = (uint64_t)-1; + if(x_1 == (beg^1)) + { + ori_1 = 1; + } + else if(x_1 == (sink^1)) + { + ori_1 = 0; + } + else + { + fprintf(stderr, "error 1\n"); + return 0; + } + ori_0 &= 1; ori_1 &= 1; ori_1 ^= 1; + + p->ul = (id0<<1) | ori_0; p->ul <<= 32; p->ul += 0; + p->v = (id1<<1) | ori_1; + p->ol = 0; p->del = 0; p->el = p->no_l_indel = p->strong = 1; + } + } + else + { + x = (uint32_t)-1; + if(x_0 != (uint32_t)-1) x = x_0; + if(x_1 != (uint32_t)-1) x = x_1; + if(x == (uint32_t)-1) return 0; + if(x_0_b_id == NULL && x_1_b_id == NULL) + { + get_bub_id(bub, x>>1, &id0, &id1, check_het); + } + + if(id0 != (uint64_t)-1) + { + get_bubbles(bub, id0, &beg, &sink, &a, &n, NULL); + if(x == (beg^1)) + { + return 1; + } + else if(x == (sink^1)) + { + return 1; + } + + return 0; + } + + if(id1 != (uint64_t)-1) + { + get_bubbles(bub, id1, &beg, &sink, &a, &n, NULL); + if(x == (beg^1)) + { + return 1; + } + else if(x == (sink^1)) + { + return 1; + } + + return 0; + } + } + + return 1; +} + +#define arc_first(g, v) ((g)->arc[(g)->idx[(v)]>>32]) +#define arc_cnt(g, v) ((uint32_t)(g)->idx[(v)]) +void debug_bub_utg(bubble_type* bub, ma_ug_t *bug, asg_t *bsg, uint32_t check_het) +{ + uint32_t i, k, rId, rId_next, ori, ori_next, root, beg, end; + uint64_t id0, id1; + ma_utg_t *u = NULL; + asg_arc_t *t = NULL; + for (i = 0; i < bug->u.n; i++) + { + u = &(bug->u.a[i]); + if(u->n == 0) continue; + for (k = 0; k < u->n; k++) + { + if(k+1 >= u->n) continue; + rId = u->a[k]>>33; + ori = u->a[k]>>32&1; + get_bubbles(bub, rId, ori == 1?&root:NULL, ori == 0?&root:NULL, NULL, NULL, NULL); + + t = &(arc_first(bsg, u->a[k]>>32)); + get_bub_id(bub, root>>1, &id0, &id1, check_het); + if(id0 == (uint64_t)-1 || (t->el == 1 && id1 == (uint64_t)-1) || (t->el == 0 && id1 != (uint64_t)-1)) + { + fprintf(stderr, "sbsbsb0sbsbsb-utg%.6d, check_het: %u\n", (int)((root>>1)+1), check_het); + fprintf(stderr, "id0: %lu, id1: %lu, t->el: %u\n", id0, id1, t->el); + continue; + } + + ///fprintf(stderr, "aaaaaaaa10aaaaaaaa-utg%.6d\n", (int)((root>>1)+1)); + + rId_next = u->a[k+1]>>33; + ori_next = u->a[k+1]>>32&1; + + + get_bubbles(bub, rId, &beg, &end, NULL, NULL, NULL); + if(ori == 1) + { + if(root != beg) fprintf(stderr, "sbsbsb1sbsbsb, root: %u, beg: %u, end: %u\n", root, beg, end); + } + else + { + if(root != end) fprintf(stderr, "sbsbsb2sbsbsb, root: %u, beg: %u, end: %u\n", root, beg, end); + } + + if(t->el == 1) + { + get_bubbles(bub, rId_next, &beg, &end, NULL, NULL, NULL); + if(ori_next == 0) + { + if(root != (beg^1)) fprintf(stderr, "sbsbsb3sbsbsb, root: %u, beg: %u, end: %u\n", root, beg, end); + } + else + { + if(root != (end^1)) fprintf(stderr, "sbsbsb4sbsbsb, root: %u, beg: %u, end: %u\n", root, beg, end); + } + } + } + } + + fprintf(stderr, "[M::%s]\n", __func__); +} + +///just change the hap status of beg/sink, but they are are still at a chain of bubble +///might be ok +inline void set_bub_idx(bubble_type* bub, ma_utg_t *bu, asg_t *untig_sg, int beg_idx, int end_idx, +uint32_t is_to_hom, uint32_t check_het) +{ + int k; + uint32_t rId, ori, root; + uint64_t id0, id1, len0, len1; + for (k = beg_idx; k <= end_idx; k++) + { + rId = bu->a[k]>>33; + ori = bu->a[k]>>32&1; + get_bubbles(bub, rId, ori == 1?&root:NULL, ori == 0?&root:NULL, NULL, NULL, NULL); + + if(is_to_hom && IF_HOM(root>>1, *bub)) continue; + if(!is_to_hom && IF_HET(root>>1, *bub)) continue; + + + get_bub_id(bub, root>>1, &id0, &id1, check_het); + if(id0 == (uint64_t)-1 || id1 == (uint64_t)-1) continue; + get_bubbles(bub, id0, NULL, NULL, NULL, NULL, &len0); + get_bubbles(bub, id1, NULL, NULL, NULL, NULL, &len1); + + if(is_to_hom) + { + if(untig_sg->seq[root>>1].len > (MIN(len0, len1)*3)) continue; + bub->index[root>>1] = (uint32_t)-1; + } + else + { + bub->index[root>>1] = bub->f_bub+1; + } + + } +} + +void determine_bub_idx(bubble_type* bub, ma_utg_t *bu, asg_t *untig_sg, uint64_t pLen, +uint64_t rLEN, uint64_t r_hetLen, int beg_idx, int end_idx, uint32_t check_het) +{ + if(beg_idx > end_idx) return; + + uint64_t r_homLen = rLEN - r_hetLen; + if(pLen > 0 && rLEN > 0 && r_hetLen > 0 && rLEN < pLen*0.5 && r_hetLen < rLEN * 0.2) ///set het to hom + { + set_bub_idx(bub, bu, untig_sg, beg_idx, end_idx, 1, bub->check_het); + } + else if(pLen > 0 && rLEN > 0 && r_homLen > 0 && rLEN > pLen*0.9 && r_homLen < rLEN * 0.1) ///set hom to het + { + set_bub_idx(bub, bu, untig_sg, beg_idx, end_idx, 0, bub->check_het); + } +} + +void detect_bub_graph(bubble_type* bub, asg_t *untig_sg) +{ + asg_t *bg = bub->b_g; + ma_ug_t *ug = NULL; + ug = ma_ug_gen(bub->b_g); + ///debug_bub_utg(bub, ug, bg, bub->check_het); + uint32_t i, k, rId, ori, root, r_root; + int beg_idx, end_idx; + uint64_t pLen, rLEN, r_hetLen; + ma_utg_t *u = NULL; + asg_arc_t *t = NULL; + for (i = 0; i < ug->u.n; i++) + { + u = &(ug->u.a[i]); + if(u->n == 0) continue; + for (k = pLen = rLEN = r_hetLen = beg_idx = 0, end_idx = -1; k < u->n; k++) + { + rId = u->a[k]>>33; + ori = u->a[k]>>32&1; + get_bubbles(bub, rId, ori == 1?&root:&r_root, ori == 0?&root:&r_root, NULL, NULL, NULL); + + t = NULL; + if(k+1 < u->n) t = &(arc_first(bg, u->a[k]>>32)); + + pLen += bg->seq[rId].len; + if(end_idx < beg_idx) ///first bubble + { + pLen += untig_sg->seq[r_root>>1].len; + rLEN += untig_sg->seq[r_root>>1].len; + if(IF_HET(r_root>>1, *bub)) r_hetLen += untig_sg->seq[r_root>>1].len; + } + + if(t) + { + if(t->el == 0) + { + if(end_idx >= beg_idx) + { + pLen += untig_sg->seq[root>>1].len; + rLEN += untig_sg->seq[root>>1].len; + if(IF_HET(root>>1, *bub)) r_hetLen += untig_sg->seq[root>>1].len; + determine_bub_idx(bub, u, untig_sg, pLen, rLEN, r_hetLen, beg_idx, end_idx, bub->check_het); + } + + pLen = rLEN = r_hetLen = 0; + beg_idx = k + 1; end_idx = k; + } + else + { + pLen += t->ol; + rLEN += t->ol; + if(IF_HET(root>>1, *bub)) r_hetLen += t->ol; + end_idx = k; + } + } + } + + if(end_idx >= beg_idx) + { + pLen += untig_sg->seq[root>>1].len; + rLEN += untig_sg->seq[root>>1].len; + if(IF_HET(root>>1, *bub)) r_hetLen += untig_sg->seq[root>>1].len; + determine_bub_idx(bub, u, untig_sg, pLen, rLEN, r_hetLen, beg_idx, end_idx, bub->check_het); + } + } + + ma_ug_destroy(ug); +} + +void get_bub_graph(ma_ug_t* ug, bubble_type* bub) +{ + asg_t *sg = ug->g; + asg_arc_t t, *p = NULL; + pdq pq; + init_pdq(&pq, sg->n_seq<<1); + uint32_t n_vtx = sg->n_seq<<1, v, k; + uint32_t *pre = NULL; MALLOC(pre, n_vtx); + uint32_t pre_id, adjecent, bub_occ; + asg_t *bub_g = asg_init(); + for (v = 0; v < bub->f_bub; v++) + { + uint64_t pathbase; + uint32_t beg, sink; + get_bubbles(bub, v, &beg, &sink, NULL, NULL, &pathbase); + asg_seq_set(bub_g, v, pathbase, (bub->check_het && IF_HET(beg>>1, *bub) && IF_HET(sink>>1, *bub))?1:0); + bub_g->seq[v].c = PRIMARY_LABLE; + } + + //check all unitigs + for (v = 0; v < n_vtx; ++v) + { + if(sg->seq[v>>1].del) continue; + if(bub->b_s_idx.a[v>>1] == (uint64_t)-1) continue; ///if (v>>1) is not a beg or sink of bubbles + bub_occ = connect_bub_occ(bub, v>>1, bub->check_het); + if(bub_occ == 0) continue; + if(bub_occ == 2) + { + if(ma_2_bub_arc(bub, v, NULL, v^1, NULL, &t, bub->check_het)) + { + t.ol = sg->seq[v>>1].len; + p = asg_arc_pushp(bub_g); + *p = t; + } + continue; + } + if(ma_2_bub_arc(bub, v, NULL, (uint32_t)-1, NULL, &t, bub->check_het) == 0) continue; + + get_shortest_path(v, &pq, sg, pre); + for (k = 0; k < pq.dis.n; k++) + { + if(pq.dis.a[k] == (uint64_t)-1) continue; + if(bub->b_s_idx.a[k>>1] == (uint64_t)-1) continue; + if(connect_bub_occ(bub, k>>1, bub->check_het) == 0) continue; + if((k>>1) == (v>>1)) continue; + pre_id = pre[k]; + adjecent = 0; + while (pre_id != v) + { + if(connect_bub_occ(bub, pre_id>>1, bub->check_het) > 0) + { + adjecent = 1; + break; + } + pre_id = pre[pre_id]; + } + + if(adjecent == 0) + { + if(ma_2_bub_arc(bub, v, NULL, k^1, NULL, &t, bub->check_het)) + { + t.el = 0; t.ol = pq.dis.a[k] + sg->seq[k>>1].len; + p = asg_arc_pushp(bub_g); + *p = t; + } + } + } + } + + free(pre); + destory_pdq(&pq); + + asg_cleanup(bub_g); + bub_g->r_seq = bub_g->n_seq; + bub->b_g = bub_g; +} + +void print_bubble_chain(bubble_type* bub, const char* command) +{ + ma_ug_t *ug = NULL; + ug = ma_ug_gen(bub->b_g); + uint32_t i, k, j, rId, beg, sink, *a, n; + ma_utg_t *u = NULL; + asg_arc_t *t = NULL; + for (i = 0; i < ug->u.n; i++) + { + u = &(ug->u.a[i]); + if(u->n == 0) continue; + fprintf(stderr,"\n%s: chain-%u\n", command, i); + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + get_bubbles(bub, rId, &beg, &sink, &a, &n, NULL); + t = NULL; + if(k+1 < u->n) t = &(arc_first(bub->b_g, u->a[k]>>32)); + fprintf(stderr, "[utg%.6dl, utg%.6dl] el=%u no_long_indel=%u, rId=%u, nv: %u, nv^: %u\n", + (int)((beg>>1)+1), (int)((sink>>1)+1), t?t->el:16, t?t->no_l_indel:16, rId, arc_cnt(bub->b_g, u->a[k]>>32), arc_cnt(bub->b_g, (u->a[k]>>32)^1)); + // if((u->a[k]>>33) == 9658) + // { + // asg_arc_t *av; + // uint32_t nv, nv_i; + // av = asg_arc_a(bub->b_g, u->a[k]>>32); + // nv = asg_arc_n(bub->b_g, u->a[k]>>32); + // for (nv_i = 0; nv_i < nv; nv_i++) + // { + // if(av[nv_i].del) continue; + // fprintf(stderr, "v--->%u\n", av[nv_i].v>>1); + // } + + + + // av = asg_arc_a(bub->b_g, (u->a[k]>>32)^1); + // nv = asg_arc_n(bub->b_g, (u->a[k]>>32)^1); + // for (nv_i = 0; nv_i < nv; nv_i++) + // { + // if(av[nv_i].del) continue; + // fprintf(stderr, "v^1--->%u\n", av[nv_i].v>>1); + // } + + // } + if(bub->b_g->seq[rId].c == HAP_LABLE) + { + for (j = 0; j < n; j++) + { + fprintf(stderr, ">>>utg%.6dl\n", (int)((a[j]>>1)+1)); + } + } + } + } + + ma_ug_destroy(ug); +} + +int is_simple_broken_bubble(ma_ug_t *unitig_ug, uint32_t x, uint32_t beg, uint32_t sink, uint32_t* new_het) +{ + uint32_t nv, v = (uint32_t)-1, u_s = (uint32_t)-1, u_e = (uint32_t)-1, i; + asg_arc_t *av = NULL; + (*new_het) = (uint32_t)-1; + + if((asg_arc_n(unitig_ug->g, x) == 1) + && (asg_arc_n(unitig_ug->g, x^1) == 0)) + { + v = x; + } + + if((asg_arc_n(unitig_ug->g, x^1) == 1) + && (asg_arc_n(unitig_ug->g, x) == 0)) + { + v = x^1; + } + + if(v == (uint32_t)-1) return 0; + + + av = asg_arc_a(unitig_ug->g, v); + nv = asg_arc_n(unitig_ug->g, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + if((av[i].v>>1) == (beg>>1)) u_s = beg, u_e = sink; + if((av[i].v>>1) == (sink>>1)) u_s = sink, u_e = beg; + } + + if(u_s == (uint32_t)-1 || u_e == (uint32_t)-1) return 0; + + av = asg_arc_a(unitig_ug->g, u_s); + nv = asg_arc_n(unitig_ug->g, u_s); + if(nv != 2) return 0; + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + if(av[i].v == (v^1)) continue; + if(av[i].v == (u_e^1)) + { + (*new_het) = u_e; + return 1; + } + } + + return 0; +} + +int double_check_broken_bubble(asg_t *g, kvec_t_u32_warp* broken, uint32_t beg, uint32_t sink, +uint8_t* vis_flag, uint32_t vis_flag_n, kvec_t_u32_warp* stack, asg_t *bsg, asg_arc_t *p_t) +{ + uint32_t cur, ncur, i, n, pre, occ; + radix_sort_u32(broken->a.a, broken->a.a + broken->a.n); + for (i = n = 0, pre = (uint32_t)-1; i < broken->a.n; i++) + { + if((broken->a.a[i]>>1) == (pre>>1)) continue; + pre = broken->a.a[i]; + broken->a.a[n] = pre; + n++; + } + broken->a.n = n; + + asg_arc_t *acur = NULL; + memset(vis_flag, 0, vis_flag_n); + stack->a.n = 0; + kv_push(uint32_t, stack->a, beg); + occ = 0; + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if(vis_flag[cur] == 0 && vis_flag[cur^1] == 0) occ++; + if(vis_flag[cur] == 0 && cur != (beg^1) && (sink == (uint32_t)-1 || cur != (sink^1))) + { + vis_flag[cur] = 1; + ncur = asg_arc_n(g, cur); + acur = asg_arc_a(g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + vis_flag[cur] = 1; + + + cur^=1; + if(vis_flag[cur] == 0 && cur != (beg^1) && (sink == (uint32_t)-1 || cur != (sink^1))) + { + vis_flag[cur] = 1; + ncur = asg_arc_n(g, cur); + acur = asg_arc_a(g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(vis_flag[acur[i].v]) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + vis_flag[cur] = 1; + } + + n = broken->a.n; + if(beg != (uint32_t)-1) n++; + if(sink != (uint32_t)-1) n++; + + if(occ > n) + { + ///fprintf(stderr, "\n+++++sb+++++beg-utg%.6ul, sink-utg%.6ul, occ: %u, n: %u\n", (beg>>1)+1, (sink>>1)+1, occ, n); + if(bsg && p_t && beg != (uint32_t)-1 && sink != (uint32_t)-1) + { + p_t->del = 1; + asg_arc_del(bsg, (p_t->v)^1, (p_t->ul>>32)^1, 1); + } + /*******************************for debug************************************/ + // memset(vis_flag, 0, vis_flag_n); + // stack->a.n = 0; + // kv_push(uint32_t, stack->a, beg); + // occ = 0; + // while (stack->a.n > 0) + // { + // occ++; + // stack->a.n--; + // cur = stack->a.a[stack->a.n]; + + // fprintf(stderr, "cur-utg%.6ul\n", (cur>>1)+1); + + // vis_flag[cur] = 1; + // if(cur == (beg^1) || cur == (sink^1)) continue; + // ncur = asg_arc_n(g, cur); + // acur = asg_arc_a(g, cur); + // for (i = 0; i < ncur; i++) + // { + // if(acur[i].del) continue; + // if(vis_flag[acur[i].v]) continue; + // kv_push(uint32_t, stack->a, acur[i].v); + // } + + // cur^=1; + // if(vis_flag[cur]) continue; + // vis_flag[cur] = 1; + // if(cur == (beg^1) || cur == (sink^1)) continue; + // ncur = asg_arc_n(g, cur); + // acur = asg_arc_a(g, cur); + // for (i = 0; i < ncur; i++) + // { + // if(acur[i].del) continue; + // if(vis_flag[acur[i].v]) continue; + // kv_push(uint32_t, stack->a, acur[i].v); + // } + // } + + // for (i = 0; i < broken->a.n; i++) + // { + // fprintf(stderr, "*****cur-utg%.6ul\n", (broken->a.a[i]>>1)+1); + // } + /*******************************for debug************************************/ + return 0; + } + + return 1; +} + +int is_local_simple_circle(asg_t *g, uint32_t v) +{ + if(asg_arc_n(g, v) != asg_arc_n(g, v^1)) return 0; + if(asg_arc_n(g, v) == 1) v = arc_first(g, v).v; + if(asg_arc_n(g, v) != asg_arc_n(g, v^1)) return 0; + if(asg_arc_n(g, v) != 2) return 0; + uint32_t ncur, i, u; + asg_arc_t *acur = NULL; + ncur = asg_arc_n(g, v); + acur = asg_arc_a(g, v); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + u = acur[i].v; + if(asg_arc_n(g, u) != 1 || asg_arc_n(g, u^1) != 1) continue; + if(arc_first(g, u).v != v) continue; + return 1; + } + return 0; +} + +void update_bub_b_s_idx(bubble_type* bub) +{ + memset(bub->b_s_idx.a, -1, bub->b_s_idx.n * sizeof(uint64_t)); + uint32_t i, v, beg, sink, n_bub = bub->num.n - 1; + for (i = 0; i < n_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, NULL, NULL, NULL); + + if(beg != (uint32_t)-1) + { + v = beg>>1; + if(bub->b_s_idx.a[v] == (uint64_t)-1) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + else if((bub->b_s_idx.a[v] & 0xffffffff00000000) == 0xffffffff00000000) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + } + + + + if(sink != (uint32_t)-1) + { + v = sink>>1; + if(bub->b_s_idx.a[v] == (uint64_t)-1) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + else if((bub->b_s_idx.a[v] & 0xffffffff00000000) == 0xffffffff00000000) + { + bub->b_s_idx.a[v] <<= 32; + bub->b_s_idx.a[v] |= i; + } + } + } +} + +void update_bubble_graph(kvec_t_u32_warp* broken, uint32_t beg, uint32_t beg_bub_id, +uint32_t sink, uint32_t sink_bub_id, bubble_type* bub, kvec_asg_arc_t_warp* edges, asg_t *bsg, +asg_arc_t *p_t, uint8_t *bsg_idx, ma_ug_t *unitig_ug, uint64_t* occ_thres, uint64_t is_b_bub) +{ + uint32_t i, pre, n, bub_id, v; + uint64_t occ; + asg_arc_t t_f, t_r; + radix_sort_u32(broken->a.a, broken->a.a + broken->a.n); + for (i = n = occ = 0, pre = (uint32_t)-1; i < broken->a.n; i++) + { + if((broken->a.a[i]>>1) == (pre>>1)) continue; + if(IF_HOM((broken->a.a[i]>>1), *bub)) + { + if(is_local_simple_circle(unitig_ug->g, broken->a.a[i])) + { + bub->index[broken->a.a[i]>>1] = bub->f_bub+1; + } + else + { + continue; + } + } + + pre = broken->a.a[i]; + broken->a.a[n] = pre; + occ += unitig_ug->u.a[broken->a.a[n]>>1].n; + n++; + } + broken->a.n = n; + ///if(broken->a.n == 0) return; + if(broken->a.n == 1) + { + ///fprintf(stderr, "+++++sb+++++utg%.6ul\n", (broken->a.a[0]>>1)+1); + if(beg != (uint32_t)-1 && sink != (uint32_t)-1 && + is_simple_broken_bubble(unitig_ug, broken->a.a[0], beg, sink, &v)) + { + if((v>>1) != (broken->a.a[0]>>1)) + { + ///fprintf(stderr, "-----sb-----utg%.6ul\n", (v>>1)+1); + kv_push(uint32_t, broken->a, v); + occ += unitig_ug->u.a[v>>1].n; + if(occ_thres && occ > (*occ_thres)) return; + bub->index[v>>1] = bub->f_bub+1; ///set to het + } + } + } + + + if(occ_thres && occ > (*occ_thres)) return; + /********************push graph node********************/ + bub_id = bub->b_g->n_seq; + asg_seq_set(bub->b_g, bub_id, 0, 0); + bub->b_g->seq[bub_id].c = HAP_LABLE; + if(is_b_bub) bub->b_bub++; + /********************push graph node********************/ + + /********************push bubble********************/ + kv_push(uint32_t, bub->num, bub->list.n); + kv_push(uint64_t, bub->pathLen, 0); + kv_push(uint32_t, bub->list, beg); + kv_push(uint32_t, bub->list, sink); + + for (i = 0; i < broken->a.n; i++) + { + kv_push(uint32_t, bub->list, broken->a.a[i]); + if(bsg_idx) bsg_idx[broken->a.a[i]>>1] = 1; + } + /********************push bubble********************/ + if(beg != (uint32_t)-1) ///beg_bub_id ----> bub_id + { + if(ma_2_bub_arc(bub, beg, &beg_bub_id, beg^1, &bub_id, &t_f, bub->check_het) && + ma_2_bub_arc(bub, beg^1, &bub_id, beg, &beg_bub_id, &t_r, bub->check_het)) + { + t_f.el = 0; t_f.no_l_indel = 0; t_f.del = 0; + kv_push(asg_arc_t, edges->a, t_f); + + t_r.el = 0; t_r.no_l_indel = 0; t_r.del = 0; + kv_push(asg_arc_t, edges->a, t_r); + } + } + + if(sink != (uint32_t)-1) ///bub_id ----> sink_bub_id + { + if(ma_2_bub_arc(bub, sink^1, &bub_id, sink, &sink_bub_id, &t_f, bub->check_het) && + ma_2_bub_arc(bub, sink, &sink_bub_id, sink^1, &bub_id, &t_r, bub->check_het)) + { + t_f.el = 0; t_f.no_l_indel = 0; t_f.del = 0; + kv_push(asg_arc_t, edges->a, t_f); + + t_r.el = 0; t_r.no_l_indel = 0; t_r.del = 0; + kv_push(asg_arc_t, edges->a, t_r); + } + } + + if(beg != (uint32_t)-1 && sink != (uint32_t)-1 && p_t) + { + p_t->del = 1; + asg_arc_del(bsg, (p_t->v)^1, (p_t->ul>>32)^1, 1); + } +} + +void get_related_bub_nodes(kvec_t_u32_warp* broken, bubble_type* bub, pdq* pq, asg_t *unitig_g, + uint32_t *pre, uint32_t src, uint32_t dest, uint8_t *bsg_idx) +{ + uint32_t j_i, pre_id, adjecent; + src ^= 1; + get_shortest_path(src, pq, unitig_g, pre); + + for (j_i = 0; j_i < pq->dis.n; j_i++) + { + if(pq->dis.a[j_i] == (uint64_t)-1) continue; + ///if(IF_HOM(j_i>>1, *bub)) continue; + if((j_i>>1) == (src>>1)) continue; + if((dest != (uint32_t)-1) && ((j_i>>1) == (dest>>1))) continue; + + pre_id = pre[j_i]; + adjecent = 0; + while (pre_id != src) + { + if(((dest != (uint32_t)-1) && ((pre_id>>1) == (dest>>1))) + || ((pre_id>>1) == (src>>1))) + { + adjecent = 1; + break; + } + pre_id = pre[pre_id]; + } + + if(adjecent == 0) + { + if(broken->a.n == 0 || (broken->a.n > 0 && (j_i>>1) != (broken->a.a[broken->a.n - 1]>>1))) + { + if(bsg_idx && bsg_idx[(j_i>>1)]) + { + broken->a.n = 0; + return; + } + kv_push(uint32_t, broken->a, j_i); + } + } + } +} + +uint64_t calculate_chain_weight(ma_utg_t *u, bubble_type* bub, ma_ug_t *unitig_ug, chain_w_type* x) +{ + x->b_occ = x->g_occ = 0; + uint32_t i, j, *a, n; + uint64_t occ, occ_n, thres; + for (i = occ = occ_n = 0; i < u->n; i++) + { + if(bub->b_g->seq[u->a[i]>>33].c != HAP_LABLE) + { + get_bubbles(bub, u->a[i]>>33, NULL, NULL, &a, &n, NULL); + for (j = 0; j < n; j++) + { + occ += unitig_ug->u.a[a[j]>>1].n; + } + occ_n++; + } + } + + thres = (uint64_t)-1; + if(occ_n > 0) thres = (occ*6)/occ_n; + + for (i = 0; i < u->n; i++) + { + occ = 0; + get_bubbles(bub, u->a[i]>>33, NULL, NULL, &a, &n, NULL); + for (j = 0; j < n; j++) + { + occ += unitig_ug->u.a[a[j]>>1].n; + } + + if(bub->b_g->seq[u->a[i]>>33].c != HAP_LABLE || occ < thres) + { + x->g_occ += occ; + } + else + { + x->b_occ += occ; + } + } + + return thres; +} + +int cmp_chain_weight(const void * a, const void * b) +{ + if((*(chain_w_type*)a).del != (*(chain_w_type*)b).del) + { + return (*(chain_w_type*)a).del > (*(chain_w_type*)b).del? 1 : -1; + } + else + { + long long a_occ = (*(chain_w_type*)a).g_occ - (*(chain_w_type*)a).b_occ; + long long b_occ = (*(chain_w_type*)b).g_occ - (*(chain_w_type*)b).b_occ; + if(a_occ != b_occ) + { + return a_occ > b_occ? -1 : 1; + } + else + { + return 0; + } + } +} + +void resolve_bubble_chain_tangle_back(ma_ug_t* ug, bubble_type* bub, hc_links* link) +{ + ma_ug_t *copy_ug = copy_untig_graph(bub->b_ug); + asg_arc_t *av = NULL; + uint32_t i, j, k, v, w, w1, w2, nw1, nw2, nv, occ_e_1, occ_e_2, occ_c; + ma_ug_t *bub_ug = copy_ug; + ///ma_utg_t *u = NULL; + buf_t b; memset(&b, 0, sizeof(buf_t)); + kvec_t_u32_warp stack, result; + kv_init(stack.a); kv_init(result.a); + uint8_t *vis = NULL; CALLOC(vis, ug->g->n_seq<<1); + uint8_t *is_vis = NULL; CALLOC(is_vis, ug->g->n_seq<<1); + kvec_t(uint64_t) occ_idx; kv_init(occ_idx); uint64_t tmp, *p = NULL; + + for (k = occ_idx.n = 0; k < bub_ug->g->n_seq; k++) + { + v = (k<<1); + av = asg_arc_a(bub_ug->g, v); + nv = asg_arc_n(bub_ug->g, v); + for (i = 0, w = (uint32_t)-1, nw1 = 0; i < nv; i++) + { + if(av[i].del) continue; + nw1++; + if((av[i].v>>1) == (v>>1)) continue; + if(w != (uint32_t)-1) break; + w = av[i].v; + } + if(i < nv) continue; + w1 = w; + + + v = (k<<1)+1; + av = asg_arc_a(bub_ug->g, v); + nv = asg_arc_n(bub_ug->g, v); + for (i = 0, w = (uint32_t)-1, nw2 = 0; i < nv; i++) + { + if(av[i].del) continue; + nw2++; + if((av[i].v>>1) == (v>>1)) continue; + if(w != (uint32_t)-1) break; + w = av[i].v; + } + if(i < nv) continue; + w2 = w; + + if(nw1 <= 1 && nw2 <= 1) continue; + if(w1 == (uint32_t)-1 && w2 == (uint32_t)-1) continue; + if(w1 != (uint32_t)-1) w1 ^=1; + if(w2 != (uint32_t)-1) w2 ^=1; + + if(w1 != (uint32_t)-1) + { + w = (uint32_t)-1; + if(w2 != (uint32_t)-1) w = w2^1; + + av = asg_arc_a(bub_ug->g, w1); + nv = asg_arc_n(bub_ug->g, w1); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + if((av[i].v>>1) == k) continue; + if(av[i].v == w) continue; + break; + } + if(i < nv) continue; + } + + + if(w2 != (uint32_t)-1) + { + w = (uint32_t)-1; + if(w1 != (uint32_t)-1) w = w1^1; + + av = asg_arc_a(bub_ug->g, w2); + nv = asg_arc_n(bub_ug->g, w2); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + if((av[i].v>>1) == k) continue; + if(av[i].v == w) continue; + break; + } + if(i < nv) continue; + } + + + if(w1 == (uint32_t)-1 && w2 != (uint32_t)-1) w1 = w2; + if(w1 == w2) w2 = (uint32_t)-1; + + occ_c = occ_e_1 = occ_e_2 = (uint32_t)-1; + set_b_utg_weight_flag(bub, &b, k<<1, NULL, 0, &occ_c); + if(w1 != (uint32_t)-1) set_b_utg_weight_flag(bub, &b, w1^1, NULL, 0, &occ_e_1); + if(w2 != (uint32_t)-1) set_b_utg_weight_flag(bub, &b, w2^1, NULL, 0, &occ_e_2); + + fprintf(stderr, "\n>>>>>>k=btg%.6ul (n=%u), w1=btg%.6ul (n=%u), w2=utg%.6ul (n=%u)\n", k+1, occ_c, + (w1>>1)+1, occ_e_1, (w2>>1)+1, occ_e_2); + + if(occ_c*5 >= occ_e_1) continue; + if(occ_c*5 >= occ_e_2) continue; + if(occ_c*10 >= (occ_e_1 + occ_e_2)) continue; + kv_pushp(uint64_t, occ_idx, &p); + (*p) = occ_e_1 + occ_e_2 - occ_c; + (*p) <<= 32; (*p) += k; + fprintf(stderr, "passed\n"); + } + + radix_sort_hc64(occ_idx.a, occ_idx.a + occ_idx.n); + + for (k = 0; k < occ_idx.n; ++k) + { + tmp = occ_idx.a[k]; + occ_idx.a[k] = occ_idx.a[occ_idx.n - k - 1]; + occ_idx.a[occ_idx.n - k - 1] = tmp; + } + + for (j = 0; j < occ_idx.n; j++) + { + k = (uint32_t)occ_idx.a[j]; + + v = (k<<1); + av = asg_arc_a(bub_ug->g, v); + nv = asg_arc_n(bub_ug->g, v); + for (i = 0, w = (uint32_t)-1, nw1 = 0; i < nv; i++) + { + if(av[i].del) continue; + nw1++; + if((av[i].v>>1) == (v>>1)) continue; + if(w != (uint32_t)-1) break; + w = av[i].v; + } + if(i < nv) continue; + w1 = w; + + + v = (k<<1)+1; + av = asg_arc_a(bub_ug->g, v); + nv = asg_arc_n(bub_ug->g, v); + for (i = 0, w = (uint32_t)-1, nw2 = 0; i < nv; i++) + { + if(av[i].del) continue; + nw2++; + if((av[i].v>>1) == (v>>1)) continue; + if(w != (uint32_t)-1) break; + w = av[i].v; + } + if(i < nv) continue; + w2 = w; + + if(nw1 <= 1 && nw2 <= 1) continue; + if(w1 == (uint32_t)-1 && w2 == (uint32_t)-1) continue; + if(w1 != (uint32_t)-1) w1 ^=1; + if(w2 != (uint32_t)-1) w2 ^=1; + + + + } + + free(vis); free(is_vis); free(b.b.a); kv_destroy(occ_idx); kv_destroy(stack.a); kv_destroy(result.a); + ma_ug_destroy(copy_ug); +} + +uint32_t get_btg_occ(bubble_type* bub, uint32_t v) +{ + ma_ug_t *bub_ug = bub->b_ug; + ma_utg_t *u = NULL; + uint32_t k_i, k_j, *a = NULL, n, tan_occ = 0; + + u = &(bub_ug->u.a[v]); + for (k_i = 0; k_i < u->n; k_i++) + { + get_bubbles(bub, u->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + for (k_j = 0; k_j < n; k_j++) + { + tan_occ += bub->ug->u.a[a[k_j]>>1].n; + } + } + return tan_occ; +} + +int check_bubble_tangle(bubble_type* bub, ma_ug_t* ug, uint32_t beg, uint32_t sink, +double side_rate, double total_rate, uint32_t beg_occ, uint32_t sink_occ, +uint8_t* is_vis, kvec_t_u32_warp* stack, kvec_t_u32_warp* res, uint8_t* chain_flag, +uint32_t* extra_check) +{ + if(extra_check) (*extra_check) = 1; + uint32_t cur, tan_occ = 0, ncur, i, no_first = 0; + asg_arc_t *acur = NULL; + memset(is_vis, 0, ug->g->n_seq<<1); + stack->a.n = 0; + kv_push(uint32_t, stack->a, beg); + if(res) res->a.n = 0; + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + // if((beg>>1) == 162) + // { + // fprintf(stderr, ">>>###0###>>>beg=btg%.6ul, beg&1: %u, cur=btg%.6ul, cur&1: %u, sink=btg%.6ul, sink&1: %u, tan_occ: %u\n", + // (beg>>1)+1, beg&1, (cur>>1)+1, cur&1, (sink>>1)+1, sink&1, tan_occ); + // } + + + if(no_first && cur == beg) return 0; + if(sink != (uint32_t)-1 && cur == sink) return 0; + + + if(is_vis[cur] == 0 && is_vis[cur^1] == 0) + { + if((cur>>1) != (beg>>1) && (sink == (uint32_t)-1 || (cur>>1) != (sink>>1))) + { + if(res) kv_push(uint32_t, res->a, cur); + if(chain_flag && chain_flag[cur>>1] != 0 && extra_check) + { + (*extra_check) = 0; + } + + if(bub) + { + tan_occ += get_btg_occ(bub, cur>>1); + if(tan_occ*side_rate >= beg_occ) return 0; + if(sink != (uint32_t)-1 && (tan_occ*side_rate >= sink_occ)) return 0; + if(tan_occ*total_rate >= (beg_occ + ((sink != (uint32_t)-1)?sink_occ : 0))) return 0; + } + } + } + + + if(is_vis[cur] == 0 && cur != (beg^1) && (sink == (uint32_t)-1 || cur != (sink^1))) + { + is_vis[cur] = 1; + ncur = asg_arc_n(ug->g, cur); + acur = asg_arc_a(ug->g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + + if(acur[i].v == beg) return 0; + if(sink != (uint32_t)-1 && acur[i].v == sink) return 0; + + if(is_vis[acur[i].v]) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + is_vis[cur] = 1; + + + cur^=1; + if(is_vis[cur] == 0 && cur != (beg^1) && (sink == (uint32_t)-1 || cur != (sink^1))) + { + is_vis[cur] = 1; + ncur = asg_arc_n(ug->g, cur); + acur = asg_arc_a(ug->g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + + if(acur[i].v == beg) return 0; + if(sink != (uint32_t)-1 && acur[i].v == sink) return 0; + + if(is_vis[acur[i].v]) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + is_vis[cur] = 1; + no_first = 1; + } + + if(bub) + { + if(tan_occ*side_rate >= beg_occ) return 0; + if(sink != (uint32_t)-1 && (tan_occ*side_rate >= sink_occ)) return 0; + if(tan_occ*total_rate >= (beg_occ + ((sink != (uint32_t)-1)?sink_occ : 0))) return 0; + } + + return 1; +} +int find_bubble_tangle(bubble_type* bub, ma_ug_t* ug, uint8_t* is_vis, uint8_t* is_vis2, +uint32_t v, double side_rate, double total_rate, kvec_t_u32_warp* stack, +kvec_t_u32_warp* stack2, kvec_t_u32_warp* res_btg, kvec_t_u32_warp* res_utg, uint8_t* chain_flag, +uint32_t* r_b_utg_beg, uint32_t* r_b_utg_sink, uint32_t* r_b_tg_beg, uint32_t* r_b_tg_sink, +uint32_t* r_utg_beg, uint32_t* r_utg_sink) +{ + (*r_b_utg_beg) = (*r_b_utg_sink) = (*r_b_tg_beg) = (*r_b_tg_sink) = (*r_utg_beg) = (*r_utg_sink) = (uint32_t)-1; + ma_ug_t *bub_ug = bub->b_ug; + ma_utg_t *u = NULL; + uint32_t tan_occ = 0, cur, ncur, i, k, no_root = 0, v_occ, c_occ, utg_occ, w, btg_beg, btg_sink, utg_beg, utg_sink, is_t, extra_check; + stack->a.n = 0; + asg_arc_t *acur = NULL; + + memset(is_vis, 0, bub_ug->g->n_seq<<1); + stack->a.n = 0; + kv_push(uint32_t, stack->a, v); + v_occ = get_btg_occ(bub, v>>1); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if(is_vis[cur]) continue; + + c_occ = 0; + if(no_root && cur == v) return 0; + if(no_root && is_vis[cur] == 0 && is_vis[cur^1] == 0) + { + c_occ = get_btg_occ(bub, cur>>1); + if((tan_occ*side_rate) < c_occ && (tan_occ*total_rate) < (c_occ + v_occ)) + { + if(check_bubble_tangle(bub, bub->b_ug, v, cur^1, side_rate, total_rate, v_occ, c_occ, + is_vis2, stack2, NULL, NULL, NULL)) + { + check_bubble_tangle(bub, bub->b_ug, v, cur^1, side_rate, total_rate, v_occ, c_occ, + is_vis2, stack2, res_btg, NULL, NULL); + + for (k = 0; k < res_btg->a.n; k++) + { + set_b_utg_weight_flag(bub, NULL, res_btg->a.a[k], chain_flag, 0, NULL); + } + btg_beg = v; btg_sink = cur^1; + + u = &(bub_ug->u.a[btg_beg>>1]); + if((btg_beg&1)==1) + { + get_bubbles(bub, (u->a[0]>>32)>>1, (((u->a[0]>>32)&1)^1)==1?&w:NULL, + (((u->a[0]>>32)&1)^1) == 0?&w:NULL, NULL, NULL, NULL); + (*r_b_tg_beg) = u->a[0]>>32; + } + else + { + get_bubbles(bub, (u->a[u->n-1]>>32)>>1, ((u->a[u->n-1]>>32)&1)==1?&w:NULL, + ((u->a[u->n-1]>>32)&1) == 0?&w:NULL, NULL, NULL, NULL); + (*r_b_tg_beg) = u->a[u->n-1]>>32; + } + utg_beg = w^1; + + + u = &(bub_ug->u.a[btg_sink>>1]); + if((btg_sink&1)==1) + { + get_bubbles(bub, (u->a[0]>>32)>>1, (((u->a[0]>>32)&1)^1)==1?&w:NULL, + (((u->a[0]>>32)&1)^1) == 0?&w:NULL, NULL, NULL, NULL); + (*r_b_tg_sink) = u->a[0]>>32; + } + else + { + get_bubbles(bub, (u->a[u->n-1]>>32)>>1, ((u->a[u->n-1]>>32)&1)==1?&w:NULL, + ((u->a[u->n-1]>>32)&1) == 0?&w:NULL, NULL, NULL, NULL); + (*r_b_tg_sink) = u->a[u->n-1]>>32; + } + utg_sink = w^1; + + // fprintf(stderr, "\nbtg_beg=btg%.6ul, utg_beg=utg%.6ul\n", + // (btg_beg>>1)+1, (utg_beg>>1)+1); + // fprintf(stderr, "btg_sink=btg%.6ul, utg_sink=utg%.6ul\n", + // (btg_sink>>1)+1, (utg_sink>>1)+1); + + is_t = check_bubble_tangle(NULL, ug, utg_beg, utg_sink, side_rate, total_rate, + (uint32_t)-1, (uint32_t)-1, is_vis2, stack2, res_utg, chain_flag, &extra_check); + + if(is_t == 1 && extra_check == 0) + { + for (k = utg_occ = 0; k < res_utg->a.n; k++) + { + if(IF_HOM((res_utg->a.a[k]>>1), *bub)) continue; + utg_occ += ug->u.a[res_utg->a.a[k]>>1].n; + } + + if(utg_occ*total_rate >= (v_occ+c_occ)) is_t = 0; + } + + for (k = 0; k < res_btg->a.n; k++) + { + set_b_utg_weight_flag(bub, NULL, res_btg->a.a[k], chain_flag, 1, NULL); + } + + if(is_t) + { + (*r_b_utg_beg) = btg_beg; + (*r_b_utg_sink) = btg_sink; + (*r_utg_beg) = utg_beg; + (*r_utg_sink) = utg_sink; + return is_t; + } + } + } + } + + is_vis[cur] = 1; + if(cur != (v^1)) + { + ncur = asg_arc_n(bub_ug->g, cur); + acur = asg_arc_a(bub_ug->g, cur); + for (i = 0; i < ncur; i++) + { + if(acur[i].del) continue; + if(acur[i].v == v) return 0; + if(is_vis[acur[i].v]) continue; + kv_push(uint32_t, stack->a, acur[i].v); + } + } + + + if(no_root) tan_occ += c_occ; + if((tan_occ*side_rate) >= v_occ) return 0; + no_root = 1; + } + + if(tan_occ*side_rate >= v_occ) return 0; + if(tan_occ*total_rate >= v_occ) return 0; + + if(check_bubble_tangle(bub, bub->b_ug, v, (uint32_t)-1, side_rate, total_rate, v_occ, (uint32_t)-1, + is_vis2, stack2, res_btg, NULL, NULL) == 0) + { + return 0; + } + + for (k = 0; k < res_btg->a.n; k++) + { + set_b_utg_weight_flag(bub, NULL, res_btg->a.a[k], chain_flag, 0, NULL); + } + + btg_beg = v; + u = &(bub_ug->u.a[btg_beg>>1]); + if((btg_beg&1)==1) + { + get_bubbles(bub, (u->a[0]>>32)>>1, (((u->a[0]>>32)&1)^1)==1?&w:NULL, + (((u->a[0]>>32)&1)^1) == 0?&w:NULL, NULL, NULL, NULL); + (*r_b_tg_beg) = u->a[0]>>32; + } + else + { + get_bubbles(bub, (u->a[u->n-1]>>32)>>1, ((u->a[u->n-1]>>32)&1)==1?&w:NULL, + ((u->a[u->n-1]>>32)&1) == 0?&w:NULL, NULL, NULL, NULL); + (*r_b_tg_beg) = u->a[u->n-1]>>32; + } + utg_beg = w^1; + + is_t = check_bubble_tangle(NULL, ug, utg_beg, (uint32_t)-1, side_rate, total_rate, + (uint32_t)-1, (uint32_t)-1, is_vis2, stack2, res_utg, chain_flag, &extra_check); + + if(is_t == 1 && extra_check == 0) + { + for (k = utg_occ = 0; k < res_utg->a.n; k++) + { + if(IF_HOM((res_utg->a.a[k]>>1), *bub)) continue; + utg_occ += ug->u.a[res_utg->a.a[k]>>1].n; + } + + if(utg_occ*total_rate >= v_occ) is_t = 0; + } + + for (k = 0; k < res_btg->a.n; k++) + { + set_b_utg_weight_flag(bub, NULL, res_btg->a.a[k], chain_flag, 1, NULL); + } + + if(is_t) + { + (*r_b_utg_beg) = btg_beg; + (*r_utg_beg) = utg_beg; + } + return is_t; +} + +uint32_t get_utg_end_from_btg(bubble_type* bub, ma_ug_t *bub_ug, uint32_t v) +{ + ma_utg_t *u = &(bub_ug->u.a[v>>1]); + + if((v&1)==1) + { + return (u->a[0]>>32)^1; + } + else + { + return u->a[u->n-1]>>32; + } +} + +void drop_g_edges_by_utg(bubble_type* bub, asg_t *bsg, ma_ug_t *bub_ug, kvec_t_u32_warp* res_btg, +uint32_t b_utg_beg, uint32_t b_utg_sink) +{ + uint32_t i, k, v, root, nv; + asg_arc_t *av = NULL; + if(b_utg_beg != (uint32_t)-1) + { + root = b_utg_beg; + v = get_utg_end_from_btg(bub, bub_ug, root); + nv = asg_arc_n(bsg, v); + av = asg_arc_a(bsg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + av[i].del = 1; + asg_arc_del(bsg, (av[i].v)^1, (av[i].ul>>32)^1, 1); + } + } + + if(b_utg_sink != (uint32_t)-1) + { + root = b_utg_sink; + v = get_utg_end_from_btg(bub, bub_ug, root); + nv = asg_arc_n(bsg, v); + av = asg_arc_a(bsg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + av[i].del = 1; + asg_arc_del(bsg, (av[i].v)^1, (av[i].ul>>32)^1, 1); + } + } + + + if(res_btg == NULL) return; + + for (k = 0; k < res_btg->a.n; k++) + { + root = res_btg->a.a[k]; + v = get_utg_end_from_btg(bub, bub_ug, root); + nv = asg_arc_n(bsg, v); + av = asg_arc_a(bsg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + av[i].del = 1; + asg_arc_del(bsg, (av[i].v)^1, (av[i].ul>>32)^1, 1); + } + + + root = res_btg->a.a[k]^1; + v = get_utg_end_from_btg(bub, bub_ug, root); + nv = asg_arc_n(bsg, v); + av = asg_arc_a(bsg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + av[i].del = 1; + asg_arc_del(bsg, (av[i].v)^1, (av[i].ul>>32)^1, 1); + } + } +} + +void debug_tangle_bubble(bubble_type* bub, long long beg_idx, long long end_idx, const char* command) +{ + // long long beg_idx = (long long)bub->b_g->n_seq - bub->tangle_bub; + // long long end_idx = (long long)bub->b_g->n_seq - 1; + long long i, j, k; + ma_utg_t *u = NULL; + uint32_t beg_utg, sink_utg, *a = NULL, n, btg_left, ori_left, btg_right, ori_right, root_0, root_1; + for (i = beg_idx; i <= end_idx; i++) + { + get_bubbles(bub, i, &beg_utg, &sink_utg, &a, &n, NULL); + fprintf(stderr, "\n(%lld) %s: beg=utg%.6ul, sink=utg%.6ul, n: %u\n", i, command, (beg_utg>>1)+1, (sink_utg>>1)+1, n); + for (k = 0; k < n; k++) + { + fprintf(stderr, "mid=utg%.6ul\n", (a[k]>>1)+1); + } + + for (j = 0; j < bub->b_ug->g->n_seq; j++) + { + u = &(bub->b_ug->u.a[j]); + if(u->n) continue; + for (k = 0; k < u->n; k++) + { + if((long long)(u->a[k]>>33) != i) continue; + fprintf(stderr, "is the %lld-th bubble at btg%.6lldl\n", k, j+1); + if(k > 0) + { + btg_left = u->a[k-1]>>33; + ori_left = u->a[k-1]>>32&1; + get_bubbles(bub, btg_left, ori_left == 1?&root_0:NULL, ori_left == 0?&root_0:NULL, NULL, NULL, NULL); + fprintf(stderr, "left-utg%.6ul\n", (ori_left>>1)+1); + } + + if(k + 1 < u->n) + { + btg_right = u->a[k+1]>>33; + ori_right = (u->a[k+1]>>32&1)^1; + get_bubbles(bub, btg_right, ori_right == 1?&root_1:NULL, ori_right == 0?&root_1:NULL, NULL, NULL, NULL); + fprintf(stderr, "right-utg%.6ul\n", (ori_right>>1)+1); + } + } + } + + } +} + + +void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub) +{ + ma_ug_t *bub_ug = bub->b_ug; + asg_t *bsg = bub->b_g; + uint32_t k, i, v, n_vx, new_bub; + n_vx = MAX((MAX(ug->g->n_seq<<1, bub->b_ug->g->n_seq<<1)), bub->b_g->n_seq<<1); + buf_t b; memset(&b, 0, sizeof(buf_t)); + kvec_t_u32_warp stack, stack2, res_btg, res_utg; + kv_init(stack.a); kv_init(stack2.a); kv_init(res_btg.a); kv_init(res_utg.a); + kvec_asg_arc_t_warp edges; kv_init(edges.a); + uint8_t *is_vis = NULL; CALLOC(is_vis, n_vx); + uint8_t *is_vis2 = NULL; CALLOC(is_vis2, n_vx); + uint8_t *is_used = NULL; CALLOC(is_used, n_vx); + uint8_t *chain_flag = NULL; CALLOC(chain_flag, n_vx); + kvec_t(uint64_t) occ_idx; kv_init(occ_idx); uint64_t tmp, *p = NULL; + double side_rate = 2.5, total_rate = 8; + uint32_t b_utg_beg, b_utg_sink, b_tg_beg, b_tg_sink, utg_beg, utg_sink; + + + + + + + while(1) + { + occ_idx.n = 0; edges.a.n = 0; + if(n_vx < (uint32_t)(MAX((MAX(ug->g->n_seq<<1, bub->b_ug->g->n_seq<<1)), bub->b_g->n_seq<<1))) + { + n_vx = MAX((MAX(ug->g->n_seq<<1, bub->b_ug->g->n_seq<<1)), bub->b_g->n_seq<<1); + is_vis = (uint8_t*)realloc(is_vis, n_vx); + is_vis2 = (uint8_t*)realloc(is_vis2, n_vx); + is_used = (uint8_t*)realloc(is_used, n_vx); + chain_flag = (uint8_t*)realloc(chain_flag, n_vx); + } + memset(is_vis, 0, n_vx); + memset(is_vis2, 0, n_vx); + memset(is_used, 0, n_vx); + memset(chain_flag, 0, n_vx); + if(bub->num.n > 0) bub->num.n--; + new_bub = bub->b_g->n_seq; + + for (k = 0; k < bub_ug->g->n_seq; k++) + { + kv_pushp(uint64_t, occ_idx, &p); + (*p) = get_btg_occ(bub, k); + (*p) <<= 32; (*p) += k; + set_b_utg_weight_flag(bub, NULL, k<<1, chain_flag, 1, NULL); + } + radix_sort_hc64(occ_idx.a, occ_idx.a + occ_idx.n); + for (k = 0; k < occ_idx.n>>1; ++k) + { + tmp = occ_idx.a[k]; + occ_idx.a[k] = occ_idx.a[occ_idx.n - k - 1]; + occ_idx.a[occ_idx.n - k - 1] = tmp; + } + + for (k = 0; k < bub_ug->g->n_seq; k++) + { + v = ((uint32_t)(occ_idx.a[k]))<<1; + if(is_used[v] == 0 && asg_arc_n(bub_ug->g, v) > 0) + { + if(find_bubble_tangle(bub, ug, is_vis, is_vis2, v, side_rate, total_rate, &stack, &stack2, + &res_btg, &res_utg, chain_flag, &b_utg_beg, &b_utg_sink, &b_tg_beg, &b_tg_sink, + &utg_beg, &utg_sink)) + { + if(utg_beg != (uint32_t)-1 && (!IF_HOM(utg_beg>>1, *bub))) + { + kv_push(uint32_t, res_utg.a, utg_beg); + } + if(utg_sink != (uint32_t)-1 && (!IF_HOM(utg_sink>>1, *bub))) + { + kv_push(uint32_t, res_utg.a, utg_sink); + } + + for (i = 0; i < res_btg.a.n; i++) + { + is_used[res_btg.a.a[i]] = 1; + is_used[res_btg.a.a[i]^1] = 1; + } + if(b_utg_beg != (uint32_t)-1) is_used[b_utg_beg] = 1; + if(b_utg_sink != (uint32_t)-1) is_used[b_utg_sink] = 1; + if(b_tg_beg != (uint32_t)-1) b_tg_beg>>=1; + if(b_tg_sink != (uint32_t)-1) b_tg_sink>>=1; + update_bubble_graph(&res_utg, utg_beg, b_tg_beg, utg_sink, b_tg_sink, + bub, &edges, bsg, NULL, NULL, ug, NULL, 0); + + drop_g_edges_by_utg(bub, bsg, bub_ug, &res_btg, b_utg_beg, b_utg_sink); + ///fprintf(stderr, "+>>>>>>beg=btg%.6ul, sink=btg%.6ul\n", (b_utg_beg>>1)+1, (b_utg_sink>>1)+1); + } + } + + + v ^= 1; + if(is_used[v] == 0 && asg_arc_n(bub_ug->g, v) > 0) + { + + if(find_bubble_tangle(bub, ug, is_vis, is_vis2, v, side_rate, total_rate, &stack, &stack2, + &res_btg, &res_utg, chain_flag, &b_utg_beg, &b_utg_sink, &b_tg_beg, &b_tg_sink, + &utg_beg, &utg_sink)) + { + if(utg_beg != (uint32_t)-1 && (!IF_HOM(utg_beg>>1, *bub))) + { + kv_push(uint32_t, res_utg.a, utg_beg); + } + if(utg_sink != (uint32_t)-1 && (!IF_HOM(utg_sink>>1, *bub))) + { + kv_push(uint32_t, res_utg.a, utg_sink); + } + + for (i = 0; i < res_btg.a.n; i++) + { + is_used[res_btg.a.a[i]] = 1; + is_used[res_btg.a.a[i]^1] = 1; + } + + if(b_utg_beg != (uint32_t)-1) is_used[b_utg_beg] = 1; + if(b_utg_sink != (uint32_t)-1) is_used[b_utg_sink] = 1; + if(b_tg_beg != (uint32_t)-1) b_tg_beg>>=1; + if(b_tg_sink != (uint32_t)-1) b_tg_sink>>=1; + + update_bubble_graph(&res_utg, utg_beg, b_tg_beg, utg_sink, b_tg_sink, + bub, &edges, bsg, NULL, NULL, ug, NULL, 0); + drop_g_edges_by_utg(bub, bsg, bub_ug, &res_btg, b_utg_beg, b_utg_sink); + ///fprintf(stderr, "->>>>>>beg=btg%.6ul, sink=btg%.6ul\n", (b_utg_beg>>1)+1, (b_utg_sink>>1)+1); + } + } + } + kv_push(uint32_t, bub->num, bub->list.n); + new_bub = bub->b_g->n_seq - new_bub; + bub->tangle_bub += new_bub; + if(new_bub) update_bub_b_s_idx(bub); + asg_arc_t *t = NULL; + for (k = 0; k < edges.a.n; k++) + { + t = asg_arc_pushp(bsg); + *t = edges.a.a[k]; + } + + bsg->is_srt = 0; free(bsg->idx); bsg->idx = 0; + asg_cleanup(bsg); + ma_ug_destroy(bub_ug); + bub_ug = ma_ug_gen(bub->b_g); + bub->b_ug = bub_ug; + ///fprintf(stderr, "new_bub: %u, bub->tangle_bub: %lu\n", new_bub, bub->tangle_bub); + if(new_bub == 0) break; + } + + kv_destroy(bub->chain_weight); + ma_utg_t *u = NULL; + bub_ug = bub->b_ug; + kv_malloc(bub->chain_weight, bub_ug->u.n); bub->chain_weight.n = bub_ug->u.n; + for (i = 0; i < bub_ug->u.n; i++) + { + u = &(bub_ug->u.a[i]); + bub->chain_weight.a[i].id = i; + // if(u->n <= 1) ///not a chain + // { + // bub->chain_weight.a[i].b_occ = bub->chain_weight.a[i].g_occ = 0; + // bub->chain_weight.a[i].del = 1; + // } + // else + { + bub->chain_weight.a[i].del = 0; + calculate_chain_weight(u, bub, ug, &(bub->chain_weight.a[i])); + } + } + qsort(bub->chain_weight.a, bub->chain_weight.n, sizeof(chain_w_type), cmp_chain_weight); + + ///debug_tangle_bubble(bub); + + free(is_vis); free(is_vis2); free(is_used); free(chain_flag); free(b.b.a); + kv_destroy(occ_idx); kv_destroy(stack.a); kv_destroy(stack2.a); + kv_destroy(res_btg.a); kv_destroy(res_utg.a); kv_destroy(edges.a); +} + + +void update_bubble_chain(ma_ug_t* ug, bubble_type* bub, uint32_t is_middle, uint32_t is_end) +{ + if(bub->b_ug) ma_ug_destroy(bub->b_ug); + if(bub->chain_weight.a) kv_destroy(bub->chain_weight); + kvec_t_u32_warp broken; + kv_init(broken.a); + kvec_asg_arc_t_warp edges; + kv_init(edges.a); + ma_utg_t *u = NULL; + asg_arc_t *t = NULL; + asg_t *sg = ug->g; + pdq pq; + init_pdq(&pq, sg->n_seq<<1); + asg_t *bsg = bub->b_g; + ma_ug_t *bub_ug = NULL; + bub_ug = ma_ug_gen(bub->b_g); + uint32_t i, j, k_i, rId_0, ori_0, root_0, rId_1, ori_1, root_1, n_vtx = sg->n_seq<<1, new_bub; + uint32_t *pre = NULL; MALLOC(pre, n_vtx); + uint8_t* vis_flag = NULL; MALLOC(vis_flag, ug->g->n_seq*2); + kvec_t_u32_warp stack; kv_init(stack.a); + ///chain_w_type x; + ///uint64_t end_thres; + + uint8_t *bsg_idx = NULL; CALLOC(bsg_idx, n_vtx>>1); + for (i = 0; i < bub_ug->u.n; i++) + { + u = &(bub_ug->u.a[i]); + if(u->n == 0) continue; + for (k_i = 0; k_i < u->n; k_i++) + { + uint32_t *a, n; + get_bubbles(bub, u->a[k_i]>>33, &root_0, &root_1, &a, &n, NULL); + for (j = 0; j < n; j++) + { + bsg_idx[a[j]>>1] = 1; + } + bsg_idx[root_0>>1] = 1; + bsg_idx[root_1>>1] = 1; + } + } + + if(bub->num.n > 0) bub->num.n--; + new_bub = bub->b_g->n_seq; + for (i = 0; i < bub_ug->u.n; i++) + { + u = &(bub_ug->u.a[i]); + if(u->n == 0) continue; + ///end_thres = calculate_chain_weight(u, bub, ug, &x); + if(is_middle) + { + for (k_i = 0; k_i < u->n; k_i++) + { + if(k_i+1 >= u->n) continue; + ///note: must igore .del here, since bsg might be changed + t = &(arc_first(bsg, u->a[k_i]>>32)); + if(t->el == 1) continue; + + rId_0 = u->a[k_i]>>33; + ori_0 = u->a[k_i]>>32&1; + get_bubbles(bub, rId_0, ori_0 == 1?&root_0:NULL, ori_0 == 0?&root_0:NULL, NULL, NULL, NULL); + + rId_1 = u->a[k_i+1]>>33; + ori_1 = (u->a[k_i+1]>>32&1)^1; + get_bubbles(bub, rId_1, ori_1 == 1?&root_1:NULL, ori_1 == 0?&root_1:NULL, NULL, NULL, NULL); + + broken.a.n = 0; + get_related_bub_nodes(&broken, bub, &pq, sg, pre, root_0, root_1, NULL); + get_related_bub_nodes(&broken, bub, &pq, sg, pre, root_1, root_0, NULL); + ///no need to cut the edge, we still have chance to flip by chain + if(double_check_broken_bubble(ug->g, &broken, root_0^1, root_1^1, vis_flag, + ug->g->n_seq*2, &stack, NULL, NULL/**bsg, t**/) == 0) + { + continue; + } + if(!IF_HOM(root_0>>1, *bub)) kv_push(uint32_t, broken.a, root_0); + if(!IF_HOM(root_1>>1, *bub)) kv_push(uint32_t, broken.a, root_1); + if(broken.a.n > 0) + { + update_bubble_graph(&broken, root_0^1, rId_0, root_1^1, rId_1, bub, &edges, bsg, t, bsg_idx, ug, NULL, 1); + } + } + } + + + if(is_end) + { + if(u->n >0 && arc_cnt(bub_ug->g, (i<<1)+1) == 0) + { + rId_0 = u->a[0]>>33; + ori_0 = (u->a[0]>>32&1)^1; + get_bubbles(bub, rId_0, ori_0 == 1?&root_0:NULL, ori_0 == 0?&root_0:NULL, NULL, NULL, NULL); + broken.a.n = 0; + get_related_bub_nodes(&broken, bub, &pq, sg, pre, root_0, (uint32_t)-1, NULL); + + if(double_check_broken_bubble(ug->g, &broken, root_0^1, (uint32_t)-1, vis_flag, + ug->g->n_seq*2, &stack, NULL, NULL)) + { + if(!IF_HOM(root_0>>1, *bub)) kv_push(uint32_t, broken.a, root_0); + if(broken.a.n > 0) + { + ///fprintf(stderr, "root_0: utg%.6ul, broken.a.n: %u\n", (root_0>>1)+1, (uint32_t)broken.a.n); + update_bubble_graph(&broken, root_0^1, rId_0, (uint32_t)-1, (uint32_t)-1, bub, &edges, bsg, NULL, bsg_idx, ug, NULL, 0); + } + } + } + + + if(u->n >0 && arc_cnt(bub_ug->g, i<<1) == 0) + { + rId_1 = u->a[u->n-1]>>33; + ori_1 = u->a[u->n-1]>>32&1; + get_bubbles(bub, rId_1, ori_1 == 1?&root_1:NULL, ori_1 == 0?&root_1:NULL, NULL, NULL, NULL); + broken.a.n = 0; + get_related_bub_nodes(&broken, bub, &pq, sg, pre, root_1, (uint32_t)-1, bsg_idx); + + if(double_check_broken_bubble(ug->g, &broken, root_1^1, (uint32_t)-1, vis_flag, + ug->g->n_seq*2, &stack, NULL, NULL)) + { + if(!IF_HOM(root_1>>1, *bub)) kv_push(uint32_t, broken.a, root_1); + if(broken.a.n > 0) + { + ///fprintf(stderr, "root_1: utg%.6ul, broken.a.n: %u\n", (root_1>>1)+1, (uint32_t)broken.a.n); + update_bubble_graph(&broken, (uint32_t)-1, (uint32_t)-1, root_1^1, rId_1, bub, &edges, bsg, NULL, bsg_idx, ug, NULL, 0); + } + + } + } + } + } + kv_push(uint32_t, bub->num, bub->list.n); + new_bub = bub->b_g->n_seq - new_bub; + if(is_end) bub->b_end_bub += new_bub; + if(new_bub) update_bub_b_s_idx(bub); + + + for (i = 0; i < edges.a.n; i++) + { + t = asg_arc_pushp(bsg); + *t = edges.a.a[i]; + } + + bsg->is_srt = 0; free(bsg->idx); bsg->idx = 0; + asg_cleanup(bsg); + ma_ug_destroy(bub_ug); + destory_pdq(&pq); + free(pre); + kv_destroy(broken.a); + kv_destroy(edges.a); + free(bsg_idx); + kv_destroy(stack.a); + free(vis_flag); + + bub->b_ug = ma_ug_gen(bub->b_g); + bub_ug = bub->b_ug; + kv_malloc(bub->chain_weight, bub_ug->u.n); bub->chain_weight.n = bub_ug->u.n; + for (i = 0; i < bub_ug->u.n; i++) + { + u = &(bub_ug->u.a[i]); + bub->chain_weight.a[i].id = i; + // if(u->n <= 1) ///not a chain + // { + // bub->chain_weight.a[i].b_occ = bub->chain_weight.a[i].g_occ = 0; + // bub->chain_weight.a[i].del = 1; + // } + // else + { + bub->chain_weight.a[i].del = 0; + calculate_chain_weight(u, bub, ug, &(bub->chain_weight.a[i])); + } + } + + qsort(bub->chain_weight.a, bub->chain_weight.n, sizeof(chain_w_type), cmp_chain_weight); + + + + /** + uint32_t d_v, d_u, v; + for (i = 0; i < bsg->n_arc; i++) + { + d_v = (uint32_t)(bsg->arc[i].ul>>32); + d_u = bsg->arc[i].v; + for (v = 0; v < bsg->n_arc; v++) + { + if(((bsg->arc[v].ul>>32) == (d_u^1)) && (bsg->arc[v].v == (d_v^1))) break; + } + + if(v == bsg->n_arc) + { + fprintf(stderr, "hahaha, el: %u, ul>>33: %lu, ul&1: %lu, v>>1: %u, v&1: %u\n", + bsg->arc[i].el, bsg->arc[i].ul>>33, (bsg->arc[i].ul>>32)&1, bsg->arc[i].v>>1, bsg->arc[i].v&1); + } + // else + // { + // fprintf(stderr, "hehehe, el: %u, ul>>33: %lu, ul&1: %lu, v>>1: %u, v&1: %u\n", + // bsg->arc[i].el, bsg->arc[i].ul>>33, (bsg->arc[i].ul>>32)&1, bsg->arc[i].v>>1, bsg->arc[i].v&1); + // } + } + + asg_arc_t *av = NULL, *au = NULL; + uint32_t nv, nu; + for (v = 0; v < (uint32_t)(bsg->n_seq<<1); v++) + { + av = asg_arc_a(bsg, v); + nv = asg_arc_n(bsg, v); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + + au = asg_arc_a(bsg, av[i].v^1); + nu = asg_arc_n(bsg, av[i].v^1); + for (k_i = 0; k_i < nu; k_i++) + { + if(au[k_i].del) continue; + if(au[k_i].v == (v^1)) break; + } + if(k_i == nu) fprintf(stderr, "hahaha: v: %u, u: %u\n", v, av[i].v); + } + } + **/ + +} + +void set_b_utg_weight_flag(bubble_type* bub, buf_t* b, uint32_t v, uint8_t* vis_flag, uint32_t flag, uint32_t* occ) +{ + ma_ug_t *bub_ug = bub->b_ug; + long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; + ma_utg_t *u = NULL; + uint32_t convex, k, k_i, k_j, *a, n, beg, sink; + if(b) + { + b->b.n = 0; + get_unitig(bub_ug->g, NULL, v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, b); + } + + if(occ) (*occ) = 0; + for (k = 0; k < (b?b->b.n:1); k++) + { + u = &(bub_ug->u.a[b?(b->b.a[k]>>1):(v>>1)]); + if(u->n == 0) continue; + for (k_i = 0; k_i < u->n; k_i++) + { + get_bubbles(bub, u->a[k_i]>>33, &beg, &sink, &a, &n, NULL); + + for (k_j = 0; k_j < n; k_j++) + { + if(vis_flag) vis_flag[a[k_j]>>1] = flag; + if(occ) (*occ) += bub->ug->u.a[a[k_j]>>1].n; + } + if(beg != (uint32_t)-1 && vis_flag) vis_flag[beg>>1] = flag; + if(sink != (uint32_t)-1 && vis_flag) vis_flag[sink>>1] = flag; + } + } +} + +void set_b_utg_weight_flag_xor(bubble_type* bub, ma_ug_t *bub_ug, buf_t* b, uint32_t v, uint8_t* vis_flag, uint32_t flag, uint32_t* occ) +{ + long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; + ma_utg_t *u = NULL; + uint32_t convex, k, k_i, k_j, *a, n, beg, sink; + if(b) + { + b->b.n = 0; + get_unitig(bub_ug->g, NULL, v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, b); + } + + if(occ) (*occ) = 0; + for (k = 0; k < (b?b->b.n:1); k++) + { + u = &(bub_ug->u.a[b?(b->b.a[k]>>1):(v>>1)]); + if(u->n == 0) continue; + for (k_i = 0; k_i < u->n; k_i++) + { + get_bubbles(bub, u->a[k_i]>>33, &beg, &sink, &a, &n, NULL); + + for (k_j = 0; k_j < n; k_j++) + { + if(vis_flag) vis_flag[a[k_j]>>1] ^= flag; + if(occ) (*occ) += bub->ug->u.a[a[k_j]>>1].n; + } + if(beg != (uint32_t)-1 && vis_flag) vis_flag[beg>>1] ^= flag; + if(sink != (uint32_t)-1 && vis_flag) vis_flag[sink>>1] ^= flag; + } + } +} + +double dfs_weight(uint32_t v, uint8_t* vis_flag, uint8_t* is_vis, hc_links* link, +kvec_t_u32_warp* stack, kvec_t_u32_warp* result, uint32_t e_flag, uint32_t ava_flag, +uint32_t* link_occ) +{ + uint32_t cur, i, next = (uint32_t)-1; + stack->a.n = 0; + kv_push(uint32_t, stack->a, v); + double w = 0; + if(link_occ) (*link_occ) = 0; + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if(is_vis[cur]) continue; + is_vis[cur] = 1; + if(cur!=v && vis_flag[cur] != ava_flag) continue; + for (i = 0; i < link->a.a[cur].e.n; i++) + { + if(link->a.a[cur].e.a[i].del) continue; + next = link->a.a[cur].e.a[i].uID; + ///if(vis_flag[next] == e_flag) + if(vis_flag[next]&e_flag) + { + w += link->a.a[cur].e.a[i].weight; + if(link_occ) (*link_occ) += link->a.a[cur].e.a[i].occ; + continue; + } + if(is_vis[next]) continue; + if(vis_flag[next] != ava_flag) continue; + kv_push(uint32_t, stack->a, next); + } + } + return w; +} + +void if_conflict_utg(uint32_t root, uint32_t* aim_0, uint32_t* aim_1, ma_ug_t* ug, uint8_t* vis_flag, +uint8_t* is_vis_2, uint32_t ava_flag, kvec_t_u32_warp* stack) +{ + uint32_t n_vx = ug->g->n_seq<<1, k, cur, ncur; + asg_arc_t *acur = NULL; + memset(is_vis_2, 0, n_vx); + + stack->a.n = 0; + kv_push(uint32_t, stack->a, root); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if(is_vis_2[cur]) continue; + is_vis_2[cur] = 1; + + ncur = asg_arc_n(ug->g, cur); + acur = asg_arc_a(ug->g, cur); + for (k = 0; k < ncur; k++) + { + if(acur[k].del) continue; + if(is_vis_2[acur[k].v]) continue; + if(vis_flag[acur[k].v>>1] != 0 && vis_flag[acur[k].v>>1] != ava_flag) + { + if(aim_0 && (acur[k].v>>1) == (*aim_0)) continue; + if(aim_1 && (acur[k].v>>1) == (*aim_1)) continue; + break; + } + kv_push(uint32_t, stack->a, acur[k].v); + } + + if(k < ncur) return; + } + + for (k = 0; k < n_vx; k++) + { + if(is_vis_2[k] && vis_flag[k>>1] == 0) + { + ///fprintf(stderr, "******************k=utg%.6ul, vis_flag: %u\n", (k>>1)+1, vis_flag[k>>1]); + vis_flag[k>>1] = ava_flag; + } + + } +} + +double get_chain_weight(bubble_type* bub, ma_ug_t *bub_ug, buf_t* b, uint32_t v, uint32_t convex_source, hc_links* link, +uint8_t* vis_flag, uint8_t* is_vis, ma_ug_t* ug, kvec_t_u32_warp* stack, kvec_t_u32_warp* result, +uint32_t e_flag, uint32_t ava_flag, kvec_t_u32_warp* res_utg, uint32_t* link_occ) +{ + long long nodeLen, baseLen, max_stop_nodeLen, max_stop_baseLen; + ma_utg_t *u = NULL; + uint32_t convex, k, k_i, k_j, *a, n, beg, sink, uID, root, cur, ncur, n_vx = ug->g->n_seq<<1, occ; + asg_arc_t *acur = NULL; + double w = 0; + b->b.n = 0; + get_unitig(bub_ug->g, NULL, v, &convex, &nodeLen, &baseLen, &max_stop_nodeLen, + &max_stop_baseLen, 1, b); + memset(is_vis, 0, n_vx); + + for (k = 0; k < b->b.n; k++) + { + u = &(bub_ug->u.a[b->b.a[k]>>1]); + if(u->n == 0) continue; + + for (k_i = 0; k_i < u->n; k_i++) + { + get_bubbles(bub, u->a[k_i]>>33, &beg, &sink, &a, &n, NULL); + for(k_j = 0; k_j < n; k_j++) is_vis[a[k_j]] = is_vis[a[k_j]^1] = 1; + if(beg != (uint32_t)-1) is_vis[beg] = is_vis[beg^1] = 1; + if(sink != (uint32_t)-1) is_vis[sink] = is_vis[sink^1] = 1; + } + } + + u = &(bub_ug->u.a[v>>1]); + if((v&1)==0) + { + get_bubbles(bub, (u->a[0]>>32)>>1, (((u->a[0]>>32)&1)^1)==1?&root:NULL, + (((u->a[0]>>32)&1)^1) == 0?&root:NULL, NULL, NULL, NULL); + } + else + { + get_bubbles(bub, (u->a[u->n-1]>>32)>>1, ((u->a[u->n-1]>>32)&1)==1?&root:NULL, + ((u->a[u->n-1]>>32)&1) == 0?&root:NULL, NULL, NULL, NULL); + } + + root ^= 1; + ///fprintf(stderr, "root=utg%.6dl\n", (root>>1)+1); + is_vis[root] = 0; + stack->a.n = 0; + kv_push(uint32_t, stack->a, root); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if(is_vis[cur]) continue; + is_vis[cur] = 1; + if(vis_flag[cur>>1] == 0) vis_flag[cur>>1] = ava_flag; + + ncur = asg_arc_n(ug->g, cur); + acur = asg_arc_a(ug->g, cur); + for (k = 0; k < ncur; k++) + { + if(acur[k].del) continue; + if(is_vis[acur[k].v]) continue; + if(vis_flag[acur[k].v>>1] != 0 && vis_flag[acur[k].v>>1] != ava_flag) continue; + kv_push(uint32_t, stack->a, acur[k].v); + } + } + + + + uint32_t aim_0, aim_1, root_source; + aim_0 = root>>1; + u = &(bub_ug->u.a[convex_source>>1]); + if((convex_source&1)==1) + { + get_bubbles(bub, (u->a[0]>>32)>>1, (((u->a[0]>>32)&1)^1)==1?&root_source:NULL, + (((u->a[0]>>32)&1)^1) == 0?&root_source:NULL, NULL, NULL, NULL); + } + else + { + get_bubbles(bub, (u->a[u->n-1]>>32)>>1, ((u->a[u->n-1]>>32)&1)==1?&root_source:NULL, + ((u->a[u->n-1]>>32)&1) == 0?&root_source:NULL, NULL, NULL, NULL); + } + root_source ^= 1; + aim_1 = root_source>>1; + + ///fprintf(stderr, "aim_0=utg%.6ul, aim_1=utg%.6ul\n", aim_0+1, aim_1+1); + + cur = root_source; + ncur = asg_arc_n(ug->g, cur); + acur = asg_arc_a(ug->g, cur); + for (k_i = 0; k_i < ncur; k_i++) + { + if(acur[k_i].del) continue; + if(vis_flag[acur[k_i].v>>1] != 0) continue; + if_conflict_utg(acur[k_i].v, &aim_0, &aim_1, ug, vis_flag, is_vis, ava_flag, stack); + } + + + for (k = 0; k < ug->g->n_seq; k++) + { + if(vis_flag[k] == ava_flag) + { + cur = k<<1; + ncur = asg_arc_n(ug->g, cur); + acur = asg_arc_a(ug->g, cur); + for (k_i = 0; k_i < ncur; k_i++) + { + if(acur[k_i].del) continue; + if(vis_flag[acur[k_i].v>>1] != 0) continue; + if_conflict_utg(acur[k_i].v, &aim_0, &aim_1, ug, vis_flag, is_vis, ava_flag, stack); + } + + + + cur = (k<<1)+1; + ncur = asg_arc_n(ug->g, cur); + acur = asg_arc_a(ug->g, cur); + for (k_i = 0; k_i < ncur; k_i++) + { + if(acur[k_i].del) continue; + if(vis_flag[acur[k_i].v>>1] != 0) continue; + if_conflict_utg(acur[k_i].v, &aim_0, &aim_1, ug, vis_flag, is_vis, ava_flag, stack); + } + } + } + + + + + + + + memset(is_vis, 0, n_vx); + if(link_occ) (*link_occ) = 0; + for (k = result->a.n = 0, w = 0; k < b->b.n; k++) + { + u = &(bub_ug->u.a[b->b.a[k]>>1]); + if(u->n == 0) continue; + for (k_i = 0; k_i < u->n; k_i++) + { + get_bubbles(bub, u->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + + for (k_j = 0; k_j < n; k_j++) + { + uID = a[k_j]>>1; + w += dfs_weight(uID, vis_flag, is_vis, link, stack, result, e_flag, ava_flag, &occ); + if(link_occ) (*link_occ) += occ; + } + } + } + + + for (k = 0; k < ug->g->n_seq; k++) + { + if(vis_flag[k] == ava_flag) + { + vis_flag[k] = 0; + if(res_utg && (!IF_HOM(k, *bub))) + { + kv_push(uint32_t, res_utg->a, k<<1); + } + } + } + + + return w; +} + +int double_check_bub_branch(asg_arc_t *t, ma_ug_t *bs_ug, double *e_w, uint32_t *e_occ, double cutoff, uint32_t max_w_occ) +{ + uint32_t v = t->v^1, w = (t->ul>>32)^1, i, nv, rv, max_i, w_i, *a_occ = NULL; + asg_arc_t *av = NULL; + double *aw = NULL, max_w = cutoff - 1, w_w = 1; + av = asg_arc_a(bs_ug->g, v); + nv = asg_arc_n(bs_ug->g, v); + aw = (&e_w[bs_ug->g->idx[v]>>32]); + a_occ = (&e_occ[bs_ug->g->idx[v]>>32]); + + if(nv <= 1) return 1; + + for (i = rv = 0, max_i = w_i = (uint32_t)-1; i < nv; i++) + { + if(av[i].del) continue; + rv++; + if(av[i].v == w) + { + w_i = i; + w_w = aw[i]; + continue; + } + if(max_i == (uint32_t)-1) + { + max_i = i; + max_w = aw[i]; + } + else if(max_w < aw[i]) + { + max_i = i; + max_w = aw[i]; + } + } + + if(rv <= 1) return 1; ///must be here + + if(max_i == (uint32_t)-1 || w_i == (uint32_t)-1) return 0; + ///if(max_w <= max_w_cutoff) return 0; //must be <= + if(a_occ[max_i] <= max_w_occ) return 0; //must be <= + + if(w_w*cutoff < max_w) return 1; + return 0; +} + +void clean_bubble_chain_by_HiC(ma_ug_t* ug, hc_links* link, bubble_type* bub) +{ + ma_ug_t *bs_ug = bub->b_ug; + uint32_t v, u, i, m, max_i, nv, rv, n_vx, root, flag_pri = 1, flag_aux = 2, flag_ava = 4, occ; + double w, cutoff = 2/**, max_w_cutoff = MAX(MIN(100*OFFSET_RATE_MIN_W, OFFSET_RATE_MAX_W/100), OFFSET_RATE_MIN_W)**/; + uint32_t max_w_occ = 4; + asg_arc_t *av = NULL; + n_vx = bs_ug->g->n_seq << 1; + uint8_t *vis = NULL; CALLOC(vis, ug->g->n_seq<<1); + uint8_t *is_vis = NULL; CALLOC(is_vis, ug->g->n_seq<<1); + uint8_t *is_used = NULL; CALLOC(is_used, n_vx); + uint8_t *dedup = NULL; CALLOC(dedup, ug->g->n_seq<<1); + buf_t b; memset(&b, 0, sizeof(buf_t)); + kvec_t_u32_warp stack, result, res_utg; + kv_init(stack.a); kv_init(result.a); kv_init(res_utg.a); + double *e_w = NULL; MALLOC(e_w, bs_ug->g->n_arc); + uint32_t *e_occ = NULL, *a_occ = NULL; CALLOC(e_occ, bs_ug->g->n_arc); + double *aw = NULL, max_w = 0; + kvec_asg_arc_t_warp edges; kv_init(edges.a); + ma_ug_t *back_bs_ug = copy_untig_graph(bs_ug); + + for (i = 0; i < bs_ug->g->n_arc; i++) + { + e_w[i] = -1; + } + + for (i = 0; i < bs_ug->g->n_seq; i++) + { + set_b_utg_weight_flag(bub, &b, i<<1, vis, flag_aux, NULL); + } + + + for (v = 0; v < n_vx; v++) + { + av = asg_arc_a(bs_ug->g, v); + nv = asg_arc_n(bs_ug->g, v); + aw = (&e_w[bs_ug->g->idx[v]>>32]); + a_occ = (&e_occ[bs_ug->g->idx[v]>>32]); + if(nv <= 1 || get_real_length(bs_ug->g, v, NULL) <= 1) continue; + set_b_utg_weight_flag_xor(bub, bs_ug, &b, v^1, vis, flag_pri, NULL); + + ///fprintf(stderr, "\n******pri>btg%.6dl\n", (v>>1)+1); + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + //fprintf(stderr, "aux>btg%.6dl\n", (av[i].v>>1)+1); + w = get_chain_weight(bub, bs_ug, &b, av[i].v, v, link, vis, is_vis, ug, &stack, &result, flag_pri, flag_ava, NULL, &occ); + ///fprintf(stderr, "aux>btg%.6dl, w: %f\n", (av[i].v>>1)+1, w); + aw[i] = w; + a_occ[i] = occ; + } + + set_b_utg_weight_flag_xor(bub, bs_ug, &b, v^1, vis, flag_pri, NULL); + } + + + for (v = 0; v < n_vx; v++) + { + av = asg_arc_a(bs_ug->g, v); + nv = asg_arc_n(bs_ug->g, v); + aw = (&e_w[bs_ug->g->idx[v]>>32]); + a_occ = (&e_occ[bs_ug->g->idx[v]>>32]); + if(nv <= 1 || get_real_length(bs_ug->g, v, NULL) <= 1) continue; + + for (i = rv = 0, max_i = (uint32_t)-1; i < nv; i++) + { + if(av[i].del) continue; + if(max_i == (uint32_t)-1) + { + max_i = i; + max_w = aw[i]; + } + else if(max_w < aw[i]) + { + max_i = i; + max_w = aw[i]; + } + rv++; + } + + if(max_i == (uint32_t)-1) continue; + ///if(max_w <= max_w_cutoff) continue; //must be <= + if(a_occ[max_i] <= max_w_occ) continue; //must be <= + if(rv < 2) continue; + + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + if(i == max_i) continue; + ///if((av[i].v>>1) == (v>>1) && aw[i] <= max_w_cutoff) continue; ///might be not reasonable + if((av[i].v>>1) == (v>>1) && a_occ[i] <= max_w_occ) continue; ///might be not reasonable + if(aw[i]*cutoff < max_w && double_check_bub_branch(&av[i], bs_ug, e_w, e_occ, cutoff, max_w_occ)) + { + av[i].del = 1; asg_arc_del(bs_ug->g, (av[i].v)^1, (av[i].ul>>32)^1, 1); + } + } + } + + uint32_t rId_0, ori_0, rId_1, ori_1, root_0, root_1, new_bub; + if(bub->num.n > 0) bub->num.n--; + new_bub = bub->b_g->n_seq; + + for (v = 0; v < n_vx; v++) + { + av = asg_arc_a(bs_ug->g, v); + nv = asg_arc_n(bs_ug->g, v); + rv = get_real_length(bs_ug->g, v, NULL); + if(nv == rv) continue; + if(rv != 1 || nv <= 1) continue; + get_real_length(bs_ug->g, v, &u); + u ^= 1; + if(get_real_length(bs_ug->g, u, NULL) != 1) continue; + drop_g_edges_by_utg(bub, bub->b_g, bs_ug, NULL, v, u); + if(is_used[v] || is_used[u]) continue; + + is_used[v] = is_used[u] = 1; + root = get_utg_end_from_btg(bub, bs_ug, v); + rId_0 = root>>1; + ori_0 = root&1; + get_bubbles(bub, rId_0, ori_0 == 1?&root_0:NULL, ori_0 == 0?&root_0:NULL, NULL, NULL, NULL); + + root = get_utg_end_from_btg(bub, bs_ug, u); + rId_1 = root>>1; + ori_1 = root&1; + get_bubbles(bub, rId_1, ori_1 == 1?&root_1:NULL, ori_1 == 0?&root_1:NULL, NULL, NULL, NULL); + + res_utg.a.n = 0; + set_b_utg_weight_flag_xor(bub, back_bs_ug, &b, v^1, vis, flag_pri, NULL); + get_chain_weight(bub, back_bs_ug, &b, u^1, v, link, vis, is_vis, ug, &stack, &result, flag_pri, flag_ava, &res_utg, NULL); + set_b_utg_weight_flag_xor(bub, back_bs_ug, &b, v^1, vis, flag_pri, NULL); + for (i = 0; i < res_utg.a.n; i++) dedup[res_utg.a.a[i]>>1] |= 1; + + set_b_utg_weight_flag_xor(bub, back_bs_ug, &b, u^1, vis, flag_pri, NULL); + get_chain_weight(bub, back_bs_ug, &b, v^1, u, link, vis, is_vis, ug, &stack, &result, flag_pri, flag_ava, &res_utg, NULL); + set_b_utg_weight_flag_xor(bub, back_bs_ug, &b, u^1, vis, flag_pri, NULL); + for (; i < res_utg.a.n; i++) dedup[res_utg.a.a[i]>>1] |= 2; + + + for (i = m = 0; i < res_utg.a.n; i++) + { + if(dedup[res_utg.a.a[i]>>1] == 3) + { + res_utg.a.a[m] = res_utg.a.a[i]; + m++; + } + dedup[res_utg.a.a[i]>>1] = 0; + } + res_utg.a.n = m; + + // fprintf(stderr, "res_utg.a.n: %u, m: %u, beg-utg%.6ul, sink-utg%.6ul\n", + // res_utg.a.n, m, (root_0>>1)+1, (root_1>>1)+1); + + if(!IF_HOM(root_0>>1, *bub)) kv_push(uint32_t, res_utg.a, root_0); + if(!IF_HOM(root_1>>1, *bub)) kv_push(uint32_t, res_utg.a, root_1); + + update_bubble_graph(&res_utg, root_0^1, rId_0, root_1^1, rId_1, bub, &edges, bub->b_g, NULL, NULL, ug, NULL, 0); + + ///fprintf(stderr, "\n******src-btg%.6ul------>dest-btg%.6ul\n", (v>>1)+1, (u>>1)+1); + } + + kv_push(uint32_t, bub->num, bub->list.n); + new_bub = bub->b_g->n_seq - new_bub; + bub->cross_bub += new_bub; + if(new_bub) update_bub_b_s_idx(bub); + + ///fprintf(stderr, "bub->cross_bub: %u\n", (uint32_t)bub->cross_bub); + ///debug_tangle_bubble(bub, bub->b_g->n_seq - bub->cross_bub, bub->b_g->n_seq - 1, "Cross-tangle"); + + asg_arc_t *t = NULL; + for (i = 0; i < edges.a.n; i++) + { + t = asg_arc_pushp(bub->b_g); + *t = edges.a.a[i]; + } + bub->b_g->is_srt = 0; + free(bub->b_g->idx); + bub->b_g->idx = 0; + asg_cleanup(bub->b_g); + ma_ug_destroy(bs_ug); + bs_ug = ma_ug_gen(bub->b_g); + bub->b_ug = bs_ug; + kv_destroy(bub->chain_weight); + ma_utg_t *u_x = NULL; + bs_ug = bub->b_ug; + kv_malloc(bub->chain_weight, bs_ug->u.n); bub->chain_weight.n = bs_ug->u.n; + for (i = 0; i < bs_ug->u.n; i++) + { + u_x = &(bs_ug->u.a[i]); + bub->chain_weight.a[i].id = i; + // if(u->n <= 1) ///not a chain + // { + // bub->chain_weight.a[i].b_occ = bub->chain_weight.a[i].g_occ = 0; + // bub->chain_weight.a[i].del = 1; + // } + // else + { + bub->chain_weight.a[i].del = 0; + calculate_chain_weight(u_x, bub, ug, &(bub->chain_weight.a[i])); + } + } + qsort(bub->chain_weight.a, bub->chain_weight.n, sizeof(chain_w_type), cmp_chain_weight); + + + + free(vis); free(is_vis); free(is_used); free(dedup); free(b.b.a); free(e_w); free(e_occ); + kv_destroy(stack.a); kv_destroy(result.a); kv_destroy(res_utg.a); kv_destroy(edges.a); + ma_ug_destroy(back_bs_ug); +} + + +void append_boundary_chain(ma_ug_t* ug, hc_links* link, bubble_type* bub) +{ + ma_ug_t *bs_ug = bub->b_ug; + uint32_t v, u, i, k, beg_idx, m, nv, n_vx, flag_pri = 1, flag_aux = 2, flag_ava = 4; + uint32_t root, rId_0, ori_0, root_0, new_bub; + asg_arc_t *av = NULL; + n_vx = bs_ug->g->n_seq << 1; + uint8_t *vis = NULL; CALLOC(vis, ug->g->n_seq<<1); + uint8_t *is_vis = NULL; CALLOC(is_vis, ug->g->n_seq<<1); + uint8_t *is_used = NULL; CALLOC(is_used, n_vx); + uint8_t *dedup = NULL; CALLOC(dedup, ug->g->n_seq<<1); + buf_t b; memset(&b, 0, sizeof(buf_t)); + kvec_t_u32_warp stack, result, res_utg; + kv_init(stack.a); kv_init(result.a); kv_init(res_utg.a); + kvec_asg_arc_t_warp edges; kv_init(edges.a); + + for (i = 0; i < bs_ug->g->n_seq; i++) + { + set_b_utg_weight_flag(bub, &b, i<<1, vis, flag_aux, NULL); + } + + if(bub->num.n > 0) bub->num.n--; + new_bub = bub->b_g->n_seq; + for (v = 0; v < n_vx; v++) + { + av = asg_arc_a(bs_ug->g, v); + nv = asg_arc_n(bs_ug->g, v); + if(nv == 0 || get_real_length(bs_ug->g, v, NULL) == 0) continue; + res_utg.a.n = 0; + for (i = 0; i < nv; i++) + { + if(av[i].del) continue; + u = av[i].v^1; + /** + beg_idx = res_utg.a.n; + set_b_utg_weight_flag_xor(bub, bs_ug, &b, v^1, vis, flag_pri, NULL); + get_chain_weight(bub, bs_ug, &b, u^1, v, link, vis, is_vis, ug, &stack, &result, flag_pri, flag_ava, &res_utg, NULL); + set_b_utg_weight_flag_xor(bub, bs_ug, &b, v^1, vis, flag_pri, NULL); + for (k = m = beg_idx; k < res_utg.a.n; k++) + { + if(dedup[res_utg.a.a[k]>>1] != 0) continue; + dedup[res_utg.a.a[k]>>1] = 1; + res_utg.a.a[m] = res_utg.a.a[k]; + m++; + } + res_utg.a.n = m; + **/ + beg_idx = res_utg.a.n; + set_b_utg_weight_flag_xor(bub, bs_ug, &b, u^1, vis, flag_pri, NULL); + get_chain_weight(bub, bs_ug, &b, v^1, u, link, vis, is_vis, ug, &stack, &result, flag_pri, flag_ava, &res_utg, NULL); + set_b_utg_weight_flag_xor(bub, bs_ug, &b, u^1, vis, flag_pri, NULL); + for (k = m = beg_idx; k < res_utg.a.n; k++) + { + if(dedup[res_utg.a.a[k]>>1] != 0) continue; + dedup[res_utg.a.a[k]>>1] = 1; + res_utg.a.a[m] = res_utg.a.a[k]; + m++; + } + res_utg.a.n = m; + } + + for (k = 0; k < res_utg.a.n; k++) dedup[res_utg.a.a[k]>>1] = 0; + /*******************************for debug************************************/ + // for (i = 0; i < res_utg.a.n; i++) + // { + // for (k = 0; k < res_utg.a.n; k++) + // { + // if(k == i) continue; + // if((res_utg.a.a[i]>>1) == (res_utg.a.a[k]>>1)) fprintf(stderr, "ERROR\n"); + // } + // } + /*******************************for debug************************************/ + + root = get_utg_end_from_btg(bub, bs_ug, v); + rId_0 = root>>1; + ori_0 = root&1; + get_bubbles(bub, rId_0, ori_0 == 1?&root_0:NULL, ori_0 == 0?&root_0:NULL, NULL, NULL, NULL); + + if(root_0 != (uint32_t)-1 && (!IF_HOM(root_0>>1, *bub))) kv_push(uint32_t, res_utg.a, root_0); + + if(v&1) + { + update_bubble_graph(&res_utg, root_0^1, rId_0, (uint32_t)-1, (uint32_t)-1, bub, &edges, bub->b_g, NULL, NULL, ug, NULL, 0); + } + else + { + update_bubble_graph(&res_utg, (uint32_t)-1, (uint32_t)-1, root_0^1, rId_0, bub, &edges, bub->b_g, NULL, NULL, ug, NULL, 0); + } + } + kv_push(uint32_t, bub->num, bub->list.n); + new_bub = bub->b_g->n_seq - new_bub; + bub->mess_bub += new_bub; + if(new_bub) update_bub_b_s_idx(bub); + + ///fprintf(stderr, "bub->mess_bub: %lu\n", bub->mess_bub); + + for (v = 0; v < n_vx; v++) + { + av = asg_arc_a(bs_ug->g, v); + nv = asg_arc_n(bs_ug->g, v); + if(nv == 0 || get_real_length(bs_ug->g, v, NULL) == 0) continue; + drop_g_edges_by_utg(bub, bub->b_g, bs_ug, NULL, v, (uint32_t)-1); + } + + + asg_arc_t *t = NULL; + for (i = 0; i < edges.a.n; i++) + { + t = asg_arc_pushp(bub->b_g); + *t = edges.a.a[i]; + } + bub->b_g->is_srt = 0; + free(bub->b_g->idx); + bub->b_g->idx = 0; + asg_cleanup(bub->b_g); + ma_ug_destroy(bs_ug); + bs_ug = ma_ug_gen(bub->b_g); + bub->b_ug = bs_ug; + kv_destroy(bub->chain_weight); + ma_utg_t *u_x = NULL; + bs_ug = bub->b_ug; + kv_malloc(bub->chain_weight, bs_ug->u.n); bub->chain_weight.n = bs_ug->u.n; + for (i = 0; i < bs_ug->u.n; i++) + { + u_x = &(bs_ug->u.a[i]); + bub->chain_weight.a[i].id = i; + // if(u->n <= 1) ///not a chain + // { + // bub->chain_weight.a[i].b_occ = bub->chain_weight.a[i].g_occ = 0; + // bub->chain_weight.a[i].del = 1; + // } + // else + { + bub->chain_weight.a[i].del = 0; + calculate_chain_weight(u_x, bub, ug, &(bub->chain_weight.a[i])); + } + } + qsort(bub->chain_weight.a, bub->chain_weight.n, sizeof(chain_w_type), cmp_chain_weight); + + + + free(vis); free(is_vis); free(is_used); free(dedup); free(b.b.a); + kv_destroy(stack.a); kv_destroy(result.a); kv_destroy(res_utg.a); + kv_destroy(edges.a); + + + /*******************************for debug************************************/ + // for (v = 0; v < (uint32_t)(bs_ug->g->n_seq<<1); v++) + // { + // if(asg_arc_n(bs_ug->g, v) > 0) fprintf(stderr, "ERROR, btg%.6ul\n", (v>>1)+1); + // ma_utg_t *utg = &(bs_ug->u.a[v>>1]); + // for (i = 0; i < utg->n; i++) + // { + // if((utg->a[i]>>33) >= + // (bub->f_bub + bub->b_bub + bub->b_end_bub + bub->tangle_bub + bub->cross_bub)) + // { + // if(i != 0 && i != utg->n - 1) fprintf(stderr, "ERROR, btg%.6ul, i: %u\n", (v>>1)+1, i); + // } + // } + // } + /*******************************for debug************************************/ +} + +int cmp_chain_hic_w_weight(const void * a, const void * b) +{ + if((*(chain_hic_w_type*)a).w != (*(chain_hic_w_type*)b).w) + { + return (*(chain_hic_w_type*)a).w > (*(chain_hic_w_type*)b).w? -1 : 1; + } + else + { + return 0; + } +} + + +#define is_useful_bub(ID, B) (((ID)>=((B).f_bub + (B).b_bub + (B).b_end_bub + (B).tangle_bub + (B).cross_bub))\ + && ((ID)<((B).f_bub + (B).b_bub + (B).b_end_bub + (B).tangle_bub + (B).cross_bub + (B).mess_bub))) +void init_chain_hic_warp(ma_ug_t* ug, hc_links* link, bubble_type* bub, chain_hic_warp* c_w) +{ + ma_ug_t *bs_ug = bub->b_ug; + uint32_t *a = NULL, n, occ, i, k_i, k_j, k_k, uID, is_del, m, bub_mess; + double w; + ma_utg_t *u_x = NULL; + + kv_init((*c_w)); + kv_malloc((*c_w), bs_ug->u.n); + (*c_w).n = bs_ug->u.n; + (*c_w).max_bub_id = 0; + (*c_w).u_n = ug->u.n; + (*c_w).chain_idx = NULL; + MALLOC((*c_w).chain_idx, ug->u.n); + memset((*c_w).chain_idx, -1, sizeof(uint32_t)*ug->u.n); + + for (i = bub_mess = 0; i < bs_ug->u.n; i++) + { + u_x = &(bs_ug->u.a[i]); + (*c_w).a[i].id = i; + (*c_w).a[i].w = 0; + (*c_w).a[i].occ = 0; + (*c_w).a[i].u = NULL; + for (k_i = 0, w = 0, occ = 0; k_i < u_x->n; k_i++) + { + if(is_useful_bub(u_x->a[k_i]>>33, *bub)) + { + bub_mess++; + continue; + } + get_bubbles(bub, u_x->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + + for (k_j = 0; k_j < n; k_j++) + { + uID = a[k_j]>>1; + occ += ug->u.a[uID].n; + for (k_k = 0; k_k < link->a.a[uID].e.n; k_k++) + { + if(link->a.a[uID].e.a[k_k].del) continue; + w += link->a.a[uID].e.a[k_k].weight; + } + } + } + (*c_w).a[i].w = w; + (*c_w).a[i].occ = occ; + } + if(bub_mess != bub->mess_bub) fprintf(stderr, "ERROR\n"); + ///fprintf(stderr, "bub_mess: %u, bub->mess_bub: %lu\n", bub_mess, bub->mess_bub); + + for (i = 0; i < bs_ug->u.n; i++) + { + u_x = &(bs_ug->u.a[i]); + for (k_i = 0; k_i < u_x->n; k_i++) + { + if(is_useful_bub(u_x->a[k_i]>>33, *bub)) + { + continue; + } + + get_bubbles(bub, u_x->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + for (k_j = 0; k_j < n; k_j++) + { + uID = a[k_j]>>1; + if((*c_w).chain_idx[uID] == (uint32_t)-1) + { + (*c_w).chain_idx[uID] = i; + } + else + { + if((*c_w).a[i].occ > (*c_w).a[(*c_w).chain_idx[uID]].occ) + { + (*c_w).chain_idx[uID] = i; + } + } + } + } + } + + for (i = m = 0; i < (*c_w).n; i++) + { + u_x = &(bs_ug->u.a[(*c_w).a[i].id]); + is_del = 1; + for (k_i = 0; k_i < u_x->n; k_i++) + { + if(is_useful_bub(u_x->a[k_i]>>33, *bub)) + { + continue; + } + + get_bubbles(bub, u_x->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + for (k_j = 0; k_j < n; k_j++) + { + uID = a[k_j]>>1; + if((*c_w).chain_idx[uID] == (*c_w).a[i].id) + { + is_del = 0; + break; + } + } + if(is_del == 0) break; + } + if(is_del == 0) + { + (*c_w).a[m] = (*c_w).a[i]; + m++; + } + } + + ///fprintf(stderr, "# chain: %u, # pre chain: %u\n", m, (uint32_t)(*c_w).n); + (*c_w).n = m; + for (i = 0; i < (*c_w).n; i++) + { + u_x = &(bs_ug->u.a[(*c_w).a[i].id]); + CALLOC((*c_w).a[i].u, 1); + for (k_i = (*c_w).a[i].u->n = 0; k_i < u_x->n; k_i++) + { + if(is_useful_bub(u_x->a[k_i]>>33, *bub)) continue; + (*c_w).a[i].u->n++; + } + (*c_w).a[i].u->m = (*c_w).a[i].u->n; + MALLOC((*c_w).a[i].u->a, (*c_w).a[i].u->m); + for (k_i = (*c_w).a[i].u->n = 0; k_i < u_x->n; k_i++) + { + if(is_useful_bub(u_x->a[k_i]>>33, *bub)) continue; + (*c_w).a[i].u->a[(*c_w).a[i].u->n] = u_x->a[k_i]; + (*c_w).a[i].u->n++; + } + } + (*c_w).max_bub_id = (*c_w).n; + + if(bub->num.n > 0) bub->num.n--; + chain_hic_w_type* p = NULL; + for (i = 0; i < ug->u.n; i++) + { + uID = i; + if(IF_HOM(uID, *bub)) continue; + if((*c_w).chain_idx[uID] == (uint32_t)-1) + { + kv_pushp(chain_hic_w_type, (*c_w), &p); + CALLOC(p->u, 1); + p->u->n = p->u->m = 1; + MALLOC(p->u->a, p->u->m); + p->u->a[0] = (bub->pathLen.n)<<33; + /********************push bubble********************/ + kv_push(uint32_t, bub->num, bub->list.n); + kv_push(uint64_t, bub->pathLen, 0); + kv_push(uint32_t, bub->list, uID<<1); + kv_push(uint32_t, bub->list, uID<<1); + kv_push(uint32_t, bub->list, uID<<1); + /********************push bubble********************/ + p->occ = ug->u.a[uID].n; + p->w = 0; + for (k_k = 0; k_k < link->a.a[uID].e.n; k_k++) + { + if(link->a.a[uID].e.a[k_k].del) continue; + p->w += link->a.a[uID].e.a[k_k].weight; + } + p->id = (*c_w).n - 1; + (*c_w).chain_idx[uID] = p->id; + } + } + kv_push(uint32_t, bub->num, bub->list.n); + + + + + memset((*c_w).chain_idx, -1, sizeof(uint32_t)*ug->u.n); + for (i = 0; i < (*c_w).n; i++) + { + (*c_w).a[i].id = i; + u_x = (*c_w).a[i].u; + for (k_i = 0; k_i < u_x->n; k_i++) + { + get_bubbles(bub, u_x->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + for (k_j = 0; k_j < n; k_j++) + { + uID = a[k_j]>>1; + if((*c_w).chain_idx[uID] == (uint32_t)-1) + { + (*c_w).chain_idx[uID] = (*c_w).a[i].id; + } + else + { + if((*c_w).a[i].occ > (*c_w).a[(*c_w).chain_idx[uID]].occ) + { + (*c_w).chain_idx[uID] = (*c_w).a[i].id; + } + } + } + } + } + + /** + uint32_t rId_0, ori_0, root_0, rId_1, ori_1, root_1; + for (i = 0; i < (*c_w).n; i++) + { + (*c_w).a[i].l_d = (*c_w).a[i].r_d = (uint64_t)-1; + u_x = (*c_w).a[i].u; + if(u_x->n == 0) continue; + + rId_0 = u_x->a[0]>>33; + ori_0 = (u_x->a[0]>>32&1)^1; + get_bubbles(bub, rId_0, ori_0 == 1?&root_0:NULL, ori_0 == 0?&root_0:NULL, NULL, NULL, NULL); + root_0 ^= 1; + } + **/ + + ///fprintf(stderr, "# chain: %u, # c_w.max_bub_id: %u\n", (uint32_t)(*c_w).n, (*c_w).max_bub_id); + ///qsort((*c_w).a, (*c_w).n, sizeof(chain_hic_w_type), cmp_chain_hic_w_weight); +} + +void destory_chain_hic_warp(chain_hic_warp* c_w) +{ + uint32_t i; + for (i = 0; i < c_w->n; i++) + { + free(c_w->a[i].u->a); + free(c_w->a[i].u); + } + kv_destroy((*c_w)); + free((*c_w).chain_idx); +} + + +void build_bub_graph(ma_ug_t* ug, bubble_type* bub) +{ + bub->check_het = 0; + get_bub_graph(ug, bub); ///just create nodes/edges from f_bub + detect_bub_graph(bub, ug->g); + asg_destroy(bub->b_g); + bub->check_het = 1; + get_bub_graph(ug, bub); + ///print_bubble_chain(bub, "first round"); + // detect_bub_graph(bub, ug->g, 1); + update_bubble_chain(ug, bub, 1, 0); + ///print_bubble_chain(bub, "second round"); +} + +void get_forward_distance(uint32_t src, uint32_t dest, asg_t *sg, hc_links* link, MT* M) +{ + hc_edge *e = NULL; + e = get_hc_edge(link, src, dest, 0); + if(e == NULL) return; + uint32_t v, j; + uint64_t d[2], db[2], q_u, min, min_i, min_b; + e->dis = (uint64_t)-1; + + for (v = ((uint64_t)(src)<<1); v < ((uint64_t)(src+1)<<1); v++) + { + d[0] = d[1] = db[0] = db[1] = (uint64_t)-1; + for (j = 0; j < M->matrix.a[v].a.n; j++) + { + q_u = M->matrix.a[v].a.a[j] >> M->uID_shift; + if((q_u>>1) == dest) d[q_u&1] = (M->matrix.a[v].a.a[j] & M->dis_mode) + sg->seq[q_u>>1].len; + if((q_u>>1) > dest) break;///just for speeding up, doesn't affect results + } + + min = min_i = min_b = (uint64_t)-1; + if(e->dis != (uint64_t)-1) min = e->dis >> 3; + + if(d[0] < min) min = d[0], min_i = 0, min_b = 0; + if(d[1] < min) min = d[1], min_i = 1, min_b = 0; + if(min_i != (uint64_t)-1 && min != (uint64_t)-1) + { + e->dis = min<<1; + e->dis += min_b; + e->dis <<=1; + e->dis += v&1; + e->dis <<=1; + e->dis += min_i; + } + } + // fprintf(stderr, "%s\t%s\tdis(%lu)\n", e->dis == (uint64_t)-1? "unreach cur": "**reach cur", + // ((e->dis>>2)&1)?"back":"forw", e->dis>>3); +} + + + + +int get_trans_rate_function(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, MT* M, H_partition* hap, trans_idx* dis) +{ + kvec_t(uint64_t) buf, buf_idx; + kv_init(buf); + kv_init(buf_idx); + uint64_t beg, end, cnt[2]; + uint64_t k, i, t_d, r_idx, f_idx, med = (uint64_t)-1; + int beg_status, end_status; + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + link->a.a[i].e.a[k].dis = (uint64_t)-1; + } + } + fill_utg_distance_multi(idx, link, M, bub); + + + buf.n = 0; + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>(64 - idx->uID_bits)); + end = ((hits->a.a[k].e<<1)>>(64 - idx->uID_bits)); + + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + + t_d = get_hic_distance(&(hits->a.a[k]), link, idx); + if(t_d == (uint64_t)-1) continue; + if(beg == end) + { + t_d = (t_d << 1); + } + else + { + beg_status = get_phase_status(hap, beg); + if(beg_status != 1 && beg_status != -1) continue; + end_status = get_phase_status(hap, end); + if(end_status != 1 && end_status != -1) continue; + if(beg_status != end_status) + { + t_d = (t_d << 1) + 1; + } + else + { + t_d = (t_d << 1); + } + } + + kv_push(uint64_t, buf, t_d); + } + + ///might have bias, we may not use right linkage larger than trans rc linkage + radix_sort_hc64(buf.a, buf.a+buf.n); + + for (k = 0, r_idx = f_idx = (uint64_t)-1; k < buf.n; k++) + { + if((buf.a[k]&1) == 0) r_idx = k; + if((buf.a[k]&1) == 1) f_idx = k; + } + buf.n = MIN(r_idx, f_idx); + + for (k = 0; k < buf.n; k++) + { + if((buf.a[k]&1) == 1) + { + kv_push(uint64_t, buf_idx, buf.a[k]>>1); + } + } + + uint64_t cutoff = buf_idx.n * 0.9, t = buf_idx.n * 0.005, pre, step; + k = 0; + if(cutoff >= t) k = cutoff - t; + pre = 0; + if(cutoff >= t + 1) pre = buf_idx.a[cutoff - t - 1]; + for (t_d = i = 0; k < cutoff + t; k++) + { + t_d += (buf_idx.a[k] - pre); + pre = buf_idx.a[k]; + i++; + } + + if(t_d == 0 || i == 0 || t == 0) + { + kv_destroy(buf); + kv_destroy(buf_idx); + return 0; + } + + step = (t_d/i)*20; + + if(step == 0) + { + kv_destroy(buf); + kv_destroy(buf_idx); + return 0; + } + + trans_p_t* p = NULL; + dis->n = 0; + uint64_t step_s = 0, step_e = step; + if(buf.n>0) step_s = buf.a[0]>>1, step_e = (buf.a[0]>>1) + step; + for (k = cnt[0] = cnt[1] = 0; k < buf.n; k++) + { + if((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s) + { + cnt[buf.a[k]&1]++; + } + + if((buf.a[k]>>1) >= step_e) + { + while (!((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s)) + { + kv_pushp(trans_p_t, *dis, &p); + p->beg = step_s; + p->end = step_e; + p->cnt_0 = cnt[0]; + p->cnt_1 = cnt[1]; + step_s += step; + step_e += step; + cnt[0] = cnt[1] = 0; + } + } + } + + if(cnt[0] > 0 || cnt[1] > 0) + { + kv_pushp(trans_p_t, *dis, &p); + p->beg = step_s; + p->end = step_e; + p->cnt_0 = cnt[0]; + p->cnt_1 = cnt[1]; + } + + + uint64_t smooth_step = 20, k_i, cnt_0; + if(dis->n > 0) med = dis->a[dis->n-1].end; + for (k = 0; k+smooth_step < dis->n; k++) + { + for (k_i = cnt_0 = 0; k_i < smooth_step; k_i++) + { + if(dis->a[k+k_i].cnt_0 == 0 || dis->a[k+k_i].cnt_1 == 0) cnt_0++; + } + + if(cnt_0 >= smooth_step * 0.2) + { + med = dis->a[k].beg; + break; + } + } + + + long long b_k = 0, b_i = 0, b_j, pass = 0; + ///for (b_k = b_i = 0; b_k < (long long)dis->n; b_k++) + while(b_k < (long long)dis->n) + { + pass = 1; + beg = dis->a[b_k].beg; + end = dis->a[b_k].end; + cnt[0] = dis->a[b_k].cnt_0; + cnt[1] = dis->a[b_k].cnt_1; + if(cnt[0] > 0 && cnt[1] > 0) + { + dis->a[b_i].beg = beg; + dis->a[b_i].end = end; + dis->a[b_i].cnt_0 = cnt[0]; + dis->a[b_i].cnt_1 = cnt[1]; + b_i++; + b_k++; + continue; + } + + b_k++; + for (b_j = b_k; b_j < (long long)dis->n; b_j++, b_k++) + { + end = dis->a[b_j].end; + cnt[0] += dis->a[b_j].cnt_0; + cnt[1] += dis->a[b_j].cnt_1; + if(cnt[0] > 0 && cnt[1] > 0) break; + } + + + if(b_j < (long long)dis->n) + { + dis->a[b_i].beg = beg; + dis->a[b_i].end = end; + dis->a[b_i].cnt_0 = cnt[0]; + dis->a[b_i].cnt_1 = cnt[1]; + b_i++; + b_k++; + continue; + } + + for(b_j = b_i-1; b_j >= 0; b_j--) + { + beg = dis->a[b_j].beg; + cnt[0] += dis->a[b_j].cnt_0; + cnt[1] += dis->a[b_j].cnt_1; + if(cnt[0] > 0 && cnt[1] > 0) break; + } + + if(b_j >= 0) + { + b_i = b_j; + dis->a[b_i].beg = beg; + dis->a[b_i].end = end; + dis->a[b_i].cnt_0 = cnt[0]; + dis->a[b_i].cnt_1 = cnt[1]; + b_i++; + b_k++; + continue; + } + + pass = 0; + break; + } + + dis->n = b_i; + if(dis->n == 0 || pass == 0) + { + kv_destroy(buf); + kv_destroy(buf_idx); + return 0; + } + + // for (i = 0; i < dis->n; i++) + // { + // if(i > 0 && dis->a[i].beg != dis->a[i-1].end) fprintf(stderr, "ERROR: dis->a[i].beg: %lu, dis->a[i-1].end: %lu\n", dis->a[i].beg, dis->a[i-1].end); + // fprintf(stderr, "beg: %lu, end: %lu, cnt_0: %lu, cnt_1: %lu, error_rate: %f\n", + // dis->a[i].beg, dis->a[i].end, dis->a[i].cnt_0, dis->a[i].cnt_1, (double)(dis->a[i].cnt_1)/(double)(dis->a[i].cnt_1 + dis->a[i].cnt_0)); + // } + + + LeastSquare_advance(dis, idx, med); + + // fprintf(stderr, "idx->a: %f, idx->b: %f, idx->frac: %f, med: %lu\n", + // (double)idx->a, (double)idx->b, (double)idx->frac, med); + + dis->max = dis->a[dis->n-1].end; + + + kv_destroy(buf); + kv_destroy(buf_idx); + + if(idx->a < 0) idx->a = 0; + if(idx->a == 0) + { + idx->b = MAX((((double)(dis->a[dis->n-1].cnt_1))/((double)(dis->a[dis->n-1].cnt_0 + dis->a[dis->n-1].cnt_1))), idx->b); + } + if(idx->b < 0 && get_trans(idx, dis->max) < 0) + { + idx->b = ((double)(dis->a[dis->n-1].cnt_1))/((double)(dis->a[dis->n-1].cnt_0 + dis->a[dis->n-1].cnt_1)); + } + + + // fprintf(stderr, "idx->a: %f, idx->b: %f, idx->frac: %f, med: %lu\n", + // (double)idx->a, (double)idx->b, (double)idx->frac, med); + + return 1; +} + +void init_hic_p(ha_ug_index* idx, kvec_pe_hit* hits, hc_links* link, bubble_type* bub, +kvec_hc_edge* back_hc_edge, MT* M, H_partition* hap, uint32_t ignore_dis) +{ + uint64_t k, i, m, uID, is_comples_weight = 0; + trans_idx dis; + kv_init(dis); + + if(bub->round_id > 0 && ignore_dis == 0) + { + is_comples_weight = get_trans_rate_function(idx, hits, link, bub, M, hap, &dis); + } + + + hc_edge *e = NULL; + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].f.n; k++) + { + if(link->a.a[i].f.a[k].del) continue; + if(link->a.a[i].f.a[k].dis == RC_0) + { + uID = link->a.a[i].f.a[k].uID; + e = get_hc_edge(link, i, uID, 0); + if(e) + { + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = 1; + } + + e = get_hc_edge(link, uID, i, 0); + if(e) + { + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = 1; + } + } + else if(link->a.a[i].f.a[k].dis == RC_1) + { + uID = link->a.a[i].f.a[k].uID; + get_forward_distance(i, uID, idx->ug->g, link, M); + get_forward_distance(uID, i, idx->ug->g, link, M); + } + } + } + + + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + if(link->a.a[i].e.a[k].dis == (uint64_t)-1) + { + e = get_hc_edge(link, link->a.a[i].e.a[k].uID, i, 0); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, link->a.a[i].e.a[k]); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = link->a.a[i].e.a[k].del = 1; + } + } + } + + for (i = 0; i < link->a.n; i++) + { + for (k = m = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[m] = link->a.a[i].e.a[k]; + link->a.a[i].e.a[m].weight = 0; + link->a.a[i].e.a[m].occ = 0; + m++; + } + link->a.a[i].e.n = m; + } + + weight_edges_advance(idx, hits, link, bub, is_comples_weight == 1? &dis : NULL); + + + for (i = 0; i < link->a.n; i++) + { + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + if(link->a.a[i].e.a[k].weight <= 0) + { + e = get_hc_edge(link, link->a.a[i].e.a[k].uID, i, 0); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, link->a.a[i].e.a[k]); + if(back_hc_edge) kv_push(hc_edge, back_hc_edge->a, *e); + e->del = link->a.a[i].e.a[k].del = 1; + } + } + } + + for (i = 0; i < link->a.n; i++) + { + for (k = m = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + link->a.a[i].e.a[m] = link->a.a[i].e.a[k]; + m++; + } + link->a.a[i].e.n = m; + } + + kv_destroy(dis); +} + + +#define is_hap_set(i, Hap) (!!((Hap).hap[(i)]&((Hap).m[0]|(Hap).m[1]|(Hap).m[2]))) +#define is_hap_set_label(i, Hap, label) (is_hap_set((i), (Hap))&&((Hap).hap[(i)]>>(Hap).label_shift)==((label)>>(Hap).label_shift)) + +double get_path_weight(uint32_t query, uint32_t v0, uint32_t root, bub_p_t_warp *b, hc_links* x) +{ + if(v0 == root) return 0; + uint32_t v, u; + hc_edge *p = NULL; + double weight = 0; + v = v0; + do { + u = b->a[v].p; // u->v + p = get_hc_edge(x, query>>1, v>>1, 0); + if(p) weight += p->weight; + v = u; + } while (v != root); + + return weight; +} + +uint32_t get_related_weight(uint32_t x, H_partition* hap, double* w0, double* w1, uint32_t* hap_label) +{ + (*w0) = (*w1) = 0; + if(x >= hap->link->a.n) return 0; + uint32_t i, a_n = hap->link->a.a[x].e.n, occ; + hc_edge* a = hap->link->a.a[x].e.a; + for (i = occ = 0; i < a_n; i++) + { + if(a[i].del) continue; + if(hap_label && (!is_hap_set_label(a[i].uID, *hap, *hap_label))) continue; + if(is_hap_set(a[i].uID, *hap)) occ++; + if((hap->hap[a[i].uID] & hap->m[0])) (*w0)+= a[i].weight; + if((hap->hap[a[i].uID] & hap->m[1])) (*w1)+= a[i].weight; + } + return occ; +} + +void set_path_hap(bub_p_t_warp *b, uint32_t root, H_partition* hap, uint32_t max_hap_label) +{ + uint32_t v, u, label; + double w0 = 0, w1 = 0, cur_w0, cur_w1; + ///v is the sink of this bubble + v = b->S.a[0]; + do { + u = b->a[v].p; // u->v + if(v != b->S.a[0]) + { + get_related_weight(v>>1, hap, &cur_w0, &cur_w1, &max_hap_label); + w0 += cur_w0; w1 += cur_w1; + } + v = u; + } while (v != root); + + if(w0 > w1) + { + label = max_hap_label | hap->m[0]; + b->exist_hap_label = hap->m[0]; + } + else if(w0 < w1) + { + label = max_hap_label | hap->m[1]; + b->exist_hap_label = hap->m[1]; + } + else + { + if(b->exist_hap_label == (uint32_t)-1) + { + label = max_hap_label | hap->m[0]; + b->exist_hap_label = hap->m[0]; + } + else + { + if(b->exist_hap_label == hap->m[0]) + { + label = max_hap_label | hap->m[1]; + b->exist_hap_label = hap->m[1]; + } + else + { + label = max_hap_label | hap->m[0]; + b->exist_hap_label = hap->m[0]; + } + } + } + + + v = b->S.a[0]; + do { + u = b->a[v].p; // u->v + if(v != b->S.a[0]) hap->hap[v>>1] |= label; + v = u; + } while (v != root); +} + + +uint64_t get_phase_path(ma_ug_t *ug, uint32_t s, uint32_t d, bub_p_t_warp *b, H_partition* hap, uint32_t max_hap_label) +{ + asg_t *g = ug->g; + if(g->seq[s>>1].del) return 0; // already deleted + if(get_real_length(g, s, NULL)<2) return 0; + uint32_t i, n_pending, is_first, to_replace, cur_nc, cur_uc, cur_ac, n_tips, tip_end, n_pop; + double cur_nh, cur_w0, cur_w1, cur_rate, max_rate, cur_weight, max_weight; + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[s].d = b->a[s].nc = b->a[s].ac = b->a[s].uc = 0; b->a[s].nh = b->a[s].w[0] = b->a[s].w[1] = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, s); + n_pop = n_tips = n_pending = 0; + tip_end = (uint32_t)-1; + is_first = 1; + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S); + uint32_t d = b->a[v].d, nc = b->a[v].nc, uc = b->a[v].uc, ac = b->a[v].ac; + double nh = b->a[v].nh; + double nw_0 = b->a[v].w[0], nw_1 = b->a[v].w[1]; + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + for (i = 0; i < nv; ++i) { + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l, not overlap length + bub_p_t *t = &b->a[w]; + //got a circle + if ((w>>1) == (s>>1)) goto pop_reset; + //important when poping at long untig graph + if(is_first) l = 0; + if (av[i].del) continue; + ///push the edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + + if (t->s == 0) + { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p is the parent node of + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l, t->nc = nc + ug->u.a[(w>>1)].n; + t->r = get_real_length(g, w^1, NULL); + /**need fix**/ + t->nh = nh + get_path_weight(w, v, s, b, hap->link); + + get_related_weight(w>>1, hap, &(t->w[0]), &(t->w[1]), &max_hap_label); + t->w[0] += nw_0; t->w[1] += nw_1; + + t->ac = ac + ((!is_hap_set_label(w>>1, *hap, max_hap_label))?ug->u.a[(w>>1)].n : 0); + t->uc = uc + ((is_hap_set_label(w>>1, *hap, max_hap_label))?ug->u.a[(w>>1)].n : 0); + + ++n_pending; + } + else { + to_replace = 0; + + cur_nc = nc + ug->u.a[(w>>1)].n; + /**need fix**/ + cur_nh = nh + get_path_weight(w, v, s, b, hap->link); + get_related_weight(w>>1, hap, &cur_w0, &cur_w1, &max_hap_label); + cur_w0 += nw_0; cur_w1 += nw_1; + cur_weight = cur_nh + MAX(cur_w0, cur_w1) - MIN(cur_w0, cur_w1); + max_weight = t->nh + MAX(t->w[0], t->w[1]) - MIN(t->w[0], t->w[1]); + + cur_ac = ac + ((!is_hap_set_label(w>>1, *hap, max_hap_label))?ug->u.a[(w>>1)].n : 0);; + cur_uc = uc + ((is_hap_set_label(w>>1, *hap, max_hap_label))?ug->u.a[(w>>1)].n : 0); + cur_rate = ((double)(cur_ac)/(double)(cur_ac+cur_uc)); + max_rate = ((double)(t->ac)/(double)(t->ac+t->uc)); + + if(cur_rate > max_rate) + { + to_replace = 1; + } + else if(cur_rate == max_rate) + { + ///if(cur_nh > t->nh) + if(cur_weight > max_weight) + { + to_replace = 1; + } + else if(cur_weight == max_weight)///(cur_nh == t->nh) + { + if(cur_nc > t->nc) + { + to_replace = 1; + } + else if(cur_nc == t->nc) + { + if(d + l > t->d) + { + to_replace = 1; + } + } + } + } + + + if(to_replace) + { + t->p = v; + t->nc = cur_nc; + t->nh = cur_nh; + t->ac = cur_ac; + t->uc = cur_uc; + t->w[0] = cur_w0; + t->w[1] = cur_w1; + } + + + if (d + l < t->d) t->d = d + l; // update dist + } + + if (--(t->r) == 0) { + uint32_t x = get_real_length(g, w, NULL); + if(x > 0) + { + kv_push(uint32_t, b->S, w); + } + else + { + ///at most one tip + if(n_tips != 0) goto pop_reset; + n_tips++; + tip_end = w; + } + --n_pending; + } + } + is_first = 0; + + + if(n_tips == 1) + { + if(tip_end != (uint32_t)-1 && n_pending == 0 && b->S.n == 0) + { + ///sink is b.S.a[0] + kv_push(uint32_t, b->S, tip_end); + break; + } + else + { + goto pop_reset; + } + } + + if (i < nv || b->S.n == 0) goto pop_reset; + }while (b->S.n > 1 || n_pending); + + + n_pop = 1; + /**need fix**/ + set_path_hap(b, s, hap, max_hap_label); + pop_reset: + + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + bub_p_t *t = &b->a[b->b.a[i]]; + t->p = t->d = t->nc = t->ac = t->uc = t->r = t->s = 0; + t->nh = t->w[0] = t->w[1] = 0; + } + + return n_pop; +} + +uint32_t get_weightest_hap_label_from_uid(uint64_t x, H_partition* hap, uint8_t* hap_label_flag, +uint32_t* max_hap_label, double* max_hap_weight) +{ + (*max_hap_label) = (uint32_t)-1; + kv_resize(double, hap->label_buffer, (hap->label>>hap->label_shift)+1); + hap->label_buffer.n = (hap->label>>hap->label_shift)+1; + uint32_t i, is_set_ava, is_unset_ava, a_n; + hc_edge* a = NULL; + for (i = 0; i < hap->label_buffer.n; i++) + { + hap->label_buffer.a[i] = 0; + } + + is_set_ava = is_unset_ava = 0; + a_n = hap->link->a.a[x].e.n; + a = hap->link->a.a[x].e.a; + for (i = 0; i < a_n; i++) + { + if(a[i].del) continue; + if(is_hap_set(a[i].uID, *hap)) + { + if(hap_label_flag && hap_label_flag[hap->hap[a[i].uID]] == 0) continue; + hap->label_buffer.a[hap->hap[a[i].uID]>>hap->label_shift] += a[i].weight; + is_set_ava = 1; + } + is_unset_ava = 1; + } + + if(hap_label_flag && is_set_ava == 0) return 0; + if(is_unset_ava == 0) return 0; + if(is_set_ava == 0 && is_unset_ava > 0) + { + hap->label += hap->label_add; + (*max_hap_label) = hap->label; + return 1; + } + + double max_weight; + uint32_t max_i; + for (i = 0, max_weight = -1, max_i = (uint32_t)-1; i < hap->label_buffer.n; i++) + { + if(hap->label_buffer.a[i] > max_weight) + { + max_weight = hap->label_buffer.a[i]; + max_i = i; + } + } + + (*max_hap_label) = max_i<label_shift; + if(max_hap_weight) (*max_hap_weight) = max_weight; + return 1; + +} + +uint32_t get_weightest_hap_label_from_bubble(uint64_t bid, H_partition* hap, bubble_type* bub, +uint8_t* hap_label_flag, uint32_t* max_hap_label, double* max_hap_weight) +{ + (*max_hap_label) = (uint32_t)-1; + + kv_resize(double, hap->label_buffer, (hap->label>>hap->label_shift)+1); + hap->label_buffer.n = (hap->label>>hap->label_shift)+1; + uint32_t i, m, is_set_ava, is_unset_ava, a_n, *x_a, x_n, x; + hc_edge* a = NULL; + for (i = 0; i < hap->label_buffer.n; i++) + { + hap->label_buffer.a[i] = 0; + } + + ///bid might be bubble or non-bubble + get_bubbles(bub, bid, NULL, NULL, &x_a, &x_n, NULL); + for (m = is_set_ava = is_unset_ava = 0; m < x_n; m++) + { + ///x is uid + x = x_a[m]>>1; + a_n = hap->link->a.a[x].e.n; + a = hap->link->a.a[x].e.a; + for (i = 0; i < a_n; i++) + { + if(a[i].del) continue; + if(is_hap_set(a[i].uID, *hap)) + { + if(hap_label_flag && hap_label_flag[hap->hap[a[i].uID]] == 0) continue; ///not at current chain + hap->label_buffer.a[hap->hap[a[i].uID]>>hap->label_shift] += a[i].weight; + is_set_ava = 1; + } + is_unset_ava = 1; + } + } + if(hap_label_flag && is_set_ava == 0) return 0; ///no connection in current chain + + if(is_unset_ava == 0) return 0; ///no any connection + if(is_set_ava == 0 && is_unset_ava > 0) ///update hap->label + { + hap->label += hap->label_add; + (*max_hap_label) = hap->label; + return 1; + } + + double max_weight; + uint32_t max_i; + for (i = 0, max_weight = -1, max_i = (uint32_t)-1; i < hap->label_buffer.n; i++) + { + if(hap->label_buffer.a[i] > max_weight) + { + max_weight = hap->label_buffer.a[i]; + max_i = i; + } + } + + (*max_hap_label) = max_i<label_shift; + if(max_hap_weight) (*max_hap_weight) = max_weight; + return 1; +} + + +uint32_t get_available_com(H_partition* hap, bubble_type* bub, ma_ug_t *ug, uint32_t check_self, uint32_t check_others, +uint8_t* hap_label_flag, uint32_t* max_hap_label) +{ + hc_links* link = hap->link; + uint32_t beg, sink, n, *a, i, j, k, uID, max_bub_i, max_non_bub_i, max_i, is_ava; + uint32_t hap_label, max_bub_label = (uint32_t)-1, max_non_bub_label = (uint32_t)-1; + double w, max_bub_w, max_non_bub_w; + max_i = (uint32_t)-1; + + for (i = 0, max_bub_w = -1, max_bub_i = (uint32_t)-1; i < bub->f_bub/**bub->s_bub**/; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, NULL); + for (j = 0, w = 0; j < n; j++) + { + uID = a[j]>>1; + if(check_self && is_hap_set(uID, *hap)) break; + } + if(j != n) continue; + + is_ava = 0; + hap_label = (uint32_t)-1; + if(check_others) + { + if(get_weightest_hap_label_from_bubble(i, hap, bub, + hap_label_flag, &hap_label, &w)>0) + { + is_ava = 1; + } + } + else + { + for (j = 0, w = 0, is_ava = 0; j < n; j++) + { + uID = a[j]>>1; + for (k = 0; k < link->a.a[uID].e.n; k++) + { + if(link->a.a[uID].e.a[k].del) continue; + w += link->a.a[uID].e.a[k].weight; + is_ava = 1; + } + } + } + + if(is_ava == 0) continue; + + if(w > max_bub_w) + { + max_bub_w = w; + max_bub_i = i; + max_bub_label = hap_label; + } + } + + for (i = 0, max_non_bub_w = -1, max_non_bub_i = (uint32_t)-1; i < ug->u.n; i++) + { + if(IF_HET(i, *bub)/** || (bub->index[i] >= bub->s_bub && bub->index[i] < bub->f_bub)**/) + { + uID = i; + is_ava = 0; + if(check_self && is_hap_set(uID, *hap)) continue; + hap_label = (uint32_t)-1; + if(check_others) + { + if(get_weightest_hap_label_from_uid(uID, hap, hap_label_flag, &hap_label, &w)>0) + { + is_ava = 1; + } + } + else + { + for (k = 0, w = 0, is_ava = 0; k < link->a.a[uID].e.n; k++) + { + if(link->a.a[uID].e.a[k].del) continue; + w += link->a.a[uID].e.a[k].weight; + is_ava = 1; + } + } + + if(is_ava == 0) continue; + + if(w > max_non_bub_w) + { + max_non_bub_w = w; + max_non_bub_i = i; + max_non_bub_label = hap_label; + } + } + } + + if(max_bub_i != (uint32_t)-1 && max_non_bub_i != (uint32_t)-1) + { + if(max_non_bub_w > max_bub_w) + { + max_i = (max_non_bub_i << 1) + 1; + w = max_non_bub_w; + (*max_hap_label) = max_non_bub_label; + } + else + { + max_i = (max_bub_i << 1); + w = max_bub_w; + (*max_hap_label) = max_bub_label; + } + } + else if(max_bub_i != (uint32_t)-1) + { + max_i = (max_bub_i << 1); + w = max_bub_w; + (*max_hap_label) = max_bub_label; + } + else if(max_non_bub_i != (uint32_t)-1) + { + max_i = (max_non_bub_i << 1) + 1; + w = max_non_bub_w; + (*max_hap_label) = max_non_bub_label; + } + + if(max_i == (uint32_t)-1) + { + for (i = 0, max_non_bub_w = -1, max_non_bub_i = (uint32_t)-1; i < ug->u.n; i++) + { + ///if(bub->index[i] < bub->s_bub) + if(IF_BUB(i, *bub)) + { + + uID = i; + is_ava = 0; + if(check_self && is_hap_set(uID, *hap)) continue; + hap_label = (uint32_t)-1; + if(check_others) + { + if(get_weightest_hap_label_from_uid(uID, hap, hap_label_flag, &hap_label, &w)>0) + { + is_ava = 1; + } + } + else + { + for (k = 0, w = 0, is_ava = 0; k < link->a.a[uID].e.n; k++) + { + if(link->a.a[uID].e.a[k].del) continue; + w += link->a.a[uID].e.a[k].weight; + is_ava = 1; + } + } + + if(is_ava == 0) continue; + + if(w > max_non_bub_w) + { + max_non_bub_w = w; + max_non_bub_i = i; + max_non_bub_label = hap_label; + } + + } + } + + if(max_non_bub_i != (uint32_t)-1) + { + max_i = (max_non_bub_i << 1) + 1; + w = max_non_bub_w; + (*max_hap_label) = max_non_bub_label; + } + } + + // if(max_i == (uint32_t)-1) + // { + // fprintf(stderr, "-Cannot find!\n"); + // } + // else if(max_i & 1) + // { + // fprintf(stderr, "-utg-%uth, phasing ID: %u, w: %f, max_bub_i: %u, max_bub_w: %f, max_non_bub_i: %u, max_non_bub_w: %f\n", + // max_i>>1, hap->label>>3, w, max_bub_i, max_bub_w, max_non_bub_i, max_non_bub_w); + // } + // else + // { + // fprintf(stderr, "-bubble-%uth, phasing ID: %u, w: %f, max_bub_i: %u, max_bub_w: %f, max_non_bub_i: %u, max_non_bub_w: %f\n", + // max_i>>1, hap->label>>3, w, max_bub_i, max_bub_w, max_non_bub_i, max_non_bub_w); + // } + + return max_i; +} + +void reset_ambiguous_label(H_partition* hap, uint8_t* hap_label_flag, uint32_t uID) +{ + uint32_t hap_label = (uint32_t)-1; + if(get_weightest_hap_label_from_uid(uID, hap, hap_label_flag, &hap_label, NULL)>0) + { + double cur_w0, cur_w1; + get_related_weight(uID, hap, &cur_w0, &cur_w1, &hap_label); + if(cur_w0 >= cur_w1) + { + hap->hap[uID] |= (hap_label | hap->m[0]); + } + else + { + hap->hap[uID] |= (hap_label | hap->m[1]); + } + } +} + +uint32_t get_unset_com(H_partition* hap, bubble_type* bub, ma_ug_t *ug, uint8_t* hap_label_flag, uint32_t* max_hap_label) +{ + uint32_t max_i = get_available_com(hap, bub, ug, 1, 1, hap_label_flag, max_hap_label); + + if(max_i == (uint32_t)-1) + { + max_i = get_available_com(hap, bub, ug, 1, 0, hap_label_flag, max_hap_label); + if(max_i != (uint32_t)-1) + { + hap->label += hap->label_add; + (*max_hap_label) = hap->label; + } + } + + return max_i; +} + +void phase_com(H_partition* hap, ma_ug_t *ug, bub_p_t_warp* b, bubble_type* bub, uint32_t bid, uint32_t max_hap_label) +{ + + if((bid & 1) == 0) ///bubble + { + uint32_t beg = (uint32_t)-1, sink = (uint32_t)-1, n, *a; + get_bubbles(bub, bid>>1, &beg, &sink, &a, &n, NULL); + ///fprintf(stderr, "+bubble-%uth, beg: %u, sink: %u, phasing ID: %u\n", bid>>1, beg>>1, sink>>1, hap->label>>3); + b->exist_hap_label = (uint32_t)-1; + get_phase_path(ug, beg, sink, b, hap, max_hap_label); + get_phase_path(ug, beg, sink, b, hap, max_hap_label); + ///fprintf(stderr, "-bubble-%uth, beg: %u, sink: %u, phasing ID: %u\n", bid>>1, beg>>1, sink>>1, hap->label>>3); + } + else + { + double cur_w0, cur_w1; + get_related_weight(bid>>1, hap, &cur_w0, &cur_w1, &max_hap_label); + ///fprintf(stderr, "utg-%uth, phasing ID: %u\n", bid>>1, hap->label>>3); + if(cur_w0 >= cur_w1) + { + hap->hap[bid>>1] |= (max_hap_label | hap->m[0]); + } + else + { + hap->hap[bid>>1] |= (max_hap_label | hap->m[1]); + } + } +} + + +double get_cluster_weight(H_partition* hap, hc_links* link, uint32_t *h, uint32_t h_n) +{ + int o_d = 0; + double weight = 0; + uint32_t j, k, m; + for (j = 0, weight = 0; j < h_n; j++) + { + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + if(link->a.a[h[j]].e.a[k].del) continue; + for (m = 0; m < h_n; m++) + { + if(h[m] == link->a.a[h[j]].e.a[k].uID) break; + } + if(m < h_n) continue; + + o_d = get_phase_status(hap, link->a.a[h[j]].e.a[k].uID); + if(o_d < -1) continue; + ///if(o_d < -1) fprintf(stderr, "ERROR\n"); + weight += (o_d*link->a.a[h[j]].e.a[k].weight); + } + } + + return weight; +} + + +double get_cluster_inner_weight(H_partition* hap, hc_links* link, uint32_t *h0, uint32_t h0_n, +uint32_t *h1, uint32_t h1_n) +{ + double weight = 0; + uint32_t j, k, m; + for (j = 0, weight = 0; j < h0_n; j++) + { + for (k = 0; k < link->a.a[h0[j]].e.n; k++) + { + if(link->a.a[h0[j]].e.a[k].del) continue; + for (m = 0; m < h1_n; m++) + { + if(h1[m] == link->a.a[h0[j]].e.a[k].uID) break; + } + if(m == h1_n) continue; + + weight += link->a.a[h0[j]].e.a[k].weight; + } + } + + return weight * 2; +} + +void update_partition_flag(H_partition* h, G_partition* g_p, hc_links* link, uint32_t id) +{ + uint32_t k, *h0, h0_n, *h1, h1_n, uID, flag = 0; + int status; + get_phased_block(g_p, NULL, id, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + + status = g_p->a[id].status[0]; + if(status == 1) flag = h->m[0]; + else if(status == -1) flag = h->m[1]; + else if(status == 0) flag = h->m[2]; + else if(status == -2) flag = 0; + for (k = 0; k < h0_n; k++) + { + uID = h0[k]; + h->hap[uID] >>= 3; + h->hap[uID] <<= 3; + h->hap[uID] |= flag; + } + + status = g_p->a[id].status[1]; + if(status == 1) flag = h->m[0]; + else if(status == -1) flag = h->m[1]; + else if(status == 0) flag = h->m[2]; + else if(status == -2) flag = 0; + for (k = 0; k < h1_n; k++) + { + uID = h1[k]; + h->hap[uID] >>= 3; + h->hap[uID] <<= 3; + h->hap[uID] |= flag; + } +} + + +void update_partition_flag_debug(H_partition* h, G_partition* g_p, hc_links* link, uint32_t id) +{ + uint32_t k, *h0, h0_n, *h1, h1_n, uID, flag = 0; + int status; + get_phased_block(g_p, NULL, id, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + + status = g_p->a[id].status[0]; + if(status == 1) flag = h->m[0]; + else if(status == -1) flag = h->m[1]; + else if(status == 0) flag = h->m[2]; + else if(status == -2) flag = 0; + for (k = 0; k < h0_n; k++) + { + uID = h0[k]; + if(flag != (h->hap[uID]&7)) fprintf(stderr, "h0, id: %u, uID: %u, pre_flag: %u, cur_flag: %u\n", id, uID, (h->hap[uID]&7), flag); + h->hap[uID] >>= 3; + h->hap[uID] <<= 3; + h->hap[uID] |= flag; + } + + status = g_p->a[id].status[1]; + if(status == 1) flag = h->m[0]; + else if(status == -1) flag = h->m[1]; + else if(status == 0) flag = h->m[2]; + else if(status == -2) flag = 0; + for (k = 0; k < h1_n; k++) + { + uID = h1[k]; + if(flag != (h->hap[uID]&7)) fprintf(stderr, "h1, id: %u, uID: %u, pre_flag: %u, cur_flag: %u\n", id, uID, (h->hap[uID]&7), flag); + h->hap[uID] >>= 3; + h->hap[uID] <<= 3; + h->hap[uID] |= flag; + } +} + +void print_contig_partition(H_partition* hap, const char* debug) +{ + uint32_t i; + int status; + for (i = 0; i < hap->n; i++) + { + status = get_phase_status(hap, i); + fprintf(stderr, "%s\tutg%.6d\tP:%u\tHG:A:%d\n", debug, (int)(i+1), hap->hap[i]>>3, status); + } +} + +void adjust_contig_partition(H_partition* hap, hc_links* link) +{ + uint32_t i, k, *h0, h0_n, *h1, h1_n; + uint32_t h0_status[4], h1_status[4], h0_status_max; + int h0_h, h1_h; + for (i = 0; i < hap->g_p.n; i++) + { + + get_phased_block(&(hap->g_p), NULL, i, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + hap->g_p.a[i].status[0] = hap->g_p.a[i].status[1] = -2; + hap->g_p.a[i].weight[0] = hap->g_p.a[i].weight[1] = hap->g_p.a[i].weight_convex = 0; + + + h0_status[0] = h0_status[1] = h0_status[2] = h0_status[3] = 0; + for (k = 0; k < h0_n; k++) + { + h0_status[get_phase_status(hap, h0[k])+2]++; + } + + hap->g_p.a[i].weight[0] = get_cluster_weight(hap, link, h0, h0_n); + if(h1_n == 0) + { + if(h0_status[0] == h0_n) ///unset, flag = -2 + { + hap->g_p.a[i].status[0] = -2; + } + else + { + if(h0_status[1] > 0 || h0_status[3] > 0) ///phased flag = 1/-1 + { + h0_status[0] = h0_status[2] = 0; + + h0_h = -2; + h0_status_max = 0; + for (k = 0; k < 4; k++) + { + if(h0_status[k] > h0_status_max) + { + h0_status_max = h0_status[k]; + h0_h = (int)(k) - 2; + } + } + hap->g_p.a[i].status[0] = h0_h; + } + else if(h0_status[2] > 0) ///hom flag + { + hap->g_p.a[i].status[0] = 0; + } + else //unset flag + { + hap->g_p.a[i].status[0] = -2; + } + } + } + else + { + hap->g_p.a[i].weight[1] = get_cluster_weight( hap, link, h1, h1_n); + h1_status[0] = h1_status[1] = h1_status[2] = h1_status[3] = 0; + for (k = 0; k < h1_n; k++) + { + h1_status[get_phase_status(hap, h1[k])+2]++; + } + + + h0_h = h1_h = 0; + for (k = 0; k < 4; k++) + { + if(h0_status[k] == h0_n) h0_h = (int)(k) - 2; + if(h1_status[k] == h1_n) h1_h = (int)(k) - 2; + } + + if(h0_h * h1_h == -1) + { + hap->g_p.a[i].status[0] = h0_h; + hap->g_p.a[i].status[1] = h1_h; + } + else + { + if(hap->g_p.a[i].weight[0] >= hap->g_p.a[i].weight[1]) + { + hap->g_p.a[i].status[0] = 1; + hap->g_p.a[i].status[1] = -1; + } + else + { + hap->g_p.a[i].status[0] = -1; + hap->g_p.a[i].status[1] = 1; + } + } + } + hap->g_p.a[i].weight_convex = get_cluster_inner_weight(hap, link, h0, h0_n, h1, h1_n); + update_partition_flag(hap, &(hap->g_p), link, i); + ///update_partition_flag_debug(hap, &(hap->g_p), link, i); + } + + for (i = 0; i < hap->g_p.n; i++) + { + get_phased_block(&(hap->g_p), NULL, i, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + hap->g_p.a[i].weight[0] = get_cluster_weight(hap, link, h0, h0_n); + hap->g_p.a[i].weight[1] = get_cluster_weight(hap, link, h1, h1_n); + hap->g_p.a[i].weight_convex = get_cluster_inner_weight(hap, link, h0, h0_n, h1, h1_n); + } +} + +uint32_t get_weightest_uid(uint32_t* a, uint32_t n, H_partition* hap, uint8_t* hap_label_flag, +uint32_t* max_hap_label) +{ + double cur_w0, cur_w1, max_weight; + uint32_t j, max_i, is_ava, hap_label; + + for (j = is_ava = 0, max_i = (uint32_t)-1; j < n; j++) + { + if(is_hap_set(a[j]>>1, *hap)) ///avoid repeat phasing + { + continue; + } + + get_weightest_hap_label_from_uid(a[j]>>1, hap, hap_label_flag, &hap_label, NULL); + + if(hap_label == (uint32_t)-1) + { + continue; + } + + if(get_related_weight(a[j]>>1, hap, &cur_w0, &cur_w1, &hap_label) > 0) + { + if(max_i == (uint32_t)-1) + { + max_i = j; + max_weight = cur_w0 + cur_w1; + (*max_hap_label) = hap_label; + } + else if((cur_w0 + cur_w1) > max_weight) + { + max_i = j; + max_weight = cur_w0 + cur_w1; + (*max_hap_label) = hap_label; + } + is_ava++; + } + } + + ///no useful unitig + if(is_ava == 0) + { + for (j = is_ava = 0, max_i = (uint32_t)-1; j < n; j++) + { + if(is_hap_set(a[j]>>1, *hap)) + { + continue; + } + + if(get_weightest_hap_label_from_uid(a[j]>>1, hap, NULL, &hap_label, NULL)==0) continue; + + if(get_related_weight(a[j]>>1, hap, &cur_w0, &cur_w1, &hap_label) > 0) + { + if(max_i == (uint32_t)-1) + { + max_i = j; + max_weight = cur_w0 + cur_w1; + (*max_hap_label) = hap_label; + } + else if((cur_w0 + cur_w1) > max_weight) + { + max_i = j; + max_weight = cur_w0 + cur_w1; + (*max_hap_label) = hap_label; + } + is_ava++; + } + } + } + if(max_i == (uint32_t)-1) return max_i; + + return a[max_i]>>1; +} + + +void get_weightest_hap_label_from_chain(ma_utg_t *u, H_partition* hap, bubble_type* bub, +uint32_t* max_hap_label, uint32_t* max_bid_idx, uint32_t* is_forward_first) +{ + (*max_hap_label) = (uint32_t)-1; + if(max_bid_idx) (*max_bid_idx) = 0; + if(is_forward_first) (*is_forward_first) = 1; + + if(u->n == 0) return; + kv_resize(double, hap->label_buffer, (hap->label>>hap->label_shift)+1); + hap->label_buffer.n = (hap->label>>hap->label_shift)+1; + uint32_t i, k, m, is_ava, a_n, *x_a, x_n, x; + uint64_t bid; + hc_edge* a = NULL; + for (i = 0; i < hap->label_buffer.n; i++) + { + hap->label_buffer.a[i] = 0; + } + + for (k = is_ava = 0; k < u->n; k++) + { + bid = u->a[k]>>33; + ///bid might be bubble or non-bubble + get_bubbles(bub, bid, NULL, NULL, &x_a, &x_n, NULL); + for (m = 0; m < x_n; m++) + { + ///x is uid + x = x_a[m]>>1; + a_n = hap->link->a.a[x].e.n; + a = hap->link->a.a[x].e.a; + for (i = 0; i < a_n; i++) + { + if(a[i].del) continue; + if(is_hap_set(a[i].uID, *hap)) + { + hap->label_buffer.a[hap->hap[a[i].uID]>>hap->label_shift] += a[i].weight; + is_ava = 1; + } + } + } + } + + if(is_ava == 0) return; ///this is a totally new chain + double max_weight; + uint32_t max_i; + for (i = 0, max_weight = -1, max_i = (uint32_t)-1; i < hap->label_buffer.n; i++) + { + if(hap->label_buffer.a[i] > max_weight) + { + max_weight = hap->label_buffer.a[i]; + max_i = i; + } + } + + (*max_hap_label) = max_i<label_shift; + if(max_bid_idx) + { + double current_weight, tot_w = 0, half_w = 0; + for (k = 0, max_weight = -1, max_i = (uint32_t)-1; k < u->n; k++) + { + bid = u->a[k]>>33; + ///bid might be bubble or non-bubble + get_bubbles(bub, bid, NULL, NULL, &x_a, &x_n, NULL); + for (m = 0, current_weight = 0; m < x_n; m++) + { + x = x_a[m]>>1; + a_n = hap->link->a.a[x].e.n; + a = hap->link->a.a[x].e.a; + for (i = 0; i < a_n; i++) + { + if(a[i].del) continue; + if(is_hap_set_label(a[i].uID, *hap, *max_hap_label)) + { + current_weight += a[i].weight; + } + } + } + + if(current_weight > max_weight) + { + max_weight = current_weight; + max_i = k; + half_w = 0; + } + tot_w += current_weight; + half_w += current_weight; + } + + (*max_bid_idx) = max_i; + if(is_forward_first) + { + if(half_w >= (tot_w - half_w)) + { + (*is_forward_first) = 1; + } + else + { + (*is_forward_first) = 0; + } + } + } + + return; +} + +void phase_bubble_chain_dir(H_partition* hap, ma_ug_t *ug, bub_p_t_warp* b, bubble_type* bub, +ma_utg_t *u, uint8_t* hap_label_flag, uint32_t beg_idx, uint32_t end_idx, uint32_t is_forward) +{ + uint32_t i, j, beg, sink, *a, n, max_hap_label, max_uid; + uint64_t bid; + double cur_w0, cur_w1; + for (i = beg_idx; i <= end_idx; i++) + { + bid = is_forward? u->a[i]>>33:u->a[u->n-i-1]>>33; + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + + if(bub->b_g->seq[bid].c != HAP_LABLE/** && bid < bub->s_bub**/) ///simple bubble + { + for (j = 0; j < n; j++) ///avoiding repeat phasing + { + if(is_hap_set(a[j]>>1, *hap)) break; + } + + if(j < n) goto complete; + + ///get current max_hap_label from current chain + get_weightest_hap_label_from_bubble(bid, hap, bub, hap_label_flag, &max_hap_label, NULL); + + + ///three levels: + ///1. has setted weight with same hap label (using current max_hap_label) + ///2. has setted weight but with different hap labels (using max max_hap_label from current weight) + ///3. has unsetted weight (add hap->hap_label) + ///4. skip, do nothing + if(max_hap_label == (uint32_t)-1 && + get_weightest_hap_label_from_bubble(bid, hap, bub, NULL, &max_hap_label, NULL) == 0) + { + goto complete; + } + + ///phase bubble + b->exist_hap_label = (uint32_t)-1; + get_phase_path(ug, beg, sink, b, hap, max_hap_label); + get_phase_path(ug, beg, sink, b, hap, max_hap_label); + for (j = 0; j < n; j++) ///set bubble as visited + { + hap_label_flag[a[j]>>1] = 1; + } + } + else + { + ///select unitig with highest related weight at one time + while (1) + { + max_uid = get_weightest_uid(a, n, hap, hap_label_flag, &max_hap_label); + if(max_uid == (uint32_t)-1) break; + + get_related_weight(max_uid, hap, &cur_w0, &cur_w1, &max_hap_label); + if(cur_w0 >= cur_w1) + { + hap->hap[max_uid] |= (max_hap_label | hap->m[0]); + } + else + { + hap->hap[max_uid] |= (max_hap_label | hap->m[1]); + } + + hap_label_flag[max_uid] = 1; + } + } + + complete:; + } +} +///ignore unitigs wihci have already been labeled in current chain (might happen) +void phase_bubble_chain(H_partition* hap, ma_ug_t *ug, bub_p_t_warp* b, bubble_type* bub, + uint8_t* hap_label_flag, uint32_t chain_id) +{ + ma_utg_t *u = &(bub->b_ug->u.a[chain_id]); + if(u->n == 0) return; + uint32_t is_forward = 1, i, max_hap_label, max_bid_idx; + memset(hap_label_flag, 0, ug->g->n_seq); + + get_weightest_hap_label_from_chain(u, hap, bub, &max_hap_label, &max_bid_idx, &is_forward); + ///fprintf(stderr, "\n######max_bid_idx: %u, is_forward: %u, max_hap_label: %u\n", max_bid_idx, is_forward, max_hap_label); + if(max_hap_label != (uint32_t)-1) ///means this is not a new chain + { + for (i = 0; i < hap->n; i++) + { + if(is_hap_set_label(i, *hap, max_hap_label)) hap_label_flag[i] = 1; + } + } + + if(max_hap_label == (uint32_t)-1) ///a totally new chain + { + phase_bubble_chain_dir(hap, ug, b, bub, u, hap_label_flag, 0, u->n - 1, 1); + } + else + { + if(max_bid_idx == 0) + { + phase_bubble_chain_dir(hap, ug, b, bub, u, hap_label_flag, 0, u->n - 1, 1); + } + else + { + if(is_forward) + { + phase_bubble_chain_dir(hap, ug, b, bub, u, hap_label_flag, max_bid_idx, u->n - 1, 1); + phase_bubble_chain_dir(hap, ug, b, bub, u, hap_label_flag, 0, max_bid_idx-1, 0); + } + else + { + phase_bubble_chain_dir(hap, ug, b, bub, u, hap_label_flag, 0, max_bid_idx-1, 0); + phase_bubble_chain_dir(hap, ug, b, bub, u, hap_label_flag, max_bid_idx, u->n - 1, 1); + } + } + } +} + + +uint32_t if_flip(H_partition* h, G_partition* g_p, hc_links* link, +bubble_type* bub, uint32_t gid) +{ + double weight = 0; + if(h->lock[gid]) return 0; + if(g_p->a[gid].h[0] > 0 && (g_p->a[gid].status[0] == 1 || g_p->a[gid].status[0] == -1)) + { + weight += (g_p->a[gid].weight[0] * g_p->a[gid].status[0]); + } + + if(g_p->a[gid].h[1] > 0 && (g_p->a[gid].status[1] == 1 || g_p->a[gid].status[1] == -1)) + { + weight += (g_p->a[gid].weight[1] * g_p->a[gid].status[1]); + } + weight += g_p->a[gid].weight_convex*2; + if(weight >= 0) return 0; + return 1; +} +void flip_unitig(G_partition* g_p, hc_links* link, bubble_type* bub, uint32_t id); +uint32_t phasing_improvement(H_partition* h, G_partition* g_p, ha_ug_index* idx, bubble_type* bub); +uint32_t get_max_unitig(H_partition* h, G_partition* g_p, hc_links* link, bubble_type* bub); +double get_cluster_weight_debug(G_partition* g_p, hc_links* link, uint32_t *h, uint32_t h_n); + +void debug_flip(G_partition* g_p, hc_links* link, bubble_type* bub, uint32_t id) +{ + fprintf(stderr, "33333333333\n"); + uint32_t *h0, h0_n, *h1, h1_n, k, wrong; + double hw0, hw1; + for (k = wrong = 0; k < g_p->n; k++) + { + hw0 = hw1 = 0; + get_phased_block(g_p, NULL, k, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + if(h0_n >0) hw0 = get_cluster_weight_debug(g_p, link, h0, h0_n); + if(h1_n >0) hw1 = get_cluster_weight_debug(g_p, link, h1, h1_n); + if(hw0 != g_p->a[k].weight[0]) + { + if((uint32_t)hw0 != (uint32_t)g_p->a[k].weight[0]) wrong = 1; + fprintf(stderr, "k: %u, ERROR(id: %u): hw0: %f, weight[0]: %f\n", k, id, hw0, g_p->a[k].weight[0]); + } + + if(hw1 != g_p->a[k].weight[1]) + { + if((uint32_t)hw1 != (uint32_t)g_p->a[k].weight[1]) wrong = 1; + fprintf(stderr, "k: %u, ERROR(id: %u): hw1: %f, weight[1]: %f\n", k, id, hw1, g_p->a[k].weight[1]); + } + + if(wrong) break; + } +} +void merge_phase_group_by_chain(H_partition* hap, G_partition* g_p, bubble_type* bub, uint32_t chain_id) +{ + uint32_t i, k; + uint32_t beg, sink, *a, n, pre_id, hap_label_id; + uint64_t bid, uid; + ma_utg_t *u = &(bub->b_ug->u.a[chain_id]); + + for (i = 0, pre_id = (uint32_t)-1; i < u->n; i++) + { + // fprintf(stderr, "inner i: %u, u->n: %u\n", i, (uint32_t)u->n); + bid = u->a[i]>>33; ///here is a bubble + // fprintf(stderr, "bid: %lu\n", bid); + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + for (k = 0; k < n; k++) + { + // fprintf(stderr, "k: %u, n: %u\n", k, n); + uid = a[k]>>1; + // fprintf(stderr, "uid: %lu\n", uid); + if(g_p->index[uid] == (uint32_t)-1) ///mean this unitig doesn't have hap label + { + pre_id = (uint32_t)-1; + continue; + } + hap_label_id = g_p->index[uid]>>1; + // fprintf(stderr, "hap_label_id: %u, hap->n: %lu\n", hap_label_id, hap->n); + if(hap_label_id == pre_id) continue; + pre_id = hap_label_id; + if(hap->lock[hap_label_id] == 1) continue; + if(if_flip(hap, g_p, hap->link, bub, hap_label_id)) + { + // fprintf(stderr, "2222222222\n"); + flip_unitig(g_p, hap->link, bub, hap_label_id); + ///debug_flip(g_p, hap->link, bub, hap_label_id); + } + } + } +} +/** +double get_add_weight(H_partition* h, G_partition* g_p, hc_links* link, block_phase_type* block, +bubble_type* bub, uint32_t gid) +{ + double weight = 0; + if(g_p->a[gid].h[0] > 0 && (g_p->a[gid].status[0] == 1 || g_p->a[gid].status[0] == -1)) + { + weight += (g_p->a[gid].weight[0] * g_p->a[gid].status[0]); + } + + if(g_p->a[gid].h[1] > 0 && (g_p->a[gid].status[1] == 1 || g_p->a[gid].status[1] == -1)) + { + weight += (g_p->a[gid].weight[1] * g_p->a[gid].status[1]); + } +} + +void update_block_weight(H_partition* hap, G_partition* g_p, bubble_type* bub, block_phase_type* block, +uint64_t bid) +{ + uint32_t beg, sink, k, uid, *a, n, gid; + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + for (k = 0; k < n; k++) + { + uid = a[k]>>1; + if(g_p->index[uid] == (uint32_t)-1) continue; + gid = g_p->index[uid]>>1; + if(block->vis.a[gid]) continue; + + } + +} + +**/ + +void print_phase_group(G_partition* g_p, bubble_type* bub, const char* command) +{ + uint32_t i, k; + partition_warp *res = NULL; + for (i = 0; i < g_p->n; i++) + { + res = &(g_p->a[i]); + fprintf(stderr, "\n%s: %u-th group: # %d = %u (weight: %f), # %d = %u (weight: %f), inner_weight: %f\n", command, i, + res->status[0], res->h[0], res->weight[0], res->status[1], res->h[1], res->weight[1], res->weight_convex); + for (k = 0; k < res->h[0]; k++) + { + fprintf(stderr, "%d: utg%.6ul\n", res->status[0], int(res->a.a[k]+1)); + } + + for (; k < res->a.n; k++) + { + fprintf(stderr, "%d: utg%.6ul\n", res->status[1], int(res->a.a[k]+1)); + } + } +} + + +void set_bubble(H_partition* hap, G_partition* g_p, bubble_type* bub, block_phase_type* block, +uint64_t bid) +{ + uint32_t beg, sink, k, uid, *a, n, gid; + block->weight = 0; + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + for (k = 0; k < n; k++) + { + uid = a[k]>>1; + if(g_p->index[uid] == (uint32_t)-1) continue; + gid = g_p->index[uid]>>1; + block->vis.a[gid] = 1; + } +} + +uint32_t next_hap_label_id(block_phase_type* b, G_partition* g_p, bubble_type* bub, ma_utg_t *u, +int is_forward, long long* c_bid, long long* c_uid) +{ + uint32_t beg, sink, uid, *a, n, gid, pre_gid; + while (1) ///while(b->bid < (long long)u->n) + { + if(is_forward == 1 && b->bid >= (long long)u->n) break; + if(is_forward == 0 && b->bid < 0) break; + + get_bubbles(bub, u->a[b->bid]>>33, &beg, &sink, &a, &n, NULL); + while (1) ///while (b->uid < (long long)n) + { + if(is_forward == 1 && b->uid >= (long long)n) break; + if(is_forward == 0 && b->uid < 0) break; + + uid = a[b->uid]>>1; + gid = (uint32_t)-1; + if(g_p->index[uid] != (uint32_t)-1) + { + gid = g_p->index[uid]>>1; + } + if(c_bid) (*c_bid) = b->bid; + if(c_uid) (*c_uid) = b->uid; + + if(is_forward == 1) b->uid++; + if(is_forward == 0) b->uid--; + if(gid == (uint32_t)-1) continue; + + + pre_gid = uid = (uint32_t)-1; + if(is_forward == 1 && (b->uid >= 2)) uid = a[b->uid - 2]>>1; + if(is_forward == 0 && (b->uid + 2 < n)) uid = a[b->uid + 2]>>1; + if(uid != (uint32_t)-1 && g_p->index[uid] != (uint32_t)-1) pre_gid = g_p->index[uid]>>1; + + if(pre_gid == gid) continue; + + return gid; + } + + if(is_forward == 1) b->bid++, b->uid = 0; + if(is_forward == 0) b->bid--, b->uid = (long long)n - (long long)1; + } + + return (uint32_t)-1; +} + + +double get_new_weight(G_partition* g_p, uint8_t* flag, hc_links* link, uint32_t gid) +{ + double total_weight = 0, weight; + int status, o_d; + uint32_t *h0, h0_n, *h1, h1_n, *h, h_n, j, k, uID; + + if(g_p->a[gid].h[0] > 0 && (g_p->a[gid].status[0] == 1 || g_p->a[gid].status[0] == -1)) + { + total_weight += (g_p->a[gid].weight[0] * g_p->a[gid].status[0]); + } + + if(g_p->a[gid].h[1] > 0 && (g_p->a[gid].status[1] == 1 || g_p->a[gid].status[1] == -1)) + { + total_weight += (g_p->a[gid].weight[1] * g_p->a[gid].status[1]); + } + total_weight += g_p->a[gid].weight_convex*2; + + + + get_phased_block(g_p, NULL, gid, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + + h = h0; h_n = h0_n; status = g_p->a[gid].status[0]; + for (j = 0, weight = 0; j < h_n; j++) + { + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + if(link->a.a[h[j]].e.a[k].del) continue; + uID = link->a.a[h[j]].e.a[k].uID; + if(g_p->index[uID] == (uint32_t)-1) continue; + if(flag[g_p->index[uID]>>1] == 0) continue; + if((g_p->index[uID]>>1) == gid) continue; + o_d = g_p->a[g_p->index[uID]>>1].status[g_p->index[uID]&1]; + weight += (status*o_d*link->a.a[h[j]].e.a[k].weight); + } + } + total_weight -= (2*weight); + + + + + h = h1; h_n = h1_n; status = g_p->a[gid].status[1]; + for (j = 0, weight = 0; j < h_n; j++) + { + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + if(link->a.a[h[j]].e.a[k].del) continue; + uID = link->a.a[h[j]].e.a[k].uID; + if(g_p->index[uID] == (uint32_t)-1) continue; + if(flag[g_p->index[uID]>>1] == 0) continue; + if((g_p->index[uID]>>1) == gid) continue; + o_d = g_p->a[g_p->index[uID]>>1].status[g_p->index[uID]&1]; + weight += (status*o_d*link->a.a[h[j]].e.a[k].weight); + } + } + total_weight -= (2*weight); + + + + return total_weight; +} + +int identify_best_interval(block_phase_type* i_buf, uint8_t* lock, G_partition* g_p, bubble_type* bub, +ma_utg_t *u, hc_links* link, long long f_bid, long long f_uid, long long* l_bid, long long* l_uid) +{ + long long c_bid, c_uid, min_bid, min_uid; + double w = 0, min_w = 1; + uint32_t gid, val = 0; + block_phase_type b; + b.bid = f_bid; b.uid = f_uid; + memset(i_buf->vis.a, 0, g_p->n); + i_buf->weight = min_w = 1; min_bid = min_uid = -1; + (*l_bid) = (*l_uid) = -1; + + while (1) + { + gid = next_hap_label_id(&b, g_p, bub, u, 1, &c_bid, &c_uid); + if(gid == (uint32_t)-1) break; + if(i_buf->vis.a[gid] == 1) continue; + w += get_new_weight(g_p, i_buf->vis.a, link, gid); + i_buf->vis.a[gid] = 1; + if(lock[gid] == 0) val = 1; + if(val == 0) continue; + + if(min_w > w) + { + min_w = w; + min_bid = c_bid; + min_uid = c_uid; + } + } + + if(min_w < 0 && min_bid != -1 && min_uid != -1) + { + (*l_bid) = min_bid; (*l_uid) = min_uid; + i_buf->weight = min_w; + ///fprintf(stderr, "+min_w: %f, min_bid: %lld, min_uid: %lld\n", min_w, min_bid, min_uid); + } + + if(val == 0) return 1; + + return 0; +} + + +void identify_best_interval_debug(block_phase_type* i_buf, G_partition* g_p, bubble_type* bub, +ma_utg_t *u, hc_links* link, long long f_bid, long long f_uid, long long* l_bid, long long* l_uid) +{ + long long c_bid, c_uid, min_bid, min_uid; + double w = 0, min_w = 1; + uint32_t gid; + block_phase_type b; + b.bid = f_bid; b.uid = f_uid; + memset(i_buf->vis.a, 0, g_p->n); + min_w = 1; min_bid = min_uid = -1; + (*l_bid) = (*l_uid) = -1; + + while (1) + { + gid = next_hap_label_id(&b, g_p, bub, u, 1, &c_bid, &c_uid); + if(gid == (uint32_t)-1) break; + if(i_buf->vis.a[gid] == 1) continue; + w += get_new_weight(g_p, i_buf->vis.a, link, gid); + if(f_bid == 689 && f_uid == 1) + { + fprintf(stderr, "gid: %u, w: %f\n", gid, w); + } + + if(min_w > w) + { + min_w = w; + min_bid = c_bid; + min_uid = c_uid; + } + i_buf->vis.a[gid] = 1; + } + + if(min_w < 0 && min_bid != -1 && min_uid != -1) + { + (*l_bid) = min_bid; (*l_uid) = min_uid; + i_buf->weight = min_w; + b.bid = min_bid; b.uid = min_uid; + + + gid = next_hap_label_id(&b, g_p, bub, u, 1, &c_bid, &c_uid); + fprintf(stderr, "-min_w: %f, f_bid: %lld, f_uid: %lld, min_bid: %lld, min_uid: %lld, gid: %u\n", + min_w, f_bid, f_uid, min_bid, min_uid, gid); + + } +} + + + +int flip_block(block_phase_type* i_buf, G_partition* g_p, bubble_type* bub, +ma_utg_t *u, hc_links* link, uint8_t* lock, long long f_bid, long long f_uid, long long l_bid, long long l_uid) +{ + long long c_bid = l_bid, c_uid = l_uid; + uint32_t gid, val = 0; + block_phase_type b; + b.bid = f_bid; b.uid = f_uid; + memset(i_buf->vis.a, 0, g_p->n); + val = 0; + while (1) + { + gid = next_hap_label_id(&b, g_p, bub, u, 1, &c_bid, &c_uid); + ////fprintf(stderr, "+gid: %u\n", gid); + if(gid == (uint32_t)-1) break; + ///fprintf(stderr, "lock[gid]: %u\n", lock[gid]); + if(lock[gid] == 0) + { + val = 1; + break; + } + if(c_bid == l_bid && c_uid == l_uid) break; + } + + if(val == 0) return 0; + + b.bid = f_bid; b.uid = f_uid; + while (1) + { + gid = next_hap_label_id(&b, g_p, bub, u, 1, &c_bid, &c_uid); + ///fprintf(stderr, "-gid: %u\n", gid); + if(gid == (uint32_t)-1) break; + ///fprintf(stderr, "vis[gid]: %u\n", i_buf->vis.a[gid]); + if(i_buf->vis.a[gid] == 1) continue; + lock[gid] = 1; + i_buf->vis.a[gid] = 1; + // if(f_bid == 689 && f_uid == 1) + // { + // fprintf(stderr, "sssssssssssssssssssss\n"); + // print_phase_group(g_p, bub, "Small-1"); + // fprintf(stderr, "sbsbsbsb-gid: %u\n", gid); + // } + flip_unitig(g_p, link, bub, gid); + ///flip_unitig_debug(g_p, link, bub, gid); + // if(f_bid == 689 && f_uid == 1) + // { + // fprintf(stderr, "sasasasa-gid: %u\n", gid); + // print_phase_group(g_p, bub, "Small-2"); + // fprintf(stderr, "eeeeeeeeeeeeeeeeeeeeee\n"); + // } + if(c_bid == l_bid && c_uid == l_uid) break; + } + + return 1; +} + + +double get_total_weight(H_partition* h, G_partition* g_p) +{ + uint32_t i, k, uID; + hc_links* link = h->link; + int o_d = 0, o_f = 0; + double w, t_w; + for (i = 0, t_w = 0; i < h->n; i++) + { + if(g_p->index[i] == (uint32_t)-1) continue; + o_f = g_p->a[g_p->index[i]>>1].status[g_p->index[i]&1]; + for (k = 0; k < link->a.a[i].e.n; k++) + { + if(link->a.a[i].e.a[k].del) continue; + uID = link->a.a[i].e.a[k].uID; + w = link->a.a[i].e.a[k].weight; + if(g_p->index[uID] == (uint32_t)-1) continue; + o_d = g_p->a[g_p->index[uID]>>1].status[g_p->index[uID]&1]; + t_w += (o_f*o_d*w); + } + } + + return t_w; +} + + +void hap_label_fliping(H_partition* hap, G_partition* g_p, bubble_type* bub, hc_links* link, uint32_t chain_id) +{ + long long c_bid, c_uid, l_bid, l_uid; + uint32_t gid; + ma_utg_t *u = &(bub->b_ug->u.a[chain_id]); + hap->b.bid = hap->b.uid = 0; + while (1) + { + gid = next_hap_label_id(&(hap->b), g_p, bub, u, 1, &c_bid, &c_uid); + if(gid == (uint32_t)-1) break; + identify_best_interval(&(hap->b), hap->lock, g_p, bub, u, link, c_bid, c_uid, &l_bid, &l_uid); + if(l_bid == -1 || l_uid == -1) continue; + + // fprintf(stderr, "\nbefore weight: %f\n", get_total_weight(hap, g_p)); + // identify_best_interval_debug(&(hap->b), g_p, bub, u, link, c_bid, c_uid, &l_bid, &l_uid); + if(flip_block(&(hap->b), g_p, bub, u, link, hap->lock, c_bid, c_uid, l_bid, l_uid)) + { + ///fprintf(stderr, "after weight +: %f\n", get_total_weight(hap, g_p)); + hap->b.bid = l_bid; + hap->b.uid = l_uid; + gid = next_hap_label_id(&(hap->b), g_p, bub, u, 1, &c_bid, &c_uid); + if(gid == (uint32_t)-1) break; + } + ///fprintf(stderr, "after weight -: %f\n", get_total_weight(hap, g_p)); + // exit(0); + } +} + +typedef struct{ + long long min_chain_id; + long long min_f_bid; + long long min_f_uid; + long long min_l_bid; + long long min_l_uid; + long long min_idx; + double min_w; +}block_res_type; + +typedef struct{ + block_phase_type* x; + uint32_t n_thread; + bubble_type* bub; + uint64_t* chain_idx; + uint64_t chain_idx_n; + uint64_t chain_ele_occ; + block_res_type* res; + H_partition* h; + G_partition* g_p; +}mul_block_phase_type; + +uint32_t shift_block_phase_type(ma_utg_t *u, G_partition* g_p, bubble_type* bub, +block_phase_type* b, uint32_t offset) +{ + long long c_bid, c_uid; + uint32_t gid, occ = 0; + b->bid = b->uid = 0; + while (1) + { + if(occ == offset) break; + gid = next_hap_label_id(b, g_p, bub, u, 1, &c_bid, &c_uid); + if(gid == (uint32_t)-1) break; + occ++; + } + return occ; +} + +void get_block_phase_type(uint64_t* chain_idx, G_partition* g_p, bubble_type* bub, uint32_t id, block_phase_type* i_b) +{ + uint64_t i; + ma_utg_t *u = NULL; + for (i = 0; i < bub->chain_weight.n; i++) + { + if(id >= chain_idx[i] && id < chain_idx[i+1]) break; + } + + u = &(bub->b_ug->u.a[bub->chain_weight.a[i].id]); + shift_block_phase_type(u, g_p, bub, i_b, id - chain_idx[i]); + i_b->chainID = bub->chain_weight.a[i].id; +} + +void init_mul_block_phase_type(mul_block_phase_type* x, G_partition* g_p, bubble_type* bub, uint32_t n_thread, H_partition* hap) +{ + ma_utg_t *u = NULL; + uint32_t i, n; + block_phase_type b; + x->bub = bub; + x->n_thread = n_thread; + CALLOC(x->res, x->n_thread); + CALLOC(x->x, x->n_thread); + for (i = 0; i < x->n_thread; i++) + { + kv_init(x->x[i].vis); + kv_malloc(x->x[i].vis, hap->n); + x->x[i].vis.n = hap->n; + } + + x->chain_idx_n = 0; + MALLOC(x->chain_idx, bub->chain_weight.n+1); + for (i = n = 0; i < bub->chain_weight.n; i++) + { + x->chain_idx[i] = n; + if(bub->chain_weight.a[i].del) continue; + u = &(bub->b_ug->u.a[bub->chain_weight.a[i].id]); + n += shift_block_phase_type(u, g_p, bub, &b, (uint32_t)-1); + x->chain_idx_n++; + } + x->chain_idx[i] = n; + x->chain_ele_occ = n; +} + +void destory_mul_block_phase_type(mul_block_phase_type* x) +{ + uint32_t i; + free(x->res); + free(x->chain_idx); + for (i = 0; i < x->n_thread; i++) + { + kv_destroy(x->x[i].vis); + } +} + +void select_max_block_by_utg_multi_thread(H_partition* h, G_partition* g_p, bubble_type* bub, +hc_links* link, block_phase_type* i_b, uint64_t* chain_idx, uint32_t id, block_res_type* res) +{ + long long c_bid, c_uid, l_bid, l_uid; + uint32_t gid; + get_block_phase_type(chain_idx, g_p, bub, id, i_b); + + ma_utg_t *u = &(bub->b_ug->u.a[i_b->chainID]); + + gid = next_hap_label_id(i_b, g_p, bub, u, 1, &c_bid, &c_uid); + + if(gid == (uint32_t)-1) return; + if(identify_best_interval(i_b, h->lock, g_p, bub, u, link, c_bid, c_uid, &l_bid, &l_uid)) return; + if(l_bid == -1 || l_uid == -1) return; + + if((res->min_w > i_b->weight) || (res->min_w == i_b->weight && id < res->min_idx)) + { + res->min_w = i_b->weight; + res->min_f_bid = c_bid; + res->min_f_uid = c_uid; + res->min_l_bid = l_bid; + res->min_l_uid = l_uid; + res->min_chain_id = i_b->chainID; + res->min_idx = id; + } +} + +static void worker_for_max_block(void *data, long i, int tid) // callback for kt_for() +{ + mul_block_phase_type* x = (mul_block_phase_type*)data; + select_max_block_by_utg_multi_thread(x->h, x->g_p, x->bub, x->h->link, + &(x->x[tid]), x->chain_idx, i, &(x->res[tid])); +} + + +void select_max_block_by_utg_multi_thread_by_chain(H_partition* h, G_partition* g_p, bubble_type* bub, +hc_links* link, block_phase_type* i_b, uint32_t id, block_res_type* res) +{ + long long c_bid, c_uid, l_bid, l_uid; + uint32_t gid; + ma_utg_t *u = &(bub->b_ug->u.a[id]); + i_b->bid = i_b->uid = 0; i_b->chainID = id; + while (1) + { + gid = next_hap_label_id(i_b, g_p, bub, u, 1, &c_bid, &c_uid); + if(gid == (uint32_t)-1) break; + + if(identify_best_interval(i_b, h->lock, g_p, bub, u, link, c_bid, c_uid, &l_bid, &l_uid)) + { + break; + } + if(l_bid == -1 || l_uid == -1) continue; + if(res->min_w > i_b->weight) + { + res->min_w = i_b->weight; + res->min_f_bid = c_bid; + res->min_f_uid = c_uid; + res->min_l_bid = l_bid; + res->min_l_uid = l_uid; + res->min_chain_id = i_b->chainID; + res->min_idx = id; + } + } +} + + +// static void worker_for_max_block_by_chain(void *data, long i, int tid) // callback for kt_for() +// { +// mul_block_phase_type* x = (mul_block_phase_type*)data; +// select_max_block_by_utg_multi_thread_by_chain(x->h, x->g_p, x->bub, x->h->link, +// &(x->x[tid]), x->bub->chain_weight.a[i].id, &(x->res[tid])); +// } + +int get_max_block_multi_thread(H_partition* h, G_partition* g_p, bubble_type* bub, mul_block_phase_type* x, +long long* min_u, long long* min_f_bid, long long* min_f_uid, long long* min_l_bid, long long* min_l_uid, +double* min_w) +{ + uint32_t i; + (*min_w) = 1; + (*min_u) = (*min_f_bid) = (*min_f_uid) = (*min_l_bid) = (*min_l_uid) = -1; + for (i = 0; i < x->n_thread; i++) + { + x->res[i].min_chain_id = x->res[i].min_f_bid = x->res[i].min_f_uid = -1; + x->res[i].min_l_bid = x->res[i].min_l_uid = x->res[i].min_idx = -1; + x->res[i].min_w = 1; + } + x->g_p = g_p; + x->h = h; + kt_for(x->n_thread, worker_for_max_block, x, x->chain_ele_occ); + ///kt_for(x->n_thread, worker_for_max_block_by_chain, x, x->chain_idx_n); + + long long min_idx = -1; + for (i = 0; i < x->n_thread; i++) + { + if(x->res[i].min_chain_id == -1) continue; + if(x->res[i].min_f_bid == -1 || x->res[i].min_f_uid == -1) continue; + if(x->res[i].min_l_bid == -1 || x->res[i].min_l_uid == -1) continue; + if(((*min_w) > x->res[i].min_w) || ((*min_w) == x->res[i].min_w && x->res[i].min_idx < min_idx)) + { + (*min_w) = x->res[i].min_w; + (*min_u) = x->res[i].min_chain_id; + (*min_f_bid) = x->res[i].min_f_bid; + (*min_f_uid) = x->res[i].min_f_uid; + (*min_l_bid) = x->res[i].min_l_bid; + (*min_l_uid) = x->res[i].min_l_uid; + min_idx = x->res[i].min_idx; + } + } + + if((*min_u) != -1 && (*min_f_bid) != -1 && (*min_f_uid) != -1 && (*min_l_bid) != -1 && (*min_l_uid) != -1) + { + return 1; + } + + return 0; +} + +void select_max_block_by_utg(H_partition* hap, G_partition* g_p, bubble_type* bub, hc_links* link, uint32_t chain_id, +long long* min_f_bid, long long* min_f_uid, long long* min_l_bid, long long* min_l_uid, double* min_w) +{ + long long c_bid, c_uid, l_bid, l_uid; + uint32_t gid; + ma_utg_t *u = &(bub->b_ug->u.a[chain_id]); + (*min_w) = 1; + hap->b.bid = hap->b.uid = 0; + (*min_f_bid) = (*min_f_uid) = (*min_l_bid) = (*min_l_uid) = -1; + while (1) + { + gid = next_hap_label_id(&(hap->b), g_p, bub, u, 1, &c_bid, &c_uid); + if(gid == (uint32_t)-1) break; + + if(identify_best_interval(&(hap->b), hap->lock, g_p, bub, u, link, c_bid, c_uid, &l_bid, &l_uid)) + { + break; + } + if(l_bid == -1 || l_uid == -1) continue; + if((*min_w) > hap->b.weight) + { + (*min_w) = hap->b.weight; + (*min_f_bid) = c_bid; + (*min_f_uid) = c_uid; + (*min_l_bid) = l_bid; + (*min_l_uid) = l_uid; + } + } +} + + +int get_max_block(H_partition* h, G_partition* g_p, bubble_type* bub, long long* min_u, +long long* min_f_bid, long long* min_f_uid, long long* min_l_bid, long long* min_l_uid, +double* min_w) +{ + uint32_t i; + long long f_bid, f_uid, l_bid, l_uid; + double w; + (*min_w) = 1; + (*min_u) = (*min_f_bid) = (*min_f_uid) = (*min_l_bid) = (*min_l_uid) = -1; + for (i = 0; i < bub->chain_weight.n; i++) + { + if(bub->chain_weight.a[i].del) continue; + select_max_block_by_utg(h, g_p, bub, h->link, bub->chain_weight.a[i].id, + &f_bid, &f_uid, &l_bid, &l_uid, &w); + if(f_bid == -1 || f_uid == -1 || l_bid == -1 || l_uid == -1) continue; + if((*min_w) > w) + { + (*min_w) = w; + (*min_u) = bub->chain_weight.a[i].id; + (*min_f_bid) = f_bid; + (*min_f_uid) = f_uid; + (*min_l_bid) = l_bid; + (*min_l_uid) = l_uid; + } + } + + if((*min_u) != -1 && (*min_f_bid) != -1 && (*min_f_uid) != -1 && (*min_l_bid) != -1 && (*min_l_uid) != -1) + { + return 1; + } + + return 0; +} + +void phasing_improvement_by_block(H_partition* h, G_partition* g_p, bubble_type* bub, mul_block_phase_type* x) +{ + long long min_u, min_f_bid, min_f_uid, min_l_bid, min_l_uid; + double min_w; + + memset(h->lock, 0, sizeof(uint8_t)*h->n); + while(get_max_block_multi_thread(h, g_p, bub, x, &min_u, &min_f_bid, &min_f_uid, &min_l_bid, &min_l_uid, &min_w)) + ///while(get_max_block(h, g_p, bub, &min_u, &min_f_bid, &min_f_uid, &min_l_bid, &min_l_uid, &min_w)) + { + ///fprintf(stderr, "\nmin_w: %f, min_u: %lld, min_f_bid: %lld, min_f_uid: %lld, min_l_bid: %lld, min_l_uid: %lld\n", min_w, min_u, min_f_bid, min_f_uid, min_l_bid, min_l_uid); + ///fprintf(stderr, "before weight: %f\n", get_total_weight(h, g_p)); + flip_block(&(h->b), g_p, bub, &(bub->b_ug->u.a[min_u]), h->link, h->lock, min_f_bid, + min_f_uid, min_l_bid, min_l_uid); + ///fprintf(stderr, "after weight: %f\n", get_total_weight(h, g_p)); + } +} + +void flip_by_chain(H_partition* h, G_partition* g_p, bubble_type* bub) +{ + uint32_t i; + memset(h->lock, 0, sizeof(uint8_t)*h->n); + for (i = 0; i < bub->chain_weight.n; i++) + { + if(bub->chain_weight.a[i].del) continue; + merge_phase_group_by_chain(h, g_p, bub, bub->chain_weight.a[i].id); + } + + double pre_w = get_total_weight(h, g_p), current_w; + uint32_t round = 0; + while (1) + { + memset(h->lock, 0, sizeof(uint8_t)*h->n); + while (1) + { + i = get_max_unitig(h, g_p, h->link, bub); + if(i == (uint32_t)-1) break; + h->lock[i] = 1; + flip_unitig(g_p, h->link, bub, i); + } + current_w = get_total_weight(h, g_p); + ///fprintf(stderr, "[M::%s::round %u, pre_w: %f, current_w: %f]\n", __func__, round, pre_w, current_w); + if(ceil(current_w) <= ceil(pre_w)) break; + round++; + pre_w = current_w; + } + + + ///print_phase_group(g_p, bub, "Large-pre"); + + ///fprintf(stderr, "[M::%s::round %u, before block flipping: %f]\n", __func__, round, get_total_weight(h, g_p)); + + mul_block_phase_type b_x; + init_mul_block_phase_type(&b_x, g_p, bub, asm_opt.thread_num, h); + + pre_w = get_total_weight(h, g_p); + while (1) + { + phasing_improvement_by_block(h, g_p, bub, &b_x); + current_w = get_total_weight(h, g_p); + ///fprintf(stderr, "[M::%s::round %u, after block flipping: %f]\n", __func__, round, get_total_weight(h, g_p)); + ///debug_flip(g_p, h->link, bub, 0); + if(ceil(current_w) <= ceil(pre_w)) break; + round++; + pre_w = current_w; + } + destory_mul_block_phase_type(&b_x); + + for (i = 0; i < g_p->n; i++) + { + update_partition_flag(h, g_p, h->link, i); + } +} + +void flip_by_node(H_partition* h, G_partition* g_p, bubble_type* bub) +{ + uint32_t i; + memset(h->lock, 0, sizeof(uint8_t)*h->n); + + + double pre_w = get_total_weight(h, g_p), current_w; + uint32_t round = 0; + while (1) + { + memset(h->lock, 0, sizeof(uint8_t)*h->n); + while (1) + { + i = get_max_unitig(h, g_p, h->link, bub); + if(i == (uint32_t)-1) break; + h->lock[i] = 1; + flip_unitig(g_p, h->link, bub, i); + } + current_w = get_total_weight(h, g_p); + ///fprintf(stderr, "[M::%s::round %u, pre_w: %f, current_w: %f]\n", __func__, round, pre_w, current_w); + if(ceil(current_w) <= ceil(pre_w)) break; + round++; + pre_w = current_w; + } + + ///fprintf(stderr, "[M::%s::round %u, before block flipping: %f]\n", __func__, round, get_total_weight(h, g_p)); + + for (i = 0; i < g_p->n; i++) + { + update_partition_flag(h, g_p, h->link, i); + } +} + + +void link_phase_group(H_partition* hap, bubble_type* bub) +{ + double index_time = yak_realtime(); + uint32_t i, k, n = (hap->label>>hap->label_shift)+1, *h0, h0_n, *h1, h1_n;; + init_G_partition(&(hap->group_g_p), hap->n); + partition_warp *res = NULL; + for (i = 0; i < n; i++) + { + kv_pushp(partition_warp, hap->group_g_p, &res); + kv_init(res->a); + res->full_bub = 0; + res->h[0] = res->h[1] = 0; + res->status[0] = 1; res->status[1] = -1; + res->weight[0] = res->weight[1] = res->weight_convex = 0; + ///all unitigs + for (k = 0; k < hap->n; k++) + { + if(get_phase_group(hap, k) == i && get_phase_status(hap, k) == 1) + { + kv_push(uint32_t, res->a, k); + res->h[0]++; + } + } + + for (k = 0; k < hap->n; k++) + { + if(get_phase_group(hap, k) == i && get_phase_status(hap, k) == -1) + { + kv_push(uint32_t, res->a, k); + res->h[1]++; + } + } + + for (k = 0; k < res->h[0]; k++) + { + ///if(hap->group_g_p.index[res->a.a[k]] != (uint32_t)-1) fprintf(stderr, "ERROR---00\n"); + hap->group_g_p.index[res->a.a[k]] = hap->group_g_p.n-1; + hap->group_g_p.index[res->a.a[k]] = hap->group_g_p.index[res->a.a[k]] << 1; + } + + for (; k < res->a.n; k++) + { + ///if(hap->group_g_p.index[res->a.a[k]] != (uint32_t)-1) fprintf(stderr, "ERROR---11\n"); + hap->group_g_p.index[res->a.a[k]] = hap->group_g_p.n-1; + hap->group_g_p.index[res->a.a[k]] = (hap->group_g_p.index[res->a.a[k]] << 1) + 1; + } + + get_phased_block(&(hap->group_g_p), NULL, i, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + if(h0_n >0) res->weight[0] = get_cluster_weight(hap, hap->link, h0, h0_n); + if(h1_n >0) res->weight[1] = get_cluster_weight(hap, hap->link, h1, h1_n); + res->weight_convex = get_cluster_inner_weight(hap, hap->link, h0, h0_n, h1, h1_n); + } + + fprintf(stderr, "[M::%s::%.3f]\n", __func__, yak_realtime()-index_time); + // for (i = 0; i < n; i++) + // { + // double w0 = 0, w1 = 0; + // res = &(hap->group_g_p.a[i]); + // fprintf(stderr, "%u-th group: # %d = %u, # %d = %u\n", i, + // res->status[0], res->h[0], res->status[1], res->h[1]); + // get_phased_block(&(hap->group_g_p), NULL, i, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + // if(h0_n >0) w0 = get_cluster_weight_debug(&(hap->group_g_p), hap->link, h0, h0_n); + // if(h1_n >0) w1 = get_cluster_weight_debug(&(hap->group_g_p), hap->link, h1, h1_n); + // if(w0 != res->weight[0]) fprintf(stderr, "i: %u, ERROR: w0: %f, weight[0]: %f\n", i, w0, res->weight[0]); + // if(w1 != res->weight[1]) fprintf(stderr, "i: %u, ERROR: w1: %f, weight[1]: %f\n", i, w1, res->weight[1]); + + + // for (k = 0; k < res->h[0]; k++) + // { + // fprintf(stderr, "%d: utg%.6ul\n", res->status[0], int(res->a.a[k]+1)); + // } + + // for (; k < res->a.n; k++) + // { + // fprintf(stderr, "%d: utg%.6ul\n", res->status[1], int(res->a.a[k]+1)); + // } + // } + + /*******************************for debug************************************/ + // for (i = 0; i < hap->n; i++) + // { + // if(hap->link->a.a[i].e.n == 0) continue; + // if(get_phase_status(hap, i) == -2) + // { + // fprintf(stderr, "ERROR+++: i: %u, group: %u, bub->index: %u\n", i, get_phase_group(hap, i), bub->index[i]); + // for (k = 0; k < hap->link->a.a[i].e.n; k++) + // { + // fprintf(stderr, "k: %u, uID: %u, weight: %f, del: %u\n", k, hap->link->a.a[i].e.a[k].uID, + // hap->link->a.a[i].e.a[k].weight, hap->link->a.a[i].e.a[k].del); + // } + // } + // } + /*******************************for debug************************************/ + + + flip_by_chain(hap, &(hap->group_g_p), bub); + ///print_phase_group(&(hap->group_g_p), bub, "Large"); +} + +void print_chain_phasing(H_partition* hap, ma_ug_t *ug, bubble_type* bub, uint32_t chain_id) +{ + uint32_t i, k; + uint32_t beg, sink, *a, n; + uint64_t bid, uid; + ma_utg_t *u = &(bub->b_ug->u.a[chain_id]); + fprintf(stderr, "\n**********chain_id: %u**********\n", chain_id); + for (i = 0; i < u->n; i++) + { + bid = u->a[i]>>33; + fprintf(stderr, "(%u) chain_id: %u, u->n: %u, bid: %u\n", i, chain_id, (uint32_t)u->n, i); + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + for (k = 0; k < n; k++) + { + uid = a[k]>>1; + fprintf(stderr, "utg%.6ul, hap: %u, group: %u, stats: %d\n", (int)(uid+1), hap->hap[uid], + get_phase_group(hap, uid), get_phase_status(hap, uid)); + } + } +} + +int graph_bipartiteness(uint32_t* b_a, uint32_t b_a_n, uint8_t *color, hc_links* link, kvec_t_u32_warp* stack) +{ + if(b_a_n == 0) return 0; + uint32_t i, uID, cur, occ = 0, sucess = 0, c; + for (i = 0; i < b_a_n; i++) color[b_a[i]>>1] = 8; + stack->a.n = 0; + kv_push(uint32_t, stack->a, b_a[0]>>1); + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + if((color[cur] & 1) == 0) occ++; + color[cur] |= 1; + for (i = 0; i < link->a.a[cur].f.n; i++) + { + if(link->a.a[cur].f.a[i].del) continue; + if(link->a.a[cur].f.a[i].dis != RC_0) continue; + uID = link->a.a[cur].f.a[i].uID; + if((color[uID] & 8) == 0) continue; + if((color[uID] & 1) == 1) continue; + kv_push(uint32_t, stack->a, uID); + } + } + if(occ != b_a_n) goto Failed; + + sucess = 1; + for (i = 0; i < b_a_n; i++) color[b_a[i]>>1] = 8; + stack->a.n = 0; + kv_push(uint32_t, stack->a, b_a[0]>>1); + color[b_a[0]>>1] |= 2;///colored + while (stack->a.n > 0) + { + stack->a.n--; + cur = stack->a.a[stack->a.n]; + color[cur] |= 1; + c = color[cur] & 4; ///get color + for (i = 0; i < link->a.a[cur].f.n; i++) + { + if(link->a.a[cur].f.a[i].del) continue; + if(link->a.a[cur].f.a[i].dis != RC_0) continue; + uID = link->a.a[cur].f.a[i].uID; + if((color[uID] & 8) == 0) continue; + if((color[uID] & 2) && ((color[uID] & 4) == c)) break; ///conflict + if((color[uID] & 1) == 1) continue; + kv_push(uint32_t, stack->a, uID); + color[uID] |= 2; color[uID] |= (c^4); + } + + if(i != link->a.a[cur].f.n) + { + sucess = -1; + break; + } + } + + Failed: + if(sucess != 1) + { + for (i = 0; i < b_a_n; i++) color[b_a[i]>>1] = 0; + } + + return sucess; +} + +void assign_per_unitig_G_partition(G_partition* g_p, uint64_t hap_n, hc_links* link, bubble_type* bub, +uint32_t bubble_first) +{ + reset_G_partition(g_p, hap_n); + + partition_warp* res = NULL; + hc_edge *a = NULL; + uint32_t i, a_n, v, u, uv = (uint32_t)-1, k, k_n, k_nv, k_nu, beg, sink, *b_a = NULL, b_a_n; + + if(bubble_first) + { + int c; + kvec_t_u32_warp stack; kv_init(stack.a); + uint8_t *color = NULL; CALLOC(color, hap_n); + uint32_t n_bub = bub->f_bub + bub->b_bub; + for (i = 0; i < n_bub; i++) + { + get_bubbles(bub, i, &beg, &sink, &b_a, &b_a_n, NULL); + if(b_a_n == 2 && i < bub->f_bub) + { + continue; + } + ///full bubble do not overlap with any others + ///broken bubbles might be, but should do nothing + c = graph_bipartiteness(b_a, b_a_n, color, link, &stack); + + if(c == 0) + { + fprintf(stderr, "too good: s-utg%.6ul && e-utg%.6ul && %s\n",(beg>>1)+1, (sink>>1)+1, b_a_n != 4? "abnormal" : "normal"); + } + if(c == -1) + { + fprintf(stderr, "too bad: s-utg%.6ul && e-utg%.6ul\n",(beg>>1)+1, (sink>>1)+1); + } + if(c == 1) + { + fprintf(stderr, "\nprefect=%u: s-utg%.6ul && e-utg%.6ul\n", b_a_n, (beg>>1)+1, (sink>>1)+1); + + + for (k = 0; k < b_a_n; k++) + { + if((color[b_a[k]>>1] & 2) == 0) fprintf(stderr, "ERROR\n"); + if((color[b_a[k]>>1] & 4) == 0) fprintf(stderr, "0: utg%.6ul\n", (b_a[k]>>1)+1); + } + + for (k = 0; k < b_a_n; k++) + { + if((color[b_a[k]>>1] & 2) == 0) fprintf(stderr, "ERROR\n"); + if((color[b_a[k]>>1] & 4) != 0) fprintf(stderr, "1: utg%.6ul\n", (b_a[k]>>1)+1); + } + + for (k = 0; k < b_a_n; k++) color[b_a[k]>>1] = 0; + } + } + free(color); + kv_destroy(stack.a); + } + + for (i = 0; i < hap_n; i++) + { + v = i; + a = link->a.a[v].f.a; + a_n = link->a.a[v].f.n; + for (k = k_n = 0; k < a_n; k++) + { + if(a[k].del) continue; + if(a[k].dis != RC_0) break; + u = a[k].uID; + k_n++; + } + if(k_n != 1) + { + u = (uint32_t)-1; + goto push_uv; + } + + a = link->a.a[u].f.a; + a_n = link->a.a[u].f.n; + for (k = k_n = 0; k < a_n; k++) + { + if(a[k].del) continue; + if(a[k].dis != RC_0) break; + uv = a[k].uID; + k_n++; + } + if(k_n != 1 || uv != v) + { + u = (uint32_t)-1; + goto push_uv; + } + + push_uv: + k_nv = 0;k_nu = 0; + + // not such easy. need to deal with here very carefully + // if(g_p->index[v] != (uint32_t)-1) continue; + // if(u != (uint32_t)-1 && g_p->index[u] != (uint32_t)-1) u = (uint32_t)-1; + + a = link->a.a[v].e.a; + a_n = link->a.a[v].e.n; + for (k = 0; k < a_n; k++) + { + if(a[k].del) continue; + k_nv++; + } + + if(u != (uint32_t)-1) + { + a = link->a.a[u].e.a; + a_n = link->a.a[u].e.n; + for (k = 0; k < a_n; k++) + { + if(a[k].del) continue; + k_nu++; + } + } + if(k_nv == 0) continue; + if(k_nv > 0 && k_nu > 0 && v > u) continue; + + kv_pushp(partition_warp, *g_p, &res); + kv_init(res->a); + res->full_bub = 0; + res->h[0] = 1; res->h[1] = 0; + kv_push(uint32_t, res->a, v); + if(u != (uint32_t)-1) + { + res->h[1] = 1; + kv_push(uint32_t, res->a, u); + } + + for (k = 0; k < res->h[0]; k++) + { + g_p->index[res->a.a[k]] = g_p->n-1; + g_p->index[res->a.a[k]] = g_p->index[res->a.a[k]] << 1; + } + + for (; k < res->a.n; k++) + { + g_p->index[res->a.a[k]] = g_p->n-1; + g_p->index[res->a.a[k]] = (g_p->index[res->a.a[k]] << 1) + 1; + } + } + +} + +typedef struct { + double weight; + uint64_t p_id, beg_idx, end_idx; + uint8_t used; +}bub_sort_type; + +typedef struct { + bub_sort_type* a; + size_t n, m; +}bub_sort_vec; + +double get_specific_weight_by_chain(uint64_t* ids, uint64_t beg_idx, uint64_t end_idx, uint64_t p_id, +hc_links* link, uint8_t* vis, uint8_t flag) +{ + uint64_t x, k; + uint32_t uid; + double w; + for (x = beg_idx, w = 0; x <= end_idx; x++) + { + uid = (uint32_t)((uint32_t)ids[x])>>1; + for (k = 0; k < link->a.a[uid].e.n; k++) + { + if(link->a.a[uid].e.a[k].del) continue; + if(vis[link->a.a[uid].e.a[k].uID] != flag) continue; + w += link->a.a[uid].e.a[k].weight; + } + } + + return w; +} + +int cmp_bubble_ele_by_chain(const void * a, const void * b) +{ + if((*(bub_sort_type*)a).weight == (*(bub_sort_type*)b).weight) + { + return (*(bub_sort_type*)a).weight > (*(bub_sort_type*)b).weight? -1 : 1; + } + + return 0; +} + +uint32_t get_max_hap_g(bub_sort_vec* w_stack, uint32_t* require_iso) +{ + uint32_t k, max_idx = (uint32_t)-1; + double max_w; + for (k = require_iso? (*require_iso)+1 : 0, max_idx = (uint32_t)-1; k < w_stack->n; k++) + { + if(w_stack->a[k].used) continue; + if(!require_iso) + { + if(w_stack->a[k].p_id == (uint32_t)-1) continue; + if((max_idx == (uint32_t)-1) || (max_idx != (uint32_t)-1 && max_w < w_stack->a[k].weight)) + { + max_w = w_stack->a[k].weight; + max_idx = k; + } + } + else + { + if(w_stack->a[k].p_id != (uint32_t)-1) continue; + return k; + } + } + + if(require_iso && (*require_iso) != 0) + { + for (k = 0; k < w_stack->n; k++) + { + if(w_stack->a[k].used) continue; + if(w_stack->a[k].p_id != (uint32_t)-1) continue; + return k; + } + } + + return max_idx; +} + +void update_bub_sort_vec(uint64_t* ids, bub_sort_vec* w_stack, uint32_t max_idx, hc_links* link, +uint32_t* set_hap) +{ + w_stack->a[max_idx].used = 1; + uint64_t i, k; + uint32_t uid, pid_idx; + for (i = w_stack->a[max_idx].beg_idx; i <= w_stack->a[max_idx].end_idx; i++) + { + uid = (uint32_t)((uint32_t)ids[i])>>1; + for (k = 0; k < link->a.a[uid].e.n; k++) + { + if(link->a.a[uid].e.a[k].del) continue; + pid_idx = set_hap[link->a.a[uid].e.a[k].uID]; + if(pid_idx == (uint32_t)-1) continue; + if(w_stack->a[pid_idx].used) continue; + w_stack->a[pid_idx].weight += link->a.a[uid].e.a[k].weight; + } + } +} + +void sort_bubble_ele_by_chain(G_partition* g_p, hc_links* link, bubble_type* bub, kvec_t_u64_warp* stack, +bub_sort_vec* w_stack, uint8_t* vis, uint32_t* set_hap, uint32_t n_utg, uint32_t chain_id) +{ + uint32_t max_idx, i, k, j, m, beg, sink, *a, n, flag_cur = 3, flag_right = 2, flag_left = 1, flag_unset = 0; + uint64_t bid, uid, pid, pre_pid; + ma_utg_t *u = &(bub->b_ug->u.a[chain_id]); + bub_sort_type *p = NULL; + memset(vis, flag_unset, n_utg); + for (i = 0; i < u->n; i++) + { + bid = u->a[i]>>33; + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + for (k = 0; k < n; k++) + { + uid = a[k]>>1; + vis[uid] = flag_right; + } + } + + for (i = 0; i < u->n; i++) + { + stack->a.n = 0; w_stack->n = 0; + bid = u->a[i]>>33; + get_bubbles(bub, bid, &beg, &sink, &a, &n, NULL); + for (k = 0; k < n; k++) + { + uid = a[k]>>1; + pid = g_p->index[uid]; + if(pid != (uint32_t)-1) pid >>= 1; + kv_push(uint64_t, stack->a, (pid<<32)|a[k]); + vis[uid] = flag_cur; + } + radix_sort_hc64(stack->a.a, stack->a.a + stack->a.n);///sort is to dedup pid + + for (k = 0, pre_pid = (uint64_t)-1; k < stack->a.n; k++) + { + if((stack->a.a[k]>>32) == pre_pid) continue; + if(w_stack->n > 0) w_stack->a[w_stack->n-1].end_idx = k - 1; + + pre_pid = stack->a.a[k]>>32; + kv_pushp(bub_sort_type, *w_stack, &p); + p->weight = 0; + p->p_id = pre_pid; + p->beg_idx = k; + p->end_idx = (uint64_t)-1; + p->used = 0; + } + if(w_stack->n > 0) w_stack->a[w_stack->n-1].end_idx = k - 1; + + ///get each hap id + for (k = 0; k < w_stack->n; k++) + { + w_stack->a[k].weight += get_specific_weight_by_chain(stack->a.a, w_stack->a[k].beg_idx, w_stack->a[k].end_idx, w_stack->a[k].p_id, + link, vis, flag_left); + w_stack->a[k].weight -= get_specific_weight_by_chain(stack->a.a, w_stack->a[k].beg_idx, w_stack->a[k].end_idx, w_stack->a[k].p_id, + link, vis, flag_right); + for (j = w_stack->a[k].beg_idx; j <= w_stack->a[k].end_idx; j++) + { + set_hap[((uint32_t)stack->a.a[j])>>1] = k; + } + } + + m = 0; + while ((max_idx = get_max_hap_g(w_stack, NULL)) != (uint32_t)-1) + { + for (j = w_stack->a[max_idx].beg_idx; j <= w_stack->a[max_idx].end_idx; j++) + { + a[m] = (uint32_t)stack->a.a[j]; + m++; + } + update_bub_sort_vec(stack->a.a, w_stack, max_idx, link, set_hap); + } + + while ((max_idx = get_max_hap_g(w_stack, &max_idx)) != (uint32_t)-1) + { + for (j = w_stack->a[max_idx].beg_idx; j <= w_stack->a[max_idx].end_idx; j++) + { + a[m] = (uint32_t)stack->a.a[j]; + m++; + } + update_bub_sort_vec(stack->a.a, w_stack, max_idx, link, set_hap); + } + + + /** + qsort(w_stack->a, w_stack->n, sizeof(bub_sort_type), cmp_bubble_ele_by_chain); + + m = 0; + + for (k = 0; k < w_stack->n; k++) + { + if(w_stack->a[k].p_id == (uint32_t)-1) continue; + for (j = w_stack->a[k].beg_idx; j <= w_stack->a[k].end_idx; j++) + { + a[m] = (uint32_t)stack->a.a[j]; + m++; + } + } + + for (k = 0; k < w_stack->n; k++) + { + if(w_stack->a[k].p_id != (uint32_t)-1) continue; + for (j = w_stack->a[k].beg_idx; j <= w_stack->a[k].end_idx; j++) + { + a[m] = (uint32_t)stack->a.a[j]; + m++; + } + } + **/ + + for (k = 0; k < n; k++) + { + uid = a[k]>>1; + vis[uid] = flag_left; + set_hap[uid] = (uint32_t)-1; + } + } +} + +void sort_bubble_ele(G_partition* g_p, hc_links* link, bubble_type* bub, uint32_t n_utg) +{ + kvec_t_u64_warp stack; kv_init(stack.a); + bub_sort_vec w_stack; kv_init(w_stack); + uint8_t* vis = NULL; MALLOC(vis, n_utg); + uint32_t* set_hap = NULL; MALLOC(set_hap, n_utg); memset(set_hap, -1, sizeof(uint32_t)*n_utg); + uint32_t i; + + for (i = 0; i < bub->chain_weight.n; i++) + { + if(bub->chain_weight.a[i].del) continue; + sort_bubble_ele_by_chain(g_p, link, bub, &stack, &w_stack, vis, set_hap, n_utg, bub->chain_weight.a[i].id); + } + + kv_destroy(stack.a); kv_destroy(w_stack); free(vis); free(set_hap); +} + +uint32_t init_contig_partition(H_partition* hap, ha_ug_index* idx, bubble_type* bub) +{ + hc_links* link = idx->link; + ma_ug_t *ug = idx->ug; + bub_p_t_warp b; + memset(&b, 0, sizeof(bub_p_t_warp)); + CALLOC(b.a, ug->g->n_seq*2); + uint32_t i, nv = ug->g->n_seq * 2, max_i, max_hap_label; + for (i = 0; i < nv; i++) + { + b.a[i].w[0] = b.a[i].w[1] = b.a[i].nh = 0; + b.a[i].p =b.a[i].d = b.a[i].nc = b.a[i].uc = b.a[i].ac = b.a[i].r = b.a[i].s = 0; + } + uint8_t* hap_label_flag = NULL; + CALLOC(hap_label_flag, ug->g->n_seq); + + + hap->n = ug->u.n; + MALLOC(hap->hap, hap->n); + memset(hap->hap, 0, hap->n*sizeof(uint32_t)); + MALLOC(hap->lock, hap->n); + memset(hap->lock, 0, hap->n); + hap->m[0] = 1; hap->m[1] = 2; hap->m[2] = 4; + hap->link = link; + hap->label = 0; + hap->label_add = 8; + for(hap->label_shift=1; (uint64_t)(1<label_shift)<(uint64_t)hap->label_add; hap->label_shift++); + kv_init(hap->label_buffer); + kv_init(hap->b.vis); kv_malloc(hap->b.vis, hap->n); hap->b.vis.n = hap->n; + + ///fprintf(stderr, "hap->label: %u, hap->label_add: %u, hap->label_shift: %u\n", hap->label, hap->label_add, hap->label_shift); + + + ///sorted by weight + for (i = 0; i < bub->chain_weight.n; i++) + { + if(bub->chain_weight.a[i].del) continue; + phase_bubble_chain(hap, ug, &b, bub, hap_label_flag, bub->chain_weight.a[i].id); + ///print_chain_phasing(hap, ug, bub, bub->chain_weight.a[i].id); + } + + memset(hap_label_flag, 1, ug->g->n_seq); + while (1) + { + max_i = get_unset_com(hap, bub, ug, hap_label_flag, &max_hap_label); + if(max_i == (uint32_t)-1) break; + phase_com(hap, ug, &b, bub, max_i, max_hap_label); + } + + for (i = 0; i < hap->n; i++) + { + if((hap->hap[i]&hap->m[0])&&(hap->hap[i]&hap->m[1])) + { + hap->hap[i] >>= hap->label_shift; + hap->hap[i] <<= hap->label_shift; + hap->hap[i] |= hap->m[2]; + reset_ambiguous_label(hap, hap_label_flag, i); + } + } + + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + + init_G_partition(&(hap->g_p), hap->n); + + link_phase_group(hap, bub); + + assign_per_unitig_G_partition(&(hap->g_p), hap->n, link, bub, 0); + + adjust_contig_partition(hap, link); + + update_bubble_chain(ug, bub, 0, 1); + + resolve_bubble_chain_tangle(ug, bub); + + clean_bubble_chain_by_HiC(ug, link, bub); + + append_boundary_chain(ug, link, bub); + + sort_bubble_ele(&(hap->g_p), link, bub, hap->n); + + free(hap_label_flag); + return 1; +} + + + +uint32_t get_max_unitig(H_partition* h, G_partition* g_p, hc_links* link, bubble_type* bub) +{ + double min, weight; + uint32_t i, min_i; + + for (i = 0, min = 1, min_i = (uint32_t)-1; i < g_p->n; i++) + { + if(h->lock[i]) continue; + weight = 0; + if(g_p->a[i].h[0] > 0 && (g_p->a[i].status[0] == 1 || g_p->a[i].status[0] == -1)) + { + weight += (g_p->a[i].weight[0] * g_p->a[i].status[0]); + } + + if(g_p->a[i].h[1] > 0 && (g_p->a[i].status[1] == 1 || g_p->a[i].status[1] == -1)) + { + weight += (g_p->a[i].weight[1] * g_p->a[i].status[1]); + } + weight += g_p->a[i].weight_convex*2; + + if(weight >= 0) continue; + if(weight < min) + { + min = weight; + min_i = i; + } + } + ///fprintf(stderr, "*****************min: %f\n", min); + return min_i; +} + +double get_cluster_weight_debug(G_partition* g_p, hc_links* link, uint32_t *h, uint32_t h_n) +{ + int o_d = 0; + double weight = 0; + uint32_t j, k, m, uID; + for (j = 0, weight = 0; j < h_n; j++) + { + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + if(link->a.a[h[j]].e.a[k].del) continue; + for (m = 0; m < h_n; m++) + { + if(h[m] == link->a.a[h[j]].e.a[k].uID) break; + } + if(m < h_n) continue; + + uID = link->a.a[h[j]].e.a[k].uID; + ///o_d = get_phase_status(hap, link->a.a[h[j]].e.a[k].uID); + o_d = g_p->a[g_p->index[uID]>>1].status[g_p->index[uID]&1]; + ///if(o_d < -1) fprintf(stderr, "ERROR\n"); + weight += (o_d*link->a.a[h[j]].e.a[k].weight); + } + } + + return weight; +} + +void flip_unitig(G_partition* g_p, hc_links* link, bubble_type* bub, uint32_t id) +{ + if(g_p->a[id].h[0] > 0 && g_p->a[id].status[0] != 1 && g_p->a[id].status[0] != -1) return; + if(g_p->a[id].h[1] > 0 && g_p->a[id].status[1] != 1 && g_p->a[id].status[1] != -1) return; + uint32_t k, j, m, *h0, h0_n, *h1, h1_n, uID, *h = NULL, h_n; + int status; + double weight; + get_phased_block(g_p, NULL, id, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + // fprintf(stderr, "h0_n: %u, h1_n: %u\n", h0_n, h1_n); + if(h0_n > 0) + { + + status = g_p->a[id].status[0]; + // fprintf(stderr, "+status: %d\n", status); + h = h0; h_n = h0_n; + for (j = 0; j < h_n; j++) + { + // fprintf(stderr, "+j: %u, h_n: %u\n", j, h_n); + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + // fprintf(stderr, "+k: %u, e_n: %u\n", k, (uint32_t)link->a.a[h[j]].e.n); + if(link->a.a[h[j]].e.a[k].del) continue; + for (m = 0; m < h_n; m++) + { + if(h[m] == link->a.a[h[j]].e.a[k].uID) break; + } + // fprintf(stderr, "+m: %u, h_n: %u\n", m, h_n); + if(m < h_n) continue; + + uID = link->a.a[h[j]].e.a[k].uID; + + weight = link->a.a[h[j]].e.a[k].weight; + + if(g_p->index[uID] == (uint32_t)-1) continue; + + + g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1] -= (2*status*weight); + + + } + } + g_p->a[id].status[0] *= -1; + } + + // fprintf(stderr, "hehehe\n"); + if(h1_n > 0) + { + status = g_p->a[id].status[1]; + // fprintf(stderr, "-status: %d\n", status); + h = h1; h_n = h1_n; + for (j = 0; j < h_n; j++) + { + // fprintf(stderr, "-j: %u, h_n: %u\n", j, h_n); + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + // fprintf(stderr, "-k: %u, e_n: %u\n", k, (uint32_t)link->a.a[h[j]].e.n); + if(link->a.a[h[j]].e.a[k].del) continue; + for (m = 0; m < h_n; m++) + { + if(h[m] == link->a.a[h[j]].e.a[k].uID) break; + } + // fprintf(stderr, "-m: %u, h_n: %u\n", m, h_n); + if(m < h_n) continue; + + uID = link->a.a[h[j]].e.a[k].uID; + // fprintf(stderr, "-uID: %u\n", uID); + + weight = link->a.a[h[j]].e.a[k].weight; + + if(g_p->index[uID] == (uint32_t)-1) continue; + + + g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1] -= (2*status*weight); + + } + } + g_p->a[id].status[1] *= -1; + } +} + +void flip_unitig_debug(G_partition* g_p, hc_links* link, bubble_type* bub, uint32_t id) +{ + if(g_p->a[id].h[0] > 0 && g_p->a[id].status[0] != 1 && g_p->a[id].status[0] != -1) return; + if(g_p->a[id].h[1] > 0 && g_p->a[id].status[1] != 1 && g_p->a[id].status[1] != -1) return; + uint32_t k, j, m, *h0, h0_n, *h1, h1_n, uID, *h = NULL, h_n; + int status; + double weight; + get_phased_block(g_p, NULL, id, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + // fprintf(stderr, "h0_n: %u, h1_n: %u\n", h0_n, h1_n); + if(h0_n > 0) + { + + status = g_p->a[id].status[0]; + // fprintf(stderr, "+status: %d\n", status); + h = h0; h_n = h0_n; + for (j = 0; j < h_n; j++) + { + // fprintf(stderr, "+j: %u, h_n: %u\n", j, h_n); + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + // fprintf(stderr, "+k: %u, e_n: %u\n", k, (uint32_t)link->a.a[h[j]].e.n); + if(link->a.a[h[j]].e.a[k].del) continue; + for (m = 0; m < h_n; m++) + { + if(h[m] == link->a.a[h[j]].e.a[k].uID) break; + } + // fprintf(stderr, "+m: %u, h_n: %u\n", m, h_n); + if(m < h_n) continue; + + uID = link->a.a[h[j]].e.a[k].uID; + + weight = link->a.a[h[j]].e.a[k].weight; + + if(g_p->index[uID] == (uint32_t)-1) continue; + + if(id == 19675) + { + fprintf(stderr, "+uID+: %u, e-weight: %f, g_p->index[uID]>>1: %u, status[0]: %d, pre_uID_weight: %f\n", + uID, weight, g_p->index[uID]>>1, status, g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1]); + } + + + g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1] -= (2*status*weight); + + if(id == 19675) + { + fprintf(stderr, "+uID+: %u, new_uID_weight: %f\n", uID, + g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1]); + } + + } + } + g_p->a[id].status[0] *= -1; + } + + // fprintf(stderr, "hehehe\n"); + if(h1_n > 0) + { + status = g_p->a[id].status[1]; + // fprintf(stderr, "-status: %d\n", status); + h = h1; h_n = h1_n; + for (j = 0; j < h_n; j++) + { + // fprintf(stderr, "-j: %u, h_n: %u\n", j, h_n); + for (k = 0; k < link->a.a[h[j]].e.n; k++) + { + // fprintf(stderr, "-k: %u, e_n: %u\n", k, (uint32_t)link->a.a[h[j]].e.n); + if(link->a.a[h[j]].e.a[k].del) continue; + for (m = 0; m < h_n; m++) + { + if(h[m] == link->a.a[h[j]].e.a[k].uID) break; + } + // fprintf(stderr, "-m: %u, h_n: %u\n", m, h_n); + if(m < h_n) continue; + + uID = link->a.a[h[j]].e.a[k].uID; + // fprintf(stderr, "-uID: %u\n", uID); + + weight = link->a.a[h[j]].e.a[k].weight; + + if(g_p->index[uID] == (uint32_t)-1) continue; + + if(id == 19675) + { + fprintf(stderr, "-uID-: %u, e-weight: %f, g_p->index[uID]>>1: %u, status[1]: %d, pre_uID_weight: %f\n", + uID, weight, g_p->index[uID]>>1, status, g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1]); + } + + g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1] -= (2*status*weight); + + if(id == 19675) + { + fprintf(stderr, "-uID-: %u, new_uID_weight: %f\n", uID, + g_p->a[g_p->index[uID]>>1].weight[g_p->index[uID]&1]); + } + } + } + g_p->a[id].status[1] *= -1; + } +} + + +uint32_t phasing_improvement(H_partition* h, G_partition* g_p, ha_ug_index* idx, bubble_type* bub) +{ + uint32_t i, occ = 0, round = 0; + double pre_w, pre_total, current_w; + mul_block_phase_type b_x; + init_mul_block_phase_type(&b_x, g_p, bub, asm_opt.thread_num, h); + ///double index_time = yak_realtime(); + + while(1) + { + pre_w = get_total_weight(h, g_p); + pre_total = pre_w; + + while (1) + { + memset(h->lock, 0, sizeof(uint8_t)*g_p->n); + while (1) + { + i = get_max_unitig(h, g_p, idx->link, bub); + if(i == (uint32_t)-1) break; + h->lock[i] = 1; + flip_unitig(g_p, idx->link, bub, i); + occ++; + } + current_w = get_total_weight(h, g_p); + ///fprintf(stderr, "[M::%s::round single %u, pre_w: %f, current_w: %f]\n", __func__, round, pre_w, current_w); + if(ceil(current_w) <= ceil(pre_w)) break; + round++; + pre_w = current_w; + } + + + pre_w = get_total_weight(h, g_p); + round = 0; + while (1) + { + ///fprintf(stderr, "[M::%s::round block %u, h->n: %lu]\n", __func__, round, h->n); + phasing_improvement_by_block(h, g_p, bub, &b_x); + current_w = get_total_weight(h, g_p); + ///fprintf(stderr, "[M::%s::round block %u, pre_w: %f, current_w: %f]\n", __func__, round, pre_w, current_w); + if(ceil(current_w) <= ceil(pre_w)) break; + round++; + pre_w = current_w; + } + + if(ceil(current_w) <= ceil(pre_total)) break; + } + + destory_mul_block_phase_type(&b_x); + ///fprintf(stderr, "[M::%s:Flipping time:%.3f]\n", __func__, yak_realtime()-index_time); + + + for (i = 0; i < g_p->n; i++) + { + update_partition_flag(h, g_p, idx->link, i); + } + + ///print_phase_group(g_p, bub, "Small"); + // double w0 = 0, w1 = 0; + // uint32_t *h0, h0_n, *h1, h1_n; + // get_phased_block(g_p, NULL, 2973, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + // w0 = get_cluster_weight_debug(g_p, h->link, h0, h0_n); + // w1 = get_cluster_weight_debug(g_p, h->link, h1, h1_n); + // fprintf(stderr, "debug-w0: %f, g_p->a[2973].weight[0]: %f\n", w0, g_p->a[2973].weight[0]); + // fprintf(stderr, "debug-w1: %f, g_p->a[2973].weight[1]: %f\n", w1, g_p->a[2973].weight[1]); + + return !!occ; +} + +void destory_contig_partition(H_partition* hap) +{ + free(hap->lock); + free(hap->hap); + destory_G_partition(&(hap->g_p)); + destory_G_partition(&(hap->group_g_p)); + kv_destroy(hap->label_buffer); + kv_destroy(hap->b.vis); +} + +void label_unitigs(G_partition* g_p, ma_ug_t* ug) +{ + memset(R_INF.trio_flag, AMBIGU, R_INF.total_reads * sizeof(uint8_t)); + uint32_t i, k, j, *h0, h0_n, *h1, h1_n, uID, *h = NULL, h_n, flag = AMBIGU; + int status; + ma_utg_t *u = NULL; + + for (i = 0; i < g_p->n; i++) + { + if(g_p->a[i].h[0] > 0 && g_p->a[i].status[0] != 1 && g_p->a[i].status[0] != -1) continue; + if(g_p->a[i].h[1] > 0 && g_p->a[i].status[1] != 1 && g_p->a[i].status[1] != -1) continue; + get_phased_block(g_p, NULL, i, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + + status = g_p->a[i].status[0]; + h = h0; h_n = h0_n; + if(status == 1) + { + flag = FATHER; + } + else if (status == -1) + { + flag = MOTHER; + } + for (j = 0; j < h_n; j++) + { + uID = h[j]; + u = &ug->u.a[uID]; + if(u->m == 0) continue; + for (k = 0; k < u->n; k++) + { + R_INF.trio_flag[u->a[k]>>33] = flag; + } + } + + + + status = g_p->a[i].status[1]; + h = h1; h_n = h1_n; + if(status == 1) + { + flag = FATHER; + } + else if (status == -1) + { + flag = MOTHER; + } + for (j = 0; j < h_n; j++) + { + uID = h[j]; + u = &ug->u.a[uID]; + if(u->m == 0) continue; + for (k = 0; k < u->n; k++) + { + R_INF.trio_flag[u->a[k]>>33] = flag; + } + } + } + + + uint64_t occ = 0; + for (i = 0; i < ug->u.n; i++) + { + occ += ug->u.a[i].n; + } + + ///fprintf(stderr, "# reads: %lu\n", occ); + + for (i = occ = 0; i < R_INF.total_reads; i++) + { + if(R_INF.trio_flag[i] == FATHER) occ++; + } + + ///fprintf(stderr, "# Father reads: %lu\n", occ); + + for (i = occ = 0; i < R_INF.total_reads; i++) + { + if(R_INF.trio_flag[i] == MOTHER) occ++; + } + + ///fprintf(stderr, "# Mother reads: %lu\n", occ); +} + + +void print_bubble_graph(bubble_type* bub, ma_ug_t* ug, const char* prefix, FILE *fp) +{ + uint32_t i, k, *a, n, beg, sink, x; + asg_t *b_g = bub->b_g; + char name[32]; + for (i = 0; i < b_g->n_seq; i++) + { + get_bubbles(bub, i, &beg, &sink, &a, &n, NULL); + sprintf(name, "%s%.6d%c", prefix, i, "fb"[if_bub?0:1]); + fprintf(stderr, "S\t%s\t*\tLN:i:%d\n", name, n); + + if(beg != (uint32_t)-1) fprintf(stderr, "A\tutg%.6d%c\t%s\n", (beg>>1)+1, "lc"[ug->u.a[(beg>>1)].circ], "beg"); + if(sink != (uint32_t)-1) fprintf(stderr, "A\tutg%.6d%c\t%s\n", (sink>>1)+1, "lc"[ug->u.a[(sink>>1)].circ], "sink"); + for (k = 0; k < n; k++) + { + x = a[k]>>1; + fprintf(stderr, "A\tutg%.6d%c\t%s\n", x+1, "lc"[ug->u.a[x].circ], "mid"); + } + } + + asg_arc_t* au = NULL; + uint32_t nu, u, v; + for (i = 0; i < b_g->n_seq; i++) + { + u = i<<1; + au = asg_arc_a(b_g, u); + nu = asg_arc_n(b_g, u); + for (k = 0; k < nu; k++) + { + if(au[k].del) continue; + v = au[k].v; + fprintf(stderr, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\n", + prefix, u>>1, "fb"[(u>>1)f_bub?0:1], "+-"[u&1], + prefix, v>>1, "fb"[(v>>1)f_bub?0:1], "+-"[v&1], 0, 0); + + + asg_arc_t* av = asg_arc_a(b_g, v^1); + uint32_t nv = asg_arc_n(b_g, v^1), m; + for (m = 0; m < nv; m++) + { + if(av[m].del) continue; + if(av[m].v == (u^1)) break; + } + + if(m == nv) fprintf(stderr, "sb1sb, nv: %u, nu: %u\n", nv, nu); + } + + + u = (i<<1) + 1; + au = asg_arc_a(ug->g, u); + nu = asg_arc_n(ug->g, u); + for (k = 0; k < nu; k++) + { + if(au[k].del) continue; + v = au[k].v; + fprintf(stderr, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\n", + prefix, u>>1, "fb"[(u>>1)f_bub?0:1], "+-"[u&1], + prefix, v>>1, "fb"[(v>>1)f_bub?0:1], "+-"[v&1], 0, 0); + + asg_arc_t* av = asg_arc_a(b_g, v^1); + uint32_t nv = asg_arc_n(b_g, v^1), m; + for (m = 0; m < nv; m++) + { + if(av[m].del) continue; + if(av[m].v == (u^1)) break; + } + + if(m == nv) fprintf(stderr, "sb2sb, nv: %u, nu: %u\n", nv, nu); + } + } +} + + + +void print_bubble_utg(bubble_type* bub, ma_ug_t* unitig_ug, const char* prefix, FILE *fp) +{ + uint32_t i, k, *a, n, beg, sink, x, occ; + ma_ug_t *b_ug = bub->b_ug; + char name[32]; + for (i = 0; i < b_ug->u.n; i++) + { + ma_utg_t *p = &b_ug->u.a[i]; + if(p->n == 0) continue; + for (k = occ = 0; k < p->n; k++) + { + x = p->a[k]>>33; + get_bubbles(bub, x, &beg, &sink, &a, &n, NULL); + occ += n; + } + sprintf(name, "%s%.6d%c", prefix, i + 1, "lc"[p->circ]); + fprintf(fp, "S\t%s\t*\tLN:i:%u\n", name, occ); + for (k = 0; k < p->n; k++) + { + x = p->a[k]>>33; + get_bubbles(bub, x, &beg, &sink, &a, &n, NULL); + if(beg != (uint32_t)-1) fprintf(fp, "A\tutg%.6d%c\t%u\t%s\n", (beg>>1)+1, "lc"[unitig_ug->u.a[(beg>>1)].circ], n, "beg"); + if(sink != (uint32_t)-1) fprintf(fp, "A\tutg%.6d%c\t%u\t%s\n", (sink>>1)+1, "lc"[unitig_ug->u.a[(sink>>1)].circ], n, "sink"); + } + } + + asg_arc_t* au = NULL; + uint32_t nu, u, v, j; + for (i = 0; i < b_ug->u.n; ++i) { + if(b_ug->u.a[i].m == 0) continue; + if(b_ug->u.a[i].circ) + { + fprintf(fp, "L\t%s%.6dc\t+\t%s%.6dc\t+\t%dM\tL1:i:%d\n", + prefix, i+1, prefix, i+1, 0, 0); + fprintf(fp, "L\t%s%.6dc\t-\t%s%.6dc\t-\t%dM\tL1:i:%d\n", + prefix, i+1, prefix, i+1, 0, 0); + } + u = i<<1; + au = asg_arc_a(b_ug->g, u); + nu = asg_arc_n(b_ug->g, u); + for (j = 0; j < nu; j++) + { + if(au[j].del) continue; + v = au[j].v; + fprintf(fp, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\n", + prefix, (u>>1)+1, "lc"[b_ug->u.a[u>>1].circ], "+-"[u&1], + prefix, (v>>1)+1, "lc"[b_ug->u.a[v>>1].circ], "+-"[v&1], 0, 0); + } + + + u = (i<<1) + 1; + au = asg_arc_a(b_ug->g, u); + nu = asg_arc_n(b_ug->g, u); + for (j = 0; j < nu; j++) + { + if(au[j].del) continue; + v = au[j].v; + fprintf(fp, "L\t%s%.6d%c\t%c\t%s%.6d%c\t%c\t%dM\tL1:i:%d\n", + prefix, (u>>1)+1, "lc"[b_ug->u.a[u>>1].circ], "+-"[u&1], + prefix, (v>>1)+1, "lc"[b_ug->u.a[v>>1].circ], "+-"[v&1], 0, 0); + } + } + + +} + + +void print_debug_bubble_graph(bubble_type* bub, ma_ug_t* ug, const char *fn) +{ + char *buf = (char*)calloc(strlen(fn) + 25, 1); + sprintf(buf, "%s.bub.gfa", fn); + FILE* fp = fopen(buf, "w"); + + print_bubble_utg(bub, ug, "btg", fp); + + fclose(fp); + free(buf); +} + +void print_bubble_chain(bubble_type* bub) +{ + uint32_t m, i; + uint32_t beg, sink; + uint64_t bid; + ma_utg_t *u = NULL; + for (m = 0; m < bub->chain_weight.n; m++) + { + if(bub->chain_weight.a[m].del) continue; + u = &(bub->b_ug->u.a[bub->chain_weight.a[m].id]); + fprintf(stderr, "\nChain_id=%lu\n", bub->chain_weight.a[m].id); + for (i = 0; i < u->n; i++) + { + bid = u->a[i]>>33; + get_bubbles(bub, bid, &beg, &sink, NULL, NULL, NULL); + fprintf(stderr, "btg%.6lu%c, beg-utg%.6ul, sink-utg%.6ul\n", + bid, "fb"[bidf_bub?0:1], (beg>>1)+1, (sink>>1)+1); + } + } +} + +void init_contig_H_partition(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit* hits, H_partition* hap) +{ + uint32_t i, k_i, k_j, uID, *a = NULL, n, *h0, h0_n, *h1, h1_n; + destory_G_partition(&(hap->group_g_p)); memset(&(hap->group_g_p), 0, sizeof(G_partition)); + init_G_partition(&(hap->group_g_p), hap->n); + partition_warp *res = NULL; + ma_utg_t *u_x = NULL; + chain_hic_warp *c_w = &(bub->c_w); + + for (i = 0; i < bub->c_w.n; i++) + { + kv_pushp(partition_warp, hap->group_g_p, &res); + kv_init(res->a); + res->full_bub = 0; + res->h[0] = res->h[1] = 0; + + res->status[0] = 1; res->status[1] = -1; + res->weight[0] = res->weight[1] = res->weight_convex = 0; + + + u_x = (*c_w).a[(*c_w).a[i].id].u; + for (k_i = 0; k_i < u_x->n; k_i++) + { + get_bubbles(bub, u_x->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + for (k_j = 0; k_j < n; k_j++) + { + uID = a[k_j]>>1; + if((*c_w).chain_idx[uID] != (*c_w).a[i].id) continue; + if(get_phase_status(hap, uID)==1) + { + kv_push(uint32_t, res->a, uID); + res->h[0]++; + } + } + } + + for (k_i = 0; k_i < u_x->n; k_i++) + { + get_bubbles(bub, u_x->a[k_i]>>33, NULL, NULL, &a, &n, NULL); + for (k_j = 0; k_j < n; k_j++) + { + uID = a[k_j]>>1; + if((*c_w).chain_idx[uID] != (*c_w).a[i].id) continue; + if(get_phase_status(hap, uID)==-1) + { + kv_push(uint32_t, res->a, uID); + res->h[1]++; + } + } + } + + + for (k_i = 0; k_i < res->h[0]; k_i++) + { + ///if(hap->group_g_p.index[res->a.a[k_i]] != (uint32_t)-1) fprintf(stderr, "ERROR---00\n"); + hap->group_g_p.index[res->a.a[k_i]] = hap->group_g_p.n-1; + hap->group_g_p.index[res->a.a[k_i]] = hap->group_g_p.index[res->a.a[k_i]] << 1; + } + + for (; k_i < res->a.n; k_i++) + { + ///if(hap->group_g_p.index[res->a.a[k_i]] != (uint32_t)-1) fprintf(stderr, "ERROR---11\n"); + hap->group_g_p.index[res->a.a[k_i]] = hap->group_g_p.n-1; + hap->group_g_p.index[res->a.a[k_i]] = (hap->group_g_p.index[res->a.a[k_i]] << 1) + 1; + } + + get_phased_block(&(hap->group_g_p), NULL, i, NULL, NULL, &h0, &h0_n, &h1, &h1_n, NULL, NULL); + if(h0_n >0) res->weight[0] = get_cluster_weight(hap, hap->link, h0, h0_n); + if(h1_n >0) res->weight[1] = get_cluster_weight(hap, hap->link, h1, h1_n); + res->weight_convex = get_cluster_inner_weight(hap, hap->link, h0, h0_n, h1, h1_n); + } + + flip_by_node(hap, &(hap->group_g_p), bub); + label_unitigs(&(hap->group_g_p), idx->ug); +} + +void cluster_contigs(bubble_type* bub, ha_ug_index* idx, kvec_pe_hit* hits, MT* M, H_partition* hap) +{ + uint64_t k, i, shif = 64 - idx->uID_bits, beg, end, t_d; + hc_links* link = idx->link; + for (i = 0; i < link->a.n; i++) link->a.a[i].e.n = 0; + for (k = 0; k < hits->a.n; ++k) + { + beg = ((hits->a.a[k].s<<1)>>shif); + end = ((hits->a.a[k].e<<1)>>shif); + + if(beg == end) continue; + if(IF_HOM(beg, *bub)) continue; + if(IF_HOM(end, *bub)) continue; + + t_d = 1; + push_hc_edge(&(link->a.a[beg]), end, 0, 0, &t_d); + push_hc_edge(&(link->a.a[end]), beg, 0, 0, &t_d); + } + + init_hic_p((ha_ug_index*)idx, hits, link, bub, NULL, M, NULL, 1); + + init_chain_hic_warp(idx->ug, link, bub, &bub->c_w); + + hap->link = link; + hap->n = idx->ug->u.n; + + init_contig_H_partition(bub, idx, hits, hap); + + destory_chain_hic_warp(&bub->c_w); +} + +void reset_H_partition(H_partition* hap, uint32_t is_init) +{ + if(!is_init) + { + hap->n = 0; + free(hap->lock); + free(hap->hap); + hap->m[0] = hap->m[1] = hap->m[2] = (uint32_t)-1; + hap->label = hap->label_add = hap->label_shift = (uint32_t)-1; + destory_G_partition(&(hap->g_p)); memset(&(hap->g_p), 0, sizeof(G_partition)); + destory_G_partition(&(hap->group_g_p)); memset(&(hap->group_g_p), 0, sizeof(G_partition)); + kv_destroy(hap->label_buffer); kv_init(hap->label_buffer); + kv_destroy(hap->b.vis); kv_init(hap->b.vis); memset(&(hap->b), 0, sizeof(block_phase_type)); + } + + memset(hap, 0, sizeof(H_partition)); +} + + +int alignment_worker_pipeline(sldat_t* sl, const enzyme *fn1, const enzyme *fn2) +{ + int i; + for (i = 0; i < fn1->n && i < fn2->n; i++) + { + gzFile fp1, fp2; + if ((fp1 = gzopen(fn1->a[i], "r")) == 0) return 0; + if ((fp2 = gzopen(fn2->a[i], "r")) == 0) return 0; + sl->ks1 = kseq_init(fp1); + sl->ks2 = kseq_init(fp2); + + kt_pipeline(3, worker_pipeline, sl, 3); + + ///fprintf(stderr, "fn1->a[i]: %s, fn2->a[i]: %s, sl->hits.a.n: %u\n", fn1->a[i], fn2->a[i], (uint32_t)sl->hits.a.n); + + + kseq_destroy(sl->ks1); + kseq_destroy(sl->ks2); + gzclose(fp1); + gzclose(fp2); + } + + ///fprintf(stderr, "+sl->hits.a.n: %u\n", (uint32_t)sl->hits.a.n); + + dedup_hits(&(sl->hits)); + + ///fprintf(stderr, "-sl->hits.a.n: %u\n", (uint32_t)sl->hits.a.n); + + return 1; +} + +int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx) +{ + double index_time = yak_realtime(); + sldat_t sl; + kvec_hc_edge back_hc_edge; + kv_init(back_hc_edge.a); + sl.idx = idx; + sl.link = idx->link; + sl.chunk_size = 20000000; + sl.n_thread = asm_opt.thread_num; + sl.total_base = sl.total_pair = 0; + idx->max_cnt = 5; + kv_init(sl.hits.a); + + if(!load_hc_hits(&sl.hits, asm_opt.output_file_name)) + { + /*******************************for debug************************************/ + // load_reads(&R1, fn1); + // test_reads(&R1, fn1); + // load_reads(&R2, fn2); + // test_reads(&R1, fn1); + /*******************************for debug************************************/ + + // kt_pipeline(3, worker_pipeline, &sl, 3); + // dedup_hits(&sl.hits); + alignment_worker_pipeline(&sl, fn1, fn2); + /*******************************for debug************************************/ + // sort_hits(&sl.hits); + // print_hits(idx, &sl.hits, fn1); + /*******************************for debug************************************/ + + write_hc_hits(&sl.hits, asm_opt.output_file_name); + } + + ///fprintf(stderr, "u.n: %d, uID_bits: %lu, pos_bits: %lu, sl.hits.a.n: %u\n", (uint32_t)idx->ug->u.n, idx->uID_bits, idx->pos_bits, (uint32_t)sl.hits.a.n); + + H_partition hap; + MT M; + init_MT(&M, idx->ug->g->n_seq<<1); + bubble_type bub; + memset(&bub, 0, sizeof(bubble_type)); + bub.round_id = 0; bub.n_round = 2; + for (bub.round_id = 0; bub.round_id < bub.n_round; bub.round_id++) + { + identify_bubbles(idx->ug, &bub, idx->link); + if(bub.round_id == 0) + { + collect_hc_links(sl.idx, &sl.hits, idx->link, &bub, &M); + collect_hc_reverse_links(idx->link, idx->ug, &bub); + } + init_hic_p((ha_ug_index*)sl.idx, &sl.hits, idx->link, &bub, &back_hc_edge, &M, &hap, 0); + ///init_hic_p_new((ha_ug_index*)sl.idx, &sl.hits, idx->link, &bub, &back_hc_edge, &M); + reset_H_partition(&hap, (bub.round_id == 0? 1 : 0)); + init_contig_partition(&hap, idx, &bub); + phasing_improvement(&hap, &(hap.g_p), idx, &bub); + label_unitigs(&(hap.g_p), idx->ug); + + ///print_hc_links(idx->link, 0, &hap); + } + + cluster_contigs(&bub, idx, &sl.hits, &M, &hap); + + destory_MT(&M); + + ///print_bubbles(idx->ug, &bub, sl.hits.a.n?&sl.hits:NULL, idx->link, idx); + ///print_hits(idx, &sl.hits, fn1); + + + ///print_debug_bubble_graph(&bub, idx->ug, asm_opt.output_file_name); + // print_bubble_chain(&bub); + // print_hc_links(idx->link, 0, &hap); + + ///print_contig_partition(&hap, "final"); + + // uint32_t i; + // for (i = 0; i < idx->ug->g->n_seq; i++) + // { + // fprintf(stderr, "utg%.6ul, index: %u\n", (int)(i+1), bub.index[i]); + // } + + + + + destory_contig_partition(&hap); + kv_destroy(back_hc_edge.a); + return 1; + + /*******************************for debug************************************/ + // destory_reads(&R1); + // destory_reads(&R2); + /*******************************for debug************************************/ + print_bubbles(idx->ug, &bub, sl.hits.a.n?&sl.hits:NULL, idx->link, idx); + collect_hc_reverse_links(idx->link, idx->ug, &bub); + normalize_hc_links(idx->link); + /*******************************for debug************************************/ + ///print_hc_links(&link); + /*******************************for debug************************************/ + min_cut_t* cut = clean_hap(idx->link, &bub, idx->ug); + ///print_bubbles(idx->ug, &bub, NULL, &link, idx); + G_partition* gp = clean_bubbles(idx->link, &bub, cut, idx->ug); + ///print_hc_links(&link); + + destory_min_cut_t(cut); free(cut); + destory_G_partition(gp); free(gp); + kv_destroy(sl.hits.a); + destory_bubbles(&bub); + + fprintf(stderr, "[M::%s::%.3f] processed %lu pairs; %lu bases\n", __func__, yak_realtime()-index_time, sl.total_pair, sl.total_base); + return 1; +} + + +void hic_analysis(ma_ug_t *ug, asg_t* read_g, hc_links* link) +{ + ug_index = NULL; + int exist = load_hc_pt_index(&ug_index, asm_opt.output_file_name); + if(exist == 0) ug_index = build_unitig_index(ug, asm_opt.hic_mer_length); + if(exist == 0) write_hc_pt_index(ug_index, asm_opt.output_file_name); + ug_index->ug = ug; + ug_index->read_g = read_g; + ug_index->link = link; + ///test_unitig_index(ug_index, ug); + hic_short_align(asm_opt.hic_reads[0], asm_opt.hic_reads[1], ug_index); + + destory_hc_pt_index(ug_index); +} + +typedef struct{ + //[uID_start, uID_end) + uint64_t uID_start; + uint64_t uID_end; + uint64_t u_n; + uint64_t r_n; + uint64_t* r_idx; +} bench_utg; + +typedef struct{ + uint64_t s, e; +}homo_interval; + +typedef struct{ + kvec_t(bench_utg) ug_idx; + uint64_t uID_bits; + uint64_t pos_mode; + hc_links link; + kvec_t(homo_interval) regions; +}bench_idx; + +uint64_t* set_bench_idx(ma_ug_t *ug, asg_t* read_g, uint64_t uID_start, uint64_t uID_end, uint64_t uID_bits, uint64_t r_n) +{ + uint64_t *idx = (uint64_t*)malloc(sizeof(uint64_t)*r_n), i, k; + memset(idx, -1, sizeof(uint64_t)*r_n); + uint64_t rId, ori, start, l; + ma_utg_t *u = NULL; + for (i = uID_start; i < uID_end; i++) + { + u = &(ug->u.a[i]); + if(u->n == 0) continue; + for (k = l = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + ori = u->a[k]>>32&1; + start = l; + l += (uint32_t)u->a[k]; + if(idx[rId] != (uint64_t)-1) + { + idx[rId] = (uint64_t)-1; + } + else + { + idx[rId] = (ori<<63) + ((i<<(64-uID_bits))>>1) + start; + if(ori) idx[rId] = idx[rId] + read_g->seq[rId].len - 1; + } + } + } + + return idx; +} + +void get_r_utg_bench(uint64_t index, bench_idx* idx, ma_ug_t *ug) +{ + + bench_utg* a_list = idx->ug_idx.a; + uint64_t a_n = idx->ug_idx.n; + bench_utg *x = &(a_list[index]), *y = NULL; + uint64_t i, k, t, rev, x_uid, y_uid, y_pos, x_pos, d; + uint64_t rId, ori; + ma_utg_t *u = NULL; + for (i = x->uID_start; i < x->uID_end; i++) + { + u = &(ug->u.a[i]); + x_uid = i; + if(u->n == 0) continue; + for (k = 0; k < u->n; k++) + { + rId = u->a[k]>>33; + ori = u->a[k]>>32&1; + if(x->r_idx[rId] == (uint64_t)-1) continue; + x_pos = x->r_idx[rId] & idx->pos_mode; + + for (t = 0; t < a_n; t++) + { + if(t == index) continue; + y = &(a_list[t]); + if(y->r_idx[rId] == (uint64_t)-1) continue; + rev = 0; + if((y->r_idx[rId]>>63) != ori) rev = 1; + y_uid = (y->r_idx[rId]<<1)>>(64 - idx->uID_bits); + y_pos = y->r_idx[rId] & idx->pos_mode; + if(rev) y_pos = ug->u.a[y_uid].len - y_pos - 1; + ///if(ori) x_pos = ug->u.a[x_uid].len - x_pos - 1, y_pos = ug->u.a[y_uid].len - y_pos - 1; + d = MAX(x_pos, y_pos) - MIN(x_pos, y_pos); + d = (d<<2) + (rev<<1); + if(y_pos > x_pos) d = d + 1; + push_hc_edge(&(idx->link.a.a[x_uid]), y_uid, 1, 0, &d); + if(x_pos != y_pos) d = d ^ 1; + push_hc_edge(&(idx->link.a.a[y_uid]), x_uid, 1, 0, &d); + } + } + } +} + +void hap_ID(bench_idx* idx, uint64_t ID, uint64_t* hapID, uint64_t* uID) +{ + uint64_t i; + (*hapID) = (*uID) = (uint64_t)-1; + for (i = 0; i < idx->ug_idx.n; i++) + { + if(ID >= idx->ug_idx.a[i].uID_start && ID < idx->ug_idx.a[i].uID_end) + { + (*hapID) = i; + (*uID) = ID - idx->ug_idx.a[i].uID_start; + return; + } + } + return; +} + +void print_bench_idx(bench_idx* idx, ma_ug_t *ug) +{ + uint64_t i, k, s_uID, s_hapID, d_uID, d_hapID; + long long x[2] = {1, -1}; + for (i = 0; i < idx->link.a.n; i++) + { + for (k = 0; k < idx->link.a.a[i].e.n; k++) + { + if(idx->link.a.a[i].e.a[k].del) continue; + hap_ID(idx, i, &s_hapID, &s_uID); + hap_ID(idx, idx->link.a.a[i].e.a[k].uID, &d_hapID, &d_uID); + fprintf(stderr, "s-hap%lu-utg%.6d\td-hap%lu-utg%.6d\t%c\t%lld\n", + s_hapID, (int)(s_uID+1), d_hapID, (int)(d_uID+1), + "+-"[!!(idx->link.a.a[i].e.a[k].dis&(uint64_t)2)], + ((long long)(idx->link.a.a[i].e.a[k].dis>>2))*x[idx->link.a.a[i].e.a[k].dis&(uint64_t)1]); + } + } + + +} + +uint64_t get_hic_distance_bench(pe_hit* hit, hc_links* link, bench_idx* idx, ma_ug_t *ug, uint64_t* is_trans) +{ + (*is_trans) = (uint64_t)-1; + uint64_t s_uid, e_uid; + long long s_pos, e_pos; + s_uid = ((hit->s<<1)>>(64 - idx->uID_bits)); s_pos = hit->s & idx->pos_mode; + e_uid = ((hit->e<<1)>>(64 - idx->uID_bits)); e_pos = hit->e & idx->pos_mode; + if(s_uid == e_uid) + { + (*is_trans) = 0; + return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + } + + uint64_t s_i, e_i, k, ori; + for (s_i = 0; s_i < idx->ug_idx.n; s_i++) + { + if(s_uid >= idx->ug_idx.a[s_i].uID_start && s_uid < idx->ug_idx.a[s_i].uID_end) break; + } + for (e_i = 0; e_i < idx->ug_idx.n; e_i++) + { + if(e_uid >= idx->ug_idx.a[e_i].uID_start && e_uid < idx->ug_idx.a[e_i].uID_end) break; + } + if(s_i == idx->ug_idx.n || e_i == idx->ug_idx.n) return (uint64_t)-1; + if(s_i == e_i) + { + (*is_trans) = 0; + return (uint64_t)-1; + } + + (*is_trans) = 1; + hc_linkeage* t = &(link->a.a[s_uid]); + long long m_x[2] = {1, -1}, dis; + for (k = 0; k < t->e.n; k++) + { + if(t->e.a[k].del || t->e.a[k].uID != e_uid) continue; + ori = !!(t->e.a[k].dis & (uint64_t)2); + dis = (long long)(t->e.a[k].dis>>2) * m_x[t->e.a[k].dis & (uint64_t)1]; + if(ori) e_pos = ug->u.a[e_uid].len - e_pos - 1; + e_pos = e_pos + dis; + return MAX(s_pos, e_pos) - MIN(s_pos, e_pos); + } + + return (uint64_t)-1; +} + +void init_bench_idx(bench_idx* idx, asg_t* read_g, ma_ug_t *ug) +{ + uint64_t i, occ; + kv_init(idx->ug_idx); + kv_init(idx->regions); + kv_malloc(idx->ug_idx, ug->occ.n); idx->ug_idx.n = ug->occ.n; + for (idx->uID_bits = 1; (uint64_t)(1<uID_bits)<(uint64_t)ug->u.n; idx->uID_bits++); + idx->pos_mode = ((uint64_t)-1)>>(idx->uID_bits+1); + for (i = occ = 0; i < ug->occ.n; i++) + { + idx->ug_idx.a[i].uID_start = occ; + occ += ug->occ.a[i]; + idx->ug_idx.a[i].uID_end = occ; + idx->ug_idx.a[i].u_n = ug->occ.a[i]; + + idx->ug_idx.a[i].r_n = read_g->n_seq; + idx->ug_idx.a[i].r_idx + = set_bench_idx(ug, read_g, idx->ug_idx.a[i].uID_start, idx->ug_idx.a[i].uID_end, + idx->uID_bits, idx->ug_idx.a[i].r_n); + } + + init_hc_links(&(idx->link), ug->u.n, ug->g->n_seq); + + for (i = 0; i < idx->ug_idx.n; i++) + { + get_r_utg_bench(i, idx, ug); + } +} + +void evaluate_bench_idx(bench_idx* idx, kvec_pe_hit* hits, ma_ug_t *ug) +{ + uint64_t k, distance, is_trans, trans[2]; + kvec_t(uint64_t) buf; + kv_init(buf); + for (k = trans[0] = trans[1] = 0; k < hits->a.n; ++k) + { + distance = get_hic_distance_bench(&(hits->a.a[k]), &(idx->link), idx, ug, &is_trans); + if(is_trans != (uint64_t)-1) trans[is_trans]++; + if(distance == (uint64_t)-1 || is_trans == (uint64_t)-1) continue; + distance = (distance << 1) + is_trans; + kv_push(uint64_t, buf, distance); + } + + radix_sort_hc64(buf.a, buf.a+buf.n); + + for (k = 0; k < buf.n; k++) + { + fprintf(stderr, "%lu\t%lu\n", buf.a[k]>>1, buf.a[k]&1); + } + /** + uint64_t up_dis = buf.a[(uint64_t)(buf.n*0.99)]>>1, step = 7240; + uint64_t step_s = 0, step_e = step, cnt[2]; + for (k = cnt[0] = cnt[1] = 0; k < buf.n; k++) + { + if(step_s > up_dis) step_e = (buf.a[buf.n-1]>>1) + 1; + if((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s) + { + cnt[buf.a[k]&1]++; + } + if((buf.a[k]>>1) >= step_e) + { + while (!((buf.a[k]>>1) < step_e && (buf.a[k]>>1) >= step_s)) + { + fprintf(stderr, "i: %lu, step_s: %lu, step_e: %lu, cnt[0]: %lu, cnt[1]: %lu, rate: %f\n", + step_s/step, step_s, step_e, cnt[0], cnt[1], ((double)cnt[1])/(double)(cnt[0] + cnt[1])); + step_s += step; + step_e += step; + cnt[0] = cnt[1] = 0; + } + } + } + + if(cnt[0] > 0 || cnt[1] > 0) + { + fprintf(stderr, "i: %lu, step_s: %lu, step_e: %lu, cnt[0]: %lu, cnt[1]: %lu, rate: %f\n", + step_s/step, step_s, step_e, cnt[0], cnt[1], ((double)cnt[1])/(double)(cnt[0] + cnt[1])); + } + **/ + kv_destroy(buf); +} + +void destory_bench_idx(bench_idx* idx) +{ + uint64_t i; + for (i = 0; i < idx->ug_idx.n; i++) + { + free(idx->ug_idx.a[i].r_idx); + } + kv_destroy(idx->ug_idx); + kv_destroy(idx->regions); + destory_hc_links(&(idx->link)); +} + + +int hic_short_align_bench(const enzyme *fn1, const enzyme *fn2, const char *output_file_name, ha_ug_index* idx) +{ + double index_time = yak_realtime(); + sldat_t sl; + sl.idx = idx; + sl.link = NULL; + sl.chunk_size = 20000000; + sl.n_thread = asm_opt.thread_num; + sl.total_base = sl.total_pair = 0; + idx->max_cnt = 5; + kv_init(sl.hits.a); + fprintf(stderr, "u.n: %d, uID_bits: %lu, pos_bits: %lu\n", (uint32_t)idx->ug->u.n, idx->uID_bits, idx->pos_bits); + + if(!load_hc_hits(&sl.hits, output_file_name)) + { + // kt_pipeline(3, worker_pipeline, &sl, 3); + // dedup_hits(&sl.hits); + alignment_worker_pipeline(&sl, fn1, fn2); + write_hc_hits(&sl.hits, output_file_name); + } + bench_idx bench; + init_bench_idx(&bench, idx->read_g, idx->ug); + ///print_bench_idx(&bench, idx->ug); + evaluate_bench_idx(&bench, &sl.hits, idx->ug); + + destory_bench_idx(&bench); + kv_destroy(sl.hits.a); + fprintf(stderr, "[M::%s::%.3f] processed %lu pairs; %lu bases\n", __func__, yak_realtime()-index_time, sl.total_pair, sl.total_base); + return 1; +} + +void hic_benchmark(ma_ug_t *ug, asg_t* read_g) +{ + char *output_file_name = (char*)calloc(strlen(asm_opt.output_file_name) + 25, 1); + sprintf(output_file_name, "%s.bench", asm_opt.output_file_name); + ug_index = NULL; + int exist = load_hc_pt_index(&ug_index, output_file_name); + if(exist == 0) ug_index = build_unitig_index(ug, asm_opt.hic_mer_length); + if(exist == 0) write_hc_pt_index(ug_index, output_file_name); + ug_index->ug = ug; + ug_index->read_g = read_g; + + hic_short_align_bench(asm_opt.hic_reads[0], asm_opt.hic_reads[1], output_file_name, ug_index); + + free(output_file_name); +} \ No newline at end of file diff --git a/hic.h b/hic.h new file mode 100644 index 0000000..b604104 --- /dev/null +++ b/hic.h @@ -0,0 +1,72 @@ +#ifndef __HIC__ +#define __HIC__ +#include +#include "Overlaps.h" + +#define kdq_clear(q) ((q)->count = (q)->front = 0) +#define kv_malloc(v, s) ((v).n = 0, (v).m = (s), MALLOC((v).a, (s))) +#define RC_0 0 +#define RC_1 1 +#define RC_2 2 + +hc_edge* get_hc_edge(hc_links* link, uint64_t src, uint64_t dest, uint64_t dir); +void push_hc_edge(hc_linkeage* x, uint64_t uID, double weight, int dir, uint64_t* d); +void hic_analysis(ma_ug_t *ug, asg_t* read_g, hc_links* link); +void hic_benchmark(ma_ug_t *ug, asg_t* read_g); + +typedef struct { + double w; + uint32_t id, occ; + ///uint32_t *bid, bid_n; + ma_utg_t *u; + uint64_t l_d, r_d; +}chain_hic_w_type; + +typedef struct { + size_t n, m; + chain_hic_w_type* a; + uint32_t max_bub_id; + uint32_t *chain_idx, u_n; +}chain_hic_warp; + +typedef struct { + long long g_occ, b_occ; + uint64_t id; + uint8_t del; +}chain_w_type; + +typedef struct { + uint32_t* index, round_id, n_round; + ma_ug_t* ug; + kvec_t(uint32_t) list; + kvec_t(uint32_t) num; + kvec_t(uint64_t) pathLen; + kvec_t(uint64_t) b_s_idx; + uint64_t s_bub, f_bub, b_bub, b_end_bub, tangle_bub, cross_bub, mess_bub; + uint32_t check_het; + asg_t *b_g; + ma_ug_t* b_ug; + kvec_t(chain_w_type) chain_weight; + chain_hic_warp c_w; +} bubble_type; +#define P_het(B) ((B).num.n) +#define M_het(B) ((B).num.n + 1) +// #define IF_BUB(ID, B) ((B).index[(ID)] < (B).num.n) +// #define IF_HET(ID, B) ((B).index[(ID)] == (B).num.n) +// #define IF_HOM(ID, B) ((B).index[(ID)] > (B).num.n) +#define IF_BUB(ID, B) ((B).index[(ID)] < (B).f_bub+1) +#define IF_HET(ID, B) ((B).index[(ID)] == (B).f_bub+1) +#define IF_HOM(ID, B) ((B).index[(ID)] > (B).f_bub+1) +#define Get_bub_num(RECORD) ((RECORD).num.n-1) +void get_bubbles(bubble_type* bub, uint64_t id, uint32_t* beg, uint32_t* sink, uint32_t** a, uint32_t* n, uint64_t* pathBase); +int load_hc_links(hc_links* link, const char *fn); +void write_hc_links(hc_links* link, const char *fn); +void destory_bubbles(bubble_type* bub); +void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link); +void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub); +uint32_t connect_bub_occ(bubble_type* bub, uint32_t root_id, uint32_t check_het); +void get_bub_id(bubble_type* bub, uint32_t root, uint64_t* id0, uint64_t* id1, uint32_t check_het); +void update_bubble_chain(ma_ug_t* ug, bubble_type* bub, uint32_t is_middle, uint32_t is_end); +void set_b_utg_weight_flag(bubble_type* bub, buf_t* b, uint32_t v, uint8_t* vis_flag, uint32_t flag, uint32_t* occ); + +#endif diff --git a/hifiasm.1 b/hifiasm.1 index 2093c6f..42bb851 100644 --- a/hifiasm.1 +++ b/hifiasm.1 @@ -1,4 +1,4 @@ -.TH hifiasm 1 "19 July 2020" "hifiasm-0.9 (r289)" "Bioinformatics tools" +.TH hifiasm 1 "13 Feb 2021" "hifiasm-0.14 (r310)" "Bioinformatics tools" .SH NAME .PP @@ -212,6 +212,32 @@ with suffix .B lowQ.bed [70]. Set 0 to disable. + +.TP +.BI --b-cov \ INT +Break contigs at potential misassemblies with INT-fold coverage [-1]. +Work with +.B --m-rate. +Set -1 to disable. + +.TP +.BI --m-rate \ FLOAT +Break contigs with <=FLOAT*coverage exact overlaps [0.75]. +Only work with +.B --b-cov +and +.B --h-cov. + + + + .SS Trio-partition options .TP 10 @@ -289,6 +315,17 @@ For ordinary samples, no need to enable this mode [experimental, not stable]. Write additional files to speed up the debugging of graph cleaning. +.SS Hi-C-partition options [experimental, not stable] + +.TP +.BI --h1 \ FILEs +File names of input Hi-C R1 [r1_1.fq,r1_2.fq,...] + +.TP +.BI --h2 \ FILEs +File names of input Hi-C R2 [r2_1.fq,r2_2.fq,...] + + .SH OUTPUTS .PP @@ -347,6 +384,21 @@ phased maternal/haplotype2 contig graph. This graph keeps the phased maternal/haplotype2 assembly. .RE +.PP +With Hi-C partition, hifiasm outputs the assembly graphs like trio partition, +but with additional prefix +.B [hic]. +In this mode, hifiasm keeps Hi-C alignment results and Hi-C index in two bin +files: +.B *hic.lk.bin +and +.B *hic.tlb.bin. +Rerunning hifiasm with different Hi-C reads needs to delete these bin files. +.RE + + + + .PP For each graph, hifiasm also outputs a simplified version without sequences for the ease of visualization. Hifiasm keeps corrected reads and overlaps in three diff --git a/htab.cpp b/htab.cpp index 37c476a..3881ceb 100644 --- a/htab.cpp +++ b/htab.cpp @@ -48,6 +48,7 @@ typedef struct { int32_t pre; int32_t n_thread; int64_t chunk_size; + int adaLen; } yak_copt_t; void yak_copt_init(yak_copt_t *o) @@ -596,7 +597,9 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip } } else { while ((ret = kseq_read(p->ks)) >= 0) { - int l = p->ks->seq.l; + int l = (int)(p->ks->seq.l) - (int)(p->opt->adaLen) - (int)(p->opt->adaLen); + if(l <= 0) continue; + if (p->n_seq >= 1<<28) { fprintf(stderr, "ERROR: this implementation supports no more than %d reads\n", 1<<28); exit(1); @@ -610,9 +613,9 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip int i, n_N; assert(l == (int)p->rs_out->read_length[p->n_seq]); for (i = n_N = 0; i < l; ++i) // count number of ambiguous bases - if (seq_nt4_table[(uint8_t)p->ks->seq.s[i]] >= 4) + if (seq_nt4_table[(uint8_t)p->ks->seq.s[i+p->opt->adaLen]] >= 4) ++n_N; - ha_compress_base(Get_READ(*p->rs_out, p->n_seq), p->ks->seq.s, l, &p->rs_out->N_site[p->n_seq], n_N); + ha_compress_base(Get_READ(*p->rs_out, p->n_seq), p->ks->seq.s+p->opt->adaLen, l, &p->rs_out->N_site[p->n_seq], n_N); memcpy(&p->rs_out->name[p->rs_out->name_index[p->n_seq]], p->ks->name.s, p->ks->name.l); } } @@ -623,7 +626,7 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip REALLOC(s->seq, s->m_seq); } MALLOC(s->seq[s->n_seq], l); - memcpy(s->seq[s->n_seq], p->ks->seq.s, l); + memcpy(s->seq[s->n_seq], p->ks->seq.s+p->opt->adaLen, l); s->len[s->n_seq++] = l; ++p->n_seq; s->sum_len += l; @@ -719,6 +722,48 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip return 0; } +void debug_adapter(const hifiasm_opt_t *asm_opt, All_reads *rs) +{ + int ret; + uint32_t i, m, pass, unpass; + gzFile fp = 0; + kseq_t *ks = NULL; + UC_Read ucr; + init_UC_Read(&ucr); + + for (i = m = pass = unpass = 0; i < (uint32_t)asm_opt->num_reads; ++i) + { + if ((fp = gzopen(asm_opt->read_file_names[i], "r")) == 0) continue; + ks = kseq_init(fp); + while ((ret = kseq_read(ks)) >= 0) + { + int l = ks->seq.l; + if((l - asm_opt->adapterLen*2) <= 0) continue; + recover_UC_Read(&ucr, rs, m); + fprintf(stderr, "l: %d, ucr.length: %lld, asm_opt->adapterLen: %d\n", + l, ucr.length, asm_opt->adapterLen); + if(memcmp(ucr.seq, ks->seq.s+asm_opt->adapterLen, ucr.length) == 0) + { + pass++; + } + else + { + unpass++; + } + m++; + } + kseq_destroy(ks); + gzclose(fp); + ks = NULL; + fp = 0; + } + + destory_UC_Read(&ucr); + + fprintf(stderr, "[M::%s::# reads: %u, # pass: %u, # unpass: %u\n]", __func__, m, pass, unpass); + exit(1); +} + static ha_ct_t *yak_count(const yak_copt_t *opt, const char *fn, int flag, ha_pt_t *p0, ha_ct_t *c0, const void *flt_tab, All_reads *rs, int64_t *n_seq) { ///for 0-th counting, flag = HAF_COUNT_ALL|HAF_RS_WRITE_LEN|HAF_CREATE_NEW @@ -791,6 +836,7 @@ ha_ct_t *ha_count(const hifiasm_opt_t *asm_opt, int flag, ha_pt_t *p0, const voi ///for ha_pt_gen, shoud be 0 opt.bf_shift = flag & HAF_COUNT_EXACT? 0 : asm_opt->bf_shift; opt.n_thread = asm_opt->thread_num; + opt.adaLen = asm_opt->adapterLen; ///asm_opt->num_reads is the number of fastq files for (i = 0; i < asm_opt->num_reads; ++i) h = yak_count(&opt, asm_opt->read_file_names[i], flag|HAF_CREATE_NEW, p0, h, flt_tab, rs, &n_seq); @@ -1008,16 +1054,17 @@ int load_ct_index(void **i_ct_idx, char* file_name) } ha_ct_t** ct_idx = (ha_ct_t**)i_ct_idx; double index_time = 0; + uint64_t flag = 0; int i; ha_ct_t *h = 0; ha_ct1_t *g; CALLOC(h, 1); - fread(&h->k, sizeof(h->k), 1, fp); - fread(&h->pre, sizeof(h->pre), 1, fp); - fread(&h->n_hash, sizeof(h->n_hash), 1, fp); - fread(&h->n_shift, sizeof(h->n_shift), 1, fp); - fread(&h->tot, sizeof(h->tot), 1, fp); + flag += fread(&h->k, sizeof(h->k), 1, fp); + flag += fread(&h->pre, sizeof(h->pre), 1, fp); + flag += fread(&h->n_hash, sizeof(h->n_hash), 1, fp); + flag += fread(&h->n_shift, sizeof(h->n_shift), 1, fp); + flag += fread(&h->tot, sizeof(h->tot), 1, fp); CALLOC(h->h, 1<pre); diff --git a/htab.h b/htab.h index 9a801a3..a809e00 100644 --- a/htab.h +++ b/htab.h @@ -60,6 +60,7 @@ void ha_triobin(const hifiasm_opt_t *opt); void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, int sample_dist, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct); int ha_analyze_count(int n_cnt, int start_cnt, const int64_t *cnt, int *peak_het); +void debug_adapter(const hifiasm_opt_t *asm_opt, All_reads *rs); static inline uint64_t yak_hash64(uint64_t key, uint64_t mask) // invertible integer hash function { diff --git a/khashl.h b/khashl.h index 4a8f2dc..4ecf294 100644 --- a/khashl.h +++ b/khashl.h @@ -147,13 +147,14 @@ static kh_inline khint_t __kh_h2b(khint_t hash, khint_t bits) { return hash * 26 SCOPE khint_t prefix##_load(HType **h, FILE* fp) { \ (*h) = prefix##_init(); \ khint_t n_buckets; \ - fread(&n_buckets, sizeof(n_buckets), 1, fp); \ - fread(&(*h)->bits, sizeof((*h)->bits), 1, fp); \ - fread(&(*h)->count, sizeof((*h)->count), 1, fp); \ + uint64_t flag = 0;\ + flag += fread(&n_buckets, sizeof(n_buckets), 1, fp); \ + flag += fread(&(*h)->bits, sizeof((*h)->bits), 1, fp); \ + flag += fread(&(*h)->count, sizeof((*h)->count), 1, fp); \ (*h)->used = (khint32_t*)kmalloc(__kh_fsize(n_buckets) * sizeof(khint32_t)); \ (*h)->keys = (khkey_t*)kmalloc(n_buckets * sizeof(khkey_t)); \ - fread((*h)->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \ - fread((*h)->keys, sizeof(khkey_t), n_buckets, fp); \ + flag += fread((*h)->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \ + flag += fread((*h)->keys, sizeof(khkey_t), n_buckets, fp); \ return 1; \ } \