Merge pull request #354 from chhylp123/hifiasm_dev_debug

fixed gap filling with UL

.github/workflows/ci.yaml

@@ -0,0 +1,21 @@
+name: CI
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        compiler: [gcc]
+
+    steps:
+    - name: Checkout minimap2
+      uses: actions/checkout@v2
+
+    - name: Compile with ${{ matrix.compiler }}
+      run:  make CC=${{ matrix.compiler }}

Assembly.cpp

@@ -10,13 +10,19 @@
 #include "Correct.h"
 #include "htab.h"
 #include "kthread.h"
+#include "rcut.h"
+#include "kalloc.h"
 
 void ha_get_candidates_interface(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_region_alloc *overlap_list, overlap_region_alloc *overlap_list_hp, Candidates_list *cl, double bw_thres, 
-int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* chain_idx, ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct);
+int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* chain_idx, ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, st_mt_t *sp);
+void ha_get_ug_candidates(ha_abuf_t *ab, int64_t rid, ma_utg_t *u, ma_utg_v *ua, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag,
+kvec_t_u64_warp* chain_idx, void *ha_flt_tab, ha_pt_t *ha_idx, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, double chain_match_rate);
 void ha_sort_list_by_anchor(overlap_region_alloc *overlap_list);
 
 All_reads R_INF;
 Debug_reads R_INF_FLAG;
+all_ul_t UL_INF, ULG_INF;
+uint32_t *het_cnt = NULL;
 
 void get_corrected_read_from_cigar(Cigar_record* cigar, char* pre_read, int pre_length, char* new_read, int* new_length)
 {
@@ -414,29 +420,37 @@ long long push_final_overlaps_increment(ma_hit_t_alloc* paf, ma_hit_t_alloc* rev
     return available_overlaps;
 }
 
-typedef struct {
-	int is_final, save_ov;
-	// chaining and overlapping related buffers
-	UC_Read self_read, ovlp_read;
-	Candidates_list clist;
-	overlap_region_alloc olist;
-    overlap_region_alloc olist_hp;
-	ha_abuf_t *ab;
-	// error correction related buffers
-	int64_t num_read_base, num_correct_base, num_recorrect_base;
-	Cigar_record cigar1;
-	Graph POA_Graph;
-	Graph DAGCon;
-	Correct_dumy correct;
-	haplotype_evdience_alloc hap;
-	Round2_alignment round2;
-    kvec_t_u32_warp b_buf;
-    kvec_t_u64_warp r_buf;
-    kvec_t_u8_warp k_flag;
-    overlap_region tmp_region;
-} ha_ovec_buf_t;
+ha_ovec_buf_t *ha_ovec_buf_init(void *km, int is_final, int save_ov, int is_ug)
+{
+	ha_ovec_buf_t *b;
+	KCALLOC(km, b, 1);
+	b->is_final = !!is_final, b->save_ov = !!save_ov;
+	init_UC_Read(&b->self_read);//set 0
+	init_UC_Read(&b->ovlp_read);//set 0
+	init_Candidates_list(&b->clist);//set 0
+    memset(&b->olist, 0, sizeof(overlap_region_alloc));
+    memset(&b->olist_hp, 0, sizeof(overlap_region_alloc));
+    // init_overlap_region_alloc(&b->olist);
+    // init_overlap_region_alloc(&b->olist_hp);
+    init_fake_cigar(&(b->tmp_region.f_cigar));//set 0
+    kv_init(b->b_buf.a);//set 0
+    kv_init(b->r_buf.a);//set 0
+    kv_init(b->k_flag.a);//set 0
+    kv_init(b->sp);//set 0
+	if(!is_ug) b->ab = ha_abuf_init_buf(km);
+    else b->abl = ha_abufl_init_buf(km);
+	if (!b->is_final) {
+        init_Cigar_record_buf(&b->cigar1, km);
+		// init_Graph(&b->POA_Graph);
+		// init_Graph(&b->DAGCon);
+		init_Correct_dumy_buf(&b->correct, km);//set 0
+        InitHaplotypeEvdience_buf(&b->hap, km);
+		init_Round2_alignment_buf(&b->round2, km);
+	}
+	return b;
+}
 
-ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov)
+ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov, int is_ug)
 {
 	ha_ovec_buf_t *b;
 	CALLOC(b, 1);
@@ -447,10 +461,15 @@ ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov)
 	init_overlap_region_alloc(&b->olist);
     init_overlap_region_alloc(&b->olist_hp);
     init_fake_cigar(&(b->tmp_region.f_cigar));
+    memset(&(b->tmp_region.w_list), 0, sizeof(b->tmp_region.w_list));
+    CALLOC(b->tmp_region.w_list.a, 1); b->tmp_region.w_list.n = b->tmp_region.w_list.m = 1;
     kv_init(b->b_buf.a);
     kv_init(b->r_buf.a);
     kv_init(b->k_flag.a);
-	b->ab = ha_abuf_init();
+    kv_init(b->sp);
+    init_bit_extz_t(&(b->exz), 31);
+	if(!is_ug) b->ab = ha_abuf_init();
+    else b->abl = ha_abufl_init();
 	if (!b->is_final) {
 		init_Cigar_record(&b->cigar1);
 		init_Graph(&b->POA_Graph);
@@ -470,10 +489,14 @@ void ha_ovec_destroy(ha_ovec_buf_t *b)
 	destory_overlap_region_alloc(&b->olist);
     destory_overlap_region_alloc(&b->olist_hp);
 	ha_abuf_destroy(b->ab);
+    ha_abufl_destroy(b->abl);
     destory_fake_cigar(&(b->tmp_region.f_cigar));
+    free(b->tmp_region.w_list.a); free(b->tmp_region.w_list.c.a);
     kv_destroy(b->b_buf.a);
     kv_destroy(b->r_buf.a);
     kv_destroy(b->k_flag.a);
+    kv_destroy(b->sp);
+    destroy_bit_extz_t(&(b->exz));
 	if (!b->is_final) {
 		destory_Cigar_record(&b->cigar1);
 		destory_Graph(&b->POA_Graph);
@@ -499,51 +522,94 @@ static int64_t ha_Graph_mem(const Graph *g)
 	return mem;
 }
 
-int64_t ha_ovec_mem(const ha_ovec_buf_t *b)
+int64_t ha_ovec_mem(const ha_ovec_buf_t *b, int64_t *mem_a)
 {
-	int64_t i, mem = 0, mem_clist, mem_olist;
-	mem_clist = b->clist.size * sizeof(k_mer_hit) + b->clist.chainDP.size * 7 * 4;
-    
+	int64_t i, mem_ab = 0, mem_clist, mem_olist, mem_hap = 0, mem_aux = 0;
+	// mem_clist = b->clist.size * sizeof(k_mer_hit) + b->clist.chainDP.size * 7 * 4;
+    mem_clist = (b->clist.size * sizeof(k_mer_hit)) + ((sizeof((*b->clist.chainDP.score)) + sizeof((*b->clist.chainDP.pre)) 
+                    + sizeof((*b->clist.chainDP.indels)) + sizeof((*b->clist.chainDP.self_length)) 
+                    + sizeof((*b->clist.chainDP.occ)) + sizeof((*b->clist.chainDP.tmp))) * b->clist.chainDP.size);
+    mem_clist += sizeof(*(b->b_buf.a.a)) * b->b_buf.a.m;
+    mem_clist += sizeof(*(b->r_buf.a.a)) * b->r_buf.a.m;
+    mem_clist += sizeof(*(b->k_flag.a.a)) * b->k_flag.a.m;
+    mem_clist += sizeof(*(b->sp.a)) * b->sp.m;
+    mem_clist += sizeof(*(b->tmp_region.f_cigar.buffer)) * b->tmp_region.f_cigar.size;
+    mem_clist += sizeof(*(b->tmp_region.w_list.a)) * b->tmp_region.w_list.n;
+    mem_clist += sizeof(*(b->tmp_region.w_list.c.a)) * b->tmp_region.w_list.c.n;
+
 	mem_olist = b->olist.size * sizeof(overlap_region);
 	for (i = 0; i < (int64_t)b->olist.size; ++i) {
 		const overlap_region *r = &b->olist.list[i];
-		mem_olist += r->w_list_size * sizeof(window_list);
+		mem_olist += (r->w_list.n*sizeof(*(r->w_list.a))) + (r->w_list.c.n*sizeof(*(r->w_list.c.a)));
 		mem_olist += r->f_cigar.size * 8;
-		mem_olist += r->boundary_cigars.size * sizeof(window_list);
+		mem_olist += (r->boundary_cigars.n*sizeof(*(r->boundary_cigars.a))) 
+                                    + (r->boundary_cigars.c.n*sizeof(*(r->boundary_cigars.c.a)));
 	}
+
     mem_olist += b->olist_hp.size * sizeof(overlap_region);
 	for (i = 0; i < (int64_t)b->olist_hp.size; ++i) {
 		const overlap_region *r = &b->olist_hp.list[i];
-		mem_olist += r->w_list_size * sizeof(window_list);
+		mem_olist += (r->w_list.n*sizeof(*(r->w_list.a))) + (r->w_list.c.n*sizeof(*(r->w_list.c.a)));
 		mem_olist += r->f_cigar.size * 8;
-		mem_olist += r->boundary_cigars.size * sizeof(window_list);
+		mem_olist += (r->boundary_cigars.n*sizeof(*(r->boundary_cigars.a))) 
+                                    + (r->boundary_cigars.c.n*sizeof(*(r->boundary_cigars.c.a)));
 	}
 
-	mem = ha_abuf_mem(b->ab) + mem_clist + mem_olist;
+    if(b->ab) mem_ab += ha_abuf_mem(b->ab);
+    if(b->abl) mem_ab += ha_abufl_mem(b->abl);
+
 	if (!b->is_final) {
-		mem += sizeof(Cigar_record) + b->cigar1.lost_base_size + b->cigar1.size * 4;
-		mem += sizeof(Correct_dumy) + b->correct.size * 8;
-		mem += sizeof(Round2_alignment) + b->round2.cigar.size * 4 + b->round2.tmp_cigar.size * 4;
-		mem += sizeof(haplotype_evdience_alloc) + b->hap.size * sizeof(haplotype_evdience) + b->hap.snp_matrix_size + b->hap.snp_stat_size * sizeof(SnpStats);
-		mem += ha_Graph_mem(&b->POA_Graph);
-		mem += ha_Graph_mem(&b->DAGCon);
+		mem_hap += sizeof(Cigar_record) + b->cigar1.lost_base_size + b->cigar1.size * 4;
+		mem_hap += sizeof(Correct_dumy) + b->correct.size * 8;
+		mem_hap += sizeof(Round2_alignment) + b->round2.cigar.size * 4 + b->round2.tmp_cigar.size * 4;
+		mem_hap += sizeof(haplotype_evdience_alloc) + b->hap.size * sizeof(haplotype_evdience) + b->hap.snp_matrix_size + b->hap.r_snp_size + b->hap.snp_stat.m * sizeof(SnpStats) + b->hap.snp_srt.m * sizeof(uint64_t);
+		mem_hap += ha_Graph_mem(&b->POA_Graph);
+		mem_hap += ha_Graph_mem(&b->DAGCon);
 	}
-	return mem;
+   
+
+    mem_aux += sizeof(*(b->self_read.seq)) * b->self_read.size;
+    mem_aux += sizeof(*(b->ovlp_read.seq)) * b->ovlp_read.size;
+
+    if(mem_a) {
+        mem_a[0] = mem_ab; mem_a[1] = mem_clist; mem_a[2] = mem_olist; 
+        mem_a[3] = mem_hap; mem_a[4] = mem_aux;
+    }
+
+	return mem_ab + mem_clist + mem_olist + mem_hap + mem_aux;
+}
+
+uint32_t get_het_cnt(haplotype_evdience_alloc *hap)
+{
+    uint32_t i, cnt;
+    for (i = cnt = 0; i < hap->snp_stat.n; i++) {
+        if(hap->snp_stat.a[i].score == 1 && (!(hap->snp_stat.a[i].occ_0 < 2 || hap->snp_stat.a[i].occ_1 < 2))) {
+            cnt++;
+        }
+    }
+    return cnt;
 }
 
 static void worker_ovec(void *data, long i, int tid)
 {
 	ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid];
 	int fully_cov, abnormal;
+    // if(i != 33) return;
+    // fprintf(stderr, "[M::%s-beg] rid->%ld\n", __func__, i);
+    // if (memcmp("m64012_190920_173625/88015004/ccs", Get_NAME((R_INF), i), Get_NAME_LENGTH((R_INF),i)) == 0) {
+    //     fprintf(stderr, "[M::%s-beg] rid->%ld\n", __func__, i);
+    // } else {
+    //     return;
+    // }
 
     ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 
-    0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL);
+    0.02, asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
 
 	clear_Cigar_record(&b->cigar1);
 	clear_Round2_alignment(&b->round2);
 
 	correct_overlap(&b->olist, &R_INF, &b->self_read, &b->correct, &b->ovlp_read, &b->POA_Graph, &b->DAGCon,
-			&b->cigar1, &b->hap, &b->round2, 0, 1, &fully_cov, &abnormal);
+			&b->cigar1, &b->hap, &b->round2, &b->r_buf, &(b->tmp_region.w_list), 0, 1, &fully_cov, &abnormal);
 
 	b->num_read_base += b->self_read.length;
 	b->num_correct_base += b->correct.corrected_base;
@@ -572,6 +638,9 @@ static void worker_ovec(void *data, long i, int tid)
 		push_overlaps(&(R_INF.paf[i]), &b->olist, 1, &R_INF, is_rev);
 		push_overlaps(&(R_INF.reverse_paf[i]), &b->olist, 2, &R_INF, is_rev);
 	}
+
+    if(het_cnt) het_cnt[i] = get_het_cnt(&b->hap);
+    // fprintf(stderr, "[M::%s-end] rid->%ld\n", __func__, i);
 }
 
 
@@ -595,13 +664,13 @@ static void worker_ovec_related_reads(void *data, long i, int tid)
         int fully_cov, abnormal, q_idx = k;
 
         ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 
-        0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]));
+        0.02, asm_opt.max_n_chain, 1, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]), &(b->sp));
 
         clear_Cigar_record(&b->cigar1);
         clear_Round2_alignment(&b->round2);
 
         correct_overlap(&b->olist, &R_INF, &b->self_read, &b->correct, &b->ovlp_read, &b->POA_Graph, &b->DAGCon,
-                &b->cigar1, &b->hap, &b->round2, 0, 1, &fully_cov, &abnormal);
+                &b->cigar1, &b->hap, &b->round2, &b->r_buf, &(b->tmp_region.w_list), 0, 1, &fully_cov, &abnormal);
 
         b->num_read_base += b->self_read.length;
         b->num_correct_base += b->correct.corrected_base;
@@ -621,7 +690,7 @@ static void worker_ovec_related_reads(void *data, long i, int tid)
 
         pthread_mutex_lock(&R_INF_FLAG.OutputMutex);
 
-        fprintf(R_INF_FLAG.fp, "\n>%.*s\n", (int)Get_NAME_LENGTH((R_INF), i), Get_NAME((R_INF), i));
+        fprintf(R_INF_FLAG.fp, "\n>%.*s, rid: %ld\n", (int)Get_NAME_LENGTH((R_INF), i), Get_NAME((R_INF), i), i);
         fprintf(R_INF_FLAG.fp, "%d-th round, len: %lu, hom_cov: %d, max_n_chain: %d\n", 
             asm_opt.number_of_round, Get_READ_LENGTH(R_INF, i), asm_opt.hom_cov, asm_opt.max_n_chain);
 
@@ -827,6 +896,20 @@ void rescue_hp_reads(ha_ovec_buf_t **b)
 }
 
 
+void print_het_cnt_log(uint32_t *het_cnt)
+{
+    if(!het_cnt) return;
+    char* gfa_name = (char*)malloc(strlen(asm_opt.output_file_name)+35);
+    sprintf(gfa_name, "%s.het_cnt.log", asm_opt.output_file_name);
+    FILE* output_file = fopen(gfa_name, "w");
+    fprintf(stderr, "[M::%s::] ==> print cnt of het sites to %s...\n", __func__, gfa_name);
+    free(gfa_name);
+    uint64_t i;
+    for (i = 0; i < R_INF.total_reads; i++){
+        fprintf(output_file, ">%.*s\t%u\n", (int)Get_NAME_LENGTH(R_INF, i), Get_NAME(R_INF, i), het_cnt[i]);
+    }
+    fclose(output_file);
+}
 
 
 void ha_overlap_and_correct(int round)
@@ -845,27 +928,35 @@ void ha_overlap_and_correct(int round)
 	// overlap and correct reads
 	CALLOC(b, asm_opt.thread_num);
 	for (i = 0; i < asm_opt.thread_num; ++i)
-		b[i] = ha_ovec_init(0, (round == asm_opt.number_of_round - 1));
+		b[i] = ha_ovec_init(0, (round == asm_opt.number_of_round - 1),0);
     if(ha_idx) hom_cov = asm_opt.hom_cov;
 	if(ha_idx == NULL) ha_idx = ha_pt_gen(&asm_opt, ha_flt_tab, round == 0? 0 : 1, 0, &R_INF, &hom_cov, &het_cov); // build the index
 	///debug_adapter(&asm_opt, &R_INF);
     if (round == 0 && ha_flt_tab == 0) // then asm_opt.hom_cov hasn't been updated
 		ha_opt_update_cov(&asm_opt, hom_cov);
+    het_cnt = NULL;
+    if(round == asm_opt.number_of_round-1 && asm_opt.is_dbg_het_cnt) CALLOC(het_cnt, R_INF.total_reads);
+    // fprintf(stderr, "[M::%s-start]\n", __func__);
 	if (asm_opt.required_read_name)
 		kt_for(asm_opt.thread_num, worker_ovec_related_reads, b, R_INF.total_reads);
 	else
-		kt_for(asm_opt.thread_num, worker_ovec, b, R_INF.total_reads);
+		kt_for(asm_opt.thread_num, worker_ovec, b, R_INF.total_reads);///debug_for_fix
+    // fprintf(stderr, "[M::%s-end]\n", __func__);
 
     if (r_out) write_pt_index(ha_flt_tab, ha_idx, &R_INF, &asm_opt, asm_opt.output_file_name);
 	ha_pt_destroy(ha_idx);
 	ha_idx = NULL;
 
+    if(het_cnt) {
+        print_het_cnt_log(het_cnt); free(het_cnt); het_cnt = NULL;
+    }
+
 	// collect statistics
 	for (i = 0; i < asm_opt.thread_num; ++i) {
 		asm_opt.num_bases += b[i]->num_read_base;
 		asm_opt.num_corrected_bases += b[i]->num_correct_base;
 		asm_opt.num_recorrected_bases += b[i]->num_recorrect_base;
-		asm_opt.mem_buf += ha_ovec_mem(b[i]);
+		asm_opt.mem_buf += ha_ovec_mem(b[i], NULL);
 		ha_ovec_destroy(b[i]);
 	}
 	free(b);
@@ -1287,7 +1378,7 @@ static void worker_ov_final(void *data, long i, int tid)
 
 	//get_new_candidates(i, &g_read, &overlap_list, &array_list, &l, 0.001, 0);
     ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 0.001, 
-    asm_opt.max_n_chain, 0, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL);
+    asm_opt.max_n_chain, 0, NULL/**&(b->k_flag)**/, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
 
 	overlap_region_sort_y_id(b->olist.list, b->olist.length);
 	ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
@@ -1348,12 +1439,13 @@ void debug_affine_gap_alignment(overlap_region_alloc *overlap_list, UC_Read* g_r
     kv_destroy(y_num);
 }
 
+/**
 static void worker_ov_final_high_het(void *data, long i, int tid)
 {
     ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid];
 
     ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, HIGH_HET_ERROR_RATE, 
-    asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL);
+    asm_opt.max_n_chain, 1, NULL, &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
 
     overlap_region_sort_y_id(b->olist.list, b->olist.length);
     ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
@@ -1377,6 +1469,7 @@ static void worker_ov_final_high_het(void *data, long i, int tid)
     correct_overlap_high_het(&b->olist, &R_INF, &b->self_read, &b->correct, &b->ovlp_read);
     push_final_overlaps_increment(&(R_INF.reverse_paf[i]), R_INF.reverse_paf, &b->olist, 2);
 }
+**/
 
 void Output_PAF()
 {
@@ -1424,6 +1517,31 @@ void Output_PAF()
     fprintf(stderr, "PAF has been written.\n");
 }
 
+void Output_yak_binning()
+{
+    fprintf(stderr, "Writing binning to disk ...... \n");
+    char* paf_name = (char*)malloc(strlen(asm_opt.output_file_name)+50);
+    sprintf(paf_name, "%s.hap1.bin.log", asm_opt.output_file_name);
+    FILE* oh1 = fopen(paf_name, "w");
+    sprintf(paf_name, "%s.hap2.bin.log", asm_opt.output_file_name);
+    FILE* oh2 = fopen(paf_name, "w");
+    uint64_t i;
+
+    for (i = 0; i < R_INF.total_reads; i++) {
+        if(R_INF.trio_flag[i]==FATHER) {
+            fprintf(oh1, "%.*s\n", (int)Get_NAME_LENGTH(R_INF, i), Get_NAME(R_INF, i));
+        }
+        if(R_INF.trio_flag[i]==MOTHER) {
+            fprintf(oh2, "%.*s\n", (int)Get_NAME_LENGTH(R_INF, i), Get_NAME(R_INF, i));
+        }
+    }
+
+    free(paf_name);
+    fclose(oh1); fclose(oh2);
+    fprintf(stderr, "Binning has been written.\n");
+}
+
+
 int check_cluster(uint64_t* list, long long listLen, ma_hit_t_alloc* paf, float threshold)
 {
     long long i, k;
@@ -1581,13 +1699,13 @@ void ha_overlap_final(void)
 
 	CALLOC(b, asm_opt.thread_num);
 	for (i = 0; i < asm_opt.thread_num; ++i)
-		b[i] = ha_ovec_init(asm_opt.flag & HA_F_HIGH_HET, 1);///b[i] = ha_ovec_init(1, 1);
+		b[i] = ha_ovec_init(asm_opt.flag & HA_F_HIGH_HET, 1,0);///b[i] = ha_ovec_init(1, 1);
 	ha_idx = ha_pt_gen(&asm_opt, ha_flt_tab, 1, 0, &R_INF, &hom_cov, &het_cov); // build the index
-    if(asm_opt.flag & HA_F_HIGH_HET)
-    {
-        kt_for(asm_opt.thread_num, worker_ov_final_high_het, b, R_INF.total_reads);
-    }
-    else
+    // if(asm_opt.flag & HA_F_HIGH_HET)
+    // {
+    //     kt_for(asm_opt.thread_num, worker_ov_final_high_het, b, R_INF.total_reads);
+    // }
+    // else
     {
         kt_for(asm_opt.thread_num, worker_ov_final, b, R_INF.total_reads);
     }
@@ -1601,10 +1719,63 @@ void ha_overlap_final(void)
     asm_opt.het_cov = het_cov;
 }
 
+static void worker_ov_utg(void *data, long i, int tid)
+{
+	ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid];
+    if(b->ua->a[i].len == 0) return;
 
+    ha_get_ug_candidates(b->ab, i, &(b->ua->a[i]), b->ua, &b->olist, &b->clist, 
+    0.3, asm_opt.polyploidy*5, 0, &(b->k_flag), &b->r_buf, ha_flt_tab, ha_idx,
+    &(b->tmp_region), NULL, /**0.3**/0);
+    
+	overlap_region_sort_y_id(b->olist.list, b->olist.length);
+	ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
+	ma_hit_sort_tn(R_INF.reverse_paf[i].buffer, R_INF.reverse_paf[i].length);
+
+	update_overlaps(&b->olist, &(R_INF.paf[i]), &b->self_read, &b->ovlp_read, 1, 1);
+	update_overlaps(&b->olist, &(R_INF.reverse_paf[i]), &b->self_read, &b->ovlp_read, 2, 0);
+	///recover missing exact overlaps
+	update_exact_overlaps(&b->olist, &b->self_read, &b->ovlp_read);
+
+	///Final_phasing(&overlap_list, &cigarline, &g_read, &overlap_read, c2n);
+	push_final_overlaps(&(R_INF.paf[i]), R_INF.reverse_paf, &b->olist, 1);
+	push_final_overlaps(&(R_INF.reverse_paf[i]), R_INF.reverse_paf, &b->olist, 2);
+}
+
+
+void ug_idx_build(ma_ug_t *ug, int hap_n)
+{
+    int flag = asm_opt.flag&HA_F_NO_HPC, i;
+    asm_opt.flag -= flag;
+    // ha_flt_tab = ha_ft_ug_gen(&asm_opt, &(ug->u), hap_n, hap_n);
+    // ha_idx = ha_pt_ug_gen(&asm_opt, ha_flt_tab, &(ug->u), hap_n);
+
+    ha_ovec_buf_t **b = NULL;
+    // overlap and correct reads
+    CALLOC(b, asm_opt.thread_num);
+    for (i = 0; i < asm_opt.thread_num; ++i)
+    {
+        b[i] = ha_ovec_init(1, 1, 0);
+        b[i]->ua = &(ug->u);
+    }
+            
+    kt_for(asm_opt.thread_num, worker_ov_utg, b, R_INF.total_reads);
+
+    for (i = 0; i < asm_opt.thread_num; ++i)
+        ha_ovec_destroy(b[i]);
+    free(b);
+
+    ha_ft_destroy(ha_flt_tab); 
+    ha_pt_destroy(ha_idx);
+    asm_opt.flag += flag;
+    exit(1);
+}
 
 int ha_assemble(void)
 {
+    // debug_mc_g_t(MC_NAME);
+    // debug_mc_gg_t(MC_NAME, 0, 0);
+    // quick_debug_phasing(MC_NAME);
 	extern void ha_extract_print_list(const All_reads *rs, int n_rounds, const char *o);
 	int r, hom_cov = -1, ovlp_loaded = 0;
 	if (asm_opt.load_index_from_disk && load_all_data_from_disk(&R_INF.paf, &R_INF.reverse_paf, asm_opt.output_file_name)) {
@@ -1614,11 +1785,13 @@ int ha_assemble(void)
 			ha_extract_print_list(&R_INF, asm_opt.extract_iter, asm_opt.extract_list);
 			exit(0);
 		}
-		if (!(asm_opt.flag & HA_F_SKIP_TRIOBIN) && !(asm_opt.flag & HA_F_VERBOSE_GFA)) ha_triobin(&asm_opt);
-        ///if (!(asm_opt.flag & HA_F_SKIP_TRIOBIN)) ha_triobin(&asm_opt), ovlp_loaded = 2;
+		// if (!(asm_opt.flag & HA_F_SKIP_TRIOBIN) && !(asm_opt.flag & HA_F_VERBOSE_GFA)) ha_triobin(&asm_opt);
+        if (!(asm_opt.flag & HA_F_SKIP_TRIOBIN)) ha_triobin(&asm_opt);
+        // if (!(asm_opt.flag & HA_F_SKIP_TRIOBIN)) ha_triobin(&asm_opt), ovlp_loaded = 2;
 		if (asm_opt.flag & HA_F_WRITE_EC) Output_corrected_reads();
 		if (asm_opt.flag & HA_F_WRITE_PAF) Output_PAF();
         if (asm_opt.het_cov == -1024) hap_recalculate_peaks(asm_opt.output_file_name), ovlp_loaded = 2;
+        if (asm_opt.fn_bin_yak[0] && asm_opt.fn_bin_yak[1]) Output_yak_binning();
 	}
 	if (!ovlp_loaded) {
         ha_flt_tab = ha_idx = NULL;

Assembly.h

@@ -1,6 +1,10 @@
 #ifndef __ASSEMBLY__
 #define __ASSEMBLY__
 #include "CommandLines.h"
+#include "Overlaps.h"
+#include "Process_Read.h"
+#include "Hash_Table.h"
+#include "Correct.h"
 
 #define FORWARD 0
 #define REVERSE_COMPLEMENT (0x8000000000000000)
@@ -13,6 +17,38 @@
 #define RESEED_LEN 2000
 #define RESEED_HP_RATE 0.9
 
+typedef struct {
+	int is_final, save_ov;
+	// chaining and overlapping related buffers
+	UC_Read self_read, ovlp_read;
+	Candidates_list clist;
+	overlap_region_alloc olist;
+    overlap_region_alloc olist_hp;
+	ha_abuf_t *ab;
+    ha_abufl_t *abl;
+	// error correction related buffers
+	int64_t num_read_base, num_correct_base, num_recorrect_base;
+	Cigar_record cigar1;
+	Graph POA_Graph;
+	Graph DAGCon;
+	Correct_dumy correct;
+	haplotype_evdience_alloc hap;
+	Round2_alignment round2;
+    kvec_t_u32_warp b_buf;
+    kvec_t_u64_warp r_buf;
+    kvec_t_u8_warp k_flag;
+    overlap_region tmp_region;
+    ma_utg_v *ua;
+    st_mt_t sp;
+	bit_extz_t exz;
+} ha_ovec_buf_t;
+
 int ha_assemble(void);
+void ug_idx_build(ma_ug_t *ug, int hap_n);
+ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov, int is_ug);
+ha_ovec_buf_t *ha_ovec_buf_init(void *km, int is_final, int save_ov, int is_ug);
+void ha_ovec_destroy(ha_ovec_buf_t *b);
+int64_t ha_ovec_mem(const ha_ovec_buf_t *b, int64_t *mem_a)
+;
 
 #endif

CommandLines.cpp

@@ -21,15 +21,34 @@ static ko_longopt_t long_options[] = {
 	{ "max-od-final",  ko_no_argument, 306 },
 	{ "ex-list",       ko_required_argument, 307 },
 	{ "ex-iter",       ko_required_argument, 308 },
-    { "purge-cov",     ko_required_argument, 309 },
+    { "hom-cov",     ko_required_argument, 309 },
     { "pri-range",     ko_required_argument, 310 },
-    { "high-het",      ko_no_argument, 311 },
     { "lowQ",          ko_required_argument, 312 },
 	{ "min-hist-cnt",  ko_required_argument, 313 },
     { "h1",      ko_required_argument, 314 },
     { "h2",      ko_required_argument, 315 },
     { "enzyme",      ko_required_argument, 316 },
     { "b-cov",      ko_required_argument, 317 },
+    { "h-cov",      ko_required_argument, 318 },
+    { "m-rate",      ko_required_argument, 319 },
+    { "primary",      ko_no_argument, 320 },
+    { "t-occ",      ko_required_argument, 321 },
+    { "seed",      ko_required_argument, 322 },
+    { "n-perturb",      ko_required_argument, 323 },
+    { "f-perturb",      ko_required_argument, 324 },
+    { "n-hap",      ko_required_argument, 325 },
+    { "n-weight",      ko_required_argument, 326 },
+    { "l-msjoin",      ko_required_argument, 327 },
+    { "purge-max",     ko_required_argument, 328 },
+    { "fast",     ko_no_argument, 329 },
+    { "dp-er",     ko_required_argument, 330},
+    { "max-kocc",     ko_required_argument, 331},
+    { "hg-size",     ko_required_argument, 332},
+    { "ul",     ko_required_argument, 333},
+    { "unskew",     ko_no_argument, 334},
+    { "kpt-rate",     ko_required_argument, 335},
+    { "ul-rate",     ko_required_argument, 336},
+    { "dbg-het-cnt",     ko_no_argument, 337},
 	{ 0, 0, 0 }
 };
 
@@ -45,64 +64,100 @@ void Print_H(hifiasm_opt_t* asm_opt)
     fprintf(stderr, "Usage: hifiasm [options] <in_1.fq> <in_2.fq> <...>\n");
     fprintf(stderr, "Options:\n");
 	fprintf(stderr, "  Input/Output:\n");
-    fprintf(stderr, "    -o STR      prefix of output files [%s]\n", asm_opt->output_file_name);
-    fprintf(stderr, "    -i          ignore saved read correction and overlaps\n");
-    fprintf(stderr, "    -t INT      number of threads [%d]\n", asm_opt->thread_num);
-    fprintf(stderr, "    -z INT      length of adapters that should be removed [%d]\n", asm_opt->adapterLen);
-    fprintf(stderr, "    --version   show version number\n");
+    fprintf(stderr, "    -o STR       prefix of output files [%s]\n", asm_opt->output_file_name);
+    fprintf(stderr, "    -t INT       number of threads [%d]\n", asm_opt->thread_num);
+    fprintf(stderr, "    -h           show help information\n");
+    fprintf(stderr, "    --version    show version number\n");
 	fprintf(stderr, "  Overlap/Error correction:\n");
-    fprintf(stderr, "    -k INT      k-mer length (must be <64) [%d]\n", asm_opt->k_mer_length);
-	fprintf(stderr, "    -w INT      minimizer window size [%d]\n", asm_opt->mz_win);
-	fprintf(stderr, "    -f INT      number of bits for bloom filter; 0 to disable [%d]\n", asm_opt->bf_shift);
-	fprintf(stderr, "    -D FLOAT    drop k-mers occurring >FLOAT*coverage times [%.1f]\n", asm_opt->high_factor);
-	fprintf(stderr, "    -N INT      consider up to max(-D*coverage,-N) overlaps for each oriented read [%d]\n", asm_opt->max_n_chain);
-    fprintf(stderr, "    -r INT      round of correction [%d]\n", asm_opt->number_of_round);
+    fprintf(stderr, "    -k INT       k-mer length (must be <64) [%d]\n", asm_opt->k_mer_length);
+	fprintf(stderr, "    -w INT       minimizer window size [%d]\n", asm_opt->mz_win);
+	fprintf(stderr, "    -f INT       number of bits for bloom filter; 0 to disable [%d]\n", asm_opt->bf_shift);
+	fprintf(stderr, "    -D FLOAT     drop k-mers occurring >FLOAT*coverage times [%.1f]\n", asm_opt->high_factor);
+	fprintf(stderr, "    -N INT       consider up to max(-D*coverage,-N) overlaps for each oriented read [%d]\n", asm_opt->max_n_chain);
+    fprintf(stderr, "    -r INT       round of correction [%d]\n", asm_opt->number_of_round);
+    fprintf(stderr, "    -z INT       length of adapters that should be removed [%d]\n", asm_opt->adapterLen);
+    fprintf(stderr, "    --max-kocc   INT\n");
+    fprintf(stderr, "                 employ k-mers occurring <INT times to rescue repetitive overlaps [%d]\n", asm_opt->max_kmer_cnt);
+    fprintf(stderr, "    --hg-size    INT(k, m or g)\n");
+    fprintf(stderr, "                 estimated haploid genome size used for inferring read coverage [auto]\n");
     fprintf(stderr, "  Assembly:\n");
-    fprintf(stderr, "    -a INT      round of assembly cleaning [%d]\n", asm_opt->clean_round);
-    fprintf(stderr, "    -m INT      pop bubbles of <INT in size in contig graphs [%lld]\n", asm_opt->large_pop_bubble_size);
-    fprintf(stderr, "    -p INT      pop bubbles of <INT in size in unitig graphs [%lld]\n", asm_opt->small_pop_bubble_size);
-    fprintf(stderr, "    -n INT      remove tip unitigs composed of <=INT reads [%d]\n", asm_opt->max_short_tip);
-    fprintf(stderr, "    -x FLOAT    max overlap drop ratio [%.2g]\n", asm_opt->max_drop_rate);
-    fprintf(stderr, "    -y FLOAT    min overlap drop ratio [%.2g]\n", asm_opt->min_drop_rate);
-    fprintf(stderr, "    -u          disable post join contigs step which may improve N50\n");
-    fprintf(stderr, "    --lowQ      INT\n");
-    fprintf(stderr, "                output contig regions with >=INT%% inconsistency in BED format; 0 to disable [%d]\n", asm_opt->bed_inconsist_rate);
-    fprintf(stderr, "    --b-cov     INT\n");
-    fprintf(stderr, "                break contigs at breakpoints with coverage drop at <INT-fold coverage [%d]\n", asm_opt->break_cov);
+    fprintf(stderr, "    -a INT       round of assembly cleaning [%d]\n", asm_opt->clean_round);
+    fprintf(stderr, "    -m INT       pop bubbles of <INT in size in contig graphs [%lld]\n", asm_opt->large_pop_bubble_size);
+    fprintf(stderr, "    -p INT       pop bubbles of <INT in size in unitig graphs [%lld]\n", asm_opt->small_pop_bubble_size);
+    fprintf(stderr, "    -n INT       remove tip unitigs composed of <=INT reads [%d]\n", asm_opt->max_short_tip);
+    fprintf(stderr, "    -x FLOAT     max overlap drop ratio [%.2g]\n", asm_opt->max_drop_rate);
+    fprintf(stderr, "    -y FLOAT     min overlap drop ratio [%.2g]\n", asm_opt->min_drop_rate);
+    fprintf(stderr, "    -i           ignore saved read correction and overlaps\n");
+    fprintf(stderr, "    -u           disable post-join step for contigs which may improve N50\n");
+    fprintf(stderr, "    --hom-cov    INT\n");
+    fprintf(stderr, "                 homozygous read coverage [auto]\n");
+    fprintf(stderr, "    --lowQ       INT\n");
+    fprintf(stderr, "                 output contig regions with >=INT%% inconsistency in BED format; 0 to disable [%d]\n", asm_opt->bed_inconsist_rate);
+    fprintf(stderr, "    --b-cov      INT\n");
+    fprintf(stderr, "                 break contigs at positions with <INT-fold coverage; work with '--m-rate'; 0 to disable [%d]\n", asm_opt->b_low_cov);
+    fprintf(stderr, "    --h-cov      INT\n");
+    fprintf(stderr, "                 break contigs at positions with >INT-fold coverage; work with '--m-rate'; -1 to disable [%d]\n", asm_opt->b_high_cov);
+    fprintf(stderr, "    --m-rate     FLOAT\n");
+    fprintf(stderr, "                 break contigs at positions with <=FLOAT*coverage exact overlaps;\n");
+    fprintf(stderr, "                 only work with '--b-cov' or '--h-cov'[%.2f]\n", asm_opt->m_rate);
+    fprintf(stderr, "    --primary    output a primary assembly and an alternate assembly\n");
     
 //	fprintf(stderr, "    --pri-range INT1[,INT2]\n");
 //	fprintf(stderr, "                keep contigs with coverage in this range in p_ctg.gfa; -1 to disable [auto,inf]\n");
 
     fprintf(stderr, "  Trio-partition:\n");
-    fprintf(stderr, "    -1 FILE     hap1/paternal k-mer dump generated by \"yak count\" []\n");
-    fprintf(stderr, "    -2 FILE     hap2/maternal k-mer dump generated by \"yak count\" []\n");
-    fprintf(stderr, "    -c INT      lower bound of the binned k-mer's frequency [%d]\n", asm_opt->min_cnt);
-    fprintf(stderr, "    -d INT      upper bound of the binned k-mer's frequency [%d]\n", asm_opt->mid_cnt);
-	fprintf(stderr, "    -3 FILE     list of hap1/paternal read names []\n");
-	fprintf(stderr, "    -4 FILE     list of hap2/maternal read names []\n");
+    fprintf(stderr, "    -1 FILE      hap1/paternal k-mer dump generated by \"yak count\" []\n");
+    fprintf(stderr, "    -2 FILE      hap2/maternal k-mer dump generated by \"yak count\" []\n");
+    fprintf(stderr, "    -3 FILE      list of hap1/paternal read names []\n");
+	fprintf(stderr, "    -4 FILE      list of hap2/maternal read names []\n");
+    fprintf(stderr, "    -c INT       lower bound of the binned k-mer's frequency [%d]\n", asm_opt->min_cnt);
+    fprintf(stderr, "    -d INT       upper bound of the binned k-mer's frequency [%d]\n", asm_opt->mid_cnt);
+    fprintf(stderr, "    --t-occ      INT\n");
+    fprintf(stderr, "                 forcedly remove unitigs with >INT unexpected haplotype-specific reads;\n");
+    fprintf(stderr, "                 ignore graph topology; [%d]\n", asm_opt->trio_flag_occ_thres);
 
     fprintf(stderr, "  Purge-dups:\n");
-    fprintf(stderr, "    -l INT      purge level. 0: no purging; 1: light; 2: aggressive [0 for trio; 2 for unzip]\n");
-    fprintf(stderr, "    -s FLOAT    similarity threshold for duplicate haplotigs [%g]\n", 
-                                     asm_opt->purge_simi_rate);
-    fprintf(stderr, "    -O INT      min number of overlapped reads for duplicate haplotigs [%d]\n", 
-                                     asm_opt->purge_overlap_len);
-    fprintf(stderr, "    --purge-cov INT\n");
-    fprintf(stderr, "                coverage upper bound of Purge-dups [auto]\n");
-    fprintf(stderr, "    --high-het  enable this mode for high heterozygosity sample [experimental, not stable]\n");
+    fprintf(stderr, "    -l INT       purge level. 0: no purging; 1: light; 2/3: aggressive [0 for trio; 3 for unzip]\n");
+    fprintf(stderr, "    -s FLOAT     similarity threshold for duplicate haplotigs [%g for -l1/-l2, %g for -l3]\n", 
+                                      asm_opt->purge_simi_rate_l2, asm_opt->purge_simi_rate_l3);
+    fprintf(stderr, "    -O INT       min number of overlapped reads for duplicate haplotigs [%d]\n", 
+                                      asm_opt->purge_overlap_len);
+    fprintf(stderr, "    --purge-max  INT\n");
+    fprintf(stderr, "                 coverage upper bound of Purge-dups [auto]\n");
+    fprintf(stderr, "    --n-hap      INT\n");
+    fprintf(stderr, "                 number of haplotypes [%d]\n", asm_opt->polyploidy);
     
-    fprintf(stderr, "  Hi-C-partition [experimental, not stable]:\n");
+    // fprintf(stderr, "  Hi-C-partition [experimental, not stable]:\n");
+    fprintf(stderr, "  Hi-C-partition:\n");
     fprintf(stderr, "    --h1 FILEs   file names of Hi-C R1  [r1_1.fq,r1_2.fq,...]\n");
     fprintf(stderr, "    --h2 FILEs   file names of Hi-C R2  [r2_1.fq,r2_2.fq,...]\n");
+    fprintf(stderr, "    --seed INT   RNG seed [%lu]\n", asm_opt->seed);
+
+    
+    fprintf(stderr, "    --n-weight   INT\n");
+    fprintf(stderr, "                 rounds of reweighting Hi-C links [%d]\n", asm_opt->n_weight);
+	fprintf(stderr, "    --n-perturb  INT\n");
+    fprintf(stderr, "                 rounds of perturbation [%d]\n", asm_opt->n_perturb);
+    fprintf(stderr, "    --f-perturb  FLOAT\n");
+	fprintf(stderr, "                 fraction to flip for perturbation [%.3g]\n", asm_opt->f_perturb);
+    fprintf(stderr, "    --l-msjoin   INT\n");
+    fprintf(stderr, "                 detect misjoined unitigs of >=INT in size; 0 to disable [%lu]\n", asm_opt->misjoin_len);
+
+    fprintf(stderr, "  Ultra-Long-integration (beta):\n");
+    fprintf(stderr, "    --ul FILEs   file names of Ultra-Long reads [r1.fq,r2.fq,...]\n");
+    fprintf(stderr, "    --ul-rate    FLOAT\n");
+    fprintf(stderr, "                 similarity threshold for UL-to-HiFi alignment [%.3g]\n", asm_opt->ul_error_rate);
+    
 
     fprintf(stderr, "Example: ./hifiasm -o NA12878.asm -t 32 NA12878.fq.gz\n");
-    fprintf(stderr, "See `man ./hifiasm.1' for detailed description of these command-line options.\n");
+    fprintf(stderr, "See `https://hifiasm.readthedocs.io/en/latest/' or `man ./hifiasm.1' for complete documentation.\n");
 }
 
 void init_opt(hifiasm_opt_t* asm_opt)
 {
 	memset(asm_opt, 0, sizeof(hifiasm_opt_t));
-	asm_opt->flag = 0;
+	///asm_opt->flag = 0;
+    asm_opt->flag = HA_F_PARTITION;
     asm_opt->coverage = -1;
     asm_opt->num_reads = 0;
     asm_opt->read_file_names = NULL;
@@ -111,11 +166,19 @@ void init_opt(hifiasm_opt_t* asm_opt)
     asm_opt->hic_enzymes = NULL;
     asm_opt->hic_reads[0] = NULL;
     asm_opt->hic_reads[1] = NULL;
+    asm_opt->fn_bin_poy = NULL;
+    asm_opt->ar = NULL;
     asm_opt->thread_num = 1;
     asm_opt->k_mer_length = 51;
     asm_opt->hic_mer_length = 31;
+    asm_opt->ul_mer_length = 19;
 	asm_opt->mz_win = 51;
+    asm_opt->ul_mz_win = 19;
+    asm_opt->mz_rewin = 1000;
+    asm_opt->ul_mz_rewin = 360;
+	asm_opt->mz_sample_dist = 500;
 	asm_opt->bf_shift = 37;
+	asm_opt->max_kmer_cnt = 2000;
 	asm_opt->high_factor = 5.0;
 	asm_opt->max_ov_diff_ec = 0.04;
 	asm_opt->max_ov_diff_final = 0.03;
@@ -128,7 +191,8 @@ void init_opt(hifiasm_opt_t* asm_opt)
     asm_opt->number_of_round = 3;
     asm_opt->adapterLen = 0;
     asm_opt->clean_round = 4;
-    asm_opt->small_pop_bubble_size = 100000;
+    ///asm_opt->small_pop_bubble_size = 100000;
+    asm_opt->small_pop_bubble_size = 0;
     asm_opt->large_pop_bubble_size = 10000000;
     asm_opt->min_drop_rate = 0.2;
     asm_opt->max_drop_rate = 0.8;
@@ -140,20 +204,45 @@ void init_opt(hifiasm_opt_t* asm_opt)
     asm_opt->max_short_tip = 3;
     asm_opt->min_cnt = 2;
     asm_opt->mid_cnt = 5;
-    asm_opt->purge_level_primary = 2;
+    asm_opt->purge_level_primary = 3;
     asm_opt->purge_level_trio = 0;
-    asm_opt->purge_simi_rate = 0.75;
-    asm_opt->purge_simi_rate_hic = 0.85;
+    asm_opt->purge_simi_rate_l2 = 0.75;
+    asm_opt->purge_simi_rate_l3 = 0.55;
     asm_opt->purge_overlap_len = 1;
-    asm_opt->purge_overlap_len_hic = 50;
+    ///asm_opt->purge_overlap_len_hic = 50;
     asm_opt->recover_atg_cov_min = -1024;
     asm_opt->recover_atg_cov_max = INT_MAX;
     asm_opt->hom_global_coverage = -1;
+    asm_opt->hom_global_coverage_set = 0;
+    asm_opt->pur_global_coverage = -1;
     asm_opt->bed_inconsist_rate = 70;
     asm_opt->hic_inconsist_rate = 30;
     ///asm_opt->bub_mer_length = 3;
     asm_opt->bub_mer_length = 1000000;
-    asm_opt->break_cov = 0;
+    asm_opt->b_low_cov = 0;
+    asm_opt->b_high_cov = -1;
+    asm_opt->m_rate = 0.75;
+    asm_opt->hap_occ = 1;
+    asm_opt->polyploidy = 2;
+    asm_opt->trio_flag_occ_thres = 60;
+    asm_opt->seed = 11;
+    asm_opt->n_perturb = 10000;
+    asm_opt->f_perturb = 0.1;
+    asm_opt->n_weight = 3;
+    asm_opt->is_alt = 0;
+    asm_opt->misjoin_len = 500000;
+    asm_opt->scffold = 0;
+    asm_opt->dp_min_len = 2000;
+    asm_opt->dp_e = 0.0025;
+    asm_opt->hg_size = -1;
+    asm_opt->kpt_rate = -1;
+    asm_opt->infor_cov = 3;
+    asm_opt->s_hap_cov = 3;
+    asm_opt->ul_error_rate = 0.2/**0.15**/;
+    asm_opt->ul_error_rate_low = 0.1;
+    asm_opt->ul_error_rate_hpc = 0.2;
+    asm_opt->ul_ec_round = 3;
+    asm_opt->is_dbg_het_cnt = 0;
 }
 
 void destory_enzyme(enzyme* f)
@@ -177,6 +266,7 @@ void destory_opt(hifiasm_opt_t* asm_opt)
     if(asm_opt->hic_enzymes != NULL) destory_enzyme(asm_opt->hic_enzymes);
     if(asm_opt->hic_reads[0] != NULL) destory_enzyme(asm_opt->hic_reads[0]);
     if(asm_opt->hic_reads[1] != NULL) destory_enzyme(asm_opt->hic_reads[1]);
+    if(asm_opt->ar != NULL) destory_enzyme(asm_opt->ar);
 }
 
 void ha_opt_reset_to_round(hifiasm_opt_t* asm_opt, int round)
@@ -350,9 +440,9 @@ int check_option(hifiasm_opt_t* asm_opt)
         return 0;
     }
 
-    if(asm_opt->purge_level_primary < 0 || asm_opt->purge_level_primary > 2)
+    if(asm_opt->purge_level_primary < 0 || asm_opt->purge_level_primary > 3)
     {
-        fprintf(stderr, "[ERROR] the level of purge-dup should be [0, 2] (-l)\n");
+        fprintf(stderr, "[ERROR] the level of purge-dup should be [0, 3] (-l)\n");
         return 0;
     }
 
@@ -364,7 +454,13 @@ int check_option(hifiasm_opt_t* asm_opt)
 
     if(asm_opt->hom_global_coverage < 0 && asm_opt->hom_global_coverage != -1)
     {
-        fprintf(stderr, "[ERROR] purge duplication coverage threshold should be >= 0 (--purge-cov)\n");
+        fprintf(stderr, "[ERROR] homozygous read coverage should be >= 0 (--hom-cov)\n");
+        return 0;
+    }
+
+    if(asm_opt->pur_global_coverage < 0 && asm_opt->pur_global_coverage != -1)
+    {
+        fprintf(stderr, "[ERROR] purge duplication coverage threshold should be >= 0 (--purge-max)\n");
         return 0;
     }
 
@@ -419,25 +515,54 @@ int check_option(hifiasm_opt_t* asm_opt)
         return 0;
     }
 
-    // fprintf(stderr, "input file num: %d\n", asm_opt->num_reads);
-    // fprintf(stderr, "output file: %s\n", asm_opt->output_file_name);
-    // fprintf(stderr, "number of threads: %d\n", asm_opt->thread_num);
-    // fprintf(stderr, "number of rounds for correction: %d\n", asm_opt->number_of_round);
-    // fprintf(stderr, "number of rounds for assembly cleaning: %d\n", asm_opt->clean_round);
-    // fprintf(stderr, "length of removed adapters: %d\n", asm_opt->adapterLen);
-    // fprintf(stderr, "length of k_mer: %d\n", asm_opt->k_mer_length);
-    // fprintf(stderr, "min overlap drop ratio: %.2g\n", asm_opt->min_drop_rate);
-    // fprintf(stderr, "max overlap drop ratio: %.2g\n", asm_opt->max_drop_rate);
-    // fprintf(stderr, "size of popped small bubbles: %lld\n", asm_opt->small_pop_bubble_size);
-    // fprintf(stderr, "size of popped large bubbles: %lld\n", asm_opt->large_pop_bubble_size);
-    // fprintf(stderr, "small removed unitig threshold: %d\n", asm_opt->max_short_tip);
-    // fprintf(stderr, "small removed unitig threshold: %d\n", asm_opt->max_short_tip);
-    // fprintf(stderr, "min_cnt: %d\n", asm_opt->min_cnt);
-    // fprintf(stderr, "mid_cnt: %d\n", asm_opt->mid_cnt);
-    // fprintf(stderr, "purge_level_primary: %d\n", asm_opt->purge_level_primary);
-    // fprintf(stderr, "purge_level_trio: %d\n", asm_opt->purge_level_trio);
-    // fprintf(stderr, "purge_simi_rate: %f\n", asm_opt->purge_simi_rate);
-    // fprintf(stderr, "purge_overlap_len: %d\n", asm_opt->purge_overlap_len);
+    if(asm_opt->ar != NULL && check_hic_reads(asm_opt->ar, "UL") == 0) return 0;
+    if(asm_opt->ar != NULL && asm_opt->ar->n == 0)
+    {
+        fprintf(stderr, "[ERROR] wrong UL reads (--ul)\n");
+        return 0;
+    }
+
+    if(asm_opt->b_low_cov < 0)
+    {
+        fprintf(stderr, "[ERROR] must >= 0 (--b-cov)\n");
+        return 0;
+    }
+
+    if(asm_opt->b_high_cov != -1 && asm_opt->b_high_cov < 0)
+    {
+        fprintf(stderr, "[ERROR] must >= 0 (--h-cov)\n");
+        return 0;
+    }
+
+    if(asm_opt->m_rate < 0)
+    {
+        fprintf(stderr, "[ERROR] must >= 0 (--m-rate)\n");
+        return 0;
+    }
+
+    if(asm_opt->b_high_cov != -1 && asm_opt->b_high_cov <= asm_opt->b_low_cov)
+    {
+        fprintf(stderr, "[ERROR] [--h-cov] must >= [--b-cov]\n");
+        return 0;
+    }
+
+    if(asm_opt->purge_simi_thres < 0)
+    {
+        fprintf(stderr, "[ERROR] [-s] must >= 0\n");
+        return 0;
+    }
+
+    if(asm_opt->max_kmer_cnt < 0)
+    {
+        fprintf(stderr, "[ERROR] [--max-kocc] must >= 0\n");
+        return 0;
+    }
+
+    if(asm_opt->hg_size < -1)
+    {
+        fprintf(stderr, "[ERROR] [--hg-size] wrong genome size\n");
+        return 0;
+    }
 
     return 1;
 }
@@ -524,13 +649,30 @@ void get_hic_enzymes(char *argv, enzyme** x, int check_name)
     (*x)->a[k][(*x)->l[k]] = '\0';
 }
 
+int64_t inter_gsize(char *argv)
+{
+    int64_t len = strlen(argv);
+    double s;
+    if(len <= 1) return -2;
+    char t = argv[len-1];
+    if(t != 'k' && t != 'K' && t != 'm' && t != 'M' && t != 'g' && t != 'G') return -2;
+    char *ss=(char*)malloc(len);
+    memcpy(ss, argv, len-1); ss[len-1] = '\0';
+    s = atof(ss);
+    free(ss);
+    if(t == 'k' || t == 'K') return s*1000;
+    if(t == 'm' || t == 'M') return s*1000000;
+    if(t == 'g' || t == 'G') return s*1000000000;
+    return s;
+}
+
 int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 {
     ketopt_t opt = KETOPT_INIT;
 
     int c;
 
-    while ((c = ketopt(&opt, argc, argv, 1, "hvt:o:k:w:m:n:r:a:b:z:x:y:p:c:d:M:P:if:D:FN:1:2:3:4:l:s:O:eu", long_options)) >= 0) {
+    while ((c = ketopt(&opt, argc, argv, 1, "hvt:o:k:w:m:n:r:a:b:z:x:y:p:c:d:M:P:if:D:FN:1:2:3:4:5:l:s:O:eu", long_options)) >= 0) {
         if (c == 'h')
         {
             Print_H(asm_opt);
@@ -560,6 +702,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
         else if (c == '2' || c == 'M') asm_opt->fn_bin_yak[1] = opt.arg;
         else if (c == '3') asm_opt->fn_bin_list[0] = opt.arg;
         else if (c == '4') asm_opt->fn_bin_list[1] = opt.arg;
+        else if (c == '5') asm_opt->fn_bin_poy = opt.arg;
         else if (c == 'x') asm_opt->max_drop_rate = atof(opt.arg);
         else if (c == 'y') asm_opt->min_drop_rate = atof(opt.arg);
         else if (c == 'p') asm_opt->small_pop_bubble_size = atoll(opt.arg);
@@ -575,7 +718,11 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 		else if (c == 306) asm_opt->max_ov_diff_final = atof(opt.arg);
 		else if (c == 307) asm_opt->extract_list = opt.arg;
 		else if (c == 308) asm_opt->extract_iter = atoi(opt.arg);
-        else if (c == 309) asm_opt->hom_global_coverage = atoi(opt.arg);
+        else if (c == 309)
+        {
+            asm_opt->hom_global_coverage = atoi(opt.arg);
+            asm_opt->hom_global_coverage_set = 1;
+        } 
         else if (c == 310)
         {
             char* s = NULL;
@@ -586,18 +733,38 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
                 asm_opt->recover_atg_cov_min = asm_opt->recover_atg_cov_max = -1;
             }
         }
-        else if (c == 311) asm_opt->flag |= HA_F_HIGH_HET;
+        ///else if (c == 311) asm_opt->flag |= HA_F_HIGH_HET;
         else if (c == 312) asm_opt->bed_inconsist_rate = atoi(opt.arg);
         else if (c == 313) asm_opt->min_hist_kmer_cnt = atoi(opt.arg);
         else if (c == 314) get_hic_enzymes(opt.arg, &(asm_opt->hic_reads[0]), 0);
         else if (c == 315) get_hic_enzymes(opt.arg, &(asm_opt->hic_reads[1]), 0);
         else if (c == 316) get_hic_enzymes(opt.arg, &(asm_opt->hic_enzymes), 1);
-        else if (c == 317) asm_opt->break_cov = atoi(opt.arg);
+        else if (c == 317) asm_opt->b_low_cov = atoi(opt.arg);
+        else if (c == 318) asm_opt->b_high_cov = atoi(opt.arg);
+        else if (c == 319) asm_opt->m_rate = atof(opt.arg);
+        else if (c == 320) asm_opt->flag -= HA_F_PARTITION, asm_opt->is_alt = 1;
+        else if (c == 321) asm_opt->trio_flag_occ_thres = atoi(opt.arg);
+        else if (c == 322) asm_opt->seed = atol(opt.arg);
+        else if (c == 323) asm_opt->n_perturb = atoi(opt.arg);
+        else if (c == 324) asm_opt->f_perturb = atof(opt.arg);
+        else if (c == 325) asm_opt->polyploidy = atoi(opt.arg);
+        else if (c == 326) asm_opt->n_weight = atoi(opt.arg);
+        else if (c == 327) asm_opt->misjoin_len = atol(opt.arg);
+        else if (c == 328) asm_opt->pur_global_coverage = atoi(opt.arg);
+        else if (c == 329) asm_opt->flag |= HA_F_FAST;
+        else if (c == 330) asm_opt->dp_e = atof(opt.arg);
+        else if (c == 331) asm_opt->max_kmer_cnt = atol(opt.arg);
+        else if (c == 332) asm_opt->hg_size = inter_gsize(opt.arg);      
+        else if (c == 333) get_hic_enzymes(opt.arg, &(asm_opt->ar), 0);
+        else if (c == 334) asm_opt->flag |= HA_F_USKEW;
+        else if (c == 335) asm_opt->kpt_rate = atof(opt.arg);
+        else if (c == 336) asm_opt->ul_error_rate = atof(opt.arg);
+        else if (c == 337) asm_opt->is_dbg_het_cnt = 1;
         else if (c == 'l')
         {   ///0: disable purge_dup; 1: purge containment; 2: purge overlap
             asm_opt->purge_level_primary = asm_opt->purge_level_trio = atoi(opt.arg);
         }
-        else if (c == 's') asm_opt->purge_simi_rate = atof(opt.arg);
+        else if (c == 's') asm_opt->purge_simi_rate_l2 = asm_opt->purge_simi_rate_l3 = atof(opt.arg);
         else if (c == 'O') asm_opt->purge_overlap_len = atoll(opt.arg);
         else if (c == ':') 
         {
@@ -611,6 +778,9 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 		}
     }
 
+    if(asm_opt->purge_level_primary > 2) asm_opt->purge_simi_thres = asm_opt->purge_simi_rate_l3;
+    else asm_opt->purge_simi_thres = asm_opt->purge_simi_rate_l2;
+
     
     if (argc == opt.ind)
     {
@@ -620,7 +790,5 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 
     get_queries(argc, argv, &opt, asm_opt);
 
-
-
     return check_option(asm_opt);
 }

CommandLines.h

@@ -2,8 +2,9 @@
 #define __COMMAND_LINE_PARSER__
 
 #include <pthread.h>
+#include <stdint.h>
 
-#define HA_VERSION "0.14-r309"
+#define HA_VERSION "0.17.5-r458"
 
 #define VERBOSE 0
 
@@ -18,6 +19,9 @@
 #define HA_F_BAN_POST_JOIN   0x100
 #define HA_F_BAN_ASSEMBLY    0x200
 #define HA_F_HIGH_HET        0x400
+#define HA_F_PARTITION       0x800
+#define HA_F_FAST            0x1000
+#define HA_F_USKEW            0x2000
 
 #define HA_MIN_OV_DIFF       0.02 // min sequence divergence in an overlap
 
@@ -34,22 +38,32 @@ typedef struct {
     char* required_read_name;
 	char *fn_bin_yak[2];
 	char *fn_bin_list[2];
+    char *fn_bin_poy;
 	char *extract_list;
     enzyme *hic_reads[2];
     enzyme *hic_enzymes;
+    enzyme *ar;
 	int extract_iter;
     int thread_num;
     int k_mer_length;
     int hic_mer_length;
+    int ul_mer_length;
     int bub_mer_length;
 	int mz_win;
+    int ul_mz_win;
+    int mz_rewin;
+    int ul_mz_rewin;
+	int mz_sample_dist;
 	int bf_shift;
+	int max_kmer_cnt;
 	double high_factor; // coverage cutoff set to high_factor*hom_cov
 	double max_ov_diff_ec;
 	double max_ov_diff_final;
 	int hom_cov;
     int het_cov;
-    int break_cov;
+    int b_low_cov;
+    int b_high_cov;
+    double m_rate;
 	int max_n_chain; // fall-back max number of chains to consider
 	int min_hist_kmer_cnt;
     int load_index_from_disk;
@@ -68,18 +82,23 @@ typedef struct {
     int purge_level_primary;
     int purge_level_trio;
     int purge_overlap_len;
-    int purge_overlap_len_hic;
+    ///int purge_overlap_len_hic;
     int recover_atg_cov_min;
     int recover_atg_cov_max;
     int hom_global_coverage;
+    int hom_global_coverage_set;
+    int pur_global_coverage;
     int bed_inconsist_rate;
     int hic_inconsist_rate;
 
     float max_hang_rate;
     float min_drop_rate;
     float max_drop_rate;
-    float purge_simi_rate;
-    float purge_simi_rate_hic;
+    float purge_simi_rate_l2;
+    float purge_simi_rate_l3;
+    float purge_simi_thres;
+
+    ///float purge_simi_rate_hic;
 
     long long small_pop_bubble_size;
     long long large_pop_bubble_size;
@@ -88,7 +107,24 @@ typedef struct {
     long long num_recorrected_bases;
 	long long mem_buf;
     long long coverage;
-    
+    int hap_occ;
+    int polyploidy;
+    int trio_flag_occ_thres;
+    uint64_t seed;
+    int32_t n_perturb;
+    double f_perturb;
+    int32_t n_weight;
+    uint32_t is_alt;
+    uint64_t misjoin_len;
+    uint64_t scffold;
+    int32_t dp_min_len;
+    float dp_e;
+    int64_t hg_size;
+    float kpt_rate;
+    int64_t infor_cov, s_hap_cov;
+    double ul_error_rate, ul_error_rate_low, ul_error_rate_hpc;
+    int32_t ul_ec_round;
+    uint8_t is_dbg_het_cnt;
 } hifiasm_opt_t;
 
 extern hifiasm_opt_t asm_opt;

Correct.h

@@ -6,6 +6,7 @@
 #include "POA.h"
 #include "Process_Read.h"
 #include "Correct.h"
+#include "kalloc.h"
 
 //#define CORRECT_THRESHOLD 0.70
 #define CORRECT_THRESHOLD 0.60
@@ -16,6 +17,10 @@
 #define MISMATCH 1
 #define INSERTION 2
 #define DELETION 3
+#define ERROR_RATE 1.5
+#define UL_TOPN 50
+#define SGAP 16
+#define MAX_LGAP(ql) ((((ql)*0.2)<256)?((ql)*0.2):256)
 
 #define WINDOW_MAX_SIZE (WINDOW + (int)(1.0 / HA_MIN_OV_DIFF) + 3) // TODO: why 1/max_ov_diff?
 
@@ -132,6 +137,7 @@ typedef struct
     uint32_t overlapSite;
     ///there are several types: 0: equal to read 1: not equal to read, but it is a mismatch 2: is a gap
     uint8_t type;
+    uint32_t cov;
     ///misbase
     char misBase;
 }haplotype_evdience;
@@ -229,7 +235,14 @@ DP_matrix;
 #define Get_SNP_Martix_Size(matrix) (matrix.snp * matrix.overlap)
 #define Get_SNP_Vector(matrix, i) (matrix.snp_matrix + matrix.overlap * i)
 #define Get_SNP_Vector_Length(matrix) (matrix.overlap)
-#define Get_Result_SNP_Vector(matrix) (matrix.snp_matrix + matrix.overlap*matrix.snp)
+// #define Get_Result_SNP_Vector(matrix) (matrix.snp_matrix + matrix.overlap*matrix.snp)
+#define Get_Result_SNP_Vector(matrix) (matrix.r_snp)
+
+typedef struct
+{
+    SnpStats* a;
+    size_t n,m;
+}kv_SnpStats_t;
 
 typedef struct
 {
@@ -243,15 +256,22 @@ typedef struct
     uint8_t flag[WINDOW_MAX_SIZE];
     /****************************may have bugs********************************/
 
-    uint32_t available_snp;
+    
     uint32_t core_snp;
-    uint32_t snp;
     uint32_t overlap;
-    int8_t* snp_matrix;
+    int8_t *snp_matrix;
     uint32_t snp_matrix_size;
-    SnpStats* snp_stat; 
+    int8_t *r_snp;
+    uint32_t r_snp_size;
     SnpStats result_stat;
-    uint32_t snp_stat_size;
+    
+    kv_SnpStats_t snp_stat;
+    uint32_t nn_snp;
+    kvec_t(uint64_t) snp_srt;
+    // SnpStats* snp_stat; 
+    // uint32_t snp;
+    // uint32_t snp_stat_size;
+    // uint32_t available_snp;
 
     DP_matrix dp;
 }
@@ -341,9 +361,9 @@ inline void count_nearby_snps(haplotype_evdience_alloc* hap, uint32_t* SNPs, int
             small_id = SNPs[i + 1];
             
 
-            if(hap->snp_stat[large_id].site - hap->snp_stat[current_id].site < distance
+            if(hap->snp_stat.a[large_id].site - hap->snp_stat.a[current_id].site < distance
               ||
-              hap->snp_stat[current_id].site - hap->snp_stat[small_id].site < distance)
+              hap->snp_stat.a[current_id].site - hap->snp_stat.a[small_id].site < distance)
             {
                 (*nearsnp)++;
             }
@@ -352,8 +372,8 @@ inline void count_nearby_snps(haplotype_evdience_alloc* hap, uint32_t* SNPs, int
                 (*non_nearsnps)++;
             }
 
-            if(hap->snp_stat[current_id].site > hap->snp_stat[large_id].site || 
-               hap->snp_stat[current_id].site < hap->snp_stat[small_id].site)
+            if(hap->snp_stat.a[current_id].site > hap->snp_stat.a[large_id].site || 
+               hap->snp_stat.a[current_id].site < hap->snp_stat.a[small_id].site)
             {
                 fprintf(stderr, "error\n");
             }
@@ -362,7 +382,7 @@ inline void count_nearby_snps(haplotype_evdience_alloc* hap, uint32_t* SNPs, int
         {
             current_id= SNPs[i];
             small_id = SNPs[i + 1];
-            if(hap->snp_stat[current_id].site - hap->snp_stat[small_id].site < distance)
+            if(hap->snp_stat.a[current_id].site - hap->snp_stat.a[small_id].site < distance)
             {
                 (*nearsnp)++;
             }
@@ -371,7 +391,7 @@ inline void count_nearby_snps(haplotype_evdience_alloc* hap, uint32_t* SNPs, int
                 (*non_nearsnps)++;
             }
 
-            if(hap->snp_stat[current_id].site < hap->snp_stat[small_id].site)
+            if(hap->snp_stat.a[current_id].site < hap->snp_stat.a[small_id].site)
             {
                 fprintf(stderr, "error\n");
             }
@@ -380,7 +400,7 @@ inline void count_nearby_snps(haplotype_evdience_alloc* hap, uint32_t* SNPs, int
         {
             large_id = SNPs[i - 1];
             current_id= SNPs[i];
-            if(hap->snp_stat[large_id].site - hap->snp_stat[current_id].site < distance)
+            if(hap->snp_stat.a[large_id].site - hap->snp_stat.a[current_id].site < distance)
             {
                 (*nearsnp)++;
             }
@@ -389,7 +409,7 @@ inline void count_nearby_snps(haplotype_evdience_alloc* hap, uint32_t* SNPs, int
                 (*non_nearsnps)++;
             }
 
-            if(hap->snp_stat[current_id].site > hap->snp_stat[large_id].site)
+            if(hap->snp_stat.a[current_id].site > hap->snp_stat.a[large_id].site)
             {
                 fprintf(stderr, "error\n");
             }
@@ -632,46 +652,61 @@ UC_Read* g_read)
 {
     if(sub_length <= 0)
         return;
-    long long i = 0;
-    h->snp_stat[h->available_snp].id = h->available_snp;
-    h->snp_stat[h->available_snp].occ_0 = 0;
-    h->snp_stat[h->available_snp].occ_1 = 0;
-    h->snp_stat[h->available_snp].occ_2 = 0;
-    h->snp_stat[h->available_snp].overlap_num = 0;
+    long long i = 0; SnpStats *p = NULL;
+    kv_pushp(SnpStats, h->snp_stat, &p);
     
-    h->snp_stat[h->available_snp].site = sub_list[0].site;
+    // h->snp_stat[h->available_snp].id = h->available_snp;
+    // h->snp_stat[h->available_snp].occ_0 = 0;
+    // h->snp_stat[h->available_snp].occ_1 = 0;
+    // h->snp_stat[h->available_snp].occ_2 = 0;
+    // h->snp_stat[h->available_snp].overlap_num = 0;
+    // h->snp_stat[h->available_snp].site = sub_list[0].site;
+    // h->snp_stat[h->available_snp].is_homopolymer = 
+    // if_is_homopolymer_strict(h->snp_stat[h->available_snp].site, g_read->seq, g_read->length);
+    // int8_t* vector = Get_SNP_Vector((*h), h->available_snp);
 
-    h->snp_stat[h->available_snp].is_homopolymer = 
-    if_is_homopolymer_strict(h->snp_stat[h->available_snp].site, g_read->seq, g_read->length);
-
-    int8_t* vector = Get_SNP_Vector((*h), h->available_snp);
+    p->id = h->snp_stat.n-1;
+    p->occ_0 = 0;
+    p->occ_1 = 0;
+    p->occ_2 = 0;
+    p->overlap_num = 0;
+    p->site = sub_list[0].site;
+    p->is_homopolymer = if_is_homopolymer_strict(p->site, g_read->seq, g_read->length);
+    int8_t* vector = Get_SNP_Vector((*h), p->id);
     for (i = 0; i < sub_length; i++)
     {
         if(sub_list[i].type == 0)
         {
             vector[sub_list[i].overlapID] = 0;
-            h->snp_stat[h->available_snp].occ_0++;
+            // h->snp_stat[h->available_snp].occ_0++;
+            h->snp_stat.a[p->id].occ_0++;
         }
         else if(sub_list[i].type == 1 && sub_list[i].misBase == misBase)
         {
             vector[sub_list[i].overlapID] = 1;
-            h->snp_stat[h->available_snp].occ_1++;
+            // h->snp_stat[h->available_snp].occ_1++;
+            h->snp_stat.a[p->id].occ_1++;
         }
         else
         {
             vector[sub_list[i].overlapID] = 2;
-            h->snp_stat[h->available_snp].occ_2++;
+            // h->snp_stat[h->available_snp].occ_2++;
+            h->snp_stat.a[p->id].occ_2++;
         }
-        h->snp_stat[h->available_snp].overlap_num++;
+        // h->snp_stat[h->available_snp].overlap_num++;
+        h->snp_stat.a[p->id].overlap_num++;
     }
 
 
-    int new_occ_0 = h->snp_stat[h->available_snp].occ_0 + 1;
-    int new_occ_1 = h->snp_stat[h->available_snp].occ_1;
+    // int new_occ_0 = h->snp_stat[h->available_snp].occ_0 + 1;
+    // int new_occ_1 = h->snp_stat[h->available_snp].occ_1;
+    int new_occ_0 = h->snp_stat.a[p->id].occ_0 + 1;
+    int new_occ_1 = h->snp_stat.a[p->id].occ_1;
 
-    if(filter_snp(new_occ_0, new_occ_1, new_occ_0 + new_occ_1) == 0)
+    if(filter_snp(new_occ_0, new_occ_1, new_occ_0 + new_occ_1) == 0) ///Fix-attention:definitely wrong
     {
-        h->snp_stat[h->available_snp].score = -1;
+        // h->snp_stat[h->available_snp].score = -1;
+        h->snp_stat.a[p->id].score = -1;
     }
     else
     {
@@ -682,7 +717,7 @@ UC_Read* g_read)
 
         consensus = consensus /((double)(new_occ_0 + new_occ_1));
 
-        ///50% vs 50%
+        ///50% vs 50%///Fix-attention:definitely wrong
         if(new_occ_0 == new_occ_1)
         {
             consensus = consensus + 0.25;
@@ -707,12 +742,13 @@ UC_Read* g_read)
 
         consensus= consensus*((double)(new_occ_0 + new_occ_1));
 
-        h->snp_stat[h->available_snp].score = consensus;
+        // h->snp_stat[h->available_snp].score = consensus;
+        h->snp_stat.a[p->id].score = consensus;
     }
 
 
     
-    h->available_snp++;
+    // h->available_snp++;
 }
 
 
@@ -760,32 +796,31 @@ inline int calculate_score(int new_occ_0, int new_occ_1)
     return consensus;
 }
 
-inline void SetSnpMatrix(haplotype_evdience_alloc* h, long long snp_num, long long overlap_num)
+inline void SetSnpMatrix(haplotype_evdience_alloc* h, uint32_t *nn_snp, uint64_t *overlap_num, int32_t set_matrix, void *km)
 {
-    long long new_size = (snp_num + 1)* overlap_num;
+    if(nn_snp && overlap_num) {
+        if(!km) kv_resize(SnpStats, h->snp_stat, *nn_snp);
+        else kv_resize_km(km, SnpStats, h->snp_stat, *nn_snp);
+        h->snp_stat.n = 0; h->overlap = *overlap_num; h->core_snp = 0;
+    } 
 
-    if(h->snp_matrix_size < new_size)
-    {
-        h->snp_matrix_size = new_size;
-        h->snp_matrix = (int8_t*)realloc(h->snp_matrix, h->snp_matrix_size);
+    if(set_matrix) {
+        uint64_t n_snp = nn_snp? *nn_snp:h->snp_stat.n;
+        uint64_t n_ovlp = overlap_num? *overlap_num:h->overlap;
+        uint64_t new_size = n_snp* n_ovlp;
+        if(h->snp_matrix_size < new_size) {
+            h->snp_matrix_size = new_size;
+            if(!km) REALLOC(h->snp_matrix, h->snp_matrix_size);
+            else KREALLOC(km, h->snp_matrix, h->snp_matrix_size);
+        }
+        memset(h->snp_matrix, -1, n_snp * n_ovlp);
+
+        if(h->r_snp_size < n_ovlp) {
+            h->r_snp_size = n_ovlp;
+            if(!km) REALLOC(h->r_snp, h->r_snp_size);
+            else KREALLOC(km, h->r_snp, h->r_snp_size);
+        }
     }
-
-    if(h->snp_stat_size < snp_num)
-    {
-        h->snp_stat_size = snp_num;
-        h->snp_stat = (SnpStats*)realloc(h->snp_stat, h->snp_stat_size * sizeof(SnpStats));
-    }
-
-    ///h->snp may be different with the number of snp vector
-    ///since some snps have been filtered
-    h->snp = snp_num;
-    h->overlap = overlap_num;
-    h->available_snp = 0;
-    h->core_snp = 0;
-
-    memset(h->snp_matrix, -1, h->snp * h->overlap);
-
-    
 }
 
 
@@ -897,15 +932,33 @@ inline void init_DP_matrix(DP_matrix* dp, uint32_t snp_num)
 
 }
 
+inline void InitHaplotypeEvdience_buf(haplotype_evdience_alloc* h, void *km)
+{
+    memset(h, 0, sizeof(haplotype_evdience_alloc));
+    /****************************may have bugs********************************/
+    memset(h->flag, 0, WINDOW_MAX_SIZE * sizeof(uint8_t));
+    /****************************may have bugs********************************/
+    // init_SNP_IDs(&(h->dp.SNP_IDs));
+    memset(&(h->dp.SNP_IDs), 0, sizeof(h->dp.SNP_IDs));
+    h->dp.SNP_IDs.max_snp_id = -1;
+}
+
+
 inline void InitHaplotypeEvdience(haplotype_evdience_alloc* h)
 {
-    h->snp = 0;
-    h->available_snp = 0;
+    
     h->overlap = 0;
-    h->snp_matrix_size = 0;
-    h->snp_stat_size = 0;
+    h->snp_matrix_size = 0;   
     h->snp_matrix = NULL;
-    h->snp_stat = NULL;
+    h->r_snp_size = 0;
+    h->r_snp = NULL;
+    kv_init(h->snp_stat);
+    kv_init(h->snp_srt);
+    h->nn_snp = 0;
+    // h->snp_stat = NULL;
+    // h->snp = 0;
+    // h->snp_stat_size = 0;
+    // h->available_snp = 0;
 
 
     h->sub_list_start = 0;
@@ -937,7 +990,6 @@ inline void InitHaplotypeEvdience(haplotype_evdience_alloc* h)
     h->dp.max_buffer = NULL;
 
     init_SNP_IDs(&(h->dp.SNP_IDs));
-
 }
 
 inline void StarSubListHaplotypeEvdience(haplotype_evdience_alloc* h)
@@ -954,8 +1006,11 @@ inline void EndSubListHaplotypeEvdience(haplotype_evdience_alloc* h)
 inline void destoryHaplotypeEvdience(haplotype_evdience_alloc* h)
 {
     free(h->list);
-    free(h->snp_stat);
+    // free(h->snp_stat);
+    kv_destroy(h->snp_stat);
+    kv_destroy(h->snp_srt);
     free(h->snp_matrix);
+    free(h->r_snp);
     free(h->dp.backtrack);
     free(h->dp.max);
     free(h->dp.max_for_sort);
@@ -966,9 +1021,12 @@ inline void destoryHaplotypeEvdience(haplotype_evdience_alloc* h)
     destory_SNP_IDs(&(h->dp.SNP_IDs));
 }
 
+
 inline void ResizeInitHaplotypeEvdience(haplotype_evdience_alloc* h)
 {
-    h->snp = 0;
+    // h->snp = 0;
+    h->nn_snp = 0;
+    h->snp_stat.n = 0;
     h->length = 0;
     h->sub_list_start = 0;
     h->sub_list_length = 0;
@@ -984,17 +1042,14 @@ inline void RsetInitHaplotypeEvdienceFlag(haplotype_evdience_alloc* h, long long
     /****************************may have bugs********************************/
 }
 
-inline void addHaplotypeEvdience(haplotype_evdience_alloc* h, haplotype_evdience* ev)
+inline void addHaplotypeEvdience(haplotype_evdience_alloc* h, haplotype_evdience* ev, void *km)
 {
-    uint32_t new_length = h->length + 1;
-    if(new_length > h->size)
-    {
-        h->size = h->size * 2;
-        if(h->size < new_length)
-        {
-            h->size = new_length;
-        }
-        h->list = (haplotype_evdience*)realloc(h->list, sizeof(haplotype_evdience)*h->size);
+    if(h->length + 1 > h->size){
+        h->size = h->length + 1;
+        kroundup32(h->size);
+        if(!km) REALLOC(h->list, h->size);
+        else KREALLOC(km, h->list, h->size);
+        // h->list = (haplotype_evdience*)realloc(h->list, sizeof(haplotype_evdience)*h->size);
     }
 
     h->list[h->length] = (*ev);
@@ -1043,24 +1098,36 @@ typedef struct
 }
 Round2_alignment;
 
+void init_Round2_alignment_buf(Round2_alignment* h, void *km);
 void init_Round2_alignment(Round2_alignment* h);
 void destory_Round2_alignment(Round2_alignment* h);
 void clear_Round2_alignment(Round2_alignment* h);
 
+typedef struct {
+	int32_t c_qs, c_qe, c_ts, c_te; //[c_qs, c_qe) && [c_ts, c_te)
+    int32_t c_wsid, c_weid, c_wsii, c_weii;//[c_wsid, c_weid] && [c_wsii, c_weii)
+    int32_t rev, sfx_e, pfx_e, mid_e, oid;
+} rtrace_t;
 
+typedef struct {
+	rtrace_t *a;
+	size_t n, m;
+} kv_rtrace_t;
 
 void correct_overlap(overlap_region_alloc* overlap_list, All_reads* R_INF, 
                         UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, Graph* g, Graph* DAGCon, 
                         Cigar_record* current_cigar, haplotype_evdience_alloc* hap,
-                        Round2_alignment* second_round, int force_repeat, int is_consensus,
-                        int* fully_cov, int* abnormal);
+                        Round2_alignment* second_round, kvec_t_u64_warp* v_idx, window_list_alloc* win_ciagr_buf, 
+                        int force_repeat, int is_consensus, int* fully_cov, int* abnormal);
+void init_Correct_dumy_buf(Correct_dumy* list, void *km);
 void init_Correct_dumy(Correct_dumy* list);
 void destory_Correct_dumy(Correct_dumy* list);
-void clear_Correct_dumy(Correct_dumy* list, overlap_region_alloc* overlap_list);
+void clear_Correct_dumy(Correct_dumy* list, overlap_region_alloc* overlap_list, void *km);
 void clear_Correct_dumy_pure(Correct_dumy* list);
 void get_seq_from_Graph(Graph* backbone, Graph* DAGCon, Correct_dumy* dumy, Cigar_record* current_cigar, char* self_string,
 char* r_string, long long r_string_length, long long r_string_site);
 void init_Cigar_record(Cigar_record* dummy);
+void init_Cigar_record_buf(Cigar_record* dummy, void *km);
 void destory_Cigar_record(Cigar_record* dummy);
 void clear_Cigar_record(Cigar_record* dummy);
 void add_new_cell_to_cigar_record(Cigar_record* dummy, uint32_t len, uint32_t type);
@@ -1069,6 +1136,30 @@ void add_new_cell_to_cigar_record_with_different_base(Cigar_record* dummy, uint3
 void add_existing_cell_to_cigar_record_with_different_base(Cigar_record* dummy, uint32_t len, uint32_t type, char* seq);
 
 
+void correct_ul_overlap(overlap_region_alloc* overlap_list, const ul_idx_t *uref, 
+                        UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, 
+                        Graph* g, Graph* DAGCon, Cigar_record* current_cigar, 
+                        haplotype_evdience_alloc* hap, Round2_alignment* second_round, 
+                        kvec_t_u64_warp* v_idx, window_list_alloc* win_ciagr_buf, 
+                        int force_repeat, int is_consensus, int* fully_cov, int* abnormal, 
+                        double max_ov_diff_ec, long long winLen, void *km);
+void ul_lalign(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *uref, const ug_opt_t *uopt, char *qstr, 
+                        uint64_t ql, UC_Read* qu, UC_Read* tu, Correct_dumy* dumy, bit_extz_t *exz,
+                        haplotype_evdience_alloc* hap, kvec_t_u64_warp* v_idx, overlap_region *aux_o,   
+                        double e_rate, int64_t wl, kv_ul_ov_t *aln, int64_t sid, uint64_t hpc_k, st_mt_t *stb, void *km);
+
+void ul_lalign_old_ed(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *uref, char *qstr, 
+                        uint64_t ql, UC_Read* qu, UC_Read* tu, Correct_dumy* dumy, 
+                        haplotype_evdience_alloc* hap, kvec_t_u64_warp* v_idx,   
+                        double e_rate, int64_t wl, uint64_t is_base, void *km);
+
+void lchain_align(overlap_region_alloc* overlap_list, const ul_idx_t *uref, 
+                        UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, 
+                        Graph* g, Graph* DAGCon, Cigar_record* current_cigar, 
+                        haplotype_evdience_alloc* hap, Round2_alignment* second_round, 
+                        kvec_t_u64_warp* v_idx, window_list_alloc* win_ciagr_buf, 
+                        int force_repeat, int is_consensus, int* fully_cov, int* abnormal, 
+                        double max_ov_diff_ec, long long winLen, void *km);
 /***
  type:
  0. match
@@ -1169,8 +1260,8 @@ long long* max_q_pos, long long* max_t_pos, long long* global_score,
 long long* extention_score, long long* q_boundary_score, long long* q_boundary_t_coordinate,
 long long* t_boundary_score, long long* t_boundary_q_coordinate,
 long long* droped, int mode);
-void correct_overlap_high_het(overlap_region_alloc* overlap_list, All_reads* R_INF, 
-                        UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read);
+// void correct_overlap_high_het(overlap_region_alloc* overlap_list, All_reads* R_INF, 
+//                         UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read);
 long long get_affine_gap_score(overlap_region* ovc, UC_Read* g_read, UC_Read* overlap_read, uint8_t* x_num,
 uint8_t* y_num, uint64_t EstimateXOlen, uint64_t EstimateYOlen);
 int collect_hp_regions(overlap_region_alloc* olist, All_reads* R_INF, kvec_t_u8_warp* k_flag, float hp_rate, int rlen, FILE* fp);
@@ -1187,7 +1278,80 @@ inline int if_exact_match(char* x, long long xLen, char* y, long long yLen, long
     return 0;
 }
 
+inline void get_cigar_cell(window_list *idx, window_list_alloc *cc, uint32_t i, uint8_t *c, uint32_t *len)
+{
+    uint16_t p = cc->c.a[idx->cidx+i];
+    (*c) = (uint8_t)(p>>14); (*len) = (p&((uint16_t)0x3fff));
+}
 
+inline void push_cigar_cell(window_list_alloc *res, uint8_t c, uint32_t len)
+{
+    uint16_t p = c; p <<= 14; p += (uint16_t)len;
+    kv_push(uint16_t, res->c, p);
+}
+int64_t get_num_wins(int64_t s, int64_t e, int64_t block_s);
+void append_unmatched_wins(overlap_region *z, int64_t block_s);
+
+///[w_s, w_e]
+inline int64_t get_win_id_by_s(overlap_region *z, int64_t w_s, int64_t block_s, int64_t *w_e)
+{
+    int64_t n_s = ((z->x_pos_s/block_s)*block_s), wid = (w_s-n_s)/block_s;
+    if(w_e) {
+        (*w_e) = n_s + (wid+1)*block_s - 1;        
+        if((*w_e) > z->x_pos_e) (*w_e) = z->x_pos_e;
+    }
+    return wid;
+}
+
+///[w_s, w_e]
+inline int64_t get_win_id_by_e(overlap_region *z, int64_t w_e, int64_t block_s, int64_t *w_s)
+{
+    int64_t n_s = ((z->x_pos_s/block_s)*block_s), wid = (w_e-n_s)/block_s;
+    if(w_s) {
+        (*w_s) = n_s + wid*block_s;        
+        if((*w_s) < z->x_pos_s) (*w_s) = z->x_pos_s;
+    }
+    return wid;
+}
+
+///[w_s, w_e]
+inline void get_win_se_by_normalize_xs(overlap_region *z, int64_t norm_w_s, int64_t block_s, int64_t *w_s, int64_t *w_e)
+{
+    int64_t n_s = ((z->x_pos_s/block_s)*block_s), wid = (norm_w_s-n_s)/block_s;
+    if(w_s) {
+        (*w_s) = n_s + wid*block_s;        
+        if((*w_s) < z->x_pos_s) (*w_s) = z->x_pos_s;
+    }
+    if(w_e) {
+        (*w_e) = n_s + (wid+1)*block_s - 1;        
+        if((*w_e) > z->x_pos_e) (*w_e) = z->x_pos_e;
+    }
+}
+
+void inline resize_UC_Read(UC_Read *z, int64_t s)
+{
+    if(z->size < s) {
+        REALLOC(z->seq, s); z->size = s;
+    }
+}
+
+void update_sketch_trace(overlap_region_alloc* ol, const ul_idx_t *uref, const ug_opt_t *uopt, 
+All_reads *rref, UC_Read* tu, asg64_v* idx, asg64_v *b0, asg64_v *b1, int64_t ql, int64_t wl, 
+kv_ul_ov_t *aln, uint64_t rid, int64_t max_lgap, double sgap_rate);
+
+int64_t infer_rovlp(ul_ov_t *li, ul_ov_t *lj, uc_block_t *bi, uc_block_t *bj, All_reads *ridx, ma_ug_t *ug);
+void convert_ul_ov_t(ul_ov_t *des, overlap_region *src, const ul_idx_t *uref);
+uint64_t check_connect_ug(const ul_idx_t *uref, uint32_t v, uint32_t w, int64_t bw, double diff_ec_ul, int64_t dq);
+uint64_t check_connect_rg(const ul_idx_t *uref, const ug_opt_t *uopt, uint32_t uv, uint32_t uw, int64_t bw, double diff_ec_ul, int64_t dq);
+uint32_t govlp_check(const ul_idx_t *uref, const ug_opt_t *uopt, int64_t bw, double diff_ec_ul, ul_ov_t *li, ul_ov_t *lj);
+void ul_rid_lalign_adv(overlap_region_alloc* ol, Candidates_list *cl, const ul_idx_t *uref, const ug_opt_t *uopt, 
+        char *qstr, uint64_t ql, UC_Read* qu, UC_Read* tu, bit_extz_t *exz, overlap_region *aux_o, double e_rate, 
+        int64_t wl, kv_ul_ov_t *aln, kv_ul_ov_t *cln, kv_rtrace_t *trace, int64_t sid, uint64_t khit, void *km);
+
+#define copy_asg_arr(des, src) ((des).a = (src).a, (des).n = (src).n, (des).m = (src).m)
+#define is_ualn_win(a) (((a).error==INT16_MAX)&&((a).clen==0)&&((a).extra_end<0))
+#define is_exact_aln(a) (((a).error<INT16_MAX)&&((a).clen>0))
+#define is_est_aln(a) (((a).error<INT16_MAX)&&((a).clen==0))
 
 #define FORWARD_KSW 0
 #define BACKWARD_KSW 1
@@ -1197,4 +1361,24 @@ inline int if_exact_match(char* x, long long xLen, char* y, long long yLen, long
 #define GAP_EXT_KSW 2
 #define Z_DROP_KSW 400
 #define BAND_KSW 500
+
+#define set_bit_extz_t(x, z, id) do {\
+        (x).cigar.a = (z).w_list.c.a+(z).w_list.a[(id)].cidx;\
+		(x).cigar.n = (x).cigar.m = (z).w_list.a[(id)].clen;\
+        (x).ts = (z).w_list.a[(id)].x_start;\
+        (x).te = (z).w_list.a[(id)].x_end;\
+        (x).ps = (z).w_list.a[(id)].y_start;\
+        (x).pe = (z).w_list.a[(id)].y_end;\
+        (x).err = (z).w_list.a[(id)].error;\
+        (x).thre = (z).w_list.a[(id)].error;\
+	} while (0)
+
+typedef struct {
+	int64_t k, q[2], t[2], cq[2], ct[2], ci[2], werr, werr0, cerr;
+	int64_t qoff, f, toff, coff, cur_qoff;
+} rtrace_iter;
+int64_t get_rid_backward_cigar_err(rtrace_iter *it, ul_ov_t *aln, kv_rtrace_t *trace, rtrace_t *tc,
+const ul_idx_t *uref, char* qstr, UC_Read *tu, overlap_region_alloc *ol, overlap_region *o, 
+bit_extz_t *exz, double e_rate, int64_t qs);
+
 #endif

Hash_Table.h

@@ -11,10 +11,20 @@
 ///for one side, the first or last WINDOW_UNCORRECT_SINGLE_SIDE_BOUNDARY bases should not be corrected
 #define WINDOW_UNCORRECT_SINGLE_SIDE_BOUNDARY 25
 #define THRESHOLD  15
-#define OVERLAP_THRESHOLD_FILTER 0.9
+#define OVERLAP_THRESHOLD_HIFI_FILTER 0.9
+#define OVERLAP_THRESHOLD_NOSI_FILTER 0.7
+#define OVERLAP_THRESHOLD_FILTER_HPC 0.75
 #define HIGH_HET_OVERLAP_THRESHOLD_FILTER 0.3
 #define HIGH_HET_ERROR_RATE 0.08
 #define THRESHOLD_MAX_SIZE  31
+#define THRESHOLD_UL_MAX 0.2
+#define WINDOW_UL 75
+#define WINDOW_UL_H 200
+// #define WINDOW_UL_H 150
+#define MIN_UL_ALIN_RATE 0.5
+#define MIN_UL_ALIN_LEN (WINDOW_UL*6)
+#define WINDOW_UL_BOUND 48
+#define WINDOW_UL_BOUND_RATE 0.55
 
 #define GROUP_SIZE 4
 ///the max cigar likes 10M10D10M10D10M
@@ -36,32 +46,21 @@ typedef struct
     uint64_t end_pos;
 } k_mer_pos_list;
 
-typedef struct
-{
-    int C_L[CIGAR_MAX_LENGTH];
-    char C_C[CIGAR_MAX_LENGTH];
-    int length;
-} CIGAR;
-
 typedef struct
 {
   ///the begining and end of a window, instead of the whole overlap
-  uint64_t x_start;
-  uint64_t x_end;
-  int y_end;
-  int y_start;
-  int extra_begin;
-  int extra_end;
-  int error_threshold;
-  int error;
-  CIGAR cigar;
+  int32_t x_start, x_end;
+  int32_t y_start, y_end;
+  int16_t extra_begin, extra_end;
+  int16_t error, error_threshold;
+  uint32_t cidx, clen;
 } window_list;
 
 typedef struct
 {
-    window_list* buffer;
-    int32_t length;
-    int32_t size;
+    size_t n, m; 
+    window_list *a;
+    kvec_t(uint16_t) c;
 } window_list_alloc;
 
 typedef struct
@@ -92,11 +91,12 @@ typedef struct
     int8_t strong;
     uint32_t non_homopolymer_errors;
 
-    window_list* w_list;
-    uint32_t w_list_size;
-    uint32_t w_list_length;
+    // window_list* w_list;
+    // uint32_t w_list_size;
+    // uint32_t w_list_length;
     Fake_Cigar f_cigar;
 
+    window_list_alloc w_list;
     window_list_alloc boundary_cigars;
 } overlap_region;
 
@@ -110,8 +110,8 @@ typedef struct
 
 typedef struct
 {
-	uint32_t readID:30, strand:1, good:1;
-	uint32_t offset, self_offset;
+	uint32_t readID:31, strand:1;
+	uint32_t offset, self_offset, cnt;
 } k_mer_hit;
 
 typedef struct {
@@ -136,23 +136,24 @@ typedef struct
 void init_Candidates_list(Candidates_list* l);
 void clear_Candidates_list(Candidates_list* l);
 void destory_Candidates_list(Candidates_list* l);
+void destory_Candidates_list_buf(void *km, Candidates_list* l, int is_z);
 
 void init_overlap_region_alloc(overlap_region_alloc* list);
 void clear_overlap_region_alloc(overlap_region_alloc* list);
 void destory_overlap_region_alloc(overlap_region_alloc* list);
 void append_window_list(overlap_region* region, uint64_t x_start, uint64_t x_end, int y_start, int y_end, int error,
-int extra_begin, int extra_end, int error_threshold);
+int extra_begin, int extra_end, int error_threshold, int blockLen, void *km);
 
 void overlap_region_sort_y_id(overlap_region *a, long long n);
 
 void calculate_overlap_region_by_chaining(Candidates_list* candidates, overlap_region_alloc* overlap_list, kvec_t_u64_warp* chain_idx,
-uint64_t readID, uint64_t readLength, All_reads* R_INF, double band_width_threshold, int add_beg_end, overlap_region* f_cigar);
+uint64_t readID, uint64_t readLength, All_reads* R_INF, const ul_idx_t *uref, double band_width_threshold, int add_beg_end, overlap_region* f_cigar, void *km);
 
 void init_fake_cigar(Fake_Cigar* x);
 void destory_fake_cigar(Fake_Cigar* x);
 void clear_fake_cigar(Fake_Cigar* x);
-void add_fake_cigar(Fake_Cigar* x, uint32_t gap_site, int32_t gap_shift);
-void resize_fake_cigar(Fake_Cigar* x, uint64_t size);
+void add_fake_cigar(Fake_Cigar* x, uint32_t gap_site, int32_t gap_shift, void *km);
+void resize_fake_cigar(Fake_Cigar* x, uint64_t size, void *km);
 int get_fake_gap_pos(Fake_Cigar* x, int index);
 int get_fake_gap_shift(Fake_Cigar* x, int index);
 
@@ -182,11 +183,56 @@ static inline long long y_start_offset(long long x_start, Fake_Cigar* o)
     return get_fake_gap_shift(o, i - 1);
 }
 
-void resize_Chain_Data(Chain_Data* x, long long size);
+void resize_Chain_Data(Chain_Data* x, long long size, void *km);
 void init_window_list_alloc(window_list_alloc* x);
 void clear_window_list_alloc(window_list_alloc* x);
 void destory_window_list_alloc(window_list_alloc* x);
-void resize_window_list_alloc(window_list_alloc* x, long long size);
-void chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, double band_width_threshold, int max_skip, int x_readLen, int y_readLen);
+void resize_window_list_alloc(window_list_alloc* x, uint64_t size);
+long long chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, double band_width_threshold, int max_skip, int x_readLen, int y_readLen, void *km);
+uint64_t lchain_dp(k_mer_hit* a, int64_t a_n, k_mer_hit* des, Chain_Data* dp, overlap_region* res, 
+              int64_t max_skip, int64_t max_iter, int64_t max_dis, double chn_pen_gap, double chn_pen_skip, double bw_rate, 
+              int64_t xl, int64_t yl, int64_t quick_check);
+int ovlp_chain_gen(overlap_region_alloc* ol, overlap_region* t, int64_t xl, int64_t yl, int64_t apend_be, k_mer_hit* hit, int64_t n_hit);
+void gen_fake_cigar(Fake_Cigar* z, overlap_region *o, int64_t apend_be, k_mer_hit* hit, int64_t n_hit);
+int append_utg_inexact_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp, 
+                                        ma_utg_v *ua, int add_beg_end, void *km);
+
+#define kv_pushp_cl(type, v, p) do {									\
+		if ((v).length == (v).size) {										\
+			(v).size = (v).size? (v).size<<1 : 2;							\
+			(v).list = (type*)realloc((v).list, sizeof(type) * (v).size);	\
+		}															\
+		*(p) = &((v).list[(v).length++]); \
+	} while (0)
+
+#define kv_resize_cl(type, v, s) do { \
+		if ((v).size < (s)) { \
+			(v).size = (s); \
+			kv_roundup32((v).size); \
+			(v).list = (type*)realloc((v).list, sizeof(type) * (v).size); \
+		} \
+	} while (0)
+
+#define is_alnw(a) (((a).readID) == ((uint32_t)(0x7fffffff)))
+#define is_pri_aln(a) ((((a).readID) == ((uint32_t)(0x7fffffff)))||((a).cnt >= (a).readID))
+
+uint64_t lchain_dp_trace(k_mer_hit* a, int64_t a_n, int64_t max_lgap, double sgap_rate, int64_t sgap);
+uint64_t lchain_qdp(k_mer_hit* a, int64_t a_n, k_mer_hit* des, Chain_Data* dp, overlap_region* res, 
+              int64_t max_skip, int64_t max_iter, int64_t max_dis, double chn_pen_gap, double chn_pen_skip, double bw_rate, 
+              int64_t xl, int64_t yl, int64_t quick_check);
+int ovlp_chain_qgen(overlap_region_alloc* ol, overlap_region* t, int64_t xl, int64_t yl, int64_t apend_be, k_mer_hit* hit, int64_t n_hit);
+uint64_t lchain_refine(k_mer_hit* a, int64_t a_n, k_mer_hit* des, Chain_Data* dp, 
+                                    int64_t max_skip, int64_t max_iter, int64_t max_dis, int64_t long_gap);
+uint64_t lchain_qdp_fix(k_mer_hit* a, int64_t a_n, Chain_Data* dp, int64_t max_skip, 
+                int64_t max_iter, int64_t max_dis, double chn_pen_gap, double chn_pen_skip, 
+                double bw_rate, int64_t xl, int64_t yl, int64_t quick_check, 
+                int64_t left_fix, int64_t right_fix);
+uint64_t lchain_simple(k_mer_hit* a, int64_t a_n, k_mer_hit* des, Chain_Data* dp, 
+                                                        int64_t max_skip, int64_t max_iter);
+uint64_t lchain_qdp_mcopy(Candidates_list *cl, int64_t a_idx, int64_t a_n, int64_t des_idx, 
+              Chain_Data* dp, overlap_region_alloc* res, int64_t max_skip, int64_t max_iter, 
+              int64_t max_dis, double chn_pen_gap, double chn_pen_skip, double bw_rate, 
+              uint32_t xid, int64_t xl, int64_t yl, int64_t quick_check, uint32_t apend_be, 
+              int64_t gen_cigar);
 
 #endif

Makefile

@@ -6,7 +6,8 @@ CPPFLAGS=
 INCLUDES=
 OBJS=		CommandLines.o Process_Read.o Assembly.o Hash_Table.o \
 			POA.o Correct.o Levenshtein_distance.o Overlaps.o Trio.o kthread.o Purge_Dups.o \
-			htab.o hist.o sketch.o anchor.o extract.o sys.o ksw2_extz2_sse.o hic.o
+			htab.o hist.o sketch.o anchor.o extract.o sys.o ksw2_extz2_sse.o hic.o rcut.o horder.o \
+			tovlp.o inter.o kalloc.o gfa_ut.o gchain_map.o
 EXE=		hifiasm
 LIBS=		-lz -lpthread -lm
 
@@ -43,7 +44,7 @@ Assembly.o: kthread.h
 CommandLines.o: CommandLines.h ketopt.h
 Correct.o: Correct.h Hash_Table.h htab.h Process_Read.h Overlaps.h kvec.h
 Correct.o: kdq.h CommandLines.h Levenshtein_distance.h POA.h Assembly.h
-Correct.o: ksw2.h
+Correct.o: ksw2.h ksort.h
 Hash_Table.o: Hash_Table.h htab.h Process_Read.h Overlaps.h kvec.h kdq.h
 Hash_Table.o: CommandLines.h ksort.h
 Levenshtein_distance.o: Levenshtein_distance.h
@@ -72,3 +73,10 @@ main.o: Levenshtein_distance.h htab.h
 sketch.o: kvec.h htab.h Process_Read.h Overlaps.h kdq.h CommandLines.h
 sys.o: htab.h Process_Read.h Overlaps.h kvec.h kdq.h CommandLines.h
 hic.o: hic.h
+rcut.o: rcut.h
+horder.o: horder.h
+tovlp.o: tovlp.h
+inter.o: inter.h Process_Read.h
+kalloc.o: kalloc.h
+gfa_ut.o: Overlaps.h
+gchain_map.o: gchain_map.h

Overlaps.h

@@ -4,6 +4,8 @@
 #include <stdint.h>
 #include "kvec.h"
 #include "kdq.h"
+#include "ksort.h"
+#include "CommandLines.h"
 
 ///#define MIN_OVERLAP_LEN 2000
 ///#define MIN_OVERLAP_LEN 500
@@ -26,10 +28,11 @@
 #define DOUBLE_CHECK_THRES 0.1
 #define FINAL_DOUBLE_CHECK_THRES 0.2
 #define CHIMERIC_TRIM_THRES 4
+#define GAP_LEN 100
 // #define PRIMARY_LABLE 1
 // #define ALTER_LABLE 2
 // #define HAP_LABLE 4
-
+#define HA_RE_UL_ID "re"
 
 #define Get_qn(RECORD) ((uint32_t)((RECORD).qns>>32))
 #define Get_qs(RECORD) ((uint32_t)((RECORD).qns))
@@ -50,13 +53,46 @@
 #define TRIM 10
 #define CUT 11
 #define CUT_DIF_HAP 12
+#define SEC_MODE ((uint32_t)(0x3fffffffU))
 
+///query is the read itself
+typedef struct {
+	uint32_t qn, qs, qe;
+	uint32_t tn, ts, te;
+	uint32_t sec:30, el:1, rev:1;
+} ul_ov_t;
+
+typedef struct {
+	ul_ov_t *a;
+	size_t n, m;
+} kv_ul_ov_t;
+
+typedef struct {
+	///off: start idx in mg128_t * a[];
+	///cnt: how many eles in this chain
+	///a[off, off+cnt) saves the eles in this chain
+	int32_t off, cnt:31, inner_pre:1;
+	///ref_id|rev
+	uint32_t v;
+	///chain in ref: [rs, re)
+	///chain in query: [qs, qe)
+	int32_t rs, re, qs, qe;
+	///score: chain score
+	int32_t score, dist_pre;
+	uint32_t hash_pre;
+} mg_lchain_t;
+
+typedef struct {
+	mg_lchain_t *a;
+	size_t n, m;
+}vec_mg_lchain_t;
 
 ///query is the read itself
 typedef struct {
 	uint64_t qns;
 	uint32_t qe, tn, ts, te;
-	uint32_t ml:31, rev:1;
+	// uint32_t ml:31, rev:1;
+	uint32_t cc:30, ml:1, rev:1;
 	uint32_t bl:31, del:1;
 	uint8_t el;
 	uint8_t no_l_indel;
@@ -98,13 +134,46 @@ int max_hang, int min_ovlp);
 long long get_specific_overlap(ma_hit_t_alloc* x, uint32_t qn, uint32_t tn);
 
 
+typedef struct {
+	uint32_t qSpre, qEpre, qScur, qEcur, qn;///[qSp, qEp) && [qSn, qEn]
+	uint32_t tSpre, tEpre, tScur, tEcur, tn;
+} u_trans_hit_t;
+
+typedef struct {
+	size_t n, m;
+	u_trans_hit_t* a;
+} kv_u_trans_hit_t;
+
+
+
+typedef struct {
+	uint32_t qs, qe, qn;
+	uint32_t ts, te, tn;
+	uint32_t occ;
+	double nw;
+	uint8_t f:6, rev:1, del:1;
+	///uint8_t qo:4, to:4;
+} u_trans_t;
+
+typedef struct {
+	size_t n, m;
+	u_trans_t* a;
+	kvec_t(uint64_t) idx;
+} kv_u_trans_t;
+
+#define u_trans_a(x, id) ((x).a + ((x).idx.a[(id)]>>32))
+#define u_trans_n(x, id) ((uint32_t)((x).idx.a[(id)]))
+#define OU_MASK (0x3fffU)
+
 typedef struct {
 	uint64_t ul;
 	uint32_t v;
 	uint32_t ol:31, del:1;
-	uint8_t strong;
+	uint16_t ou:14, strong:1, no_l_indel:1;
 	uint8_t el;
-	uint8_t no_l_indel;
+	// uint8_t strong;
+	// uint8_t el;
+	// uint8_t no_l_indel;
 } asg_arc_t;
 
 typedef struct {
@@ -143,6 +212,20 @@ typedef struct {
 	ma_utg_t* F_seq;
 } asg_t;
 
+typedef struct {
+	ma_hit_t_alloc* src;
+	int64_t min_ovlp, max_hang, max_hang_rate, need_srt, gap_fuzz;
+	asg_t *g;
+	uint32_t *idx;
+	kvec_t(uint32_t) pi;
+	asg_arc_t *a;
+	size_t n, m;
+} flex_asg_t;
+
+typedef struct {
+	uint32_t i[2];
+}flex_asg_e_retrive_t;
+
 asg_t *asg_init(void);
 void asg_destroy(asg_t *g);
 void asg_arc_sort(asg_t *g);
@@ -156,7 +239,7 @@ void print_gfa(asg_t *g);
 typedef struct { size_t n, m; uint64_t *a; } asg64_v;
 
 
-typedef struct { size_t n, m; ma_utg_t *a; } ma_utg_v;
+typedef struct { size_t n, m; ma_utg_t *a;} ma_utg_v;
 
 typedef struct {
 	ma_utg_v u;
@@ -168,7 +251,65 @@ typedef struct {
 	uint32_t utg:31, ori:1, start, len;
 } utg_intv_t;
 
+typedef struct {
+	uint32_t x, s, e;
+} utg_ct_t;
 
+typedef struct {
+	uint32_t *idx;
+	kvec_t(uint64_t) interval;
+} ucov_t;
+
+typedef struct {
+	uint32_t u, off, pos;
+} utg_rid_dt;
+
+typedef struct {
+	uint32_t *idx;
+	kvec_t(utg_rid_dt) p;
+	asg_t *rg;
+} utg_rid_t;
+
+typedef struct {
+	kvec_t(uint64_t) idx;
+	kvec_t(utg_ct_t) rids;
+	kvec_t(uint8_t) is_c;
+} ul_contain;
+
+typedef struct {
+	ma_ug_t *ug;
+	asg_t *rg;
+	uint64_t *idx;
+} cvert_t;
+
+typedef struct {
+	size_t n, m;
+    uint8_t *a;
+	uint64_t *idx;
+} hmap_t;
+
+typedef struct {
+	ma_ug_t *hg;
+	size_t n, m; 
+	uint64_t *a;
+	hmap_t *mm;
+} hpc_t;
+
+#define hpc_len(x, id) ((x).hg->u.a[(id)].len>>1)
+#define hpc_str(x, id, rev) (((x).hg->u.a[(id)].s)+((rev)?((x).hg->u.a[(id)].len>>1):(0)))
+
+typedef struct {
+	ma_ug_t *ug;
+	hpc_t *hpc_g;
+	ucov_t *cc;
+	ucov_t *cr;
+	ul_contain *ct;
+	utg_rid_t *r_ug;
+	// cvert_t *nug;
+	// kv_ul_ov_t *ov;
+} ul_idx_t;
+
+#define MA_HT_DOUBLE     (-1024)
 #define MA_HT_INT        (-1)
 #define MA_HT_QCONT      (-2)
 #define MA_HT_TCONT      (-3)
@@ -368,7 +509,6 @@ typedef struct {
 	kvec_t(uint32_t) e; // visited edges/arcs
 } buf_t;
 
-
 typedef struct {
 	kvec_t(uint64_t) Nodes; 
 	kvec_t(uint64_t) Edges; 
@@ -450,7 +590,7 @@ long long max_hang_length, long long clean_round, long long gap_fuzz,
 float min_ovlp_drop_ratio, float max_ovlp_drop_ratio, char* output_file_name, 
 long long bubble_dist, int read_graph, int write);
 
-void debug_info_of_specfic_read(char* name, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, int id, char* command);
+void debug_info_of_specfic_read(const char* name, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, int id, const char* command);
 void collect_abnormal_edges(ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, long long readNum);
 void add_overlaps(ma_hit_t_alloc* source_paf, ma_hit_t_alloc* dest_paf, uint64_t* source_index, long long listLen);
 void remove_overlaps(ma_hit_t_alloc* source_paf, uint64_t* source_index, long long listLen);
@@ -470,6 +610,7 @@ uint64_t* source_index, long long listLen);
 typedef struct {
 	uint64_t len;
 	uint32_t* index;
+	uint8_t* is_het;
 } R_to_U;
 
 void init_R_to_U(R_to_U* x, uint64_t len);
@@ -477,8 +618,7 @@ void destory_R_to_U(R_to_U* x);
 void set_R_to_U(R_to_U* x, uint32_t rID, uint32_t uID, uint32_t is_Unitig, uint8_t* flag);
 void get_R_to_U(R_to_U* x, uint32_t rID, uint32_t* uID, uint32_t* is_Unitig);
 void transfor_R_to_U(R_to_U* x);
-void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_tangle);
-int asg_pop_bubble_primary(asg_t *g, int max_dist);
+void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, kvec_asg_arc_t_warp* edge, int require_equal_nv, int test_tangle);
 long long asg_arc_del_simple_circle_untig(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *g, long long circleLen, int is_drop);
 
 typedef struct {
@@ -493,9 +633,21 @@ typedef struct {
 	uint32_t new_edges_i;
 } Edge_iter;
 
+typedef struct {
+	asg_arc_t x;
+    uint64_t Off;
+    uint64_t weight;
+}asg_arc_t_offset;
+
+typedef struct {
+	kvec_t(asg_arc_t_offset) a;
+	uint64_t i;
+}kvec_asg_arc_t_offset;
+
+
+
 void init_Edge_iter(asg_t* g, uint32_t v, asg_arc_t* new_edges, uint32_t new_edges_n, Edge_iter* x);
 int get_arc_t(Edge_iter* x, asg_arc_t* get);
-int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag);
 
 
 inline int get_real_length(asg_t *g, uint32_t v, uint32_t* v_s)
@@ -540,55 +692,6 @@ inline uint32_t check_tip(asg_t *sg, uint32_t begNode, uint32_t* endNode, buf_t*
     }
 }
 
-inline uint32_t get_unitig_back(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* endNode, 
-long long* nodeLen, long long* baseLen, buf_t* b)
-{
-	ma_utg_v* u = NULL; 
-    uint32_t v = begNode, w, k;
-    uint32_t kv;
-	(*nodeLen) = (*baseLen) = 0;
-    (*endNode) = (uint32_t)-1;
-	if(ug!=NULL) u = &(ug->u);
-
-    while (1)
-    {
-        kv = get_real_length(sg, v, NULL);
-        (*endNode) = v;
-		if(u == NULL) 
-		{
-			(*nodeLen)++;
-		}
-		else
-		{
-			(*nodeLen) += EvaluateLen((*u), v>>1);
-		}
-        if(b) kv_push(uint32_t, b->b, v);
-		///means reach the end of a unitig
-		if(kv!=1) (*baseLen) += sg->seq[v>>1].len;
-		if(kv==0) return END_TIPS;
-        if(kv>1) return MUL_OUTPUT;
-        ///kv must be 1 here
-        kv = get_real_length(sg, v, &w);
-		///means reach the end of a unitig
-        if(get_real_length(sg, w^1, NULL)!=1)
-		{
-			(*baseLen) += sg->seq[v>>1].len;
-			return MUL_INPUT;
-		}
-
-		for (k = 0; k < asg_arc_n(sg, v); k++)
-        {
-			if(asg_arc_a(sg, v)[k].del) continue;
-			///here is just one undeleted edge
-			(*baseLen) += asg_arc_len(asg_arc_a(sg, v)[k]);
-			break;
-		}
-
-        v = w;
-        if(v == begNode) return LOOP;
-    }
-}
-
 inline uint32_t get_unitig(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* endNode, 
 long long* nodeLen, long long* baseLen, long long* max_stop_nodeLen, long long* max_stop_baseLen, 
 uint32_t stops_threshold, buf_t* b)
@@ -714,302 +817,12 @@ uint32_t stops_threshold, buf_t* b)
 #define UNAVAILABLE (uint32_t)-1
 #define PLOID 0
 #define NON_PLOID 1
-#define DIFF_HAP_RATE 0.75
+// #define DIFF_HAP_RATE 0.75
 #define TRIO_DROP_THRES 0.9
 #define TRIO_DROP_LENGTH_THRES 0.8
 #define MAX_STOP_RATE 0.6
 #define TANGLE_MISSED_THRES 0.6
-///if ug == NULL, nsg should be equal to read_sg
-inline uint32_t check_different_haps(asg_t *nsg, ma_ug_t *ug, asg_t *read_sg, 
-uint32_t v_0, uint32_t v_1, ma_hit_t_alloc* reverse_sources, buf_t* b_0, buf_t* b_1, 
-R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold)
-{
-	uint32_t vEnd, qn, tn, j, is_Unitig, uId;
-	long long ELen_0, ELen_1, tmp, max_stop_nodeLen, max_stop_baseLen;
-
-	b_0->b.n = b_1->b.n = 0;
-	if(get_unitig(nsg, ug, v_0, &vEnd, &ELen_0, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
-	stops_threshold, b_0) == LOOP)
-	{
-		return UNAVAILABLE;
-	}
-	if(get_unitig(nsg, ug, v_1, &vEnd, &ELen_1, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
-	stops_threshold, b_1) == LOOP)
-	{
-		return UNAVAILABLE;
-	}
-	if(ELen_0<=min_edge_length || ELen_1<=min_edge_length) return UNAVAILABLE;
-
-	rIdContig b_max, b_min;
-	b_max.b_0 = b_min.b_0 = NULL;
-	b_max.offset = b_max.readI = b_max.untigI = 0;
-	b_min.offset = b_min.readI = b_min.untigI = 0;
-
-	if(ELen_0<=ELen_1)
-	{
-        b_min.b_0 = b_0;
-        b_max.b_0 = b_1;
-    }
-    else
-    {
-        b_min.b_0 = b_1;
-        b_max.b_0 = b_0;
-    }
-
-	uint32_t max_count = 0, min_count = 0;
-	ma_utg_t *node_min = NULL, *node_max = NULL;
-	if(ug != NULL)
-	{
-		/*****************************label all unitigs****************************************/
-		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
-		{
-			node_max = &(ug->u.a[b_max.b_0->b.a[b_max.untigI]>>1]);
-			///each read
-			for (b_max.readI = 0; b_max.readI < node_max->n; b_max.readI++)
-			{
-				qn = (node_max->a[b_max.readI]>>33);
-				set_R_to_U(ruIndex, qn, (b_max.b_0->b.a[b_max.untigI]>>1), 1, &(read_sg->seq[qn].c));
-			}
-		}
-		/*****************************label all unitigs****************************************/
-
-		///each unitig
-		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
-		{
-			
-			node_min = &(ug->u.a[(b_min.b_0->b.a[b_min.untigI]>>1)]);
-
-			///each read
-			for (b_min.readI = 0; b_min.readI < node_min->n; b_min.readI++)
-        	{
-				qn = node_min->a[b_min.readI]>>33;
-
-				/************************BUG: don't forget****************************/
-				if(reverse_sources[qn].length > 0) min_count++;
-				///if(reverse_sources[qn].length >= 0) min_count++;
-				/************************BUG: don't forget****************************/
-				for (j = 0; j < (long long)reverse_sources[qn].length; j++)
-				{
-					tn = Get_tn(reverse_sources[qn].buffer[j]);
-					if(read_sg->seq[tn].del == 1)
-					{
-						get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
-						if(tn == (uint32_t)-1 || is_Unitig == 1 || read_sg->seq[tn].del == 1) continue;
-					}
-
-					get_R_to_U(ruIndex, tn, &uId, &is_Unitig);
-					if(uId!=(uint32_t)-1 && is_Unitig == 1)
-					{
-						max_count++;
-						break;
-					}
-				}
-			}
-		}
-		/*****************************label all unitigs****************************************/
-		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
-		{
-			node_max = &(ug->u.a[b_max.b_0->b.a[b_max.untigI]>>1]);
-			///each read
-			for (b_max.readI = 0; b_max.readI < node_max->n; b_max.readI++)
-			{
-				qn = (node_max->a[b_max.readI]>>33);
-				ruIndex->index[qn] = (uint32_t)-1;
-			}
-		}
-		/*****************************label all unitigs****************************************/
-	}
-	else
-	{
-		/*****************************label all reads****************************************/
-		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
-		{
-			qn = (b_max.b_0->b.a[b_max.untigI]>>1);
-			set_R_to_U(ruIndex, qn, 1, 1, &(read_sg->seq[qn].c));
-		}
-		/*****************************label all reads****************************************/
-
-		///each read
-		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
-		{
-			qn = (b_min.b_0->b.a[b_min.untigI]>>1);
-
-			/************************BUG: don't forget****************************/
-			if(reverse_sources[qn].length > 0) min_count++;
-			///if(reverse_sources[qn].length >= 0) min_count++;
-			/************************BUG: don't forget****************************/
-
-			for (j = 0; j < (long long)reverse_sources[qn].length; j++)
-        	{
-            	tn = Get_tn(reverse_sources[qn].buffer[j]);
-				if(nsg->seq[tn].del == 1)
-				{
-					get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
-					if(tn == (uint32_t)-1 || is_Unitig == 1 || nsg->seq[tn].del == 1) continue;
-				}
-
-
-				get_R_to_U(ruIndex, tn, &uId, &is_Unitig);
-				if(uId!=(uint32_t)-1 && is_Unitig == 1)
-				{
-					max_count++;
-					break;
-				}
-			}
-		}
-
-		/*****************************label all reads****************************************/
-		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
-		{
-			qn = (b_max.b_0->b.a[b_max.untigI]>>1);
-			ruIndex->index[qn] = (uint32_t)-1;
-		}
-		/*****************************label all reads****************************************/
-	}
-
-	// if(((v_0==7707) && (v_1==26867))||((v_1==7707) && (v_0==26867)))
-	// {
-	// 	fprintf(stderr, "******\nv_0>>1: %u, v_0&1: %u, ELen_0: %u\n", v_0>>1, v_0&1, (uint32_t)ELen_0);
-	// 	fprintf(stderr, "v_1>>1: %u, v_1&1: %u, ELen_1: %u\n", v_1>>1, v_1&1, (uint32_t)ELen_1);
-	// 	fprintf(stderr, "min_count: %u, max_count: %u, DIFF_HAP_RATE: %f\n\n", 
-	// 	min_count, max_count, DIFF_HAP_RATE);
-	// }
-	
-	if(min_count == 0) return UNAVAILABLE;
-	if(max_count > min_count*DIFF_HAP_RATE) return PLOID;
-	return NON_PLOID;
-}
-
-inline uint32_t check_different_haps_naive(asg_t *nsg, ma_ug_t *ug, asg_t *read_sg, 
-uint32_t v_0, uint32_t v_1, ma_hit_t_alloc* reverse_sources, buf_t* b_0, buf_t* b_1, 
-R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold)
-{
-	uint32_t vEnd, qn, tn, j, is_Unitig;
-	long long ELen_0, ELen_1, tmp, max_stop_nodeLen, max_stop_baseLen;
-
-	b_0->b.n = b_1->b.n = 0;
-	if(get_unitig(nsg, ug, v_0, &vEnd, &ELen_0, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
-	stops_threshold, b_0) == LOOP)
-	{
-		return UNAVAILABLE;
-	} 
-	if(get_unitig(nsg, ug, v_1, &vEnd, &ELen_1, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
-	stops_threshold, b_1) == LOOP)
-	{
-		return UNAVAILABLE;
-	}
-
-	if(ELen_0<=min_edge_length || ELen_1<=min_edge_length) return UNAVAILABLE;
-
-	rIdContig b_max, b_min;
-	b_max.b_0 = b_min.b_0 = NULL;
-	b_max.offset = b_max.readI = b_max.untigI = 0;
-	b_min.offset = b_min.readI = b_min.untigI = 0;
-
-	if(ELen_0<=ELen_1)
-	{
-        b_min.b_0 = b_0;
-        b_max.b_0 = b_1;
-    }
-    else
-    {
-        b_min.b_0 = b_1;
-        b_max.b_0 = b_0;
-    }
-
-	uint32_t max_count = 0, min_count = 0;
-	ma_utg_t *node_min = NULL, *node_max = NULL;
-
-	if(ug != NULL)
-	{
-		///each unitig
-		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
-		{
-			
-			node_min = &(ug->u.a[(b_min.b_0->b.a[b_min.untigI]>>1)]);
-
-			///each read
-			for (b_min.readI = 0; b_min.readI < node_min->n; b_min.readI++)
-        	{
-				qn = node_min->a[b_min.readI]>>33;
-
-				if(reverse_sources[qn].length > 0) min_count++;
-				for (j = 0; j < (long long)reverse_sources[qn].length; j++)
-				{
-					tn = Get_tn(reverse_sources[qn].buffer[j]);
-					if(read_sg->seq[tn].del == 1)
-					{
-						get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
-						if(tn == (uint32_t)-1 || is_Unitig == 1 || read_sg->seq[tn].del == 1) continue;
-					}
-
-					///each unitig
-					for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
-					{
-						node_max = &(ug->u.a[b_max.b_0->b.a[b_max.untigI]>>1]);
-						///each read
-						for (b_max.readI = 0; b_max.readI < node_max->n; b_max.readI++)
-						{
-							if(tn == (node_max->a[b_max.readI]>>33))
-							{
-								max_count++;
-								goto end_check_different_haps_ug;
-							}
-						}
-					}
-				}
-
-				end_check_different_haps_ug:;
-			}
-		}
-	}
-	else
-	{
-		///each read
-		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
-		{
-			qn = (b_min.b_0->b.a[b_min.untigI]>>1);
-
-			if(reverse_sources[qn].length > 0) min_count++;
-
-			for (j = 0; j < (long long)reverse_sources[qn].length; j++)
-        	{
-            	tn = Get_tn(reverse_sources[qn].buffer[j]);
-				if(nsg->seq[tn].del == 1)
-				{
-					get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
-					if(tn == (uint32_t)-1 || is_Unitig == 1 || nsg->seq[tn].del == 1) continue;
-				}
-
-				///each read
-				for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
-                {
-					if((b_max.b_0->b.a[b_max.untigI]>>1) == tn)
-					{
-						max_count++;
-						goto end_check_different_haps_non_ug;
-					}
-				}
-			}
-
-			end_check_different_haps_non_ug:;
-		}
-	}
-
-	// if(((v_0==7707) && (v_1==26867))||((v_1==7707) && (v_0==26867)))
-	// {
-	// 	fprintf(stderr, "******\nv_0>>1: %u, v_0&1: %u, ELen_0: %u\n", v_0>>1, v_0&1, (uint32_t)ELen_0);
-	// 	fprintf(stderr, "v_1>>1: %u, v_1&1: %u, ELen_1: %u\n", v_1>>1, v_1&1, (uint32_t)ELen_1);
-	// 	fprintf(stderr, "min_count: %u, max_count: %u, DIFF_HAP_RATE: %f\n\n", 
-	// 	min_count, max_count, DIFF_HAP_RATE);
-	// }
-	
-	if(min_count == 0) return UNAVAILABLE;
-	if(max_count > min_count*DIFF_HAP_RATE) return PLOID;
-	return NON_PLOID;
-}
-
-
+#define HET_HOM_RATE 0.7
 
 typedef struct {
 	uint32_t father_occ;
@@ -1019,36 +832,61 @@ typedef struct {
 	uint32_t total;
 } Trio_counter;
 
+typedef struct {
+	uint32_t p; // the optimal parent vertex
+	uint32_t d; // the shortest distance from the initial vertex
+	uint32_t r:31, s:1; // r: the number of remaining incoming arc; s: state
+} binfo_s_t;
+
+typedef struct {
+	///all information for each node
+	binfo_s_t *a;
+	kvec_t(uint32_t) S; // set of vertices without parents, nodes with all incoming edges visited
+	kvec_t(uint32_t) b; // visited vertices
+	kvec_t(uint32_t) e; // visited edges/arcs
+} buf_s_t;
+
+typedef struct{
+    buf_s_t *b;
+    uint32_t n_thres, n_reads;
+    asg_t *g;
+    uint32_t check_cross;
+	uint64_t bub_dist;
+} bub_label_t;
+
 void resolve_tangles(ma_ug_t *src, asg_t *read_g, ma_hit_t_alloc* reverse_sources, long long minLongUntig, 
-long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, uint32_t trio_flag, 
-float drop_ratio);
-void adjust_utg_advance(asg_t *sg, ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex);
+long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, uint8_t* is_r_het, 
+uint32_t trio_flag, float drop_ratio);
+void adjust_utg_advance(asg_t *sg, ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, bub_label_t* b_mask_t, uint8_t* is_r_het);
 void rescue_contained_reads_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut,
-R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t chainLenThres, uint32_t is_bubble_check, 
-uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes);
+R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t is_bubble_check, 
+uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t);
 void rescue_missing_overlaps_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut,
-R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t is_bubble_check, 
-uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges);
+R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t);
 void all_to_all_deduplicate(ma_ug_t* ug, asg_t* read_g, ma_sub_t* coverage_cut, 
-ma_hit_t_alloc* sources, uint8_t postive_flag, float drop_rate, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, float double_check_rate);
-void drop_semi_circle(ma_ug_t *ug, asg_t* nsg, asg_t* read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex);
+ma_hit_t_alloc* sources, uint8_t postive_flag, float drop_rate, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, uint8_t* is_r_het, float double_check_rate, int non_tig_occ);
+void drop_semi_circle(ma_ug_t *ug, asg_t* nsg, asg_t* read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, uint8_t* is_r_het);
 void rescue_wrong_overlaps_to_unitigs(ma_ug_t *i_ug, asg_t *r_g,  ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, 
-ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, kvec_asg_arc_t_warp* keep_edges);
+ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, kvec_asg_arc_t_warp* keep_edges, bub_label_t* b_mask_t);
 void get_unitig_trio_flag(ma_utg_t* nsu, uint32_t flag, uint32_t* require, uint32_t* non_require, uint32_t* ambigious);
 void rescue_missing_overlaps_backward(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut,
-R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t backward_steps, 
-uint32_t is_bubble_check, uint32_t is_primary_check);
+R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t);
 uint32_t get_edge_from_source(ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, 
 R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t target, asg_arc_t* t);
-uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, 
-uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes);
 int unitig_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio);
-
+void asg_bub_backtrack_primary(asg_t *g, uint32_t v0, buf_t *b);
+void set_hom_global_coverage(hifiasm_opt_t *opt, asg_t *sg, ma_sub_t* coverage_cut, 
+ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, int max_hang, int min_ovlp);
+void rescue_bubble_by_chain(asg_t *sg, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, 
+long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, 
+float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, uint32_t chainLenThres, long long gap_fuzz, 
+bub_label_t* b_mask_t, long long no_trio_recover);
 
 typedef struct{
     double weight;
-    uint32_t uID:31, del:1;
+    uint32_t uID;
     uint64_t dis;
+	uint8_t is_cc:7, del:1;
 	uint64_t occ;
 	///uint64_t occ:63, scaff:1;
     ///uint32_t enzyme;
@@ -1071,35 +909,124 @@ typedef struct{
 typedef struct{
     kvec_t(hc_linkeage) a;
     kvec_t(uint64_t) enzymes;
-	kvec_t(bed_in) bed;
-    uint32_t* u_idx;
-	uint64_t r_num;
 } hc_links;
 
+#define N_HET 0
+#define C_HET 1
+#define P_HET 2
+#define S_HET 4
+
+typedef struct {
+	uint32_t p_x_p, p_y_p, p_x, p_y;
+	uint32_t c_x_p, c_y_p;
+	uint8_t c_rev;
+} ca_buf_t;
+
+typedef struct {
+	size_t n, m;
+	ca_buf_t* a;
+} kv_ca_buf_t;
+
+typedef struct {
+	kvec_t(uint32_t) uIDs;
+	kvec_t(uint32_t) iDXs;
+	uint32_t chain_num;
+} sub_tran_t;
+
+typedef struct{
+	uint32_t* rUidx;
+	uint64_t* rUpos;
+	uint8_t* ir_het;
+	uint32_t r_num, u_num;
+	kvec_t(bed_in) bed;
+	kvec_t(uint32_t) topo_buf;
+	kvec_t(uint32_t) topo_res;
+	buf_t b_buf_0, b_buf_1;
+	///uint32_t* uLen;
+	kv_u_trans_t k_trans;
+	kv_u_trans_hit_t k_t_b;
+	kv_ca_buf_t c_buf;
+	sub_tran_t st;
+}trans_chain;
+
+typedef struct {
+    uint32_t n;
+    uint32_t* cov;
+    uint64_t* pos_idx;
+    ma_hit_t_alloc* reverse_sources;
+    ma_sub_t *coverage_cut;
+    R_to_U* ruIndex;
+	asg_t *read_g;
+    int max_hang;
+    int min_ovlp;
+    kvec_asg_arc_t_offset u_buffer; 
+    kvec_t_i32_warp tailIndex;
+    kvec_t_i32_warp prevIndex;
+	uint8_t* is_r_het;
+	trans_chain* t_ch;
+}hap_cov_t;
+
 typedef struct{
     ///kvec_t(hc_edge) a;
     size_t n, m; 
     hc_edge *a;
 }hc_edge_warp;
 
-void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num);
+typedef struct {
+	uint32_t qs, qe, qn, qus, que;
+	uint32_t ts, te, tn, tus, tue;
+} utg_thit_t;
+
+typedef struct {
+	size_t n, m;
+	utg_thit_t* a;
+} kv_utg_thit_t_t;
+
+typedef struct {
+	ma_hit_t_alloc* reverse_sources;
+    ma_sub_t *coverage_cut;
+    R_to_U* ruIndex;
+    asg_t *read_g;	
+	kvec_asg_arc_t_offset u_buffer; 
+    kvec_t_i32_warp tailIndex;
+    kvec_t_i32_warp prevIndex;
+	kv_utg_thit_t_t k_t_b;
+	kv_ca_buf_t c_buf;
+	kv_u_trans_t k_trans;
+	uint64_t *pos_idx, rn;
+	kvec_t(uint32_t) topo_res;
+	ma_ug_t *cug;
+	int max_hang;
+	int min_ovlp;
+
+	ma_utg_v u;
+	kv_u_trans_t t;
+	buf_t b0, b1;
+} utg_trans_t;
+
+typedef struct {
+	ma_ug_t *ug;
+	kvec_t(uint64_t) idx;
+	kvec_t(uint32_t) dst;
+} spg_t;
+
+void init_hc_links(hc_links* link, uint64_t ug_num, trans_chain* t_ch);
 void destory_hc_links(hc_links* link);
-void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources,
-long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
-R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, 
-uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, 
-float drop_ratio, hc_links* link);
+uint64_t get_bub_pop_max_dist(asg_t *g, buf_t *b);
+uint64_t get_bub_pop_max_dist_advance(asg_t *g, buf_t *b);
+uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, uint64_t max_dist, buf_t *b, uint32_t positive_flag, 
+uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, hap_cov_t *cov, uint32_t is_update_chain, uint32_t keep_d, utg_trans_t *o);
+
 void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate,
 ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, 
-long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
+long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
 R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp,
-kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link);
-void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg);
+kvec_asg_arc_t_warp* new_rtg_edges, hap_cov_t **i_cov, bub_label_t* b_mask_t, uint32_t collect_p_trans, uint32_t collect_p_trans_f);
 ma_ug_t* copy_untig_graph(ma_ug_t *src);
 ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, 
-uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, 
+uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, 
 long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, 
-float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench);
+float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, bub_label_t* b_mask_t, char *f_prefix, uint8_t *kpt_buf, kvec_asg_arc_t_warp *r_edges);
 asg_t* copy_read_graph(asg_t *src);
 ma_ug_t *ma_ug_gen(asg_t *g);
 void ma_ug_destroy(ma_ug_t *ug);
@@ -1112,6 +1039,129 @@ inline int inter_interval(int a_s, int a_e, int b_s, int b_e, int* i_s, int* i_e
     return 1;
 }
 
+inline uint32_t get_origin_uid(uint32_t v, trans_chain* t_ch, uint32_t *off, uint32_t *idx)
+{
+	if(off) (*off) = (t_ch->rUpos[v>>1]>>32);
+	if(idx) (*idx) = (uint32_t)(t_ch->rUpos[v>>1]);
+    if(t_ch->rUpos[v>>1] == (uint64_t)-1) return (uint32_t)-1;
+    return (uint32_t)(((t_ch->rUidx[v>>1]>>1)<<1) + ((t_ch->rUidx[v>>1]^v)&1));
+}
+void chain_origin_trans_uid_by_distance(hap_cov_t *cov, asg_t *read_sg, 
+uint32_t *pri_a, uint32_t pri_n, uint32_t pri_beg, uint64_t *i_pri_len, 
+uint32_t *aux_a, uint32_t aux_n, uint32_t aux_beg, uint64_t *i_aux_len,
+ma_ug_t *ug, uint32_t flag, double overall_score, const char* cmd);
+int asg_arc_del_trans(asg_t *g, int fuzz);
+void kt_u_trans_t_idx(kv_u_trans_t *ta, uint32_t n);
+void kt_u_trans_t_simple_symm(kv_u_trans_t *ta, uint32_t un, uint32_t symm_add);
+uint32_t get_u_trans_spec(kv_u_trans_t *ta, uint32_t qn, uint32_t tn, u_trans_t **r_a, uint32_t *occ);
+int ma_ug_seq(ma_ug_t *g, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, 
+kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, kvec_asg_arc_t_warp *E, uint32_t is_polish);
+
+
+typedef struct{
+    ma_sub_t* coverage_cut;
+    ma_hit_t_alloc* sources; 
+    ma_hit_t_alloc* reverse_sources; 
+    long long tipsLen; 
+    float tip_drop_ratio; 
+    long long stops_threshold; 
+    R_to_U* ruIndex; 
+    float chimeric_rate;
+    float drop_ratio; 
+    int max_hang;
+    int min_ovlp;
+    int is_bench;
+	long long gap_fuzz;
+	int64_t min_dp;
+    bub_label_t* b_mask_t;
+	uint64_t* readLen;
+}ug_opt_t;
+
+typedef struct{
+	ma_hit_t_alloc **src;
+	ma_hit_t_alloc **r_src;
+	long long *n_read;
+	uint64_t **readLen;
+	asg_t **sg;
+	R_to_U *ruIndex;
+	ma_sub_t **cov;
+	bub_label_t *b_mask_t;
+	int64_t max_hang;
+	int64_t mini_ovlp;
+}ul_renew_t;
+
+
+void adjust_utg_by_trio(ma_ug_t **ug, asg_t* read_g, uint8_t flag, float drop_rate,
+ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, 
+long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
+R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp,
+kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t);
+uint32_t cmp_untig_graph(ma_ug_t *src, ma_ug_t *dest);
+void reduce_hamming_error(asg_t *sg, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, 
+int max_hang, int min_ovlp, long long gap_fuzz);
+int ma_ug_seq_scaffold(ma_ug_t *g, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, 
+kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, kvec_asg_arc_t_warp *E, uint32_t is_polish);
+void ma_ug_print(const ma_ug_t *ug, asg_t* read_g, const ma_sub_t *coverage_cut, 
+ma_hit_t_alloc* sources, R_to_U* ruIndex, const char* prefix, FILE *fp);
+void ma_ug_print_simple(const ma_ug_t *ug, asg_t* read_g, const ma_sub_t *coverage_cut, 
+ma_hit_t_alloc* sources, R_to_U* ruIndex, const char* prefix, FILE *fp);
+trans_chain* init_trans_chain(ma_ug_t *ug, uint64_t r_num);
+void destory_trans_chain(trans_chain **x);
+
+typedef struct {///[cBeg, cEnd)
+	uint32_t u_i, r_i, len, s_pos_cur, s_pre_v, s_pre_w, p_v, p_idx, p_uId, cBeg, cEnd;
+    ///buf_t* x;
+    uint32_t *a, an;
+    ma_ug_t *ug;
+    asg_t *read_sg;
+    trans_chain* t_ch;
+} u_trans_hit_idx;
+void reset_u_trans_hit_idx(u_trans_hit_idx *t, uint32_t* i_x_a, uint32_t i_x_n, ma_ug_t *i_ug, 
+asg_t *i_read_sg, trans_chain* i_t_ch, uint32_t i_cBeg, uint32_t i_cEnd);
+uint32_t get_u_trans_hit(u_trans_hit_idx *t, u_trans_hit_t *hit);
+inline uint32_t get_offset_adjust(uint32_t offset, uint32_t offsetLen, uint32_t targetLen)
+{
+    return ((double)(offset)/(double)(offsetLen))*targetLen;
+}
+
+uint32_t set_utg_offset(uint32_t *a, uint32_t a_n, ma_ug_t *ug, asg_t *read_sg, uint64_t* pos_idx, uint32_t is_clear,
+uint32_t only_len);
+uint64_t get_utg_cov(ma_ug_t *ug, uint32_t uID, asg_t* read_g, 
+const ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, R_to_U* ruIndex, uint8_t* r_flag);
+trans_chain* load_hc_trans(const char *fn);
+char *get_outfile_name(char* output_file_name);
+void reset_u_trans_hit_idx(u_trans_hit_idx *t, uint32_t* i_x_a, uint32_t i_x_n, ma_ug_t *i_ug, 
+asg_t *i_read_sg, trans_chain* i_t_ch, uint32_t i_cBeg, uint32_t i_cEnd);
+void extract_sub_overlaps(uint32_t i_tScur, uint32_t i_tEcur, uint32_t i_tSpre, uint32_t i_tEpre,
+uint32_t tn, kv_u_trans_hit_t* ktb, uint32_t bn);
+void clean_u_trans_t_idx(kv_u_trans_t *ta, ma_ug_t *ug, asg_t *read_g);
+uint32_t test_dbug(ma_ug_t* ug, FILE* fp);
+void write_dbug(ma_ug_t* ug, FILE* fp);
+int asg_arc_identify_simple_bubbles_multi(asg_t *g, bub_label_t* x, int check_cross);
+uint8_t get_tip_trio_infor(asg_t *sg, uint32_t begNode);
+int asg_topocut_aux(asg_t *g, uint32_t v, int max_ext);
+int asg_arc_del_triangular_directly(asg_t *g, long long min_edge_length, 
+ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex);
+int asg_arc_del_short_diploid_by_exact(asg_t *g, int max_ext, ma_hit_t_alloc* sources);
+uint32_t print_debug_gfa(asg_t *read_g, ma_ug_t *ug, ma_sub_t* coverage_cut, const char* output_file_name, 
+ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp, int is_update_ou, int is_check_alter_lable, int is_seq);
+void debug_info_of_specfic_node(const char* name, asg_t *g, R_to_U* ruIndex, const char* command);
+ma_ug_t *gen_polished_ug(const ug_opt_t *uopt, asg_t *sg);
+void output_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, 
+ma_hit_t_alloc* sources, R_to_U* ruIndex, int max_hang, int min_ovlp);
+void flat_soma_v(asg_t *sg, ma_hit_t_alloc* sources, R_to_U* ruIndex);
+void hic_clean(asg_t* read_g);
+int64_t count_edges_v_w(asg_t *g, uint32_t v, uint32_t w);
+void renew_utg(ma_ug_t **ug, asg_t* read_g, kvec_asg_arc_t_warp* edge);
+void merge_unitig_content(ma_utg_t* collection, ma_ug_t* ug, asg_t* read_g, kvec_asg_arc_t_warp* edge);
+void reset_bub_label_t(bub_label_t* x, asg_t *g, uint64_t bub_dist, uint32_t check_cross);
+void set_reverse_overlap(ma_hit_t* dest, ma_hit_t* source);
+// void break_ug_contig(ma_ug_t **ug, asg_t *read_g, All_reads *RNF, ma_sub_t *coverage_cut,
+// ma_hit_t_alloc* sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, 
+// int* b_low_cov, int* b_high_cov, double m_rate);
+void ma_hit_contained_advance(ma_hit_t_alloc* sources, long long n_read, ma_sub_t *coverage_cut, 
+R_to_U* ruIndex, int max_hang, int min_ovlp);
+
 #define JUNK_COV 5
 #define DISCARD_RATE 0.8
 

POA.cpp

@@ -307,66 +307,53 @@ void addUnmatchedSeqToGraph(Graph* g, char* g_read_seq, long long g_read_length,
 }
 
 void addmatchedSeqToGraph(Graph* backbone, long long currentNodeID, char* x_string, long long x_length, 
-        char* y_string, long long y_length, CIGAR* cigar, long long backbone_start, long long backbone_end)
+        char* y_string, long long y_length, window_list *cigar_idx, window_list_alloc *cigar_s, long long backbone_start, long long backbone_end)
 {
     
-    int x_i, y_i, cigar_i;
-    x_i = 0;
-    y_i = 0;
-    cigar_i = 0;
-    int operation;
-    int operationLen;
-    int i;
-    int last_operation = -1;
-
+    int64_t x_i = 0, y_i = 0, c_i = 0, c_n = cigar_idx->clen;
+    uint32_t i, operLen; uint8_t oper; int8_t last_oper = -1;
+    // if(currentNodeID == 366 && x_length == 9 && y_length == 9) {
+    //     fprintf(stderr, "[M::%s] currentNodeID::%lld, x_length::%lld, c_n::%ld, cidx::%u, cigar_s_n::%lld\n", __func__, 
+    //     currentNodeID, x_length, c_n, cigar_idx->cidx, (long long)cigar_s->c.n);
+    // }
     
     ///note that node 0 is the start node
     ///0 is match, 1 is mismatch, 2 is up, 3 is left
     ///2 mean y has more bases, while 3 means x has more bases
-    while (cigar_i < cigar->length)
-    {
-        operation = cigar->C_C[cigar_i];
-        operationLen = cigar->C_L[cigar_i];
+    for (c_i = 0; c_i < c_n; c_i++) {
+        get_cigar_cell(cigar_idx, cigar_s, c_i, &oper, &operLen);
 
-        ///match/mismatch
-        if (operation == 0 || operation == 1)
-        {
-            
-            for (i = 0; i < operationLen; i++)
-            {
+        // if(currentNodeID == 366 && x_length == 9 && y_length == 9) {
+        //     fprintf(stderr, "[M::%s] c_i::%ld, oper::%u, operLen::%u, last_oper::%d\n", __func__, c_i, oper, operLen, last_oper);
+        // }
+        
+        if (oper == 0 || oper == 1) { ///match/mismatch
+            for (i = 0; i < operLen; i++) {
                 ///if the previous node is insertion, this node might be mismatch/match
-                add_mismatchEdge_weight(backbone, currentNodeID, y_string[y_i], last_operation);    
-                x_i++;
-                y_i++;
-                currentNodeID++;
+                add_mismatchEdge_weight(backbone, currentNodeID, y_string[y_i], last_oper);    
+                x_i++; y_i++; currentNodeID++;
             }
-        }///insertion
-        else if (operation == 2)
-        {
+        } else if (oper == 2) { ///insertion
             ///the begin and end of cigar cannot be 2, so -1 is right here
             ///if (operationLen <= CORRECT_INDEL_LENGTH)
             {
-                add_insertionEdge_weight(backbone, currentNodeID, y_string + y_i, operationLen);
+                add_insertionEdge_weight(backbone, currentNodeID, y_string + y_i, operLen);
                 backbone->g_nodes.list[currentNodeID].num_insertions++;
             }
-            y_i += operationLen;
-        }
-        else if (operation == 3)
-        {
+            y_i += operLen;
+        } else if (oper == 3) {
             ///3 means x has more bases, that means backbone has more bases
             ///like a mismatch (-)
             ///if (operationLen <= CORRECT_INDEL_LENGTH)
             {
-                add_deletionEdge_weight(backbone, currentNodeID, operationLen);
+                add_deletionEdge_weight(backbone, currentNodeID, operLen);
             }
 
             
-            currentNodeID += operationLen;
-            x_i += operationLen;
+            currentNodeID += operLen;
+            x_i += operLen;
         }
 
-        last_operation = operation;
-        
-        cigar_i++;
+        last_oper = oper;
     }
 }

POA.h

@@ -431,7 +431,7 @@ uint64_t* get_Topo_Sort_Order(Node_alloc* list, int need_sort);
 void init_Graph(Graph* g);
 void addUnmatchedSeqToGraph(Graph* g, char* g_read_seq, long long g_read_length, long long* startID, long long* endID);
 void addmatchedSeqToGraph(Graph* backbone, long long currentNodeID, char* x_string, long long x_length, 
-        char* y_string, long long y_length, CIGAR* cigar, long long backbone_start, long long backbone_end);
+        char* y_string, long long y_length, window_list *cigar_idx, window_list_alloc *cigar_s, long long backbone_start, long long backbone_end);
 void destory_Graph(Graph* g);
 void clear_Graph(Graph* g);
 void Perform_POA(Graph* g, overlap_region_alloc* overlap_list, All_reads* R_INF, UC_Read* g_read);

Process_Read.h

@@ -29,7 +29,7 @@ extern uint8_t seq_nt6_table[256];
 extern char bit_t_seq_table[256][4];
 extern char bit_t_seq_table_rc[256][4];
 extern char s_H[5];
-extern char rc_Table[5];
+extern char rc_Table[6];
 
 
 #define RC_CHAR(x) rc_Table[seq_nt6_table[(uint8_t)x]]
@@ -101,6 +101,9 @@ typedef struct
 #define MIX_TRIO 3
 #define NON_TRIO 4
 #define DROP 5
+#define SET_TRIO 8
+#define CHAIN_MATCH 1
+#define CHAIN_UNMATCH 0.334
 
 typedef struct
 {
@@ -153,6 +156,55 @@ typedef struct
     pthread_mutex_t OutputMutex;
 } Debug_reads;
 
+
+typedef struct
+{
+    uint32_t hid;
+    uint32_t qs, qe, ts, te; uint32_t pidx, pdis, aidx;///TODO: enable pdis
+    uint8_t pchain:5, rev:1, base:1, el:1;
+} uc_block_t;
+
+typedef struct
+{
+    uint32_t *a;
+    size_t n, m;
+} N_t;
+
+typedef struct
+{
+    char *a; uint32_t n;
+} nid_t;
+
+typedef struct
+{
+    kvec_t(uint8_t) r_base;
+    uint32_t rlen; 
+
+    kvec_t(uc_block_t) bb;
+    N_t N_site;
+
+    uint8_t dd;
+} ul_vec_t;
+
+typedef struct{
+    kvec_t(uint32_t) idx;
+    kvec_t(uint64_t) occ;
+} ul_vec_rid_t;
+
+typedef struct
+{
+    kvec_t(nid_t) nid;
+    ul_vec_rid_t ridx;
+    ul_vec_t *a;
+    size_t n, m;
+    All_reads *hR;
+    // uint32_t mm;
+} all_ul_t;
+
+extern all_ul_t UL_INF;
+extern all_ul_t ULG_INF;
+// extern uint32_t *het_cnt;
+
 void init_All_reads(All_reads* r);
 void malloc_All_reads(All_reads* r);
 void ha_insert_read_len(All_reads *r, int read_len, int name_len);
@@ -160,7 +212,7 @@ void ha_compress_base(uint8_t* dest, char* src, uint64_t src_l, uint64_t** N_sit
 void init_UC_Read(UC_Read* r);
 void recover_UC_Read(UC_Read* r, const All_reads *R_INF, uint64_t ID);
 void recover_UC_Read_RC(UC_Read* r, All_reads* R_INF, uint64_t ID);
-void recover_UC_Read_sub_region(char* r, long long start_pos, long long length, uint8_t strand, All_reads* R_INF, long long ID);
+void recover_UC_Read_sub_region(char* r, int64_t start_pos, int64_t length, uint8_t strand, All_reads* R_INF, int64_t ID);
 void destory_UC_Read(UC_Read* r);
 void reverse_complement(char* pattern, uint64_t length);
 void write_All_reads(All_reads* r, char* read_file_name);
@@ -170,4 +222,18 @@ int destory_read_bin(All_reads* r);
 void init_Debug_reads(Debug_reads* x, const char* file);
 void destory_Debug_reads(Debug_reads* x);
 void recover_UC_sub_Read(UC_Read* i_r, long long start_pos, long long length, uint8_t strand, All_reads* R_INF, long long ID);
+
+void init_all_ul_t(all_ul_t *x, All_reads *hR);
+void destory_all_ul_t(all_ul_t *x);
+void append_ul_t(all_ul_t *x, uint64_t *rid, char* id, int64_t id_l, char* str, int64_t str_l, ul_ov_t *o, int64_t on, float p_chain_rate, const ug_opt_t *uopt, uint32_t save_bases);
+void retrieve_ul_t(UC_Read* i_r, char *i_s, all_ul_t *ref, uint64_t ID, uint8_t strand, int64_t s, int64_t l);
+void retrieve_u_seq(UC_Read* i_r, char* i_s, ma_utg_t *u, uint8_t strand, int64_t s, int64_t l, void *km);
+void debug_retrieve_rc_sub(const ug_opt_t *uopt, all_ul_t *ref, const All_reads *R_INF, ul_idx_t *ul, uint32_t n_step);
+uint32_t retrieve_u_cov(const ul_idx_t *ul, uint64_t id, uint8_t strand, uint64_t pos, uint8_t dir, int64_t *pi);
+uint64_t retrieve_u_cov_region(const ul_idx_t *ul, uint64_t id, uint8_t strand, uint64_t s, uint64_t e, int64_t *pi);
+uint64_t retrieve_r_cov_region(const ul_idx_t *ul, uint64_t id, uint8_t strand, uint64_t s, uint64_t e, int64_t *pi);
+void append_ul_t_back(all_ul_t *x, uint64_t *rid, char* id, int64_t id_l, char* str, int64_t str_l, ul_ov_t *o, int64_t on, float p_chain_rate);
+void write_compress_base_disk(FILE *fp, uint64_t ul_rid, char *str, uint32_t len, ul_vec_t *buf);
+int64_t load_compress_base_disk(FILE *fp, uint64_t *ul_rid, char *dest, uint32_t *len, ul_vec_t *buf);
+
 #endif

Purge_Dups.h

@@ -10,15 +10,85 @@
 #define HOM_PEAK_RATE 1.25
 #define HET_PEAK_RATE (HOM_PEAK_RATE*2)
 #define ALTER_COV_THRES 0.9
-#define REAL_ALTER_THRES 0.1
+#define REAL_ALTER_THRES 0.25
+#define CHAIN_FILTER_RATE 0.7
+#define REV_W 8
+
+#define SELF_EXIST 0
+#define REVE_EXIST 1
+#define DELETE 2
+#define MIXED 3
+#define FLIP 4
+
+#define X2Y 0
+#define Y2X 1
+#define XCY 2
+#define YCX 3
+
+#define Cal_Off(OFF) ((long long)((uint32_t)((OFF)>>32)) - (long long)((uint32_t)((OFF))))
+#define Get_xOff(OFF) ((long long)((uint32_t)((OFF)>>32)))
+#define Get_yOff(OFF) ((long long)((uint32_t)((OFF))))
+#define Get_match(x) ((x).weight)
+#define Get_total(x) ((x).index_beg)
+#define Get_type(x) ((x).index_end)
+#define Get_x_beg(x) ((x).x_beg_pos)
+#define Get_x_end(x) ((x).x_end_pos)
+#define Get_y_beg(x) ((x).y_beg_pos)
+#define Get_y_end(x) ((x).y_end_pos)
+#define Get_rev(x) ((x).rev)
+
+typedef struct {
+    uint8_t rev;
+    uint8_t type;
+    uint8_t status;
+    uint32_t x_beg_pos;
+    uint32_t x_end_pos;
+    uint32_t y_beg_pos;
+    uint32_t y_end_pos;
+    uint32_t x_beg_id;
+    uint32_t x_end_id;
+    uint32_t y_beg_id;
+    uint32_t y_end_id;
+    uint32_t xUid;
+    uint32_t yUid;
+    uint32_t weight;
+    long long score;
+    float s;
+}hap_overlaps;
+
+typedef struct {
+    kvec_t(hap_overlaps) a;
+}kvec_hap_overlaps;
+
+typedef struct {
+    kvec_hap_overlaps* x;
+    uint32_t num;
+}hap_overlaps_list;
 
 void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, 
 ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, 
-uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, 
-uint32_t just_contain, uint32_t just_coverage, hc_links* link);
+uint32_t purege_minLen, int max_hang, int min_ovlp, float drop_ratio, uint32_t just_contain, 
+uint32_t just_coverage, hap_cov_t *cov, uint32_t collect_p_trans, uint32_t collect_p_trans_f);
 void fill_unitig(uint64_t* buffer, uint32_t bufferLen, asg_t* read_g, kvec_asg_arc_t_warp* edge,
 uint32_t is_circle, uint64_t* rLen);
 void get_contig_length(ma_ug_t *ug, asg_t *g, uint64_t* primaryLen, uint64_t* alterLen);
 void enable_debug_mode(uint32_t mode);
+hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, 
+ma_sub_t *coverage_cut, int max_hang, int min_ovlp, uint32_t is_collect_trans);
+void destory_hap_cov_t(hap_cov_t **x);
+void chain_trans_ovlp(hap_cov_t *cov, utg_trans_t *o, ma_ug_t *ug, asg_t *read_sg, buf_t* xReads, uint32_t targetBaseLen, uint32_t* xEnd);
+int get_specific_hap_overlap(kvec_hap_overlaps* x, uint32_t qn, uint32_t tn);
+void set_reverse_hap_overlap(hap_overlaps* dest, hap_overlaps* source, uint32_t* types);
+void print_hap_paf(ma_ug_t *ug, hap_overlaps* ovlp);
+uint64_t get_xy_pos_by_pos(asg_t *read_g, asg_arc_t* t, uint32_t v_in_unitig, uint32_t w_in_unitig, 
+uint32_t v_in_pos, uint32_t w_in_pos, uint32_t xUnitigLen, uint32_t yUnitigLen, uint8_t* rev);
+void quick_LIS(asg_arc_t_offset* x, uint32_t n, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex);
+uint32_t classify_hap_overlap(long long xBeg, long long xEnd, long long xLen,
+long long yBeg, long long yEnd, long long yLen, long long* r_xBeg, long long* r_xEnd, 
+long long* r_yBeg, long long* r_yEnd);
+int cmp_hap_alignment_chaining(const void * a, const void * b);
+uint32_t classify_hap_overlap(long long xBeg, long long xEnd, long long xLen,
+long long yBeg, long long yEnd, long long yLen, long long* r_xBeg, long long* r_xEnd, 
+long long* r_yBeg, long long* r_yEnd);
 
 #endif

README.md

@@ -1,4 +1,4 @@
-## Getting Started
+## <a name="started"></a>Getting Started
 
 ```sh
 # Install hifiasm (requiring g++ and zlib)
@@ -8,34 +8,53 @@ cd hifiasm && make
 # Run on test data (use -f0 for small datasets)
 wget https://github.com/chhylp123/hifiasm/releases/download/v0.7/chr11-2M.fa.gz
 ./hifiasm -o test -t4 -f0 chr11-2M.fa.gz 2> test.log
-awk '/^S/{print ">"$2;print $3}' test.p_ctg.gfa > test.p_ctg.fa  # get primary contigs in FASTA
+awk '/^S/{print ">"$2;print $3}' test.bp.p_ctg.gfa > test.p_ctg.fa  # get primary contigs in FASTA
 
 # Assemble inbred/homozygous genomes (-l0 disables duplication purging)
 hifiasm -o CHM13.asm -t32 -l0 CHM13-HiFi.fa.gz 2> CHM13.asm.log
-# Assemble heterozygous with built-in duplication purging
+# Assemble heterozygous genomes with built-in duplication purging
 hifiasm -o HG002.asm -t32 HG002-file1.fq.gz HG002-file2.fq.gz
 
+# Hi-C phasing with paired-end short reads in two FASTQ files
+hifiasm -o HG002.asm --h1 read1.fq.gz --h2 read2.fq.gz HG002-HiFi.fq.gz
+
 # Trio binning assembly (requiring https://github.com/lh3/yak)
 yak count -b37 -t16 -o pat.yak <(cat pat_1.fq.gz pat_2.fq.gz) <(cat pat_1.fq.gz pat_2.fq.gz)
 yak count -b37 -t16 -o mat.yak <(cat mat_1.fq.gz mat_2.fq.gz) <(cat mat_1.fq.gz mat_2.fq.gz)
 hifiasm -o HG002.asm -t32 -1 pat.yak -2 mat.yak HG002-HiFi.fa.gz
 ```
+See [tutorial][tutorial] for more details. 
 
-## Introduction
+## Table of Contents
 
-Hifiasm is a fast haplotype-resolved de novo assembler for PacBio Hifi reads. 
-It can assemble a human genome in several hours and works with the California
-redwood genome, one of the most complex genomes sequenced so far. Hifiasm can
-produce primary/alternate assemblies of quality competitive with the best
-assemblers. It also introduces a new graph binning algorithm and achieves
-the best haplotype-resolved assembly given trio data.
+- [Getting Started](#started)
+- [Introduction](#intro)
+- [Why Hifiasm?](#why)
+- [Usage](#use)
+  - [Assembling HiFi reads without additional data types](#hifionly)
+  - [Hi-C integration](#hic)
+  - [Trio binning](#trio)
+  - [Ultra-long ONT integration](#ul)
+  - [Output files](#output)
+- [Results](#results)
+- [Getting Help](#help)
+- [Limitations](#limit)
+- [Citing Hifiasm](#cite)
 
-## Why Hifiasm?
+## <a name="intro"></a>Introduction
+
+Hifiasm is a fast haplotype-resolved de novo assembler for PacBio HiFi reads.
+It can assemble a human genome in several hours and assemble a ~30Gb California
+redwood genome in a few days. Hifiasm emits partially phased assemblies of
+quality competitive with the best assemblers. Given parental short reads or
+Hi-C data, it produces arguably the best haplotype-resolved assemblies so far.
+
+## <a name="why"></a>Why Hifiasm?
 
 * Hifiasm delivers high-quality assemblies. It tends to generate longer contigs
   and resolve more segmental duplications than other assemblers.
 
-* Given sequence reads from the parents, hifiasm can produce overall the best
+* Given Hi-C reads or short reads from the parents, hifiasm can produce overall the best
   haplotype-resolved assembly so far. It is the assembler of choice by the
   [Human Pangenome Project][hpp] for the first batch of samples.
 
@@ -47,13 +66,15 @@ the best haplotype-resolved assembly given trio data.
 * Hifiasm is fast. It can assemble a human genome in half a day and assemble a
   ~30Gb redwood genome in three days. No genome is too large for hifiasm.
 
-* Hifiasm is trivial to install and easy to use. It does not required python,
-  R or C++11 compilers and can be compiled into a single executable. The
+* Hifiasm is trivial to install and easy to use. It does not required Python,
+  R or C++11 compilers, and can be compiled into a single executable. The
   default setting works well with a variety of genomes.
 
 [hpp]: https://humanpangenome.org
 
-## Usage
+## <a name="use"></a>Usage
+
+### <a name="hifionly"></a>Assembling HiFi reads without additional data types
 
 A typical hifiasm command line looks like:
 ```sh
@@ -61,11 +82,21 @@ hifiasm -o NA12878.asm -t 32 NA12878.fq.gz
 ```
 where `NA12878.fq.gz` provides the input reads, `-t` sets the number of CPUs in
 use and `-o` specifies the prefix of output files. For this example, the
-primary contigs are written to `NA12878.asm.p_ctg.gfa` and alternate contigs to
-`NA12878.asm.a_ctg.gfa`. At the first run, hifiasm saves corrected reads and
+primary contigs are written to `NA12878.asm.bp.p_ctg.gfa`. 
+Since v0.15, hifiasm also produces two sets of
+partially phased contigs at `NA12878.asm.bp.hap?.p_ctg.gfa`. This pair of files
+can be thought to represent the two haplotypes in a diploid genome, though with
+occasional switch errors. The frequency of switches is determined by the
+heterozygosity of the input sample.
+
+At the first run, hifiasm saves corrected reads and
 overlaps to disk as `NA12878.asm.*.bin`. It reuses the saved results to avoid
 the time-consuming all-vs-all overlap calculation next time. You may specify
 `-i` to ignore precomputed overlaps and redo overlapping from raw reads.
+You can also dump error corrected reads in FASTA and read overlaps in PAF with
+```sh
+hifiasm -o NA12878.asm -t 32 --write-paf --write-ec /dev/null
+```
 
 Hifiasm purges haplotig duplications by default. For inbred or homozygous
 genomes, you may disable purging with option `-l0`. Old HiFi reads may contain
@@ -75,7 +106,27 @@ bloom filter which takes 16GB memory at the beginning. For genomes much larger
 than human, applying `-f38` or even `-f39` is preferred to save memory on k-mer
 counting.
 
-When parental short reads are available, hifiasm can generate a pair of
+### <a name="hic"></a>Hi-C integration
+
+Hifiasm can generate a pair of haplotype-resolved assemblies with paired-end
+Hi-C reads:
+```sh
+hifiasm -o NA12878.asm -t32 --h1 read1.fq.gz --h2 read2.fq.gz HiFi-reads.fq.gz
+```
+In this mode, each contig is supposed to be a haplotig, which by definition
+comes from one parental haplotype only. Hifiasm often puts all contigs from the
+same parental chromosome in one assembly. It has cleanly separated chrX and
+chrY for a human male dataset. Nonetheless, phasing across centromeres is
+challenging. Hifiasm is often able to phase entire chromosomes but it may fail 
+in rare cases. Also, contigs from different parental chromosomes are randomly mixed as
+it is just not possible to phase across chromosomes with Hi-C.
+
+Hifiasm does not perform scaffolding for now. You need to run a standalone
+scaffolder such as SALSA or 3D-DNA to scaffold phased haplotigs.
+
+### <a name="trio"></a>Trio binning
+
+When parental short reads are available, hifiasm can also generate a pair of
 haplotype-resolved assemblies with trio binning. To perform such assembly, you
 need to count k-mers first with [yak][yak] first and then do assembly:
 ```sh
@@ -83,52 +134,34 @@ yak count -k31 -b37 -t16 -o pat.yak paternal.fq.gz
 yak count -k31 -b37 -t16 -o mat.yak maternal.fq.gz
 hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak NA12878.fq.gz
 ```
-Here `NA12878.asm.hap1.p_ctg.gfa` and `NA12878.asm.hap2.p_ctg.gfa` give the two
+Here `NA12878.asm.dip.hap1.p_ctg.gfa` and `NA12878.asm.dip.hap2.p_ctg.gfa` give the two
 haplotype assemblies. In the binning mode, hifiasm does not purge haplotig
-duplications by default. Because hifiasm reuses saved overlaps, you can
+duplicates by default. Because hifiasm reuses saved overlaps, you can
 generate both primary/alternate assemblies and trio binning assemblies with
 ```sh
 hifiasm -o NA12878.asm -t 32 NA12878.fq.gz 2> NA12878.asm.pri.log
 hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak /dev/null 2> NA12878.asm.trio.log
 ```
-The second command line will run much faster than the first. You can also dump
-error corrected in FASTA and/or overlaps in PAF with
+The second command line will run much faster than the first.
+
+### <a name="ul"></a>Ultra-long ONT integration
+
+Hifiasm could integrate ultra-long ONT reads to improve the assembly quality:
 ```sh
-hifiasm -o NA12878.asm -t 32 --write-paf --write-ec /dev/null
+hifiasm -o NA12878.asm -t32 --ul ul.fq.gz HiFi-reads.fq.gz
 ```
+Please note that this mode is not stable right now. We have only tested with >=100kb UL reads.
 
-## Output files
+### <a name="output"></a>Output files
 
-For non-trio assembly, hifiasm generates the following files:
-
-1. Haplotype-resolved raw [unitig][unitig] graph in [GFA][gfa] format
-   (*prefix*.r\_utg.gfa). This graph keeps all haplotype information, including
-   somatic mutations and recurrent sequencing errors.
-2. Haplotype-resolved processed unitig graph without small bubbles
-   (*prefix*.p\_utg.gfa). Small bubbles might be caused by somatic mutations or noise in data, 
-   which are not the real haplotype information.
-3. Primary assembly [contig][unitig] graph (*prefix*.p\_ctg.gfa). This graph collapses different
-   haplotypes.
-4. Alternate assembly contig graph (*prefix*.a\_ctg.gfa). This graph consists of all assemblies that
-   are discarded in primary contig graph.
-
-For trio assembly, hifiasm generates the following files:
-
-1. Haplotype-resolved raw [unitig][unitig] graph in [GFA][gfa] format
-   (*prefix*.r\_utg.gfa). This graph keeps all haplotype information. 
-
-2. Phased paternal/haplotype1 contig graph (*prefix*.hap1.p\_ctg.gfa). This graph keeps the phased
-   paternal/haplotype1 assembly.
-
-3. Phased maternal/haplotype2 contig graph (*prefix*.hap2.p\_ctg.gfa). This graph keeps the phased
-   maternal/haplotype2 assembly.
-
-Hifiasm writes error corrected reads to the *prefix*.ec.bin binary file and
+Hifiasm generates different types of assemblies based on the input data. 
+It also writes error corrected reads to the *prefix*.ec.bin binary file and
 writes overlaps to *prefix*.ovlp.source.bin and *prefix*.ovlp.reverse.bin.
+For more details, please see the complete [documentation][tutorial_output].
 
-## Results
+## <a name="results"></a>Results
 
-The following table shows the statistics of several hifiasm primary assemblies:
+The following table shows the statistics of several hifiasm primary assemblies assembled with v0.12:
 
 |<sub>Dataset<sub>|<sub>Size<sub>|<sub>Cov.<sub>|<sub>Asm options<sub>|<sub>CPU time<sub>|<sub>Wall time<sub>|<sub>RAM<sub>|<sub> N50<sub>|
 |:---------------|-----:|-----:|:---------------------|-------:|--------:|----:|----------------:|
@@ -155,7 +188,10 @@ redwood genome in a few days on a single machine. For trio binning assembly:
 |:---------------|-----:|-------:|--------:|----:|----------------:|
 |<sub>[HG00733][HG00733-data], [\[father\]][HG00731-data], [\[mother\]][HG00732-data]</sub>|<sub>&times;33</sub>|<sub>269.1h</sub>|<sub>6.9h</sub>|<sub>135G</sub>|<sub>35.1Mb (paternal), 34.9Mb (maternal)</sub>|
 |<sub>[HG002][NA24385-data],   [\[father\]][NA24149-data], [\[mother\]][NA24143-data]</sup>|<sub>&times;36</sub>|<sub>305.4h</sub>|<sub>7.7h</sub>|<sub>137G</sub>|<sub>41.0Mb (paternal), 40.8Mb (maternal)</sub>|
+
+<!--
 |<sub>[NA12878][NA12878-data], [\[father\]][NA12891-data], [\[mother\]][NA12892-data]</sub>|<sub>&times;30</sub>|<sub>180.8h</sub>|<sub>4.9h</sub>|<sub>123G</sub>|<sub>27.7Mb (paternal), 27.0Mb (maternal)</sub>|
+-->
 
 [HG00733-data]: https://www.ebi.ac.uk/ena/data/view/ERX3831682
 [HG00731-data]: https://www.ebi.ac.uk/ena/data/view/ERR3241754
@@ -167,29 +203,40 @@ redwood genome in a few days on a single machine. For trio binning assembly:
 [NA12891-data]: https://www.ebi.ac.uk/ena/data/view/ERR194160
 [NA12892-data]: https://www.ebi.ac.uk/ena/data/view/ERR194161
 
-Except NA12878, the assemblies above were produced by hifiasm v0.12 and can be
-downloaded at
-```txt
-ftp://ftp.dfci.harvard.edu/pub/hli/hifiasm/submission/hifiasm-0.12/
-```
-NA12878 was assembled with an older version of hifiasm and is available at
-```txt
-ftp://ftp.dfci.harvard.edu/pub/hli/hifiasm/NA12878-r253/
-```
-
+Human assemblies above can be acquired [from Zenodo][zenodo-human] and
+non-human ones are available [here][zenodo-nonh].
 
+[zenodo-human]: https://zenodo.org/record/4393631
+[zenodo-nonh]: https://zenodo.org/record/4393750
 [unitig]: http://wgs-assembler.sourceforge.net/wiki/index.php/Celera_Assembler_Terminology
 [gfa]: https://github.com/pmelsted/GFA-spec/blob/master/GFA-spec.md
 [paf]: https://github.com/lh3/miniasm/blob/master/PAF.md
 [yak]: https://github.com/lh3/yak
+[tutorial]: https://hifiasm.readthedocs.io/en/latest/index.html
+[tutorial_output]: https://hifiasm.readthedocs.io/en/latest/interpreting-output.html#interpreting-output
 
-## Getting Help
 
-For detailed description of options, please see `man ./hifiasm.1`. The `-h`
+## <a name="help"></a>Getting Help
+
+For detailed description of options, please see [tutorial][tutorial] or `man ./hifiasm.1`. The `-h`
 option of hifiasm also provides brief description of options. If you have
 further questions, please raise an issue at the [issue
 page](https://github.com/chhylp123/hifiasm/issues).
 
-## Limitations
+## <a name="limit"></a>Limitations
 
-1. Purging haplotig duplications may introduce misassemblies.
+1. Purging haplotig duplications may introduce misassemblies.
+
+## <a name="cite"></a>Citating Hifiasm
+
+If you use hifiasm in your work, please cite:
+
+> Cheng, H., Concepcion, G.T., Feng, X., Zhang, H., Li H. (2021)
+> Haplotype-resolved de novo assembly using phased assembly graphs with
+> hifiasm. *Nat Methods*, **18**:170-175.
+> https://doi.org/10.1038/s41592-020-01056-5
+
+> Cheng, H., Jarvis, E.D., Fedrigo, O., Koepfli, K.P., Urban, L., Gemmell, N.J., Li, H. (2022)
+> Haplotype-resolved assembly of diploid genomes without parental data. 
+> *Nature Biotechnology*, **40**:1332–1335.
+> https://doi.org/10.1038/s41587-022-01261-x

Trio.cpp

@@ -349,6 +349,83 @@ static void ha_triobin_list(const hifiasm_opt_t *opt)
 	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> partitioned reads with external lists\n", __func__, yak_realtime(), yak_cpu_usage());
 }
 
+inline void phrase_hstatus(char *s, char **rname, uint32_t *hid)
+{
+	char *p = NULL, *id = NULL; *rname = NULL; *hid = (uint32_t)-1;
+	uint32_t tot; 
+	for (p = s, tot = 0; *p; ++p){
+		if (*p == '\t' || *p == ' '){
+			*p = 0;
+			if(!tot) *rname = p+1;
+			else break;
+			tot++;
+		}
+	}
+	for (p = s, tot = 0; *p; ++p){
+		if (*p == '_') id = p + 1;
+	}
+	*hid = atoi(id);
+}
+
+uint32_t *ha_polybin_list(const hifiasm_opt_t *opt)
+{
+	int64_t i;
+	khint_t k;
+	cstr_ht_t *h;
+	assert(R_INF.total_reads < (uint32_t)-1);
+	h = cstr_ht_init();
+	for (i = 0; i < (int64_t)R_INF.total_reads; ++i) {
+		int absent;
+		char *str = (char*)calloc(Get_NAME_LENGTH(R_INF, i) + 1, 1);
+		strncpy(str, Get_NAME(R_INF, i), Get_NAME_LENGTH(R_INF, i));
+		k = cstr_ht_put(h, str, &absent);
+		if (absent) kh_val(h, k) = i;
+	}
+	fprintf(stderr, "[M::%s::%.3f*%.2f] created the hash table for read names\n", __func__, yak_realtime(), yak_cpu_usage());
+
+	gzFile fp;
+	kstream_t *ks;
+	kstring_t str = {0,0,0};
+	char *rname = NULL;
+	uint32_t hid, *ss = NULL;
+	int dret;
+	int64_t n_tot = 0, n_bin = 0;
+	fp = gzopen(opt->fn_bin_poy, "r");
+	if (fp == 0) {
+		fprintf(stderr, "ERROR: failed to open file '%s'\n", opt->fn_bin_poy);
+		for (k = 0; k < kh_end(h); ++k)
+			if (kh_exist(h, k))
+				free((char*)kh_key(h, k));
+		cstr_ht_destroy(h);
+		return NULL;
+	}
+	CALLOC(ss, R_INF.total_reads);
+	ks = ks_init(fp);
+	while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) >= 0) {
+		khint_t k; ++n_tot;
+		phrase_hstatus(str.s, &rname, &hid);
+		if((!(*rname)) || hid == (uint32_t)-1) {
+			fprintf(stderr, "ERROR: wrong hap status\n");
+			continue;
+		}
+		k = cstr_ht_get(h, rname);
+		if (k != kh_end(h)) {
+			ss[kh_val(h, k)] |= (((uint32_t)1)<<(hid-1));
+			++n_bin;
+		}
+	}
+	free(str.s);
+	ks_destroy(ks);
+	gzclose(fp);
+	
+	for (k = 0; k < kh_end(h); ++k)
+		if (kh_exist(h, k))
+			free((char*)kh_key(h, k));
+	cstr_ht_destroy(h);
+	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> partitioned reads with external lists\n", __func__, yak_realtime(), yak_cpu_usage());
+	return ss;
+}
+
 void ha_triobin(const hifiasm_opt_t *opt)
 {
 	memset(R_INF.trio_flag, AMBIGU, R_INF.total_reads * sizeof(uint8_t));

code_of_conduct.md

@@ -0,0 +1,30 @@
+## Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all
+people who contribute through reporting issues, posting feature requests,
+updating documentation, submitting pull requests or patches, and other
+activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual
+language or imagery, derogatory comments or personal attacks, trolling, public
+or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct. Project maintainers or
+contributors who do not follow the Code of Conduct may be removed from the
+project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by opening an issue or contacting the maintainer via email.
+
+This Code of Conduct is adapted from the [Contributor Covenant][cc], [version
+1.0.0][v1].
+
+[cc]: http://contributor-covenant.org/
+[v1]: http://contributor-covenant.org/version/1/0/0/

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

@@ -0,0 +1,257 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.todo',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.ifconfig',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'hifiasm'
+copyright = u'2021, Haoyu Cheng, Heng Li'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.16.0-r369'
+# The full version, including alpha/beta/rc tags.
+release = '0.16.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+#  Build using the RTD theme, if not on RTD.
+#    https://read-the-docs.readthedocs.org/en/latest/theme.html
+#    https://github.com/snide/sphinx_rtd_theme
+#
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [ "/usr/local/lib/python2.7/site-packages", ]
+
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'hifiasm-doc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+  ('index', 'hifiasm.tex', u'hifiasm Documentation',
+   u'Haoyu Cheng, Heng Li', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'hifiasm', u'hifiasm Documentation',
+     [u'Haoyu Cheng, Heng Li'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'hifiasm', u'hifiasm Documentation',
+   u'Haoyu Cheng, Heng Li', 'hifiasm', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False

docs/source/faq.rst

@@ -0,0 +1,123 @@
+
+.. _faq:
+
+Hifiasm FAQ
+===========
+
+
+.. contents::
+  :local:
+
+
+How do I get contigs in FASTA?
+-------------------------------------
+    The FASTA file can be produced from GFA as follows:
+    ::
+
+        awk '/^S/{print ">"$2;print $3}' test.p_ctg.gfa > test.p_ctg.fa
+
+Which types of assemblies should I use?
+----------------------------------------
+    If parental data is available, ``*dip.hap*.p_ctg.gfa`` produced in trio-binning mode should be always preferred. Otherwise if Hi-C data is available, ``*hic.hap*.p_ctg.gfa`` produced in Hi-C mode is the best choice. Both trio-binning mode and Hi-C mode generate fully-phased assemblies. 
+
+    If you only have HiFi reads, hifiasm in default outputs ``*bp.hap*.p_ctg.gfa``. The primary/alternate assemblies can be also produced by using ``--primary``. All these HiFi-only assemblies are not fully-phased. See `blog <https://lh3.github.io/2021/04/17/concepts-in-phased-assemblies>`_ here for more details.
+
+Are inbred/homozygous genomes supported?
+--------------------------------------------------------------------------
+
+    Yes, please use the ``-l0`` option to disable purge duplication step.
+
+Are diploid genomes supported?
+-------------------------------------
+    Yes, most modules of hifiasm are designed for diploid samples, including purge duplication step, partially phased assembly and fully-phased assembly with trio-binning or Hi-C.
+
+Are polyploid genomes supported?
+-------------------------------------
+
+    The ``*r_utg.gfa`` and ``*p_utg.gfa`` are lossless so that they also work for polyploid genomes. However, currently the contig-generation modules of hifiasm are designed for diploid samples, which means both the partially phased assembly and the fully-phased assembly does not directly support polyploid genomes. If it is set to >2, the quality of primary assembly for polyploid genomes might be improved. Please use primary assembly for polyploid samples and run multiple rounds of purging steps using third-party tools such as purge_dups. 
+
+Why one Hi-C integrated assembly is larger than another one?
+------------------------------------------------------------
+
+    For some samples like human male, the paternal haplotype should be larger than the maternal haplotype. However, if one assembly is much larger than another one, it should be the issues of hifiasm. To fix it, please set smaller value for ``-s`` (default: 0.55). 
+
+    Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. For instance, hifiasm prints the following information during assembly: 
+    ::
+
+        [M::purge_dups] homozygous read coverage threshold: 36
+
+    In this example, hifiasm identifies the coverage threshold for homozygous reads as ``36``. If it is significantly smaller than the homozygous coverage peak, hifiasm will generate two unbalanced assemblies. In this case, please set ``--hom-cov`` to homozygous coverage peak. Please note that tuning ``--hom-cov`` may affect ``*p_utg*gfa`` so that ``*hic*.bin`` should be deleted. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically.
+
+    
+
+For Hi-C integrated assembly, why the assembly size of both haplotypes are much larger than the estimated genome size?
+------------------------------------------------------------------------------------------------------------------------------
+    It is likely that hifiasm misidentifies coverage threshold for homozygous reads. Hifiasm prints the following information for debugging:
+    ::
+
+        [M::stat] # heterozygous bases: 645155110; # homozygous bases: 1495396634
+
+    If most bases of a diploid sample are homozygous, the coverage threshold is wrongly determined by hifiasm. For instance, hifiasm prints the following information during assembly: 
+    ::
+
+        [M::purge_dups] homozygous read coverage threshold: 36
+
+    In this example, hifiasm identifies the coverage threshold for homozygous reads as ``36``. If it is much smaller than homozygous coverage peak, hifiasm thinks most reads are homozygous and assign them to both assemblies, making both of them much larger than the estimated haploid genome size. In this case, please set ``--hom-cov`` to homozygous coverage peak. Please note that tuning ``--hom-cov`` may affect ``*p_utg*gfa`` so that ``*hic*.bin`` should be deleted. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically.
+
+
+.. _hic-iss:
+
+How can I tweak parameters to improve Hi-C integrated assembly?
+---------------------------------------------------------------
+    Compared with the HiFi-only assembly or the trio-binning assembly, the Hi-C integrated assembly is a little bit more complex so that you need to take care of the results. See `Why one Hi-C integrated assembly is larger than another one?`_ and `For Hi-C integrated assembly, why the assembly size of both haplotypes are much larger than the estimated genome size?`_ for details on how to fix potential issues.
+
+    There are several other options that may affect the Hi-C integrated assembly. Increasing the values of ``--n-weight``, ``--n-perturb`` and ``--f-perturb`` may improve phasing results but takes longer time. However, tuning ``--l-msjoin`` is tricky. All these options do not affect ``*p_utg*gfa`` so that ``*hic*.bin`` can be reused.
+
+.. _p-large:
+
+Why the size of primary assembly or partially phased assembly is much larger than the estimated genome size? 
+---------------------------------------------------------------------------------------------------------------
+    It could be because the estimated genome size is incorrect. Another possibility is that hifiasm does not perform enough purging. Setting smaller value for ``-s`` (default: 0.55) or turning ``--hom-cov`` should be helpful. See :ref:`loginter` for more details.
+
+
+.. _p-hamming:
+
+Why the hamming error rate or the swith error rate of trio-binning assembly is very high?
+---------------------------------------------------------------------------------------------------------------
+    In rare cases, a potential issue is that a few contigs may misjoin two haplotypes. For example, half of a contig come from mother while another half come from father. Such misjoined contigs can be fixed by manually breaking. The coordinates of problematic regions can be found by A-lines in GFA file or ``yak trioeval -e`` (see `issue 37 <https://github.com/chhylp123/hifiasm/issues/37>`_ for more details). However, if there are many misjoined contigs or the switch/hamming error rate reported by ``yak trioeval`` is very high, users should check if the parental data is correct (see `issue 130 <https://github.com/chhylp123/hifiasm/issues/130#issuecomment-862347943>`_ for more details).
+
+    Another possibility is that there are some unitigs in unitig graph misjoining two haplotypes. Such problematic unitigs might be ignored by the graph-binning strategy. Set smaller value for ``--t-occ`` forcedly remove unitig including unexpected haplotype-specific reads.
+
+Why does hifiasm stuck or crash? 
+-------------------------------------
+    In most cases, it is caused by the low quality HiFi reads. A good HiFi dataset should have a k-mer plot like `issue10 <https://github.com/chhylp123/hifiasm/issues/10#issuecomment-616213684>`_ or `issue49 <https://github.com/chhylp123/hifiasm/issues/49#issue-729106823>`_. In contrast, low quality HiFi data often lead to weird k-mer plot like `issue93 <https://github.com/chhylp123/hifiasm/issues/93#issue-852259042>`_. Such weird k-mer plots usually indicate insufficient coverage or presence of contaminants. See :ref:`loginter` for more details. If the HiFi data look fine, please raise an issue at the `issue page <https://github.com/chhylp123/hifiasm/issues>`_. 
+
+What's the usage of different bin files in hifiasm?
+----------------------------------------------------
+    ``*ec.bin``, ``*ovlp.reverse.bin`` and ``*ovlp.source.bin`` save the results of error correction step. ``*hic*bin`` saves the results of Hi-C alignment. Please note that ``*hic*.bin`` should be deleted when tuning any parameters affecting ``*p_utg*gfa``. There are several parameters which does not change ``*p_utg*gfa``, including ``-s``, ``--seed``, ``--n-weight``, ``--n-perturb``, ``--f-perturb`` and ``--l-msjoin``. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. 
+
+Can I generate HiFi-only assembly first, and then add Hi-C or trio data later?
+----------------------------------------------------------------------------------------
+    Yes, the HiFi-only assembly, Hi-C phased assembly and trio-binning assembly share the same ``*ec.bin``, ``*ovlp.reverse.bin`` and ``*ovlp.source.bin``.
+
+What is the minimum read coverage required for hifiasm?
+-------------------------------------------------------
+    Usually >=13x HiFi reads per haplotype. Higher coverage might be able to improve the contiguity of assembly.
+
+Why the primary assembly is more contiguous than the fully-phased assemblies and the partially phased assemblies (i.e. ``*.hap*.p_ctg.gfa``)?
+----------------------------------------------------------------------------------------------------------------------------------------------------
+
+    For diploid samples, primary assembly usually has greater N50 but at the expense of highly fragmented alternate assembly. From the method view, the primary assembly has an extra joining step, which joins two haplotypes to make primary assembly more contiguous.
+
+    When producing fully-phased assemblies and partially phased assemblies, hifiasm is designed to keep both haplotypes contiguous. It is important for many downstream applications like SV calling.
+
+My assembly is fragmented or not contiguous enough, how do I improve it?
+--------------------------------------------------------------------------
+
+    Raising ``-D`` or ``-N`` may improve the resolution of repetitive regions but takes longer time. These two options affect all types of assemblies and usually do not have a negative impact on the assembly quality. In contrast, ``--purge-max`` only affects primary assembly. Setting larger value for ``--purge-max`` makes primary assembly more contiguous but may collapse repeats or segmental duplications.
+
+    If the assembly is too fragmented, users should check if HiFi data is good enough. See `Why does hifiasm stuck or crash?`_ for details. 
+
+How do I avoid misassemblies?
+--------------------------------------------------------------------------
+    Set smaller value for ``--purge-max``, ``-s`` and ``-O``, or use the ``-u`` option.

docs/source/hic-assembly.rst

@@ -0,0 +1,16 @@
+
+.. _hic-assembly:
+
+Hi-C Integrated Assembly
+========================
+
+Hifiasm can generate a pair of haplotype-resolved assemblies with paired-end Hi-C reads::
+
+ hifiasm -o NA12878.asm -t32 --h1 read1.fq.gz --h2 read2.fq.gz HiFi-reads.fq.gz
+
+In this mode, each contig is supposed to be a haplotig, which by definition comes from one parental haplotype only. Hifiasm often puts all contigs from the same parental chromosome in one assembly. It has cleanly separated chrX and chrY for a human male dataset. Nonetheless, phasing across centromeres is challenging. Hifiasm is often able to phase entire chromosomes but it may fail in rare cases. Also, contigs from different parental chromosomes are randomly mixed as it is just not possible to phase across chromosomes with Hi-C. Hifiasm does not perform scaffolding for now. You need to run a standalone scaffolder such as SALSA or 3D-DNA to scaffold phased haplotigs. 
+
+
+For samples with high heterozygosity rate, a common issue is that one assembly is much larger than another one. To fix this issue, please set smaller value for ``-s`` (default: 0.55). Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. In this case, please set ``--hom-cov`` to homozygous coverage peak. See :ref:`hic-iss` for more details.
+
+At the first run, hifiasm saves the alignment of Hi-C reads to disk as ``*hic*.bin``. It reuses the saved results to avoid Hi-C alignment next time. Please note that ``*hic*.bin`` should be deleted when tuning any parameters affecting ``*p_utg*gfa``. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. There are several parameters which do not change ``*p_utg*gfa``, including ``-s``, ``--seed``, ``--n-weight``, ``--n-perturb``, ``--f-perturb`` and ``--l-msjoin``.

docs/source/index.rst

@@ -0,0 +1,80 @@
+Hifiasm
+=======
+
+.. toctree::
+   :hidden:
+
+   pa-assembly
+   trio-assembly
+   hic-assembly
+   interpreting-output
+   faq
+   parameter-reference
+
+
+
+
+`Hifiasm <https://github.com/chhylp123/hifiasm>`_ is a fast haplotype-resolved de novo assembler for PacBio HiFi reads. It can assemble a human genome in several hours and assemble a ~30Gb California redwood genome in a few days. Hifiasm emits partially phased assemblies of quality competitive with the best assemblers. Given parental short reads or Hi-C data, it produces arguably the best haplotype-resolved assemblies so far.
+
+Publications
+============
+
+Hifiasm
+  Haoyu Cheng, Gregory T. Concepcion, Xiaowen Feng, Haowen Zhang & Heng Li.
+  `Haplotype-resolved de novo assembly using phased assembly graphs with hifiasm <https://doi.org/10.1038/s41592-020-01056-5>`_. Nature Methods. (2021).
+
+Install
+=======
+The easiest way to get started is to download a `release <https://github.com/chhylp123/hifiasm/releases>`_. Please report any issues on `github issues <https://github.com/chhylp123/hifiasm/issues>`_ page.
+
+In addition, the latest unreleased version can be found from github:
+
+::
+
+  git clone https://github.com/chhylp123/hifiasm
+  cd hifiasm && make
+
+Another way is to install hifiasm via `bioconda <https://anaconda.org/bioconda/hifiasm>`_:
+
+::
+
+  conda install -c bioconda hifiasm
+
+Assembly Concepts 
+=================
+There are different types of assemblies which are commonly used in practice (see 
+`details <https://lh3.github.io/2021/04/17/concepts-in-phased-assemblies>`_). 
+Hifiasm produces primary/alternate assemblies or partially phased assemblies 
+only with HiFi reads. Given Hi-C data or trio-binning data, hifiasm produces 
+contiguous fully-phased assemblies, i.e. haplotype-resolved assemblies.
+
+Why Hifiasm?
+============
+* Hifiasm delivers high-quality assemblies. It tends to generate longer contigs
+  and resolve more segmental duplications than other assemblers.
+
+* Given Hi-C reads or short reads from the parents, hifiasm can produce overall the best
+  haplotype-resolved assembly so far. It is the assembler of choice by the 
+  `Human Pangenome Project <https://humanpangenome.org/>`_ for the first batch of samples.
+
+* Hifiasm can purge duplications between haplotigs without relying on
+  third-party tools such as purge\_dups. Hifiasm does not need polishing tools
+  like pilon or racon, either. This simplifies the assembly pipeline and saves
+  running time.
+
+* Hifiasm is fast. It can assemble a human genome in half a day and assemble a
+  ~30Gb redwood genome in three days. No genome is too large for hifiasm.
+
+* Hifiasm is trivial to install and easy to use. It does not required Python,
+  R or C++11 compilers, and can be compiled into a single executable. The
+  default setting works well with a variety of genomes.
+
+Learn
+=====
+
+*  :ref:`HiFi-only Assembly        <pa-assembly>` - Assembling HiFi reads without additional data types
+*  :ref:`Trio-binning Assembly     <trio-assembly>` - Producing fully phased assemblies with HiFi and trio-binning data
+*  :ref:`Hi-C Integrated Assembly   <hic-assembly>`   - Producing fully phased assemblies with HiFi and Hi-C data
+*  :ref:`Hifiasm Output           <interpreting-output>`   - Interpreting results
+*  :ref:`Hifiasm FAQ           <faq>`   - Frequently asked questions
+*  :ref:`Hifiasm Parameters <parameter-reference>` - Parameter reference of hifiasm

docs/source/interpreting-output.rst

@@ -0,0 +1,102 @@
+
+.. _interpreting-output:
+
+Hifiasm Output 
+===============
+
+.. _outfile:
+
+Output files
+---------------------------------------
+
+In general, hifiasm generates the following assembly graphs in the GFA format:
+
+* ```prefix`.r_utg.gfa``: haplotype-resolved raw unitig graph. This graph keeps all haplotype information.
+* ```prefix`.p_utg.gfa``: haplotype-resolved  processed  unitig  graph  without small bubbles. Small bubbles might be caused by somatic mutations or noise in data, which are not the real haplotype information. Hifiasm automatically pops such small bubbles based on coverage. The option ``--hom-cov`` affects the result. See :ref:`homozygous coverage setting <homcov>` for more details. In addition, the option ``-p`` forcedly pops bubbles.
+* ```prefix`.p_ctg.gfa``: assembly graph of primary contigs. This graph includes a complete assembly with long stretches of phased blocks.
+* ```prefix`.a_ctg.gfa``: assembly graph of alternate contigs. This graph consists of all contigs that are discarded in primary contig graph.
+* ```prefix`.*hap*.p_ctg.gfa``: phased contig graph. This graph keeps the phased contigs.
+
+
+Hifiasm outputs ``*.r_utg.gfa`` and ``*.p_utg.gfa`` in any cases. Specifically, hifiasm outputs the following assembly graphs in trio-binning mode:
+
+* ```prefix`.dip.hap1.p_ctg.gfa``: fully phased paternal/haplotype1 contig graph keeping the phased paternal/haplotype1 assembly.
+* ```prefix`.dip.hap2.p_ctg.gfa``: fully phased maternal/haplotype2 contig graph keeping the phased maternal/haplotype2 assembly.
+
+With Hi-C partition options, hifiasm outputs:
+
+* ```prefix`.hic.p_ctg.gfa``: assembly graph of primary contigs.
+* ```prefix`.hic.hap1.p_ctg.gfa``: fully phased contig graph of haplotype1 where each contig is fully phased.
+* ```prefix`.hic.hap2.p_ctg.gfa``: fully phased contig graph of haplotype2 where each contig is fully phased.
+* ```prefix`.hic.a_ctg.gfa`` (optional with ``--primary``): assembly graph of alternate contigs.
+
+Hifiasm generates the following assembly graphs only with HiFi reads in default:
+
+* ```prefix`.bp.p_ctg.gfa``: assembly graph of primary contigs.
+* ```prefix`.bp.hap1.p_ctg.gfa``: partially phased contig graph of haplotype1.
+* ```prefix`.bp.hap2.p_ctg.gfa``: partially phased contig graph of haplotype2.
+
+If the option ``--primary`` or ``-l0`` is specified, hifiasm outputs:
+
+* ```prefix`.p_ctg.gfa``: assembly graph of primary contigs.
+* ```prefix`.a_ctg.gfa``: assembly graph of alternate contigs.
+
+For each graph, hifiasm also outputs a simplified version (``*noseq*gfa``) without sequences for the ease of visualization. The coordinates of low quality regions are written to ``*lowQ.bed`` in BED format.
+The concepts of different types of assemblies can be found `here <https://lh3.github.io/2021/04/17/concepts-in-phased-assemblies>`_.
+
+.. _outformat:
+
+Output file formats
+---------------------------------------
+Hifiasm broadly follows the specification for `GFA 1.0 <https://github.com/GFA-spec/GFA-spec/blob/master/GFA1.md>`_. There are several fields that are specifically used by hifiasm. For ``S`` segment line:
+
+* ``rd:i:``: read coverage. It is calculated by the reads coming from the same contig/unitig.
+
+Hifiasm outputs ``A`` lines including the information of reads which are used to construct contig/unitig. Each ``A`` line is plain-text, tab-separated, and the columns appear in the following order:
+
+.. list-table::
+   :widths: 10 25 50
+   :header-rows: 1
+
+   * - Col
+     - Type
+     - Description
+   * - 1
+     - string
+     - Should be always ``A``
+   * - 2
+     - string
+     - Contig/unitig name
+   * - 3
+     - int
+     - Contig/unitig start coordinate of subregion constructed by read
+   * - 4
+     - char
+     - Read strand: "+" or "-"
+   * - 5
+     - string
+     - Read name
+   * - 6
+     - int
+     - Read start coordinate of subregion which is used to construct contig/unitig
+   * - 7
+     - int
+     - Read end coordinate of subregion which is used to construct contig/unitig
+   * - 8
+     - id:i:int
+     - Read ID
+   * - 9
+     - HG:A:char
+     - Haplotype status of read. ``HG:A:a``, ``HG:A:p``, ``HG:A:m`` indicate read is non-binnable, father/hap1-specific and mother/hap2-specific, respectively.
+
+.. _loginter:
+
+Hifiasm log interpretation
+---------------------------------------
+Hifiasm prints several information for quick debugging, including:
+
+.. _homcov:
+
+* k-mer plot: showing how many k-mers appear a certain number of times. For homozygous samples, there should be one peak around read coverage. For heterozygous samples, there should two peaks, where the smaller peak is around the heterozygous read coverage and the larger peak is around the homozygous read coverage. For example, `issue10 <https://github.com/chhylp123/hifiasm/issues/10#issuecomment-616213684>`_ indicates the heterozygous read coverage and the homozygous read coverage are 28 and 57, respectively. `Issue49 <https://github.com/chhylp123/hifiasm/issues/49#issue-729106823>`_ is another good example. Weird k-mer plot like `issue93 <https://github.com/chhylp123/hifiasm/issues/93#issue-852259042>`_ is often caused by insufficient coverage or presence of contaminants. 
+* homozygous coverage: coverage threshold for homozygous reads. Hifiasm prints it as: ``[M::purge_dups] homozygous read coverage threshold: X``. If it is not around homozygous coverage, the final assembly might be either too large or too small. To fix this issue, please set ``--hom-cov`` to homozygous coverage.  
+* number of het/hom bases: how many bases in unitig graph are heterozygous and homozygous during Hi-C phased assembly. Hifiasm prints it as: ``[M::stat] # heterozygous bases: X; # homozygous bases: Y``. Given a heterozygous sample, if there are much more homozygous bases than heterozygous bases, hifiasm fails to identify correct coverage threshold for homozygous reads. In this case, please set ``--hom-cov`` to homozygous coverage. 

docs/source/pa-assembly.rst

@@ -0,0 +1,47 @@
+
+.. _pa-assembly:
+
+HiFi-only Assembly
+==================
+
+A typical hifiasm command line looks like::
+
+ hifiasm -o NA12878.asm -t 32 NA12878.fq.gz
+
+where ``NA12878.fq.gz`` provides the input reads, ``-t`` sets the number of CPUs in
+use and ``-o`` specifies the prefix of output files. Input sequences should be FASTA 
+or FASTQ format, uncompressed or compressed with gzip (.gz). The quality scores of reads 
+in FASTQ are ignored by hifiasm. Hifiasm outputs assemblies in `GFA <https://github.com/pmelsted/GFA-spec/blob/master/GFA-spec.md>`_ format.
+
+At the first run, hifiasm saves corrected reads and overlaps to disk as ``NA12878.asm.*.bin``. It reuses the saved results to avoid the time-consuming all-vs-all overlap calculation next time. You may specify ``-i`` to ignore precomputed overlaps and redo overlapping from raw reads. You can also dump error corrected reads in FASTA and read overlaps in PAF with::
+
+ hifiasm -o NA12878.asm -t 32 --write-paf --write-ec /dev/null
+
+Hifiasm purges haplotig duplications by default. For inbred or homozygous genomes, you may disable purging with option ``-l0``. Old HiFi reads may contain short adapter sequences at the ends of reads. You can specify ``-z20`` to trim both ends of reads by 20bp. For small genomes, use ``-f0`` to disable the initial bloom filter which takes 16GB memory at the beginning. For genomes much larger than human, applying ``-f38`` or even ``-f39`` is preferred to save memory on k-mer counting.
+
+
+Produce two partially phased assemblies
+---------------------------------------
+
+
+Since v0.15, hifiasm produces two sets of partially phased contigs in default like::
+
+ hifiasm -o NA12878.asm -t 32 NA12878.fq.gz
+
+In this example, the partially phased contigs are written to ``NA12878.asm.bp.hap*.p_ctg.gfa``. 
+This pair of files can be thought to represent the two haplotypes in a diploid genome, though with occasional switch errors. The frequency of switches is determined by the heterozygosity of the input sample. Hifiasm also writes the primary contigs to ``NA12878.asm.bp.p_ctg.gfa``. 
+
+For samples with high heterozygosity rate, a common issue is that one set of partially phased contigs is much larger than another set. To fix this issue, please set smaller value for ``-s`` (default: 0.55). Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. 
+In this case, please set ``--hom-cov`` to homozygous coverage. See :ref:`p-large` for more details.
+
+
+Produce primary/alternate assemblies
+------------------------------------
+
+To get primary/alternate assemblies, the option ``--primary`` should be set::
+
+ hifiasm -o NA12878.asm --primary -t 32 NA12878.fq.gz
+
+The primary contigs and the alternate contigs are written to ``NA12878.asm.p_ctg.gfa`` and ``NA12878.asm.a_ctg.gfa``, respectively. For inbred or homozygous genomes, the primary/alternate assemblies can be also produced by ``-l0``. Similarly, turning ``-s`` or ``--hom-cov`` should 
+be helpful if the primary assembly is too large. See :ref:`p-large` for more details.
+

docs/source/parameter-reference.rst

@@ -0,0 +1,298 @@
+
+.. _parameter-reference:
+
+Hifiasm Parameter Reference
+============================
+
+Synopsis
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Assembly only with HiFi reads:
+::
+
+  hifiasm -o [prefix] -t [nThreads] [options] input1.fq [input2.fq [...]]
+
+Trio binning assembly with yak dumps:
+::
+
+  yak count -o paternal.yak -b37 [-t nThreads] [-k kmerLen] paternal.fq.gz
+  yak count -o maternal.yak -b37 [-t nThreads] [-k kmerLen] maternal.fq.gz
+  hifiasm [-o prefix] [-t nThreads] [options] -1 paternal.yak -2 maternal.yak child.hifi.fq.gz
+
+Hi-C integrated assembly:
+::
+
+  hifiasm -o [prefix] -t [nThreads] --h1 [hic_r1.fq.gz,...] --h2 [hic_r2.fq.gz,...] [options] HiFi.read.fq.gz
+
+To get detailed description of options, run:
+::
+
+  hifiasm -h
+
+or:
+::
+
+  man ./hifiasm.1
+
+
+General options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _oopt:
+
+**\-o <FILE=hifiasm.asm>**
+  Prefix of output files. See :ref:`outfile` and :ref:`outformat` for more details.
+
+.. _topt:
+
+**\-t <INT=1>**
+  Number of CPU threads used by hifiasm.
+
+.. _hopt:
+
+**\-h** 
+  Show help information.
+
+.. _versionopt:
+
+**\-\-version** 
+  Show version number.
+
+
+Error correction options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _kopt:
+
+**\-k <INT=51>**
+  K-mer length. This option must be less than 64.
+
+.. _wopt:
+
+**\-w <INT=51>**    
+  Minimizer window size.
+
+.. _fopt:
+
+**\-f <INT=37>**    
+  Number  of  bits for bloom filter; 0 to disable. This bloom filter is used to filter out singleton k-mers when counting all k-mers. It takes 2\ :sup:`(INT-3)` bytes of memory. A proper setting saves memory. ``-f37`` is recommended for human assembly. For small genomes, use ``-f0`` to disable the initial bloom filter which takes 16GB memory at the beginning. For genomes much larger than human, applying ``-f38`` or even ``-f39`` is preferred to save memory on k-mer counting.
+
+.. _Dopt:
+
+**\-D <FLOAT=5.0>**    
+  Drop k-mers occurring ``>FLOAT*coverage`` times. Hifiasm discards these high-frequency k-mers during error correction to reduce running time. The ``coverage`` is determined automatically by hifiasm based on k-mer plot, representing homozygous read coverage. Raising this option may improve the resolution of repetitive regions but takes longer time.
+
+.. _NEopt:
+
+**\-N <INT=100>**
+  Consider up to ``max(-D*coverage,-N)`` overlaps for each oriented read. The ``coverage`` is determined automatically by hifiasm based on k-mer plot, representing homozygous read coverage. Raising this option may improve the resolution of repetitive regions but takes longer time.
+
+.. _ropt:
+
+**\-r <INT=3>**
+  Rounds of haplotype-aware error correction. This option affects all outputs of hifiasm. Odd rounds of correction are preferred in practice.
+
+
+.. _zopt:
+
+**\-z <INT=0>**
+  Length  of  adapters that should be removed. This option remove ``INT`` bases from both ends of each read.  Some old HiFi reads may consist of short adapters (e.g. 20bp adapter at one end). For such data, trimming short adapters would significantly improve the assembly quality.
+
+.. _max-kocc-opt:
+
+**\-\-max-kocc <INT=2000>**
+  Employ k-mers occurring < ``INT`` times to rescue repetitive overlaps. This option may improve the resolution of repeats.
+
+
+.. _hg-size-opt:
+
+**\-\-hg-size <INT(k/m/g)>**
+  Estimated haploid genome size used for inferring read coverage. This option is used to get accurate homozygous read coverage during error correction. Common suffices are required, for example, 100m or 3g.
+
+
+.. _min-hist-cnt-opt:
+
+**\-\-min-hist-cnt <INT=5>**
+  When analyzing the k-mer spectrum, ignore counts below ``INT``. For very low coverage of HiFi data, set smaller value for this option. See `issue 45 <https://github.com/chhylp123/hifiasm/issues/49>`_ for example.
+
+
+
+Assembly options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _aopt:
+
+**\-a <INT=4>**
+  Rounds of assembly graph cleaning. This option is used with ``-x`` and ``-y``. Note that unlike -r, this option does not affect error corrected reads and all-to-all overlaps.
+
+
+.. _mopt:
+
+**\-m <INT=10000000>**
+  Maximal probing distance for bubble popping when generating primary/alternate contig graphs. Bubbles longer than ``INT`` bases will not be popped.
+
+.. _popt:
+
+**\-p <INT=0>**
+  Maximal probing distance for bubble popping when generating haplotype-resolved processed unitig graph without small bubbles. Bubbles longer than ``INT`` bases will not be popped. Small bubbles might be caused by somatic mutations or noise in data. Please note that hifiasm automatically pops small bubbles based on coverage, which can be tweaked by ``--hom-cov``.
+
+.. _nopt:
+
+**\-n <INT=3>**
+  A unitig is considered small if it is composed of less than ``INT`` reads. Hifiasm may try to remove small unitigs at various steps.
+
+.. _xyopt:
+
+**\-x <FLOAT1=0.8>, \-y <FLOAT2=0.2>**
+  Max and min overlap drop ratio. This option is used with ``-a``. Given a node N in the assembly graph, let max(N) be the length of the longest overlap of N. Hifiasm iteratively drops overlaps of N if their length/max(N) is below a threshold controlled by ``-x`` and ``-y``. Hifiasm applies ``-a`` rounds of short overlap removal with an increasing threshold between ``FLOAT1`` and ``FLOAT2``.
+
+.. _iopt:
+
+**\-i**
+  Ignore all bin files so that hifiasm will start again from scratch.
+
+.. _uopt:
+
+**\-u**
+  Disable post-join step for contigs which may improve N50. The post-join step of hifiasm improves contig N50 but may introduce misassemblies. 
+
+
+.. _hom-cov-opt:
+
+**\-\-hom-cov <INT>**
+  Homozygous read coverage inferred automatically in default. This option affects different types of outputs, including Hi-C phased assembly and HiFi-only assembly. For more details, see :ref:`hic-iss`, :ref:`p-large` and :ref:`loginter`.
+
+.. _pri-range-opt:
+
+**\-\-pri-range <INT1[,INT2]>**
+  Min and max coverage cutoffs of primary contigs. Keep contigs with coverage in this range at p_ctg.gfa. Inferred automatically in default. If ``INT2`` is not specified, it is set to infinity. Set -1 to disable.
+
+.. _lowQ-opt:
+
+**\-\-lowQ <INT=70>**
+  Output contig regions with ``>=INT%`` inconsistency to the bed file with suffix lowQ.bed. Set 0 to disable.
+
+.. _b-cov-opt:
+
+**\-\-b-cov <INT=0>**
+  Break contigs at potential misassemblies with ``<INT``-fold coverage. Work with ``--m-rate``. Set 0 to disable.
+
+.. _h-cov-opt:
+
+**\-\-h-cov <INT=-1>**
+  Break contigs at potential misassemblies with ``>INT``-fold coverage. Work with ``--m-rate``. Set -1 to disable.
+
+.. _m-rate-opt:
+
+**\-\-m-rate <FLOAT=0.75>**
+  Break contigs with ``<=FLOAT*coverage`` exact overlaps. Only work when ``--b-cov`` and ``--h-cov`` are specified.
+
+.. _primary-opt:
+
+**\-\-primary**
+  Output a primary assembly and an alternate assembly. Enable this option or ``-l0`` outputs a primary assembly and an alternate assembly.
+
+
+Trio-binning options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _1opt:
+
+**\-1 <FILE>**
+  K-mer dump generated by `yak count <https://github.com/lh3/yak>`_ from the paternal/haplotype1 reads.
+
+.. _2opt:
+
+**\-2 <FILE>**
+  K-mer dump generated by `yak count <https://github.com/lh3/yak>`_ from the maternal/haplotype2 reads.
+
+.. _3opt:
+
+**\-3 <FILE>**
+  List of paternal/haplotype1 read names.
+
+.. _4opt:
+
+**\-4 <FILE>**
+  List of maternal/haplotype2 read names.
+
+.. _cdopt:
+
+**\-c <INT1=2>, -d <INT2=5>**
+  Lower bound and upper bound of the binned k-mer's frequency. When doing trio binning, a k-mer is said to be differentiating if it occurs >= ``INT2`` times in one sample but occurs < ``INT1`` times in the other sample.
+
+
+.. _t-occ-opt:
+
+**\-\-t-occ <INT=60>**
+  Forcedly remove unitig including ``>INT`` unexpected haplotype-specific reads without considering graph topology. For more details, see :ref:`p-hamming`.
+
+
+Purge duplication options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _ldopt:
+
+**\-l <INT=3>**
+  Level of purge duplication. 0 to disable, 1 to only purge contained haplotigs, 2 to purge all types of haplotigs, 3 to purge all types of haplotigs in the most aggressive way. In default, 3 for non-trio assembly, 0 for trio-binning assembly. For trio-binning assembly, only level 0 and level 1 are allowed.
+
+.. _sdopt:
+
+**\-s <FLOAT=0.55>**
+  Similarity threshold for duplicate haplotigs that should be purged. In default, 0.75 for ``-l1/-l2``, 0.55 for ``-l3``. This option affects both HiFi-only assembly and Hi-C phased assembly. For more details, see :ref:`hic-iss` and :ref:`p-large`.
+
+.. _ovlpdopt:
+
+**\-O <INT=1>**
+  Min number of overlapped reads for duplicate haplotigs that should be purged.
+
+.. _purgeopt:
+
+**\-\-purge-max <INT>**
+  Coverage upper bound of purge duplication, which is inferred automatically in default. If the coverage of a contig is higher than this bound, don't apply purge duplication. Larger value makes assembly more contiguous but may collapse repeats or segmental duplications.
+
+.. _nhapopt:
+
+**\-\-n\-hap <INT=2>**
+  Assumption of haplotype number. If it is set to >2, the quality of primary assembly for polyploid genomes might be improved.
+
+
+
+Hi-C integration options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _h1opt:
+
+**\-\-h1 <FILEs>**
+  File names of input Hi-C R1 ``[r1_1.fq,r1_2.fq,...]``.
+
+.. _h2opt:
+
+**\-\-h2 <FILEs>**
+  File names of input Hi-C R2 ``[r2_1.fq,r2_2.fq,...]``.
+
+.. _n-weightopt:
+
+**\-\-n-weight <INT=3>**
+  Rounds of reweighting Hi-C links. Raising this option may improve phasing results but takes longer time.
+
+.. _n-perturbopt:
+
+**\-\-n-perturb <INT=10000>**
+  Rounds of perturbation. Increasing this option may improve phasing results but takes longer time.
+
+.. _f-perturbopt:
+
+**\-\-f-perturb <FLOAT=0.1>**
+  Fraction to flip for perturbation. Increasing this option may improve phasing results but takes longer time.
+
+.. _seedopt:
+
+**\-\-seed <INT=11>**
+  RNG seed.
+
+
+.. _l-msjoin:
+
+**\-\-l-msjoin <INT=500000>**
+  Detect misjoined unitigs of ``>=INT`` in size; 0 to disable.

docs/source/trio-assembly.rst

@@ -0,0 +1,29 @@
+
+.. _trio-assembly:
+
+Trio-binning Assembly
+=====================
+
+When parental short reads are available, hifiasm can also generate a pair of haplotype-resolved assemblies with trio binning. To perform such assembly, you need to count k-mers first with `yak <https://github.com/lh3/yak>`_ and then do assembly::
+
+ yak count -k31 -b37 -t16 -o pat.yak paternal.fq.gz
+ yak count -k31 -b37 -t16 -o mat.yak maternal.fq.gz
+ hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak NA12878.fq.gz
+
+Here ``NA12878.asm.hap1.p_ctg.gfa`` and ``NA12878.asm.hap2.p_ctg.gfa`` give the assemblies for two haplotypes. In the binning mode, hifiasm does not purge haplotig duplicates by default. Because hifiasm reuses saved overlaps, you can generate both primary/alternate assemblies and trio binning assemblies with::
+
+ hifiasm -o NA12878.asm --primary -t 32 NA12878.fq.gz 2> NA12878.asm.pri.log
+ hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak /dev/null 2> NA12878.asm.trio.log
+
+The second command line will run much faster than the first. The phasing switch error rate and hamming error rate are able to be evaluated quickly by `yak <https://github.com/lh3/yak>`_::
+
+ yak trioeval -t16 pat.yak mat.yak assembly.fa
+
+The W-line and H-line reported by ``yak trioeval`` indicate switch error rate and hamming error rate respectively::
+
+ W       26714   3029448 0.008818
+ H       24315   3029885 0.008025
+
+For this example, the switch error rate is 0.8818% and the hamming error rate is 0.8025%. If the hamming error rate or the swith error rate of trio-binning assembly is very high, it might be caused by hifiasm or the incorrect parental data. To fix it, see :ref:`p-hamming` for more details.
+
+

gchain_map.cpp

@@ -0,0 +1,204 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <zlib.h>
+#include <math.h>
+#include "kseq.h" // FASTA/Q parser
+#include "kavl.h"
+#include "khash.h"
+#include "kalloc.h"
+#include "kthread.h"
+#include "inter.h"
+#include "Overlaps.h"
+#include "CommandLines.h"
+#include "htab.h"
+#include "Hash_Table.h"
+#include "Correct.h"
+#include "Process_Read.h"
+#include "Assembly.h"
+#include "gchain_map.h"
+KSEQ_INIT(gzFile, gzread)
+void ul_map_lchain(ha_abufl_t *ab, int64_t rid, char* rs, uint64_t rl, uint64_t mz_w, uint64_t mz_k, const ul_idx_t *uref, overlap_region_alloc *overlap_list, overlap_region_alloc *overlap_list_hp, Candidates_list *cl, double bw_thres, 
+								 int max_n_chain, int apend_be, kvec_t_u8_warp* k_flag, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, st_mt_t *sp, uint32_t *high_occ, uint32_t *low_occ, uint32_t is_accurate, uint32_t gen_off);
+
+typedef struct { // global data structure for kt_pipeline()
+    const void *ha_flt_tab;
+    const ha_pt_t *ha_idx;
+    const mg_idxopt_t *opt;
+    const ma_ug_t *ug;
+	const asg_t *rg;
+	const ug_opt_t *uopt;
+	const ul_idx_t *uu;
+	ucr_file_t *ucr_s;
+	kseq_t *ks;
+    int64_t chunk_size;
+    uint64_t n_thread;
+    uint64_t total_base;
+    uint64_t total_pair;
+	uint64_t num_bases, num_corrected_bases, num_recorrected_bases;
+	uint64_t remap, mini_cut;
+} gmap_t;
+
+typedef struct { // data structure for each step in kt_pipeline()
+    const mg_idxopt_t *opt;
+    const void *ha_flt_tab;
+    const ha_pt_t *ha_idx;
+    const ma_ug_t *ug;
+	const asg_t *rg;
+	const ug_opt_t *uopt;
+	const ul_idx_t *uu;
+	int n, m, sum_len;
+	uint64_t *len, id;
+	char **seq;
+    // ha_mzl_v *mzs;///useless
+    // st_mt_t *sps;///useless
+	// mg_gchains_t **gcs;///useless
+    mg_tbuf_t **buf;///useless
+	ha_ovec_buf_t **hab;
+	kv_ul_ov_t *res;
+	// glchain_t *ll;
+	// gdpchain_t *gdp;
+	// glchain_t *sec_ll;
+	uint64_t num_bases, num_corrected_bases, num_recorrected_bases, mini_cut;
+	int64_t n_thread;
+} sstep_t;
+
+/**
+static void worker_ul_map(void *data, long i, int tid) // callback for kt_for()
+{
+    sstep_t *s = (sstep_t*)data;
+    ha_ovec_buf_t *b = s->hab[tid];
+    kv_ul_ov_t *res = (s->res?(&(s->res[tid])):(NULL));
+    mg_tbuf_t *buf = (s->buf?s->buf[tid]:NULL);
+    int64_t winLen = MIN((((double)THRESHOLD_MAX_SIZE)/s->opt->diff_ec_ul), WINDOW);
+    int fully_cov, abnormal; 
+    assert(UL_INF.a[s->id+i].rlen == s->len[i]);
+    // if(s->id+i!=43) return;
+
+    ul_map_lchain(b->abl, i, s->seq[i], s->len[i], s->opt->w, s->opt->k, s->uu, &b->olist, &b->olist_hp, &b->clist, s->opt->bw_thres, 
+        s->opt->max_n_chain, 1, NULL, &(b->tmp_region), NULL, &(b->sp), s->mini_cut, 0);
+    
+    clear_Cigar_record(&b->cigar1);
+    clear_Round2_alignment(&b->round2);
+
+    b->self_read.seq = s->seq[i]; b->self_read.length = s->len[i]; b->self_read.size = 0;
+    lchain_align(&b->olist, s->uu, &b->self_read, &b->correct, &b->ovlp_read, &b->POA_Graph, &b->DAGCon,
+            &b->cigar1, &b->hap, &b->round2, &b->r_buf, &(b->tmp_region.w_list), 0, 1, &fully_cov, &abnormal, s->opt->diff_ec_ul, winLen, NULL);
+
+    // gl_chain_refine(&b->olist, &b->correct, &b->hap, bl, s->uu, s->opt->diff_ec_ul, winLen, s->len[i], km);
+    gl_chain_refine_advance_combine(s->buf[tid], &(UL_INF.a[s->id+i]), &b->olist, &b->correct, &b->hap, &(s->sps[tid]), bl, &(s->gdp[tid]), s->uu, s->opt->diff_ec_ul, winLen, s->len[i], s->uopt, s->id+i, tid, NULL);
+    
+    memset(&b->self_read, 0, sizeof(b->self_read));
+    if(UL_INF.a[s->id+i].dd) {
+        free(s->seq[i]); s->seq[i] = NULL; b->num_correct_base++;
+    }
+    s->hab[tid]->num_read_base++;
+}
+
+static void *worker_gmap_work_ovec_pip(void *data, int step, void *in) // callback for kt_pipeline()
+{
+    gmap_t *p = (gmap_t*)data;
+    if (step == 0) { // step 1: read a block of sequences
+        int32_t ret; uint64_t l; sstep_t *s; CALLOC(s, 1);
+        s->ha_flt_tab = p->ha_flt_tab; s->ha_idx = p->ha_idx; s->id = p->total_pair; 
+        s->opt = p->opt; s->uu = p->uu; s->uopt = p->uopt; s->rg = p->rg; s->mini_cut = p->mini_cut;///need set
+        while ((ret = kseq_read(p->ks)) >= 0) {
+            if (p->ks->seq.l < (uint64_t)p->opt->k) continue;
+            if (s->n == s->m) {
+                s->m = s->m < 16? 16 : s->m + (s->n>>1);
+                REALLOC(s->len, s->m);
+                REALLOC(s->seq, s->m);
+            }
+            if(!(p->remap)) {
+				append_ul_t(&UL_INF, NULL, p->ks->name.s, p->ks->name.l, NULL, 0, NULL, 0, P_CHAIN_COV, s->uopt, 0);        
+			}
+			l = p->ks->seq.l;
+            MALLOC(s->seq[s->n], l);
+            s->sum_len += l;
+            memcpy(s->seq[s->n], p->ks->seq.s, l);
+            s->len[s->n++] = l;
+            if (s->sum_len >= p->chunk_size) break;            
+        }
+        p->total_pair += s->n;
+        if (s->sum_len == 0) free(s);
+        else return s;
+    }
+    else if (step == 1) { // step 2: alignment
+        sstep_t *s = (sstep_t*)in; uint64_t i;
+        CALLOC(s->hab, p->n_thread); 
+		CALLOC(s->buf, p->n_thread); 
+		if(!(p->remap)) CALLOC(s->res, p->n_thread);//for results
+        for (i = 0; i < p->n_thread; ++i) {
+			s->hab[i] = ha_ovec_init(0, 0, 1); s->buf[i] = mg_tbuf_init();
+		}
+        // kt_for(p->n_thread, worker_for_ul_scall_alignment, s, s->n);
+       
+        for (i = 0; i < p->n_thread; ++i) {
+            s->num_bases += s->hab[i]->num_read_base;
+            s->num_corrected_bases += s->hab[i]->num_correct_base;
+            s->num_recorrected_bases += s->hab[i]->num_recorrect_base;
+			ha_ovec_destroy(s->hab[i]); mg_tbuf_destroy(s->buf[i]); 
+        }
+        free(s->hab); free(s->buf); 
+        return s;
+    }
+    else if (step == 2) { // step 3: dump
+        sstep_t *s = (sstep_t*)in;
+        uint64_t i, rid, sn = s->n;
+        p->num_bases += s->num_bases;
+        p->num_corrected_bases += s->num_corrected_bases;
+        p->num_recorrected_bases += s->num_recorrected_bases;
+		if(!(p->remap)) {
+			for (i = 0; i < p->n_thread; ++i) {
+				push_uc_block_t(s->uopt, &(s->res[i]), s->seq, s->len, s->id);
+				kv_destroy(s->res[i]);
+			}
+			free(s->res);
+        
+			for (i = 0; i < sn; ++i) {
+				rid = s->id + i; 
+				if((UL_INF.n <= rid) || (UL_INF.n > rid && UL_INF.a[rid].rlen != s->len[i])) {///reads without alignment
+					append_ul_t(&UL_INF, &rid, NULL, 0, s->seq[i], s->len[i], NULL, 0, P_CHAIN_COV, s->uopt, 0);
+				} 
+				free(s->seq[i]); 
+			}
+		} else {
+			for (i = 0; i < sn; ++i) {
+				rid = s->id + i;
+				if(UL_INF.a[rid].dd == 0 && p->ucr_s && p->ucr_s->flag == 1) {
+					assert(s->seq[i]);
+					///for debug interval
+					write_compress_base_disk(p->ucr_s->fp, rid, s->seq[i], s->len[i], &(p->ucr_s->u));
+				}
+				free(s->seq[i]);
+			}
+		}
+		free(s->len); free(s->seq); free(s);
+    }
+    return 0;
+}
+**/
+
+// int gmap_work_ovec(gmap_t* sl, const enzyme *fn)
+// {
+//     double index_time = yak_realtime();
+//     int i;
+
+// 	init_all_ul_t(&UL_INF, &R_INF);
+//     for (i = 0; i < fn->n; i++){
+//         gzFile fp;
+//         if ((fp = gzopen(fn->a[i], "r")) == 0) return 0;
+//         sl->ks = kseq_init(fp);
+//         kt_pipeline(3, worker_gmap_work_ovec_pip, sl, 3);
+//         kseq_destroy(sl->ks);
+//         gzclose(fp);
+//     }
+// 	sl->hits.total_base = sl->total_base;
+// 	sl->hits.total_pair = sl->total_pair;
+//     fprintf(stderr, "[M::%s::%.3f] ==> Qualification\n", __func__, yak_realtime()-index_time);
+// 	fprintf(stderr, "[M::%s::] ==> # reads: %lu, # bases: %lu\n", __func__, UL_INF.n, sl->total_base);
+// 	fprintf(stderr, "[M::%s::] ==> # bases: %lu; # corrected bases: %lu; # recorrected bases: %lu\n", 
+// 	__func__, sl->num_bases, sl->num_corrected_bases, sl->num_recorrected_bases);
+// 	gen_ul_vec_rid_t(&UL_INF, &R_INF, NULL);
+//     return 1;
+// }

gchain_map.h

@@ -0,0 +1,6 @@
+#ifndef __INTER__
+#define __INTER__
+#include "Overlaps.h"
+#include "Process_Read.h"
+
+#endif

gfa_ut.h

@@ -0,0 +1,35 @@
+#ifndef __GFA_UT__
+#define __GFA_UT__
+#include "Overlaps.h"
+
+typedef struct {
+	asg_t *g; 
+	ma_hit_t_alloc *src;
+    R_to_U* ruIndex;
+    int64_t max_hang; 
+    int64_t min_ovlp; 
+    int64_t ul_occ;
+} sset_aux;
+
+void ul_clean_gfa(ug_opt_t *uopt, asg_t *sg, ma_hit_t_alloc *src, ma_hit_t_alloc *rev, R_to_U* rI, int64_t clean_round, double min_ovlp_drop_ratio, double max_ovlp_drop_ratio, 
+double ou_drop_rate, int64_t max_tip, int64_t gap_fuzz, bub_label_t *b_mask_t, int32_t is_ou, int32_t is_trio, uint32_t ou_thres, char *o_file);
+uint32_t asg_arc_cut_tips(asg_t *g, uint32_t max_ext, asg64_v *in, uint32_t is_ou, R_to_U *ru);
+void asg_iterative_semi_circ(asg_t *g, ma_hit_t_alloc* src, asg64_v *in, uint32_t normal_len, uint32_t pop_chimer, asg64_v *dbg);
+void asg_arc_cut_chimeric(asg_t *g, ma_hit_t_alloc* src, asg64_v *in, uint32_t ou_thres);
+void asg_arc_cut_inexact(asg_t *g, ma_hit_t_alloc* src, asg64_v *in, int32_t max_ext, uint32_t is_ou, uint32_t is_trio, float ou_rat/**, asg64_v *dbg**/);
+void asg_arc_cut_length(asg_t *g, asg64_v *in, int32_t max_ext, float len_rat, float ou_rat, uint32_t is_ou, uint32_t is_trio, 
+uint32_t is_topo, ma_hit_t_alloc *rev, R_to_U* rI, uint32_t *max_drop_len);
+void asg_arc_cut_bub_links(asg_t *g, asg64_v *in, float len_rat, float sec_len_rat, float ou_rat, uint32_t is_ou, uint64_t check_dist, ma_hit_t_alloc *rev, R_to_U* rI, int32_t max_ext);
+void asg_arc_cut_complex_bub_links(asg_t *g, asg64_v *in, float len_rat, float ou_rat, uint32_t is_ou, bub_label_t *b_mask_t);
+uint32_t asg_cut_large_indel(asg_t *g, asg64_v *in, int32_t max_ext, float ou_rat, uint32_t is_ou);
+uint32_t asg_cut_semi_circ(asg_t *g, uint32_t lim_len, uint32_t is_clean);
+void ul_realignment_gfa(ug_opt_t *uopt, asg_t *sg, int64_t clean_round, double min_ovlp_drop_ratio, 
+double max_ovlp_drop_ratio, int64_t max_tip, bub_label_t *b_mask_t, uint32_t is_trio, char *o_file, ul_renew_t *ropt);
+void recover_contain_g(asg_t *g, ma_hit_t_alloc *src, R_to_U* ruIndex, int64_t max_hang, int64_t min_ovlp, int64_t ul_occ);
+void normalize_gou(asg_t *g);
+void prt_specfic_sge(asg_t *g, uint32_t src, uint32_t dst, const char* cmd);
+asg_t *gen_ng(ma_ug_t *ug, asg_t *sg, ug_opt_t *uopt, ma_sub_t **cov, R_to_U *ruI, uint64_t scaffold_len);
+void post_rescue(ug_opt_t *uopt, asg_t *sg, ma_hit_t_alloc *src, ma_hit_t_alloc *rev, R_to_U* rI, bub_label_t *b_mask_t, long long no_trio_recover);
+// void print_raw_u2rgfa_seq(all_ul_t *aln, R_to_U* rI, uint32_t is_detail);
+
+#endif

hic.h

@@ -10,8 +10,7 @@
 #define RC_2 2
 
 hc_edge* get_hc_edge(hc_links* link, uint64_t src, uint64_t dest, uint64_t dir);
-void push_hc_edge(hc_linkeage* x, uint64_t uID, double weight, int dir, uint64_t* d);
-void hic_analysis(ma_ug_t *ug, asg_t* read_g, hc_links* link);
+hc_edge* push_hc_edge(hc_linkeage* x, uint64_t uID, double weight, int dir, uint64_t* d);
 void hic_benchmark(ma_ug_t *ug, asg_t* read_g);
 
 typedef struct {
@@ -49,6 +48,34 @@ typedef struct {
     kvec_t(chain_w_type) chain_weight;
     chain_hic_warp c_w;
 } bubble_type;
+
+typedef struct {
+    int8_t *s;
+    uint64_t xs;
+    uint64_t n;
+} ps_t;
+
+typedef struct {
+	uint64_t s, e, id, len;
+} pe_hit;
+
+typedef struct {
+    kvec_t(pe_hit) a;
+    kvec_t(uint64_t) idx;
+    kvec_t(uint64_t) occ;
+    uint64_t uID_bits;
+    uint64_t pos_mode;
+} kvec_pe_hit;
+
+typedef struct{
+    kvec_t(uint8_t) vis;
+    kvec_t(uint64_t) x;
+    kvec_t(uint64_t) dis;
+    uint64_t uID_mode, uID_shift, tmp_v, tmp_d;
+}pdq;
+
+
+
 #define P_het(B) ((B).num.n)
 #define M_het(B) ((B).num.n + 1)
 // #define IF_BUB(ID, B) ((B).index[(ID)] < (B).num.n)
@@ -62,11 +89,25 @@ void get_bubbles(bubble_type* bub, uint64_t id, uint32_t* beg, uint32_t* sink, u
 int load_hc_links(hc_links* link, const char *fn);
 void write_hc_links(hc_links* link, const char *fn);
 void destory_bubbles(bubble_type* bub);
-void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link);
+void identify_bubbles(ma_ug_t* ug, bubble_type* bub, uint8_t *r_het_flag, kv_u_trans_t *ref);
+void identify_bubbles_recal(asg_t* sg, ma_ug_t* ug, bubble_type* bub, uint8_t *r_het_flag, ma_hit_t_alloc* sources, R_to_U* ruIndex,
+kv_u_trans_t *ref);
 void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub);
 uint32_t connect_bub_occ(bubble_type* bub, uint32_t root_id, uint32_t check_het);
 void get_bub_id(bubble_type* bub, uint32_t root, uint64_t* id0, uint64_t* id1, uint32_t check_het);
 void update_bubble_chain(ma_ug_t* ug, bubble_type* bub, uint32_t is_middle, uint32_t is_end);
 void set_b_utg_weight_flag(bubble_type* bub, buf_t* b, uint32_t v, uint8_t* vis_flag, uint32_t flag, uint32_t* occ);
-
+void debug_gfa_space(ma_ug_t* ug, hap_cov_t *cov);
+void init_ug_idx(ma_ug_t *ug, uint64_t k, uint64_t up_bound, uint64_t low_bound, uint64_t build_idx);
+void des_ug_idx();
+uint64_t count_unique_k_mers(char *r, uint64_t len, uint64_t query, uint64_t target, uint64_t *all, uint64_t *found);
+void init_pdq(pdq* q, uint64_t utg_num);
+void destory_pdq(pdq* q);
+uint32_t check_trans_relation_by_path(uint32_t v, uint32_t w, pdq* pqv, uint32_t* path_v, buf_t *resv,
+pdq* pqw, uint32_t* path_w, buf_t *resw, asg_t *sg, uint8_t *dest, uint8_t df, uint32_t df_occ, double rate,
+long long *dis);
+void set_utg_by_dis(uint32_t v, pdq* pq, asg_t *g, kvec_t_u32_warp *res, uint32_t dis);
+void dedup_hits(kvec_pe_hit* hits, uint64_t is_dup);
+void hic_analysis(ma_ug_t *ug, asg_t* read_g, trans_chain* t_ch, ug_opt_t *opt, uint32_t is_poy, kvec_pe_hit **rhits);
+spg_t *hic_pre_analysis(ma_ug_t *ug, asg_t* read_g, trans_chain* t_ch, ug_opt_t *opt, kvec_pe_hit **rhits);
 #endif

hifiasm.1

@@ -1,4 +1,4 @@
-.TH hifiasm 1 "19 July 2020" "hifiasm-0.9 (r289)" "Bioinformatics tools"
+.TH hifiasm 1 "22 August 2021" "hifiasm-0.16.0 (r369)" "Bioinformatics tools"
 
 .SH NAME
 .PP
@@ -62,7 +62,7 @@ Hifiasm is an ultrafast haplotype-resolved de novo assembler for PacBio
 Hifi reads. Unlike most existing assemblers, hifiasm starts from uncollapsed
 genome. Thus, it is able to keep the haplotype information as much as possible.
 The input of hifiasm is the PacBio Hifi reads in fasta/fastq format, and its
-outputs consist of multiple types of assembly graph in GFA format.
+outputs consist of multiple types of assembly graphs in GFA format.
 
 
 .SH OPTIONS
@@ -104,17 +104,82 @@ Minimizer window size [51].
 Number of bits for bloom filter; 0 to disable [37]. This bloom filter is used
 to filter out singleton k-mers when counting all k-mers. It takes
 .RI 2^( INT -3)
-bytes of memory. A proper setting saves memory. 37 is recommended for human
-assembly.
+bytes of memory. A proper setting saves memory. 
+.BR -f37 
+is recommended for human
+assembly. For small genomes, use 
+.BR -f0 
+to disable the initial bloom filter 
+which takes 16GB memory at the beginning. For genomes much larger 
+than human, applying 
+.BR -f38 
+or even 
+.BR -f39 
+is preferred to save memory on k-mer counting.
+
+.TP
+.BI -D \ INT
+Drop k-mers occurring 
+.I >FLOAT*coverage 
+times [5.0]. 
+Hifiasm discards these high-frequency k-mers 
+during error correction to reduce running time. 
+The 
+.I coverage 
+is determined automatically 
+by hifiasm based on k-mer plot, representing 
+homozygous read coverage. Raising this option 
+may improve the resolution of repetitive regions 
+but takes longer time.
+
+.TP
+.BI -N \ INT
+Consider up to 
+.I max(-D*coverage,-N) 
+overlaps for each oriented read [100]. 
+The 
+.I coverage 
+is determined automatically 
+by hifiasm based on k-mer plot, representing 
+homozygous read coverage. Raising this option may 
+improve the resolution of repetitive regions but 
+takes longer time.
+
 
 .TP
 .BI -r \ INT
-Rounds of haplotype-aware error corrections [3]. This option affects all outputs of hifiasm.
+Rounds of haplotype-aware error corrections [3]. 
+This option affects all outputs of hifiasm. 
+Odd rounds of correction are preferred in practice.
+
+.TP
+.BI -z \ INT
+Length of adapters that should be removed [0]. This option remove
+.I INT
+bases from both ends of each read.
+Some old Hifi reads may consist of
+short adapters (e.g., 20bp adapter at one end). For such data, trimming short adapters would 
+significantly improve the assembly quality.
+
+.TP
+.BI --max-kocc \ INT
+Employ k-mers occurring < 
+.IR INT 
+times to rescue repetitive overlaps [2000]. 
+This option may improve the resolution of repeats.
+
+.TP
+.BI --hg-size \ INT (k/m/g)
+Estimated haploid genome size used for inferring read coverage [auto]. 
+This option is used to get accurate homozygous read coverage during 
+error correction. Common suffices are required, for example, 100m or 3g.
 
 .TP
 .BI --min-hist-cnt \ INT
 When analyzing the k-mer spectrum, ignore counts below
-.IR INT .
+.IR INT 
+[5]. For very low coverage of HiFi data, set smaller 
+value for this option.
 
 .SS Assembly options
 
@@ -128,14 +193,6 @@ Note that unlike
 .BR -r ,
 this option does not affect error corrected reads and all-to-all overlaps.
 
-.TP
-.BI -z \ INT
-Length of adapters that should be removed [0]. This option remove
-.I INT
-bases from both ends of each read.
-Some old Hifi reads may consist of
-short adapters (e.g., 20bp adapter at one end). For such data, trimming short adapters would 
-significantly improve the assembly quality.
 
 .TP
 .BI -m \ INT
@@ -149,10 +206,13 @@ section of this man-page.
 .TP
 .BI -p \ INT
 Maximal probing distance for bubble popping when generating haplotype-resolved processed unitig graph
-without small bubbles [100000]. Bubbles longer than
+without small bubbles [0]. Bubbles longer than
 .I INT
-bases will not be popped. Small bubbles might be caused by somatic mutations or noise in data, which
-are not the real haplotype information. For detailed description of this graph, please see the
+bases will not be popped. Small bubbles might be caused by somatic mutations or noise in data. 
+Please note that hifiasm automatically pops small bubbles based on coverage, 
+which can be tweaked by 
+.BR --hom-cov .
+For detailed description of this graph, please see the
 .B OUTPUTS
 section of this man-page. 
 
@@ -163,13 +223,13 @@ A unitig is considered small if it is composed of less than
 reads [3]. Hifiasm may try to remove small unitigs at various steps.
 
 .TP
-.BI -x \ FLOAT, -y \ FLOAT
+.BI -x \ FLOAT1, -y \ FLOAT2
 Max and min overlap drop ratio [0.8, 0.2]. This option is used with
-.BR -r .
+.BR -a .
 Given a node
 .I N
 in the assembly graph, let max(N)
-be the length of the largest overlap of
+be the length of the longest overlap of
 .I N.
 Hifiasm iteratively drops overlaps of
 .I N
@@ -179,17 +239,17 @@ are below a threshold controlled by
 and
 .BR -y .
 Hifiasm applies
-.B -r
+.B -a
 rounds of short overlap removal with an increasing threshold between
-.B -x
+.I FLOAT1
 and
-.BR -y .
+.I FLOAT2.
 
 .TP
 .BI -i
 Ignore error corrected reads and overlaps saved in
 .IR prefix .*.bin
-files.
+files so that hifiasm will start again from scratch.
 Apart from assembly graphs, hifiasm also outputs three binary files
 that save all overlap information during assembly step.
 With these files, hifiasm can avoid the time-consuming all-to-all overlap calculation step,
@@ -197,12 +257,24 @@ and do the assembly directly and quickly.
 This might be helpful when users want to get an optimized assembly by multiple rounds of experiments
 with different parameters.
 
+.TP
+.BI -u
+Disable post-join step for contigs which may improve N50. 
+The post-join step of hifiasm improves contig N50 but may introduce misassemblies.
+
+.TP
+.BI --hom-cov \ INT
+Homozygous read coverage inferred automatically in default. This option affects different types of outputs, 
+including Hi-C phased assembly and HiFi-only assembly. 
+
 .TP
 .BI --pri-range \ INT1[,INT2]
 Min and max coverage cutoff of primary contigs.
 Keep contigs with coverage in this range at p_ctg.gfa. 
 Inferred automatically in default.
-If INT2 is not specified, it is set to infinity. 
+If 
+.I INT2 
+is not specified, it is set to infinity. 
 Set -1 to disable.
 
 .TP
@@ -212,6 +284,38 @@ with suffix
 .B lowQ.bed
 [70]. Set 0 to disable. 
 
+
+.TP
+.BI --b-cov \ INT
+Break contigs at potential misassemblies with <INT-fold coverage [0].
+Work with 
+.B --m-rate.
+Set 0 to disable. 
+
+.TP
+.BI --h-cov \ INT
+Break contigs at potential misassemblies with >INT-fold coverage [-1].
+Work with 
+.B --m-rate.
+Set -1 to disable. 
+
+.TP
+.BI --m-rate \ FLOAT
+Break contigs with <=FLOAT*coverage exact overlaps [0.75].
+Only work with 
+.B --b-cov
+and
+.B --h-cov.
+
+.TP
+.BI --primary
+Output a primary assembly and an alternate assembly.
+Hifiasm outputs two balanced assemblies and a primary
+assembly in default. Enable this option or 
+.B -l0
+outputs a primary assembly and an alternate assembly.
+
+
 .SS Trio-partition options
 
 .TP 10
@@ -254,49 +358,86 @@ but occurs <
 .B -c
 times in the other sample.
 
+.TP
+.BI --t-occ \ INT
+Forcedly remove unitig including >
+.I INT 
+unexpected haplotype-specific reads 
+without considering graph topology [60].
+
 
 .SS Purge-dups options
 
 .TP 10
 .BI -l \ INT
 Level of purge-dup. 0 to disable purge-dup, 1 to only purge contained haplotigs, 
-2 to purge all types of haplotigs. In default, [2] for non-trio assembly, [0] for trio assembly.
+2 to purge all types of haplotigs, 3 to purge all types of haplotigs in most aggressive way. 
+In default, [3] for non-trio assembly, [0] for trio assembly.
 For trio assembly, only level 0 and level 1 are allowed.
 
 .TP
 .BI -s \ FLOAT
-Similarity threshold for duplicate haplotigs that should be purged [0.75].
+Similarity threshold for duplicate haplotigs that should be purged [0.75 for 
+.BR -l1/-l2 , 
+0.55 for 
+.BR -l3 ].
 
 .TP
 .BI -O \ FLOAT
 Min number of overlapped reads for duplicate haplotigs that should be purged [1].
 
 .TP
-.BI --purge-cov \ INT
+.BI --purge-max \ INT
 Coverage upper bound of Purge-dups, which is inferred automatically in default.
-If the coverage of a contig is higher than this bound, don't apply Purge-dups.
+If the coverage of a contig is higher than this bound, don't apply Purge-dups. 
+Larger value makes assembly more contiguous but may collapse repeats or segmental duplications.
 
 .TP
-.BI --high-het \ INT
-Enable this mode for high heterozygosity sample, which will increase running time. 
-For ordinary samples, no need to enable this mode [experimental, not stable].
+.BI   --n-hap \ INT
+Assumption of haplotype number. If it is set to >2, the quality of 
+primary assembly for polyploid genomes might be improved.
 
 
-.SS Debugging options
 
-.TP 10
-.B --dbg-gfa
-Write additional files to speed up the debugging of graph cleaning.
+.SS Hi-C-partition options [experimental, not stable]
 
+.TP
+.BI --h1 \ FILEs 
+File names of input Hi-C R1 [r1_1.fq,r1_2.fq,...].
+
+.TP
+.BI --h2 \ FILEs 
+File names of input Hi-C R2 [r2_1.fq,r2_2.fq,...].
+
+.TP
+.BI --n-weight \ INT 
+Rounds of reweighting Hi-C links [3]. Increasing this may improves
+phasing results but takes longer time.
+
+.TP
+.BI --n-perturb \ INT 
+Rounds of perturbation [10000]. Increasing this may improves
+phasing results but takes longer time.
+
+.TP
+.BI --f-perturb \ FLOAT 
+Fraction to flip for perturbation [0.1]. Increasing this may improves
+phasing results but takes longer time.
+
+.TP
+.BI --l-msjoin \ INT 
+Detect misjoined unitigs of >=
+.I INT 
+in size; 0 to disable [500000].
+
+.TP
+.BI --seed \ INT 
+RNG seed [11].
 
 .SH OUTPUTS
 
 .PP
-Without trio partition options
-.B -1
-and
-.BR -2 ,
-hifiasm generates the following assembly graphs in the GFA format:
+In general, hifiasm generates the following assembly graphs in the GFA format:
 
 .RS 2
 .TP 2
@@ -309,44 +450,138 @@ haplotype-resolved raw unitig graph. This graph keeps all haplotype information.
 .IR prefix .p_utg.gfa:
 haplotype-resolved processed unitig graph without small bubbles.  Small bubbles
 might be caused by somatic mutations or noise in data, which are not the real
-haplotype information.  The size of popped small bubbles should be specified by
-.BR -p .
+haplotype information. Hifiasm automatically pops such small bubbles based on coverage. 
+The option 
+.BR --hom-cov 
+affects the result.
+In addition, the option 
+.BR -p 
+forcedly pops bubbles.
 
 .TP
 *
 .IR prefix .p_ctg.gfa:
-assembly graph of primary contigs. This graph collapses different haplotypes.
+assembly graph of primary contigs. This graph includes a complete assembly with 
+long stretches of phased blocks.
 
 .TP
 *
 .IR prefix .a_ctg.gfa:
-assembly graph of alternate contigs. This graph consists of all assemblies that
+assembly graph of alternate contigs. This graph consists of all contigs that
 are discarded in primary contig graph.
 
+.TP
+*
+.IR prefix .*hap*.p_ctg.gfa:
+phased contig graph. This graph keeps the phased assembly.
+
 .RE
 
 .PP
-With trio partition, hifiasm outputs the following assembly graphs:
+Hifiasm outputs
+.B *.r_utg.gfa
+and
+.B *.p_utg.gfa
+in any cases.
+Specifically, hifiasm outputs the following assembly graphs
+with trio-binning options:
 
 .RS 2
 .TP 2
 *
-.IR prefix .dip.r_utg.gfa:
-haplotype-resolved raw unitig graph. This graph keeps all haplotype information.
-
-.TP
-*
-.IR prefix .hap1.p_ctg.gfa:
-phased paternal/haplotype1 contig graph. This graph keeps the phased
+.IR prefix .dip.hap1.p_ctg.gfa:
+fully phased paternal/haplotype1 contig graph keeping the phased
 paternal/haplotype1 assembly.
 
 .TP
 *
-.IR prefix .hap2.p_ctg.gfa:
-phased maternal/haplotype2 contig graph. This graph keeps the phased
+.IR prefix .dip.hap2.p_ctg.gfa:
+fully phased maternal/haplotype2 contig graph keeping the phased
 maternal/haplotype2 assembly.
 .RE
 
+.PP
+With Hi-C partition options, hifiasm outputs:
+
+.RS 2
+.TP 2
+*
+.IR prefix .hic.p_ctg.gfa:
+assembly graph of primary contigs.
+
+.TP
+*
+.IR prefix .hic.hap1.p_ctg.gfa:
+fully phased contig graph where each contig is fully phased.
+
+.TP
+*
+.IR prefix .hic.hap2.p_ctg.gfa:
+fully phased contig graph where each contig is fully phased.
+
+.TP
+*
+.IR prefix .hic.a_ctg.gfa 
+(optional with 
+.BR --primary): 
+assembly graph of alternate contigs.
+
+.RE
+
+
+.PP
+Hifiasm keeps Hi-C alignment results and Hi-C index in two bin 
+files: 
+.B *hic.lk.bin
+and 
+.B *hic.tlb.bin.
+Rerunning hifiasm with different Hi-C reads needs to delete these bin files
+or use 
+.BR -i .
+.RE
+
+.PP
+Hifiasm generates the following assembly graphs only with HiFi reads:
+
+.RS 2
+.TP 2
+*
+.IR prefix .bp.p_ctg.gfa:
+assembly graph of primary contigs. 
+
+.TP
+*
+.IR prefix .bp.hap1.p_ctg.gfa:
+partially phased contig graph of haplotype1.
+
+.TP
+*
+.IR prefix .bp.hap2.p_ctg.gfa:
+partially phased contig graph of haplotype2.
+.RE
+
+.PP
+If the option
+.BR -l0
+or
+.BR --primary
+is specified, hifiasm outputs:
+
+.RS 2
+.TP 2
+*
+.IR prefix .p_ctg.gfa:
+assembly graph of primary contigs.
+
+.TP
+*
+.IR prefix .a_ctg.gfa:
+assembly graph of alternate contigs.
+.RE
+
+
+
+
 .PP
 For each graph, hifiasm also outputs a simplified version without sequences for
 the ease of visualization. Hifiasm keeps corrected reads and overlaps in three

hist.cpp

@@ -12,7 +12,66 @@ static void ha_hist_line(int c, int x, int exceed, int64_t cnt)
 	fprintf(stderr, " %lld\n", (long long)cnt);
 }
 
-int ha_analyze_count(int n_cnt, int start_cnt, const int64_t *cnt, int *peak_het)
+void print_hist_lines(int n_cnt, int start_cnt, const int64_t *cnt)
+{
+	const int hist_max = 100;
+	int i, start, low_i, max_i, max;
+	// determine the start point
+	assert(n_cnt > start_cnt);
+	start = cnt[1] > 0? 1 : 2;
+
+	// find the low point from the left
+	low_i = start > start_cnt? start : start_cnt;
+	for (i = low_i; i < n_cnt; ++i)
+		if (cnt[i] > cnt[i-1]) break;
+	low_i = i - 1;
+	fprintf(stderr, "[M::%s] lowest: count[%d] = %ld\n", __func__, low_i, (long)cnt[low_i]);
+
+	// find the highest peak
+	max_i = start > start_cnt? start : start_cnt, max = cnt[max_i];
+	for (i = max_i; i < n_cnt; ++i)
+		if (cnt[i] > max)
+			max = cnt[i], max_i = i;
+	fprintf(stderr, "[M::%s] highest: count[%d] = %ld\n", __func__, max_i, (long)cnt[max_i]);
+
+	for (i = start; i < n_cnt; ++i) {
+		int x, exceed = 0;
+		x = (int)((double)hist_max * cnt[i] / cnt[max_i] + .499);
+		if (x > hist_max) exceed = 1, x = hist_max; // may happen if cnt[2] is higher
+		if (i > max_i && x == 0) break;
+		ha_hist_line(i, x, exceed, cnt[i]);
+	}
+}
+
+int adj_m_peak_hom(int m_peak_hom, int max_i, int max2_i, int max3_i, int *peak_het)
+{
+	int64_t mm[3], d, min_i, min_d, i;
+	mm[0] = max2_i; mm[1] = max_i; mm[2] = max3_i;
+	for (i = 0, min_i = -1, min_d = -1; i < 3; i++){
+		if(mm[i] <= 0) continue;
+		d = (mm[i] >= m_peak_hom?mm[i]-m_peak_hom:m_peak_hom-mm[i]);
+		if(min_d == -1 || min_d > d || (min_d == d && i == 1)){
+			min_d = d; min_i = i;
+		}
+	}
+	if(min_i < 0) return m_peak_hom;
+	if(mm[min_i] < m_peak_hom){
+		d = m_peak_hom - mm[min_i];
+		if(d >= mm[min_i]*0.51) {
+			*peak_het = mm[min_i];
+			return m_peak_hom;
+		}
+	}
+
+	for (i = min_i-1; i >= 0; i--){
+		if(mm[i] <= 0) continue;
+		*peak_het = mm[i];
+		break;
+	}
+	return mm[min_i];
+}
+
+int ha_analyze_count(int n_cnt, int start_cnt, int m_peak_hom, const int64_t *cnt, int *peak_het)
 {
 	const int hist_max = 100;
 	int i, start, low_i, max_i, max2_i, max3_i;
@@ -86,6 +145,8 @@ int ha_analyze_count(int n_cnt, int start_cnt, const int64_t *cnt, int *peak_het
 	}
 	if (max3 > 0) fprintf(stderr, "[M::%s] right: count[%d] = %ld\n", __func__, max3_i, (long)cnt[max3_i]);
 	else fprintf(stderr, "[M::%s] right: none\n", __func__);
+
+	if(m_peak_hom > 0) return adj_m_peak_hom(m_peak_hom, max_i, max2_i, max3_i, peak_het);
 	if (max3_i > 0) {
 		*peak_het = max_i;
 		return max3_i;

horder.h

@@ -0,0 +1,72 @@
+#ifndef __HORDER__
+#define __HORDER__
+#include <stdint.h>
+#include "hic.h"
+
+#define get_hit_srev(x, k) ((x).a.a[(k)].s>>63)
+#define get_hit_slen(x, k) ((x).a.a[(k)].len>>32)
+#define get_hit_suid(x, k) (((x).a.a[(k)].s<<1)>>(64 - (x).uID_bits))
+#define get_hit_spos(x, k) ((x).a.a[(k)].s & (x).pos_mode)
+#define get_hit_spos_e(x, k) (get_hit_srev((x),(k))?\
+        ((get_hit_spos((x),(k))+1>=get_hit_slen((x),(k)))?\
+            (get_hit_spos((x),(k))+1-get_hit_slen((x),(k))):0)\
+                    :(get_hit_spos((x),(k))+get_hit_slen((x),(k))-1))
+
+#define get_hit_erev(x, k) ((x).a.a[(k)].e>>63)
+#define get_hit_elen(x, k) ((uint32_t)((x).a.a[(k)].len))
+#define get_hit_euid(x, k) (((x).a.a[(k)].e<<1)>>(64 - (x).uID_bits))
+#define get_hit_epos(x, k) ((x).a.a[(k)].e & (x).pos_mode)
+#define get_hit_epos_e(x, k) (get_hit_erev((x),(k))?\
+        ((get_hit_epos((x),(k))+1>=get_hit_elen((x),(k)))?\
+            (get_hit_epos((x),(k))+1-get_hit_elen((x),(k))):0)\
+                    :(get_hit_epos((x),(k))+get_hit_elen((x),(k))-1))
+
+typedef struct {
+	uint32_t v;
+    uint32_t u;
+	uint32_t occ:31, del:1;
+	double w, nw;
+} osg_arc_t;
+
+typedef struct {
+	double mw[2], ez[2];
+    uint8_t del;
+} osg_seq_t;
+
+typedef struct {
+	uint32_t m_arc, n_arc:31, is_srt:1;
+	osg_arc_t *arc;
+
+	uint32_t m_seq, n_seq:31, is_symm:1;
+	osg_seq_t *seq;
+
+	uint64_t *idx;
+} osg_t;
+
+typedef struct {
+    osg_t *g;
+}scg_t;
+typedef struct {
+	kvec_t(uint64_t) avoid;
+    // kvec_t(uint64_t) occ;
+    // kvec_t(uint8_t) hf;
+    kvec_pe_hit r_hits, u_hits;
+    ma_ug_t *ug;
+    asg_t *r_g;
+    scg_t sg;
+}horder_t;
+
+horder_t *init_horder_t(kvec_pe_hit *i_hits, uint64_t i_hits_uid_bits, uint64_t i_hits_pos_mode, 
+asg_t *i_rg, ma_ug_t* i_ug, bubble_type* bub, kv_u_trans_t *ref, ug_opt_t *opt, uint32_t round);
+void destory_horder_t(horder_t **h);
+void horder_clean_sg_by_utg(asg_t *sg, ma_ug_t *ug);
+kvec_pe_hit *get_r_hits_for_trio(kvec_pe_hit *u_hits, asg_t* r_g, ma_ug_t* ug, bubble_type* bub, uint64_t uID_bits, uint64_t pos_mode);
+void update_switch_unitig(ma_ug_t *ug, asg_t *rg, kvec_pe_hit *hits, kv_u_trans_t *k_trans, uint64_t cutoff_s, uint64_t cutoff_e,
+uint64_t min_ulen, double boundaryRate);
+kvec_pe_hit *get_r_hits_order(kvec_pe_hit *uhits, uint64_t hits_uid_bits, uint64_t hits_pos_mode, 
+asg_t *rg, ma_ug_t* ug, bubble_type* bub);
+void ha_aware_order(kvec_pe_hit *r_hits, asg_t *rg, ma_ug_t *ug_fa, ma_ug_t *ug_mo, kv_u_trans_t *ref, 
+ug_opt_t *opt, uint32_t round);
+spg_t *horder_utg(kvec_pe_hit *i_hits, uint64_t i_hits_uid_bits, uint64_t i_hits_pos_mode, 
+asg_t *i_rg, ma_ug_t* i_ug, bubble_type* bub, ug_opt_t *opt);
+#endif

htab.h

@@ -5,6 +5,11 @@
 #include "Process_Read.h"
 #include "CommandLines.h"
 
+typedef struct {
+	size_t n, m;
+	uint64_t *a;
+} st_mt_t;
+
 typedef struct {
 	uint64_t x; ///x is the hash key
 	///rid is the read id, pos is the end pos of this minimizer, rev is the direction
@@ -18,12 +23,48 @@ typedef struct {
 
 typedef struct { uint32_t n, m; ha_mz1_t *a; } ha_mz1_v;
 
+typedef struct {
+	uint64_t x; ///x is the hash key
+	uint64_t rid:31, rev:1, pos:32;
+	uint8_t span;
+} ha_mzl_t;
+
+typedef struct {
+	uint64_t rid:31, rev:1, pos:32;
+	uint8_t span;
+} ha_idxposl_t;
+
+typedef struct { uint32_t n, m; ha_mzl_t *a; } ha_mzl_v;
+
+typedef struct { // a simplified version of kdq
+	int front, count;
+	int a[64];
+} tiny_queue_t;
+
+static inline void tq_push(tiny_queue_t *q, int x)
+{
+	q->a[((q->count++) + q->front) & 0x3f] = x;
+}
+
+static inline int tq_shift(tiny_queue_t *q)
+{
+	int x;
+	if (q->count == 0) return -1;
+	x = q->a[q->front++];
+	q->front &= 0x3f;
+	--q->count;
+	return x;
+}
+
 struct ha_pt_s;
 typedef struct ha_pt_s ha_pt_t;
 
 struct ha_abuf_s;
 typedef struct ha_abuf_s ha_abuf_t;
 
+struct ha_abufl_s;
+typedef struct ha_abufl_s ha_abufl_t;
+
 extern const unsigned char seq_nt4_table[256];
 extern void *ha_flt_tab;
 extern ha_pt_t *ha_idx;
@@ -31,24 +72,39 @@ extern void *ha_flt_tab_hp;
 extern ha_pt_t *ha_idx_hp;
 extern void *ha_ct_table;
 
-
+void *ha_ft_ul_gen(const hifiasm_opt_t *asm_opt, ma_utg_v *us, int k, int w, int cutoff);
+void *ha_ft_ug_gen(const hifiasm_opt_t *asm_opt, ma_utg_v *us, int is_HPC, int k, int w, int min_freq, int max_freq);
 void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int is_hp_mode);
-int ha_ft_isflt(const void *hh, uint64_t y);
+int32_t ha_ft_cnt(const void *hh, uint64_t y);
 void ha_ft_destroy(void *h);
 
+ha_pt_t *ha_pt_ul_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, ma_utg_v *us, int k, int w, int cutoff);
+ha_pt_t *ha_pt_ug_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, ma_utg_v *us, int is_HPC, int k, int w, int min_freq);
 ha_pt_t *ha_pt_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, int read_from_store, int is_hp_mode, All_reads *rs, int *hom_cov, int *het_cov);
 void ha_pt_destroy(ha_pt_t *h);
 const ha_idxpos_t *ha_pt_get(const ha_pt_t *h, uint64_t hash, int *n);
+const ha_idxposl_t *ha_ptl_get(const ha_pt_t *h, uint64_t hash, int *n);
+const int ha_pt_cnt(const ha_pt_t *h, uint64_t hash);
 
 int write_pt_index(void *flt_tab, ha_pt_t *ha_idx, All_reads* r, hifiasm_opt_t* opt, char* file_name);
 int load_pt_index(void **r_flt_tab, ha_pt_t **r_ha_idx, All_reads* r, hifiasm_opt_t* opt, char* file_name);
+int uidx_write(void *flt_tab, ha_pt_t *ha_idx, char* file_name, ma_ug_t *ug);
+int uidx_load(void **r_flt_tab, ha_pt_t **r_ha_idx, char* file_name, ma_ug_t *ug);
 int write_ct_index(void *ct_idx, char* file_name);
 int load_ct_index(void **ct_idx, char* file_name);
 int query_ct_index(void* ct_idx, uint64_t hash);
 
+ha_abuf_t *ha_abuf_init_buf(void *km);
+ha_abufl_t *ha_abufl_init_buf(void *km);
+void ha_abuf_destroy_buf(void *km, ha_abuf_t *ab);
+void ha_abufl_destroy_buf(void *km, ha_abufl_t *ab);
+void ha_abufl_free_buf(void *km, ha_abufl_t *ab, int is_z);
 ha_abuf_t *ha_abuf_init(void);
 void ha_abuf_destroy(ha_abuf_t *ab);
 uint64_t ha_abuf_mem(const ha_abuf_t *ab);
+ha_abufl_t *ha_abufl_init(void);
+void ha_abufl_destroy(ha_abufl_t *ab);
+uint64_t ha_abufl_mem(const ha_abufl_t *ab);
 
 double yak_cputime(void);
 void yak_reset_realtime(void);
@@ -58,12 +114,24 @@ double yak_peakrss_in_gb(void);
 double yak_cpu_usage(void);
 
 void ha_triobin(const hifiasm_opt_t *opt);
+uint32_t *ha_polybin_list(const hifiasm_opt_t *opt);
 
-void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf);
-void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct);
-int ha_analyze_count(int n_cnt, int start_cnt, const int64_t *cnt, int *peak_het);
+void mz1_ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, int sample_dist, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct, ha_pt_t *pt, int min_freq, int32_t dp_min_len, float dp_e, st_mt_t *mt, int32_t ws, int32_t is_unique, void *km);
+void mz2_ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mzl_v *p, const void *hf, int sample_dist, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct, ha_pt_t *pt, int min_freq, int32_t dp_min_len, float dp_e, st_mt_t *mt, int32_t ws, int32_t is_unique, void *km);
+int ha_analyze_count(int n_cnt, int start_cnt, int m_peak_hom, const int64_t *cnt, int *peak_het);
+int adj_m_peak_hom(int m_peak_hom, int max_i, int max2_i, int max3_i, int *peak_het);
+void print_hist_lines(int n_cnt, int start_cnt, const int64_t *cnt);
 void debug_adapter(const hifiasm_opt_t *asm_opt, All_reads *rs);
 
+inline int mz_low_b(int peak_hom, int peak_het)
+{
+	int low_freq = 2;
+	if(peak_het > 0) low_freq = peak_het/2;
+	else if(peak_hom > 0) low_freq = peak_hom/4;
+	if(low_freq < 2) low_freq = 2;
+	return low_freq;
+}
+
 static inline uint64_t yak_hash64(uint64_t key, uint64_t mask) // invertible integer hash function
 {
 	key = (~key + (key << 21)) & mask; // key = (key << 21) - key - 1;
@@ -98,6 +166,7 @@ static inline uint64_t yak_hash_long(uint64_t x[4])
 #define CALLOC(ptr, len) ((ptr) = (__typeof__(ptr))calloc((len), sizeof(*(ptr))))
 #define MALLOC(ptr, len) ((ptr) = (__typeof__(ptr))malloc((len) * sizeof(*(ptr))))
 #define REALLOC(ptr, len) ((ptr) = (__typeof__(ptr))realloc((ptr), (len) * sizeof(*(ptr))))
+#define MEMCPY(dest, src, len) (memcpy((dest), (src), (len) * sizeof(*(src))))
 
 #ifndef kroundup32
 #define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))

inter.h

@@ -0,0 +1,120 @@
+#ifndef __INTER__
+#define __INTER__
+#include "Overlaps.h"
+#include "Process_Read.h"
+
+#define G_CHAIN_BW 16//128
+#define FLANK_M (0x7fffU)
+#define P_CHAIN_COV 0.985
+#define P_FRAGEMENT_CHAIN_COV 0.20
+#define P_FRAGEMENT_PRIMARY_CHAIN_COV 0.70
+#define P_FRAGEMENT_PRIMARY_SECOND_COV 0.25
+#define P_CHAIN_SCORE 0.6
+#define G_CHAIN_GAP 0.1
+#define UG_SKIP 5
+#define RG_SKIP 25
+#define UG_SKIP_GRAPH_N 72
+#define UG_SKIP_N 100
+#define UG_ITER_N 5000
+#define UG_DIS_N 50000
+// #define UG_TRANS_W 2
+#define UG_TRANS_W 2
+// #define UG_TRANS_ERR_W 512
+#define UG_TRANS_ERR_W 64
+#define G_CHAIN_TRANS_RATE 0.25
+#define G_CHAIN_TRANS_WEIGHT -1
+#define G_CHAIN_INDEL 128
+#define W_CHN_PEN_GAP 0.1
+#define N_GCHAIN_RATE 0.04
+#define PRIMARY_UL_CHAIN_MIN 75000
+
+typedef struct {
+	int w, k, bw, max_gap, is_HPC, hap_n, occ_weight, max_gap_pre, max_gc_seq_ext, seed;
+	int max_lc_skip, max_lc_iter, min_lc_cnt, min_lc_score, max_gc_skip, ref_bonus;
+	int min_gc_cnt, min_gc_score, sub_diff, best_n;
+    float chn_pen_gap, mask_level, pri_ratio;
+	///base-alignment
+	double bw_thres, diff_ec_ul, diff_ec_ul_low, diff_ec_ul_hpc; int max_n_chain, ec_ul_round;
+} mg_idxopt_t;
+
+struct mg_tbuf_s {
+	void *km;
+	int frag_gap;
+};
+typedef struct mg_tbuf_s mg_tbuf_t;
+
+
+mg_tbuf_t *mg_tbuf_init(void);
+
+void mg_tbuf_destroy(mg_tbuf_t *b);
+
+void *mg_tbuf_get_km(mg_tbuf_t *b);
+
+typedef struct {
+	FILE *fp;
+	ul_vec_t u;
+	uint64_t flag;
+} ucr_file_t;
+
+typedef struct {
+	int32_t off, cnt;
+	uint32_t v;
+	int32_t score;
+} mg_llchain_t;
+
+typedef struct {
+	int32_t id, parent;
+	int32_t off, cnt;
+	int32_t n_anchor, score;
+	int32_t qs, qe;
+	int32_t plen, ps, pe;
+	int32_t blen, mlen;
+	float div;
+	uint32_t hash;
+	int32_t subsc, n_sub;
+	uint32_t mapq:8, flt:1, dummy:23;
+} mg_gchain_t;
+
+typedef struct {
+	size_t n,m;
+    uint64_t *a, tl;
+	kvec_t(char) cc;
+} mg_dbn_t;
+
+typedef struct {
+	int32_t cnt;
+	uint32_t v;
+	int32_t score;
+	uint32_t qs, qe, ts, te;
+} mg_lres_t;
+
+typedef struct {
+    int32_t n_gc, n_lc;
+    mg_gchain_t *gc;///g_chain; idx in l_chains
+    mg_lres_t *lc;///l_chain
+    uint64_t qid, qlen;
+} mg_gres_t;
+
+typedef struct {
+	size_t n,m;
+    mg_gres_t *a;
+	uint64_t total_base;
+    uint64_t total_pair;
+} mg_gres_a;
+
+void push_uc_block_t(const ug_opt_t *uopt, kv_ul_ov_t *z, char **seq, uint64_t *len, uint64_t b_id);
+void ul_resolve(ma_ug_t *ug, const asg_t *rg, const ug_opt_t *uopt, int hap_n);
+void ul_load(const ug_opt_t *uopt);
+uint64_t* get_hifi2ul_list(all_ul_t *x, uint64_t hid, uint64_t* a_n);
+uint64_t ul_refine_alignment(const ug_opt_t *uopt, asg_t *sg);
+ma_ug_t *ul_realignment(const ug_opt_t *uopt, asg_t *sg, uint32_t double_check_cache);
+int32_t write_all_ul_t(all_ul_t *x, char* file_name, ma_ug_t *ug);
+int32_t load_all_ul_t(all_ul_t *x, char* file_name, All_reads *hR, ma_ug_t *ug);
+uint32_t ugl_cover_check(uint64_t is, uint64_t ie, ma_utg_t *u);
+void filter_ul_ug(ma_ug_t *ug);
+void gen_ul_vec_rid_t(all_ul_t *x, All_reads *rdb, ma_ug_t *ug);
+void update_ug_arch_ul_mul(ma_ug_t *ug);
+void print_ul_alignment(ma_ug_t *ug, all_ul_t *aln, uint32_t id, const char* cmd);
+void clear_all_ul_t(all_ul_t *x);
+
+#endif

kalloc.cpp

@@ -0,0 +1,205 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "kalloc.h"
+
+/* In kalloc, a *core* is a large chunk of contiguous memory. Each core is
+ * associated with a master header, which keeps the size of the current core
+ * and the pointer to next core. Kalloc allocates small *blocks* of memory from
+ * the cores and organizes free memory blocks in a circular single-linked list.
+ *
+ * In the following diagram, "@" stands for the header of a free block (of type
+ * header_t), "#" for the header of an allocated block (of type size_t), "-"
+ * for free memory, and "+" for allocated memory.
+ *
+ * master        This region is core 1.          master           This region is core 2.
+ *      |                                             |
+ *      *@-------#++++++#++++++++++++@--------        *@----------#++++++++++++#+++++++@------------
+ *       |                           |                 |                               |
+ *       p=p->ptr->ptr->ptr->ptr     p->ptr            p->ptr->ptr                     p->ptr->ptr->ptr
+ */
+typedef struct header_t {
+	size_t size;
+	struct header_t *ptr;
+} header_t;
+
+typedef struct {
+	void *par;
+	size_t min_core_size;
+	header_t base, *loop_head, *core_head; /* base is a zero-sized block always kept in the loop */
+} kmem_t;
+
+static void panic(const char *s)
+{
+	fprintf(stderr, "%s\n", s);
+	abort();
+}
+
+void *km_init2(void *km_par, size_t min_core_size)
+{
+	kmem_t *km;
+	km = (kmem_t*)kcalloc(km_par, 1, sizeof(kmem_t));
+	km->par = km_par;
+	km->min_core_size = min_core_size > 0? min_core_size : 0x80000;
+	return (void*)km;
+}
+
+void *km_init(void) { return km_init2(0, 0); }
+
+void km_destroy(void *_km)
+{
+	kmem_t *km = (kmem_t*)_km;
+	void *km_par;
+	header_t *p, *q;
+	if (km == NULL) return;
+	km_par = km->par;
+	for (p = km->core_head; p != NULL;) {
+		q = p->ptr;
+		kfree(km_par, p);
+		p = q;
+	}
+	kfree(km_par, km);
+}
+
+static header_t *morecore(kmem_t *km, size_t nu)
+{
+	header_t *q;
+	size_t bytes, *p;
+	nu = (nu + 1 + (km->min_core_size - 1)) / km->min_core_size * km->min_core_size; /* the first +1 for core header */
+	bytes = nu * sizeof(header_t);
+	q = (header_t*)kmalloc(km->par, bytes);
+	if (!q) panic("[morecore] insufficient memory");
+	q->ptr = km->core_head, q->size = nu, km->core_head = q;
+	p = (size_t*)(q + 1);
+	*p = nu - 1; /* the size of the free block; -1 because the first unit is used for the core header */
+	kfree(km, p + 1); /* initialize the new "core"; NB: the core header is not looped. */
+	return km->loop_head;
+}
+
+void kfree(void *_km, void *ap) /* kfree() also adds a new core to the circular list */
+{
+	header_t *p, *q;
+	kmem_t *km = (kmem_t*)_km;
+	
+	if (!ap) return;
+	if (km == NULL) {
+		free(ap);
+		return;
+	}
+	p = (header_t*)((size_t*)ap - 1);
+	p->size = *((size_t*)ap - 1);
+	/* Find the pointer that points to the block to be freed. The following loop can stop on two conditions:
+	 *
+	 * a) "p>q && p<q->ptr": @------#++++++++#+++++++@-------    @---------------#+++++++@-------
+	 *    (can also be in    |      |                |        -> |                       |
+	 *     two cores)        q      p           q->ptr           q                  q->ptr
+	 *
+	 *                       @--------    #+++++++++@--------    @--------    @------------------
+	 *                       |            |         |         -> |            |
+	 *                       q            p    q->ptr            q       q->ptr
+	 *
+	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------#+++++   @--------#+++++++     @-------#+++++   @----------------
+	 *                                       |                |        |         -> |                |
+	 *                                  q->ptr                q        p       q->ptr                q
+	 *
+	 *                                       #+++++++@-----   #++++++++@-------     @-------------   #++++++++@-------
+	 *                                       |       |                 |         -> |                         |
+	 *                                       p  q->ptr                 q       q->ptr                         q
+	 */
+	for (q = km->loop_head; !(p > q && p < q->ptr); q = q->ptr)
+		if (q >= q->ptr && (p > q || p < q->ptr)) break;
+	if (p + p->size == q->ptr) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
+		p->size += q->ptr->size;
+		p->ptr = q->ptr->ptr;
+	} else if (p + p->size > q->ptr && q->ptr >= p) {
+		panic("[kfree] The end of the allocated block enters a free block.");
+	} else p->ptr = q->ptr; /* backup q->ptr */
+
+	if (q + q->size == p) { /* two adjacent blocks, merge q and p (the other two cases) */
+		q->size += p->size;
+		q->ptr = p->ptr;
+		km->loop_head = q;
+	} else if (q + q->size > p && p >= q) {
+		panic("[kfree] The end of a free block enters the allocated block.");
+	} else km->loop_head = p, q->ptr = p; /* in two cores, cannot be merged; create a new block in the list */
+}
+
+void *kmalloc(void *_km, size_t n_bytes)
+{
+	kmem_t *km = (kmem_t*)_km;
+	size_t n_units;
+	header_t *p, *q;
+
+	if (n_bytes == 0) return 0;
+	if (km == NULL) return malloc(n_bytes);
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t); /* header+n_bytes requires at least this number of units */
+
+	if (!(q = km->loop_head)) /* the first time when kmalloc() is called, intialize it */
+		q = km->loop_head = km->base.ptr = &km->base;
+	for (p = q->ptr;; q = p, p = p->ptr) { /* search for a suitable block */
+		if (p->size >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
+			if (p->size == n_units) q->ptr = p->ptr; /* no need to split the block */
+			else { /* split the block. NB: memory is allocated at the end of the block! */
+				p->size -= n_units; /* reduce the size of the free block */
+				p += p->size; /* p points to the allocated block */
+				*(size_t*)p = n_units; /* set the size */
+			}
+			km->loop_head = q; /* set the end of chain */
+			return (size_t*)p + 1;
+		}
+		if (p == km->loop_head) { /* then ask for more "cores" */
+			if ((p = morecore(km, n_units)) == 0) return 0;
+		}
+	}
+}
+
+void *kcalloc(void *_km, size_t count, size_t size)
+{
+	kmem_t *km = (kmem_t*)_km;
+	void *p;
+	if (size == 0 || count == 0) return 0;
+	if (km == NULL) return calloc(count, size);
+	p = kmalloc(km, count * size);
+	memset(p, 0, count * size);
+	return p;
+}
+
+void *krealloc(void *_km, void *ap, size_t n_bytes) // TODO: this can be made more efficient in principle
+{
+	kmem_t *km = (kmem_t*)_km;
+	size_t cap, *p, *q;
+
+	if (n_bytes == 0) {
+		kfree(km, ap); return 0;
+	}
+	if (km == NULL) return realloc(ap, n_bytes);
+	if (ap == NULL) return kmalloc(km, n_bytes);
+	p = (size_t*)ap - 1;
+	cap = (*p) * sizeof(header_t) - sizeof(size_t);
+	if (cap >= n_bytes) return ap; /* TODO: this prevents shrinking */
+	q = (size_t*)kmalloc(km, n_bytes);
+	memcpy(q, ap, cap);
+	kfree(km, ap);
+	return q;
+}
+
+void km_stat(const void *_km, km_stat_t *s)
+{
+	kmem_t *km = (kmem_t*)_km;
+	header_t *p;
+	memset(s, 0, sizeof(km_stat_t));
+	if (km == NULL || km->loop_head == NULL) return;
+	for (p = km->loop_head;; p = p->ptr) {
+		s->available += p->size * sizeof(header_t);
+		if (p->size != 0) ++s->n_blocks; /* &kmem_t::base is always one of the cores. It is zero-sized. */
+		if (p->ptr > p && p + p->size > p->ptr)
+			panic("[km_stat] The end of a free block enters another free block.");
+		if (p->ptr == km->loop_head) break;
+	}
+	for (p = km->core_head; p != NULL; p = p->ptr) {
+		size_t size = p->size * sizeof(header_t);
+		++s->n_cores;
+		s->capacity += size;
+		s->largest = s->largest > size? s->largest : size;
+	}
+}

kalloc.h

@@ -0,0 +1,107 @@
+#ifndef _KALLOC_H_
+#define _KALLOC_H_
+
+#include <stddef.h> /* for size_t */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+	size_t capacity, available, n_blocks, n_cores, largest;
+} km_stat_t;
+
+void *kmalloc(void *km, size_t size);
+void *krealloc(void *km, void *ptr, size_t size);
+void *kcalloc(void *km, size_t count, size_t size);
+void kfree(void *km, void *ptr);
+
+void *km_init(void);
+void *km_init2(void *km_par, size_t min_core_size);
+void km_destroy(void *km);
+void km_stat(const void *_km, km_stat_t *s);
+
+#ifdef __cplusplus
+}
+#endif
+
+#define KMALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kmalloc((km), (len) * sizeof(*(ptr))))
+#define KCALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kcalloc((km), (len), sizeof(*(ptr))))
+#define KREALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))krealloc((km), (ptr), (len) * sizeof(*(ptr))))
+
+#define KEXPAND(km, a, m) do { \
+		(m) = (m) >= 4? (m) + ((m)>>1) : 16; \
+		KREALLOC((km), (a), (m)); \
+	} while (0)
+
+#define kv_resize_km(km, type, v, s) do { \
+        if ((v).m < (s)) { \
+            (v).m = (s); \
+            kv_roundup32((v).m); \
+			KREALLOC((km), (v).a, (v).m); \
+        } \
+    } while (0)
+
+#define kv_copy_km(km, type, v1, v0) do {                          \
+        if ((v1).m < (v0).n) kv_resize_km((km), type, v1, (v0).n);   \
+        (v1).n = (v0).n;                                    \
+        memcpy((v1).a, (v0).a, sizeof(type) * (v0).n);      \
+    } while (0)                                             \
+
+#define kv_push_km(km, type, v, x) do {                                    \
+        if ((v).n == (v).m) {                                       \
+            (v).m = (v).m? (v).m<<1 : 2;                            \
+            KREALLOC((km), (v).a, (v).m);							    \
+        }                                                           \
+        (v).a[(v).n++] = (x);                                       \
+    } while (0)
+
+#define kv_pushp_km(km, type, v, p) do { \
+        if ((v).n == (v).m) { \
+            (v).m = (v).m? (v).m<<1 : 2; \
+            KREALLOC((km), (v).a, (v).m);	 \
+        } \
+        *(p) = &(v).a[(v).n++]; \
+    } while (0)
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+// adapted from klist.h
+#define KALLOC_POOL_INIT2(SCOPE, name, kmptype_t) \
+	typedef struct { \
+		size_t cnt, n, max; \
+		kmptype_t **buf; \
+		void *km; \
+	} kmp_##name##_t; \
+	SCOPE kmp_##name##_t *kmp_init_##name(void *km) { \
+		kmp_##name##_t *mp; \
+		KCALLOC(km, mp, 1); \
+		mp->km = km; \
+		return mp; \
+	} \
+	SCOPE void kmp_destroy_##name(kmp_##name##_t *mp) { \
+		size_t k; \
+		for (k = 0; k < mp->n; ++k) kfree(mp->km, mp->buf[k]); \
+		kfree(mp->km, mp->buf); kfree(mp->km, mp); \
+	} \
+	SCOPE kmptype_t *kmp_alloc_##name(kmp_##name##_t *mp) { \
+		++mp->cnt; \
+		if (mp->n == 0) return (kmptype_t*)kcalloc(mp->km, 1, sizeof(kmptype_t)); \
+		return mp->buf[--mp->n]; \
+	} \
+	SCOPE void kmp_free_##name(kmp_##name##_t *mp, kmptype_t *p) { \
+		--mp->cnt; \
+		if (mp->n == mp->max) KEXPAND(mp->km, mp->buf, mp->max); \
+		mp->buf[mp->n++] = p; \
+	}
+
+#define KALLOC_POOL_INIT(name, kmptype_t) \
+	KALLOC_POOL_INIT2(static inline klib_unused, name, kmptype_t)
+
+#endif

kavl.h

@@ -0,0 +1,414 @@
+/* The MIT License
+
+   Copyright (c) 2018 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* An example:
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "kavl.h"
+
+struct my_node {
+  char key;
+  KAVL_HEAD(struct my_node) head;
+};
+#define my_cmp(p, q) (((q)->key < (p)->key) - ((p)->key < (q)->key))
+KAVL_INIT(my, struct my_node, head, my_cmp)
+
+int main(void) {
+  const char *str = "MNOLKQOPHIA"; // from wiki, except a duplicate
+  struct my_node *root = 0;
+  int i, l = strlen(str);
+  for (i = 0; i < l; ++i) {        // insert in the input order
+    struct my_node *q, *p = malloc(sizeof(*p));
+    p->key = str[i];
+    q = kavl_insert(my, &root, p, 0);
+    if (p != q) free(p);           // if already present, free
+  }
+  kavl_itr_t(my) itr;
+  kavl_itr_first(my, root, &itr);  // place at first
+  do {                             // traverse
+    const struct my_node *p = kavl_at(&itr);
+    putchar(p->key);
+    free((void*)p);                // free node
+  } while (kavl_itr_next(my, &itr));
+  putchar('\n');
+  return 0;
+}
+*/
+
+#ifndef KAVL_H
+#define KAVL_H
+
+#ifdef __STRICT_ANSI__
+#define inline __inline__
+#endif
+
+#define KAVL_MAX_DEPTH 64
+
+#define kavl_size(head, p) ((p)? (p)->head.size : 0)
+#define kavl_size_child(head, q, i) ((q)->head.p[(i)]? (q)->head.p[(i)]->head.size : 0)
+
+#define KAVL_HEAD(__type) \
+	struct { \
+		__type *p[2]; \
+		signed char balance; /* balance factor */ \
+		unsigned size; /* #elements in subtree */ \
+	}
+
+#define __KAVL_FIND(suf, __scope, __type, __head,  __cmp) \
+	__scope __type *kavl_find_##suf(const __type *root, const __type *x, unsigned *cnt_) { \
+		const __type *p = root; \
+		unsigned cnt = 0; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += kavl_size_child(__head, p, 0) + 1; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		return (__type*)p; \
+	} \
+	__scope __type *kavl_interval_##suf(const __type *root, const __type *x, __type **lower, __type **upper) { \
+		const __type *p = root, *l = 0, *u = 0; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) u = p, p = p->__head.p[0]; \
+			else if (cmp > 0) l = p, p = p->__head.p[1]; \
+			else { l = u = p; break; } \
+		} \
+		if (lower) *lower = (__type*)l; \
+		if (upper) *upper = (__type*)u; \
+		return (__type*)p; \
+	}
+
+#define __KAVL_ROTATE(suf, __type, __head) \
+	/* one rotation: (a,(b,c)q)p => ((a,b)p,c)q */ \
+	static inline __type *kavl_rotate1_##suf(__type *p, int dir) { /* dir=0 to left; dir=1 to right */ \
+		int opp = 1 - dir; /* opposite direction */ \
+		__type *q = p->__head.p[opp]; \
+		unsigned size_p = p->__head.size; \
+		p->__head.size -= q->__head.size - kavl_size_child(__head, q, dir); \
+		q->__head.size = size_p; \
+		p->__head.p[opp] = q->__head.p[dir]; \
+		q->__head.p[dir] = p; \
+		return q; \
+	} \
+	/* two consecutive rotations: (a,((b,c)r,d)q)p => ((a,b)p,(c,d)q)r */ \
+	static inline __type *kavl_rotate2_##suf(__type *p, int dir) { \
+		int b1, opp = 1 - dir; \
+		__type *q = p->__head.p[opp], *r = q->__head.p[dir]; \
+		unsigned size_x_dir = kavl_size_child(__head, r, dir); \
+		r->__head.size = p->__head.size; \
+		p->__head.size -= q->__head.size - size_x_dir; \
+		q->__head.size -= size_x_dir + 1; \
+		p->__head.p[opp] = r->__head.p[dir]; \
+		r->__head.p[dir] = p; \
+		q->__head.p[dir] = r->__head.p[opp]; \
+		r->__head.p[opp] = q; \
+		b1 = dir == 0? +1 : -1; \
+		if (r->__head.balance == b1) q->__head.balance = 0, p->__head.balance = -b1; \
+		else if (r->__head.balance == 0) q->__head.balance = p->__head.balance = 0; \
+		else q->__head.balance = b1, p->__head.balance = 0; \
+		r->__head.balance = 0; \
+		return r; \
+	}
+
+#define __KAVL_INSERT(suf, __scope, __type, __head, __cmp) \
+	__scope __type *kavl_insert_##suf(__type **root_, __type *x, unsigned *cnt_) { \
+		unsigned char stack[KAVL_MAX_DEPTH]; \
+		__type *path[KAVL_MAX_DEPTH]; \
+		__type *bp, *bq; \
+		__type *p, *q, *r = 0; /* _r_ is potentially the new root */ \
+		int i, which = 0, top, b1, path_len; \
+		unsigned cnt = 0; \
+		bp = *root_, bq = 0; \
+		/* find the insertion location */ \
+		for (p = bp, q = bq, top = path_len = 0; p; q = p, p = p->__head.p[which]) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += kavl_size_child(__head, p, 0) + 1; \
+			if (cmp == 0) { \
+				if (cnt_) *cnt_ = cnt; \
+				return p; \
+			} \
+			if (p->__head.balance != 0) \
+				bq = q, bp = p, top = 0; \
+			stack[top++] = which = (cmp > 0); \
+			path[path_len++] = p; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		x->__head.balance = 0, x->__head.size = 1, x->__head.p[0] = x->__head.p[1] = 0; \
+		if (q == 0) *root_ = x; \
+		else q->__head.p[which] = x; \
+		if (bp == 0) return x; \
+		for (i = 0; i < path_len; ++i) ++path[i]->__head.size; \
+		for (p = bp, top = 0; p != x; p = p->__head.p[stack[top]], ++top) /* update balance factors */ \
+			if (stack[top] == 0) --p->__head.balance; \
+			else ++p->__head.balance; \
+		if (bp->__head.balance > -2 && bp->__head.balance < 2) return x; /* no re-balance needed */ \
+		/* re-balance */ \
+		which = (bp->__head.balance < 0); \
+		b1 = which == 0? +1 : -1; \
+		q = bp->__head.p[1 - which]; \
+		if (q->__head.balance == b1) { \
+			r = kavl_rotate1_##suf(bp, which); \
+			q->__head.balance = bp->__head.balance = 0; \
+		} else r = kavl_rotate2_##suf(bp, which); \
+		if (bq == 0) *root_ = r; \
+		else bq->__head.p[bp != bq->__head.p[0]] = r; \
+		return x; \
+	}
+
+#define __KAVL_ERASE(suf, __scope, __type, __head, __cmp) \
+	__scope __type *kavl_erase_##suf(__type **root_, const __type *x, unsigned *cnt_) { \
+		__type *p, *path[KAVL_MAX_DEPTH], fake; \
+		unsigned char dir[KAVL_MAX_DEPTH]; \
+		int i, d = 0, cmp; \
+		unsigned cnt = 0; \
+		fake.__head.p[0] = *root_, fake.__head.p[1] = 0; \
+		if (cnt_) *cnt_ = 0; \
+		if (x) { \
+			for (cmp = -1, p = &fake; cmp; cmp = __cmp(x, p)) { \
+				int which = (cmp > 0); \
+				if (cmp > 0) cnt += kavl_size_child(__head, p, 0) + 1; \
+				dir[d] = which; \
+				path[d++] = p; \
+				p = p->__head.p[which]; \
+				if (p == 0) { \
+					if (cnt_) *cnt_ = 0; \
+					return 0; \
+				} \
+			} \
+			cnt += kavl_size_child(__head, p, 0) + 1; /* because p==x is not counted */ \
+		} else { \
+			for (p = &fake, cnt = 1; p; p = p->__head.p[0]) \
+				dir[d] = 0, path[d++] = p; \
+			p = path[--d]; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		for (i = 1; i < d; ++i) --path[i]->__head.size; \
+		if (p->__head.p[1] == 0) { /* ((1,.)2,3)4 => (1,3)4; p=2 */ \
+			path[d-1]->__head.p[dir[d-1]] = p->__head.p[0]; \
+		} else { \
+			__type *q = p->__head.p[1]; \
+			if (q->__head.p[0] == 0) { /* ((1,2)3,4)5 => ((1)2,4)5; p=3 */ \
+				q->__head.p[0] = p->__head.p[0]; \
+				q->__head.balance = p->__head.balance; \
+				path[d-1]->__head.p[dir[d-1]] = q; \
+				path[d] = q, dir[d++] = 1; \
+				q->__head.size = p->__head.size - 1; \
+			} else { /* ((1,((.,2)3,4)5)6,7)8 => ((1,(2,4)5)3,7)8; p=6 */ \
+				__type *r; \
+				int e = d++; /* backup _d_ */\
+				for (;;) { \
+					dir[d] = 0; \
+					path[d++] = q; \
+					r = q->__head.p[0]; \
+					if (r->__head.p[0] == 0) break; \
+					q = r; \
+				} \
+				r->__head.p[0] = p->__head.p[0]; \
+				q->__head.p[0] = r->__head.p[1]; \
+				r->__head.p[1] = p->__head.p[1]; \
+				r->__head.balance = p->__head.balance; \
+				path[e-1]->__head.p[dir[e-1]] = r; \
+				path[e] = r, dir[e] = 1; \
+				for (i = e + 1; i < d; ++i) --path[i]->__head.size; \
+				r->__head.size = p->__head.size - 1; \
+			} \
+		} \
+		while (--d > 0) { \
+			__type *q = path[d]; \
+			int which, other, b1 = 1, b2 = 2; \
+			which = dir[d], other = 1 - which; \
+			if (which) b1 = -b1, b2 = -b2; \
+			q->__head.balance += b1; \
+			if (q->__head.balance == b1) break; \
+			else if (q->__head.balance == b2) { \
+				__type *r = q->__head.p[other]; \
+				if (r->__head.balance == -b1) { \
+					path[d-1]->__head.p[dir[d-1]] = kavl_rotate2_##suf(q, which); \
+				} else { \
+					path[d-1]->__head.p[dir[d-1]] = kavl_rotate1_##suf(q, which); \
+					if (r->__head.balance == 0) { \
+						r->__head.balance = -b1; \
+						q->__head.balance = b1; \
+						break; \
+					} else r->__head.balance = q->__head.balance = 0; \
+				} \
+			} \
+		} \
+		*root_ = fake.__head.p[0]; \
+		return p; \
+	}
+
+#define kavl_free(__type, __head, __root, __free) do { \
+		__type *_p, *_q; \
+		for (_p = __root; _p; _p = _q) { \
+			if (_p->__head.p[0] == 0) { \
+				_q = _p->__head.p[1]; \
+				__free(_p); \
+			} else { \
+				_q = _p->__head.p[0]; \
+				_p->__head.p[0] = _q->__head.p[1]; \
+				_q->__head.p[1] = _p; \
+			} \
+		} \
+	} while (0)
+
+#define __KAVL_ITR(suf, __scope, __type, __head, __cmp) \
+	struct kavl_itr_##suf { \
+		const __type *stack[KAVL_MAX_DEPTH], **top; \
+	}; \
+	__scope void kavl_itr_first_##suf(const __type *root, struct kavl_itr_##suf *itr) { \
+		const __type *p; \
+		for (itr->top = itr->stack - 1, p = root; p; p = p->__head.p[0]) \
+			*++itr->top = p; \
+	} \
+	__scope int kavl_itr_find_##suf(const __type *root, const __type *x, struct kavl_itr_##suf *itr) { \
+		const __type *p = root; \
+		itr->top = itr->stack - 1; \
+		while (p != 0) { \
+			int cmp; \
+			*++itr->top = p; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		return p? 1 : 0; \
+	} \
+	__scope int kavl_itr_next_bidir_##suf(struct kavl_itr_##suf *itr, int dir) { \
+		const __type *p; \
+		if (itr->top < itr->stack) return 0; \
+		dir = !!dir; \
+		p = (*itr->top)->__head.p[dir]; \
+		if (p) { /* go down */ \
+			for (; p; p = p->__head.p[!dir]) \
+				*++itr->top = p; \
+			return 1; \
+		} else { /* go up */ \
+			do { \
+				p = *itr->top--; \
+			} while (itr->top >= itr->stack && p == (*itr->top)->__head.p[dir]); \
+			return itr->top < itr->stack? 0 : 1; \
+		} \
+	} \
+
+/**
+ * Insert a node to the tree
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node to insert (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return _x_ if not present in the tree, or the node equal to x.
+ */
+#define kavl_insert(suf, proot, x, cnt) kavl_insert_##suf(proot, x, cnt)
+
+/**
+ * Find a node in the tree
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param root    root of the tree
+ * @param x       node value to find (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return node equal to _x_ if present, or NULL if absent
+ */
+#define kavl_find(suf, root, x, cnt) kavl_find_##suf(root, x, cnt)
+#define kavl_interval(suf, root, x, lower, upper) kavl_interval_##suf(root, x, lower, upper)
+
+/**
+ * Delete a node from the tree
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node value to delete; if NULL, delete the first node (in)
+ *
+ * @return node removed from the tree if present, or NULL if absent
+ */
+#define kavl_erase(suf, proot, x, cnt) kavl_erase_##suf(proot, x, cnt)
+#define kavl_erase_first(suf, proot) kavl_erase_##suf(proot, 0, 0)
+
+#define kavl_itr_t(suf) struct kavl_itr_##suf
+
+/**
+ * Place the iterator at the smallest object
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param root    root of the tree
+ * @param itr     iterator
+ */
+#define kavl_itr_first(suf, root, itr) kavl_itr_first_##suf(root, itr)
+
+/**
+ * Place the iterator at the object equal to or greater than the query
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param root    root of the tree
+ * @param x       query (in)
+ * @param itr     iterator (out)
+ *
+ * @return 1 if find; 0 otherwise. kavl_at(itr) is NULL if and only if query is
+ *         larger than all objects in the tree
+ */
+#define kavl_itr_find(suf, root, x, itr) kavl_itr_find_##suf(root, x, itr)
+
+/**
+ * Move to the next object in order
+ *
+ * @param itr     iterator (modified)
+ *
+ * @return 1 if there is a next object; 0 otherwise
+ */
+#define kavl_itr_next(suf, itr) kavl_itr_next_bidir_##suf(itr, 1)
+#define kavl_itr_prev(suf, itr) kavl_itr_next_bidir_##suf(itr, 0)
+
+/**
+ * Return the pointer at the iterator
+ *
+ * @param itr     iterator
+ *
+ * @return pointer if present; NULL otherwise
+ */
+#define kavl_at(itr) ((itr)->top < (itr)->stack? 0 : *(itr)->top)
+
+#define KAVL_INIT2(suf, __scope, __type, __head, __cmp) \
+	__KAVL_FIND(suf, __scope, __type, __head,  __cmp) \
+	__KAVL_ROTATE(suf, __type, __head) \
+	__KAVL_INSERT(suf, __scope, __type, __head, __cmp) \
+	__KAVL_ERASE(suf, __scope, __type, __head, __cmp) \
+	__KAVL_ITR(suf, __scope, __type, __head, __cmp)
+
+#define KAVL_INIT(suf, __type, __head, __cmp) \
+	KAVL_INIT2(suf,, __type, __head, __cmp)
+
+#endif

khash.h

@@ -0,0 +1,635 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+	int ret, is_missing;
+	khiter_t k;
+	khash_t(32) *h = kh_init(32);
+	k = kh_put(32, h, 5, &ret);
+	kh_value(h, k) = 10;
+	k = kh_get(32, h, 10);
+	is_missing = (k == kh_end(h));
+	k = kh_get(32, h, 5);
+	kh_del(32, h, k);
+	for (k = kh_begin(h); k != kh_end(h); ++k)
+		if (kh_exist(h, k)) kh_value(h, k) = 1;
+	kh_destroy(32, h);
+	return 0;
+}
+*/
+
+/*
+  2013-05-02 (0.2.8):
+
+	* Use quadratic probing. When the capacity is power of 2, stepping function
+	  i*(i+1)/2 guarantees to traverse each bucket. It is better than double
+	  hashing on cache performance and is more robust than linear probing.
+
+	  In theory, double hashing should be more robust than quadratic probing.
+	  However, my implementation is probably not for large hash tables, because
+	  the second hash function is closely tied to the first hash function,
+	  which reduce the effectiveness of double hashing.
+
+	Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php
+
+  2011-12-29 (0.2.7):
+
+    * Minor code clean up; no actual effect.
+
+  2011-09-16 (0.2.6):
+
+	* The capacity is a power of 2. This seems to dramatically improve the
+	  speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
+
+	   - http://code.google.com/p/ulib/
+	   - http://nothings.org/computer/judy/
+
+	* Allow to optionally use linear probing which usually has better
+	  performance for random input. Double hashing is still the default as it
+	  is more robust to certain non-random input.
+
+	* Added Wang's integer hash function (not used by default). This hash
+	  function is more robust to certain non-random input.
+
+  2011-02-14 (0.2.5):
+
+    * Allow to declare global functions.
+
+  2009-09-26 (0.2.4):
+
+    * Improve portability
+
+  2008-09-19 (0.2.3):
+
+	* Corrected the example
+	* Improved interfaces
+
+  2008-09-11 (0.2.2):
+
+	* Improved speed a little in kh_put()
+
+  2008-09-10 (0.2.1):
+
+	* Added kh_clear()
+	* Fixed a compiling error
+
+  2008-09-02 (0.2.0):
+
+	* Changed to token concatenation which increases flexibility.
+
+  2008-08-31 (0.1.2):
+
+	* Fixed a bug in kh_get(), which has not been tested previously.
+
+  2008-08-31 (0.1.1):
+
+	* Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+/*!
+  @header
+
+  Generic hash table library.
+ */
+
+#define AC_VERSION_KHASH_H "0.2.8"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include "kalloc.h"
+
+/* compiler specific configuration */
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifndef kh_inline
+#ifdef _MSC_VER
+#define kh_inline __inline
+#else
+#define kh_inline inline
+#endif
+#endif /* kh_inline */
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+typedef khint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define __KHASH_TYPE(name, khkey_t, khval_t) \
+	typedef struct kh_##name##_s { \
+		khint_t n_buckets, size, n_occupied, upper_bound; \
+		khint32_t *flags; \
+		khkey_t *keys; \
+		khval_t *vals; \
+		void *km; \
+	} kh_##name##_t;
+
+#define __KHASH_PROTOTYPES(name, khkey_t, khval_t)	 					\
+	extern kh_##name##_t *kh_init_##name(void);							\
+	extern void kh_destroy_##name(kh_##name##_t *h);					\
+	extern void kh_clear_##name(kh_##name##_t *h);						\
+	extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); 	\
+	extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+	extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
+	extern void kh_del_##name(kh_##name##_t *h, khint_t x);
+
+#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	SCOPE kh_##name##_t *kh_init2_##name(void *km) { \
+		kh_##name##_t *h; \
+		h = (kh_##name##_t*)kcalloc(km, 1, sizeof(kh_##name##_t)); \
+		h->km = km; \
+		return h; \
+	} \
+	SCOPE kh_##name##_t *kh_init_##name(void) {	return kh_init2_##name(0); } \
+	SCOPE void kh_destroy_##name(kh_##name##_t *h) \
+	{ \
+		if (h) { \
+			void *km = h->km; \
+			kfree(km, (void *)h->keys); kfree(km, h->flags); \
+			kfree(km, (void *)h->vals); \
+			kfree(km, h); \
+		} \
+	} \
+	SCOPE void kh_clear_##name(kh_##name##_t *h)						\
+	{																	\
+		if (h && h->flags) {											\
+			memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
+			h->size = h->n_occupied = 0;								\
+		}																\
+	}																	\
+	SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) 	\
+	{																	\
+		if (h->n_buckets) {												\
+			khint_t k, i, last, mask, step = 0; \
+			mask = h->n_buckets - 1;									\
+			k = __hash_func(key); i = k & mask;							\
+			last = i; \
+			while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+				i = (i + (++step)) & mask; \
+				if (i == last) return h->n_buckets;						\
+			}															\
+			return __ac_iseither(h->flags, i)? h->n_buckets : i;		\
+		} else return 0;												\
+	}																	\
+	SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+	{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
+		khint32_t *new_flags = 0;										\
+		khint_t j = 1;													\
+		{																\
+			kroundup32(new_n_buckets); 									\
+			if (new_n_buckets < 4) new_n_buckets = 4;					\
+			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	/* requested size is too small */ \
+			else { /* hash table size to be changed (shrink or expand); rehash */ \
+				new_flags = (khint32_t*)kmalloc(h->km, __ac_fsize(new_n_buckets) * sizeof(khint32_t));	\
+				if (!new_flags) return -1;								\
+				memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+				if (h->n_buckets < new_n_buckets) {	/* expand */		\
+					khkey_t *new_keys = (khkey_t*)krealloc(h->km, (void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+					if (!new_keys) { kfree(h->km, new_flags); return -1; }	\
+					h->keys = new_keys;									\
+					if (kh_is_map) {									\
+						khval_t *new_vals = (khval_t*)krealloc(h->km, (void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+						if (!new_vals) { kfree(h->km, new_flags); return -1; } \
+						h->vals = new_vals;								\
+					}													\
+				} /* otherwise shrink */								\
+			}															\
+		}																\
+		if (j) { /* rehashing is needed */								\
+			for (j = 0; j != h->n_buckets; ++j) {						\
+				if (__ac_iseither(h->flags, j) == 0) {					\
+					khkey_t key = h->keys[j];							\
+					khval_t val;										\
+					khint_t new_mask;									\
+					new_mask = new_n_buckets - 1; 						\
+					if (kh_is_map) val = h->vals[j];					\
+					__ac_set_isdel_true(h->flags, j);					\
+					while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+						khint_t k, i, step = 0; \
+						k = __hash_func(key);							\
+						i = k & new_mask;								\
+						while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \
+						__ac_set_isempty_false(new_flags, i);			\
+						if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
+							{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+							if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+							__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
+						} else { /* write the element and jump out of the loop */ \
+							h->keys[i] = key;							\
+							if (kh_is_map) h->vals[i] = val;			\
+							break;										\
+						}												\
+					}													\
+				}														\
+			}															\
+			if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
+				h->keys = (khkey_t*)krealloc(h->km, (void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+				if (kh_is_map) h->vals = (khval_t*)krealloc(h->km, (void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+			}															\
+			kfree(h->km, h->flags); /* free the working space */ \
+			h->flags = new_flags;										\
+			h->n_buckets = new_n_buckets;								\
+			h->n_occupied = h->size;									\
+			h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+		}																\
+		return 0;														\
+	}																	\
+	SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+	{																	\
+		khint_t x;														\
+		if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
+			if (h->n_buckets > (h->size<<1)) {							\
+				if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
+					*ret = -1; return h->n_buckets;						\
+				}														\
+			} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
+				*ret = -1; return h->n_buckets;							\
+			}															\
+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+		{																\
+			khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \
+			x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
+			if (__ac_isempty(h->flags, i)) x = i; /* for speed up */	\
+			else {														\
+				last = i; \
+				while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+					if (__ac_isdel(h->flags, i)) site = i;				\
+					i = (i + (++step)) & mask; \
+					if (i == last) { x = site; break; }					\
+				}														\
+				if (x == h->n_buckets) {								\
+					if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+					else x = i;											\
+				}														\
+			}															\
+		}																\
+		if (__ac_isempty(h->flags, x)) { /* not present at all */		\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size; ++h->n_occupied;									\
+			*ret = 1;													\
+		} else if (__ac_isdel(h->flags, x)) { /* deleted */				\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size;													\
+			*ret = 2;													\
+		} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+		return x;														\
+	}																	\
+	SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x)				\
+	{																	\
+		if (x != h->n_buckets && !__ac_iseither(h->flags, x)) {			\
+			__ac_set_isdel_true(h->flags, x);							\
+			--h->size;													\
+		}																\
+	}
+
+#define KHASH_DECLARE(name, khkey_t, khval_t)		 					\
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_PROTOTYPES(name, khkey_t, khval_t)
+
+#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	KHASH_INIT2(name, static kh_inline klib_unused, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+/*! @function
+  @abstract     Integer hash function
+  @param  key   The integer [khint32_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int_hash_func(key) (khint32_t)(key)
+/*! @function
+  @abstract     Integer comparison function
+ */
+#define kh_int_hash_equal(a, b) ((a) == (b))
+/*! @function
+  @abstract     64-bit integer hash function
+  @param  key   The integer [khint64_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
+/*! @function
+  @abstract     64-bit integer comparison function
+ */
+#define kh_int64_hash_equal(a, b) ((a) == (b))
+/*! @function
+  @abstract     const char* hash function
+  @param  s     Pointer to a null terminated string
+  @return       The hash value
+ */
+static kh_inline khint_t __ac_X31_hash_string(const char *s)
+{
+	khint_t h = (khint_t)*s;
+	if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
+	return h;
+}
+/*! @function
+  @abstract     Another interface to const char* hash function
+  @param  key   Pointer to a null terminated string [const char*]
+  @return       The hash value [khint_t]
+ */
+#define kh_str_hash_func(key) __ac_X31_hash_string(key)
+/*! @function
+  @abstract     Const char* comparison function
+ */
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+static kh_inline khint_t __ac_Wang_hash(khint_t key)
+{
+    key += ~(key << 15);
+    key ^=  (key >> 10);
+    key +=  (key << 3);
+    key ^=  (key >> 6);
+    key += ~(key << 11);
+    key ^=  (key >> 16);
+    return key;
+}
+#define kh_int_hash_func2(key) __ac_Wang_hash((khint_t)key)
+
+static kh_inline khint64_t __ac_Wang_hash64(khint64_t key)
+{
+	key = ~key + (key << 21);
+	key = key ^ key >> 24;
+	key = (key + (key << 3)) + (key << 8);
+	key = key ^ key >> 14;
+	key = (key + (key << 2)) + (key << 4);
+	key = key ^ key >> 28;
+	key = key + (key << 31);
+	return key;
+}
+#define kh_int_hash64_func2(key) __ac_Wang_hash64((khint64_t)key)
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other convenient macros... */
+
+/*!
+  @abstract Type of the hash table.
+  @param  name  Name of the hash table [symbol]
+ */
+#define khash_t(name) kh_##name##_t
+
+/*! @function
+  @abstract     Initiate a hash table.
+  @param  name  Name of the hash table [symbol]
+  @return       Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_init(name) kh_init_##name()
+#define kh_init2(name, km) kh_init2_##name(km)
+
+/*! @function
+  @abstract     Destroy a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_destroy(name, h) kh_destroy_##name(h)
+
+/*! @function
+  @abstract     Reset a hash table without deallocating memory.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_clear(name, h) kh_clear_##name(h)
+
+/*! @function
+  @abstract     Resize a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  s     New size [khint_t]
+ */
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+
+/*! @function
+  @abstract     Insert a key to the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @param  r     Extra return code: -1 if the operation failed;
+                0 if the key is present in the hash table;
+                1 if the bucket is empty (never used); 2 if the element in
+				the bucket has been deleted [int*]
+  @return       Iterator to the inserted element [khint_t]
+ */
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+
+/*! @function
+  @abstract     Retrieve a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @return       Iterator to the found element, or kh_end(h) if the element is absent [khint_t]
+ */
+#define kh_get(name, h, k) kh_get_##name(h, k)
+
+/*! @function
+  @abstract     Remove a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Iterator to the element to be deleted [khint_t]
+ */
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+/*! @function
+  @abstract     Test whether a bucket contains data.
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       1 if containing data; 0 otherwise [int]
+ */
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+
+/*! @function
+  @abstract     Get key given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Key [type of keys]
+ */
+#define kh_key(h, x) ((h)->keys[x])
+
+/*! @function
+  @abstract     Get value given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Value [type of values]
+  @discussion   For hash sets, calling this results in segfault.
+ */
+#define kh_val(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Alias of kh_val()
+ */
+#define kh_value(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Get the start iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The start iterator [khint_t]
+ */
+#define kh_begin(h) (khint_t)(0)
+
+/*! @function
+  @abstract     Get the end iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The end iterator [khint_t]
+ */
+#define kh_end(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Get the number of elements in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of elements in the hash table [khint_t]
+ */
+#define kh_size(h) ((h)->size)
+
+/*! @function
+  @abstract     Get the number of buckets in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of buckets in the hash table [khint_t]
+ */
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Iterate over the entries in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  kvar  Variable to which key will be assigned
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach(h, kvar, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(kvar) = kh_key(h,__i);								\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/*! @function
+  @abstract     Iterate over the values in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach_value(h, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/* More conenient interfaces */
+
+/*! @function
+  @abstract     Instantiate a hash set containing integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT(name)										\
+	KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT(name, khval_t)								\
+	KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT64(name)										\
+	KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT64(name, khval_t)								\
+	KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+/*! @function
+  @abstract     Instantiate a hash map containing const char* keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_STR(name)										\
+	KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing const char* keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_STR(name, khval_t)								\
+	KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#endif /* __AC_KHASH_H */

khashl.h

@@ -136,25 +136,33 @@ static kh_inline khint_t __kh_h2b(khint_t hash, khint_t bits) { return hash * 26
 #define __KHASHL_IMPL_S_L(SCOPE, HType, prefix, khkey_t) \
 	SCOPE khint_t prefix##_save(HType *h, FILE* fp) { \
 		if (!h) return 0; \
+		uint8_t ff; \
 		khint_t n_buckets = (h->keys? 1U<<h->bits : 0U); \
 		fwrite(&n_buckets, sizeof(n_buckets), 1, fp); \
 		fwrite(&h->bits, sizeof(h->bits), 1, fp); \
 		fwrite(&h->count, sizeof(h->count), 1, fp); \
-		fwrite(h->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \
-		fwrite(h->keys, sizeof(khkey_t), n_buckets, fp); \
+		ff = h->used? 1:0; fwrite(&ff, sizeof(ff), 1, fp); \
+		if(ff) fwrite(h->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \
+		ff = h->keys? 1:0; fwrite(&ff, sizeof(ff), 1, fp); \
+		if(ff) fwrite(h->keys, sizeof(khkey_t), n_buckets, fp); \
 		return 1; \
 	} \
 	SCOPE khint_t prefix##_load(HType **h, FILE* fp) { \
 		(*h) = prefix##_init(); \
 		khint_t n_buckets; \
 		uint64_t flag = 0;\
+		uint8_t ff; \
 		flag += fread(&n_buckets, sizeof(n_buckets), 1, fp); \
 		flag += fread(&(*h)->bits, sizeof((*h)->bits), 1, fp); \
 		flag += fread(&(*h)->count, sizeof((*h)->count), 1, fp); \
+		flag += fread(&ff, sizeof(ff), 1, fp); \
+		if(ff) {\
 		(*h)->used = (khint32_t*)kmalloc(__kh_fsize(n_buckets) * sizeof(khint32_t)); \
+		flag += fread((*h)->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); }\
+		flag += fread(&ff, sizeof(ff), 1, fp); \
+		if(ff) {\
 		(*h)->keys = (khkey_t*)kmalloc(n_buckets * sizeof(khkey_t)); \
-		flag += fread((*h)->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \
-		flag += fread((*h)->keys, sizeof(khkey_t), n_buckets, fp); \
+		flag += fread((*h)->keys, sizeof(khkey_t), n_buckets, fp); }\
 		return 1; \
 	} \
 

ksort.h

@@ -40,7 +40,43 @@ typedef struct {
 
 #define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; }
 
-#define KSORT_INIT(name, type_t, __sort_lt)								\
+#define KSORT_INIT(name, type_t, __sort_lt) \
+	void ks_heapdown_##name(size_t i, size_t n, type_t l[]) \
+	{ \
+		size_t k = i; \
+		type_t tmp = l[i]; \
+		while ((k = (k << 1) + 1) < n) { \
+			if (k != n - 1 && __sort_lt(l[k], l[k+1])) ++k; \
+			if (__sort_lt(l[k], tmp)) break; \
+			l[i] = l[k]; i = k; \
+		} \
+		l[i] = tmp; \
+	} \
+	void ks_heapup_##name(size_t n, type_t l[]) \
+	{ \
+		size_t i, k = n - 1; \
+		type_t tmp = l[k]; \
+		while (k) { \
+			i = (k - 1) >> 1; \
+			if (__sort_lt(tmp, l[i])) break; \
+			l[k] = l[i]; k = i; \
+		} \
+		l[k] = tmp; \
+	} \
+	void ks_heapmake_##name(size_t lsize, type_t l[]) \
+	{ \
+		size_t i; \
+		for (i = (lsize >> 1) - 1; i != (size_t)(-1); --i) \
+			ks_heapdown_##name(i, lsize, l); \
+	} \
+	void ks_heapsort_##name(size_t lsize, type_t l[]) \
+	{ \
+		size_t i; \
+		for (i = lsize - 1; i > 0; --i) { \
+			type_t tmp; \
+			tmp = *l; *l = l[i]; l[i] = tmp; ks_heapdown_##name(0, i, l); \
+		} \
+	} \
 	static inline void __ks_insertsort_##name(type_t *s, type_t *t)		\
 	{																	\
 		type_t *i, *j, swap_tmp;										\

main.cpp

@@ -11,7 +11,57 @@ int main(int argc, char *argv[])
 	int i, ret;
 	yak_reset_realtime();
     init_opt(&asm_opt);
-    if (!CommandLine_process(argc, argv, &asm_opt)) return 1;
+    if (!CommandLine_process(argc, argv, &asm_opt)) return 0;
+	
+	// bit_extz_t exz, exz64; init_bit_extz_t(&exz, 2); init_bit_extz_t(&exz64, 2);
+	
+	// char *pstr = "GACCCAG", *tsrt = "GTTGTTAATTCCAT"; int32_t thre = 14; clear_align(exz); clear_align(exz64);
+	// ed_band_cal_extension_64_0_w_trace((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, &exz);
+	// // // ed_band_cal_semi_64_w_absent_diag((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, 0, &exz);
+	// fprintf(stderr, "\n[M::%s::] exz.err::%d, exz.ps::%d, exz.pe::%d, exz.ts::%d, exz.te::%d\n", __func__, 
+    //         exz.err, exz.ps, exz.pe, exz.ts, exz.te);
+	// cigar_check((char*)pstr, (char*)tsrt, &(exz));
+	// ed_band_cal_extension_64_0_w((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, &exz);
+	// fprintf(stderr, "\n[M::%s::] exz.err::%d, exz.ps::%d, exz.pe::%d, exz.ts::%d, exz.te::%d\n", __func__, 
+    //         exz.err, exz.ps, exz.pe, exz.ts, exz.te);
+	// ed_band_cal_semi_infi_w((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, NULL, &exz);
+	// ed_band_cal_semi_64_w((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, &exz64);
+	// fprintf(stderr, "\n[M::%s::] exz.err::%d, exz64.err::%d, exz.ps::%d, exz64.ps::%d, exz.pe::%d, exz64.pe::%d, exz.ts::%d, exz64.ts::%d, exz.te::%d, exz64.te::%d\n", __func__, 
+    //         exz.err, exz64.err, exz.ps, exz64.ps, exz.pe, exz64.pe, exz.ts, exz64.ts, exz.te, exz64.te);
+	// ed_band_cal_semi_64_w_trace((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, &exz64);
+	// cigar_check((char*)pstr, (char*)tsrt, &(exz64));
+	
+	
+	// char *pstr = "TGT", *tsrt = "CTGT"; int32_t thre = 1;
+	// ed_band_cal_global_infi_w((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, NULL, &exz);
+	// ed_band_cal_global_64_w((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, &exz64);
+	// ed_band_cal_global_64_w_trace((char*)pstr, strlen(pstr), (char*)tsrt, strlen(tsrt), thre, &exz64);
+	// fprintf(stderr, "\n[M::%s::] exz.err::%d, exz64.err::%d, exz.ps::%d, exz64.ps::%d, exz.pe::%d, exz64.pe::%d, exz.ts::%d, exz64.ts::%d, exz.te::%d, exz64.te::%d\n", __func__, 
+    //         exz.err, exz64.err, exz.ps, exz64.ps, exz.pe, exz64.pe, exz.ts, exz64.ts, exz.te, exz64.te);
+	// cigar_check((char*)pstr, (char*)tsrt, &(exz64));
+	
+	// ed_band_cal_extension_infi0_w((char *)"AAT", 3, (char *)"ACTTTTTT", 8, 2, NULL, &exz);
+    // ed_band_cal_extension_64_w((char *)"AAT", 3, (char *)"ACTTTTTT", 8, 2, &exz64);
+	// fprintf(stderr, "\n[M::%s::] exz.err::%d, exz64.err::%d, exz.ps::%d, exz64.ps::%d, exz.pe::%d, exz64.pe::%d, exz.ts::%d, exz64.ts::%d, exz.te::%d, exz64.te::%d\n", __func__, 
+    //         exz.err, exz64.err, exz.ps, exz64.ps, exz.pe, exz64.pe, exz.ts, exz64.ts, exz.te, exz64.te);
+
+	//bit_extz_t exz; ///ed_band_cal_global_128bit(t_string+r_ts, t_end+1-r_ts, q_string, ql, thres, &exz);
+	// ed_band_cal_extension_128bit((char *)"AAGTTTA", 7, (char *)"CCTTTTTT", 8, 4, &exz);
+	// ed_band_cal_extension_128bit((char *)"AA", 2, (char *)"ACTTTTTT", 8, 1, &exz);
+	// fprintf(stderr, "ed_extension::%d, pe::%d, te::%d\n", exz.err, exz.pe, exz.te);
+	// exit(1);
+	
+	
+
+	// fprintf(stderr, "[M::%s::] ed_global::%d, ed_global_128bit::%d\n", __func__, 
+	// ed_band_cal_global((char *)"ACT", 3, (char *)"AAT", 3, 1), 
+	// ed_band_cal_global_128bit((char *)"ACT", 3, (char *)"AAT", 3, 1));
+
+	// fprintf(stderr, "[M::%s::] ed_global::%d, ed_global_128bit::%d\n", __func__, 
+	// ed_band_cal_global((char*)"ACTTTTTT", 8, (char*)"AATTTT", 6, 3),
+	// ed_band_cal_global_128bit((char*)"ACTTTTTT", 8, (char*)"AATTTT", 6, 3));
+	// exit(1);
+	
 	ret = ha_assemble();
     destory_opt(&asm_opt);
 	fprintf(stderr, "[M::%s] Version: %s\n", __func__, HA_VERSION);

rcut.h

@@ -0,0 +1,129 @@
+#ifndef __RCUT__
+#define __RCUT__
+#include <stdio.h>
+#include <stdint.h>
+#include "kvec.h"
+#include "Overlaps.h"
+#include "Purge_Dups.h"
+#include "hic.h"
+
+typedef struct {
+    uint32_t bS, bE;
+    uint32_t nS, nE;
+    uint32_t uID;
+    uint8_t hs;
+}mc_interval_t;
+
+#define mc_node_t int8_t
+#define mcg_node_t uint32_t
+// #define w_t int64_t
+// #define t_w_t int64_t
+// #define w_cast(x) ((t_w_t)((x) < 0 ? (x) - 0.5 : (x) + 0.5))
+
+#define w_t double
+#define t_w_t double
+#define w_cast(x) ((t_w_t)((x)))
+#define MC_NAME "debug_mc.bin"
+
+typedef struct {
+	uint64_t x; ///(uint64_t)nid1 << 32 | nid2;
+	w_t w;  ///might be negative or positive
+} mc_edge_t;
+
+typedef struct {
+    kvec_t(uint64_t) idx;
+    kvec_t(mc_edge_t) ma;
+    uint64_t* cc;
+    uint32_t n_seq;
+} mc_match_t;
+
+typedef struct {
+    kvec_t(mc_node_t) s;
+    ma_ug_t *ug;
+    asg_t *rg;
+    mc_match_t* e;
+}mc_g_t;
+
+typedef struct {
+    uint32_t a[2], occ[2]; 
+    mc_node_t s[2];
+    t_w_t z[4];
+}mb_node_t;
+
+typedef struct {
+    kvec_t(uint32_t) bid;
+    kvec_t(uint32_t) idx;
+    kvec_t(mb_node_t) u;
+}mb_nodes_t;
+
+typedef struct {
+	uint64_t x; ///(uint64_t)nid1 << 32 | nid2;
+	t_w_t w[4];  ///might be negative or positive
+} mb_edge_t;
+
+typedef struct {
+    kvec_t(uint64_t) idx;
+    kvec_t(mb_edge_t) ma;
+    uint64_t* cc;
+    uint32_t n_seq;
+} mb_match_t;
+
+typedef struct {
+    mb_nodes_t* u; 
+    mb_match_t* e;
+}mb_g_t;
+
+typedef struct {
+	mcg_node_t s;
+	uint16_t h[2], hc;
+	double hw[2];
+}mc_gg_status;
+
+typedef struct {
+	mc_gg_status *a;
+	size_t n, m;
+}kv_gg_status;
+
+typedef struct {
+	mcg_node_t *a;
+	size_t n, m;
+}mcb_t;
+
+
+typedef struct {
+	kv_gg_status *s;
+    // ma_ug_t *ug;
+    // asg_t *rg;
+    uint32_t un;
+    mc_match_t* e;
+	kvec_t(mcb_t) m;
+	mcg_node_t mask;
+	uint16_t hN;
+}mc_gg_t;
+
+
+static inline uint64_t kr_splitmix64(uint64_t x)
+{
+	uint64_t z = (x += 0x9E3779B97F4A7C15ULL);
+	z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
+	z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
+	return z ^ (z >> 31);
+}
+
+static inline double kr_drand_r(uint64_t *x)
+{
+    union { uint64_t i; double d; } u;
+	*x = kr_splitmix64(*x);
+    u.i = 0x3FFULL << 52 | (*x) >> 12;
+    return u.d - 1.0;
+}
+
+void mc_solve(hap_overlaps_list* ovlp, trans_chain* t_ch, kv_u_trans_t *ta, ma_ug_t *ug, asg_t *read_g, double f_rate, uint8_t* trio_flag, uint32_t renew_s, int8_t *s, uint32_t is_sys, bubble_type* bub, kv_u_trans_t *ref, int clean_ov, int is_dump);
+void debug_mc_g_t(const char* name);
+void mc_solve_general(kv_u_trans_t *ta, uint32_t un, kv_gg_status *s, uint16_t hapN, uint16_t update_ta, uint16_t write_dump);
+kv_gg_status *init_mc_gg_status(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, 
+ma_hit_t_alloc* sources, R_to_U* ruIndex, uint64_t t_cov, uint16_t hapN);
+void destory_mc_gg_t(mc_gg_t **p);
+void debug_mc_gg_t(const char* fn, uint32_t update_ta, uint32_t convert_mc_g_t);
+void quick_debug_phasing(const char* fn);
+#endif

sketch.cpp

@@ -4,258 +4,579 @@
 #include <string.h>
 #include "kvec.h"
 #include "htab.h"
+#include "ksort.h"
+#include "Correct.h"
+#include "kalloc.h"
 
-typedef struct { // a simplified version of kdq
-	int front, count;
-	int a[64];
-} tiny_queue_t;
+#define MAX_HIGH_OCC     8   // TODO: don't hard code if we need to tune this parameter
+#define MAX_MAX_HIGH_OCC 16
+#define GMC(a, x,y,xn) ((a)[(x)*(xn)+(y)])
+#define GL(x, i) ((int64_t)((uint32_t)((x).a[(i)])))
+#define A_M(p, i) ((i) >= 0 && (p).a[(i)].rid > 0)
 
-static inline void tq_push(tiny_queue_t *q, int x)
+void debug_refine(ha_mz1_t *ma, uint64_t *mmt, int32_t sn, int32_t n, int32_t m, int32_t end)
 {
-	q->a[((q->count++) + q->front) & 0x3f] = x;
-}
-
-static inline int tq_shift(tiny_queue_t *q)
-{
-	int x;
-	if (q->count == 0) return -1;
-	x = q->a[q->front++];
-	q->front &= 0x3f;
-	--q->count;
-	return x;
-}
-
-/**
- * Find symmetric (w,k)-minimizers on a DNA sequence
- *
- * @param str    DNA sequence
- * @param len    length of $str
- * @param w      find a minimizer for every $w consecutive k-mers
- * @param k      k-mer size
- * @param rid    reference ID; will be copied to the output $p array
- * @param is_hpc homopolymer-compressed or not
- * @param p      minimizers
- */
-void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf)
-{	///in default, w = 51, k = 51, is_hpc = 1
-	/**
-	 uint64_t x;
-	 uint64_t rid:28, pos:27, rev:1, span:8;
-	 **/
-	static const ha_mz1_t dummy = { UINT64_MAX, 0, 0, 0 };
-	uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0};
-	int i, j, l, buf_pos, min_pos, kmer_span = 0;
-	ha_mz1_t buf[256], min = dummy;
-	tiny_queue_t tq;
-
-	assert(len > 0 && len < 1<<27 && rid < 1<<28 && (w > 0 && w < 256) && (k > 0 && k <= 63));
-	///sizeof(ha_mz1_t) = 16
-	memset(buf, 0xff, w * 16);
-	memset(&tq, 0, sizeof(tiny_queue_t));
-	///len/w is the evaluated minimizer numbers
-	kv_resize(ha_mz1_t, *p, p->n + len/w);
-
-	for (i = l = buf_pos = min_pos = 0; i < len; ++i) {
-		int c = seq_nt4_table[(uint8_t)str[i]];
-		ha_mz1_t info = dummy;
-		if (c < 4) { // not an ambiguous base
-			int z;
-			if (is_hpc) {
-				int skip_len = 1;
-				if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {
-					for (skip_len = 2; i + skip_len < len; ++skip_len)
-						if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)
-							break;
-					i += skip_len - 1; // put $i at the end of the current homopolymer run
-				}
-				tq_push(&tq, skip_len);
-				kmer_span += skip_len;
-				if (tq.count > k) kmer_span -= tq_shift(&tq);
-			} else kmer_span = l + 1 < k? l + 1 : k; 
-			///kmer_span should be used for HPC k-mer
-			///so for non-HPC k-mer, kmer_span should be k in any case?
-			///kmer_span is used to calculate anchor pos on reverse complementary strand
-
-			kmer[0] = (kmer[0] << 1 | (c&1))  & mask;                  // forward k-mer
-			kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
-			kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1))  << shift1; // reverse k-mer
-			kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;
-			if (kmer[1] == kmer[3]) continue; // skip "symmetric k-mers" as we don't know it strand
-			z = kmer[1] < kmer[3]? 0 : 1; // strand
-			++l;
-			if (l >= k && kmer_span < 256) {
-				uint64_t y;
-				y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
-				if (hf == 0 || ha_ft_isflt(hf, y) == 0)
-					info.x = y, info.rid = rid, info.pos = i, info.rev = z, info.span = kmer_span;
-			}
-		} else l = 0, tq.count = tq.front = 0, kmer_span = 0;
-
-
-		//for non-HPC k-mer, l = i; but for HPC k-mer, l is always less than i
-		//i is the real base iterator, while l is the HPC base iterator
-		//only if l >= k, info is a useful minimizer (ha_mz1_t.x != UINT64_MAX)
-		//but even if l < k, infor is still stored into buf
-		buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
-		if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
-			for (j = buf_pos + 1; j < w; ++j)
-				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
-			for (j = 0; j < buf_pos; ++j)
-				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
-		}
-		/**
-		 * There are three cases:
-		 * 1. info.x <= min.x, means info is a new minimizer
-		 * 2. info.x > min.x, info is not a new minimizer
-		 *    (1) buf_pos != min_pos, do nothing
-		 *    (2) buf_pos == min_pos, means current minimizer has moved outside the window
-		 * **/
-		///three cases: 1. 
-		if (info.x <= min.x) { // a new minimum; then write the old min
-			if (l >= w + k && min.x != UINT64_MAX) kv_push(ha_mz1_t, *p, min);
-			min = info, min_pos = buf_pos;
-		} else if (buf_pos == min_pos) { // old min has moved outside the window
-			if (l >= w + k - 1 && min.x != UINT64_MAX) kv_push(ha_mz1_t, *p, min);
-			///buf_pos == min_pos, means current minimizer has moved outside the window
-			///so for now we need to find a new minimizer at the current window (w k-mers)
-			for (j = buf_pos + 1, min.x = UINT64_MAX; j < w; ++j) // the two loops are necessary when there are identical k-mers
-				if (min.x >= buf[j].x) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
-			for (j = 0; j <= buf_pos; ++j)
-				if (min.x >= buf[j].x) min = buf[j], min_pos = j;
-
-			if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
-				for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
-					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
-				for (j = 0; j <= buf_pos; ++j)
-					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
-			}
-		}
-		if (++buf_pos == w) buf_pos = 0;
-	}
-	if (min.x != UINT64_MAX)
-		kv_push(ha_mz1_t, *p, min);
-}
-
-
-
-void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, 
-kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct)
-{	///in default, w = 51, k = 51, is_hpc = 1
-	/**
-	 uint64_t x;
-	 uint64_t rid:28, pos:27, rev:1, span:8;
-	 **/
-	extern void *ha_ct_table;
-	if(dbg_ct != NULL) dbg_ct->a.n = 0;
-
-	static const ha_mz1_t dummy = { UINT64_MAX, 0, 0, 0 };
-	uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0}, filtered;
-	int i, j, l, buf_pos, min_pos, kmer_span = 0;
-	ha_mz1_t buf[256], min = dummy;
-	tiny_queue_t tq;
-	if(k_flag != NULL)
+	uint64_t ks = end;
+	int64_t t = 0, i, k, sp = -1, ep = -1, ovlp, tot = mmt[end]&0xffffffff, nt = 0;;
+	while (ks != 0xffffffff)
 	{
-		kv_resize(uint8_t, k_flag->a, (uint64_t)len);
-		k_flag->a.n = len; 
-		memset(k_flag->a.a, 0, k_flag->a.n);
-	} 
-
-
-	assert(len > 0 && len < 1<<27 && rid < 1<<28 && (w > 0 && w < 256) && (k > 0 && k <= 63));
-	///sizeof(ha_mz1_t) = 16
-	memset(buf, 0xff, w * 16);
-	memset(&tq, 0, sizeof(tiny_queue_t));
-	///len/w is the evaluated minimizer numbers
-	kv_resize(ha_mz1_t, *p, p->n + len/w);
-
-	for (i = l = buf_pos = min_pos = 0; i < len; ++i) {
-		int c = seq_nt4_table[(uint8_t)str[i]];
-		ha_mz1_t info = dummy;
-		if (c < 4) { // not an ambiguous base
-			int z;
-			if (is_hpc) {
-				int skip_len = 1;
-				if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {
-					for (skip_len = 2; i + skip_len < len; ++skip_len)
-						if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)
-							break;
-					i += skip_len - 1; // put $i at the end of the current homopolymer run
-				}
-				tq_push(&tq, skip_len);
-				kmer_span += skip_len;
-				///how many bases that are covered by this HPC k-mer
-				///kmer_span includes at most k HPC elements
-				if (tq.count > k) kmer_span -= tq_shift(&tq);
-			} else kmer_span = l + 1 < k? l + 1 : k; 
-			///kmer_span should be used for HPC k-mer
-			///non-HPC k-mer, kmer_span should be k
-			///kmer_span is used to calculate anchor pos on reverse complementary strand
-
-			if(k_flag != NULL) k_flag->a.a[i] = 1;///lable all useful base, which are not ignored by HPC
-
-			kmer[0] = (kmer[0] << 1 | (c&1))  & mask;                  // forward k-mer
-			kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
-			kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1))  << shift1; // reverse k-mer
-			kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;
-			if (kmer[1] == kmer[3]) continue; // skip "symmetric k-mers" as we don't know it strand
-			z = kmer[1] < kmer[3]? 0 : 1; // strand
-			++l;
-			if (l >= k && kmer_span < 256) {
-				uint64_t y;
-				y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
-
-				filtered = 0;
-				if(hf != 0) filtered = ha_ft_isflt(hf, y);
-				if(dbg_ct != NULL) kv_push(uint64_t, dbg_ct->a, ((((uint64_t)(query_ct_index(ha_ct_table, y))<<1)|filtered)<<32)|(uint64_t)(i));
-				///if (hf == 0 || ha_ft_isflt(hf, y) == 0)
-				if(filtered == 0)
-					info.x = y, info.rid = rid, info.pos = i, info.rev = z, info.span = kmer_span;
-				if(k_flag != NULL) k_flag->a.a[i]++;
-				if(k_flag != NULL && filtered == 1) k_flag->a.a[i]++;
-			}
-		} else l = 0, tq.count = tq.front = 0, kmer_span = 0;
-
-
-		//for non-HPC k-mer, l = i; but for HPC k-mer, l is always less than i
-		//i is the real base iterator, while l is the HPC base iterator
-		//only if l >= k, info is a useful minimizer (ha_mz1_t.x != UINT64_MAX)
-		//but even if l < k, infor is still stored into buf
-		buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
-		if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
-			for (j = buf_pos + 1; j < w; ++j)
-				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
-			for (j = 0; j < buf_pos; ++j)
-				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
+		i = ks/m; k = ks%m;
+		ks = mmt[ks]>>32;
+		if(ks == 0xffffffff || (int32_t)(ks/m) == (i-1))
+		{
+			t++;
+			ovlp = ((MIN(ep, (int64_t)ma[k].pos) >= MAX(sp, (int64_t)(ma[k].pos+1-ma[k].span)))? 
+						MIN(ep, (int64_t)ma[k].pos) - MAX(sp, (int64_t)(ma[k].pos+1-ma[k].span)) + 1:0);
+			if(ovlp != 0) fprintf(stderr, "ERROR-OVLP\n");
+			if(sp == -1 || sp > (ma[k].pos+1-ma[k].span)) sp = ma[k].pos+1-ma[k].span;
+			if(ep == -1 || ep < ma[k].pos) ep = ma[k].pos;
+			nt += (ma[k].rid);
 		}
-		/**
-		 * There are three cases:
-		 * 1. info.x <= min.x, means info is a new minimizer
-		 * 2. info.x > min.x, info is not a new minimizer
-		 *    (1) buf_pos != min_pos, do nothing
-		 *    (2) buf_pos == min_pos, means current minimizer has moved outside the window
-		 * **/
-		///three cases: 1. 
-		if (info.x <= min.x) { // a new minimum; then write the old min
-			if (l >= w + k && min.x != UINT64_MAX) kv_push(ha_mz1_t, *p, min);
-			min = info, min_pos = buf_pos;
-		} else if (buf_pos == min_pos) { // old min has moved outside the window
-			if (l >= w + k - 1 && min.x != UINT64_MAX) kv_push(ha_mz1_t, *p, min);
-			///buf_pos == min_pos, means current minimizer has moved outside the window
-			///so for now we need to find a new minimizer at the current window (w k-mers)
-			for (j = buf_pos + 1, min.x = UINT64_MAX; j < w; ++j) // the two loops are necessary when there are identical k-mers
-				if (min.x >= buf[j].x) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
-			for (j = 0; j <= buf_pos; ++j)
-				if (min.x >= buf[j].x) min = buf[j], min_pos = j;
-
-			if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
-				for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
-					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
-				for (j = 0; j <= buf_pos; ++j)
-					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
-			}
-		}
-		if (++buf_pos == w) buf_pos = 0;
 	}
-	if (min.x != UINT64_MAX)
-		kv_push(ha_mz1_t, *p, min);
-}
+	if(t != sn) fprintf(stderr, "ERROR-TN, t: %ld, sn: %d\n", t, sn);
+	if(nt != tot) fprintf(stderr, "ERROR-TOT, nt: %ld, tot: %ld\n", nt, tot);
+}
+
+void dbg_boundary(ha_mz1_v *p, st_mt_t *mt, int32_t w, int32_t k, int32_t tot_l)
+{
+	if(tot_l < w + k -1) return;
+	int32_t i, m, n = p->n, s, a;
+	for (i = 0; i < n; i++){
+		if(GL(*mt, i) >= w+k-1){
+			for (m = s = a = 0; m <= i; m++){
+				if(!A_M(*p, m)) continue;
+				if(GL(*mt, m) <= w+k-1){
+					a++;
+					if(mt->a[m]&0x100000000) s++;
+				}
+			}
+			if(a > 0 && s == 0){
+				fprintf(stderr, "\nERROR1, s: %d, n: %d, tot_l: %d, end_l: %ld\n", s, n, tot_l, GL(*mt, i));
+				for (m = s = a = 0; m <= i; m++){
+					if(!A_M(*p, m)) continue;
+					if(GL(*mt, m) <= w+k-1){
+						fprintf(stderr, "lp: %ld\n", GL(*mt, m));
+						a++;
+						if(mt->a[m]&0x100000000) s++;
+					}
+				}
+				
+			} 
+			
+			break;
+		}
+	}
+	if(i == n){
+		for (m = s = a = 0; m < n; m++){
+			if(!A_M(*p, m)) continue;
+			if(GL(*mt, m) <= w+k-1){
+				a++;
+				if(mt->a[m]&0x100000000) s++;
+			}
+		}
+		if(a > 0 && s == 0) fprintf(stderr, "ERROR2\n");
+	}
+
+	for (i = n-1; i >= 0; i--)
+	{
+		if (GL(*mt, i) + w <= tot_l + 1) {
+			for (m = i, s = a = 0; m < n; m++){
+				if(!A_M(*p, m)) continue;
+				if(GL(*mt, m) + w >= tot_l + 1){
+					a++;
+					if(mt->a[m]&0x100000000) s++;
+				} 				
+			}
+			if(a > 0 && s == 0) {
+				fprintf(stderr, "\nERROR3, s: %d, n: %d, tot_l: %d, end_l: %ld\n", s, n, tot_l, GL(*mt, i));
+				for (m = i, s = a = 0; m < n; m++){
+					if(!A_M(*p, m)) continue;
+					if(GL(*mt, m) + w >= tot_l + 1){
+						fprintf(stderr, "lp: %ld\n", GL(*mt, m));
+						a++;
+						if(mt->a[m]&0x100000000) s++;
+					} 				
+				}
+			}
+			
+			break;
+		}
+	}
+	
+	if(i < 0){
+		for (m = s = a = 0; m < n; m++){
+			if(!A_M(*p, m)) continue;
+			if(GL(*mt, m) + w >= tot_l + 1){
+				a++;
+				if(mt->a[m]&0x100000000) s++;
+			} 				
+		}
+		if(a > 0 && s == 0) fprintf(stderr, "ERROR4\n");
+	}
+}
+
+void debug_pl(const char *str, int len, int w, int k, int is_hpc, ha_mz1_v *p, const void *hf, st_mt_t *mt)
+{
+	int i, l, dbi, dbcnt = 0, kmer_span = 0;
+	tiny_queue_t tq;
+	memset(&tq, 0, sizeof(tiny_queue_t));
+	uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0};
+	
+    for (i = l = dbi = 0; i < len; ++i) {
+        int c = seq_nt4_table[(uint8_t)str[i]];
+        if (c < 4) { // not an ambiguous base
+            int z;
+            if (is_hpc) {
+                int skip_len = 1;
+                if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {
+                    for (skip_len = 2; i + skip_len < len; ++skip_len)
+                        if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)
+                            break;
+                    i += skip_len - 1; // put $i at the end of the current homopolymer run
+                }
+                tq_push(&tq, skip_len);
+                kmer_span += skip_len;
+                ///how many bases that are covered by this HPC k-mer
+                ///kmer_span includes at most k HPC elements
+                if (tq.count > k) kmer_span -= tq_shift(&tq);
+            } else kmer_span = l + 1 < k? l + 1 : k; 
+            ///kmer_span should be used for HPC k-mer
+            ///non-HPC k-mer, kmer_span should be k
+            ///kmer_span is used to calculate anchor pos on reverse complementary strand
+
+            kmer[0] = (kmer[0] << 1 | (c&1))  & mask;                  // forward k-mer
+            kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
+            kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1))  << shift1; // reverse k-mer
+            kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;
+            if (kmer[1] == kmer[3]) continue; // skip "symmetric k-mers" as we don't know it strand
+            z = kmer[1] < kmer[3]? 0 : 1; // strand
+            ++l;
+
+            if (l >= k && kmer_span < 256) {
+                uint64_t y;
+                int32_t cnt;
+                y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
+                cnt = hf? ha_ft_cnt(hf, y) : 0;
+
+				for (dbi = 0; dbi < (int32_t)mt->n; dbi++)
+				{
+					if(p->a[dbi].x == y && p->a[dbi].rid == cnt && p->a[dbi].pos == i && p->a[dbi].rev == z && p->a[dbi].span == kmer_span)
+					{
+						if(l != (int)mt->a[dbi]) fprintf(stderr, "ERROR\n");
+						dbcnt++;
+					}
+				}
+            }
+        } else l = 0, tq.count = tq.front = 0, kmer_span = 0;
+	}
+
+	if(dbcnt != (int32_t)mt->n) fprintf(stderr, "ERROR\n");
+	if(mt->n != p->n) fprintf(stderr, "ERROR\n");
+	for (dbi = 1; dbi < (int32_t)mt->n; dbi++)
+	{
+		if(p->a[dbi].pos <= p->a[dbi-1].pos || (int)mt->a[dbi] <= (int)mt->a[dbi-1])
+		{
+			fprintf(stderr, "ERROR\n");
+		}
+	}
+}
+
+static inline int mz1_mzcmp(const ha_mz1_t *a, const ha_mz1_t *b){return a->rid < b->rid? -1 : a->rid > b->rid? 1 : ((a->x > b->x) - (a->x < b->x));}
+#define mz1_mz_lt(a, b) (mz1_mzcmp(&(a), &(b)) < 0)
+KSORT_INIT(mz1_mz, ha_mz1_t, mz1_mz_lt)
+
+static inline int mz2_mzcmp(const ha_mzl_t *a, const ha_mzl_t *b){return a->rid < b->rid? -1 : a->rid > b->rid? 1 : ((a->x > b->x) - (a->x < b->x));}
+#define mz2_mz_lt(a, b) (mz2_mzcmp(&(a), &(b)) < 0)
+KSORT_INIT(mz2_mz, ha_mzl_t, mz2_mz_lt)
+
+
+#define HA_SC_INIT(sf, HType, VType, RidBits, PosBits)\
+inline void sf##_hf_select(VType *p, int32_t si, int32_t ei, int32_t n, int32_t len, int32_t sample_dist, HType *b, int32_t force)\
+{\
+	if(ei - si <= 1) return;\
+	int32_t ps = si < 0? 0 : p->a[si].pos;\
+	int32_t pe = ei == n? len : p->a[ei].pos;\
+	int32_t j, k, st = si + 1, en = ei;\
+	int32_t max_high_occ = (int32_t)((double)(pe - ps) / sample_dist + .499);\
+	if (max_high_occ > MAX_MAX_HIGH_OCC)\
+		max_high_occ = MAX_MAX_HIGH_OCC;\
+	for (j = st, k = 0; j < en && k < max_high_occ; ++j, ++k)\
+		b[k] = p->a[j], b[k].pos = j; /** b[].pos keeps the index in p->a[]**/\
+	ks_heapmake_##sf##_mz(k, b); /** initialize the binomial heap**/\
+	for (; j < en; ++j) { /** if there are more, choose top max_high_occ**/\
+		if (sf##_mz_lt(p->a[j], b[0])) { /** then update the heap**/\
+			b[0] = p->a[j], b[0].pos = j;\
+			ks_heapdown_##sf##_mz(0, k, b);\
+		}\
+	}\
+	/**ks_heapsort_mz(k, b); // sorting is not needed for now**/\
+	for (j = 0; j < k; ++j)\
+		if (b[j].rid < pe - ps || force)\
+			p->a[b[j].pos].rid = 0;\
+}\
+static inline int sf##_mzcmp_l(const VType *p, int32_t ai, int32_t bi)\
+{\
+	if(ai >= 0 && bi >= 0){\
+		HType *a = &(p->a[ai]), *b = &(p->a[bi]);\
+		if(a->rid > 0 && b->rid > 0) return sf##_mzcmp(a, b);\
+		return (a->rid == 0) - (b->rid == 0);\
+	}\
+	return (ai < 0) - (bi < 0);\
+}\
+int32_t sf##_qfw(VType *p, st_mt_t *mt, int32_t n, int32_t tot_l, int32_t ws, int32_t i, int32_t *mi)\
+{\
+	int32_t m, si;\
+	for (si = i, (*mi) = -1; i < n; i++){\
+		if(GL(*mt, i) >= ws || (i+1 < n && GL(*mt, i) < ws && GL(*mt, i+1) > ws) || \
+				(i+1 == n && tot_l >= ws && GL(*mt, i) < ws)){\
+			for (m = si; m <= i; m++){\
+				if(!A_M(*p, m)) continue;\
+				if(sf##_mzcmp_l(p, *mi, m) >= 0) (*mi) = m;\
+			}\
+			if((*mi) >= 0 && A_M(*p, *mi)){\
+				for (m = si; m <= i; m++){\
+					if(!A_M(*p, m)) continue;\
+					if(sf##_mzcmp_l(p, *mi, m) == 0) mt->a[m] |= 0x100000000;\
+				}\
+			}\
+			break;\
+		}\
+	}\
+	return i;\
+}\
+static void sf##_select_mz_h(VType *p, st_mt_t *mt, int len, int sample_dist, int32_t w, int32_t k, int32_t tot_l)\
+{ /**for high-occ minimizers, choose up to max_high_occ in each high-occ streak**/\
+	int32_t i, mi = -1, si, last0 = -1, n = (int32_t)p->n, m = 0, ws = w + k - 1;\
+	if (n == 0) return;\
+	assert((int64_t)(n) < (int64_t)((((uint64_t)1)<<PosBits)));\
+	for (i = m = 0, last0 = -1; i <= n; ++i) {\
+		if (i == n || p->a[i].rid == 0) {\
+			if (i - last0 > 1) {\
+				int32_t ps = last0 < 0? 0 : p->a[last0].pos;\
+				int32_t pe = i == n? len : p->a[i].pos;\
+				if(((int32_t)((double)(pe - ps) / sample_dist + .499)) > 0){\
+					last0 = -2;\
+					m++;\
+					break;\
+				}\
+			}\
+			last0 = i;\
+		}\
+	}\
+	if (m == 0) return; /**no high-frequency k-mers; do nothing**/\
+	if(last0 >= -1) goto sf##_ff;\
+	i = 0;\
+	i = sf##_qfw(p, mt, n, tot_l, ws, i, &mi);\
+	if(i == n) goto sf##_ff;\
+	for (si = 0, i++; i < n; i++){\
+		for (; si < i; si++){\
+			if(GL(*mt, si) + w > GL(*mt, i)) break;\
+		}\
+		/**a new minimum; then write the old min**/\
+		if(sf##_mzcmp_l(p, i, mi) <= 0) {\
+			if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;\
+			mi = i;\
+		}/**old min has moved outside the window**/\
+		else if(si > mi){\
+			if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;\
+			for (m = si, mi = -1; m <= i; m++){\
+				if(sf##_mzcmp_l(p, mi, m) >= 0) mi = m;\
+            }\
+			if(A_M(*p, mi)){\
+				for (m = si; m <= i; m++){\
+					if(!A_M(*p, m)) continue;\
+					if(sf##_mzcmp_l(p, mi, m) == 0) mt->a[m] |= 0x100000000;\
+				}\
+			}\
+		}\
+	}\
+	if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;\
+	for (i = n - 1; si < n && GL(*mt, si) + w <= tot_l + 1; si++){\
+		if(si > mi){\
+			if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;\
+			for (m = si, mi = -1; m <= i; m++){\
+				if(sf##_mzcmp_l(p, mi, m) >= 0) mi = m;\
+            }\
+			if(A_M(*p, mi)){\
+				for (m = si; m <= i; m++){\
+					if(!A_M(*p, m)) continue;\
+					if(sf##_mzcmp_l(p, mi, m) == 0) mt->a[m] |= 0x100000000;\
+				}\
+			}\
+		}\
+	}\
+	/**dbg_boundary(p, mt, w, k, tot_l);**/\
+	HType b[MAX_MAX_HIGH_OCC];\
+	for (i = 0, last0 = -1; i <= n; ++i) {\
+		if (i == n || p->a[i].rid == 0) {\
+			if (i - last0 > 1) {\
+				int32_t ps = last0 < 0? 0 : p->a[last0].pos;\
+				int32_t pe = i == n? len : p->a[i].pos;\
+				if(((int32_t)((double)(pe - ps) / sample_dist + .499)) > 0){\
+					for (m = last0 + 1, mi = 0; m < i; ++m){\
+						if(mt->a[m]&0x100000000) p->a[m].rid = 0, mi++;\
+					}\
+					if(mi == 0) sf##_hf_select(p, last0, i, n, len, sample_dist, b, 0);\
+				}\
+			}\
+			last0 = i;\
+		}\
+	}\
+	sf##_ff:\
+	for (i = n = 0; i < (int32_t)p->n; ++i) /**squeeze out filtered minimizers**/\
+		if (p->a[i].rid == 0)\
+			p->a[n++] = p->a[i];\
+	p->n = n;\
+}\
+void sf##_refine_select(VType *mz, int32_t sidx, int32_t eidx, int32_t sn, int32_t min_freq, st_mt_t *mm, int32_t *rsi, int32_t *rei, void *km)\
+{\
+	int32_t n = sn, m = eidx + 1 - sidx, i, k, t, mk=-1;\
+	uint64_t ix, kx, ks;\
+	kv_resize_km(km, uint64_t, *mm, mm->n+n*m);\
+	HType *ma = mz->a + sidx;\
+	uint64_t *mmt = mm->a + mm->n;\
+	/**fprintf(stderr, "[M::%s::] ==> +n: %d, m: %d, sn: %d, sidx: %d, eidx: %d\n", __func__, n, m, sn, sidx, eidx);**/\
+	for (i = 0; i < n; i++) /**how many selected minimizers**/\
+	{\
+		for (k = 0, mk = -1; k < m; k++) /**how many minimizers in total**/\
+		{\
+			if((int32_t)(ma[k].rid)<min_freq) continue;\
+			ks = ma[k].pos + 1 - ma[k].span; t = -1;\
+			if(i > 0){\
+				for (t = k-1; t >= 0 && (ma[t].pos >= ks||(int32_t)(ma[t].rid)<min_freq); t--);\
+			}\
+			ix = (i <= 0?0:(t<0?0xffffffff:(GMC(mmt, i-1,t,m)&0xffffffff)));\
+			if(ix < 0xffffffff) ix += (ma[k].rid);\
+			kx = (mk < 0?0xffffffff:(GMC(mmt, i, mk,m)&0xffffffff));\
+			ks = MIN(ix, kx);\
+			/**fprintf(stderr, "ks: %lu, i: %d (n-%d), k: %d (m-%d), ix: %lu, kx: %lu, t: %d, mk: %d\n", ks, i, n, k, m, ix, kx, t, mk);**/\
+			if((ks&0xffffffff) == 0xffffffff) ks |= ((uint64_t)0xffffffff)<<32;\
+			else if(ks == ix) ks |= (uint64_t)(i>0?(i-1)*m+t:0xffffffff)<<32;\
+			else if(ks == kx) ks |= (uint64_t)(mk>=0?i*m+mk:0xffffffff)<<32;\
+			GMC(mmt, i,k,m) = ks;\
+			mk = k;\
+		}\
+	}\
+	/**fprintf(stderr, "[M::%s::] ==> ++n: %d, m: %d, sn: %d, sidx: %d, eidx: %d\n", __func__, n, m, sn, sidx, eidx);**/\
+	ks = (n-1)*m + mk; ix = (uint64_t)-1; kx = 0;\
+	while (ks != 0xffffffff)\
+	{\
+		i = ks/m; k = ks%m;\
+		ks = mmt[ks]>>32;\
+		/**fprintf(stderr, "i: %d, k: %d, ks: %lu\n", i, k, ks);**/\
+		if(ks == 0xffffffff || (int32_t)(ks/m) == (i-1)){\
+			mm->a[sidx+k] = 1;\
+			ix = MIN((uint64_t)k, ix); kx = MAX((uint64_t)k, kx);\
+		}\
+	}\
+	/**debug_refine(ma, mmt, sn, n, m, (n-1)*m + mk);**/\
+	if(rsi) (*rsi) = ix + sidx;\
+	if(rei) (*rei) = kx + sidx;\
+}\
+void sf##_refine_sketch(VType *p, ha_pt_t *pt, int32_t rlen, int32_t dp_min_len, float er, int32_t min_freq, st_mt_t *mt, void *km)\
+{\
+	/**fprintf(stderr, "[M::%s::] ==> #########10#########, rlen: %d\n", __func__, rlen);**/\
+	int32_t i, n = p->n, bd, len = MIN(rlen, dp_min_len), sublen, cnt, ei, li, ri;\
+	int32_t sn =  len*er + 1;\
+	kv_resize_km(km, uint64_t, *mt, (int64_t)p->n);\
+	mt->n = p->n; memset(mt->a, 0, sizeof(uint64_t)*p->n);\
+	for (i = 0; i < n; i++) p->a[i].rid = ha_pt_cnt(pt, p->a[i].x);\
+	for (i = cnt = 0, bd = -1, ei = -1; i < n; i++){\
+		if((int32_t)(p->a[i].rid)<min_freq) continue;\
+		sublen = p->a[i].pos + 1;\
+		if(sublen > len) break;\
+		else ei = i;\
+		if((int32_t)(p->a[i].pos + 1 - p->a[i].span) > bd){\
+			bd = p->a[i].pos;\
+			cnt++;\
+		}\
+	}\
+	/**fprintf(stderr, "[M::%s::] ==> +cnt: %d, sn: %d, ei: %d, n: %d\n", __func__, cnt, sn, ei, n);**/\
+	if(cnt >= sn) sf##_refine_select(p, 0, ei, sn, min_freq, mt, NULL, &li, km);\
+	else{\
+		li = i-1;\
+		for (i = 0; i <= li; i++) mt->a[i] = 1;\
+	}\
+	if(len < rlen){\
+		for (i = n-1, cnt = 0, bd = rlen+1, ei = -1; i >= 0; i--){\
+			if((int32_t)(p->a[i].rid)<min_freq) continue;\
+			sublen = rlen - (p->a[i].pos + 1 - p->a[i].span);\
+			if(sublen > len) break;\
+			else ei = i;\
+			if((int32_t)(p->a[i].pos) < bd){\
+				bd = p->a[i].pos + 1 - p->a[i].span;\
+				cnt++;\
+			}\
+		}\
+		/**fprintf(stderr, "[M::%s::] ==> -cnt: %d, sn: %d, ei: %d, n: %d\n", __func__, cnt, sn, ei, n);**/\
+		if(cnt >= sn) sf##_refine_select(p, ei, n-1, sn, min_freq, mt, &ri, NULL, km);\
+		else {\
+			ri = i+1;\
+			for (i = ri; i <= n-1; i++) mt->a[i] = 1;\
+		}\
+		/**fprintf(stderr, "[M::%s::] ==> --cnt: %d, sn: %d, ei: %d, n: %d\n", __func__, cnt, sn, ei, n);**/\
+		if(ri - li >= 2){\
+			li++; ri--;\
+			sn =  (p->a[ri].pos - p->a[li].pos + p->a[li].span)*er + 1;\
+			for (i = li, cnt = 0, bd = -1; i <= ri; i++){\
+				if((int32_t)(p->a[i].rid)<min_freq) continue;\
+				if((int32_t)(p->a[i].pos + 1 - p->a[i].span) > bd){\
+					bd = p->a[i].pos;\
+					cnt++;\
+					if(cnt >= sn) break;\
+				}\
+			}\
+			if(cnt >= sn) sf##_refine_select(p, li, ri, sn, min_freq, mt, NULL, NULL, km);\
+			else for (i = li; i <= ri; i++) mt->a[i] = 1;\
+		}\
+	}\
+	/**fprintf(stderr, "[M::%s::] ==> #########20#########, p->n: %u, n: %d\n", __func__, p->n, n);**/\
+	for (i = sn = 0; i < n; i++){\
+		if(mt->a[i]){\
+			p->a[sn] = p->a[i];\
+			sn++;\
+		}\
+	}\
+	/**if(p->n != sn) fprintf(stderr, "[M::%s::] ==> #########21#########, p->n: %u, sn: %d\n", __func__, p->n, sn);**/\
+	p->n = sn;\
+}\
+/**\
+ * Find symmetric (w,k)-minimizers on a DNA sequence\
+ *\
+ * @param str    DNA sequence\
+ * @param len    length of $str\
+ * @param w      find a minimizer for every $w consecutive k-mers\
+ * @param k      k-mer size\
+ * @param rid    reference ID; will be copied to the output $p array\
+ * @param is_hpc homopolymer-compressed or not\
+ * @param p      minimizers\
+ */\
+void sf##_ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, VType *p, const void *hf, int sample_dist, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct, ha_pt_t *pt, int min_freq, int32_t dp_min_len, float dp_e, st_mt_t *mt, int32_t ws, int32_t is_unique, void *km)\
+{   /**in default, w = 51, k = 51, is_hpc = 1**/\
+    extern void *ha_ct_table;\
+    static const HType dummy = { UINT64_MAX, (((uint64_t)1)<<RidBits) - 1, 0, 0, 0};\
+    uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0};\
+    int i, j, l, tl = 0, buf_pos, min_pos, kmer_span = 0;\
+    HType buf[256], min = dummy;\
+    uint32_t buf_p[256], min_s = (uint32_t)-1;\
+    tiny_queue_t tq;\
+    assert(len > 0 && (int64_t)(len) < (int64_t)((((uint64_t)1)<<PosBits)) && (int64_t)(rid) < (int64_t)((((uint64_t)1)<<RidBits)) && (w > 0 && w < 256) && (k > 0 && k <= 63));\
+    if (dbg_ct != NULL) dbg_ct->a.n = 0;\
+    if (k_flag != NULL) {\
+        kv_resize_km(km, uint8_t, k_flag->a, (uint64_t)len);\
+        k_flag->a.n = len;\
+        memset(k_flag->a.a, 0, k_flag->a.n);\
+    }\
+    memset(buf, 0xff, w * sizeof(HType));\
+    memset(&tq, 0, sizeof(tiny_queue_t));\
+    /**len/w is the evaluated minimizer numbers**/\
+    kv_resize_km(km, HType, *p, p->n + len/w);\
+    kv_resize_km(km, uint64_t, *mt, (int64_t)p->m); mt->n = p->n;\
+    for (i = l = tl = buf_pos = min_pos = 0; i < len; ++i) {\
+        int c = seq_nt4_table[(uint8_t)str[i]];\
+        HType info = dummy;\
+        if (c < 4) { /**not an ambiguous base**/\
+            int z;\
+            if (is_hpc) {\
+                int skip_len = 1;\
+                if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {\
+                    for (skip_len = 2; i + skip_len < len; ++skip_len)\
+                        if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)\
+                            break;\
+                    i += skip_len - 1; /**put $i at the end of the current homopolymer run**/\
+                }\
+                tq_push(&tq, skip_len);\
+                kmer_span += skip_len;\
+                /**how many bases that are covered by this HPC k-mer\
+                kmer_span includes at most k HPC elements**/\
+                if (tq.count > k) kmer_span -= tq_shift(&tq);\
+            } else kmer_span = l + 1 < k? l + 1 : k;\
+            /**kmer_span should be used for HPC k-mer\
+            non-HPC k-mer, kmer_span should be k\
+            kmer_span is used to calculate anchor pos on reverse complementary strand**/\
+            if (k_flag != NULL) k_flag->a.a[i] = 1;/**lable all useful base, which are not ignored by HPC**/\
+            kmer[0] = (kmer[0] << 1 | (c&1))  & mask;/**forward k-mer**/\
+            kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;\
+            kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1))  << shift1; /**reverse k-mer**/\
+            kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;\
+            if (kmer[1] == kmer[3]) continue; /** skip "symmetric k-mers" as we don't know it strand**/\
+            z = kmer[1] < kmer[3]? 0 : 1; /** strand**/\
+            ++l; tl++;\
+            if (l >= k && kmer_span < 256) {\
+                uint64_t y;\
+                int32_t cnt, filtered;\
+                y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);\
+                cnt = hf? ha_ft_cnt(hf, y) : 0;\
+				filtered = (cnt >= 1<<28);\
+                if(is_unique && (!filtered)) {\
+					filtered = (cnt == 0);\
+					cnt = (cnt == 1? 0:cnt);\
+				}\
+                if (dbg_ct != NULL) kv_push_km(km, uint64_t, dbg_ct->a, ((((uint64_t)(query_ct_index(ha_ct_table, y))<<1)|filtered)<<32)|(uint64_t)(i));\
+                if (!filtered) info.x = y, info.rid = cnt, info.pos = i, info.rev = z, info.span = kmer_span; /** initially ha_mz1_t::rid keeps the k-mer count**/\
+                if (k_flag != NULL) k_flag->a.a[i]++;\
+                if (k_flag != NULL && filtered > 0) k_flag->a.a[i]++;\
+            }\
+        } else l = 0, tq.count = tq.front = 0, kmer_span = 0;\
+        buf[buf_pos] = info; /**need to do this here as appropriate buf_pos and buf[buf_pos] are needed below**/\
+        buf_p[buf_pos] = l;\
+        if (l == w + k - 1 && min.x != UINT64_MAX) { /**special case for the first window - because identical k-mers are not stored yet**/\
+			for (j = buf_pos + 1; j < w; ++j){\
+                if (sf##_mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos){\
+                    kv_push_km(km, HType, *p, buf[j]); kv_push_km(km, uint64_t, *mt, buf_p[j]);\
+                }\
+            }\
+            for (j = 0; j < buf_pos; ++j){\
+                if (sf##_mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos){\
+                    kv_push_km(km, HType, *p, buf[j]); kv_push_km(km, uint64_t, *mt, buf_p[j]);\
+                }\
+            }\
+        }\
+        /**\
+         * There are three cases:\
+         * 1. info.x <= min.x, means info is a new minimizer\
+         * 2. info.x > min.x, info is not a new minimizer\
+         *    (1) buf_pos != min_pos, do nothing\
+         *    (2) buf_pos == min_pos, means current minimizer has moved outside the window\
+         * **/\
+        /**three cases: 1.**/\
+        if (sf##_mzcmp(&min, &info) >= 0) { /**a new minimum; then write the old min**/\
+            if (l >= w + k && min.x != UINT64_MAX){\
+                kv_push_km(km, HType, *p, min); kv_push_km(km, uint64_t, *mt, min_s);\
+            }\
+            min = info, min_pos = buf_pos, min_s = buf_p[buf_pos];\
+        } else if (buf_pos == min_pos) { /**old min has moved outside the window**/\
+            if (l >= w + k - 1 && min.x != UINT64_MAX){\
+                kv_push_km(km, HType, *p, min); kv_push_km(km, uint64_t, *mt, min_s);\
+            }\
+            /**buf_pos == min_pos, means current minimizer has moved outside the window\
+            so for now we need to find a new minimizer at the current window (w k-mers)**/\
+            for (j = buf_pos + 1, min = dummy; j < w; ++j) /**the two loops are necessary when there are identical k-mers**/\
+                if (sf##_mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j, min_s = buf_p[j]; /** >= is important s.t. min is always the closest k-mer**/\
+            for (j = 0; j <= buf_pos; ++j)\
+                if (sf##_mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j, min_s = buf_p[j];\
+            if (l >= w + k - 1 && min.x != UINT64_MAX) { /**write identical k-mers**/\
+                for (j = buf_pos + 1; j < w; ++j) /**these two loops make sure the output is sorted**/\
+                    if (sf##_mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos){\
+                        kv_push_km(km, HType, *p, buf[j]); kv_push_km(km, uint64_t, *mt, buf_p[j]);\
+                    }\
+                for (j = 0; j <= buf_pos; ++j)\
+                    if (sf##_mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos){\
+                        kv_push_km(km, HType, *p, buf[j]); kv_push_km(km, uint64_t, *mt, buf_p[j]);\
+                    }\
+            }\
+        }\
+        if (++buf_pos == w) buf_pos = 0;\
+    }\
+    if (min.x != UINT64_MAX){\
+        kv_push_km(km, HType, *p, min); kv_push_km(km, uint64_t, *mt, min_s);\
+    }\
+	/**debug_pl(str, len, w, k, is_hpc, p, hf, mt);**/\
+    if (sample_dist > w) sf##_select_mz_h(p, mt, len, sample_dist, ws, k, tl);\
+    if (dp_min_len > 0 && pt && mt) sf##_refine_sketch(p, pt, len, dp_min_len, dp_e, min_freq, mt, km);\
+	for (i = 0; i < (int)p->n; ++i) /**populate .rid as this was keeping counts**/\
+		p->a[i].rid = rid;\
+}
+
+HA_SC_INIT(mz1, ha_mz1_t, ha_mz1_v, 28, 27)
+HA_SC_INIT(mz2, ha_mzl_t, ha_mzl_v, 31, 32)

tovlp.h

@@ -0,0 +1,26 @@
+#ifndef __TOVLP__
+#define __TOVLP__
+#include <stdint.h>
+#include "Overlaps.h"
+
+typedef struct {///[cBeg, cEnd)
+	uint32_t ui, len, cBeg, cEnd;
+    uint32_t *a, an;
+    ma_ug_t *ug;
+    utg_trans_t *o;
+} utg_trans_hit_idx;
+
+utg_trans_t *init_utg_trans_t(ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, asg_t *read_g, int max_hang, int min_ovlp);
+void destroy_utg_trans_t(utg_trans_t **o);
+void asg_bub_collect_ovlp(ma_ug_t *ug, uint32_t v0, buf_t *b, utg_trans_t *o);
+void collect_trans_ovlp(const char* cmd, buf_t* pri, uint64_t pri_offset, buf_t* aux, uint64_t aux_offset,
+ma_ug_t *ug, utg_trans_t *o);
+int asg_arc_decompress(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources,
+R_to_U* ruIndex, utg_trans_t *o);
+int asg_arc_decompress_mul(asg_t *g, ma_ug_t *ug, asg_t *read_sg, uint32_t positive_flag, uint32_t negative_flag,
+ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, utg_trans_t *o);
+kv_u_trans_t *pt_pdist(ma_ug_t *ug, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, 
+kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, uint32_t min_chain_cnt);
+void reset_utg_trans_hit_idx(utg_trans_hit_idx *t, uint32_t* i_x_a, uint32_t i_x_n, ma_ug_t *i_ug, 
+utg_trans_t *i_o, uint32_t i_cBeg, uint32_t i_cEnd);
+#endif