Merge branch 'master' into hifiasm-v0.14

.github/workflows/ci.yaml

@@ -1,21 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        compiler: [gcc]
-
-    steps:
-    - name: Checkout minimap2
-      uses: actions/checkout@v2
-
-    - name: Compile with ${{ matrix.compiler }}
-      run:  make CC=${{ matrix.compiler }}

Assembly.cpp

@@ -10,12 +10,9 @@
 #include "Correct.h"
 #include "htab.h"
 #include "kthread.h"
-#include "rcut.h"
 
 void ha_get_candidates_interface(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_region_alloc *overlap_list, overlap_region_alloc *overlap_list_hp, Candidates_list *cl, double bw_thres, 
-int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* chain_idx, ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, st_mt_t *sp);
-void ha_get_ug_candidates(ha_abuf_t *ab, int64_t rid, ma_utg_t *u, ma_utg_v *ua, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag,
-kvec_t_u64_warp* chain_idx, void *ha_flt_tab, ha_pt_t *ha_idx, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, double chain_match_rate);
+int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* chain_idx, ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct);
 void ha_sort_list_by_anchor(overlap_region_alloc *overlap_list);
 
 All_reads R_INF;
@@ -437,8 +434,6 @@ typedef struct {
     kvec_t_u64_warp r_buf;
     kvec_t_u8_warp k_flag;
     overlap_region tmp_region;
-    ma_utg_v *ua;
-    st_mt_t sp;
 } ha_ovec_buf_t;
 
 ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov)
@@ -455,7 +450,6 @@ ha_ovec_buf_t *ha_ovec_init(int is_final, int save_ov)
     kv_init(b->b_buf.a);
     kv_init(b->r_buf.a);
     kv_init(b->k_flag.a);
-    kv_init(b->sp);
 	b->ab = ha_abuf_init();
 	if (!b->is_final) {
 		init_Cigar_record(&b->cigar1);
@@ -480,7 +474,6 @@ void ha_ovec_destroy(ha_ovec_buf_t *b)
     kv_destroy(b->b_buf.a);
     kv_destroy(b->r_buf.a);
     kv_destroy(b->k_flag.a);
-    kv_destroy(b->sp);
 	if (!b->is_final) {
 		destory_Cigar_record(&b->cigar1);
 		destory_Graph(&b->POA_Graph);
@@ -544,7 +537,7 @@ static void worker_ovec(void *data, long i, int tid)
 	int fully_cov, abnormal;
 
     ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 
-    0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
+    0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL);
 
 	clear_Cigar_record(&b->cigar1);
 	clear_Round2_alignment(&b->round2);
@@ -602,7 +595,7 @@ static void worker_ovec_related_reads(void *data, long i, int tid)
         int fully_cov, abnormal, q_idx = k;
 
         ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 
-        0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]), &(b->sp));
+        0.02, asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), &(R_INF_FLAG.candidate_count[q_idx]));
 
         clear_Cigar_record(&b->cigar1);
         clear_Round2_alignment(&b->round2);
@@ -628,7 +621,7 @@ static void worker_ovec_related_reads(void *data, long i, int tid)
 
         pthread_mutex_lock(&R_INF_FLAG.OutputMutex);
 
-        fprintf(R_INF_FLAG.fp, "\n>%.*s, rid: %ld\n", (int)Get_NAME_LENGTH((R_INF), i), Get_NAME((R_INF), i), i);
+        fprintf(R_INF_FLAG.fp, "\n>%.*s\n", (int)Get_NAME_LENGTH((R_INF), i), Get_NAME((R_INF), i));
         fprintf(R_INF_FLAG.fp, "%d-th round, len: %lu, hom_cov: %d, max_n_chain: %d\n", 
             asm_opt.number_of_round, Get_READ_LENGTH(R_INF, i), asm_opt.hom_cov, asm_opt.max_n_chain);
 
@@ -1294,7 +1287,7 @@ static void worker_ov_final(void *data, long i, int tid)
 
 	//get_new_candidates(i, &g_read, &overlap_list, &array_list, &l, 0.001, 0);
     ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, 0.001, 
-    asm_opt.max_n_chain, 0, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
+    asm_opt.max_n_chain, 0, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL);
 
 	overlap_region_sort_y_id(b->olist.list, b->olist.length);
 	ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
@@ -1360,7 +1353,7 @@ static void worker_ov_final_high_het(void *data, long i, int tid)
     ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid];
 
     ha_get_candidates_interface(b->ab, i, &b->self_read, &b->olist, &b->olist_hp, &b->clist, HIGH_HET_ERROR_RATE, 
-    asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL, &(b->sp));
+    asm_opt.max_n_chain, 1, &(b->k_flag), &b->r_buf, &(R_INF.paf[i]), &(R_INF.reverse_paf[i]), &(b->tmp_region), NULL);
 
     overlap_region_sort_y_id(b->olist.list, b->olist.length);
     ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
@@ -1608,62 +1601,10 @@ void ha_overlap_final(void)
     asm_opt.het_cov = het_cov;
 }
 
-static void worker_ov_utg(void *data, long i, int tid)
-{
-	ha_ovec_buf_t *b = ((ha_ovec_buf_t**)data)[tid];
-    if(b->ua->a[i].len == 0) return;
 
-    ha_get_ug_candidates(b->ab, i, &(b->ua->a[i]), b->ua, &b->olist, &b->clist, 
-    0.3, asm_opt.polyploidy*5, 0, &(b->k_flag), &b->r_buf, ha_flt_tab, ha_idx,
-    &(b->tmp_region), NULL, /**0.3**/0);
-    
-	overlap_region_sort_y_id(b->olist.list, b->olist.length);
-	ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length);
-	ma_hit_sort_tn(R_INF.reverse_paf[i].buffer, R_INF.reverse_paf[i].length);
-
-	update_overlaps(&b->olist, &(R_INF.paf[i]), &b->self_read, &b->ovlp_read, 1, 1);
-	update_overlaps(&b->olist, &(R_INF.reverse_paf[i]), &b->self_read, &b->ovlp_read, 2, 0);
-	///recover missing exact overlaps
-	update_exact_overlaps(&b->olist, &b->self_read, &b->ovlp_read);
-
-	///Final_phasing(&overlap_list, &cigarline, &g_read, &overlap_read, c2n);
-	push_final_overlaps(&(R_INF.paf[i]), R_INF.reverse_paf, &b->olist, 1);
-	push_final_overlaps(&(R_INF.reverse_paf[i]), R_INF.reverse_paf, &b->olist, 2);
-}
-
-
-void ug_idx_build(ma_ug_t *ug, int hap_n)
-{
-    int flag = asm_opt.flag&HA_F_NO_HPC, i;
-    asm_opt.flag -= flag;
-    ha_flt_tab = ha_ft_ug_gen(&asm_opt, &(ug->u), hap_n);
-    ha_idx = ha_pt_ug_gen(&asm_opt, ha_flt_tab, &(ug->u), hap_n);
-
-    ha_ovec_buf_t **b = NULL;
-    // overlap and correct reads
-    CALLOC(b, asm_opt.thread_num);
-    for (i = 0; i < asm_opt.thread_num; ++i)
-    {
-        b[i] = ha_ovec_init(1, 1);
-        b[i]->ua = &(ug->u);
-    }
-            
-    kt_for(asm_opt.thread_num, worker_ov_utg, b, R_INF.total_reads);
-
-    for (i = 0; i < asm_opt.thread_num; ++i)
-        ha_ovec_destroy(b[i]);
-    free(b);
-
-    ha_ft_destroy(ha_flt_tab); 
-    ha_pt_destroy(ha_idx);
-    asm_opt.flag += flag;
-    exit(1);
-}
 
 int ha_assemble(void)
 {
-    // debug_mc_g_t(MC_NAME);
-    // debug_mc_gg_t(MC_NAME, 0, 0);
 	extern void ha_extract_print_list(const All_reads *rs, int n_rounds, const char *o);
 	int r, hom_cov = -1, ovlp_loaded = 0;
 	if (asm_opt.load_index_from_disk && load_all_data_from_disk(&R_INF.paf, &R_INF.reverse_paf, asm_opt.output_file_name)) {

Assembly.h

@@ -1,7 +1,6 @@
 #ifndef __ASSEMBLY__
 #define __ASSEMBLY__
 #include "CommandLines.h"
-#include "Overlaps.h"
 
 #define FORWARD 0
 #define REVERSE_COMPLEMENT (0x8000000000000000)
@@ -15,6 +14,5 @@
 #define RESEED_HP_RATE 0.9
 
 int ha_assemble(void);
-void ug_idx_build(ma_ug_t *ug, int hap_n);
 
 #endif

CommandLines.cpp

@@ -21,29 +21,15 @@ static ko_longopt_t long_options[] = {
 	{ "max-od-final",  ko_no_argument, 306 },
 	{ "ex-list",       ko_required_argument, 307 },
 	{ "ex-iter",       ko_required_argument, 308 },
-    { "hom-cov",     ko_required_argument, 309 },
+    { "purge-cov",     ko_required_argument, 309 },
     { "pri-range",     ko_required_argument, 310 },
+    { "high-het",      ko_no_argument, 311 },
     { "lowQ",          ko_required_argument, 312 },
 	{ "min-hist-cnt",  ko_required_argument, 313 },
     { "h1",      ko_required_argument, 314 },
     { "h2",      ko_required_argument, 315 },
     { "enzyme",      ko_required_argument, 316 },
     { "b-cov",      ko_required_argument, 317 },
-    { "h-cov",      ko_required_argument, 318 },
-    { "m-rate",      ko_required_argument, 319 },
-    { "primary",      ko_no_argument, 320 },
-    { "t-occ",      ko_required_argument, 321 },
-    { "seed",      ko_required_argument, 322 },
-    { "n-perturb",      ko_required_argument, 323 },
-    { "f-perturb",      ko_required_argument, 324 },
-    { "n-hap",      ko_required_argument, 325 },
-    { "n-weight",      ko_required_argument, 326 },
-    { "l-msjoin",      ko_required_argument, 327 },
-    { "purge-max",     ko_required_argument, 328 },
-    { "fast",     ko_no_argument, 329 },
-    { "dp-er",     ko_required_argument, 330},
-    { "max-kocc",     ko_required_argument, 331},
-    { "hg-size",     ko_required_argument, 332},
 	{ 0, 0, 0 }
 };
 
@@ -59,94 +45,64 @@ void Print_H(hifiasm_opt_t* asm_opt)
     fprintf(stderr, "Usage: hifiasm [options] <in_1.fq> <in_2.fq> <...>\n");
     fprintf(stderr, "Options:\n");
 	fprintf(stderr, "  Input/Output:\n");
-    fprintf(stderr, "    -o STR       prefix of output files [%s]\n", asm_opt->output_file_name);
-    fprintf(stderr, "    -t INT       number of threads [%d]\n", asm_opt->thread_num);
-    fprintf(stderr, "    -h           show help information\n");
-    fprintf(stderr, "    --version    show version number\n");
+    fprintf(stderr, "    -o STR      prefix of output files [%s]\n", asm_opt->output_file_name);
+    fprintf(stderr, "    -i          ignore saved read correction and overlaps\n");
+    fprintf(stderr, "    -t INT      number of threads [%d]\n", asm_opt->thread_num);
+    fprintf(stderr, "    -z INT      length of adapters that should be removed [%d]\n", asm_opt->adapterLen);
+    fprintf(stderr, "    --version   show version number\n");
 	fprintf(stderr, "  Overlap/Error correction:\n");
-    fprintf(stderr, "    -k INT       k-mer length (must be <64) [%d]\n", asm_opt->k_mer_length);
-	fprintf(stderr, "    -w INT       minimizer window size [%d]\n", asm_opt->mz_win);
-	fprintf(stderr, "    -f INT       number of bits for bloom filter; 0 to disable [%d]\n", asm_opt->bf_shift);
-	fprintf(stderr, "    -D FLOAT     drop k-mers occurring >FLOAT*coverage times [%.1f]\n", asm_opt->high_factor);
-	fprintf(stderr, "    -N INT       consider up to max(-D*coverage,-N) overlaps for each oriented read [%d]\n", asm_opt->max_n_chain);
-    fprintf(stderr, "    -r INT       round of correction [%d]\n", asm_opt->number_of_round);
-    fprintf(stderr, "    -z INT       length of adapters that should be removed [%d]\n", asm_opt->adapterLen);
-    fprintf(stderr, "    --max-kocc   INT\n");
-    fprintf(stderr, "                 employ k-mers occurring <INT times to rescue repetitive overlaps [%d]\n", asm_opt->max_kmer_cnt);
-    fprintf(stderr, "    --hg-size    INT(k, m or g)\n");
-    fprintf(stderr, "                 estimated haploid genome size used for inferring read coverage [auto]\n");
+    fprintf(stderr, "    -k INT      k-mer length (must be <64) [%d]\n", asm_opt->k_mer_length);
+	fprintf(stderr, "    -w INT      minimizer window size [%d]\n", asm_opt->mz_win);
+	fprintf(stderr, "    -f INT      number of bits for bloom filter; 0 to disable [%d]\n", asm_opt->bf_shift);
+	fprintf(stderr, "    -D FLOAT    drop k-mers occurring >FLOAT*coverage times [%.1f]\n", asm_opt->high_factor);
+	fprintf(stderr, "    -N INT      consider up to max(-D*coverage,-N) overlaps for each oriented read [%d]\n", asm_opt->max_n_chain);
+    fprintf(stderr, "    -r INT      round of correction [%d]\n", asm_opt->number_of_round);
     fprintf(stderr, "  Assembly:\n");
-    fprintf(stderr, "    -a INT       round of assembly cleaning [%d]\n", asm_opt->clean_round);
-    fprintf(stderr, "    -m INT       pop bubbles of <INT in size in contig graphs [%lld]\n", asm_opt->large_pop_bubble_size);
-    fprintf(stderr, "    -p INT       pop bubbles of <INT in size in unitig graphs [%lld]\n", asm_opt->small_pop_bubble_size);
-    fprintf(stderr, "    -n INT       remove tip unitigs composed of <=INT reads [%d]\n", asm_opt->max_short_tip);
-    fprintf(stderr, "    -x FLOAT     max overlap drop ratio [%.2g]\n", asm_opt->max_drop_rate);
-    fprintf(stderr, "    -y FLOAT     min overlap drop ratio [%.2g]\n", asm_opt->min_drop_rate);
-    fprintf(stderr, "    -i           ignore saved read correction and overlaps\n");
-    fprintf(stderr, "    -u           disable post-join step for contigs which may improve N50\n");
-    fprintf(stderr, "    --hom-cov    INT\n");
-    fprintf(stderr, "                 homozygous read coverage [auto]\n");
-    fprintf(stderr, "    --lowQ       INT\n");
-    fprintf(stderr, "                 output contig regions with >=INT%% inconsistency in BED format; 0 to disable [%d]\n", asm_opt->bed_inconsist_rate);
-    fprintf(stderr, "    --b-cov      INT\n");
-    fprintf(stderr, "                 break contigs at positions with <INT-fold coverage; work with '--m-rate'; 0 to disable [%d]\n", asm_opt->b_low_cov);
-    fprintf(stderr, "    --h-cov      INT\n");
-    fprintf(stderr, "                 break contigs at positions with >INT-fold coverage; work with '--m-rate'; -1 to disable [%d]\n", asm_opt->b_high_cov);
-    fprintf(stderr, "    --m-rate     FLOAT\n");
-    fprintf(stderr, "                 break contigs at positions with <=FLOAT*coverage exact overlaps;\n");
-    fprintf(stderr, "                 only work with '--b-cov' or '--h-cov'[%.2f]\n", asm_opt->m_rate);
-    fprintf(stderr, "    --primary    output a primary assembly and an alternate assembly\n");
+    fprintf(stderr, "    -a INT      round of assembly cleaning [%d]\n", asm_opt->clean_round);
+    fprintf(stderr, "    -m INT      pop bubbles of <INT in size in contig graphs [%lld]\n", asm_opt->large_pop_bubble_size);
+    fprintf(stderr, "    -p INT      pop bubbles of <INT in size in unitig graphs [%lld]\n", asm_opt->small_pop_bubble_size);
+    fprintf(stderr, "    -n INT      remove tip unitigs composed of <=INT reads [%d]\n", asm_opt->max_short_tip);
+    fprintf(stderr, "    -x FLOAT    max overlap drop ratio [%.2g]\n", asm_opt->max_drop_rate);
+    fprintf(stderr, "    -y FLOAT    min overlap drop ratio [%.2g]\n", asm_opt->min_drop_rate);
+    fprintf(stderr, "    -u          disable post join contigs step which may improve N50\n");
+    fprintf(stderr, "    --lowQ      INT\n");
+    fprintf(stderr, "                output contig regions with >=INT%% inconsistency in BED format; 0 to disable [%d]\n", asm_opt->bed_inconsist_rate);
+    fprintf(stderr, "    --b-cov     INT\n");
+    fprintf(stderr, "                break contigs at breakpoints with coverage drop at <INT-fold coverage [%d]\n", asm_opt->break_cov);
     
 //	fprintf(stderr, "    --pri-range INT1[,INT2]\n");
 //	fprintf(stderr, "                keep contigs with coverage in this range in p_ctg.gfa; -1 to disable [auto,inf]\n");
 
     fprintf(stderr, "  Trio-partition:\n");
-    fprintf(stderr, "    -1 FILE      hap1/paternal k-mer dump generated by \"yak count\" []\n");
-    fprintf(stderr, "    -2 FILE      hap2/maternal k-mer dump generated by \"yak count\" []\n");
-    fprintf(stderr, "    -3 FILE      list of hap1/paternal read names []\n");
-	fprintf(stderr, "    -4 FILE      list of hap2/maternal read names []\n");
-    fprintf(stderr, "    -c INT       lower bound of the binned k-mer's frequency [%d]\n", asm_opt->min_cnt);
-    fprintf(stderr, "    -d INT       upper bound of the binned k-mer's frequency [%d]\n", asm_opt->mid_cnt);
-    fprintf(stderr, "    --t-occ      INT\n");
-    fprintf(stderr, "                 forcedly remove unitigs with >INT unexpected haplotype-specific reads;\n");
-    fprintf(stderr, "                 ignore graph topology; [%d]\n", asm_opt->trio_flag_occ_thres);
+    fprintf(stderr, "    -1 FILE     hap1/paternal k-mer dump generated by \"yak count\" []\n");
+    fprintf(stderr, "    -2 FILE     hap2/maternal k-mer dump generated by \"yak count\" []\n");
+    fprintf(stderr, "    -c INT      lower bound of the binned k-mer's frequency [%d]\n", asm_opt->min_cnt);
+    fprintf(stderr, "    -d INT      upper bound of the binned k-mer's frequency [%d]\n", asm_opt->mid_cnt);
+	fprintf(stderr, "    -3 FILE     list of hap1/paternal read names []\n");
+	fprintf(stderr, "    -4 FILE     list of hap2/maternal read names []\n");
 
     fprintf(stderr, "  Purge-dups:\n");
-    fprintf(stderr, "    -l INT       purge level. 0: no purging; 1: light; 2/3: aggressive [0 for trio; 3 for unzip]\n");
-    fprintf(stderr, "    -s FLOAT     similarity threshold for duplicate haplotigs [%g for -l1/-l2, %g for -l3]\n", 
-                                      asm_opt->purge_simi_rate_l2, asm_opt->purge_simi_rate_l3);
-    fprintf(stderr, "    -O INT       min number of overlapped reads for duplicate haplotigs [%d]\n", 
-                                      asm_opt->purge_overlap_len);
-    fprintf(stderr, "    --purge-max  INT\n");
-    fprintf(stderr, "                 coverage upper bound of Purge-dups [auto]\n");
-    fprintf(stderr, "    --n-hap      INT\n");
-    fprintf(stderr, "                 number of haplotypes [%d]\n", asm_opt->polyploidy);
+    fprintf(stderr, "    -l INT      purge level. 0: no purging; 1: light; 2: aggressive [0 for trio; 2 for unzip]\n");
+    fprintf(stderr, "    -s FLOAT    similarity threshold for duplicate haplotigs [%g]\n", 
+                                     asm_opt->purge_simi_rate);
+    fprintf(stderr, "    -O INT      min number of overlapped reads for duplicate haplotigs [%d]\n", 
+                                     asm_opt->purge_overlap_len);
+    fprintf(stderr, "    --purge-cov INT\n");
+    fprintf(stderr, "                coverage upper bound of Purge-dups [auto]\n");
+    fprintf(stderr, "    --high-het  enable this mode for high heterozygosity sample [experimental, not stable]\n");
     
-    // fprintf(stderr, "  Hi-C-partition [experimental, not stable]:\n");
-    fprintf(stderr, "  Hi-C-partition:\n");
+    fprintf(stderr, "  Hi-C-partition [experimental, not stable]:\n");
     fprintf(stderr, "    --h1 FILEs   file names of Hi-C R1  [r1_1.fq,r1_2.fq,...]\n");
     fprintf(stderr, "    --h2 FILEs   file names of Hi-C R2  [r2_1.fq,r2_2.fq,...]\n");
-    fprintf(stderr, "    --seed INT   RNG seed [%lu]\n", asm_opt->seed);
-
-    
-    fprintf(stderr, "    --n-weight   INT\n");
-    fprintf(stderr, "                 rounds of reweighting Hi-C links [%d]\n", asm_opt->n_weight);
-	fprintf(stderr, "    --n-perturb  INT\n");
-    fprintf(stderr, "                 rounds of perturbation [%d]\n", asm_opt->n_perturb);
-    fprintf(stderr, "    --f-perturb  FLOAT\n");
-	fprintf(stderr, "                 fraction to flip for perturbation [%.3g]\n", asm_opt->f_perturb);
-    fprintf(stderr, "    --l-msjoin   INT\n");
-    fprintf(stderr, "                 detect misjoined unitigs of >=INT in size; 0 to disable [%lu]\n", asm_opt->misjoin_len);
 
     fprintf(stderr, "Example: ./hifiasm -o NA12878.asm -t 32 NA12878.fq.gz\n");
-    fprintf(stderr, "See `https://hifiasm.readthedocs.io/en/latest/' or `man ./hifiasm.1' for complete documentation.\n");
+    fprintf(stderr, "See `man ./hifiasm.1' for detailed description of these command-line options.\n");
 }
 
 void init_opt(hifiasm_opt_t* asm_opt)
 {
 	memset(asm_opt, 0, sizeof(hifiasm_opt_t));
-	///asm_opt->flag = 0;
-    asm_opt->flag = HA_F_PARTITION;
+	asm_opt->flag = 0;
     asm_opt->coverage = -1;
     asm_opt->num_reads = 0;
     asm_opt->read_file_names = NULL;
@@ -159,10 +115,7 @@ void init_opt(hifiasm_opt_t* asm_opt)
     asm_opt->k_mer_length = 51;
     asm_opt->hic_mer_length = 31;
 	asm_opt->mz_win = 51;
-    asm_opt->mz_rewin = 1000;
-	asm_opt->mz_sample_dist = 500;
 	asm_opt->bf_shift = 37;
-	asm_opt->max_kmer_cnt = 2000;
 	asm_opt->high_factor = 5.0;
 	asm_opt->max_ov_diff_ec = 0.04;
 	asm_opt->max_ov_diff_final = 0.03;
@@ -175,8 +128,7 @@ void init_opt(hifiasm_opt_t* asm_opt)
     asm_opt->number_of_round = 3;
     asm_opt->adapterLen = 0;
     asm_opt->clean_round = 4;
-    ///asm_opt->small_pop_bubble_size = 100000;
-    asm_opt->small_pop_bubble_size = 0;
+    asm_opt->small_pop_bubble_size = 100000;
     asm_opt->large_pop_bubble_size = 10000000;
     asm_opt->min_drop_rate = 0.2;
     asm_opt->max_drop_rate = 0.8;
@@ -188,37 +140,20 @@ void init_opt(hifiasm_opt_t* asm_opt)
     asm_opt->max_short_tip = 3;
     asm_opt->min_cnt = 2;
     asm_opt->mid_cnt = 5;
-    asm_opt->purge_level_primary = 3;
+    asm_opt->purge_level_primary = 2;
     asm_opt->purge_level_trio = 0;
-    asm_opt->purge_simi_rate_l2 = 0.75;
-    asm_opt->purge_simi_rate_l3 = 0.55;
+    asm_opt->purge_simi_rate = 0.75;
+    asm_opt->purge_simi_rate_hic = 0.85;
     asm_opt->purge_overlap_len = 1;
-    ///asm_opt->purge_overlap_len_hic = 50;
+    asm_opt->purge_overlap_len_hic = 50;
     asm_opt->recover_atg_cov_min = -1024;
     asm_opt->recover_atg_cov_max = INT_MAX;
     asm_opt->hom_global_coverage = -1;
-    asm_opt->hom_global_coverage_set = 0;
-    asm_opt->pur_global_coverage = -1;
     asm_opt->bed_inconsist_rate = 70;
     asm_opt->hic_inconsist_rate = 30;
     ///asm_opt->bub_mer_length = 3;
     asm_opt->bub_mer_length = 1000000;
-    asm_opt->b_low_cov = 0;
-    asm_opt->b_high_cov = -1;
-    asm_opt->m_rate = 0.75;
-    asm_opt->hap_occ = 1;
-    asm_opt->polyploidy = 2;
-    asm_opt->trio_flag_occ_thres = 60;
-    asm_opt->seed = 11;
-    asm_opt->n_perturb = 10000;
-    asm_opt->f_perturb = 0.1;
-    asm_opt->n_weight = 3;
-    asm_opt->is_alt = 0;
-    asm_opt->misjoin_len = 500000;
-    asm_opt->scffold = 0;
-    asm_opt->dp_min_len = 2000;
-    asm_opt->dp_e = 0.0025;
-    asm_opt->hg_size = -1;
+    asm_opt->break_cov = 0;
 }
 
 void destory_enzyme(enzyme* f)
@@ -415,9 +350,9 @@ int check_option(hifiasm_opt_t* asm_opt)
         return 0;
     }
 
-    if(asm_opt->purge_level_primary < 0 || asm_opt->purge_level_primary > 3)
+    if(asm_opt->purge_level_primary < 0 || asm_opt->purge_level_primary > 2)
     {
-        fprintf(stderr, "[ERROR] the level of purge-dup should be [0, 3] (-l)\n");
+        fprintf(stderr, "[ERROR] the level of purge-dup should be [0, 2] (-l)\n");
         return 0;
     }
 
@@ -429,13 +364,7 @@ int check_option(hifiasm_opt_t* asm_opt)
 
     if(asm_opt->hom_global_coverage < 0 && asm_opt->hom_global_coverage != -1)
     {
-        fprintf(stderr, "[ERROR] homozygous read coverage should be >= 0 (--hom-cov)\n");
-        return 0;
-    }
-
-    if(asm_opt->pur_global_coverage < 0 && asm_opt->pur_global_coverage != -1)
-    {
-        fprintf(stderr, "[ERROR] purge duplication coverage threshold should be >= 0 (--purge-max)\n");
+        fprintf(stderr, "[ERROR] purge duplication coverage threshold should be >= 0 (--purge-cov)\n");
         return 0;
     }
 
@@ -490,47 +419,25 @@ int check_option(hifiasm_opt_t* asm_opt)
         return 0;
     }
 
-    if(asm_opt->b_low_cov < 0)
-    {
-        fprintf(stderr, "[ERROR] must >= 0 (--b-cov)\n");
-        return 0;
-    }
-
-    if(asm_opt->b_high_cov != -1 && asm_opt->b_high_cov < 0)
-    {
-        fprintf(stderr, "[ERROR] must >= 0 (--h-cov)\n");
-        return 0;
-    }
-
-    if(asm_opt->m_rate < 0)
-    {
-        fprintf(stderr, "[ERROR] must >= 0 (--m-rate)\n");
-        return 0;
-    }
-
-    if(asm_opt->b_high_cov != -1 && asm_opt->b_high_cov <= asm_opt->b_low_cov)
-    {
-        fprintf(stderr, "[ERROR] [--h-cov] must >= [--b-cov]\n");
-        return 0;
-    }
-
-    if(asm_opt->purge_simi_thres < 0)
-    {
-        fprintf(stderr, "[ERROR] [-s] must >= 0\n");
-        return 0;
-    }
-
-    if(asm_opt->max_kmer_cnt < 0)
-    {
-        fprintf(stderr, "[ERROR] [--max-kocc] must >= 0\n");
-        return 0;
-    }
-
-    if(asm_opt->hg_size < -1)
-    {
-        fprintf(stderr, "[ERROR] [--hg-size] wrong genome size\n");
-        return 0;
-    }
+    // fprintf(stderr, "input file num: %d\n", asm_opt->num_reads);
+    // fprintf(stderr, "output file: %s\n", asm_opt->output_file_name);
+    // fprintf(stderr, "number of threads: %d\n", asm_opt->thread_num);
+    // fprintf(stderr, "number of rounds for correction: %d\n", asm_opt->number_of_round);
+    // fprintf(stderr, "number of rounds for assembly cleaning: %d\n", asm_opt->clean_round);
+    // fprintf(stderr, "length of removed adapters: %d\n", asm_opt->adapterLen);
+    // fprintf(stderr, "length of k_mer: %d\n", asm_opt->k_mer_length);
+    // fprintf(stderr, "min overlap drop ratio: %.2g\n", asm_opt->min_drop_rate);
+    // fprintf(stderr, "max overlap drop ratio: %.2g\n", asm_opt->max_drop_rate);
+    // fprintf(stderr, "size of popped small bubbles: %lld\n", asm_opt->small_pop_bubble_size);
+    // fprintf(stderr, "size of popped large bubbles: %lld\n", asm_opt->large_pop_bubble_size);
+    // fprintf(stderr, "small removed unitig threshold: %d\n", asm_opt->max_short_tip);
+    // fprintf(stderr, "small removed unitig threshold: %d\n", asm_opt->max_short_tip);
+    // fprintf(stderr, "min_cnt: %d\n", asm_opt->min_cnt);
+    // fprintf(stderr, "mid_cnt: %d\n", asm_opt->mid_cnt);
+    // fprintf(stderr, "purge_level_primary: %d\n", asm_opt->purge_level_primary);
+    // fprintf(stderr, "purge_level_trio: %d\n", asm_opt->purge_level_trio);
+    // fprintf(stderr, "purge_simi_rate: %f\n", asm_opt->purge_simi_rate);
+    // fprintf(stderr, "purge_overlap_len: %d\n", asm_opt->purge_overlap_len);
 
     return 1;
 }
@@ -617,23 +524,6 @@ void get_hic_enzymes(char *argv, enzyme** x, int check_name)
     (*x)->a[k][(*x)->l[k]] = '\0';
 }
 
-int64_t inter_gsize(char *argv)
-{
-    int64_t len = strlen(argv);
-    double s;
-    if(len <= 1) return -2;
-    char t = argv[len-1];
-    if(t != 'k' && t != 'K' && t != 'm' && t != 'M' && t != 'g' && t != 'G') return -2;
-    char *ss=(char*)malloc(len);
-    memcpy(ss, argv, len-1); ss[len-1] = '\0';
-    s = atof(ss);
-    free(ss);
-    if(t == 'k' || t == 'K') return s*1000;
-    if(t == 'm' || t == 'M') return s*1000000;
-    if(t == 'g' || t == 'G') return s*1000000000;
-    return s;
-}
-
 int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 {
     ketopt_t opt = KETOPT_INIT;
@@ -685,11 +575,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 		else if (c == 306) asm_opt->max_ov_diff_final = atof(opt.arg);
 		else if (c == 307) asm_opt->extract_list = opt.arg;
 		else if (c == 308) asm_opt->extract_iter = atoi(opt.arg);
-        else if (c == 309)
-        {
-            asm_opt->hom_global_coverage = atoi(opt.arg);
-            asm_opt->hom_global_coverage_set = 1;
-        } 
+        else if (c == 309) asm_opt->hom_global_coverage = atoi(opt.arg);
         else if (c == 310)
         {
             char* s = NULL;
@@ -700,33 +586,18 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
                 asm_opt->recover_atg_cov_min = asm_opt->recover_atg_cov_max = -1;
             }
         }
-        ///else if (c == 311) asm_opt->flag |= HA_F_HIGH_HET;
+        else if (c == 311) asm_opt->flag |= HA_F_HIGH_HET;
         else if (c == 312) asm_opt->bed_inconsist_rate = atoi(opt.arg);
         else if (c == 313) asm_opt->min_hist_kmer_cnt = atoi(opt.arg);
         else if (c == 314) get_hic_enzymes(opt.arg, &(asm_opt->hic_reads[0]), 0);
         else if (c == 315) get_hic_enzymes(opt.arg, &(asm_opt->hic_reads[1]), 0);
         else if (c == 316) get_hic_enzymes(opt.arg, &(asm_opt->hic_enzymes), 1);
-        else if (c == 317) asm_opt->b_low_cov = atoi(opt.arg);
-        else if (c == 318) asm_opt->b_high_cov = atoi(opt.arg);
-        else if (c == 319) asm_opt->m_rate = atof(opt.arg);
-        else if (c == 320) asm_opt->flag -= HA_F_PARTITION, asm_opt->is_alt = 1;
-        else if (c == 321) asm_opt->trio_flag_occ_thres = atoi(opt.arg);
-        else if (c == 322) asm_opt->seed = atol(opt.arg);
-        else if (c == 323) asm_opt->n_perturb = atoi(opt.arg);
-        else if (c == 324) asm_opt->f_perturb = atof(opt.arg);
-        else if (c == 325) asm_opt->polyploidy = atoi(opt.arg);
-        else if (c == 326) asm_opt->n_weight = atoi(opt.arg);
-        else if (c == 327) asm_opt->misjoin_len = atol(opt.arg);
-        else if (c == 328) asm_opt->pur_global_coverage = atoi(opt.arg);
-        else if (c == 329) asm_opt->flag |= HA_F_FAST;
-        else if (c == 330) asm_opt->dp_e = atof(opt.arg);
-        else if (c == 331) asm_opt->max_kmer_cnt = atol(opt.arg);
-        else if (c == 332) asm_opt->hg_size = inter_gsize(opt.arg);      
+        else if (c == 317) asm_opt->break_cov = atoi(opt.arg);
         else if (c == 'l')
         {   ///0: disable purge_dup; 1: purge containment; 2: purge overlap
             asm_opt->purge_level_primary = asm_opt->purge_level_trio = atoi(opt.arg);
         }
-        else if (c == 's') asm_opt->purge_simi_rate_l2 = asm_opt->purge_simi_rate_l3 = atof(opt.arg);
+        else if (c == 's') asm_opt->purge_simi_rate = atof(opt.arg);
         else if (c == 'O') asm_opt->purge_overlap_len = atoll(opt.arg);
         else if (c == ':') 
         {
@@ -740,9 +611,6 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 		}
     }
 
-    if(asm_opt->purge_level_primary > 2) asm_opt->purge_simi_thres = asm_opt->purge_simi_rate_l3;
-    else asm_opt->purge_simi_thres = asm_opt->purge_simi_rate_l2;
-
     
     if (argc == opt.ind)
     {
@@ -752,5 +620,7 @@ int CommandLine_process(int argc, char *argv[], hifiasm_opt_t* asm_opt)
 
     get_queries(argc, argv, &opt, asm_opt);
 
+
+
     return check_option(asm_opt);
 }

CommandLines.h

@@ -2,9 +2,8 @@
 #define __COMMAND_LINE_PARSER__
 
 #include <pthread.h>
-#include <stdint.h>
 
-#define HA_VERSION "0.16.0-r369"
+#define HA_VERSION "0.14-r309"
 
 #define VERBOSE 0
 
@@ -19,8 +18,6 @@
 #define HA_F_BAN_POST_JOIN   0x100
 #define HA_F_BAN_ASSEMBLY    0x200
 #define HA_F_HIGH_HET        0x400
-#define HA_F_PARTITION       0x800
-#define HA_F_FAST            0x1000
 
 #define HA_MIN_OV_DIFF       0.02 // min sequence divergence in an overlap
 
@@ -46,18 +43,13 @@ typedef struct {
     int hic_mer_length;
     int bub_mer_length;
 	int mz_win;
-    int mz_rewin;
-	int mz_sample_dist;
 	int bf_shift;
-	int max_kmer_cnt;
 	double high_factor; // coverage cutoff set to high_factor*hom_cov
 	double max_ov_diff_ec;
 	double max_ov_diff_final;
 	int hom_cov;
     int het_cov;
-    int b_low_cov;
-    int b_high_cov;
-    double m_rate;
+    int break_cov;
 	int max_n_chain; // fall-back max number of chains to consider
 	int min_hist_kmer_cnt;
     int load_index_from_disk;
@@ -76,23 +68,18 @@ typedef struct {
     int purge_level_primary;
     int purge_level_trio;
     int purge_overlap_len;
-    ///int purge_overlap_len_hic;
+    int purge_overlap_len_hic;
     int recover_atg_cov_min;
     int recover_atg_cov_max;
     int hom_global_coverage;
-    int hom_global_coverage_set;
-    int pur_global_coverage;
     int bed_inconsist_rate;
     int hic_inconsist_rate;
 
     float max_hang_rate;
     float min_drop_rate;
     float max_drop_rate;
-    float purge_simi_rate_l2;
-    float purge_simi_rate_l3;
-    float purge_simi_thres;
-
-    ///float purge_simi_rate_hic;
+    float purge_simi_rate;
+    float purge_simi_rate_hic;
 
     long long small_pop_bubble_size;
     long long large_pop_bubble_size;
@@ -101,19 +88,7 @@ typedef struct {
     long long num_recorrected_bases;
 	long long mem_buf;
     long long coverage;
-    int hap_occ;
-    int polyploidy;
-    int trio_flag_occ_thres;
-    uint64_t seed;
-    int32_t n_perturb;
-    double f_perturb;
-    int32_t n_weight;
-    uint32_t is_alt;
-    uint64_t misjoin_len;
-    uint64_t scffold;
-    int32_t dp_min_len;
-    float dp_e;
-    int64_t hg_size;
+    
 } hifiasm_opt_t;
 
 extern hifiasm_opt_t asm_opt;

Hash_Table.cpp

@@ -9,8 +9,6 @@ pthread_mutex_t output_mutex;
 #define overlap_region_key(a) ((a).y_id)
 KRADIX_SORT_INIT(overlap_region_sort, overlap_region, overlap_region_key, member_size(overlap_region, y_id))
 
-#define normal_w(x, y) ((x)>=(y)?(x)/(y):1)
-
 void overlap_region_sort_y_id(overlap_region *a, long long n)
 {
 	radix_sort_overlap_region_sort(a, a + n);
@@ -195,168 +193,6 @@ int append_inexact_overlap_region_alloc(overlap_region_alloc* list, overlap_regi
 
 
 
-        resize_fake_cigar(&(list->list[list->length].f_cigar), (tmp->f_cigar.length + 2));
-        if(add_beg_end == 1)
-        {
-            add_fake_cigar(&(list->list[list->length].f_cigar), list->list[list->length].x_pos_s, 0);
-        }
-        
-        long long distance_self_pos = tmp->x_pos_e - tmp->x_pos_s;
-        long long distance_pos = tmp->y_pos_e - tmp->y_pos_s;
-        long long init_distance_gap = distance_pos - distance_self_pos;
-        /****************************may have bugs********************************/
-        ///long long pre_distance_gap = init_distance_gap;
-        long long pre_distance_gap = 0xfffffffffffffff;
-        /****************************may have bugs********************************/
-        long long distance_gap;
-        long long i = 0;
-        for (i = tmp->f_cigar.length - 1; i >= 0; i--)
-        {
-            distance_gap = get_fake_gap_shift(&(tmp->f_cigar), i);
-            if(distance_gap != pre_distance_gap)
-            {
-                pre_distance_gap = distance_gap;
-
-                add_fake_cigar(&(list->list[list->length].f_cigar), 
-                get_fake_gap_pos(&(tmp->f_cigar), i), init_distance_gap - pre_distance_gap);
-            }
-        }
-
-        if(add_beg_end == 1 && get_fake_gap_pos(&(list->list[list->length].f_cigar), 
-        list->list[list->length].f_cigar.length - 1) != (long long)list->list[list->length].x_pos_e)
-        {
-            add_fake_cigar(&(list->list[list->length].f_cigar), 
-            list->list[list->length].x_pos_e, 
-            get_fake_gap_shift(&(list->list[list->length].f_cigar), 
-            list->list[list->length].f_cigar.length - 1));
-        } 
-    }
-
-    list->list[list->length].shared_seed = tmp->shared_seed;
-    list->list[list->length].align_length = 0;
-    list->list[list->length].is_match = 0;
-    list->list[list->length].non_homopolymer_errors = 0;
-    list->list[list->length].strong = 0;
-
-    list->length++;
-
-    return 1;
-}
-
-
-int append_utg_inexact_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp, 
-                                        ma_utg_v *ua, int add_beg_end)
-{
-   
-    if (list->length + 1 > list->size)
-    {
-        list->size = list->size * 2;
-        list->list = (overlap_region*)realloc(list->list, sizeof(overlap_region)*list->size);
-        /// need to set new space to be 0
-        memset(list->list + (list->size/2), 0, sizeof(overlap_region)*(list->size/2));
-    }
-
-    if (list->length!=0 && list->list[list->length - 1].y_id==tmp->y_id)
-    {    
-        ///if(list->list[list->length - 1].shared_seed >= tmp->shared_seed)
-        if((list->list[list->length - 1].shared_seed > tmp->shared_seed)
-           ||
-           ((list->list[list->length - 1].shared_seed == tmp->shared_seed) && 
-           (list->list[list->length - 1].overlapLen <= tmp->overlapLen)))
-        {
-            return 0;
-        }
-        else
-        {
-            list->length--;
-        }
-    }
-
-    if(tmp->x_pos_s <= tmp->y_pos_s)
-    {
-        tmp->y_pos_s = tmp->y_pos_s - tmp->x_pos_s;
-        tmp->x_pos_s = 0;
-    }
-    else
-    {
-        tmp->x_pos_s = tmp->x_pos_s - tmp->y_pos_s;
-        tmp->y_pos_s = 0;
-    }
-
-
-    long long x_right_length = ua->a[tmp->x_id].len - tmp->x_pos_e - 1;
-    long long y_right_length = ua->a[tmp->y_id].len - tmp->y_pos_e - 1;
-
-    if(x_right_length <= y_right_length)
-    {
-        tmp->x_pos_e = ua->a[tmp->x_id].len - 1;
-        tmp->y_pos_e = tmp->y_pos_e + x_right_length;        
-    }
-    else
-    {
-        tmp->x_pos_e = tmp->x_pos_e + y_right_length;
-        tmp->y_pos_e = ua->a[tmp->y_id].len - 1;
-    }
-    
-    if (tmp->x_pos_strand == 1)
-    {
-        list->list[list->length].x_id = tmp->x_id;
-        list->list[list->length].x_pos_e = ua->a[tmp->x_id].len - tmp->x_pos_s - 1;
-        list->list[list->length].x_pos_s = ua->a[tmp->x_id].len - tmp->x_pos_e - 1;
-        list->list[list->length].x_pos_strand = 0;
-
-        list->list[list->length].y_id = tmp->y_id;
-        list->list[list->length].y_pos_e = ua->a[tmp->y_id].len - tmp->y_pos_s - 1;
-        list->list[list->length].y_pos_s = ua->a[tmp->y_id].len - tmp->y_pos_e - 1;
-        list->list[list->length].y_pos_strand = 1;
-
-        resize_fake_cigar(&(list->list[list->length].f_cigar), (tmp->f_cigar.length + 2));
-        if(add_beg_end == 1)
-        {
-            add_fake_cigar(&(list->list[list->length].f_cigar), list->list[list->length].x_pos_s, 0);
-        }
-        
-        long long distance_gap;
-        /****************************may have bugs********************************/
-        ///long long pre_distance_gap = 0;
-        long long pre_distance_gap = 0xfffffffffffffff;
-        /****************************may have bugs********************************/
-        long long i = 0;
-        for (i = 0; i < (long long)tmp->f_cigar.length; i++)
-        {
-            distance_gap = get_fake_gap_shift(&(tmp->f_cigar), i);
-            if(distance_gap != pre_distance_gap)
-            {
-                pre_distance_gap = distance_gap;
-                add_fake_cigar(&(list->list[list->length].f_cigar), 
-                ua->a[tmp->x_id].len - get_fake_gap_pos(&(tmp->f_cigar), i) - 1, 
-                pre_distance_gap);
-            }
-        }
-
-        if(add_beg_end == 1 && get_fake_gap_pos(&(list->list[list->length].f_cigar), 
-            list->list[list->length].f_cigar.length - 1) != (long long)list->list[list->length].x_pos_e)
-        {
-            add_fake_cigar(&(list->list[list->length].f_cigar), 
-            list->list[list->length].x_pos_e, 
-            get_fake_gap_shift(&(list->list[list->length].f_cigar), 
-            list->list[list->length].f_cigar.length - 1));
-        }
-    }
-    else
-    {
-        list->list[list->length].x_id = tmp->x_id;
-        list->list[list->length].x_pos_e = tmp->x_pos_e;
-        list->list[list->length].x_pos_s = tmp->x_pos_s;
-        list->list[list->length].x_pos_strand = tmp->x_pos_strand;
-
-        list->list[list->length].y_id = tmp->y_id;
-        list->list[list->length].y_pos_e = tmp->y_pos_e;
-        list->list[list->length].y_pos_s = tmp->y_pos_s;
-        list->list[list->length].y_pos_strand = tmp->y_pos_strand;
-
-
-
         resize_fake_cigar(&(list->list[list->length].f_cigar), (tmp->f_cigar.length + 2));
         if(add_beg_end == 1)
         {
@@ -469,46 +305,7 @@ void debug_chain(k_mer_hit* a, long long a_n, Chain_Data* dp)
     }
 }
 
-void print_chain(k_mer_hit* a, long long a_n, Chain_Data* dp, long long topN)
-{
-    fprintf(stderr, "topN: %lld\n", topN);
-    long long max_score = -1, max_i = -1, max_n = 0;;
-    long long ss, i, j, current_j;
-    kvec_t(long long) si; kv_init(si);
-    for (ss = 0; ss < topN && ss < a_n; ss++){
-        for (i = 0, max_i = -1, max_score = -1; i < a_n; ++i) {
-            for (j = 0; j < (long long)si.n; j++){
-                if(i == si.a[j]) break;
-            }
-            if(j < (long long)si.n) continue;
-            if(dp->score[i] > max_score) max_score = dp->score[i], max_i = i;
-        }
-        if(max_i < 0) continue;
-        j = max_i; max_n = 0;
-        while (j >= 0)
-        {
-            current_j = j;
-            if(current_j == -1) continue;
-            j = dp->pre[j];
-            max_n++;
-        }
 
-        fprintf(stderr, "\nmax_i: %lld, max_score: %lld, max_n: %lld\n", max_i, max_score, max_n);
-        
-        j = max_i; 
-        while (j >= 0)
-        {
-            current_j = j;
-            if(current_j == -1) continue;
-
-            kv_push(long long, si, current_j);
-            j = dp->pre[j];
-            fprintf(stderr, "self_offset: %u, offset: %u, cnt: %u, score: %d\n", 
-            a[current_j].self_offset, a[current_j].offset, a[current_j].cnt, dp->score[current_j]);
-        }
-    }
-    kv_destroy(si);
-}
 
 
 long long get_chainLen(long long x_beg, long long x_end, long long xLen, 
@@ -597,8 +394,7 @@ int32_t ha_chain_check(k_mer_hit *a, int32_t n_a, Chain_Data *dp, int32_t min_sc
 			break;
 	if (i < n_a) return -1;
 	bw_pen = 1.0 / bw_thres;
-	// dp->score[0] = a[0].good? min_sc : min_sc>>1;
-    dp->score[0] = normal_w(min_sc, (int64_t)a[0].cnt);
+	dp->score[0] = a[0].good? min_sc : min_sc>>1;
 	dp->pre[0] = -1, dp->indels[0] = 0, dp->self_length[0] = 0, dp->occ[0] = 1;
 	for (i = 1; i < n_a; ++i) {
 		int32_t score, dg;
@@ -612,7 +408,7 @@ int32_t ha_chain_check(k_mer_hit *a, int32_t n_a, Chain_Data *dp, int32_t min_sc
 		dg = dx < dy? dx : dy;
 		if (dd > THRESHOLD_MAX_SIZE && dd > dg * bw_thres) break;
 		score = dg < min_sc? dg : min_sc;
-        score = normal_w(score, (int64_t)a[i].cnt);
+		if (!a[i].good) score >>= 1;
 		gap_rate = (double)tot_indel / tot_len;
 		score -= (int)(gap_rate * score * bw_pen);
 		dp->score[i] = dp->score[i-1] + score;
@@ -626,7 +422,7 @@ int32_t ha_chain_check(k_mer_hit *a, int32_t n_a, Chain_Data *dp, int32_t min_sc
 }
 
 ///double band_width_threshold = 0.05;
-long long chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, 
+void chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, 
               double band_width_threshold, int max_skip, int x_readLen, int y_readLen)
 {
     long long i, j;
@@ -643,6 +439,7 @@ long long chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region*
     resize_Chain_Data(dp, a_n);
 
 	ret = ha_chain_check(a, a_n, dp, min_score, band_width_threshold);
+
 	if (ret > 0) {
 		a_n = ret;
 		goto skip_dp;
@@ -658,7 +455,7 @@ long long chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region*
         pos = a[i].offset;
         self_pos = a[i].self_offset;
         max_j = -1;
-        max_score = normal_w(min_score, (int64_t)a[i].cnt);
+        max_score = a[i].good? min_score : min_score>>1;
         max_indels = 0;
         max_self_length = 0;
 
@@ -689,7 +486,7 @@ long long chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region*
             score = distance_min < min_score? distance_min : min_score;
             ///need to be fixed in r305
 			///if (!a[j].good) score = (score >> 1) + (score & 1);
-            score = normal_w(score, (int64_t)a[j].cnt);
+            if (!a[j].good) score >>= 1;
 
             gap_rate = (double)((double)(total_indels)/(double)(total_self_length));
             ///if the gap rate > 0.06, score will be negative
@@ -725,7 +522,6 @@ long long chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region*
     }
 
     ///debug_chain(a, a_n, dp);
-    // if((*result).x_id == 2162668 && (*result).y_id == 182804) print_chain(a, a_n, dp, 10);
     
 skip_dp:
 
@@ -817,9 +613,72 @@ skip_dp:
             i = dp->pre[i];
         }
     }
-    return chainLen;
 }
 
+void calculate_overlap_region_by_chaining_back(Candidates_list* candidates, overlap_region_alloc* overlap_list, 
+                                          uint64_t readID, uint64_t readLength, All_reads* R_INF, double band_width_threshold, int add_beg_end)
+{
+    overlap_region tmp_region;
+    long long i = 0;
+    uint64_t current_ID;
+    uint64_t current_stand;
+
+    if (candidates->length == 0)
+    {
+        return;
+    }
+
+    long long sub_region_beg;
+    long long sub_region_end;
+
+    init_fake_cigar(&(tmp_region.f_cigar));
+
+    i = 0;
+    while (i < candidates->length)
+    {
+        current_ID = candidates->list[i].readID;
+        current_stand = candidates->list[i].strand;
+
+        ///reference read
+        tmp_region.x_id = readID;
+        tmp_region.x_pos_strand = current_stand;
+        ///query read
+        tmp_region.y_id = current_ID;
+        ///here the strand of query is always 0
+        tmp_region.y_pos_strand = 0;  
+
+        sub_region_beg = i;
+        sub_region_end = i;
+        i++;
+
+        while (i < candidates->length 
+        && 
+        current_ID == candidates->list[i].readID
+        &&
+        current_stand == candidates->list[i].strand)
+        {
+            sub_region_end = i;
+            i++;
+        }
+
+        if (tmp_region.x_id == tmp_region.y_id)
+        {
+            continue;
+        }
+
+		chain_DP(candidates->list + sub_region_beg,
+				sub_region_end - sub_region_beg + 1, &(candidates->chainDP), &tmp_region, band_width_threshold,
+				25, Get_READ_LENGTH((*R_INF), tmp_region.x_id), Get_READ_LENGTH((*R_INF), tmp_region.y_id));
+
+        ///if (tmp_region.x_id != tmp_region.y_id && tmp_region.shared_seed > 1)
+        if (tmp_region.x_id != tmp_region.y_id)
+        {
+            append_inexact_overlap_region_alloc(overlap_list, &tmp_region, R_INF, add_beg_end);
+        }
+    }
+
+    destory_fake_cigar(&(tmp_region.f_cigar));
+}
 
 
 void calculate_overlap_region_by_chaining(Candidates_list* candidates, overlap_region_alloc* overlap_list, kvec_t_u64_warp* chain_idx,

Hash_Table.h

@@ -110,8 +110,8 @@ typedef struct
 
 typedef struct
 {
-	uint32_t readID:31, strand:1;
-	uint32_t offset, self_offset, cnt;
+	uint32_t readID:30, strand:1, good:1;
+	uint32_t offset, self_offset;
 } k_mer_hit;
 
 typedef struct {
@@ -187,7 +187,6 @@ void init_window_list_alloc(window_list_alloc* x);
 void clear_window_list_alloc(window_list_alloc* x);
 void destory_window_list_alloc(window_list_alloc* x);
 void resize_window_list_alloc(window_list_alloc* x, long long size);
-long long chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, double band_width_threshold, int max_skip, int x_readLen, int y_readLen);
-int append_utg_inexact_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp, 
-                                        ma_utg_v *ua, int add_beg_end);
+void chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, double band_width_threshold, int max_skip, int x_readLen, int y_readLen);
+
 #endif

Makefile

@@ -6,8 +6,7 @@ CPPFLAGS=
 INCLUDES=
 OBJS=		CommandLines.o Process_Read.o Assembly.o Hash_Table.o \
 			POA.o Correct.o Levenshtein_distance.o Overlaps.o Trio.o kthread.o Purge_Dups.o \
-			htab.o hist.o sketch.o anchor.o extract.o sys.o ksw2_extz2_sse.o hic.o rcut.o horder.o \
-			tovlp.o
+			htab.o hist.o sketch.o anchor.o extract.o sys.o ksw2_extz2_sse.o hic.o
 EXE=		hifiasm
 LIBS=		-lz -lpthread -lm
 
@@ -44,7 +43,7 @@ Assembly.o: kthread.h
 CommandLines.o: CommandLines.h ketopt.h
 Correct.o: Correct.h Hash_Table.h htab.h Process_Read.h Overlaps.h kvec.h
 Correct.o: kdq.h CommandLines.h Levenshtein_distance.h POA.h Assembly.h
-Correct.o: ksw2.h ksort.h
+Correct.o: ksw2.h
 Hash_Table.o: Hash_Table.h htab.h Process_Read.h Overlaps.h kvec.h kdq.h
 Hash_Table.o: CommandLines.h ksort.h
 Levenshtein_distance.o: Levenshtein_distance.h
@@ -73,6 +72,3 @@ main.o: Levenshtein_distance.h htab.h
 sketch.o: kvec.h htab.h Process_Read.h Overlaps.h kdq.h CommandLines.h
 sys.o: htab.h Process_Read.h Overlaps.h kvec.h kdq.h CommandLines.h
 hic.o: hic.h
-rcut.o: rcut.h
-horder.o: horder.h
-tovlp.o: tovlp.h

Overlaps.h

@@ -4,7 +4,6 @@
 #include <stdint.h>
 #include "kvec.h"
 #include "kdq.h"
-#include "ksort.h"
 
 ///#define MIN_OVERLAP_LEN 2000
 ///#define MIN_OVERLAP_LEN 500
@@ -27,7 +26,6 @@
 #define DOUBLE_CHECK_THRES 0.1
 #define FINAL_DOUBLE_CHECK_THRES 0.2
 #define CHIMERIC_TRIM_THRES 4
-#define GAP_LEN 100
 // #define PRIMARY_LABLE 1
 // #define ALTER_LABLE 2
 // #define HAP_LABLE 4
@@ -54,8 +52,6 @@
 #define CUT_DIF_HAP 12
 
 
-
-
 ///query is the read itself
 typedef struct {
 	uint64_t qns;
@@ -102,36 +98,6 @@ int max_hang, int min_ovlp);
 long long get_specific_overlap(ma_hit_t_alloc* x, uint32_t qn, uint32_t tn);
 
 
-typedef struct {
-	uint32_t qSpre, qEpre, qScur, qEcur, qn;///[qSp, qEp) && [qSn, qEn]
-	uint32_t tSpre, tEpre, tScur, tEcur, tn;
-} u_trans_hit_t;
-
-typedef struct {
-	size_t n, m;
-	u_trans_hit_t* a;
-} kv_u_trans_hit_t;
-
-
-
-typedef struct {
-	uint32_t qs, qe, qn;
-	uint32_t ts, te, tn;
-	uint32_t occ;
-	double nw;
-	uint8_t f:6, rev:1, del:1;
-	///uint8_t qo:4, to:4;
-} u_trans_t;
-
-typedef struct {
-	size_t n, m;
-	u_trans_t* a;
-	kvec_t(uint64_t) idx;
-} kv_u_trans_t;
-
-#define u_trans_a(x, id) ((x).a + ((x).idx.a[(id)]>>32))
-#define u_trans_n(x, id) ((uint32_t)((x).idx.a[(id)]))
-
 typedef struct {
 	uint64_t ul;
 	uint32_t v;
@@ -402,6 +368,7 @@ typedef struct {
 	kvec_t(uint32_t) e; // visited edges/arcs
 } buf_t;
 
+
 typedef struct {
 	kvec_t(uint64_t) Nodes; 
 	kvec_t(uint64_t) Edges; 
@@ -503,7 +470,6 @@ uint64_t* source_index, long long listLen);
 typedef struct {
 	uint64_t len;
 	uint32_t* index;
-	uint8_t* is_het;
 } R_to_U;
 
 void init_R_to_U(R_to_U* x, uint64_t len);
@@ -511,7 +477,8 @@ void destory_R_to_U(R_to_U* x);
 void set_R_to_U(R_to_U* x, uint32_t rID, uint32_t uID, uint32_t is_Unitig, uint8_t* flag);
 void get_R_to_U(R_to_U* x, uint32_t rID, uint32_t* uID, uint32_t* is_Unitig);
 void transfor_R_to_U(R_to_U* x);
-void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, kvec_asg_arc_t_warp* edge, int require_equal_nv, int test_tangle);
+void debug_utg_graph(ma_ug_t *ug, asg_t* read_g, int require_equal_nv, int test_tangle);
+int asg_pop_bubble_primary(asg_t *g, int max_dist);
 long long asg_arc_del_simple_circle_untig(ma_hit_t_alloc* sources, ma_sub_t* coverage_cut, asg_t *g, long long circleLen, int is_drop);
 
 typedef struct {
@@ -526,21 +493,9 @@ typedef struct {
 	uint32_t new_edges_i;
 } Edge_iter;
 
-typedef struct {
-	asg_arc_t x;
-    uint64_t Off;
-    uint64_t weight;
-}asg_arc_t_offset;
-
-typedef struct {
-	kvec_t(asg_arc_t_offset) a;
-	uint64_t i;
-}kvec_asg_arc_t_offset;
-
-
-
 void init_Edge_iter(asg_t* g, uint32_t v, asg_arc_t* new_edges, uint32_t new_edges_n, Edge_iter* x);
 int get_arc_t(Edge_iter* x, asg_arc_t* get);
+int asg_pop_bubble_primary_trio(ma_ug_t *ug, int max_dist, uint32_t positive_flag, uint32_t negative_flag);
 
 
 inline int get_real_length(asg_t *g, uint32_t v, uint32_t* v_s)
@@ -585,6 +540,55 @@ inline uint32_t check_tip(asg_t *sg, uint32_t begNode, uint32_t* endNode, buf_t*
     }
 }
 
+inline uint32_t get_unitig_back(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* endNode, 
+long long* nodeLen, long long* baseLen, buf_t* b)
+{
+	ma_utg_v* u = NULL; 
+    uint32_t v = begNode, w, k;
+    uint32_t kv;
+	(*nodeLen) = (*baseLen) = 0;
+    (*endNode) = (uint32_t)-1;
+	if(ug!=NULL) u = &(ug->u);
+
+    while (1)
+    {
+        kv = get_real_length(sg, v, NULL);
+        (*endNode) = v;
+		if(u == NULL) 
+		{
+			(*nodeLen)++;
+		}
+		else
+		{
+			(*nodeLen) += EvaluateLen((*u), v>>1);
+		}
+        if(b) kv_push(uint32_t, b->b, v);
+		///means reach the end of a unitig
+		if(kv!=1) (*baseLen) += sg->seq[v>>1].len;
+		if(kv==0) return END_TIPS;
+        if(kv>1) return MUL_OUTPUT;
+        ///kv must be 1 here
+        kv = get_real_length(sg, v, &w);
+		///means reach the end of a unitig
+        if(get_real_length(sg, w^1, NULL)!=1)
+		{
+			(*baseLen) += sg->seq[v>>1].len;
+			return MUL_INPUT;
+		}
+
+		for (k = 0; k < asg_arc_n(sg, v); k++)
+        {
+			if(asg_arc_a(sg, v)[k].del) continue;
+			///here is just one undeleted edge
+			(*baseLen) += asg_arc_len(asg_arc_a(sg, v)[k]);
+			break;
+		}
+
+        v = w;
+        if(v == begNode) return LOOP;
+    }
+}
+
 inline uint32_t get_unitig(asg_t *sg, ma_ug_t *ug, uint32_t begNode, uint32_t* endNode, 
 long long* nodeLen, long long* baseLen, long long* max_stop_nodeLen, long long* max_stop_baseLen, 
 uint32_t stops_threshold, buf_t* b)
@@ -710,12 +714,302 @@ uint32_t stops_threshold, buf_t* b)
 #define UNAVAILABLE (uint32_t)-1
 #define PLOID 0
 #define NON_PLOID 1
-// #define DIFF_HAP_RATE 0.75
+#define DIFF_HAP_RATE 0.75
 #define TRIO_DROP_THRES 0.9
 #define TRIO_DROP_LENGTH_THRES 0.8
 #define MAX_STOP_RATE 0.6
 #define TANGLE_MISSED_THRES 0.6
-#define HET_HOM_RATE 0.7
+///if ug == NULL, nsg should be equal to read_sg
+inline uint32_t check_different_haps(asg_t *nsg, ma_ug_t *ug, asg_t *read_sg, 
+uint32_t v_0, uint32_t v_1, ma_hit_t_alloc* reverse_sources, buf_t* b_0, buf_t* b_1, 
+R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold)
+{
+	uint32_t vEnd, qn, tn, j, is_Unitig, uId;
+	long long ELen_0, ELen_1, tmp, max_stop_nodeLen, max_stop_baseLen;
+
+	b_0->b.n = b_1->b.n = 0;
+	if(get_unitig(nsg, ug, v_0, &vEnd, &ELen_0, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
+	stops_threshold, b_0) == LOOP)
+	{
+		return UNAVAILABLE;
+	}
+	if(get_unitig(nsg, ug, v_1, &vEnd, &ELen_1, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
+	stops_threshold, b_1) == LOOP)
+	{
+		return UNAVAILABLE;
+	}
+	if(ELen_0<=min_edge_length || ELen_1<=min_edge_length) return UNAVAILABLE;
+
+	rIdContig b_max, b_min;
+	b_max.b_0 = b_min.b_0 = NULL;
+	b_max.offset = b_max.readI = b_max.untigI = 0;
+	b_min.offset = b_min.readI = b_min.untigI = 0;
+
+	if(ELen_0<=ELen_1)
+	{
+        b_min.b_0 = b_0;
+        b_max.b_0 = b_1;
+    }
+    else
+    {
+        b_min.b_0 = b_1;
+        b_max.b_0 = b_0;
+    }
+
+	uint32_t max_count = 0, min_count = 0;
+	ma_utg_t *node_min = NULL, *node_max = NULL;
+	if(ug != NULL)
+	{
+		/*****************************label all unitigs****************************************/
+		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
+		{
+			node_max = &(ug->u.a[b_max.b_0->b.a[b_max.untigI]>>1]);
+			///each read
+			for (b_max.readI = 0; b_max.readI < node_max->n; b_max.readI++)
+			{
+				qn = (node_max->a[b_max.readI]>>33);
+				set_R_to_U(ruIndex, qn, (b_max.b_0->b.a[b_max.untigI]>>1), 1, &(read_sg->seq[qn].c));
+			}
+		}
+		/*****************************label all unitigs****************************************/
+
+		///each unitig
+		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
+		{
+			
+			node_min = &(ug->u.a[(b_min.b_0->b.a[b_min.untigI]>>1)]);
+
+			///each read
+			for (b_min.readI = 0; b_min.readI < node_min->n; b_min.readI++)
+        	{
+				qn = node_min->a[b_min.readI]>>33;
+
+				/************************BUG: don't forget****************************/
+				if(reverse_sources[qn].length > 0) min_count++;
+				///if(reverse_sources[qn].length >= 0) min_count++;
+				/************************BUG: don't forget****************************/
+				for (j = 0; j < (long long)reverse_sources[qn].length; j++)
+				{
+					tn = Get_tn(reverse_sources[qn].buffer[j]);
+					if(read_sg->seq[tn].del == 1)
+					{
+						get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
+						if(tn == (uint32_t)-1 || is_Unitig == 1 || read_sg->seq[tn].del == 1) continue;
+					}
+
+					get_R_to_U(ruIndex, tn, &uId, &is_Unitig);
+					if(uId!=(uint32_t)-1 && is_Unitig == 1)
+					{
+						max_count++;
+						break;
+					}
+				}
+			}
+		}
+		/*****************************label all unitigs****************************************/
+		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
+		{
+			node_max = &(ug->u.a[b_max.b_0->b.a[b_max.untigI]>>1]);
+			///each read
+			for (b_max.readI = 0; b_max.readI < node_max->n; b_max.readI++)
+			{
+				qn = (node_max->a[b_max.readI]>>33);
+				ruIndex->index[qn] = (uint32_t)-1;
+			}
+		}
+		/*****************************label all unitigs****************************************/
+	}
+	else
+	{
+		/*****************************label all reads****************************************/
+		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
+		{
+			qn = (b_max.b_0->b.a[b_max.untigI]>>1);
+			set_R_to_U(ruIndex, qn, 1, 1, &(read_sg->seq[qn].c));
+		}
+		/*****************************label all reads****************************************/
+
+		///each read
+		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
+		{
+			qn = (b_min.b_0->b.a[b_min.untigI]>>1);
+
+			/************************BUG: don't forget****************************/
+			if(reverse_sources[qn].length > 0) min_count++;
+			///if(reverse_sources[qn].length >= 0) min_count++;
+			/************************BUG: don't forget****************************/
+
+			for (j = 0; j < (long long)reverse_sources[qn].length; j++)
+        	{
+            	tn = Get_tn(reverse_sources[qn].buffer[j]);
+				if(nsg->seq[tn].del == 1)
+				{
+					get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
+					if(tn == (uint32_t)-1 || is_Unitig == 1 || nsg->seq[tn].del == 1) continue;
+				}
+
+
+				get_R_to_U(ruIndex, tn, &uId, &is_Unitig);
+				if(uId!=(uint32_t)-1 && is_Unitig == 1)
+				{
+					max_count++;
+					break;
+				}
+			}
+		}
+
+		/*****************************label all reads****************************************/
+		for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
+		{
+			qn = (b_max.b_0->b.a[b_max.untigI]>>1);
+			ruIndex->index[qn] = (uint32_t)-1;
+		}
+		/*****************************label all reads****************************************/
+	}
+
+	// if(((v_0==7707) && (v_1==26867))||((v_1==7707) && (v_0==26867)))
+	// {
+	// 	fprintf(stderr, "******\nv_0>>1: %u, v_0&1: %u, ELen_0: %u\n", v_0>>1, v_0&1, (uint32_t)ELen_0);
+	// 	fprintf(stderr, "v_1>>1: %u, v_1&1: %u, ELen_1: %u\n", v_1>>1, v_1&1, (uint32_t)ELen_1);
+	// 	fprintf(stderr, "min_count: %u, max_count: %u, DIFF_HAP_RATE: %f\n\n", 
+	// 	min_count, max_count, DIFF_HAP_RATE);
+	// }
+	
+	if(min_count == 0) return UNAVAILABLE;
+	if(max_count > min_count*DIFF_HAP_RATE) return PLOID;
+	return NON_PLOID;
+}
+
+inline uint32_t check_different_haps_naive(asg_t *nsg, ma_ug_t *ug, asg_t *read_sg, 
+uint32_t v_0, uint32_t v_1, ma_hit_t_alloc* reverse_sources, buf_t* b_0, buf_t* b_1, 
+R_to_U* ruIndex, uint32_t min_edge_length, uint32_t stops_threshold)
+{
+	uint32_t vEnd, qn, tn, j, is_Unitig;
+	long long ELen_0, ELen_1, tmp, max_stop_nodeLen, max_stop_baseLen;
+
+	b_0->b.n = b_1->b.n = 0;
+	if(get_unitig(nsg, ug, v_0, &vEnd, &ELen_0, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
+	stops_threshold, b_0) == LOOP)
+	{
+		return UNAVAILABLE;
+	} 
+	if(get_unitig(nsg, ug, v_1, &vEnd, &ELen_1, &tmp, &max_stop_nodeLen, &max_stop_baseLen, 
+	stops_threshold, b_1) == LOOP)
+	{
+		return UNAVAILABLE;
+	}
+
+	if(ELen_0<=min_edge_length || ELen_1<=min_edge_length) return UNAVAILABLE;
+
+	rIdContig b_max, b_min;
+	b_max.b_0 = b_min.b_0 = NULL;
+	b_max.offset = b_max.readI = b_max.untigI = 0;
+	b_min.offset = b_min.readI = b_min.untigI = 0;
+
+	if(ELen_0<=ELen_1)
+	{
+        b_min.b_0 = b_0;
+        b_max.b_0 = b_1;
+    }
+    else
+    {
+        b_min.b_0 = b_1;
+        b_max.b_0 = b_0;
+    }
+
+	uint32_t max_count = 0, min_count = 0;
+	ma_utg_t *node_min = NULL, *node_max = NULL;
+
+	if(ug != NULL)
+	{
+		///each unitig
+		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
+		{
+			
+			node_min = &(ug->u.a[(b_min.b_0->b.a[b_min.untigI]>>1)]);
+
+			///each read
+			for (b_min.readI = 0; b_min.readI < node_min->n; b_min.readI++)
+        	{
+				qn = node_min->a[b_min.readI]>>33;
+
+				if(reverse_sources[qn].length > 0) min_count++;
+				for (j = 0; j < (long long)reverse_sources[qn].length; j++)
+				{
+					tn = Get_tn(reverse_sources[qn].buffer[j]);
+					if(read_sg->seq[tn].del == 1)
+					{
+						get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
+						if(tn == (uint32_t)-1 || is_Unitig == 1 || read_sg->seq[tn].del == 1) continue;
+					}
+
+					///each unitig
+					for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
+					{
+						node_max = &(ug->u.a[b_max.b_0->b.a[b_max.untigI]>>1]);
+						///each read
+						for (b_max.readI = 0; b_max.readI < node_max->n; b_max.readI++)
+						{
+							if(tn == (node_max->a[b_max.readI]>>33))
+							{
+								max_count++;
+								goto end_check_different_haps_ug;
+							}
+						}
+					}
+				}
+
+				end_check_different_haps_ug:;
+			}
+		}
+	}
+	else
+	{
+		///each read
+		for (b_min.untigI = 0; b_min.untigI < b_min.b_0->b.n; b_min.untigI++)
+		{
+			qn = (b_min.b_0->b.a[b_min.untigI]>>1);
+
+			if(reverse_sources[qn].length > 0) min_count++;
+
+			for (j = 0; j < (long long)reverse_sources[qn].length; j++)
+        	{
+            	tn = Get_tn(reverse_sources[qn].buffer[j]);
+				if(nsg->seq[tn].del == 1)
+				{
+					get_R_to_U(ruIndex, tn, &tn, &is_Unitig);
+					if(tn == (uint32_t)-1 || is_Unitig == 1 || nsg->seq[tn].del == 1) continue;
+				}
+
+				///each read
+				for (b_max.untigI = 0; b_max.untigI < b_max.b_0->b.n; b_max.untigI++)
+                {
+					if((b_max.b_0->b.a[b_max.untigI]>>1) == tn)
+					{
+						max_count++;
+						goto end_check_different_haps_non_ug;
+					}
+				}
+			}
+
+			end_check_different_haps_non_ug:;
+		}
+	}
+
+	// if(((v_0==7707) && (v_1==26867))||((v_1==7707) && (v_0==26867)))
+	// {
+	// 	fprintf(stderr, "******\nv_0>>1: %u, v_0&1: %u, ELen_0: %u\n", v_0>>1, v_0&1, (uint32_t)ELen_0);
+	// 	fprintf(stderr, "v_1>>1: %u, v_1&1: %u, ELen_1: %u\n", v_1>>1, v_1&1, (uint32_t)ELen_1);
+	// 	fprintf(stderr, "min_count: %u, max_count: %u, DIFF_HAP_RATE: %f\n\n", 
+	// 	min_count, max_count, DIFF_HAP_RATE);
+	// }
+	
+	if(min_count == 0) return UNAVAILABLE;
+	if(max_count > min_count*DIFF_HAP_RATE) return PLOID;
+	return NON_PLOID;
+}
+
+
 
 typedef struct {
 	uint32_t father_occ;
@@ -725,55 +1019,36 @@ typedef struct {
 	uint32_t total;
 } Trio_counter;
 
-typedef struct {
-	uint32_t p; // the optimal parent vertex
-	uint32_t d; // the shortest distance from the initial vertex
-	uint32_t r:31, s:1; // r: the number of remaining incoming arc; s: state
-} binfo_s_t;
-
-typedef struct {
-	///all information for each node
-	binfo_s_t *a;
-	kvec_t(uint32_t) S; // set of vertices without parents, nodes with all incoming edges visited
-	kvec_t(uint32_t) b; // visited vertices
-	kvec_t(uint32_t) e; // visited edges/arcs
-} buf_s_t;
-
-typedef struct{
-    buf_s_t *b;
-    uint32_t n_thres, n_reads;
-    asg_t *g;
-    uint32_t check_cross;
-	uint64_t bub_dist;
-} bub_label_t;
-
 void resolve_tangles(ma_ug_t *src, asg_t *read_g, ma_hit_t_alloc* reverse_sources, long long minLongUntig, 
-long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, uint8_t* is_r_het, 
-uint32_t trio_flag, float drop_ratio);
-void adjust_utg_advance(asg_t *sg, ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, bub_label_t* b_mask_t, uint8_t* is_r_het);
+long long maxShortUntig, float l_untig_rate, float max_node_threshold, R_to_U* ruIndex, uint32_t trio_flag, 
+float drop_ratio);
+void adjust_utg_advance(asg_t *sg, ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex);
 void rescue_contained_reads_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut,
-R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t chainLenThres, uint32_t is_bubble_check, 
-uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes, bub_label_t* b_mask_t);
+R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t chainLenThres, uint32_t is_bubble_check, 
+uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, kvec_t_u32_warp* new_rtg_nodes);
 void rescue_missing_overlaps_aggressive(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut,
-R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t is_bubble_check, uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t);
+R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t is_bubble_check, 
+uint32_t is_primary_check, kvec_asg_arc_t_warp* new_rtg_edges);
 void all_to_all_deduplicate(ma_ug_t* ug, asg_t* read_g, ma_sub_t* coverage_cut, 
-ma_hit_t_alloc* sources, uint8_t postive_flag, float drop_rate, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, uint8_t* is_r_het, float double_check_rate, int non_tig_occ);
-void drop_semi_circle(ma_ug_t *ug, asg_t* nsg, asg_t* read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, uint8_t* is_r_het);
+ma_hit_t_alloc* sources, uint8_t postive_flag, float drop_rate, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, float double_check_rate);
+void drop_semi_circle(ma_ug_t *ug, asg_t* nsg, asg_t* read_g, ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex);
 void rescue_wrong_overlaps_to_unitigs(ma_ug_t *i_ug, asg_t *r_g,  ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, 
-ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, kvec_asg_arc_t_warp* keep_edges, bub_label_t* b_mask_t);
+ma_sub_t *coverage_cut, R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, kvec_asg_arc_t_warp* keep_edges);
 void get_unitig_trio_flag(ma_utg_t* nsu, uint32_t flag, uint32_t* require, uint32_t* non_require, uint32_t* ambigious);
 void rescue_missing_overlaps_backward(ma_ug_t *i_ug, asg_t *r_g, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut,
-R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t backward_steps, uint32_t is_bubble_check, uint32_t is_primary_check, bub_label_t* b_mask_t);
+R_to_U* ruIndex, int max_hang, int min_ovlp, long long bubble_dist, uint32_t backward_steps, 
+uint32_t is_bubble_check, uint32_t is_primary_check);
 uint32_t get_edge_from_source(ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, 
 R_to_U* ruIndex, int max_hang, int min_ovlp, uint32_t query, uint32_t target, asg_arc_t* t);
+uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, int max_dist, buf_t *b, 
+uint32_t positive_flag, uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes);
 int unitig_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio);
-void asg_bub_backtrack_primary(asg_t *g, uint32_t v0, buf_t *b);
+
 
 typedef struct{
     double weight;
-    uint32_t uID;
+    uint32_t uID:31, del:1;
     uint64_t dis;
-	uint8_t is_cc:7, del:1;
 	uint64_t occ;
 	///uint64_t occ:63, scaff:1;
     ///uint32_t enzyme;
@@ -796,62 +1071,10 @@ typedef struct{
 typedef struct{
     kvec_t(hc_linkeage) a;
     kvec_t(uint64_t) enzymes;
-} hc_links;
-
-#define N_HET 0
-#define C_HET 1
-#define P_HET 2
-#define S_HET 4
-
-typedef struct {
-	uint32_t p_x_p, p_y_p, p_x, p_y;
-	uint32_t c_x_p, c_y_p;
-	uint8_t c_rev;
-} ca_buf_t;
-
-typedef struct {
-	size_t n, m;
-	ca_buf_t* a;
-} kv_ca_buf_t;
-
-typedef struct {
-	kvec_t(uint32_t) uIDs;
-	kvec_t(uint32_t) iDXs;
-	uint32_t chain_num;
-} sub_tran_t;
-
-typedef struct{
-	uint32_t* rUidx;
-	uint64_t* rUpos;
-	uint8_t* ir_het;
-	uint32_t r_num, u_num;
 	kvec_t(bed_in) bed;
-	kvec_t(uint32_t) topo_buf;
-	kvec_t(uint32_t) topo_res;
-	buf_t b_buf_0, b_buf_1;
-	///uint32_t* uLen;
-	kv_u_trans_t k_trans;
-	kv_u_trans_hit_t k_t_b;
-	kv_ca_buf_t c_buf;
-	sub_tran_t st;
-}trans_chain;
-
-typedef struct {
-    uint32_t n;
-    uint32_t* cov;
-    uint64_t* pos_idx;
-    ma_hit_t_alloc* reverse_sources;
-    ma_sub_t *coverage_cut;
-    R_to_U* ruIndex;
-	asg_t *read_g;
-    int max_hang;
-    int min_ovlp;
-    kvec_asg_arc_t_offset u_buffer; 
-    kvec_t_i32_warp tailIndex;
-    kvec_t_i32_warp prevIndex;
-	uint8_t* is_r_het;
-	trans_chain* t_ch;
-}hap_cov_t;
+    uint32_t* u_idx;
+	uint64_t r_num;
+} hc_links;
 
 typedef struct{
     ///kvec_t(hc_edge) a;
@@ -859,55 +1082,24 @@ typedef struct{
     hc_edge *a;
 }hc_edge_warp;
 
-typedef struct {
-	uint32_t qs, qe, qn, qus, que;
-	uint32_t ts, te, tn, tus, tue;
-} utg_thit_t;
-
-typedef struct {
-	size_t n, m;
-	utg_thit_t* a;
-} kv_utg_thit_t_t;
-
-typedef struct {
-	ma_hit_t_alloc* reverse_sources;
-    ma_sub_t *coverage_cut;
-    R_to_U* ruIndex;
-    asg_t *read_g;	
-	kvec_asg_arc_t_offset u_buffer; 
-    kvec_t_i32_warp tailIndex;
-    kvec_t_i32_warp prevIndex;
-	kv_utg_thit_t_t k_t_b;
-	kv_ca_buf_t c_buf;
-	kv_u_trans_t k_trans;
-	uint64_t *pos_idx, rn;
-	kvec_t(uint32_t) topo_res;
-	ma_ug_t *cug;
-	int max_hang;
-	int min_ovlp;
-
-	ma_utg_v u;
-	kv_u_trans_t t;
-	buf_t b0, b1;
-} utg_trans_t;
-
-void init_hc_links(hc_links* link, uint64_t ug_num, trans_chain* t_ch);
+void init_hc_links(hc_links* link, uint64_t ug_num, uint64_t r_num);
 void destory_hc_links(hc_links* link);
-uint64_t get_bub_pop_max_dist(asg_t *g, buf_t *b);
-uint64_t get_bub_pop_max_dist_advance(asg_t *g, buf_t *b);
-uint64_t asg_bub_pop1_primary_trio(asg_t *g, ma_ug_t *utg, uint32_t v0, uint64_t max_dist, buf_t *b, uint32_t positive_flag, 
-uint32_t negative_flag, uint32_t is_pop, uint64_t* path_base_len, uint64_t* path_nodes, hap_cov_t *cov, uint32_t is_update_chain, uint32_t keep_d, utg_trans_t *o);
-
+void clean_primary_untig_graph(ma_ug_t *ug, asg_t *read_g, ma_hit_t_alloc* reverse_sources,
+long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
+R_to_U* ruIndex, buf_t* b_0, uint8_t* visit, float density, uint32_t miniHapLen, 
+uint32_t miniBiGraph, float chimeric_rate, int is_final_clean, int just_bubble_pop, 
+float drop_ratio, hc_links* link);
 void adjust_utg_by_primary(ma_ug_t **ug, asg_t* read_g, float drop_rate,
 ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, 
-long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
+long long bubble_dist, long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
 R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp,
-kvec_asg_arc_t_warp* new_rtg_edges, hap_cov_t **i_cov, bub_label_t* b_mask_t, uint32_t collect_p_trans, uint32_t collect_p_trans_f);
+kvec_asg_arc_t_warp* new_rtg_edges, hc_links* link);
+void collect_reverse_unitigs(buf_t* b_0, buf_t* b_1, hc_links* link, ma_ug_t *ug, asg_t *read_sg);
 ma_ug_t* copy_untig_graph(ma_ug_t *src);
 ma_ug_t* output_trio_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, 
-uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, 
+uint8_t flag, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long bubble_dist, 
 long long tipsLen, float tip_drop_ratio, long long stops_threshold, R_to_U* ruIndex, 
-float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, bub_label_t* b_mask_t);
+float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench);
 asg_t* copy_read_graph(asg_t *src);
 ma_ug_t *ma_ug_gen(asg_t *g);
 void ma_ug_destroy(ma_ug_t *ug);
@@ -920,88 +1112,6 @@ inline int inter_interval(int a_s, int a_e, int b_s, int b_e, int* i_s, int* i_e
     return 1;
 }
 
-inline uint32_t get_origin_uid(uint32_t v, trans_chain* t_ch, uint32_t *off, uint32_t *idx)
-{
-	if(off) (*off) = (t_ch->rUpos[v>>1]>>32);
-	if(idx) (*idx) = (uint32_t)(t_ch->rUpos[v>>1]);
-    if(t_ch->rUpos[v>>1] == (uint64_t)-1) return (uint32_t)-1;
-    return (uint32_t)(((t_ch->rUidx[v>>1]>>1)<<1) + ((t_ch->rUidx[v>>1]^v)&1));
-}
-void chain_origin_trans_uid_by_distance(hap_cov_t *cov, asg_t *read_sg, 
-uint32_t *pri_a, uint32_t pri_n, uint32_t pri_beg, uint64_t *i_pri_len, 
-uint32_t *aux_a, uint32_t aux_n, uint32_t aux_beg, uint64_t *i_aux_len,
-ma_ug_t *ug, uint32_t flag, double overall_score, const char* cmd);
-int asg_arc_del_trans(asg_t *g, int fuzz);
-void kt_u_trans_t_idx(kv_u_trans_t *ta, uint32_t n);
-void kt_u_trans_t_simple_symm(kv_u_trans_t *ta, uint32_t un, uint32_t symm_add);
-uint32_t get_u_trans_spec(kv_u_trans_t *ta, uint32_t qn, uint32_t tn, u_trans_t **r_a, uint32_t *occ);
-int ma_ug_seq(ma_ug_t *g, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, 
-kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, kvec_asg_arc_t_warp *E, uint32_t is_polish);
-
-
-typedef struct{
-    ma_sub_t* coverage_cut;
-    ma_hit_t_alloc* sources; 
-    ma_hit_t_alloc* reverse_sources; 
-    long long tipsLen; 
-    float tip_drop_ratio; 
-    long long stops_threshold; 
-    R_to_U* ruIndex; 
-    float chimeric_rate;
-    float drop_ratio; 
-    int max_hang;
-    int min_ovlp;
-    int is_bench;
-	long long gap_fuzz;
-    bub_label_t* b_mask_t;
-}ug_opt_t;
-
-void adjust_utg_by_trio(ma_ug_t **ug, asg_t* read_g, uint8_t flag, float drop_rate,
-ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, ma_sub_t* coverage_cut, 
-long long tipsLen, float tip_drop_ratio, long long stops_threshold, 
-R_to_U* ruIndex, float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp,
-kvec_asg_arc_t_warp* new_rtg_edges, bub_label_t* b_mask_t);
-uint32_t cmp_untig_graph(ma_ug_t *src, ma_ug_t *dest);
-void reduce_hamming_error(asg_t *sg, ma_hit_t_alloc* sources, ma_sub_t *coverage_cut, 
-int max_hang, int min_ovlp, long long gap_fuzz);
-int ma_ug_seq_scaffold(ma_ug_t *g, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, 
-kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, kvec_asg_arc_t_warp *E, uint32_t is_polish);
-void ma_ug_print(const ma_ug_t *ug, asg_t* read_g, const ma_sub_t *coverage_cut, 
-ma_hit_t_alloc* sources, R_to_U* ruIndex, const char* prefix, FILE *fp);
-void ma_ug_print_simple(const ma_ug_t *ug, asg_t* read_g, const ma_sub_t *coverage_cut, 
-ma_hit_t_alloc* sources, R_to_U* ruIndex, const char* prefix, FILE *fp);
-trans_chain* init_trans_chain(ma_ug_t *ug, uint64_t r_num);
-void destory_trans_chain(trans_chain **x);
-
-typedef struct {///[cBeg, cEnd)
-	uint32_t u_i, r_i, len, s_pos_cur, s_pre_v, s_pre_w, p_v, p_idx, p_uId, cBeg, cEnd;
-    ///buf_t* x;
-    uint32_t *a, an;
-    ma_ug_t *ug;
-    asg_t *read_sg;
-    trans_chain* t_ch;
-} u_trans_hit_idx;
-void reset_u_trans_hit_idx(u_trans_hit_idx *t, uint32_t* i_x_a, uint32_t i_x_n, ma_ug_t *i_ug, 
-asg_t *i_read_sg, trans_chain* i_t_ch, uint32_t i_cBeg, uint32_t i_cEnd);
-uint32_t get_u_trans_hit(u_trans_hit_idx *t, u_trans_hit_t *hit);
-inline uint32_t get_offset_adjust(uint32_t offset, uint32_t offsetLen, uint32_t targetLen)
-{
-    return ((double)(offset)/(double)(offsetLen))*targetLen;
-}
-
-uint32_t set_utg_offset(uint32_t *a, uint32_t a_n, ma_ug_t *ug, asg_t *read_sg, uint64_t* pos_idx, uint32_t is_clear,
-uint32_t only_len);
-uint64_t get_utg_cov(ma_ug_t *ug, uint32_t uID, asg_t* read_g, 
-const ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, R_to_U* ruIndex, uint8_t* r_flag);
-trans_chain* load_hc_trans(const char *fn);
-char *get_outfile_name(char* output_file_name);
-void reset_u_trans_hit_idx(u_trans_hit_idx *t, uint32_t* i_x_a, uint32_t i_x_n, ma_ug_t *i_ug, 
-asg_t *i_read_sg, trans_chain* i_t_ch, uint32_t i_cBeg, uint32_t i_cEnd);
-void extract_sub_overlaps(uint32_t i_tScur, uint32_t i_tEcur, uint32_t i_tSpre, uint32_t i_tEpre,
-uint32_t tn, kv_u_trans_hit_t* ktb, uint32_t bn);
-void clean_u_trans_t_idx(kv_u_trans_t *ta, ma_ug_t *ug, asg_t *read_g);
-
-
 #define JUNK_COV 5
 #define DISCARD_RATE 0.8
 

Process_Read.h

@@ -101,9 +101,6 @@ typedef struct
 #define MIX_TRIO 3
 #define NON_TRIO 4
 #define DROP 5
-#define SET_TRIO 8
-#define CHAIN_MATCH 1
-#define CHAIN_UNMATCH 0.334
 
 typedef struct
 {

Purge_Dups.h

@@ -10,84 +10,15 @@
 #define HOM_PEAK_RATE 1.25
 #define HET_PEAK_RATE (HOM_PEAK_RATE*2)
 #define ALTER_COV_THRES 0.9
-#define REAL_ALTER_THRES 0.25
-#define CHAIN_FILTER_RATE 0.7
-
-#define SELF_EXIST 0
-#define REVE_EXIST 1
-#define DELETE 2
-#define MIXED 3
-#define FLIP 4
-
-#define X2Y 0
-#define Y2X 1
-#define XCY 2
-#define YCX 3
-
-#define Cal_Off(OFF) ((long long)((uint32_t)((OFF)>>32)) - (long long)((uint32_t)((OFF))))
-#define Get_xOff(OFF) ((long long)((uint32_t)((OFF)>>32)))
-#define Get_yOff(OFF) ((long long)((uint32_t)((OFF))))
-#define Get_match(x) ((x).weight)
-#define Get_total(x) ((x).index_beg)
-#define Get_type(x) ((x).index_end)
-#define Get_x_beg(x) ((x).x_beg_pos)
-#define Get_x_end(x) ((x).x_end_pos)
-#define Get_y_beg(x) ((x).y_beg_pos)
-#define Get_y_end(x) ((x).y_end_pos)
-#define Get_rev(x) ((x).rev)
-
-typedef struct {
-    uint8_t rev;
-    uint8_t type;
-    uint8_t status;
-    uint32_t x_beg_pos;
-    uint32_t x_end_pos;
-    uint32_t y_beg_pos;
-    uint32_t y_end_pos;
-    uint32_t x_beg_id;
-    uint32_t x_end_id;
-    uint32_t y_beg_id;
-    uint32_t y_end_id;
-    uint32_t xUid;
-    uint32_t yUid;
-    uint32_t weight;
-    long long score;
-    float s;
-}hap_overlaps;
-
-typedef struct {
-    kvec_t(hap_overlaps) a;
-}kvec_hap_overlaps;
-
-typedef struct {
-    kvec_hap_overlaps* x;
-    uint32_t num;
-}hap_overlaps_list;
+#define REAL_ALTER_THRES 0.1
 
 void purge_dups(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, 
 ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, kvec_asg_arc_t_warp* edge, float density, 
-uint32_t purege_minLen, int max_hang, int min_ovlp, float drop_ratio, uint32_t just_contain, 
-uint32_t just_coverage, hap_cov_t *cov, uint32_t collect_p_trans, uint32_t collect_p_trans_f);
+uint32_t purege_minLen, int max_hang, int min_ovlp, long long bubble_dist, float drop_ratio, 
+uint32_t just_contain, uint32_t just_coverage, hc_links* link);
 void fill_unitig(uint64_t* buffer, uint32_t bufferLen, asg_t* read_g, kvec_asg_arc_t_warp* edge,
 uint32_t is_circle, uint64_t* rLen);
 void get_contig_length(ma_ug_t *ug, asg_t *g, uint64_t* primaryLen, uint64_t* alterLen);
 void enable_debug_mode(uint32_t mode);
-hap_cov_t* init_hap_cov_t(ma_ug_t *ug, asg_t* read_g, ma_hit_t_alloc* sources, R_to_U* ruIndex, ma_hit_t_alloc* reverse_sources, 
-ma_sub_t *coverage_cut, int max_hang, int min_ovlp, uint32_t is_collect_trans);
-void destory_hap_cov_t(hap_cov_t **x);
-void chain_trans_ovlp(hap_cov_t *cov, utg_trans_t *o, ma_ug_t *ug, asg_t *read_sg, buf_t* xReads, uint32_t targetBaseLen, uint32_t* xEnd);
-int get_specific_hap_overlap(kvec_hap_overlaps* x, uint32_t qn, uint32_t tn);
-void set_reverse_hap_overlap(hap_overlaps* dest, hap_overlaps* source, uint32_t* types);
-void print_hap_paf(ma_ug_t *ug, hap_overlaps* ovlp);
-uint64_t get_xy_pos_by_pos(asg_t *read_g, asg_arc_t* t, uint32_t v_in_unitig, uint32_t w_in_unitig, 
-uint32_t v_in_pos, uint32_t w_in_pos, uint32_t xUnitigLen, uint32_t yUnitigLen, uint8_t* rev);
-void quick_LIS(asg_arc_t_offset* x, uint32_t n, kvec_t_i32_warp* tailIndex, kvec_t_i32_warp* prevIndex);
-uint32_t classify_hap_overlap(long long xBeg, long long xEnd, long long xLen,
-long long yBeg, long long yEnd, long long yLen, long long* r_xBeg, long long* r_xEnd, 
-long long* r_yBeg, long long* r_yEnd);
-int cmp_hap_alignment_chaining(const void * a, const void * b);
-uint32_t classify_hap_overlap(long long xBeg, long long xEnd, long long xLen,
-long long yBeg, long long yEnd, long long yLen, long long* r_xBeg, long long* r_xEnd, 
-long long* r_yBeg, long long* r_yEnd);
 
 #endif

README.md

@@ -1,4 +1,4 @@
-## <a name="started"></a>Getting Started
+## Getting Started
 
 ```sh
 # Install hifiasm (requiring g++ and zlib)
@@ -8,52 +8,34 @@ cd hifiasm && make
 # Run on test data (use -f0 for small datasets)
 wget https://github.com/chhylp123/hifiasm/releases/download/v0.7/chr11-2M.fa.gz
 ./hifiasm -o test -t4 -f0 chr11-2M.fa.gz 2> test.log
-awk '/^S/{print ">"$2;print $3}' test.bp.p_ctg.gfa > test.p_ctg.fa  # get primary contigs in FASTA
+awk '/^S/{print ">"$2;print $3}' test.p_ctg.gfa > test.p_ctg.fa  # get primary contigs in FASTA
 
 # Assemble inbred/homozygous genomes (-l0 disables duplication purging)
 hifiasm -o CHM13.asm -t32 -l0 CHM13-HiFi.fa.gz 2> CHM13.asm.log
-# Assemble heterozygous genomes with built-in duplication purging
+# Assemble heterozygous with built-in duplication purging
 hifiasm -o HG002.asm -t32 HG002-file1.fq.gz HG002-file2.fq.gz
 
-# Hi-C phasing with paired-end short reads in two FASTQ files
-hifiasm -o HG002.asm --h1 read1.fq.gz --h2 read2.fq.gz HG002-HiFi.fq.gz
-
 # Trio binning assembly (requiring https://github.com/lh3/yak)
 yak count -b37 -t16 -o pat.yak <(cat pat_1.fq.gz pat_2.fq.gz) <(cat pat_1.fq.gz pat_2.fq.gz)
 yak count -b37 -t16 -o mat.yak <(cat mat_1.fq.gz mat_2.fq.gz) <(cat mat_1.fq.gz mat_2.fq.gz)
 hifiasm -o HG002.asm -t32 -1 pat.yak -2 mat.yak HG002-HiFi.fa.gz
 ```
-See [tutorial][tutorial] for more details. 
 
-## Table of Contents
+## Introduction
 
-- [Getting Started](#started)
-- [Introduction](#intro)
-- [Why Hifiasm?](#why)
-- [Usage](#use)
-  - [Assembling HiFi reads without additional data types](#hifionly)
-  - [Hi-C integration](#hic)
-  - [Trio binning](#trio)
-  - [Output files](#output)
-- [Results](#results)
-- [Getting Help](#help)
-- [Limitations](#limit)
-- [Citing Hifiasm](#cite)
+Hifiasm is a fast haplotype-resolved de novo assembler for PacBio Hifi reads. 
+It can assemble a human genome in several hours and works with the California
+redwood genome, one of the most complex genomes sequenced so far. Hifiasm can
+produce primary/alternate assemblies of quality competitive with the best
+assemblers. It also introduces a new graph binning algorithm and achieves
+the best haplotype-resolved assembly given trio data.
 
-## <a name="intro"></a>Introduction
-
-Hifiasm is a fast haplotype-resolved de novo assembler for PacBio HiFi reads.
-It can assemble a human genome in several hours and assemble a ~30Gb California
-redwood genome in a few days. Hifiasm emits partially phased assemblies of
-quality competitive with the best assemblers. Given parental short reads or
-Hi-C data, it produces arguably the best haplotype-resolved assemblies so far.
-
-## <a name="why"></a>Why Hifiasm?
+## Why Hifiasm?
 
 * Hifiasm delivers high-quality assemblies. It tends to generate longer contigs
   and resolve more segmental duplications than other assemblers.
 
-* Given Hi-C reads or short reads from the parents, hifiasm can produce overall the best
+* Given sequence reads from the parents, hifiasm can produce overall the best
   haplotype-resolved assembly so far. It is the assembler of choice by the
   [Human Pangenome Project][hpp] for the first batch of samples.
 
@@ -65,15 +47,13 @@ Hi-C data, it produces arguably the best haplotype-resolved assemblies so far.
 * Hifiasm is fast. It can assemble a human genome in half a day and assemble a
   ~30Gb redwood genome in three days. No genome is too large for hifiasm.
 
-* Hifiasm is trivial to install and easy to use. It does not required Python,
-  R or C++11 compilers, and can be compiled into a single executable. The
+* Hifiasm is trivial to install and easy to use. It does not required python,
+  R or C++11 compilers and can be compiled into a single executable. The
   default setting works well with a variety of genomes.
 
 [hpp]: https://humanpangenome.org
 
-## <a name="use"></a>Usage
-
-### <a name="hifionly"></a>Assembling HiFi reads without additional data types
+## Usage
 
 A typical hifiasm command line looks like:
 ```sh
@@ -81,21 +61,11 @@ hifiasm -o NA12878.asm -t 32 NA12878.fq.gz
 ```
 where `NA12878.fq.gz` provides the input reads, `-t` sets the number of CPUs in
 use and `-o` specifies the prefix of output files. For this example, the
-primary contigs are written to `NA12878.asm.bp.p_ctg.gfa`. 
-Since v0.15, hifiasm also produces two sets of
-partially phased contigs at `NA12878.asm.bp.hap?.p_ctg.gfa`. This pair of files
-can be thought to represent the two haplotypes in a diploid genome, though with
-occasional switch errors. The frequency of switches is determined by the
-heterozygosity of the input sample.
-
-At the first run, hifiasm saves corrected reads and
+primary contigs are written to `NA12878.asm.p_ctg.gfa` and alternate contigs to
+`NA12878.asm.a_ctg.gfa`. At the first run, hifiasm saves corrected reads and
 overlaps to disk as `NA12878.asm.*.bin`. It reuses the saved results to avoid
 the time-consuming all-vs-all overlap calculation next time. You may specify
 `-i` to ignore precomputed overlaps and redo overlapping from raw reads.
-You can also dump error corrected reads in FASTA and read overlaps in PAF with
-```sh
-hifiasm -o NA12878.asm -t 32 --write-paf --write-ec /dev/null
-```
 
 Hifiasm purges haplotig duplications by default. For inbred or homozygous
 genomes, you may disable purging with option `-l0`. Old HiFi reads may contain
@@ -105,27 +75,7 @@ bloom filter which takes 16GB memory at the beginning. For genomes much larger
 than human, applying `-f38` or even `-f39` is preferred to save memory on k-mer
 counting.
 
-### <a name="hic"></a>Hi-C integration
-
-Hifiasm can generate a pair of haplotype-resolved assemblies with paired-end
-Hi-C reads:
-```sh
-hifiasm -o NA12878.asm -t32 --h1 read1.fq.gz --h2 read2.fq.gz HiFi-reads.fq.gz
-```
-In this mode, each contig is supposed to be a haplotig, which by definition
-comes from one parental haplotype only. Hifiasm often puts all contigs from the
-same parental chromosome in one assembly. It has cleanly separated chrX and
-chrY for a human male dataset. Nonetheless, phasing across centromeres is
-challenging. Hifiasm is often able to phase entire chromosomes but it may fail 
-in rare cases. Also, contigs from different parental chromosomes are randomly mixed as
-it is just not possible to phase across chromosomes with Hi-C.
-
-Hifiasm does not perform scaffolding for now. You need to run a standalone
-scaffolder such as SALSA or 3D-DNA to scaffold phased haplotigs.
-
-### <a name="trio"></a>Trio binning
-
-When parental short reads are available, hifiasm can also generate a pair of
+When parental short reads are available, hifiasm can generate a pair of
 haplotype-resolved assemblies with trio binning. To perform such assembly, you
 need to count k-mers first with [yak][yak] first and then do assembly:
 ```sh
@@ -133,26 +83,52 @@ yak count -k31 -b37 -t16 -o pat.yak paternal.fq.gz
 yak count -k31 -b37 -t16 -o mat.yak maternal.fq.gz
 hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak NA12878.fq.gz
 ```
-Here `NA12878.asm.dip.hap1.p_ctg.gfa` and `NA12878.asm.dip.hap2.p_ctg.gfa` give the two
+Here `NA12878.asm.hap1.p_ctg.gfa` and `NA12878.asm.hap2.p_ctg.gfa` give the two
 haplotype assemblies. In the binning mode, hifiasm does not purge haplotig
-duplicates by default. Because hifiasm reuses saved overlaps, you can
+duplications by default. Because hifiasm reuses saved overlaps, you can
 generate both primary/alternate assemblies and trio binning assemblies with
 ```sh
 hifiasm -o NA12878.asm -t 32 NA12878.fq.gz 2> NA12878.asm.pri.log
 hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak /dev/null 2> NA12878.asm.trio.log
 ```
-The second command line will run much faster than the first.
+The second command line will run much faster than the first. You can also dump
+error corrected in FASTA and/or overlaps in PAF with
+```sh
+hifiasm -o NA12878.asm -t 32 --write-paf --write-ec /dev/null
+```
 
-### <a name="output"></a>Output files
+## Output files
 
-Hifiasm generates different types of assemblies based on the input data. 
-It also writes error corrected reads to the *prefix*.ec.bin binary file and
+For non-trio assembly, hifiasm generates the following files:
+
+1. Haplotype-resolved raw [unitig][unitig] graph in [GFA][gfa] format
+   (*prefix*.r\_utg.gfa). This graph keeps all haplotype information, including
+   somatic mutations and recurrent sequencing errors.
+2. Haplotype-resolved processed unitig graph without small bubbles
+   (*prefix*.p\_utg.gfa). Small bubbles might be caused by somatic mutations or noise in data, 
+   which are not the real haplotype information.
+3. Primary assembly [contig][unitig] graph (*prefix*.p\_ctg.gfa). This graph collapses different
+   haplotypes.
+4. Alternate assembly contig graph (*prefix*.a\_ctg.gfa). This graph consists of all assemblies that
+   are discarded in primary contig graph.
+
+For trio assembly, hifiasm generates the following files:
+
+1. Haplotype-resolved raw [unitig][unitig] graph in [GFA][gfa] format
+   (*prefix*.r\_utg.gfa). This graph keeps all haplotype information. 
+
+2. Phased paternal/haplotype1 contig graph (*prefix*.hap1.p\_ctg.gfa). This graph keeps the phased
+   paternal/haplotype1 assembly.
+
+3. Phased maternal/haplotype2 contig graph (*prefix*.hap2.p\_ctg.gfa). This graph keeps the phased
+   maternal/haplotype2 assembly.
+
+Hifiasm writes error corrected reads to the *prefix*.ec.bin binary file and
 writes overlaps to *prefix*.ovlp.source.bin and *prefix*.ovlp.reverse.bin.
-For more details, please see the complete [documentation][tutorial_output].
 
-## <a name="results"></a>Results
+## Results
 
-The following table shows the statistics of several hifiasm primary assemblies assembled with v0.12:
+The following table shows the statistics of several hifiasm primary assemblies:
 
 |<sub>Dataset<sub>|<sub>Size<sub>|<sub>Cov.<sub>|<sub>Asm options<sub>|<sub>CPU time<sub>|<sub>Wall time<sub>|<sub>RAM<sub>|<sub> N50<sub>|
 |:---------------|-----:|-----:|:---------------------|-------:|--------:|----:|----------------:|
@@ -179,10 +155,7 @@ redwood genome in a few days on a single machine. For trio binning assembly:
 |:---------------|-----:|-------:|--------:|----:|----------------:|
 |<sub>[HG00733][HG00733-data], [\[father\]][HG00731-data], [\[mother\]][HG00732-data]</sub>|<sub>&times;33</sub>|<sub>269.1h</sub>|<sub>6.9h</sub>|<sub>135G</sub>|<sub>35.1Mb (paternal), 34.9Mb (maternal)</sub>|
 |<sub>[HG002][NA24385-data],   [\[father\]][NA24149-data], [\[mother\]][NA24143-data]</sup>|<sub>&times;36</sub>|<sub>305.4h</sub>|<sub>7.7h</sub>|<sub>137G</sub>|<sub>41.0Mb (paternal), 40.8Mb (maternal)</sub>|
-
-<!--
 |<sub>[NA12878][NA12878-data], [\[father\]][NA12891-data], [\[mother\]][NA12892-data]</sub>|<sub>&times;30</sub>|<sub>180.8h</sub>|<sub>4.9h</sub>|<sub>123G</sub>|<sub>27.7Mb (paternal), 27.0Mb (maternal)</sub>|
--->
 
 [HG00733-data]: https://www.ebi.ac.uk/ena/data/view/ERX3831682
 [HG00731-data]: https://www.ebi.ac.uk/ena/data/view/ERR3241754
@@ -194,35 +167,29 @@ redwood genome in a few days on a single machine. For trio binning assembly:
 [NA12891-data]: https://www.ebi.ac.uk/ena/data/view/ERR194160
 [NA12892-data]: https://www.ebi.ac.uk/ena/data/view/ERR194161
 
-Human assemblies above can be acquired [from Zenodo][zenodo-human] and
-non-human ones are available [here][zenodo-nonh].
+Except NA12878, the assemblies above were produced by hifiasm v0.12 and can be
+downloaded at
+```txt
+ftp://ftp.dfci.harvard.edu/pub/hli/hifiasm/submission/hifiasm-0.12/
+```
+NA12878 was assembled with an older version of hifiasm and is available at
+```txt
+ftp://ftp.dfci.harvard.edu/pub/hli/hifiasm/NA12878-r253/
+```
+
 
-[zenodo-human]: https://zenodo.org/record/4393631
-[zenodo-nonh]: https://zenodo.org/record/4393750
 [unitig]: http://wgs-assembler.sourceforge.net/wiki/index.php/Celera_Assembler_Terminology
 [gfa]: https://github.com/pmelsted/GFA-spec/blob/master/GFA-spec.md
 [paf]: https://github.com/lh3/miniasm/blob/master/PAF.md
 [yak]: https://github.com/lh3/yak
-[tutorial]: https://hifiasm.readthedocs.io/en/latest/index.html
-[tutorial_output]: https://hifiasm.readthedocs.io/en/latest/interpreting-output.html#interpreting-output
 
+## Getting Help
 
-## <a name="help"></a>Getting Help
-
-For detailed description of options, please see [tutorial][tutorial] or `man ./hifiasm.1`. The `-h`
+For detailed description of options, please see `man ./hifiasm.1`. The `-h`
 option of hifiasm also provides brief description of options. If you have
 further questions, please raise an issue at the [issue
 page](https://github.com/chhylp123/hifiasm/issues).
 
-## <a name="limit"></a>Limitations
+## Limitations
 
-1. Purging haplotig duplications may introduce misassemblies.
-
-## <a name="cite"></a>Citating Hifiasm
-
-If you use hifiasm in your work, please cite:
-
-> Cheng, H., Concepcion, G.T., Feng, X., Zhang, H., Li H. (2021)
-> Haplotype-resolved de novo assembly using phased assembly graphs with
-> hifiasm. *Nat Methods*, **18**:170-175.
-> https://doi.org/10.1038/s41592-020-01056-5
+1. Purging haplotig duplications may introduce misassemblies.

anchor.cpp

@@ -1,5 +1,4 @@
 #include <stdio.h>
-#include <math.h>
 #include "htab.h"
 #include "ksort.h"
 #include "Hash_Table.h"
@@ -8,9 +7,8 @@
 
 typedef struct { // this struct is not strictly necessary; we can use k_mer_pos instead, with modifications
 	uint64_t srt;
-	uint32_t self_off;
+	uint32_t self_off:31, good:1;
 	uint32_t other_off;
-	uint32_t cnt;
 } anchor1_t;
 
 #define an_key1(a) ((a).srt)
@@ -25,7 +23,7 @@ KSORT_INIT(or_xs, overlap_region, oreg_xs_lt)
 KSORT_INIT(or_ss, overlap_region, oreg_ss_lt)
 
 typedef struct {
-	int n;
+	int n, good;
 	const ha_idxpos_t *a;
 } seed1_t;
 
@@ -59,14 +57,13 @@ int ha_ov_type(const overlap_region *r, uint32_t len)
 	else return r->x_pos_s == 0? 0 : 1;
 }
 
-void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, int max_n_chain, int keep_whole_chain,
-						   kvec_t_u8_warp* k_flag, kvec_t_u64_warp* chain_idx, void *ha_flt_tab, ha_pt_t *ha_idx, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, st_mt_t *sp)
+void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag,
+kvec_t_u64_warp* chain_idx, void *ha_flt_tab, ha_pt_t *ha_idx, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct)
 {
 	uint32_t i, rlen;
 	uint64_t k, l;
-	uint32_t low_occ = asm_opt.hom_cov * HA_KMER_GOOD_RATIO;
-	uint32_t high_occ = asm_opt.hom_cov * (2.0 - HA_KMER_GOOD_RATIO);
-	if(low_occ < 2) low_occ = 2;
+	double low_occ = asm_opt.hom_cov * HA_KMER_GOOD_RATIO;
+	double high_occ = asm_opt.hom_cov * (2.0 - HA_KMER_GOOD_RATIO);
 
 	// prepare
     clear_Candidates_list(cl);
@@ -76,7 +73,7 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg
 	rlen = Get_READ_LENGTH(R_INF, rid); // read length
 
 	// get the list of anchors
-	ha_sketch(ucr->seq, ucr->length, asm_opt.mz_win, asm_opt.k_mer_length, 0, !(asm_opt.flag & HA_F_NO_HPC), &ab->mz, ha_flt_tab, asm_opt.mz_sample_dist, k_flag, dbg_ct, NULL, -1, asm_opt.dp_min_len, -1, sp, asm_opt.mz_rewin);
+	ha_sketch_query(ucr->seq, ucr->length, asm_opt.mz_win, asm_opt.k_mer_length, 0, !(asm_opt.flag & HA_F_NO_HPC), &ab->mz, ha_flt_tab, k_flag, dbg_ct);
 	// minimizer of queried read
 	if (ab->mz.m > ab->old_mz_m) {
 		ab->old_mz_m = ab->mz.m;
@@ -86,6 +83,7 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg
 		int n;
 		ab->seed[i].a = ha_pt_get(ha_idx, ab->mz.a[i].x, &n);
 		ab->seed[i].n = n;
+		ab->seed[i].good = (n > low_occ && n < high_occ);
 		ab->n_a += n;
 	}
 	if (ab->n_a > ab->m_a) {
@@ -104,7 +102,7 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg
 			uint8_t rev = z->rev == y->rev? 0 : 1;
 			an->other_off = y->pos;
 			an->self_off = rev? ucr->length - 1 - (z->pos + 1 - z->span) : z->pos;
-			an->cnt = s->n;
+			an->good = s->good;
 			an->srt = (uint64_t)y->rid<<33 | (uint64_t)rev<<32 | an->other_off;
 		}
 	}
@@ -131,16 +129,7 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg
 		p->strand = ab->a[k].srt >> 32 & 1;
 		p->offset = ab->a[k].other_off;
 		p->self_offset = ab->a[k].self_off;
-		if(ab->a[k].cnt > low_occ && ab->a[k].cnt < high_occ){
-			p->cnt = 1;
-		}
-		else if(ab->a[k].cnt <= low_occ){
-			p->cnt = 2;
-		}
-		else{
-			p->cnt = 1 + ((ab->a[k].cnt + (high_occ<<1) - 1)/(high_occ<<1));
-			p->cnt = pow(p->cnt, 1.1);
-		}
+		p->good = ab->a[k].good;
 	}
 	cl->length = ab->n_a;
 
@@ -156,7 +145,7 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg
 		}
 	}
 	#endif
-	
+
 	if ((int)overlap_list->length > max_n_chain) {
 		int32_t w, n[4], s[4];
 		n[0] = n[1] = n[2] = n[3] = 0, s[0] = s[1] = s[2] = s[3] = 0;
@@ -168,12 +157,9 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg
 			if ((int)n[w] == max_n_chain) s[w] = r->shared_seed;
 		}
 		if (s[0] > 0 || s[1] > 0 || s[2] > 0 || s[3] > 0) {
-			// n[0] = n[1] = n[2] = n[3] = 0;
 			for (i = 0, k = 0; i < (uint32_t)overlap_list->length; ++i) {
 				overlap_region *r = &overlap_list->list[i];
 				w = ha_ov_type(r, rlen);
-				// ++n[w];
-				// if (((int)n[w] <= max_n_chain) || (r->shared_seed >= s[w] && s[w] >= (asm_opt.k_mer_length<<1))) {
 				if (r->shared_seed >= s[w]) {
 					if ((uint32_t)k != i) {
 						overlap_region t;
@@ -192,187 +178,6 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg
 }
 
 
-void calculate_ug_chaining(Candidates_list* candidates, overlap_region_alloc* overlap_list, kvec_t_u64_warp* chain_idx,
-                                          uint64_t readID, ma_utg_v *ua, double band_width_threshold, int add_beg_end, overlap_region* f_cigar, long long mz_occ, double mz_rate)
-{
-    long long i = 0;
-    uint64_t current_ID;
-    uint64_t current_stand;
-
-    if (candidates->length == 0)
-    {
-        return;
-    }
-
-    long long sub_region_beg;
-    long long sub_region_end;
-	long long chain_len;
-
-    clear_fake_cigar(&((*f_cigar).f_cigar));
-
-    i = 0;
-    while (i < candidates->length)
-    {
-        chain_idx->a.n = 0;
-        current_ID = candidates->list[i].readID;
-        current_stand = candidates->list[i].strand;
-
-        ///reference read
-        (*f_cigar).x_id = readID;
-        (*f_cigar).x_pos_strand = current_stand;
-        ///query read
-        (*f_cigar).y_id = current_ID;
-        ///here the strand of query is always 0
-        (*f_cigar).y_pos_strand = 0;  
-
-        sub_region_beg = i;
-        sub_region_end = i;
-        i++;
-
-        while (i < candidates->length 
-        && 
-        current_ID == candidates->list[i].readID
-        &&
-        current_stand == candidates->list[i].strand)
-        {
-            sub_region_end = i;
-            i++;
-        }
-
-        if ((*f_cigar).x_id == (*f_cigar).y_id)
-        {
-            continue;
-        }
-
-		chain_len = chain_DP(candidates->list + sub_region_beg,
-				sub_region_end - sub_region_beg + 1, &(candidates->chainDP), f_cigar, band_width_threshold,
-				50, ua->a[(*f_cigar).x_id].len, ua->a[(*f_cigar).y_id].len);
-		
-		
-		// if ((*f_cigar).x_id != (*f_cigar).y_id)
-		if ((*f_cigar).x_id != (*f_cigar).y_id && chain_len > mz_occ*mz_rate)
-        {
-			append_utg_inexact_overlap_region_alloc(overlap_list, f_cigar, ua, add_beg_end);
-        }
-    }
-}
-
-void ha_get_ug_candidates(ha_abuf_t *ab, int64_t rid, ma_utg_t *u, ma_utg_v *ua, overlap_region_alloc *overlap_list, Candidates_list *cl, double bw_thres, int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag,
-kvec_t_u64_warp* chain_idx, void *ha_flt_tab, ha_pt_t *ha_idx, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct, double chain_match_rate)
-{
-	uint32_t i;
-	uint64_t k, l;
-
-	// prepare
-    clear_Candidates_list(cl);
-    clear_overlap_region_alloc(overlap_list);
-	ab->mz.n = 0, ab->n_a = 0;
-
-	// get the list of anchors
-	//should use the new version...
-	///ha_sketch_query(u->s, u->len, asm_opt.mz_win, asm_opt.k_mer_length, 0, !(asm_opt.flag & HA_F_NO_HPC), &ab->mz, ha_flt_tab, k_flag, dbg_ct);
-	// minimizer of queried read
-	if (ab->mz.m > ab->old_mz_m) {
-		ab->old_mz_m = ab->mz.m;
-		REALLOC(ab->seed, ab->old_mz_m);
-	}
-	for (i = 0, ab->n_a = 0; i < ab->mz.n; ++i) {
-		int n;
-		ab->seed[i].a = ha_pt_get(ha_idx, ab->mz.a[i].x, &n);
-		ab->seed[i].n = n;
-		ab->n_a += n;
-	}
-	if (ab->n_a > ab->m_a) {
-		ab->m_a = ab->n_a;
-		kroundup64(ab->m_a);
-		REALLOC(ab->a, ab->m_a);
-	}
-	for (i = 0, k = 0; i < ab->mz.n; ++i) {
-		int j;
-		///z is one of the minimizer
-		ha_mz1_t *z = &ab->mz.a[i];
-		seed1_t *s = &ab->seed[i];
-		for (j = 0; j < s->n; ++j) {
-			const ha_idxpos_t *y = &s->a[j];
-			anchor1_t *an = &ab->a[k++];
-			uint8_t rev = z->rev == y->rev? 0 : 1;
-			an->other_off = y->pos;
-			an->self_off = rev? u->len - 1 - (z->pos + 1 - z->span) : z->pos;
-			an->cnt = 1;
-			an->srt = (uint64_t)y->rid<<33 | (uint64_t)rev<<32 | an->other_off;
-		}
-	}
-
-	// sort anchors
-	radix_sort_ha_an1(ab->a, ab->a + ab->n_a);
-	for (k = 1, l = 0; k <= ab->n_a; ++k) {
-		if (k == ab->n_a || ab->a[k].srt != ab->a[l].srt) {
-			if (k - l > 1)
-				radix_sort_ha_an2(ab->a + l, ab->a + k);
-			l = k;
-		}
-	}
-
-
-	// copy over to _cl_
-	if (ab->m_a >= (uint64_t)cl->size) {
-		cl->size = ab->m_a;
-		REALLOC(cl->list, cl->size);
-	}
-	for (k = 0; k < ab->n_a; ++k) {
-		k_mer_hit *p = &cl->list[k];
-		p->readID = ab->a[k].srt >> 33;
-		p->strand = ab->a[k].srt >> 32 & 1;
-		p->offset = ab->a[k].other_off;
-		p->self_offset = ab->a[k].self_off;
-		p->cnt = 1;
-	}
-	cl->length = ab->n_a;
-
-	calculate_ug_chaining(cl, overlap_list, chain_idx, rid, ua, bw_thres, keep_whole_chain, f_cigar, ab->mz.n, chain_match_rate);
-
-	#if 0
-	if (overlap_list->length > 0) {
-		fprintf(stderr, "B\t%ld\t%ld\t%d\n", (long)rid, (long)overlap_list->length, rlen);
-		for (int i = 0; i < (int)overlap_list->length; ++i) {
-			overlap_region *r = &overlap_list->list[i];
-			fprintf(stderr, "C\t%d\t%d\t%d\t%c\t%d\t%ld\t%d\t%d\t%c\t%d\t%d\n", (int)r->x_id, (int)r->x_pos_s, (int)r->x_pos_e, "+-"[r->x_pos_strand],
-					(int)r->y_id, (long)Get_READ_LENGTH(R_INF, r->y_id), (int)r->y_pos_s, (int)r->y_pos_e, "+-"[r->y_pos_strand], (int)r->shared_seed, ha_ov_type(r, rlen));
-		}
-	}
-	#endif
-
-	if ((int)overlap_list->length > max_n_chain) {
-		int32_t w, n[4], s[4];
-		n[0] = n[1] = n[2] = n[3] = 0, s[0] = s[1] = s[2] = s[3] = 0;
-		ks_introsort_or_ss(overlap_list->length, overlap_list->list);
-		for (i = 0; i < (uint32_t)overlap_list->length; ++i) {
-			const overlap_region *r = &overlap_list->list[i];
-			w = ha_ov_type(r, u->len);
-			++n[w];
-			if ((int)n[w] == max_n_chain) s[w] = r->shared_seed;
-		}
-		if (s[0] > 0 || s[1] > 0 || s[2] > 0 || s[3] > 0) {
-			for (i = 0, k = 0; i < (uint32_t)overlap_list->length; ++i) {
-				overlap_region *r = &overlap_list->list[i];
-				w = ha_ov_type(r, u->len);
-				if (r->shared_seed >= s[w]) {
-					if ((uint32_t)k != i) {
-						overlap_region t;
-						t = overlap_list->list[k];
-						overlap_list->list[k] = overlap_list->list[i];
-						overlap_list->list[i] = t;
-					}
-					++k;
-				}
-			}
-			overlap_list->length = k;
-		}
-	}
-
-	///ks_introsort_or_xs(overlap_list->length, overlap_list->list);
-}
-
 void lable_matched_ovlp(overlap_region_alloc* overlap_list, ma_hit_t_alloc* paf)
 {
 	uint64_t j = 0, inner_j = 0;
@@ -400,15 +205,14 @@ void lable_matched_ovlp(overlap_region_alloc* overlap_list, ma_hit_t_alloc* paf)
 
 
 void ha_get_candidates_interface(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_region_alloc *overlap_list, overlap_region_alloc *overlap_list_hp, Candidates_list *cl, double bw_thres, 
-								 int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* chain_idx, ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, overlap_region* f_cigar,
-								 kvec_t_u64_warp* dbg_ct, st_mt_t *sp)
+int max_n_chain, int keep_whole_chain, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* chain_idx, ma_hit_t_alloc* paf, ma_hit_t_alloc* rev_paf, overlap_region* f_cigar, kvec_t_u64_warp* dbg_ct)
 {
 	extern void *ha_flt_tab;
 	extern ha_pt_t *ha_idx;
 	extern void *ha_flt_tab_hp;
 	extern ha_pt_t *ha_idx_hp;
 
-	ha_get_new_candidates(ab, rid, ucr, overlap_list, cl, bw_thres, max_n_chain, keep_whole_chain, k_flag, chain_idx, ha_flt_tab, ha_idx, f_cigar, dbg_ct, sp);
+	ha_get_new_candidates(ab, rid, ucr, overlap_list, cl, bw_thres, max_n_chain, keep_whole_chain, k_flag, chain_idx, ha_flt_tab, ha_idx, f_cigar, dbg_ct);
 
 	if(ha_idx_hp)
 	{
@@ -438,7 +242,7 @@ void ha_get_candidates_interface(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overl
 		overlap_list->length = k;
 
 
-		ha_get_new_candidates(ab, rid, ucr, overlap_list_hp, cl, bw_thres, max_n_chain, keep_whole_chain, k_flag, chain_idx, ha_flt_tab_hp, ha_idx_hp, f_cigar, dbg_ct, sp);	
+		ha_get_new_candidates(ab, rid, ucr, overlap_list_hp, cl, bw_thres, max_n_chain, keep_whole_chain, k_flag, chain_idx, ha_flt_tab_hp, ha_idx_hp, f_cigar, dbg_ct);	
 		
 		if(overlap_list->length + overlap_list_hp->length > overlap_list->size)
 		{
@@ -498,4 +302,4 @@ void ha_get_candidates_interface(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overl
 void ha_sort_list_by_anchor(overlap_region_alloc *overlap_list)
 {
 	ks_introsort_or_xs(overlap_list->length, overlap_list->list);
-}
+}

code_of_conduct.md

@@ -1,30 +0,0 @@
-## Contributor Code of Conduct
-
-As contributors and maintainers of this project, we pledge to respect all
-people who contribute through reporting issues, posting feature requests,
-updating documentation, submitting pull requests or patches, and other
-activities.
-
-We are committed to making participation in this project a harassment-free
-experience for everyone, regardless of level of experience, gender, gender
-identity and expression, sexual orientation, disability, personal appearance,
-body size, race, age, or religion.
-
-Examples of unacceptable behavior by participants include the use of sexual
-language or imagery, derogatory comments or personal attacks, trolling, public
-or private harassment, insults, or other unprofessional conduct.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct. Project maintainers or
-contributors who do not follow the Code of Conduct may be removed from the
-project team.
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by opening an issue or contacting the maintainer via email.
-
-This Code of Conduct is adapted from the [Contributor Covenant][cc], [version
-1.0.0][v1].
-
-[cc]: http://contributor-covenant.org/
-[v1]: http://contributor-covenant.org/version/1/0/0/

docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

@@ -1,257 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import sys
-import os
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-    'sphinx.ext.todo',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.ifconfig',
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix of source filenames.
-source_suffix = '.rst'
-
-# The encoding of source files.
-#source_encoding = 'utf-8-sig'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = u'hifiasm'
-copyright = u'2021, Haoyu Cheng, Heng Li'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = '0.16.0-r369'
-# The full version, including alpha/beta/rc tags.
-release = '0.16.0'
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#today = ''
-# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-exclude_patterns = []
-
-# The reST default role (used for this markup: `text`) to use for all
-# documents.
-#default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-html_theme = 'default'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
-
-#  Build using the RTD theme, if not on RTD.
-#    https://read-the-docs.readthedocs.org/en/latest/theme.html
-#    https://github.com/snide/sphinx_rtd_theme
-#
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
-    html_theme_path = [ "/usr/local/lib/python2.7/site-packages", ]
-
-
-# The name for this set of Sphinx documents.  If None, it defaults to
-# "<project> v<release> documentation".
-#html_title = None
-
-# A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#html_logo = None
-
-# The name of an image file (within the static path) to use as favicon of the
-# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-#html_favicon = None
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#html_extra_path = []
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#html_additional_pages = {}
-
-# If false, no module index is generated.
-#html_domain_indices = True
-
-# If false, no index is generated.
-#html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-#html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it.  The value of this option must be the
-# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'hifiasm-doc'
-
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-  ('index', 'hifiasm.tex', u'hifiasm Documentation',
-   u'Haoyu Cheng, Heng Li', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# If true, show page references after internal links.
-#latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_domain_indices = True
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'hifiasm', u'hifiasm Documentation',
-     [u'Haoyu Cheng, Heng Li'], 1)
-]
-
-# If true, show URL addresses after external links.
-#man_show_urls = False
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-  ('index', 'hifiasm', u'hifiasm Documentation',
-   u'Haoyu Cheng, Heng Li', 'hifiasm', 'One line description of project.',
-   'Miscellaneous'),
-]
-
-# Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
-
-# If false, no module index is generated.
-#texinfo_domain_indices = True
-
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False

docs/source/faq.rst

@@ -1,123 +0,0 @@
-
-.. _faq:
-
-Hifiasm FAQ
-===========
-
-
-.. contents::
-  :local:
-
-
-How do I get contigs in FASTA?
--------------------------------------
-    The FASTA file can be produced from GFA as follows:
-    ::
-
-        awk '/^S/{print ">"$2;print $3}' test.p_ctg.gfa > test.p_ctg.fa
-
-Which types of assemblies should I use?
-----------------------------------------
-    If parental data is available, ``*dip.hap*.p_ctg.gfa`` produced in trio-binning mode should be always preferred. Otherwise if Hi-C data is available, ``*hic.hap*.p_ctg.gfa`` produced in Hi-C mode is the best choice. Both trio-binning mode and Hi-C mode generate fully-phased assemblies. 
-
-    If you only have HiFi reads, hifiasm in default outputs ``*bp.hap*.p_ctg.gfa``. The primary/alternate assemblies can be also produced by using ``--primary``. All these HiFi-only assemblies are not fully-phased. See `blog <https://lh3.github.io/2021/04/17/concepts-in-phased-assemblies>`_ here for more details.
-
-Are inbred/homozygous genomes supported?
---------------------------------------------------------------------------
-
-    Yes, please use the ``-l0`` option to disable purge duplication step.
-
-Are diploid genomes supported?
--------------------------------------
-    Yes, most modules of hifiasm are designed for diploid samples, including purge duplication step, partially phased assembly and fully-phased assembly with trio-binning or Hi-C.
-
-Are polyploid genomes supported?
--------------------------------------
-
-    The ``*r_utg.gfa`` and ``*p_utg.gfa`` are lossless so that they also work for polyploid genomes. However, currently the contig-generation modules of hifiasm are designed for diploid samples, which means both the partially phased assembly and the fully-phased assembly does not directly support polyploid genomes. If it is set to >2, the quality of primary assembly for polyploid genomes might be improved. Please use primary assembly for polyploid samples and run multiple rounds of purging steps using third-party tools such as purge_dups. 
-
-Why one Hi-C integrated assembly is larger than another one?
-------------------------------------------------------------
-
-    For some samples like human male, the paternal haplotype should be larger than the maternal haplotype. However, if one assembly is much larger than another one, it should be the issues of hifiasm. To fix it, please set smaller value for ``-s`` (default: 0.55). 
-
-    Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. For instance, hifiasm prints the following information during assembly: 
-    ::
-
-        [M::purge_dups] homozygous read coverage threshold: 36
-
-    In this example, hifiasm identifies the coverage threshold for homozygous reads as ``36``. If it is significantly smaller than the homozygous coverage peak, hifiasm will generate two unbalanced assemblies. In this case, please set ``--hom-cov`` to homozygous coverage peak. Please note that tuning ``--hom-cov`` may affect ``*p_utg*gfa`` so that ``*hic*.bin`` should be deleted. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically.
-
-    
-
-For Hi-C integrated assembly, why the assembly size of both haplotypes are much larger than the estimated genome size?
-------------------------------------------------------------------------------------------------------------------------------
-    It is likely that hifiasm misidentifies coverage threshold for homozygous reads. Hifiasm prints the following information for debugging:
-    ::
-
-        [M::stat] # heterozygous bases: 645155110; # homozygous bases: 1495396634
-
-    If most bases of a diploid sample are homozygous, the coverage threshold is wrongly determined by hifiasm. For instance, hifiasm prints the following information during assembly: 
-    ::
-
-        [M::purge_dups] homozygous read coverage threshold: 36
-
-    In this example, hifiasm identifies the coverage threshold for homozygous reads as ``36``. If it is much smaller than homozygous coverage peak, hifiasm thinks most reads are homozygous and assign them to both assemblies, making both of them much larger than the estimated haploid genome size. In this case, please set ``--hom-cov`` to homozygous coverage peak. Please note that tuning ``--hom-cov`` may affect ``*p_utg*gfa`` so that ``*hic*.bin`` should be deleted. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically.
-
-
-.. _hic-iss:
-
-How can I tweak parameters to improve Hi-C integrated assembly?
----------------------------------------------------------------
-    Compared with the HiFi-only assembly or the trio-binning assembly, the Hi-C integrated assembly is a little bit more complex so that you need to take care of the results. See `Why one Hi-C integrated assembly is larger than another one?`_ and `For Hi-C integrated assembly, why the assembly size of both haplotypes are much larger than the estimated genome size?`_ for details on how to fix potential issues.
-
-    There are several other options that may affect the Hi-C integrated assembly. Increasing the values of ``--n-weight``, ``--n-perturb`` and ``--f-perturb`` may improve phasing results but takes longer time. However, tuning ``--l-msjoin`` is tricky. All these options do not affect ``*p_utg*gfa`` so that ``*hic*.bin`` can be reused.
-
-.. _p-large:
-
-Why the size of primary assembly or partially phased assembly is much larger than the estimated genome size? 
----------------------------------------------------------------------------------------------------------------
-    It could be because the estimated genome size is incorrect. Another possibility is that hifiasm does not perform enough purging. Setting smaller value for ``-s`` (default: 0.55) or turning ``--hom-cov`` should be helpful. See :ref:`loginter` for more details.
-
-
-.. _p-hamming:
-
-Why the hamming error rate or the swith error rate of trio-binning assembly is very high?
----------------------------------------------------------------------------------------------------------------
-    In rare cases, a potential issue is that a few contigs may misjoin two haplotypes. For example, half of a contig come from mother while another half come from father. Such misjoined contigs can be fixed by manually breaking. The coordinates of problematic regions can be found by A-lines in GFA file or ``yak trioeval -e`` (see `issue 37 <https://github.com/chhylp123/hifiasm/issues/37>`_ for more details). However, if there are many misjoined contigs or the switch/hamming error rate reported by ``yak trioeval`` is very high, users should check if the parental data is correct (see `issue 130 <https://github.com/chhylp123/hifiasm/issues/130#issuecomment-862347943>`_ for more details).
-
-    Another possibility is that there are some unitigs in unitig graph misjoining two haplotypes. Such problematic unitigs might be ignored by the graph-binning strategy. Set smaller value for ``--t-occ`` forcedly remove unitig including unexpected haplotype-specific reads.
-
-Why does hifiasm stuck or crash? 
--------------------------------------
-    In most cases, it is caused by the low quality HiFi reads. A good HiFi dataset should have a k-mer plot like `issue10 <https://github.com/chhylp123/hifiasm/issues/10#issuecomment-616213684>`_ or `issue49 <https://github.com/chhylp123/hifiasm/issues/49#issue-729106823>`_. In contrast, low quality HiFi data often lead to weird k-mer plot like `issue93 <https://github.com/chhylp123/hifiasm/issues/93#issue-852259042>`_. Such weird k-mer plots usually indicate insufficient coverage or presence of contaminants. See :ref:`loginter` for more details. If the HiFi data look fine, please raise an issue at the `issue page <https://github.com/chhylp123/hifiasm/issues>`_. 
-
-What's the usage of different bin files in hifiasm?
-----------------------------------------------------
-    ``*ec.bin``, ``*ovlp.reverse.bin`` and ``*ovlp.source.bin`` save the results of error correction step. ``*hic*bin`` saves the results of Hi-C alignment. Please note that ``*hic*.bin`` should be deleted when tuning any parameters affecting ``*p_utg*gfa``. There are several parameters which does not change ``*p_utg*gfa``, including ``-s``, ``--seed``, ``--n-weight``, ``--n-perturb``, ``--f-perturb`` and ``--l-msjoin``. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. 
-
-Can I generate HiFi-only assembly first, and then add Hi-C or trio data later?
-----------------------------------------------------------------------------------------
-    Yes, the HiFi-only assembly, Hi-C phased assembly and trio-binning assembly share the same ``*ec.bin``, ``*ovlp.reverse.bin`` and ``*ovlp.source.bin``.
-
-What is the minimum read coverage required for hifiasm?
--------------------------------------------------------
-    Usually >=13x HiFi reads per haplotype. Higher coverage might be able to improve the contiguity of assembly.
-
-Why the primary assembly is more contiguous than the fully-phased assemblies and the partially phased assemblies (i.e. ``*.hap*.p_ctg.gfa``)?
-----------------------------------------------------------------------------------------------------------------------------------------------------
-
-    For diploid samples, primary assembly usually has greater N50 but at the expense of highly fragmented alternate assembly. From the method view, the primary assembly has an extra joining step, which joins two haplotypes to make primary assembly more contiguous.
-
-    When producing fully-phased assemblies and partially phased assemblies, hifiasm is designed to keep both haplotypes contiguous. It is important for many downstream applications like SV calling.
-
-My assembly is fragmented or not contiguous enough, how do I improve it?
---------------------------------------------------------------------------
-
-    Raising ``-D`` or ``-N`` may improve the resolution of repetitive regions but takes longer time. These two options affect all types of assemblies and usually do not have a negative impact on the assembly quality. In contrast, ``--purge-max`` only affects primary assembly. Setting larger value for ``--purge-max`` makes primary assembly more contiguous but may collapse repeats or segmental duplications.
-
-    If the assembly is too fragmented, users should check if HiFi data is good enough. See `Why does hifiasm stuck or crash?`_ for details. 
-
-How do I avoid misassemblies?
---------------------------------------------------------------------------
-    Set smaller value for ``--purge-max``, ``-s`` and ``-O``, or use the ``-u`` option.

docs/source/hic-assembly.rst

@@ -1,16 +0,0 @@
-
-.. _hic-assembly:
-
-Hi-C Integrated Assembly
-========================
-
-Hifiasm can generate a pair of haplotype-resolved assemblies with paired-end Hi-C reads::
-
- hifiasm -o NA12878.asm -t32 --h1 read1.fq.gz --h2 read2.fq.gz HiFi-reads.fq.gz
-
-In this mode, each contig is supposed to be a haplotig, which by definition comes from one parental haplotype only. Hifiasm often puts all contigs from the same parental chromosome in one assembly. It has cleanly separated chrX and chrY for a human male dataset. Nonetheless, phasing across centromeres is challenging. Hifiasm is often able to phase entire chromosomes but it may fail in rare cases. Also, contigs from different parental chromosomes are randomly mixed as it is just not possible to phase across chromosomes with Hi-C. Hifiasm does not perform scaffolding for now. You need to run a standalone scaffolder such as SALSA or 3D-DNA to scaffold phased haplotigs. 
-
-
-For samples with high heterozygosity rate, a common issue is that one assembly is much larger than another one. To fix this issue, please set smaller value for ``-s`` (default: 0.55). Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. In this case, please set ``--hom-cov`` to homozygous coverage peak. See :ref:`hic-iss` for more details.
-
-At the first run, hifiasm saves the alignment of Hi-C reads to disk as ``*hic*.bin``. It reuses the saved results to avoid Hi-C alignment next time. Please note that ``*hic*.bin`` should be deleted when tuning any parameters affecting ``*p_utg*gfa``. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. There are several parameters which do not change ``*p_utg*gfa``, including ``-s``, ``--seed``, ``--n-weight``, ``--n-perturb``, ``--f-perturb`` and ``--l-msjoin``.

docs/source/index.rst

@@ -1,80 +0,0 @@
-Hifiasm
-=======
-
-.. toctree::
-   :hidden:
-
-   pa-assembly
-   trio-assembly
-   hic-assembly
-   interpreting-output
-   faq
-   parameter-reference
-
-
-
-
-`Hifiasm <https://github.com/chhylp123/hifiasm>`_ is a fast haplotype-resolved de novo assembler for PacBio HiFi reads. It can assemble a human genome in several hours and assemble a ~30Gb California redwood genome in a few days. Hifiasm emits partially phased assemblies of quality competitive with the best assemblers. Given parental short reads or Hi-C data, it produces arguably the best haplotype-resolved assemblies so far.
-
-Publications
-============
-
-Hifiasm
-  Haoyu Cheng, Gregory T. Concepcion, Xiaowen Feng, Haowen Zhang & Heng Li.
-  `Haplotype-resolved de novo assembly using phased assembly graphs with hifiasm <https://doi.org/10.1038/s41592-020-01056-5>`_. Nature Methods. (2021).
-
-Install
-=======
-The easiest way to get started is to download a `release <https://github.com/chhylp123/hifiasm/releases>`_. Please report any issues on `github issues <https://github.com/chhylp123/hifiasm/issues>`_ page.
-
-In addition, the latest unreleased version can be found from github:
-
-::
-
-  git clone https://github.com/chhylp123/hifiasm
-  cd hifiasm && make
-
-Another way is to install hifiasm via `bioconda <https://anaconda.org/bioconda/hifiasm>`_:
-
-::
-
-  conda install -c bioconda hifiasm
-
-Assembly Concepts 
-=================
-There are different types of assemblies which are commonly used in practice (see 
-`details <https://lh3.github.io/2021/04/17/concepts-in-phased-assemblies>`_). 
-Hifiasm produces primary/alternate assemblies or partially phased assemblies 
-only with HiFi reads. Given Hi-C data or trio-binning data, hifiasm produces 
-contiguous fully-phased assemblies, i.e. haplotype-resolved assemblies.
-
-Why Hifiasm?
-============
-* Hifiasm delivers high-quality assemblies. It tends to generate longer contigs
-  and resolve more segmental duplications than other assemblers.
-
-* Given Hi-C reads or short reads from the parents, hifiasm can produce overall the best
-  haplotype-resolved assembly so far. It is the assembler of choice by the 
-  `Human Pangenome Project <https://humanpangenome.org/>`_ for the first batch of samples.
-
-* Hifiasm can purge duplications between haplotigs without relying on
-  third-party tools such as purge\_dups. Hifiasm does not need polishing tools
-  like pilon or racon, either. This simplifies the assembly pipeline and saves
-  running time.
-
-* Hifiasm is fast. It can assemble a human genome in half a day and assemble a
-  ~30Gb redwood genome in three days. No genome is too large for hifiasm.
-
-* Hifiasm is trivial to install and easy to use. It does not required Python,
-  R or C++11 compilers, and can be compiled into a single executable. The
-  default setting works well with a variety of genomes.
-
-Learn
-=====
-
-*  :ref:`HiFi-only Assembly        <pa-assembly>` - Assembling HiFi reads without additional data types
-*  :ref:`Trio-binning Assembly     <trio-assembly>` - Producing fully phased assemblies with HiFi and trio-binning data
-*  :ref:`Hi-C Integrated Assembly   <hic-assembly>`   - Producing fully phased assemblies with HiFi and Hi-C data
-*  :ref:`Hifiasm Output           <interpreting-output>`   - Interpreting results
-*  :ref:`Hifiasm FAQ           <faq>`   - Frequently asked questions
-*  :ref:`Hifiasm Parameters <parameter-reference>` - Parameter reference of hifiasm

docs/source/interpreting-output.rst

@@ -1,102 +0,0 @@
-
-.. _interpreting-output:
-
-Hifiasm Output 
-===============
-
-.. _outfile:
-
-Output files
----------------------------------------
-
-In general, hifiasm generates the following assembly graphs in the GFA format:
-
-* ```prefix`.r_utg.gfa``: haplotype-resolved raw unitig graph. This graph keeps all haplotype information.
-* ```prefix`.p_utg.gfa``: haplotype-resolved  processed  unitig  graph  without small bubbles. Small bubbles might be caused by somatic mutations or noise in data, which are not the real haplotype information. Hifiasm automatically pops such small bubbles based on coverage. The option ``--hom-cov`` affects the result. See :ref:`homozygous coverage setting <homcov>` for more details. In addition, the option ``-p`` forcedly pops bubbles.
-* ```prefix`.p_ctg.gfa``: assembly graph of primary contigs. This graph includes a complete assembly with long stretches of phased blocks.
-* ```prefix`.a_ctg.gfa``: assembly graph of alternate contigs. This graph consists of all contigs that are discarded in primary contig graph.
-* ```prefix`.*hap*.p_ctg.gfa``: phased contig graph. This graph keeps the phased contigs.
-
-
-Hifiasm outputs ``*.r_utg.gfa`` and ``*.p_utg.gfa`` in any cases. Specifically, hifiasm outputs the following assembly graphs in trio-binning mode:
-
-* ```prefix`.dip.hap1.p_ctg.gfa``: fully phased paternal/haplotype1 contig graph keeping the phased paternal/haplotype1 assembly.
-* ```prefix`.dip.hap2.p_ctg.gfa``: fully phased maternal/haplotype2 contig graph keeping the phased maternal/haplotype2 assembly.
-
-With Hi-C partition options, hifiasm outputs:
-
-* ```prefix`.hic.p_ctg.gfa``: assembly graph of primary contigs.
-* ```prefix`.hic.hap1.p_ctg.gfa``: fully phased contig graph of haplotype1 where each contig is fully phased.
-* ```prefix`.hic.hap2.p_ctg.gfa``: fully phased contig graph of haplotype2 where each contig is fully phased.
-* ```prefix`.hic.a_ctg.gfa`` (optional with ``--primary``): assembly graph of alternate contigs.
-
-Hifiasm generates the following assembly graphs only with HiFi reads in default:
-
-* ```prefix`.bp.p_ctg.gfa``: assembly graph of primary contigs.
-* ```prefix`.bp.hap1.p_ctg.gfa``: partially phased contig graph of haplotype1.
-* ```prefix`.bp.hap2.p_ctg.gfa``: partially phased contig graph of haplotype2.
-
-If the option ``--primary`` or ``-l0`` is specified, hifiasm outputs:
-
-* ```prefix`.p_ctg.gfa``: assembly graph of primary contigs.
-* ```prefix`.a_ctg.gfa``: assembly graph of alternate contigs.
-
-For each graph, hifiasm also outputs a simplified version (``*noseq*gfa``) without sequences for the ease of visualization. The coordinates of low quality regions are written to ``*lowQ.bed`` in BED format.
-The concepts of different types of assemblies can be found `here <https://lh3.github.io/2021/04/17/concepts-in-phased-assemblies>`_.
-
-.. _outformat:
-
-Output file formats
----------------------------------------
-Hifiasm broadly follows the specification for `GFA 1.0 <https://github.com/GFA-spec/GFA-spec/blob/master/GFA1.md>`_. There are several fields that are specifically used by hifiasm. For ``S`` segment line:
-
-* ``rd:i:``: read coverage. It is calculated by the reads coming from the same contig/unitig.
-
-Hifiasm outputs ``A`` lines including the information of reads which are used to construct contig/unitig. Each ``A`` line is plain-text, tab-separated, and the columns appear in the following order:
-
-.. list-table::
-   :widths: 10 25 50
-   :header-rows: 1
-
-   * - Col
-     - Type
-     - Description
-   * - 1
-     - string
-     - Should be always ``A``
-   * - 2
-     - string
-     - Contig/unitig name
-   * - 3
-     - int
-     - Contig/unitig start coordinate of subregion constructed by read
-   * - 4
-     - char
-     - Read strand: "+" or "-"
-   * - 5
-     - string
-     - Read name
-   * - 6
-     - int
-     - Read start coordinate of subregion which is used to construct contig/unitig
-   * - 7
-     - int
-     - Read end coordinate of subregion which is used to construct contig/unitig
-   * - 8
-     - id:i:int
-     - Read ID
-   * - 9
-     - HG:A:char
-     - Haplotype status of read. ``HG:A:a``, ``HG:A:p``, ``HG:A:m`` indicate read is non-binnable, father/hap1-specific and mother/hap2-specific, respectively.
-
-.. _loginter:
-
-Hifiasm log interpretation
----------------------------------------
-Hifiasm prints several information for quick debugging, including:
-
-.. _homcov:
-
-* k-mer plot: showing how many k-mers appear a certain number of times. For homozygous samples, there should be one peak around read coverage. For heterozygous samples, there should two peaks, where the smaller peak is around the heterozygous read coverage and the larger peak is around the homozygous read coverage. For example, `issue10 <https://github.com/chhylp123/hifiasm/issues/10#issuecomment-616213684>`_ indicates the heterozygous read coverage and the homozygous read coverage are 28 and 57, respectively. `Issue49 <https://github.com/chhylp123/hifiasm/issues/49#issue-729106823>`_ is another good example. Weird k-mer plot like `issue93 <https://github.com/chhylp123/hifiasm/issues/93#issue-852259042>`_ is often caused by insufficient coverage or presence of contaminants. 
-* homozygous coverage: coverage threshold for homozygous reads. Hifiasm prints it as: ``[M::purge_dups] homozygous read coverage threshold: X``. If it is not around homozygous coverage, the final assembly might be either too large or too small. To fix this issue, please set ``--hom-cov`` to homozygous coverage.  
-* number of het/hom bases: how many bases in unitig graph are heterozygous and homozygous during Hi-C phased assembly. Hifiasm prints it as: ``[M::stat] # heterozygous bases: X; # homozygous bases: Y``. Given a heterozygous sample, if there are much more homozygous bases than heterozygous bases, hifiasm fails to identify correct coverage threshold for homozygous reads. In this case, please set ``--hom-cov`` to homozygous coverage. 

docs/source/pa-assembly.rst

@@ -1,47 +0,0 @@
-
-.. _pa-assembly:
-
-HiFi-only Assembly
-==================
-
-A typical hifiasm command line looks like::
-
- hifiasm -o NA12878.asm -t 32 NA12878.fq.gz
-
-where ``NA12878.fq.gz`` provides the input reads, ``-t`` sets the number of CPUs in
-use and ``-o`` specifies the prefix of output files. Input sequences should be FASTA 
-or FASTQ format, uncompressed or compressed with gzip (.gz). The quality scores of reads 
-in FASTQ are ignored by hifiasm. Hifiasm outputs assemblies in `GFA <https://github.com/pmelsted/GFA-spec/blob/master/GFA-spec.md>`_ format.
-
-At the first run, hifiasm saves corrected reads and overlaps to disk as ``NA12878.asm.*.bin``. It reuses the saved results to avoid the time-consuming all-vs-all overlap calculation next time. You may specify ``-i`` to ignore precomputed overlaps and redo overlapping from raw reads. You can also dump error corrected reads in FASTA and read overlaps in PAF with::
-
- hifiasm -o NA12878.asm -t 32 --write-paf --write-ec /dev/null
-
-Hifiasm purges haplotig duplications by default. For inbred or homozygous genomes, you may disable purging with option ``-l0``. Old HiFi reads may contain short adapter sequences at the ends of reads. You can specify ``-z20`` to trim both ends of reads by 20bp. For small genomes, use ``-f0`` to disable the initial bloom filter which takes 16GB memory at the beginning. For genomes much larger than human, applying ``-f38`` or even ``-f39`` is preferred to save memory on k-mer counting.
-
-
-Produce two partially phased assemblies
----------------------------------------
-
-
-Since v0.15, hifiasm produces two sets of partially phased contigs in default like::
-
- hifiasm -o NA12878.asm -t 32 NA12878.fq.gz
-
-In this example, the partially phased contigs are written to ``NA12878.asm.bp.hap*.p_ctg.gfa``. 
-This pair of files can be thought to represent the two haplotypes in a diploid genome, though with occasional switch errors. The frequency of switches is determined by the heterozygosity of the input sample. Hifiasm also writes the primary contigs to ``NA12878.asm.bp.p_ctg.gfa``. 
-
-For samples with high heterozygosity rate, a common issue is that one set of partially phased contigs is much larger than another set. To fix this issue, please set smaller value for ``-s`` (default: 0.55). Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. 
-In this case, please set ``--hom-cov`` to homozygous coverage. See :ref:`p-large` for more details.
-
-
-Produce primary/alternate assemblies
-------------------------------------
-
-To get primary/alternate assemblies, the option ``--primary`` should be set::
-
- hifiasm -o NA12878.asm --primary -t 32 NA12878.fq.gz
-
-The primary contigs and the alternate contigs are written to ``NA12878.asm.p_ctg.gfa`` and ``NA12878.asm.a_ctg.gfa``, respectively. For inbred or homozygous genomes, the primary/alternate assemblies can be also produced by ``-l0``. Similarly, turning ``-s`` or ``--hom-cov`` should 
-be helpful if the primary assembly is too large. See :ref:`p-large` for more details.
-

docs/source/parameter-reference.rst

@@ -1,298 +0,0 @@
-
-.. _parameter-reference:
-
-Hifiasm Parameter Reference
-============================
-
-Synopsis
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Assembly only with HiFi reads:
-::
-
-  hifiasm -o [prefix] -t [nThreads] [options] input1.fq [input2.fq [...]]
-
-Trio binning assembly with yak dumps:
-::
-
-  yak count -o paternal.yak -b37 [-t nThreads] [-k kmerLen] paternal.fq.gz
-  yak count -o maternal.yak -b37 [-t nThreads] [-k kmerLen] maternal.fq.gz
-  hifiasm [-o prefix] [-t nThreads] [options] -1 paternal.yak -2 maternal.yak child.hifi.fq.gz
-
-Hi-C integrated assembly:
-::
-
-  hifiasm -o [prefix] -t [nThreads] --h1 [hic_r1.fq.gz,...] --h2 [hic_r2.fq.gz,...] [options] HiFi.read.fq.gz
-
-To get detailed description of options, run:
-::
-
-  hifiasm -h
-
-or:
-::
-
-  man ./hifiasm.1
-
-
-General options
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. _oopt:
-
-**\-o <FILE=hifiasm.asm>**
-  Prefix of output files. See :ref:`outfile` and :ref:`outformat` for more details.
-
-.. _topt:
-
-**\-t <INT=1>**
-  Number of CPU threads used by hifiasm.
-
-.. _hopt:
-
-**\-h** 
-  Show help information.
-
-.. _versionopt:
-
-**\-\-version** 
-  Show version number.
-
-
-Error correction options
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. _kopt:
-
-**\-k <INT=51>**
-  K-mer length. This option must be less than 64.
-
-.. _wopt:
-
-**\-w <INT=51>**    
-  Minimizer window size.
-
-.. _fopt:
-
-**\-f <INT=37>**    
-  Number  of  bits for bloom filter; 0 to disable. This bloom filter is used to filter out singleton k-mers when counting all k-mers. It takes 2\ :sup:`(INT-3)` bytes of memory. A proper setting saves memory. ``-f37`` is recommended for human assembly. For small genomes, use ``-f0`` to disable the initial bloom filter which takes 16GB memory at the beginning. For genomes much larger than human, applying ``-f38`` or even ``-f39`` is preferred to save memory on k-mer counting.
-
-.. _Dopt:
-
-**\-D <FLOAT=5.0>**    
-  Drop k-mers occurring ``>FLOAT*coverage`` times. Hifiasm discards these high-frequency k-mers during error correction to reduce running time. The ``coverage`` is determined automatically by hifiasm based on k-mer plot, representing homozygous read coverage. Raising this option may improve the resolution of repetitive regions but takes longer time.
-
-.. _NEopt:
-
-**\-N <INT=100>**
-  Consider up to ``max(-D*coverage,-N)`` overlaps for each oriented read. The ``coverage`` is determined automatically by hifiasm based on k-mer plot, representing homozygous read coverage. Raising this option may improve the resolution of repetitive regions but takes longer time.
-
-.. _ropt:
-
-**\-r <INT=3>**
-  Rounds of haplotype-aware error correction. This option affects all outputs of hifiasm. Odd rounds of correction are preferred in practice.
-
-
-.. _zopt:
-
-**\-z <INT=0>**
-  Length  of  adapters that should be removed. This option remove ``INT`` bases from both ends of each read.  Some old HiFi reads may consist of short adapters (e.g. 20bp adapter at one end). For such data, trimming short adapters would significantly improve the assembly quality.
-
-.. _max-kocc-opt:
-
-**\-\-max-kocc <INT=2000>**
-  Employ k-mers occurring < ``INT`` times to rescue repetitive overlaps. This option may improve the resolution of repeats.
-
-
-.. _hg-size-opt:
-
-**\-\-hg-size <INT(k/m/g)>**
-  Estimated haploid genome size used for inferring read coverage. This option is used to get accurate homozygous read coverage during error correction. Common suffices are required, for example, 100m or 3g.
-
-
-.. _min-hist-cnt-opt:
-
-**\-\-min-hist-cnt <INT=5>**
-  When analyzing the k-mer spectrum, ignore counts below ``INT``. For very low coverage of HiFi data, set smaller value for this option. See `issue 45 <https://github.com/chhylp123/hifiasm/issues/49>`_ for example.
-
-
-
-Assembly options
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. _aopt:
-
-**\-a <INT=4>**
-  Rounds of assembly graph cleaning. This option is used with ``-x`` and ``-y``. Note that unlike -r, this option does not affect error corrected reads and all-to-all overlaps.
-
-
-.. _mopt:
-
-**\-m <INT=10000000>**
-  Maximal probing distance for bubble popping when generating primary/alternate contig graphs. Bubbles longer than ``INT`` bases will not be popped.
-
-.. _popt:
-
-**\-p <INT=0>**
-  Maximal probing distance for bubble popping when generating haplotype-resolved processed unitig graph without small bubbles. Bubbles longer than ``INT`` bases will not be popped. Small bubbles might be caused by somatic mutations or noise in data. Please note that hifiasm automatically pops small bubbles based on coverage, which can be tweaked by ``--hom-cov``.
-
-.. _nopt:
-
-**\-n <INT=3>**
-  A unitig is considered small if it is composed of less than ``INT`` reads. Hifiasm may try to remove small unitigs at various steps.
-
-.. _xyopt:
-
-**\-x <FLOAT1=0.8>, \-y <FLOAT2=0.2>**
-  Max and min overlap drop ratio. This option is used with ``-a``. Given a node N in the assembly graph, let max(N) be the length of the longest overlap of N. Hifiasm iteratively drops overlaps of N if their length/max(N) is below a threshold controlled by ``-x`` and ``-y``. Hifiasm applies ``-a`` rounds of short overlap removal with an increasing threshold between ``FLOAT1`` and ``FLOAT2``.
-
-.. _iopt:
-
-**\-i**
-  Ignore all bin files so that hifiasm will start again from scratch.
-
-.. _uopt:
-
-**\-u**
-  Disable post-join step for contigs which may improve N50. The post-join step of hifiasm improves contig N50 but may introduce misassemblies. 
-
-
-.. _hom-cov-opt:
-
-**\-\-hom-cov <INT>**
-  Homozygous read coverage inferred automatically in default. This option affects different types of outputs, including Hi-C phased assembly and HiFi-only assembly. For more details, see :ref:`hic-iss`, :ref:`p-large` and :ref:`loginter`.
-
-.. _pri-range-opt:
-
-**\-\-pri-range <INT1[,INT2]>**
-  Min and max coverage cutoffs of primary contigs. Keep contigs with coverage in this range at p_ctg.gfa. Inferred automatically in default. If ``INT2`` is not specified, it is set to infinity. Set -1 to disable.
-
-.. _lowQ-opt:
-
-**\-\-lowQ <INT=70>**
-  Output contig regions with ``>=INT%`` inconsistency to the bed file with suffix lowQ.bed. Set 0 to disable.
-
-.. _b-cov-opt:
-
-**\-\-b-cov <INT=0>**
-  Break contigs at potential misassemblies with ``<INT``-fold coverage. Work with ``--m-rate``. Set 0 to disable.
-
-.. _h-cov-opt:
-
-**\-\-h-cov <INT=-1>**
-  Break contigs at potential misassemblies with ``>INT``-fold coverage. Work with ``--m-rate``. Set -1 to disable.
-
-.. _m-rate-opt:
-
-**\-\-m-rate <FLOAT=0.75>**
-  Break contigs with ``<=FLOAT*coverage`` exact overlaps. Only work when ``--b-cov`` and ``--h-cov`` are specified.
-
-.. _primary-opt:
-
-**\-\-primary**
-  Output a primary assembly and an alternate assembly. Enable this option or ``-l0`` outputs a primary assembly and an alternate assembly.
-
-
-Trio-binning options
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. _1opt:
-
-**\-1 <FILE>**
-  K-mer dump generated by `yak count <https://github.com/lh3/yak>`_ from the paternal/haplotype1 reads.
-
-.. _2opt:
-
-**\-2 <FILE>**
-  K-mer dump generated by `yak count <https://github.com/lh3/yak>`_ from the maternal/haplotype2 reads.
-
-.. _3opt:
-
-**\-3 <FILE>**
-  List of paternal/haplotype1 read names.
-
-.. _4opt:
-
-**\-4 <FILE>**
-  List of maternal/haplotype2 read names.
-
-.. _cdopt:
-
-**\-c <INT1=2>, -d <INT2=5>**
-  Lower bound and upper bound of the binned k-mer's frequency. When doing trio binning, a k-mer is said to be differentiating if it occurs >= ``INT2`` times in one sample but occurs < ``INT1`` times in the other sample.
-
-
-.. _t-occ-opt:
-
-**\-\-t-occ <INT=60>**
-  Forcedly remove unitig including ``>INT`` unexpected haplotype-specific reads without considering graph topology. For more details, see :ref:`p-hamming`.
-
-
-Purge duplication options
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. _ldopt:
-
-**\-l <INT=3>**
-  Level of purge duplication. 0 to disable, 1 to only purge contained haplotigs, 2 to purge all types of haplotigs, 3 to purge all types of haplotigs in the most aggressive way. In default, 3 for non-trio assembly, 0 for trio-binning assembly. For trio-binning assembly, only level 0 and level 1 are allowed.
-
-.. _sdopt:
-
-**\-s <FLOAT=0.55>**
-  Similarity threshold for duplicate haplotigs that should be purged. In default, 0.75 for ``-l1/-l2``, 0.55 for ``-l3``. This option affects both HiFi-only assembly and Hi-C phased assembly. For more details, see :ref:`hic-iss` and :ref:`p-large`.
-
-.. _ovlpdopt:
-
-**\-O <INT=1>**
-  Min number of overlapped reads for duplicate haplotigs that should be purged.
-
-.. _purgeopt:
-
-**\-\-purge-max <INT>**
-  Coverage upper bound of purge duplication, which is inferred automatically in default. If the coverage of a contig is higher than this bound, don't apply purge duplication. Larger value makes assembly more contiguous but may collapse repeats or segmental duplications.
-
-.. _nhapopt:
-
-**\-\-n\-hap <INT=2>**
-  Assumption of haplotype number. If it is set to >2, the quality of primary assembly for polyploid genomes might be improved.
-
-
-
-Hi-C integration options
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. _h1opt:
-
-**\-\-h1 <FILEs>**
-  File names of input Hi-C R1 ``[r1_1.fq,r1_2.fq,...]``.
-
-.. _h2opt:
-
-**\-\-h2 <FILEs>**
-  File names of input Hi-C R2 ``[r2_1.fq,r2_2.fq,...]``.
-
-.. _n-weightopt:
-
-**\-\-n-weight <INT=3>**
-  Rounds of reweighting Hi-C links. Raising this option may improve phasing results but takes longer time.
-
-.. _n-perturbopt:
-
-**\-\-n-perturb <INT=10000>**
-  Rounds of perturbation. Increasing this option may improve phasing results but takes longer time.
-
-.. _f-perturbopt:
-
-**\-\-f-perturb <FLOAT=0.1>**
-  Fraction to flip for perturbation. Increasing this option may improve phasing results but takes longer time.
-
-.. _seedopt:
-
-**\-\-seed <INT=11>**
-  RNG seed.
-
-
-.. _l-msjoin:
-
-**\-\-l-msjoin <INT=500000>**
-  Detect misjoined unitigs of ``>=INT`` in size; 0 to disable.

docs/source/trio-assembly.rst

@@ -1,29 +0,0 @@
-
-.. _trio-assembly:
-
-Trio-binning Assembly
-=====================
-
-When parental short reads are available, hifiasm can also generate a pair of haplotype-resolved assemblies with trio binning. To perform such assembly, you need to count k-mers first with `yak <https://github.com/lh3/yak>`_ and then do assembly::
-
- yak count -k31 -b37 -t16 -o pat.yak paternal.fq.gz
- yak count -k31 -b37 -t16 -o mat.yak maternal.fq.gz
- hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak NA12878.fq.gz
-
-Here ``NA12878.asm.hap1.p_ctg.gfa`` and ``NA12878.asm.hap2.p_ctg.gfa`` give the assemblies for two haplotypes. In the binning mode, hifiasm does not purge haplotig duplicates by default. Because hifiasm reuses saved overlaps, you can generate both primary/alternate assemblies and trio binning assemblies with::
-
- hifiasm -o NA12878.asm --primary -t 32 NA12878.fq.gz 2> NA12878.asm.pri.log
- hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak /dev/null 2> NA12878.asm.trio.log
-
-The second command line will run much faster than the first. The phasing switch error rate and hamming error rate are able to be evaluated quickly by `yak <https://github.com/lh3/yak>`_::
-
- yak trioeval -t16 pat.yak mat.yak assembly.fa
-
-The W-line and H-line reported by ``yak trioeval`` indicate switch error rate and hamming error rate respectively::
-
- W       26714   3029448 0.008818
- H       24315   3029885 0.008025
-
-For this example, the switch error rate is 0.8818% and the hamming error rate is 0.8025%. If the hamming error rate or the swith error rate of trio-binning assembly is very high, it might be caused by hifiasm or the incorrect parental data. To fix it, see :ref:`p-hamming` for more details.
-
-

hic.h

@@ -10,7 +10,8 @@
 #define RC_2 2
 
 hc_edge* get_hc_edge(hc_links* link, uint64_t src, uint64_t dest, uint64_t dir);
-hc_edge* push_hc_edge(hc_linkeage* x, uint64_t uID, double weight, int dir, uint64_t* d);
+void push_hc_edge(hc_linkeage* x, uint64_t uID, double weight, int dir, uint64_t* d);
+void hic_analysis(ma_ug_t *ug, asg_t* read_g, hc_links* link);
 void hic_benchmark(ma_ug_t *ug, asg_t* read_g);
 
 typedef struct {
@@ -48,33 +49,6 @@ typedef struct {
     kvec_t(chain_w_type) chain_weight;
     chain_hic_warp c_w;
 } bubble_type;
-
-typedef struct {
-    int8_t *s;
-    uint64_t xs;
-    uint64_t n;
-} ps_t;
-
-typedef struct {
-	uint64_t s, e, id, len;
-} pe_hit;
-
-typedef struct {
-    kvec_t(pe_hit) a;
-    kvec_t(uint64_t) idx;
-    kvec_t(uint64_t) occ;
-    uint64_t uID_bits;
-    uint64_t pos_mode;
-} kvec_pe_hit;
-
-typedef struct{
-    kvec_t(uint8_t) vis;
-    kvec_t(uint64_t) x;
-    kvec_t(uint64_t) dis;
-    uint64_t uID_mode, uID_shift, tmp_v, tmp_d;
-}pdq;
-
-
 #define P_het(B) ((B).num.n)
 #define M_het(B) ((B).num.n + 1)
 // #define IF_BUB(ID, B) ((B).index[(ID)] < (B).num.n)
@@ -88,23 +62,11 @@ void get_bubbles(bubble_type* bub, uint64_t id, uint32_t* beg, uint32_t* sink, u
 int load_hc_links(hc_links* link, const char *fn);
 void write_hc_links(hc_links* link, const char *fn);
 void destory_bubbles(bubble_type* bub);
-void identify_bubbles(ma_ug_t* ug, bubble_type* bub, uint8_t *r_het_flag, kv_u_trans_t *ref);
+void identify_bubbles(ma_ug_t* ug, bubble_type* bub, hc_links* link);
 void resolve_bubble_chain_tangle(ma_ug_t* ug, bubble_type* bub);
 uint32_t connect_bub_occ(bubble_type* bub, uint32_t root_id, uint32_t check_het);
 void get_bub_id(bubble_type* bub, uint32_t root, uint64_t* id0, uint64_t* id1, uint32_t check_het);
 void update_bubble_chain(ma_ug_t* ug, bubble_type* bub, uint32_t is_middle, uint32_t is_end);
 void set_b_utg_weight_flag(bubble_type* bub, buf_t* b, uint32_t v, uint8_t* vis_flag, uint32_t flag, uint32_t* occ);
-void debug_gfa_space(ma_ug_t* ug, hap_cov_t *cov);
-void init_ug_idx(ma_ug_t *ug, uint64_t k, uint64_t up_bound, uint64_t low_bound, uint64_t build_idx);
-void des_ug_idx();
-uint64_t count_unique_k_mers(char *r, uint64_t len, uint64_t query, uint64_t target, uint64_t *all, uint64_t *found);
-void init_pdq(pdq* q, uint64_t utg_num);
-void destory_pdq(pdq* q);
-uint32_t check_trans_relation_by_path(uint32_t v, uint32_t w, pdq* pqv, uint32_t* path_v, buf_t *resv,
-pdq* pqw, uint32_t* path_w, buf_t *resw, asg_t *sg, uint8_t *dest, uint8_t df, uint32_t df_occ, double rate,
-long long *dis);
-void set_utg_by_dis(uint32_t v, pdq* pq, asg_t *g, kvec_t_u32_warp *res, uint32_t dis);
-void dedup_hits(kvec_pe_hit* hits, uint64_t is_dup);
-void hic_analysis(ma_ug_t *ug, asg_t* read_g, trans_chain* t_ch, ug_opt_t *opt, uint32_t is_poy, kvec_pe_hit **rhits);
 
 #endif

hifiasm.1

@@ -1,4 +1,4 @@
-.TH hifiasm 1 "22 August 2021" "hifiasm-0.16.0 (r369)" "Bioinformatics tools"
+.TH hifiasm 1 "19 July 2020" "hifiasm-0.9 (r289)" "Bioinformatics tools"
 
 .SH NAME
 .PP
@@ -62,7 +62,7 @@ Hifiasm is an ultrafast haplotype-resolved de novo assembler for PacBio
 Hifi reads. Unlike most existing assemblers, hifiasm starts from uncollapsed
 genome. Thus, it is able to keep the haplotype information as much as possible.
 The input of hifiasm is the PacBio Hifi reads in fasta/fastq format, and its
-outputs consist of multiple types of assembly graphs in GFA format.
+outputs consist of multiple types of assembly graph in GFA format.
 
 
 .SH OPTIONS
@@ -104,82 +104,17 @@ Minimizer window size [51].
 Number of bits for bloom filter; 0 to disable [37]. This bloom filter is used
 to filter out singleton k-mers when counting all k-mers. It takes
 .RI 2^( INT -3)
-bytes of memory. A proper setting saves memory. 
-.BR -f37 
-is recommended for human
-assembly. For small genomes, use 
-.BR -f0 
-to disable the initial bloom filter 
-which takes 16GB memory at the beginning. For genomes much larger 
-than human, applying 
-.BR -f38 
-or even 
-.BR -f39 
-is preferred to save memory on k-mer counting.
-
-.TP
-.BI -D \ INT
-Drop k-mers occurring 
-.I >FLOAT*coverage 
-times [5.0]. 
-Hifiasm discards these high-frequency k-mers 
-during error correction to reduce running time. 
-The 
-.I coverage 
-is determined automatically 
-by hifiasm based on k-mer plot, representing 
-homozygous read coverage. Raising this option 
-may improve the resolution of repetitive regions 
-but takes longer time.
-
-.TP
-.BI -N \ INT
-Consider up to 
-.I max(-D*coverage,-N) 
-overlaps for each oriented read [100]. 
-The 
-.I coverage 
-is determined automatically 
-by hifiasm based on k-mer plot, representing 
-homozygous read coverage. Raising this option may 
-improve the resolution of repetitive regions but 
-takes longer time.
-
+bytes of memory. A proper setting saves memory. 37 is recommended for human
+assembly.
 
 .TP
 .BI -r \ INT
-Rounds of haplotype-aware error corrections [3]. 
-This option affects all outputs of hifiasm. 
-Odd rounds of correction are preferred in practice.
-
-.TP
-.BI -z \ INT
-Length of adapters that should be removed [0]. This option remove
-.I INT
-bases from both ends of each read.
-Some old Hifi reads may consist of
-short adapters (e.g., 20bp adapter at one end). For such data, trimming short adapters would 
-significantly improve the assembly quality.
-
-.TP
-.BI --max-kocc \ INT
-Employ k-mers occurring < 
-.IR INT 
-times to rescue repetitive overlaps [2000]. 
-This option may improve the resolution of repeats.
-
-.TP
-.BI --hg-size \ INT (k/m/g)
-Estimated haploid genome size used for inferring read coverage [auto]. 
-This option is used to get accurate homozygous read coverage during 
-error correction. Common suffices are required, for example, 100m or 3g.
+Rounds of haplotype-aware error corrections [3]. This option affects all outputs of hifiasm.
 
 .TP
 .BI --min-hist-cnt \ INT
 When analyzing the k-mer spectrum, ignore counts below
-.IR INT 
-[5]. For very low coverage of HiFi data, set smaller 
-value for this option.
+.IR INT .
 
 .SS Assembly options
 
@@ -193,6 +128,14 @@ Note that unlike
 .BR -r ,
 this option does not affect error corrected reads and all-to-all overlaps.
 
+.TP
+.BI -z \ INT
+Length of adapters that should be removed [0]. This option remove
+.I INT
+bases from both ends of each read.
+Some old Hifi reads may consist of
+short adapters (e.g., 20bp adapter at one end). For such data, trimming short adapters would 
+significantly improve the assembly quality.
 
 .TP
 .BI -m \ INT
@@ -206,13 +149,10 @@ section of this man-page.
 .TP
 .BI -p \ INT
 Maximal probing distance for bubble popping when generating haplotype-resolved processed unitig graph
-without small bubbles [0]. Bubbles longer than
+without small bubbles [100000]. Bubbles longer than
 .I INT
-bases will not be popped. Small bubbles might be caused by somatic mutations or noise in data. 
-Please note that hifiasm automatically pops small bubbles based on coverage, 
-which can be tweaked by 
-.BR --hom-cov .
-For detailed description of this graph, please see the
+bases will not be popped. Small bubbles might be caused by somatic mutations or noise in data, which
+are not the real haplotype information. For detailed description of this graph, please see the
 .B OUTPUTS
 section of this man-page. 
 
@@ -223,13 +163,13 @@ A unitig is considered small if it is composed of less than
 reads [3]. Hifiasm may try to remove small unitigs at various steps.
 
 .TP
-.BI -x \ FLOAT1, -y \ FLOAT2
+.BI -x \ FLOAT, -y \ FLOAT
 Max and min overlap drop ratio [0.8, 0.2]. This option is used with
-.BR -a .
+.BR -r .
 Given a node
 .I N
 in the assembly graph, let max(N)
-be the length of the longest overlap of
+be the length of the largest overlap of
 .I N.
 Hifiasm iteratively drops overlaps of
 .I N
@@ -239,17 +179,17 @@ are below a threshold controlled by
 and
 .BR -y .
 Hifiasm applies
-.B -a
+.B -r
 rounds of short overlap removal with an increasing threshold between
-.I FLOAT1
+.B -x
 and
-.I FLOAT2.
+.BR -y .
 
 .TP
 .BI -i
 Ignore error corrected reads and overlaps saved in
 .IR prefix .*.bin
-files so that hifiasm will start again from scratch.
+files.
 Apart from assembly graphs, hifiasm also outputs three binary files
 that save all overlap information during assembly step.
 With these files, hifiasm can avoid the time-consuming all-to-all overlap calculation step,
@@ -257,24 +197,12 @@ and do the assembly directly and quickly.
 This might be helpful when users want to get an optimized assembly by multiple rounds of experiments
 with different parameters.
 
-.TP
-.BI -u
-Disable post-join step for contigs which may improve N50. 
-The post-join step of hifiasm improves contig N50 but may introduce misassemblies.
-
-.TP
-.BI --hom-cov \ INT
-Homozygous read coverage inferred automatically in default. This option affects different types of outputs, 
-including Hi-C phased assembly and HiFi-only assembly. 
-
 .TP
 .BI --pri-range \ INT1[,INT2]
 Min and max coverage cutoff of primary contigs.
 Keep contigs with coverage in this range at p_ctg.gfa. 
 Inferred automatically in default.
-If 
-.I INT2 
-is not specified, it is set to infinity. 
+If INT2 is not specified, it is set to infinity. 
 Set -1 to disable.
 
 .TP
@@ -284,38 +212,6 @@ with suffix
 .B lowQ.bed
 [70]. Set 0 to disable. 
 
-
-.TP
-.BI --b-cov \ INT
-Break contigs at potential misassemblies with <INT-fold coverage [0].
-Work with 
-.B --m-rate.
-Set 0 to disable. 
-
-.TP
-.BI --h-cov \ INT
-Break contigs at potential misassemblies with >INT-fold coverage [-1].
-Work with 
-.B --m-rate.
-Set -1 to disable. 
-
-.TP
-.BI --m-rate \ FLOAT
-Break contigs with <=FLOAT*coverage exact overlaps [0.75].
-Only work with 
-.B --b-cov
-and
-.B --h-cov.
-
-.TP
-.BI --primary
-Output a primary assembly and an alternate assembly.
-Hifiasm outputs two balanced assemblies and a primary
-assembly in default. Enable this option or 
-.B -l0
-outputs a primary assembly and an alternate assembly.
-
-
 .SS Trio-partition options
 
 .TP 10
@@ -358,86 +254,49 @@ but occurs <
 .B -c
 times in the other sample.
 
-.TP
-.BI --t-occ \ INT
-Forcedly remove unitig including >
-.I INT 
-unexpected haplotype-specific reads 
-without considering graph topology [60].
-
 
 .SS Purge-dups options
 
 .TP 10
 .BI -l \ INT
 Level of purge-dup. 0 to disable purge-dup, 1 to only purge contained haplotigs, 
-2 to purge all types of haplotigs, 3 to purge all types of haplotigs in most aggressive way. 
-In default, [3] for non-trio assembly, [0] for trio assembly.
+2 to purge all types of haplotigs. In default, [2] for non-trio assembly, [0] for trio assembly.
 For trio assembly, only level 0 and level 1 are allowed.
 
 .TP
 .BI -s \ FLOAT
-Similarity threshold for duplicate haplotigs that should be purged [0.75 for 
-.BR -l1/-l2 , 
-0.55 for 
-.BR -l3 ].
+Similarity threshold for duplicate haplotigs that should be purged [0.75].
 
 .TP
 .BI -O \ FLOAT
 Min number of overlapped reads for duplicate haplotigs that should be purged [1].
 
 .TP
-.BI --purge-max \ INT
+.BI --purge-cov \ INT
 Coverage upper bound of Purge-dups, which is inferred automatically in default.
-If the coverage of a contig is higher than this bound, don't apply Purge-dups. 
-Larger value makes assembly more contiguous but may collapse repeats or segmental duplications.
+If the coverage of a contig is higher than this bound, don't apply Purge-dups.
 
 .TP
-.BI   --n-hap \ INT
-Assumption of haplotype number. If it is set to >2, the quality of 
-primary assembly for polyploid genomes might be improved.
+.BI --high-het \ INT
+Enable this mode for high heterozygosity sample, which will increase running time. 
+For ordinary samples, no need to enable this mode [experimental, not stable].
 
 
+.SS Debugging options
 
-.SS Hi-C-partition options [experimental, not stable]
+.TP 10
+.B --dbg-gfa
+Write additional files to speed up the debugging of graph cleaning.
 
-.TP
-.BI --h1 \ FILEs 
-File names of input Hi-C R1 [r1_1.fq,r1_2.fq,...].
-
-.TP
-.BI --h2 \ FILEs 
-File names of input Hi-C R2 [r2_1.fq,r2_2.fq,...].
-
-.TP
-.BI --n-weight \ INT 
-Rounds of reweighting Hi-C links [3]. Increasing this may improves
-phasing results but takes longer time.
-
-.TP
-.BI --n-perturb \ INT 
-Rounds of perturbation [10000]. Increasing this may improves
-phasing results but takes longer time.
-
-.TP
-.BI --f-perturb \ FLOAT 
-Fraction to flip for perturbation [0.1]. Increasing this may improves
-phasing results but takes longer time.
-
-.TP
-.BI --l-msjoin \ INT 
-Detect misjoined unitigs of >=
-.I INT 
-in size; 0 to disable [500000].
-
-.TP
-.BI --seed \ INT 
-RNG seed [11].
 
 .SH OUTPUTS
 
 .PP
-In general, hifiasm generates the following assembly graphs in the GFA format:
+Without trio partition options
+.B -1
+and
+.BR -2 ,
+hifiasm generates the following assembly graphs in the GFA format:
 
 .RS 2
 .TP 2
@@ -450,138 +309,44 @@ haplotype-resolved raw unitig graph. This graph keeps all haplotype information.
 .IR prefix .p_utg.gfa:
 haplotype-resolved processed unitig graph without small bubbles.  Small bubbles
 might be caused by somatic mutations or noise in data, which are not the real
-haplotype information. Hifiasm automatically pops such small bubbles based on coverage. 
-The option 
-.BR --hom-cov 
-affects the result.
-In addition, the option 
-.BR -p 
-forcedly pops bubbles.
+haplotype information.  The size of popped small bubbles should be specified by
+.BR -p .
 
 .TP
 *
 .IR prefix .p_ctg.gfa:
-assembly graph of primary contigs. This graph includes a complete assembly with 
-long stretches of phased blocks.
+assembly graph of primary contigs. This graph collapses different haplotypes.
 
 .TP
 *
 .IR prefix .a_ctg.gfa:
-assembly graph of alternate contigs. This graph consists of all contigs that
+assembly graph of alternate contigs. This graph consists of all assemblies that
 are discarded in primary contig graph.
 
-.TP
-*
-.IR prefix .*hap*.p_ctg.gfa:
-phased contig graph. This graph keeps the phased assembly.
-
 .RE
 
 .PP
-Hifiasm outputs
-.B *.r_utg.gfa
-and
-.B *.p_utg.gfa
-in any cases.
-Specifically, hifiasm outputs the following assembly graphs
-with trio-binning options:
+With trio partition, hifiasm outputs the following assembly graphs:
 
 .RS 2
 .TP 2
 *
-.IR prefix .dip.hap1.p_ctg.gfa:
-fully phased paternal/haplotype1 contig graph keeping the phased
+.IR prefix .dip.r_utg.gfa:
+haplotype-resolved raw unitig graph. This graph keeps all haplotype information.
+
+.TP
+*
+.IR prefix .hap1.p_ctg.gfa:
+phased paternal/haplotype1 contig graph. This graph keeps the phased
 paternal/haplotype1 assembly.
 
 .TP
 *
-.IR prefix .dip.hap2.p_ctg.gfa:
-fully phased maternal/haplotype2 contig graph keeping the phased
+.IR prefix .hap2.p_ctg.gfa:
+phased maternal/haplotype2 contig graph. This graph keeps the phased
 maternal/haplotype2 assembly.
 .RE
 
-.PP
-With Hi-C partition options, hifiasm outputs:
-
-.RS 2
-.TP 2
-*
-.IR prefix .hic.p_ctg.gfa:
-assembly graph of primary contigs.
-
-.TP
-*
-.IR prefix .hic.hap1.p_ctg.gfa:
-fully phased contig graph where each contig is fully phased.
-
-.TP
-*
-.IR prefix .hic.hap2.p_ctg.gfa:
-fully phased contig graph where each contig is fully phased.
-
-.TP
-*
-.IR prefix .hic.a_ctg.gfa 
-(optional with 
-.BR --primary): 
-assembly graph of alternate contigs.
-
-.RE
-
-
-.PP
-Hifiasm keeps Hi-C alignment results and Hi-C index in two bin 
-files: 
-.B *hic.lk.bin
-and 
-.B *hic.tlb.bin.
-Rerunning hifiasm with different Hi-C reads needs to delete these bin files
-or use 
-.BR -i .
-.RE
-
-.PP
-Hifiasm generates the following assembly graphs only with HiFi reads:
-
-.RS 2
-.TP 2
-*
-.IR prefix .bp.p_ctg.gfa:
-assembly graph of primary contigs. 
-
-.TP
-*
-.IR prefix .bp.hap1.p_ctg.gfa:
-partially phased contig graph of haplotype1.
-
-.TP
-*
-.IR prefix .bp.hap2.p_ctg.gfa:
-partially phased contig graph of haplotype2.
-.RE
-
-.PP
-If the option
-.BR -l0
-or
-.BR --primary
-is specified, hifiasm outputs:
-
-.RS 2
-.TP 2
-*
-.IR prefix .p_ctg.gfa:
-assembly graph of primary contigs.
-
-.TP
-*
-.IR prefix .a_ctg.gfa:
-assembly graph of alternate contigs.
-.RE
-
-
-
-
 .PP
 For each graph, hifiasm also outputs a simplified version without sequences for
 the ease of visualization. Hifiasm keeps corrected reads and overlaps in three

hist.cpp

@@ -12,66 +12,7 @@ static void ha_hist_line(int c, int x, int exceed, int64_t cnt)
 	fprintf(stderr, " %lld\n", (long long)cnt);
 }
 
-void print_hist_lines(int n_cnt, int start_cnt, const int64_t *cnt)
-{
-	const int hist_max = 100;
-	int i, start, low_i, max_i, max;
-	// determine the start point
-	assert(n_cnt > start_cnt);
-	start = cnt[1] > 0? 1 : 2;
-
-	// find the low point from the left
-	low_i = start > start_cnt? start : start_cnt;
-	for (i = low_i; i < n_cnt; ++i)
-		if (cnt[i] > cnt[i-1]) break;
-	low_i = i - 1;
-	fprintf(stderr, "[M::%s] lowest: count[%d] = %ld\n", __func__, low_i, (long)cnt[low_i]);
-
-	// find the highest peak
-	max_i = start > start_cnt? start : start_cnt, max = cnt[max_i];
-	for (i = max_i; i < n_cnt; ++i)
-		if (cnt[i] > max)
-			max = cnt[i], max_i = i;
-	fprintf(stderr, "[M::%s] highest: count[%d] = %ld\n", __func__, max_i, (long)cnt[max_i]);
-
-	for (i = start; i < n_cnt; ++i) {
-		int x, exceed = 0;
-		x = (int)((double)hist_max * cnt[i] / cnt[max_i] + .499);
-		if (x > hist_max) exceed = 1, x = hist_max; // may happen if cnt[2] is higher
-		if (i > max_i && x == 0) break;
-		ha_hist_line(i, x, exceed, cnt[i]);
-	}
-}
-
-int adj_m_peak_hom(int m_peak_hom, int max_i, int max2_i, int max3_i, int *peak_het)
-{
-	int64_t mm[3], d, min_i, min_d, i;
-	mm[0] = max2_i; mm[1] = max_i; mm[2] = max3_i;
-	for (i = 0, min_i = -1, min_d = -1; i < 3; i++){
-		if(mm[i] <= 0) continue;
-		d = (mm[i] >= m_peak_hom?mm[i]-m_peak_hom:m_peak_hom-mm[i]);
-		if(min_d == -1 || min_d > d || (min_d == d && i == 1)){
-			min_d = d; min_i = i;
-		}
-	}
-	if(min_i < 0) return m_peak_hom;
-	if(mm[min_i] < m_peak_hom){
-		d = m_peak_hom - mm[min_i];
-		if(d >= mm[min_i]*0.51) {
-			*peak_het = mm[min_i];
-			return m_peak_hom;
-		}
-	}
-
-	for (i = min_i-1; i >= 0; i--){
-		if(mm[i] <= 0) continue;
-		*peak_het = mm[i];
-		break;
-	}
-	return mm[min_i];
-}
-
-int ha_analyze_count(int n_cnt, int start_cnt, int m_peak_hom, const int64_t *cnt, int *peak_het)
+int ha_analyze_count(int n_cnt, int start_cnt, const int64_t *cnt, int *peak_het)
 {
 	const int hist_max = 100;
 	int i, start, low_i, max_i, max2_i, max3_i;
@@ -145,8 +86,6 @@ int ha_analyze_count(int n_cnt, int start_cnt, int m_peak_hom, const int64_t *cn
 	}
 	if (max3 > 0) fprintf(stderr, "[M::%s] right: count[%d] = %ld\n", __func__, max3_i, (long)cnt[max3_i]);
 	else fprintf(stderr, "[M::%s] right: none\n", __func__);
-
-	if(m_peak_hom > 0) return adj_m_peak_hom(m_peak_hom, max_i, max2_i, max3_i, peak_het);
 	if (max3_i > 0) {
 		*peak_het = max_i;
 		return max3_i;

horder.h

@@ -1,70 +0,0 @@
-#ifndef __HORDER__
-#define __HORDER__
-#include <stdint.h>
-#include "hic.h"
-
-#define get_hit_srev(x, k) ((x).a.a[(k)].s>>63)
-#define get_hit_slen(x, k) ((x).a.a[(k)].len>>32)
-#define get_hit_suid(x, k) (((x).a.a[(k)].s<<1)>>(64 - (x).uID_bits))
-#define get_hit_spos(x, k) ((x).a.a[(k)].s & (x).pos_mode)
-#define get_hit_spos_e(x, k) (get_hit_srev((x),(k))?\
-        ((get_hit_spos((x),(k))+1>=get_hit_slen((x),(k)))?\
-            (get_hit_spos((x),(k))+1-get_hit_slen((x),(k))):0)\
-                    :(get_hit_spos((x),(k))+get_hit_slen((x),(k))-1))
-
-#define get_hit_erev(x, k) ((x).a.a[(k)].e>>63)
-#define get_hit_elen(x, k) ((uint32_t)((x).a.a[(k)].len))
-#define get_hit_euid(x, k) (((x).a.a[(k)].e<<1)>>(64 - (x).uID_bits))
-#define get_hit_epos(x, k) ((x).a.a[(k)].e & (x).pos_mode)
-#define get_hit_epos_e(x, k) (get_hit_erev((x),(k))?\
-        ((get_hit_epos((x),(k))+1>=get_hit_elen((x),(k)))?\
-            (get_hit_epos((x),(k))+1-get_hit_elen((x),(k))):0)\
-                    :(get_hit_epos((x),(k))+get_hit_elen((x),(k))-1))
-
-typedef struct {
-	uint32_t v;
-    uint32_t u;
-	uint32_t occ:31, del:1;
-	double w, nw;
-} osg_arc_t;
-
-typedef struct {
-	double mw[2], ez[2];
-    uint8_t del;
-} osg_seq_t;
-
-typedef struct {
-	uint32_t m_arc, n_arc:31, is_srt:1;
-	osg_arc_t *arc;
-
-	uint32_t m_seq, n_seq:31, is_symm:1;
-	osg_seq_t *seq;
-
-	uint64_t *idx;
-} osg_t;
-
-typedef struct {
-    osg_t *g;
-}scg_t;
-typedef struct {
-	kvec_t(uint64_t) avoid;
-    // kvec_t(uint64_t) occ;
-    // kvec_t(uint8_t) hf;
-    kvec_pe_hit r_hits, u_hits;
-    ma_ug_t *ug;
-    asg_t *r_g;
-    scg_t sg;
-}horder_t;
-
-horder_t *init_horder_t(kvec_pe_hit *i_hits, uint64_t i_hits_uid_bits, uint64_t i_hits_pos_mode, 
-asg_t *i_rg, ma_ug_t* i_ug, bubble_type* bub, kv_u_trans_t *ref, ug_opt_t *opt, uint32_t round);
-void destory_horder_t(horder_t **h);
-void horder_clean_sg_by_utg(asg_t *sg, ma_ug_t *ug);
-kvec_pe_hit *get_r_hits_for_trio(kvec_pe_hit *u_hits, asg_t* r_g, ma_ug_t* ug, bubble_type* bub, uint64_t uID_bits, uint64_t pos_mode);
-void update_switch_unitig(ma_ug_t *ug, asg_t *rg, kvec_pe_hit *hits, kv_u_trans_t *k_trans, uint64_t cutoff_s, uint64_t cutoff_e,
-uint64_t min_ulen, double boundaryRate);
-kvec_pe_hit *get_r_hits_order(kvec_pe_hit *uhits, uint64_t hits_uid_bits, uint64_t hits_pos_mode, 
-asg_t *rg, ma_ug_t* ug, bubble_type* bub);
-void ha_aware_order(kvec_pe_hit *r_hits, asg_t *rg, ma_ug_t *ug_fa, ma_ug_t *ug_mo, kv_u_trans_t *ref, 
-ug_opt_t *opt, uint32_t round);
-#endif

htab.cpp

@@ -1,3 +1,4 @@
+#include <stdint.h>
 #include <zlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -48,7 +49,7 @@ typedef struct {
 	int32_t pre;
 	int32_t n_thread;
 	int64_t chunk_size;
-	int adaLen, min_rcnt;
+	int adaLen;
 } yak_copt_t;
 
 void yak_copt_init(yak_copt_t *o)
@@ -130,7 +131,6 @@ typedef struct {
 typedef struct {
 	int k, pre, n_hash, n_shift;
 	uint64_t tot;  ///number of distinct k-mers
-	uint64_t bs;
 	ha_ct1_t *h;
 } ha_ct_t;
 
@@ -191,10 +191,12 @@ static int ha_ct_insert_list(ha_ct_t *h, int create_new, int n, const uint64_t *
 		///so low 12 bits are not useful
 		uint64_t x = a[j] >> h->pre;
 		khint_t k;
-		assert((a[j]&mask) == (a[0]&mask));
+		if ((a[j]&mask) != (a[0]&mask)) continue;
 		if (create_new) {
+			///for 0-th counting, g->b = NULL
 			if (g->b)
 				ins = (yak_bf_insert(g->b, x) == h->n_hash);
+			///for 0-th counting, g->b = NULL
 			///x = the high 52 bits of a[j] + low 12 bits 0
 			///the low 12 bits are used for counting
 			if (ins) {
@@ -287,8 +289,6 @@ static void ha_ct_shrink(ha_ct_t *h, int min, int max, int n_thread)
 	kt_for(n_thread, worker_ct_shrink, &a, 1<<h->pre);
 	for (i = 0, h->tot = 0; i < 1<<h->pre; ++i)
 		h->tot += kh_size(h->h[i].h);
-	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> counted %ld distinct minimizer k-mers\n", __func__,
-			yak_realtime(), yak_cpu_usage(), (long)h->tot);
 }
 
 /***********************
@@ -316,42 +316,6 @@ typedef struct {
 	ha_pt_t *pt;
 } pt_gen_aux_t;
 
-
-static void worker_pt_shrink(void *data, long i, int tid) // callback for kt_for()
-{
-    ha_pt_t *h = (ha_pt_t*)data;
-	ha_pt1_t *b = &h->h[i];
-    yak_pt_t *f = NULL;
-    khint_t k;
-    f = yak_pt_init();
-    for (k = 0, b->n = 0; k < kh_end(b->h); ++k) {
-        if (kh_exist(b->h, k)) {
-			if(kh_val(b->h, k) <= 0) continue;
-			int absent; khint_t l;
-			l = yak_pt_put(f, (kh_key(b->h, k) >> h->pre) << YAK_COUNTER_BITS, &absent);
-			kh_val(f, l) = b->n; 
-			b->n += kh_key(b->h, k) & YAK_MAX_COUNT;
-        }
-    }
-    yak_pt_destroy(b->h);
-    h->h[i].h = f;
-	CALLOC(b->a, b->n);///need fix
-}
-
-static uint64_t ha_pt_shrink(ha_pt_t *h, int n_thread)
-{
-    int i;
-	uint64_t occ;
-    ///still start 4096 threads
-    kt_for(n_thread, worker_pt_shrink, h, 1<<h->pre);
-    for (i = 0, occ = 0, h->tot = 0; i < 1<<h->pre; ++i)
-	{
-		h->tot += kh_size(h->h[i].h);
-		occ += h->h[i].n;
-	}
-	return occ;
-}
-
 static void worker_pt_gen(void *data, long i, int tid) // callback for kt_for()
 {
 	pt_gen_aux_t *a = (pt_gen_aux_t*)data;
@@ -392,44 +356,6 @@ ha_pt_t *ha_pt_gen(ha_ct_t *ct, int n_thread)
 	return pt;
 }
 
-static void worker_pt_gen_count(void *data, long i, int tid) // callback for kt_for()
-{
-	pt_gen_aux_t *a = (pt_gen_aux_t*)data;
-	ha_pt1_t *b = &a->pt->h[i];
-	yak_ct_t *g = a->ct->h[i].h;
-	khint_t k;
-	for (k = 0, b->n = 0; k != kh_end(g); ++k) {
-		if (kh_exist(g, k)) {
-			int absent;
-			khint_t l;
-			l = yak_pt_put(b->h, kh_key(g, k) >> a->ct->pre << YAK_COUNTER_BITS, &absent);
-			kh_val(b->h, l) = 0; kh_key(b->h, l) |= kh_key(g, k) & YAK_MAX_COUNT;
-		}
-	}
-	yak_ct_destroy(g);
-	a->ct->h[i].h = 0;
-}
-
-ha_pt_t *ha_pt_gen_count(ha_ct_t *ct, int n_thread)
-{
-	pt_gen_aux_t a;
-	int i;
-	ha_pt_t *pt;
-	ha_ct_destroy_bf(ct);
-	CALLOC(pt, 1);
-	pt->k = ct->k, pt->pre = ct->pre, pt->tot = ct->tot;
-	CALLOC(pt->h, 1<<pt->pre);
-	for (i = 0; i < 1<<pt->pre; ++i) {
-		pt->h[i].h = yak_pt_init();
-		yak_pt_resize(pt->h[i].h, kh_size(ct->h[i].h));
-	}
-	a.ct = ct, a.pt = pt;
-	kt_for(n_thread, worker_pt_gen_count, &a, 1<<pt->pre);
-	free(ct->h); free(ct);
-	return pt;
-}
-
-
 int ha_pt_insert_list(ha_pt_t *h, int n, const ha_mz1_t *a)
 {
 	int j, mask = (1<<h->pre) - 1, n_ins = 0;
@@ -441,9 +367,9 @@ int ha_pt_insert_list(ha_pt_t *h, int n, const ha_mz1_t *a)
 		khint_t k;
 		int n;
 		ha_idxpos_t *p;
-		assert((a[j].x&mask) == (a[0].x&mask));
+		if ((a[j].x&mask) != (a[0].x&mask)) continue;
 		k = yak_pt_get(g->h, x<<YAK_COUNTER_BITS);
-		if (k == kh_end(g->h)) continue; // TODO: understand why we sometimes come here
+		if (k == kh_end(g->h)) continue;
 		n = kh_key(g->h, k) & YAK_MAX_COUNT;
 		assert(n < YAK_MAX_COUNT);
 		p = &g->a[kh_val(g->h, k) + n];
@@ -454,26 +380,6 @@ int ha_pt_insert_list(ha_pt_t *h, int n, const ha_mz1_t *a)
 	}
 	return n_ins;
 }
-
-
-int ha_pt_cnt_insert_list(ha_pt_t *h, int n, const uint64_t *a)
-{
-	int j, mask = (1<<h->pre) - 1, n_ins = 0;
-	ha_pt1_t *g;
-	if (n == 0) return 0;
-	g = &h->h[a[0]&mask];
-	for (j = 0; j < n; ++j) {
-		uint64_t x = a[j] >> h->pre;
-		khint_t k;
-		assert((a[j]&mask) == (a[0]&mask));
-		k = yak_pt_get(g->h, x<<YAK_COUNTER_BITS);
-		if (k == kh_end(g->h)) continue; // TODO: understand why we sometimes come here
-		++kh_val(g->h, k);
-		++n_ins;
-	}
-	// fprintf(stderr, "n: %d, n_ins: %d\n", n, n_ins);
-	return n_ins;
-}
 /*
 static void worker_pt_sort(void *data, long i, int tid)
 {
@@ -517,15 +423,6 @@ const ha_idxpos_t *ha_pt_get(const ha_pt_t *h, uint64_t hash, int *n)
 	return &g->a[kh_val(g->h, k)];
 }
 
-const int ha_pt_cnt(const ha_pt_t *h, uint64_t hash)
-{
-	khint_t k;
-	const ha_pt1_t *g = &h->h[hash & ((1ULL<<h->pre) - 1)];
-	k = yak_pt_get(g->h, hash >> h->pre << YAK_COUNTER_BITS);
-	if (k == kh_end(g->h)) return 0;
-	return kh_key(g->h, k) & YAK_MAX_COUNT;
-}
-
 /**********************************
  * Buffer for counting all k-mers *
  **********************************/
@@ -618,21 +515,18 @@ KSEQ_INIT(gzFile, gzread)
 #define HAF_RS_READ      0x10
 #define HAF_CREATE_NEW   0x20
 #define HAF_SKIP_READ    0x40
-#define HAF_UG_READ      0x80
-#define HAF_COUNT_REFINE 0x100
 
 typedef struct { // global data structure for kt_pipeline()
 	const yak_copt_t *opt;
 	const void *flt_tab;
 	int flag, create_new, is_store;
-	uint64_t n_mz, n_seq; ///number of total reads
+	uint64_t n_seq; ///number of total reads
 	kseq_t *ks;
 	UC_Read ucr;
 	ha_ct_t *ct;
 	ha_pt_t *pt;
 	const All_reads *rs_in;
 	All_reads *rs_out;
-	const ma_utg_v *us_in;
 } pl_data_t;
 
 typedef struct { // data structure for each step in kt_pipeline()
@@ -645,7 +539,6 @@ typedef struct { // data structure for each step in kt_pipeline()
 	ha_mz1_v *mz_buf;
 	ha_mz1_v *mz;
 	ch_buf_t *buf;
-	st_mt_t *mt;
 } st_data_t;
 
 static void worker_for_insert(void *data, long i, int tid) // callback for kt_for()
@@ -653,20 +546,9 @@ static void worker_for_insert(void *data, long i, int tid) // callback for kt_fo
 	st_data_t *s = (st_data_t*)data;
 	ch_buf_t *b = &s->buf[i];
 	if (s->p->pt)
-	{
-		if(s->p->flag&HAF_COUNT_REFINE)
-		{
-			b->n_ins += ha_pt_cnt_insert_list(s->p->pt, b->n, b->a);
-		}
-		else
-		{
-			b->n_ins += ha_pt_insert_list(s->p->pt, b->n, b->b);
-		}
-	}
+		b->n_ins += ha_pt_insert_list(s->p->pt, b->n, b->b);
 	else///for 0-th count, go into here
-	{
 		b->n_ins += ha_ct_insert_list(s->p->ct, s->p->create_new, b->n, b->a);
-	}		
 }
 
 static void worker_for_mz(void *data, long i, int tid)
@@ -675,8 +557,8 @@ static void worker_for_mz(void *data, long i, int tid)
 	///get the corresponding minimzer vector of this read
 	ha_mz1_v *b = &s->mz_buf[tid];
 	s->mz_buf[tid].n = 0;
-	ha_sketch(s->seq[i], s->len[i], s->p->opt->w, s->p->opt->k, s->n_seq0 + i, s->p->opt->is_HPC, b, s->p->flt_tab, asm_opt.mz_sample_dist, 0, 0, 
-	(s->p->pt&&(s->p->flag&HAF_COUNT_REFINE))?s->p->pt:NULL, s->p->opt->min_rcnt, asm_opt.dp_min_len, asm_opt.dp_e, &(s->mt[tid]), asm_opt.mz_rewin);
+	///s->p->opt->w = 51, s->p->opt->k
+	ha_sketch(s->seq[i], s->len[i], s->p->opt->w, s->p->opt->k, s->n_seq0 + i, s->p->opt->is_HPC, b, s->p->flt_tab);
 	s->mz[i].n = s->mz[i].m = b->n;
 	MALLOC(s->mz[i].a, b->n);
 	memcpy(s->mz[i].a, b->a, b->n * sizeof(ha_mz1_t));
@@ -693,7 +575,8 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip
 		s->n_seq0 = p->n_seq;
 		if (p->rs_in && (p->flag & HAF_RS_READ)) {
 			while (p->n_seq < p->rs_in->total_reads) {
-				if ((p->flag & HAF_SKIP_READ) && p->rs_in->trio_flag[p->n_seq] != AMBIGU) {
+				if((p->flag & HAF_SKIP_READ) && p->rs_in->trio_flag[p->n_seq] != AMBIGU)
+				{
 					++p->n_seq;
 					continue;
 				}
@@ -714,24 +597,6 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip
 				if (s->sum_len >= p->opt->chunk_size)
 					break;
 			}
-		} else if(p->us_in) {
-			ma_utg_t *u;
-			while (p->n_seq < p->us_in->n) {
-				u = &(p->us_in->a[p->n_seq]);
-				if (s->n_seq == s->m_seq) {
-					s->m_seq = s->m_seq < 16? 16 : s->m_seq + (s->m_seq>>1);
-					REALLOC(s->len, s->m_seq);
-					REALLOC(s->seq, s->m_seq);
-				}
-				MALLOC(s->seq[s->n_seq], u->len);
-				memcpy(s->seq[s->n_seq], u->s, u->len);
-				s->len[s->n_seq++] = u->len;
-				++p->n_seq;
-				s->sum_len += u->len;
-				s->nk += u->len >= p->opt->k? u->len - p->opt->k + 1 : 0;
-				if (s->sum_len >= p->opt->chunk_size)
-					break;
-			}
 		} else {
 			while ((ret = kseq_read(p->ks)) >= 0) {
 				int l = (int)(p->ks->seq.l) - (int)(p->opt->adaLen) - (int)(p->opt->adaLen);
@@ -787,7 +652,7 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip
 		for (i = 0; i < n_pre; ++i) {
 			s->buf[i].m = m;
 			///for 0-th counting, p->pt = NULL
-			if (p->pt && !(p->flag&HAF_COUNT_REFINE)) MALLOC(s->buf[i].b, m);
+			if (p->pt) MALLOC(s->buf[i].b, m);
 			else MALLOC(s->buf[i].a, m);
 		}
 		// fill the buffer
@@ -808,19 +673,13 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip
 			// s->mz && s->mz_buf are lists of minimzer vectors
 			CALLOC(s->mz, s->n_seq);
 			CALLOC(s->mz_buf, p->opt->n_thread);
-			CALLOC(s->mt, p->opt->n_thread);						
 			///calculate minimzers for each read, each read corresponds to one thread
 			kt_for(p->opt->n_thread, worker_for_mz, s, s->n_seq);
-
 			for (i = 0; i < p->opt->n_thread; ++i)
-			{
-				free(s->mt[i].a);
 				free(s->mz_buf[i].a);
-			}
-			free(s->mt);
 			free(s->mz_buf);
 			// insert minimizers
-			if (p->pt && !(p->flag&HAF_COUNT_REFINE)) {///insert whole minimizer
+			if (p->pt) {///insert whole minimizer
 				for (i = 0; i < s->n_seq; ++i)
 					for (j = 0; j < s->mz[i].n; ++j)
 						pt_insert_buf(s->buf, p->opt->pre, &s->mz[i].a[j]);
@@ -830,7 +689,6 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip
 						ct_insert_buf(s->buf, p->opt->pre, s->mz[i].a[j].x);
 			}
 			for (i = 0; i < s->n_seq; ++i) {
-				p->n_mz += s->mz[i].n;
 				free(s->mz[i].a);
 				if (!p->is_store) free(s->seq[i]);
 			}
@@ -849,10 +707,10 @@ static void *worker_count(void *data, int step, void *in) // callback for kt_pip
 		///n_ins is number of distinct k-mers
 		for (i = 0; i < n; ++i) {
 			n_ins += s->buf[i].n_ins;
-			if (p->pt && !(p->flag&HAF_COUNT_REFINE)) free(s->buf[i].b);
+			if (p->pt) free(s->buf[i].b);
 			else free(s->buf[i].a);
 		}
-		if (p->ct) p->ct->tot += n_ins, p->ct->bs += s->sum_len;
+		if (p->ct) p->ct->tot += n_ins;
 		if (p->pt) p->pt->tot_pos += n_ins;
 		free(s->buf);
 		#if 0
@@ -907,18 +765,15 @@ void debug_adapter(const hifiasm_opt_t *asm_opt, All_reads *rs)
 	exit(1);
 }
 
-static ha_ct_t *yak_count(const yak_copt_t *opt, const char *fn, int flag, ha_pt_t *p0, ha_ct_t *c0, const void *flt_tab, All_reads *rs, ma_utg_v *us, int64_t *n_seq)
+static ha_ct_t *yak_count(const yak_copt_t *opt, const char *fn, int flag, ha_pt_t *p0, ha_ct_t *c0, const void *flt_tab, All_reads *rs, int64_t *n_seq)
 {
 	///for 0-th counting, flag = HAF_COUNT_ALL|HAF_RS_WRITE_LEN|HAF_CREATE_NEW
 	int read_rs = (rs && (flag & HAF_RS_READ));
-	int ug_rs = (us && (flag & HAF_UG_READ));
 	pl_data_t pl;
 	gzFile fp = 0;
 	memset(&pl, 0, sizeof(pl_data_t));
 	pl.n_seq = *n_seq;
-	if(ug_rs) {
-		pl.us_in = us;
-	} else if (read_rs) {
+	if (read_rs) {
 		pl.rs_in = rs;
 		init_UC_Read(&pl.ucr);
 	} else {///for 0-th counting, go into here
@@ -946,24 +801,21 @@ static ha_ct_t *yak_count(const yak_copt_t *opt, const char *fn, int flag, ha_pt
 		///building a large hash table consisting of 4096 small hash tables
 		pl.ct = ha_ct_init(opt->k, opt->pre, opt->bf_n_hash, opt->bf_shift);
 	}
-	if(pl.ct) pl.ct->bs = 0;
 	kt_pipeline(3, worker_count, &pl, 3);
 	if (read_rs) {
 		destory_UC_Read(&pl.ucr);
-	} else if(!read_rs && !ug_rs) {
+	} else {
 		kseq_destroy(pl.ks);
 		gzclose(fp);
 	}
 	*n_seq = pl.n_seq;
-	if (pl.opt->w > 1) fprintf(stderr, "[M::%s] collected %ld minimizers\n", __func__, (long)pl.n_mz);
 	return pl.ct;
 }
 
-ha_ct_t *ha_count(const hifiasm_opt_t *asm_opt, int flag, ha_pt_t *p0, const void *flt_tab, All_reads *rs, ma_utg_v *us, int keep_adapter, int *low_freq)
+ha_ct_t *ha_count(const hifiasm_opt_t *asm_opt, int flag, ha_pt_t *p0, const void *flt_tab, All_reads *rs)
 {
 	int i;
 	int64_t n_seq = 0;
-	uint64_t n_bs = 0;
 	yak_copt_t opt;
 	ha_ct_t *h = 0;
 	assert(!(flag & HAF_RS_WRITE_LEN) || !(flag & HAF_RS_WRITE_SEQ)); // not both
@@ -984,14 +836,10 @@ ha_ct_t *ha_count(const hifiasm_opt_t *asm_opt, int flag, ha_pt_t *p0, const voi
 	///for ha_pt_gen, shoud be 0
 	opt.bf_shift = flag & HAF_COUNT_EXACT? 0 : asm_opt->bf_shift;
 	opt.n_thread = asm_opt->thread_num;
-	opt.adaLen = (keep_adapter? asm_opt->adapterLen : 0);
-	opt.min_rcnt = (low_freq?*low_freq:-1);
+	opt.adaLen = asm_opt->adapterLen;
 	///asm_opt->num_reads is the number of fastq files
-	for (i = n_bs = 0; i < asm_opt->num_reads; ++i){
-		h = yak_count(&opt, asm_opt->read_file_names[i], flag|HAF_CREATE_NEW, p0, h, flt_tab, rs, us, &n_seq);
-		if(h) n_bs += h->bs;
-	}
-	if(h) h->bs = n_bs;	
+	for (i = 0; i < asm_opt->num_reads; ++i)
+		h = yak_count(&opt, asm_opt->read_file_names[i], flag|HAF_CREATE_NEW, p0, h, flt_tab, rs, &n_seq);
 	if (h && opt.bf_shift > 0)
 		ha_ct_destroy_bf(h);
 	return h;
@@ -1001,41 +849,34 @@ ha_ct_t *ha_count(const hifiasm_opt_t *asm_opt, int flag, ha_pt_t *p0, const voi
  * High count filter table *
  ***************************/
 
-// Warning: the max count is 32767
-KHASHL_MAP_INIT(static klib_unused, yak_ft_t, yak_ft, uint64_t, int16_t, kh_hash_dummy, kh_eq_generic)
+KHASHL_SET_INIT(static klib_unused, yak_ft_t, yak_ft, uint64_t, kh_hash_dummy, kh_eq_generic)
 
-static yak_ft_t *gen_hh(const ha_ct_t *h, int max_cnt)
+static yak_ft_t *gen_hh(const ha_ct_t *h)
 {
 	int i;
 	yak_ft_t *hh;
-	if (max_cnt > YAK_MAX_COUNT - 1) max_cnt = YAK_MAX_COUNT - 1;
-	if (max_cnt > INT16_MAX - 1) max_cnt = INT16_MAX - 1;
 	hh = yak_ft_init();
 	yak_ft_resize(hh, h->tot * 2);
 	for (i = 0; i < 1<<h->pre; ++i) {
 		yak_ct_t *ht = h->h[i].h;
-		khint_t k, l;
+		khint_t k;
 		for (k = 0; k < kh_end(ht); ++k) {
 			if (kh_exist(ht, k)) {
 				uint64_t y = kh_key(ht, k) >> h->pre << YAK_COUNTER_BITS | i;
 				int absent;
-				l = yak_ft_put(hh, y, &absent);
-				if (absent) {
-					int cnt = kh_key(ht, k) & YAK_MAX_COUNT;
-					kh_val(hh, l) = cnt > max_cnt? INT16_MAX : cnt;
-				}
+				yak_ft_put(hh, y, &absent);
 			}
 		}
 	}
 	return hh;
 }
 
-int32_t ha_ft_cnt(const void *hh, uint64_t y)
+int ha_ft_isflt(const void *hh, uint64_t y)
 {
 	yak_ft_t *h = (yak_ft_t*)hh;
 	khint_t k;
 	k = yak_ft_get(h, y);
-	return k == kh_end(h)? 0 : kh_val(h, k) == INT16_MAX? INT32_MAX : kh_val(h, k);
+	return k == kh_end(h)? 0 : 1;
 }
 
 void ha_ft_destroy(void *h)
@@ -1073,59 +914,6 @@ void debug_ct_index(void* q_ct_idx, void* r_ct_idx)
  * High-level interfaces *
  *************************/
 
-void *ha_ft_ug_gen(const hifiasm_opt_t *asm_opt, ma_utg_v *us, int hap_n)
-{
-	yak_ft_t *flt_tab;
-	int64_t cnt[YAK_N_COUNTS];
-	int cutoff = hap_n + 1;
-	ha_ct_t *h;
-	h = ha_count(asm_opt, HAF_COUNT_ALL|HAF_UG_READ|HAF_COUNT_EXACT, NULL, NULL, NULL, us, 0, NULL);
-
-	ha_ct_hist(h, cnt, asm_opt->thread_num);
-	print_hist_lines(YAK_N_COUNTS, 1, cnt);
-	
-	ha_ct_shrink(h, cutoff, YAK_MAX_COUNT, asm_opt->thread_num);
-	flt_tab = gen_hh(h, asm_opt->max_kmer_cnt);
-	ha_ct_destroy(h);
-	fprintf(stderr, "[M::%s::%.3f*%.2f@%.3fGB] ==> filtered out %ld k-mers occurring %d or more times\n", __func__,
-			yak_realtime(), yak_cpu_usage(), yak_peakrss_in_gb(), (long)kh_size(flt_tab), cutoff);
-	return (void*)flt_tab;
-}
-
-
-ha_pt_t *ha_pt_ug_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, ma_utg_v *us, int hap_n)
-{
-	int64_t cnt[YAK_N_COUNTS], tot_cnt;
-	int i;
-	ha_ct_t *ct;
-	ha_pt_t *pt;
-	///HAF_COUNT_EXACT: no bf
-	ct = ha_count(asm_opt, HAF_COUNT_EXACT|HAF_UG_READ, NULL, flt_tab, NULL, us, 0, NULL);
-	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> counted %ld distinct minimizer k-mers\n", __func__,
-			yak_realtime(), yak_cpu_usage(), (long)ct->tot);
-
-	ha_ct_hist(ct, cnt, asm_opt->thread_num);
-	print_hist_lines(YAK_N_COUNTS, 1, cnt);
-	
-	///here ha_ct_shrink is mostly used to remove k-mer appearing only 1 time
-	if (flt_tab == 0) {
-		ha_ct_shrink(ct, 2, hap_n, asm_opt->thread_num);
-		for (i = 2, tot_cnt = 0; i <= hap_n; ++i) tot_cnt += cnt[i] * i;
-	} else {
-		///Note: here is just to remove minimizer appearing YAK_MAX_COUNT times
-		///minimizer with YAK_MAX_COUNT occ may apper > YAK_MAX_COUNT times, so it may lead to overflow at ha_pt_gen
-		ha_ct_shrink(ct, 2, YAK_MAX_COUNT - 1, asm_opt->thread_num);
-		for (i = 2, tot_cnt = 0; i <= YAK_MAX_COUNT - 1; ++i) tot_cnt += cnt[i] * i;
-	}
-	pt = ha_pt_gen(ct, asm_opt->thread_num);
-	ha_count(asm_opt, HAF_COUNT_EXACT|HAF_UG_READ, pt, flt_tab, NULL, us, 0, NULL);
-	assert((uint64_t)tot_cnt == pt->tot_pos);
-	//ha_pt_sort(pt, asm_opt->thread_num);
-	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> indexed %ld positions\n", __func__,
-			yak_realtime(), yak_cpu_usage(), (long)pt->tot_pos);
-	return pt;
-}
-
 void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int is_hp_mode)
 {
 	yak_ft_t *flt_tab;
@@ -1133,7 +921,7 @@ void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int i
 	int peak_hom, peak_het, cutoff = YAK_MAX_COUNT - 1, ex_flag = 0;
 	if(is_hp_mode) ex_flag = HAF_RS_READ|HAF_SKIP_READ;
 	ha_ct_t *h;
-	h = ha_count(asm_opt, HAF_COUNT_ALL|HAF_RS_WRITE_LEN|ex_flag, NULL, NULL, rs, NULL, 1, NULL);
+	h = ha_count(asm_opt, HAF_COUNT_ALL|HAF_RS_WRITE_LEN|ex_flag, NULL, NULL, rs);
 	if((asm_opt->flag & HA_F_VERBOSE_GFA))
 	{
 		write_ct_index((void*)h, asm_opt->output_file_name);
@@ -1146,7 +934,7 @@ void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int i
 	if(!(ex_flag & HAF_SKIP_READ))
 	{
 		ha_ct_hist(h, cnt, asm_opt->thread_num);
-		peak_hom = ha_analyze_count(YAK_N_COUNTS, asm_opt->min_hist_kmer_cnt, asm_opt->hg_size>0?(h->bs/asm_opt->hg_size):(-1), cnt, &peak_het);
+		peak_hom = ha_analyze_count(YAK_N_COUNTS, asm_opt->min_hist_kmer_cnt, cnt, &peak_het);
 		if (hom_cov) *hom_cov = peak_hom;
 		if (peak_hom > 0) fprintf(stderr, "[M::%s] peak_hom: %d; peak_het: %d\n", __func__, peak_hom, peak_het);
 		///in default, asm_opt->high_factor = 5.0
@@ -1154,29 +942,13 @@ void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int i
 		if (cutoff > YAK_MAX_COUNT - 1) cutoff = YAK_MAX_COUNT - 1;
 	}
 	ha_ct_shrink(h, cutoff, YAK_MAX_COUNT, asm_opt->thread_num);
-	flt_tab = gen_hh(h, asm_opt->max_kmer_cnt);
+	flt_tab = gen_hh(h);
 	ha_ct_destroy(h);
 	fprintf(stderr, "[M::%s::%.3f*%.2f@%.3fGB] ==> filtered out %ld k-mers occurring %d or more times\n", __func__,
 			yak_realtime(), yak_cpu_usage(), yak_peakrss_in_gb(), (long)kh_size(flt_tab), cutoff);
 	return (void*)flt_tab;
 }
 
-ha_pt_t *ha_pt_gen_dp(const hifiasm_opt_t *asm_opt, ha_ct_t *ct, int flag, int n_thread, const void *flt_tab, All_reads *rs, int peak_hom, int peak_het)
-{
-	int low_freq = mz_low_b(peak_hom, peak_het);
-	ha_pt_t *pt = ha_pt_gen_count(ct, n_thread); ///key = cnt, val = 0
-	ha_count(asm_opt, HAF_COUNT_EXACT|HAF_COUNT_REFINE|flag, pt, flt_tab, rs, NULL, 1, &low_freq);
-	uint64_t occ = ha_pt_shrink(pt, n_thread);
-	if(flag&HAF_RS_WRITE_LEN) flag -= HAF_RS_WRITE_LEN;
-	if(flag&HAF_RS_WRITE_SEQ) flag -= HAF_RS_WRITE_SEQ;
-	flag |= HAF_RS_READ; pt->tot_pos = 0;
-	ha_count(asm_opt, HAF_COUNT_EXACT|flag, pt, flt_tab, rs, NULL, 1, NULL);
-	// fprintf(stderr, "[M::%s::] counted %lu distinct minimizer k-mers\n", __func__, pt->tot);
-	// fprintf(stderr, "[M::%s::] collected %lu minimizers\n\n\n", __func__, pt->tot_pos);
-	assert(occ == pt->tot_pos);
-	return pt;
-}
-
 ha_pt_t *ha_pt_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, int read_from_store, int is_hp_mode, All_reads *rs, int *hom_cov, int *het_cov)
 {
 	int64_t cnt[YAK_N_COUNTS], tot_cnt;
@@ -1194,12 +966,12 @@ ha_pt_t *ha_pt_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, int read_f
 	}
 	if(is_hp_mode) extra_flag1 |= HAF_SKIP_READ, extra_flag2 |= HAF_SKIP_READ;
 
-	ct = ha_count(asm_opt, HAF_COUNT_EXACT|extra_flag1, NULL, flt_tab, rs, NULL, 1, NULL);
+	ct = ha_count(asm_opt, HAF_COUNT_EXACT|extra_flag1, NULL, flt_tab, rs);
 	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> counted %ld distinct minimizer k-mers\n", __func__,
 			yak_realtime(), yak_cpu_usage(), (long)ct->tot);
 	ha_ct_hist(ct, cnt, asm_opt->thread_num);
 	fprintf(stderr, "[M::%s] count[%d] = %ld (for sanity check)\n", __func__, YAK_MAX_COUNT, (long)cnt[YAK_MAX_COUNT]);
-	peak_hom = ha_analyze_count(YAK_N_COUNTS, asm_opt->min_hist_kmer_cnt, asm_opt->hg_size>0?(ct->bs/asm_opt->hg_size):(-1), cnt, &peak_het);
+	peak_hom = ha_analyze_count(YAK_N_COUNTS, asm_opt->min_hist_kmer_cnt, cnt, &peak_het);
 	if (hom_cov) *hom_cov = peak_hom;
 	if (het_cov) *het_cov = peak_het;
 	if (peak_hom > 0) fprintf(stderr, "[M::%s] peak_hom: %d; peak_het: %d\n", __func__, peak_hom, peak_het);
@@ -1216,21 +988,12 @@ ha_pt_t *ha_pt_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, int read_f
 		ha_ct_shrink(ct, 2, YAK_MAX_COUNT - 1, asm_opt->thread_num);
 		for (i = 2, tot_cnt = 0; i <= YAK_MAX_COUNT - 1; ++i) tot_cnt += cnt[i] * i;
 	}
-	if(!(asm_opt->flag & HA_F_FAST))
-	{
-		fprintf(stderr, "[M::%s::] counting in normal mode\n", __func__);
-		pt = ha_pt_gen(ct, asm_opt->thread_num);
-		ha_count(asm_opt, HAF_COUNT_EXACT|extra_flag2, pt, flt_tab, rs, NULL, 1, NULL);
-		assert((uint64_t)tot_cnt == pt->tot_pos);
-	}
-	else
-	{
-		fprintf(stderr, "[M::%s::] counting in fast mode\n", __func__);
-		pt = ha_pt_gen_dp(asm_opt, ct, HAF_COUNT_EXACT|extra_flag2, asm_opt->thread_num, flt_tab, rs, peak_hom, peak_het);
-	}
+	pt = ha_pt_gen(ct, asm_opt->thread_num);
+	ha_count(asm_opt, HAF_COUNT_EXACT|extra_flag2, pt, flt_tab, rs);
+	assert((uint64_t)tot_cnt == pt->tot_pos);
 	//ha_pt_sort(pt, asm_opt->thread_num);
-	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> indexed %ld positions, counted %ld distinct minimizer k-mers\n", __func__,
-			yak_realtime(), yak_cpu_usage(), (long)pt->tot_pos, (long)pt->tot);
+	fprintf(stderr, "[M::%s::%.3f*%.2f] ==> indexed %ld positions\n", __func__,
+			yak_realtime(), yak_cpu_usage(), (long)pt->tot_pos);
 	return pt;
 }
 
@@ -1240,9 +1003,10 @@ int query_ct_index(void* ct_idx, uint64_t hash)
 	khint_t k;
 	k = yak_ct_get(g->h, hash);
 	if (k == kh_end(g->h)) return 0;
-	return ((kh_key(g->h, k)&YAK_MAX_COUNT)==YAK_MAX_COUNT)?-1:(kh_key(g->h, k)&YAK_MAX_COUNT);
+	return kh_key(g->h, k)&YAK_MAX_COUNT;
 }
 
+
 int write_ct_index(void *i_ct_idx, char* file_name)
 {
 	char* gfa_name = (char*)malloc(strlen(file_name)+25);
@@ -1375,7 +1139,7 @@ int load_pt_index(void **r_flt_tab, ha_pt_t **r_ha_idx, All_reads* r, hifiasm_op
 
 	ha_pt_t *ha_idx = NULL;
 	char mode = 0;
-	int f_flag = 0, absent, i;
+	int f_flag, absent, i;
 	double index_time, index_s_time, pos_time, pos_s_time;
 
 	

htab.h

@@ -5,11 +5,6 @@
 #include "Process_Read.h"
 #include "CommandLines.h"
 
-typedef struct {
-	int n, m;
-	uint64_t *a;
-} st_mt_t;
-
 typedef struct {
 	uint64_t x; ///x is the hash key
 	///rid is the read id, pos is the end pos of this minimizer, rev is the direction
@@ -23,41 +18,6 @@ typedef struct {
 
 typedef struct { uint32_t n, m; ha_mz1_t *a; } ha_mz1_v;
 
-typedef struct {
-	uint64_t x; ///x is the hash key
-	///rid is the read id, pos is the end pos of this minimizer, rev is the direction
-	///span is the length of this k-mer. For non-HPC k-mer, span may not be equal to k
-	uint64_t rid:30, pos:34;
-	uint16_t rev:1, span:15;
-} ha_mzl_t;
-
-typedef struct {
-	uint64_t rid:30, pos:34;
-	uint16_t rev:1, span:15;
-} ha_mzl_idxpos_t;
-
-typedef struct { uint32_t n, m; ha_mzl_t *a; } ha_mzl_v;
-
-typedef struct { // a simplified version of kdq
-	int front, count;
-	int a[64];
-} tiny_queue_t;
-
-static inline void tq_push(tiny_queue_t *q, int x)
-{
-	q->a[((q->count++) + q->front) & 0x3f] = x;
-}
-
-static inline int tq_shift(tiny_queue_t *q)
-{
-	int x;
-	if (q->count == 0) return -1;
-	x = q->a[q->front++];
-	q->front &= 0x3f;
-	--q->count;
-	return x;
-}
-
 struct ha_pt_s;
 typedef struct ha_pt_s ha_pt_t;
 
@@ -71,16 +31,14 @@ extern void *ha_flt_tab_hp;
 extern ha_pt_t *ha_idx_hp;
 extern void *ha_ct_table;
 
-void *ha_ft_ug_gen(const hifiasm_opt_t *asm_opt, ma_utg_v *us, int hap_n);
+
 void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int is_hp_mode);
-int32_t ha_ft_cnt(const void *hh, uint64_t y);
+int ha_ft_isflt(const void *hh, uint64_t y);
 void ha_ft_destroy(void *h);
 
-ha_pt_t *ha_pt_ug_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, ma_utg_v *us, int hap_n);
 ha_pt_t *ha_pt_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, int read_from_store, int is_hp_mode, All_reads *rs, int *hom_cov, int *het_cov);
 void ha_pt_destroy(ha_pt_t *h);
 const ha_idxpos_t *ha_pt_get(const ha_pt_t *h, uint64_t hash, int *n);
-const int ha_pt_cnt(const ha_pt_t *h, uint64_t hash);
 
 int write_pt_index(void *flt_tab, ha_pt_t *ha_idx, All_reads* r, hifiasm_opt_t* opt, char* file_name);
 int load_pt_index(void **r_flt_tab, ha_pt_t **r_ha_idx, All_reads* r, hifiasm_opt_t* opt, char* file_name);
@@ -101,21 +59,11 @@ double yak_cpu_usage(void);
 
 void ha_triobin(const hifiasm_opt_t *opt);
 
-void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, int sample_dist, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct, ha_pt_t *pt, int min_freq, int32_t dp_min_len, float dp_e, st_mt_t *mt, int32_t ws);
-int ha_analyze_count(int n_cnt, int start_cnt, int m_peak_hom, const int64_t *cnt, int *peak_het);
-int adj_m_peak_hom(int m_peak_hom, int max_i, int max2_i, int max3_i, int *peak_het);
-void print_hist_lines(int n_cnt, int start_cnt, const int64_t *cnt);
+void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf);
+void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct);
+int ha_analyze_count(int n_cnt, int start_cnt, const int64_t *cnt, int *peak_het);
 void debug_adapter(const hifiasm_opt_t *asm_opt, All_reads *rs);
 
-inline int mz_low_b(int peak_hom, int peak_het)
-{
-	int low_freq = 2;
-	if(peak_het > 0) low_freq = peak_het/2;
-	else if(peak_hom > 0) low_freq = peak_hom/4;
-	if(low_freq < 2) low_freq = 2;
-	return low_freq;
-}
-
 static inline uint64_t yak_hash64(uint64_t key, uint64_t mask) // invertible integer hash function
 {
 	key = (~key + (key << 21)) & mask; // key = (key << 21) - key - 1;

khashl.h

@@ -136,33 +136,25 @@ static kh_inline khint_t __kh_h2b(khint_t hash, khint_t bits) { return hash * 26
 #define __KHASHL_IMPL_S_L(SCOPE, HType, prefix, khkey_t) \
 	SCOPE khint_t prefix##_save(HType *h, FILE* fp) { \
 		if (!h) return 0; \
-		uint8_t ff; \
 		khint_t n_buckets = (h->keys? 1U<<h->bits : 0U); \
 		fwrite(&n_buckets, sizeof(n_buckets), 1, fp); \
 		fwrite(&h->bits, sizeof(h->bits), 1, fp); \
 		fwrite(&h->count, sizeof(h->count), 1, fp); \
-		ff = h->used? 1:0; fwrite(&ff, sizeof(ff), 1, fp); \
-		if(ff) fwrite(h->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \
-		ff = h->keys? 1:0; fwrite(&ff, sizeof(ff), 1, fp); \
-		if(ff) fwrite(h->keys, sizeof(khkey_t), n_buckets, fp); \
+		fwrite(h->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \
+		fwrite(h->keys, sizeof(khkey_t), n_buckets, fp); \
 		return 1; \
 	} \
 	SCOPE khint_t prefix##_load(HType **h, FILE* fp) { \
 		(*h) = prefix##_init(); \
 		khint_t n_buckets; \
 		uint64_t flag = 0;\
-		uint8_t ff; \
 		flag += fread(&n_buckets, sizeof(n_buckets), 1, fp); \
 		flag += fread(&(*h)->bits, sizeof((*h)->bits), 1, fp); \
 		flag += fread(&(*h)->count, sizeof((*h)->count), 1, fp); \
-		flag += fread(&ff, sizeof(ff), 1, fp); \
-		if(ff) {\
 		(*h)->used = (khint32_t*)kmalloc(__kh_fsize(n_buckets) * sizeof(khint32_t)); \
-		flag += fread((*h)->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); }\
-		flag += fread(&ff, sizeof(ff), 1, fp); \
-		if(ff) {\
 		(*h)->keys = (khkey_t*)kmalloc(n_buckets * sizeof(khkey_t)); \
-		flag += fread((*h)->keys, sizeof(khkey_t), n_buckets, fp); }\
+		flag += fread((*h)->used, sizeof(khint32_t), __kh_fsize(n_buckets), fp); \
+		flag += fread((*h)->keys, sizeof(khkey_t), n_buckets, fp); \
 		return 1; \
 	} \
 

ksort.h

@@ -40,32 +40,7 @@ typedef struct {
 
 #define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; }
 
-#define KSORT_INIT(name, type_t, __sort_lt) \
-	void ks_heapdown_##name(size_t i, size_t n, type_t l[]) \
-	{ \
-		size_t k = i; \
-		type_t tmp = l[i]; \
-		while ((k = (k << 1) + 1) < n) { \
-			if (k != n - 1 && __sort_lt(l[k], l[k+1])) ++k; \
-			if (__sort_lt(l[k], tmp)) break; \
-			l[i] = l[k]; i = k; \
-		} \
-		l[i] = tmp; \
-	} \
-	void ks_heapmake_##name(size_t lsize, type_t l[]) \
-	{ \
-		size_t i; \
-		for (i = (lsize >> 1) - 1; i != (size_t)(-1); --i) \
-			ks_heapdown_##name(i, lsize, l); \
-	} \
-	void ks_heapsort_##name(size_t lsize, type_t l[]) \
-	{ \
-		size_t i; \
-		for (i = lsize - 1; i > 0; --i) { \
-			type_t tmp; \
-			tmp = *l; *l = l[i]; l[i] = tmp; ks_heapdown_##name(0, i, l); \
-		} \
-	} \
+#define KSORT_INIT(name, type_t, __sort_lt)								\
 	static inline void __ks_insertsort_##name(type_t *s, type_t *t)		\
 	{																	\
 		type_t *i, *j, swap_tmp;										\

main.cpp

@@ -11,7 +11,7 @@ int main(int argc, char *argv[])
 	int i, ret;
 	yak_reset_realtime();
     init_opt(&asm_opt);
-    if (!CommandLine_process(argc, argv, &asm_opt)) return 0;
+    if (!CommandLine_process(argc, argv, &asm_opt)) return 1;
 	ret = ha_assemble();
     destory_opt(&asm_opt);
 	fprintf(stderr, "[M::%s] Version: %s\n", __func__, HA_VERSION);

rcut.h

@@ -1,128 +0,0 @@
-#ifndef __RCUT__
-#define __RCUT__
-#include <stdio.h>
-#include <stdint.h>
-#include "kvec.h"
-#include "Overlaps.h"
-#include "Purge_Dups.h"
-#include "hic.h"
-
-typedef struct {
-    uint32_t bS, bE;
-    uint32_t nS, nE;
-    uint32_t uID;
-    uint8_t hs;
-}mc_interval_t;
-
-#define mc_node_t int8_t
-#define mcg_node_t uint32_t
-// #define w_t int64_t
-// #define t_w_t int64_t
-// #define w_cast(x) ((t_w_t)((x) < 0 ? (x) - 0.5 : (x) + 0.5))
-
-#define w_t double
-#define t_w_t double
-#define w_cast(x) ((t_w_t)((x)))
-#define MC_NAME "debug_mc.bin"
-
-typedef struct {
-	uint64_t x; ///(uint64_t)nid1 << 32 | nid2;
-	w_t w;  ///might be negative or positive
-} mc_edge_t;
-
-typedef struct {
-    kvec_t(uint64_t) idx;
-    kvec_t(mc_edge_t) ma;
-    uint64_t* cc;
-    uint32_t n_seq;
-} mc_match_t;
-
-typedef struct {
-    kvec_t(mc_node_t) s;
-    ma_ug_t *ug;
-    asg_t *rg;
-    mc_match_t* e;
-}mc_g_t;
-
-typedef struct {
-    uint32_t a[2], occ[2]; 
-    mc_node_t s[2];
-    t_w_t z[4];
-}mb_node_t;
-
-typedef struct {
-    kvec_t(uint32_t) bid;
-    kvec_t(uint32_t) idx;
-    kvec_t(mb_node_t) u;
-}mb_nodes_t;
-
-typedef struct {
-	uint64_t x; ///(uint64_t)nid1 << 32 | nid2;
-	t_w_t w[4];  ///might be negative or positive
-} mb_edge_t;
-
-typedef struct {
-    kvec_t(uint64_t) idx;
-    kvec_t(mb_edge_t) ma;
-    uint64_t* cc;
-    uint32_t n_seq;
-} mb_match_t;
-
-typedef struct {
-    mb_nodes_t* u; 
-    mb_match_t* e;
-}mb_g_t;
-
-typedef struct {
-	mcg_node_t s;
-	uint16_t h[2], hc;
-	double hw[2];
-}mc_gg_status;
-
-typedef struct {
-	mc_gg_status *a;
-	size_t n, m;
-}kv_gg_status;
-
-typedef struct {
-	mcg_node_t *a;
-	size_t n, m;
-}mcb_t;
-
-
-typedef struct {
-	kv_gg_status *s;
-    // ma_ug_t *ug;
-    // asg_t *rg;
-    uint32_t un;
-    mc_match_t* e;
-	kvec_t(mcb_t) m;
-	mcg_node_t mask;
-	uint16_t hN;
-}mc_gg_t;
-
-
-static inline uint64_t kr_splitmix64(uint64_t x)
-{
-	uint64_t z = (x += 0x9E3779B97F4A7C15ULL);
-	z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
-	z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
-	return z ^ (z >> 31);
-}
-
-static inline double kr_drand_r(uint64_t *x)
-{
-    union { uint64_t i; double d; } u;
-	*x = kr_splitmix64(*x);
-    u.i = 0x3FFULL << 52 | (*x) >> 12;
-    return u.d - 1.0;
-}
-
-void mc_solve(hap_overlaps_list* ovlp, trans_chain* t_ch, kv_u_trans_t *ta, ma_ug_t *ug, asg_t *read_g, double f_rate, uint8_t* trio_flag, uint32_t renew_s, int8_t *s, uint32_t is_sys, bubble_type* bub, kv_u_trans_t *ref);
-void debug_mc_g_t(const char* name);
-void mc_solve_general(kv_u_trans_t *ta, uint32_t un, kv_gg_status *s, uint16_t hapN, uint16_t update_ta, uint16_t write_dump);
-kv_gg_status *init_mc_gg_status(ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut, 
-ma_hit_t_alloc* sources, R_to_U* ruIndex, uint64_t t_cov, uint16_t hapN);
-void destory_mc_gg_t(mc_gg_t **p);
-void debug_mc_gg_t(const char* fn, uint32_t update_ta, uint32_t convert_mc_g_t);
-#endif

sketch.cpp

@@ -4,571 +4,25 @@
 #include <string.h>
 #include "kvec.h"
 #include "htab.h"
-#include "ksort.h"
-#include "Correct.h"
 
-#define MAX_HIGH_OCC     8   // TODO: don't hard code if we need to tune this parameter
-#define MAX_MAX_HIGH_OCC 16
-#define GMC(a, x,y,xn) ((a)[(x)*(xn)+(y)])
+typedef struct { // a simplified version of kdq
+	int front, count;
+	int a[64];
+} tiny_queue_t;
 
-static inline int mzcmp(const ha_mz1_t *a, const ha_mz1_t *b)
+static inline void tq_push(tiny_queue_t *q, int x)
 {
-	return a->rid < b->rid? -1 : a->rid > b->rid? 1 : ((a->x > b->x) - (a->x < b->x));
+	q->a[((q->count++) + q->front) & 0x3f] = x;
 }
 
-#define mz_lt(a, b) (mzcmp(&(a), &(b)) < 0)
-KSORT_INIT(mz, ha_mz1_t, mz_lt)
-
-void debug_refine(ha_mz1_t *ma, uint64_t *mmt, int32_t sn, int32_t n, int32_t m, int32_t end)
+static inline int tq_shift(tiny_queue_t *q)
 {
-	uint64_t ks = end;
-	int64_t t = 0, i, k, sp = -1, ep = -1, ovlp, tot = mmt[end]&0xffffffff, nt = 0;;
-	while (ks != 0xffffffff)
-	{
-		i = ks/m; k = ks%m;
-		ks = mmt[ks]>>32;
-		if(ks == 0xffffffff || (int32_t)(ks/m) == (i-1))
-		{
-			t++;
-			ovlp = ((MIN(ep, (int64_t)ma[k].pos) >= MAX(sp, (int64_t)(ma[k].pos+1-ma[k].span)))? 
-						MIN(ep, (int64_t)ma[k].pos) - MAX(sp, (int64_t)(ma[k].pos+1-ma[k].span)) + 1:0);
-			if(ovlp != 0) fprintf(stderr, "ERROR-OVLP\n");
-			if(sp == -1 || sp > (ma[k].pos+1-ma[k].span)) sp = ma[k].pos+1-ma[k].span;
-			if(ep == -1 || ep < ma[k].pos) ep = ma[k].pos;
-			nt += (ma[k].rid);
-		}
-	}
-	if(t != sn) fprintf(stderr, "ERROR-TN, t: %ld, sn: %d\n", t, sn);
-	if(nt != tot) fprintf(stderr, "ERROR-TOT, nt: %ld, tot: %ld\n", nt, tot);
-}
-
-void refine_select(ha_mz1_v *mz, int32_t sidx, int32_t eidx, int32_t sn, int32_t min_freq, st_mt_t *mm, 
-int32_t *rsi, int32_t *rei)
-{
-	int32_t n = sn, m = eidx + 1 - sidx, i, k, t, mk=-1;
-	uint64_t ix, kx, ks;
-	kv_resize(uint64_t, *mm, mm->n+n*m);
-	ha_mz1_t *ma = mz->a + sidx;
-	uint64_t *mmt = mm->a + mm->n;
-	// fprintf(stderr, "[M::%s::] ==> +n: %d, m: %d, sn: %d, sidx: %d, eidx: %d\n", __func__, n, m, sn, sidx, eidx);
-
-	for (i = 0; i < n; i++) ///how many selected minimizers 
-	{
-		for (k = 0, mk = -1; k < m; k++) ///how many minimizers in total
-		{
-			if((int32_t)(ma[k].rid)<min_freq) continue;
-			
-			ks = ma[k].pos + 1 - ma[k].span; t = -1;
-			if(i > 0)
-			{
-				for (t = k-1; t >= 0 && (ma[t].pos >= ks||(int32_t)(ma[t].rid)<min_freq); t--);
-			} 
-			
-			ix = (i <= 0?0:(t<0?0xffffffff:(GMC(mmt, i-1,t,m)&0xffffffff)));
-			if(ix < 0xffffffff) ix += (ma[k].rid);
-			kx = (mk < 0?0xffffffff:(GMC(mmt, i, mk,m)&0xffffffff));
-
-			
-
-			ks = MIN(ix, kx);
-			// fprintf(stderr, "ks: %lu, i: %d (n-%d), k: %d (m-%d), ix: %lu, kx: %lu, t: %d, mk: %d\n", 
-			// ks, i, n, k, m, ix, kx, t, mk);
-			if((ks&0xffffffff) == 0xffffffff) ks |= ((uint64_t)0xffffffff)<<32;
-			else if(ks == ix) ks |= (uint64_t)(i>0?(i-1)*m+t:0xffffffff)<<32;
-			else if(ks == kx) ks |= (uint64_t)(mk>=0?i*m+mk:0xffffffff)<<32;
-			
-			GMC(mmt, i,k,m) = ks;	
-			mk = k;
-		}
-	}
-	// fprintf(stderr, "[M::%s::] ==> ++n: %d, m: %d, sn: %d, sidx: %d, eidx: %d\n", __func__, n, m, sn, sidx, eidx);
-
-	ks = (n-1)*m + mk; ix = (uint64_t)-1; kx = 0;
-	while (ks != 0xffffffff)
-	{
-		i = ks/m; k = ks%m;
-		ks = mmt[ks]>>32;
-		// fprintf(stderr, "i: %d, k: %d, ks: %lu\n", i, k, ks);
-		if(ks == 0xffffffff || (int32_t)(ks/m) == (i-1))
-		{
-			mm->a[sidx+k] = 1;
-			ix = MIN((uint64_t)k, ix); kx = MAX((uint64_t)k, kx);
-		}
-	}
-	///debug
-	// debug_refine(ma, mmt, sn, n, m, (n-1)*m + mk);
-
-	if(rsi) (*rsi) = ix + sidx;
-	if(rei) (*rei) = kx + sidx;	
-}
-
-void refine_sketch(ha_mz1_v *p, ha_pt_t *pt, int32_t rlen, int32_t dp_min_len, float er, int32_t min_freq, st_mt_t *mt)
-{
-	// fprintf(stderr, "[M::%s::] ==> #########10#########, rlen: %d\n", __func__, rlen);
-
-	int32_t i, n = p->n, bd, len = MIN(rlen, dp_min_len), sublen, cnt, ei, li, ri;
-	int32_t sn =  len*er + 1;
-	kv_resize(uint64_t, *mt, (int64_t)p->n); 
-	mt->n = p->n; memset(mt->a, 0, sizeof(uint64_t)*p->n);
-	for (i = 0; i < n; i++) p->a[i].rid = ha_pt_cnt(pt, p->a[i].x);
-
-	for (i = cnt = 0, bd = -1, ei = -1; i < n; i++)
-	{
-		if((int32_t)(p->a[i].rid)<min_freq) continue;
-		sublen = p->a[i].pos + 1;
-		if(sublen > len) break;
-		else ei = i;
-		
-		if((int32_t)(p->a[i].pos + 1 - p->a[i].span) > bd)
-		{
-			bd = p->a[i].pos;
-			cnt++;
-		}
-	}
-
-	// fprintf(stderr, "[M::%s::] ==> +cnt: %d, sn: %d, ei: %d, n: %d\n", __func__, cnt, sn, ei, n);
-
-	
-	if(cnt >= sn) refine_select(p, 0, ei, sn, min_freq, mt, NULL, &li);
-	else
-	{
-		li = i-1; 
-		for (i = 0; i <= li; i++) mt->a[i] = 1;
-	} 
-	
-	
-	if(len < rlen)
-	{
-		for (i = n-1, cnt = 0, bd = rlen+1, ei = -1; i >= 0; i--)
-		{
-			if((int32_t)(p->a[i].rid)<min_freq) continue;
-			sublen = rlen - (p->a[i].pos + 1 - p->a[i].span);
-			if(sublen > len) break;
-			else ei = i;
-
-			if((int32_t)(p->a[i].pos) < bd)
-			{
-				bd = p->a[i].pos + 1 - p->a[i].span;
-				cnt++;
-			}
-		}
-
-		// fprintf(stderr, "[M::%s::] ==> -cnt: %d, sn: %d, ei: %d, n: %d\n", __func__, cnt, sn, ei, n);
-
-		
-		if(cnt >= sn) refine_select(p, ei, n-1, sn, min_freq, mt, &ri, NULL);
-		else 
-		{
-			ri = i+1;
-			for (i = ri; i <= n-1; i++) mt->a[i] = 1;
-		} 
-		
-		// fprintf(stderr, "[M::%s::] ==> --cnt: %d, sn: %d, ei: %d, n: %d\n", __func__, cnt, sn, ei, n);
-
-		if(ri - li >= 2)
-		{
-			li++; ri--;
-			sn =  (p->a[ri].pos - p->a[li].pos + p->a[li].span)*er + 1;
-			for (i = li, cnt = 0, bd = -1; i <= ri; i++)
-			{
-				if((int32_t)(p->a[i].rid)<min_freq) continue;
-				if((int32_t)(p->a[i].pos + 1 - p->a[i].span) > bd)
-				{
-					bd = p->a[i].pos;
-					cnt++;
-					if(cnt >= sn) break;
-				}
-			}
-
-			if(cnt >= sn) refine_select(p, li, ri, sn, min_freq, mt, NULL, NULL);
-			else for (i = li; i <= ri; i++) mt->a[i] = 1;
-		}
-	}
-
-	// fprintf(stderr, "[M::%s::] ==> #########20#########, p->n: %u, n: %d\n", __func__, p->n, n);
-	for (i = sn = 0; i < n; i++)
-	{
-		if(mt->a[i])
-		{
-			p->a[sn] = p->a[i];
-			sn++;
-		}
-	}
-	// if(p->n != sn) fprintf(stderr, "[M::%s::] ==> #########21#########, p->n: %u, sn: %d\n", __func__, p->n, sn);
-	p->n = sn;
-	
-}
-
-inline int hf_dp(ha_mz1_v *mz, int32_t sidx, int32_t eidx, int32_t sn, int32_t min_freq, st_mt_t *mm, 
-int32_t *rsi, int32_t *rei)
-{
-	return 0;
-}
-
-inline void hf_select(ha_mz1_v *p, int32_t si, int32_t ei, int32_t n, int32_t len, int32_t sample_dist, ha_mz1_t *b, int32_t force)
-{
-	if(ei - si <= 1) return;
-	int32_t ps = si < 0? 0 : p->a[si].pos;
-	int32_t pe = ei == n? len : p->a[ei].pos;
-	int32_t j, k, st = si + 1, en = ei;
-	int32_t max_high_occ = (int32_t)((double)(pe - ps) / sample_dist + .499);
-	if (max_high_occ > MAX_MAX_HIGH_OCC)
-		max_high_occ = MAX_MAX_HIGH_OCC;
-	for (j = st, k = 0; j < en && k < max_high_occ; ++j, ++k)
-		b[k] = p->a[j], b[k].pos = j; // b[].pos keeps the index in p->a[]
-	ks_heapmake_mz(k, b); // initialize the binomial heap
-	for (; j < en; ++j) { // if there are more, choose top max_high_occ
-		if (mz_lt(p->a[j], b[0])) { // then update the heap
-			b[0] = p->a[j], b[0].pos = j;
-			ks_heapdown_mz(0, k, b);
-		}
-	}
-	//ks_heapsort_mz(k, b); // sorting is not needed for now
-	for (j = 0; j < k; ++j)
-		if (b[j].rid < pe - ps || force)
-			p->a[b[j].pos].rid = 0;
-}
-
-void select_mz(ha_mz1_v *p, int len, int sample_dist, int32_t dp_min_len)
-{ // for high-occ minimizers, choose up to max_high_occ in each high-occ streak
-	int32_t i, last0 = -1, n = (int32_t)p->n, m = 0, nw[2], min_len;
-	ha_mz1_t b[MAX_MAX_HIGH_OCC]; // this is to avoid a heap allocation
-
-	if (n == 0 || n == 1) return;
-	assert(n < 1<<27); // 27 is the number of bits for ha_mz1_t::pos; this should be safe as there are more bases than minimizers
-	for (i = 0; i < n; ++i)
-		if (p->a[i].rid != 0) ++m;
-	if (m == 0) return; // no high-frequency k-mers; do nothing
-	for (i = 0; i <= n; ++i) {
-		if (i == n || p->a[i].rid == 0) {
-			if (i - last0 > 1) {
-				hf_select(p, last0, i, n, len, sample_dist, b, 0);
-				// int32_t ps = last0 < 0? 0 : p->a[last0].pos;
-				// int32_t pe = i == n? len : p->a[i].pos;
-				// int32_t j, k, st = last0 + 1, en = i;
-				// int32_t max_high_occ = (int32_t)((double)(pe - ps) / sample_dist + .499);
-				// if (max_high_occ > MAX_MAX_HIGH_OCC)
-				// 	max_high_occ = MAX_MAX_HIGH_OCC;
-				// for (j = st, k = 0; j < en && k < max_high_occ; ++j, ++k)
-				// 	b[k] = p->a[j], b[k].pos = j; // b[].pos keeps the index in p->a[]
-				// ks_heapmake_mz(k, b); // initialize the binomial heap
-				// for (; j < en; ++j) { // if there are more, choose top max_high_occ
-				// 	if (mz_lt(p->a[j], b[0])) { // then update the heap
-				// 		b[0] = p->a[j], b[0].pos = j;
-				// 		ks_heapdown_mz(0, k, b);
-				// 	}
-				// }
-				// //ks_heapsort_mz(k, b); // sorting is not needed for now
-				// for (j = 0; j < k; ++j)
-				// 	if (b[j].rid < pe - ps)
-				// 		p->a[b[j].pos].rid = 0;
-			}
-			last0 = i;
-		}
-	}
-
-	min_len = MAX(dp_min_len, (p->a[0].pos+1)+sample_dist);
-	for (i = 0, nw[0] = nw[1] = 0; i < n; i++)
-    {
-		nw[(p->a[i].rid!=0)]++;
-        if((p->a[i].pos + 1) > min_len) break;
-    }
-	if(nw[0]==0 && nw[1]>0) hf_select(p, -1, i, n, len, sample_dist, b, 1);
-
-	min_len = MAX(dp_min_len, (len - (p->a[n-1].pos + 1 - p->a[n-1].span))+sample_dist);
-	for (i = n-1, nw[0] = nw[1] = 0; i >= 0; i--)
-	{
-		nw[(p->a[i].rid!=0)]++;
-		if((len - (p->a[i].pos + 1 - p->a[i].span)) > min_len) break;
-	}
-	if(nw[0]==0 && nw[1]>0) hf_select(p, i, n, n, len, sample_dist, b, 1);
-
-	for (i = n = 0; i < (int32_t)p->n; ++i) // squeeze out filtered minimizers
-		if (p->a[i].rid == 0)
-			p->a[n++] = p->a[i];
-	p->n = n;
-}
-
-static inline int mzcmp_l(const ha_mz1_v *p, int32_t ai, int32_t bi)
-{
-	if(ai >= 0 && bi >= 0){
-		ha_mz1_t *a = &(p->a[ai]), *b = &(p->a[bi]);
-		if(a->rid > 0 && b->rid > 0) return mzcmp(a, b);
-		return (a->rid == 0) - (b->rid == 0);
-	}
-	return (ai < 0) - (bi < 0);
-}
-
-#define GL(x, i) ((int64_t)((uint32_t)((x).a[(i)])))
-#define A_M(p, i) ((i) >= 0 && (p).a[(i)].rid > 0)
-int32_t qfw(ha_mz1_v *p, st_mt_t *mt, int32_t n, int32_t tot_l, int32_t ws, int32_t i, int32_t *mi)
-{
-	int32_t m, si;
-	for (si = i, (*mi) = -1; i < n; i++){
-		if(GL(*mt, i) >= ws || (i+1 < n && GL(*mt, i) < ws && GL(*mt, i+1) > ws) || 
-				(i+1 == n && tot_l >= ws && GL(*mt, i) < ws)){
-			for (m = si; m <= i; m++){
-				if(!A_M(*p, m)) continue;
-				if(mzcmp_l(p, *mi, m) >= 0) (*mi) = m;
-			}
-			if((*mi) >= 0 && A_M(*p, *mi)){
-				for (m = si; m <= i; m++){
-					if(!A_M(*p, m)) continue;
-					if(mzcmp_l(p, *mi, m) == 0) mt->a[m] |= 0x100000000;
-				}
-			}
-			break;
-		}
-	}
-	return i;
-}
-
-void dbg_boundary(ha_mz1_v *p, st_mt_t *mt, int32_t w, int32_t k, int32_t tot_l)
-{
-	if(tot_l < w + k -1) return;
-	int32_t i, m, n = p->n, s, a;
-	for (i = 0; i < n; i++){
-		if(GL(*mt, i) >= w+k-1){
-			for (m = s = a = 0; m <= i; m++){
-				if(!A_M(*p, m)) continue;
-				if(GL(*mt, m) <= w+k-1){
-					a++;
-					if(mt->a[m]&0x100000000) s++;
-				}
-			}
-			if(a > 0 && s == 0){
-				fprintf(stderr, "\nERROR1, s: %d, n: %d, tot_l: %d, end_l: %ld\n", s, n, tot_l, GL(*mt, i));
-				for (m = s = a = 0; m <= i; m++){
-					if(!A_M(*p, m)) continue;
-					if(GL(*mt, m) <= w+k-1){
-						fprintf(stderr, "lp: %ld\n", GL(*mt, m));
-						a++;
-						if(mt->a[m]&0x100000000) s++;
-					}
-				}
-				
-			} 
-			
-			break;
-		}
-	}
-	if(i == n){
-		for (m = s = a = 0; m < n; m++){
-			if(!A_M(*p, m)) continue;
-			if(GL(*mt, m) <= w+k-1){
-				a++;
-				if(mt->a[m]&0x100000000) s++;
-			}
-		}
-		if(a > 0 && s == 0) fprintf(stderr, "ERROR2\n");
-	}
-
-	for (i = n-1; i >= 0; i--)
-	{
-		if (GL(*mt, i) + w <= tot_l + 1) {
-			for (m = i, s = a = 0; m < n; m++){
-				if(!A_M(*p, m)) continue;
-				if(GL(*mt, m) + w >= tot_l + 1){
-					a++;
-					if(mt->a[m]&0x100000000) s++;
-				} 				
-			}
-			if(a > 0 && s == 0) {
-				fprintf(stderr, "\nERROR3, s: %d, n: %d, tot_l: %d, end_l: %ld\n", s, n, tot_l, GL(*mt, i));
-				for (m = i, s = a = 0; m < n; m++){
-					if(!A_M(*p, m)) continue;
-					if(GL(*mt, m) + w >= tot_l + 1){
-						fprintf(stderr, "lp: %ld\n", GL(*mt, m));
-						a++;
-						if(mt->a[m]&0x100000000) s++;
-					} 				
-				}
-			}
-			
-			break;
-		}
-	}
-	
-	if(i < 0){
-		for (m = s = a = 0; m < n; m++){
-			if(!A_M(*p, m)) continue;
-			if(GL(*mt, m) + w >= tot_l + 1){
-				a++;
-				if(mt->a[m]&0x100000000) s++;
-			} 				
-		}
-		if(a > 0 && s == 0) fprintf(stderr, "ERROR4\n");
-	}
-}
-
-static void select_mz_h(ha_mz1_v *p, st_mt_t *mt, int len, int sample_dist, int32_t w, int32_t k, int32_t tot_l)
-{ // for high-occ minimizers, choose up to max_high_occ in each high-occ streak
-	int32_t i, mi = -1, si, last0 = -1, n = (int32_t)p->n, m = 0, ws = w + k - 1;
-
-	if (n == 0) return;
-	assert(n < 1<<27); // 27 is the number of bits for ha_mz1_t::pos; this should be safe as there are more bases than minimizers
-
-	for (i = m = 0, last0 = -1; i <= n; ++i) {
-		if (i == n || p->a[i].rid == 0) {
-			if (i - last0 > 1) {
-				int32_t ps = last0 < 0? 0 : p->a[last0].pos;
-				int32_t pe = i == n? len : p->a[i].pos;
-				if(((int32_t)((double)(pe - ps) / sample_dist + .499)) > 0){
-					last0 = -2;
-					m++;
-					break;
-				}
-			}
-			last0 = i;
-		}
-	}
-	if (m == 0) return; // no high-frequency k-mers; do nothing
-	if(last0 >= -1) goto ff;
-	i = 0;
-	i = qfw(p, mt, n, tot_l, ws, i, &mi);
-	if(i == n) goto ff;
-	
-	for (si = 0, i++; i < n; i++){
-		for (; si < i; si++){
-			if(GL(*mt, si) + w > GL(*mt, i)) break;
-		}
-
-		// a new minimum; then write the old min
-		if(mzcmp_l(p, i, mi) <= 0) {
-			if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;
-			mi = i;
-		}// old min has moved outside the window
-		else if(si > mi){
-			if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;
-			for (m = si, mi = -1; m <= i; m++){
-				if(mzcmp_l(p, mi, m) >= 0) mi = m;
-            }
-			if(A_M(*p, mi)){
-				for (m = si; m <= i; m++){
-					if(!A_M(*p, m)) continue;
-					if(mzcmp_l(p, mi, m) == 0) mt->a[m] |= 0x100000000;
-				}
-			}
-		}
-	}	
-
-	if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;
-
-	for (i = n - 1; si < n && GL(*mt, si) + w <= tot_l + 1; si++){
-		if(si > mi){
-			if(A_M(*p, mi)) mt->a[mi] |= 0x100000000;
-			for (m = si, mi = -1; m <= i; m++){
-				if(mzcmp_l(p, mi, m) >= 0) mi = m;
-            }
-			if(A_M(*p, mi)){
-				for (m = si; m <= i; m++){
-					if(!A_M(*p, m)) continue;
-					if(mzcmp_l(p, mi, m) == 0) mt->a[m] |= 0x100000000;
-				}
-			}
-		}
-	}
-	
-	/**
-	dbg_boundary(p, mt, w, k, tot_l);
-	fprintf(stderr, "\n");
-	for (i = 0; i < (int32_t)p->n; ++i){
-		if(p->a[i].rid == 0) continue;
-		fprintf(stderr, "%cl: %u, pos: %lu, cnt: %lu, key: %lu, i: %d\n", "+-"[!!(mt->a[i]&0x100000000)], 
-													(uint32_t)mt->a[i], p->a[i].pos, p->a[i].rid, p->a[i].x, i);
-		// if (mt->a[i]&0x100000000){
-		// 	fprintf(stderr, "+l: %u, pos: %lu, cnt: %lu\n", (uint32_t)mt->a[i], p->a[i].pos, p->a[i].rid);
-		// }
-	}
-	**/
-	ha_mz1_t b[MAX_MAX_HIGH_OCC];
-	for (i = 0, last0 = -1; i <= n; ++i) {
-		if (i == n || p->a[i].rid == 0) {
-			if (i - last0 > 1) {
-				int32_t ps = last0 < 0? 0 : p->a[last0].pos;
-				int32_t pe = i == n? len : p->a[i].pos;
-				if(((int32_t)((double)(pe - ps) / sample_dist + .499)) > 0){
-					for (m = last0 + 1, mi = 0; m < i; ++m){
-						if(mt->a[m]&0x100000000) p->a[m].rid = 0, mi++;
-					}
-					if(mi == 0) hf_select(p, last0, i, n, len, sample_dist, b, 0);
-				} 
-			}
-			last0 = i;
-		}
-	}
-
-	ff:
-	for (i = n = 0; i < (int32_t)p->n; ++i) // squeeze out filtered minimizers
-		if (p->a[i].rid == 0)
-			p->a[n++] = p->a[i];
-	p->n = n;
-}
-
-void debug_pl(const char *str, int len, int w, int k, int is_hpc, ha_mz1_v *p, const void *hf, st_mt_t *mt)
-{
-	int i, l, dbi, dbcnt = 0, kmer_span = 0;
-	tiny_queue_t tq;
-	memset(&tq, 0, sizeof(tiny_queue_t));
-	uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0};
-	
-    for (i = l = dbi = 0; i < len; ++i) {
-        int c = seq_nt4_table[(uint8_t)str[i]];
-        if (c < 4) { // not an ambiguous base
-            int z;
-            if (is_hpc) {
-                int skip_len = 1;
-                if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {
-                    for (skip_len = 2; i + skip_len < len; ++skip_len)
-                        if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)
-                            break;
-                    i += skip_len - 1; // put $i at the end of the current homopolymer run
-                }
-                tq_push(&tq, skip_len);
-                kmer_span += skip_len;
-                ///how many bases that are covered by this HPC k-mer
-                ///kmer_span includes at most k HPC elements
-                if (tq.count > k) kmer_span -= tq_shift(&tq);
-            } else kmer_span = l + 1 < k? l + 1 : k; 
-            ///kmer_span should be used for HPC k-mer
-            ///non-HPC k-mer, kmer_span should be k
-            ///kmer_span is used to calculate anchor pos on reverse complementary strand
-
-            kmer[0] = (kmer[0] << 1 | (c&1))  & mask;                  // forward k-mer
-            kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
-            kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1))  << shift1; // reverse k-mer
-            kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;
-            if (kmer[1] == kmer[3]) continue; // skip "symmetric k-mers" as we don't know it strand
-            z = kmer[1] < kmer[3]? 0 : 1; // strand
-            ++l;
-
-            if (l >= k && kmer_span < 256) {
-                uint64_t y;
-                int32_t cnt;
-                y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
-                cnt = hf? ha_ft_cnt(hf, y) : 0;
-
-				for (dbi = 0; dbi < mt->n; dbi++)
-				{
-					if(p->a[dbi].x == y && p->a[dbi].rid == cnt && p->a[dbi].pos == i && p->a[dbi].rev == z && p->a[dbi].span == kmer_span)
-					{
-						if(l != (int)mt->a[dbi]) fprintf(stderr, "ERROR\n");
-						dbcnt++;
-					}
-				}
-            }
-        } else l = 0, tq.count = tq.front = 0, kmer_span = 0;
-	}
-
-	if(dbcnt != mt->n) fprintf(stderr, "ERROR\n");
-	if(mt->n != (int)p->n) fprintf(stderr, "ERROR\n");
-	for (dbi = 1; dbi < mt->n; dbi++)
-	{
-		if(p->a[dbi].pos <= p->a[dbi-1].pos || (int)mt->a[dbi] <= (int)mt->a[dbi-1])
-		{
-			fprintf(stderr, "ERROR\n");
-		}
-	}
+	int x;
+	if (q->count == 0) return -1;
+	x = q->a[q->front++];
+	q->front &= 0x3f;
+	--q->count;
+	return x;
 }
 
 /**
@@ -582,164 +36,134 @@ void debug_pl(const char *str, int len, int w, int k, int is_hpc, ha_mz1_v *p, c
  * @param is_hpc homopolymer-compressed or not
  * @param p      minimizers
  */
-void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, int sample_dist, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct, ha_pt_t *pt, int min_freq, int32_t dp_min_len, float dp_e, st_mt_t *mt, int32_t ws)
-{   ///in default, w = 51, k = 51, is_hpc = 1
-    /**
-     uint64_t x;
-     uint64_t rid:28, pos:27, rev:1, span:8;
-     **/
-    extern void *ha_ct_table;
-    static const ha_mz1_t dummy = { UINT64_MAX, (1<<28) - 1, 0, 0, 0};
-    uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0};
-    int i, j, l, tl = 0, buf_pos, min_pos, kmer_span = 0;
-    ha_mz1_t buf[256], min = dummy;
-    uint32_t buf_p[256], min_s = (uint32_t)-1;
-    tiny_queue_t tq;
-
-    assert(len > 0 && len < 1<<27 && rid < 1<<28 && (w > 0 && w < 256) && (k > 0 && k <= 63));
-    if (dbg_ct != NULL) dbg_ct->a.n = 0;
-    if (k_flag != NULL) {
-        kv_resize(uint8_t, k_flag->a, (uint64_t)len);
-        k_flag->a.n = len; 
-        memset(k_flag->a.a, 0, k_flag->a.n);
-    } 
-
-    memset(buf, 0xff, w * sizeof(ha_mz1_t));
-    memset(&tq, 0, sizeof(tiny_queue_t));
-    ///len/w is the evaluated minimizer numbers
-    kv_resize(ha_mz1_t, *p, p->n + len/w);
-    kv_resize(uint64_t, *mt, (int64_t)p->m); mt->n = p->n;
-
-    for (i = l = tl = buf_pos = min_pos = 0; i < len; ++i) {
-        int c = seq_nt4_table[(uint8_t)str[i]];
-        ha_mz1_t info = dummy;
-        if (c < 4) { // not an ambiguous base
-            int z;
-            if (is_hpc) {
-                int skip_len = 1;
-                if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {
-                    for (skip_len = 2; i + skip_len < len; ++skip_len)
-                        if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)
-                            break;
-                    i += skip_len - 1; // put $i at the end of the current homopolymer run
-                }
-                tq_push(&tq, skip_len);
-                kmer_span += skip_len;
-                ///how many bases that are covered by this HPC k-mer
-                ///kmer_span includes at most k HPC elements
-                if (tq.count > k) kmer_span -= tq_shift(&tq);
-            } else kmer_span = l + 1 < k? l + 1 : k; 
-            ///kmer_span should be used for HPC k-mer
-            ///non-HPC k-mer, kmer_span should be k
-            ///kmer_span is used to calculate anchor pos on reverse complementary strand
-
-            if (k_flag != NULL) k_flag->a.a[i] = 1;///lable all useful base, which are not ignored by HPC
-
-            kmer[0] = (kmer[0] << 1 | (c&1))  & mask;                  // forward k-mer
-            kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
-            kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1))  << shift1; // reverse k-mer
-            kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;
-            if (kmer[1] == kmer[3]) continue; // skip "symmetric k-mers" as we don't know it strand
-            z = kmer[1] < kmer[3]? 0 : 1; // strand
-            ++l; tl++;
-            if (l >= k && kmer_span < 256) {
-                uint64_t y;
-                int32_t cnt, filtered;
-                y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
-                cnt = hf? ha_ft_cnt(hf, y) : 0;
-                filtered = (cnt >= 1<<28);
-                if (dbg_ct != NULL) kv_push(uint64_t, dbg_ct->a, ((((uint64_t)(query_ct_index(ha_ct_table, y))<<1)|filtered)<<32)|(uint64_t)(i));
-                if (!filtered) info.x = y, info.rid = cnt, info.pos = i, info.rev = z, info.span = kmer_span; // initially ha_mz1_t::rid keeps the k-mer count
-                if (k_flag != NULL) k_flag->a.a[i]++;
-                if (k_flag != NULL && filtered > 0) k_flag->a.a[i]++;
-            }
-        } else l = 0, tq.count = tq.front = 0, kmer_span = 0;
-
-        buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
-        buf_p[buf_pos] = l;
-        if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
-			for (j = buf_pos + 1; j < w; ++j){
-                if (mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos){
-                    kv_push(ha_mz1_t, *p, buf[j]); kv_push(uint64_t, *mt, buf_p[j]);
-                }
-            }
-            for (j = 0; j < buf_pos; ++j){
-                if (mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos){
-                    kv_push(ha_mz1_t, *p, buf[j]); kv_push(uint64_t, *mt, buf_p[j]);
-                } 
-            }
-        }
-
-        /**
-         * There are three cases:
-         * 1. info.x <= min.x, means info is a new minimizer
-         * 2. info.x > min.x, info is not a new minimizer
-         *    (1) buf_pos != min_pos, do nothing
-         *    (2) buf_pos == min_pos, means current minimizer has moved outside the window
-         * **/
-        ///three cases: 1. 
-        if (mzcmp(&min, &info) >= 0) { // a new minimum; then write the old min
-            if (l >= w + k && min.x != UINT64_MAX){
-                kv_push(ha_mz1_t, *p, min); kv_push(uint64_t, *mt, min_s);
-            } 
-            min = info, min_pos = buf_pos, min_s = buf_p[buf_pos];
-        } else if (buf_pos == min_pos) { // old min has moved outside the window
-            if (l >= w + k - 1 && min.x != UINT64_MAX){
-                kv_push(ha_mz1_t, *p, min); kv_push(uint64_t, *mt, min_s);
-            } 
-            ///buf_pos == min_pos, means current minimizer has moved outside the window
-            ///so for now we need to find a new minimizer at the current window (w k-mers)
-            for (j = buf_pos + 1, min = dummy; j < w; ++j) // the two loops are necessary when there are identical k-mers
-                if (mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j, min_s = buf_p[j]; // >= is important s.t. min is always the closest k-mer
-            for (j = 0; j <= buf_pos; ++j)
-                if (mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j, min_s = buf_p[j];
-
-            if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
-                for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
-                    if (mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos){
-                        kv_push(ha_mz1_t, *p, buf[j]); kv_push(uint64_t, *mt, buf_p[j]);
-                    }
-                for (j = 0; j <= buf_pos; ++j)
-                    if (mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos){
-                        kv_push(ha_mz1_t, *p, buf[j]); kv_push(uint64_t, *mt, buf_p[j]);
-                    } 
-            }
-        }
-        if (++buf_pos == w) buf_pos = 0;
-    }
-    if (min.x != UINT64_MAX){
-        kv_push(ha_mz1_t, *p, min); kv_push(uint64_t, *mt, min_s); 
-    }
-	// debug_pl(str, len, w, k, is_hpc, p, hf, mt);
-    // if (sample_dist > w) select_mz(p, len, MAX_HIGH_OCC, dp_min_len);
-    select_mz_h(p, mt, len, sample_dist, ws, k, tl);
-    if (dp_min_len > 0 && pt && mt) refine_sketch(p, pt, len, dp_min_len, dp_e, min_freq, mt);
-    for (i = 0; i < (int)p->n; ++i) // populate .rid as this was keeping counts
-        p->a[i].rid = rid;
-}
-
-void ha_sketch_worse(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, int sample_dist, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct, ha_pt_t *pt, int min_freq, int32_t dp_min_len, float dp_e, st_mt_t *mt)
+void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf)
 {	///in default, w = 51, k = 51, is_hpc = 1
 	/**
 	 uint64_t x;
 	 uint64_t rid:28, pos:27, rev:1, span:8;
 	 **/
-	extern void *ha_ct_table;
-	static const ha_mz1_t dummy = { UINT64_MAX, (1<<28) - 1, 0, 0 };
+	static const ha_mz1_t dummy = { UINT64_MAX, 0, 0, 0 };
 	uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0};
 	int i, j, l, buf_pos, min_pos, kmer_span = 0;
 	ha_mz1_t buf[256], min = dummy;
 	tiny_queue_t tq;
 
 	assert(len > 0 && len < 1<<27 && rid < 1<<28 && (w > 0 && w < 256) && (k > 0 && k <= 63));
-	if (dbg_ct != NULL) dbg_ct->a.n = 0;
-	if (k_flag != NULL) {
+	///sizeof(ha_mz1_t) = 16
+	memset(buf, 0xff, w * 16);
+	memset(&tq, 0, sizeof(tiny_queue_t));
+	///len/w is the evaluated minimizer numbers
+	kv_resize(ha_mz1_t, *p, p->n + len/w);
+
+	for (i = l = buf_pos = min_pos = 0; i < len; ++i) {
+		int c = seq_nt4_table[(uint8_t)str[i]];
+		ha_mz1_t info = dummy;
+		if (c < 4) { // not an ambiguous base
+			int z;
+			if (is_hpc) {
+				int skip_len = 1;
+				if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {
+					for (skip_len = 2; i + skip_len < len; ++skip_len)
+						if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)
+							break;
+					i += skip_len - 1; // put $i at the end of the current homopolymer run
+				}
+				tq_push(&tq, skip_len);
+				kmer_span += skip_len;
+				if (tq.count > k) kmer_span -= tq_shift(&tq);
+			} else kmer_span = l + 1 < k? l + 1 : k; 
+			///kmer_span should be used for HPC k-mer
+			///so for non-HPC k-mer, kmer_span should be k in any case?
+			///kmer_span is used to calculate anchor pos on reverse complementary strand
+
+			kmer[0] = (kmer[0] << 1 | (c&1))  & mask;                  // forward k-mer
+			kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
+			kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1))  << shift1; // reverse k-mer
+			kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;
+			if (kmer[1] == kmer[3]) continue; // skip "symmetric k-mers" as we don't know it strand
+			z = kmer[1] < kmer[3]? 0 : 1; // strand
+			++l;
+			if (l >= k && kmer_span < 256) {
+				uint64_t y;
+				y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
+				if (hf == 0 || ha_ft_isflt(hf, y) == 0)
+					info.x = y, info.rid = rid, info.pos = i, info.rev = z, info.span = kmer_span;
+			}
+		} else l = 0, tq.count = tq.front = 0, kmer_span = 0;
+
+
+		//for non-HPC k-mer, l = i; but for HPC k-mer, l is always less than i
+		//i is the real base iterator, while l is the HPC base iterator
+		//only if l >= k, info is a useful minimizer (ha_mz1_t.x != UINT64_MAX)
+		//but even if l < k, infor is still stored into buf
+		buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
+		if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
+			for (j = buf_pos + 1; j < w; ++j)
+				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
+			for (j = 0; j < buf_pos; ++j)
+				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
+		}
+		/**
+		 * There are three cases:
+		 * 1. info.x <= min.x, means info is a new minimizer
+		 * 2. info.x > min.x, info is not a new minimizer
+		 *    (1) buf_pos != min_pos, do nothing
+		 *    (2) buf_pos == min_pos, means current minimizer has moved outside the window
+		 * **/
+		///three cases: 1. 
+		if (info.x <= min.x) { // a new minimum; then write the old min
+			if (l >= w + k && min.x != UINT64_MAX) kv_push(ha_mz1_t, *p, min);
+			min = info, min_pos = buf_pos;
+		} else if (buf_pos == min_pos) { // old min has moved outside the window
+			if (l >= w + k - 1 && min.x != UINT64_MAX) kv_push(ha_mz1_t, *p, min);
+			///buf_pos == min_pos, means current minimizer has moved outside the window
+			///so for now we need to find a new minimizer at the current window (w k-mers)
+			for (j = buf_pos + 1, min.x = UINT64_MAX; j < w; ++j) // the two loops are necessary when there are identical k-mers
+				if (min.x >= buf[j].x) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
+			for (j = 0; j <= buf_pos; ++j)
+				if (min.x >= buf[j].x) min = buf[j], min_pos = j;
+
+			if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
+				for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
+					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
+				for (j = 0; j <= buf_pos; ++j)
+					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
+			}
+		}
+		if (++buf_pos == w) buf_pos = 0;
+	}
+	if (min.x != UINT64_MAX)
+		kv_push(ha_mz1_t, *p, min);
+}
+
+
+
+void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, 
+kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct)
+{	///in default, w = 51, k = 51, is_hpc = 1
+	/**
+	 uint64_t x;
+	 uint64_t rid:28, pos:27, rev:1, span:8;
+	 **/
+	extern void *ha_ct_table;
+	if(dbg_ct != NULL) dbg_ct->a.n = 0;
+
+	static const ha_mz1_t dummy = { UINT64_MAX, 0, 0, 0 };
+	uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0}, filtered;
+	int i, j, l, buf_pos, min_pos, kmer_span = 0;
+	ha_mz1_t buf[256], min = dummy;
+	tiny_queue_t tq;
+	if(k_flag != NULL)
+	{
 		kv_resize(uint8_t, k_flag->a, (uint64_t)len);
 		k_flag->a.n = len; 
 		memset(k_flag->a.a, 0, k_flag->a.n);
 	} 
 
-	memset(buf, 0xff, w * sizeof(ha_mz1_t));
+
+	assert(len > 0 && len < 1<<27 && rid < 1<<28 && (w > 0 && w < 256) && (k > 0 && k <= 63));
+	///sizeof(ha_mz1_t) = 16
+	memset(buf, 0xff, w * 16);
 	memset(&tq, 0, sizeof(tiny_queue_t));
 	///len/w is the evaluated minimizer numbers
 	kv_resize(ha_mz1_t, *p, p->n + len/w);
@@ -767,7 +191,7 @@ void ha_sketch_worse(const char *str, int len, int w, int k, uint32_t rid, int i
 			///non-HPC k-mer, kmer_span should be k
 			///kmer_span is used to calculate anchor pos on reverse complementary strand
 
-			if (k_flag != NULL) k_flag->a.a[i] = 1;///lable all useful base, which are not ignored by HPC
+			if(k_flag != NULL) k_flag->a.a[i] = 1;///lable all useful base, which are not ignored by HPC
 
 			kmer[0] = (kmer[0] << 1 | (c&1))  & mask;                  // forward k-mer
 			kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
@@ -778,17 +202,20 @@ void ha_sketch_worse(const char *str, int len, int w, int k, uint32_t rid, int i
 			++l;
 			if (l >= k && kmer_span < 256) {
 				uint64_t y;
-				int32_t cnt, filtered;
 				y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
-				cnt = hf? ha_ft_cnt(hf, y) : 0;
-				filtered = (cnt >= 1<<28);
-				if (dbg_ct != NULL) kv_push(uint64_t, dbg_ct->a, ((((uint64_t)(query_ct_index(ha_ct_table, y))<<1)|filtered)<<32)|(uint64_t)(i));
-				if (!filtered) info.x = y, info.rid = pt? ha_pt_cnt(pt, y):cnt, info.pos = i, info.rev = z, info.span = kmer_span; // initially ha_mz1_t::rid keeps the k-mer count
-				if (k_flag != NULL) k_flag->a.a[i]++;
-				if (k_flag != NULL && filtered > 0) k_flag->a.a[i]++;
+
+				filtered = 0;
+				if(hf != 0) filtered = ha_ft_isflt(hf, y);
+				if(dbg_ct != NULL) kv_push(uint64_t, dbg_ct->a, ((((uint64_t)(query_ct_index(ha_ct_table, y))<<1)|filtered)<<32)|(uint64_t)(i));
+				///if (hf == 0 || ha_ft_isflt(hf, y) == 0)
+				if(filtered == 0)
+					info.x = y, info.rid = rid, info.pos = i, info.rev = z, info.span = kmer_span;
+				if(k_flag != NULL) k_flag->a.a[i]++;
+				if(k_flag != NULL && filtered == 1) k_flag->a.a[i]++;
 			}
 		} else l = 0, tq.count = tq.front = 0, kmer_span = 0;
 
+
 		//for non-HPC k-mer, l = i; but for HPC k-mer, l is always less than i
 		//i is the real base iterator, while l is the HPC base iterator
 		//only if l >= k, info is a useful minimizer (ha_mz1_t.x != UINT64_MAX)
@@ -796,9 +223,9 @@ void ha_sketch_worse(const char *str, int len, int w, int k, uint32_t rid, int i
 		buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
 		if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
 			for (j = buf_pos + 1; j < w; ++j)
-				if (mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
+				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
 			for (j = 0; j < buf_pos; ++j)
-				if (mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
+				if (min.x == buf[j].x && buf[j].pos != min.pos) kv_push(ha_mz1_t, *p, buf[j]);
 		}
 		/**
 		 * There are three cases:
@@ -816,23 +243,19 @@ void ha_sketch_worse(const char *str, int len, int w, int k, uint32_t rid, int i
 			///buf_pos == min_pos, means current minimizer has moved outside the window
 			///so for now we need to find a new minimizer at the current window (w k-mers)
 			for (j = buf_pos + 1, min.x = UINT64_MAX; j < w; ++j) // the two loops are necessary when there are identical k-mers
-				if (mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
+				if (min.x >= buf[j].x) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
 			for (j = 0; j <= buf_pos; ++j)
-				if (mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j;
+				if (min.x >= buf[j].x) min = buf[j], min_pos = j;
 
 			if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
 				for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
-					if (mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
+					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
 				for (j = 0; j <= buf_pos; ++j)
-					if (mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
+					if (min.x == buf[j].x && min.pos != buf[j].pos) kv_push(ha_mz1_t, *p, buf[j]);
 			}
 		}
 		if (++buf_pos == w) buf_pos = 0;
 	}
 	if (min.x != UINT64_MAX)
 		kv_push(ha_mz1_t, *p, min);
-	if (dp_min_len > 0 && pt && mt) refine_sketch(p, pt, len, dp_min_len, dp_e, min_freq, mt);
-	// if (sample_dist > w) select_mz(p, len, MAX_HIGH_OCC);
-	for (i = 0; i < (int)p->n; ++i) // populate .rid as this was keeping counts
-		p->a[i].rid = rid;
-}
+}

tovlp.h

@@ -1,26 +0,0 @@
-#ifndef __TOVLP__
-#define __TOVLP__
-#include <stdint.h>
-#include "Overlaps.h"
-
-typedef struct {///[cBeg, cEnd)
-	uint32_t ui, len, cBeg, cEnd;
-    uint32_t *a, an;
-    ma_ug_t *ug;
-    utg_trans_t *o;
-} utg_trans_hit_idx;
-
-utg_trans_t *init_utg_trans_t(ma_ug_t *ug, ma_hit_t_alloc* reverse_sources, ma_sub_t *coverage_cut, R_to_U* ruIndex, asg_t *read_g, int max_hang, int min_ovlp);
-void destroy_utg_trans_t(utg_trans_t **o);
-void asg_bub_collect_ovlp(ma_ug_t *ug, uint32_t v0, buf_t *b, utg_trans_t *o);
-void collect_trans_ovlp(const char* cmd, buf_t* pri, uint64_t pri_offset, buf_t* aux, uint64_t aux_offset,
-ma_ug_t *ug, utg_trans_t *o);
-int asg_arc_decompress(asg_t *g, ma_ug_t *ug, asg_t *read_sg, ma_hit_t_alloc* reverse_sources,
-R_to_U* ruIndex, utg_trans_t *o);
-int asg_arc_decompress_mul(asg_t *g, ma_ug_t *ug, asg_t *read_sg, uint32_t positive_flag, uint32_t negative_flag,
-ma_hit_t_alloc* reverse_sources, R_to_U* ruIndex, utg_trans_t *o);
-kv_u_trans_t *pt_pdist(ma_ug_t *ug, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources, 
-kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, uint32_t min_chain_cnt);
-void reset_utg_trans_hit_idx(utg_trans_hit_idx *t, uint32_t* i_x_a, uint32_t i_x_n, ma_ug_t *i_ug, 
-utg_trans_t *i_o, uint32_t i_cBeg, uint32_t i_cEnd);
-#endif