diff --git a/CommandLines.cpp b/CommandLines.cpp index b7de8b8..739e44a 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -199,6 +199,7 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->n_weight = 3; asm_opt->is_alt = 0; asm_opt->misjoin_len = 500000; + asm_opt->scffold = 0; } void destory_enzyme(enzyme* f) diff --git a/CommandLines.h b/CommandLines.h index e7b9b1c..0da1f99 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -4,7 +4,7 @@ #include #include -#define HA_VERSION "0.15.4-r347" +#define HA_VERSION "0.15.4-r348" #define VERBOSE 0 @@ -105,6 +105,7 @@ typedef struct { int32_t n_weight; uint32_t is_alt; uint64_t misjoin_len; + uint64_t scffold; } hifiasm_opt_t; extern hifiasm_opt_t asm_opt; diff --git a/Overlaps.cpp b/Overlaps.cpp index 300929c..c224a11 100644 --- a/Overlaps.cpp +++ b/Overlaps.cpp @@ -13220,6 +13220,8 @@ long long gap_fuzz, bub_label_t* b_mask_t) { hic_clean(sg); ug_opt_t opt; memset(&opt, 0, sizeof(opt)); + kvec_pe_hit *rhits = NULL; + ma_ug_t *ug_fa = NULL, *ug_mo = NULL; opt.coverage_cut = coverage_cut; opt.sources = sources; opt.reverse_sources = reverse_sources; @@ -13289,18 +13291,18 @@ long long gap_fuzz, bub_label_t* b_mask_t) // fclose(output_file); // free(gfa_name); - hic_analysis(ug, sg, cov?cov->t_ch:t_ch, &opt, 0); + hic_analysis(ug, sg, cov?cov->t_ch:t_ch, &opt, 0, asm_opt.scffold?&rhits:NULL); - if(cov) destory_hap_cov_t(&cov); - if(t_ch) destory_trans_chain(&t_ch); + if(!rhits && cov) destory_hap_cov_t(&cov); + if(!rhits && t_ch) destory_trans_chain(&t_ch); - char* gfa_name = (char*)malloc(strlen(output_file_name)+50); - sprintf(gfa_name, "%s.after.clean_d_utg.noseq.gfa", output_file_name); - FILE* output_file = fopen(gfa_name, "w"); - ma_ug_print_simple(ug, sg, coverage_cut, sources, ruIndex, "utg", output_file); - fclose(output_file); - free(gfa_name); + // char* gfa_name = (char*)malloc(strlen(output_file_name)+50); + // sprintf(gfa_name, "%s.after.clean_d_utg.noseq.gfa", output_file_name); + // FILE* output_file = fopen(gfa_name, "w"); + // ma_ug_print_simple(ug, sg, coverage_cut, sources, ruIndex, "utg", output_file); + // fclose(output_file); + // free(gfa_name); ma_ug_destroy(ug); kv_destroy(new_rtg_edges.a); @@ -13329,10 +13331,18 @@ long long gap_fuzz, bub_label_t* b_mask_t) reduce_hamming_error(sg, sources, coverage_cut, max_hang, min_ovlp, gap_fuzz); - output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang, min_ovlp, 0, b_mask_t); - output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, - 0.05, 0.9, max_hang, min_ovlp, 0, b_mask_t); + ug_fa = output_trio_unitig_graph(sg, coverage_cut, output_file_name, FATHER, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + 0.05, 0.9, max_hang, min_ovlp, rhits?1:0, b_mask_t); + ug_mo = output_trio_unitig_graph(sg, coverage_cut, output_file_name, MOTHER, sources, reverse_sources, (asm_opt.max_short_tip*2), 0.15, 3, ruIndex, + 0.05, 0.9, max_hang, min_ovlp, rhits?1:0, b_mask_t); + if(rhits) + { + ha_aware_order(rhits, sg, ug_fa, ug_mo, cov?&(cov->t_ch->k_trans):&(t_ch->k_trans), &opt, 3); + kv_destroy(rhits->a); kv_destroy(rhits->idx); kv_destroy(rhits->occ); free(rhits); + if(cov) destory_hap_cov_t(&cov); + if(t_ch) destory_trans_chain(&t_ch); + ma_ug_destroy(ug_fa); ma_ug_destroy(ug_mo); + } } ma_ug_t *get_poly_ug(asg_t *sg, ma_sub_t* coverage_cut, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, @@ -13416,7 +13426,7 @@ long long gap_fuzz, bub_label_t* b_mask_t) if((asm_opt.flag & HA_F_VERBOSE_GFA)) write_trans_chain(t_ch, output_file_name); } - hic_analysis(ug, sg, t_ch, &opt, 0); + hic_analysis(ug, sg, t_ch, &opt, 0, NULL); destory_trans_chain(&t_ch); ma_ug_destroy(ug); asg_cleanup(sg); @@ -17102,7 +17112,6 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, { char* gfa_name = (char*)malloc(strlen(output_file_name)+100); sprintf(gfa_name, "%s.%s.p_ctg.gfa", output_file_name, (flag==FATHER?"hap1":"hap2")); - fprintf(stderr, "Writing %s to disk... \n", gfa_name); FILE* output_file = NULL; if(is_bench == 0) output_file = fopen(gfa_name, "w"); @@ -17128,16 +17137,18 @@ float chimeric_rate, float drop_ratio, int max_hang, int min_ovlp, int is_bench, NULL, &asm_opt.b_high_cov, asm_opt.m_rate); } - ///debug_utg_graph(ug, sg, 0, 0); - ///debug_untig_length(ug, tipsLen, gfa_name); - ///print_untig_by_read(ug, "m64011_190901_095311/125831121/ccs", 2310925, "end"); - ma_ug_seq(ug, sg, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp, 0, 1); if(is_bench) { free(gfa_name); kv_destroy(new_rtg_edges.a); return ug; } + fprintf(stderr, "Writing %s to disk... \n", gfa_name); + ///debug_utg_graph(ug, sg, 0, 0); + ///debug_untig_length(ug, tipsLen, gfa_name); + ///print_untig_by_read(ug, "m64011_190901_095311/125831121/ccs", 2310925, "end"); + ma_ug_seq(ug, sg, coverage_cut, sources, &new_rtg_edges, max_hang, min_ovlp, 0, 1); + ma_ug_print(ug, sg, coverage_cut, sources, ruIndex, (flag==FATHER?"h1tg":"h2tg"), output_file); fclose(output_file); diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..6311774 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,257 @@ +# -*- coding: utf-8 -*- + +import sys +import os + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.todo', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'hifiasm' +copyright = u'2021, Haoyu Cheng, Heng Li' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.15.5-r348' +# The full version, including alpha/beta/rc tags. +release = '0.15.5' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# Build using the RTD theme, if not on RTD. +# https://read-the-docs.readthedocs.org/en/latest/theme.html +# https://github.com/snide/sphinx_rtd_theme +# +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [ "/usr/local/lib/python2.7/site-packages", ] + + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'hifiasm-doc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'hifiasm.tex', u'hifiasm Documentation', + u'Haoyu Cheng, Heng Li', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'hifiasm', u'hifiasm Documentation', + [u'Haoyu Cheng, Heng Li'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'hifiasm', u'hifiasm Documentation', + u'Haoyu Cheng, Heng Li', 'hifiasm', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False \ No newline at end of file diff --git a/docs/source/faq.rst b/docs/source/faq.rst new file mode 100644 index 0000000..0ec25c7 --- /dev/null +++ b/docs/source/faq.rst @@ -0,0 +1,109 @@ + +.. _faq: + +Hifiasm FAQ +=========== + + +.. contents:: + :local: + + +How do I get contigs in FASTA? +------------------------------------- + The FASTA file can be produced from GFA as follows: + :: + + awk '/^S/{print ">"$2;print $3}' test.p_ctg.gfa > test.p_ctg.fa + +Which types of assemblies should I use? +---------------------------------------- + If parental data is available, ``*dip.hap*.p_ctg.gfa`` produced in trio-binning mode should be always preferred. Otherwise if Hi-C data is available, ``*hic.hap*.p_ctg.gfa`` produced in Hi-C mode is the best choice. Both trio-binning mode and Hi-C mode generate fully-phased assemblies. + + If you only have HiFi reads, hifiasm in default outputs ``*bp.hap*.p_ctg.gfa``. The primary/alternate assemblies can be also produced by using ``--primary``. All these HiFi-only assemblies are not fully-phased. See `blog `_ here for more details. + +Are inbred/homozygous genomes supported? +-------------------------------------------------------------------------- + + Yes, please use the ``-l0`` option to disable purge duplication step. + +Are diploid genomes supported? +------------------------------------- + Yes, most modules of hifiasm are designed for diploid samples, including purge duplication step, partially phased assembly and fully-phased assembly with trio-binning or Hi-C. + +Are polyploid genomes supported? +------------------------------------- + + The ``*r_utg.gfa`` and ``*p_utg.gfa`` are lossless so that they also work for polyploid genomes. However, currently the contig-generation modules of hifiasm are designed for diploid samples, which means both the partially phased assembly and the fully-phased assembly does not directly support polyploid genomes. Please use primary assembly for polyploid samples and run multiple rounds of purging steps using third-party tools such as purge_dups. + +Why one Hi-C integrated assembly is larger than another one? +------------------------------------------------------------ + + For some samples like human male, the paternal haplotype should be larger than the maternal haplotype. However, if one assembly is much larger than another one, it should be the issues of hifiasm. To fix it, please set smaller value for ``-s`` (default: 0.55). + + Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. For instance, hifiasm prints the following information during assembly without setting the option ``--purge-cov``: + :: + + [M::purge_dups] purge duplication coverage threshold: 46 + + In this example, hifiasm identifies the coverage threshold for homozygous reads as ``46/1.25 = 36``. If it is significantly smaller than the homozygous coverage peak, hifiasm will generate two unbalanced assemblies. In this case, please set ``--purge-cov`` to homozygous coverage peak. Please note that tuning ``--purge-cov`` may affect ``*p_utg*gfa`` so that ``*hic*.bin`` should be deleted. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. + + + +For Hi-C integrated assembly, why the assembly size of both haplotypes are much larger than the estimated genome size? +------------------------------------------------------------------------------------------------------------------------------ + It is likely that hifiasm misidentifies coverage threshold for homozygous reads. Hifiasm prints the following information for debugging: + :: + + [M::stat] # heterozygous bases: 645155110; # homozygous bases: 1495396634 + + If most bases of a diploid sample are homozygous, the coverage threshold is wrongly determined by hifiasm. For instance, hifiasm prints the following information during assembly without setting the option ``--purge-cov``: + :: + + [M::purge_dups] purge duplication coverage threshold: 46 + + In this example, hifiasm identifies the coverage threshold for homozygous reads as ``46/1.25 = 36``. If it is much smaller than homozygous coverage peak, hifiasm thinks most reads are homozygous and assign them to both assemblies, making both of them much larger than the estimated haploid genome size. In this case, please set ``--purge-cov`` to homozygous coverage peak. Please note that tuning ``--purge-cov`` may affect ``*p_utg*gfa`` so that ``*hic*.bin`` should be deleted. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. + + +.. _hic-iss: + +How can I tweak parameters to improve Hi-C integrated assembly? +--------------------------------------------------------------- + Compared with the HiFi-only assembly or the trio-binning assembly, the Hi-C integrated assembly is a little bit more complex so that you need to take care of the results. See `Why one Hi-C integrated assembly is larger than another one?`_ and `For Hi-C integrated assembly, why the assembly size of both haplotypes are much larger than the estimated genome size?`_ for details on how to fix potential issues. + + There are several other options that may affect the Hi-C integrated assembly. Increasing the values of ``--n-weight``, ``--n-perturb`` and ``--f-perturb`` may improve phasing results but takes longer time. However, tuning ``--l-msjoin`` is tricky. All these options do not affect ``*p_utg*gfa`` so that ``*hic*.bin`` can be reused. + +.. _p-large: + +Why the size of primary assembly or partially phased assembly is much larger than the estimated genome size? +--------------------------------------------------------------------------------------------------------------- + It could be because the estimated genome size is incorrect. Another possibility is that hifiasm does not perform enough purging. Setting smaller value for ``-s`` (default: 0.55) or turning ``--purge-cov`` should be helpful. See :ref:`loginter` for more details. + + +Why the hamming error rate or the swith error rate of trio-binning assembly is very high? +--------------------------------------------------------------------------------------------------------------- + In rare cases, a potential issue is that a few contigs may misjoin two haplotypes. For example, half of a contig come from mother while another half come from father. Such misjoined contigs can be fixed by manually breaking. The coordinates of problematic regions can be found by A-lines in GFA file or ``yak trioeval -e`` (see `issue 37 `_ for more details). However, if there are many misjoined contigs or the switch/hamming error rate reported by ``yak trioeval`` is very high, users should check if the parental data is correct (see `issue 130 `_ for more details). + +Why does hifiasm stuck or crash? +------------------------------------- + In most cases, it is caused by the low quality HiFi reads. A good HiFi dataset should have a k-mer plot like `issue10 `_ or `issue49 `_. In contrast, low quality HiFi data often lead to weird k-mer plot like `issue93 `_. Such weird k-mer plots usually indicate insufficient coverage or presence of contaminants. See :ref:`loginter` for more details. If the HiFi data look fine, please raise an issue at the `issue page `_. + +What's the usage of different bin files in hifiasm? +---------------------------------------------------- + ``*ec.bin``, ``*ovlp.reverse.bin`` and ``*ovlp.source.bin`` save the results of error correction step. ``*hic*bin`` saves the results of Hi-C alignment. Please note that ``*hic*.bin`` should be deleted when tuning any parameters affecting ``*p_utg*gfa``. There are several parameters which does not change ``*p_utg*gfa``, including ``-s``, ``--seed``, ``--n-weight``, ``--n-perturb``, ``--f-perturb`` and ``--l-msjoin``. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. + +Can I generate HiFi-only assembly first, and then add Hi-C or trio data later? +---------------------------------------------------------------------------------------- + Yes, the HiFi-only assembly, Hi-C phased assembly and trio-binning assembly share the same ``*ec.bin``, ``*ovlp.reverse.bin`` and ``*ovlp.source.bin``. + +What is the minimum read coverage required for hifiasm? +------------------------------------------------------- + Usually >=14x HiFi reads per haplotype. Higher coverage might be able to improve the contiguity of assembly. + +Why the primary assembly is more contiguous than the fully-phased assemblies and the partially phased assemblies (i.e. ``*.hap*.p_ctg.gfa``)? +---------------------------------------------------------------------------------------------------------------------------------------------------- + + For diploid samples, primary assembly usually has greater N50 but at the expense of highly fragmented alternate assembly. From the method view, the primary assembly has an extra joining step, which joins two haplotypes to make primary assembly more contiguous. + + When producing fully-phased assemblies and partially phased assemblies, hifiasm is designed to keep both haplotypes contiguous. It is important for many downstream applications like SV calling. + diff --git a/docs/source/hic-assembly.rst b/docs/source/hic-assembly.rst new file mode 100644 index 0000000..e395356 --- /dev/null +++ b/docs/source/hic-assembly.rst @@ -0,0 +1,16 @@ + +.. _hic-assembly: + +Hi-C Integrated Assembly +======================== + +Hifiasm can generate a pair of haplotype-resolved assemblies with paired-end Hi-C reads:: + + hifiasm -o NA12878.asm -t32 --h1 read1.fq.gz --h2 read2.fq.gz HiFi-reads.fq.gz + +In this mode, each contig is supposed to be a haplotig, which by definition comes from one parental haplotype only. Hifiasm often puts all contigs from the same parental chromosome in one assembly. It has cleanly separated chrX and chrY for a human male dataset. Nonetheless, phasing across centromeres is challenging. Hifiasm is often able to phase entire chromosomes but it may fail in rare cases. Also, contigs from different parental chromosomes are randomly mixed as it is just not possible to phase across chromosomes with Hi-C. Hifiasm does not perform scaffolding for now. You need to run a standalone scaffolder such as SALSA or 3D-DNA to scaffold phased haplotigs. + + +For samples with high heterozygosity rate, a common issue is that one assembly is much larger than another one. To fix this issue, please set smaller value for ``-s`` (default: 0.55). Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. In this case, please set ``--purge-cov`` to homozygous coverage peak. See :ref:`hic-iss` for more details. + +At the first run, hifiasm saves the alignment of Hi-C reads to disk as ``*hic*.bin``. It reuses the saved results to avoid Hi-C alignment next time. Please note that ``*hic*.bin`` should be deleted when tuning any parameters affecting ``*p_utg*gfa``. Since v0.15.5, hifiasm can detect such changes and renew Hi-C bin files automatically. There are several parameters which do not change ``*p_utg*gfa``, including ``-s``, ``--seed``, ``--n-weight``, ``--n-perturb``, ``--f-perturb`` and ``--l-msjoin``. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..4746972 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,80 @@ +Hifiasm +======= + +.. toctree:: + :hidden: + + pa-assembly + trio-assembly + hic-assembly + interpreting-output + faq + parameter-reference + + + + +`Hifiasm `_ is a fast haplotype-resolved de novo assembler for PacBio HiFi reads. It can assemble a human genome in several hours and assemble a ~30Gb California redwood genome in a few days. Hifiasm emits partially phased assemblies of quality competitive with the best assemblers. Given parental short reads or Hi-C data, it produces arguably the best haplotype-resolved assemblies so far. + +Publications +============ + +Hifiasm + Haoyu Cheng, Gregory T. Concepcion, Xiaowen Feng, Haowen Zhang & Heng Li. + `Haplotype-resolved de novo assembly using phased assembly graphs with hifiasm `_. Nature Methods. (2021). + +Install +======= +The easiest way to get started is to download a `release `_. Please report any issues on `github issues `_ page. + +In addition, the latest unreleased version can be found from github: + +:: + + git clone https://github.com/chhylp123/hifiasm + cd hifiasm && make + +Another way is to install hifiasm via `bioconda `_: + +:: + + conda install -c bioconda hifiasm + +Assembly Concepts +================= +There are different types of assemblies which are commonly used in practice (see +`details `_). +Hifiasm produces primary/alternate assemblies or partially phased assemblies +only with HiFi reads. Given Hi-C data or trio-binning data, hifiasm produces +contiguous fully-phased assemblies, i.e. haplotype-resolved assemblies. + +Why Hifiasm? +============ +* Hifiasm delivers high-quality assemblies. It tends to generate longer contigs + and resolve more segmental duplications than other assemblers. + +* Given Hi-C reads or short reads from the parents, hifiasm can produce overall the best + haplotype-resolved assembly so far. It is the assembler of choice by the + `Human Pangenome Project `_ for the first batch of samples. + +* Hifiasm can purge duplications between haplotigs without relying on + third-party tools such as purge\_dups. Hifiasm does not need polishing tools + like pilon or racon, either. This simplifies the assembly pipeline and saves + running time. + +* Hifiasm is fast. It can assemble a human genome in half a day and assemble a + ~30Gb redwood genome in three days. No genome is too large for hifiasm. + +* Hifiasm is trivial to install and easy to use. It does not required Python, + R or C++11 compilers, and can be compiled into a single executable. The + default setting works well with a variety of genomes. + +Learn +===== + +* :ref:`HiFi-only Assembly ` - Assembling HiFi reads without additional data types +* :ref:`Trio-binning Assembly ` - Producing fully phased assemblies with HiFi and trio-binning data +* :ref:`Hi-C Integrated Assembly ` - Producing fully phased assemblies with HiFi and Hi-C data +* :ref:`Hifiasm Output ` - Interpreting results +* :ref:`Hifiasm FAQ ` - Frequently asked questions +* :ref:`Hifiasm Parameters ` - Parameter reference of hifiasm diff --git a/docs/source/interpreting-output.rst b/docs/source/interpreting-output.rst new file mode 100644 index 0000000..828746a --- /dev/null +++ b/docs/source/interpreting-output.rst @@ -0,0 +1,102 @@ + +.. _interpreting-output: + +Hifiasm Output +=============== + +.. _outfile: + +Output files +--------------------------------------- + +In general, hifiasm generates the following assembly graphs in the GFA format: + +* ```prefix`.r_utg.gfa``: haplotype-resolved raw unitig graph. This graph keeps all haplotype information. +* ```prefix`.p_utg.gfa``: haplotype-resolved processed unitig graph without small bubbles. Small bubbles might be caused by somatic mutations or noise in data, which are not the real haplotype information. Hifiasm automatically pops such small bubbles based on coverage. The option ``--purge-cov`` affects the result. See :ref:`homozygous coverage setting ` for more details. In addition, the option ``-p`` forcedly pops bubbles. +* ```prefix`.p_ctg.gfa``: assembly graph of primary contigs. This graph includes a complete assembly with long stretches of phased blocks. +* ```prefix`.a_ctg.gfa``: assembly graph of alternate contigs. This graph consists of all assemblies that are discarded in primary contig graph. +* ```prefix`.*hap*.p_ctg.gfa``: phased contig graph. This graph keeps the phased contigs. + + +Hifiasm outputs ``*.r_utg.gfa`` and ``*.p_utg.gfa`` in any cases. Specifically, hifiasm outputs the following assembly graphs in trio-binning mode: + +* ```prefix`.dip.hap1.p_ctg.gfa``: fully phased paternal/haplotype1 contig graph keeping the phased paternal/haplotype1 assembly. +* ```prefix`.dip.hap2.p_ctg.gfa``: fully phased maternal/haplotype2 contig graph keeping the phased maternal/haplotype2 assembly. + +With Hi-C partition options, hifiasm outputs: + +* ```prefix`.hic.p_ctg.gfa``: assembly graph of primary contigs. +* ```prefix`.hic.hap1.p_ctg.gfa``: fully phased contig graph of haplotype1 where each contig is fully phased. +* ```prefix`.hic.hap2.p_ctg.gfa``: fully phased contig graph of haplotype2 where each contig is fully phased. +* ```prefix`.hic.a_ctg.gfa`` (optional with ``--primary``): assembly graph of alternate contigs. + +Hifiasm generates the following assembly graphs only with HiFi reads in default: + +* ```prefix`.bp.p_ctg.gfa``: assembly graph of primary contigs. +* ```prefix`.bp.hap1.p_ctg.gfa``: partially phased contig graph of haplotype1. +* ```prefix`.bp.hap2.p_ctg.gfa``: partially phased contig graph of haplotype2. + +If the option ``--primary`` or ``-l0`` is specified, hifiasm outputs: + +* ```prefix`.p_ctg.gfa``: assembly graph of primary contigs. +* ```prefix`.a_ctg.gfa``: assembly graph of alternate contigs. + +For each graph, hifiasm also outputs a simplified version (``*noseq*gfa``) without sequences for the ease of visualization. The coordinates of low quality regions are written to ``*lowQ.bed`` in BED format. +The concepts of different types of assemblies can be found `here `_. + +.. _outformat: + +Output file formats +--------------------------------------- +Hifiasm broadly follows the specification for `GFA 1.0 `_. There are several fields that are specifically used by hifiasm. For ``S`` segment line: + +* ``rd:i:``: read coverage. It is calculated by the reads coming from the same contig/unitig. + +Hifiasm outputs ``A`` lines including the information of reads which are used to construct contig/unitig. Each ``A`` line is plain-text, tab-separated, and the columns appear in the following order: + +.. list-table:: + :widths: 10 25 50 + :header-rows: 1 + + * - Col + - Type + - Description + * - 1 + - string + - Should be always ``A`` + * - 2 + - string + - Contig/unitig name + * - 3 + - int + - Contig/unitig start coordinate of subregion constructed by read + * - 4 + - char + - Read strand: "+" or "-" + * - 5 + - string + - Read name + * - 6 + - int + - Read start coordinate of subregion which is used to construct contig/unitig + * - 7 + - int + - Read end coordinate of subregion which is used to construct contig/unitig + * - 8 + - id:i:int + - Read ID + * - 9 + - HG:A:char + - Haplotype status of read. ``HG:A:a``, ``HG:A:p``, ``HG:A:m`` indicate read is non-binnable, father/hap1-specific and mother/hap2-specific, respectively. + +.. _loginter: + +Hifiasm log interpretation +--------------------------------------- +Hifiasm prints several information for quick debugging, including: + +.. _homcov: + +* k-mer plot: showing how many k-mers appear a certain number of times. For homozygous samples, there should be one peak around read coverage. For heterozygous samples, there should two peaks, where the smaller peak is around the heterozygous read coverage and the larger peak is around the homozygous read coverage. For example, `issue10 `_ indicates the heterozygous read coverage and the homozygous read coverage are 28 and 57, respectively. `Issue49 `_ is another good example. Weird k-mer plot like `issue93 `_ is often caused by insufficient coverage or presence of contaminants. +* homozygous coverage: coverage threshold for homozygous reads. Hifiasm prints it as: ``[M::purge_dups] purge duplication coverage threshold: X``. Without setting the option ``--purge-cov``, the homozygous coverage threshold determined by hifiasm is: ``X/1.25``. If it is not around homozygous coverage, the final assembly might be either too large or too small. To fix this issue, please set ``--purge-cov`` to homozygous coverage. +* number of het/hom bases: how many bases in unitig graph are heterozygous and homozygous during Hi-C phased assembly. Hifiasm prints it as: ``[M::stat] # heterozygous bases: X; # homozygous bases: Y``. Given a heterozygous sample, if there are much more homozygous bases than heterozygous bases, hifiasm fails to identify correct coverage threshold for homozygous reads. In this case, please set ``--purge-cov`` to homozygous coverage. diff --git a/docs/source/pa-assembly.rst b/docs/source/pa-assembly.rst new file mode 100644 index 0000000..58984b9 --- /dev/null +++ b/docs/source/pa-assembly.rst @@ -0,0 +1,47 @@ + +.. _pa-assembly: + +HiFi-only Assembly +================== + +A typical hifiasm command line looks like:: + + hifiasm -o NA12878.asm -t 32 NA12878.fq.gz + +where ``NA12878.fq.gz`` provides the input reads, ``-t`` sets the number of CPUs in +use and ``-o`` specifies the prefix of output files. Input sequences should be FASTA +or FASTQ format, uncompressed or compressed with gzip (.gz). The quality scores of reads +in FASTQ are ignored by hifiasm. Hifiasm outputs assemblies in `GFA `_ format. + +At the first run, hifiasm saves corrected reads and overlaps to disk as ``NA12878.asm.*.bin``. It reuses the saved results to avoid the time-consuming all-vs-all overlap calculation next time. You may specify ``-i`` to ignore precomputed overlaps and redo overlapping from raw reads. You can also dump error corrected reads in FASTA and read overlaps in PAF with:: + + hifiasm -o NA12878.asm -t 32 --write-paf --write-ec /dev/null + +Hifiasm purges haplotig duplications by default. For inbred or homozygous genomes, you may disable purging with option ``-l0``. Old HiFi reads may contain short adapter sequences at the ends of reads. You can specify ``-z20`` to trim both ends of reads by 20bp. For small genomes, use ``-f0`` to disable the initial bloom filter which takes 16GB memory at the beginning. For genomes much larger than human, applying ``-f38`` or even ``-f39`` is preferred to save memory on k-mer counting. + + +Produce two partially phased assemblies +--------------------------------------- + + +Since v0.15, hifiasm produces two sets of partially phased contigs in default like:: + + hifiasm -o NA12878.asm -t 32 NA12878.fq.gz + +In this example, the partially phased contigs are written to ``NA12878.asm.bp.hap*.p_ctg.gfa``. +This pair of files can be thought to represent the two haplotypes in a diploid genome, though with occasional switch errors. The frequency of switches is determined by the heterozygosity of the input sample. Hifiasm also writes the primary contigs to ``NA12878.asm.bp.p_ctg.gfa``. + +For samples with high heterozygosity rate, a common issue is that one set of partially phased contigs is much larger than another set. To fix this issue, please set smaller value for ``-s`` (default: 0.55). Another possibility is that hifiasm misidentifies coverage threshold for homozygous reads. +In this case, please set ``--purge-cov`` to homozygous coverage. See :ref:`p-large` for more details. + + +Produce primary/alternate assemblies +------------------------------------ + +To get primary/alternate assemblies, the option ``--primary`` should be set:: + + hifiasm -o NA12878.asm --primary -t 32 NA12878.fq.gz + +The primary contigs and the alternate contigs are written to ``NA12878.asm.p_ctg.gfa`` and ``NA12878.asm.a_ctg.gfa``, respectively. For inbred or homozygous genomes, the primary/alternate assemblies can be also produced by ``-l0``. Similarly, turning ``-s`` or ``--purge-cov`` should +be helpful if the primary assembly is too large. See :ref:`p-large` for more details. + diff --git a/docs/source/parameter-reference.rst b/docs/source/parameter-reference.rst new file mode 100644 index 0000000..7e124ca --- /dev/null +++ b/docs/source/parameter-reference.rst @@ -0,0 +1,250 @@ + +.. _parameter-reference: + +Hifiasm Parameter Reference +============================ + +Synopsis +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Assembly only with HiFi reads: +:: + + hifiasm -o [prefix] -t [nThreads] [options] input1.fq [input2.fq [...]] + +Trio binning assembly with yak dumps: +:: + + yak count -o paternal.yak -b37 [-t nThreads] [-k kmerLen] paternal.fq.gz + yak count -o maternal.yak -b37 [-t nThreads] [-k kmerLen] maternal.fq.gz + hifiasm [-o prefix] [-t nThreads] [options] -1 paternal.yak -2 maternal.yak child.hifi.fq.gz + +Hi-C integrated assembly: +:: + + hifiasm -o [prefix] -t [nThreads] --h1 [hic_r1.fq.gz,...] --h2 [hic_r2.fq.gz,...] [options] HiFi.read.fq.gz + +To get detailed description of options, run: +:: + + hifiasm -h + +or: +:: + + man ./hifiasm.1 + + +General options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _oopt: + +**\-o ** + Prefix of output files. See :ref:`outfile` and :ref:`outformat` for more details. + +.. _topt: + +**\-t ** + Number of CPU threads used by hifiasm. + +.. _hopt: + +**\-h** + Show help information. + +.. _versionopt: + +**\-\-version** + Show version number. + + +Error correction options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _kopt: + +**\-k ** + K-mer length. This option must be less than 64. + +.. _wopt: + +**\-w ** + Minimizer window size. + +.. _fopt: + +**\-f ** + Number of bits for bloom filter; 0 to disable. This bloom filter is used to filter out singleton k-mers when counting all k-mers. It takes 2\ :sup:`(INT-3)` bytes of memory. A proper setting saves memory. For small genomes, use ``-f0`` to disable the initial bloom filter which takes 16GB memory at the beginning. For genomes much larger than human, applying ``-f38`` or even ``-f39`` is preferred to save memory on k-mer counting. + +.. _ropt: + +**\-r ** + Rounds of haplotype-aware error correction. This option affects all outputs of hifiasm. Odd rounds of correction are preferred in practice. + +.. _min-hist-cnt-opt: + +**\-\-min-hist-cnt ** + When analyzing the k-mer spectrum, ignore counts below ``INT``. See `issue 45 `_ for example. + + + +Assembly options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _aopt: + +**\-a ** + Rounds of assembly graph cleaning. This option is used with ``-x`` and ``-y``. Note that unlike -r, this option does not affect error corrected reads and all-to-all overlaps. + +.. _zopt: + +**\-z ** + Length of adapters that should be removed. This option remove ``INT`` bases from both ends of each read. Some old HiFi reads may consist of short adapters (e.g. 20bp adapter at one end). For such data, trimming short adapters would significantly improve the assembly quality. + +.. _mopt: + +**\-m ** + Maximal probing distance for bubble popping when generating primary/alternate contig graphs. Bubbles longer than ``INT`` bases will not be popped. + +.. _popt: + +**\-p ** + Maximal probing distance for bubble popping when generating haplotype-resolved processed unitig graph without small bubbles. Bubbles longer than ``INT`` bases will not be popped. Small bubbles might be caused by somatic mutations or noise in data. Please note that hifiasm automatically pops small bubbles based on coverage, which can be tweaked by ``--purge-cov``. + +.. _nopt: + +**\-n ** + A unitig is considered small if it is composed of less than ``INT`` reads. Hifiasm may try to remove small unitigs at various steps. + +.. _xyopt: + +**\-x , \-y ** + Max and min overlap drop ratio. This option is used with ``-a``. Given a node N in the assembly graph, let max(N) be the length of the largest overlap of N. Hifiasm iteratively drops overlaps of N if their length/max(N) is below a threshold controlled by ``-x`` and ``-y``. Hifiasm applies ``-a`` rounds of short overlap removal with an increasing threshold between ``FLOAT1`` and ``FLOAT2``. + +.. _iopt: + +**\-i** + Ignore all bin files so that hifiasm will start again from scratch. + +.. _pri-range-opt: + +**\-\-pri-range ** + Min and max coverage cutoffs of primary contigs. Keep contigs with coverage in this range at p_ctg.gfa. Inferred automatically in default. If ``INT2`` is not specified, it is set to infinity. Set -1 to disable. + +.. _lowQ-opt: + +**\-\-lowQ ** + Output contig regions with ``>=INT%`` inconsistency to the bed file with suffix lowQ.bed. Set 0 to disable. + +.. _b-cov-opt: + +**\-\-b-cov ** + Break contigs at potential misassemblies with ``** + Break contigs at potential misassemblies with ``>INT``-fold coverage. Work with ``--m-rate``. Set -1 to disable. + +.. _m-rate-opt: + +**\-\-m-rate ** + Break contigs with ``<=FLOAT*coverage`` exact overlaps. Only work when ``--b-cov`` and ``--h-cov`` are specified. + +.. _primary-opt: + +**\-\-primary** + Output a primary assembly and an alternate assembly. Enable this option or ``-l0`` outputs a primary assembly and an alternate assembly. + + +Trio-binning options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _1opt: + +**\-1 ** + K-mer dump generated by `yak count `_ from the paternal/haplotype1 reads. + +.. _2opt: + +**\-2 ** + K-mer dump generated by `yak count `_ from the maternal/haplotype2 reads. + +.. _3opt: + +**\-3 ** + List of paternal/haplotype1 read names. + +.. _4opt: + +**\-4 ** + List of maternal/haplotype2 read names. + +.. _cdopt: + +**\-c , -d ** + Lower bound and upper bound of the binned k-mer's frequency. When doing trio binning, a k-mer is said to be differentiating if it occurs >= ``INT2`` times in one sample but occurs < ``INT1`` times in the other sample. + +Purge duplication options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _ldopt: + +**\-l ** + Level of purge duplication. 0 to disable, 1 to only purge contained haplotigs, 2 to purge all types of haplotigs, 3 to purge all types of haplotigs in the most aggressive way. In default, 3 for non-trio assembly, 0 for trio-binning assembly. For trio-binning assembly, only level 0 and level 1 are allowed. + +.. _sdopt: + +**\-s ** + Similarity threshold for duplicate haplotigs that should be purged. In default, 0.75 for ``-l1/-l2``, 0.55 for ``-l3``. This option affects both HiFi-only assembly and Hi-C phased assembly. For more details, see :ref:`hic-iss` and :ref:`p-large`. + +.. _ovlpdopt: + +**\-O ** + Min number of overlapped reads for duplicate haplotigs that should be purged. + +.. _purgeopt: + +**\-\-purge-cov ** + Coverage upper bound of purge duplication, which is inferred automatically in default. If the coverage of a contig is higher than this bound, don't apply purge duplication. Moreover, this option also affects Hi-C phased assembly. For more details, see :ref:`hic-iss`, :ref:`p-large` and :ref:`loginter`. + +.. _nhapopt: + +**\-\-n\-hap ** + Assumption of haplotype number. + + + +Hi-C integration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _h1opt: + +**\-\-h1 ** + File names of input Hi-C R1 ``[r1_1.fq,r1_2.fq,...]``. + +.. _h2opt: + +**\-\-h2 ** + File names of input Hi-C R2 ``[r2_1.fq,r2_2.fq,...]``. + +.. _n-weightopt: + +**\-\-n-weight ** + Rounds of reweighting Hi-C links. Raising this option may improve phasing results but takes longer time. + +.. _n-perturbopt: + +**\-\-n-perturb ** + Rounds of perturbation. Increasing this option may improve phasing results but takes longer time. + +.. _f-perturbopt: + +**\-\-f-perturb ** + Fraction to flip for perturbation. Increasing this option may improve phasing results but takes longer time. + +.. _seedopt: + +**\-\-seed ** + RNG seed. \ No newline at end of file diff --git a/docs/source/trio-assembly.rst b/docs/source/trio-assembly.rst new file mode 100644 index 0000000..2533514 --- /dev/null +++ b/docs/source/trio-assembly.rst @@ -0,0 +1,31 @@ + +.. _trio-assembly: + +Trio-binning Assembly +===================== + +When parental short reads are available, hifiasm can also generate a pair of haplotype-resolved assemblies with trio binning. To perform such assembly, you need to count k-mers first with `yak `_ and then do assembly:: + + yak count -k31 -b37 -t16 -o pat.yak paternal.fq.gz + yak count -k31 -b37 -t16 -o mat.yak maternal.fq.gz + hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak NA12878.fq.gz + +Here ``NA12878.asm.hap1.p_ctg.gfa`` and ``NA12878.asm.hap2.p_ctg.gfa`` give the assemblies for two haplotypes. In the binning mode, hifiasm does not purge haplotig duplicates by default. Because hifiasm reuses saved overlaps, you can generate both primary/alternate assemblies and trio binning assemblies with:: + + hifiasm -o NA12878.asm --primary -t 32 NA12878.fq.gz 2> NA12878.asm.pri.log + hifiasm -o NA12878.asm -t 32 -1 pat.yak -2 mat.yak /dev/null 2> NA12878.asm.trio.log + +The second command line will run much faster than the first. The phasing switch error rate and hamming error rate are able to be evaluated quickly by `yak `_:: + + yak trioeval -t16 pat.yak mat.yak assembly.fa + +The W-line and H-line reported by ``yak trioeval`` indicate switch error rate and hamming error rate respectively:: + + W 26714 3029448 0.008818 + H 24315 3029885 0.008025 + +For this example, the switch error rate is 0.8818% and the hamming error rate is 0.8025%. + +In rare cases, a potential issue is that a few contigs may misjoin two haplotypes. For example, half of a contig come from mother while another half come from father. Such misjoined contigs can be fixed by +manually breaking. The coordinates of problematic regions can be found by A-lines in GFA file or ``yak trioeval -e``. However, if there are many misjoined contigs or the switch/hamming error rate reported by ``yak trioeval`` is very high, users should check if the parental data is correct (see `example `_ for more details). + diff --git a/hic.cpp b/hic.cpp index 9ab63e9..242a9a1 100644 --- a/hic.cpp +++ b/hic.cpp @@ -493,6 +493,59 @@ void hc_pt_t_gen_single(hc_pt1_t* pt, uint64_t* up_bound, uint64_t* low_bound) CALLOC(pt->a, pt->n); } +void write_dbug(ma_ug_t* ug, FILE* fp) +{ + ma_utg_t *u = NULL; + uint32_t t, i; + fwrite(&(ug->u.n), sizeof(ug->u.n), 1, fp); + for (i = 0; i < ug->u.n; i++) + { + u = &(ug->u.a[i]); + t = u->len; + fwrite(&t, sizeof(t), 1, fp); + t = u->circ; + fwrite(&t, sizeof(t), 1, fp); + fwrite(&(u->start), sizeof(u->start), 1, fp); + fwrite(&(u->end), sizeof(u->end), 1, fp); + fwrite(&(u->n), sizeof(u->n), 1, fp); + fwrite(u->a, sizeof(uint64_t), u->n, fp); + } +} + +uint32_t test_dbug(ma_ug_t* ug, FILE* fp) +{ + uint32_t f_flag = 0, t, i, r_flag = 0; + size_t tt; + ma_utg_t ua, *ub = NULL; memset(&ua, 0, sizeof(ua)); + f_flag = fread(&tt, sizeof(tt), 1, fp); + if(f_flag == 0 || tt != ug->u.n) goto DES; + for (i = 0; i < tt; i++) + { + ub = &(ug->u.a[i]); + f_flag = fread(&t, sizeof(t), 1, fp); + if(f_flag == 0 || t != ub->len) goto DES; + f_flag = fread(&t, sizeof(t), 1, fp); + if(f_flag == 0 || t != ub->circ) goto DES; + f_flag = fread(&(ua.start), sizeof(ua.start), 1, fp); + if(f_flag == 0 || ua.start != ub->start) goto DES; + f_flag = fread(&(ua.end), sizeof(ua.end), 1, fp); + if(f_flag == 0 || ua.end != ub->end) goto DES; + f_flag = fread(&(ua.n), sizeof(ua.n), 1, fp); + if(f_flag == 0 || ua.n != ub->n) goto DES; + t = ua.n; + ua.n = 0; + kv_resize(uint64_t, ua, t); + ua.n = t; + f_flag = fread(ua.a, sizeof(uint64_t), ua.n, fp); + if(f_flag == 0 || memcmp(ua.a, ub->a, ua.n)) goto DES; + } + r_flag = 1; + + DES: + free(ua.a); + return r_flag; +} + int write_hc_pt_index(ha_ug_index* idx, char* file_name) { char* gfa_name = (char*)malloc(strlen(file_name)+25); @@ -522,13 +575,16 @@ int write_hc_pt_index(ha_ug_index* idx, char* file_name) hc_pt_save(idx->idx_buf[i].h, fp); } + write_dbug(idx->ug, fp); + fprintf(stderr, "[M::%s] Index has been written.\n", __func__); free(gfa_name); fclose(fp); return 1; } -int load_hc_pt_index(ha_ug_index** r_idx, char* file_name) +void destory_hc_pt_index(ha_ug_index* idx); +int load_hc_pt_index(ha_ug_index** r_idx, ma_ug_t *ug, char* file_name) { uint64_t flag = 0; // double index_time = yak_realtime(); @@ -565,6 +621,16 @@ int load_hc_pt_index(ha_ug_index** r_idx, char* file_name) (*r_idx) = idx; free(gfa_name); + if(!test_dbug(ug, fp)) + { + destory_hc_pt_index(idx); + free(idx); + (*r_idx) = NULL; + fclose(fp); + fprintf(stderr, "[M::%s::] ==> Renew Hi-C index\n", __func__); + return 0; + } + fclose(fp); // fprintf(stderr, "[M::%s::%.3f] ==> HiC index has been loaded\n", __func__, yak_realtime()-index_time); return 1; @@ -4640,7 +4706,7 @@ int load_hc_links(hc_links* link, const char *fn) return 1; } -void write_hc_hits(kvec_pe_hit* hits, const char *fn) +void write_hc_hits(kvec_pe_hit* hits, ma_ug_t* ug, const char *fn) { char *buf = (char*)calloc(strlen(fn) + 25, 1); sprintf(buf, "%s.hic.lk.bin", fn); @@ -4648,6 +4714,7 @@ void write_hc_hits(kvec_pe_hit* hits, const char *fn) fwrite(&hits->a.n, sizeof(hits->a.n), 1, fp); fwrite(hits->a.a, sizeof(pe_hit), hits->a.n, fp); + write_dbug(ug, fp); fclose(fp); free(buf); @@ -4802,7 +4869,7 @@ void debug_hc_hits_v14(kvec_pe_hit_hap* i_hits, const char *fn, const ha_ug_inde exit(1); } -int load_hc_hits(kvec_pe_hit* hits, const char *fn) +int load_hc_hits(kvec_pe_hit* hits, ma_ug_t* ug, const char *fn) { uint64_t flag = 0; char *buf = (char*)calloc(strlen(fn) + 25, 1); @@ -4816,9 +4883,18 @@ int load_hc_hits(kvec_pe_hit* hits, const char *fn) flag += fread(&hits->a.n, sizeof(hits->a.n), 1, fp); hits->a.m = hits->a.n; MALLOC(hits->a.a, hits->a.n); flag += fread(hits->a.a, sizeof(pe_hit), hits->a.n, fp); + free(buf); + + if(!test_dbug(ug, fp)) + { + free(hits->a.a); + kv_init(hits->a); + fclose(fp); + fprintf(stderr, "[M::%s::] ==> Renew Hi-C linkages\n", __func__); + return 0; + } fclose(fp); - free(buf); // fprintf(stderr, "[M::%s::] ==> Hi-C linkages have been loaded\n", __func__); return 1; } @@ -16078,12 +16154,10 @@ void renew_idx_para(ha_ug_index* idx, ma_ug_t* ug) idx->rev_mode = ((uint64_t)1) << 63; } -int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx, ug_opt_t *opt) +int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx, ug_opt_t *opt, kvec_pe_hit **rhits) { double index_time = yak_realtime(); sldat_t sl; - kvec_hc_edge back_hc_edge; - kv_init(back_hc_edge.a); sl.idx = idx; sl.t_ch = idx->t_ch; sl.chunk_size = 20000000; @@ -16093,10 +16167,10 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx, ug_o kv_init(sl.hits.a); kv_init(sl.hits.idx); kv_init(sl.hits.occ); - if(!load_hc_hits(&sl.hits, asm_opt.output_file_name)) + if(!load_hc_hits(&sl.hits, idx->ug, asm_opt.output_file_name)) { alignment_worker_pipeline(&sl, fn1, fn2); - write_hc_hits(&sl.hits, asm_opt.output_file_name); + write_hc_hits(&sl.hits, idx->ug, asm_opt.output_file_name); } filter_kv_u_trans_t(&(idx->t_ch->k_trans), idx->ug, 0.5); // flter_by_cov(idx, &sl.hits, 2); @@ -16166,6 +16240,8 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx, ug_o // skip_flipping: verbose_het_stat(&bub); + if(rhits) (*rhits) = get_r_hits_order(&sl.hits, idx->uID_bits, idx->pos_mode, idx->read_g, idx->ug, &bub); + // tag_reads(idx, &sl.hits, &bub, s->s, opt->sources); // print_kv_weight(&k_trans, s->s); @@ -16184,15 +16260,12 @@ int hic_short_align(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx, ug_o // print_bubble_chain(&bub); // destory_contig_partition(&hap); // destory_horder_t(&ho); - kv_destroy(back_hc_edge.a); - kv_destroy(sl.hits.a); - kv_destroy(sl.hits.idx); - kv_destroy(sl.hits.occ); + kv_destroy(sl.hits.a); kv_destroy(sl.hits.idx); kv_destroy(sl.hits.occ); destory_hc_links(&link); - kv_destroy(k_trans); - kv_destroy(k_trans.idx); + kv_destroy(k_trans); kv_destroy(k_trans.idx); destory_ps_t(&s); kv_destroy(u.bid); kv_destroy(u.idx); kv_destroy(u.u); + destory_bubbles(&bub); return 1; @@ -16280,10 +16353,10 @@ int hic_short_align_poy(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx, kv_init(sl.hits.a); kv_init(sl.hits.idx); kv_init(sl.hits.occ); - if(!load_hc_hits(&sl.hits, asm_opt.output_file_name)) + if(!load_hc_hits(&sl.hits, idx->ug, asm_opt.output_file_name)) { alignment_worker_pipeline(&sl, fn1, fn2); - write_hc_hits(&sl.hits, asm_opt.output_file_name); + write_hc_hits(&sl.hits, idx->ug, asm_opt.output_file_name); } hc_links link; @@ -16360,18 +16433,18 @@ int hic_short_align_poy(const enzyme *fn1, const enzyme *fn2, ha_ug_index* idx, } -void hic_analysis(ma_ug_t *ug, asg_t* read_g, trans_chain* t_ch, ug_opt_t *opt, uint32_t is_poy) +void hic_analysis(ma_ug_t *ug, asg_t* read_g, trans_chain* t_ch, ug_opt_t *opt, uint32_t is_poy, kvec_pe_hit **rhits) { ug_index = NULL; int exist = (asm_opt.load_index_from_disk? - load_hc_pt_index(&ug_index, asm_opt.output_file_name) : 0); + load_hc_pt_index(&ug_index, ug, asm_opt.output_file_name) : 0); if(exist == 0) ug_index = build_unitig_index(ug, asm_opt.hic_mer_length, asm_opt.hap_occ, 0, asm_opt.thread_num); if(exist == 0) write_hc_pt_index(ug_index, asm_opt.output_file_name); ug_index->ug = ug; ug_index->read_g = read_g; ug_index->t_ch = t_ch; ///test_unitig_index(ug_index, ug); - if(!is_poy) hic_short_align(asm_opt.hic_reads[0], asm_opt.hic_reads[1], ug_index, opt); + if(!is_poy) hic_short_align(asm_opt.hic_reads[0], asm_opt.hic_reads[1], ug_index, opt, rhits); else hic_short_align_poy(asm_opt.hic_reads[0], asm_opt.hic_reads[1], ug_index, opt); @@ -16820,12 +16893,12 @@ int hic_short_align_bench(const enzyme *fn1, const enzyme *fn2, const char *outp kv_init(sl.hits.a); fprintf(stderr, "u.n: %d, uID_bits: %lu, pos_bits: %lu\n", (uint32_t)idx->ug->u.n, idx->uID_bits, idx->pos_bits); - if(!load_hc_hits(&sl.hits, output_file_name)) + if(!load_hc_hits(&sl.hits, idx->ug, output_file_name)) { // kt_pipeline(3, worker_pipeline, &sl, 3); // dedup_hits(&sl.hits); alignment_worker_pipeline(&sl, fn1, fn2); - write_hc_hits(&sl.hits, output_file_name); + write_hc_hits(&sl.hits, idx->ug, output_file_name); } bench_idx bench; init_bench_idx(&bench, idx->read_g, idx->ug); @@ -16843,7 +16916,7 @@ void hic_benchmark(ma_ug_t *ug, asg_t* read_g) char *output_file_name = (char*)calloc(strlen(asm_opt.output_file_name) + 25, 1); sprintf(output_file_name, "%s.bench", asm_opt.output_file_name); ug_index = NULL; - int exist = load_hc_pt_index(&ug_index, output_file_name); + int exist = load_hc_pt_index(&ug_index, ug, output_file_name); if(exist == 0) ug_index = build_unitig_index(ug, asm_opt.hic_mer_length, asm_opt.hap_occ, 0, asm_opt.thread_num); if(exist == 0) write_hc_pt_index(ug_index, output_file_name); ug_index->ug = ug; diff --git a/hic.h b/hic.h index f664e7e..f0ebbc9 100644 --- a/hic.h +++ b/hic.h @@ -11,7 +11,6 @@ hc_edge* get_hc_edge(hc_links* link, uint64_t src, uint64_t dest, uint64_t dir); hc_edge* push_hc_edge(hc_linkeage* x, uint64_t uID, double weight, int dir, uint64_t* d); -void hic_analysis(ma_ug_t *ug, asg_t* read_g, trans_chain* t_ch, ug_opt_t *opt, uint32_t is_poy); void hic_benchmark(ma_ug_t *ug, asg_t* read_g); typedef struct { @@ -106,5 +105,6 @@ pdq* pqw, uint32_t* path_w, buf_t *resw, asg_t *sg, uint8_t *dest, uint8_t df, u long long *dis); void set_utg_by_dis(uint32_t v, pdq* pq, asg_t *g, kvec_t_u32_warp *res, uint32_t dis); void dedup_hits(kvec_pe_hit* hits, uint64_t is_dup); +void hic_analysis(ma_ug_t *ug, asg_t* read_g, trans_chain* t_ch, ug_opt_t *opt, uint32_t is_poy, kvec_pe_hit **rhits); #endif diff --git a/horder.cpp b/horder.cpp index 0abc054..1313c61 100644 --- a/horder.cpp +++ b/horder.cpp @@ -454,6 +454,26 @@ void resolve_hit(uint64_t x, uint32_t rLen, uint64_t uID_bits, uint64_t pos_mode if(end) (*end) = p_end + 1; } +void idx_hits(kvec_pe_hit* hits, uint64_t n) +{ + uint64_t k, l; + kv_resize(uint64_t, hits->idx, n); + hits->idx.n = n; + memset(hits->idx.a, 0, hits->idx.n*sizeof(uint64_t)); + + radix_sort_pe_hit_idx_hn1(hits->a.a, hits->a.a + hits->a.n); + for (k = 1, l = 0; k <= hits->a.n; ++k) + { + if (k == hits->a.n || (get_hit_suid(*hits, k) != get_hit_suid(*hits, l))) + { + if (k - l > 1) radix_sort_pe_hit_idx_hn2(hits->a.a + l, hits->a.a + k); + + hits->idx.a[get_hit_suid(*hits, l)] = (uint64_t)l << 32 | (k - l); + l = k; + } + } +} + kvec_pe_hit *get_r_hits_for_trio(kvec_pe_hit *u_hits, asg_t* r_g, ma_ug_t* ug, bubble_type* bub, uint64_t uID_bits, uint64_t pos_mode) { kvec_pe_hit *r_hits = NULL; @@ -674,6 +694,7 @@ void get_r_hits(kvec_pe_hit *u_hits, kvec_pe_hit *r_hits, asg_t* r_g, ma_ug_t* u r_hits->uID_bits = ubits; r_hits->pos_mode = p_mode; + idx_hits(r_hits, r_g->n_seq); } uint64_t get_corresp_usite(uint64_t rid, uint64_t rpos, uint64_t rev, uint64_t rlen, u_hits_t *x, uint64_t ubits, uint64_t p_mode, kvec_t_u64_warp *buf) @@ -694,26 +715,6 @@ uint64_t get_corresp_usite(uint64_t rid, uint64_t rpos, uint64_t rev, uint64_t r return a_n; } -void idx_hits(kvec_pe_hit* hits, uint64_t n) -{ - uint64_t k, l; - kv_resize(uint64_t, hits->idx, n); - hits->idx.n = n; - memset(hits->idx.a, 0, hits->idx.n*sizeof(uint64_t)); - - radix_sort_pe_hit_idx_hn1(hits->a.a, hits->a.a + hits->a.n); - for (k = 1, l = 0; k <= hits->a.n; ++k) - { - if (k == hits->a.n || (get_hit_suid(*hits, k) != get_hit_suid(*hits, l))) - { - if (k - l > 1) radix_sort_pe_hit_idx_hn2(hits->a.a + l, hits->a.a + k); - - hits->idx.a[get_hit_suid(*hits, l)] = (uint64_t)l << 32 | (k - l); - l = k; - } - } -} - void update_u_hits(kvec_pe_hit *u_hits, kvec_pe_hit *r_hits, ma_ug_t* ug, asg_t* r_g) { u_hits_t x; memset(&x, 0, sizeof(x)); @@ -744,7 +745,7 @@ void update_u_hits(kvec_pe_hit *u_hits, kvec_pe_hit *r_hits, ma_ug_t* ug, asg_t* } radix_sort_hit_aux_ruid(x.a, x.a + x.n); - x.idx.n = x.idx.m = (x.n?(x.a[x.n-1].ruid>>33)+1:0); + x.idx.n = x.idx.m = (x.n?(x.a[x.n-1].ruid>>33)+1:0);///how many unitigs CALLOC(x.idx.a, x.idx.n); for (k = 1, l = 0; k <= x.n; ++k) { @@ -2329,6 +2330,9 @@ void generate_haplotypes(horder_t *h, ug_opt_t *opt) ug_1 = get_trio_unitig_graph(h->r_g, FATHER, opt); ug_2 = get_trio_unitig_graph(h->r_g, MOTHER, opt); + // kv_push(uint64_t, h->occ, ug_1->u.n); + // kv_push(uint64_t, h->occ, ug_2->u.n); + h->ug = ug_1; ///update unitigs @@ -2345,6 +2349,9 @@ void generate_haplotypes(horder_t *h, ug_opt_t *opt) ma_ug_destroy(ug_2); asg_destroy(h->ug->g); h->ug->g = NULL; + + // MALLOC(h->hf.a, h->ug->u.n); h->hf.n = h->hf.m = h->ug->u.n; + // for (i = 0; i < h->ug->u.n; i++) h->hf.a[i] = (i < h->occ.a[0]? 0 : 1); } osg_t *osg_init(void) @@ -2726,7 +2733,7 @@ void update_scg(horder_t *h, trans_col_t *t_idx) for (i = 0, e.n = 0; i < hits->a.n; i++) { - if(!hits->a.a[i].id) continue; + if(!hits->a.a[i].id) continue;//hom hits suid = get_hit_suid(*hits, i); euid = get_hit_euid(*hits, i); if(suid == euid) continue; @@ -3773,6 +3780,8 @@ void scaffold_hap(horder_t *h, ug_opt_t *opt, trans_col_t *t_idx, uint32_t round h->ug = get_trio_unitig_graph(h->r_g, flag, opt); asg_destroy(h->ug->g); h->ug->g = NULL; + // kv_push(uint64_t, h->occ, h->ug->u.n); + // CALLOC(h->hf.a, h->ug->u.n); h->hf.n = h->hf.m = h->ug->u.n; print_N50(h->ug); @@ -3795,6 +3804,51 @@ void scaffold_hap(horder_t *h, ug_opt_t *opt, trans_col_t *t_idx, uint32_t round free(gfa_name); } +void scaffold_ug(horder_t *h, ma_ug_t *ug, ug_opt_t *opt, uint32_t round, char *output_file_name, uint8_t flag) +{ + uint32_t i; + kv_destroy(h->u_hits.a); + kv_destroy(h->u_hits.idx); + kv_destroy(h->u_hits.occ); + memset(&(h->u_hits), 0, sizeof(h->u_hits)); + kv_destroy(h->avoid); + h->avoid.m = h->avoid.n = 0; + h->avoid.a = NULL; + osg_destroy(h->sg.g); + h->sg.g = NULL; + + h->ug = ug; + asg_destroy(h->ug->g); + h->ug->g = NULL; + // kv_push(uint64_t, h->occ, h->ug->u.n); + // CALLOC(h->hf.a, h->ug->u.n); h->hf.n = h->hf.m = h->ug->u.n; + + print_N50(h->ug); + fprintf(stderr, "[M::%s::]***0***\n", __func__); + for (i = 0; i < round; i++) + { + fprintf(stderr, "[M::%s::]*i->%u*\n", __func__, i); + update_u_hits(&(h->u_hits), &(h->r_hits), h->ug, h->r_g); + fprintf(stderr, "[M::%s::]**i->%u**\n", __func__, i); + update_scg(h, NULL); + fprintf(stderr, "[M::%s::]***i->%u***\n", __func__, i); + layout_scg(h, 1.001, 19); + fprintf(stderr, "[M::%s::]****i->%u****\n", __func__, i); + renew_scaffold(h); + fprintf(stderr, "[M::%s::]*****i->%u*****\n", __func__, i); + } + + char* gfa_name = (char*)malloc(strlen(output_file_name)+100); + sprintf(gfa_name, "%s.%s", output_file_name, (flag==FATHER?"hap1":"hap2")); + + print_scaffold(h->ug, h->r_g, opt->coverage_cut, gfa_name, + opt->sources, opt->reverse_sources, opt->tipsLen, opt->tip_drop_ratio, + opt->stops_threshold, opt->ruIndex, opt->chimeric_rate, opt->drop_ratio, + opt->max_hang, opt->min_ovlp); + + free(gfa_name); +} + void output_hic_rtg(ma_ug_t *ug, asg_t *rg, ug_opt_t *opt, char* output_file_name) { char* gfa_name = (char*)malloc(strlen(output_file_name)+50); @@ -3817,13 +3871,13 @@ asg_t *i_rg, ma_ug_t* i_ug, bubble_type* bub, kv_u_trans_t *ref, ug_opt_t *opt, t_idx = init_trans_col(i_ug, h->r_g->n_seq, ref); // output_hic_rtg(i_ug, h->r_g, opt, asm_opt.output_file_name); - // reduce_hamming_error(h->r_g, opt->sources, opt->coverage_cut, opt->max_hang, opt->min_ovlp, opt->gap_fuzz); + reduce_hamming_error(h->r_g, opt->sources, opt->coverage_cut, opt->max_hang, opt->min_ovlp, opt->gap_fuzz); /** scaffold_hap(h, t_idx, opt, round, asm_opt.output_file_name, FATHER); scaffold_hap(h, t_idx, opt, round, asm_opt.output_file_name, MOTHER); **/ - + generate_haplotypes(h, opt); print_N50(h->ug); @@ -3848,6 +3902,18 @@ asg_t *i_rg, ma_ug_t* i_ug, bubble_type* bub, kv_u_trans_t *ref, ug_opt_t *opt, return h; } + + +void ha_aware_order(kvec_pe_hit *r_hits, asg_t *rg, ma_ug_t *ug_fa, ma_ug_t *ug_mo, kv_u_trans_t *ref, +ug_opt_t *opt, uint32_t round) +{ + horder_t *h = NULL; CALLOC(h, 1); + h->r_hits = *r_hits; + h->r_g = rg; + scaffold_ug(h, ug_fa, opt, round, asm_opt.output_file_name, FATHER); + scaffold_ug(h, ug_mo, opt, round, asm_opt.output_file_name, MOTHER); +} + void destory_horder_t(horder_t **h) { kv_destroy((*h)->r_hits.a); @@ -3865,4 +3931,12 @@ void destory_horder_t(horder_t **h) ma_ug_destroy((*h)->ug); asg_destroy((*h)->r_g); free((*h)); +} + +kvec_pe_hit *get_r_hits_order(kvec_pe_hit *uhits, uint64_t hits_uid_bits, uint64_t hits_pos_mode, +asg_t *rg, ma_ug_t* ug, bubble_type* bub) +{ + kvec_pe_hit *r_hits = NULL; CALLOC(r_hits, 1); + get_r_hits(uhits, r_hits, rg, ug, bub, hits_uid_bits, hits_pos_mode); + return r_hits; } \ No newline at end of file diff --git a/horder.h b/horder.h index 980c9c2..e262075 100644 --- a/horder.h +++ b/horder.h @@ -48,6 +48,8 @@ typedef struct { }scg_t; typedef struct { kvec_t(uint64_t) avoid; + // kvec_t(uint64_t) occ; + // kvec_t(uint8_t) hf; kvec_pe_hit r_hits, u_hits; ma_ug_t *ug; asg_t *r_g; @@ -61,4 +63,8 @@ void horder_clean_sg_by_utg(asg_t *sg, ma_ug_t *ug); kvec_pe_hit *get_r_hits_for_trio(kvec_pe_hit *u_hits, asg_t* r_g, ma_ug_t* ug, bubble_type* bub, uint64_t uID_bits, uint64_t pos_mode); void update_switch_unitig(ma_ug_t *ug, asg_t *rg, kvec_pe_hit *hits, kv_u_trans_t *k_trans, uint64_t cutoff_s, uint64_t cutoff_e, uint64_t min_ulen, double boundaryRate); +kvec_pe_hit *get_r_hits_order(kvec_pe_hit *uhits, uint64_t hits_uid_bits, uint64_t hits_pos_mode, +asg_t *rg, ma_ug_t* ug, bubble_type* bub); +void ha_aware_order(kvec_pe_hit *r_hits, asg_t *rg, ma_ug_t *ug_fa, ma_ug_t *ug_mo, kv_u_trans_t *ref, +ug_opt_t *opt, uint32_t round); #endif