Released minimap2-2.7 (r654)

This happened when the last operation during backtracking is an intron.

.gitignore

@@ -1,4 +1,8 @@
+.cproject
+.project
 .*.swp
 *.a
 *.o
 *.dSYM
+minimap2
+mappy.c

.travis.yml

@@ -1,5 +1,20 @@
-language: c
-compiler:
-  - gcc
-  - clang
-script: make
+matrix:
+  include:
+    - language: c
+      compiler: gcc
+      script: make
+    - language: c
+      compiler: clang
+      script: make
+    - language: python
+      python: "2.7"
+      before_install: pip install cython
+      script: python setup.py build_ext
+    - language: python
+      python: "3.5"
+      before_install: pip install cython
+      script: python setup.py build_ext
+    - language: python
+      python: "3.6"
+      before_install: pip install cython
+      script: python setup.py build_ext

MANIFEST.in

@@ -0,0 +1,11 @@
+include *.h
+include Makefile
+include ksw2_dispatch.c
+include getopt.c
+include main.c
+include README.md
+include python/mappy.c
+include python/cmappy.h
+include python/cmappy.pxd
+include python/mappy.pyx
+include python/README.rst

Makefile

@@ -1,17 +1,24 @@
-CC=			gcc
 CFLAGS=		-g -Wall -O2 -Wc++-compat
 CPPFLAGS=	-DHAVE_KALLOC
-INCLUDES=	-I.
-OBJS=		kthread.o kalloc.o ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_exts2_sse.o ksw2_ll_sse.o \
-			misc.o bseq.o sketch.o sdust.o index.o chain.o align.o hit.o map.o format.o
+INCLUDES=
+OBJS=		kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o index.o chain.o align.o hit.o map.o format.o pe.o esterr.o ksw2_ll_sse.o
 PROG=		minimap2
 PROG_EXTRA=	sdust minimap2-lite
 LIBS=		-lm -lz -lpthread
 
+ifeq ($(arm_neon),)
 ifeq ($(sse2only),)
-	CFLAGS+=-msse4
+	OBJS+=ksw2_extz2_sse41.o ksw2_extd2_sse41.o ksw2_exts2_sse41.o ksw2_extz2_sse2.o ksw2_extd2_sse2.o ksw2_exts2_sse2.o ksw2_dispatch.o
+else
+	OBJS+=ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_exts2_sse.o
+endif
+else
+    OBJS+=ksw2_extz2_neon.o ksw2_extd2_neon.o ksw2_exts2_neon.o
+    CFLAGS+=-D_FILE_OFFSET_BITS=64 -mfpu=neon -fsigned-char
+    INCLUDES+=-I sse2neon
 endif
 
+.PHONY:all extra clean depend
 .SUFFIXES:.c .o
 
 .c.o:
@@ -21,8 +28,8 @@ all:$(PROG)
 
 extra:all $(PROG_EXTRA)
 
-minimap2:main.o libminimap2.a
-		$(CC) $(CFLAGS) $< -o $@ -L. -lminimap2 $(LIBS)
+minimap2:main.o getopt.o libminimap2.a
+		$(CC) $(CFLAGS) main.o getopt.o -o $@ -L. -lminimap2 $(LIBS)
 
 minimap2-lite:example.o libminimap2.a
 		$(CC) $(CFLAGS) $< -o $@ -L. -lminimap2 $(LIBS)
@@ -30,11 +37,47 @@ minimap2-lite:example.o libminimap2.a
 libminimap2.a:$(OBJS)
 		$(AR) -csru $@ $(OBJS)
 
-sdust:sdust.c kalloc.o kalloc.h kdq.h kvec.h kseq.h sdust.h
-		$(CC) -D_SDUST_MAIN $(CFLAGS) $< kalloc.o -o $@ -lz
+sdust:sdust.c getopt.o kalloc.o kalloc.h kdq.h kvec.h kseq.h sdust.h
+		$(CC) -D_SDUST_MAIN $(CFLAGS) $< getopt.o kalloc.o -o $@ -lz
+
+# SSE-specific targets on x86/x86_64
+
+ksw2_extz2_sse41.o:ksw2_extz2_sse.c ksw2.h kalloc.h
+		$(CC) -c -msse4 $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+ksw2_extz2_sse2.o:ksw2_extz2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
+
+ksw2_extd2_sse41.o:ksw2_extd2_sse.c ksw2.h kalloc.h
+		$(CC) -c -msse4 $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+ksw2_extd2_sse2.o:ksw2_extd2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
+
+ksw2_exts2_sse41.o:ksw2_exts2_sse.c ksw2.h kalloc.h
+		$(CC) -c -msse4 $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+ksw2_exts2_sse2.o:ksw2_exts2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
+
+ksw2_dispatch.o:ksw2_dispatch.c ksw2.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+# NEON-specific targets on ARM
+
+ksw2_extz2_neon.o:ksw2_extz2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_SSE2_ONLY -D__SSE2__ $(INCLUDES) $< -o $@
+
+ksw2_extd2_neon.o:ksw2_extd2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_SSE2_ONLY -D__SSE2__ $(INCLUDES) $< -o $@
+
+ksw2_exts2_neon.o:ksw2_exts2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_SSE2_ONLY -D__SSE2__ $(INCLUDES) $< -o $@
+
+# other non-file targets
 
 clean:
-		rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM session*
+		rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM build dist mappy.so mappy.c python/mappy.c mappy.egg*
 
 depend:
 		(LC_ALL=C; export LC_ALL; makedepend -Y -- $(CFLAGS) $(CPPFLAGS) -- *.c)
@@ -42,19 +85,22 @@ depend:
 # DO NOT DELETE
 
 align.o: minimap.h mmpriv.h bseq.h ksw2.h kalloc.h
-bseq.o: bseq.h kseq.h
+bseq.o: bseq.h kvec.h kalloc.h kseq.h
 chain.o: minimap.h mmpriv.h bseq.h kalloc.h
+esterr.o: mmpriv.h minimap.h bseq.h
 example.o: minimap.h kseq.h
 format.o: kalloc.h mmpriv.h minimap.h bseq.h
-hit.o: mmpriv.h minimap.h bseq.h kalloc.h
+getopt.o: getopt.h
+hit.o: mmpriv.h minimap.h bseq.h kalloc.h khash.h
 index.o: kthread.h bseq.h minimap.h mmpriv.h kvec.h kalloc.h khash.h
 kalloc.o: kalloc.h
 ksw2_extd2_sse.o: ksw2.h kalloc.h
 ksw2_exts2_sse.o: ksw2.h kalloc.h
 ksw2_extz2_sse.o: ksw2.h kalloc.h
 ksw2_ll_sse.o: ksw2.h kalloc.h
-main.o: bseq.h minimap.h mmpriv.h
-map.o: kthread.h kvec.h kalloc.h sdust.h mmpriv.h minimap.h bseq.h
+main.o: bseq.h minimap.h mmpriv.h getopt.h
+map.o: kthread.h kvec.h kalloc.h sdust.h mmpriv.h minimap.h bseq.h khash.h
 misc.o: minimap.h ksort.h
+pe.o: mmpriv.h minimap.h bseq.h kvec.h kalloc.h ksort.h
 sdust.o: kalloc.h kdq.h kvec.h sdust.h
 sketch.o: kvec.h kalloc.h minimap.h

NEWS.md

@@ -1,3 +1,207 @@
+Release 2.7-r654 (9 January 2018)
+---------------------------------
+
+This release fixed a bug in the splice mode and added a few minor features:
+
+ * Fixed a bug that occasionally takes an intron as a long deletion in the
+   splice mode. This was caused by wrong backtracking at the last CIGAR
+   operator. The current fix eliminates the error, but it is not optimal in
+   that it often produces a wrong junction when the last operator is an intron.
+   A future version of minimap2 may improve upon this.
+
+ * Support high-end ARM CPUs that implement the NEON instruction set (#81).
+   This enables minimap2 to work on Raspberry Pi 3 and Odroid XU4.
+
+ * Added a C API to construct a minimizer index from a set of C strings (#80).
+
+ * Check scoring specified on the command line (#79). Due to the 8-bit limit,
+   excessively large score penalties fail minimap2.
+
+For genomic sequences, minimap2 should give identical alignments to v2.6.
+
+(2.7: 9 January 2018, r654)
+
+
+
+Release 2.6-r623 (12 December 2017)
+-----------------------------------
+
+This release adds several features and fixes two minor bugs:
+
+ * Optionally build an index without sequences. This helps to reduce the
+   peak memory for read overlapping and is automatically applied when
+   base-level alignment is not requested.
+
+ * Approximately estimate per-base sequence divergence (i.e. 1-identity)
+   without performing base-level alignment, using a MashMap-like method. The
+   estimate is written to a new dv:f tag.
+
+ * Reduced the number of tiny terminal exons in RNA-seq alignment. The current
+   setting is conservative. Increase --end-seed-pen to drop more such exons.
+
+ * Reduced the peak memory when aligning long query sequences.
+
+ * Fixed a bug that is caused by HPC minimizers longer than 256bp. This should
+   have no effect in practice, but it is recommended to rebuild HPC indices if
+   possible.
+
+ * Fixed a bug when identifying identical hits (#71). This should only affect
+   artifactual reference consisting of near identical sequences.
+
+For genomic sequences, minimap2 should give nearly identical alignments to
+v2.5, except the new dv:f tag.
+
+(2.6: 12 December 2017, r623)
+
+
+
+Release 2.5-r572 (11 November 2017)
+-----------------------------------
+
+This release fixes several bugs and brings a couple of minor improvements:
+
+ * Fixed a severe bug that leads to incorrect mapping coordinates in rare
+   corner cases.
+
+ * Fixed underestimated mapping quality for chimeric alignments when the whole
+   query sequence contain many repetitive minimizers, and for chimeric
+   alignments caused by Z-drop.
+
+ * Fixed two bugs in Python binding: incorrect strand field (#57) and incorrect
+   sequence names for Python3 (#55).
+
+ * Improved mapping accuracy for highly overlapping paired ends.
+
+ * Added option -Y to use soft clipping for supplementary alignments (#56).
+
+(2.5: 11 November 2017, r572)
+
+
+
+Release 2.4-r555 (6 November 2017)
+----------------------------------
+
+As is planned, this release focuses on fine tuning the base algorithm. Notable
+changes include
+
+ * Changed the mapping quality scale to match the scale of BWA-MEM. This makes
+   minimap2 and BWA-MEM achieve similar sensitivity-specificity balance on real
+   short-read data.
+
+ * Improved the accuracy of splice alignment by modeling one additional base
+   close to the GT-AG signal. This model is used by default with `-x splice`.
+   For SIRV control data, however, it is recommended to add `--splice-flank=no`
+   to disable this feature as the SIRV splice signals are slightly different.
+
+ * Tuned the parameters for Nanopore Direct RNA reads. The recommended command
+   line is `-axsplice -k14 -uf` (#46).
+
+ * Fixed a segmentation fault when aligning PacBio reads (#47 and #48). This
+   bug is very rare but it affects all versions of minimap2. It is also
+   recommended to re-index reference genomes created with `map-pb`. For human,
+   two minimizers in an old index are wrong.
+
+ * Changed option `-L` in sync with the final decision of hts-specs: a fake
+   CIGAR takes the form of `<readLen>S<refLen>N`. Note that `-L` only enables
+   future tools to recognize long CIGARs. It is not possible for older tools to
+   work with such alignments in BAM (#43 and #51).
+
+ * Fixed a tiny issue whereby minimap2 may waste 8 bytes per candidate
+   alignment.
+
+The minimap2 technical note hosted at arXiv has also been updated to reflect
+recent changes.
+
+(2.4: 6 November 2017, r555)
+
+
+
+Release 2.3-r531 (22 October 2017)
+----------------------------------
+
+This release come with many improvements and bug fixes:
+
+ * The **sr** preset now supports paired-end short-read alignment. Minimap2 is
+   3-4 times as fast as BWA-MEM, but is slightly less accurate on simulated
+   reads.
+
+ * Meticulous improvements to assembly-to-assembly alignment (special thanks to
+   Alexey Gurevich from the QUAST team): a) apply a small penalty to matches
+   between ambiguous bases; b) reduce missing alignments due to spurious
+   overlaps; c) introduce the short form of the `cs` tag, an improvement to the
+   SAM MD tag.
+
+ * Make sure gaps are always left-aligned.
+
+ * Recognize `U` bases from Oxford Nanopore Direct RNA-seq (#33).
+
+ * Fixed slightly wrong chaining score. Fixed slightly inaccurate coordinates
+   for split alignment.
+
+ * Fixed multiple reported bugs: 1) wrong reference name for inversion
+   alignment (#30); 2) redundant SQ lines when multiple query files are
+   specified (#39); 3) non-functioning option `-K` (#36).
+
+This release has implemented all the major features I planned five months ago,
+with the addition of spliced long-read alignment. The next couple of releases
+will focus on fine tuning of the base algorithms.
+
+(2.3: 22 October 2017, r531)
+
+
+
+Release 2.2-r409 (17 September 2017)
+------------------------------------
+
+This is a feature release. It improves single-end short-read alignment and
+comes with Python bindings. Detailed changes include:
+
+ * Added the **sr** preset for single-end short-read alignment. In this mode,
+   minimap2 runs faster than BWA-MEM, but is slightly less accurate on
+   simulated data sets. Paired-end alignment is not supported as of now.
+
+ * Improved mapping quality estimate with more accurate identification of
+   repetitive hits. This mainly helps short-read alignment.
+
+ * Implemented **mappy**, a Python binding for minimap2, which is available
+   from PyPI and can be installed with `pip install --user mappy`. Python users
+   can perform read alignment without the minimap2 executable.
+
+ * Restructured the indexing APIs and documented key minimap2 APIs in the
+   header file minimap.h. Updated example.c with the new APIs. Old APIs still
+   work but may become deprecated in future.
+
+This release may output alignments different from the previous version, though
+the overall alignment statistics, such as the number of aligned bases and long
+gaps, remain close.
+
+(2.2: 17 September 2017, r409)
+
+
+
+Release 2.1.1-r341 (6 September 2017)
+-------------------------------------
+
+This is a maintenance release that is expected to output identical alignment to
+v2.1. Detailed changes include:
+
+ * Support CPU dispatch. By default, minimap2 is compiled with both SSE2 and
+   SSE4 based implementation of alignment and automatically chooses the right
+   one at runtime. This avoids unexpected errors on older CPUs (#21).
+
+ * Improved Windows support as is requested by Oxford Nanopore (#19). Minimap2
+   now avoids variable-length stacked arrays, eliminates alloca(), ships with
+   getopt_long() and provides timing functions implemented with Windows APIs.
+
+ * Fixed a potential segmentation fault when specifying -k/-w/-H with
+   multi-part index (#23).
+
+ * Fixed two memory leaks in example.c
+
+(2.1.1: 6 September 2017, r341)
+
+
+
 Release 2.1-r311 (25 August 2017)
 ---------------------------------
 

README.md

@@ -1,54 +1,286 @@
+[![GitHub Downloads](https://img.shields.io/github/downloads/lh3/minimap2/total.svg?style=social&logo=github&label=Download)](https://github.com/lh3/minimap2/releases)
+[![BioConda Install](https://img.shields.io/conda/dn/bioconda/minimap2.svg?style=flag&label=BioConda%20install)](https://anaconda.org/bioconda/minimap2)
+[![PyPI](https://img.shields.io/pypi/v/mappy.svg?style=flat)](https://pypi.python.org/pypi/mappy)
 [![Build Status](https://travis-ci.org/lh3/minimap2.svg?branch=master)](https://travis-ci.org/lh3/minimap2)
-## Getting Started
+## <a name="started"></a>Getting Started
 ```sh
 git clone https://github.com/lh3/minimap2
 cd minimap2 && make
-# long reads against a reference genome
-./minimap2 -ax map10k test/MT-human.fa test/MT-orang.fa > test.sam
+# long sequences against a reference genome
+./minimap2 -a test/MT-human.fa test/MT-orang.fa > test.sam
 # create an index first and then map
-./minimap2 -x map10k -d MT-human.mmi test/MT-human.fa
-./minimap2 -ax map10k MT-human.mmi test/MT-orang.fa > test.sam
-# long-read overlap (no test data)
-./minimap2 -x ava-pb your-reads.fa your-reads.fa > overlaps.paf
-# man page
+./minimap2 -d MT-human.mmi test/MT-human.fa
+./minimap2 -a MT-human.mmi test/MT-orang.fa > test.sam
+# use presets (no test data)
+./minimap2 -ax map-pb ref.fa pacbio.fq.gz > aln.sam       # PacBio genomic reads
+./minimap2 -ax map-ont ref.fa ont.fq.gz > aln.sam         # Oxford Nanopore genomic reads
+./minimap2 -ax sr ref.fa read1.fa read2.fa > aln.sam      # short genomic paired-end reads
+./minimap2 -ax splice ref.fa rna-reads.fa > aln.sam       # spliced long reads
+./minimap2 -ax splice -k14 -uf ref.fa reads.fa > aln.sam  # Nanopore Direct RNA-seq
+./minimap2 -cx asm5 asm1.fa asm2.fa > aln.paf             # intra-species asm-to-asm alignment
+./minimap2 -x ava-pb reads.fa reads.fa > overlaps.paf     # PacBio read overlap
+./minimap2 -x ava-ont reads.fa reads.fa > overlaps.paf    # Nanopore read overlap
+# man page for detailed command line options
 man ./minimap2.1
 ```
+## Table of Contents
 
-## Introduction
+- [Getting Started](#started)
+- [Users' Guide](#uguide)
+  - [Installation](#install)
+  - [General usage](#general)
+  - [Use cases](#cases)
+    - [Map long noisy genomic reads](#map-long-genomic)
+    - [Map long mRNA/cDNA reads](#map-long-splice)
+    - [Find overlaps between long reads](#long-overlap)
+    - [Map short accurate genomic reads](#short-genomic)
+    - [Full genome/assembly alignment](#full-genome)
+  - [Advanced features](#advanced)
+    - [Working with >65535 CIGAR operations](#long-cigar)
+    - [The cs optional tag](#cs)
+    - [Evaluation scripts](#eval)
+  - [Algorithm overview](#algo)
+  - [Getting help](#help)
+  - [Citing minimap2](#cite)
+- [Developers' Guide](#dguide)
+- [Limitations](#limit)
 
-Minimap2 is a fast sequence mapping and alignment program that can find
-overlaps between long noisy reads, or map long reads or their assemblies to a
-reference genome optionally with detailed alignment (i.e. CIGAR). At present,
-it works efficiently with query sequences from a few kilobases to ~100
-megabases in length at an error rate ~15%. Minimap2 outputs in the [PAF][paf] or
-the [SAM format][sam]. On limited test data sets, minimap2 is over 20 times
-faster than most other long-read aligners. It will replace BWA-MEM for long
-reads and contig alignment.
+## <a name="uguide"></a>Users' Guide
 
-Minimap2 is the successor of [minimap][minimap]. It uses a similar
-minimizer-based indexing and seeding algorithm, and improves the original
-minimap with homopolyer-compressed k-mers (see also [SMARTdenovo][smartdenovo]
-and [longISLND][longislnd]), better chaining and the ability to produce CIGAR
-with fast extension alignment (see also [libgaba][gaba] and [ksw2][ksw2]) and
-piece-wise affine gap cost.
+Minimap2 is a versatile sequence alignment program that aligns DNA or mRNA
+sequences against a large reference database. Typical use cases include: (1)
+mapping PacBio or Oxford Nanopore genomic reads to the human genome; (2)
+finding overlaps between long reads with error rate up to ~15%; (3)
+splice-aware alignment of PacBio Iso-Seq or Nanopore cDNA or Direct RNA reads
+against a reference genome; (4) aligning Illumina single- or paired-end reads;
+(5) assembly-to-assembly alignment; (6) full-genome alignment between two
+closely related species with divergence below ~15%.
 
-If you use minimap2 in your work, please consider to cite:
+For ~10kb noisy reads sequences, minimap2 is tens of times faster than
+mainstream long-read mappers such as BLASR, BWA-MEM, NGMLR and GMAP. It is more
+accurate on simulated long reads and produces biologically meaningful alignment
+ready for downstream analyses. For >100bp Illumina short reads, minimap2 is
+three times as fast as BWA-MEM and Bowtie2, and as accurate on simulated data.
+Detailed evaluations are available from the [minimap2 preprint][preprint].
 
-> Li, H. (2017). Minimap2: fast pairwise alignment for long DNA sequences. [arXiv:1708.01492](https://arxiv.org/abs/1708.01492).
+### <a name="install"></a>Installation
 
-## Installation
+Minimap2 is optimized for x86-64 CPUs. You can acquire precompiled binaries from
+the [release page][release] with:
+```sh
+curl -L https://github.com/lh3/minimap2/releases/download/v2.7/minimap2-2.7_x64-linux.tar.bz2 \
+  | tar -jxvf -
+./minimap2-2.7_x64-linux/minimap2
+```
+If you want to compile from the source, you need to have a C compiler, GNU make
+and zlib development files installed. Then type `make` in the source code
+directory to compile. If you see compilation errors, try `make sse2only=1`
+to disable SSE4 code, which will make minimap2 slightly slower.
 
-For modern x86-64 CPUs, just type `make` in the source code directory. This
-will compile a binary `minimap2` which you can copy to your desired location.
-If you see compilation errors, try `make sse2only=1` to disable SSE4. Minimap2
-will run a little slower. At present, minimap2 does not work with non-x86 CPUs
-or ancient CPUs that do not support SSE2. SSE2 is critical to the performance
-of minimap2.
+Minimap2 also works with ARM CPUs supporting the NEON instruction sets. To
+compile, use `make arm_neon=1`.
 
-## Algorithm Overview
+### <a name="general"></a>General usage
+
+Without any options, minimap2 takes a reference database and a query sequence
+file as input and produce approximate mapping, without base-level alignment
+(i.e. no CIGAR), in the [PAF format][paf]:
+```sh
+minimap2 ref.fa query.fq > approx-mapping.paf
+```
+You can ask minimap2 to generate CIGAR at the `cg` tag of PAF with:
+```sh
+minimap2 -c ref.fa query.fq > alignment.paf
+```
+or to output alignments in the [SAM format][sam]:
+```sh
+minimap2 -a ref.fa query.fq > alignment.sam
+```
+Minimap2 seamlessly works with gzip'd FASTA and FASTQ formats as input. You
+don't need to convert between FASTA and FASTQ or decompress gzip'd files first.
+
+For the human reference genome, minimap2 takes a few minutes to generate a
+minimizer index for the reference before mapping. To reduce indexing time, you
+can optionally save the index with option **-d** and replace the reference
+sequence file with the index file on the minimap2 command line:
+```sh
+minimap2 -d ref.mmi ref.fa                     # indexing
+minimap2 -a ref.mmi reads.fq > alignment.sam   # alignment
+```
+***Importantly***, it should be noted that once you build the index, indexing
+parameters such as **-k**, **-w**, **-H** and **-I** can't be changed during
+mapping. If you are running minimap2 for different data types, you will
+probably need to keep multiple indexes generated with different parameters.
+This makes minimap2 different from BWA which always uses the same index
+regardless of query data types.
+
+### <a name="cases"></a>Use cases
+
+Minimap2 uses the same base algorithm for all applications. However, due to the
+different data types it supports (e.g. short vs long reads; DNA vs mRNA reads),
+minimap2 needs to be tuned for optimal performance and accuracy. It is usually
+recommended to choose a preset with option **-x**, which sets multiple
+parameters at the same time. The default setting is the same as `map-ont`.
+
+#### <a name="map-long-genomic"></a>Map long noisy genomic reads
+
+```sh
+minimap2 -ax map-pb  ref.fa pacbio-reads.fq > aln.sam   # for PacBio subreads
+minimap2 -ax map-ont ref.fa ont-reads.fq > aln.sam      # for Oxford Nanopore reads
+```
+The difference between `map-pb` and `map-ont` is that `map-pb` uses
+homopolymer-compressed (HPC) minimizers as seeds, while `map-ont` uses ordinary
+minimizers as seeds. Emperical evaluation suggests HPC minimizers improve
+performance and sensitivity when aligning PacBio reads, but hurt when aligning
+Nanopore reads.
+
+#### <a name="map-long-splice"></a>Map long mRNA/cDNA reads
+
+```sh
+minimap2 -ax splice -uf ref.fa iso-seq.fq > aln.sam          # PacBio Iso-seq/traditional cDNA
+minimap2 -ax splice ref.fa nanopore-cdna.fa > aln.sam        # Nanopore 2D cDNA-seq
+minimap2 -ax splice -uf -k14 ref.fa direct-rna.fq > aln.sam  # Nanopore Direct RNA-seq
+minimap2 -ax splice --splice-flank=no SIRV.fa SIRV-seq.fa    # mapping against SIRV control
+```
+There are different long-read RNA-seq technologies, including tranditional
+full-length cDNA, EST, PacBio Iso-seq, Nanopore 2D cDNA-seq and Direct RNA-seq.
+They produce data of varying quality and properties. By default, `-x splice`
+assumes the read orientation relative to the transcript strand is unknown. It
+tries two rounds of alignment to infer the orientation and write the strand to
+the `ts` SAM/PAF tag if possible. For Iso-seq, Direct RNA-seq and tranditional
+full-length cDNAs, it would be desired to apply `-u f` to force minimap2 to
+consider the forward transcript strand only. This speeds up alignment with
+slight improvement to accuracy. For noisy Nanopore Direct RNA-seq reads, it is
+recommended to use a smaller k-mer size for increased sensitivity to the first
+or the last exons.
+
+Minimap2 rates an alignment by the score of the max-scoring sub-segment,
+*excluding* introns, and marks the best alignment as primary in SAM. When a
+spliced gene also has unspliced pseudogenes, minimap2 does not intentionally
+prefer spliced alignment, though in practice it more often marks the spliced
+alignment as the primary. By default, minimap2 outputs up to five secondary
+alignments (i.e. likely pseudogenes in the context of RNA-seq mapping). This
+can be tuned with option **-N**.
+
+For long RNA-seq reads, minimap2 may produce chimeric alignments potentially
+caused by gene fusions/structural variations or by an intron longer than the
+max intron length **-G** (200k by default). For now, it is not recommended to
+apply an excessively large **-G** as this slows down minimap2 and sometimes
+leads to false alignments.
+
+It is worth noting that by default `-x splice` prefers GT[A/G]..[C/T]AG
+over GT[C/T]..[A/G]AG, and then over other splicing signals. Considering
+one additional base improves the junction accuracy for noisy reads, but
+reduces the accuracy when aligning against the widely used SIRV control data.
+This is because SIRV does not honor the evolutionarily conservative splicing
+signal. If you are studying SIRV, you may apply `--splice-flank=no` to let
+minimap2 only model GT..AG, ignoring the additional base.
+
+#### <a name="long-overlap"></a>Find overlaps between long reads
+
+```sh
+minimap2 -x ava-pb  reads.fq reads.fq > ovlp.paf    # PacBio read overlap
+minimap2 -x ava-ont reads.fq reads.fq > ovlp.paf    # Oxford Nanopore read overlap
+```
+Similarly, `ava-pb` uses HPC minimizers while `ava-ont` uses ordinary
+minimizers. It is usually not recommended to perform base-level alignment in
+the overlapping mode because it is slow and may produce false positive
+overlaps. However, if performance is not a concern, you may try to add `-a` or
+`-c` anyway.
+
+#### <a name="short-genomic"></a>Map short accurate genomic reads
+
+```sh
+minimap2 -ax sr ref.fa reads-se.fq > aln.sam           # single-end alignment
+minimap2 -ax sr ref.fa read1.fq read2.fq > aln.sam     # paired-end alignment
+minimap2 -ax sr ref.fa reads-interleaved.fq > aln.sam  # paired-end alignment
+```
+When two read files are specified, minimap2 reads from each file in turn and
+merge them into an interleaved stream internally. Two reads are considered to
+be paired if they are adjacent in the input stream and have the same name (with
+the `/[0-9]` suffix trimmed if present). Single- and paired-end reads can be
+mixed.
+
+Minimap2 does not work well with short spliced reads. There are many capable
+RNA-seq mappers for short reads.
+
+#### <a name="full-genome"></a>Full genome/assembly alignment
+
+```sh
+minimap2 -ax asm5 ref.fa asm.fa > aln.sam       # assembly to assembly/ref alignment
+```
+For cross-species full-genome alignment, the scoring system needs to be tuned
+according to the sequence divergence.
+
+### <a name="advanced"></a>Advanced features
+
+#### <a name="long-cigar"></a>Working with >65535 CIGAR operations
+
+Due to a design flaw, BAM does not work with CIGAR strings with >65535
+operations (SAM and CRAM work). However, for ultra-long nanopore reads minimap2
+may align ~1% of read bases with long CIGARs beyond the capability of BAM. If
+you convert such SAM/CRAM to BAM, Picard and recent samtools will throw an
+error and abort. Older samtools and other tools may create corrupted BAM.
+
+To avoid this issue, you can add option `-L` at the minimap2 command line.
+This option moves a long CIGAR to the `CG` tag and leaves a fully clipped CIGAR
+at the SAM CIGAR column. Current tools that don't read CIGAR (e.g. merging and
+sorting) still work with such BAM records; tools that read CIGAR will
+effectively ignore these records. It has been decided that future tools will
+will seamlessly recognize long-cigar records generated by option `-L`.
+
+**TD;DR**: if you work with ultra-long reads and use tools that only process
+BAM files, please add option `-L`.
+
+#### <a name="cs"></a>The cs optional tag
+
+The `cs` SAM/PAF tag encodes bases at mismatches and INDELs. It matches regular
+expression `/(:[0-9]+|\*[a-z][a-z]|[=\+\-][A-Za-z]+)+/`. Like CIGAR, `cs`
+consists of series of operations.  Each leading character specifies the
+operation; the following sequence is the one involved in the operation.
+
+The `cs` tag is enabled by command line option `--cs`. The following alignment,
+for example:
+```txt
+CGATCGATAAATAGAGTAG---GAATAGCA
+||||||   ||||||||||   |||| |||
+CGATCG---AATAGAGTAGGTCGAATtGCA
+```
+is represented as `:6-ata:10+gtc:4*at:3`, where `:[0-9]+` represents an
+identical block, `-ata` represents a deltion, `+gtc` an insertion and `*at`
+indicates reference base `a` is substituted with a query base `t`. It is
+similar to the `MD` SAM tag but is standalone and easier to parse.
+
+If `--cs=long` is used, the `cs` string also contains identical sequences in
+the alignment. The above example will become
+`=CGATCG-ata=AATAGAGTAG+gtc=GAAT*at=GCA`. The long form of `cs` encodes both
+reference and query sequences in one string.
+
+#### <a name="eval"></a>Evaluation scripts
+
+Minimap2 comes with several (java)scripts for evaluating the accuracy of
+minimap2. These scripts require the [k8][k8] javascript shell to run.
+Recent minimap2 binary release tar-balls contain a copy of k8 executable, a
+single file. Here are a few examples on how to use these scripts:
+
+```sh
+# Generate reads from PBSIM alignment (truth encoded in read names)
+k8 misc/sim-pbsim.js ref.fa.fai pbsim-aln.maf > pbsim-reads.fq
+# Generate reads from mason2 alignment (not tested for simulated SVs)
+k8 misc/sim-mason2.js mason2-aln.sam > mason2-reads.fq
+# Evaluate mapping accuracy with ROC-like curve
+k8 misc/sim-eval.js my-aln.sam.gz > result.txt
+k8 misc/sim-eval.js my-aln.paf.gz > result.txt
+# Collect alignment statistics
+k8 misc/mapstat.js my-aln.sam > result.txt
+# Compare spliced junctions to existing gene annotations
+k8 misc/intron-eval.js anno.gtf my-spliced-aln.sam > result.txt
+```
+
+### <a name="algo"></a>Algorithm overview
 
 In the following, minimap2 command line options have a dash ahead and are
-highlighted in bold.
+highlighted in bold. The description may help to tune minimap2 parameters.
 
 1. Read **-I** [=*4G*] reference bases, extract (**-k**,**-w**)-minimizers and
    index them in a hash table.
@@ -89,16 +321,43 @@ highlighted in bold.
 9. If there are more reference sequences, reopen the query file from the start
    and go to step 1; otherwise stop.
 
-## Limitations
+### <a name="help"></a>Getting help
+
+Manpage [minimap2.1][manpage] provides detailed description of minimap2
+command line options and optional tags. If you encounter bugs or have further
+questions or requests, you can raise an issue at the [issue page][issue].
+There is not a specific mailing list for the time being.
+
+### <a name="cite"></a>Citing minimap2
+
+If you use minimap2 in your work, please consider to cite:
+
+> Li, H. (2017). Minimap2: fast pairwise alignment for long nucleotide sequences. [arXiv:1708.01492][preprint]
+
+## <a name="dguide"></a>Developers' Guide
+
+Minimap2 is not only a command line tool, but also a programming library.
+It provides C APIs to build/load index and to align sequences against the
+index. File [example.c](example.c) demonstrates typical uses of C APIs. Header
+file [minimap.h](minimap.h) gives more detailed API documentation. Minimap2
+aims to keep APIs in this header stable. File [mmpriv.h](mmpriv.h) contains
+additional private APIs which may be subjected to changes frequently.
+
+This repository also provides Python bindings to a subset of C APIs. File
+[python/README.rst](python/README.rst) gives the full documentation;
+[python/minimap2.py](python/minimap2.py) shows an example. This Python
+extension, mappy, is also [available from PyPI][mappypypi] via `pip install
+mappy` or [from BioConda][mappyconda] via `conda install -c bioconda mappy`.
+
+## <a name="limit"></a>Limitations
 
 * Minimap2 may produce suboptimal alignments through long low-complexity
   regions where seed positions may be suboptimal. This should not be a big
   concern because even the optimal alignment may be wrong in such regions.
 
-* Minimap2 does not work well with Illumina short reads as of now.
-
-* Minimap2 requires SSE2 instructions to compile. It is possible to add
-  non-SSE2 support, but it would make minimap2 slower by several times.
+* Minimap2 requires SSE2 instructions on x86 CPUs or NEON on ARM CPUs. It is
+  possible to add non-SIMD support, but it would make minimap2 slower by
+  several times.
 
 In general, minimap2 is a young project with most code written since June, 2017.
 It may have bugs and room for improvements. Bug reports and suggestions are
@@ -113,3 +372,10 @@ warmly welcomed.
 [longislnd]: https://www.ncbi.nlm.nih.gov/pubmed/27667791
 [gaba]: https://github.com/ocxtal/libgaba
 [ksw2]: https://github.com/lh3/ksw2
+[preprint]: https://arxiv.org/abs/1708.01492
+[release]: https://github.com/lh3/minimap2/releases
+[mappypypi]: https://pypi.python.org/pypi/mappy
+[mappyconda]: https://anaconda.org/bioconda/mappy
+[issue]: https://github.com/lh3/minimap2/issues
+[k8]: https://github.com/attractivechaos/k8
+[manpage]: https://lh3.github.io/minimap2/minimap2.html

align.c

@@ -1,5 +1,7 @@
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
+#include <math.h>
 #include "minimap.h"
 #include "mmpriv.h"
 #include "ksw2.h"
@@ -61,51 +63,109 @@ static int mm_check_zdrop(const uint8_t *qseq, const uint8_t *tseq, uint32_t n_c
 	return 0;
 }
 
-static void mm_update_extra(mm_extra_t *p, const uint8_t *qseq, const uint8_t *tseq, const int8_t *mat, int8_t q, int8_t e)
+static void mm_fix_cigar(mm_reg1_t *r, const uint8_t *qseq, const uint8_t *tseq, int *qshift, int *tshift)
+{
+	mm_extra_t *p = r->p;
+	int32_t k, toff = 0, qoff = 0, to_shrink = 0;
+	*qshift = *tshift = 0;
+	if (p->n_cigar <= 1) return;
+	for (k = 0; k < p->n_cigar; ++k) { // indel left alignment
+		uint32_t op = p->cigar[k]&0xf, len = p->cigar[k]>>4;
+		if (len == 0) to_shrink = 1;
+		if (op == 0) {
+			toff += len, qoff += len;
+		} else if (op == 1 || op == 2) { // insertion or deletion
+			if (k > 0 && k < p->n_cigar - 1 && (p->cigar[k-1]&0xf) == 0 && (p->cigar[k+1]&0xf) == 0) {
+				int l, prev_len = p->cigar[k-1] >> 4;
+				if (op == 1) {
+					for (l = 0; l < prev_len; ++l)
+						if (qseq[qoff - 1 - l] != qseq[qoff + len - 1 - l])
+							break;
+				} else {
+					for (l = 0; l < prev_len; ++l)
+						if (tseq[toff - 1 - l] != tseq[toff + len - 1 - l])
+							break;
+				}
+				if (l > 0)
+					p->cigar[k-1] -= l<<4, p->cigar[k+1] += l<<4, qoff -= l, toff -= l;
+				if (l == prev_len) to_shrink = 1;
+			}
+			if (op == 1) qoff += len;
+			else toff += len;
+		} else if (op == 3) {
+			toff += len;
+		}
+	}
+	assert(qoff == r->qe - r->qs && toff == r->re - r->rs);
+	if (to_shrink) { // squeeze out zero-length operations
+		int32_t l = 0;
+		for (k = 0; k < p->n_cigar; ++k) // squeeze out zero-length operations
+			if (p->cigar[k]>>4 != 0)
+				p->cigar[l++] = p->cigar[k];
+		p->n_cigar = l;
+		for (k = l = 0; k < p->n_cigar; ++k) // merge two adjacent operations if they are the same
+			if (k == p->n_cigar - 1 || (p->cigar[k]&0xf) != (p->cigar[k+1]&0xf))
+				p->cigar[l++] = p->cigar[k];
+			else p->cigar[k+1] += p->cigar[k]>>4<<4; // add length to the next CIGAR operator
+		p->n_cigar = l;
+	}
+	if ((p->cigar[0]&0xf) == 1 || (p->cigar[0]&0xf) == 2) { // get rid of leading I or D
+		int32_t l = p->cigar[0] >> 4;
+		if ((p->cigar[0]&0xf) == 1) {
+			if (r->rev) r->qe -= l;
+			else r->qs += l;
+			*qshift = l;
+		} else r->rs += l, *tshift = l;
+		--p->n_cigar;
+		memmove(p->cigar, p->cigar + 1, p->n_cigar * 4);
+	}
+}
+
+static void mm_update_extra(mm_reg1_t *r, const uint8_t *qseq, const uint8_t *tseq, const int8_t *mat, int8_t q, int8_t e)
 {
 	uint32_t k, l, toff = 0, qoff = 0;
-	int32_t s = 0, max = 0, n_gtag = 0, n_ctac = 0;
+	int32_t s = 0, max = 0, qshift, tshift;
+	mm_extra_t *p = r->p;
 	if (p == 0) return;
+	mm_fix_cigar(r, qseq, tseq, &qshift, &tshift);
+	qseq += qshift, tseq += tshift; // qseq and tseq may be shifted due to the removal of leading I/D
+	r->blen = r->mlen = 0;
 	for (k = 0; k < p->n_cigar; ++k) {
 		uint32_t op = p->cigar[k]&0xf, len = p->cigar[k]>>4;
 		if (op == 0) { // match/mismatch
+			int n_ambi = 0, n_diff = 0;
 			for (l = 0; l < len; ++l) {
 				int cq = qseq[qoff + l], ct = tseq[toff + l];
-				if (ct > 3 || cq > 3) ++p->n_ambi;
-				else if (ct != cq) ++p->n_diff;
+				if (ct > 3 || cq > 3) ++n_ambi;
+				else if (ct != cq) ++n_diff;
 				s += mat[ct * 5 + cq];
 				if (s < 0) s = 0;
 				else max = max > s? max : s;
 			}
-			toff += len, qoff += len, p->blen += len;
+			r->blen += len - n_ambi, r->mlen += len - (n_ambi + n_diff), p->n_ambi += n_ambi;
+			toff += len, qoff += len;
 		} else if (op == 1) { // insertion
 			int n_ambi = 0;
 			for (l = 0; l < len; ++l)
 				if (qseq[qoff + l] > 3) ++n_ambi;
-			qoff += len, p->blen += len;
-			p->n_ambi += n_ambi, p->n_diff += len - n_ambi;
+			r->blen += len - n_ambi, p->n_ambi += n_ambi;
 			s -= q + e * len;
 			if (s < 0) s = 0;
+			qoff += len;
 		} else if (op == 2) { // deletion
 			int n_ambi = 0;
 			for (l = 0; l < len; ++l)
 				if (tseq[toff + l] > 3) ++n_ambi;
-			toff += len, p->blen += len;
-			p->n_ambi += n_ambi, p->n_diff += len - n_ambi;
+			r->blen += len - n_ambi, p->n_ambi += n_ambi;
 			s -= q + e * len;
 			if (s < 0) s = 0;
+			toff += len;
 		} else if (op == 3) { // intron
-			uint8_t b[4];
-			b[0] = tseq[toff], b[1] = tseq[toff+1];
-			b[2] = tseq[toff+len-2], b[3] = tseq[toff+len-1];
-			if (memcmp(b, "\2\3\0\2", 4) == 0) ++n_gtag;
-			else if (memcmp(b, "\1\3\0\1", 4) == 0) ++n_ctac;
-			toff += len, p->blen += len;
+			toff += len;
 		}
 	}
 	p->dp_max = max;
-	if (n_gtag > n_ctac) p->trans_strand = 1;
-	else if (n_gtag < n_ctac) p->trans_strand = 2;
+	assert(qoff == r->qe - r->qs && toff == r->re - r->rs);
 }
 
 static void mm_append_cigar(mm_reg1_t *r, uint32_t n_cigar, uint32_t *cigar) // TODO: this calls the libc realloc()
@@ -133,20 +193,30 @@ static void mm_append_cigar(mm_reg1_t *r, uint32_t n_cigar, uint32_t *cigar) //
 	}
 }
 
-static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint8_t *qseq, int tlen, const uint8_t *tseq, const int8_t *mat, int w, int flag, ksw_extz_t *ez)
+static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint8_t *qseq, int tlen, const uint8_t *tseq, const int8_t *mat, int w, int end_bonus, int flag, ksw_extz_t *ez)
 {
+	int zdrop = opt->zdrop;
 	if (mm_dbg_flag & MM_DBG_PRINT_ALN_SEQ) {
 		int i;
 		fprintf(stderr, "===> q=(%d,%d), e=(%d,%d), bw=%d, flag=%d, zdrop=%d <===\n", opt->q, opt->q2, opt->e, opt->e2, w, flag, opt->zdrop);
-		for (i = 0; i < tlen; ++i) fputc("ACGTN"[tseq[i]], stderr); fputc('\n', stderr);
-		for (i = 0; i < qlen; ++i) fputc("ACGTN"[qseq[i]], stderr); fputc('\n', stderr);
+		for (i = 0; i < tlen; ++i) fputc("ACGTN"[tseq[i]], stderr);
+		fputc('\n', stderr);
+		for (i = 0; i < qlen; ++i) fputc("ACGTN"[qseq[i]], stderr);
+		fputc('\n', stderr);
 	}
 	if (opt->flag & MM_F_SPLICE)
-		ksw_exts2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->noncan, opt->zdrop, flag, ez);
+		ksw_exts2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->noncan, zdrop, flag, ez);
 	else if (opt->q == opt->q2 && opt->e == opt->e2)
-		ksw_extz2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, w, opt->zdrop, flag, ez);
+		ksw_extz2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, w, zdrop, end_bonus, flag, ez);
 	else
-		ksw_extd2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->e2, w, opt->zdrop, flag, ez);
+		ksw_extd2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->e2, w, zdrop, end_bonus, flag, ez);
+	if (mm_dbg_flag & MM_DBG_PRINT_ALN_SEQ) {
+		int i;
+		fprintf(stderr, "score=%d, cigar=", ez->score);
+		for (i = 0; i < ez->n_cigar; ++i)
+			fprintf(stderr, "%d%c", ez->cigar[i]>>4, "MIDN"[ez->cigar[i]&0xf]);
+		fprintf(stderr, "\n");
+	}
 }
 
 static inline int mm_get_hplen_back(const mm_idx_t *mi, uint32_t rid, uint32_t x)
@@ -160,7 +230,7 @@ static inline int mm_get_hplen_back(const mm_idx_t *mi, uint32_t rid, uint32_t x
 
 static inline void mm_adjust_minier(const mm_idx_t *mi, uint8_t *const qseq0[2], mm128_t *a, int32_t *r, int32_t *q)
 {
-	if (mi->is_hpc) {
+	if (mi->flag & MM_I_HPC) {
 		const uint8_t *qseq = qseq0[a->x>>63];
 		int i, c;
 		*q = (int32_t)a->y;
@@ -251,8 +321,79 @@ static void mm_fix_bad_ends(const mm_reg1_t *r, const mm128_t *a, int bw, int32_
 	}
 }
 
-static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, uint8_t *qseq0[2], mm_reg1_t *r, mm_reg1_t *r2, mm128_t *a, ksw_extz_t *ez, int splice_flag)
+static void mm_max_stretch(const mm_mapopt_t *opt, const mm_reg1_t *r, const mm128_t *a, int32_t *as, int32_t *cnt)
 {
+	int32_t i, score, max_score, len, max_i, max_len;
+
+	*as = r->as, *cnt = r->cnt;
+	if (r->cnt < 2) return;
+
+	max_score = -1, max_i = -1, max_len = 0;
+	score = a[r->as].y >> 32 & 0xff, len = 1;
+	for (i = r->as + 1; i < r->as + r->cnt; ++i) {
+		int32_t lq, lr, q_span;
+		q_span = a[i].y >> 32 & 0xff;
+		lr = (int32_t)a[i].x - (int32_t)a[i-1].x;
+		lq = (int32_t)a[i].y - (int32_t)a[i-1].y;
+		if (lq == lr) {
+			score += lq < q_span? lq : q_span;
+			++len;
+		} else {
+			if (score > max_score)
+				max_score = score, max_len = len, max_i = i - len;
+			score = q_span, len = 1;
+		}
+	}
+	if (score > max_score)
+		max_score = score, max_len = len, max_i = i - len;
+	*as = max_i, *cnt = max_len;
+}
+
+static int mm_seed_ext_score(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, const int8_t mat[25], int qlen, uint8_t *qseq0[2], const mm128_t *a)
+{
+	uint8_t *qseq, *tseq;
+	int q_span = a->y>>32&0xff, qs, qe, rs, re, rid, score, q_off, t_off, ext_len = opt->anchor_ext_len;
+	void *qp;
+	rid = a->x<<1>>33;
+	re = (uint32_t)a->x + 1, rs = re - q_span;
+	qe = (uint32_t)a->y + 1, qs = qe - q_span;
+	rs = rs - ext_len > 0? rs - ext_len : 0;
+	qs = qs - ext_len > 0? qs - ext_len : 0;
+	re = re + ext_len < mi->seq[rid].len? re + ext_len : mi->seq[rid].len;
+	qe = qe + ext_len < qlen? qe + ext_len : qlen;
+	tseq = (uint8_t*)kmalloc(km, re - rs);
+	mm_idx_getseq(mi, rid, rs, re, tseq);
+	qseq = qseq0[a->x>>63] + qs;
+	qp = ksw_ll_qinit(km, 2, qe - qs, qseq, 5, mat);
+	score = ksw_ll_i16(qp, re - rs, tseq, opt->q, opt->e, &q_off, &t_off);
+	kfree(km, tseq);
+	kfree(km, qp);
+	return score;
+}
+
+static void mm_fix_bad_ends_splice(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, const mm_reg1_t *r, const int8_t mat[25], int qlen, uint8_t *qseq0[2], const mm128_t *a, int *as1, int *cnt1)
+{ // this assumes a very crude k-mer based mode; it is not necessary to use a good model just for filtering bounary exons
+	int score;
+	double log_gap;
+	*as1 = r->as, *cnt1 = r->cnt;
+	if (r->cnt < 3) return;
+	log_gap = log((int32_t)a[r->as + 1].x - (int32_t)a[r->as].x);
+	if ((a[r->as].y>>32&0xff) < log_gap + opt->anchor_ext_shift) {
+		score = mm_seed_ext_score(km, opt, mi, mat, qlen, qseq0, &a[r->as]);
+		if ((double)score / mat[0] < log_gap + opt->anchor_ext_shift) // a more exact format is "score < log_4(gap) + shift"
+			++(*as1), --(*cnt1);
+	}
+	log_gap = log((int32_t)a[r->as + r->cnt - 1].x - (int32_t)a[r->as + r->cnt - 2].x);
+	if ((a[r->as + r->cnt - 1].y>>32&0xff) < log_gap + opt->anchor_ext_shift) {
+		score = mm_seed_ext_score(km, opt, mi, mat, qlen, qseq0, &a[r->as + r->cnt - 1]);
+		if ((double)score / mat[0] < log_gap + opt->anchor_ext_shift)
+			--(*cnt1);
+	}
+}
+
+static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, uint8_t *qseq0[2], mm_reg1_t *r, mm_reg1_t *r2, int n_a, mm128_t *a, ksw_extz_t *ez, int splice_flag)
+{
+	int is_sr = !!(opt->flag & MM_F_SR), is_splice = !!(opt->flag & MM_F_SPLICE);
 	int32_t rid = a[r->as].x<<1>>33, rev = a[r->as].x>>63, as1, cnt1;
 	uint8_t *tseq, *qseq;
 	int32_t i, l, bw, dropped = 0, extra_flag = 0, rs0, re0, qs0, qe0;
@@ -260,47 +401,104 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 	int32_t rs1, qs1, re1, qe1;
 	int8_t mat[25];
 
+	if (is_sr) assert(!(mi->flag & MM_I_HPC)); // HPC won't work with SR because with HPC we can't easily tell if there is a gap
+
+	r2->cnt = 0;
 	if (r->cnt == 0) return;
 	ksw_gen_simple_mat(5, mat, opt->a, opt->b);
 	bw = (int)(opt->bw * 1.5 + 1.);
 
-	r2->cnt = 0;
-	if (!(opt->flag & MM_F_SPLICE))
-		mm_fix_bad_ends(r, a, opt->bw, &as1, &cnt1);
-	else as1 = r->as, cnt1 = r->cnt;
-	mm_filter_bad_seeds(km, as1, cnt1, a, 10, 40, opt->max_gap>>1, 10);
-	mm_adjust_minier(mi, qseq0, &a[as1], &rs, &qs);
-	mm_adjust_minier(mi, qseq0, &a[as1 + cnt1 - 1], &re, &qe);
+	if (is_sr && !(mi->flag & MM_I_HPC)) {
+		mm_max_stretch(opt, r, a, &as1, &cnt1);
+		rs = (int32_t)a[as1].x + 1 - (int32_t)(a[as1].y>>32&0xff);
+		qs = (int32_t)a[as1].y + 1 - (int32_t)(a[as1].y>>32&0xff);
+		re = (int32_t)a[as1+cnt1-1].x + 1;
+		qe = (int32_t)a[as1+cnt1-1].y + 1;
+	} else {
+		if (is_splice) {
+			mm_fix_bad_ends_splice(km, opt, mi, r, mat, qlen, qseq0, a, &as1, &cnt1);
+		} else {
+			mm_fix_bad_ends(r, a, opt->bw, &as1, &cnt1);
+		}
+		mm_filter_bad_seeds(km, as1, cnt1, a, 10, 40, opt->max_gap>>1, 10);
+		mm_adjust_minier(mi, qseq0, &a[as1], &rs, &qs);
+		mm_adjust_minier(mi, qseq0, &a[as1 + cnt1 - 1], &re, &qe);
+	}
+	assert(cnt1 > 0);
 
-	if (opt->flag & MM_F_SPLICE) {
+	if (is_splice) {
 		if (splice_flag & MM_F_SPLICE_FOR) extra_flag |= rev? KSW_EZ_SPLICE_REV : KSW_EZ_SPLICE_FOR;
 		if (splice_flag & MM_F_SPLICE_REV) extra_flag |= rev? KSW_EZ_SPLICE_FOR : KSW_EZ_SPLICE_REV;
-		if (splice_flag & MM_F_SPLICE_BOTH) extra_flag |= KSW_EZ_SPLICE_FOR|KSW_EZ_SPLICE_REV;
+		if (opt->flag & MM_F_SPLICE_FLANK) extra_flag |= KSW_EZ_SPLICE_FLANK;
 	}
 
-	// compute rs0 and qs0
-	if (r->split && as1 > 0) {
-		mm_adjust_minier(mi, qseq0, &a[as1-1], &rs0, &qs0);
+	/* Look for the start and end of regions to perform DP. This sounds easy
+	 * but is in fact tricky. Excessively small regions lead to unnecessary
+	 * clippings and lose alignable sequences. Excessively large regions
+	 * occasionally lead to large overlaps between two chains and may cause
+	 * loss of alignments in corner cases. */
+	if (is_sr) {
+		qs0 = 0, qe0 = qlen;
+		l = qs;
+		l += l * opt->a + opt->end_bonus > opt->q? (l * opt->a + opt->end_bonus - opt->q) / opt->e : 0;
+		rs0 = rs - l > 0? rs - l : 0;
+		l = qlen - qe;
+		l += l * opt->a + opt->end_bonus > opt->q? (l * opt->a + opt->end_bonus - opt->q) / opt->e : 0;
+		re0 = re + l < mi->seq[rid].len? re + l : mi->seq[rid].len;
 	} else {
-		if (qs > 0 && rs > 0) { // actually this is always true
+		// compute rs0 and qs0
+		rs0 = (int32_t)a[r->as].x + 1 - (int32_t)(a[r->as].y>>32&0xff);
+		qs0 = (int32_t)a[r->as].y + 1 - (int32_t)(a[r->as].y>>32&0xff);
+		if (rs0 < 0) rs0 = 0; // this may happen when HPC is in use
+		assert(qs0 >= 0); // this should never happen, or it is logic error
+		rs1 = qs1 = 0;
+		for (i = r->as - 1, l = 0; i >= 0 && a[i].x>>32 == a[r->as].x>>32; --i) { // inspect nearby seeds
+			int32_t x = (int32_t)a[i].x + 1 - (int32_t)(a[i].y>>32&0xff);
+			int32_t y = (int32_t)a[i].y + 1 - (int32_t)(a[i].y>>32&0xff);
+			if (x < rs0 && y < qs0) {
+				if (++l > opt->min_cnt) {
+					l = rs0 - x > qs0 - y? rs0 - x : qs0 - y;
+					rs1 = rs0 - l, qs1 = qs0 - l;
+					break;
+				}
+			}
+		}
+		if (qs > 0 && rs > 0) {
 			l = qs < opt->max_gap? qs : opt->max_gap;
-			qs0 = qs - l;
+			qs1 = qs1 > qs - l? qs1 : qs - l;
+			qs0 = qs0 < qs1? qs0 : qs1; // at least include qs0
 			l += l * opt->a > opt->q? (l * opt->a - opt->q) / opt->e : 0;
 			l = l < opt->max_gap? l : opt->max_gap;
 			l = l < rs? l : rs;
-			rs0 = rs - l;
+			rs1 = rs1 > rs - l? rs1 : rs - l;
+			rs0 = rs0 < rs1? rs0 : rs1;
 		} else rs0 = rs, qs0 = qs;
-
+		// compute re0 and qe0
+		re0 = (int32_t)a[r->as + r->cnt - 1].x + 1;
+		qe0 = (int32_t)a[r->as + r->cnt - 1].y + 1;
+		re1 = mi->seq[rid].len, qe1 = qlen;
+		for (i = r->as + r->cnt, l = 0; i < n_a && a[i].x>>32 == a[r->as].x>>32; ++i) { // inspect nearby seeds
+			int32_t x = (int32_t)a[i].x + 1;
+			int32_t y = (int32_t)a[i].y + 1;
+			if (x > re0 && y > qe0) {
+				if (++l > opt->min_cnt) {
+					l = x - re0 > y - qe0? x - re0 : y - qe0;
+					re1 = re0 + l, qe1 = qe0 + l;
+					break;
+				}
+			}
+		}
+		if (qe < qlen && re < mi->seq[rid].len) {
+			l = qlen - qe < opt->max_gap? qlen - qe : opt->max_gap;
+			qe1 = qe1 < qe + l? qe1 : qe + l;
+			qe0 = qe0 > qe1? qe0 : qe1; // at least include qe0
+			l += l * opt->a > opt->q? (l * opt->a - opt->q) / opt->e : 0;
+			l = l < opt->max_gap? l : opt->max_gap;
+			l = l < mi->seq[rid].len - re? l : mi->seq[rid].len - re;
+			re1 = re1 < re + l? re1 : re + l;
+			re0 = re0 > re1? re0 : re1;
+		} else re0 = re, qe0 = qe;
 	}
-	// compute re0 and qe0
-	if (qe < qlen && re < mi->seq[rid].len) {
-		l = qlen - qe < opt->max_gap? qlen - qe : opt->max_gap;
-		qe0 = qe + l;
-		l += l * opt->a > opt->q? (l * opt->a - opt->q) / opt->e : 0;
-		l = l < opt->max_gap? l : opt->max_gap;
-		l = l < mi->seq[rid].len - re? l : mi->seq[rid].len - re;
-		re0 = re + l;
-	} else re0 = re, qe0 = qe;
 
 	assert(re0 > rs0);
 	tseq = (uint8_t*)kmalloc(km, re0 - rs0);
@@ -310,44 +508,57 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 		mm_idx_getseq(mi, rid, rs0, rs, tseq);
 		mm_seq_rev(qs - qs0, qseq);
 		mm_seq_rev(rs - rs0, tseq);
-		mm_align_pair(km, opt, qs - qs0, qseq, rs - rs0, tseq, mat, bw, extra_flag|KSW_EZ_EXTZ_ONLY|KSW_EZ_RIGHT|KSW_EZ_REV_CIGAR, ez);
+		mm_align_pair(km, opt, qs - qs0, qseq, rs - rs0, tseq, mat, bw, opt->end_bonus, extra_flag|KSW_EZ_EXTZ_ONLY|KSW_EZ_RIGHT|KSW_EZ_REV_CIGAR, ez);
 		if (ez->n_cigar > 0) {
 			mm_append_cigar(r, ez->n_cigar, ez->cigar);
 			r->p->dp_score += ez->max;
 		}
-		rs1 = rs - (ez->max_t + 1);
-		qs1 = qs - (ez->max_q + 1);
+		rs1 = rs - (ez->reach_end? ez->mqe_t + 1 : ez->max_t + 1);
+		qs1 = qs - (ez->reach_end? qs - qs0 : ez->max_q + 1);
 		mm_seq_rev(qs - qs0, qseq);
 	} else rs1 = rs, qs1 = qs;
 	re1 = rs, qe1 = qs;
 	assert(qs1 >= 0 && rs1 >= 0);
 
-	for (i = 1; i < cnt1; ++i) { // gap filling
+	for (i = is_sr? cnt1 - 1 : 1; i < cnt1; ++i) { // gap filling
 		if ((a[as1+i].y & (MM_SEED_IGNORE|MM_SEED_TANDEM)) && i != cnt1 - 1) continue;
-		mm_adjust_minier(mi, qseq0, &a[as1 + i], &re, &qe);
+		if (is_sr && !(mi->flag & MM_I_HPC)) {
+			re = (int32_t)a[as1 + i].x + 1;
+			qe = (int32_t)a[as1 + i].y + 1;
+		} else mm_adjust_minier(mi, qseq0, &a[as1 + i], &re, &qe);
 		re1 = re, qe1 = qe;
 		if (i == cnt1 - 1 || (a[as1+i].y&MM_SEED_LONG_JOIN) || (qe - qs >= opt->min_ksw_len && re - rs >= opt->min_ksw_len)) {
-			int bw1 = bw;
+			int j, bw1 = bw;
 			if (a[as1+i].y & MM_SEED_LONG_JOIN)
 				bw1 = qe - qs > re - rs? qe - qs : re - rs;
 			qseq = &qseq0[rev][qs];
 			mm_idx_getseq(mi, rid, rs, re, tseq);
-			mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, extra_flag|KSW_EZ_APPROX_MAX, ez);
+			if (is_sr) { // perform ungapped alignment
+				assert(qe - qs == re - rs);
+				ksw_reset_extz(ez);
+				for (j = 0, ez->score = 0; j < qe - qs; ++j) {
+					if (qseq[j] >= 4 || tseq[j] >= 4) ez->score += opt->e2;
+					else ez->score += qseq[j] == tseq[j]? opt->a : -opt->b;
+				}
+				ez->cigar = ksw_push_cigar(km, &ez->n_cigar, &ez->m_cigar, ez->cigar, 0, qe - qs);
+			} else { // perform normal gapped alignment
+				mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, -1, extra_flag|KSW_EZ_APPROX_MAX, ez); // first pass: with approximate Z-drop
+			}
 			if (mm_check_zdrop(qseq, tseq, ez->n_cigar, ez->cigar, mat, opt->q, opt->e, opt->zdrop))
-				mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, extra_flag, ez);
+				mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, -1, extra_flag, ez); // second pass: lift approximate
 			if (ez->n_cigar > 0)
 				mm_append_cigar(r, ez->n_cigar, ez->cigar);
 			if (ez->zdropped) { // truncated by Z-drop; TODO: sometimes Z-drop kicks in because the next seed placement is wrong. This can be fixed in principle.
-				int j;
 				for (j = i - 1; j >= 0; --j)
-					if ((int32_t)a[as1 + j].x < re + ez->max_t)
+					if ((int32_t)a[as1 + j].x <= rs + ez->max_t)
 						break;
 				dropped = 1;
+				if (j < 0) j = 0;
 				r->p->dp_score += ez->max;
 				re1 = rs + (ez->max_t + 1);
 				qe1 = qs + (ez->max_q + 1);
 				if (cnt1 - (j + 1) >= opt->min_cnt)
-					mm_split_reg(r, r2, j + 1, qlen, a);
+					mm_split_reg(r, r2, as1 + j + 1 - r->as, qlen, a);
 				break;
 			} else r->p->dp_score += ez->score;
 			rs = re, qs = qe;
@@ -357,13 +568,13 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 	if (!dropped && qe < qe0 && re < re0) { // right extension
 		qseq = &qseq0[rev][qe];
 		mm_idx_getseq(mi, rid, re, re0, tseq);
-		mm_align_pair(km, opt, qe0 - qe, qseq, re0 - re, tseq, mat, bw, extra_flag|KSW_EZ_EXTZ_ONLY, ez);
+		mm_align_pair(km, opt, qe0 - qe, qseq, re0 - re, tseq, mat, bw, opt->end_bonus, extra_flag|KSW_EZ_EXTZ_ONLY, ez);
 		if (ez->n_cigar > 0) {
 			mm_append_cigar(r, ez->n_cigar, ez->cigar);
 			r->p->dp_score += ez->max;
 		}
-		re1 = re + (ez->max_t + 1);
-		qe1 = qe + (ez->max_q + 1);
+		re1 = re + (ez->reach_end? ez->mqe_t + 1 : ez->max_t + 1);
+		qe1 = qe + (ez->reach_end? qe0 - qe : ez->max_q + 1);
 	}
 	assert(qe1 <= qlen);
 
@@ -374,7 +585,7 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 	assert(re1 - rs1 <= re0 - rs0);
 	if (r->p) {
 		mm_idx_getseq(mi, rid, rs1, re1, tseq);
-		mm_update_extra(r->p, &qseq0[r->rev][qs1], tseq, mat, opt->q, opt->e);
+		mm_update_extra(r, &qseq0[r->rev][qs1], tseq, mat, opt->q, opt->e);
 		if (rev && r->p->trans_strand)
 			r->p->trans_strand ^= 3; // flip to the read strand
 	}
@@ -413,17 +624,19 @@ static int mm_align1_inv(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, i
 	mm_seq_rev(tl, tseq);
 	if (score < opt->min_dp_max) goto end_align1_inv;
 	q_off = ql - (q_off + 1), t_off = tl - (t_off + 1);
-	mm_align_pair(km, opt, ql - q_off, qseq + q_off, tl - t_off, tseq + t_off, mat, (int)(opt->bw * 1.5), KSW_EZ_EXTZ_ONLY, ez);
+	mm_align_pair(km, opt, ql - q_off, qseq + q_off, tl - t_off, tseq + t_off, mat, (int)(opt->bw * 1.5), -1, KSW_EZ_EXTZ_ONLY, ez);
 	if (ez->n_cigar == 0) goto end_align1_inv; // should never be here
 	mm_append_cigar(r_inv, ez->n_cigar, ez->cigar);
 	r_inv->p->dp_score = ez->max;
-	mm_update_extra(r_inv->p, qseq + q_off, tseq + t_off, mat, opt->q, opt->e);
 	r_inv->id = -1;
 	r_inv->parent = MM_PARENT_UNSET;
 	r_inv->inv = 1;
 	r_inv->rev = !r1->rev;
+	r_inv->rid = r1->rid;
+	r_inv->div = -1.0f;
 	r_inv->qs = r1->qe + q_off, r_inv->qe = r_inv->qs + ez->max_q + 1;
 	r_inv->rs = r1->re + t_off, r_inv->re = r_inv->rs + ez->max_t + 1;
+	mm_update_extra(r_inv, &qseq[q_off], &tseq[t_off], mat, opt->q, opt->e);
 	ret = 1;
 end_align1_inv:
 	kfree(km, tseq);
@@ -443,28 +656,29 @@ static inline mm_reg1_t *mm_insert_reg(const mm_reg1_t *r, int i, int *n_regs, m
 mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, int *n_regs_, mm_reg1_t *regs, mm128_t *a)
 {
 	extern unsigned char seq_nt4_table[256];
-	int32_t i, n_regs = *n_regs_;
+	int32_t i, n_regs = *n_regs_, n_a;
 	uint8_t *qseq0[2];
 	ksw_extz_t ez;
 
 	// encode the query sequence
-	qseq0[0] = (uint8_t*)kmalloc(km, qlen);
-	qseq0[1] = (uint8_t*)kmalloc(km, qlen);
+	qseq0[0] = (uint8_t*)kmalloc(km, qlen * 2);
+	qseq0[1] = qseq0[0] + qlen;
 	for (i = 0; i < qlen; ++i) {
 		qseq0[0][i] = seq_nt4_table[(uint8_t)qstr[i]];
 		qseq0[1][qlen - 1 - i] = qseq0[0][i] < 4? 3 - qseq0[0][i] : 4;
 	}
 
 	// align through seed hits
+	n_a = mm_squeeze_a(km, n_regs, regs, a);
 	memset(&ez, 0, sizeof(ksw_extz_t));
 	for (i = 0; i < n_regs; ++i) {
 		mm_reg1_t r2;
-		if ((opt->flag&MM_F_SPLICE) && (opt->flag&MM_F_SPLICE_FOR) && (opt->flag&MM_F_SPLICE_REV)) {
+		if ((opt->flag&MM_F_SPLICE) && (opt->flag&MM_F_SPLICE_FOR) && (opt->flag&MM_F_SPLICE_REV)) { // then do two rounds of alignments for both strands
 			mm_reg1_t s[2], s2[2];
 			int which, trans_strand;
 			s[0] = s[1] = regs[i];
-			mm_align1(km, opt, mi, qlen, qseq0, &s[0], &s2[0], a, &ez, MM_F_SPLICE_FOR);
-			mm_align1(km, opt, mi, qlen, qseq0, &s[1], &s2[1], a, &ez, MM_F_SPLICE_REV);
+			mm_align1(km, opt, mi, qlen, qseq0, &s[0], &s2[0], n_a, a, &ez, MM_F_SPLICE_FOR);
+			mm_align1(km, opt, mi, qlen, qseq0, &s[1], &s2[1], n_a, a, &ez, MM_F_SPLICE_REV);
 			if (s[0].p->dp_score > s[1].p->dp_score) which = 0, trans_strand = 1;
 			else if (s[0].p->dp_score < s[1].p->dp_score) which = 1, trans_strand = 2;
 			else trans_strand = 3, which = (qlen + s[0].p->dp_score) & 1; // randomly choose a strand, effectively
@@ -476,19 +690,21 @@ mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *m
 				free(s[0].p);
 			}
 			regs[i].p->trans_strand = trans_strand;
-		} else {
-			mm_align1(km, opt, mi, qlen, qseq0, &regs[i], &r2, a, &ez, opt->flag);
-			if ((opt->flag&MM_F_SPLICE) && !(opt->flag&MM_F_SPLICE_BOTH))
+		} else { // one round of alignment
+			mm_align1(km, opt, mi, qlen, qseq0, &regs[i], &r2, n_a, a, &ez, opt->flag);
+			if (opt->flag&MM_F_SPLICE)
 				regs[i].p->trans_strand = opt->flag&MM_F_SPLICE_FOR? 1 : 2;
 		}
 		if (r2.cnt > 0) regs = mm_insert_reg(&r2, i, &n_regs, regs);
-		if (i > 0 && mm_align1_inv(km, opt, mi, qlen, qseq0, &regs[i-1], &regs[i], &r2, &ez)) {
-			regs = mm_insert_reg(&r2, i, &n_regs, regs);
-			++i; // skip the inserted INV alignment
+		if (!(opt->flag&MM_F_SPLICE) && !(opt->flag&MM_F_SR) && i > 0) { // don't try inversion alignment for -xsplice or -xsr
+			if (mm_align1_inv(km, opt, mi, qlen, qseq0, &regs[i-1], &regs[i], &r2, &ez)) {
+				regs = mm_insert_reg(&r2, i, &n_regs, regs);
+				++i; // skip the inserted INV alignment
+			}
 		}
 	}
 	*n_regs_ = n_regs;
-	kfree(km, qseq0[0]); kfree(km, qseq0[1]);
+	kfree(km, qseq0[0]);
 	kfree(km, ez.cigar);
 	mm_filter_regs(km, opt, n_regs_, regs);
 	mm_hit_sort_by_dp(km, n_regs_, regs);

bseq.c

@@ -1,15 +1,37 @@
 #include <zlib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 #include <assert.h>
 #include "bseq.h"
+#include "kvec.h"
 #include "kseq.h"
-KSEQ_INIT(gzFile, gzread)
+KSEQ_INIT2(, gzFile, gzread)
+
+unsigned char seq_comp_table[256] = {
+	  0,   1,	2,	 3,	  4,   5,	6,	 7,	  8,   9,  10,	11,	 12,  13,  14,	15,
+	 16,  17,  18,	19,	 20,  21,  22,	23,	 24,  25,  26,	27,	 28,  29,  30,	31,
+	 32,  33,  34,	35,	 36,  37,  38,	39,	 40,  41,  42,	43,	 44,  45,  46,	47,
+	 48,  49,  50,	51,	 52,  53,  54,	55,	 56,  57,  58,	59,	 60,  61,  62,	63,
+	 64, 'T', 'V', 'G', 'H', 'E', 'F', 'C', 'D', 'I', 'J', 'M', 'L', 'K', 'N', 'O',
+	'P', 'Q', 'Y', 'S', 'A', 'A', 'B', 'W', 'X', 'R', 'Z',	91,	 92,  93,  94,	95,
+	 64, 't', 'v', 'g', 'h', 'e', 'f', 'c', 'd', 'i', 'j', 'm', 'l', 'k', 'n', 'o',
+	'p', 'q', 'y', 's', 'a', 'a', 'b', 'w', 'x', 'r', 'z', 123, 124, 125, 126, 127,
+	128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
+	144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+	160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
+	176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191,
+	192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
+	208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223,
+	224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+	240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255
+};
+
+#define CHECK_PAIR_THRES 1000000
 
 struct mm_bseq_file_s {
 	gzFile fp;
 	kseq_t *ks;
+	mm_bseq1_t s;
 };
 
 mm_bseq_file_t *mm_bseq_open(const char *fn)
@@ -31,32 +53,85 @@ void mm_bseq_close(mm_bseq_file_t *fp)
 	free(fp);
 }
 
-mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_)
+static inline void kseq2bseq(kseq_t *ks, mm_bseq1_t *s, int with_qual)
 {
-	int size = 0, m, n;
-	mm_bseq1_t *seqs;
+	int i;
+	s->name = strdup(ks->name.s);
+	s->seq = strdup(ks->seq.s);
+	for (i = 0; i < ks->seq.l; ++i) // convert U to T
+		if (s->seq[i] == 'u' || s->seq[i] == 'U')
+			--s->seq[i];
+	s->qual = with_qual && ks->qual.l? strdup(ks->qual.s) : 0;
+	s->l_seq = ks->seq.l;
+}
+
+mm_bseq1_t *mm_bseq_read2(mm_bseq_file_t *fp, int chunk_size, int with_qual, int frag_mode, int *n_)
+{
+	int64_t size = 0;
+	kvec_t(mm_bseq1_t) a = {0,0,0};
 	kseq_t *ks = fp->ks;
-	m = n = 0; seqs = 0;
+	*n_ = 0;
+	if (fp->s.seq) {
+		kv_resize(mm_bseq1_t, 0, a, 256);
+		kv_push(mm_bseq1_t, 0, a, fp->s);
+		size = fp->s.l_seq;
+		memset(&fp->s, 0, sizeof(mm_bseq1_t));
+	}
 	while (kseq_read(ks) >= 0) {
 		mm_bseq1_t *s;
 		assert(ks->seq.l <= INT32_MAX);
-		if (n >= m) {
-			m = m? m<<1 : 256;
-			seqs = (mm_bseq1_t*)realloc(seqs, m * sizeof(mm_bseq1_t));
+		if (a.m == 0) kv_resize(mm_bseq1_t, 0, a, 256);
+		kv_pushp(mm_bseq1_t, 0, a, &s);
+		kseq2bseq(ks, s, with_qual);
+		size += s->l_seq;
+		if (size >= chunk_size) {
+			if (frag_mode && a.a[a.n-1].l_seq < CHECK_PAIR_THRES) {
+				while (kseq_read(ks) >= 0) {
+					kseq2bseq(ks, &fp->s, with_qual);
+					if (mm_qname_same(fp->s.name, a.a[a.n-1].name)) {
+						kv_push(mm_bseq1_t, 0, a, fp->s);
+						memset(&fp->s, 0, sizeof(mm_bseq1_t));
+					} else break;
+				}
+			}
+			break;
+		}
+	}
+	*n_ = a.n;
+	return a.a;
+}
+
+mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_)
+{
+	return mm_bseq_read2(fp, chunk_size, with_qual, 0, n_);
+}
+
+mm_bseq1_t *mm_bseq_read_frag(int n_fp, mm_bseq_file_t **fp, int chunk_size, int with_qual, int *n_)
+{
+	int i;
+	int64_t size = 0;
+	kvec_t(mm_bseq1_t) a = {0,0,0};
+	*n_ = 0;
+	if (n_fp < 1) return 0;
+	while (1) {
+		for (i = 0; i < n_fp; ++i)
+			if (kseq_read(fp[i]->ks) < 0)
+				break;
+		if (i != n_fp) break; // some file reaches the end
+		if (a.m == 0) kv_resize(mm_bseq1_t, 0, a, 256);
+		for (i = 0; i < n_fp; ++i) {
+			mm_bseq1_t *s;
+			kv_pushp(mm_bseq1_t, 0, a, &s);
+			kseq2bseq(fp[i]->ks, s, with_qual);
+			size += s->l_seq;
 		}
-		s = &seqs[n];
-		s->name = strdup(ks->name.s);
-		s->seq = strdup(ks->seq.s);
-		s->qual = with_qual && ks->qual.l? strdup(ks->qual.s) : 0;
-		s->l_seq = ks->seq.l;
-		size += seqs[n++].l_seq;
 		if (size >= chunk_size) break;
 	}
-	*n_ = n;
-	return seqs;
+	*n_ = a.n;
+	return a.a;
 }
 
 int mm_bseq_eof(mm_bseq_file_t *fp)
 {
-	return ks_eof(fp->ks->f);
+	return (ks_eof(fp->ks->f) && fp->s.seq == 0);
 }

bseq.h

@@ -2,6 +2,7 @@
 #define MM_BSEQ_H
 
 #include <stdint.h>
+#include <string.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,10 +18,42 @@ typedef struct {
 
 mm_bseq_file_t *mm_bseq_open(const char *fn);
 void mm_bseq_close(mm_bseq_file_t *fp);
+mm_bseq1_t *mm_bseq_read2(mm_bseq_file_t *fp, int chunk_size, int with_qual, int frag_mode, int *n_);
 mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_);
+mm_bseq1_t *mm_bseq_read_frag(int n_fp, mm_bseq_file_t **fp, int chunk_size, int with_qual, int *n_);
 int mm_bseq_eof(mm_bseq_file_t *fp);
 
 extern unsigned char seq_nt4_table[256];
+extern unsigned char seq_comp_table[256];
+
+static inline int mm_qname_len(const char *s)
+{
+	int l;
+	l = strlen(s);
+	return l >= 3 && s[l-1] >= '0' && s[l-1] <= '9' && s[l-2] == '/'? l - 2 : l;
+}
+
+static inline int mm_qname_same(const char *s1, const char *s2)
+{
+	int l1, l2;
+	l1 = mm_qname_len(s1);
+	l2 = mm_qname_len(s2);
+	return (l1 == l2 && strncmp(s1, s2, l1) == 0);
+}
+
+static inline void mm_revcomp_bseq(mm_bseq1_t *s)
+{
+	int i, t, l = s->l_seq;
+	for (i = 0; i < l>>1; ++i) {
+		t = s->seq[l - i - 1];
+		s->seq[l - i - 1] = seq_comp_table[(uint8_t)s->seq[i]];
+		s->seq[i] = seq_comp_table[t];
+	}
+	if (l&1) s->seq[l>>1] = seq_comp_table[(uint8_t)s->seq[l>>1]];
+	if (s->qual)
+		for (i = 0; i < l>>1; ++i)
+			t = s->qual[l - i - 1], s->qual[l - i - 1] = s->qual[i], s->qual[i] = t;
+}
 
 #ifdef __cplusplus
 }

chain.c

@@ -19,15 +19,15 @@ static inline int ilog2_32(uint32_t v)
 	return (t = v>>8) ? 8 + LogTable256[t] : LogTable256[v];
 }
 
-int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int64_t n, mm128_t *a, uint64_t **_u, void *km)
+mm128_t *mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int n_segs, int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km)
 { // TODO: make sure this works when n has more than 32 bits
-	int32_t st = 0, k, *f, *p, *t, *v, n_u, n_v;
-	int64_t i, j;
+	int32_t k, *f, *p, *t, *v, n_u, n_v;
+	int64_t i, j, st = 0;
 	uint64_t *u, *u2, sum_qspan = 0;
 	float avg_qspan;
 	mm128_t *b, *w;
 
-	if (_u) *_u = 0;
+	if (_u) *_u = 0, *n_u_ = 0;
 	f = (int32_t*)kmalloc(km, n * 4);
 	p = (int32_t*)kmalloc(km, n * 4);
 	t = (int32_t*)kmalloc(km, n * 4);
@@ -40,25 +40,31 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 	// fill the score and backtrack arrays
 	for (i = 0; i < n; ++i) {
 		uint64_t ri = a[i].x;
+		int64_t max_j = -1;
 		int32_t qi = (int32_t)a[i].y, q_span = a[i].y>>32&0xff; // NB: only 8 bits of span is used!!!
-		int32_t max_f = q_span, max_j = -1, n_skip = 0, min_d, max_f_past = -INT32_MAX;
+		int32_t max_f = q_span, n_skip = 0, min_d;
+		int32_t sidi = (a[i].y & MM_SEED_SEG_MASK) >> MM_SEED_SEG_SHIFT;
 		while (st < i && ri - a[st].x > max_dist_x) ++st;
 		for (j = i - 1; j >= st; --j) {
 			int64_t dr = ri - a[j].x;
-			int32_t dq = qi - (int32_t)a[j].y, dd, sc;
-			if (dr == 0 || dq <= 0 || dq > max_dist_y) continue;
+			int32_t dq = qi - (int32_t)a[j].y, dd, sc, log_dd;
+			int32_t sidj = (a[j].y & MM_SEED_SEG_MASK) >> MM_SEED_SEG_SHIFT;
+			if ((sidi == sidj && dr == 0) || dq <= 0) continue; // don't skip if an anchor is used by multiple segments; see below
+			if ((sidi == sidj && dq > max_dist_y) || dq > max_dist_x) continue;
 			dd = dr > dq? dr - dq : dq - dr;
-			if (dd > bw) continue;
-			max_f_past = max_f_past > f[j]? max_f_past : f[j];
+			if (sidi == sidj && dd > bw) continue;
+			if (n_segs > 1 && !is_cdna && sidi == sidj && dr > max_dist_y) continue;
 			min_d = dq < dr? dq : dr;
 			sc = min_d > q_span? q_span : dq < dr? dq : dr;
-			if (is_cdna) {
+			log_dd = dd? ilog2_32(dd) : 0;
+			if (is_cdna || sidi != sidj) {
 				int c_log, c_lin;
 				c_lin = (int)(dd * .01 * avg_qspan);
-				c_log = ilog2_32(dd);
-				if (dr > dq) sc -= c_lin < c_log? c_lin : c_log;
+				c_log = log_dd;
+				if (sidi != sidj && dr == 0) ++sc; // possibly due to overlapping paired ends; give a minor bonus
+				else if (dr > dq || sidi != sidj) sc -= c_lin < c_log? c_lin : c_log;
 				else sc -= c_lin + (c_log>>1);
-			} else sc -= (int)(dd * .01 * avg_qspan) + (ilog2_32(dd)>>1);
+			} else sc -= (int)(dd * .01 * avg_qspan) + (log_dd>>1);
 			sc += f[j];
 			if (sc > max_f) {
 				max_f = sc, max_j = j;
@@ -69,7 +75,8 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 			}
 			if (p[j] >= 0) t[p[j]] = i;
 		}
-		f[i] = max_f, p[i] = max_j, v[i] = max_f_past; // v[] keeps the max score in the previous chain
+		f[i] = max_f, p[i] = max_j;
+		v[i] = max_j >= 0 && v[max_j] > max_f? v[max_j] : max_f; // v[] keeps the peak score up to i; f[] is the score ending at i, not always the peak
 	}
 
 	// find the ending positions of chains
@@ -80,14 +87,14 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 		if (t[i] == 0 && v[i] >= min_sc)
 			++n_u;
 	if (n_u == 0) {
-		kfree(km, f); kfree(km, p); kfree(km, t); kfree(km, v);
+		kfree(km, a); kfree(km, f); kfree(km, p); kfree(km, t); kfree(km, v);
 		return 0;
 	}
 	u = (uint64_t*)kmalloc(km, n_u * 8);
 	for (i = n_u = 0; i < n; ++i) {
 		if (t[i] == 0 && v[i] >= min_sc) {
 			j = i;
-			while (j >= 0 && f[j] < v[j]) j = p[j]; // find the point that maximizes f[]
+			while (j >= 0 && f[j] < v[j]) j = p[j]; // find the peak that maximizes f[]
 			if (j < 0) j = i; // TODO: this should really be assert(j>=0)
 			u[n_u++] = (uint64_t)f[j] << 32 | j;
 		}
@@ -115,9 +122,9 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 		}
 		if (k0 == k) n_v = n_v0; // no new chain added, reset
 	}
-	n_u = k, *_u = u; // NB: note that u[] may not be sorted by score here
+	*n_u_ = n_u = k, *_u = u; // NB: note that u[] may not be sorted by score here
 
-	// free
+	// free temporary arrays
 	kfree(km, f); kfree(km, p); kfree(km, t);
 
 	// write the result to b[]
@@ -144,6 +151,7 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 		k += n;
 	}
 	memcpy(u, u2, n_u * 8);
-	kfree(km, b); kfree(km, w); kfree(km, u2);
-	return n_u;
+	memcpy(b, a, k * sizeof(mm128_t)); // write _a_ to _b_ and deallocate _a_ because _a_ is oversized, sometimes a lot
+	kfree(km, a); kfree(km, w); kfree(km, u2);
+	return b;
 }

esterr.c

@@ -0,0 +1,64 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "mmpriv.h"
+
+static inline int32_t get_for_qpos(int32_t qlen, const mm128_t *a)
+{
+	int32_t x = (int32_t)a->y;
+	int32_t q_span = a->y>>32 & 0xff;
+	if (a->x>>63)
+		x = qlen - 1 - (x + 1 - q_span); // revert the position to the forward strand of query
+	return x;
+}
+
+static int get_mini_idx(int qlen, const mm128_t *a, int32_t n, const uint64_t *mini_pos)
+{
+	int32_t x, L = 0, R = n - 1;
+	x = get_for_qpos(qlen, a);
+	while (L <= R) { // binary search
+		int32_t m = ((uint64_t)L + R) >> 1;
+		int32_t y = (int32_t)mini_pos[m];
+		if (y < x) L = m + 1;
+		else if (y > x) R = m - 1;
+		else return m;
+	}
+	return -1;
+}
+
+void mm_est_err(const mm_idx_t *mi, int qlen, int n_regs, mm_reg1_t *regs, const mm128_t *a, int32_t n, const uint64_t *mini_pos)
+{
+	int i;
+	uint64_t sum_k = 0;
+	float avg_k;
+
+	if (n == 0) return;
+	for (i = 0; i < n; ++i)
+		sum_k += mini_pos[i] >> 32 & 0xff;
+	avg_k = (float)sum_k / n;
+
+	for (i = 0; i < n_regs; ++i) {
+		mm_reg1_t *r = &regs[i];
+		int32_t st, en, j, k, n_match, n_tot, l_ref;
+		r->div = -1.0f;
+		if (r->cnt == 0) continue;
+		st = en = get_mini_idx(qlen, r->rev? &a[r->as + r->cnt - 1] : &a[r->as], n, mini_pos);
+		if (st < 0) {
+			if (mm_verbose >= 2)
+				fprintf(stderr, "[WARNING] logic inconsistency in mm_est_err(). Please contact the developer.\n");
+			continue;
+		}
+		l_ref = mi->seq[r->rid].len;
+		for (k = 1, j = st + 1, n_match = 1; j < n && k < r->cnt; ++j) {
+			int32_t x;
+			x = get_for_qpos(qlen, r->rev? &a[r->as + r->cnt - 1 - k] : &a[r->as + k]);
+			if (x == (int32_t)mini_pos[j])
+				++k, en = j, ++n_match;
+		}
+		n_tot = en - st + 1;
+		if (r->qs > avg_k && r->rs > avg_k) ++n_tot;
+		if (qlen - r->qs > avg_k && l_ref - r->re > avg_k) ++n_tot;
+		r->div = logf((float)n_tot / n_match) / avg_k;
+	}
+}

example.c

@@ -11,51 +11,51 @@ KSEQ_INIT(gzFile, gzread)
 
 int main(int argc, char *argv[])
 {
+	mm_idxopt_t iopt;
+	mm_mapopt_t mopt;
+	int n_threads = 3;
+
 	mm_verbose = 2; // disable message output to stderr
+	mm_set_opt(0, &iopt, &mopt);
+	mopt.flag |= MM_F_CIGAR; // perform alignment
 
 	if (argc < 3) {
 		fprintf(stderr, "Usage: minimap2-lite <target.fa> <query.fa>\n");
 		return 1;
 	}
-	
+
 	// open query file for reading; you may use your favorite FASTA/Q parser
 	gzFile f = gzopen(argv[2], "r");
 	assert(f);
 	kseq_t *ks = kseq_init(f);
 
-	// create index for target; we are creating one index for all target sequence
-	int n_threads = 4, w = 10, k = 15, is_hpc = 0;
-	mm_idx_t *mi = mm_idx_build(argv[1], w, k, is_hpc, n_threads);
-	assert(mi);
-
-	// mapping
-	mm_mapopt_t opt;
-	mm_mapopt_init(&opt); // initialize mapping parameters
-	mm_mapopt_update(&opt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()!
-	opt.flag |= MM_F_CIGAR; // perform alignment
-	mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread
-	while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence
-		const mm_reg1_t *reg;
-		int j, i, n_reg;
-		// get all hits for the query
-		reg = mm_map(mi, ks->seq.l, ks->seq.s, &n_reg, tbuf, &opt, 0);
-		// traverse hits and print them out
-		for (j = 0; j < n_reg; ++j) {
-			const mm_reg1_t *r = &reg[j];
-			assert(r->p); // with MM_F_CIGAR, this should not be NULL
-			printf("%s\t%d\t%d\t%d\t%c\t", ks->name.s, ks->seq.l, r->qs, r->qe, "+-"[r->rev]);
-			printf("%s\t%d\t%d\t%d\t%d\t%d\t%d\tcg:Z:", mi->seq[r->rid].name, mi->seq[r->rid].len, r->rs, r->re,
-				   r->p->blen - r->p->n_ambi - r->p->n_diff, r->p->blen, r->mapq);
-			for (i = 0; i < r->p->n_cigar; ++i) // IMPORTANT: this gives the CIGAR in the aligned regions. NO soft/hard clippings!
-				printf("%d%c", r->p->cigar[i]>>4, "MIDSHN"[r->p->cigar[i]&0xf]);
-			putchar('\n');
+	// open index reader
+	mm_idx_reader_t *r = mm_idx_reader_open(argv[1], &iopt, 0);
+	mm_idx_t *mi;
+	while ((mi = mm_idx_reader_read(r, n_threads)) != 0) { // traverse each part of the index
+		mm_mapopt_update(&mopt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()!
+		mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread
+		while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence
+			mm_reg1_t *reg;
+			int j, i, n_reg;
+			reg = mm_map(mi, ks->seq.l, ks->seq.s, &n_reg, tbuf, &mopt, 0); // get all hits for the query
+			for (j = 0; j < n_reg; ++j) { // traverse hits and print them out
+				mm_reg1_t *r = &reg[j];
+				assert(r->p); // with MM_F_CIGAR, this should not be NULL
+				printf("%s\t%d\t%d\t%d\t%c\t", ks->name.s, ks->seq.l, r->qs, r->qe, "+-"[r->rev]);
+				printf("%s\t%d\t%d\t%d\t%d\t%d\t%d\tcg:Z:", mi->seq[r->rid].name, mi->seq[r->rid].len, r->rs, r->re, r->mlen, r->blen, r->mapq);
+				for (i = 0; i < r->p->n_cigar; ++i) // IMPORTANT: this gives the CIGAR in the aligned regions. NO soft/hard clippings!
+					printf("%d%c", r->p->cigar[i]>>4, "MIDSHN"[r->p->cigar[i]&0xf]);
+				putchar('\n');
+				free(r->p);
+			}
+			free(reg);
 		}
+		mm_tbuf_destroy(tbuf);
+		mm_idx_destroy(mi);
 	}
-	mm_tbuf_destroy(tbuf);
-
-	// deallocate index and close the query file
-	mm_idx_destroy(mi);
-	kseq_destroy(ks);
+	mm_idx_reader_close(r); // close the index reader
+	kseq_destroy(ks); // close the query file
 	gzclose(f);
 	return 0;
 }

format.c

@@ -43,6 +43,13 @@ static void mm_sprintf_lite(kstring_t *s, const char *fmt, ...)
 				if (c < 0) buf[l++] = '-';
 				str_enlarge(s, l);
 				for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
+			} else if (*p == 'u') {
+				int i, l = 0;
+				uint32_t x;
+				x = va_arg(ap, uint32_t);
+				do { buf[l++] = x%10 + '0'; x /= 10; } while (x > 0);
+				str_enlarge(s, l);
+				for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
 			} else if (*p == 's') {
 				char *r = va_arg(ap, char*);
 				str_copy(s, r, r + strlen(r));
@@ -105,10 +112,15 @@ err_set_rg:
 	free(rg_line);
 }
 
-void mm_write_sam_hdr_no_SQ(const char *rg, const char *ver, int argc, char *argv[])
+void mm_write_sam_hdr(const mm_idx_t *idx, const char *rg, const char *ver, int argc, char *argv[])
 {
 	kstring_t str = {0,0,0};
-	sam_write_rg_line(&str, rg);
+	if (idx) {
+		uint32_t i;
+		for (i = 0; i < idx->n_seq; ++i)
+			printf("@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len);
+	}
+	if (rg) sam_write_rg_line(&str, rg);
 	mm_sprintf_lite(&str, "@PG\tID:minimap2\tPN:minimap2");
 	if (ver) mm_sprintf_lite(&str, "\tVN:%s", ver);
 	if (argc > 1) {
@@ -122,7 +134,7 @@ void mm_write_sam_hdr_no_SQ(const char *rg, const char *ver, int argc, char *arg
 	free(str.s);
 }
 
-static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r)
+static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int no_iden)
 {
 	extern unsigned char seq_nt4_table[256];
 	int i, q_off, t_off;
@@ -145,22 +157,26 @@ static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_
 	}
 	for (i = q_off = t_off = 0; i < r->p->n_cigar; ++i) {
 		int j, op = r->p->cigar[i]&0xf, len = r->p->cigar[i]>>4;
-		assert(op >= 0 && op <= 2);
+		assert(op >= 0 && op <= 3);
 		if (op == 0) {
 			int l_tmp = 0;
 			for (j = 0; j < len; ++j) {
 				if (qseq[q_off + j] != tseq[t_off + j]) {
 					if (l_tmp > 0) {
-						tmp[l_tmp] = 0;
-						mm_sprintf_lite(s, "=%s", tmp);
+						if (!no_iden) {
+							tmp[l_tmp] = 0;
+							mm_sprintf_lite(s, "=%s", tmp);
+						} else mm_sprintf_lite(s, ":%d", l_tmp);
 						l_tmp = 0;
 					}
 					mm_sprintf_lite(s, "*%c%c", "acgtn"[tseq[t_off + j]], "acgtn"[qseq[q_off + j]]);
 				} else tmp[l_tmp++] = "ACGTN"[qseq[q_off + j]];
 			}
 			if (l_tmp > 0) {
-				tmp[l_tmp] = 0;
-				mm_sprintf_lite(s, "=%s", tmp);
+				if (!no_iden) {
+					tmp[l_tmp] = 0;
+					mm_sprintf_lite(s, "=%s", tmp);
+				} else mm_sprintf_lite(s, ":%d", l_tmp);
 			}
 			q_off += len, t_off += len;
 		} else if (op == 1) {
@@ -173,6 +189,11 @@ static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_
 				tmp[j] = "acgtn"[tseq[t_off + j]];
 			mm_sprintf_lite(s, "-%s", tmp);
 			t_off += len;
+		} else {
+			assert(len >= 2);
+			mm_sprintf_lite(s, "~%c%c%d%c%c", "acgtn"[tseq[t_off]], "acgtn"[tseq[t_off+1]],
+				len, "acgtn"[tseq[t_off+len-2]], "acgtn"[tseq[t_off+len-1]]);
+			t_off += len;
 		}
 	}
 	assert(t_off == r->re - r->rs && q_off == r->qe - r->qs);
@@ -181,15 +202,23 @@ static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_
 
 static inline void write_tags(kstring_t *s, const mm_reg1_t *r)
 {
-	int type = r->inv? 'I' : r->id == r->parent? 'P' : 'S';
-	mm_sprintf_lite(s, "\ttp:A:%c\tcm:i:%d\ts1:i:%d", type, r->cnt, r->score);
-	if (r->parent == r->id) mm_sprintf_lite(s, "\ts2:i:%d", r->subsc);
-	if (r->split) mm_sprintf_lite(s, "\tzd:i:%d", r->split);
+	int type;
+	if (r->id == r->parent) type = r->inv? 'I' : 'P';
+	else type = r->inv? 'i' : 'S';
 	if (r->p) {
-		mm_sprintf_lite(s, "\tNM:i:%d\tms:i:%d\tAS:i:%d\tnn:i:%d", r->p->n_diff, r->p->dp_max, r->p->dp_score, r->p->n_ambi);
+		mm_sprintf_lite(s, "\tNM:i:%d\tms:i:%d\tAS:i:%d\tnn:i:%d", r->blen - r->mlen + r->p->n_ambi, r->p->dp_max, r->p->dp_score, r->p->n_ambi);
 		if (r->p->trans_strand == 1 || r->p->trans_strand == 2)
 			mm_sprintf_lite(s, "\tts:A:%c", "?+-?"[r->p->trans_strand]);
 	}
+	mm_sprintf_lite(s, "\ttp:A:%c\tcm:i:%d\ts1:i:%d", type, r->cnt, r->score);
+	if (r->parent == r->id) mm_sprintf_lite(s, "\ts2:i:%d", r->subsc);
+	if (r->div >= 0.0f && r->div <= 1.0f) {
+		char buf[8];
+		if (r->div == 0.0f) buf[0] = '0', buf[1] = 0;
+		else sprintf(buf, "%.4f", r->div);
+		mm_sprintf_lite(s, "\tdv:f:%s", buf);
+	}
+	if (r->split) mm_sprintf_lite(s, "\tzd:i:%d", r->split);
 }
 
 void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag)
@@ -199,8 +228,7 @@ void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 	if (mi->seq[r->rid].name) mm_sprintf_lite(s, "%s", mi->seq[r->rid].name);
 	else mm_sprintf_lite(s, "%d", r->rid);
 	mm_sprintf_lite(s, "\t%d\t%d\t%d", mi->seq[r->rid].len, r->rs, r->re);
-	if (r->p) mm_sprintf_lite(s, "\t%d\t%d", r->p->blen - r->p->n_ambi - r->p->n_diff, r->p->blen);
-	else mm_sprintf_lite(s, "\t%d\t%d", r->fuzzy_mlen, r->fuzzy_blen);
+	mm_sprintf_lite(s, "\t%d\t%d", r->mlen, r->blen);
 	mm_sprintf_lite(s, "\t%d", r->mapq);
 	write_tags(s, r);
 	if (r->p && (opt_flag & MM_F_OUT_CG)) {
@@ -210,66 +238,158 @@ void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 			mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MIDN"[r->p->cigar[k]&0xf]);
 	}
 	if (r->p && (opt_flag & MM_F_OUT_CS))
-		write_cs(km, s, mi, t, r);
-}
-
-static char comp_tab[] = {
-	  0,   1,	2,	 3,	  4,   5,	6,	 7,	  8,   9,  10,	11,	 12,  13,  14,	15,
-	 16,  17,  18,	19,	 20,  21,  22,	23,	 24,  25,  26,	27,	 28,  29,  30,	31,
-	 32,  33,  34,	35,	 36,  37,  38,	39,	 40,  41,  42,	43,	 44,  45,  46,	47,
-	 48,  49,  50,	51,	 52,  53,  54,	55,	 56,  57,  58,	59,	 60,  61,  62,	63,
-	 64, 'T', 'V', 'G', 'H', 'E', 'F', 'C', 'D', 'I', 'J', 'M', 'L', 'K', 'N', 'O',
-	'P', 'Q', 'Y', 'S', 'A', 'A', 'B', 'W', 'X', 'R', 'Z',	91,	 92,  93,  94,	95,
-	 64, 't', 'v', 'g', 'h', 'e', 'f', 'c', 'd', 'i', 'j', 'm', 'l', 'k', 'n', 'o',
-	'p', 'q', 'y', 's', 'a', 'a', 'b', 'w', 'x', 'r', 'z', 123, 124, 125, 126, 127
-};
-
-void mm_write_sam_SQ(const mm_idx_t *idx)
-{
-	uint32_t i;
-	for (i = 0; i < idx->n_seq; ++i)
-		printf("@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len);
+		write_cs(km, s, mi, t, r, !(opt_flag&MM_F_OUT_CS_LONG));
 }
 
 static void sam_write_sq(kstring_t *s, char *seq, int l, int rev, int comp)
 {
+	extern unsigned char seq_comp_table[256];
 	if (rev) {
 		int i;
 		str_enlarge(s, l);
 		for (i = 0; i < l; ++i) {
 			int c = seq[l - 1 - i];
-			s->s[s->l + i] = c < 128 && comp? comp_tab[c] : c;
+			s->s[s->l + i] = c < 128 && comp? seq_comp_table[c] : c;
 		}
 		s->l += l;
 	} else str_copy(s, seq, seq + l);
 }
 
-void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs)
+static inline const mm_reg1_t *get_sam_pri(int n_regs, const mm_reg1_t *regs)
 {
-	int flag = 0;
+	int i;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].sam_pri)
+			return &regs[i];
+	assert(n_regs == 0);
+	return NULL;
+}
+
+static void write_sam_cigar(kstring_t *s, int sam_flag, int in_tag, int qlen, const mm_reg1_t *r, int opt_flag)
+{
+	if (r->p == 0) {
+		mm_sprintf_lite(s, "*");
+	} else {
+		uint32_t k, clip_len[2];
+		clip_len[0] = r->rev? qlen - r->qe : r->qs;
+		clip_len[1] = r->rev? r->qs : qlen - r->qe;
+		if (in_tag) {
+			int clip_char = (sam_flag&0x800) && !(opt_flag&MM_F_SOFTCLIP)? 5 : 4;
+			mm_sprintf_lite(s, "\tCG:B:I");
+			if (clip_len[0]) mm_sprintf_lite(s, ",%u", clip_len[0]<<4|clip_char);
+			for (k = 0; k < r->p->n_cigar; ++k)
+				mm_sprintf_lite(s, ",%u", r->p->cigar[k]);
+			if (clip_len[1]) mm_sprintf_lite(s, ",%u", clip_len[1]<<4|clip_char);
+		} else {
+			int clip_char = (sam_flag&0x800) && !(opt_flag&MM_F_SOFTCLIP)? 'H' : 'S';
+			if (clip_len[0]) mm_sprintf_lite(s, "%d%c", clip_len[0], clip_char);
+			for (k = 0; k < r->p->n_cigar; ++k)
+				mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MIDN"[r->p->cigar[k]&0xf]);
+			if (clip_len[1]) mm_sprintf_lite(s, "%d%c", clip_len[1], clip_char);
+		}
+	}
+}
+
+void mm_write_sam2(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regss, const mm_reg1_t *const* regss, void *km, int opt_flag)
+{
+	const int max_bam_cigar_op = 65535;
+	int flag, n_regs = n_regss[seg_idx], cigar_in_tag = 0;
+	int this_rid = -1, this_pos = -1, this_rev = 0;
+	const mm_reg1_t *regs = regss[seg_idx], *r_prev = NULL, *r_next;
+	const mm_reg1_t *r = n_regs > 0 && reg_idx < n_regs && reg_idx >= 0? &regs[reg_idx] : NULL;
+
+	// find the primary of the previous and the next segments, if they are mapped
+	if (n_seg > 1) {
+		int i, next_sid = (seg_idx + 1) % n_seg;
+		r_next = get_sam_pri(n_regss[next_sid], regss[next_sid]);
+		if (n_seg > 2) {
+			for (i = 1; i <= n_seg - 1; ++i) {
+				int prev_sid = (seg_idx + n_seg - i) % n_seg;
+				if (n_regss[prev_sid] > 0) {
+					r_prev = get_sam_pri(n_regss[prev_sid], regss[prev_sid]);
+					break;
+				}
+			}
+		} else r_prev = r_next;
+	} else r_prev = r_next = NULL;
+
+	// write QNAME
 	s->l = 0;
+	mm_sprintf_lite(s, "%s", t->name);
+	if (n_seg > 1) s->l = mm_qname_len(t->name); // trim the suffix like /1 or /2
+
+	// write flag
+	flag = n_seg > 1? 0x1 : 0x0;
+	if (r == 0) {
+		flag |= 0x4;
+	} else {
+		if (r->rev) flag |= 0x10;
+		if (r->parent != r->id) flag |= 0x100;
+		else if (!r->sam_pri) flag |= 0x800;
+	}
+	if (n_seg > 1) {
+		if (r && r->proper_frag) flag |= 0x2; // TODO: this doesn't work when there are more than 2 segments
+		if (seg_idx == 0) flag |= 0x40;
+		else if (seg_idx == n_seg - 1) flag |= 0x80;
+		if (r_next == NULL) flag |= 0x8;
+		else if (r_next->rev) flag |= 0x20;
+	}
+	mm_sprintf_lite(s, "\t%d", flag);
+
+	// write coordinate, MAPQ and CIGAR
+	if (r == 0) {
+		if (r_prev) {
+			this_rid = r_prev->rid, this_pos = r_prev->rs;
+			mm_sprintf_lite(s, "\t%s\t%d\t0\t*", mi->seq[this_rid].name, this_pos+1);
+		} else mm_sprintf_lite(s, "\t*\t0\t0\t*");
+	} else {
+		this_rid = r->rid, this_pos = r->rs, this_rev = r->rev;
+		mm_sprintf_lite(s, "\t%s\t%d\t%d\t", mi->seq[r->rid].name, r->rs+1, r->mapq);
+		if ((opt_flag & MM_F_LONG_CIGAR) && r->p && r->p->n_cigar > max_bam_cigar_op - 2) {
+			int n_cigar = r->p->n_cigar;
+			if (r->qs != 0) ++n_cigar;
+			if (r->qe != t->l_seq) ++n_cigar;
+			if (n_cigar > max_bam_cigar_op)
+				cigar_in_tag = 1;
+		}
+		if (cigar_in_tag) {
+			if (flag & 0x100) mm_sprintf_lite(s, "0S"); // secondary alignment
+			else if (flag & 0x800) mm_sprintf_lite(s, "%dS", r->re - r->rs); // supplementary alignment
+			else mm_sprintf_lite(s, "%dS", t->l_seq);
+		} else write_sam_cigar(s, flag, 0, t->l_seq, r, opt_flag);
+	}
+
+	// write mate positions
+	if (n_seg > 1) {
+		int tlen = 0;
+		if (this_rid >= 0 && r_next) {
+			if (this_rid == r_next->rid) {
+				int this_pos5 = r && r->rev? r->re - 1 : this_pos;
+				int next_pos5 = r_next->rev? r_next->re - 1 : r_next->rs;
+				tlen = next_pos5 - this_pos5;
+				mm_sprintf_lite(s, "\t=\t");
+			} else mm_sprintf_lite(s, "\t%s\t", mi->seq[r_next->rid].name);
+			mm_sprintf_lite(s, "%d\t", r_next->rs + 1);
+		} else if (r_next) { // && this_rid < 0
+			mm_sprintf_lite(s, "\t%s\t%d\t", mi->seq[r_next->rid].name, r_next->rs + 1);
+		} else if (this_rid >= 0) { // && r_next == NULL
+			int this_pos5 = this_rev? r->re - 1 : this_pos; // this_rev is only true when r != NULL
+			tlen = this_pos - this_pos5; // next_pos5 will be this_pos
+			mm_sprintf_lite(s, "\t=\t%d\t", this_pos + 1); // next segment will take r's coordinate
+		} else mm_sprintf_lite(s, "\t*\t0\t"); // neither has coordinates
+		if (tlen > 0) ++tlen;
+		else if (tlen < 0) --tlen;
+		mm_sprintf_lite(s, "%d\t", tlen);
+	} else mm_sprintf_lite(s, "\t*\t0\t0\t");
+
+	// write SEQ and QUAL
 	if (r == 0) {
-		mm_sprintf_lite(s, "%s\t4\t*\t0\t0\t*\t*\t0\t0\t", t->name);
 		sam_write_sq(s, t->seq, t->l_seq, 0, 0);
 		mm_sprintf_lite(s, "\t");
 		if (t->qual) sam_write_sq(s, t->qual, t->l_seq, 0, 0);
 		else mm_sprintf_lite(s, "*");
 	} else {
-		if (r->rev) flag |= 0x10;
-		if (r->parent != r->id) flag |= 0x100;
-		else if (!r->sam_pri) flag |= 0x800;
-		mm_sprintf_lite(s, "%s\t%d\t%s\t%d\t%d\t", t->name, flag, mi->seq[r->rid].name, r->rs+1, r->mapq);
-		if (r->p) { // actually this should always be true for SAM output
-			uint32_t k, clip_len = r->rev? t->l_seq - r->qe : r->qs;
-			int clip_char = (flag&0x800)? 'H' : 'S';
-			if (clip_len) mm_sprintf_lite(s, "%d%c", clip_len, clip_char);
-			for (k = 0; k < r->p->n_cigar; ++k)
-				mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MIDN"[r->p->cigar[k]&0xf]);
-			clip_len = r->rev? r->qs : t->l_seq - r->qe;
-			if (clip_len) mm_sprintf_lite(s, "%d%c", clip_len, clip_char);
-		} else mm_sprintf_lite(s, "*");
-		mm_sprintf_lite(s, "\t*\t0\t0\t");
-		if ((flag & 0x900) == 0) {
+		if ((flag & 0x900) == 0 || (opt_flag & MM_F_SOFTCLIP)) {
 			sam_write_sq(s, t->seq, t->l_seq, r->rev, r->rev);
 			mm_sprintf_lite(s, "\t");
 			if (t->qual) sam_write_sq(s, t->qual, t->l_seq, r->rev, 0);
@@ -282,8 +402,13 @@ void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 			if (t->qual) sam_write_sq(s, t->qual + r->qs, r->qe - r->qs, r->rev, 0);
 			else mm_sprintf_lite(s, "*");
 		}
+	}
+
+	// write tags
+	if (mm_rg_id[0]) mm_sprintf_lite(s, "\tRG:Z:%s", mm_rg_id);
+	if (n_seg > 2) mm_sprintf_lite(s, "\tFI:i:%d", seg_idx);
+	if (r) {
 		write_tags(s, r);
-		if (mm_rg_id[0]) mm_sprintf_lite(s, "\tRG:Z:%s", mm_rg_id);
 		if (r->parent == r->id && r->p && n_regs > 1 && regs && r >= regs && r - regs < n_regs) { // supplementary aln may exist
 			int i, n_sa = 0; // n_sa: number of SA fields
 			for (i = 0; i < n_regs; ++i)
@@ -305,10 +430,23 @@ void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 					if (l_I) mm_sprintf_lite(s, "%dI", l_I);
 					if (l_D) mm_sprintf_lite(s, "%dD", l_D);
 					if (clip3) mm_sprintf_lite(s, "%dS", clip3);
-					mm_sprintf_lite(s, ",%d,%d;", q->mapq, q->p->n_diff);
+					mm_sprintf_lite(s, ",%d,%d;", q->mapq, q->blen - q->mlen + q->p->n_ambi);
 				}
 			}
 		}
+		if (r->p && (opt_flag & MM_F_OUT_CS))
+			write_cs(km, s, mi, t, r, !(opt_flag&MM_F_OUT_CS_LONG));
+		if (cigar_in_tag)
+			write_sam_cigar(s, flag, 1, t->l_seq, r, opt_flag);
 	}
+
 	s->s[s->l] = 0; // we always have room for an extra byte (see str_enlarge)
 }
+
+void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs)
+{
+	int i;
+	for (i = 0; i < n_regs; ++i)
+		if (r == &regs[i]) break;
+	mm_write_sam2(s, mi, t, 0, i, 1, &n_regs, &regs, NULL, 0);
+}

getopt.c

@@ -0,0 +1,216 @@
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+#include "getopt.h"
+
+char *optarg;
+int optind=1, opterr=1, optopt, __optpos, optreset=0;
+
+#define optpos __optpos
+
+static void __getopt_msg(const char *a, const char *b, const char *c, size_t l)
+{
+	FILE *f = stderr;
+#if !defined(WIN32) && !defined(_WIN32)
+	flockfile(f);
+#endif
+	fputs(a, f);
+	fwrite(b, strlen(b), 1, f);
+	fwrite(c, 1, l, f);
+	fputc('\n', f);
+#if !defined(WIN32) && !defined(_WIN32)
+	funlockfile(f);
+#endif
+}
+
+int getopt(int argc, char * const argv[], const char *optstring)
+{
+	int i, c, d;
+	int k, l;
+	char *optchar;
+
+	if (!optind || optreset) {
+		optreset = 0;
+		__optpos = 0;
+		optind = 1;
+	}
+
+	if (optind >= argc || !argv[optind])
+		return -1;
+
+	if (argv[optind][0] != '-') {
+		if (optstring[0] == '-') {
+			optarg = argv[optind++];
+			return 1;
+		}
+		return -1;
+	}
+
+	if (!argv[optind][1])
+		return -1;
+
+	if (argv[optind][1] == '-' && !argv[optind][2])
+		return optind++, -1;
+
+	if (!optpos) optpos++;
+	c = argv[optind][optpos], k = 1;
+	optchar = argv[optind]+optpos;
+	optopt = c;
+	optpos += k;
+
+	if (!argv[optind][optpos]) {
+		optind++;
+		optpos = 0;
+	}
+
+	if (optstring[0] == '-' || optstring[0] == '+')
+		optstring++;
+
+	i = 0;
+	d = 0;
+	do {
+		d = optstring[i], l = 1;
+		if (l>0) i+=l; else i++;
+	} while (l && d != c);
+
+	if (d != c) {
+		if (optstring[0] != ':' && opterr)
+			__getopt_msg(argv[0], ": unrecognized option: ", optchar, k);
+		return '?';
+	}
+	if (optstring[i] == ':') {
+		if (optstring[i+1] == ':') optarg = 0;
+		else if (optind >= argc) {
+			if (optstring[0] == ':') return ':';
+			if (opterr) __getopt_msg(argv[0],
+				": option requires an argument: ",
+				optchar, k);
+			return '?';
+		}
+		if (optstring[i+1] != ':' || optpos) {
+			optarg = argv[optind++] + optpos;
+			optpos = 0;
+		}
+	}
+	return c;
+}
+
+static void permute(char *const *argv, int dest, int src)
+{
+	char **av = (char **)argv;
+	char *tmp = av[src];
+	int i;
+	for (i=src; i>dest; i--)
+		av[i] = av[i-1];
+	av[dest] = tmp;
+}
+
+static int __getopt_long_core(int argc, char *const *argv, const char *optstring, const struct option *longopts, int *idx, int longonly)
+{
+	optarg = 0;
+	if (longopts && argv[optind][0] == '-' &&
+		((longonly && argv[optind][1] && argv[optind][1] != '-') ||
+		 (argv[optind][1] == '-' && argv[optind][2])))
+	{
+		int colon = optstring[optstring[0]=='+'||optstring[0]=='-']==':';
+		int i, cnt, match = -1;
+		char *opt;
+		for (cnt=i=0; longopts[i].name; i++) {
+			const char *name = longopts[i].name;
+			opt = argv[optind]+1;
+			if (*opt == '-') opt++;
+			for (; *name && *name == *opt; name++, opt++);
+			if (*opt && *opt != '=') continue;
+			match = i;
+			if (!*name) {
+				cnt = 1;
+				break;
+			}
+			cnt++;
+		}
+		if (cnt==1) {
+			i = match;
+			optind++;
+			optopt = longopts[i].val;
+			if (*opt == '=') {
+				if (!longopts[i].has_arg) {
+					if (colon || !opterr)
+						return '?';
+					__getopt_msg(argv[0],
+						": option does not take an argument: ",
+						longopts[i].name,
+						strlen(longopts[i].name));
+					return '?';
+				}
+				optarg = opt+1;
+			} else if (longopts[i].has_arg == required_argument) {
+				if (!(optarg = argv[optind])) {
+					if (colon) return ':';
+					if (!opterr) return '?';
+					__getopt_msg(argv[0],
+						": option requires an argument: ",
+						longopts[i].name,
+						strlen(longopts[i].name));
+					return '?';
+				}
+				optind++;
+			}
+			if (idx) *idx = i;
+			if (longopts[i].flag) {
+				*longopts[i].flag = longopts[i].val;
+				return 0;
+			}
+			return longopts[i].val;
+		}
+		if (argv[optind][1] == '-') {
+			if (!colon && opterr)
+				__getopt_msg(argv[0], cnt ?
+					": option is ambiguous: " :
+					": unrecognized option: ",
+					argv[optind]+2,
+					strlen(argv[optind]+2));
+			optind++;
+			return '?';
+		}
+	}
+	return getopt(argc, argv, optstring);
+}
+
+static int __getopt_long(int argc, char *const *argv, const char *optstring, const struct option *longopts, int *idx, int longonly)
+{
+	int ret, skipped, resumed;
+	if (!optind || optreset) {
+		optreset = 0;
+		__optpos = 0;
+		optind = 1;
+	}
+	if (optind >= argc || !argv[optind]) return -1;
+	skipped = optind;
+	if (optstring[0] != '+' && optstring[0] != '-') {
+		int i;
+		for (i=optind; ; i++) {
+			if (i >= argc || !argv[i]) return -1;
+			if (argv[i][0] == '-' && argv[i][1]) break;
+		}
+		optind = i;
+	}
+	resumed = optind;
+	ret = __getopt_long_core(argc, argv, optstring, longopts, idx, longonly);
+	if (resumed > skipped) {
+		int i, cnt = optind-resumed;
+		for (i=0; i<cnt; i++)
+			permute(argv, skipped, optind-1);
+		optind = skipped + cnt;
+	}
+	return ret;
+}
+
+int getopt_long(int argc, char *const *argv, const char *optstring, const struct option *longopts, int *idx)
+{
+	return __getopt_long(argc, argv, optstring, longopts, idx, 0);
+}
+
+int getopt_long_only(int argc, char *const *argv, const char *optstring, const struct option *longopts, int *idx)
+{
+	return __getopt_long(argc, argv, optstring, longopts, idx, 1);
+}

getopt.h

@@ -0,0 +1,53 @@
+/*
+  Copyright 2005-2014 Rich Felker, et al.
+  
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+  
+  The above copyright notice and this permission notice shall be
+  included in all copies or substantial portions of the Software.
+  
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+#ifndef _GETOPT_H
+#define _GETOPT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int getopt(int, char * const [], const char *);
+extern char *optarg;
+extern int optind, opterr, optopt, optreset;
+
+struct option {
+	const char *name;
+	int has_arg;
+	int *flag;
+	int val;
+};
+
+int getopt_long(int, char *const *, const char *, const struct option *, int *);
+int getopt_long_only(int, char *const *, const char *, const struct option *, int *);
+
+#define no_argument        0
+#define required_argument  1
+#define optional_argument  2
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

hit.c

@@ -1,20 +1,22 @@
 #include <string.h>
+#include <stdlib.h>
 #include <math.h>
 #include "mmpriv.h"
 #include "kalloc.h"
+#include "khash.h"
 
 static inline void mm_cal_fuzzy_len(mm_reg1_t *r, const mm128_t *a)
 {
 	int i;
-	r->fuzzy_mlen = r->fuzzy_blen = 0;
+	r->mlen = r->blen = 0;
 	if (r->cnt <= 0) return;
-	r->fuzzy_mlen = r->fuzzy_blen = a[r->as].y>>32&0xff;
+	r->mlen = r->blen = a[r->as].y>>32&0xff;
 	for (i = r->as + 1; i < r->as + r->cnt; ++i) {
 		int span = a[i].y>>32&0xff;
 		int tl = (int32_t)a[i].x - (int32_t)a[i-1].x;
 		int ql = (int32_t)a[i].y - (int32_t)a[i-1].y;
-		r->fuzzy_blen += tl > ql? tl : ql;
-		r->fuzzy_mlen += tl > span && ql > span? span : tl < ql? tl : ql;
+		r->blen += tl > ql? tl : ql;
+		r->mlen += tl > span && ql > span? span : tl < ql? tl : ql;
 	}
 }
 
@@ -35,7 +37,19 @@ static inline void mm_reg_set_coor(mm_reg1_t *r, int32_t qlen, const mm128_t *a)
 	mm_cal_fuzzy_len(r, a);
 }
 
-mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) // convert chains to hits
+static inline uint64_t hash64(uint64_t key)
+{
+	key = (~key + (key << 21));
+	key = key ^ key >> 24;
+	key = ((key + (key << 3)) + (key << 8));
+	key = key ^ key >> 14;
+	key = ((key + (key << 2)) + (key << 4));
+	key = key ^ key >> 28;
+	key = (key + (key << 31));
+	return key;
+}
+
+mm_reg1_t *mm_gen_regs(void *km, uint32_t hash, int qlen, int n_u, uint64_t *u, mm128_t *a) // convert chains to hits
 {
 	mm128_t *z, tmp;
 	mm_reg1_t *r;
@@ -46,7 +60,9 @@ mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) //
 	// sort by score
 	z = (mm128_t*)kmalloc(km, n_u * 16);
 	for (i = k = 0; i < n_u; ++i) {
-		z[i].x = u[i] >> 32;
+		uint32_t h;
+		h = (uint32_t)hash64((hash64(a[k].x) + hash64(a[k].y)) ^ hash);
+		z[i].x = u[i] ^ h; // u[i] -- higher 32 bits: chain score; lower 32 bits: number of seeds in the chain
 		z[i].y = (uint64_t)k << 32 | (int32_t)u[i];
 		k += (int32_t)u[i];
 	}
@@ -60,9 +76,11 @@ mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) //
 		mm_reg1_t *ri = &r[i];
 		ri->id = i;
 		ri->parent = MM_PARENT_UNSET;
-		ri->score = z[i].x;
+		ri->score = ri->score0 = z[i].x >> 32;
+		ri->hash = (uint32_t)z[i].x;
 		ri->cnt = (int32_t)z[i].y;
 		ri->as = z[i].y >> 32;
+		ri->div = -1.0f;
 		mm_reg_set_coor(ri, qlen, a);
 	}
 	kfree(km, z);
@@ -87,55 +105,86 @@ void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a)
 	r->split |= 1, r2->split |= 2;
 }
 
-void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r) // and compute mm_reg1_t::subsc
+void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r, int sub_diff) // and compute mm_reg1_t::subsc
 {
 	int i, j, k, *w;
+	uint64_t *cov;
 	if (n <= 0) return;
 	for (i = 0; i < n; ++i) r[i].id = i;
+	cov = (uint64_t*)kmalloc(km, n * sizeof(uint64_t));
 	w = (int*)kmalloc(km, n * sizeof(int));
 	w[0] = 0, r[0].parent = 0;
 	for (i = 1, k = 1; i < n; ++i) {
 		mm_reg1_t *ri = &r[i];
-		int si = ri->qs, ei = ri->qe;
-		for (j = 0; j < k; ++j) {
+		int si = ri->qs, ei = ri->qe, n_cov = 0, uncov_len = 0;
+		for (j = 0; j < k; ++j) { // traverse existing primary hits to find overlapping hits
 			mm_reg1_t *rp = &r[w[j]];
 			int sj = rp->qs, ej = rp->qe;
-			int min = ej - sj < ei - si? ej - sj : ei - si;
-			int ol = si < sj? (ei < sj? 0 : ei < ej? ei - sj : ej - sj) : (ej < si? 0 : ej < ei? ej - si : ei - si);
-			if (ol > mask_level * min) {
+			if (ej <= si || sj >= ei) continue;
+			if (sj < si) sj = si;
+			if (ej > ei) ej = ei;
+			cov[n_cov++] = (uint64_t)sj<<32 | ej;
+		}
+		if (n_cov == 0) {
+			goto set_parent_test; // no overlapping primary hits; then i is a new primary hit
+		} else if (n_cov > 0) { // there are overlapping primary hits; find the length not covered by existing primary hits
+			int j, x = si;
+			radix_sort_64(cov, cov + n_cov);
+			for (j = 0; j < n_cov; ++j) {
+				if (cov[j]>>32 > x) uncov_len += (cov[j]>>32) - x;
+				x = (int32_t)cov[j] > x? (int32_t)cov[j] : x;
+			}
+			if (ei > x) uncov_len += ei - x;
+		}
+		for (j = 0; j < k; ++j) { // traverse existing primary hits again
+			mm_reg1_t *rp = &r[w[j]];
+			int sj = rp->qs, ej = rp->qe, min, max, ol;
+			if (ej <= si || sj >= ei) continue; // no overlap
+			min = ej - sj < ei - si? ej - sj : ei - si;
+			max = ej - sj > ei - si? ej - sj : ei - si;
+			ol = si < sj? (ei < sj? 0 : ei < ej? ei - sj : ej - sj) : (ej < si? 0 : ej < ei? ej - si : ei - si); // overlap length
+			if ((float)ol / min - (float)uncov_len / max > mask_level) {
+				int cnt_sub = 0;
 				ri->parent = rp->parent;
 				rp->subsc = rp->subsc > ri->score? rp->subsc : ri->score;
-				if (rp->p && ri->p)
+				if (ri->cnt >= rp->cnt) cnt_sub = 1;
+				if (rp->p && ri->p && (rp->rid != ri->rid || rp->rs != ri->rs || rp->re != ri->re || ol != min)) { // the last condition excludes identical hits after DP
 					rp->p->dp_max2 = rp->p->dp_max2 > ri->p->dp_max? rp->p->dp_max2 : ri->p->dp_max;
+					if (rp->p->dp_max - ri->p->dp_max <= sub_diff) cnt_sub = 1;
+				}
+				if (cnt_sub) ++rp->n_sub;
 				break;
 			}
 		}
-		if (j == k) w[k++] = i, ri->parent = i;
+set_parent_test:
+		if (j == k) w[k++] = i, ri->parent = i, ri->n_sub = 0;
 	}
+	kfree(km, cov);
 	kfree(km, w);
 }
 
 void mm_hit_sort_by_dp(void *km, int *n_regs, mm_reg1_t *r)
 {
 	int32_t i, n_aux, n = *n_regs;
-	uint64_t *aux;
+	mm128_t *aux;
 	mm_reg1_t *t;
 
 	if (n <= 1) return;
-	aux = (uint64_t*)kmalloc(km, n * 8);
+	aux = (mm128_t*)kmalloc(km, n * 16);
 	t = (mm_reg1_t*)kmalloc(km, n * sizeof(mm_reg1_t));
 	for (i = n_aux = 0; i < n; ++i) {
 		if (r[i].inv || r[i].cnt > 0) { // squeeze out elements with cnt==0 (soft deleted)
 			assert(r[i].p);
-			aux[n_aux++] = (uint64_t)r[i].p->dp_max << 32 | i;
+			aux[n_aux].x = (uint64_t)r[i].p->dp_max << 32 | r[i].hash;
+			aux[n_aux++].y = i;
 		} else if (r[i].p) {
 			free(r[i].p);
 			r[i].p = 0;
 		}
 	}
-	radix_sort_64(aux, aux + n_aux);
+	radix_sort_128x(aux, aux + n_aux);
 	for (i = n_aux - 1; i >= 0; --i)
-		t[n_aux - 1 - i] = r[(int32_t)aux[i]];
+		t[n_aux - 1 - i] = r[aux[i].y];
 	memcpy(r, t, sizeof(mm_reg1_t) * n_aux);
 	*n_regs = n_aux;
 	kfree(km, aux);
@@ -177,15 +226,20 @@ void mm_sync_regs(void *km, int n_regs, mm_reg1_t *regs) // keep mm_reg1_t::{id,
 	mm_set_sam_pri(n_regs, regs);
 }
 
-void mm_select_sub(void *km, float mask_level, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r)
+void mm_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r)
 {
 	if (pri_ratio > 0.0f && *n_ > 0) {
 		int i, k, n = *n_, n_2nd = 0;
-		for (i = k = 0; i < n; ++i)
-			if (r[i].parent == i) r[k++] = r[i];
-			else if ((r[i].score >= r[r[i].parent].score * pri_ratio || r[i].score + min_diff >= r[r[i].parent].score) && n_2nd++ < best_n)
+		for (i = k = 0; i < n; ++i) {
+			int p = r[i].parent;
+			if (p == i || r[i].inv) { // primary or inversion
 				r[k++] = r[i];
-			else if (r[i].p) free(r[i].p);
+			} else if ((r[i].score >= r[p].score * pri_ratio || r[i].score + min_diff >= r[p].score) && n_2nd < best_n) {
+				if (!(r[i].qs == r[p].qs && r[i].qe == r[p].qe && r[i].rid == r[p].rid && r[i].rs == r[p].rs && r[i].re == r[p].re)) // not identical hits
+					r[k++] = r[i], ++n_2nd;
+				else if (r[i].p) free(r[i].p);
+			} else if (r[i].p) free(r[i].p);
+		}
 		if (k != n) mm_sync_regs(km, k, r); // removing hits requires sync()
 		*n_ = k;
 	}
@@ -197,9 +251,9 @@ void mm_filter_regs(void *km, const mm_mapopt_t *opt, int *n_regs, mm_reg1_t *re
 	for (i = k = 0; i < *n_regs; ++i) {
 		mm_reg1_t *r = &regs[i];
 		int flt = 0;
-		if (!r->inv && r->cnt < opt->min_cnt) flt = 1;
+		if (!r->inv && !r->seg_split && r->cnt < opt->min_cnt) flt = 1;
 		if (r->p) {
-			if (r->p->blen - r->p->n_ambi - r->p->n_diff < opt->min_chain_score) flt = 1;
+			if (r->mlen < opt->min_chain_score) flt = 1;
 			else if (r->p->dp_max < opt->min_dp_max) flt = 1;
 			if (flt) free(r->p);
 		}
@@ -288,24 +342,109 @@ void mm_join_long(void *km, const mm_mapopt_t *opt, int qlen, int *n_regs_, mm_r
 	}
 }
 
-void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc)
+mm_seg_t *mm_seg_gen(void *km, uint32_t hash, int n_segs, const int *qlens, int n_regs0, const mm_reg1_t *regs0, int *n_regs, mm_reg1_t **regs, const mm128_t *a)
+{
+	int s, i, j, acc_qlen[MM_MAX_SEG+1], qlen_sum = 0;
+	mm_seg_t *seg;
+
+	assert(n_segs <= MM_MAX_SEG);
+	for (s = 1, acc_qlen[0] = 0; s < n_segs; ++s)
+		acc_qlen[s] = acc_qlen[s-1] + qlens[s-1];
+	qlen_sum = acc_qlen[n_segs - 1] + qlens[n_segs - 1];
+
+	seg = (mm_seg_t*)kcalloc(km, n_segs, sizeof(mm_seg_t));
+	for (s = 0; s < n_segs; ++s) {
+		seg[s].u = (uint64_t*)kmalloc(km, n_regs0 * 8);
+		for (i = 0; i < n_regs0; ++i)
+			seg[s].u[i] = (uint64_t)regs0[i].score << 32;
+	}
+	for (i = 0; i < n_regs0; ++i) {
+		const mm_reg1_t *r = &regs0[i];
+		for (j = 0; j < r->cnt; ++j) {
+			int sid = (a[r->as + j].y&MM_SEED_SEG_MASK)>>MM_SEED_SEG_SHIFT;
+			++seg[sid].u[i];
+			++seg[sid].n_a;
+		}
+	}
+	for (s = 0; s < n_segs; ++s) {
+		mm_seg_t *sr = &seg[s];
+		for (i = 0, sr->n_u = 0; i < n_regs0; ++i) // squeeze out zero-length per-segment chains
+			if ((int32_t)sr->u[i] != 0)
+				sr->u[sr->n_u++] = sr->u[i];
+		sr->a = (mm128_t*)kmalloc(km, sr->n_a * sizeof(mm128_t));
+		sr->n_a = 0;
+	}
+
+	for (i = 0; i < n_regs0; ++i) {
+		const mm_reg1_t *r = &regs0[i];
+		for (j = 0; j < r->cnt; ++j) {
+			int sid = (a[r->as + j].y&MM_SEED_SEG_MASK)>>MM_SEED_SEG_SHIFT;
+			mm128_t a1 = a[r->as + j];
+			// on reverse strand, the segment position is:
+			//   x_for_cat = qlen_sum - 1 - (int32_t)a1.y - 1 + q_span
+			//   (int32_t)new_a1.y = qlens[sid] - (x_for_cat - acc_qlen[sid] + 1 - q_span) - 1 = (int32_t)a1.y - (qlen_sum - (qlens[sid] + acc_qlen[sid]))
+			a1.y -= a1.x>>63? qlen_sum - (qlens[sid] + acc_qlen[sid]) : acc_qlen[sid];
+			seg[sid].a[seg[sid].n_a++] = a1;
+		}
+	}
+	for (s = 0; s < n_segs; ++s) {
+		regs[s] = mm_gen_regs(km, hash, qlens[s], seg[s].n_u, seg[s].u, seg[s].a);
+		n_regs[s] = seg[s].n_u;
+		for (i = 0; i < n_regs[s]; ++i)
+			regs[s][i].seg_split = 1;
+	}
+	return seg;
+}
+
+void mm_seg_free(void *km, int n_segs, mm_seg_t *segs)
 {
-	static const float q_coef = 30.0f;
 	int i;
+	for (i = 0; i < n_segs; ++i) kfree(km, segs[i].u);
+	for (i = 0; i < n_segs; ++i) kfree(km, segs[i].a);
+	kfree(km, segs);
+}
+
+void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc, int match_sc, int rep_len, int is_sr)
+{
+	static const float q_coef = 40.0f;
+	int64_t sum_sc = 0;
+	float uniq_ratio;
+	int i;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].parent == regs[i].id)
+			sum_sc += regs[i].score;
+	uniq_ratio = (float)sum_sc / (sum_sc + rep_len);
 	for (i = 0; i < n_regs; ++i) {
 		mm_reg1_t *r = &regs[i];
 		if (r->inv) {
 			r->mapq = 0;
 		} else if (r->parent == r->id) {
 			int mapq, subsc;
-			float pen_cm = r->cnt >= 10? 1.0f : 0.1f * r->cnt;
+			float pen_s1 = (r->score > 100? 1.0f : 0.01f * r->score) * uniq_ratio;
+			float pen_cm = r->cnt > 10? 1.0f : 0.1f * r->cnt;
+			pen_cm = pen_s1 < pen_cm? pen_s1 : pen_cm;
 			subsc = r->subsc > min_chain_sc? r->subsc : min_chain_sc;
 			if (r->p && r->p->dp_max2 > 0 && r->p->dp_max > 0) {
-				float identity = (float)(r->p->blen - r->p->n_diff - r->p->n_ambi) / (r->p->blen - r->p->n_ambi);
-				mapq = (int)(identity * pen_cm * q_coef * (1. - (float)r->p->dp_max2 * subsc / r->p->dp_max / r->score) * logf(r->score));
-			} else mapq = (int)(pen_cm * q_coef * (1. - (float)subsc / r->score) * logf(r->score));
+				float identity = (float)r->mlen / r->blen;
+				float x = (float)r->p->dp_max2 * subsc / r->p->dp_max / r->score0;
+				mapq = (int)(identity * pen_cm * q_coef * (1.0f - x * x) * logf((float)r->p->dp_max / match_sc));
+				if (!is_sr) {
+					int mapq_alt = (int)(6.02f * identity * identity * (r->p->dp_max - r->p->dp_max2) / match_sc + .499f); // BWA-MEM like mapQ, mostly for short reads
+					mapq = mapq < mapq_alt? mapq : mapq_alt; // in case the long-read heuristic fails
+				}
+			} else {
+				float x = (float)subsc / r->score0;
+				if (r->p) {
+					float identity = (float)r->mlen / r->blen;
+					mapq = (int)(identity * pen_cm * q_coef * (1.0f - x) * logf((float)r->p->dp_max / match_sc));
+				} else {
+					mapq = (int)(pen_cm * q_coef * (1.0f - x) * logf(r->score));
+				}
+			}
+			mapq -= (int)(4.343f * logf(r->n_sub + 1) + .499f);
 			mapq = mapq > 0? mapq : 0;
 			r->mapq = mapq < 60? mapq : 60;
+			if (r->p && r->p->dp_max > r->p->dp_max2 && r->mapq == 0) r->mapq = 1;
 		} else r->mapq = 0;
 	}
 }

index.c

@@ -1,6 +1,10 @@
 #include <stdlib.h>
 #include <assert.h>
+#if defined(WIN32) || defined(_WIN32)
+#include <io.h> // for open(2)
+#else
 #include <unistd.h>
+#endif
 #include <fcntl.h>
 #include <stdio.h>
 #include "kthread.h"
@@ -17,13 +21,29 @@ typedef khash_t(idx) idxhash_t;
 
 #define kroundup64(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, (x)|=(x)>>32, ++(x))
 
-mm_idx_t *mm_idx_init(int w, int k, int b, int is_hpc)
+typedef struct mm_idx_bucket_s {
+	mm128_v a;   // (minimizer, position) array
+	int32_t n;   // size of the _p_ array
+	uint64_t *p; // position array for minimizers appearing >1 times
+	void *h;     // hash table indexing _p_ and minimizers appearing once
+} mm_idx_bucket_t;
+
+void mm_idxopt_init(mm_idxopt_t *opt)
+{
+	memset(opt, 0, sizeof(mm_idxopt_t));
+	opt->k = 15, opt->w = 10, opt->flag = 0;
+	opt->bucket_bits = 14;
+	opt->mini_batch_size = 50000000;
+	opt->batch_size = 4000000000ULL;
+}
+
+mm_idx_t *mm_idx_init(int w, int k, int b, int flag)
 {
 	mm_idx_t *mi;
 	if (k*2 < b) b = k * 2;
 	if (w < 1) w = 1;
 	mi = (mm_idx_t*)calloc(1, sizeof(mm_idx_t));
-	mi->w = w, mi->k = k, mi->b = b, mi->is_hpc = is_hpc;
+	mi->w = w, mi->k = k, mi->b = b, mi->flag = flag;
 	mi->B = (mm_idx_bucket_t*)calloc(1<<b, sizeof(mm_idx_bucket_t));
 	if (!(mm_dbg_flag & 1)) mi->km = km_init();
 	return mi;
@@ -69,7 +89,7 @@ void mm_idx_stat(const mm_idx_t *mi)
 {
 	int i, n = 0, n1 = 0;
 	uint64_t sum = 0, len = 0;
-	fprintf(stderr, "[M::%s] kmer size: %d; skip: %d; is_HPC: %d; #seq: %d\n", __func__, mi->k, mi->w, mi->is_hpc, mi->n_seq);
+	fprintf(stderr, "[M::%s] kmer size: %d; skip: %d; is_hpc: %d; #seq: %d\n", __func__, mi->k, mi->w, mi->flag&MM_I_HPC, mi->n_seq);
 	for (i = 0; i < mi->n_seq; ++i)
 		len += mi->seq[i].len;
 	for (i = 0; i < 1<<mi->b; ++i)
@@ -100,13 +120,13 @@ int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, ui
 	return en - st;
 }
 
-uint32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
+int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
 {
 	int i;
 	size_t n = 0;
 	uint32_t thres;
 	khint_t *a, k;
-	if (f <= 0.) return UINT32_MAX;
+	if (f <= 0.) return INT32_MAX;
 	for (i = 0; i < 1<<mi->b; ++i)
 		if (mi->B[i].h) n += kh_size((idxhash_t*)mi->B[i].h);
 	a = (uint32_t*)malloc(n * 4);
@@ -176,7 +196,7 @@ static void worker_post(void *g, long i, int tid)
 	assert(b->n == start_p);
 
 	// deallocate and clear b->a
-	free(b->a.a);
+	kfree(0, b->a.a);
 	b->a.n = b->a.m = 0, b->a.a = 0;
 }
  
@@ -194,7 +214,7 @@ static void mm_idx_post(mm_idx_t *mi, int n_threads)
 #include "bseq.h"
 
 typedef struct {
-	int mini_batch_size, keep_name;
+	int mini_batch_size;
 	uint64_t batch_size, sum_len;
 	mm_bseq_file_t *fp;
 	mm_idx_t *mi;
@@ -226,7 +246,6 @@ static void *worker_pipeline(void *shared, int step, void *in)
 		s->seq = mm_bseq_read(p->fp, p->mini_batch_size, 0, &s->n_seq); // read a mini-batch
 		if (s->seq) {
 			uint32_t old_m, m;
-			uint64_t sum_len, old_max_len, max_len;
 			assert((uint64_t)p->mi->n_seq + s->n_seq <= UINT32_MAX); // to prevent integer overflow
 			// make room for p->mi->seq
 			old_m = p->mi->n_seq, m = p->mi->n_seq + s->n_seq;
@@ -234,30 +253,34 @@ static void *worker_pipeline(void *shared, int step, void *in)
 			if (old_m != m)
 				p->mi->seq = (mm_idx_seq_t*)krealloc(p->mi->km, p->mi->seq, m * sizeof(mm_idx_seq_t));
 			// make room for p->mi->S
-			for (i = 0, sum_len = 0; i < s->n_seq; ++i) sum_len += s->seq[i].l_seq;
-			old_max_len = (p->sum_len + 7) / 8;
-			max_len = (p->sum_len + sum_len + 7) / 8;
-			kroundup64(old_max_len); kroundup64(max_len);
-			if (old_max_len != max_len) {
-				p->mi->S = (uint32_t*)realloc(p->mi->S, max_len * 4);
-				memset(&p->mi->S[old_max_len], 0, 4 * (max_len - old_max_len));
+			if (!(p->mi->flag & MM_I_NO_SEQ)) {
+				uint64_t sum_len, old_max_len, max_len;
+				for (i = 0, sum_len = 0; i < s->n_seq; ++i) sum_len += s->seq[i].l_seq;
+				old_max_len = (p->sum_len + 7) / 8;
+				max_len = (p->sum_len + sum_len + 7) / 8;
+				kroundup64(old_max_len); kroundup64(max_len);
+				if (old_max_len != max_len) {
+					p->mi->S = (uint32_t*)realloc(p->mi->S, max_len * 4);
+					memset(&p->mi->S[old_max_len], 0, 4 * (max_len - old_max_len));
+				}
 			}
 			// populate p->mi->seq
 			for (i = 0; i < s->n_seq; ++i) {
 				mm_idx_seq_t *seq = &p->mi->seq[p->mi->n_seq];
 				uint32_t j;
-				if (p->keep_name) {
-					assert(strlen(s->seq[i].name) <= 254); // a long query name breaks BAM
+				if (!(p->mi->flag & MM_I_NO_NAME)) {
 					seq->name = (char*)kmalloc(p->mi->km, strlen(s->seq[i].name) + 1);
 					strcpy(seq->name, s->seq[i].name);
 				} else seq->name = 0;
 				seq->len = s->seq[i].l_seq;
 				seq->offset = p->sum_len;
 				// copy the sequence
-				for (j = 0; j < seq->len; ++j) { // TODO: this is not the fastest way, but let's first see if speed matters here
-					uint64_t o = p->sum_len + j;
-					int c = seq_nt4_table[(uint8_t)s->seq[i].seq[j]];
-					mm_seq4_set(p->mi->S, o, c);
+				if (!(p->mi->flag & MM_I_NO_SEQ)) {
+					for (j = 0; j < seq->len; ++j) { // TODO: this is not the fastest way, but let's first see if speed matters here
+						uint64_t o = p->sum_len + j;
+						int c = seq_nt4_table[(uint8_t)s->seq[i].seq[j]];
+						mm_seq4_set(p->mi->S, o, c);
+					}
 				}
 				// update p->sum_len and p->mi->n_seq
 				p->sum_len += seq->len;
@@ -269,7 +292,10 @@ static void *worker_pipeline(void *shared, int step, void *in)
         step_t *s = (step_t*)in;
 		for (i = 0; i < s->n_seq; ++i) {
 			mm_bseq1_t *t = &s->seq[i];
-			mm_sketch(0, t->seq, t->l_seq, p->mi->w, p->mi->k, t->rid, p->mi->is_hpc, &s->a);
+			if (t->l_seq > 0)
+				mm_sketch(0, t->seq, t->l_seq, p->mi->w, p->mi->k, t->rid, p->mi->flag&MM_I_HPC, &s->a);
+			else if (mm_verbose >= 2)
+				fprintf(stderr, "[WARNING] the length database sequence '%s' is 0\n", t->name);
 			free(t->seq); free(t->name);
 		}
 		free(s->seq); s->seq = 0;
@@ -277,21 +303,20 @@ static void *worker_pipeline(void *shared, int step, void *in)
     } else if (step == 2) { // dispatch sketch to buckets
         step_t *s = (step_t*)in;
 		mm_idx_add(p->mi, s->a.n, s->a.a);
-		free(s->a.a); free(s);
+		kfree(0, s->a.a); free(s);
 	}
     return 0;
 }
 
-mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int is_hpc, int mini_batch_size, int n_threads, uint64_t batch_size, int keep_name)
+mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size)
 {
 	pipeline_t pl;
 	if (fp == 0 || mm_bseq_eof(fp)) return 0;
 	memset(&pl, 0, sizeof(pipeline_t));
 	pl.mini_batch_size = mini_batch_size < batch_size? mini_batch_size : batch_size;
-	pl.keep_name = keep_name;
 	pl.batch_size = batch_size;
 	pl.fp = fp;
-	pl.mi = mm_idx_init(w, k, b, is_hpc);
+	pl.mi = mm_idx_init(w, k, b, flag);
 
 	kt_pipeline(n_threads < 3? n_threads : 3, worker_pipeline, &pl, 3);
 	if (mm_verbose >= 3)
@@ -304,17 +329,60 @@ mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int is_hpc, int mi
 	return pl.mi;
 }
 
-mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads) // a simpler interface
+mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads) // a simpler interface; deprecated
 {
 	mm_bseq_file_t *fp;
 	mm_idx_t *mi;
 	fp = mm_bseq_open(fn);
 	if (fp == 0) return 0;
-	mi = mm_idx_gen(fp, w, k, MM_IDX_DEF_B, is_hpc, 1<<18, n_threads, UINT64_MAX, 1);
+	mi = mm_idx_gen(fp, w, k, 14, flag, 1<<18, n_threads, UINT64_MAX);
 	mm_bseq_close(fp);
 	return mi;
 }
 
+mm_idx_t *mm_idx_str(int w, int k, int is_hpc, int bucket_bits, int n, const char **seq, const char **name)
+{
+	uint64_t sum_len = 0;
+	mm128_v a = {0,0,0};
+	mm_idx_t *mi;
+	int i, flag = 0;
+	if (n <= 0) return 0;
+	for (i = 0; i < n; ++i) // get the total length
+		sum_len += strlen(seq[i]);
+	if (is_hpc) flag |= MM_I_HPC;
+	if (name == 0) flag |= MM_I_NO_NAME;
+	if (bucket_bits < 0) bucket_bits = 14;
+	mi = mm_idx_init(w, k, bucket_bits, flag);
+	mi->n_seq = n;
+	mi->seq = (mm_idx_seq_t*)kcalloc(mi->km, n, sizeof(mm_idx_seq_t)); // ->seq is allocated from km
+	mi->S = (uint32_t*)calloc((sum_len + 7) / 8, 4);
+	for (i = 0, sum_len = 0; i < n; ++i) {
+		const char *s = seq[i];
+		mm_idx_seq_t *p = &mi->seq[i];
+		uint32_t j;
+		if (name && name[i]) {
+			p->name = (char*)kmalloc(mi->km, strlen(name[i]) + 1);
+			strcpy(p->name, name[i]);
+		}
+		p->offset = sum_len;
+		p->len = strlen(s);
+		for (j = 0; j < p->len; ++j) {
+			int c = seq_nt4_table[(uint8_t)s[j]];
+			uint64_t o = sum_len + j;
+			mm_seq4_set(mi->S, o, c);
+		}
+		sum_len += p->len;
+		if (p->len > 0) {
+			a.n = 0;
+			mm_sketch(0, s, p->len, w, k, i, is_hpc, &a);
+			mm_idx_add(mi, a.n, a.a);
+		}
+	}
+	free(a.a);
+	mm_idx_post(mi, 1);
+	return mi;
+}
+
 /*************
  * index I/O *
  *************/
@@ -325,7 +393,7 @@ void mm_idx_dump(FILE *fp, const mm_idx_t *mi)
 	uint32_t x[5];
 	int i;
 
-	x[0] = mi->w, x[1] = mi->k, x[2] = mi->b, x[3] = mi->n_seq, x[4] = mi->is_hpc;
+	x[0] = mi->w, x[1] = mi->k, x[2] = mi->b, x[3] = mi->n_seq, x[4] = mi->flag;
 	fwrite(MM_IDX_MAGIC, 1, 4, fp);
 	fwrite(x, 4, 5, fp);
 	for (i = 0; i < mi->n_seq; ++i) {
@@ -352,7 +420,8 @@ void mm_idx_dump(FILE *fp, const mm_idx_t *mi)
 			fwrite(x, 8, 2, fp);
 		}
 	}
-	fwrite(mi->S, 4, (sum_len + 7) / 8, fp);
+	if (!(mi->flag & MM_I_NO_SEQ))
+		fwrite(mi->S, 4, (sum_len + 7) / 8, fp);
 	fflush(fp);
 }
 
@@ -402,26 +471,75 @@ mm_idx_t *mm_idx_load(FILE *fp)
 			kh_val(h, k) = x[1];
 		}
 	}
-	mi->S = (uint32_t*)malloc((sum_len + 7) / 8 * 4);
-	fread(mi->S, 4, (sum_len + 7) / 8, fp);
+	if (!(mi->flag & MM_I_NO_SEQ)) {
+		mi->S = (uint32_t*)malloc((sum_len + 7) / 8 * 4);
+		fread(mi->S, 4, (sum_len + 7) / 8, fp);
+	}
 	return mi;
 }
 
-int mm_idx_is_idx(const char *fn)
+int64_t mm_idx_is_idx(const char *fn)
 {
 	int fd, is_idx = 0;
-	off_t ret;
+	off_t ret, off_end;
 	char magic[4];
 
 	if (strcmp(fn, "-") == 0) return 0; // read from pipe; not an index
 	fd = open(fn, O_RDONLY);
 	if (fd < 0) return -1; // error
-	if ((ret = lseek(fd, 0, SEEK_END)) >= 4) {
+	if ((off_end = lseek(fd, 0, SEEK_END)) >= 4) {
 		lseek(fd, 0, SEEK_SET);
 		ret = read(fd, magic, 4);
 		if (ret == 4 && strncmp(magic, MM_IDX_MAGIC, 4) == 0)
 			is_idx = 1;
 	}
 	close(fd);
-	return is_idx;
+	return is_idx? off_end : 0;
+}
+
+mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out)
+{
+	int64_t is_idx;
+	mm_idx_reader_t *r;
+	is_idx = mm_idx_is_idx(fn);
+	if (is_idx < 0) return 0; // failed to open the index
+	r = (mm_idx_reader_t*)calloc(1, sizeof(mm_idx_reader_t));
+	r->is_idx = is_idx;
+	if (opt) r->opt = *opt;
+	else mm_idxopt_init(&r->opt);
+	if (r->is_idx) {
+		r->fp.idx = fopen(fn, "rb");
+		r->idx_size = is_idx;
+	} else r->fp.seq = mm_bseq_open(fn);
+	if (fn_out) r->fp_out = fopen(fn_out, "wb");
+	return r;
+}
+
+void mm_idx_reader_close(mm_idx_reader_t *r)
+{
+	if (r->is_idx) fclose(r->fp.idx);
+	else mm_bseq_close(r->fp.seq);
+	if (r->fp_out) fclose(r->fp_out);
+	free(r);
+}
+
+mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
+{
+	mm_idx_t *mi;
+	if (r->is_idx) {
+		mi = mm_idx_load(r->fp.idx);
+		if (mi && mm_verbose >= 2 && (mi->k != r->opt.k || mi->w != r->opt.w || (mi->flag&MM_I_HPC) != (r->opt.flag&MM_I_HPC)))
+			fprintf(stderr, "[WARNING]\033[1;31m Indexing parameters (-k, -w or -H) overridden by parameters used in the prebuilt index.\033[0m\n");
+	} else
+		mi = mm_idx_gen(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.flag, r->opt.mini_batch_size, n_threads, r->opt.batch_size);
+	if (mi) {
+		if (r->fp_out) mm_idx_dump(r->fp_out, mi);
+		++r->n_parts;
+	}
+	return mi;
+}
+
+int mm_idx_reader_eof(const mm_idx_reader_t *r) // TODO: in extremely rare cases, mm_bseq_eof() might not work
+{
+	return r->is_idx? (feof(r->fp.idx) || ftell(r->fp.idx) == r->idx_size) : mm_bseq_eof(r->fp.seq);
 }

kalloc.c

@@ -1,175 +1,144 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <limits.h>
 #include "kalloc.h"
 
-/* The whole thing is: ("@" for the kheader_t of the block, "-" for free
- * memory, and "+" for allocated memory. One char for one unit.)
- *                        
- *           This region is core 1.                             This region is core 2.
+/* In kalloc, a *core* is a large chunk of contiguous memory. Each core is
+ * associated with a master header, which keeps the size of the current core
+ * and the pointer to next core. Kalloc allocates small *blocks* of memory from
+ * the cores and organizes free memory blocks in a circular single-linked list.
  *
- *   @-------@++++++@++++++++++++@------------           @----------@++++++++++++@+++++++@------------
- *   |                           |                       |                               |
- *   p=p->ptr->ptr->ptr->ptr     p->ptr             p->ptr->ptr                p->ptr->ptr->ptr
+ * In the following diagram, "@" stands for the header of a free block (of type
+ * header_t), "#" for the header of an allocated block (of type size_t), "-"
+ * for free memory, and "+" for allocated memory.
+ *
+ * master        This region is core 1.          master           This region is core 2.
+ *      |                                             |
+ *      *@-------#++++++#++++++++++++@--------        *@----------#++++++++++++#+++++++@------------
+ *       |                           |                 |                               |
+ *       p=p->ptr->ptr->ptr->ptr     p->ptr            p->ptr->ptr                     p->ptr->ptr->ptr
  */
 
-#define PTR(p) ((size_t*)((size_t*)p)[1])
+#define MIN_CORE_SIZE 0x80000
 
-typedef struct _allocated_t {
-	struct _allocated_t *next;
-	size_t *ptr;
-} allocated_t;
+typedef struct header_t {
+	size_t size;
+	struct header_t *ptr;
+} header_t;
 
 typedef struct {
-	size_t base[2], *loop_head;
-	allocated_t list_head, *list_tail;
-	size_t total_allocated;
+	header_t base, *loop_head, *core_head; /* base is a zero-sized block always kept in the loop */
 } kmem_t;
 
-void *km_init()
-{
-	return calloc(1, sizeof(kmem_t));
-}
-
-static void kerror(const char *s)
+static void panic(const char *s)
 {
 	fprintf(stderr, "%s\n", s);
-	exit(1);
+	abort();
 }
 
-static size_t *morecore(kmem_t *km, size_t nu)
+void *km_init(void)
 {
-	size_t rnu, *up;
-
-	rnu = (nu + 0xfffff) & (~(size_t)0xfffff);
-	up = (size_t*)malloc(rnu * sizeof(size_t));
-	if (!up) { /* fail to allocate memory */
-		km_stat(km);
-		fprintf(stderr, "[morecore] %lu bytes requested but not available.\n", rnu * sizeof(size_t));
-		exit(1);
-	}
-	/* put the pointer in km->list_head */
-	if (km->list_tail == 0) km->list_tail = &km->list_head;
-	km->list_tail->ptr = up;
-	km->list_tail->next = (allocated_t*)calloc(1, sizeof(allocated_t));
-	km->list_tail = km->list_tail->next;
-
-	km->total_allocated += rnu * sizeof(size_t);
-	*up = rnu; /* the size of the current block, and in this case the block is the same as the new core */
-	kfree(km, up + 1); /* initialize the new "core" */
-	return km->loop_head;
+	return calloc(1, sizeof(kmem_t));
 }
 
 void km_destroy(void *_km)
 {
 	kmem_t *km = (kmem_t*)_km;
-	allocated_t *p, *q;
-	if (km == 0) return;
-	p = &km->list_head;
-	do {
-		q = p->next;
-		free(p->ptr);
-		if (p != &km->list_head) free(p);
+	header_t *p, *q;
+	if (km == NULL) return;
+	for (p = km->core_head; p != NULL;) {
+		q = p->ptr;
+		free(p);
 		p = q;
-	} while (p && p->next);
-	if (p != &km->list_head) free(p);
+	}
 	free(km);
 }
 
-void kfree(void *_km, void *ap)
+static header_t *morecore(kmem_t *km, size_t nu)
 {
-	size_t *p, *q;
+	header_t *q;
+	size_t bytes, *p;
+	nu = (nu + 1 + (MIN_CORE_SIZE - 1)) / MIN_CORE_SIZE * MIN_CORE_SIZE; /* the first +1 for core header */
+	bytes = nu * sizeof(header_t);
+	q = (header_t*)malloc(bytes);
+	if (!q) panic("[morecore] insufficient memory");
+	q->ptr = km->core_head, q->size = nu, km->core_head = q;
+	p = (size_t*)(q + 1);
+	*p = nu - 1; /* the size of the free block; -1 because the first unit is used for the core header */
+	kfree(km, p + 1); /* initialize the new "core"; NB: the core header is not looped. */
+	return km->loop_head;
+}
+
+void kfree(void *_km, void *ap) /* kfree() also adds a new core to the circular list */
+{
+	header_t *p, *q;
 	kmem_t *km = (kmem_t*)_km;
 	
 	if (!ap) return;
-	if (km == 0) {
+	if (km == NULL) {
 		free(ap);
 		return;
 	}
-	p = (size_t*)ap - 1; /* *p is the size of the current block */
+	p = (header_t*)((size_t*)ap - 1);
+	p->size = *((size_t*)ap - 1);
 	/* Find the pointer that points to the block to be freed. The following loop can stop on two conditions:
 	 *
-	 * a) "p>q && p<q->ptr": @------@++++++++@+++++++@-------    @---------------@+++++++@-------
+	 * a) "p>q && p<q->ptr": @------#++++++++#+++++++@-------    @---------------#+++++++@-------
 	 *    (can also be in    |      |                |        -> |                       |
 	 *     two cores)        q      p           q->ptr           q                  q->ptr
 	 *
-	 *                       @--------    @+++++++++@--------    @--------    @------------------
+	 *                       @--------    #+++++++++@--------    @--------    @------------------
 	 *                       |            |         |         -> |            |
 	 *                       q            p    q->ptr            q       q->ptr
 	 *
-	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------@+++++   @--------@+++++++     @-------@+++++   @----------------
+	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------#+++++   @--------#+++++++     @-------#+++++   @----------------
 	 *                                       |                |        |         -> |                |
 	 *                                  q->ptr                q        p       q->ptr                q
 	 *
-	 *                                       @+++++++@-----   @++++++++@-------     @-------------   @++++++++@-------
+	 *                                       #+++++++@-----   #++++++++@-------     @-------------   #++++++++@-------
 	 *                                       |       |                 |         -> |                         |
 	 *                                       p  q->ptr                 q       q->ptr                         q
 	 */
-	for (q = km->loop_head; !(p > q && p < PTR(q)); q = PTR(q))
-		if (q >= PTR(q) && (p > q || p < PTR(q))) break;
-	if (p + (*p) == PTR(q)) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
-		*p += *PTR(q); /* this is the new q->ptr size */
-		p[1] = (size_t)PTR(PTR(q)); /* this is the new q->ptr->ptr */
-		/* p is actually the new q->ptr. The actual change happens a few lines below. */
-	} else if (p + (*p) > PTR(q) && PTR(q) >= p) { /* the end of the allocated block is in the next free block */
-		kerror("[kfree] The end of the allocated block enters a free block.");
-	} else p[1] = (size_t)PTR(q); /* backup q->ptr */
+	for (q = km->loop_head; !(p > q && p < q->ptr); q = q->ptr)
+		if (q >= q->ptr && (p > q || p < q->ptr)) break;
+	if (p + p->size == q->ptr) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
+		p->size += q->ptr->size;
+		p->ptr = q->ptr->ptr;
+	} else if (p + p->size > q->ptr && q->ptr >= p) {
+		panic("[kfree] The end of the allocated block enters a free block.");
+	} else p->ptr = q->ptr; /* backup q->ptr */
 
-	if (q + (*q) == p) { /* two adjacent blocks, merge q and p (the other two cases) */
-		*q += *p;
-		q[1] = (size_t)PTR(p);
+	if (q + q->size == p) { /* two adjacent blocks, merge q and p (the other two cases) */
+		q->size += p->size;
+		q->ptr = p->ptr;
 		km->loop_head = q;
-	} else if (q + (*q) > p && p >= q) { /* the end of a free block in the allocated block */
-		kerror("[kfree] The end of a free block enters the allocated block.");
-	} else km->loop_head = p, q[1] = (size_t)p; /* in two cores, cannot be merged */
-}
-
-void *krealloc(void *_km, void *ap, size_t n_bytes)
-{
-	kmem_t *km = (kmem_t*)_km;
-	size_t n_units, *p, *q;
-
-	if (n_bytes == 0) {
-		kfree(km, ap); return 0;
-	}
-	if (km == 0) return realloc(ap, n_bytes);
-	if (!ap) return kmalloc(km, n_bytes);
-	n_units = 1 + (n_bytes + sizeof(size_t) - 1) / sizeof(size_t);
-	p = (size_t*)ap - 1;
-	if (*p >= n_units) return ap; /* TODO: this prevents shrinking */
-	q = (size_t*)kmalloc(km, n_bytes);
-	memcpy(q, ap, (*p - 1) * sizeof(size_t));
-	kfree(km, ap);
-	return q;
+	} else if (q + q->size > p && p >= q) {
+		panic("[kfree] The end of a free block enters the allocated block.");
+	} else km->loop_head = p, q->ptr = p; /* in two cores, cannot be merged; create a new block in the list */
 }
 
 void *kmalloc(void *_km, size_t n_bytes)
 {
 	kmem_t *km = (kmem_t*)_km;
-	size_t n_units, *p, *q;
+	size_t n_units;
+	header_t *p, *q;
 
 	if (n_bytes == 0) return 0;
-	if (km == 0) return malloc(n_bytes);
-	/* "n_units" means the number of units. The size of one unit equals to sizeof(kheader_t).
-	 * "1" is the kheader_t of a block, which is always required. */
-	n_units = 1 + (n_bytes + sizeof(size_t) - 1) / sizeof(size_t);
-	if (n_units&1) ++n_units; /* make n_units an even number, or it will segfault if only one unit remains */
+	if (km == NULL) return malloc(n_bytes);
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t) + 1;
 
-	if (!(q = km->loop_head)) { /* the first time when kmalloc() is called, intialization */
-		km->base[1] = (size_t)(km->loop_head = q = km->base); *q = 0;
-	}
-	for (p = PTR(q);; q = p, p = PTR(p)) { /* search for a suitable block */
-		if (*p >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
-			if (*p == n_units) q[1] = (size_t)PTR(p); /* no need to split the block */
-			else { /* split the block */
-				/* memory is allocated at the end of the block */
-				*p -= n_units; /* reduce the size of the free block */
-				p += *p; /* skip to the kheader_t of the allocated block */
-				*p = n_units; /* set the size */
+	if (!(q = km->loop_head)) /* the first time when kmalloc() is called, intialize it */
+		q = km->loop_head = km->base.ptr = &km->base;
+	for (p = q->ptr;; q = p, p = p->ptr) { /* search for a suitable block */
+		if (p->size >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
+			if (p->size == n_units) q->ptr = p->ptr; /* no need to split the block */
+			else { /* split the block. NB: memory is allocated at the end of the block! */
+				p->size -= n_units; /* reduce the size of the free block */
+				p += p->size; /* p points to the allocated block */
+				*(size_t*)p = n_units; /* set the size */
 			}
 			km->loop_head = q; /* set the end of chain */
-			return p + 1; /* skip the kheader_t */
+			return (size_t*)p + 1;
 		}
 		if (p == km->loop_head) { /* then ask for more "cores" */
 			if ((p = morecore(km, n_units)) == 0) return 0;
@@ -182,33 +151,48 @@ void *kcalloc(void *_km, size_t count, size_t size)
 	kmem_t *km = (kmem_t*)_km;
 	void *p;
 	if (size == 0 || count == 0) return 0;
-	if (km == 0) return calloc(count, size);
+	if (km == NULL) return calloc(count, size);
 	p = kmalloc(km, count * size);
 	memset(p, 0, count * size);
 	return p;
 }
 
-void km_stat(const void *_km)
+void *krealloc(void *_km, void *ap, size_t n_bytes) // TODO: this can be made more efficient in principle
 {
 	kmem_t *km = (kmem_t*)_km;
-	unsigned n_blocks, n_units;
-	size_t max_block = 0, *p, *q;
-	float frag;
+	size_t n_units, *p, *q;
 
-	if (km == 0 || !(p = km->loop_head)) return;
-	n_blocks = n_units = 0;
-	do {
-		q = PTR(p);
-		if (*p > max_block) max_block = *p;
-		n_units += *p;
-		if (p + (*p) > q && q > p)
-			kerror("[kr_stat] The end of a free block enters another free block.");
-		p = q;
-		++n_blocks;
-	} while (p != km->loop_head);
-	
-	--n_blocks;
-	frag = 1.0/1024.0 * n_units * sizeof(size_t) / n_blocks;
-	fprintf(stderr, "[kr_stat] tot=%lu, free=%lu, n_block=%u, max_block=%lu, frag_len=%.3fK\n",
-			km->total_allocated, n_units * sizeof(size_t), n_blocks, max_block * sizeof(size_t), frag);
+	if (n_bytes == 0) {
+		kfree(km, ap); return 0;
+	}
+	if (km == NULL) return realloc(ap, n_bytes);
+	if (ap == NULL) return kmalloc(km, n_bytes);
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t);
+	p = (size_t*)ap - 1;
+	if (*p >= n_units) return ap; /* TODO: this prevents shrinking */
+	q = (size_t*)kmalloc(km, n_bytes);
+	memcpy(q, ap, (*p - 1) * sizeof(header_t));
+	kfree(km, ap);
+	return q;
+}
+
+void km_stat(const void *_km, km_stat_t *s)
+{
+	kmem_t *km = (kmem_t*)_km;
+	header_t *p;
+	memset(s, 0, sizeof(km_stat_t));
+	if (km == NULL || km->loop_head == NULL) return;
+	for (p = km->loop_head;; p = p->ptr) {
+		s->available += p->size * sizeof(header_t);
+		if (p->size != 0) ++s->n_blocks; /* &kmem_t::base is always one of the cores. It is zero-sized. */
+		if (p->ptr > p && p + p->size > p->ptr)
+			panic("[km_stat] The end of a free block enters another free block.");
+		if (p->ptr == km->loop_head) break;
+	}
+	for (p = km->core_head; p != NULL; p = p->ptr) {
+		size_t size = p->size * sizeof(header_t);
+		++s->n_cores;
+		s->capacity += size;
+		s->largest = s->largest > size? s->largest : size;
+	}
 }

kalloc.h

@@ -1,14 +1,16 @@
 #ifndef _KALLOC_H_
 #define _KALLOC_H_
 
-#include <stdlib.h>
-
-#define km_size(x) (*(((size_t*)(x))-1) * sizeof(size_t))
+#include <stddef.h> /* for size_t */
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct {
+	size_t capacity, available, n_blocks, n_cores, largest;
+} km_stat_t;
+
 void *kmalloc(void *km, size_t size);
 void *krealloc(void *km, void *ptr, size_t size);
 void *kcalloc(void *km, size_t count, size_t size);
@@ -16,8 +18,7 @@ void kfree(void *km, void *ptr);
 
 void *km_init(void);
 void km_destroy(void *km);
-
-void km_stat(const void *km); // TODO: return numbers instead of print to stderr
+void km_stat(const void *_km, km_stat_t *s);
 
 #ifdef __cplusplus
 }

kdq.h

@@ -3,11 +3,12 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <stdint.h>
 #include "kalloc.h"
 
 #define __KDQ_TYPE(type) \
 	typedef struct { \
-		size_t front:58, bits:6, count, mask; \
+		uint64_t front:58, bits:6, count, mask; \
 		type *a; \
 		void *km; \
 	} kdq_##type##_t;

ksort.h

@@ -30,6 +30,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 
 typedef struct {
 	void *left, *right;
@@ -78,6 +79,7 @@ typedef const char *ksstr_t;
 #define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str)
 
 #define RS_MIN_SIZE 64
+#define RS_MAX_BITS 8
 
 #define KRADIX_SORT_INIT(name, rstype_t, rskey, sizeof_key) \
 	typedef struct { \
@@ -98,7 +100,8 @@ typedef const char *ksstr_t;
 	{ \
 		rstype_t *i; \
 		int size = 1<<n_bits, m = size - 1; \
-		rsbucket_##name##_t *k, b[size], *be = b + size; \
+		rsbucket_##name##_t *k, b[1<<RS_MAX_BITS], *be = b + size; \
+		assert(n_bits <= RS_MAX_BITS); \
 		for (k = b; k != be; ++k) k->b = k->e = beg; \
 		for (i = beg; i != end; ++i) ++b[rskey(*i)>>s&m].e; \
 		for (k = b + 1; k != be; ++k) \
@@ -127,7 +130,7 @@ typedef const char *ksstr_t;
 	void radix_sort_##name(rstype_t *beg, rstype_t *end) \
 	{ \
 		if (end - beg <= RS_MIN_SIZE) rs_insertsort_##name(beg, end); \
-		else rs_sort_##name(beg, end, 8, sizeof_key * 8 - 8); \
+		else rs_sort_##name(beg, end, RS_MAX_BITS, (sizeof_key - 1) * RS_MAX_BITS); \
 	}
 
 #endif

ksw2.h

@@ -14,6 +14,7 @@
 #define KSW_EZ_REV_CIGAR   0x80 // reverse CIGAR in the output
 #define KSW_EZ_SPLICE_FOR  0x100
 #define KSW_EZ_SPLICE_REV  0x200
+#define KSW_EZ_SPLICE_FLANK 0x400
 
 #ifdef __cplusplus
 extern "C" {
@@ -26,6 +27,7 @@ typedef struct {
 	int mte, mte_q;        // max score when reaching the end of target
 	int score;             // max score reaching both ends; may be KSW_NEG_INF
 	int m_cigar, n_cigar;
+	int reach_end;
 	uint32_t *cigar;
 } ksw_extz_t;
 
@@ -46,14 +48,17 @@ typedef struct {
  * @param flag      flag (see KSW_EZ_* macros)
  * @param ez        (out) scores and cigar
  */
-void ksw_extz(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
-void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+void ksw_extz(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+			  int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 
 void ksw_extd(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
 			  int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int flag, ksw_extz_t *ez);
 
 void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int flag, ksw_extz_t *ez);
+				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 
 void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
 				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez);
@@ -111,7 +116,7 @@ static inline uint32_t *ksw_push_cigar(void *km, int *n_cigar, int *m_cigar, uin
 //   bit 0-2: which type gets the max - 0 for H, 1 for E, 2 for F, 3 for \tilde{E} and 4 for \tilde{F}
 //   bit 3/0x08: 1 if a continuation on the E state (bit 5/0x20 for a continuation on \tilde{E})
 //   bit 4/0x10: 1 if a continuation on the F state (bit 6/0x40 for a continuation on \tilde{F})
-static inline void ksw_backtrack(void *km, int is_rot, int is_rev, int with_N, const uint8_t *p, const int *off, const int *off_end, int n_col, int i0, int j0,
+static inline void ksw_backtrack(void *km, int is_rot, int is_rev, int min_intron_len, const uint8_t *p, const int *off, const int *off_end, int n_col, int i0, int j0,
 								 int *m_cigar_, int *n_cigar_, uint32_t **cigar_)
 { // p[] - lower 3 bits: which type gets the max; bit
 	int n_cigar = 0, m_cigar = *m_cigar_, i = i0, j = j0, r, state = 0;
@@ -133,11 +138,11 @@ static inline void ksw_backtrack(void *km, int is_rot, int is_rev, int with_N, c
 		if (state == 0) state = tmp & 7; // TODO: probably this line can be merged into the "else if" line right above; not 100% sure
 		if (force_state >= 0) state = force_state;
 		if (state == 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 0, 1), --i, --j; // match
-		else if (state == 1 || (state == 3 && !with_N)) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 2, 1), --i; // deletion
-		else if (state == 3 && with_N) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 3, 1), --i; // intron
+		else if (state == 1 || (state == 3 && min_intron_len <= 0)) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 2, 1), --i; // deletion
+		else if (state == 3 && min_intron_len > 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 3, 1), --i; // intron
 		else cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 1, 1), --j; // insertion
 	}
-	if (i >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 2, i + 1); // first deletion
+	if (i >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, min_intron_len > 0 && i >= min_intron_len? 3 : 2, i + 1); // first deletion
 	if (j >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 1, j + 1); // first insertion
 	if (!is_rev)
 		for (i = 0; i < n_cigar>>1; ++i) // reverse CIGAR
@@ -149,7 +154,7 @@ static inline void ksw_reset_extz(ksw_extz_t *ez)
 {
 	ez->max_q = ez->max_t = ez->mqe_t = ez->mte_q = -1;
 	ez->max = 0, ez->score = ez->mqe = ez->mte = KSW_NEG_INF;
-	ez->n_cigar = 0, ez->zdropped = 0;
+	ez->n_cigar = 0, ez->zdropped = 0, ez->reach_end = 0;
 }
 
 static inline int ksw_apply_zdrop(ksw_extz_t *ez, int is_rot, int32_t H, int a, int b, int zdrop, int8_t e)
@@ -169,5 +174,4 @@ static inline int ksw_apply_zdrop(ksw_extz_t *ez, int is_rot, int32_t H, int a,
 	}
 	return 0;
 }
-
 #endif

ksw2_dispatch.c

@@ -0,0 +1,97 @@
+#ifdef KSW_CPU_DISPATCH
+#include <stdlib.h>
+#include "ksw2.h"
+
+#define SIMD_SSE     0x1
+#define SIMD_SSE2    0x2
+#define SIMD_SSE3    0x4
+#define SIMD_SSSE3   0x8
+#define SIMD_SSE4_1  0x10
+#define SIMD_SSE4_2  0x20
+#define SIMD_AVX     0x40
+#define SIMD_AVX2    0x80
+#define SIMD_AVX512F 0x100
+
+#ifndef _MSC_VER
+// adapted from https://github.com/01org/linux-sgx/blob/master/common/inc/internal/linux/cpuid_gnu.h
+void __cpuidex(int cpuid[4], int func_id, int subfunc_id)
+{
+#if defined(__x86_64__)
+	asm volatile ("cpuid"
+			: "=a" (cpuid[0]), "=b" (cpuid[1]), "=c" (cpuid[2]), "=d" (cpuid[3])
+			: "0" (func_id), "2" (subfunc_id));
+#else // on 32bit, ebx can NOT be used as PIC code
+	asm volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
+			: "=a" (cpuid[0]), "=r" (cpuid[1]), "=c" (cpuid[2]), "=d" (cpuid[3])
+			: "0" (func_id), "2" (subfunc_id));
+#endif
+}
+#endif
+
+int x86_simd(void)
+{
+	int flag = 0, cpuid[4], max_id;
+	__cpuidex(cpuid, 0, 0);
+	max_id = cpuid[0];
+	if (max_id == 0) return 0;
+	__cpuidex(cpuid, 1, 0);
+	if (cpuid[3]>>25&1) flag |= SIMD_SSE;
+	if (cpuid[3]>>26&1) flag |= SIMD_SSE2;
+	if (cpuid[2]>>0 &1) flag |= SIMD_SSE3;
+	if (cpuid[2]>>9 &1) flag |= SIMD_SSSE3;
+	if (cpuid[2]>>19&1) flag |= SIMD_SSE4_1;
+	if (cpuid[2]>>20&1) flag |= SIMD_SSE4_2;
+	if (cpuid[2]>>28&1) flag |= SIMD_AVX;
+	if (max_id >= 7) {
+		__cpuidex(cpuid, 7, 0);
+		if (cpuid[1]>>5 &1) flag |= SIMD_AVX2;
+		if (cpuid[1]>>16&1) flag |= SIMD_AVX512F;
+	}
+	return flag;
+}
+
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+{
+	extern void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	extern void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	unsigned simd;
+	simd = x86_simd();
+	if (simd & SIMD_SSE4_1)
+		ksw_extz2_sse41(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, end_bonus, flag, ez);
+	else if (simd & SIMD_SSE2)
+		ksw_extz2_sse2(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, end_bonus, flag, ez);
+	else abort();
+}
+
+void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+{
+	extern void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	extern void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	unsigned simd;
+	simd = x86_simd();
+	if (simd & SIMD_SSE4_1)
+		ksw_extd2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
+	else if (simd & SIMD_SSE2)
+		ksw_extd2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
+	else abort();
+}
+
+void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez)
+{
+	extern void ksw_exts2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez);
+	extern void ksw_exts2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez);
+	unsigned simd;
+	simd = x86_simd();
+	if (simd & SIMD_SSE4_1)
+		ksw_exts2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, flag, ez);
+	else if (simd & SIMD_SSE2)
+		ksw_exts2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, flag, ez);
+	else abort();
+}
+#endif

ksw2_extd2_sse.c

@@ -6,12 +6,26 @@
 #ifdef __SSE2__
 #include <emmintrin.h>
 
+#ifdef KSW_SSE2_ONLY
+#undef __SSE4_1__
+#endif
+
 #ifdef __SSE4_1__
 #include <smmintrin.h>
 #endif
 
+#ifdef KSW_CPU_DISPATCH
+#ifdef __SSE4_1__
+void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#else
+void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif
+#else
 void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez)
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif // ~KSW_CPU_DISPATCH
 {
 #define __dp_code_block1 \
 	z = _mm_load_si128(&s[t]); \
@@ -47,7 +61,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_;
+	__m128i q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_, sc_N_;
 	__m128i *u, *v, *x, *y, *x2, *y2, *s, *p = 0;
 
 	ksw_reset_extz(ez);
@@ -62,6 +76,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	qe2_    = _mm_set1_epi8(q2 + e2);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = _mm_set1_epi8(-e2);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 
 	if (w < 0) w = tlen > qlen? tlen : qlen;
@@ -148,10 +163,11 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp);
 			}
 		} else {
@@ -364,10 +380,14 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	if (!approx_max) kfree(km, H);
 	if (with_cigar) { // backtrack
 		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
-		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
+		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
-		else if (ez->max_t >= 0 && ez->max_q >= 0)
+		} else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > ez->max) {
+			ez->reach_end = 1;
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		} else if (ez->max_t >= 0 && ez->max_q >= 0) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		}
 		kfree(km, mem2); kfree(km, off);
 	}
 }

ksw2_exts2_sse.c

@@ -6,12 +6,26 @@
 #ifdef __SSE2__
 #include <emmintrin.h>
 
+#ifdef KSW_SSE2_ONLY
+#undef __SSE4_1__
+#endif
+
 #ifdef __SSE4_1__
 #include <smmintrin.h>
 #endif
 
+#ifdef KSW_CPU_DISPATCH
+#ifdef __SSE4_1__
+void ksw_exts2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez)
+#else
+void ksw_exts2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez)
+#endif
+#else
 void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
 				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez)
+#endif // ~KSW_CPU_DISPATCH
 {
 #define __dp_code_block1 \
 	z = _mm_load_si128(&s[t]); \
@@ -45,7 +59,7 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, q2_, qe_, zero_, sc_mch_, sc_mis_, m1_;
+	__m128i q_, q2_, qe_, zero_, sc_mch_, sc_mis_, sc_N_, m1_;
 	__m128i *u, *v, *x, *y, *x2, *s, *p = 0, *donor, *acceptor;
 
 	ksw_reset_extz(ez);
@@ -57,6 +71,7 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	qe_     = _mm_set1_epi8(q + e);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = _mm_set1_epi8(-e);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 
 	tlen_ = (tlen + 15) / 16;
@@ -96,19 +111,22 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 
 	// set the donor and acceptor arrays. TODO: this assumes 0/1/2/3 encoding!
 	if (flag & (KSW_EZ_SPLICE_FOR|KSW_EZ_SPLICE_REV)) {
+		int semi_cost = flag&KSW_EZ_SPLICE_FLANK? -noncan/2 : 0; // GTr or yAG is worth 0.5 bit; see PMID:18688272
 		memset(donor, -noncan, tlen_ * 16);
-		for (t = 0; t < tlen - 2; ++t) {
-			int is_can = 0; // is a canonical site
-			if ((flag & KSW_EZ_SPLICE_FOR) && target[t+1] == 2 && target[t+2] == 3) is_can = 1;
-			if ((flag & KSW_EZ_SPLICE_REV) && target[t+1] == 1 && target[t+2] == 3) is_can = 1;
-			if (is_can) ((int8_t*)donor)[t] = 0;
+		for (t = 0; t < tlen - 4; ++t) {
+			int can_type = 0; // type of canonical site: 0=none, 1=GT/AG only, 2=GTr/yAG
+			if ((flag & KSW_EZ_SPLICE_FOR) && target[t+1] == 2 && target[t+2] == 3) can_type = 1; // GTr...
+			if ((flag & KSW_EZ_SPLICE_REV) && target[t+1] == 1 && target[t+2] == 3) can_type = 1; // CTr...
+			if (can_type && (target[t+3] == 0 || target[t+3] == 2)) can_type = 2;
+			if (can_type) ((int8_t*)donor)[t] = can_type == 2? 0 : semi_cost;
 		}
 		memset(acceptor, -noncan, tlen_ * 16);
 		for (t = 2; t < tlen; ++t) {
-			int is_can = 0;
-			if ((flag & KSW_EZ_SPLICE_FOR) && target[t-1] == 0 && target[t] == 2) is_can = 1;
-			if ((flag & KSW_EZ_SPLICE_REV) && target[t-1] == 0 && target[t] == 1) is_can = 1;
-			if (is_can) ((int8_t*)acceptor)[t] = 0;
+			int can_type = 0;
+			if ((flag & KSW_EZ_SPLICE_FOR) && target[t-1] == 0 && target[t] == 2) can_type = 1; // ...yAG
+			if ((flag & KSW_EZ_SPLICE_REV) && target[t-1] == 0 && target[t] == 1) can_type = 1; // ...yAC
+			if (can_type && (target[t-2] == 1 || target[t-2] == 3)) can_type = 2;
+			if (can_type) ((int8_t*)acceptor)[t] = can_type == 2? 0 : semi_cost;
 		}
 	}
 
@@ -145,10 +163,11 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp);
 			}
 		} else {
@@ -348,9 +367,9 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	if (with_cigar) { // backtrack
 		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
 		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
-			ksw_backtrack(km, 1, rev_cigar, 1, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+			ksw_backtrack(km, 1, rev_cigar, long_thres, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
 		else if (ez->max_t >= 0 && ez->max_q >= 0)
-			ksw_backtrack(km, 1, rev_cigar, 1, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+			ksw_backtrack(km, 1, rev_cigar, long_thres, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
 		kfree(km, mem2); kfree(km, off);
 	}
 }

ksw2_extz2_sse.c

@@ -5,11 +5,23 @@
 #ifdef __SSE2__
 #include <emmintrin.h>
 
+#ifdef KSW_SSE2_ONLY
+#undef __SSE4_1__
+#endif
+
 #ifdef __SSE4_1__
 #include <smmintrin.h>
 #endif
 
-void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez)
+#ifdef KSW_CPU_DISPATCH
+#ifdef __SSE4_1__
+void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#else
+void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif
+#else
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif // ~KSW_CPU_DISPATCH
 {
 #define __dp_code_block1 \
 	z = _mm_add_epi8(_mm_load_si128(&s[t]), qe2_); \
@@ -38,7 +50,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, qe2_, zero_, flag1_, flag2_, flag8_, flag16_, sc_mch_, sc_mis_, m1_, max_sc_;
+	__m128i q_, qe2_, zero_, flag1_, flag2_, flag8_, flag16_, sc_mch_, sc_mis_, sc_N_, m1_, max_sc_;
 	__m128i *u, *v, *x, *y, *s, *p = 0;
 
 	ksw_reset_extz(ez);
@@ -53,6 +65,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	flag16_ = _mm_set1_epi8(0x10);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = _mm_set1_epi8(-e);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 	max_sc_ = _mm_set1_epi8(mat[0] + (q + e) * 2);
 
@@ -118,10 +131,11 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((uint8_t*)s + t), tmp);
 			}
 		} else {
@@ -277,10 +291,14 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	if (!approx_max) kfree(km, H);
 	if (with_cigar) { // backtrack
 		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
-		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
+		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
-		else if (ez->max_t >= 0 && ez->max_q >= 0)
+		} else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > ez->max) {
+			ez->reach_end = 1;
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		} else if (ez->max_t >= 0 && ez->max_q >= 0) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		}
 		kfree(km, mem2); kfree(km, off);
 	}
 }

ksw2_ll_sse.c

@@ -122,7 +122,7 @@ int ksw_ll_i16(void *q_, int tlen, const uint8_t *target, int _gapo, int _gape,
 			f = _mm_max_epi16(f, h);
 			h = _mm_load_si128(H0 + j);
 		}
-		for (k = 0; LIKELY(k < 16); ++k) {
+		for (k = 0; LIKELY(k < 8); ++k) {
 			f = _mm_slli_si128(f, 2);
 			for (j = 0; LIKELY(j < slen); ++j) {
 				h = _mm_load_si128(H1 + j);

kthread.c

@@ -1,6 +1,11 @@
 #include <pthread.h>
 #include <stdlib.h>
 #include <limits.h>
+#include <stdint.h>
+
+#if (defined(WIN32) || defined(_WIN32)) && defined(_MSC_VER)
+#define __sync_fetch_and_add(ptr, addend)     _InterlockedExchangeAdd((void*)ptr, addend)
+#endif
 
 /************
  * kt_for() *
@@ -52,12 +57,13 @@ void kt_for(int n_threads, void (*func)(void*,long,int), void *data, long n)
 		kt_for_t t;
 		pthread_t *tid;
 		t.func = func, t.data = data, t.n_threads = n_threads, t.n = n;
-		t.w = (ktf_worker_t*)alloca(n_threads * sizeof(ktf_worker_t));
-		tid = (pthread_t*)alloca(n_threads * sizeof(pthread_t));
+		t.w = (ktf_worker_t*)calloc(n_threads, sizeof(ktf_worker_t));
+		tid = (pthread_t*)calloc(n_threads, sizeof(pthread_t));
 		for (i = 0; i < n_threads; ++i)
 			t.w[i].t = &t, t.w[i].i = i;
 		for (i = 0; i < n_threads; ++i) pthread_create(&tid[i], 0, ktf_worker, &t.w[i]);
 		for (i = 0; i < n_threads; ++i) pthread_join(tid[i], 0);
+		free(tid); free(t.w);
 	} else {
 		long j;
 		for (j = 0; j < n; ++j) func(data, j, 0);
@@ -135,16 +141,17 @@ void kt_pipeline(int n_threads, void *(*func)(void*, int, void*), void *shared_d
 	pthread_mutex_init(&aux.mutex, 0);
 	pthread_cond_init(&aux.cv, 0);
 
-	aux.workers = (ktp_worker_t*)alloca(n_threads * sizeof(ktp_worker_t));
+	aux.workers = (ktp_worker_t*)calloc(n_threads, sizeof(ktp_worker_t));
 	for (i = 0; i < n_threads; ++i) {
 		ktp_worker_t *w = &aux.workers[i];
 		w->step = 0; w->pl = &aux; w->data = 0;
 		w->index = aux.index++;
 	}
 
-	tid = (pthread_t*)alloca(n_threads * sizeof(pthread_t));
+	tid = (pthread_t*)calloc(n_threads, sizeof(pthread_t));
 	for (i = 0; i < n_threads; ++i) pthread_create(&tid[i], 0, ktp_worker, &aux.workers[i]);
 	for (i = 0; i < n_threads; ++i) pthread_join(tid[i], 0);
+	free(tid); free(aux.workers);
 
 	pthread_mutex_destroy(&aux.mutex);
 	pthread_cond_destroy(&aux.cv);

main.c

@@ -1,39 +1,50 @@
-#include <getopt.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <sys/resource.h>
-#include <sys/time.h>
 #include "bseq.h"
 #include "minimap.h"
 #include "mmpriv.h"
+#include "getopt.h"
 
-#define MM_VERSION "2.1-r311"
+#define MM_VERSION "2.7-r654"
 
+#ifdef __linux__
+#include <sys/resource.h>
+#include <sys/time.h>
 void liftrlimit()
 {
-#ifdef __linux__
 	struct rlimit r;
 	getrlimit(RLIMIT_AS, &r);
 	r.rlim_cur = r.rlim_max;
 	setrlimit(RLIMIT_AS, &r);
-#endif
 }
+#else
+void liftrlimit() {}
+#endif
 
 static struct option long_options[] = {
 	{ "bucket-bits",    required_argument, 0, 0 },
 	{ "mb-size",        required_argument, 0, 'K' },
-	{ "int-rname",      no_argument,       0, 0 },
+	{ "seed",           required_argument, 0, 0 },
 	{ "no-kalloc",      no_argument,       0, 0 },
 	{ "print-qname",    no_argument,       0, 0 },
 	{ "no-self",        no_argument,       0, 0 },
-	{ "print-seed",     no_argument,       0, 0 },
+	{ "print-seeds",    no_argument,       0, 0 },
 	{ "max-chain-skip", required_argument, 0, 0 },
 	{ "min-dp-len",     required_argument, 0, 0 },
 	{ "print-aln-seq",  no_argument,       0, 0 },
 	{ "splice",         no_argument,       0, 0 },
-	{ "cost-non-gt-ag", required_argument, 0, 0 },
-	{ "no-sam-sq",      no_argument,       0, 0 },
+	{ "cost-non-gt-ag", required_argument, 0, 'C' },
+	{ "no-long-join",   no_argument,       0, 0 },
+	{ "sr",             no_argument,       0, 0 },
+	{ "frag",           optional_argument, 0, 0 },
+	{ "secondary",      optional_argument, 0, 0 },
+	{ "cs",             optional_argument, 0, 0 },
+	{ "end-bonus",      required_argument, 0, 0 },
+	{ "no-pairing",     no_argument,       0, 0 },
+	{ "splice-flank",   optional_argument, 0, 0 },
+	{ "idx-no-seq",     no_argument,       0, 0 },
+	{ "end-seed-pen",   required_argument, 0, 0 },   // 21
 	{ "help",           no_argument,       0, 'h' },
 	{ "max-intron-len", required_argument, 0, 'G' },
 	{ "version",        no_argument,       0, 'V' },
@@ -58,37 +69,50 @@ static inline int64_t mm_parse_num(const char *str)
 
 int main(int argc, char *argv[])
 {
+	const char *opt_str = "2aSw:k:K:t:r:f:Vv:g:G:I:d:XT:s:x:Hcp:M:n:z:A:B:O:E:m:N:Qu:R:hF:LC:";
 	mm_mapopt_t opt;
-	int i, c, k = 15, w = -1, bucket_bits = MM_IDX_DEF_B, n_threads = 3, keep_name = 1, is_idx, is_hpc = 0, long_idx, idx_par_set = 0, max_intron_len = 0, n_idx_part = 0;
-	int minibatch_size = 200000000;
-	uint64_t batch_size = 4000000000ULL;
-	mm_bseq_file_t *fp = 0;
+	mm_idxopt_t ipt;
+	int i, c, n_threads = 3, long_idx;
 	char *fnw = 0, *rg = 0, *s;
-	FILE *fpr = 0, *fpw = 0, *fp_help = stderr;
+	FILE *fp_help = stderr;
+	mm_idx_reader_t *idx_rdr;
+	mm_idx_t *mi;
 
+	mm_verbose = 3;
 	liftrlimit();
 	mm_realtime0 = realtime();
-	mm_mapopt_init(&opt);
+	mm_set_opt(0, &ipt, &opt);
 
-	while ((c = getopt_long(argc, argv, "aSw:k:K:t:r:f:Vv:g:G:I:d:XT:s:x:Hcp:M:n:z:A:B:O:E:m:N:Qu:R:h", long_options, &long_idx)) >= 0) {
-		if (c == 'w') w = atoi(optarg), idx_par_set = 1;
-		else if (c == 'k') k = atoi(optarg), idx_par_set = 1;
-		else if (c == 'H') is_hpc = 1, idx_par_set = 1;
+	while ((c = getopt_long(argc, argv, opt_str, long_options, &long_idx)) >= 0) // apply option -x/preset first
+		if (c == 'x') {
+			if (mm_set_opt(optarg, &ipt, &opt) < 0) {
+				fprintf(stderr, "[ERROR] unknown preset '%s'\n", optarg);
+				return 1;
+			}
+			break;
+		}
+	optreset = 1;
+
+	while ((c = getopt_long(argc, argv, opt_str, long_options, &long_idx)) >= 0) {
+		if (c == 'w') ipt.w = atoi(optarg);
+		else if (c == 'k') ipt.k = atoi(optarg);
+		else if (c == 'H') ipt.flag |= MM_I_HPC;
 		else if (c == 'd') fnw = optarg; // the above are indexing related options, except -I
 		else if (c == 'r') opt.bw = (int)mm_parse_num(optarg);
-		else if (c == 'f') opt.mid_occ_frac = atof(optarg);
 		else if (c == 't') n_threads = atoi(optarg);
 		else if (c == 'v') mm_verbose = atoi(optarg);
 		else if (c == 'g') opt.max_gap = (int)mm_parse_num(optarg);
-		else if (c == 'G') max_intron_len = (int)mm_parse_num(optarg);
+		else if (c == 'G') mm_mapopt_max_intron_len(&opt, (int)mm_parse_num(optarg));
+		else if (c == 'F') opt.max_frag_len = (int)mm_parse_num(optarg);
 		else if (c == 'N') opt.best_n = atoi(optarg);
 		else if (c == 'p') opt.pri_ratio = atof(optarg);
 		else if (c == 'M') opt.mask_level = atof(optarg);
 		else if (c == 'c') opt.flag |= MM_F_OUT_CG | MM_F_CIGAR;
-		else if (c == 'S') opt.flag |= MM_F_OUT_CS | MM_F_CIGAR;
 		else if (c == 'X') opt.flag |= MM_F_AVA | MM_F_NO_SELF;
 		else if (c == 'a') opt.flag |= MM_F_OUT_SAM | MM_F_CIGAR;
 		else if (c == 'Q') opt.flag |= MM_F_NO_QUAL;
+		else if (c == 'Y') opt.flag |= MM_F_SOFTCLIP;
+		else if (c == 'L') opt.flag |= MM_F_LONG_CIGAR;
 		else if (c == 'T') opt.sdust_thres = atoi(optarg);
 		else if (c == 'n') opt.min_cnt = atoi(optarg);
 		else if (c == 'm') opt.min_chain_score = atoi(optarg);
@@ -96,33 +120,72 @@ int main(int argc, char *argv[])
 		else if (c == 'B') opt.b = atoi(optarg);
 		else if (c == 'z') opt.zdrop = atoi(optarg);
 		else if (c == 's') opt.min_dp_max = atoi(optarg);
-		else if (c == 'I') batch_size = mm_parse_num(optarg);
-		else if (c == 'K') minibatch_size = (int)mm_parse_num(optarg);
+		else if (c == 'C') opt.noncan = atoi(optarg);
+		else if (c == 'I') ipt.batch_size = mm_parse_num(optarg);
+		else if (c == 'K') opt.mini_batch_size = (int)mm_parse_num(optarg);
 		else if (c == 'R') rg = optarg;
 		else if (c == 'h') fp_help = stdout;
-		else if (c == 0 && long_idx == 0) bucket_bits = atoi(optarg); // --bucket-bits
-		else if (c == 0 && long_idx == 2) keep_name = 0; // --int-rname
+		else if (c == '2') opt.flag |= MM_F_2_IO_THREADS;
+		else if (c == 0 && long_idx == 0) ipt.bucket_bits = atoi(optarg); // --bucket-bits
+		else if (c == 0 && long_idx == 2) opt.seed = atoi(optarg); // --seed
 		else if (c == 0 && long_idx == 3) mm_dbg_flag |= MM_DBG_NO_KALLOC; // --no-kalloc
 		else if (c == 0 && long_idx == 4) mm_dbg_flag |= MM_DBG_PRINT_QNAME; // --print-qname
 		else if (c == 0 && long_idx == 5) opt.flag |= MM_F_NO_SELF; // --no-self
-		else if (c == 0 && long_idx == 6) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_SEED; // --print-seed
+		else if (c == 0 && long_idx == 6) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_SEED, n_threads = 1; // --print-seed
 		else if (c == 0 && long_idx == 7) opt.max_chain_skip = atoi(optarg); // --max-chain-skip
 		else if (c == 0 && long_idx == 8) opt.min_ksw_len = atoi(optarg); // --min-dp-len
-		else if (c == 0 && long_idx == 9) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_ALN_SEQ; // --print-aln-seq
+		else if (c == 0 && long_idx == 9) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_ALN_SEQ, n_threads = 1; // --print-aln-seq
 		else if (c == 0 && long_idx ==10) opt.flag |= MM_F_SPLICE; // --splice
-		else if (c == 0 && long_idx ==11) opt.noncan = atoi(optarg); // --cost-non-gt-ag
-		else if (c == 0 && long_idx ==12) opt.flag |= MM_F_NO_SAM_SQ; // --no-sam-sq
-		else if (c == 'V') {
+		else if (c == 0 && long_idx ==12) opt.flag |= MM_F_NO_LJOIN; // --no-long-join
+		else if (c == 0 && long_idx ==13) opt.flag |= MM_F_SR; // --sr
+		else if (c == 0 && long_idx ==17) opt.end_bonus = atoi(optarg); // --end-bonus
+		else if (c == 0 && long_idx ==18) opt.flag |= MM_F_INDEPEND_SEG; // --no-pairing
+		else if (c == 0 && long_idx ==20) ipt.flag |= MM_I_NO_SEQ; // --idx-no-seq
+		else if (c == 0 && long_idx ==21) opt.anchor_ext_shift = atoi(optarg); // --end-seed-pen
+		else if (c == 0 && long_idx == 14) { // --frag
+			if (optarg == 0 || strcmp(optarg, "yes") == 0 || strcmp(optarg, "y") == 0)
+				opt.flag |= MM_F_FRAG_MODE;
+			else opt.flag &= ~MM_F_FRAG_MODE;
+		} else if (c == 0 && long_idx == 15) { // --secondary
+			if (optarg == 0 || strcmp(optarg, "yes") == 0 || strcmp(optarg, "y") == 0)
+				opt.flag &= ~MM_F_NO_PRINT_2ND;
+			else opt.flag |= MM_F_NO_PRINT_2ND;
+		} else if (c == 0 && long_idx == 16) { // --cs
+			opt.flag |= MM_F_OUT_CS | MM_F_CIGAR;
+			if (optarg == 0 || strcmp(optarg, "short") == 0) {
+				opt.flag &= ~MM_F_OUT_CS_LONG;
+			} else if (strcmp(optarg, "long") == 0) {
+				opt.flag |= MM_F_OUT_CS_LONG;
+			} else if (strcmp(optarg, "none") == 0) {
+				opt.flag &= ~MM_F_OUT_CS;
+			} else if (mm_verbose >= 2) {
+				fprintf(stderr, "[WARNING]\033[1;31m --cs only takes 'short' or 'long'. Invalid values are assumed to be 'short'.\033[0m\n");
+			}
+		} else if (c == 0 && long_idx == 19) { // --splice-flank
+			if (optarg == 0 || strcmp(optarg, "yes") == 0 || strcmp(optarg, "y") == 0)
+				opt.flag |= MM_F_SPLICE_FLANK;
+			else opt.flag &= ~MM_F_SPLICE_FLANK;
+		} else if (c == 'S') {
+			opt.flag |= MM_F_OUT_CS | MM_F_CIGAR | MM_F_OUT_CS_LONG;
+			if (mm_verbose >= 2)
+				fprintf(stderr, "[WARNING]\033[1;31m option -S is deprecated and may be removed in future. Please use --cs=long instead.\033[0m\n");
+		} else if (c == 'V') {
 			puts(MM_VERSION);
 			return 0;
+		} else if (c == 'f') {
+			double x;
+			char *p;
+			x = strtod(optarg, &p);
+			if (x < 1.0) opt.mid_occ_frac = x, opt.mid_occ = 0;
+			else opt.mid_occ = (int)(x + .499);
+			if (*p == ',') opt.max_occ = (int)(strtod(p+1, &p) + .499);
 		} else if (c == 'u') {
-			if (*optarg == 'b') opt.flag |= MM_F_SPLICE_FOR|MM_F_SPLICE_REV;
-			else if (*optarg == 'B') opt.flag |= MM_F_SPLICE_BOTH;
-			else if (*optarg == 'f') opt.flag |= MM_F_SPLICE_FOR, opt.flag &= ~MM_F_SPLICE_REV;
-			else if (*optarg == 'r') opt.flag |= MM_F_SPLICE_REV, opt.flag &= ~MM_F_SPLICE_FOR;
-			else if (*optarg == 'n') opt.flag &= ~(MM_F_SPLICE_FOR|MM_F_SPLICE_REV);
+			if (*optarg == 'b') opt.flag |= MM_F_SPLICE_FOR|MM_F_SPLICE_REV; // both strands
+			else if (*optarg == 'f') opt.flag |= MM_F_SPLICE_FOR, opt.flag &= ~MM_F_SPLICE_REV; // match GT-AG
+			else if (*optarg == 'r') opt.flag |= MM_F_SPLICE_REV, opt.flag &= ~MM_F_SPLICE_FOR; // match CT-AC (reverse complement of GT-AG)
+			else if (*optarg == 'n') opt.flag &= ~(MM_F_SPLICE_FOR|MM_F_SPLICE_REV); // don't try to match the GT-AG signal
 			else {
-				fprintf(stderr, "[E::%s] unrecognized cDNA direction\n", __func__);
+				fprintf(stderr, "[ERROR]\033[1;31m unrecognized cDNA direction\033[0m\n");
 				return 1;
 			}
 		} else if (c == 'O') {
@@ -131,66 +194,38 @@ int main(int argc, char *argv[])
 		} else if (c == 'E') {
 			opt.e = opt.e2 = strtol(optarg, &s, 10);
 			if (*s == ',') opt.e2 = strtol(s + 1, &s, 10);
-		} else if (c == 'x') {
-			if (strcmp(optarg, "ava-ont") == 0) {
-				opt.flag |= MM_F_AVA | MM_F_NO_SELF;
-				opt.min_chain_score = 100, opt.pri_ratio = 0.0f, opt.max_gap = 10000, opt.max_chain_skip = 25;
-				minibatch_size = 500000000;
-				k = 15, w = 5;
-			} else if (strcmp(optarg, "ava-pb") == 0) {
-				opt.flag |= MM_F_AVA | MM_F_NO_SELF;
-				opt.min_chain_score = 100, opt.pri_ratio = 0.0f, opt.max_gap = 10000, opt.max_chain_skip = 25;
-				minibatch_size = 500000000;
-				is_hpc = 1, k = 19, w = 5;
-			} else if (strcmp(optarg, "map10k") == 0 || strcmp(optarg, "map-pb") == 0) {
-				is_hpc = 1, k = 19;
-			} else if (strcmp(optarg, "map-ont") == 0) {
-				is_hpc = 0, k = 15;
-			} else if (strcmp(optarg, "asm5") == 0) {
-				k = 19, w = 19;
-				opt.a = 1, opt.b = 19, opt.q = 39, opt.q2 = 81, opt.e = 3, opt.e2 = 1, opt.zdrop = 200;
-				opt.min_dp_max = 200;
-			} else if (strcmp(optarg, "asm10") == 0) {
-				k = 19, w = 19;
-				opt.a = 1, opt.b = 9, opt.q = 16, opt.q2 = 41, opt.e = 2, opt.e2 = 1, opt.zdrop = 200;
-				opt.min_dp_max = 200;
-			} else if (strcmp(optarg, "splice") == 0 || strcmp(optarg, "cdna") == 0) {
-				k = 15, w = 5;
-				opt.flag |= MM_F_SPLICE | MM_F_SPLICE_FOR | MM_F_SPLICE_REV;
-				opt.max_gap = 2000, opt.max_gap_ref = opt.bw = 200000;
-				opt.a = 1, opt.b = 2, opt.q = 2, opt.e = 1, opt.q2 = 32, opt.e2 = 0;
-				opt.noncan = 5;
-				opt.zdrop = 200;
-			} else {
-				fprintf(stderr, "[E::%s] unknown preset '%s'\n", __func__, optarg);
-				return 1;
-			}
 		}
 	}
-	if (w < 0) w = (int)(.6666667 * k + .499);
-	if ((opt.flag & MM_F_SPLICE) && max_intron_len > 0)
-		opt.max_gap_ref = opt.bw = max_intron_len;
+	if ((opt.flag & MM_F_SPLICE) && (opt.flag & MM_F_FRAG_MODE)) {
+		fprintf(stderr, "[ERROR]\033[1;31m --splice and --frag should not be specified at the same time.\033[0m\n");
+		return 1;
+	}
+	if (!fnw && !(opt.flag&MM_F_CIGAR))
+		ipt.flag |= MM_I_NO_SEQ;
+	if (mm_check_opt(&ipt, &opt) < 0)
+		return 1;
 
 	if (argc == optind || fp_help == stdout) {
 		fprintf(fp_help, "Usage: minimap2 [options] <target.fa>|<target.idx> [query.fa] [...]\n");
 		fprintf(fp_help, "Options:\n");
 		fprintf(fp_help, "  Indexing:\n");
 		fprintf(fp_help, "    -H           use homopolymer-compressed k-mer\n");
-		fprintf(fp_help, "    -k INT       k-mer size (no larger than 28) [%d]\n", k);
-		fprintf(fp_help, "    -w INT       minizer window size [{-k}*2/3]\n");
+		fprintf(fp_help, "    -k INT       k-mer size (no larger than 28) [%d]\n", ipt.k);
+		fprintf(fp_help, "    -w INT       minizer window size [%d]\n", ipt.w);
 		fprintf(fp_help, "    -I NUM       split index for every ~NUM input bases [4G]\n");
 		fprintf(fp_help, "    -d FILE      dump index to FILE []\n");
 		fprintf(fp_help, "  Mapping:\n");
 		fprintf(fp_help, "    -f FLOAT     filter out top FLOAT fraction of repetitive minimizers [%g]\n", opt.mid_occ_frac);
-		fprintf(fp_help, "    -g INT       stop chain enlongation if there are no minimizers in INT-bp [%d]\n", opt.max_gap);
-		fprintf(fp_help, "    -r INT       bandwidth used in chaining and DP-based alignment [%d]\n", opt.bw);
+		fprintf(fp_help, "    -g NUM       stop chain enlongation if there are no minimizers in INT-bp [%d]\n", opt.max_gap);
+		fprintf(fp_help, "    -G NUM       max intron length (effective with -xsplice; changing -r) [200k]\n");
+		fprintf(fp_help, "    -F NUM       max fragment length (effective with -xsr or in the fragment mode) [800]\n");
+		fprintf(fp_help, "    -r NUM       bandwidth used in chaining and DP-based alignment [%d]\n", opt.bw);
 		fprintf(fp_help, "    -n INT       minimal number of minimizers on a chain [%d]\n", opt.min_cnt);
 		fprintf(fp_help, "    -m INT       minimal chaining score (matching bases minus log gap penalty) [%d]\n", opt.min_chain_score);
 //		fprintf(fp_help, "    -T INT       SDUST threshold; 0 to disable SDUST [%d]\n", opt.sdust_thres); // TODO: this option is never used; might be buggy
 		fprintf(fp_help, "    -X           skip self and dual mappings (for the all-vs-all mode)\n");
 		fprintf(fp_help, "    -p FLOAT     min secondary-to-primary score ratio [%g]\n", opt.pri_ratio);
 		fprintf(fp_help, "    -N INT       retain at most INT secondary alignments [%d]\n", opt.best_n);
-		fprintf(fp_help, "    -G NUM       max intron length (only effective following -x splice) [200k]\n");
 		fprintf(fp_help, "  Alignment:\n");
 		fprintf(fp_help, "    -A INT       matching score [%d]\n", opt.a);
 		fprintf(fp_help, "    -B INT       mismatch penalty [%d]\n", opt.b);
@@ -202,70 +237,71 @@ int main(int argc, char *argv[])
 		fprintf(fp_help, "  Input/Output:\n");
 		fprintf(fp_help, "    -a           output in the SAM format (PAF by default)\n");
 		fprintf(fp_help, "    -Q           don't output base quality in SAM\n");
+		fprintf(fp_help, "    -L           write CIGAR with >65535 ops at the CG tag\n");
 		fprintf(fp_help, "    -R STR       SAM read group line in a format like '@RG\\tID:foo\\tSM:bar' []\n");
 		fprintf(fp_help, "    -c           output CIGAR in PAF\n");
-		fprintf(fp_help, "    -S           output the cs tag in PAF (cs encodes both query and ref sequences)\n");
+		fprintf(fp_help, "    --cs[=STR]   output the cs tag; STR is 'short' (if absent) or 'long' [none]\n");
+		fprintf(fp_help, "    -Y           use soft clipping for supplementary alignments\n");
 		fprintf(fp_help, "    -t INT       number of threads [%d]\n", n_threads);
-		fprintf(fp_help, "    -K NUM       minibatch size [200M]\n");
+		fprintf(fp_help, "    -K NUM       minibatch size for mapping [500M]\n");
 //		fprintf(fp_help, "    -v INT       verbose level [%d]\n", mm_verbose);
 		fprintf(fp_help, "    --version    show version number\n");
 		fprintf(fp_help, "  Preset:\n");
-		fprintf(fp_help, "    -x STR       preset (recommended to be applied before other options) []\n");
-		fprintf(fp_help, "                 map10k/map-pb: -Hk19 (PacBio/ONT vs reference mapping)\n");
-		fprintf(fp_help, "                 map-ont: -k15 (slightly more sensitive than 'map10k' for ONT vs reference)\n");
+		fprintf(fp_help, "    -x STR       preset (always applied before other options) []\n");
+		fprintf(fp_help, "                 map-pb: -Hk19 (PacBio vs reference mapping)\n");
+		fprintf(fp_help, "                 map-ont: -k15 (Oxford Nanopore vs reference mapping)\n");
 		fprintf(fp_help, "                 asm5: -k19 -w19 -A1 -B19 -O39,81 -E3,1 -s200 -z200 (asm to ref mapping; break at 5%% div.)\n");
 		fprintf(fp_help, "                 asm10: -k19 -w19 -A1 -B9 -O16,41 -E2,1 -s200 -z200 (asm to ref mapping; break at 10%% div.)\n");
-		fprintf(fp_help, "                 ava-pb: -Hk19 -w5 -Xp0 -m100 -g10000 -K500m --max-chain-skip 25 (PacBio read overlap)\n");
-		fprintf(fp_help, "                 ava-ont: -k15 -w5 -Xp0 -m100 -g10000 -K500m --max-chain-skip 25 (ONT read overlap)\n");
+		fprintf(fp_help, "                 ava-pb: -Hk19 -w5 -Xp0 -m100 -g10000 --max-chain-skip 25 (PacBio read overlap)\n");
+		fprintf(fp_help, "                 ava-ont: -k15 -w5 -Xp0 -m100 -g10000 --max-chain-skip 25 (ONT read overlap)\n");
 		fprintf(fp_help, "                 splice: long-read spliced alignment (see minimap2.1 for details)\n");
+		fprintf(fp_help, "                 sr: short single-end reads without splicing (see minimap2.1 for details)\n");
 		fprintf(fp_help, "\nSee `man ./minimap2.1' for detailed description of command-line options.\n");
 		return fp_help == stdout? 0 : 1;
 	}
 
-	is_idx = mm_idx_is_idx(argv[optind]);
-	if (is_idx < 0) {
+	idx_rdr = mm_idx_reader_open(argv[optind], &ipt, fnw);
+	if (idx_rdr == 0) {
 		fprintf(stderr, "[ERROR] failed to open file '%s'\n", argv[optind]);
 		return 1;
 	}
-	if (!is_idx && fnw == 0 && argc - optind < 2) {
+	if (!idx_rdr->is_idx && fnw == 0 && argc - optind < 2) {
 		fprintf(stderr, "[ERROR] missing input: please specify a query file to map or option -d to keep the index\n");
+		mm_idx_reader_close(idx_rdr);
 		return 1;
 	}
-	if (is_idx) fpr = fopen(argv[optind], "rb");
-	else fp = mm_bseq_open(argv[optind]);
-	if (fnw) fpw = fopen(fnw, "wb");
-	if (opt.flag & MM_F_OUT_SAM)
-		mm_write_sam_hdr_no_SQ(rg, MM_VERSION, argc, argv);
-	for (;;) {
-		mm_idx_t *mi;
-		if (fpr) {
-			mi = mm_idx_load(fpr);
-			if (idx_par_set && mm_verbose >= 2 && (mi->k != k || mi->w != w || mi->is_hpc != is_hpc))
-				fprintf(stderr, "[WARNING] \033[1;31mIndexing parameters on the command line (-k/-w/-H) overridden by parameters in the prebuilt index.\033[0m\n");
-		} else {
-			mi = mm_idx_gen(fp, w, k, bucket_bits, is_hpc, minibatch_size, n_threads, batch_size, keep_name);
+	if (opt.best_n == 0 && (opt.flag&MM_F_CIGAR) && mm_verbose >= 2)
+		fprintf(stderr, "[WARNING]\033[1;31m `-N 0' reduces alignment accuracy. Please use --secondary=no to suppress secondary alignments.\033[0m\n");
+	while ((mi = mm_idx_reader_read(idx_rdr, n_threads)) != 0) {
+		if ((opt.flag & MM_F_CIGAR) && (mi->flag & MM_I_NO_SEQ)) {
+			fprintf(stderr, "[ERROR] the prebuilt index doesn't contain sequences.\n");
+			mm_idx_destroy(mi);
+			mm_idx_reader_close(idx_rdr);
+			return 1;
+		}
+		if ((opt.flag & MM_F_OUT_SAM) && idx_rdr->n_parts == 1) {
+			if (mm_idx_reader_eof(idx_rdr)) {
+				mm_write_sam_hdr(mi, rg, MM_VERSION, argc, argv);
+			} else {
+				mm_write_sam_hdr(0, rg, MM_VERSION, argc, argv);
+				if (mm_verbose >= 2)
+					fprintf(stderr, "[WARNING]\033[1;31m For a multi-part index, no @SQ lines will be outputted.\033[0m\n");
+			}
 		}
-		if (mi == 0) break;
-		++n_idx_part;
-		if (mm_verbose >= 2 && n_idx_part > 1 && (opt.flag&MM_F_OUT_SAM) && !(opt.flag&MM_F_NO_SAM_SQ))
-			fprintf(stderr, "[WARNING] \033[1;31mSAM output is malformated due to internal @SQ lines. Please add option --no-sam-sq or filter afterwards.\033[0m\n");
 		if (mm_verbose >= 3)
 			fprintf(stderr, "[M::%s::%.3f*%.2f] loaded/built the index for %d target sequence(s)\n",
 					__func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), mi->n_seq);
-		if (fpw) {
-			mm_idx_dump(fpw, mi);
-			if (mm_verbose >= 3)
-				fprintf(stderr, "[M::%s::%.3f*%.2f] dumpped the (partial) index to disk\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
-		}
 		if (argc != optind + 1) mm_mapopt_update(&opt, mi);
 		if (mm_verbose >= 3) mm_idx_stat(mi);
-		for (i = optind + 1; i < argc; ++i)
-			mm_map_file(mi, argv[i], &opt, n_threads, minibatch_size);
+		if (!(opt.flag & MM_F_FRAG_MODE)) {
+			for (i = optind + 1; i < argc; ++i)
+				mm_map_file(mi, argv[i], &opt, n_threads);
+		} else {
+			mm_map_file_frag(mi, argc - (optind + 1), (const char**)&argv[optind + 1], &opt, n_threads);
+		}
 		mm_idx_destroy(mi);
 	}
-	if (fpw) fclose(fpw);
-	if (fpr) fclose(fpr);
-	if (fp)  mm_bseq_close(fp);
+	mm_idx_reader_close(idx_rdr);
 
 	fprintf(stderr, "[M::%s] Version: %s\n", __func__, MM_VERSION);
 	fprintf(stderr, "[M::%s] CMD:", __func__);

map.c

@@ -7,13 +7,14 @@
 #include "sdust.h"
 #include "mmpriv.h"
 #include "bseq.h"
+#include "khash.h"
 
 void mm_mapopt_init(mm_mapopt_t *opt)
 {
 	memset(opt, 0, sizeof(mm_mapopt_t));
-	opt->max_occ_frac = 1e-5f;
+	opt->seed = 11;
 	opt->mid_occ_frac = 2e-4f;
-	opt->sdust_thres = 0;
+	opt->sdust_thres = 0; // no SDUST masking
 
 	opt->min_cnt = 3;
 	opt->min_chain_score = 40;
@@ -32,33 +33,111 @@ void mm_mapopt_init(mm_mapopt_t *opt)
 
 	opt->a = 2, opt->b = 4, opt->q = 4, opt->e = 2, opt->q2 = 24, opt->e2 = 1;
 	opt->zdrop = 400;
+	opt->end_bonus = -1;
 	opt->min_dp_max = opt->min_chain_score * opt->a;
 	opt->min_ksw_len = 200;
+	opt->anchor_ext_len = 20, opt->anchor_ext_shift = 6;
+	opt->mini_batch_size = 500000000;
+
+	opt->pe_ori = 0; // FF
+	opt->pe_bonus = 33;
 }
 
 void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi)
 {
-	if (opt->flag & MM_F_SPLICE_BOTH)
-		opt->flag &= ~(MM_F_SPLICE_FOR|MM_F_SPLICE_REV);
-	opt->max_occ = mm_idx_cal_max_occ(mi, opt->max_occ_frac);
-	opt->mid_occ = mm_idx_cal_max_occ(mi, opt->mid_occ_frac);
+	if ((opt->flag & MM_F_SPLICE_FOR) && (opt->flag & MM_F_SPLICE_REV))
+		opt->flag |= MM_F_SPLICE;
+	if (opt->mid_occ <= 0)
+		opt->mid_occ = mm_idx_cal_max_occ(mi, opt->mid_occ_frac);
 	if (mm_verbose >= 3)
-		fprintf(stderr, "[M::%s::%.3f*%.2f] mid_occ = %d; max_occ = %d\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0),
-				opt->mid_occ, opt->max_occ);
+		fprintf(stderr, "[M::%s::%.3f*%.2f] mid_occ = %d\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), opt->mid_occ);
+}
+
+void mm_mapopt_max_intron_len(mm_mapopt_t *opt, int max_intron_len)
+{
+	if ((opt->flag & MM_F_SPLICE) && max_intron_len > 0)
+		opt->max_gap_ref = opt->bw = max_intron_len;
+}
+
+int mm_set_opt(const char *preset, mm_idxopt_t *io, mm_mapopt_t *mo)
+{
+	if (preset == 0) {
+		mm_idxopt_init(io);
+		mm_mapopt_init(mo);
+	} else if (strcmp(preset, "ava-ont") == 0) {
+		io->flag = 0, io->k = 15, io->w = 5;
+		mo->flag |= MM_F_AVA | MM_F_NO_SELF;
+		mo->min_chain_score = 100, mo->pri_ratio = 0.0f, mo->max_gap = 10000, mo->max_chain_skip = 25;
+	} else if (strcmp(preset, "ava-pb") == 0) {
+		io->flag |= MM_I_HPC, io->k = 19, io->w = 5;
+		mo->flag |= MM_F_AVA | MM_F_NO_SELF;
+		mo->min_chain_score = 100, mo->pri_ratio = 0.0f, mo->max_gap = 10000, mo->max_chain_skip = 25;
+	} else if (strcmp(preset, "map10k") == 0 || strcmp(preset, "map-pb") == 0) {
+		io->flag |= MM_I_HPC, io->k = 19;
+	} else if (strcmp(preset, "map-ont") == 0) {
+		io->flag = 0, io->k = 15;
+	} else if (strcmp(preset, "asm5") == 0) {
+		io->flag = 0, io->k = 19, io->w = 19;
+		mo->a = 1, mo->b = 19, mo->q = 39, mo->q2 = 81, mo->e = 3, mo->e2 = 1, mo->zdrop = 200;
+		mo->min_dp_max = 200;
+		mo->best_n = 50;
+	} else if (strcmp(preset, "asm10") == 0) {
+		io->flag = 0, io->k = 19, io->w = 19;
+		mo->a = 1, mo->b = 9, mo->q = 16, mo->q2 = 41, mo->e = 2, mo->e2 = 1, mo->zdrop = 200;
+		mo->min_dp_max = 200;
+		mo->best_n = 50;
+	} else if (strcmp(preset, "short") == 0 || strcmp(preset, "sr") == 0) {
+		io->flag = 0, io->k = 21, io->w = 11;
+		mo->flag |= MM_F_SR | MM_F_FRAG_MODE | MM_F_NO_PRINT_2ND | MM_F_2_IO_THREADS;
+		mo->pe_ori = 0<<1|1; // FR
+		mo->a = 2, mo->b = 8, mo->q = 12, mo->e = 2, mo->q2 = 24, mo->e2 = 1;
+		mo->zdrop = 100;
+		mo->end_bonus = 10;
+		mo->max_frag_len = 800;
+		mo->max_gap = 100;
+		mo->bw = 100;
+		mo->pri_ratio = 0.5f;
+		mo->min_cnt = 2;
+		mo->min_chain_score = 25;
+		mo->min_dp_max = 40;
+		mo->best_n = 20;
+		mo->mid_occ = 1000;
+		mo->max_occ = 5000;
+		mo->mini_batch_size = 50000000;
+	} else if (strcmp(preset, "splice") == 0 || strcmp(preset, "cdna") == 0) {
+		io->flag = 0, io->k = 15, io->w = 5;
+		mo->flag |= MM_F_SPLICE | MM_F_SPLICE_FOR | MM_F_SPLICE_REV | MM_F_SPLICE_FLANK;
+		mo->max_gap = 2000, mo->max_gap_ref = mo->bw = 200000;
+		mo->a = 1, mo->b = 2, mo->q = 2, mo->e = 1, mo->q2 = 32, mo->e2 = 0;
+		mo->noncan = 9;
+		mo->zdrop = 200;
+	} else return -1;
+	return 0;
+}
+
+int mm_check_opt(const mm_idxopt_t *io, const mm_mapopt_t *mo)
+{
+	if ((mo->q != mo->q2 || mo->e != mo->e2) && !(mo->e > mo->e2 && mo->q + mo->e < mo->q2 + mo->e2)) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m dual gap penalties violating E1>E2 and O1+E1<O2+E2\033[0m\n");
+		return -2;
+	}
+	if ((mo->q + mo->e) + (mo->q2 + mo->e2) > 127) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m scoring system violating ({-O}+{-E})+({-O2}+{-E2}) <= 127\033[0m\n");
+		return -1;
+	}
+	return 0;
 }
 
 typedef struct {
-	uint32_t n:31, is_alloc:1;
+	uint32_t n;
 	uint32_t qpos;
-	union {
-		const uint64_t *cr;
-		uint64_t *r;
-	} x;
+	uint32_t seg_id;
+	const uint64_t *cr;
 } mm_match_t;
 
 struct mm_tbuf_s {
-	sdust_buf_t *sdb;
-	mm128_v mini;
 	void *km;
 };
 
@@ -67,27 +146,26 @@ mm_tbuf_t *mm_tbuf_init(void)
 	mm_tbuf_t *b;
 	b = (mm_tbuf_t*)calloc(1, sizeof(mm_tbuf_t));
 	if (!(mm_dbg_flag & 1)) b->km = km_init();
-	b->sdb = sdust_buf_init(b->km);
 	return b;
 }
 
 void mm_tbuf_destroy(mm_tbuf_t *b)
 {
 	if (b == 0) return;
-	kfree(b->km, b->mini.a);
-	sdust_buf_destroy(b->sdb);
 	km_destroy(b->km);
 	free(b);
 }
 
-static void mm_dust_minier(mm128_v *mini, int l_seq, const char *seq, int sdust_thres, sdust_buf_t *sdb)
+static int mm_dust_minier(void *km, int n, mm128_t *a, int l_seq, const char *seq, int sdust_thres)
 {
 	int n_dreg, j, k, u = 0;
 	const uint64_t *dreg;
-	if (sdust_thres <= 0 || sdb == 0) return;
+	sdust_buf_t *sdb;
+	if (sdust_thres <= 0) return n;
+	sdb = sdust_buf_init(km);
 	dreg = sdust_core((const uint8_t*)seq, l_seq, sdust_thres, 64, &n_dreg, sdb);
-	for (j = k = 0; j < mini->n; ++j) { // squeeze out minimizers that significantly overlap with LCRs
-		int32_t qpos = (uint32_t)mini->a[j].y>>1, span = mini->a[j].x&0xff;
+	for (j = k = 0; j < n; ++j) { // squeeze out minimizers that significantly overlap with LCRs
+		int32_t qpos = (uint32_t)a[j].y>>1, span = a[j].x&0xff;
 		int32_t s = qpos - (span - 1), e = s + span;
 		while (u < n_dreg && (uint32_t)dreg[u] <= s) ++u;
 		if (u < n_dreg && dreg[u]>>32 < e) {
@@ -97,196 +175,235 @@ static void mm_dust_minier(mm128_v *mini, int l_seq, const char *seq, int sdust_
 				int ee = e < (uint32_t)dreg[v]? e : (uint32_t)dreg[v];
 				l += ee - ss;
 			}
-			if (l <= span>>1) mini->a[k++] = mini->a[j]; // keep the minimizer if less than half of it falls in masked region
-		}
+			if (l <= span>>1) a[k++] = a[j]; // keep the minimizer if less than half of it falls in masked region
+		} else a[k++] = a[j];
 	}
-	mini->n = k;
-}
-#if 0
-int mm_pair_thin_core(mm_tbuf_t *b, uint64_t x, int radius, int rel, int st0, int n, const uint64_t *z, uint64_v *a)
-{
-	int i, st = st0, en = n, mid = en - 1;
-	while (st < en) {
-		uint64_t y;
-		mid = st + ((en - st) >> 1);
-		y = z[mid];
-		if (y < x && (x - y)>>1 > radius) st = mid + 1;
-		else if (y >= x && (y - x)>>1 > radius) en = mid;
-		else break;
-	}
-	if (st < en) {
-		for (en = mid + 1; en < n; ++en)
-			if (z[en] > x && (z[en] - x)>>1 > radius)
-				break;
-		for (st = mid - 1; st >= st0; --st)
-			if (z[st] < x && (x - z[st])>>1 > radius)
-				break;
-		++st;
-		for (i = st; i < en; ++i) {
-			uint64_t y = z[i];
-			if (((x ^ y) & 1) == rel) {
-//				printf("* %d,%d\n", (uint32_t)x>>1, (uint32_t)y>>1);
-				kv_push(uint64_t, b->km, *a, y);
-			}
-		}
-		return en;
-	} else return st < n && z[st] < x? st + 1 : en;
+	sdust_buf_destroy(sdb);
+	return k; // the new size
 }
 
-void mm_pair_thin(mm_tbuf_t *b, int radius, mm_match_t *m1, mm_match_t *m2)
+static void collect_minimizers(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int n_segs, const int *qlens, const char **seqs, mm128_v *mv)
 {
-	mm_match_t *m[2];
-	const uint64_t *z[2];
-	uint64_v a[2];
-	int i, n[2], k[2], u = 0, rel = (m1->qpos ^ m2->qpos) & 1;
-
-	m[0] = m1, m[1] = m2;
-	for (i = 0; i < 2; ++i) {
-		n[i] = m[i]->n;
-		z[i] = m[i]->x.cr;
-		k[i] = 0;
-		kv_init(a[i]);
-		kv_resize(uint64_t, b->km, a[i], 256);
+	int i, j, n, sum = 0;
+	mv->n = 0;
+	for (i = n = 0; i < n_segs; ++i) {
+		mm_sketch(km, seqs[i], qlens[i], mi->w, mi->k, i, mi->flag&MM_I_HPC, mv);
+		for (j = n; j < mv->n; ++j)
+			mv->a[j].y += sum << 1;
+		if (opt->sdust_thres > 0) // mask low-complexity minimizers
+			mv->n = n + mm_dust_minier(km, mv->n - n, mv->a + n, qlens[i], seqs[i], opt->sdust_thres);
+		sum += qlens[i], n = mv->n;
 	}
-	while (k[0] < n[0] && k[1] < n[1]) {
-		//printf("%d; %d,%d\n", u, k[0], k[1]);
-		int v = u^1, dist = (int)(m[v]->qpos>>1) - (int)(m[u]->qpos>>1);
-		uint64_t x = z[u][k[u]];
-		int uori = (x ^ m[u]->qpos) & 1, last;
-		int64_t tpos = x>>1 & 0x7fffffff;
-		tpos = uori == 0? tpos + dist : tpos - dist;
-		if (tpos < 0) tpos = 0;
-		x = x>>32<<32 | tpos<<1 | (x&1);
-		last = a[v].n;
-		k[v] = mm_pair_thin_core(b, x, radius, rel, k[v], n[v], z[v], &a[v]);
-		if (a[v].n > last) kv_push(uint64_t, b->km, a[u], z[u][k[u]]);
-		++k[u];
-		u ^= 1;
-	}
-	for (i = 0; i < 2; ++i)
-		m[i]->n = a[i].n, m[i]->x.r = a[i].a, m[i]->is_alloc = 1;
-//	printf("%d,%d; %d,%d\n", m[0]->qpos>>1, m[1]->qpos>>1, m[0]->n, m[1]->n);
 }
-#endif
-mm_reg1_t *mm_map_frag(const mm_mapopt_t *opt, const mm_idx_t *mi, mm_tbuf_t *b, uint32_t m_st, uint32_t m_en, const char *qname, int qlen, const char *seq, int *n_regs)
+
+static mm128_t *collect_seed_hits(void *km, const mm_mapopt_t *opt, int max_occ, const mm_idx_t *mi, const char *qname, const mm128_v *mv, int qlen, int64_t *n_a, int *rep_len,
+								  int *n_mini_pos, uint64_t **mini_pos)
 {
-	int i, n = m_en - m_st, j, n_u, max_gap_ref;
-	int64_t n_a;
-	uint64_t *u;
+	int rep_st = 0, rep_en = 0, i;
 	mm_match_t *m;
 	mm128_t *a;
-	mm_reg1_t *regs;
 
-	// convert to local representation
-	m = (mm_match_t*)kmalloc(b->km, n * sizeof(mm_match_t));
-	for (i = 0; i < n; ++i) {
+	*n_mini_pos = 0;
+	*mini_pos = (uint64_t*)kmalloc(km, mv->n * sizeof(uint64_t));
+	m = (mm_match_t*)kmalloc(km, mv->n * sizeof(mm_match_t));
+	for (i = 0; i < mv->n; ++i) {
 		int t;
-		mm128_t *p = &b->mini.a[i + m_st];
-		m[i].is_alloc = 0;
+		mm128_t *p = &mv->a[i];
 		m[i].qpos = (uint32_t)p->y;
-		m[i].x.cr = mm_idx_get(mi, p->x>>8, &t);
+		m[i].cr = mm_idx_get(mi, p->x>>8, &t);
 		m[i].n = t;
+		m[i].seg_id = p->y >> 32;
 	}
-#if 0
-	int last = -1, last2 = -1;
-	// pair k-mer thinning
-	for (i = 0; i < n; ++i) {
-		if (m[i].n >= opt->mid_occ && m[i].n < opt->max_occ) {
-			if (last2 < 0) last2 = i;
-			if (last < 0 || m[last].n < m[i].n) last = i;
-			if (last >= 0 && (m[last].qpos>>1) + (m[last].span>>1) <= m[i].qpos>>1) {
-				mm_pair_thin(b, opt->bw, &m[last], &m[i]);
-				last2 = last = -1;
-			} else if (last2 >= 0 && (m[last2].qpos>>1) + (m[last2].span>>1) <= m[i].qpos>>1) {
-				mm_pair_thin(b, opt->bw, &m[last2], &m[i]);
-				last2 = last = -1;
-			}
-		}
-	}
-#endif
-	// fill the _a_ array
-	for (i = 0, n_a = 0; i < n; ++i) // find the length of a[]
-		if (m[i].n < opt->mid_occ) n_a += m[i].n;
-	a = (mm128_t*)kmalloc(b->km, n_a * sizeof(mm128_t));
-	for (i = j = 0; i < n; ++i) {
-		mm128_t *p = &b->mini.a[i + m_st];
+	for (i = 0, *n_a = 0; i < mv->n; ++i) // find the length of a[]
+		if (m[i].n < max_occ) *n_a += m[i].n;
+	a = (mm128_t*)kmalloc(km, *n_a * sizeof(mm128_t));
+	for (i = *rep_len = 0, *n_a = 0; i < mv->n; ++i) {
+		mm128_t *p = &mv->a[i];
 		mm_match_t *q = &m[i];
-		const uint64_t *r = q->x.cr;
+		const uint64_t *r = q->cr;
 		int k, q_span = p->x & 0xff, is_tandem = 0;
-		if (q->n >= opt->mid_occ) continue;
-		if (i > 0 && p->x>>8 == b->mini.a[m_st + i - 1].x>>8) is_tandem = 1;
-		if (i < n - 1 && p->x>>8 == b->mini.a[m_st + i + 1].x>>8) is_tandem = 1;
+		if (q->n >= max_occ) {
+			int en = (q->qpos>>1) + 1, st = en - q_span;
+			if (st > rep_en) {
+				*rep_len += rep_en - rep_st;
+				rep_st = st, rep_en = en;
+			} else rep_en = en;
+			continue;
+		}
+		(*mini_pos)[(*n_mini_pos)++] = (uint64_t)q_span<<32 | q->qpos>>1;
+		if (i > 0 && p->x>>8 == mv->a[i - 1].x>>8) is_tandem = 1;
+		if (i < mv->n - 1 && p->x>>8 == mv->a[i + 1].x>>8) is_tandem = 1;
 		for (k = 0; k < q->n; ++k) {
-			const char *tname = mi->seq[r[k]>>32].name;
 			int32_t rpos = (uint32_t)r[k] >> 1;
 			mm128_t *p;
-			if (qname && (opt->flag&MM_F_NO_SELF) && strcmp(qname, tname) == 0 && rpos == (q->qpos>>1)) // avoid the diagonal
-				continue;
-			if (qname && (opt->flag&MM_F_AVA) && strcmp(qname, tname) > 0) // all-vs-all mode: map once
-				continue;
-			p = &a[j++];
+			if (qname && (opt->flag&(MM_F_NO_SELF|MM_F_AVA))) {
+				const char *tname = mi->seq[r[k]>>32].name;
+				int cmp;
+				cmp = strcmp(qname, tname);
+				if ((opt->flag&MM_F_NO_SELF) && cmp == 0 && rpos == (q->qpos>>1)) // avoid the diagonal
+					continue;
+				if ((opt->flag&MM_F_AVA) && cmp > 0) // all-vs-all mode: map once
+					continue;
+			}
+			p = &a[(*n_a)++];
 			if ((r[k]&1) == (q->qpos&1)) { // forward strand
-				p->x = (r[k]&0xffffffff00000000ULL) | (uint32_t)r[k]>>1;
+				p->x = (r[k]&0xffffffff00000000ULL) | rpos;
 				p->y = (uint64_t)q_span << 32 | q->qpos >> 1;
 			} else { // reverse strand
-				p->x = 1ULL<<63 | (r[k]&0xffffffff00000000ULL) | (uint32_t)r[k]>>1;
+				p->x = 1ULL<<63 | (r[k]&0xffffffff00000000ULL) | rpos;
 				p->y = (uint64_t)q_span << 32 | (qlen - ((q->qpos>>1) + 1 - q_span) - 1);
 			}
+			p->y |= (uint64_t)q->seg_id << MM_SEED_SEG_SHIFT;
 			if (is_tandem) p->y |= MM_SEED_TANDEM;
 		}
 	}
-	n_a = j;
-	radix_sort_128x(a, a + n_a);
-	for (i = 0; i < n; ++i)
-		if (m[i].is_alloc) kfree(b->km, m[i].x.r);
-	kfree(b->km, m);
-
-	if (mm_dbg_flag & MM_DBG_PRINT_SEED)
-		for (i = 0; i < n_a; ++i)
-			fprintf(stderr, "SD\t%s\t%d\t%c\t%d\t%d\t%d\n", mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
-					i == 0? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
-
-	max_gap_ref = opt->max_gap_ref >= 0? opt->max_gap_ref : opt->max_gap;
-	n_u = mm_chain_dp(max_gap_ref, opt->max_gap, opt->bw, opt->max_chain_skip, opt->min_cnt, opt->min_chain_score, !!(opt->flag&MM_F_SPLICE), n_a, a, &u, b->km);
-	regs = mm_gen_regs(b->km, qlen, n_u, u, a);
-	*n_regs = n_u;
-
-	if (mm_dbg_flag & MM_DBG_PRINT_SEED)
-		for (j = 0; j < n_u; ++j)
-			for (i = regs[j].as; i < regs[j].as + regs[j].cnt; ++i)
-				fprintf(stderr, "CN\t%d\t%s\t%d\t%c\t%d\t%d\t%d\n", j, mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
-						i == regs[j].as? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
+	*rep_len += rep_en - rep_st;
+	kfree(km, m);
+	return a;
+}
 
+static void chain_post(const mm_mapopt_t *opt, int max_chain_gap_ref, const mm_idx_t *mi, void *km, int qlen, int n_segs, const int *qlens, int *n_regs, mm_reg1_t *regs, mm128_t *a)
+{
 	if (!(opt->flag & MM_F_AVA)) { // don't choose primary mapping(s) for read overlap
-		mm_set_parent(b->km, opt->mask_level, *n_regs, regs);
-		mm_select_sub(b->km, opt->mask_level, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
-		if (!(opt->flag & MM_F_SPLICE))
-			mm_join_long(b->km, opt, qlen, n_regs, regs, a); // TODO: this can be applied to all-vs-all in principle
+		mm_set_parent(km, opt->mask_level, *n_regs, regs, opt->a * 2 + opt->b);
+		if (n_segs <= 1) mm_select_sub(km, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
+		else mm_select_sub_multi(km, opt->pri_ratio, 0.2f, 0.7f, max_chain_gap_ref, mi->k*2, opt->best_n, n_segs, qlens, n_regs, regs);
+		if (!(opt->flag & MM_F_SPLICE) && !(opt->flag & MM_F_SR) && !(opt->flag & MM_F_NO_LJOIN))
+			mm_join_long(km, opt, qlen, n_regs, regs, a);
 	}
-	if (opt->flag & MM_F_CIGAR) {
-		regs = mm_align_skeleton(b->km, opt, mi, qlen, seq, n_regs, regs, a); // this calls mm_filter_regs()
-		if (!(opt->flag & MM_F_AVA)) {
-			mm_set_parent(b->km, opt->mask_level, *n_regs, regs);
-			mm_select_sub(b->km, opt->mask_level, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
-			mm_set_sam_pri(*n_regs, regs);
-		}
-	}
-	mm_set_mapq(*n_regs, regs, opt->min_chain_score);
+}
 
-	// free
-	kfree(b->km, a);
-	kfree(b->km, u);
+static mm_reg1_t *align_regs(const mm_mapopt_t *opt, const mm_idx_t *mi, void *km, int qlen, const char *seq, const char *qual, int *n_regs, mm_reg1_t *regs, mm128_t *a)
+{
+	if (!(opt->flag & MM_F_CIGAR)) return regs;
+	regs = mm_align_skeleton(km, opt, mi, qlen, seq, n_regs, regs, a); // this calls mm_filter_regs()
+	if (!(opt->flag & MM_F_AVA)) {
+		mm_set_parent(km, opt->mask_level, *n_regs, regs, opt->a * 2 + opt->b);
+		mm_select_sub(km, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
+		mm_set_sam_pri(*n_regs, regs);
+	}
 	return regs;
 }
 
-mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
+void mm_map_frag(const mm_idx_t *mi, int n_segs, const int *qlens, const char **seqs, const char **quals, int *n_regs, mm_reg1_t **regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
+{
+	int i, j, rep_len, qlen_sum, n_regs0, n_mini_pos;
+	int max_chain_gap_qry, max_chain_gap_ref, is_splice = !!(opt->flag & MM_F_SPLICE), is_sr = !!(opt->flag & MM_F_SR);
+	uint32_t hash;
+	int64_t n_a;
+	uint64_t *u, *mini_pos;
+	mm128_t *a;
+	mm128_v mv = {0,0,0};
+	mm_reg1_t *regs0;
+	km_stat_t kmst;
+
+	for (i = 0, qlen_sum = 0; i < n_segs; ++i)
+		qlen_sum += qlens[i], n_regs[i] = 0, regs[i] = 0;
+
+	if (qlen_sum == 0 || n_segs <= 0 || n_segs > MM_MAX_SEG) return;
+
+	hash  = qname? __ac_X31_hash_string(qname) : 0;
+	hash ^= __ac_Wang_hash(qlen_sum) + __ac_Wang_hash(opt->seed);
+	hash  = __ac_Wang_hash(hash);
+
+	collect_minimizers(b->km, opt, mi, n_segs, qlens, seqs, &mv);
+	a = collect_seed_hits(b->km, opt, opt->mid_occ, mi, qname, &mv, qlen_sum, &n_a, &rep_len, &n_mini_pos, &mini_pos);
+	radix_sort_128x(a, a + n_a);
+
+	if (mm_dbg_flag & MM_DBG_PRINT_SEED) {
+		fprintf(stderr, "RS\t%d\n", rep_len);
+		for (i = 0; i < n_a; ++i)
+			fprintf(stderr, "SD\t%s\t%d\t%c\t%d\t%d\t%d\n", mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
+					i == 0? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
+	}
+
+	// set max chaining gap on the query and the reference sequence
+	if (is_sr)
+		max_chain_gap_qry = qlen_sum > opt->max_gap? qlen_sum : opt->max_gap;
+	else max_chain_gap_qry = opt->max_gap;
+	if (opt->max_gap_ref > 0) {
+		max_chain_gap_ref = opt->max_gap_ref; // always honor mm_mapopt_t::max_gap_ref if set
+	} else if (opt->max_frag_len > 0) {
+		max_chain_gap_ref = opt->max_frag_len - qlen_sum;
+		if (max_chain_gap_ref < opt->max_gap) max_chain_gap_ref = opt->max_gap;
+	} else max_chain_gap_ref = opt->max_gap;
+
+	a = mm_chain_dp(max_chain_gap_ref, max_chain_gap_qry, opt->bw, opt->max_chain_skip, opt->min_cnt, opt->min_chain_score, is_splice, n_segs, n_a, a, &n_regs0, &u, b->km);
+
+	if (opt->max_occ > opt->mid_occ && rep_len > 0) {
+		int rechain = 0;
+		if (n_regs0 > 0) { // test if the best chain has all the segments
+			int n_chained_segs = 1, max = 0, max_i = -1, max_off = -1, off = 0;
+			for (i = 0; i < n_regs0; ++i) { // find the best chain
+				if (max < u[i]>>32) max = u[i]>>32, max_i = i, max_off = off;
+				off += (uint32_t)u[i];
+			}
+			for (i = 1; i < (uint32_t)u[max_i]; ++i) // count the number of segments in the best chain
+				if ((a[max_off+i].y&MM_SEED_SEG_MASK) != (a[max_off+i-1].y&MM_SEED_SEG_MASK))
+					++n_chained_segs;
+			if (n_chained_segs < n_segs)
+				rechain = 1;
+		} else rechain = 1;
+		if (rechain) { // redo chaining with a higher max_occ threshold
+			kfree(b->km, a);
+			kfree(b->km, u);
+			kfree(b->km, mini_pos);
+			a = collect_seed_hits(b->km, opt, opt->max_occ, mi, qname, &mv, qlen_sum, &n_a, &rep_len, &n_mini_pos, &mini_pos);
+			radix_sort_128x(a, a + n_a);
+			a = mm_chain_dp(max_chain_gap_ref, max_chain_gap_qry, opt->bw, opt->max_chain_skip, opt->min_cnt, opt->min_chain_score, is_splice, n_segs, n_a, a, &n_regs0, &u, b->km);
+		}
+	}
+
+	regs0 = mm_gen_regs(b->km, hash, qlen_sum, n_regs0, u, a);
+
+	if (mm_dbg_flag & MM_DBG_PRINT_SEED)
+		for (j = 0; j < n_regs0; ++j)
+			for (i = regs0[j].as; i < regs0[j].as + regs0[j].cnt; ++i)
+				fprintf(stderr, "CN\t%d\t%s\t%d\t%c\t%d\t%d\t%d\n", j, mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
+						i == regs0[j].as? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
+
+	chain_post(opt, max_chain_gap_ref, mi, b->km, qlen_sum, n_segs, qlens, &n_regs0, regs0, a);
+	if (!is_sr) mm_est_err(mi, qlen_sum, n_regs0, regs0, a, n_mini_pos, mini_pos);
+
+	if (n_segs == 1) { // uni-segment
+		regs0 = align_regs(opt, mi, b->km, qlens[0], seqs[0], quals? quals[0] : 0, &n_regs0, regs0, a);
+		mm_set_mapq(n_regs0, regs0, opt->min_chain_score, opt->a, rep_len, is_sr);
+		n_regs[0] = n_regs0, regs[0] = regs0;
+	} else { // multi-segment
+		mm_seg_t *seg;
+		seg = mm_seg_gen(b->km, hash, n_segs, qlens, n_regs0, regs0, n_regs, regs, a); // split fragment chain to separate segment chains
+		free(regs0);
+		for (i = 0; i < n_segs; ++i) {
+			mm_set_parent(b->km, opt->mask_level, n_regs[i], regs[i], opt->a * 2 + opt->b); // update mm_reg1_t::parent
+			regs[i] = align_regs(opt, mi, b->km, qlens[i], seqs[i], quals? quals[i] : 0, &n_regs[i], regs[i], seg[i].a);
+			mm_set_mapq(n_regs[i], regs[i], opt->min_chain_score, opt->a, rep_len, is_sr);
+		}
+		mm_seg_free(b->km, n_segs, seg);
+		if (n_segs == 2 && opt->pe_ori >= 0 && (opt->flag&MM_F_CIGAR))
+			mm_pair(b->km, max_chain_gap_ref, opt->pe_bonus, opt->a * 2 + opt->b, opt->a, qlens, n_regs, regs); // pairing
+	}
+
+	kfree(b->km, mv.a);
+	kfree(b->km, a);
+	kfree(b->km, u);
+	kfree(b->km, mini_pos);
+
+	if (b->km) {
+		km_stat(b->km, &kmst);
+		if (mm_dbg_flag & MM_DBG_PRINT_QNAME)
+			fprintf(stderr, "QM\t%s\t%d\tcap=%ld,nCore=%ld,largest=%ld\n", qname, qlen_sum, kmst.capacity, kmst.n_cores, kmst.largest);
+		assert(kmst.n_blocks == kmst.n_cores); // otherwise, there is a memory leak
+		if (kmst.largest > 1U<<28) {
+			km_destroy(b->km);
+			b->km = km_init();
+		}
+	}
+}
+
+mm_reg1_t *mm_map(const mm_idx_t *mi, int qlen, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
 {
 	mm_reg1_t *regs;
-	b->mini.n = 0;
-	mm_sketch(b->km, seq, l_seq, mi->w, mi->k, 0, mi->is_hpc, &b->mini);
-	if (opt->sdust_thres > 0)
-		mm_dust_minier(&b->mini, l_seq, seq, opt->sdust_thres, b->sdb);
-	regs = mm_map_frag(opt, mi, b, 0, b->mini.n, qname, l_seq, seq, n_regs);
+	mm_map_frag(mi, 1, &qlen, &seq, 0, n_regs, &regs, b, opt, qname);
 	return regs;
 }
 
@@ -295,39 +412,70 @@ mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, m
  **************************/
 
 typedef struct {
-	int mini_batch_size, n_processed, n_threads;
+	int mini_batch_size, n_processed, n_threads, n_fp;
 	const mm_mapopt_t *opt;
-	mm_bseq_file_t *fp;
+	mm_bseq_file_t **fp;
 	const mm_idx_t *mi;
 	kstring_t str;
 } pipeline_t;
 
 typedef struct {
 	const pipeline_t *p;
-    int n_seq;
+    int n_seq, n_frag;
 	mm_bseq1_t *seq;
-	int *n_reg;
+	int *n_reg, *seg_off, *n_seg;
 	mm_reg1_t **reg;
 	mm_tbuf_t **buf;
 } step_t;
 
 static void worker_for(void *_data, long i, int tid) // kt_for() callback
 {
-    step_t *step = (step_t*)_data;
+    step_t *s = (step_t*)_data;
+	int qlens[MM_MAX_SEG], j, off = s->seg_off[i], pe_ori = s->p->opt->pe_ori, is_sr = !!(s->p->opt->flag & MM_F_SR);
+	const char *qseqs[MM_MAX_SEG], *quals[MM_MAX_SEG];
+	mm_tbuf_t *b = s->buf[tid];
+	assert(s->n_seg[i] <= MM_MAX_SEG);
+	memset(quals, 0, sizeof(char*) * MM_MAX_SEG);
 	if (mm_dbg_flag & MM_DBG_PRINT_QNAME)
-		fprintf(stderr, "QR\t%s\t%d\n", step->seq[i].name, tid);
-	step->reg[i] = mm_map(step->p->mi, step->seq[i].l_seq, step->seq[i].seq, &step->n_reg[i], step->buf[tid], step->p->opt, step->seq[i].name);
+		fprintf(stderr, "QR\t%s\t%d\t%d\n", s->seq[off].name, tid, s->seq[off].l_seq);
+	for (j = 0; j < s->n_seg[i]; ++j) {
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1))))
+			mm_revcomp_bseq(&s->seq[off + j]);
+		qlens[j] = s->seq[off + j].l_seq;
+		qseqs[j] = s->seq[off + j].seq;
+		quals[j] = is_sr? s->seq[off + j].qual : 0;
+	}
+	if (s->p->opt->flag & MM_F_INDEPEND_SEG) {
+		for (j = 0; j < s->n_seg[i]; ++j)
+			mm_map_frag(s->p->mi, 1, &qlens[j], &qseqs[j], &quals[j], &s->n_reg[off+j], &s->reg[off+j], b, s->p->opt, s->seq[off+j].name);
+	} else {
+		mm_map_frag(s->p->mi, s->n_seg[i], qlens, qseqs, quals, &s->n_reg[off], &s->reg[off], b, s->p->opt, s->seq[off].name);
+	}
+	for (j = 0; j < s->n_seg[i]; ++j) // flip the query strand and coordinate to the original read strand
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1)))) {
+			int k, t;
+			mm_revcomp_bseq(&s->seq[off + j]);
+			for (k = 0; k < s->n_reg[off + j]; ++k) {
+				mm_reg1_t *r = &s->reg[off + j][k];
+				t = r->qs;
+				r->qs = qlens[j] - r->qe;
+				r->qe = qlens[j] - t;
+				r->rev = !r->rev;
+			}
+		}
 }
 
 static void *worker_pipeline(void *shared, int step, void *in)
 {
-	int i, j;
+	int i, j, k;
     pipeline_t *p = (pipeline_t*)shared;
     if (step == 0) { // step 0: read sequences
 		int with_qual = (!!(p->opt->flag & MM_F_OUT_SAM) && !(p->opt->flag & MM_F_NO_QUAL));
+		int frag_mode = (p->n_fp > 1 || !!(p->opt->flag & MM_F_FRAG_MODE));
         step_t *s;
         s = (step_t*)calloc(1, sizeof(step_t));
-		s->seq = mm_bseq_read(p->fp, p->mini_batch_size, with_qual, &s->n_seq);
+		if (p->n_fp > 1) s->seq = mm_bseq_read_frag(p->n_fp, p->fp, p->mini_batch_size, with_qual, &s->n_seq);
+		else s->seq = mm_bseq_read2(p->fp[0], p->mini_batch_size, with_qual, frag_mode, &s->n_seq);
 		if (s->seq) {
 			s->p = p;
 			for (i = 0; i < s->n_seq; ++i)
@@ -335,12 +483,20 @@ static void *worker_pipeline(void *shared, int step, void *in)
 			s->buf = (mm_tbuf_t**)calloc(p->n_threads, sizeof(mm_tbuf_t*));
 			for (i = 0; i < p->n_threads; ++i)
 				s->buf[i] = mm_tbuf_init();
-			s->n_reg = (int*)calloc(s->n_seq, sizeof(int));
+			s->n_reg = (int*)calloc(3 * s->n_seq, sizeof(int));
+			s->seg_off = s->n_reg + s->n_seq; // seg_off and n_seg are allocated together with n_reg
+			s->n_seg = s->seg_off + s->n_seq;
 			s->reg = (mm_reg1_t**)calloc(s->n_seq, sizeof(mm_reg1_t*));
+			for (i = 1, j = 0; i <= s->n_seq; ++i)
+				if (i == s->n_seq || !frag_mode || !mm_qname_same(s->seq[i-1].name, s->seq[i].name)) {
+					s->n_seg[s->n_frag] = i - j;
+					s->seg_off[s->n_frag++] = j;
+					j = i;
+				}
 			return s;
 		} else free(s);
     } else if (step == 1) { // step 1: map
-		kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_seq);
+		kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_frag);
 		return in;
     } else if (step == 2) { // step 2: output
 		void *km = 0;
@@ -349,26 +505,34 @@ static void *worker_pipeline(void *shared, int step, void *in)
 		for (i = 0; i < p->n_threads; ++i) mm_tbuf_destroy(s->buf[i]);
 		free(s->buf);
 		if ((p->opt->flag & MM_F_OUT_CS) && !(mm_dbg_flag & MM_DBG_NO_KALLOC)) km = km_init();
-		for (i = 0; i < s->n_seq; ++i) {
-			mm_bseq1_t *t = &s->seq[i];
-			for (j = 0; j < s->n_reg[i]; ++j) {
-				mm_reg1_t *r = &s->reg[i][j];
-				if (p->opt->flag & MM_F_OUT_SAM)
-					mm_write_sam(&p->str, mi, t, r, s->n_reg[i], s->reg[i]);
-				else
-					mm_write_paf(&p->str, mi, t, r, km, p->opt->flag);
-				puts(p->str.s);
+		for (k = 0; k < s->n_frag; ++k) {
+			int seg_st = s->seg_off[k], seg_en = s->seg_off[k] + s->n_seg[k];
+			for (i = seg_st; i < seg_en; ++i) {
+				mm_bseq1_t *t = &s->seq[i];
+				for (j = 0; j < s->n_reg[i]; ++j) {
+					mm_reg1_t *r = &s->reg[i][j];
+					assert(!r->sam_pri || r->id == r->parent);
+					if ((p->opt->flag & MM_F_NO_PRINT_2ND) && r->id != r->parent)
+						continue;
+					if (p->opt->flag & MM_F_OUT_SAM)
+						mm_write_sam2(&p->str, mi, t, i - seg_st, j, s->n_seg[k], &s->n_reg[seg_st], (const mm_reg1_t*const*)&s->reg[seg_st], km, p->opt->flag);
+					else
+						mm_write_paf(&p->str, mi, t, r, km, p->opt->flag);
+					puts(p->str.s);
+				}
+				if (s->n_reg[i] == 0 && (p->opt->flag & MM_F_OUT_SAM)) {
+					mm_write_sam2(&p->str, mi, t, i - seg_st, -1, s->n_seg[k], &s->n_reg[seg_st], (const mm_reg1_t*const*)&s->reg[seg_st], km, p->opt->flag);
+					puts(p->str.s);
+				}
 			}
-			if (s->n_reg[i] == 0 && (p->opt->flag & MM_F_OUT_SAM)) {
-				mm_write_sam(&p->str, 0, t, 0, 0, 0);
-				puts(p->str.s);
+			for (i = seg_st; i < seg_en; ++i) {
+				for (j = 0; j < s->n_reg[i]; ++j) free(s->reg[i][j].p);
+				free(s->reg[i]);
+				free(s->seq[i].seq); free(s->seq[i].name);
+				if (s->seq[i].qual) free(s->seq[i].qual);
 			}
-			for (j = 0; j < s->n_reg[i]; ++j) free(s->reg[i][j].p);
-			free(s->reg[i]);
-			free(s->seq[i].seq); free(s->seq[i].name);
-			if (s->seq[i].qual) free(s->seq[i].qual);
 		}
-		free(s->reg); free(s->n_reg); free(s->seq);
+		free(s->reg); free(s->n_reg); free(s->seq); // seg_off and n_seg were allocated with reg; no memory leak here
 		km_destroy(km);
 		if (mm_verbose >= 3)
 			fprintf(stderr, "[M::%s::%.3f*%.2f] mapped %d sequences\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), s->n_seq);
@@ -377,18 +541,38 @@ static void *worker_pipeline(void *shared, int step, void *in)
     return 0;
 }
 
-int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads, int mini_batch_size)
+int mm_map_file_frag(const mm_idx_t *idx, int n_segs, const char **fn, const mm_mapopt_t *opt, int n_threads)
 {
+	int i, j, pl_threads;
 	pipeline_t pl;
+	if (n_segs < 1) return -1;
 	memset(&pl, 0, sizeof(pipeline_t));
-	pl.fp = mm_bseq_open(fn);
-	if (pl.fp == 0) return -1;
+	pl.n_fp = n_segs;
+	pl.fp = (mm_bseq_file_t**)calloc(n_segs, sizeof(mm_bseq_file_t*));
+	for (i = 0; i < n_segs; ++i) {
+		pl.fp[i] = mm_bseq_open(fn[i]);
+		if (pl.fp[i] == 0) {
+			if (mm_verbose >= 1)
+				fprintf(stderr, "ERROR: failed to open file '%s'\n", fn[i]);
+			for (j = 0; j < i; ++j)
+				mm_bseq_close(pl.fp[j]);
+			free(pl.fp);
+			return -1;
+		}
+	}
 	pl.opt = opt, pl.mi = idx;
-	pl.n_threads = n_threads, pl.mini_batch_size = mini_batch_size;
-	if ((opt->flag & MM_F_OUT_SAM) && !(opt->flag & MM_F_NO_SAM_SQ))
-		mm_write_sam_SQ(idx);
-	kt_pipeline(n_threads == 1? 1 : 2, worker_pipeline, &pl, 3);
+	pl.n_threads = n_threads > 1? n_threads : 1;
+	pl.mini_batch_size = opt->mini_batch_size;
+	pl_threads = n_threads == 1? 1 : (opt->flag&MM_F_2_IO_THREADS)? 3 : 2;
+	kt_pipeline(pl_threads, worker_pipeline, &pl, 3);
 	free(pl.str.s);
-	mm_bseq_close(pl.fp);
+	for (i = 0; i < n_segs; ++i)
+		mm_bseq_close(pl.fp[i]);
+	free(pl.fp);
 	return 0;
 }
+
+int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads)
+{
+	return mm_map_file_frag(idx, 1, &fn, opt, n_threads);
+}

minimap.h

@@ -5,8 +5,6 @@
 #include <stdio.h>
 #include <sys/types.h>
 
-#define MM_IDX_DEF_B     14
-
 #define MM_F_NO_SELF     0x001
 #define MM_F_AVA         0x002
 #define MM_F_CIGAR       0x004
@@ -14,33 +12,37 @@
 #define MM_F_NO_QUAL     0x010
 #define MM_F_OUT_CG      0x020
 #define MM_F_OUT_CS      0x040
-#define MM_F_SPLICE      0x080
-#define MM_F_SPLICE_FOR  0x100
-#define MM_F_SPLICE_REV  0x200
-#define MM_F_SPLICE_BOTH 0x400
-#define MM_F_NO_SAM_SQ   0x800
+#define MM_F_SPLICE      0x080 // splice mode
+#define MM_F_SPLICE_FOR  0x100 // match GT-AG
+#define MM_F_SPLICE_REV  0x200 // match CT-AC, the reverse complement of GT-AG
+#define MM_F_NO_LJOIN    0x400
+#define MM_F_OUT_CS_LONG 0x800
+#define MM_F_SR          0x1000
+#define MM_F_FRAG_MODE   0x2000
+#define MM_F_NO_PRINT_2ND  0x4000
+#define MM_F_2_IO_THREADS  0x8000
+#define MM_F_LONG_CIGAR    0x10000
+#define MM_F_INDEPEND_SEG  0x20000
+#define MM_F_SPLICE_FLANK  0x40000
+#define MM_F_SOFTCLIP      0x80000
+
+#define MM_I_HPC          0x1
+#define MM_I_NO_SEQ       0x2
+#define MM_I_NO_NAME      0x4
 
 #define MM_IDX_MAGIC   "MMI\2"
 
+#define MM_MAX_SEG       255
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct {
- 	uint64_t x, y;
-} mm128_t;
-
+// emulate 128-bit integers and arrays
+typedef struct { uint64_t x, y; } mm128_t;
 typedef struct { size_t n, m; mm128_t *a; } mm128_v;
-typedef struct { size_t n, m; uint64_t *a; } uint64_v;
-typedef struct { size_t n, m; uint32_t *a; } uint32_v;
-
-typedef struct {
-	mm128_v a;   // (minimizer, position) array
-	int32_t n;   // size of the _p_ array
-	uint64_t *p; // position array for minimizers appearing >1 times
-	void *h;     // hash table indexing _p_ and minimizers appearing once
-} mm_idx_bucket_t;
 
+// minimap2 index
 typedef struct {
 	char *name;      // name of the db sequence
 	uint64_t offset; // offset in mm_idx_t::S
@@ -48,104 +50,249 @@ typedef struct {
 } mm_idx_seq_t;
 
 typedef struct {
-	int32_t b, w, k, is_hpc;
-	uint32_t n_seq;     // number of reference sequences
-	mm_idx_seq_t *seq;  // sequence name, length and offset
-	uint32_t *S;        // 4-bit packed sequence
-	mm_idx_bucket_t *B; // index
+	int32_t b, w, k, flag;
+	uint32_t n_seq;            // number of reference sequences
+	mm_idx_seq_t *seq;         // sequence name, length and offset
+	uint32_t *S;               // 4-bit packed sequence
+	struct mm_idx_bucket_s *B; // index (hidden)
 	void *km;
 } mm_idx_t;
 
+// minimap2 alignment
 typedef struct {
-	uint32_t capacity;
-	int32_t dp_score, dp_max, dp_max2;
-	uint32_t blen;
-	uint32_t n_diff;
-	uint32_t n_ambi:30, trans_strand:2;
-	uint32_t n_cigar;
+	uint32_t capacity;                  // the capacity of cigar[]
+	int32_t dp_score, dp_max, dp_max2;  // DP score; score of the max-scoring segment; score of the best alternate mappings
+	uint32_t n_ambi:30, trans_strand:2; // number of ambiguous bases; transcript strand: 0 for unknown, 1 for +, 2 for -
+	uint32_t n_cigar;                   // number of cigar operations in cigar[]
 	uint32_t cigar[];
 } mm_extra_t;
 
 typedef struct {
-	int32_t id;
-	uint32_t cnt:31, rev:1;
-	uint32_t rid:31, inv:1;
-	int32_t score;
-	int32_t qs, qe, rs, re;
-	int32_t parent, subsc;
-	int32_t as;
-	int32_t fuzzy_mlen, fuzzy_blen;
-	uint32_t mapq:8, split:2, sam_pri:1, n_sub:21; // TODO: n_sub is not used for now
+	int32_t id;             // ID for internal uses (see also parent below)
+	int32_t cnt;            // number of minimizers; if on the reverse strand
+	int32_t rid;            // reference index; if this is an alignment from inversion rescue
+	int32_t score;          // DP alignment score
+	int32_t qs, qe, rs, re; // query start and end; reference start and end
+	int32_t parent, subsc;  // parent==id if primary; best alternate mapping score
+	int32_t as;             // offset in the a[] array (for internal uses only)
+	int32_t mlen, blen;     // seeded exact match length; seeded alignment block length
+	int32_t n_sub;          // number of suboptimal mappings
+	int32_t score0;         // initial chaining score (before chain merging/spliting)
+	uint32_t mapq:8, split:2, rev:1, inv:1, sam_pri:1, proper_frag:1, pe_thru:1, seg_split:1, dummy:16;
+	uint32_t hash;
+	float div;
 	mm_extra_t *p;
 } mm_reg1_t;
 
+// indexing and mapping options
 typedef struct {
-	float max_occ_frac;
-	float mid_occ_frac;
-	int sdust_thres;  // score threshold for SDUST; 0 to disable
-	int flag;    // see MM_F_* macros
+	short k, w, flag, bucket_bits;
+	int mini_batch_size;
+	uint64_t batch_size;
+} mm_idxopt_t;
 
-	int bw;  // bandwidth
+typedef struct {
+	int seed;
+	int sdust_thres; // score threshold for SDUST; 0 to disable
+	int flag;        // see MM_F_* macros
+
+	int bw;          // bandwidth
 	int max_gap, max_gap_ref; // break a chain if there are no minimizers in a max_gap window
+	int max_frag_len;
 	int max_chain_skip;
-	int min_cnt;
-	int min_chain_score;
+	int min_cnt;         // min number of minimizers on each chain
+	int min_chain_score; // min chaining score
 
 	float mask_level;
 	float pri_ratio;
-	int best_n;
+	int best_n;      // top best_n chains are subjected to DP alignment
 
 	int max_join_long, max_join_short;
 	int min_join_flank_sc;
 
 	int a, b, q, e, q2, e2; // matching score, mismatch, gap-open and gap-ext penalties
-	int noncan;
-	int zdrop;
-	int min_dp_max;
+	int noncan;      // cost of non-canonical splicing sites
+	int zdrop;       // break alignment if alignment score drops too fast along the diagonal
+	int end_bonus;
+	int min_dp_max;  // drop an alignment if the score of the max scoring segment is below this threshold
 	int min_ksw_len;
+	int anchor_ext_len, anchor_ext_shift;
 
-	int max_occ;
-	int mid_occ;
+	int pe_ori, pe_bonus;
+
+	float mid_occ_frac;  // only used by mm_mapopt_update(); see below
+	int32_t mid_occ;     // ignore seeds with occurrences above this threshold
+	int32_t max_occ;
+	int mini_batch_size; // size of a batch of query bases to process in parallel
 } mm_mapopt_t;
 
-extern int mm_verbose, mm_dbg_flag;
-extern double mm_realtime0;
+// index reader
+typedef struct {
+	int is_idx, n_parts;
+	int64_t idx_size;
+	mm_idxopt_t opt;
+	FILE *fp_out;
+	union {
+		struct mm_bseq_file_s *seq;
+		FILE *idx;
+	} fp;
+} mm_idx_reader_t;
 
-struct mm_tbuf_s;
+// memory buffer for thread-local storage during mapping
 typedef struct mm_tbuf_s mm_tbuf_t;
 
-struct mm_bseq_file_s;
+// global variables
+extern int mm_verbose, mm_dbg_flag; // verbose level: 0 for no info, 1 for error, 2 for warning, 3 for message (default); debugging flag
+extern double mm_realtime0; // wall-clock timer
 
-#define mm_seq4_set(s, i, c) ((s)[(i)>>3] |= (uint32_t)(c) << (((i)&7)<<2))
-#define mm_seq4_get(s, i)    ((s)[(i)>>3] >> (((i)&7)<<2) & 0xf)
+/**
+ * Set default or preset parameters
+ *
+ * @param preset     NULL to set all parameters as default; otherwise apply preset to affected parameters
+ * @param io         pointer to indexing parameters
+ * @param mo         pointer to mapping parameters
+ *
+ * @return 0 if success; -1 if _present_ unknown
+ */
+int mm_set_opt(const char *preset, mm_idxopt_t *io, mm_mapopt_t *mo);
+int mm_check_opt(const mm_idxopt_t *io, const mm_mapopt_t *mo);
 
-// compute minimizers
-void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p);
-
-// minimizer indexing
-mm_idx_t *mm_idx_init(int w, int k, int b, int is_hpc);
-void mm_idx_destroy(mm_idx_t *mi);
-mm_idx_t *mm_idx_gen(struct mm_bseq_file_s *fp, int w, int k, int b, int is_hpc, int mini_batch_size, int n_threads, uint64_t batch_size, int keep_name);
-uint32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f);
-void mm_idx_stat(const mm_idx_t *idx);
-const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n);
-int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq);
-
-mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads);
-int mm_idx_is_idx(const char *fn);
-
-// minimizer index I/O
-void mm_idx_dump(FILE *fp, const mm_idx_t *mi);
-mm_idx_t *mm_idx_load(FILE *fp);
-
-// mapping
-void mm_mapopt_init(mm_mapopt_t *opt);
+/**
+ * Update mm_mapopt_t::mid_occ via mm_mapopt_t::mid_occ_frac
+ *
+ * If mm_mapopt_t::mid_occ is 0, this function sets it to a number such that no
+ * more than mm_mapopt_t::mid_occ_frac of minimizers in the index have a higher
+ * occurrence.
+ *
+ * @param opt        mapping parameters
+ * @param mi         minimap2 index
+ */
 void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi);
+
+void mm_mapopt_max_intron_len(mm_mapopt_t *opt, int max_intron_len);
+
+/**
+ * Initialize an index reader
+ *
+ * @param fn         index or fasta/fastq file name (this function tests the file type)
+ * @param opt        indexing parameters
+ * @param fn_out     if not NULL, write built index to this file
+ *
+ * @return an index reader on success; NULL if fail to open _fn_
+ */
+mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out);
+
+/**
+ * Read/build an index
+ *
+ * If the input file is an index file, this function reads one part of the
+ * index and returns. If the input file is a sequence file (fasta or fastq),
+ * this function constructs the index for about mm_idxopt_t::batch_size bases.
+ * Importantly, for a huge collection of sequences, this function may only
+ * return an index for part of sequences. It needs to be repeatedly called
+ * to traverse the entire index/sequence file.
+ *
+ * @param r          index reader
+ * @param n_threads  number of threads for constructing index
+ *
+ * @return an index on success; NULL if reaching the end of the input file
+ */
+mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads);
+
+/**
+ * Destroy/deallocate an index reader
+ *
+ * @param r          index reader
+ */
+void mm_idx_reader_close(mm_idx_reader_t *r);
+
+int mm_idx_reader_eof(const mm_idx_reader_t *r);
+
+/**
+ * Create an index from strings in memory
+ *
+ * @param w            minimizer window size
+ * @param k            minimizer k-mer size
+ * @param is_hpc       use HPC k-mer if true
+ * @param bucket_bits  number of bits for the first level of the hash table
+ * @param n            number of sequences
+ * @param seq          sequences in A/C/G/T
+ * @param name         sequence names; could be NULL
+ *
+ * @return minimap2 index
+ */
+mm_idx_t *mm_idx_str(int w, int k, int is_hpc, int bucket_bits, int n, const char **seq, const char **name);
+
+/**
+ * Print index statistics to stderr
+ *
+ * @param mi         minimap2 index
+ */
+void mm_idx_stat(const mm_idx_t *idx);
+
+/**
+ * Destroy/deallocate an index
+ *
+ * @param r          minimap2 index
+ */
+void mm_idx_destroy(mm_idx_t *mi);
+
+/**
+ * Initialize a thread-local buffer for mapping
+ *
+ * Each mapping thread requires a buffer specific to the thread (see mm_map()
+ * below). The primary purpose of this buffer is to reduce frequent heap
+ * allocations across threads. A buffer shall not be used by two or more
+ * threads.
+ *
+ * @return pointer to a thread-local buffer
+ */
 mm_tbuf_t *mm_tbuf_init(void);
+
+/**
+ * Destroy/deallocate a thread-local buffer for mapping
+ *
+ * @param b          the buffer
+ */
 void mm_tbuf_destroy(mm_tbuf_t *b);
+
+/**
+ * Align a query sequence against an index
+ *
+ * This function possibly finds multiple alignments of the query sequence.
+ * The returned array and the mm_reg1_t::p field of each element are allocated
+ * with malloc().
+ *
+ * @param mi         minimap2 index
+ * @param l_seq      length of the query sequence
+ * @param seq        the query sequence
+ * @param n_regs     number of hits (out)
+ * @param b          thread-local buffer; two mm_map() calls shall not use one buffer at the same time!
+ * @param opt        mapping parameters
+ * @param name       query name, used for all-vs-all overlapping and debugging
+ *
+ * @return an array of hits which need to be deallocated with free() together
+ *         with mm_reg1_t::p of each element. The size is written to _n_regs_.
+ */
 mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *name);
 
-int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads, int tbatch_size);
+/**
+ * Align a fasta/fastq file and print alignments to stdout
+ *
+ * @param idx        minimap2 index
+ * @param fn         fasta/fastq file name
+ * @param opt        mapping parameters
+ * @param n_threads  number of threads
+ *
+ * @return 0 on success; -1 if _fn_ can't be read
+ */
+int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads);
+
+int mm_map_file_frag(const mm_idx_t *idx, int n_segs, const char **fn, const mm_mapopt_t *opt, int n_threads);
+
+// deprecated APIs for backward compatibility
+void mm_mapopt_init(mm_mapopt_t *opt);
+mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads);
 
 #ifdef __cplusplus
 }

minimap2.1

@@ -1,4 +1,4 @@
-.TH minimap2 1 "25 August 2017" "minimap2-2.1-r311" "Bioinformatics tools"
+.TH minimap2 1 "9 January 2018" "minimap2-2.7 (r654)" "Bioinformatics tools"
 .SH NAME
 .PP
 minimap2 - mapping and alignment between collections of DNA sequences
@@ -99,6 +99,14 @@ multiple times to map it against each batch of target sequences.
 may be ending with k/K/m/M/g/G. NB: mapping quality is incorrect given a
 multi-part index.
 .TP
+.B --idx-no-seq
+Don't store target sequences in the index. It saves disk space and memory but
+the index generated with this option will not work with
+.B -a
+or
+.BR -c .
+When base-level alignment is not requested, this option is automatically applied.
+.TP
 .BI -d \ FILE
 Save the minimizer index of
 .I target.fa
@@ -126,7 +134,7 @@ Stop chain enlongation if there are no minimizers in
 [10000].
 .TP
 .BI -r \ INT
-Bandwidth used in chaining and DP-based alignment [1000]. This option
+Bandwidth used in chaining and DP-based alignment [500]. This option
 approximately controls the maximum gap size.
 .TP
 .BI -n \ INT
@@ -148,7 +156,7 @@ diagonal minimizer hits will also be suppressed.
 .TP
 .BI -p \ FLOAT
 Minimal secondary-to-primary score ratio to output secondary mappings [0.8].
-Between two chains overlaping over half of the shorter chain (controled by
+Between two chains overlaping over half of the shorter chain (controlled by
 .BR --mask-level ),
 the chain with a lower score is secondary to the chain with a higher score.
 If the ratio of the scores is below
@@ -163,10 +171,16 @@ secondary alignments [5]. This option has no effect when
 is applied.
 .TP
 .BI -G \ NUM
-Maximal intron length in the splice mode [200k]. This option also changes the
-bandwidth to
+Maximum gap on the reference (effective with
+.BR -xsplice / --splice ).
+This option also changes the chaining and alignment band width to
 .IR NUM .
-Increasing this option slows down spliced alignment.
+Increasing this option slows down spliced alignment. [200k]
+.TP
+.BI -F \ NUM
+Maximum fragment length (aka insert size; effective with
+.BR -xsr / --frag)
+[800]
 .TP
 .BI --max-chain-skip \ INT
 A heuristics that stops chaining early [50]. Minimap2 uses dynamic programming
@@ -175,6 +189,23 @@ option makes minimap2 exits the inner loop if it repeatedly sees seeds already
 on chains. Set
 .I INT
 to a large number to switch off this heurstics.
+.TP
+.B --no-long-join
+Disable the long gap patching heuristic. When this option is applied, the
+maximum alignment gap is mostly controlled by
+.BR -r .
+.TP
+.B --splice
+Enable the splice alignment mode.
+.TP
+.B --sr
+Enable short-read alignment heuristics. In the short-read mode, minimap2
+applies a second round of chaining with a higher minimizer occurrence threshold
+if no good chain is found. In addition, minimap2 attempts to patch gaps between
+seeds with ungapped alignment.
+.TP
+.BR --frag [= no | yes ]
+Whether to enable the fragment mode [no]
 .SS Alignment options
 .TP 10
 .BI -A \ INT
@@ -194,6 +225,12 @@ Gap extension penalty [2,1]. A gap of length
 .I k
 costs
 .RI min{ O1 + k * E1 , O2 + k * E2 }.
+In the splice mode, the second gap penalties are not used.
+.TP
+.BI -C \ INT
+Cost for a non-canonical GT-AG splicing (effective with
+.BR --splice )
+[0]
 .TP
 .BI -z \ INT
 Break an alignment if the running score drops too quickly along the diagonal of
@@ -215,8 +252,38 @@ both strands;
 .BR n :
 no attempt to match GT-AG [n]
 .TP
-.BI --cost-non-gt-ag \ INT
-Cost of non-canonical splicing sites [0].
+.BI --end-bonus \ INT
+Score bonus when alignment extends to the end of the query sequence [0].
+.TP
+.BR --splice-flank [= yes | no ]
+Assume the next base to a
+.B GT
+donor site tends to be A/G (91% in human and 92% in mouse) and the preceding
+base to a
+.B AG
+acceptor tends to be C/T [yes with
+.BR --splice ].
+This trend is evolutionarily conservative, all the way to S. cerevisiae
+(PMID:18688272). Specifying this option generally leads to higher junction
+accuracy by several percents, so it is applied by default with
+.BR --splice .
+However, the SIRV control does not honor this trend
+(only ~60%). This option reduces accuracy. If you are benchmarking minimap2
+on SIRV data, please add
+.B --splice-flank=no
+to the command line.
+.TP
+.BI --end-seed-pen \ INT
+Drop a terminal anchor if
+.IR s <log( g )+ INT ,
+where
+.I s
+is the local alignment score around the anchor and
+.I g
+the length of the terminal gap in the chain. This option is only effective
+with
+.BR --splice .
+It helps to avoid tiny terminal exons. [6]
 .SS Input/output options
 .TP 10
 .B -a
@@ -226,6 +293,12 @@ by default.
 .B -Q
 Ignore base quality in the input file.
 .TP
+.B -L
+Write CIGAR with >65535 operators at the CG tag. Older tools are unable to
+convert alignments with >65535 CIGAR ops to BAM. This option makes minimap2 SAM
+compatible with older tools. Newer tools recognizes this tag and reconstruct
+the real CIGAR in memory.
+.TP
 .BI -R \ STR
 SAM read group line in a format like
 .B @RG\\\\tID:foo\\\\tSM:bar
@@ -234,6 +307,29 @@ SAM read group line in a format like
 .B -c
 Generate CIGAR. In PAF, the CIGAR is written to the `cg' custom tag.
 .TP
+.BI --cs[= STR ]
+Output the
+.B cs
+tag.
+.I STR
+can be either
+.I short
+or
+.IR long .
+If no
+.I STR
+is given,
+.I short
+is assumed. [none]
+.TP
+.B -Y
+In SAM output, use soft clipping for supplementary alignments.
+.TP
+.BI --seed \ INT
+Integer seed for randomizing equally best hits. Minimap2 hashes
+.I INT
+and read name when choosing between equally best hits. [11]
+.TP
 .BI -t \ INT
 Number of threads [3]. Minimap2 uses at most three threads when indexing target
 sequences, and uses up to
@@ -241,27 +337,26 @@ sequences, and uses up to
 threads when mapping (the extra thread is for I/O, which is frequently idle and
 takes little CPU time).
 .TP
+.B -2
+Use two I/O threads during mapping. By default, minimap2 uses one I/O thread.
+When I/O is slow (e.g. piping to gzip, or reading from a slow pipe), the I/O
+thread may become the bottleneck. Apply this option to use one thread for input
+and another thread for output, at the cost of increased peak RAM.
+.TP
 .BI -K \ NUM
-Number of bases loaded into memory to process in a mini-batch [200M].
+Number of bases loaded into memory to process in a mini-batch [500M].
 Similar to option
 .BR -I ,
 K/M/G/k/m/g suffix is accepted. A large
 .I NUM
 helps load balancing in the multi-threading mode, at the cost of increased
-memory. Preset
-.B ava-pb
-and
-.B ava-ont
-use
-.BR -K500m .
+memory.
+.TP
+.BR --secondary [= yes | no ]
+Whether to output secondary alignments [yes]
 .TP
 .B --version
 Print version number to stdout
-.TP
-.B --no-sam-hdr
-Don't output SAM header lines. Use this option if the index consists of
-multiple parts; otherwise the SAM output is malformated due to internal header
-lines.
 .SS Preset options
 .TP 10
 .BI -x \ STR
@@ -278,11 +373,6 @@ are:
 PacBio/Oxford Nanopore read to reference mapping
 .RB ( -Hk19 )
 .TP
-.B map10k
-The same as
-.B map-pb
-.RB ( -Hk19 )
-.TP
 .B map-ont
 Slightly more sensitive for Oxford Nanopore to reference mapping
 .RB ( -k15 ).
@@ -309,13 +399,13 @@ Up to 10% sequence divergence.
 .B ava-pb
 PacBio all-vs-all overlap mapping
 .RB ( -Hk19
-.B -w5 -Xp0 -m100 -K500m -g10000 --max-chain-skip
+.B -w5 -Xp0 -m100 -g10000 --max-chain-skip
 .BR 25 ).
 .TP
 .B ava-ont
 Oxford Nanopore all-vs-all overlap mapping
 .RB ( -k15
-.B -w5 -Xp0 -m100 -K500m -g10000 --max-chain-skip
+.B -w5 -Xp0 -m100 -g10000 --max-chain-skip
 .BR 25 ).
 Similarly, the major difference from
 .B ava-pb
@@ -324,8 +414,8 @@ is that this preset is not using HPC minimizers.
 .B splice
 Long-read spliced alignment
 .RB ( -k15
-.B -w5 --splice -g2000 -G200k -A1 -B2 -O2,32 -E1,0 -z200 -ub --cost-non-gt-ag
-.BR 5 ).
+.B -w5 --splice -g2000 -G200k -A1 -B2 -O2,32 -E1,0 -C9 -z200 -ub
+.BR --splice-flank=yes ).
 In the splice mode, 1) long deletions are taken as introns and represented as
 the
 .RB ` N '
@@ -333,6 +423,13 @@ CIGAR operator; 2) long insertions are disabled; 3) deletion and insertion gap
 costs are different during chaining; 4) the computation of the
 .RB ` ms '
 tag ignores introns to demote hits to pseudogenes.
+.TP
+.B sr
+Short single-end reads without splicing
+.RB ( -k21
+.B -w11 --sr --frag -A2 -B8 -O12,32 -E2,1 -r50 -p.5 -N20 -f1000,5000 -n2 -m20
+.B -s40 -g200 -2K50m
+.BR --secondary=no ).
 .RE
 .SS Miscellaneous options
 .TP 10
@@ -345,7 +442,7 @@ multi-threading mode.
 .B --print-qname
 Print query names to stderr, mostly to see which query is crashing minimap2.
 .TP
-.B --print-seed
+.B --print-seeds
 Print seed positions to stderr, for debugging only.
 .SH OUTPUT FORMAT
 .PP
@@ -384,7 +481,7 @@ cb | cb | cb
 r | c | l .
 Tag	Type	Description
 _
-tp	A	Type of aln: P/primary, S/secondary and I/inversion
+tp	A	Type of aln: P/primary, S/secondary and I,i/inversion
 cm	i	Number of minimizers on the chain
 s1	i	Chaining score
 s2	i	Chaining score of the best secondary chain
@@ -392,7 +489,29 @@ NM	i	Total number of mismatches and gaps in the alignment
 AS	i	DP alignment score
 ms	i	DP score of the max scoring segment in the alignment
 nn	i	Number of ambiguous bases in the alignment
+ts	A	Transcript strand (splice mode only)
 cg	Z	CIGAR string (only in PAF)
+cs	Z	Difference string
+.TE
+
+.PP
+The
+.B cs
+tag encodes difference sequences in the short form or the entire query
+.I AND
+reference sequences in the long form. It consists of a series of operations:
+.TS
+center box;
+cb | cb |cb
+r | l | l .
+Op	Regex	Description
+_
+ =	[ACGTN]+	Identical sequence (long form)
+ :	[0-9]+	Identical sequence length
+ *	[acgtn][acgtn]	Substitution: ref to query
+ +	[acgtn]+	Insertion to the reference
+ -	[acgtn]+	Deletion from the reference
+ ~	[acgtn]{2}[0-9]+[acgtn]{2}	Intron length and splice signal
 .TE
 
 .SH LIMITATIONS
@@ -403,9 +522,6 @@ where seed positions may be suboptimal. This should not be a big concern
 because even the optimal alignment may be wrong in such regions.
 .TP
 *
-Minimap2 does not work well with Illumina short reads as of now.
-.TP
-*
 Minimap2 requires SSE2 instructions to compile. It is possible to add
 non-SSE2 support, but it would make minimap2 slower by several times.
 .SH SEE ALSO

misc.c

@@ -1,19 +1,104 @@
-#include <sys/resource.h>
-#include <sys/time.h>
 #include "minimap.h"
 
-int mm_verbose = 3;
+int mm_verbose = 1;
 int mm_dbg_flag = 0;
 double mm_realtime0;
 
+#if defined(WIN32) || defined(_WIN32)
+#include <windows.h>
+
+struct timezone
+{
+  __int32  tz_minuteswest; /* minutes W of Greenwich */
+  int  tz_dsttime;     /* type of dst correction */
+};
+
+/*
+ * gettimeofday.c
+ *    Win32 gettimeofday() replacement
+ *    taken from PostgreSQL, according to
+ *    https://stackoverflow.com/questions/1676036/what-should-i-use-to-replace-gettimeofday-on-windows
+ *
+ * src/port/gettimeofday.c
+ *
+ * Copyright (c) 2003 SRA, Inc.
+ * Copyright (c) 2003 SKC, Inc.
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose, without fee, and without a
+ * written agreement is hereby granted, provided that the above
+ * copyright notice and this paragraph and the following two
+ * paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT,
+ * INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS
+ * IS" BASIS, AND THE AUTHOR HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE,
+ * SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+/* FILETIME of Jan 1 1970 00:00:00. */
+static const unsigned __int64 epoch = ((unsigned __int64) 116444736000000000ULL);
+
+/*
+ * timezone information is stored outside the kernel so tzp isn't used anymore.
+ *
+ * Note: this function is not for Win32 high precision timing purpose. See
+ * elapsed_time().
+ */
+int gettimeofday(struct timeval * tp, struct timezone *tzp)
+{
+    FILETIME    file_time;
+    SYSTEMTIME  system_time;
+    ULARGE_INTEGER ularge;
+
+    GetSystemTime(&system_time);
+    SystemTimeToFileTime(&system_time, &file_time);
+    ularge.LowPart = file_time.dwLowDateTime;
+    ularge.HighPart = file_time.dwHighDateTime;
+
+    tp->tv_sec = (long) ((ularge.QuadPart - epoch) / 10000000L);
+    tp->tv_usec = (long) (system_time.wMilliseconds * 1000);
+
+    return 0;
+}
+
+// taken from https://stackoverflow.com/questions/5272470/c-get-cpu-usage-on-linux-and-windows
 double cputime()
+{
+	HANDLE hProcess = GetCurrentProcess();
+	FILETIME ftCreation, ftExit, ftKernel, ftUser;
+	SYSTEMTIME stKernel;
+	SYSTEMTIME stUser;
+
+	GetProcessTimes(hProcess, &ftCreation, &ftExit, &ftKernel, &ftUser);
+	FileTimeToSystemTime(&ftKernel, &stKernel);
+	FileTimeToSystemTime(&ftUser, &stUser);
+
+	double kernelModeTime = ((stKernel.wHour * 60.) + stKernel.wMinute * 60.) + stKernel.wSecond * 1. + stKernel.wMilliseconds / 1000.;
+	double userModeTime = ((stUser.wHour * 60.) + stUser.wMinute * 60.) + stUser.wSecond * 1. + stUser.wMilliseconds / 1000.;
+
+	return kernelModeTime + userModeTime;
+}
+#else
+#include <sys/resource.h>
+#include <sys/time.h>
+
+double cputime(void)
 {
 	struct rusage r;
 	getrusage(RUSAGE_SELF, &r);
 	return r.ru_utime.tv_sec + r.ru_stime.tv_sec + 1e-6 * (r.ru_utime.tv_usec + r.ru_stime.tv_usec);
 }
+#endif /* WIN32 || _WIN32 */
 
-double realtime()
+double realtime(void)
 {
 	struct timeval tp;
 	struct timezone tzp;

misc/cnt-feat.js

@@ -0,0 +1,258 @@
+/*******************************
+ * Command line option parsing *
+ *******************************/
+
+var getopt = function(args, ostr) {
+	var oli; // option letter list index
+	if (typeof(getopt.place) == 'undefined')
+		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
+	if (getopt.place == -1) { // update scanning pointer
+		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
+			getopt.place = -1;
+			return null;
+		}
+		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
+			++getopt.ind;
+			getopt.place = -1;
+			return null;
+		}
+	}
+	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
+	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
+		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
+		if (getopt.place < 0) ++getopt.ind;
+		return '?';
+	}
+	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
+		getopt.arg = null;
+		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
+	} else { // need an argument
+		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
+			getopt.arg = args[getopt.ind].substr(getopt.place);
+		else if (args.length <= ++getopt.ind) { // no arg
+			getopt.place = -1;
+			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
+			return '?';
+		} else getopt.arg = args[getopt.ind]; // white space
+		getopt.place = -1;
+		++getopt.ind;
+	}
+	return optopt;
+}
+
+/***********************
+ * Interval operations *
+ ***********************/
+
+Interval = {};
+
+Interval.sort = function(a)
+{
+	if (typeof a[0] == 'number')
+		a.sort(function(x, y) { return x - y });
+	else a.sort(function(x, y) { return x[0] != y[0]? x[0] - y[0] : x[1] - y[1] });
+}
+
+Interval.merge = function(a, sorted)
+{
+	if (typeof sorted == 'undefined') sorted = true;
+	if (!sorted) Interval.sort(a);
+	var k = 0;
+	for (var i = 1; i < a.length; ++i) {
+		if (a[k][1] >= a[i][0])
+			a[k][1] = a[k][1] > a[i][1]? a[k][1] : a[i][1];
+		else a[++k] = a[i].slice(0);
+	}
+	a.length = k + 1;
+}
+
+Interval.dedup = function(a, sorted)
+{
+	if (typeof sorted == 'undefined') sorted = true;
+	if (!sorted) Interval.sort(a);
+	var k = 0;
+	for (var i = 1; i < a.length; ++i)
+		if (a[k][0] != a[i][0] || a[k][1] != a[i][1])
+			a[++k] = a[i].slice(0);
+	a.length = k + 1;
+}
+
+Interval.index_end = function(a, sorted)
+{
+	if (a.length == 0) return;
+	if (typeof sorted == 'undefined') sorted = true;
+	if (!sorted) Interval.sort(a);
+	a[0].push(0);
+	var k = 0, k_en = a[0][1];
+	for (var i = 1; i < a.length; ++i) {
+		if (k_en <= a[i][0]) {
+			for (++k; k < i; ++k)
+				if (a[k][1] > a[i][0])
+					break;
+			k_en = a[k][1];
+		}
+		a[i].push(k);
+	}
+}
+
+Interval.find_intv = function(a, x)
+{
+	var left = -1, right = a.length;
+	if (typeof a[0] == 'number') {
+		while (right - left > 1) {
+			var mid = left + ((right - left) >> 1);
+			if (a[mid] > x) right = mid;
+			else if (a[mid] < x) left = mid;
+			else return mid;
+		}
+	} else {
+		while (right - left > 1) {
+			var mid = left + ((right - left) >> 1);
+			if (a[mid][0] > x) right = mid;
+			else if (a[mid][0] < x) left = mid;
+			else return mid;
+		}
+	}
+	return left;
+}
+
+Interval.find_ovlp = function(a, st, en)
+{
+	if (a.length == 0 || st >= en) return [];
+	var l = Interval.find_intv(a, st);
+	var k = l < 0? 0 : a[l][a[l].length - 1];
+	var b = [];
+	for (var i = k; i < a.length; ++i) {
+		if (a[i][0] >= en) break;
+		else if (st < a[i][1])
+			b.push(a[i]);
+	}
+	return b;
+}
+
+/*****************
+ * Main function *
+ *****************/
+
+function read_bed(fn, to_merge, to_dedup)
+{
+	var file = new File(fn);
+	var buf = new Bytes();
+	var h = {};
+	while (file.readline(buf) >= 0) {
+		var t = buf.toString().split("\t");
+		if (h[t[0]] == null)
+			h[t[0]] = [];
+		var bst = parseInt(t[1]);
+		var ben = parseInt(t[2]);
+		if (t.length >= 12 && /^\d+$/.test(t[9])) {
+			t[9] = parseInt(t[9]);
+			var sz = t[10].split(",");
+			var st = t[11].split(",");
+			for (var i = 0; i < t[9]; ++i) {
+				st[i] = parseInt(st[i]);
+				sz[i] = parseInt(sz[i]);
+				h[t[0]].push([bst + st[i], bst + st[i] + sz[i], 0, 0, 0]);
+			}
+		} else {
+			h[t[0]].push([bst, ben, 0, 0, 0]);
+		}
+	}
+	buf.destroy();
+	file.close();
+	for (var chr in h) {
+		if (to_merge) Interval.merge(h[chr], false);
+		else if (to_dedup) Interval.dedup(h[chr], false);
+		else Interval.sort(h[chr]);
+		Interval.index_end(h[chr]);
+	}
+	return h;
+}
+
+function main(args)
+{
+	var c, print_len = false, to_merge = true, to_dedup = false, fn_excl = null;
+	while ((c = getopt(args, "pde:")) != null) {
+		if (c == 'p') print_len = true;
+		else if (c == 'd') to_dedup = true, to_merge = false;
+		else if (c == 'e') fn_excl = getopt.arg;
+	}
+
+	if (args.length - getopt.ind < 2) {
+		print("Usage: k8 cnt-feat.js [options] <target.bed> <feature.bed>");
+		print("Options:");
+		print("  -e FILE    exclude features overlapping regions in BED FILE []");
+		print("  -p         print number of covered bases for each feature");
+		exit(1);
+	}
+
+	var excl = fn_excl != null? read_bed(fn_excl, true, false) : null;
+	var target = read_bed(args[getopt.ind], to_merge, to_dedup);
+
+	var file, buf = new Bytes();
+	var tot_len = 0, hit_len = 0;
+	file = args[getopt.ind+1] != '-'? new File(args[getopt.ind+1]) : new File();
+	while (file.readline(buf) >= 0) {
+		var t = buf.toString().split("\t");
+		var a = [];
+		var bst = parseInt(t[1]);
+		var ben = parseInt(t[2]);
+		if (t.length >= 12 && /^\d+$/.test(t[9])) { // BED12
+			t[9] = parseInt(t[9]);
+			var sz = t[10].split(",");
+			var st = t[11].split(",");
+			for (var i = 0; i < t[9]; ++i) {
+				st[i] = parseInt(st[i]);
+				sz[i] = parseInt(sz[i]);
+				a.push([bst + st[i], bst + st[i] + sz[i], false]);
+			}
+		} else a.push([bst, ben, false]); // 3-column BED
+		var feat_len = 0;
+		for (var i = 0; i < a.length; ++i) {
+			if (excl != null && excl[t[0]] != null) {
+				var oe = Interval.find_ovlp(excl[t[0]], a[i][0], a[i][1]);
+				if (oe.length > 0)
+					continue;
+			}
+			a[i][2] = true;
+			feat_len += a[i][1] - a[i][0];
+		}
+		tot_len += feat_len;
+		if (target[t[0]] == null) continue;
+		var b = [];
+		for (var i = 0; i < a.length; ++i) {
+			if (!a[i][2]) continue;
+			var o = Interval.find_ovlp(target[t[0]], a[i][0], a[i][1]);
+			for (var j = 0; j < o.length; ++j) {
+				var max_st = o[j][0] > a[i][0]? o[j][0] : a[i][0];
+				var min_en = o[j][1] < a[i][1]? o[j][1] : a[i][1];
+				b.push([max_st, min_en]);
+				o[j][2] += min_en - max_st;
+				++o[j][3];
+				if (max_st == o[j][0] && min_en == o[j][1])
+					++o[j][4];
+			}
+		}
+		// find the length covered
+		var feat_hit_len = 0;
+		if (b.length > 0) {
+			b.sort(function(a,b) {return a[0]-b[0]});
+			var st = b[0][0], en = b[0][1];
+			for (var i = 1; i < b.length; ++i) {
+				if (b[i][0] <= en) en = en > b[i][1]? en : b[i][1];
+				else feat_hit_len += en - st, st = b[i][0], en = b[i][1];
+			}
+			feat_hit_len += en - st;
+		}
+		hit_len += feat_hit_len;
+		if (print_len) print('F', t.slice(0, 4).join("\t"), feat_len, feat_hit_len);
+	}
+	file.close();
+
+	buf.destroy();
+
+	warn("# feature bases: " + tot_len);
+	warn("# feature bases overlapping targets: " + hit_len + ' (' + (100.0 * hit_len / tot_len).toFixed(2) + '%)');
+}
+
+main(arguments);

misc/gff2bed.js

@@ -0,0 +1,150 @@
+var getopt = function(args, ostr) {
+	var oli; // option letter list index
+	if (typeof(getopt.place) == 'undefined')
+		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
+	if (getopt.place == -1) { // update scanning pointer
+		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
+			getopt.place = -1;
+			return null;
+		}
+		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
+			++getopt.ind;
+			getopt.place = -1;
+			return null;
+		}
+	}
+	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
+	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
+		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
+		if (getopt.place < 0) ++getopt.ind;
+		return '?';
+	}
+	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
+		getopt.arg = null;
+		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
+	} else { // need an argument
+		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
+			getopt.arg = args[getopt.ind].substr(getopt.place);
+		else if (args.length <= ++getopt.ind) { // no arg
+			getopt.place = -1;
+			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
+			return '?';
+		} else getopt.arg = args[getopt.ind]; // white space
+		getopt.place = -1;
+		++getopt.ind;
+	}
+	return optopt;
+}
+
+var c, fn_ucsc_fai = null, is_short = false;
+while ((c = getopt(arguments, "u:s")) != null) {
+	if (c == 'u') fn_ucsc_fai = getopt.arg;
+	else if (c == 's') is_short = true;
+}
+
+if (getopt.ind == arguments.length) {
+	print("Usage: k8 gff2bed.js [-u ucsc-genome.fa.fai] <in.gff>");
+	exit(1);
+}
+
+var ens2ucsc = {};
+if (fn_ucsc_fai != null) {
+	var buf = new Bytes();
+	var file = new File(fn_ucsc_fai);
+	while (file.readline(buf) >= 0) {
+		var t = buf.toString().split("\t");
+		var s = t[0];
+		if (/_(random|alt|decoy)$/.test(s)) {
+			s = s.replace(/_(random|alt|decoy)$/, '');
+			s = s.replace(/^chr\S+_/, '');
+		} else {
+			s = s.replace(/^chrUn_/, '');
+		}
+		s = s.replace(/v(\d+)/, ".$1");
+		if (s != t[0]) ens2ucsc[s] = t[0];
+	}
+	file.close();
+	buf.destroy();
+}
+
+var colors = {
+		'protein_coding':'0,128,255',
+		'lincRNA':'0,192,0',
+		'snRNA':'0,192,0',
+		'miRNA':'0,192,0',
+		'misc_RNA':'0,192,0'
+	};
+
+function print_bed12(exons, cds_st, cds_en, is_short)
+{
+	if (exons.length == 0) return;
+	var name = is_short? exons[0][7] + "|" + exons[0][5] : exons[0].slice(4, 7).join("|");
+	var a = exons.sort(function(a,b) {return a[1]-b[1]});
+	var sizes = [], starts = [], st, en;
+	st = a[0][1];
+	en = a[a.length - 1][2];
+	if (cds_st == 1<<30) cds_st = st;
+	if (cds_en == 0) cds_en = en;
+	if (cds_st < st || cds_en > en)
+		throw Error("inconsistent thick start or end for transcript " + a[0][4]);
+	for (var i = 0; i < a.length; ++i) {
+		sizes.push(a[i][2] - a[i][1]);
+		starts.push(a[i][1] - st);
+	}
+	var color = colors[a[0][5]];
+	if (color == null) color = '196,196,196';
+	print(a[0][0], st, en, name, 1000, a[0][3], cds_st, cds_en, color, a.length, sizes.join(",") + ",", starts.join(",") + ",");
+}
+
+var re_gtf = /(transcript_id|transcript_type|transcript_biotype|gene_name|transcript_name) "([^"]+)";/g;
+var re_gff3 = /(transcript_id|transcript_type|transcript_biotype|gene_name|transcript_name)=([^;]+)/g;
+var buf = new Bytes();
+var file = new File(arguments[getopt.ind]);
+
+var exons = [], cds_st = 1<<30, cds_en = 0, last_id = null;
+while (file.readline(buf) >= 0) {
+	var t = buf.toString().split("\t");
+	if (t[0].charAt(0) == '#') continue;
+	if (t[2] != "CDS" && t[2] != "exon") continue;
+	t[3] = parseInt(t[3]) - 1;
+	t[4] = parseInt(t[4]);
+	var id = null, type = "", gname = "N/A", biotype = "", m, tname = "N/A";
+	while ((m = re_gtf.exec(t[8])) != null) {
+		if (m[1] == "transcript_id") id = m[2];
+		else if (m[1] == "transcript_type") type = m[2];
+		else if (m[1] == "transcript_biotype") biotype = m[2];
+		else if (m[1] == "gene_name") name = m[2];
+		else if (m[1] == "transcript_name") tname = m[2];
+	}
+	while ((m = re_gff3.exec(t[8])) != null) {
+		if (m[1] == "transcript_id") id = m[2];
+		else if (m[1] == "transcript_type") type = m[2];
+		else if (m[1] == "transcript_biotype") biotype = m[2];
+		else if (m[1] == "gene_name") name = m[2];
+		else if (m[1] == "transcript_name") tname = m[2];
+	}
+	if (type == "" && biotype != "") type = biotype;
+	if (id == null) throw Error("No transcript_id");
+	if (id != last_id) {
+		print_bed12(exons, cds_st, cds_en, is_short);
+		exons = [], cds_st = 1<<30, cds_en = 0;
+		last_id = id;
+	}
+	if (t[2] == "CDS") {
+		cds_st = cds_st < t[3]? cds_st : t[3];
+		cds_en = cds_en > t[4]? cds_en : t[4];
+	} else if (t[2] == "exon") {
+		if (fn_ucsc_fai != null) {
+			if (ens2ucsc[t[0]] != null)
+				t[0] = ens2ucsc[t[0]];
+			else if (/^[A-Z]+\d+\.\d+$/.test(t[0]))
+				t[0] = t[0].replace(/([A-Z]+\d+)\.(\d+)/, "chrUn_$1v$2");
+		}
+		exons.push([t[0], t[3], t[4], t[6], id, type, name, tname]);
+	}
+}
+if (last_id != null)
+	print_bed12(exons, cds_st, cds_en, is_short);
+
+file.close();
+buf.destroy();

misc/intron-eval.js

@@ -161,7 +161,8 @@ for (var tid in tr) {
 	if (anno[t[0]] == null) anno[t[0]] = [];
 	var s = t[4];
 	for (var i = 0; i < s.length - 1; ++i) {
-		if (s[i][1] >= s[i+1][0]) throw Error("ERROR: wrong annotation!");
+		if (s[i][1] >= s[i+1][0])
+			warn("WARNING: incorrect annotation for transcript "+tid+" ("+s[i][1]+" >= "+s[i+1][0]+")")
 		anno[t[0]].push([s[i][1], s[i+1][0]]);
 	}
 }

misc/ov-eval.js

@@ -0,0 +1,105 @@
+var getopt = function(args, ostr) {
+	var oli; // option letter list index
+	if (typeof(getopt.place) == 'undefined')
+		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
+	if (getopt.place == -1) { // update scanning pointer
+		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
+			getopt.place = -1;
+			return null;
+		}
+		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
+			++getopt.ind;
+			getopt.place = -1;
+			return null;
+		}
+	}
+	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
+	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
+		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
+		if (getopt.place < 0) ++getopt.ind;
+		return '?';
+	}
+	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
+		getopt.arg = null;
+		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
+	} else { // need an argument
+		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
+			getopt.arg = args[getopt.ind].substr(getopt.place);
+		else if (args.length <= ++getopt.ind) { // no arg
+			getopt.place = -1;
+			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
+			return '?';
+		} else getopt.arg = args[getopt.ind]; // white space
+		getopt.place = -1;
+		++getopt.ind;
+	}
+	return optopt;
+}
+
+var c, min_ovlp = 2000, min_frac = 0.95, min_mapq = 10;
+while ((c = getopt(arguments, "q:l:f:")) != null) {
+	if (c == 'q') min_mapq = parseInt(getopt.arg);
+	else if (c == 'l') min_ovlp = parseInt(getopt.arg);
+	else if (c == 'f') min_frac = parseFloat(getopt.arg);
+}
+if (arguments.length - getopt.ind < 2) {
+	print("Usage: sort -k6,6 -k8,8n to-ref.paf | k8 ov-eval.js [options] - <ovlp.paf>");
+	print("Options:");
+	print("  -l INT     min overlap length [2000]");
+	print("  -q INT     min mapping quality [10]");
+	print("  -f FLOAT   min fraction of mapped length [0.95]");
+	exit(1);
+}
+
+var buf = new Bytes();
+var file = arguments[getopt.ind] == '-'? new File() : new File(arguments[getopt.ind]);
+var a = [], h = {};
+while (file.readline(buf) >= 0) {
+	var t = buf.toString().split("\t");
+	var is_pri = false;
+	if (parseInt(t[11]) < min_mapq) continue;
+	for (var i = 12; i < t.length; ++i)
+		if (t[i] == 'tp:A:P')
+			is_pri = true;
+	if (!is_pri) continue;
+	for (var i = 1; i <= 3; ++i)
+		t[i] = parseInt(t[i]);
+	for (var i = 6; i <= 8; ++i)
+		t[i] = parseInt(t[i]);
+	if (t[3] - t[2] < min_ovlp || t[8] - t[7] < min_ovlp || (t[3] - t[2]) / t[1] < min_frac)
+		continue;
+	var ctg = t[5], st = t[7], en = t[8];
+	while (a.length > 0) {
+		if (a[0][0] == ctg && a[0][2] > st)
+			break;
+		else a.shift();
+	}
+	for (var j = 0; j < a.length; ++j) {
+		if (a[j][3] == t[0]) continue;
+		var len = (en > a[j][2]? a[j][2] : en) - st;
+		if (len >= min_ovlp) {
+			var key = a[j][3] < t[0]? a[j][3] + "\t" + t[0] : t[0] + "\t" + a[j][3];
+			h[key] = len;
+		}
+	}
+	a.push([ctg, st, en, t[0]]);
+}
+file.close();
+
+file = new File(arguments[getopt.ind + 1]);
+while (file.readline(buf) >= 0) {
+	var t = buf.toString().split("\t");
+	var key = t[0] < t[5]? t[0] + "\t" + t[5] : t[5] + "\t" + t[0];
+	if (h[key] > 0) h[key] = -h[key];
+}
+file.close();
+buf.destroy();
+
+var n_ovlp = 0, n_missing = 0;
+for (var key in h) {
+	++n_ovlp;
+	if (h[key] > 0) ++n_missing;
+}
+print(n_ovlp + " overlaps inferred from the reference mapping");
+print(n_missing + " missed by the read overlapper");
+print((100 * (1 - n_missing / n_ovlp)).toFixed(2) + "% sensitivity");

misc/paf2aln.js

@@ -36,20 +36,21 @@ var getopt = function(args, ostr) {
 	return optopt;
 }
 
-var c, maf_out = false, line_len = 80;
-while ((c = getopt(arguments, "ml:")) != null) {
-	if (c == 'm') maf_out = true;
-	else if (c == 'l') line_len = parseInt(getopt.arg); // TODO: not implemented yet
+var c, line_len = 80, fmt = "aln";
+while ((c = getopt(arguments, "f:l:")) != null) {
+	if (c == 'f') {
+		fmt = getopt.arg;
+		if (fmt != "aln" && fmt != "lastz-cigar" && fmt != "maf")
+			throw Error("format must be one of aln, lastz-cigar and maf");
+	} else if (c == 'l') line_len = parseInt(getopt.arg);
 }
 if (line_len == 0) line_len = 0x7fffffff;
 
 if (getopt.ind == arguments.length) {
-	print("Usage: k8 paf2aln.js [options] <with-cs.paf>");
+	print("Usage: k8 paf2aln.js [options] <in.paf>");
 	print("Options:");
-	print("  -m        MAF output (BLAST-like output by default)");
+	print("  -f STR    output format: aln (BLAST-like), maf or lastz-cigar [aln]");
 	print("  -l INT    line length in BLAST-like output [80]");
-	print("");
-	print("Note: this script only works when minimap2 is run with option '-S'");
 	exit(1);
 }
 
@@ -66,7 +67,7 @@ function padding_str(x, len, right)
 function update_aln(s_ref, s_qry, s_mid, type, seq, slen)
 {
 	var l = type == '*'? 1 : seq.length;
-	if (type == '=') {
+	if (type == '=' || type == ':') {
 		s_ref.set(seq);
 		s_qry.set(seq);
 		s_mid.set(Array(l+1).join("|"));
@@ -96,49 +97,73 @@ function print_aln(rs, qs, strand, slen, elen, s_ref, s_qry, s_mid)
 	var st, en;
 	if (strand == '+') st = qs + slen[1] + 1, en = qs + elen[1];
 	else st = qs - slen[1], en = qs - elen[1] + 1;
-	print(["Qry" + strand + ":", padding_str(st,               10, false), s_qry.toString(), padding_str(en          , 10, true)].join(" "));
+	print(["Qry" + strand + ":", padding_str(st, 10, false), s_qry.toString(), padding_str(en, 10, true)].join(" "));
 }
 
-var s_ref = new Bytes(), s_qry = new Bytes(), s_mid = new Bytes();
-var re = /([=\-\+\*])([A-Za-z]+)/g;
+var s_ref = new Bytes(), s_qry = new Bytes(), s_mid = new Bytes(); // these are used to show padded alignment
+var re_cs = /([:=\-\+\*])(\d+|[A-Za-z]+)/g;
+var re_cg = /(\d+)([MIDNSH])/g;
 
 var buf = new Bytes();
-var file = new File(arguments[getopt.ind]);
-if (maf_out) print("##maf version=1\n");
+var file = arguments[getopt.ind] == "-"? new File() : new File(arguments[getopt.ind]);
+var lineno = 0;
+if (fmt == "maf") print("##maf version=1\n");
 while (file.readline(buf) >= 0) {
 	var m, line = buf.toString();
 	var t = line.split("\t", 12);
-	if ((m = /\tcs:Z:(\S+)/.exec(line)) == null) continue;
-	var cs = m[1];
+	++lineno;
 	s_ref.length = s_qry.length = s_mid.length = 0;
 	var slen = [0, 0], elen = [0, 0];
-	if (maf_out) {
-		while ((m = re.exec(cs)) != null)
-			update_aln(s_ref, s_qry, s_mid, m[1], m[2], elen);
-		if (maf_out) {
-			var score = (m = /\tAS:i:(\d+)/.exec(line)) != null? parseInt(m[1]) : 0;
-			var len = t[0].length > t[5].length? t[0].length : t[5].length;
-			print("a " + score);
-			print(["s", padding_str(t[5], len, true), padding_str(t[7], 10, false), padding_str(parseInt(t[8]) - parseInt(t[7]), 10, false),
-				   "+", padding_str(t[6], 10, false), s_ref.toString()].join(" "));
-			var qs, qe, ql = parseInt(t[1]);
-			if (t[4] == '+') {
-				qs = parseInt(t[2]);
-				qe = parseInt(t[3]);
-			} else {
-				qs = ql - parseInt(t[3]);
-				qe = ql - parseInt(t[2]);
-			}
-			print(["s", padding_str(t[0], len, true), padding_str(qs, 10, false), padding_str(qe - qs, 10, false),
-				   t[4], padding_str(ql, 10, false), s_qry.toString()].join(" "));
-			print("");
+	if (fmt == "lastz-cigar") { // LASTZ-cigar output
+		var cg = (m = /\tcg:Z:(\S+)/.exec(line)) != null? m[1] : null;
+		if (cg == null) {
+			warn("WARNING: converting to LASTZ-cigar format requires the 'cg' tag, which is absent on line " + lineno);
+			continue;
 		}
-	} else {
-		line = line.replace(/\tc[sg]:Z:\S+/g, "");
+		var score = (m = /\tAS:i:(\d+)/.exec(line)) != null? m[1] : 0;
+		var out = ['cigar:', t[0], t[2], t[3], t[4], t[5], t[7], t[8], '+', score];
+		while ((m = re_cg.exec(cg)) != null)
+			out.push(m[2], m[1]);
+		print(out.join(" "));
+	} else if (fmt == "maf") { // MAF output
+		var cs = (m = /\tcs:Z:(\S+)/.exec(line)) != null? m[1] : null;
+		if (cs == null) {
+			warn("WARNING: converting to MAF requires the 'cs' tag, which is absent on line " + lineno);
+			continue;
+		}
+		while ((m = re_cs.exec(cs)) != null) {
+			if (m[1] == ':')
+				throw Error("converting to MAF only works with 'minimap2 --cs=long'");
+			update_aln(s_ref, s_qry, s_mid, m[1], m[2], elen);
+		}
+		var score = (m = /\tAS:i:(\d+)/.exec(line)) != null? parseInt(m[1]) : 0;
+		var len = t[0].length > t[5].length? t[0].length : t[5].length;
+		print("a " + score);
+		print(["s", padding_str(t[5], len, true), padding_str(t[7], 10, false), padding_str(parseInt(t[8]) - parseInt(t[7]), 10, false),
+				"+", padding_str(t[6], 10, false), s_ref.toString()].join(" "));
+		var qs, qe, ql = parseInt(t[1]);
+		if (t[4] == '+') {
+			qs = parseInt(t[2]);
+			qe = parseInt(t[3]);
+		} else {
+			qs = ql - parseInt(t[3]);
+			qe = ql - parseInt(t[2]);
+		}
+		print(["s", padding_str(t[0], len, true), padding_str(qs, 10, false), padding_str(qe - qs, 10, false),
+				t[4], padding_str(ql, 10, false), s_qry.toString()].join(" "));
+		print("");
+	} else { // BLAST-like output
+		var cs = (m = /\tcs:Z:(\S+)/.exec(line)) != null? m[1] : null;
+		if (cs == null) {
+			warn("WARNING: converting to BLAST-like alignment requires the 'cs' tag, which is absent on line " + lineno);
+			continue;
+		}
+		line = line.replace(/\tc[sg]:Z:\S+/g, ""); // get rid of cs or cg tags
 		print('>' + line);
 		var rs = parseInt(t[7]), qs = t[4] == '+'? parseInt(t[2]) : parseInt(t[3]);
 		var n_blocks = 0;
-		while ((m = re.exec(cs)) != null) {
+		while ((m = re_cs.exec(cs)) != null) {
+			if (m[1] == ':') m[2] = Array(parseInt(m[2]) + 1).join("=");
 			var start = 0, rest = m[1] == '*'? 1 : m[2].length;
 			while (rest > 0) {
 				var l_proc;

misc/paf2diff.js

@@ -0,0 +1,188 @@
+var getopt = function(args, ostr) {
+	var oli; // option letter list index
+	if (typeof(getopt.place) == 'undefined')
+		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
+	if (getopt.place == -1) { // update scanning pointer
+		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
+			getopt.place = -1;
+			return null;
+		}
+		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
+			++getopt.ind;
+			getopt.place = -1;
+			return null;
+		}
+	}
+	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
+	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
+		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
+		if (getopt.place < 0) ++getopt.ind;
+		return '?';
+	}
+	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
+		getopt.arg = null;
+		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
+	} else { // need an argument
+		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
+			getopt.arg = args[getopt.ind].substr(getopt.place);
+		else if (args.length <= ++getopt.ind) { // no arg
+			getopt.place = -1;
+			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
+			return '?';
+		} else getopt.arg = args[getopt.ind]; // white space
+		getopt.place = -1;
+		++getopt.ind;
+	}
+	return optopt;
+}
+
+var re_cs = /([:=*+-])(\d+|[A-Za-z]+)/g;
+var c, min_cov_len = 10000, min_var_len = 50000, gap_thres = 50, min_mapq = 5;
+while ((c = getopt(arguments, "l:L:g:q:")) != null) {
+	if (c == 'l') min_cov_len = parseInt(getopt.arg);
+	else if (c == 'L') min_var_len = parseInt(optarg.arg);
+	else if (c == 'g') gap_thres = parseInt(optarg.arg);
+	else if (c == 'q') min_mapq = parseInt(optarg.arg);
+}
+
+if (arguments.length == getopt.ind) {
+	print("Usage: k8 paf2diff.js [options] <with-cs.paf>");
+	print("Options:");
+	print("  -l INT    min alignment length to compute coverage ["+min_cov_len+"]");
+	print("  -L INT    min alignment length to call variants ["+min_var_len+"]");
+	print("  -q INT    min mapping quality ["+min_mapq+"]");
+	print("  -g INT    short/long gap threshold (for statistics only) ["+gap_thres+"]");
+	exit(1);
+}
+
+var file = new File(arguments[getopt.ind]);
+var buf = new Bytes();
+var tot_len = 0, n_sub = [0, 0, 0], n_ins = [0, 0, 0, 0], n_del = [0, 0, 0, 0];
+
+function count_var(o)
+{
+	if (o[3] > 1) return;
+	if (o[5] == '-' && o[6] == '-') return;
+	if (o[5] == '-') { // insertion
+		var l = o[6].length;
+		if (l == 1) ++n_ins[0];
+		else if (l == 2) ++n_ins[1];
+		else if (l < gap_thres) ++n_ins[2];
+		else ++n_ins[3];
+	} else if (o[6] == '-') { // deletion
+		var l = o[5].length;
+		if (l == 1) ++n_del[0];
+		else if (l == 2) ++n_del[1];
+		else if (l < gap_thres) ++n_del[2];
+		else ++n_del[3];
+	} else {
+		++n_sub[0];
+		var s = o[5] + o[6];
+		if (s == 'ag' || s == 'ga' || s == 'ct' || s == 'tc')
+			++n_sub[1];
+		else ++n_sub[2];
+	}
+}
+
+var a = [], out = [];
+var c1_ctg = null, c1_start = 0, c1_end = 0, c1_counted = false, c1_len = 0;
+while (file.readline(buf) >= 0) {
+	var line = buf.toString();
+	if (!/\ts2:i:/.test(line)) continue; // skip secondary alignments
+	var m, t = line.split("\t", 12);
+	for (var i = 6; i <= 11; ++i)
+		t[i] = parseInt(t[i]);
+	if (t[10] < min_cov_len || t[11] < min_mapq) continue;
+	var ctg = t[5], x = t[7], end = t[8];
+	// compute regions covered by 1 contig
+	if (ctg != c1_ctg || x >= c1_end) {
+		if (c1_counted && c1_end > c1_start) {
+			c1_len += c1_end - c1_start;
+			print('R', c1_ctg, c1_start, c1_end);
+		}
+		c1_ctg = ctg, c1_start = x, c1_end = end;
+		c1_counted = (t[10] >= min_var_len);
+	} else if (end > c1_end) { // overlap
+		if (c1_counted && x > c1_start) {
+			c1_len += x - c1_start;
+			print('R', c1_ctg, c1_start, x);
+		}
+		c1_start = c1_end, c1_end = end;
+		c1_counted = (t[10] >= min_var_len);
+	} else { // contained
+		if (c1_counted && x > c1_start) {
+			c1_len += x - c1_start;
+			print('R', c1_ctg, c1_start, x);
+		}
+		c1_start = end;
+	}
+	// output variants ahead of this alignment
+	while (out.length) {
+		if (out[0][0] != ctg || out[0][2] <= x) {
+			count_var(out[0]);
+			print('V', out[0].join("\t"));
+			out.shift();
+		} else break;
+	}
+	// update coverage
+	for (var i = 0; i < out.length; ++i)
+		if (out[i][1] >= x && out[i][2] <= end)
+			++out[i][3];
+	// drop alignments that don't overlap with the current one
+	var k = 0;
+	for (var i = 0; i < a.length; ++i)
+		if (a[0][0] == ctg && a[0][2] > x)
+			a[k++] = a[i];
+	a.length = k;
+	// core loop
+	if (t[10] >= min_var_len) {
+		if ((m = /\tcs:Z:(\S+)/.exec(line)) == null) continue; // no cs tag
+		var cs = m[1];
+		var blen = 0, n_diff = 0;
+		tot_len += t[10];
+		while ((m = re_cs.exec(cs)) != null) {
+			var cov = 1;
+			if (m[1] == '*' || m[1] == '+' || m[1] == '-')
+				for (var i = 0; i < a.length; ++i)
+					if (a[0][2] > x) ++cov;
+			if (m[1] == '=' || m[1] == ':') {
+				var l = m[1] == '='? m[2].length : parseInt(m[2]);
+				x += l, blen += l;
+			} else if (m[1] == '*') {
+				out.push([t[5], x, x+1, cov, t[11], m[2].charAt(0), m[2].charAt(1)]);
+				++x, ++blen, ++n_diff;
+			} else if (m[1] == '+') {
+				out.push([t[5], x, x, cov, t[11], '-', m[2]]);
+				++blen, ++n_diff;
+			} else if (m[1] == '-') {
+				out.push([t[5], x, x + m[2].length, cov, t[11], m[2], '-']);
+				x += m[2].length, ++blen, ++n_diff;
+			}
+		}
+	}
+	a.push([t[5], t[7], t[8]]);
+}
+if (c1_counted && c1_end > c1_start) {
+	c1_len += c1_end - c1_start;
+	print('R', c1_ctg, c1_start, c1_end);
+}
+while (out.length) {
+	count_var(out[0]);
+	print('V', out[0].join("\t"));
+	out.shift();
+}
+
+//warn(tot_len + " alignment columns considered in calling");
+warn(c1_len + " reference bases covered by exactly one contig");
+warn(n_sub[0] + " substitutions; ts/tv = " + (n_sub[1]/n_sub[2]).toFixed(3));
+warn(n_del[0] + " 1bp deletions");
+warn(n_ins[0] + " 1bp insertions");
+warn(n_del[1] + " 2bp deletions");
+warn(n_ins[1] + " 2bp insertions");
+warn(n_del[2] + " [3,"+gap_thres+") deletions");
+warn(n_ins[2] + " [3,"+gap_thres+") insertions");
+warn(n_del[3] + " >="+gap_thres+" deletions");
+warn(n_ins[3] + " >="+gap_thres+" insertions");
+
+buf.destroy();
+file.close();

misc/sam2paf.js

@@ -103,7 +103,10 @@ while (file.readline(buf) >= 0) {
 	if (flag&16) qs = clip[1], qe = qlen - clip[0];
 	else qs = clip[0], qe = qlen - clip[1];
 	var ts = parseInt(t[3]) - 1, te = ts + M + D[1] + N;
-	var a = [t[0], qlen, qs, qe, flag&16? '-' : '+', t[2], tlen, ts, te, match, blen, t[4]];
+	var qname = t[0];
+	if ((flag&1) && (flag&0x40)) qname += '/1';
+	if ((flag&1) && (flag&0x80)) qname += '/2';
+	var a = [qname, qlen, qs, qe, flag&16? '-' : '+', t[2], tlen, ts, te, match, blen, t[4]];
 	print(a.join("\t"), extra.join("\t"));
 }
 

misc/sim-eval.js

@@ -142,7 +142,9 @@ while (file.readline(buf) >= 0) {
 		} else { // SAM
 			var flag = parseInt(t[1]);
 			var read_no = flag>>6&0x3;
-			var qname = read_no == 1 || read_no == 2? t[0] + '/' + read_no : t[0];
+			var qname = t[0];
+			if (!/\/[12]$/.test(qname))
+				qname = read_no == 1 || read_no == 2? t[0] + '/' + read_no : t[0];
 			if (last != qname) {
 				if (last != null) count_err(last, a, tot, err, mode);
 				a = [], last = qname;
@@ -181,11 +183,11 @@ var sum_tot = 0, sum_err = 0, q_out = -1, sum_tot2 = 0, sum_err2 = 0;
 for (var q = max_mapq; q >= 0; --q) {
 	if (tot[q] == 0) continue;
 	if (q_out < 0 || err[q] > 0) {
-		if (q_out >= 0) print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9));
+		if (q_out >= 0) print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9), sum_tot2);
 		sum_tot = sum_err = 0, q_out = q;
 	}
 	sum_tot += tot[q], sum_err += err[q];
 	sum_tot2 += tot[q], sum_err2 += err[q];
 }
-print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9));
+print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9), sum_tot2);
 if (n_unmapped != null) print('U', n_unmapped);

misc/splice2bed.js

@@ -0,0 +1,148 @@
+var getopt = function(args, ostr) {
+	var oli; // option letter list index
+	if (typeof(getopt.place) == 'undefined')
+		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
+	if (getopt.place == -1) { // update scanning pointer
+		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
+			getopt.place = -1;
+			return null;
+		}
+		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
+			++getopt.ind;
+			getopt.place = -1;
+			return null;
+		}
+	}
+	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
+	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
+		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
+		if (getopt.place < 0) ++getopt.ind;
+		return '?';
+	}
+	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
+		getopt.arg = null;
+		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
+	} else { // need an argument
+		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
+			getopt.arg = args[getopt.ind].substr(getopt.place);
+		else if (args.length <= ++getopt.ind) { // no arg
+			getopt.place = -1;
+			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
+			return '?';
+		} else getopt.arg = args[getopt.ind]; // white space
+		getopt.place = -1;
+		++getopt.ind;
+	}
+	return optopt;
+}
+
+var colors = ["0,128,255", "255,0,0", "0,192,0"];
+
+function print_lines(a, fmt) {
+	if (a.length == 0) return;
+	if (fmt == "bed") {
+		var n_pri = 0;
+		for (var i = 0; i < a.length; ++i)
+			if (a[i][8] == 0) ++n_pri;
+		if (n_pri > 1) {
+			for (var i = 0; i < a.length; ++i)
+				if (a[i][8] == 0) a[i][8] = 1;
+		} else if (n_pri == 0) {
+			warn("Warning: " + a[0][3] + " doesn't have a primary alignment");
+		}
+		for (var i = 0; i < a.length; ++i) {
+			a[i][8] = colors[a[i][8]];
+			print(a[i].join("\t"));
+		}
+	}
+	a.length = 0;
+}
+
+function main(args) {
+	var re = /(\d+)([MIDNSH])/g;
+	var c, fmt = "bed", fn_name_conv = null;
+	while ((c = getopt(args, "f:n:")) != null) {
+		if (c == 'f') fmt = getopt.arg;
+		else if (c == 'n') fn_name_conv = getopt.arg;
+	}
+	if (getopt.ind == args.length) {
+		warn("Usage: k8 splice2bed.js <in.paf>");
+		exit(1);
+	}
+
+	var conv = null;
+	if (fn_name_conv != null) {
+		conv = new Map();
+		var file = new File(fn_name_conv);
+		var buf = new Bytes();
+		while (file.readline(buf) >= 0) {
+			var t = buf.toString().split("\t");
+			conv.put(t[0], t[1]);
+		}
+		buf.destroy();
+		file.close();
+	}
+
+	var file = new File(args[getopt.ind]);
+	var buf = new Bytes();
+	var a = [];
+	while (file.readline(buf) >= 0) {
+		var line = buf.toString();
+		if (line.charAt(0) == '@') continue; // skip SAM header lines
+		var t = line.split("\t");
+		var is_pri = false, cigar = null, a1;
+		var qname = conv != null? conv.get(t[0]) : null;
+		if (qname != null) t[0] = qname;
+		if (t.length >= 10 && t[4] != '+' && t[4] != '-' && /^\d+/.test(t[1])) { // SAM
+			var flag = parseInt(t[1]);
+			if (flag&1) t[0] += '/' + (flag>>6&3);
+		}
+		if (a.length && a[0][3] != t[0]) {
+			print_lines(a, fmt);
+			a = [];
+		}
+		if (t.length >= 12 && (t[4] == '+' || t[4] == '-')) { // PAF
+			for (var i = 12; i < t.length; ++i) {
+				if (t[i].substr(0, 5) == 'cg:Z:') {
+					cigar = t[i].substr(5);
+				} else if (t[i].substr(0, 5) == 's2:i:') {
+					is_pri = true;
+				}
+			}
+			a1 = [t[5], t[7], t[8], t[0], Math.floor(t[9]/t[10]*1000), t[4]];
+		} else if (t.length >= 10) { // SAM
+			var flag = parseInt(t[1]);
+			if ((flag&4) || a[2] == '*') continue;
+			cigar = t[5];
+			is_pri = (flag&0x100)? false : true;
+			a1 = [t[2], parseInt(t[3])-1, null, t[0], 1000, (flag&16)? '-' : '+'];
+		} else {
+			throw Error("unrecognized input format");
+		}
+		if (cigar == null) throw Error("missing CIGAR");
+		var m, x0 = 0, x = 0, bs = [], bl = [];
+		while ((m = re.exec(cigar)) != null) {
+			if (m[2] == 'M' || m[2] == 'D') {
+				x += parseInt(m[1]);
+			} else if (m[2] == 'N') {
+				bs.push(x0);
+				bl.push(x - x0);
+				x += parseInt(m[1]);
+				x0 = x;
+			}
+		}
+		bs.push(x0);
+		bl.push(x - x0);
+		// write the BED12 line
+		if (a1[2] == null) a1[2] = a1[1] + x;
+		a1.push(a1[1], a1[2]); // thick start/end is the same as start/end
+		a1.push(is_pri? 0 : 2, bs.length, bl.join(",")+",", bs.join(",")+",");
+		a.push(a1);
+	}
+	print_lines(a, fmt);
+	buf.destroy();
+	file.close();
+	if (conv != null) conv.destroy();
+}
+
+main(arguments);

mmpriv.h

@@ -17,10 +17,16 @@
 #define MM_SEED_IGNORE     (1ULL<<41)
 #define MM_SEED_TANDEM     (1ULL<<42)
 
+#define MM_SEED_SEG_SHIFT  48
+#define MM_SEED_SEG_MASK   (0xffULL<<(MM_SEED_SEG_SHIFT))
+
 #ifndef kroundup32
 #define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
 #endif
 
+#define mm_seq4_set(s, i, c) ((s)[(i)>>3] |= (uint32_t)(c) << (((i)&7)<<2))
+#define mm_seq4_get(s, i)    ((s)[(i)>>3] >> (((i)&7)<<2) & 0xf)
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -33,6 +39,12 @@ typedef struct __kstring_t {
 } kstring_t;
 #endif
 
+typedef struct {
+	int n_u, n_a;
+	uint64_t *u;
+	mm128_t *a;
+} mm_seg_t;
+
 double cputime(void);
 double realtime(void);
 
@@ -40,23 +52,38 @@ void radix_sort_128x(mm128_t *beg, mm128_t *end);
 void radix_sort_64(uint64_t *beg, uint64_t *end);
 uint32_t ks_ksmall_uint32_t(size_t n, uint32_t arr[], size_t kk);
 
-void mm_write_sam_SQ(const mm_idx_t *idx);
-void mm_write_sam_hdr_no_SQ(const char *rg, const char *ver, int argc, char *argv[]);
+void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p);
+
+void mm_write_sam_hdr(const mm_idx_t *mi, const char *rg, const char *ver, int argc, char *argv[]);
 void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag);
 void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs);
-int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int64_t n, mm128_t *a, uint64_t **_u, void *km);
+void mm_write_sam2(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regs, const mm_reg1_t *const* regs, void *km, int opt_flag);
+
+void mm_idxopt_init(mm_idxopt_t *opt);
+const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n);
+int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq);
+int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f);
+mm128_t *mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int n_segs, int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km);
 mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, int *n_regs_, mm_reg1_t *regs, mm128_t *a);
 
-mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a);
+mm_reg1_t *mm_gen_regs(void *km, uint32_t hash, int qlen, int n_u, uint64_t *u, mm128_t *a);
 void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a);
 void mm_sync_regs(void *km, int n_regs, mm_reg1_t *regs);
+int mm_squeeze_a(void *km, int n_regs, mm_reg1_t *regs, mm128_t *a);
 int mm_set_sam_pri(int n, mm_reg1_t *r);
-void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r);
-void mm_select_sub(void *km, float mask_level, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r);
+void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r, int sub_diff);
+void mm_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r);
+void mm_select_sub_multi(void *km, float pri_ratio, float pri1, float pri2, int max_gap_ref, int min_diff, int best_n, int n_segs, const int *qlens, int *n_, mm_reg1_t *r);
 void mm_filter_regs(void *km, const mm_mapopt_t *opt, int *n_regs, mm_reg1_t *regs);
 void mm_join_long(void *km, const mm_mapopt_t *opt, int qlen, int *n_regs, mm_reg1_t *regs, mm128_t *a);
 void mm_hit_sort_by_dp(void *km, int *n_regs, mm_reg1_t *r);
-void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc);
+void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc, int match_sc, int rep_len, int is_sr);
+
+void mm_est_err(const mm_idx_t *mi, int qlen, int n_regs, mm_reg1_t *regs, const mm128_t *a, int32_t n, const uint64_t *mini_pos);
+
+mm_seg_t *mm_seg_gen(void *km, uint32_t hash, int n_segs, const int *qlens, int n_regs0, const mm_reg1_t *regs0, int *n_regs, mm_reg1_t **regs, const mm128_t *a);
+void mm_seg_free(void *km, int n_segs, mm_seg_t *segs);
+void mm_pair(void *km, int max_gap_ref, int dp_bonus, int sub_diff, int match_sc, const int *qlens, int *n_regs, mm_reg1_t **regs);
 
 #ifdef __cplusplus
 }

pe.c

@@ -0,0 +1,177 @@
+#include <stdlib.h>
+#include <math.h>
+#include "mmpriv.h"
+#include "kvec.h"
+
+void mm_select_sub_multi(void *km, float pri_ratio, float pri1, float pri2, int max_gap_ref, int min_diff, int best_n, int n_segs, const int *qlens, int *n_, mm_reg1_t *r)
+{
+	if (pri_ratio > 0.0f && *n_ > 0) {
+		int i, k, n = *n_, n_2nd = 0;
+		int max_dist = n_segs == 2? qlens[0] + qlens[1] + max_gap_ref : 0;
+		for (i = k = 0; i < n; ++i) {
+			int to_keep = 0;
+			if (r[i].parent == i) { // primary
+				to_keep = 1;
+			} else if (r[i].score + min_diff >= r[r[i].parent].score) {
+				to_keep = 1;
+			} else {
+				mm_reg1_t *p = &r[r[i].parent], *q = &r[i];
+				if (p->rev == q->rev && p->rid == q->rid && q->re - p->rs < max_dist && p->re - q->rs < max_dist) { // child and parent are close on the ref
+					if (q->score >= p->score * pri1)
+						to_keep = 1;
+				} else {
+					int is_par_both = (n_segs == 2 && p->qs < qlens[0] && p->qe > qlens[0]);
+					int is_chi_both = (n_segs == 2 && q->qs < qlens[0] && q->qe > qlens[0]);
+					if (is_chi_both || is_chi_both == is_par_both) {
+						if (q->score >= p->score * pri_ratio)
+							to_keep = 1;
+					} else { // the remaining case: is_chi_both == 0 && is_par_both == 1
+						if (q->score >= p->score * pri2)
+							to_keep = 1;
+					}
+				}
+			}
+			if (to_keep && r[i].parent != i) {
+				if (n_2nd++ >= best_n) to_keep = 0; // don't keep if there are too many secondary hits
+			}
+			if (to_keep) r[k++] = r[i];
+			else if (r[i].p) free(r[i].p);
+		}
+		if (k != n) mm_sync_regs(km, k, r); // removing hits requires sync()
+		*n_ = k;
+	}
+}
+
+void mm_set_pe_thru(const int *qlens, int *n_regs, mm_reg1_t **regs)
+{
+	int s, i, n_pri[2], pri[2];
+	n_pri[0] = n_pri[1] = 0;
+	pri[0] = pri[1] = -1;
+	for (s = 0; s < 2; ++s)
+		for (i = 0; i < n_regs[s]; ++i)
+			if (regs[s][i].id == regs[s][i].parent)
+				++n_pri[s], pri[s] = i;
+	if (n_pri[0] == 1 && n_pri[1] == 1) {
+		mm_reg1_t *p = &regs[0][pri[0]];
+		mm_reg1_t *q = &regs[1][pri[1]];
+		if (p->rid == q->rid && p->rev == q->rev && abs(p->rs - q->rs) < 3 && abs(p->re - p->re) < 3
+			&& ((p->qs == 0 && qlens[1] - q->qe == 0) || (q->qs == 0 && qlens[0] - p->qe == 0)))
+		{
+			p->pe_thru = q->pe_thru = 1;
+		}
+	}
+}
+
+#include "ksort.h"
+
+typedef struct {
+	int s, rev;
+	uint64_t key;
+	mm_reg1_t *r;
+} pair_arr_t;
+
+#define sort_key_pair(a) ((a).key)
+KRADIX_SORT_INIT(pair, pair_arr_t, sort_key_pair, 8)
+
+void mm_pair(void *km, int max_gap_ref, int pe_bonus, int sub_diff, int match_sc, const int *qlens, int *n_regs, mm_reg1_t **regs)
+{
+	int i, j, s, n, last[2], dp_thres, segs = 0, max_idx[2];
+	int64_t max;
+	pair_arr_t *a;
+	kvec_t(uint64_t) sc = {0,0,0};
+
+	a = (pair_arr_t*)kmalloc(km, (n_regs[0] + n_regs[1]) * sizeof(pair_arr_t));
+	for (s = n = 0, dp_thres = 0; s < 2; ++s) {
+		int max = 0;
+		for (i = 0; i < n_regs[s]; ++i) {
+			a[n].s = s;
+			a[n].r = &regs[s][i];
+			a[n].rev = a[n].r->rev;
+			a[n].key = (uint64_t)a[n].r->rid << 32 | a[n].r->rs<<1 | (s^a[n].rev);
+			max = max > a[n].r->p->dp_max? max : a[n].r->p->dp_max;
+			++n;
+			segs |= 1<<s;
+		}
+		dp_thres += max;
+	}
+	if (segs != 3) {
+		kfree(km, a); // only one end is mapped
+		return;
+	}
+	dp_thres -= pe_bonus;
+	if (dp_thres < 0) dp_thres = 0;
+	radix_sort_pair(a, a + n);
+
+	max = -1;
+	max_idx[0] = max_idx[1] = -1;
+	last[0] = last[1] = -1;
+	kv_resize(uint64_t, km, sc, n);
+	for (i = 0; i < n; ++i) {
+		if (a[i].key & 1) { // reverse first read or forward second read
+			mm_reg1_t *q, *r;
+			if (last[a[i].rev] < 0) continue;
+			r = a[i].r;
+			q = a[last[a[i].rev]].r;
+			if (r->rid != q->rid || r->rs - q->re > max_gap_ref) continue;
+			for (j = last[a[i].rev]; j >= 0; --j) {
+				int64_t score;
+				if (a[j].rev != a[i].rev || a[j].s == a[i].s) continue;
+				q = a[j].r;
+				if (r->rid != q->rid || r->rs - q->re > max_gap_ref) break;
+				if (r->p->dp_max + q->p->dp_max < dp_thres) continue;
+				score = (int64_t)(r->p->dp_max + q->p->dp_max) << 32 | (r->hash + q->hash);
+				if (score > max)
+					max = score, max_idx[a[j].s] = j, max_idx[a[i].s] = i;
+				kv_push(uint64_t, km, sc, score);
+			}
+		} else { // forward first read or reverse second read
+			last[a[i].rev] = i;
+		}
+	}
+	if (sc.n > 1)
+		radix_sort_64(sc.a, sc.a + sc.n);
+
+	if (sc.n > 0 && max > 0) { // found at least one pair
+		int n_sub = 0, mapq_pe;
+		mm_reg1_t *r[2];
+		r[0] = a[max_idx[0]].r, r[1] = a[max_idx[1]].r;
+		r[0]->proper_frag = r[1]->proper_frag = 1;
+		for (s = 0; s < 2; ++s) {
+			if (r[s]->id != r[s]->parent) { // then lift to primary and update parent
+				mm_reg1_t *p = &regs[s][r[s]->parent];
+				for (i = 0; i < n_regs[s]; ++i)
+					if (regs[s][i].parent == p->id)
+						regs[s][i].parent = r[s]->id;
+				p->mapq = 0;
+			}
+			if (!r[s]->sam_pri) { // then sync sam_pri
+				for (i = 0; i < n_regs[s]; ++i)
+					regs[s][i].sam_pri = 0;
+				r[s]->sam_pri = 1;
+			}
+		}
+		mapq_pe = r[0]->mapq > r[1]->mapq? r[0]->mapq : r[1]->mapq;
+		for (i = 0; i < sc.n; ++i)
+			if ((sc.a[i]>>32) + sub_diff >= max>>32)
+				++n_sub;
+		if (sc.n > 1) {
+			int mapq_pe_alt;
+			mapq_pe_alt = (int)(6.02f * ((max>>32) - (sc.a[sc.n - 2]>>32)) / match_sc - 4.343f * logf(n_sub)); // n_sub > 0 because it counts the optimal, too
+			mapq_pe = mapq_pe < mapq_pe_alt? mapq_pe : mapq_pe_alt;
+		}
+		if (r[0]->mapq < mapq_pe) r[0]->mapq = (int)(.2f * r[0]->mapq + .8f * mapq_pe + .499f);
+		if (r[1]->mapq < mapq_pe) r[1]->mapq = (int)(.2f * r[1]->mapq + .8f * mapq_pe + .499f);
+		if (sc.n == 1) {
+			if (r[0]->mapq < 2) r[0]->mapq = 2;
+			if (r[1]->mapq < 2) r[1]->mapq = 2;
+		} else if (max>>32 > sc.a[sc.n - 2]>>32) {
+			if (r[0]->mapq < 1) r[0]->mapq = 1;
+			if (r[1]->mapq < 1) r[1]->mapq = 1;
+		}
+	}
+
+	kfree(km, a);
+	kfree(km, sc.a);
+
+	mm_set_pe_thru(qlens, n_regs, regs);
+}

python/README.rst

@@ -0,0 +1,145 @@
+==============================
+Mappy: Minimap2 Python Binding
+==============================
+
+Mappy provides a convenient interface to `minimap2
+<https://github.com/lh3/minimap2>`_, a fast and accurate C program to align
+genomic and transcribe nucleotide sequences.
+
+Installation
+------------
+
+Mappy depends on `zlib <http://zlib.net>`_. It can be installed with `pip
+<https://en.wikipedia.org/wiki/Pip_(package_manager)>`_:
+
+.. code:: shell
+
+	pip install --user mappy
+
+or from the minimap2 github repo (`Cython <http://cython.org>`_ required):
+
+.. code:: shell
+
+	git clone https://github.com/lh3/minimap2
+	cd minimap2
+	python setup.py install
+
+Usage
+-----
+
+The following Python script demonstrates the key functionality of mappy:
+
+.. code:: python
+
+	import mappy as mp
+	a = mp.Aligner("test/MT-human.fa")  # load or build index
+	if not a: raise Exception("ERROR: failed to load/build index")
+	for name, seq, qual in mp.fastx_read("test/MT-orang.fa"): # read a fasta/q sequence
+		for hit in a.map(seq): # traverse alignments
+			print("{}\t{}\t{}\t{}".format(hit.ctg, hit.r_st, hit.r_en, hit.cigar_str))
+
+APIs
+----
+
+Mappy implements two classes and one global function.
+
+Class mappy.Aligner
+~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+	mappy.Aligner(fn_idx_in, preset=None, ...)
+
+This constructor accepts the following arguments:
+
+* **fn_idx_in**: index or sequence file name. Minimap2 automatically tests the
+  file type. If a sequence file is provided, minimap2 builds an index. The
+  sequence file can be optionally gzip'd.
+
+* **preset**: minimap2 preset. Currently, minimap2 supports the following
+  presets: **sr** for single-end short reads; **map-pb** for PacBio
+  read-to-reference mapping; **map-ont** for Oxford Nanopore read mapping;
+  **splice** for long-read spliced alignment; **asm5** for assembly-to-assembly
+  alignment; **asm10** for full genome alignment of closely related species. Note
+  that the Python module does not support all-vs-all read overlapping.
+
+* **k**: k-mer length, no larger than 28
+
+* **w**: minimizer window size, no larger than 255
+
+* **min_cnt**: mininum number of minimizers on a chain
+
+* **min_chain_score**: minimum chaing score
+
+* **bw**: chaining and alignment band width
+
+* **best_n**: max number of alignments to return
+
+* **n_threads**: number of indexing threads; 3 by default
+
+* **fn_idx_out**: name of file to which the index is written
+
+.. code:: python
+
+	mappy.Aligner.map(seq)
+
+This method aligns :code:`seq` against the index. It is a generator, *yielding*
+a series of :code:`mappy.Alignment` objects.
+
+Class mappy.Alignment
+~~~~~~~~~~~~~~~~~~~~~
+
+This class describes an alignment. An object of this class has the following
+properties:
+
+* **ctg**: name of the reference sequence the query is mapped to
+
+* **ctg_len**: total length of the reference sequence
+
+* **r_st** and **r_en**: start and end positions on the reference
+
+* **q_st** and **q_en**: start and end positions on the query
+
+* **strand**: +1 if on the forward strand; -1 if on the reverse strand
+
+* **mapq**: mapping quality
+
+* **blen**: length of the alignment, including both alignment matches and gaps
+  but excluding ambiguous bases.
+
+* **mlen**: length of the matching bases in the alignment, excluding ambiguous
+  base matches.
+
+* **NM**: number of mismatches, gaps and ambiguous poistions in the alignment
+
+* **trans_strand**: transcript strand. +1 if on the forward strand; -1 if on the
+  reverse strand; 0 if unknown
+
+* **is_primary**: if the alignment is primary (typically the best and the first
+  to generate)
+
+* **cigar_str**: CIGAR string
+
+* **cigar**: CIGAR returned as an array of shape :code:`(n_cigar,2)`. The two
+  numbers give the length and the operator of each CIGAR operation.
+
+An :code:`Alignment` object can be converted to a string with :code:`str()` in
+the following format:
+
+::
+
+	q_st  q_en  strand  ctg  ctg_len  r_st  r_en  mlen  blen  mapq  cg:Z:cigar_str
+
+It is effectively the PAF format without the QueryName and QueryLength columns
+(the first two columns in PAF).
+
+Function mappy.fastx_read
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+	mappy.fastx_read(fn)
+
+This generator function opens a FASTA/FASTQ file and *yields* a
+:code:`(name,seq,qual)` tuple for each sequence entry. The input file may be
+optionally gzip'd.

python/cmappy.h

@@ -0,0 +1,71 @@
+#ifndef CMAPPY_H
+#define CMAPPY_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include "minimap.h"
+#include "kseq.h"
+KSEQ_DECLARE(gzFile)
+
+typedef struct {
+	const char *ctg;
+	int32_t ctg_start, ctg_end;
+	int32_t qry_start, qry_end;
+	int32_t blen, mlen, NM, ctg_len;
+	uint8_t mapq, is_primary;
+	int8_t strand, trans_strand;
+	int32_t n_cigar32;
+	uint32_t *cigar32;
+} mm_hitpy_t;
+
+static inline void mm_reg2hitpy(const mm_idx_t *mi, mm_reg1_t *r, mm_hitpy_t *h)
+{
+	h->ctg = mi->seq[r->rid].name;
+	h->ctg_len = mi->seq[r->rid].len;
+	h->ctg_start = r->rs, h->ctg_end = r->re;
+	h->qry_start = r->qs, h->qry_end = r->qe;
+	h->strand = r->rev? -1 : 1;
+	h->mapq = r->mapq;
+	h->mlen = r->mlen;
+	h->blen = r->blen;
+	h->NM = r->blen - r->mlen + r->p->n_ambi;
+	h->trans_strand = r->p->trans_strand == 1? 1 : r->p->trans_strand == 2? -1 : 0;
+	h->is_primary = (r->id == r->parent);
+	h->n_cigar32 = r->p->n_cigar;
+	h->cigar32 = r->p->cigar;
+}
+
+static inline void mm_free_reg1(mm_reg1_t *r)
+{
+	free(r->p);
+}
+
+static inline kseq_t *mm_fastx_open(const char *fn)
+{
+	gzFile fp;
+	fp = fn && strcmp(fn, "-") != 0? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+	return kseq_init(fp);
+}
+
+static inline void mm_fastx_close(kseq_t *ks)
+{
+	gzFile fp;
+	fp = ks->f->f;
+	kseq_destroy(ks);
+	gzclose(fp);
+}
+
+static inline int mm_verbose_level(int v)
+{
+	if (v >= 0) mm_verbose = v;
+	return mm_verbose;
+}
+
+static inline void mm_reset_timer(void)
+{
+	extern double realtime(void);
+	mm_realtime0 = realtime();
+}
+
+#endif

python/cmappy.pxd

@@ -0,0 +1,118 @@
+from libc.stdint cimport int8_t, uint8_t, int32_t, int64_t, uint32_t, uint64_t
+
+cdef extern from "minimap.h":
+	#
+	# Options
+	#
+	ctypedef struct mm_idxopt_t:
+		short k, w, flag, bucket_bits
+		int mini_batch_size
+		uint64_t batch_size
+
+	ctypedef struct mm_mapopt_t:
+		int seed
+		int sdust_thres
+		int flag
+		int bw
+		int max_gap, max_gap_ref
+		int max_frag_len
+		int max_chain_skip
+		int min_cnt
+		int min_chain_score
+		float mask_level
+		float pri_ratio
+		int best_n
+		int max_join_long, max_join_short
+		int min_join_flank_sc
+		int a, b, q, e, q2, e2
+		int noncan
+		int zdrop
+		int end_bonus
+		int min_dp_max
+		int min_ksw_len
+		int anchor_ext_len, anchor_ext_shift
+		int pe_ori, pe_bonus
+		float mid_occ_frac
+		int32_t mid_occ
+		int32_t max_occ
+		int mini_batch_size
+
+	int mm_set_opt(char *preset, mm_idxopt_t *io, mm_mapopt_t *mo)
+	int mm_verbose
+
+	#
+	# Indexing
+	#
+	ctypedef struct mm_idx_seq_t:
+		char *name
+		uint64_t offset
+		uint32_t len
+
+	ctypedef struct mm_idx_bucket_t:
+		pass
+
+	ctypedef struct mm_idx_t:
+		int32_t b, w, k, flag
+		uint32_t n_seq
+		mm_idx_seq_t *seq
+		uint32_t *S
+		mm_idx_bucket_t *B
+		void *km
+
+	ctypedef struct mm_idx_reader_t:
+		pass
+
+	mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out)
+	mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
+	void mm_idx_reader_close(mm_idx_reader_t *r)
+	void mm_idx_destroy(mm_idx_t *mi)
+	void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi)
+
+	#
+	# Mapping (key struct defined in cmappy.h below)
+	#
+	ctypedef struct mm_reg1_t:
+		pass
+
+	ctypedef struct mm_tbuf_t:
+		pass
+
+	mm_tbuf_t *mm_tbuf_init()
+	void mm_tbuf_destroy(mm_tbuf_t *b)
+	mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *name)
+
+#
+# Helper header (because it is hard to expose mm_reg1_t with Cython)
+#
+cdef extern from "cmappy.h":
+	ctypedef struct mm_hitpy_t:
+		const char *ctg
+		int32_t ctg_start, ctg_end
+		int32_t qry_start, qry_end
+		int32_t blen, mlen, NM, ctg_len
+		uint8_t mapq, is_primary
+		int8_t strand, trans_strand
+		int32_t n_cigar32
+		uint32_t *cigar32
+
+	void mm_reg2hitpy(const mm_idx_t *mi, mm_reg1_t *r, mm_hitpy_t *h)
+	void mm_free_reg1(mm_reg1_t *r)
+
+	ctypedef struct kstring_t:
+		unsigned l, m
+		char *s
+
+	ctypedef struct kstream_t:
+		pass
+
+	ctypedef struct kseq_t:
+		kstring_t name, comment, seq, qual
+		int last_char
+		kstream_t *f
+
+	kseq_t *mm_fastx_open(const char *fn)
+	void mm_fastx_close(kseq_t *ks)
+	int kseq_read(kseq_t *seq)
+
+	int mm_verbose_level(int v)
+	void mm_reset_timer()

python/mappy.pyx

@@ -0,0 +1,163 @@
+from libc.stdint cimport uint8_t, int8_t
+from libc.stdlib cimport free
+cimport cmappy
+
+cmappy.mm_reset_timer()
+
+cdef class Alignment:
+	cdef int _ctg_len, _r_st, _r_en
+	cdef int _q_st, _q_en
+	cdef int _NM, _mlen, _blen
+	cdef int8_t _strand, _trans_strand
+	cdef uint8_t _mapq, _is_primary
+	cdef _ctg, _cigar # these are python objects
+
+	def __cinit__(self, ctg, cl, cs, ce, strand, qs, qe, mapq, cigar, is_primary, mlen, blen, NM, trans_strand):
+		self._ctg = ctg if isinstance(ctg, str) else ctg.decode()
+		self._ctg_len, self._r_st, self._r_en = cl, cs, ce
+		self._strand, self._q_st, self._q_en = strand, qs, qe
+		self._NM, self._mlen, self._blen = NM, mlen, blen
+		self._mapq = mapq
+		self._cigar = cigar
+		self._is_primary = is_primary
+		self._trans_strand = trans_strand
+
+	@property
+	def ctg(self): return self._ctg
+
+	@property
+	def ctg_len(self): return self._ctg_len
+
+	@property
+	def r_st(self): return self._r_st
+
+	@property
+	def r_en(self): return self._r_en
+
+	@property
+	def strand(self): return self._strand
+
+	@property
+	def trans_strand(self): return self._trans_strand
+
+	@property
+	def blen(self): return self._blen
+
+	@property
+	def mlen(self): return self._mlen
+
+	@property
+	def NM(self): return self._NM
+
+	@property
+	def is_primary(self): return (self._is_primary != 0)
+
+	@property
+	def q_st(self): return self._q_st
+
+	@property
+	def q_en(self): return self._q_en
+
+	@property
+	def mapq(self): return self._mapq
+
+	@property
+	def cigar(self): return self._cigar
+
+	@property
+	def cigar_str(self):
+		return "".join(map(lambda x: str(x[0]) + 'MIDNSH'[x[1]], self._cigar))
+
+	def __str__(self):
+		if self._strand > 0: strand = '+'
+		elif self._strand < 0: strand = '-'
+		else: strand = '?'
+		if self._is_primary != 0: tp = 'tp:A:P'
+		else: tp = 'tp:A:S'
+		if self._trans_strand > 0: ts = 'ts:A:+'
+		elif self._trans_strand < 0: ts = 'ts:A:-'
+		else: ts = 'ts:A:.'
+		return "\t".join([str(self._q_st), str(self._q_en), strand, self._ctg, str(self._ctg_len), str(self._r_st), str(self._r_en),
+				str(self._mlen), str(self._blen), str(self._mapq), tp, ts, "cg:Z:" + self.cigar_str])
+
+cdef class ThreadBuffer:
+	cdef cmappy.mm_tbuf_t *_b
+
+	def __cinit__(self):
+		self._b = cmappy.mm_tbuf_init()
+
+	def __dealloc__(self):
+		cmappy.mm_tbuf_destroy(self._b)
+
+cdef class Aligner:
+	cdef cmappy.mm_idx_t *_idx
+	cdef cmappy.mm_idxopt_t idx_opt
+	cdef cmappy.mm_mapopt_t map_opt
+
+	def __cinit__(self, fn_idx_in, preset=None, k=None, w=None, min_cnt=None, min_chain_score=None, min_dp_score=None, bw=None, best_n=None, n_threads=3, fn_idx_out=None):
+		cmappy.mm_set_opt(NULL, &self.idx_opt, &self.map_opt) # set the default options
+		if preset is not None:
+			cmappy.mm_set_opt(str.encode(preset), &self.idx_opt, &self.map_opt) # apply preset
+		self.map_opt.flag |= 4 # always perform alignment
+		self.idx_opt.batch_size = 0x7fffffffffffffffL # always build a uni-part index
+		if k is not None: self.idx_opt.k = k
+		if w is not None: self.idx_opt.w = w
+		if min_cnt is not None: self.map_opt.min_cnt = min_cnt
+		if min_chain_score is not None: self.map_opt.min_chain_score = min_chain_score
+		if min_dp_score is not None: self.map_opt.min_dp_max = min_dp_score
+		if bw is not None: self.map_opt.bw = bw
+		if best_n is not None: self.best_n = best_n
+
+		cdef cmappy.mm_idx_reader_t *r;
+		if fn_idx_out is None:
+			r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, NULL)
+		else:
+			r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, fn_idx_out)
+		if r is not NULL:
+			self._idx = cmappy.mm_idx_reader_read(r, n_threads) # NB: ONLY read the first part
+			cmappy.mm_idx_reader_close(r)
+			cmappy.mm_mapopt_update(&self.map_opt, self._idx)
+
+	def __dealloc__(self):
+		if self._idx is not NULL:
+			cmappy.mm_idx_destroy(self._idx)
+
+	def __bool__(self):
+		return (self._idx != NULL)
+
+	def map(self, seq, buf=None):
+		cdef cmappy.mm_reg1_t *regs
+		cdef cmappy.mm_hitpy_t h
+		cdef ThreadBuffer b
+		cdef int n_regs
+
+		if self._idx is NULL: return None
+		if buf is None: b = ThreadBuffer()
+		else: b = buf
+		regs = cmappy.mm_map(self._idx, len(seq), str.encode(seq), &n_regs, b._b, &self.map_opt, NULL)
+
+		for i in range(n_regs):
+			cmappy.mm_reg2hitpy(self._idx, &regs[i], &h)
+			cigar = []
+			for k in range(h.n_cigar32):
+				c = h.cigar32[k]
+				cigar.append([c>>4, c&0xf])
+			yield Alignment(h.ctg, h.ctg_len, h.ctg_start, h.ctg_end, h.strand, h.qry_start, h.qry_end, h.mapq, cigar, h.is_primary, h.mlen, h.blen, h.NM, h.trans_strand)
+			cmappy.mm_free_reg1(&regs[i])
+		free(regs)
+
+def fastx_read(fn):
+	cdef cmappy.kseq_t *ks
+	ks = cmappy.mm_fastx_open(str.encode(fn))
+	if ks is NULL: return None
+	while cmappy.kseq_read(ks) >= 0:
+		if ks.qual.l > 0: qual = ks.qual.s if isinstance(ks.qual.s, str) else ks.qual.s.decode()
+		else: qual = None
+		name = ks.name.s if isinstance(ks.name.s, str) else ks.name.s.decode()
+		seq = ks.seq.s if isinstance(ks.seq.s, str) else ks.seq.s.decode()
+		yield name, seq, qual
+	cmappy.mm_fastx_close(ks)
+
+def verbose(v=None):
+	if v is None: v = -1
+	return cmappy.mm_verbose_level(v)

python/minimap2.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import sys, getopt
+import mappy as mp
+
+def main(argv):
+	opts, args = getopt.getopt(argv[1:], "x:n:m:k:w:r:")
+	if len(args) < 2:
+		print("Usage: minimap2.py [options] <ref.fa>|<ref.mmi> <query.fq>")
+		print("Options:")
+		print("  -x STR      preset: sr, map-pb, map-ont, asm5, asm10 or splice")
+		print("  -n INT      mininum number of minimizers")
+		print("  -m INT      mininum chaining score")
+		print("  -k INT      k-mer length")
+		print("  -w INT      minimizer window length")
+		print("  -r INT      band width")
+		sys.exit(1)
+
+	preset, min_cnt, min_sc, k, w, bw = None, None, None, None, None, None
+	for opt, arg in opts:
+		if opt == '-x': preset = arg
+		elif opt == '-n': min_cnt = int(arg)
+		elif opt == '-m': min_chain_score = int(arg)
+		elif opt == '-r': bw = int(arg)
+		elif opt == '-k': k = int(arg)
+		elif opt == '-w': w = int(arg)
+
+	a = mp.Aligner(args[0], preset=preset, min_cnt=min_cnt, min_chain_score=min_sc, k=k, w=w, bw=bw)
+	if not a: raise Exception("ERROR: failed to load/build index file '{}'".format(args[0]))
+	for name, seq, qual in mp.fastx_read(args[1]): # read one sequence
+		for h in a.map(seq): # traverse hits
+			print('{}\t{}\t{}'.format(name, len(seq), h))
+
+if __name__ == "__main__":
+	main(sys.argv)

sdust.c

@@ -56,6 +56,7 @@ sdust_buf_t *sdust_buf_init(void *km)
 	buf = (sdust_buf_t*)kcalloc(km, 1, sizeof(sdust_buf_t));
 	buf->km = km;
 	buf->w = kdq_init(int, buf->km);
+	kdq_resize(int, buf->w, 8);
 	return buf;
 }
 
@@ -176,7 +177,7 @@ uint64_t *sdust(void *km, const uint8_t *seq, int l_seq, int T, int W, int *n)
 #ifdef _SDUST_MAIN
 #include <zlib.h>
 #include <stdio.h>
-#include <unistd.h>
+#include "getopt.h"
 #include "kseq.h"
 KSEQ_INIT(gzFile, gzread)
 

setup.py

@@ -0,0 +1,55 @@
+try:
+	from setuptools import setup, Extension
+except ImportError:
+	from distutils.core import setup
+	from distutils.extension import Extension
+
+cmdclass = {}
+
+try:
+	from Cython.Build import build_ext
+except ImportError: # without Cython
+	module_src = 'python/mappy.c'
+else: # with Cython
+	module_src = 'python/mappy.pyx'
+	cmdclass['build_ext'] = build_ext
+
+import sys
+sys.path.append('python')
+
+def readme():
+    with open('python/README.rst') as f:
+        return f.read()
+
+setup(
+	name = 'mappy',
+	version = '2.7',
+	url = 'https://github.com/lh3/minimap2',
+	description = 'Minimap2 python binding',
+	long_description = readme(),
+	author = 'Heng Li',
+	author_email = 'lh3@me.com',
+	license = 'MIT',
+	keywords = 'sequence-alignment',
+	scripts = ['python/minimap2.py'],
+    ext_modules = [Extension('mappy',
+		sources = [module_src, 'align.c', 'bseq.c', 'chain.c', 'format.c', 'hit.c', 'index.c', 'pe.c',
+				   'ksw2_extd2_sse.c', 'ksw2_exts2_sse.c', 'ksw2_extz2_sse.c', 'ksw2_ll_sse.c',
+				   'kalloc.c', 'kthread.c', 'map.c', 'misc.c', 'sdust.c', 'sketch.c', 'esterr.c'],
+		depends = ['minimap.h', 'bseq.h', 'kalloc.h', 'kdq.h', 'khash.h', 'kseq.h', 'ksort.h',
+				   'ksw2.h', 'kthread.h', 'kvec.h', 'mmpriv.h', 'sdust.h',
+				   'python/cmappy.h', 'python/cmappy.pxd'],
+		extra_compile_args = ['-msse4'], # WARNING: ancient x86_64 CPUs don't have SSE4
+		include_dirs = ['.'],
+		libraries = ['z', 'm', 'pthread'])],
+	classifiers = [
+		'Development Status :: 4 - Beta',
+		'License :: OSI Approved :: MIT License',
+		'Operating System :: POSIX',
+		'Programming Language :: C',
+		'Programming Language :: Cython',
+		'Programming Language :: Python :: 2.7',
+		'Programming Language :: Python :: 3',
+		'Intended Audience :: Science/Research',
+		'Topic :: Scientific/Engineering :: Bio-Informatics'],
+	cmdclass = cmdclass)

sketch.c

@@ -6,21 +6,21 @@
 #include "minimap.h"
 
 unsigned char seq_nt4_table[256] = {
-	0, 1, 2, 3,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	0, 1, 2, 3,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4
 };
 
@@ -77,11 +77,10 @@ void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, i
 {
 	uint64_t shift1 = 2 * (k - 1), mask = (1ULL<<2*k) - 1, kmer[2] = {0,0};
 	int i, j, l, buf_pos, min_pos, kmer_span = 0;
-	mm128_t *buf, min = { UINT64_MAX, UINT64_MAX };
+	mm128_t buf[256], min = { UINT64_MAX, UINT64_MAX };
 	tiny_queue_t tq;
 
-	assert(len > 0 && w > 0 && k > 0 && k <= 28); // 56 bits for k-mer; could use long k-mers, but 28 enough in practice
-	buf = (mm128_t*)alloca(w * 16);
+	assert(len > 0 && (w > 0 && w < 256) && (k > 0 && k <= 28)); // 56 bits for k-mer; could use long k-mers, but 28 enough in practice
 	memset(buf, 0xff, w * 16);
 	memset(&tq, 0, sizeof(tiny_queue_t));
 	kv_resize(mm128_t, km, *p, p->n + len/w);
@@ -102,34 +101,34 @@ void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, i
 				tq_push(&tq, skip_len);
 				kmer_span += skip_len;
 				if (tq.count > k) kmer_span -= tq_shift(&tq);
-				if (kmer_span >= 256) continue; // make sure $kmer_span does not take more than 8 bits
 			} else kmer_span = l + 1 < k? l + 1 : k;
 			kmer[0] = (kmer[0] << 2 | c) & mask;           // forward k-mer
 			kmer[1] = (kmer[1] >> 2) | (3ULL^c) << shift1; // reverse k-mer
 			if (kmer[0] == kmer[1]) continue; // skip "symmetric k-mers" as we don't know it strand
 			z = kmer[0] < kmer[1]? 0 : 1; // strand
-			if (++l >= k) {
+			++l;
+			if (l >= k && kmer_span < 256) {
 				info.x = hash64(kmer[z], mask) << 8 | kmer_span;
 				info.y = (uint64_t)rid<<32 | (uint32_t)i<<1 | z;
 			}
 		} else l = 0, tq.count = tq.front = 0, kmer_span = 0;
 		buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
-		if (l == w + k - 1) { // special case for the first window - because identical k-mers are not stored yet
+		if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
 			for (j = buf_pos + 1; j < w; ++j)
 				if (min.x == buf[j].x && buf[j].y != min.y) kv_push(mm128_t, km, *p, buf[j]);
 			for (j = 0; j < buf_pos; ++j)
 				if (min.x == buf[j].x && buf[j].y != min.y) kv_push(mm128_t, km, *p, buf[j]);
 		}
 		if (info.x <= min.x) { // a new minimum; then write the old min
-			if (l >= w + k) kv_push(mm128_t, km, *p, min);
+			if (l >= w + k && min.x != UINT64_MAX) kv_push(mm128_t, km, *p, min);
 			min = info, min_pos = buf_pos;
 		} else if (buf_pos == min_pos) { // old min has moved outside the window
-			if (l >= w + k - 1) kv_push(mm128_t, km, *p, min);
+			if (l >= w + k - 1 && min.x != UINT64_MAX) kv_push(mm128_t, km, *p, min);
 			for (j = buf_pos + 1, min.x = UINT64_MAX; j < w; ++j) // the two loops are necessary when there are identical k-mers
 				if (min.x >= buf[j].x) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
 			for (j = 0; j <= buf_pos; ++j)
 				if (min.x >= buf[j].x) min = buf[j], min_pos = j;
-			if (l >= w + k - 1) { // write identical k-mers
+			if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
 				for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
 					if (min.x == buf[j].x && min.y != buf[j].y) kv_push(mm128_t, km, *p, buf[j]);
 				for (j = 0; j <= buf_pos; ++j)

tex/bowtie2-s3.sam.eval

@@ -0,0 +1,28 @@
+Q	42	16872292	669	0.000039651	16872292
+Q	40	835329	636	0.000073697	17707621
+Q	31	6544	2	0.000073783	17714165
+Q	30	8882	6	0.000074084	17723047
+Q	27	68499	9	0.000074305	17791546
+Q	26	132041	81	0.000078277	17923587
+Q	25	129378	96	0.000083033	18052965
+Q	24	92056	382	0.000103665	18145021
+Q	23	14341	402	0.000125720	18159362
+Q	22	132838	146	0.000132789	18292200
+Q	21	122274	124	0.000138641	18414474
+Q	18	112183	103	0.000143361	18526657
+Q	17	126981	213	0.000153804	18653638
+Q	16	16356	208	0.000164810	18669994
+Q	15	42804	782	0.000206223	18712798
+Q	14	16026	318	0.000223025	18728824
+Q	12	170250	814	0.000264087	18899074
+Q	11	48351	1409	0.000337777	18947425
+Q	8	1843	311	0.000354156	18949268
+Q	7	62266	4435	0.000586276	19011534
+Q	6	413997	50057	0.003150647	19425531
+Q	5	404	58	0.003153568	19425935
+Q	4	704	154	0.003161381	19426639
+Q	3	1473	681	0.003196193	19428112
+Q	2	17541	16462	0.004039875	19445653
+Q	1	534344	354879	0.021693547	19979997
+Q	0	11939	9917	0.022176642	19991936
+U	8064

tex/bwa-s3.sam.eval

@@ -0,0 +1,52 @@
+Q	60	18784147	3	0.000000160	18784147
+Q	52	19002	1	0.000000213	18803149
+Q	50	7152	2	0.000000319	18810301
+Q	49	6797	1	0.000000372	18817098
+Q	48	52188	2	0.000000477	18869286
+Q	47	48775	3	0.000000634	18918061
+Q	46	19447	2	0.000000739	18937508
+Q	45	25983	3	0.000000896	18963491
+Q	44	13455	1	0.000000949	18976946
+Q	43	14573	2	0.000001053	18991519
+Q	42	8697	4	0.000001263	19000216
+Q	41	8645	2	0.000001368	19008861
+Q	40	176603	75	0.000005264	19185464
+Q	38	2503	2	0.000005368	19187967
+Q	37	4117	3	0.000005523	19192084
+Q	36	2924	16	0.000006356	19195008
+Q	35	2323	8	0.000006772	19197331
+Q	34	2344	10	0.000007292	19199675
+Q	33	4279	6	0.000007603	19203954
+Q	32	2092	4	0.000007810	19206046
+Q	31	2625	11	0.000008382	19208671
+Q	30	2828	13	0.000009057	19211499
+Q	29	1581	1	0.000009108	19213080
+Q	28	1543	6	0.000009420	19214623
+Q	27	70916	223	0.000020948	19285539
+Q	26	1288	16	0.000021777	19286827
+Q	25	25551	122	0.000028065	19312378
+Q	24	14345	84	0.000032390	19326723
+Q	23	7308	87	0.000036878	19334031
+Q	22	8358	125	0.000043325	19342389
+Q	21	4836	71	0.000046983	19347225
+Q	20	5888	123	0.000053325	19353113
+Q	19	4656	83	0.000057600	19357769
+Q	18	3948	87	0.000062081	19361717
+Q	17	4418	114	0.000067954	19366135
+Q	16	4226	131	0.000074702	19370361
+Q	15	5760	164	0.000083144	19376121
+Q	14	4697	257	0.000096384	19380818
+Q	13	5246	313	0.000112503	19386064
+Q	12	4170	241	0.000124908	19390234
+Q	11	4095	304	0.000140557	19394329
+Q	10	3857	360	0.000159087	19398186
+Q	9	5300	438	0.000181617	19403486
+Q	8	4206	572	0.000211050	19407692
+Q	7	4676	787	0.000251541	19412368
+Q	6	3923	688	0.000286924	19416291
+Q	5	3294	708	0.000323333	19419585
+Q	4	2936	693	0.000358965	19422521
+Q	3	3928	816	0.000400897	19426449
+Q	2	2613	810	0.000442533	19429062
+Q	1	3515	1188	0.000503587	19432577
+Q	0	567423	376636	0.019321100	20000000

tex/hs38-simu.sh

@@ -2,3 +2,9 @@
 
 bin/mason_variator -ir hs38.fa -s 1 -ov hs38-s1.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
 bin/mason_simulator -ir hs38.fa -iv hs38-s1.vcf -n 1000000 --seed 1 -o s1_1.fq -or s1_2.fq -oa s1.sam --illumina-prob-mismatch-scale 2.5
+
+bin/mason_variator -ir hs38.fa -s 2 -ov hs38-s2.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
+bin/mason_simulator -ir hs38.fa -iv hs38-s2.vcf -n 1000000 --seed 2 -o mason-s2_1.fq -or mason-s2_2.fq -oa mason-s2.sam --illumina-prob-mismatch-scale 2.5 --illumina-read-length 150
+
+bin/mason_variator -ir hs38.fa -s 3 -ov hs38-s3.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
+bin/mason_simulator -ir hs38.fa -iv hs38-s3.vcf -n 10000000 --seed 3 -o mason-s3_1.fq -or mason-s3_2.fq -oa mason-s3.sam --illumina-prob-mismatch-scale 2.5 --illumina-read-length 150

tex/minimap2.bib

@@ -259,3 +259,57 @@
 	Title = {{Striped Smith-Waterman speeds database searches six times over other SIMD implementations}},
 	Volume = {23},
 	Year = {2007}}
+
+@techreport{Holtgrewe:2010aa,
+	Address = {Freie Universit{\"a}t Berlin},
+	Author = {Holtgrewe, M.},
+	Institution = {Institut f{\"u}r Mathematik und Informatik},
+	Number = {TR-B-10-06},
+	Title = {Mason -- a read simulator for second generation sequencing data},
+	Year = {2010}}
+
+@article{Zaharia:2011aa,
+	Author = {Zaharia, Matei and others},
+	Journal = {arXiv:1111:5572},
+	Title = {Faster and More Accurate Sequence Alignment with {SNAP}},
+	Year = {2011}}
+
+@article{Irimia:2008aa,
+	Author = {Irimia, Manuel and Roy, Scott William},
+	Journal = {PLoS Genet},
+	Pages = {e1000148},
+	Title = {Evolutionary convergence on highly-conserved 3' intron structures in intron-poor eukaryotes and insights into the ancestral eukaryotic genome},
+	Volume = {4},
+	Year = {2008}}
+
+@article{Depristo:2011vn,
+	Author = {Depristo, Mark A and others},
+	Journal = {Nat Genet},
+	Pages = {491-8},
+	Title = {A framework for variation discovery and genotyping using next-generation {DNA} sequencing data},
+	Volume = {43},
+	Year = {2011}}
+
+@article{Kurtz:2004zr,
+	Author = {Kurtz, Stefan and others},
+	Journal = {Genome Biol},
+	Pages = {R12},
+	Title = {Versatile and open software for comparing large genomes},
+	Volume = {5},
+	Year = {2004}}
+
+@article {Li223297,
+	author = {Li, Heng and others},
+	title = {New synthetic-diploid benchmark for accurate variant calling evaluation},
+	year = {2017},
+	note = {doi:10.1101/223297},
+	journal = {bioRxiv}
+}
+
+@article{Berlin:2015xy,
+	Author = {Berlin, Konstantin and others},
+	Journal = {Nat Biotechnol},
+	Pages = {623-30},
+	Title = {Assembling large genomes with single-molecule sequencing and locality-sensitive hashing},
+	Volume = {33},
+	Year = {2015}}

tex/minimap2.tex

@@ -13,29 +13,36 @@
 
 \usepackage{natbib}
 \bibliographystyle{apalike}
-\usepackage{hyperref}
 
 \DeclareMathOperator*{\argmax}{argmax}
 
 \begin{document}
 \firstpage{1}
 
-\title[Aligning long nucleotide sequences with minimap2]{Minimap2: fast pairwise alignment for long nucleotide sequences}
+\title[Aligning nucleotide sequences with minimap2]{Minimap2: versatile pairwise alignment for nucleotide sequences}
 \author[Li]{Heng Li}
 \address{Broad Institute, 415 Main Street, Cambridge, MA 02142, USA}
 
 \maketitle
 
 \begin{abstract}
-\section{Summary:} Minimap2 is a general-purpose mapper to align long noisy DNA
-or mRNA sequences against a large reference database. It targets query
-sequences of 1kb--100Mb in length with per-base divergence typically below
-25\%. For DNA sequence reads, minimap2 is $\sim$30 times faster than many
-mainstream long-read aligners and achieves higher accuracy on simulated data.
-It also employs concave gap cost and rescues inversions for improved alignment
-around potential structural variations. For real long RNA-seq reads, minimap2
-is $\sim$40 times faster than peers and produces alignment more consistent with
-existing gene annotations.
+
+\section{Motivation:} Recent advances in sequencing technologies promise
+ultra-long reads of $\sim$100 kilo bases (kb) in average, full-length mRNA or
+cDNA reads in high throughput and genomic contigs over 100 mega bases (Mb) in
+length. Existing alignment programs are unable or inefficient to process such data
+at scale, which presses for the development of new alignment algorithms.
+
+\section{Results:} Minimap2 is a general-purpose alignment program to map DNA or long
+mRNA sequences against a large reference database. It works with accurate short
+reads of $\ge$100bp in length, $\ge$1kb genomic reads at error rate $\sim$15\%,
+full-length noisy Direct RNA or cDNA reads, and assembly contigs or closely
+related full chromosomes of hundreds of megabases in length. Minimap2 does
+split-read alignment, employs concave gap cost for long insertions and
+deletions (INDELs) and introduces new heuristics to reduce spurious alignments.
+It is 3--4 times faster than mainstream short-read mappers at comparable
+accuracy and $\ge$30 times faster at higher accuracy for both genomic and mRNA
+reads, surpassing most aligners specialized in one type of alignment.
 
 \section{Availability and implementation:}
 \href{https://github.com/lh3/minimap2}{https://github.com/lh3/minimap2}
@@ -56,17 +63,24 @@ the thought that 10kb long sequences should be easier to map than 100bp reads
 because we can more effectively skip repetitive regions, which are often the
 bottleneck of short-read alignment. We confirmed our speculation by achieving
 approximate mapping 50 times faster than BWA-MEM~\citep{Li:2016aa}.
-\citet{Suzuki:2016} extended our work with a fast and novel algorithm on
+\citet{Suzuki130633} extended our work with a fast and novel algorithm on
 generating base-level alignment, which in turn inspired us to develop minimap2
-towards higher accuracy and more practical functionality.
+with added functionality.
 
-Both SMRT and ONT have been applied to sequence spliced mRNAs (RNA-seq). While
+Both SMRT and ONT have been applied to the sequencing of spliced mRNAs (RNA-seq). While
 traditional mRNA aligners work~\citep{Wu:2005vn,Iwata:2012aa}, they are not
 optimized for long noisy sequence reads and are tens of times slower than
 dedicated long-read aligners. When developing minimap2 initially for aligning
-genomic DNA only, we realized minor modifications could make it competitive for
-aligning mRNAs as well. Minimap2 is a first RNA-seq aligner specifically
-designed for long noisy reads.
+genomic DNA only, we realized minor modifications could enable the base
+algorithm to map mRNAs as well. Minimap2 becomes a first RNA-seq aligner
+specifically designed for long noisy reads. We have also extended the original
+algorithm to map short reads at a speed faster than several mainstream
+short-read mappers.
+
+In this article, we will describe the minimap2 algorithm and its applications
+to different types of input sequences. We will evaluate the performance and
+accuracy of minimap2 on several simulated and real data sets and demonstrate
+the versatility of minimap2.
 
 \begin{methods}
 \section{Methods}
@@ -103,8 +117,11 @@ distance between two anchors is too large); otherwise
 \begin{equation}\label{eq:chain-gap}
 \beta(j,i)=\gamma_c\big((y_i-y_j)-(x_i-x_j)\big)
 \end{equation}
-In implementation, a gap of length $l$ costs $\gamma_c(l)=0.01\cdot \bar{w}\cdot
-|l|+0.5\log_2|l|$, where $\bar{w}$ is the average seed length. For $m$ anchors, directly computing all $f(\cdot)$ with
+In implementation, a gap of length $l$ costs
+\[
+\gamma_c(l)=0.01\cdot \bar{w}\cdot|l|+0.5\log_2|l|
+\]
+where $\bar{w}$ is the average seed length. For $m$ anchors, directly computing all $f(\cdot)$ with
 Eq.~(\ref{eq:chain}) takes $O(m^2)$ time. Although theoretically faster
 chaining algorithms exist~\citep{Abouelhoda:2005aa}, they
 are inapplicable to generic gap cost, complex to implement and usually
@@ -120,13 +137,13 @@ $h=50$; even if the heuristic fails, the optimal chain is often close.
 
 \subsubsection{Backtracking}
 Let $P(i)$ be the index of the best predecessor of anchor $i$. It equals 0 if
-$f(i)=w_i$ or $\argmax_j\{f(j)+\eta(j,i)-\gamma(j,i)\}$ otherwise. For each
+$f(i)=w_i$ or $\argmax_j\{f(j)+\alpha(j,i)-\beta(j,i)\}$ otherwise. For each
 anchor $i$ in the descending order of $f(i)$, we apply $P(\cdot)$ repeatedly to
 find its predecessor and mark each visited $i$ as `used', until $P(i)=0$ or we
 reach an already `used' $i$. This way we find all chains with no anchors used
 in more than one chains.
 
-\subsubsection{Identifying primary chains}
+\subsubsection{Identifying primary chains}\label{sec:primary}
 In the absence of copy number changes, each query segment should not be mapped
 to two places in the reference. However, chains found at the previous step may
 have significant or complete overlaps due to repeats in the reference.
@@ -139,7 +156,54 @@ add the chain to $Q$. In the end, $Q$ contains all the primary chains. We did
 not choose a more sophisticated data structure (e.g. range tree or k-d tree)
 because this step is not the performance bottleneck.
 
-\subsection{Aligning genomic DNA}
+\subsubsection{Estimating per-base sequence divergence}
+Suppose a query sequence harbors $n$ seeds of length $k$, $m$ of which are
+present in a chain. We want to estimate the sequence divergence $\epsilon$
+between the query and the reference sequences in the chain. This is useful
+when base-level alignment is too expensive to perform.
+
+If we model substitutions with a homogeneous Poisson process along the query
+sequence, the probablity of seeing $k$ consecutive bases without substitutions
+is $e^{-k\epsilon}$. On the assumption that all $k$-mers are independent of
+each other, the likelihood function of $\epsilon$ is
+\[
+\mathcal{L}(\epsilon|n,m,k)=e^{-m\cdot k\epsilon}(1-e^{-k\epsilon})^{n-m}
+\]
+The maximum likelihood estimate of $\epsilon$ is
+\[
+\hat{\epsilon}=\frac{1}{k}\log\frac{n}{m}
+\]
+In reality, sequencing errors are sometimes clustered and $k$-mers are not
+independent of each other, especially when we take minimizers as seeds. These
+violate the assumptions in the derivation above. As a result, $\hat{\epsilon}$
+is only approximate and can be biased. It also ignores long deletions from the
+reference sequence. In practice, fortunately, $\hat{\epsilon}$ is often close
+to and strongly correlated with the sequence divergence estimated from
+base-level alignments. On the several datasets used in
+Section~\ref{sec:long-genomic}, the Spearman correlation coefficient is around
+$0.9$.
+
+\subsubsection{Indexing with homopolymer compressed $k$-mers}
+SmartDenovo
+(\href{https://github.com/ruanjue/smartdenovo}{https://github.com/ruanjue/smartdenovo};
+J Ruan, personal communication) indexes reads with homopolymer-compressed (HPC)
+$k$-mers and finds the strategy improves overlap sensitivity for SMRT reads.
+Minimap2 adopts the same heuristic.
+
+The HPC string of a string $s$, denoted by ${\rm HPC}(s)$, is constructed by
+contracting homopolymers in $s$ to a single base.  An HPC $k$-mer of $s$ is a
+$k$-long substring of ${\rm HPC}(s)$. For example, suppose $s={\tt GGATTTTCCA}$,
+${\rm HPC}(s)={\tt GATCA}$ and the first HPC 4-mer is ${\tt GATC}$.
+
+To demonstrate the effectiveness of HPC $k$-mers, we performed read overlapping
+for the example {\it E. coli} SMRT reads from PBcR~\citep{Berlin:2015xy}, using
+different types of $k$-mers. With normal 15bp minimizers per 5bp window,
+minimap2 finds 90.9\% of $\ge$2kb overlaps inferred from the read-to-reference
+alignment. With HPC 19-mers, minimap2 finds 97.4\% of overlaps. It achieves this
+higher sensitivity by indexing 1/3 fewer minimizers, which further helps
+performance. HPC-based indexing reduces the sensitivity for ONT reads, though.
+
+\subsection{Aligning genomic DNA}\label{sec:genomic}
 
 \subsubsection{Alignment with 2-piece affine gap cost}
 
@@ -168,7 +232,7 @@ where $s(i,j)$ is the score between the $i$-th reference base and $j$-th query
 base. Eq.~(\ref{eq:ae86}) is a natural extension to the equation under affine
 gap cost~\citep{Gotoh:1982aa,Altschul:1986aa}.
 
-\subsubsection{Suzuki's formulation}
+\subsubsection{The Suzuki-Kasahara formulation}
 
 When we allow gaps longer than several hundred base pairs, nucleotide-level
 alignment is much slower than chaining. SSE acceleration is critical to the
@@ -176,9 +240,9 @@ performance of minimap2. Traditional SSE implementations~\citep{Farrar:2007hs}
 based on Eq.~(\ref{eq:ae86}) can achieve 16-way parallelization for short
 sequences, but only 4-way parallelization when the peak alignment score reaches
 32767. Long sequence alignment may exceed this threshold. Inspired by
-\citet{Wu:1996aa} and the following work, \citet{Suzuki:2016} proposed a
-difference-based formulation that lifted this limitation. In case of 2-piece
-gap cost, define
+\citet{Wu:1996aa} and the following work, \citet{Suzuki130633} proposed a
+difference-based formulation that lifted this limitation.
+In case of 2-piece gap cost, define
 \[
 \left\{\begin{array}{ll}
 u_{ij}\triangleq H_{ij}-H_{i-1,j} & v_{ij}\triangleq H_{ij}-H_{i,j-1} \\
@@ -199,9 +263,10 @@ y_{ij}&=&\max\{0,y_{i,j-1}+u_{i,j-1}-z_{ij}+q\}-q-e\\
 \tilde{y}_{ij}&=&\max\{0,\tilde{y}_{i,j-1}+u_{i,j-1}-z_{ij}+\tilde{q}\}-\tilde{q}-\tilde{e}
 \end{array}\right.
 \end{equation}
-where $z_{ij}$ is a temporary variable that does not need to be stored.  An
-important property of Eq.~(\ref{eq:suzuki}) is that all values are bounded. To
-see that,
+where $z_{ij}$ is a temporary variable that does not need to be stored.
+
+An important property of Eq.~(\ref{eq:suzuki}) is that all values are bounded
+by scoring parameters. To see that,
 \[
 x_{ij}=E_{i+1,j}-H_{ij}=\max\{-q,E_{ij}-H_{ij}\}-e
 \]
@@ -245,8 +310,8 @@ each other. This allows us to fully vectorize the computation of all cells on
 the same anti-diagonal in one inner loop. It also simplifies banded alignment,
 which would be difficult with striped vectorization~\citep{Farrar:2007hs}.
 
-On the condition that $q+e<\tilde{q}+\tilde{e}$ and $e>\tilde{e}$, the boundary
-condition of the equation above is
+On the condition that $q+e<\tilde{q}+\tilde{e}$ and $e>\tilde{e}$, the initial
+values in the diagonal-antidiagonal formuation is
 \[
 \left\{\begin{array}{l}
 x_{r-1,-1}=y_{r-1,r}=-q-e\\
@@ -263,7 +328,7 @@ r\cdot(e-\tilde{e})-(\tilde{q}-q)-\tilde{e} & (r=\lceil\frac{\tilde{q}-q}{e-\til
 -\tilde{e} & (r>\lceil\frac{\tilde{q}-q}{e-\tilde{e}}-1\rceil)
 \end{array}\right.
 \]
-These can be derived from the initial conditions of Eq.~(\ref{eq:ae86}).
+These can be derived from the initial values for Eq.~(\ref{eq:ae86}).
 
 In practice, our 16-way vectorized implementation of global alignment is three
 times as fast as Parasail's 4-way vectorization~\citep{Daily:2016aa}.  Without
@@ -285,9 +350,9 @@ $j'<j$, such that
 S(i',j')-S(i,j)>Z+e\cdot|(i-i')-(j-j')|
 \]
 where $e$ is the gap extension cost and $Z$ is an arbitrary threshold.
-This strategy is similar to X-drop employed in BLAST~\citep{Altschul:1997vn}.
-However, unlike X-drop, it would not break the alignment in the presence of a
-single long gap.
+This strategy is first used in BWA-MEM. It is similar to X-drop employed in
+BLAST~\citep{Altschul:1997vn}, but unlike X-drop, it would not break the
+alignment in the presence of a single long gap.
 
 When minimap2 breaks a global alignment between two anchors, it performs local
 alignment between the two subsequences involved in the global alignment, but
@@ -325,18 +390,24 @@ F_{i,j+1}= \max\{H_{ij}-q,F_{ij}\}-e\\
 \tilde{E}_{i+1,j}= \max\{H_{ij}-d(i)-\tilde{q},\tilde{E}_{ij}\}\\
 \end{array}\right.
 \end{equation}
-Let $T$ be the reference sequence. $d(i)$ is the cost of a non-canonical donor
-site, which takes 0 if $T[i+1,i+2]={\tt GT}$, or a postive number $p$
-otherwise. Similarly, $a(i)$ is the cost of a non-canonical acceptor site, which
-takes 0 if $T[i-1,i]={\tt AG}$, or $p$ otherwise. Eq.~(\ref{eq:splice}) is
-almost equivalent to the equation used by EXALIN~\citep{Zhang:2006aa} except
-that we allow insertions immediately followed by deletions and vice versa; in
-addition, we use Suzuki's diagonal formulation in actual implementation.
-
-%Given that $d_i$ and $a_i$
-%are a function of the reference sequence, it is possible to incorporate
-%splicing signals with more sophisticated models, such as positional weight
-%matrices. We have not tried this approach.
+Let $T$ be the reference sequence. $d(i)$ is computed as
+\[d(i)=\left\{\begin{array}{ll}
+0 & \mbox{if $T[i+1,i+3]$ is ${\tt GTA}$ or ${\tt GTG}$} \\
+p/2 & \mbox{if $T[i+1,i+3]$ is ${\tt GTC}$ or ${\tt GTT}$} \\
+p & \mbox{otherwise}
+\end{array}\right.\]
+where $T[i,j]$ extracts a substring of $T$ between $i$ and $j$ inclusively.
+$d(i)$ penalizes non-canonical donor sites with $p$ and less frequent Eukayotic
+splicing signal ${\tt GT[C/T]}$ with $p/2$~\citep{Irimia:2008aa}. Similarly,
+\[a(i)=\left\{\begin{array}{ll}
+0 & \mbox{if $T[i-2,i]$ is ${\tt CAG}$ or ${\tt TAG}$} \\
+p/2 & \mbox{if $T[i-2,i]$ is ${\tt AAG}$ or ${\tt GAG}$} \\
+p & \mbox{otherwise}
+\end{array}\right.\]
+models the acceptor signal. Eq.~(\ref{eq:splice}) is close to an equation in
+\citet{Zhang:2006aa} except that we allow insertions immediately followed by
+deletions and vice versa; in addition, we use the Suzuki-Kasahara diagonal
+formulation in actual implementation.
 
 If RNA-seq reads are not sequenced from stranded libraries, the read strand
 relative to the underlying transcript is unknown. By default, minimap2 aligns
@@ -348,31 +419,51 @@ reads that span canonical splicing sites.
 
 In the spliced alignment mode, minimap2 further increases the density of
 minimizers and disables banded alignment. Together with the two-round DP-based
-alignment, spliced alignment is several times slower than DNA sequence
+alignment, spliced alignment is several times slower than genomic DNA
 alignment.
 
+\subsection{Aligning short paired-end reads}
+
+During chainging, minimap2 takes a pair of reads as one fragment with a gap of
+unknown length in the middle. It applies a normal gap cost between seeds on the
+same read but is a more permissive gap cost between seeds on different reads.
+More precisely, the gap cost during chaining is:
+\[
+\gamma_c(l)=\left\{\begin{array}{ll}
+0.01\cdot\bar{w}\cdot l+0.5\log_2 l & \mbox{if two seeds on the same read} \\
+\min\{0.01\cdot\bar{w}\cdot|l|,\log_2|l|\} & \mbox{otherwise}
+\end{array}\right.
+\]
+After identifying primary chains (Section~\ref{sec:primary}), we split each
+fragment chain into two read chains and perform alignment for each read as in
+Section~\ref{sec:genomic}.  Finally, we pair hits of each read end to find
+consistent paired-end alignments.
+
 \end{methods}
 
 \section{Results}
 
-\subsection{Aligning genomic reads}
+\subsection{Aligning long genomic reads}\label{sec:long-genomic}
 
 \begin{figure}[!tb]
 \centering
 \includegraphics[width=.5\textwidth]{roc-color.pdf}
-\caption{Evaluation on simulated SMRT reads aligned against human genome
-GRCh38. (a) ROC-like curve. Alignments are sorted by mapping quality in the
+\caption{Evaluation on aligning simulated reads. Simulated reads were mapped 
+to the primary assembly of human genome GRCh38. A read is considered correctly
+mapped if the true position overlaps with the best mapping position by 10\% of
+the read length. Read alignments are sorted by mapping quality in the
 descending order. For each mapping quality threshold, the fraction of
-alignments with mapping quality above the threshold and their error rate
-are plotted. (b) Accumulative mapping error rate as a function of mapping
-quality. 33,088 $\ge$1000bp reads were simulated using pbsim~\citep{Ono:2013aa}
-with error profile sampled from file `m131017\_060208\_42213\_*.1.*' downloaded
-at \href{http://bit.ly/chm1p5c3}{http://bit.ly/chm1p5c3}. The N50 read length
-is 11,628. A read is considered correctly mapped if the true position overlaps
-with the best mapping position by 10\% of the read length. All aligners were
-run under the default setting for SMRT reads. Kart outputted all alignments at
-mapping quality 60, so is not shown in the figure. It mapped nearly all reads
-with 4.1\% of alignments being wrong, less accurate than others.}\label{fig:eval}
+alignments with mapping quality above the threshold and their error rate are
+plotted along the curve. (a) long-read alignment evaluation. 33,088 $\ge$1000bp
+reads were simulated using pbsim~\citep{Ono:2013aa} with error profile sampled
+from file `m131017\_060208\_42213\_*.1.*' downloaded at
+\href{http://bit.ly/chm1p5c3}{http://bit.ly/chm1p5c3}. The N50 read length is
+11,628. Aligners were run under the default setting for SMRT reads.
+(b) short-read alignment evaluation. 10 million pairs of 150bp reads were
+simulated using mason2~\citep{Holtgrewe:2010aa} with option
+`\mbox{--illumina-prob-mismatch-scale 2.5}'. Short-read aligners were run under the
+default setting except for changing the maximum fragment length to
+800bp.}\label{fig:eval}
 \end{figure}
 
 As a sanity check, we evaluated minimap2 on simulated human reads along with
@@ -383,13 +474,11 @@ Kart~(v2.2.5; \citealp{Lin:2017aa}),
 minialign~(v0.5.3; \citealp{Suzuki:2016}) and
 NGMLR~(v0.2.5; \citealp{Sedlazeck169557}). We excluded rHAT~\citep{Liu:2016ab}
 and LAMSA~\citep{Liu:2017aa} because they either
-crashed or produced malformatted output. In this evaluation, Minimap2 has
+crashed or produced malformatted output. In this evaluation, minimap2 has
 higher power to distinguish unique and repetitive hits, and achieves overall
-higher mapping accuracy (Fig.~\ref{fig:eval}a). It is still the most accurate
-even if we skip DP-based alignment (data not shown), confirming chaining alone
-is sufficient to achieve high accuracy for approximate mapping. Minimap2 and
+higher mapping accuracy (Fig.~\ref{fig:eval}a). Minimap2 and
 NGMLR provide better mapping quality estimate: they rarely give repetitive hits
-high mapping quality (Fig.~\ref{fig:eval}b).  Apparently, other aligners may
+high mapping quality.  Apparently, other aligners may
 occasionally miss close suboptimal hits and be overconfident in wrong mappings.
 On run time, minialign is slightly faster than minimap2 and Kart. They are over
 30 times faster than the rest.  Minimap2 consumed 6.1GB memory at the peak,
@@ -405,19 +494,19 @@ confirm the observation by~\citet{Sedlazeck169557} that BWA-MEM often breaks
 them into shorter gaps. The issue is much alleviated with minimap2, thanks
 to the 2-piece affine gap cost.
 
-\subsection{Aligning spliced reads}
+\subsection{Aligning long spliced reads}
 
 We evaluated minimap2 on SIRV control data~(AC:SRR5286959;
-\citealp{Byrne:2017aa}) where the truth is known. Minimap2 predicted 59\,916
-introns from 11\,017 reads. 93.0\% of splice juctions are precise. We examined
+\citealp{Byrne:2017aa}) where the truth is known. Minimap2 predicted 59\,918
+introns from 11\,018 reads. 93.8\% of splice juctions are precise. We examined
 wrongly predicted junctions and found the majority were caused by clustered
 splicing signals (e.g. two adjacent ${\tt GT}$ sites). When INDEL sequencing
 errors are frequent, it is difficult to find precise splicing sites in this
 case. If we allow up to 10bp distance from true splicing sites, 98.4\% of
-aligned introns are approximately correct.  Given this observation, we might be
-able to improve boundary detection by initializing $d(\cdot)$ and $a(\cdot)$ in
-Eq.~(\ref{eq:splice}) with position-specific scoring matrices or more
-sophisticated models. We have not tried this approach.
+aligned introns are approximately correct. It is worth noting that for SIRV, we
+asked minimap2 to model the ${\tt GT..AG}$ splicing signal only without extra
+bases. This is because SIRV does not honor the evolutionarily prevalent signal
+${\tt GT[A/G]..[C/T]AG}$~\citep{Irimia:2008aa}.
 
 \begin{table}[!tb]
 \processtable{Evaluation of junction accuracy on 2D ONT reads}
@@ -426,15 +515,15 @@ sophisticated models. We have not tried this approach.
 \toprule
 & GMAP & minimap2 & SpAln & STAR\\
 \midrule
-Run time (CPU min) & 631 & 15.5 & 2\,076 & 33.9 \\
-Peak RAM (GByte) & 8.9 & 14.5 & 3.2 & 29.2\vspace{1em}\\
-\# aligned reads & 103\,669 & 103\,917 & 103\,711 & 26\,479\\
-\# chimeric alignments & 1\,904 & 1\,671 & 0 & 0\\
-\# non-spliced alignments & 15\,854 & 14\,483 & 17\,033 & 10\,545\vspace{1em}\\
-\# aligned introns & 692\,275 & 694\,237 & 692\,945 & 78\,603 \\
-\# novel introns & 11\,239 & 3\,217 & 8\,550 & 1\,214 \\
-\% exact introns & 83.8\% & 91.8\% & 87.9\% & 55.2\% \\
-\% approx. introns & 91.8\% & 96.5\% & 92.5\% & 82.4\% \\
+Run time (CPU min)        & 631      & 15.9     & 2\,076   & 33.9 \\
+Peak RAM (GByte)          & 8.9      & 14.5     & 3.2      & 29.2\vspace{1em}\\
+\# aligned reads          & 103\,669 & 104\,199 & 103\,711 & 26\,479 \\
+\# chimeric alignments    & 1\,904   & 1\,488   & 0        & 0 \\
+\# non-spliced alignments & 15\,854  & 14\,798  & 17\,033  & 10\,545\vspace{1em}\\
+\# aligned introns        & 692\,275 & 693\,553 & 692\,945 & 78\,603 \\
+\# novel introns          & 11\,239  & 3\,113   & 8\,550   & 1\,214 \\
+\% exact introns          & 83.8\%   & 94.0\%   & 87.9\%   & 55.2\% \\
+\% approx. introns        & 91.8\%   & 96.9\%   & 92.5\%   & 82.4\% \\
 \botrule
 \end{tabular}
 }{Mouse reads (AC:SRR5286960) were mapped to the primary assembly of mouse
@@ -455,10 +544,20 @@ STAR~(v2.5.3a; \citealp{Dobin:2013kx}). In general, minimap2 is more
 consistent with existing annotations (Table~\ref{tab:intron}): it finds
 more junctions with a higher percentage being exactly or approximately correct.
 Minimap2 is over 40 times faster than GMAP and SpAln. While STAR is close to
-minimap2 in speed, it does not work well with noisy reads.  We have also
-evaluated spliced aligners on public Iso-Seq data (human Alzheimer brain
-from \href{http://bit.ly/isoseqpub}{http://bit.ly/isoseqpub}). The observation
-is similar: minimap2 is faster at higher junction accuracy.
+minimap2 in speed, it does not work well with noisy reads.
+
+We have also evaluated spliced aligners on a human Nanopore Direct RNA-seq
+dataset (\href{http://bit.ly/na12878ont}{http://bit.ly/na12878ont}). Minimap2
+aligned 10 million reads in $<$1 wall-clock hour using 16 CPU cores. 94.2\% of
+aligned splice junctions consistent with gene annotations. In comparison,
+GMAP under option `-k 14 -n 0 --min-intronlength 30 --cross-species' is 160
+times slower; 68.7\% of GMAP junctions are found in known gene annotations. The
+percentage increases to 84.1\% if an aligned junction within 10bp from an
+annotated junction is considered to be correct. On a public Iso-Seq dataset
+(human Alzheimer brain from
+\href{http://bit.ly/isoseqpub}{http://bit.ly/isoseqpub}), minimap2 is also
+faster at higher junction accuracy in comparison to other aligners in
+Table~\ref{tab:intron}.
 
 We noted that GMAP and SpAln have not been optimized for noisy reads. We are
 showing the best setting we have experimented, but their developers should be
@@ -469,39 +568,94 @@ able to improve their accuracy further.
 %{\footnotesize
 %\begin{tabular}{lrrrr}
 %\toprule
-%& GMAP & minimap2 & SpAln & STAR\\
+%                          &      GMAP &  minimap2 &     SpAln &      STAR \\ % one GMAP thread took 14 days to align a tiny fraction of reads
 %\midrule
-%Run time (CPU min) & & 243 & 2\,352 & 1\,647 \\
-%\# aligned reads & & 1\,123\,025 & 1\,094\,092 & 682\,452\\
-%\# chimeric alignments & & 33\,091 & 0 & 0\\
-%\# non-spliced alignments & & 339\,081 & 291\,447 & 272\,536\vspace{1em}\\
-%\# aligned introns & & 9\,071\,755 & 9\,208\,564 & 3\,029\,121 \\
-%\# novel introns & & 42\,773 & 82\,230 & 17\,791 \\
-%\% exact introns & & 94.9\% & 91.7\% & 84.7\% \\
-%\% approx. introns&& 96.9\% & 93.4\% & 93.8\% \\
+%Run time (CPU min)        &         - &       243 &     2,352 &     1,647 \\
+%\# aligned reads          & 1,113,502 & 1,123,025 & 1,094,092 &   682,452 \\
+%\# chimeric alignments    &    48,927 &    33,091 &         0 &         0 \\
+%\# non-spliced alignments &   334,097 &   339,081 &   291,447 &   272,536 \vspace{1em}\\
+%\# aligned introns        & 8,922,221 & 9,071,755 & 9,208,564 & 3,029,121 \\
+%\# novel introns          &    48,927 &    42,773 &    82,230 &    17,791 \\
+%\% exact introns          &    90.6\% &    94.9\% &    91.7\% &    84.7\% \\
+%\% approx. introns        &    94.0\% &    96.9\% &    93.4\% &    93.8\% \\
 %\botrule
 %\end{tabular}
 %}{}
 %\end{table}
 
+\subsection{Aligning short genomic reads}
 
-\section{Conclusion}
+We evaluated minimap2 along with Bowtie2~(v2.3.3; \citealt{Langmead:2012fk}), BWA-MEM and
+SNAP (v1.0beta23; \citealt{Zaharia:2011aa}). Minimap2 is 3--4 times as fast as Bowtie2 and
+BWA-MEM, but is 1.3 times slower than SNAP. Minimap2 is more accurate on this
+simulated data set than Bowtie2 and SNAP but less accurate than BWA-MEM
+(Fig.~\ref{fig:eval}b). Closer investigation reveals that BWA-MEM achieves
+a higher accuracy partly because it tries to locally align a read in a small
+region close to its mate. If we disable this feature, BWA-MEM becomes slightly
+less accurate than minimap2. We might consider to implement a similar heuristic
+in minimap2 in future.
 
-Minimap2 is a fast, accurate and versatile aligner for long nucleotide
-sequences.  In addition to reference-based read mapping, minimap2 inherits
-minimap's functionality to search against huge multi-species databases and to
-find read overlaps. On a few test data sets, minimap2 appears to yield slightly
-better miniasm assembly~\citep{Li:2016aa}. Minimap2 can also align similar
-genomes or different assemblies of the same species. However, full-genome
-alignment is an intricate research topic. More thorough evaluations would be
-necessary to justify the use of minimap2 for such applications.
+To evaluate the accuracy of minimap2 on real data, we aligned human reads
+(AC:ERR1341796) with BWA-MEM and minimap2, and called SNPs and small INDELs
+with GATK HaplotypeCaller v3.5~\citep{Depristo:2011vn}. This run was sequenced
+from experimentally mixed CHM1 and CHM13 cell lines. Both of them are homozygous
+across the whole genome and have been \emph{de novo} assembled with SMRT reads
+to high quality. This allowed us to construct an independent truth variant
+dataset~\citep{Li223297} for
+ERR1341796. In this evaluation, minimap2 has higher SNP false negative rate
+(FNR; 2.5\% of minimap2 vs 2.2\% of BWA-MEM), but fewer false positive SNPs per
+million bases (FPPM; 3.0 vs 3.9), lower 2--50bp INDEL FNR (7.3\% vs 7.5\%) and
+similar INDEL FPPM (both 1.0). Minimap2 is broadly similar to BWA-MEM in the
+context of small variant calling.
+
+\subsection{Other applications}
+
+Minimap2 retains minimap's functionality to find overlaps between long reads
+and to search against large multi-species databases such as \emph{nt} from
+NCBI. Minimap2 can also align similar genomes or different assemblies of the
+same species. It took 7 wall-clock minutes over 8 CPU cores to align a human
+SMRT assembly (AC:GCA\_001297185.1) to GRCh38, over 20 times faster
+MUMmer4~\citep{Kurtz:2004zr}.
+
+\section{Discussions}
+
+Minimap2 is a versatile mapper and pairwise aligner for nucleotide sequences.
+It works with short reads, assembly contigs and long noisy genomic and RNA-seq
+reads, and can be used as a read mapper, long-read overlapper or a full-genome
+aligner. Minimap2 is also accurate and efficient, often outperforming other
+domain-specific alignment tools in terms of both speed and accuracy.
+
+The capability of minimap2 comes from a fast base-level alignment algorithm and
+an accurate chaining algorithm. When aligning long query sequences, base-level
+alignment is often the performance bottleneck. The Suzuki-Kasahara algorithm
+greatly alleviates the bottleneck and enables DP-based splice alignment
+involving $>$100kb introns, which was impractically slow ten years ago.  The
+minimap2 chaining algorithm is fast and highly accurate by itself.  In fact,
+chaining alone is more accurate than all the other long-read mappers in
+Fig.~\ref{fig:eval}a (data not shown). This accuracy helps to reduce downstream
+base-level alignment of candidate chains, which is still times slower than
+chaining even with the Suzuki-Kasahara improvement. In addition, taking a
+general form, minimap2 chaining can be adapted to non-typical data types such
+spliced reads and multiple reads per fragment. This gives us the opportunity to
+extend the same base algorithm to a variety of use cases.
+
+Modern mainstream aligners often use a full-text index, such as suffix array or
+FM-index, to index reference sequences. An advantage of this approach is that
+we can use exact seeds of arbitrary lengths, which helps to increase seed
+uniqueness and reduce unsuccessful extensions. Minimap2 indexes reference
+k-mers with a hash table instead. Such fixed-length seeds are inferior to
+variable-length seeds in theory, but can be computed much more efficiently in
+practice. When a query sequence has multiple seed hits, we can afford to skip
+highly repetitive seeds without affecting the final accuracy. This further
+alleviates the concern with the uniqueness of seeds. Hash table is the ideal
+data structure for mapping long query sequences.
 
 \section*{Acknowledgements}
-We owe a debt of gratitude to Hajime Suzuki for releasing his masterpiece and
-insightful notes before formal publication. We thank M. Schatz, P. Rescheneder
-and F.  Sedlazeck for pointing out the limitation of BWA-MEM. We are also
-grateful to early minimap2 testers who have greatly helped to fix various
-issues.
+We owe a debt of gratitude to H. Suzuki and M. Kasahara for releasing their
+masterpiece and insightful notes before formal publication. We thank M.
+Schatz, P. Rescheneder and F. Sedlazeck for pointing out the limitation of
+BWA-MEM. We are also grateful to minimap2 users who have greatly helped to
+suggest features and to fix various issues.
 
 \bibliography{minimap2}
 

tex/mm2-s3.sam.eval

@@ -0,0 +1,62 @@
+Q	60	18579866	27	0.000001453	18579866
+Q	59	27087	4	0.000001666	18606953
+Q	58	21435	1	0.000001718	18628388
+Q	57	45663	3	0.000001874	18674051
+Q	56	36031	2	0.000001978	18710082
+Q	55	18499	2	0.000002082	18728581
+Q	54	14754	2	0.000002187	18743335
+Q	53	25541	2	0.000002291	18768876
+Q	52	26397	5	0.000002554	18795273
+Q	51	15090	3	0.000002711	18810363
+Q	50	13425	11	0.000003294	18823788
+Q	49	15175	2	0.000003397	18838963
+Q	48	19407	4	0.000003606	18858370
+Q	47	11538	16	0.000004452	18869908
+Q	46	12558	17	0.000005349	18882466
+Q	45	40362	28	0.000006817	18922828
+Q	44	10465	13	0.000007500	18933293
+Q	43	10098	20	0.000008552	18943391
+Q	42	10682	19	0.000009549	18954073
+Q	41	9823	11	0.000010125	18963896
+Q	40	9685	16	0.000010963	18973581
+Q	39	10273	18	0.000011905	18983854
+Q	38	9515	18	0.000012847	18993369
+Q	37	9474	27	0.000014261	19002843
+Q	36	10430	25	0.000015568	19013273
+Q	35	9241	34	0.000017348	19022514
+Q	34	9162	31	0.000018968	19031676
+Q	33	10164	49	0.000021532	19041840
+Q	32	9152	55	0.000024408	19050992
+Q	31	9252	35	0.000026233	19060244
+Q	30	9872	55	0.000029103	19070116
+Q	29	8938	65	0.000032496	19079054
+Q	28	8951	73	0.000036306	19088005
+Q	27	9949	95	0.000041261	19097954
+Q	26	9784	97	0.000046316	19107738
+Q	25	10126	97	0.000051366	19117864
+Q	24	11260	123	0.000057765	19129124
+Q	23	10047	114	0.000063691	19139171
+Q	22	9661	123	0.000070083	19148832
+Q	21	10339	168	0.000078813	19159171
+Q	20	17928	193	0.000088804	19177099
+Q	19	9842	193	0.000098817	19186941
+Q	18	14737	247	0.000111605	19201678
+Q	17	10218	238	0.000123934	19211896
+Q	16	10271	242	0.000136457	19222167
+Q	15	12241	333	0.000153683	19234408
+Q	14	9189	336	0.000171070	19243597
+Q	13	9493	515	0.000197734	19253090
+Q	12	11502	743	0.000236185	19264592
+Q	11	8211	507	0.000262390	19272803
+Q	10	9133	606	0.000293695	19281936
+Q	9	10014	931	0.000341801	19291950
+Q	8	8436	698	0.000377816	19300386
+Q	7	8443	705	0.000414163	19308829
+Q	6	10203	944	0.000462808	19319032
+Q	5	6936	756	0.000501760	19325968
+Q	4	6732	843	0.000545190	19332700
+Q	3	8215	1104	0.000602040	19340915
+Q	2	21201	5440	0.000882342	19362116
+Q	1	82328	22186	0.002019600	19444444
+Q	0	553853	371953	0.020562901	19998297
+U	1703

tex/mm2.approx.eval

@@ -1,30 +1,12 @@
-Q	60	32066	0	0.000000000
-Q	40	32	1	0.000031155
-Q	38	19	1	0.000062272
-Q	36	11	1	0.000093376
-Q	35	32	1	0.000124378
-Q	33	15	1	0.000155400
-Q	32	58	1	0.000186145
-Q	27	11	1	0.000217095
-Q	26	80	1	0.000247494
-Q	21	19	2	0.000309186
-Q	20	16	1	0.000339936
-Q	19	19	1	0.000370622
-Q	18	22	2	0.000432099
-Q	17	37	5	0.000585751
-Q	15	24	2	0.000646930
-Q	14	18	3	0.000738939
-Q	13	30	6	0.000922821
-Q	12	18	1	0.000953054
-Q	11	29	2	0.001013638
-Q	10	30	1	0.001043393
-Q	9	20	5	0.001196099
-Q	8	25	8	0.001440348
-Q	7	28	6	0.001622830
-Q	6	35	12	0.001988132
-Q	5	34	12	0.002352725
-Q	4	29	8	0.002594865
-Q	3	36	14	0.003018937
-Q	2	46	15	0.003471482
-Q	1	69	36	0.004558162
-Q	0	167	94	0.007377173
+Q	60	32084	0	0.000000000	32084
+Q	24	318	2	0.000061725	32402
+Q	11	98	2	0.000123077	32500
+Q	8	37	2	0.000184405	32537
+Q	7	37	3	0.000276294	32574
+Q	6	40	3	0.000367940	32614
+Q	5	34	2	0.000428816	32648
+Q	4	37	5	0.000581306	32685
+Q	3	28	6	0.000764222	32713
+Q	2	38	6	0.000946536	32751
+Q	1	50	21	0.001585318	32801
+Q	0	286	150	0.006105117	33087

tex/mm2.eval

@@ -1,17 +1,13 @@
-Q	60	32072	0	0.000000000
-Q	43	206	1	0.000030981
-Q	27	201	1	0.000061578
-Q	15	59	1	0.000092200
-Q	12	25	1	0.000122839
-Q	11	16	1	0.000153473
-Q	10	24	1	0.000184032
-Q	9	17	2	0.000245248
-Q	8	27	3	0.000336938
-Q	7	23	1	0.000367309
-Q	6	20	1	0.000397675
-Q	5	18	4	0.000519751
-Q	4	17	1	0.000550038
-Q	3	29	5	0.000702204
-Q	2	32	4	0.000823522
-Q	1	54	6	0.001004872
-Q	0	234	106	0.004202697
+Q	60	32477	0	0.000000000	32477
+Q	22	16	1	0.000030776	32493
+Q	21	44	1	0.000061468	32537
+Q	19	73	1	0.000091996	32610
+Q	14	66	1	0.000122414	32676
+Q	10	26	3	0.000214054	32702
+Q	8	14	1	0.000244529	32716
+Q	7	13	2	0.000305539	32729
+Q	6	47	1	0.000335611	32776
+Q	3	10	1	0.000366010	32786
+Q	2	20	2	0.000426751	32806
+Q	1	248	94	0.003267381	33054
+Q	0	31	17	0.003778147	33085

tex/roc.gp

@@ -14,7 +14,7 @@ set size 1.59,1.04
 set multiplot layout 1,2
 
 set label "(a)" at graph -0.245,1.06 font "Helvetica-bold,40"
-set xlab "Error rate of mapped reads"
+set xlab "Error rate of mapped PacBio reads"
 set ylab "Fraction of mapped reads" off +1.8
 set ytics 0.02
 set yran [0.9:1]
@@ -34,19 +34,27 @@ unset label
 set origin 0.8,0
 set size 0.79,1
 set label "(b)" at graph -0.245,1.06 font "Helvetica-bold,40"
-unset log
-unset format
-unset key
-set log y
-set ylab "Accumulative mapping error rate" off +0
-set xlab "Mapping quality"
-set yran [1e-5:0.1]
-set ytics 1e-5,0.1
-set format y "10^{%L}"
-set xran [60:0] reverse
-plot "<./eval2roc.pl blasr-mc.eval" u 1:2 w lp ls 4, \
-     "<./eval2roc.pl bwa.eval" u 1:2 t "bwa-mem" w lp ls 2, \
-     "<./eval2roc.pl graphmap.eval" u 1:2 t "graphmap" w lp ls 3, \
-     "<./eval2roc.pl minialign.eval" u 1:2 t "minialign" w lp ls 1, \
-     "<./eval2roc.pl mm2.eval" u 1:2 t "minimap2" w lp ls 6, \
-     "<./eval2roc.pl ngmlr.eval" u 1:2 t "ngm-lr" w lp ls 5
+set xlab "Error rate of mapped short reads"
+
+set key top left
+plot "<./eval2roc.pl -n2e7 bowtie2-s3.sam.eval" u 2:3 t "bowtie2" w lp ls 5, \
+	 "<./eval2roc.pl -n2e7 bwa-s3.sam.eval" u 2:3 t "bwa-mem" w lp ls 2, \
+	 "<./eval2roc.pl -n2e7 mm2-s3.sam.eval" u 2:3 t "minimap2" w lp ls 6, \
+	 "<./eval2roc.pl -n2e7 snap-s3.sam.eval" u 2:3 t "snap" w lp ls 3
+
+#unset log
+#unset format
+#unset key
+#set log y
+#set ylab "Accumulative mapping error rate" off +0
+#set xlab "Mapping quality"
+#set yran [1e-5:0.1]
+#set ytics 1e-5,0.1
+#set format y "10^{%L}"
+#set xran [60:0] reverse
+#plot "<./eval2roc.pl blasr-mc.eval" u 1:2 w lp ls 4, \
+#     "<./eval2roc.pl bwa.eval" u 1:2 t "bwa-mem" w lp ls 2, \
+#     "<./eval2roc.pl graphmap.eval" u 1:2 t "graphmap" w lp ls 3, \
+#     "<./eval2roc.pl minialign.eval" u 1:2 t "minialign" w lp ls 1, \
+#     "<./eval2roc.pl mm2.eval" u 1:2 t "minimap2" w lp ls 6, \
+#     "<./eval2roc.pl ngmlr.eval" u 1:2 t "ngm-lr" w lp ls 5

tex/snap-s3.sam.eval

@@ -0,0 +1,62 @@
+Q	60	18993268	10320	0.000543350	18993268
+Q	59	33156	216	0.000553756	19026424
+Q	58	29982	295	0.000568365	19056406
+Q	57	9412	278	0.000582666	19065818
+Q	56	11012	228	0.000594281	19076830
+Q	55	9968	235	0.000606283	19086798
+Q	54	8602	292	0.000621301	19095400
+Q	53	6094	259	0.000634662	19101494
+Q	52	5026	257	0.000647946	19106520
+Q	51	4278	224	0.000659522	19110798
+Q	50	3682	178	0.000668708	19114480
+Q	49	2750	156	0.000676772	19117230
+Q	48	2314	112	0.000682548	19119544
+Q	47	2056	96	0.000687495	19121600
+Q	46	1658	62	0.000690677	19123258
+Q	45	1492	74	0.000694493	19124750
+Q	44	1150	56	0.000697379	19125900
+Q	43	1062	48	0.000699850	19126962
+Q	42	976	60	0.000702951	19127938
+Q	41	884	36	0.000704800	19128822
+Q	40	708	52	0.000707493	19129530
+Q	39	870	26	0.000708819	19130400
+Q	38	598	26	0.000710156	19130998
+Q	37	542	34	0.000711913	19131540
+Q	36	846	50	0.000714495	19132386
+Q	35	590	50	0.000717087	19132976
+Q	34	550	42	0.000719261	19133526
+Q	33	2174	66	0.000722628	19135700
+Q	32	876	86	0.000727089	19136576
+Q	31	638	104	0.000732500	19137214
+Q	30	1718	196	0.000742675	19138932
+Q	29	91022	968	0.000789497	19229954
+Q	28	12864	781	0.000829556	19242818
+Q	27	5806	427	0.000851489	19248624
+Q	26	25274	728	0.000888144	19273898
+Q	25	7418	680	0.000923070	19281316
+Q	24	11800	701	0.000958839	19293116
+Q	23	57328	3933	0.001159250	19350444
+Q	22	7662	846	0.001202494	19358106
+Q	21	5924	617	0.001233989	19364030
+Q	20	4623	574	0.001263330	19368653
+Q	19	4988	942	0.001311627	19373641
+Q	18	3968	793	0.001352282	19377609
+Q	17	3630	681	0.001387166	19381239
+Q	16	2921	513	0.001413422	19384160
+Q	15	2716	424	0.001435095	19386876
+Q	14	2366	365	0.001453744	19389242
+Q	13	2169	412	0.001474828	19391411
+Q	12	2077	360	0.001493233	19393488
+Q	11	2016	441	0.001515815	19395504
+Q	10	2292	738	0.001553682	19397796
+Q	9	4165	1832	0.001647772	19401961
+Q	8	3963	1862	0.001743385	19405924
+Q	7	3927	1793	0.001835408	19409851
+Q	6	3572	1639	0.001919497	19413423
+Q	5	3270	1533	0.001998126	19416693
+Q	4	3046	1610	0.002080718	19419739
+Q	3	251447	125550	0.008436553	19671186
+Q	2	24390	13537	0.009113417	19695576
+Q	1	124406	86780	0.013434624	19819982
+Q	0	171254	153874	0.021016609	19991236
+U	8764