Released minimap2-2.3 (r531)

The tech note still needs improvement. Will do that after the release of v2.3.

.gitignore

@@ -4,3 +4,5 @@
 *.a
 *.o
 *.dSYM
+minimap2
+mappy.c

.travis.yml

@@ -1,5 +1,24 @@
-language: c
-compiler:
-  - gcc
-  - clang
-script: make
+matrix:
+  include:
+    - language: c
+      compiler: gcc
+      script: make
+    - language: c
+      compiler: clang
+      script: make
+    - language: python
+      python: "2.7"
+      before_install: pip install cython
+      script: python setup.py build_ext
+    - language: python
+      python: "3.3"
+      before_install: pip install cython
+      script: python setup.py build_ext
+    - language: python
+      python: "3.5"
+      before_install: pip install cython
+      script: python setup.py build_ext
+    - language: python
+      python: "3.6"
+      before_install: pip install cython
+      script: python setup.py build_ext

MANIFEST.in

@@ -0,0 +1,11 @@
+include *.h
+include Makefile
+include ksw2_dispatch.c
+include getopt.c
+include main.c
+include README.md
+include python/mappy.c
+include python/cmappy.h
+include python/cmappy.pxd
+include python/mappy.pyx
+include python/README.rst

Makefile

@@ -1,8 +1,7 @@
-CC=			gcc
 CFLAGS=		-g -Wall -O2 -Wc++-compat
 CPPFLAGS=	-DHAVE_KALLOC
 INCLUDES=
-OBJS=		kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o index.o chain.o align.o hit.o map.o format.o ksw2_ll_sse.o
+OBJS=		kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o index.o chain.o align.o hit.o map.o format.o pe.o ksw2_ll_sse.o
 PROG=		minimap2
 PROG_EXTRA=	sdust minimap2-lite
 LIBS=		-lm -lz -lpthread
@@ -56,7 +55,7 @@ ksw2_dispatch.o:ksw2_dispatch.c ksw2.h
 		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
 
 clean:
-		rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM session*
+		rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM build dist mappy.so mappy.c python/mappy.c mappy.egg*
 
 depend:
 		(LC_ALL=C; export LC_ALL; makedepend -Y -- $(CFLAGS) $(CPPFLAGS) -- *.c)
@@ -64,12 +63,12 @@ depend:
 # DO NOT DELETE
 
 align.o: minimap.h mmpriv.h bseq.h ksw2.h kalloc.h
-bseq.o: bseq.h kseq.h
+bseq.o: bseq.h kvec.h kalloc.h kseq.h
 chain.o: minimap.h mmpriv.h bseq.h kalloc.h
 example.o: minimap.h kseq.h
 format.o: kalloc.h mmpriv.h minimap.h bseq.h
 getopt.o: getopt.h
-hit.o: mmpriv.h minimap.h bseq.h kalloc.h
+hit.o: mmpriv.h minimap.h bseq.h kalloc.h khash.h
 index.o: kthread.h bseq.h minimap.h mmpriv.h kvec.h kalloc.h khash.h
 kalloc.o: kalloc.h
 ksw2_extd2_sse.o: ksw2.h kalloc.h
@@ -77,7 +76,8 @@ ksw2_exts2_sse.o: ksw2.h kalloc.h
 ksw2_extz2_sse.o: ksw2.h kalloc.h
 ksw2_ll_sse.o: ksw2.h kalloc.h
 main.o: bseq.h minimap.h mmpriv.h getopt.h
-map.o: kthread.h kvec.h kalloc.h sdust.h mmpriv.h minimap.h bseq.h
+map.o: kthread.h kvec.h kalloc.h sdust.h mmpriv.h minimap.h bseq.h khash.h
 misc.o: minimap.h ksort.h
+pe.o: mmpriv.h minimap.h bseq.h kvec.h kalloc.h ksort.h
 sdust.o: kalloc.h kdq.h kvec.h sdust.h
 sketch.o: kvec.h kalloc.h minimap.h

NEWS.md

@@ -1,3 +1,66 @@
+Release 2.3-r531 (22 October 2017)
+----------------------------------
+
+This release come with many improvements and bug fixes:
+
+ * The **sr** preset now supports paired-end short-read alignment. Minimap2 is
+   3-4 times as fast as BWA-MEM, but is slightly less accurate on simulated
+   reads.
+
+ * Meticulous improvements to assembly-to-assembly alignment (special thanks to
+   Alexey Gurevich from the QUAST team): a) apply a small penalty to matches
+   between ambiguous bases; b) reduce missing alignments due to spurious
+   overlaps; c) introduce the short form of the `cs` tag, an improvement to the
+   SAM MD tag.
+
+ * Make sure gaps are always left-aligned.
+
+ * Recognize `U` bases from Oxford Nanopore Direct RNA-seq (#33).
+
+ * Fixed slightly wrong chaining score. Fixed slightly inaccurate coordinates
+   for split alignment.
+
+ * Fixed multiple reported bugs: 1) wrong reference name for inversion
+   alignment (#30); 2) redundant SQ lines when multiple query files are
+   specified (#39); 3) non-functioning option `-K` (#36).
+
+This release has implemented all the major features I planned five months ago,
+with the addition of spliced long-read alignment. The next couple of releases
+will focus on fine tuning of base algorithms.
+
+(2.3: 22 October 2017, r531)
+
+
+
+Release 2.2-r409 (17 September 2017)
+------------------------------------
+
+This is a feature release. It improves single-end short-read alignment and
+comes with Python bindings. Detailed changes include:
+
+ * Added the **sr** preset for single-end short-read alignment. In this mode,
+   minimap2 runs faster than BWA-MEM, but is slightly less accurate on
+   simulated data sets. Paired-end alignment is not supported as of now.
+
+ * Improved mapping quality estimate with more accurate identification of
+   repetitive hits. This mainly helps short-read alignment.
+
+ * Implemented **mappy**, a Python binding for minimap2, which is available
+   from PyPI and can be installed with `pip install --user mappy`. Python users
+   can perform read alignment without the minimap2 executable.
+
+ * Restructured the indexing APIs and documented key minimap2 APIs in the
+   header file minimap.h. Updated example.c with the new APIs. Old APIs still
+   work but may become deprecated in future.
+
+This release may output alignments different from the previous version, though
+the overall alignment statistics, such as the number of aligned bases and long
+gaps, remain close.
+
+(2.2: 17 September 2017, r409)
+
+
+
 Release 2.1.1-r341 (6 September 2017)
 -------------------------------------
 

README.md

@@ -1,56 +1,253 @@
+[![Release](https://img.shields.io/badge/Release-v2.3-blue.svg?style=flat)](https://github.com/lh3/minimap2/releases)
+[![BioConda](https://img.shields.io/conda/vn/bioconda/minimap2.svg?style=flat)](https://anaconda.org/bioconda/minimap2)
+[![PyPI](https://img.shields.io/pypi/v/mappy.svg?style=flat)](https://pypi.python.org/pypi/mappy)
+[![Python Version](https://img.shields.io/pypi/pyversions/mappy.svg?style=flat)](https://pypi.python.org/pypi/mappy)
+[![License](https://img.shields.io/badge/License-MIT-blue.svg?style=flat)](LICENSE.txt)
 [![Build Status](https://travis-ci.org/lh3/minimap2.svg?branch=master)](https://travis-ci.org/lh3/minimap2)
-## Getting Started
+[![Downloads](https://img.shields.io/github/downloads/lh3/minimap2/total.svg?style=flat)](https://github.com/lh3/minimap2/releases)
+## <a name="started"></a>Getting Started
 ```sh
 git clone https://github.com/lh3/minimap2
 cd minimap2 && make
 # long reads against a reference genome
-./minimap2 -ax map10k test/MT-human.fa test/MT-orang.fa > test.sam
+./minimap2 -a test/MT-human.fa test/MT-orang.fa > test.sam
 # create an index first and then map
-./minimap2 -x map10k -d MT-human.mmi test/MT-human.fa
-./minimap2 -ax map10k MT-human.mmi test/MT-orang.fa > test.sam
+./minimap2 -d MT-human.mmi test/MT-human.fa
+./minimap2 -a MT-human.mmi test/MT-orang.fa > test.sam
 # long-read overlap (no test data)
 ./minimap2 -x ava-pb your-reads.fa your-reads.fa > overlaps.paf
 # spliced alignment (no test data)
 ./minimap2 -ax splice ref.fa rna-seq-reads.fa > spliced.sam
-# man page
+# man page for detailed command line options
 man ./minimap2.1
 ```
+## Table of Contents
 
-## Introduction
+- [Getting Started](#started)
+- [Users' Guide](#uguide)
+  - [Installation](#install)
+  - [General usage](#general)
+  - [Use cases](#cases)
+    - [Map long noisy genomic reads](#map-long-genomic)
+    - [Map long mRNA/cDNA reads](#map-long-splice)
+    - [Find overlaps between long reads](#long-overlap)
+    - [Map short accurate genomic reads](#short-genomic)
+    - [Full genome/assembly alignment](#full-genome)
+  - [Advanced features](#advanced)
+    - [Working CIGARs with >65535 operations](#long-cigar)
+    - [The cs optional tag](#cs)
+    - [Evaluation scripts](#eval)
+  - [Algorithm overview](#algo)
+  - [Getting help](#help)
+  - [Citing minimap2](#cite)
+- [Developers' Guide](#dguide)
+- [Limitations](#limit)
 
-Minimap2 is a fast sequence mapping and alignment program that can find
-overlaps between long noisy reads, or map long reads or their assemblies to a
-reference genome optionally with detailed alignment (i.e. CIGAR). At present,
-it works efficiently with query sequences from a few kilobases to ~100
-megabases in length at an error rate ~15%. Minimap2 outputs in the [PAF][paf] or
-the [SAM format][sam]. On limited test data sets, minimap2 is over 20 times
-faster than most other long-read aligners. It will replace BWA-MEM for long
-reads and contig alignment.
+## <a name="uguide"></a>Users' Guide
 
-Minimap2 is the successor of [minimap][minimap]. It uses a similar
-minimizer-based indexing and seeding algorithm, and improves the original
-minimap with homopolyer-compressed k-mers (see also [SMARTdenovo][smartdenovo]
-and [longISLND][longislnd]), better chaining and the ability to produce CIGAR
-with fast extension alignment (see also [libgaba][gaba] and [ksw2][ksw2]) and
-piece-wise affine gap cost.
+Minimap2 is a versatile sequence alignment program that aligns DNA or mRNA
+sequences against a large reference database. Typical use cases include: (1)
+mapping PacBio or Oxford Nanopore genomic reads to the human genome; (2)
+finding overlaps between long reads with error rate up to ~15%; (3)
+splice-aware alignment of PacBio Iso-Seq or Nanopore cDNA or Direct RNA reads
+against a reference genome; (4) aligning Illumina single- or paired-end reads;
+(5) assembly-to-assembly alignment; (6) full-genome alignment between two
+closely related species with divergence below ~15%.
 
-If you use minimap2 in your work, please consider to cite:
+For ~10kb noisy reads sequences, minimap2 is tens of times faster than
+mainstream long-read mappers such as BLASR, BWA-MEM, NGMLR and GMAP. It is more
+accurate on simulated long reads and produces biologically meaningful alignment
+ready for downstream analyses. For >100bp Illumina short reads, minimap2 is
+three times as fast as BWA-MEM and Bowtie2, and as accurate on simulated data.
+Detailed evaluations are available from the [minimap2 preprint][preprint].
 
-> Li, H. (2017). Minimap2: fast pairwise alignment for long DNA sequences. [arXiv:1708.01492](https://arxiv.org/abs/1708.01492).
+### <a name="install"></a>Installation
 
-## Installation
+Minimap2 only works on x86-64 CPUs. You can acquire precompiled binaries from
+the [release page][release] with:
+```sh
+wget --no-check-certificate -O- https://github.com/lh3/minimap2/releases/download/v2.2/minimap2-2.2_x64-linux.tar.bz2 \
+  | tar -jxvf -
+./minimap2-2.2_x64-linux/minimap2
+```
+If you want to compile from the source, you need to have a C compiler, GNU make
+and zlib development files installed. Then type `make` in the source code
+directory to compile. If you see compilation errors, try `make sse2only=1`
+to disable SSE4 code, which will make minimap2 slightly slower.
 
-For modern x86-64 CPUs, just type `make` in the source code directory. This
-will compile a binary `minimap2` which you can copy to your desired location.
-If you see compilation errors, try `make sse2only=1` to disable SSE4. Minimap2
-will run a little slower. At present, minimap2 does not work with non-x86 CPUs
-or ancient CPUs that do not support SSE2. SSE2 is critical to the performance
-of minimap2.
+### <a name="general"></a>General usage
 
-## Algorithm Overview
+Without any options, minimap2 takes a reference database and a query sequence
+file as input and produce approximate mapping, without base-level alignment
+(i.e. no CIGAR), in the [PAF format][paf]:
+```sh
+minimap2 ref.fa query.fq > approx-mapping.paf
+```
+You can ask minimap2 to generate CIGAR at the `cg` tag of PAF with:
+```sh
+minimap2 -c ref.fa query.fq > alignment.paf
+```
+or to output alignments in the [SAM format][sam]:
+```sh
+minimap2 -a ref.fa query.fq > alignment.sam
+```
+Minimap2 seamlessly works with gzip'd FASTA and FASTQ formats as input. You
+don't need to convert between FASTA and FASTQ or decompress gzip'd files first.
+
+For the human reference genome, minimap2 takes a few minutes to generate a
+minimizer index for the reference before mapping. To reduce indexing time, you
+can optionally save the index with option **-d** and replace the reference
+sequence file with the index file on the minimap2 command line:
+```sh
+minimap2 -d ref.mmi ref.fa                     # indexing
+minimap2 -a ref.mmi reads.fq > alignment.sam   # alignment
+```
+***Importantly***, it should be noted that once you build the index, indexing
+parameters such as **-k**, **-w**, **-H** and **-I** can't be changed during
+mapping. If you are running minimap2 for different data types, you will
+probably need to keep multiple indexes generated with different parameters.
+This makes minimap2 different from BWA which always uses the same index
+regardless of query data types.
+
+### <a name="cases"></a>Use cases
+
+Minimap2 uses the same base algorithm for all applications. However, due to the
+different data types it supports (e.g. short vs long reads; DNA vs mRNA reads),
+minimap2 needs to be tuned for optimal performance and accuracy. It is usually
+recommended to choose a preset with option **-x**, which sets multiple
+parameters at the same time. The default setting is the same as `map-ont`.
+
+#### <a name="map-long-genomic"></a>Map long noisy genomic reads
+
+```sh
+minimap2 -ax map-pb  ref.fa pacbio-reads.fq > aln.sam   # for PacBio subreads
+minimap2 -ax map-ont ref.fa ont-reads.fq > aln.sam      # for Oxford Nanopore reads
+```
+The difference between `map-pb` and `map-ont` is that `map-pb` uses
+homopolymer-compressed (HPC) minimizers as seeds, while `map-ont` uses ordinary
+minimizers as seeds. Emperical evaluation suggests HPC minimizers improve
+performance and sensitivity when aligning PacBio reads, but hurt when aligning
+Nanopore reads.
+
+#### <a name="map-long-splice"></a>Map long mRNA/cDNA reads
+
+```sh
+minimap2 -ax splice ref.fa spliced.fq > aln.sam      # strand unknown
+minimap2 -ax splice -uf ref.fa spliced.fq > aln.sam  # assuming transcript strand
+```
+This command line has been tested on PacBio Iso-Seq reads and Nanopore 2D cDNA
+reads, and been shown to work with Nanopore 1D Direct RNA reads by others. Like
+typical RNA-seq mappers, minimap2 represents an intron with the `N` CIGAR
+operator. For spliced reads, minimap2 will try to infer the strand relative to
+transcript and may write the strand to the `ts` SAM/PAF tag.
+
+#### <a name="long-overlap"></a>Find overlaps between long reads
+
+```sh
+minimap2 -x ava-pb  reads.fq reads.fq > ovlp.paf    # PacBio read overlap
+minimap2 -x ava-ont reads.fq reads.fq > ovlp.paf    # Oxford Nanopore read overlap
+```
+Similarly, `ava-pb` uses HPC minimizers while `ava-ont` uses ordinary
+minimizers. It is usually not recommended to perform base-level alignment in
+the overlapping mode because it is slow and may produce false positive
+overlaps. However, if performance is not a concern, you may try to add `-a` or
+`-c` anyway.
+
+#### <a name="short-genomic"></a>Map short accurate genomic reads
+
+```sh
+minimap2 -ax sr ref.fa reads-se.fq > aln.sam           # single-end alignment
+minimap2 -ax sr ref.fa read1.fq read2.fq > aln.sam     # paired-end alignment
+minimap2 -ax sr ref.fa reads-interleaved.fq > aln.sam  # paired-end alignment
+```
+When two read files are specified, minimap2 reads from each file in turn and
+merge them into an interleaved stream internally. Two reads are considered to
+be paired if they are adjacent in the input stream and have the same name (with
+the `/[0-9]` suffix trimmed if present). Single- and paired-end reads can be
+mixed.
+
+Minimap2 does not work well with short spliced reads. There are many capable
+RNA-seq mappers for short reads.
+
+#### <a name="full-genome"></a>Full genome/assembly alignment
+
+```sh
+minimap2 -ax asm5 ref.fa asm.fa > aln.sam       # assembly to assembly/ref alignment
+```
+For cross-species full-genome alignment, the scoring system needs to be tuned
+according to the sequence divergence.
+
+### <a name="advanced"></a>Advanced features
+
+#### <a name="long-cigar"></a>Working CIGARs with >65535 operations
+
+Due to a design flaw, BAM does not work with CIGAR strings with >65535
+operations (SAM and CRAM work). However, for ultra-long nanopore reads minimap2
+may align ~1% of read bases with long CIGARs beyond the capability of BAM. If
+you convert such SAM/CRAM to BAM, Picard and recent samtools will throw an
+error and abort. Older samtools and other tools may create corrupted BAM.
+
+To avoid this issue, you can add option `-L` at the minimap2 command line.
+This option moves a long CIGAR to the `CG` tag and leaves a fully clipped CIGAR
+at the SAM CIGAR column. Current tools that don't read CIGAR (e.g. merging and
+sorting) still work with such BAM records; tools that read CIGAR will
+effectively ignore these records. I have pull requests to the SAM spec, htslib,
+htsjdk, bedtools2, Rsamtools and igv.js. If they are accepted, future versions
+of these tools will seamlessly recognize long-cigar records generated by option
+`-L`.
+
+**TD;DR**: if you work with ultra-long reads and use tools that only process
+BAM files, please add option `-L`.
+
+#### <a name="cs"></a>The cs optional tag
+
+The `cs` SAM/PAF tag encodes bases at mismatches and INDELs. It matches regular
+expression `/(:[0-9]+|\*[a-z][a-z]|[=\+\-][A-Za-z]+)+/`. Like CIGAR, `cs`
+consists of series of operations.  Each leading character specifies the
+operation; the following sequence is the one involved in the operation.
+
+The `cs` tag is enabled by command line option `--cs`. The following alignment,
+for example:
+```txt
+CGATCGATAAATAGAGTAG---GAATAGCA
+||||||   ||||||||||   |||| |||
+CGATCG---AATAGAGTAGGTCGAATtGCA
+```
+is represented as `:6-ata:10+gtc:4*at:3`, where `:[0-9]+` represents an
+identical block, `-ata` represents a deltion, `+gtc` an insertion and `*at`
+indicates reference base `a` is substituted with a query base `t`. It is
+similar to the `MD` SAM tag but is standalone and easier to parse.
+
+If `--cs=long` is used, the `cs` string also contains identical sequences in
+the alignment. The above example will become
+`=CGATCG-ata=AATAGAGTAG+gtc=GAAT*at=GCA`. The long form of `cs` encodes both
+reference and query sequences in one string.
+
+#### <a name="eval"></a>Evaluation scripts
+
+Minimap2 comes with several (java)scripts for evaluating the accuracy of
+minimap2. These scripts require the [k8][k8] javascript shell to run.
+Recent minimap2 binary release tar-balls contain a copy of k8 executable, a
+single file. Here are a few examples on how to use these scripts:
+
+```sh
+# Generate reads from PBSIM alignment (truth encoded in read names)
+k8 misc/sim-pbsim.js ref.fa.fai pbsim-aln.maf > pbsim-reads.fq
+# Generate reads from mason2 alignment (not tested for simulated SVs)
+k8 misc/sim-mason2.js mason2-aln.sam > mason2-reads.fq
+# Evaluate mapping accuracy with ROC-like curve
+k8 misc/sim-eval.js my-aln.sam.gz > result.txt
+k8 misc/sim-eval.js my-aln.paf.gz > result.txt
+# Collect alignment statistics
+k8 misc/mapstat.js my-aln.sam > result.txt
+# Compare spliced junctions to existing gene annotations
+k8 misc/intron-eval.js anno.gtf my-spliced-aln.sam > result.txt
+```
+
+### <a name="algo"></a>Algorithm overview
 
 In the following, minimap2 command line options have a dash ahead and are
-highlighted in bold.
+highlighted in bold. The description may help to tune minimap2 parameters.
 
 1. Read **-I** [=*4G*] reference bases, extract (**-k**,**-w**)-minimizers and
    index them in a hash table.
@@ -91,14 +288,40 @@ highlighted in bold.
 9. If there are more reference sequences, reopen the query file from the start
    and go to step 1; otherwise stop.
 
-## Limitations
+### <a name="help"></a>Getting help
+
+Manpage [minimap2.1](minimap2.1) provides detailed description of minimap2
+command line options and optional tags. If you encounter bugs or have further
+questions or requests, you can raise an issue at the [issue page][issue].
+There is not a specific mailing list for the time being.
+
+### <a name="cite"></a>Citing minimap2
+
+If you use minimap2 in your work, please consider to cite:
+
+> Li, H. (2017). Minimap2: fast pairwise alignment for long nucleotide sequences. [arXiv:1708.01492][preprint]
+
+## <a name="dguide"></a>Developers' Guide
+
+Minimap2 is not only a command line tool, but also a programming library.
+It provides C APIs to build/load index and to align sequences against the
+index. File [example.c](example.c) demonstrates typical uses of C APIs. Header
+file [minimap.h](minimap.h) gives more detailed API documentation. Minimap2
+aims to keep APIs in this header stable. File [mmpriv.h](mmpriv.h) contains
+additional private APIs which may be subjected to changes frequently.
+
+This repository also provides Python bindings to a subset of C APIs. File
+[python/README.rst](python/README.rst) gives the full documentation;
+[python/minimap2.py](python/minimap2.py) shows an example. This Python
+extension, mappy, is also [available from PyPI][mappypypi] via `pip install
+mappy` or [from BioConda][mappyconda] via `conda install -c bioconda mappy`.
+
+## <a name="limit"></a>Limitations
 
 * Minimap2 may produce suboptimal alignments through long low-complexity
   regions where seed positions may be suboptimal. This should not be a big
   concern because even the optimal alignment may be wrong in such regions.
 
-* Minimap2 does not work well with Illumina short reads as of now.
-
 * Minimap2 requires SSE2 instructions to compile. It is possible to add
   non-SSE2 support, but it would make minimap2 slower by several times.
 
@@ -115,3 +338,9 @@ warmly welcomed.
 [longislnd]: https://www.ncbi.nlm.nih.gov/pubmed/27667791
 [gaba]: https://github.com/ocxtal/libgaba
 [ksw2]: https://github.com/lh3/ksw2
+[preprint]: https://arxiv.org/abs/1708.01492
+[release]: https://github.com/lh3/minimap2/releases
+[mappypypi]: https://pypi.python.org/pypi/mappy
+[mappyconda]: https://anaconda.org/bioconda/mappy
+[issue]: https://github.com/lh3/minimap2/issues
+[k8]: https://github.com/attractivechaos/k8

align.c

@@ -1,5 +1,6 @@
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
 #include "minimap.h"
 #include "mmpriv.h"
 #include "ksw2.h"
@@ -61,51 +62,113 @@ static int mm_check_zdrop(const uint8_t *qseq, const uint8_t *tseq, uint32_t n_c
 	return 0;
 }
 
-static void mm_update_extra(mm_extra_t *p, const uint8_t *qseq, const uint8_t *tseq, const int8_t *mat, int8_t q, int8_t e)
+static void mm_fix_cigar(mm_reg1_t *r, const uint8_t *qseq, const uint8_t *tseq, int *qshift, int *tshift)
+{
+	mm_extra_t *p = r->p;
+	int32_t k, toff = 0, qoff = 0, to_shrink = 0;
+	*qshift = *tshift = 0;
+	if (p->n_cigar <= 1) return;
+	for (k = 0; k < p->n_cigar; ++k) { // indel left alignment
+		uint32_t op = p->cigar[k]&0xf, len = p->cigar[k]>>4;
+		if (len == 0) to_shrink = 1;
+		if (op == 0) {
+			toff += len, qoff += len;
+		} else if (op == 1 || op == 2) { // insertion or deletion
+			if (k > 0 && k < p->n_cigar - 1 && (p->cigar[k-1]&0xf) == 0 && (p->cigar[k+1]&0xf) == 0) {
+				int l, prev_len = p->cigar[k-1] >> 4;
+				if (op == 1) {
+					for (l = 0; l < prev_len; ++l)
+						if (qseq[qoff - 1 - l] != qseq[qoff + len - 1 - l])
+							break;
+				} else {
+					for (l = 0; l < prev_len; ++l)
+						if (tseq[toff - 1 - l] != tseq[toff + len - 1 - l])
+							break;
+				}
+				if (l > 0)
+					p->cigar[k-1] -= l<<4, p->cigar[k+1] += l<<4, qoff -= l, toff -= l;
+				if (l == prev_len) to_shrink = 1;
+			}
+			if (op == 1) qoff += len;
+			else toff += len;
+		} else if (op == 3) {
+			toff += len;
+		}
+	}
+	assert(qoff == r->qe - r->qs && toff == r->re - r->rs);
+	if (to_shrink) { // squeeze out zero-length operations
+		int32_t l = 0;
+		for (k = 0; k < p->n_cigar; ++k) // squeeze out zero-length operations
+			if (p->cigar[k]>>4 != 0)
+				p->cigar[l++] = p->cigar[k];
+		p->n_cigar = l;
+		for (k = l = 0; k < p->n_cigar; ++k) // merge two adjacent operations if they are the same
+			if (k == p->n_cigar - 1 || (p->cigar[k]&0xf) != (p->cigar[k+1]&0xf))
+				p->cigar[l++] = p->cigar[k];
+			else p->cigar[k+1] += p->cigar[k]>>4<<4; // add length to the next CIGAR operator
+		p->n_cigar = l;
+	}
+	if ((p->cigar[0]&0xf) == 1 || (p->cigar[0]&0xf) == 2) { // get rid of leading I or D
+		int32_t l = p->cigar[0] >> 4;
+		if ((p->cigar[0]&0xf) == 1) r->qs += l, *qshift = l;
+		else r->rs += l, *tshift = l;
+		--p->n_cigar;
+		memmove(p->cigar, p->cigar + 1, p->n_cigar * 4);
+	}
+}
+
+static void mm_update_extra(mm_reg1_t *r, const uint8_t *qseq, const uint8_t *qual, const uint8_t *tseq, const int8_t *mat, int8_t q, int8_t e)
 {
 	uint32_t k, l, toff = 0, qoff = 0;
-	int32_t s = 0, max = 0, n_gtag = 0, n_ctac = 0;
+	int32_t s = 0, max = 0, qshift, tshift;
+	mm_extra_t *p = r->p;
 	if (p == 0) return;
+	mm_fix_cigar(r, qseq, tseq, &qshift, &tshift);
+	qseq += qshift, tseq += tshift; // qseq and tseq may be shifted due to the removal of leading I/D
+	r->blen = r->mlen = 0;
 	for (k = 0; k < p->n_cigar; ++k) {
 		uint32_t op = p->cigar[k]&0xf, len = p->cigar[k]>>4;
 		if (op == 0) { // match/mismatch
+			int n_ambi = 0, n_diff = 0;
+			float n_diff2 = 0.0f;
 			for (l = 0; l < len; ++l) {
 				int cq = qseq[qoff + l], ct = tseq[toff + l];
-				if (ct > 3 || cq > 3) ++p->n_ambi;
-				else if (ct != cq) ++p->n_diff;
+				if (ct > 3 || cq > 3) ++n_ambi;
+				else if (ct != cq) {
+					++n_diff;
+					n_diff2 += qual == 0 || qual[qoff + l] >= 20? 1.0f : .05f * qual[qoff + l];
+				}
 				s += mat[ct * 5 + cq];
 				if (s < 0) s = 0;
 				else max = max > s? max : s;
 			}
-			toff += len, qoff += len, p->blen += len;
+			r->blen += len - n_ambi, r->mlen += len - (n_ambi + n_diff), p->n_ambi += n_ambi;
+			p->n_diff2 += n_diff2, p->blen2 += len - n_ambi;
+			toff += len, qoff += len;
 		} else if (op == 1) { // insertion
 			int n_ambi = 0;
 			for (l = 0; l < len; ++l)
 				if (qseq[qoff + l] > 3) ++n_ambi;
-			qoff += len, p->blen += len;
-			p->n_ambi += n_ambi, p->n_diff += len - n_ambi;
+			r->blen += len - n_ambi, p->n_ambi += n_ambi;
+			p->n_diff2 += 1.0f, ++p->blen2;
 			s -= q + e * len;
 			if (s < 0) s = 0;
+			qoff += len;
 		} else if (op == 2) { // deletion
 			int n_ambi = 0;
 			for (l = 0; l < len; ++l)
 				if (tseq[toff + l] > 3) ++n_ambi;
-			toff += len, p->blen += len;
-			p->n_ambi += n_ambi, p->n_diff += len - n_ambi;
+			r->blen += len - n_ambi, p->n_ambi += n_ambi;
+			p->n_diff2 += 1.0f, ++p->blen2;
 			s -= q + e * len;
 			if (s < 0) s = 0;
+			toff += len;
 		} else if (op == 3) { // intron
-			uint8_t b[4];
-			b[0] = tseq[toff], b[1] = tseq[toff+1];
-			b[2] = tseq[toff+len-2], b[3] = tseq[toff+len-1];
-			if (memcmp(b, "\2\3\0\2", 4) == 0) ++n_gtag;
-			else if (memcmp(b, "\1\3\0\1", 4) == 0) ++n_ctac;
-			toff += len, p->blen += len;
+			toff += len;
 		}
 	}
 	p->dp_max = max;
-	if (n_gtag > n_ctac) p->trans_strand = 1;
-	else if (n_gtag < n_ctac) p->trans_strand = 2;
+	assert(qoff == r->qe - r->qs && toff == r->re - r->rs);
 }
 
 static void mm_append_cigar(mm_reg1_t *r, uint32_t n_cigar, uint32_t *cigar) // TODO: this calls the libc realloc()
@@ -133,8 +196,9 @@ static void mm_append_cigar(mm_reg1_t *r, uint32_t n_cigar, uint32_t *cigar) //
 	}
 }
 
-static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint8_t *qseq, int tlen, const uint8_t *tseq, const int8_t *mat, int w, int flag, ksw_extz_t *ez)
+static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint8_t *qseq, int tlen, const uint8_t *tseq, const int8_t *mat, int w, int end_bonus, int flag, ksw_extz_t *ez)
 {
+	int zdrop = opt->zdrop;
 	if (mm_dbg_flag & MM_DBG_PRINT_ALN_SEQ) {
 		int i;
 		fprintf(stderr, "===> q=(%d,%d), e=(%d,%d), bw=%d, flag=%d, zdrop=%d <===\n", opt->q, opt->q2, opt->e, opt->e2, w, flag, opt->zdrop);
@@ -144,11 +208,11 @@ static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint
 		fputc('\n', stderr);
 	}
 	if (opt->flag & MM_F_SPLICE)
-		ksw_exts2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->noncan, opt->zdrop, flag, ez);
+		ksw_exts2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->noncan, zdrop, flag, ez);
 	else if (opt->q == opt->q2 && opt->e == opt->e2)
-		ksw_extz2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, w, opt->zdrop, flag, ez);
+		ksw_extz2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, w, zdrop, end_bonus, flag, ez);
 	else
-		ksw_extd2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->e2, w, opt->zdrop, flag, ez);
+		ksw_extd2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->e2, w, zdrop, end_bonus, flag, ez);
 }
 
 static inline int mm_get_hplen_back(const mm_idx_t *mi, uint32_t rid, uint32_t x)
@@ -253,8 +317,37 @@ static void mm_fix_bad_ends(const mm_reg1_t *r, const mm128_t *a, int bw, int32_
 	}
 }
 
-static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, uint8_t *qseq0[2], mm_reg1_t *r, mm_reg1_t *r2, mm128_t *a, ksw_extz_t *ez, int splice_flag)
+static void mm_max_stretch(const mm_mapopt_t *opt, const mm_reg1_t *r, const mm128_t *a, int32_t *as, int32_t *cnt)
 {
+	int32_t i, score, max_score, len, max_i, max_len;
+
+	*as = r->as, *cnt = r->cnt;
+	if (r->cnt < 2) return;
+
+	max_score = -1, max_i = -1, max_len = 0;
+	score = a[r->as].y >> 32 & 0xff, len = 1;
+	for (i = r->as + 1; i < r->as + r->cnt; ++i) {
+		int32_t lq, lr, q_span;
+		q_span = a[i].y >> 32 & 0xff;
+		lr = (int32_t)a[i].x - (int32_t)a[i-1].x;
+		lq = (int32_t)a[i].y - (int32_t)a[i-1].y;
+		if (lq == lr) {
+			score += lq < q_span? lq : q_span;
+			++len;
+		} else {
+			if (score > max_score)
+				max_score = score, max_len = len, max_i = i - len;
+			score = q_span, len = 1;
+		}
+	}
+	if (score > max_score)
+		max_score = score, max_len = len, max_i = i - len;
+	*as = max_i, *cnt = max_len;
+}
+
+static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, uint8_t *qseq0[2], uint8_t *qual0[2], mm_reg1_t *r, mm_reg1_t *r2, int n_a, mm128_t *a, ksw_extz_t *ez, int splice_flag)
+{
+	int is_sr = !!(opt->flag & MM_F_SR), is_splice = !!(opt->flag & MM_F_SPLICE);
 	int32_t rid = a[r->as].x<<1>>33, rev = a[r->as].x>>63, as1, cnt1;
 	uint8_t *tseq, *qseq;
 	int32_t i, l, bw, dropped = 0, extra_flag = 0, rs0, re0, qs0, qe0;
@@ -262,47 +355,101 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 	int32_t rs1, qs1, re1, qe1;
 	int8_t mat[25];
 
+	if (is_sr) assert(!mi->is_hpc); // HPC won't work with SR because with HPC we can't easily tell if there is a gap
+
+	r2->cnt = 0;
 	if (r->cnt == 0) return;
 	ksw_gen_simple_mat(5, mat, opt->a, opt->b);
 	bw = (int)(opt->bw * 1.5 + 1.);
 
-	r2->cnt = 0;
-	if (!(opt->flag & MM_F_SPLICE))
-		mm_fix_bad_ends(r, a, opt->bw, &as1, &cnt1);
-	else as1 = r->as, cnt1 = r->cnt;
-	mm_filter_bad_seeds(km, as1, cnt1, a, 10, 40, opt->max_gap>>1, 10);
-	mm_adjust_minier(mi, qseq0, &a[as1], &rs, &qs);
-	mm_adjust_minier(mi, qseq0, &a[as1 + cnt1 - 1], &re, &qe);
+	if (is_sr && !mi->is_hpc) {
+		mm_max_stretch(opt, r, a, &as1, &cnt1);
+		rs = (int32_t)a[as1].x + 1 - (int32_t)(a[as1].y>>32&0xff);
+		qs = (int32_t)a[as1].y + 1 - (int32_t)(a[as1].y>>32&0xff);
+		re = (int32_t)a[as1+cnt1-1].x + 1;
+		qe = (int32_t)a[as1+cnt1-1].y + 1;
+	} else {
+		if (!is_splice)
+			mm_fix_bad_ends(r, a, opt->bw, &as1, &cnt1);
+		else as1 = r->as, cnt1 = r->cnt;
+		mm_filter_bad_seeds(km, as1, cnt1, a, 10, 40, opt->max_gap>>1, 10);
+		mm_adjust_minier(mi, qseq0, &a[as1], &rs, &qs);
+		mm_adjust_minier(mi, qseq0, &a[as1 + cnt1 - 1], &re, &qe);
+	}
+	assert(cnt1 > 0);
 
-	if (opt->flag & MM_F_SPLICE) {
+	if (is_splice) {
 		if (splice_flag & MM_F_SPLICE_FOR) extra_flag |= rev? KSW_EZ_SPLICE_REV : KSW_EZ_SPLICE_FOR;
 		if (splice_flag & MM_F_SPLICE_REV) extra_flag |= rev? KSW_EZ_SPLICE_FOR : KSW_EZ_SPLICE_REV;
-		if (splice_flag & MM_F_SPLICE_BOTH) extra_flag |= KSW_EZ_SPLICE_FOR|KSW_EZ_SPLICE_REV;
 	}
 
-	// compute rs0 and qs0
-	if (r->split && as1 > 0) {
-		mm_adjust_minier(mi, qseq0, &a[as1-1], &rs0, &qs0);
+	/* Look for the start and end of regions to perform DP. This sounds easy
+	 * but is in fact tricky. Excessively small regions lead to unnecessary
+	 * clippings and lose alignable sequences. Excessively large regions
+	 * occasionally lead to large overlaps between two chains and may cause
+	 * loss of alignments in corner cases. */
+	if (is_sr) {
+		qs0 = 0, qe0 = qlen;
+		l = qs;
+		l += l * opt->a + opt->end_bonus > opt->q? (l * opt->a + opt->end_bonus - opt->q) / opt->e : 0;
+		rs0 = rs - l > 0? rs - l : 0;
+		l = qlen - qe;
+		l += l * opt->a + opt->end_bonus > opt->q? (l * opt->a + opt->end_bonus - opt->q) / opt->e : 0;
+		re0 = re + l < mi->seq[rid].len? re + l : mi->seq[rid].len;
 	} else {
-		if (qs > 0 && rs > 0) { // actually this is always true
+		// compute rs0 and qs0
+		rs0 = (int32_t)a[r->as].x + 1 - (int32_t)(a[r->as].y>>32&0xff);
+		qs0 = (int32_t)a[r->as].y + 1 - (int32_t)(a[r->as].y>>32&0xff);
+		if (rs0 < 0) rs0 = 0; // this may happen when HPC is in use
+		assert(qs0 >= 0); // this should never happen, or it is logic error
+		rs1 = qs1 = 0;
+		for (i = r->as - 1, l = 0; i >= 0 && a[i].x>>32 == a[r->as].x>>32; --i) { // inspect nearby seeds
+			int32_t x = (int32_t)a[i].x + 1 - (int32_t)(a[i].y>>32&0xff);
+			int32_t y = (int32_t)a[i].y + 1 - (int32_t)(a[i].y>>32&0xff);
+			if (x < rs0 && y < qs0) {
+				if (++l > opt->min_cnt) {
+					l = rs0 - x > qs0 - y? rs0 - x : qs0 - y;
+					rs1 = rs0 - l, qs1 = qs0 - l;
+					break;
+				}
+			}
+		}
+		if (qs > 0 && rs > 0) {
 			l = qs < opt->max_gap? qs : opt->max_gap;
-			qs0 = qs - l;
+			qs1 = qs1 > qs - l? qs1 : qs - l;
+			qs0 = qs0 < qs1? qs0 : qs1; // at least include qs0
 			l += l * opt->a > opt->q? (l * opt->a - opt->q) / opt->e : 0;
 			l = l < opt->max_gap? l : opt->max_gap;
 			l = l < rs? l : rs;
-			rs0 = rs - l;
+			rs1 = rs1 > rs - l? rs1 : rs - l;
+			rs0 = rs0 < rs1? rs0 : rs1;
 		} else rs0 = rs, qs0 = qs;
-
+		// compute re0 and qe0
+		re0 = (int32_t)a[r->as + r->cnt - 1].x + 1;
+		qe0 = (int32_t)a[r->as + r->cnt - 1].y + 1;
+		re1 = mi->seq[rid].len, qe1 = qlen;
+		for (i = r->as + r->cnt, l = 0; i < n_a && a[i].x>>32 == a[r->as].x>>32; ++i) { // inspect nearby seeds
+			int32_t x = (int32_t)a[i].x + 1;
+			int32_t y = (int32_t)a[i].y + 1;
+			if (x > re0 && y > qe0) {
+				if (++l > opt->min_cnt) {
+					l = x - re0 > y - qe0? x - re0 : y - qe0;
+					re1 = re0 + l, qe1 = qe0 + l;
+					break;
+				}
+			}
+		}
+		if (qe < qlen && re < mi->seq[rid].len) {
+			l = qlen - qe < opt->max_gap? qlen - qe : opt->max_gap;
+			qe1 = qe1 < qe + l? qe1 : qe + l;
+			qe0 = qe0 > qe1? qe0 : qe1; // at least include qe0
+			l += l * opt->a > opt->q? (l * opt->a - opt->q) / opt->e : 0;
+			l = l < opt->max_gap? l : opt->max_gap;
+			l = l < mi->seq[rid].len - re? l : mi->seq[rid].len - re;
+			re1 = re1 < re + l? re1 : re + l;
+			re0 = re0 > re1? re0 : re1;
+		} else re0 = re, qe0 = qe;
 	}
-	// compute re0 and qe0
-	if (qe < qlen && re < mi->seq[rid].len) {
-		l = qlen - qe < opt->max_gap? qlen - qe : opt->max_gap;
-		qe0 = qe + l;
-		l += l * opt->a > opt->q? (l * opt->a - opt->q) / opt->e : 0;
-		l = l < opt->max_gap? l : opt->max_gap;
-		l = l < mi->seq[rid].len - re? l : mi->seq[rid].len - re;
-		re0 = re + l;
-	} else re0 = re, qe0 = qe;
 
 	assert(re0 > rs0);
 	tseq = (uint8_t*)kmalloc(km, re0 - rs0);
@@ -312,44 +459,57 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 		mm_idx_getseq(mi, rid, rs0, rs, tseq);
 		mm_seq_rev(qs - qs0, qseq);
 		mm_seq_rev(rs - rs0, tseq);
-		mm_align_pair(km, opt, qs - qs0, qseq, rs - rs0, tseq, mat, bw, extra_flag|KSW_EZ_EXTZ_ONLY|KSW_EZ_RIGHT|KSW_EZ_REV_CIGAR, ez);
+		mm_align_pair(km, opt, qs - qs0, qseq, rs - rs0, tseq, mat, bw, opt->end_bonus, extra_flag|KSW_EZ_EXTZ_ONLY|KSW_EZ_RIGHT|KSW_EZ_REV_CIGAR, ez);
 		if (ez->n_cigar > 0) {
 			mm_append_cigar(r, ez->n_cigar, ez->cigar);
 			r->p->dp_score += ez->max;
 		}
-		rs1 = rs - (ez->max_t + 1);
-		qs1 = qs - (ez->max_q + 1);
+		rs1 = rs - (ez->reach_end? ez->mqe_t + 1 : ez->max_t + 1);
+		qs1 = qs - (ez->reach_end? qs - qs0 : ez->max_q + 1);
 		mm_seq_rev(qs - qs0, qseq);
 	} else rs1 = rs, qs1 = qs;
 	re1 = rs, qe1 = qs;
 	assert(qs1 >= 0 && rs1 >= 0);
 
-	for (i = 1; i < cnt1; ++i) { // gap filling
+	for (i = is_sr? cnt1 - 1 : 1; i < cnt1; ++i) { // gap filling
 		if ((a[as1+i].y & (MM_SEED_IGNORE|MM_SEED_TANDEM)) && i != cnt1 - 1) continue;
-		mm_adjust_minier(mi, qseq0, &a[as1 + i], &re, &qe);
+		if (is_sr && !mi->is_hpc) {
+			re = (int32_t)a[as1 + i].x + 1;
+			qe = (int32_t)a[as1 + i].y + 1;
+		} else mm_adjust_minier(mi, qseq0, &a[as1 + i], &re, &qe);
 		re1 = re, qe1 = qe;
 		if (i == cnt1 - 1 || (a[as1+i].y&MM_SEED_LONG_JOIN) || (qe - qs >= opt->min_ksw_len && re - rs >= opt->min_ksw_len)) {
-			int bw1 = bw;
+			int j, bw1 = bw;
 			if (a[as1+i].y & MM_SEED_LONG_JOIN)
 				bw1 = qe - qs > re - rs? qe - qs : re - rs;
 			qseq = &qseq0[rev][qs];
 			mm_idx_getseq(mi, rid, rs, re, tseq);
-			mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, extra_flag|KSW_EZ_APPROX_MAX, ez);
+			if (is_sr) { // perform ungapped alignment
+				assert(qe - qs == re - rs);
+				ksw_reset_extz(ez);
+				for (j = 0, ez->score = 0; j < qe - qs; ++j) {
+					if (qseq[j] >= 4 || tseq[j] >= 4) ez->score += opt->e2;
+					else ez->score += qseq[j] == tseq[j]? opt->a : -opt->b;
+				}
+				ez->cigar = ksw_push_cigar(km, &ez->n_cigar, &ez->m_cigar, ez->cigar, 0, qe - qs);
+			} else { // perform normal gapped alignment
+				mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, -1, extra_flag|KSW_EZ_APPROX_MAX, ez); // first pass: with approximate Z-drop
+			}
 			if (mm_check_zdrop(qseq, tseq, ez->n_cigar, ez->cigar, mat, opt->q, opt->e, opt->zdrop))
-				mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, extra_flag, ez);
+				mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, -1, extra_flag, ez); // second pass: lift approximate
 			if (ez->n_cigar > 0)
 				mm_append_cigar(r, ez->n_cigar, ez->cigar);
 			if (ez->zdropped) { // truncated by Z-drop; TODO: sometimes Z-drop kicks in because the next seed placement is wrong. This can be fixed in principle.
-				int j;
 				for (j = i - 1; j >= 0; --j)
-					if ((int32_t)a[as1 + j].x < re + ez->max_t)
+					if ((int32_t)a[as1 + j].x <= rs + ez->max_t)
 						break;
 				dropped = 1;
+				if (j < 0) j = 0;
 				r->p->dp_score += ez->max;
 				re1 = rs + (ez->max_t + 1);
 				qe1 = qs + (ez->max_q + 1);
 				if (cnt1 - (j + 1) >= opt->min_cnt)
-					mm_split_reg(r, r2, j + 1, qlen, a);
+					mm_split_reg(r, r2, as1 + j + 1 - r->as, qlen, a);
 				break;
 			} else r->p->dp_score += ez->score;
 			rs = re, qs = qe;
@@ -359,13 +519,13 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 	if (!dropped && qe < qe0 && re < re0) { // right extension
 		qseq = &qseq0[rev][qe];
 		mm_idx_getseq(mi, rid, re, re0, tseq);
-		mm_align_pair(km, opt, qe0 - qe, qseq, re0 - re, tseq, mat, bw, extra_flag|KSW_EZ_EXTZ_ONLY, ez);
+		mm_align_pair(km, opt, qe0 - qe, qseq, re0 - re, tseq, mat, bw, opt->end_bonus, extra_flag|KSW_EZ_EXTZ_ONLY, ez);
 		if (ez->n_cigar > 0) {
 			mm_append_cigar(r, ez->n_cigar, ez->cigar);
 			r->p->dp_score += ez->max;
 		}
-		re1 = re + (ez->max_t + 1);
-		qe1 = qe + (ez->max_q + 1);
+		re1 = re + (ez->reach_end? ez->mqe_t + 1 : ez->max_t + 1);
+		qe1 = qe + (ez->reach_end? qe0 - qe : ez->max_q + 1);
 	}
 	assert(qe1 <= qlen);
 
@@ -376,7 +536,7 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 	assert(re1 - rs1 <= re0 - rs0);
 	if (r->p) {
 		mm_idx_getseq(mi, rid, rs1, re1, tseq);
-		mm_update_extra(r->p, &qseq0[r->rev][qs1], tseq, mat, opt->q, opt->e);
+		mm_update_extra(r, &qseq0[r->rev][qs1], qual0[r->rev]? &qual0[r->rev][qs1] : 0, tseq, mat, opt->q, opt->e);
 		if (rev && r->p->trans_strand)
 			r->p->trans_strand ^= 3; // flip to the read strand
 	}
@@ -384,10 +544,10 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int
 	kfree(km, tseq);
 }
 
-static int mm_align1_inv(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, uint8_t *qseq0[2], const mm_reg1_t *r1, const mm_reg1_t *r2, mm_reg1_t *r_inv, ksw_extz_t *ez)
+static int mm_align1_inv(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, uint8_t *qseq0[2], uint8_t *qual0[2], const mm_reg1_t *r1, const mm_reg1_t *r2, mm_reg1_t *r_inv, ksw_extz_t *ez)
 {
 	int tl, ql, score, ret = 0, q_off, t_off;
-	uint8_t *tseq, *qseq;
+	uint8_t *tseq, *qseq, *qual;
 	int8_t mat[25];
 	void *qp;
 
@@ -405,6 +565,7 @@ static int mm_align1_inv(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, i
 	tseq = (uint8_t*)kmalloc(km, tl);
 	mm_idx_getseq(mi, r1->rid, r1->re, r2->rs, tseq);
 	qseq = &qseq0[!r1->rev][qlen - r2->qs];
+	qual = qual0[!r1->rev]? &qseq0[!r1->rev][qlen - r2->qs] : 0;
 
 	mm_seq_rev(ql, qseq);
 	mm_seq_rev(tl, tseq);
@@ -415,17 +576,18 @@ static int mm_align1_inv(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, i
 	mm_seq_rev(tl, tseq);
 	if (score < opt->min_dp_max) goto end_align1_inv;
 	q_off = ql - (q_off + 1), t_off = tl - (t_off + 1);
-	mm_align_pair(km, opt, ql - q_off, qseq + q_off, tl - t_off, tseq + t_off, mat, (int)(opt->bw * 1.5), KSW_EZ_EXTZ_ONLY, ez);
+	mm_align_pair(km, opt, ql - q_off, qseq + q_off, tl - t_off, tseq + t_off, mat, (int)(opt->bw * 1.5), -1, KSW_EZ_EXTZ_ONLY, ez);
 	if (ez->n_cigar == 0) goto end_align1_inv; // should never be here
 	mm_append_cigar(r_inv, ez->n_cigar, ez->cigar);
 	r_inv->p->dp_score = ez->max;
-	mm_update_extra(r_inv->p, qseq + q_off, tseq + t_off, mat, opt->q, opt->e);
 	r_inv->id = -1;
 	r_inv->parent = MM_PARENT_UNSET;
 	r_inv->inv = 1;
 	r_inv->rev = !r1->rev;
+	r_inv->rid = r1->rid;
 	r_inv->qs = r1->qe + q_off, r_inv->qe = r_inv->qs + ez->max_q + 1;
 	r_inv->rs = r1->re + t_off, r_inv->re = r_inv->rs + ez->max_t + 1;
+	mm_update_extra(r_inv, &qseq[q_off], qual? &qual[q_off] : 0, &tseq[t_off], mat, opt->q, opt->e);
 	ret = 1;
 end_align1_inv:
 	kfree(km, tseq);
@@ -442,31 +604,38 @@ static inline mm_reg1_t *mm_insert_reg(const mm_reg1_t *r, int i, int *n_regs, m
 	return regs;
 }
 
-mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, int *n_regs_, mm_reg1_t *regs, mm128_t *a)
+mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, const char *qual, int *n_regs_, mm_reg1_t *regs, mm128_t *a)
 {
 	extern unsigned char seq_nt4_table[256];
-	int32_t i, n_regs = *n_regs_;
-	uint8_t *qseq0[2];
+	int32_t i, n_regs = *n_regs_, n_a;
+	uint8_t *qseq0[2], *qual0[2];
 	ksw_extz_t ez;
 
 	// encode the query sequence
-	qseq0[0] = (uint8_t*)kmalloc(km, qlen);
-	qseq0[1] = (uint8_t*)kmalloc(km, qlen);
+	qseq0[0] = (uint8_t*)kmalloc(km, qlen * 2);
+	qseq0[1] = qseq0[0] + qlen;
 	for (i = 0; i < qlen; ++i) {
 		qseq0[0][i] = seq_nt4_table[(uint8_t)qstr[i]];
 		qseq0[1][qlen - 1 - i] = qseq0[0][i] < 4? 3 - qseq0[0][i] : 4;
 	}
+	if (qual) {
+		qual0[0] = (uint8_t*)kmalloc(km, qlen * 2);
+		qual0[1] = qual0[0] + qlen;
+		for (i = 0; i < qlen; ++i)
+			qual0[0][i] = qual0[1][qlen - 1 - i] = qual[i] - 33;
+	} else qual0[0] = qual0[1] = 0;
 
 	// align through seed hits
+	n_a = mm_squeeze_a(km, n_regs, regs, a);
 	memset(&ez, 0, sizeof(ksw_extz_t));
 	for (i = 0; i < n_regs; ++i) {
 		mm_reg1_t r2;
-		if ((opt->flag&MM_F_SPLICE) && (opt->flag&MM_F_SPLICE_FOR) && (opt->flag&MM_F_SPLICE_REV)) {
+		if ((opt->flag&MM_F_SPLICE) && (opt->flag&MM_F_SPLICE_FOR) && (opt->flag&MM_F_SPLICE_REV)) { // then do two rounds of alignments for both strands
 			mm_reg1_t s[2], s2[2];
 			int which, trans_strand;
 			s[0] = s[1] = regs[i];
-			mm_align1(km, opt, mi, qlen, qseq0, &s[0], &s2[0], a, &ez, MM_F_SPLICE_FOR);
-			mm_align1(km, opt, mi, qlen, qseq0, &s[1], &s2[1], a, &ez, MM_F_SPLICE_REV);
+			mm_align1(km, opt, mi, qlen, qseq0, qual0, &s[0], &s2[0], n_a, a, &ez, MM_F_SPLICE_FOR);
+			mm_align1(km, opt, mi, qlen, qseq0, qual0, &s[1], &s2[1], n_a, a, &ez, MM_F_SPLICE_REV);
 			if (s[0].p->dp_score > s[1].p->dp_score) which = 0, trans_strand = 1;
 			else if (s[0].p->dp_score < s[1].p->dp_score) which = 1, trans_strand = 2;
 			else trans_strand = 3, which = (qlen + s[0].p->dp_score) & 1; // randomly choose a strand, effectively
@@ -478,19 +647,22 @@ mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *m
 				free(s[0].p);
 			}
 			regs[i].p->trans_strand = trans_strand;
-		} else {
-			mm_align1(km, opt, mi, qlen, qseq0, &regs[i], &r2, a, &ez, opt->flag);
-			if ((opt->flag&MM_F_SPLICE) && !(opt->flag&MM_F_SPLICE_BOTH))
+		} else { // one round of alignment
+			mm_align1(km, opt, mi, qlen, qseq0, qual0, &regs[i], &r2, n_a, a, &ez, opt->flag);
+			if (opt->flag&MM_F_SPLICE)
 				regs[i].p->trans_strand = opt->flag&MM_F_SPLICE_FOR? 1 : 2;
 		}
 		if (r2.cnt > 0) regs = mm_insert_reg(&r2, i, &n_regs, regs);
-		if (i > 0 && mm_align1_inv(km, opt, mi, qlen, qseq0, &regs[i-1], &regs[i], &r2, &ez)) {
-			regs = mm_insert_reg(&r2, i, &n_regs, regs);
-			++i; // skip the inserted INV alignment
+		if (!(opt->flag&MM_F_SPLICE) && !(opt->flag&MM_F_SR) && i > 0) { // don't try inversion alignment for -xsplice or -xsr
+			if (mm_align1_inv(km, opt, mi, qlen, qseq0, qual0, &regs[i-1], &regs[i], &r2, &ez)) {
+				regs = mm_insert_reg(&r2, i, &n_regs, regs);
+				++i; // skip the inserted INV alignment
+			}
 		}
 	}
 	*n_regs_ = n_regs;
-	kfree(km, qseq0[0]); kfree(km, qseq0[1]);
+	kfree(km, qseq0[0]);
+	if (qual0[0]) kfree(km, qual0[0]);
 	kfree(km, ez.cigar);
 	mm_filter_regs(km, opt, n_regs_, regs);
 	mm_hit_sort_by_dp(km, n_regs_, regs);

bseq.c

@@ -1,15 +1,37 @@
 #include <zlib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 #include <assert.h>
 #include "bseq.h"
+#include "kvec.h"
 #include "kseq.h"
-KSEQ_INIT(gzFile, gzread)
+KSEQ_INIT2(, gzFile, gzread)
+
+unsigned char seq_comp_table[256] = {
+	  0,   1,	2,	 3,	  4,   5,	6,	 7,	  8,   9,  10,	11,	 12,  13,  14,	15,
+	 16,  17,  18,	19,	 20,  21,  22,	23,	 24,  25,  26,	27,	 28,  29,  30,	31,
+	 32,  33,  34,	35,	 36,  37,  38,	39,	 40,  41,  42,	43,	 44,  45,  46,	47,
+	 48,  49,  50,	51,	 52,  53,  54,	55,	 56,  57,  58,	59,	 60,  61,  62,	63,
+	 64, 'T', 'V', 'G', 'H', 'E', 'F', 'C', 'D', 'I', 'J', 'M', 'L', 'K', 'N', 'O',
+	'P', 'Q', 'Y', 'S', 'A', 'A', 'B', 'W', 'X', 'R', 'Z',	91,	 92,  93,  94,	95,
+	 64, 't', 'v', 'g', 'h', 'e', 'f', 'c', 'd', 'i', 'j', 'm', 'l', 'k', 'n', 'o',
+	'p', 'q', 'y', 's', 'a', 'a', 'b', 'w', 'x', 'r', 'z', 123, 124, 125, 126, 127,
+	128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
+	144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+	160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
+	176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191,
+	192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
+	208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223,
+	224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+	240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255
+};
+
+#define CHECK_PAIR_THRES 1000000
 
 struct mm_bseq_file_s {
 	gzFile fp;
 	kseq_t *ks;
+	mm_bseq1_t s;
 };
 
 mm_bseq_file_t *mm_bseq_open(const char *fn)
@@ -31,32 +53,85 @@ void mm_bseq_close(mm_bseq_file_t *fp)
 	free(fp);
 }
 
-mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_)
+static inline void kseq2bseq(kseq_t *ks, mm_bseq1_t *s, int with_qual)
 {
-	int size = 0, m, n;
-	mm_bseq1_t *seqs;
+	int i;
+	s->name = strdup(ks->name.s);
+	s->seq = strdup(ks->seq.s);
+	for (i = 0; i < ks->seq.l; ++i) // convert U to T
+		if (s->seq[i] == 'u' || s->seq[i] == 'U')
+			--s->seq[i];
+	s->qual = with_qual && ks->qual.l? strdup(ks->qual.s) : 0;
+	s->l_seq = ks->seq.l;
+}
+
+mm_bseq1_t *mm_bseq_read2(mm_bseq_file_t *fp, int chunk_size, int with_qual, int frag_mode, int *n_)
+{
+	int64_t size = 0;
+	kvec_t(mm_bseq1_t) a = {0,0,0};
 	kseq_t *ks = fp->ks;
-	m = n = 0; seqs = 0;
+	*n_ = 0;
+	if (fp->s.seq) {
+		kv_resize(mm_bseq1_t, 0, a, 256);
+		kv_push(mm_bseq1_t, 0, a, fp->s);
+		size = fp->s.l_seq;
+		memset(&fp->s, 0, sizeof(mm_bseq1_t));
+	}
 	while (kseq_read(ks) >= 0) {
 		mm_bseq1_t *s;
 		assert(ks->seq.l <= INT32_MAX);
-		if (n >= m) {
-			m = m? m<<1 : 256;
-			seqs = (mm_bseq1_t*)realloc(seqs, m * sizeof(mm_bseq1_t));
+		if (a.m == 0) kv_resize(mm_bseq1_t, 0, a, 256);
+		kv_pushp(mm_bseq1_t, 0, a, &s);
+		kseq2bseq(ks, s, with_qual);
+		size += s->l_seq;
+		if (size >= chunk_size) {
+			if (frag_mode && a.a[a.n-1].l_seq < CHECK_PAIR_THRES) {
+				while (kseq_read(ks) >= 0) {
+					kseq2bseq(ks, &fp->s, with_qual);
+					if (mm_qname_same(fp->s.name, a.a[a.n-1].name)) {
+						kv_push(mm_bseq1_t, 0, a, fp->s);
+						memset(&fp->s, 0, sizeof(mm_bseq1_t));
+					} else break;
+				}
+			}
+			break;
+		}
+	}
+	*n_ = a.n;
+	return a.a;
+}
+
+mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_)
+{
+	return mm_bseq_read2(fp, chunk_size, with_qual, 0, n_);
+}
+
+mm_bseq1_t *mm_bseq_read_frag(int n_fp, mm_bseq_file_t **fp, int chunk_size, int with_qual, int *n_)
+{
+	int i;
+	int64_t size = 0;
+	kvec_t(mm_bseq1_t) a = {0,0,0};
+	*n_ = 0;
+	if (n_fp < 1) return 0;
+	while (1) {
+		for (i = 0; i < n_fp; ++i)
+			if (kseq_read(fp[i]->ks) < 0)
+				break;
+		if (i != n_fp) break; // some file reaches the end
+		if (a.m == 0) kv_resize(mm_bseq1_t, 0, a, 256);
+		for (i = 0; i < n_fp; ++i) {
+			mm_bseq1_t *s;
+			kv_pushp(mm_bseq1_t, 0, a, &s);
+			kseq2bseq(fp[i]->ks, s, with_qual);
+			size += s->l_seq;
 		}
-		s = &seqs[n];
-		s->name = strdup(ks->name.s);
-		s->seq = strdup(ks->seq.s);
-		s->qual = with_qual && ks->qual.l? strdup(ks->qual.s) : 0;
-		s->l_seq = ks->seq.l;
-		size += seqs[n++].l_seq;
 		if (size >= chunk_size) break;
 	}
-	*n_ = n;
-	return seqs;
+	*n_ = a.n;
+	return a.a;
 }
 
 int mm_bseq_eof(mm_bseq_file_t *fp)
 {
-	return ks_eof(fp->ks->f);
+	return (ks_eof(fp->ks->f) && fp->s.seq == 0);
 }

bseq.h

@@ -2,6 +2,7 @@
 #define MM_BSEQ_H
 
 #include <stdint.h>
+#include <string.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,10 +18,42 @@ typedef struct {
 
 mm_bseq_file_t *mm_bseq_open(const char *fn);
 void mm_bseq_close(mm_bseq_file_t *fp);
+mm_bseq1_t *mm_bseq_read2(mm_bseq_file_t *fp, int chunk_size, int with_qual, int frag_mode, int *n_);
 mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_);
+mm_bseq1_t *mm_bseq_read_frag(int n_fp, mm_bseq_file_t **fp, int chunk_size, int with_qual, int *n_);
 int mm_bseq_eof(mm_bseq_file_t *fp);
 
 extern unsigned char seq_nt4_table[256];
+extern unsigned char seq_comp_table[256];
+
+static inline int mm_qname_len(const char *s)
+{
+	int l;
+	l = strlen(s);
+	return l >= 3 && s[l-1] >= '0' && s[l-1] <= '9' && s[l-2] == '/'? l - 2 : l;
+}
+
+static inline int mm_qname_same(const char *s1, const char *s2)
+{
+	int l1, l2;
+	l1 = mm_qname_len(s1);
+	l2 = mm_qname_len(s2);
+	return (l1 == l2 && strncmp(s1, s2, l1) == 0);
+}
+
+static inline void mm_revcomp_bseq(mm_bseq1_t *s)
+{
+	int i, t, l = s->l_seq;
+	for (i = 0; i < l>>1; ++i) {
+		t = s->seq[l - i - 1];
+		s->seq[l - i - 1] = seq_comp_table[(uint8_t)s->seq[i]];
+		s->seq[i] = seq_comp_table[t];
+	}
+	if (l&1) s->seq[l>>1] = seq_comp_table[(uint8_t)s->seq[l>>1]];
+	if (s->qual)
+		for (i = 0; i < l>>1; ++i)
+			t = s->qual[l - i - 1], s->qual[l - i - 1] = s->qual[i], s->qual[i] = t;
+}
 
 #ifdef __cplusplus
 }

chain.c

@@ -19,15 +19,15 @@ static inline int ilog2_32(uint32_t v)
 	return (t = v>>8) ? 8 + LogTable256[t] : LogTable256[v];
 }
 
-int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int64_t n, mm128_t *a, uint64_t **_u, void *km)
+mm128_t *mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int n_segs, int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km)
 { // TODO: make sure this works when n has more than 32 bits
-	int32_t st = 0, k, *f, *p, *t, *v, n_u, n_v;
-	int64_t i, j;
+	int32_t k, *f, *p, *t, *v, n_u, n_v;
+	int64_t i, j, st = 0;
 	uint64_t *u, *u2, sum_qspan = 0;
 	float avg_qspan;
 	mm128_t *b, *w;
 
-	if (_u) *_u = 0;
+	if (_u) *_u = 0, *n_u_ = 0;
 	f = (int32_t*)kmalloc(km, n * 4);
 	p = (int32_t*)kmalloc(km, n * 4);
 	t = (int32_t*)kmalloc(km, n * 4);
@@ -40,25 +40,30 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 	// fill the score and backtrack arrays
 	for (i = 0; i < n; ++i) {
 		uint64_t ri = a[i].x;
+		int64_t max_j = -1;
 		int32_t qi = (int32_t)a[i].y, q_span = a[i].y>>32&0xff; // NB: only 8 bits of span is used!!!
-		int32_t max_f = q_span, max_j = -1, n_skip = 0, min_d, max_f_past = -INT32_MAX;
+		int32_t max_f = q_span, n_skip = 0, min_d;
+		int32_t sidi = (a[i].y & MM_SEED_SEG_MASK) >> MM_SEED_SEG_SHIFT;
 		while (st < i && ri - a[st].x > max_dist_x) ++st;
 		for (j = i - 1; j >= st; --j) {
 			int64_t dr = ri - a[j].x;
-			int32_t dq = qi - (int32_t)a[j].y, dd, sc;
-			if (dr == 0 || dq <= 0 || dq > max_dist_y) continue;
+			int32_t dq = qi - (int32_t)a[j].y, dd, sc, log_dd;
+			int32_t sidj = (a[j].y & MM_SEED_SEG_MASK) >> MM_SEED_SEG_SHIFT;
+			if (dr == 0 || dq <= 0) continue;
+			if ((sidi == sidj && dq > max_dist_y) || dq > max_dist_x) continue;
 			dd = dr > dq? dr - dq : dq - dr;
-			if (dd > bw) continue;
-			max_f_past = max_f_past > f[j]? max_f_past : f[j];
+			if (sidi == sidj && dd > bw) continue;
+			if (n_segs > 1 && !is_cdna && sidi == sidj && dr > max_dist_y) continue;
 			min_d = dq < dr? dq : dr;
 			sc = min_d > q_span? q_span : dq < dr? dq : dr;
-			if (is_cdna) {
+			log_dd = dd? ilog2_32(dd) : 0;
+			if (is_cdna || sidi != sidj) {
 				int c_log, c_lin;
 				c_lin = (int)(dd * .01 * avg_qspan);
-				c_log = ilog2_32(dd);
-				if (dr > dq) sc -= c_lin < c_log? c_lin : c_log;
+				c_log = log_dd;
+				if (dr > dq || sidi != sidj) sc -= c_lin < c_log? c_lin : c_log;
 				else sc -= c_lin + (c_log>>1);
-			} else sc -= (int)(dd * .01 * avg_qspan) + (ilog2_32(dd)>>1);
+			} else sc -= (int)(dd * .01 * avg_qspan) + (log_dd>>1);
 			sc += f[j];
 			if (sc > max_f) {
 				max_f = sc, max_j = j;
@@ -69,7 +74,8 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 			}
 			if (p[j] >= 0) t[p[j]] = i;
 		}
-		f[i] = max_f, p[i] = max_j, v[i] = max_f_past; // v[] keeps the max score in the previous chain
+		f[i] = max_f, p[i] = max_j;
+		v[i] = max_j >= 0 && v[max_j] > max_f? v[max_j] : max_f; // v[] keeps the peak score up to i; f[] is the score ending at i, not always the peak
 	}
 
 	// find the ending positions of chains
@@ -80,14 +86,14 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 		if (t[i] == 0 && v[i] >= min_sc)
 			++n_u;
 	if (n_u == 0) {
-		kfree(km, f); kfree(km, p); kfree(km, t); kfree(km, v);
+		kfree(km, a); kfree(km, f); kfree(km, p); kfree(km, t); kfree(km, v);
 		return 0;
 	}
 	u = (uint64_t*)kmalloc(km, n_u * 8);
 	for (i = n_u = 0; i < n; ++i) {
 		if (t[i] == 0 && v[i] >= min_sc) {
 			j = i;
-			while (j >= 0 && f[j] < v[j]) j = p[j]; // find the point that maximizes f[]
+			while (j >= 0 && f[j] < v[j]) j = p[j]; // find the peak that maximizes f[]
 			if (j < 0) j = i; // TODO: this should really be assert(j>=0)
 			u[n_u++] = (uint64_t)f[j] << 32 | j;
 		}
@@ -115,9 +121,9 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 		}
 		if (k0 == k) n_v = n_v0; // no new chain added, reset
 	}
-	n_u = k, *_u = u; // NB: note that u[] may not be sorted by score here
+	*n_u_ = n_u = k, *_u = u; // NB: note that u[] may not be sorted by score here
 
-	// free
+	// free temporary arrays
 	kfree(km, f); kfree(km, p); kfree(km, t);
 
 	// write the result to b[]
@@ -144,6 +150,7 @@ int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cn
 		k += n;
 	}
 	memcpy(u, u2, n_u * 8);
-	kfree(km, b); kfree(km, w); kfree(km, u2);
-	return n_u;
+	memcpy(b, a, k * sizeof(mm128_t)); // write _a_ to _b_ and deallocate _a_ because _a_ is oversized, sometimes a lot
+	kfree(km, a); kfree(km, w); kfree(km, u2);
+	return b;
 }

example.c

@@ -11,53 +11,51 @@ KSEQ_INIT(gzFile, gzread)
 
 int main(int argc, char *argv[])
 {
+	mm_idxopt_t iopt;
+	mm_mapopt_t mopt;
+	int n_threads = 3;
+
 	mm_verbose = 2; // disable message output to stderr
+	mm_set_opt(0, &iopt, &mopt);
+	mopt.flag |= MM_F_CIGAR; // perform alignment
 
 	if (argc < 3) {
 		fprintf(stderr, "Usage: minimap2-lite <target.fa> <query.fa>\n");
 		return 1;
 	}
-	
+
 	// open query file for reading; you may use your favorite FASTA/Q parser
 	gzFile f = gzopen(argv[2], "r");
 	assert(f);
 	kseq_t *ks = kseq_init(f);
 
-	// create index for target; we are creating one index for all target sequence
-	int n_threads = 4, w = 10, k = 15, is_hpc = 0;
-	mm_idx_t *mi = mm_idx_build(argv[1], w, k, is_hpc, n_threads);
-	assert(mi);
-
-	// mapping
-	mm_mapopt_t opt;
-	mm_mapopt_init(&opt); // initialize mapping parameters
-	mm_mapopt_update(&opt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()!
-	opt.flag |= MM_F_CIGAR; // perform alignment
-	mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread
-	while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence
-		mm_reg1_t *reg;
-		int j, i, n_reg;
-		// get all hits for the query
-		reg = mm_map(mi, ks->seq.l, ks->seq.s, &n_reg, tbuf, &opt, 0);
-		// traverse hits and print them out
-		for (j = 0; j < n_reg; ++j) {
-			mm_reg1_t *r = &reg[j];
-			assert(r->p); // with MM_F_CIGAR, this should not be NULL
-			printf("%s\t%d\t%d\t%d\t%c\t", ks->name.s, ks->seq.l, r->qs, r->qe, "+-"[r->rev]);
-			printf("%s\t%d\t%d\t%d\t%d\t%d\t%d\tcg:Z:", mi->seq[r->rid].name, mi->seq[r->rid].len, r->rs, r->re,
-				   r->p->blen - r->p->n_ambi - r->p->n_diff, r->p->blen, r->mapq);
-			for (i = 0; i < r->p->n_cigar; ++i) // IMPORTANT: this gives the CIGAR in the aligned regions. NO soft/hard clippings!
-				printf("%d%c", r->p->cigar[i]>>4, "MIDSHN"[r->p->cigar[i]&0xf]);
-			putchar('\n');
-			free(r->p);
+	// open index reader
+	mm_idx_reader_t *r = mm_idx_reader_open(argv[1], &iopt, 0);
+	mm_idx_t *mi;
+	while ((mi = mm_idx_reader_read(r, n_threads)) != 0) { // traverse each part of the index
+		mm_mapopt_update(&mopt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()!
+		mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread
+		while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence
+			mm_reg1_t *reg;
+			int j, i, n_reg;
+			reg = mm_map(mi, ks->seq.l, ks->seq.s, &n_reg, tbuf, &mopt, 0); // get all hits for the query
+			for (j = 0; j < n_reg; ++j) { // traverse hits and print them out
+				mm_reg1_t *r = &reg[j];
+				assert(r->p); // with MM_F_CIGAR, this should not be NULL
+				printf("%s\t%d\t%d\t%d\t%c\t", ks->name.s, ks->seq.l, r->qs, r->qe, "+-"[r->rev]);
+				printf("%s\t%d\t%d\t%d\t%d\t%d\t%d\tcg:Z:", mi->seq[r->rid].name, mi->seq[r->rid].len, r->rs, r->re, r->mlen, r->blen, r->mapq);
+				for (i = 0; i < r->p->n_cigar; ++i) // IMPORTANT: this gives the CIGAR in the aligned regions. NO soft/hard clippings!
+					printf("%d%c", r->p->cigar[i]>>4, "MIDSHN"[r->p->cigar[i]&0xf]);
+				putchar('\n');
+				free(r->p);
+			}
+			free(reg);
 		}
-		free(reg);
+		mm_tbuf_destroy(tbuf);
+		mm_idx_destroy(mi);
 	}
-	mm_tbuf_destroy(tbuf);
-
-	// deallocate index and close the query file
-	mm_idx_destroy(mi);
-	kseq_destroy(ks);
+	mm_idx_reader_close(r); // close the index reader
+	kseq_destroy(ks); // close the query file
 	gzclose(f);
 	return 0;
 }

format.c

@@ -43,6 +43,13 @@ static void mm_sprintf_lite(kstring_t *s, const char *fmt, ...)
 				if (c < 0) buf[l++] = '-';
 				str_enlarge(s, l);
 				for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
+			} else if (*p == 'u') {
+				int i, l = 0;
+				uint32_t x;
+				x = va_arg(ap, uint32_t);
+				do { buf[l++] = x%10 + '0'; x /= 10; } while (x > 0);
+				str_enlarge(s, l);
+				for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
 			} else if (*p == 's') {
 				char *r = va_arg(ap, char*);
 				str_copy(s, r, r + strlen(r));
@@ -105,10 +112,15 @@ err_set_rg:
 	free(rg_line);
 }
 
-void mm_write_sam_hdr_no_SQ(const char *rg, const char *ver, int argc, char *argv[])
+void mm_write_sam_hdr(const mm_idx_t *idx, const char *rg, const char *ver, int argc, char *argv[])
 {
 	kstring_t str = {0,0,0};
-	sam_write_rg_line(&str, rg);
+	if (idx) {
+		uint32_t i;
+		for (i = 0; i < idx->n_seq; ++i)
+			printf("@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len);
+	}
+	if (rg) sam_write_rg_line(&str, rg);
 	mm_sprintf_lite(&str, "@PG\tID:minimap2\tPN:minimap2");
 	if (ver) mm_sprintf_lite(&str, "\tVN:%s", ver);
 	if (argc > 1) {
@@ -122,7 +134,7 @@ void mm_write_sam_hdr_no_SQ(const char *rg, const char *ver, int argc, char *arg
 	free(str.s);
 }
 
-static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r)
+static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int no_iden)
 {
 	extern unsigned char seq_nt4_table[256];
 	int i, q_off, t_off;
@@ -145,22 +157,26 @@ static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_
 	}
 	for (i = q_off = t_off = 0; i < r->p->n_cigar; ++i) {
 		int j, op = r->p->cigar[i]&0xf, len = r->p->cigar[i]>>4;
-		assert(op >= 0 && op <= 2);
+		assert(op >= 0 && op <= 3);
 		if (op == 0) {
 			int l_tmp = 0;
 			for (j = 0; j < len; ++j) {
 				if (qseq[q_off + j] != tseq[t_off + j]) {
 					if (l_tmp > 0) {
-						tmp[l_tmp] = 0;
-						mm_sprintf_lite(s, "=%s", tmp);
+						if (!no_iden) {
+							tmp[l_tmp] = 0;
+							mm_sprintf_lite(s, "=%s", tmp);
+						} else mm_sprintf_lite(s, ":%d", l_tmp);
 						l_tmp = 0;
 					}
 					mm_sprintf_lite(s, "*%c%c", "acgtn"[tseq[t_off + j]], "acgtn"[qseq[q_off + j]]);
 				} else tmp[l_tmp++] = "ACGTN"[qseq[q_off + j]];
 			}
 			if (l_tmp > 0) {
-				tmp[l_tmp] = 0;
-				mm_sprintf_lite(s, "=%s", tmp);
+				if (!no_iden) {
+					tmp[l_tmp] = 0;
+					mm_sprintf_lite(s, "=%s", tmp);
+				} else mm_sprintf_lite(s, ":%d", l_tmp);
 			}
 			q_off += len, t_off += len;
 		} else if (op == 1) {
@@ -173,6 +189,11 @@ static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_
 				tmp[j] = "acgtn"[tseq[t_off + j]];
 			mm_sprintf_lite(s, "-%s", tmp);
 			t_off += len;
+		} else {
+			assert(len >= 2);
+			mm_sprintf_lite(s, "~%c%c%d%c%c", "acgtn"[tseq[t_off]], "acgtn"[tseq[t_off+1]],
+				len, "acgtn"[tseq[t_off+len-2]], "acgtn"[tseq[t_off+len-1]]);
+			t_off += len;
 		}
 	}
 	assert(t_off == r->re - r->rs && q_off == r->qe - r->qs);
@@ -182,14 +203,15 @@ static void write_cs(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_
 static inline void write_tags(kstring_t *s, const mm_reg1_t *r)
 {
 	int type = r->inv? 'I' : r->id == r->parent? 'P' : 'S';
-	mm_sprintf_lite(s, "\ttp:A:%c\tcm:i:%d\ts1:i:%d", type, r->cnt, r->score);
-	if (r->parent == r->id) mm_sprintf_lite(s, "\ts2:i:%d", r->subsc);
-	if (r->split) mm_sprintf_lite(s, "\tzd:i:%d", r->split);
+	if (r->iden_flt) mm_sprintf_lite(s, "\tom:i:%d", r->mapq);
 	if (r->p) {
-		mm_sprintf_lite(s, "\tNM:i:%d\tms:i:%d\tAS:i:%d\tnn:i:%d", r->p->n_diff, r->p->dp_max, r->p->dp_score, r->p->n_ambi);
+		mm_sprintf_lite(s, "\tNM:i:%d\tms:i:%d\tAS:i:%d\tnn:i:%d", r->blen - r->mlen + r->p->n_ambi, r->p->dp_max, r->p->dp_score, r->p->n_ambi);
 		if (r->p->trans_strand == 1 || r->p->trans_strand == 2)
 			mm_sprintf_lite(s, "\tts:A:%c", "?+-?"[r->p->trans_strand]);
 	}
+	mm_sprintf_lite(s, "\ttp:A:%c\tcm:i:%d\ts1:i:%d", type, r->cnt, r->score);
+	if (r->parent == r->id) mm_sprintf_lite(s, "\ts2:i:%d", r->subsc);
+	if (r->split) mm_sprintf_lite(s, "\tzd:i:%d", r->split);
 }
 
 void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag)
@@ -199,8 +221,7 @@ void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 	if (mi->seq[r->rid].name) mm_sprintf_lite(s, "%s", mi->seq[r->rid].name);
 	else mm_sprintf_lite(s, "%d", r->rid);
 	mm_sprintf_lite(s, "\t%d\t%d\t%d", mi->seq[r->rid].len, r->rs, r->re);
-	if (r->p) mm_sprintf_lite(s, "\t%d\t%d", r->p->blen - r->p->n_ambi - r->p->n_diff, r->p->blen);
-	else mm_sprintf_lite(s, "\t%d\t%d", r->fuzzy_mlen, r->fuzzy_blen);
+	mm_sprintf_lite(s, "\t%d\t%d", r->mlen, r->blen);
 	mm_sprintf_lite(s, "\t%d", r->mapq);
 	write_tags(s, r);
 	if (r->p && (opt_flag & MM_F_OUT_CG)) {
@@ -210,65 +231,155 @@ void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 			mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MIDN"[r->p->cigar[k]&0xf]);
 	}
 	if (r->p && (opt_flag & MM_F_OUT_CS))
-		write_cs(km, s, mi, t, r);
-}
-
-static char comp_tab[] = {
-	  0,   1,	2,	 3,	  4,   5,	6,	 7,	  8,   9,  10,	11,	 12,  13,  14,	15,
-	 16,  17,  18,	19,	 20,  21,  22,	23,	 24,  25,  26,	27,	 28,  29,  30,	31,
-	 32,  33,  34,	35,	 36,  37,  38,	39,	 40,  41,  42,	43,	 44,  45,  46,	47,
-	 48,  49,  50,	51,	 52,  53,  54,	55,	 56,  57,  58,	59,	 60,  61,  62,	63,
-	 64, 'T', 'V', 'G', 'H', 'E', 'F', 'C', 'D', 'I', 'J', 'M', 'L', 'K', 'N', 'O',
-	'P', 'Q', 'Y', 'S', 'A', 'A', 'B', 'W', 'X', 'R', 'Z',	91,	 92,  93,  94,	95,
-	 64, 't', 'v', 'g', 'h', 'e', 'f', 'c', 'd', 'i', 'j', 'm', 'l', 'k', 'n', 'o',
-	'p', 'q', 'y', 's', 'a', 'a', 'b', 'w', 'x', 'r', 'z', 123, 124, 125, 126, 127
-};
-
-void mm_write_sam_SQ(const mm_idx_t *idx)
-{
-	uint32_t i;
-	for (i = 0; i < idx->n_seq; ++i)
-		printf("@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len);
+		write_cs(km, s, mi, t, r, !(opt_flag&MM_F_OUT_CS_LONG));
 }
 
 static void sam_write_sq(kstring_t *s, char *seq, int l, int rev, int comp)
 {
+	extern unsigned char seq_comp_table[256];
 	if (rev) {
 		int i;
 		str_enlarge(s, l);
 		for (i = 0; i < l; ++i) {
 			int c = seq[l - 1 - i];
-			s->s[s->l + i] = c < 128 && comp? comp_tab[c] : c;
+			s->s[s->l + i] = c < 128 && comp? seq_comp_table[c] : c;
 		}
 		s->l += l;
 	} else str_copy(s, seq, seq + l);
 }
 
-void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs)
+static inline const mm_reg1_t *get_sam_pri(int n_regs, const mm_reg1_t *regs)
 {
-	int flag = 0;
+	int i;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].sam_pri)
+			return &regs[i];
+	assert(n_regs == 0);
+	return NULL;
+}
+
+static void write_sam_cigar(kstring_t *s, int sam_flag, int in_tag, int qlen, const mm_reg1_t *r)
+{
+	if (r->p == 0) {
+		mm_sprintf_lite(s, "*");
+	} else {
+		uint32_t k, clip_len[2];
+		clip_len[0] = r->rev? qlen - r->qe : r->qs;
+		clip_len[1] = r->rev? r->qs : qlen - r->qe;
+		if (in_tag) {
+			int clip_char = (sam_flag&0x800)? 5 : 4;
+			mm_sprintf_lite(s, "\tCG:B:I");
+			if (clip_len[0]) mm_sprintf_lite(s, ",%u", clip_len[0]<<4|clip_char);
+			for (k = 0; k < r->p->n_cigar; ++k)
+				mm_sprintf_lite(s, ",%u", r->p->cigar[k]);
+			if (clip_len[1]) mm_sprintf_lite(s, ",%u", clip_len[1]<<4|clip_char);
+		} else {
+			int clip_char = (sam_flag&0x800)? 'H' : 'S';
+			if (clip_len[0]) mm_sprintf_lite(s, "%d%c", clip_len[0], clip_char);
+			for (k = 0; k < r->p->n_cigar; ++k)
+				mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MIDN"[r->p->cigar[k]&0xf]);
+			if (clip_len[1]) mm_sprintf_lite(s, "%d%c", clip_len[1], clip_char);
+		}
+	}
+}
+
+void mm_write_sam2(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regss, const mm_reg1_t *const* regss, void *km, int opt_flag)
+{
+	const int max_bam_cigar_op = 65535;
+	int flag, n_regs = n_regss[seg_idx], cigar_in_tag = 0;
+	int this_rid = -1, this_pos = -1, this_rev = 0;
+	const mm_reg1_t *regs = regss[seg_idx], *r_prev = NULL, *r_next;
+	const mm_reg1_t *r = n_regs > 0 && reg_idx < n_regs && reg_idx >= 0? &regs[reg_idx] : NULL;
+
+	// find the primary of the previous and the next segments, if they are mapped
+	if (n_seg > 1) {
+		int i, next_sid = (seg_idx + 1) % n_seg;
+		r_next = get_sam_pri(n_regss[next_sid], regss[next_sid]);
+		if (n_seg > 2) {
+			for (i = 1; i <= n_seg - 1; ++i) {
+				int prev_sid = (seg_idx + n_seg - i) % n_seg;
+				if (n_regss[prev_sid] > 0) {
+					r_prev = get_sam_pri(n_regss[prev_sid], regss[prev_sid]);
+					break;
+				}
+			}
+		} else r_prev = r_next;
+	} else r_prev = r_next = NULL;
+
+	// write QNAME
 	s->l = 0;
+	mm_sprintf_lite(s, "%s", t->name);
+	if (n_seg > 1) s->l = mm_qname_len(t->name); // trim the suffix like /1 or /2
+
+	// write flag
+	flag = n_seg > 1? 0x1 : 0x0;
+	if (r == 0) {
+		flag |= 0x4;
+	} else {
+		if (r->rev) flag |= 0x10;
+		if (r->parent != r->id) flag |= 0x100;
+		else if (!r->sam_pri) flag |= 0x800;
+	}
+	if (n_seg > 1) {
+		if (r && r->proper_frag) flag |= 0x2; // TODO: this doesn't work when there are more than 2 segments
+		if (seg_idx == 0) flag |= 0x40;
+		else if (seg_idx == n_seg - 1) flag |= 0x80;
+		if (r_next == NULL) flag |= 0x8;
+		else if (r_next->rev) flag |= 0x20;
+	}
+	mm_sprintf_lite(s, "\t%d", flag);
+
+	// write coordinate, MAPQ and CIGAR
+	if (r == 0) {
+		if (r_prev) {
+			this_rid = r_prev->rid, this_pos = r_prev->rs;
+			mm_sprintf_lite(s, "\t%s\t%d\t0\t*", mi->seq[this_rid].name, this_pos+1);
+		} else mm_sprintf_lite(s, "\t*\t0\t0\t*");
+	} else {
+		int mapq = !r->iden_flt? r->mapq : r->mapq < 3? r->mapq : 3;
+		this_rid = r->rid, this_pos = r->rs, this_rev = r->rev;
+		mm_sprintf_lite(s, "\t%s\t%d\t%d\t", mi->seq[r->rid].name, r->rs+1, mapq);
+		if ((opt_flag & MM_F_LONG_CIGAR) && r->p && r->p->n_cigar > max_bam_cigar_op - 2) {
+			int n_cigar = r->p->n_cigar;
+			if (r->qs != 0) ++n_cigar;
+			if (r->qe != t->l_seq) ++n_cigar;
+			if (n_cigar > max_bam_cigar_op)
+				cigar_in_tag = 1;
+		}
+		if (cigar_in_tag) mm_sprintf_lite(s, "%dS", t->l_seq);
+		else write_sam_cigar(s, flag, 0, t->l_seq, r);
+	}
+
+	// write mate positions
+	if (n_seg > 1) {
+		int tlen = 0;
+		if (this_rid >= 0 && r_next) {
+			if (this_rid == r_next->rid) {
+				int this_pos5 = r && r->rev? r->re - 1 : this_pos;
+				int next_pos5 = r_next->rev? r_next->re - 1 : r_next->rs;
+				tlen = next_pos5 - this_pos5;
+				mm_sprintf_lite(s, "\t=\t");
+			} else mm_sprintf_lite(s, "\t%s\t", mi->seq[r_next->rid].name);
+			mm_sprintf_lite(s, "%d\t", r_next->rs + 1);
+		} else if (r_next) { // && this_rid < 0
+			mm_sprintf_lite(s, "\t%s\t%d\t", mi->seq[r_next->rid].name, r_next->rs + 1);
+		} else if (this_rid >= 0) { // && r_next == NULL
+			int this_pos5 = this_rev? r->re - 1 : this_pos; // this_rev is only true when r != NULL
+			tlen = this_pos - this_pos5; // next_pos5 will be this_pos
+			mm_sprintf_lite(s, "\t=\t%d\t", this_pos + 1); // next segment will take r's coordinate
+		} else mm_sprintf_lite(s, "\t*\t0\t"); // neither has coordinates
+		if (tlen > 0) ++tlen;
+		else if (tlen < 0) --tlen;
+		mm_sprintf_lite(s, "%d\t", tlen);
+	} else mm_sprintf_lite(s, "\t*\t0\t0\t");
+
+	// write SEQ and QUAL
 	if (r == 0) {
-		mm_sprintf_lite(s, "%s\t4\t*\t0\t0\t*\t*\t0\t0\t", t->name);
 		sam_write_sq(s, t->seq, t->l_seq, 0, 0);
 		mm_sprintf_lite(s, "\t");
 		if (t->qual) sam_write_sq(s, t->qual, t->l_seq, 0, 0);
 		else mm_sprintf_lite(s, "*");
 	} else {
-		if (r->rev) flag |= 0x10;
-		if (r->parent != r->id) flag |= 0x100;
-		else if (!r->sam_pri) flag |= 0x800;
-		mm_sprintf_lite(s, "%s\t%d\t%s\t%d\t%d\t", t->name, flag, mi->seq[r->rid].name, r->rs+1, r->mapq);
-		if (r->p) { // actually this should always be true for SAM output
-			uint32_t k, clip_len = r->rev? t->l_seq - r->qe : r->qs;
-			int clip_char = (flag&0x800)? 'H' : 'S';
-			if (clip_len) mm_sprintf_lite(s, "%d%c", clip_len, clip_char);
-			for (k = 0; k < r->p->n_cigar; ++k)
-				mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MIDN"[r->p->cigar[k]&0xf]);
-			clip_len = r->rev? r->qs : t->l_seq - r->qe;
-			if (clip_len) mm_sprintf_lite(s, "%d%c", clip_len, clip_char);
-		} else mm_sprintf_lite(s, "*");
-		mm_sprintf_lite(s, "\t*\t0\t0\t");
 		if ((flag & 0x900) == 0) {
 			sam_write_sq(s, t->seq, t->l_seq, r->rev, r->rev);
 			mm_sprintf_lite(s, "\t");
@@ -282,8 +393,13 @@ void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 			if (t->qual) sam_write_sq(s, t->qual + r->qs, r->qe - r->qs, r->rev, 0);
 			else mm_sprintf_lite(s, "*");
 		}
+	}
+
+	// write tags
+	if (mm_rg_id[0]) mm_sprintf_lite(s, "\tRG:Z:%s", mm_rg_id);
+	if (n_seg > 2) mm_sprintf_lite(s, "\tFI:i:%d", seg_idx);
+	if (r) {
 		write_tags(s, r);
-		if (mm_rg_id[0]) mm_sprintf_lite(s, "\tRG:Z:%s", mm_rg_id);
 		if (r->parent == r->id && r->p && n_regs > 1 && regs && r >= regs && r - regs < n_regs) { // supplementary aln may exist
 			int i, n_sa = 0; // n_sa: number of SA fields
 			for (i = 0; i < n_regs; ++i)
@@ -305,10 +421,23 @@ void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 					if (l_I) mm_sprintf_lite(s, "%dI", l_I);
 					if (l_D) mm_sprintf_lite(s, "%dD", l_D);
 					if (clip3) mm_sprintf_lite(s, "%dS", clip3);
-					mm_sprintf_lite(s, ",%d,%d;", q->mapq, q->p->n_diff);
+					mm_sprintf_lite(s, ",%d,%d;", q->mapq, q->blen - q->mlen + q->p->n_ambi);
 				}
 			}
 		}
+		if (r->p && (opt_flag & MM_F_OUT_CS))
+			write_cs(km, s, mi, t, r, !(opt_flag&MM_F_OUT_CS_LONG));
+		if (cigar_in_tag)
+			write_sam_cigar(s, flag, 1, t->l_seq, r);
 	}
+
 	s->s[s->l] = 0; // we always have room for an extra byte (see str_enlarge)
 }
+
+void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs)
+{
+	int i;
+	for (i = 0; i < n_regs; ++i)
+		if (r == &regs[i]) break;
+	mm_write_sam2(s, mi, t, 0, i, 1, &n_regs, &regs, NULL, 0);
+}

getopt.c

@@ -113,7 +113,7 @@ static int __getopt_long_core(int argc, char *const *argv, const char *optstring
 		 (argv[optind][1] == '-' && argv[optind][2])))
 	{
 		int colon = optstring[optstring[0]=='+'||optstring[0]=='-']==':';
-		int i, cnt, match;
+		int i, cnt, match = -1;
 		char *opt;
 		for (cnt=i=0; longopts[i].name; i++) {
 			const char *name = longopts[i].name;

hit.c

@@ -1,20 +1,22 @@
 #include <string.h>
+#include <stdlib.h>
 #include <math.h>
 #include "mmpriv.h"
 #include "kalloc.h"
+#include "khash.h"
 
 static inline void mm_cal_fuzzy_len(mm_reg1_t *r, const mm128_t *a)
 {
 	int i;
-	r->fuzzy_mlen = r->fuzzy_blen = 0;
+	r->mlen = r->blen = 0;
 	if (r->cnt <= 0) return;
-	r->fuzzy_mlen = r->fuzzy_blen = a[r->as].y>>32&0xff;
+	r->mlen = r->blen = a[r->as].y>>32&0xff;
 	for (i = r->as + 1; i < r->as + r->cnt; ++i) {
 		int span = a[i].y>>32&0xff;
 		int tl = (int32_t)a[i].x - (int32_t)a[i-1].x;
 		int ql = (int32_t)a[i].y - (int32_t)a[i-1].y;
-		r->fuzzy_blen += tl > ql? tl : ql;
-		r->fuzzy_mlen += tl > span && ql > span? span : tl < ql? tl : ql;
+		r->blen += tl > ql? tl : ql;
+		r->mlen += tl > span && ql > span? span : tl < ql? tl : ql;
 	}
 }
 
@@ -35,7 +37,19 @@ static inline void mm_reg_set_coor(mm_reg1_t *r, int32_t qlen, const mm128_t *a)
 	mm_cal_fuzzy_len(r, a);
 }
 
-mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) // convert chains to hits
+static inline uint64_t hash64(uint64_t key)
+{
+	key = (~key + (key << 21));
+	key = key ^ key >> 24;
+	key = ((key + (key << 3)) + (key << 8));
+	key = key ^ key >> 14;
+	key = ((key + (key << 2)) + (key << 4));
+	key = key ^ key >> 28;
+	key = (key + (key << 31));
+	return key;
+}
+
+mm_reg1_t *mm_gen_regs(void *km, uint32_t hash, int qlen, int n_u, uint64_t *u, mm128_t *a) // convert chains to hits
 {
 	mm128_t *z, tmp;
 	mm_reg1_t *r;
@@ -46,7 +60,9 @@ mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) //
 	// sort by score
 	z = (mm128_t*)kmalloc(km, n_u * 16);
 	for (i = k = 0; i < n_u; ++i) {
-		z[i].x = u[i] >> 32;
+		uint32_t h;
+		h = (uint32_t)hash64((hash64(a[k].x) + hash64(a[k].y)) ^ hash);
+		z[i].x = u[i] ^ h; // u[i] -- higher 32 bits: chain score; lower 32 bits: number of seeds in the chain
 		z[i].y = (uint64_t)k << 32 | (int32_t)u[i];
 		k += (int32_t)u[i];
 	}
@@ -60,7 +76,8 @@ mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) //
 		mm_reg1_t *ri = &r[i];
 		ri->id = i;
 		ri->parent = MM_PARENT_UNSET;
-		ri->score = z[i].x;
+		ri->score = z[i].x >> 32;
+		ri->hash = (uint32_t)z[i].x;
 		ri->cnt = (int32_t)z[i].y;
 		ri->as = z[i].y >> 32;
 		mm_reg_set_coor(ri, qlen, a);
@@ -87,55 +104,86 @@ void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a)
 	r->split |= 1, r2->split |= 2;
 }
 
-void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r) // and compute mm_reg1_t::subsc
+void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r, int sub_diff) // and compute mm_reg1_t::subsc
 {
 	int i, j, k, *w;
+	uint64_t *cov;
 	if (n <= 0) return;
 	for (i = 0; i < n; ++i) r[i].id = i;
+	cov = (uint64_t*)kmalloc(km, n * sizeof(uint64_t));
 	w = (int*)kmalloc(km, n * sizeof(int));
 	w[0] = 0, r[0].parent = 0;
 	for (i = 1, k = 1; i < n; ++i) {
 		mm_reg1_t *ri = &r[i];
-		int si = ri->qs, ei = ri->qe;
-		for (j = 0; j < k; ++j) {
+		int si = ri->qs, ei = ri->qe, n_cov = 0, uncov_len = 0;
+		for (j = 0; j < k; ++j) { // traverse existing primary hits to find overlapping hits
 			mm_reg1_t *rp = &r[w[j]];
 			int sj = rp->qs, ej = rp->qe;
-			int min = ej - sj < ei - si? ej - sj : ei - si;
-			int ol = si < sj? (ei < sj? 0 : ei < ej? ei - sj : ej - sj) : (ej < si? 0 : ej < ei? ej - si : ei - si);
-			if (ol > mask_level * min) {
+			if (ej <= si || sj >= ei) continue;
+			if (sj < si) sj = si;
+			if (ej > ei) ej = ei;
+			cov[n_cov++] = (uint64_t)sj<<32 | ej;
+		}
+		if (n_cov == 0) {
+			goto set_parent_test; // no overlapping primary hits; then i is a new primary hit
+		} else if (n_cov > 0) { // there are overlapping primary hits; find the length not covered by existing primary hits
+			int j, x = si;
+			radix_sort_64(cov, cov + n_cov);
+			for (j = 0; j < n_cov; ++j) {
+				if (cov[j]>>32 > x) uncov_len += (cov[j]>>32) - x;
+				x = (int32_t)cov[j] > x? (int32_t)cov[j] : x;
+			}
+			if (ei > x) uncov_len += ei - x;
+		}
+		for (j = 0; j < k; ++j) { // traverse existing primary hits again
+			mm_reg1_t *rp = &r[w[j]];
+			int sj = rp->qs, ej = rp->qe, min, max, ol;
+			if (ej <= si || sj >= ei) continue; // no overlap
+			min = ej - sj < ei - si? ej - sj : ei - si;
+			max = ej - sj > ei - si? ej - sj : ei - si;
+			ol = si < sj? (ei < sj? 0 : ei < ej? ei - sj : ej - sj) : (ej < si? 0 : ej < ei? ej - si : ei - si); // overlap length
+			if ((float)ol / min - (float)uncov_len / max > mask_level) {
+				int cnt_sub = 0;
 				ri->parent = rp->parent;
 				rp->subsc = rp->subsc > ri->score? rp->subsc : ri->score;
-				if (rp->p && ri->p)
+				if (ri->cnt >= rp->cnt) cnt_sub = 1;
+				if (rp->p && ri->p && (rp->rs != ri->rs || rp->re != ri->re || ol != min)) { // the last condition excludes identical hits after DP
 					rp->p->dp_max2 = rp->p->dp_max2 > ri->p->dp_max? rp->p->dp_max2 : ri->p->dp_max;
+					if (rp->p->dp_max - ri->p->dp_max <= sub_diff) cnt_sub = 1;
+				}
+				if (cnt_sub) ++rp->n_sub;
 				break;
 			}
 		}
-		if (j == k) w[k++] = i, ri->parent = i;
+set_parent_test:
+		if (j == k) w[k++] = i, ri->parent = i, ri->n_sub = 0;
 	}
+	kfree(km, cov);
 	kfree(km, w);
 }
 
 void mm_hit_sort_by_dp(void *km, int *n_regs, mm_reg1_t *r)
 {
 	int32_t i, n_aux, n = *n_regs;
-	uint64_t *aux;
+	mm128_t *aux;
 	mm_reg1_t *t;
 
 	if (n <= 1) return;
-	aux = (uint64_t*)kmalloc(km, n * 8);
+	aux = (mm128_t*)kmalloc(km, n * 16);
 	t = (mm_reg1_t*)kmalloc(km, n * sizeof(mm_reg1_t));
 	for (i = n_aux = 0; i < n; ++i) {
 		if (r[i].inv || r[i].cnt > 0) { // squeeze out elements with cnt==0 (soft deleted)
 			assert(r[i].p);
-			aux[n_aux++] = (uint64_t)r[i].p->dp_max << 32 | i;
+			aux[n_aux].x = (uint64_t)r[i].p->dp_max << 32 | r[i].hash;
+			aux[n_aux++].y = i;
 		} else if (r[i].p) {
 			free(r[i].p);
 			r[i].p = 0;
 		}
 	}
-	radix_sort_64(aux, aux + n_aux);
+	radix_sort_128x(aux, aux + n_aux);
 	for (i = n_aux - 1; i >= 0; --i)
-		t[n_aux - 1 - i] = r[(int32_t)aux[i]];
+		t[n_aux - 1 - i] = r[aux[i].y];
 	memcpy(r, t, sizeof(mm_reg1_t) * n_aux);
 	*n_regs = n_aux;
 	kfree(km, aux);
@@ -177,15 +225,20 @@ void mm_sync_regs(void *km, int n_regs, mm_reg1_t *regs) // keep mm_reg1_t::{id,
 	mm_set_sam_pri(n_regs, regs);
 }
 
-void mm_select_sub(void *km, float mask_level, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r)
+void mm_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r)
 {
 	if (pri_ratio > 0.0f && *n_ > 0) {
 		int i, k, n = *n_, n_2nd = 0;
-		for (i = k = 0; i < n; ++i)
-			if (r[i].parent == i) r[k++] = r[i];
-			else if ((r[i].score >= r[r[i].parent].score * pri_ratio || r[i].score + min_diff >= r[r[i].parent].score) && n_2nd++ < best_n)
+		for (i = k = 0; i < n; ++i) {
+			int p = r[i].parent;
+			if (p == i || r[i].inv) { // primary or inversion
 				r[k++] = r[i];
-			else if (r[i].p) free(r[i].p);
+			} else if ((r[i].score >= r[p].score * pri_ratio || r[i].score + min_diff >= r[p].score) && n_2nd < best_n) {
+				if (!(r[i].qs == r[p].qs && r[i].qe == r[p].qe && r[i].rs == r[p].rs && r[i].re == r[p].re)) // not identical hits
+					r[k++] = r[i], ++n_2nd;
+				else if (r[i].p) free(r[i].p);
+			} else if (r[i].p) free(r[i].p);
+		}
 		if (k != n) mm_sync_regs(km, k, r); // removing hits requires sync()
 		*n_ = k;
 	}
@@ -197,9 +250,9 @@ void mm_filter_regs(void *km, const mm_mapopt_t *opt, int *n_regs, mm_reg1_t *re
 	for (i = k = 0; i < *n_regs; ++i) {
 		mm_reg1_t *r = &regs[i];
 		int flt = 0;
-		if (!r->inv && r->cnt < opt->min_cnt) flt = 1;
+		if (!r->inv && !r->seg_split && r->cnt < opt->min_cnt) flt = 1;
 		if (r->p) {
-			if (r->p->blen - r->p->n_ambi - r->p->n_diff < opt->min_chain_score) flt = 1;
+			if (r->mlen < opt->min_chain_score) flt = 1;
 			else if (r->p->dp_max < opt->min_dp_max) flt = 1;
 			if (flt) free(r->p);
 		}
@@ -211,6 +264,45 @@ void mm_filter_regs(void *km, const mm_mapopt_t *opt, int *n_regs, mm_reg1_t *re
 	*n_regs = k;
 }
 
+void mm_filter_by_identity(void *km, int n_regs, mm_reg1_t *regs, float min_iden, int qlen, const char *qual) // TODO: make sure it is not beyond the ends of contigs
+{
+	int i, j, n_aux = 0, en, blen = 0;
+	uint64_t *aux;
+	float n_diff = 0.0f;
+	if (n_regs <= 0) return;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].id == regs[i].parent && regs[i].pe_thru) // sequenced through the fragment; don't filter
+			return;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].id == regs[i].parent)
+			++n_aux;
+	assert(n_aux >= 1);
+	aux = (uint64_t*)kmalloc(km, n_aux * 8);
+	for (i = 0, n_aux = 0; i < n_regs; ++i)
+		if (regs[i].id == regs[i].parent)
+			aux[n_aux++] = (uint64_t)regs[i].qs<<32 | i;
+	radix_sort_64(aux, aux + n_aux);
+	for (i = 0, en = 0; i < n_aux; ++i) {
+		mm_reg1_t *r = &regs[(int32_t)aux[i]];
+		if (r->qs > en) {
+			for (j = en; j < r->qs; ++j)
+				n_diff += qual == 0 || qual[j] >= 53? .25f : .05f * .25f * (qual[j] - 33);
+			blen += r->qs - en;
+		}
+		assert(r->p);
+		blen += r->p->blen2;
+		n_diff += r->p->n_diff2;
+		en = en > r->qe? en : r->qe;
+	}
+	for (j = en; j < qlen; ++j)
+		n_diff += qual == 0 || qual[j] >= 53? .25f : .05f * .25f * (qual[j] - 33);
+	blen += qlen - en;
+	kfree(km, aux);
+	if (1.0f - n_diff / blen < min_iden)
+		for (i = 0; i < n_regs; ++i)
+			regs[i].iden_flt = 1;
+}
+
 int mm_squeeze_a(void *km, int n_regs, mm_reg1_t *regs, mm128_t *a)
 { // squeeze out regions in a[] that are not referenced by regs[]
 	int i, as = 0;
@@ -288,9 +380,71 @@ void mm_join_long(void *km, const mm_mapopt_t *opt, int qlen, int *n_regs_, mm_r
 	}
 }
 
-void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc)
+mm_seg_t *mm_seg_gen(void *km, uint32_t hash, int n_segs, const int *qlens, int n_regs0, const mm_reg1_t *regs0, int *n_regs, mm_reg1_t **regs, const mm128_t *a)
 {
-	static const float q_coef = 30.0f;
+	int s, i, j, acc_qlen[MM_MAX_SEG+1], qlen_sum = 0;
+	mm_seg_t *seg;
+
+	assert(n_segs <= MM_MAX_SEG);
+	for (s = 1, acc_qlen[0] = 0; s < n_segs; ++s)
+		acc_qlen[s] = acc_qlen[s-1] + qlens[s-1];
+	qlen_sum = acc_qlen[n_segs - 1] + qlens[n_segs - 1];
+
+	seg = (mm_seg_t*)kcalloc(km, n_segs, sizeof(mm_seg_t));
+	for (s = 0; s < n_segs; ++s) {
+		seg[s].u = (uint64_t*)kmalloc(km, n_regs0 * 8);
+		for (i = 0; i < n_regs0; ++i)
+			seg[s].u[i] = (uint64_t)regs0[i].score << 32;
+	}
+	for (i = 0; i < n_regs0; ++i) {
+		const mm_reg1_t *r = &regs0[i];
+		for (j = 0; j < r->cnt; ++j) {
+			int sid = (a[r->as + j].y&MM_SEED_SEG_MASK)>>MM_SEED_SEG_SHIFT;
+			++seg[sid].u[i];
+			++seg[sid].n_a;
+		}
+	}
+	for (s = 0; s < n_segs; ++s) {
+		mm_seg_t *sr = &seg[s];
+		for (i = 0, sr->n_u = 0; i < n_regs0; ++i) // squeeze out zero-length per-segment chains
+			if ((int32_t)sr->u[i] != 0)
+				sr->u[sr->n_u++] = sr->u[i];
+		sr->a = (mm128_t*)kmalloc(km, sr->n_a * sizeof(mm128_t));
+		sr->n_a = 0;
+	}
+
+	for (i = 0; i < n_regs0; ++i) {
+		const mm_reg1_t *r = &regs0[i];
+		for (j = 0; j < r->cnt; ++j) {
+			int sid = (a[r->as + j].y&MM_SEED_SEG_MASK)>>MM_SEED_SEG_SHIFT;
+			mm128_t a1 = a[r->as + j];
+			// on reverse strand, the segment position is:
+			//   x_for_cat = qlen_sum - 1 - (int32_t)a1.y - 1 + q_span
+			//   (int32_t)new_a1.y = qlens[sid] - (x_for_cat - acc_qlen[sid] + 1 - q_span) - 1 = (int32_t)a1.y - (qlen_sum - (qlens[sid] + acc_qlen[sid]))
+			a1.y -= a1.x>>63? qlen_sum - (qlens[sid] + acc_qlen[sid]) : acc_qlen[sid];
+			seg[sid].a[seg[sid].n_a++] = a1;
+		}
+	}
+	for (s = 0; s < n_segs; ++s) {
+		regs[s] = mm_gen_regs(km, hash, qlens[s], seg[s].n_u, seg[s].u, seg[s].a);
+		n_regs[s] = seg[s].n_u;
+		for (i = 0; i < n_regs[s]; ++i)
+			regs[s][i].seg_split = 1;
+	}
+	return seg;
+}
+
+void mm_seg_free(void *km, int n_segs, mm_seg_t *segs)
+{
+	int i;
+	for (i = 0; i < n_segs; ++i) kfree(km, segs[i].u);
+	for (i = 0; i < n_segs; ++i) kfree(km, segs[i].a);
+	kfree(km, segs);
+}
+
+void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc, int match_sc, int rep_len)
+{
+	static const float q_coef = 40.0f;
 	int i;
 	for (i = 0; i < n_regs; ++i) {
 		mm_reg1_t *r = &regs[i];
@@ -298,14 +452,20 @@ void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc)
 			r->mapq = 0;
 		} else if (r->parent == r->id) {
 			int mapq, subsc;
-			float pen_cm = r->cnt >= 10? 1.0f : 0.1f * r->cnt;
+			float pen_s1 = (r->score > 100? 1.0f : 0.01f * r->score) * ((float)r->score / (r->score + rep_len));
+			float pen_cm = r->cnt > 10? 1.0f : 0.1f * r->cnt;
+			pen_cm = pen_s1 < pen_cm? pen_s1 : pen_cm;
 			subsc = r->subsc > min_chain_sc? r->subsc : min_chain_sc;
 			if (r->p && r->p->dp_max2 > 0 && r->p->dp_max > 0) {
-				float identity = (float)(r->p->blen - r->p->n_diff - r->p->n_ambi) / (r->p->blen - r->p->n_ambi);
-				mapq = (int)(identity * pen_cm * q_coef * (1. - (float)r->p->dp_max2 * subsc / r->p->dp_max / r->score) * logf(r->score));
+				float identity = (float)r->mlen / r->blen;
+				int mapq_alt = (int)(6.02f * identity * identity * (r->p->dp_max - r->p->dp_max2) / match_sc + .499f); // BWA-MEM like mapQ, mostly for short reads
+				mapq = (int)(identity * pen_cm * q_coef * (1. - (float)r->p->dp_max2 * subsc / r->p->dp_max / r->score) * logf(r->score)); // more for long reads
+				mapq = mapq < mapq_alt? mapq : mapq_alt; // in case the long-read heuristic fails
 			} else mapq = (int)(pen_cm * q_coef * (1. - (float)subsc / r->score) * logf(r->score));
+			mapq -= (int)(4.343f * logf(r->n_sub + 1) + .499f);
 			mapq = mapq > 0? mapq : 0;
 			r->mapq = mapq < 60? mapq : 60;
+			if (r->p && r->p->dp_max > r->p->dp_max2 && r->mapq == 0) r->mapq = 1;
 		} else r->mapq = 0;
 	}
 }

index.c

@@ -21,6 +21,22 @@ typedef khash_t(idx) idxhash_t;
 
 #define kroundup64(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, (x)|=(x)>>32, ++(x))
 
+typedef struct mm_idx_bucket_s {
+	mm128_v a;   // (minimizer, position) array
+	int32_t n;   // size of the _p_ array
+	uint64_t *p; // position array for minimizers appearing >1 times
+	void *h;     // hash table indexing _p_ and minimizers appearing once
+} mm_idx_bucket_t;
+
+void mm_idxopt_init(mm_idxopt_t *opt)
+{
+	memset(opt, 0, sizeof(mm_idxopt_t));
+	opt->k = 15, opt->w = 10, opt->is_hpc = 0;
+	opt->bucket_bits = 14;
+	opt->mini_batch_size = 50000000;
+	opt->batch_size = 4000000000ULL;
+}
+
 mm_idx_t *mm_idx_init(int w, int k, int b, int is_hpc)
 {
 	mm_idx_t *mi;
@@ -104,13 +120,13 @@ int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, ui
 	return en - st;
 }
 
-uint32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
+int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
 {
 	int i;
 	size_t n = 0;
 	uint32_t thres;
 	khint_t *a, k;
-	if (f <= 0.) return UINT32_MAX;
+	if (f <= 0.) return INT32_MAX;
 	for (i = 0; i < 1<<mi->b; ++i)
 		if (mi->B[i].h) n += kh_size((idxhash_t*)mi->B[i].h);
 	a = (uint32_t*)malloc(n * 4);
@@ -180,7 +196,7 @@ static void worker_post(void *g, long i, int tid)
 	assert(b->n == start_p);
 
 	// deallocate and clear b->a
-	free(b->a.a);
+	kfree(0, b->a.a);
 	b->a.n = b->a.m = 0, b->a.a = 0;
 }
  
@@ -281,7 +297,7 @@ static void *worker_pipeline(void *shared, int step, void *in)
     } else if (step == 2) { // dispatch sketch to buckets
         step_t *s = (step_t*)in;
 		mm_idx_add(p->mi, s->a.n, s->a.a);
-		free(s->a.a); free(s);
+		kfree(0, s->a.a); free(s);
 	}
     return 0;
 }
@@ -314,7 +330,7 @@ mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads)
 	mm_idx_t *mi;
 	fp = mm_bseq_open(fn);
 	if (fp == 0) return 0;
-	mi = mm_idx_gen(fp, w, k, MM_IDX_DEF_B, is_hpc, 1<<18, n_threads, UINT64_MAX, 1);
+	mi = mm_idx_gen(fp, w, k, 14, is_hpc, 1<<18, n_threads, UINT64_MAX, 1);
 	mm_bseq_close(fp);
 	return mi;
 }
@@ -411,21 +427,68 @@ mm_idx_t *mm_idx_load(FILE *fp)
 	return mi;
 }
 
-int mm_idx_is_idx(const char *fn)
+int64_t mm_idx_is_idx(const char *fn)
 {
 	int fd, is_idx = 0;
-	off_t ret;
+	off_t ret, off_end;
 	char magic[4];
 
 	if (strcmp(fn, "-") == 0) return 0; // read from pipe; not an index
 	fd = open(fn, O_RDONLY);
 	if (fd < 0) return -1; // error
-	if ((ret = lseek(fd, 0, SEEK_END)) >= 4) {
+	if ((off_end = lseek(fd, 0, SEEK_END)) >= 4) {
 		lseek(fd, 0, SEEK_SET);
 		ret = read(fd, magic, 4);
 		if (ret == 4 && strncmp(magic, MM_IDX_MAGIC, 4) == 0)
 			is_idx = 1;
 	}
 	close(fd);
-	return is_idx;
+	return is_idx? off_end : 0;
+}
+
+mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out)
+{
+	int64_t is_idx;
+	mm_idx_reader_t *r;
+	is_idx = mm_idx_is_idx(fn);
+	if (is_idx < 0) return 0; // failed to open the index
+	r = (mm_idx_reader_t*)calloc(1, sizeof(mm_idx_reader_t));
+	r->is_idx = is_idx;
+	if (opt) r->opt = *opt;
+	else mm_idxopt_init(&r->opt);
+	if (r->is_idx) {
+		r->fp.idx = fopen(fn, "rb");
+		r->idx_size = is_idx;
+	} else r->fp.seq = mm_bseq_open(fn);
+	if (fn_out) r->fp_out = fopen(fn_out, "wb");
+	return r;
+}
+
+void mm_idx_reader_close(mm_idx_reader_t *r)
+{
+	if (r->is_idx) fclose(r->fp.idx);
+	else mm_bseq_close(r->fp.seq);
+	if (r->fp_out) fclose(r->fp_out);
+	free(r);
+}
+
+mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
+{
+	mm_idx_t *mi;
+	if (r->is_idx) {
+		mi = mm_idx_load(r->fp.idx);
+		if (mi && mm_verbose >= 2 && (mi->k != r->opt.k || mi->w != r->opt.w || mi->is_hpc != r->opt.is_hpc))
+			fprintf(stderr, "[WARNING]\033[1;31m Indexing parameters (-k, -w or -H) overridden by parameters used in the prebuilt index.\033[0m\n");
+	} else
+		mi = mm_idx_gen(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.is_hpc, r->opt.mini_batch_size, n_threads, r->opt.batch_size, 1);
+	if (mi) {
+		if (r->fp_out) mm_idx_dump(r->fp_out, mi);
+		++r->n_parts;
+	}
+	return mi;
+}
+
+int mm_idx_reader_eof(const mm_idx_reader_t *r) // TODO: in extremely rare cases, mm_bseq_eof() might not work
+{
+	return r->is_idx? (feof(r->fp.idx) || ftell(r->fp.idx) == r->idx_size) : mm_bseq_eof(r->fp.seq);
 }

kalloc.c

@@ -1,175 +1,144 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <limits.h>
 #include "kalloc.h"
 
-/* The whole thing is: ("@" for the kheader_t of the block, "-" for free
- * memory, and "+" for allocated memory. One char for one unit.)
- *                        
- *           This region is core 1.                             This region is core 2.
+/* In kalloc, a *core* is a large chunk of contiguous memory. Each core is
+ * associated with a master header, which keeps the size of the current core
+ * and the pointer to next core. Kalloc allocates small *blocks* of memory from
+ * the cores and organizes free memory blocks in a circular single-linked list.
  *
- *   @-------@++++++@++++++++++++@------------           @----------@++++++++++++@+++++++@------------
- *   |                           |                       |                               |
- *   p=p->ptr->ptr->ptr->ptr     p->ptr             p->ptr->ptr                p->ptr->ptr->ptr
+ * In the following diagram, "@" stands for the header of a free block (of type
+ * header_t), "#" for the header of an allocated block (of type size_t), "-"
+ * for free memory, and "+" for allocated memory.
+ *
+ * master        This region is core 1.          master           This region is core 2.
+ *      |                                             |
+ *      *@-------#++++++#++++++++++++@--------        *@----------#++++++++++++#+++++++@------------
+ *       |                           |                 |                               |
+ *       p=p->ptr->ptr->ptr->ptr     p->ptr            p->ptr->ptr                     p->ptr->ptr->ptr
  */
 
-#define PTR(p) ((size_t*)((size_t*)p)[1])
+#define MIN_CORE_SIZE 0x80000
 
-typedef struct _allocated_t {
-	struct _allocated_t *next;
-	size_t *ptr;
-} allocated_t;
+typedef struct header_t {
+	size_t size;
+	struct header_t *ptr;
+} header_t;
 
 typedef struct {
-	size_t base[2], *loop_head;
-	allocated_t list_head, *list_tail;
-	size_t total_allocated;
+	header_t base, *loop_head, *core_head; /* base is a zero-sized block always kept in the loop */
 } kmem_t;
 
-void *km_init()
-{
-	return calloc(1, sizeof(kmem_t));
-}
-
-static void kerror(const char *s)
+static void panic(const char *s)
 {
 	fprintf(stderr, "%s\n", s);
-	exit(1);
+	abort();
 }
 
-static size_t *morecore(kmem_t *km, size_t nu)
+void *km_init(void)
 {
-	size_t rnu, *up;
-
-	rnu = (nu + 0xfffff) & (~(size_t)0xfffff);
-	up = (size_t*)malloc(rnu * sizeof(size_t));
-	if (!up) { /* fail to allocate memory */
-		km_stat(km);
-		fprintf(stderr, "[morecore] %lu bytes requested but not available.\n", (unsigned long)rnu * sizeof(size_t));
-		exit(1);
-	}
-	/* put the pointer in km->list_head */
-	if (km->list_tail == 0) km->list_tail = &km->list_head;
-	km->list_tail->ptr = up;
-	km->list_tail->next = (allocated_t*)calloc(1, sizeof(allocated_t));
-	km->list_tail = km->list_tail->next;
-
-	km->total_allocated += rnu * sizeof(size_t);
-	*up = rnu; /* the size of the current block, and in this case the block is the same as the new core */
-	kfree(km, up + 1); /* initialize the new "core" */
-	return km->loop_head;
+	return calloc(1, sizeof(kmem_t));
 }
 
 void km_destroy(void *_km)
 {
 	kmem_t *km = (kmem_t*)_km;
-	allocated_t *p, *q;
-	if (km == 0) return;
-	p = &km->list_head;
-	do {
-		q = p->next;
-		free(p->ptr);
-		if (p != &km->list_head) free(p);
+	header_t *p, *q;
+	if (km == NULL) return;
+	for (p = km->core_head; p != NULL;) {
+		q = p->ptr;
+		free(p);
 		p = q;
-	} while (p && p->next);
-	if (p != &km->list_head) free(p);
+	}
 	free(km);
 }
 
-void kfree(void *_km, void *ap)
+static header_t *morecore(kmem_t *km, size_t nu)
 {
-	size_t *p, *q;
+	header_t *q;
+	size_t bytes, *p;
+	nu = (nu + 1 + (MIN_CORE_SIZE - 1)) / MIN_CORE_SIZE * MIN_CORE_SIZE; /* the first +1 for core header */
+	bytes = nu * sizeof(header_t);
+	q = (header_t*)malloc(bytes);
+	if (!q) panic("[morecore] insufficient memory");
+	q->ptr = km->core_head, q->size = nu, km->core_head = q;
+	p = (size_t*)(q + 1);
+	*p = nu - 1; /* the size of the free block; -1 because the first unit is used for the core header */
+	kfree(km, p + 1); /* initialize the new "core"; NB: the core header is not looped. */
+	return km->loop_head;
+}
+
+void kfree(void *_km, void *ap) /* kfree() also adds a new core to the circular list */
+{
+	header_t *p, *q;
 	kmem_t *km = (kmem_t*)_km;
 	
 	if (!ap) return;
-	if (km == 0) {
+	if (km == NULL) {
 		free(ap);
 		return;
 	}
-	p = (size_t*)ap - 1; /* *p is the size of the current block */
+	p = (header_t*)((size_t*)ap - 1);
+	p->size = *((size_t*)ap - 1);
 	/* Find the pointer that points to the block to be freed. The following loop can stop on two conditions:
 	 *
-	 * a) "p>q && p<q->ptr": @------@++++++++@+++++++@-------    @---------------@+++++++@-------
+	 * a) "p>q && p<q->ptr": @------#++++++++#+++++++@-------    @---------------#+++++++@-------
 	 *    (can also be in    |      |                |        -> |                       |
 	 *     two cores)        q      p           q->ptr           q                  q->ptr
 	 *
-	 *                       @--------    @+++++++++@--------    @--------    @------------------
+	 *                       @--------    #+++++++++@--------    @--------    @------------------
 	 *                       |            |         |         -> |            |
 	 *                       q            p    q->ptr            q       q->ptr
 	 *
-	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------@+++++   @--------@+++++++     @-------@+++++   @----------------
+	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------#+++++   @--------#+++++++     @-------#+++++   @----------------
 	 *                                       |                |        |         -> |                |
 	 *                                  q->ptr                q        p       q->ptr                q
 	 *
-	 *                                       @+++++++@-----   @++++++++@-------     @-------------   @++++++++@-------
+	 *                                       #+++++++@-----   #++++++++@-------     @-------------   #++++++++@-------
 	 *                                       |       |                 |         -> |                         |
 	 *                                       p  q->ptr                 q       q->ptr                         q
 	 */
-	for (q = km->loop_head; !(p > q && p < PTR(q)); q = PTR(q))
-		if (q >= PTR(q) && (p > q || p < PTR(q))) break;
-	if (p + (*p) == PTR(q)) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
-		*p += *PTR(q); /* this is the new q->ptr size */
-		p[1] = (size_t)PTR(PTR(q)); /* this is the new q->ptr->ptr */
-		/* p is actually the new q->ptr. The actual change happens a few lines below. */
-	} else if (p + (*p) > PTR(q) && PTR(q) >= p) { /* the end of the allocated block is in the next free block */
-		kerror("[kfree] The end of the allocated block enters a free block.");
-	} else p[1] = (size_t)PTR(q); /* backup q->ptr */
+	for (q = km->loop_head; !(p > q && p < q->ptr); q = q->ptr)
+		if (q >= q->ptr && (p > q || p < q->ptr)) break;
+	if (p + p->size == q->ptr) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
+		p->size += q->ptr->size;
+		p->ptr = q->ptr->ptr;
+	} else if (p + p->size > q->ptr && q->ptr >= p) {
+		panic("[kfree] The end of the allocated block enters a free block.");
+	} else p->ptr = q->ptr; /* backup q->ptr */
 
-	if (q + (*q) == p) { /* two adjacent blocks, merge q and p (the other two cases) */
-		*q += *p;
-		q[1] = (size_t)PTR(p);
+	if (q + q->size == p) { /* two adjacent blocks, merge q and p (the other two cases) */
+		q->size += p->size;
+		q->ptr = p->ptr;
 		km->loop_head = q;
-	} else if (q + (*q) > p && p >= q) { /* the end of a free block in the allocated block */
-		kerror("[kfree] The end of a free block enters the allocated block.");
-	} else km->loop_head = p, q[1] = (size_t)p; /* in two cores, cannot be merged */
-}
-
-void *krealloc(void *_km, void *ap, size_t n_bytes)
-{
-	kmem_t *km = (kmem_t*)_km;
-	size_t n_units, *p, *q;
-
-	if (n_bytes == 0) {
-		kfree(km, ap); return 0;
-	}
-	if (km == 0) return realloc(ap, n_bytes);
-	if (!ap) return kmalloc(km, n_bytes);
-	n_units = 1 + (n_bytes + sizeof(size_t) - 1) / sizeof(size_t);
-	p = (size_t*)ap - 1;
-	if (*p >= n_units) return ap; /* TODO: this prevents shrinking */
-	q = (size_t*)kmalloc(km, n_bytes);
-	memcpy(q, ap, (*p - 1) * sizeof(size_t));
-	kfree(km, ap);
-	return q;
+	} else if (q + q->size > p && p >= q) {
+		panic("[kfree] The end of a free block enters the allocated block.");
+	} else km->loop_head = p, q->ptr = p; /* in two cores, cannot be merged; create a new block in the list */
 }
 
 void *kmalloc(void *_km, size_t n_bytes)
 {
 	kmem_t *km = (kmem_t*)_km;
-	size_t n_units, *p, *q;
+	size_t n_units;
+	header_t *p, *q;
 
 	if (n_bytes == 0) return 0;
-	if (km == 0) return malloc(n_bytes);
-	/* "n_units" means the number of units. The size of one unit equals to sizeof(kheader_t).
-	 * "1" is the kheader_t of a block, which is always required. */
-	n_units = 1 + (n_bytes + sizeof(size_t) - 1) / sizeof(size_t);
-	if (n_units&1) ++n_units; /* make n_units an even number, or it will segfault if only one unit remains */
+	if (km == NULL) return malloc(n_bytes);
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t) + 1;
 
-	if (!(q = km->loop_head)) { /* the first time when kmalloc() is called, intialization */
-		km->base[1] = (size_t)(km->loop_head = q = km->base); *q = 0;
-	}
-	for (p = PTR(q);; q = p, p = PTR(p)) { /* search for a suitable block */
-		if (*p >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
-			if (*p == n_units) q[1] = (size_t)PTR(p); /* no need to split the block */
-			else { /* split the block */
-				/* memory is allocated at the end of the block */
-				*p -= n_units; /* reduce the size of the free block */
-				p += *p; /* skip to the kheader_t of the allocated block */
-				*p = n_units; /* set the size */
+	if (!(q = km->loop_head)) /* the first time when kmalloc() is called, intialize it */
+		q = km->loop_head = km->base.ptr = &km->base;
+	for (p = q->ptr;; q = p, p = p->ptr) { /* search for a suitable block */
+		if (p->size >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
+			if (p->size == n_units) q->ptr = p->ptr; /* no need to split the block */
+			else { /* split the block. NB: memory is allocated at the end of the block! */
+				p->size -= n_units; /* reduce the size of the free block */
+				p += p->size; /* p points to the allocated block */
+				*(size_t*)p = n_units; /* set the size */
 			}
 			km->loop_head = q; /* set the end of chain */
-			return p + 1; /* skip the kheader_t */
+			return (size_t*)p + 1;
 		}
 		if (p == km->loop_head) { /* then ask for more "cores" */
 			if ((p = morecore(km, n_units)) == 0) return 0;
@@ -182,33 +151,44 @@ void *kcalloc(void *_km, size_t count, size_t size)
 	kmem_t *km = (kmem_t*)_km;
 	void *p;
 	if (size == 0 || count == 0) return 0;
-	if (km == 0) return calloc(count, size);
+	if (km == NULL) return calloc(count, size);
 	p = kmalloc(km, count * size);
 	memset(p, 0, count * size);
 	return p;
 }
 
-void km_stat(const void *_km)
+void *krealloc(void *_km, void *ap, size_t n_bytes) // TODO: this can be made more efficient in principle
 {
 	kmem_t *km = (kmem_t*)_km;
-	unsigned n_blocks, n_units;
-	size_t max_block = 0, *p, *q;
-	float frag;
+	size_t n_units, *p, *q;
 
-	if (km == 0 || !(p = km->loop_head)) return;
-	n_blocks = n_units = 0;
-	do {
-		q = PTR(p);
-		if (*p > max_block) max_block = *p;
-		n_units += *p;
-		if (p + (*p) > q && q > p)
-			kerror("[kr_stat] The end of a free block enters another free block.");
-		p = q;
-		++n_blocks;
-	} while (p != km->loop_head);
-	
-	--n_blocks;
-	frag = 1.0/1024.0 * n_units * sizeof(size_t) / n_blocks;
-	fprintf(stderr, "[kr_stat] tot=%lu, free=%lu, n_block=%u, max_block=%lu, frag_len=%.3fK\n",
-			(unsigned long)km->total_allocated, (unsigned long)n_units * sizeof(size_t), n_blocks, (unsigned long)max_block * sizeof(size_t), frag);
+	if (n_bytes == 0) {
+		kfree(km, ap); return 0;
+	}
+	if (km == NULL) return realloc(ap, n_bytes);
+	if (ap == NULL) return kmalloc(km, n_bytes);
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t);
+	p = (size_t*)ap - 1;
+	if (*p >= n_units) return ap; /* TODO: this prevents shrinking */
+	q = (size_t*)kmalloc(km, n_bytes);
+	memcpy(q, ap, (*p - 1) * sizeof(header_t));
+	kfree(km, ap);
+	return q;
+}
+
+void km_stat(const void *_km, km_stat_t *s)
+{
+	kmem_t *km = (kmem_t*)_km;
+	header_t *p;
+	memset(s, 0, sizeof(km_stat_t));
+	if (km == NULL || km->loop_head == NULL) return;
+	for (p = km->loop_head;; p = p->ptr) {
+		s->available += p->size * sizeof(header_t);
+		if (p->size != 0) ++s->n_blocks; /* &kmem_t::base is always one of the cores. It is zero-sized. */
+		if (p->ptr > p && p + p->size > p->ptr)
+			panic("[km_stat] The end of a free block enters another free block.");
+		if (p->ptr == km->loop_head) break;
+	}
+	for (p = km->core_head; p != NULL; p = p->ptr)
+		++s->n_cores, s->capacity += p->size * sizeof(header_t);
 }

kalloc.h

@@ -1,14 +1,16 @@
 #ifndef _KALLOC_H_
 #define _KALLOC_H_
 
-#include <stdlib.h>
-
-#define km_size(x) (*(((size_t*)(x))-1) * sizeof(size_t))
+#include <stddef.h> /* for size_t */
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct {
+	size_t capacity, available, n_blocks, n_cores;
+} km_stat_t;
+
 void *kmalloc(void *km, size_t size);
 void *krealloc(void *km, void *ptr, size_t size);
 void *kcalloc(void *km, size_t count, size_t size);
@@ -16,8 +18,7 @@ void kfree(void *km, void *ptr);
 
 void *km_init(void);
 void km_destroy(void *km);
-
-void km_stat(const void *km); // TODO: return numbers instead of print to stderr
+void km_stat(const void *_km, km_stat_t *s);
 
 #ifdef __cplusplus
 }

ksw2.h

@@ -26,6 +26,7 @@ typedef struct {
 	int mte, mte_q;        // max score when reaching the end of target
 	int score;             // max score reaching both ends; may be KSW_NEG_INF
 	int m_cigar, n_cigar;
+	int reach_end;
 	uint32_t *cigar;
 } ksw_extz_t;
 
@@ -46,14 +47,17 @@ typedef struct {
  * @param flag      flag (see KSW_EZ_* macros)
  * @param ez        (out) scores and cigar
  */
-void ksw_extz(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
-void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+void ksw_extz(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+			  int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 
 void ksw_extd(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
 			  int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int flag, ksw_extz_t *ez);
 
 void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int flag, ksw_extz_t *ez);
+				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 
 void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
 				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez);
@@ -149,7 +153,7 @@ static inline void ksw_reset_extz(ksw_extz_t *ez)
 {
 	ez->max_q = ez->max_t = ez->mqe_t = ez->mte_q = -1;
 	ez->max = 0, ez->score = ez->mqe = ez->mte = KSW_NEG_INF;
-	ez->n_cigar = 0, ez->zdropped = 0;
+	ez->n_cigar = 0, ez->zdropped = 0, ez->reach_end = 0;
 }
 
 static inline int ksw_apply_zdrop(ksw_extz_t *ez, int is_rot, int32_t H, int a, int b, int zdrop, int8_t e)

ksw2_dispatch.c

@@ -50,32 +50,32 @@ int x86_simd(void)
 	return flag;
 }
 
-void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez)
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 {
-	extern void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
-	extern void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+	extern void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	extern void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 	unsigned simd;
 	simd = x86_simd();
 	if (simd & SIMD_SSE4_1)
-		ksw_extz2_sse41(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, flag, ez);
+		ksw_extz2_sse41(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, end_bonus, flag, ez);
 	else if (simd & SIMD_SSE2)
-		ksw_extz2_sse2(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, flag, ez);
+		ksw_extz2_sse2(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, end_bonus, flag, ez);
 	else abort();
 }
 
 void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez)
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 {
 	extern void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez);
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 	extern void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez);
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 	unsigned simd;
 	simd = x86_simd();
 	if (simd & SIMD_SSE4_1)
-		ksw_extd2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, flag, ez);
+		ksw_extd2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
 	else if (simd & SIMD_SSE2)
-		ksw_extd2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, flag, ez);
+		ksw_extd2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
 	else abort();
 }
 

ksw2_extd2_sse.c

@@ -17,14 +17,14 @@
 #ifdef KSW_CPU_DISPATCH
 #ifdef __SSE4_1__
 void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez)
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 #else
 void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez)
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 #endif
 #else
 void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez)
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 #endif // ~KSW_CPU_DISPATCH
 {
 #define __dp_code_block1 \
@@ -61,7 +61,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_;
+	__m128i q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_, sc_N_;
 	__m128i *u, *v, *x, *y, *x2, *y2, *s, *p = 0;
 
 	ksw_reset_extz(ez);
@@ -76,6 +76,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	qe2_    = _mm_set1_epi8(q2 + e2);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = _mm_set1_epi8(-e2);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 
 	if (w < 0) w = tlen > qlen? tlen : qlen;
@@ -162,10 +163,11 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp);
 			}
 		} else {
@@ -378,10 +380,14 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	if (!approx_max) kfree(km, H);
 	if (with_cigar) { // backtrack
 		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
-		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
+		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
-		else if (ez->max_t >= 0 && ez->max_q >= 0)
+		} else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > ez->max) {
+			ez->reach_end = 1;
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		} else if (ez->max_t >= 0 && ez->max_q >= 0) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		}
 		kfree(km, mem2); kfree(km, off);
 	}
 }

ksw2_exts2_sse.c

@@ -59,7 +59,7 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, q2_, qe_, zero_, sc_mch_, sc_mis_, m1_;
+	__m128i q_, q2_, qe_, zero_, sc_mch_, sc_mis_, sc_N_, m1_;
 	__m128i *u, *v, *x, *y, *x2, *s, *p = 0, *donor, *acceptor;
 
 	ksw_reset_extz(ez);
@@ -71,6 +71,7 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	qe_     = _mm_set1_epi8(q + e);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = _mm_set1_epi8(-e);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 
 	tlen_ = (tlen + 15) / 16;
@@ -159,10 +160,11 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp);
 			}
 		} else {

ksw2_extz2_sse.c

@@ -15,12 +15,12 @@
 
 #ifdef KSW_CPU_DISPATCH
 #ifdef __SSE4_1__
-void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez)
+void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 #else
-void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez)
+void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 #endif
 #else
-void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez)
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
 #endif // ~KSW_CPU_DISPATCH
 {
 #define __dp_code_block1 \
@@ -50,7 +50,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, qe2_, zero_, flag1_, flag2_, flag8_, flag16_, sc_mch_, sc_mis_, m1_, max_sc_;
+	__m128i q_, qe2_, zero_, flag1_, flag2_, flag8_, flag16_, sc_mch_, sc_mis_, sc_N_, m1_, max_sc_;
 	__m128i *u, *v, *x, *y, *s, *p = 0;
 
 	ksw_reset_extz(ez);
@@ -65,6 +65,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	flag16_ = _mm_set1_epi8(0x10);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = _mm_set1_epi8(-e);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 	max_sc_ = _mm_set1_epi8(mat[0] + (q + e) * 2);
 
@@ -130,10 +131,11 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((uint8_t*)s + t), tmp);
 			}
 		} else {
@@ -289,10 +291,14 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	if (!approx_max) kfree(km, H);
 	if (with_cigar) { // backtrack
 		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
-		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
+		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
-		else if (ez->max_t >= 0 && ez->max_q >= 0)
+		} else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > ez->max) {
+			ez->reach_end = 1;
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		} else if (ez->max_t >= 0 && ez->max_q >= 0) {
 			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		}
 		kfree(km, mem2); kfree(km, off);
 	}
 }

ksw2_ll_sse.c

@@ -122,7 +122,7 @@ int ksw_ll_i16(void *q_, int tlen, const uint8_t *target, int _gapo, int _gape,
 			f = _mm_max_epi16(f, h);
 			h = _mm_load_si128(H0 + j);
 		}
-		for (k = 0; LIKELY(k < 16); ++k) {
+		for (k = 0; LIKELY(k < 8); ++k) {
 			f = _mm_slli_si128(f, 2);
 			for (j = 0; LIKELY(j < slen); ++j) {
 				h = _mm_load_si128(H1 + j);

main.c

@@ -6,7 +6,7 @@
 #include "mmpriv.h"
 #include "getopt.h"
 
-#define MM_VERSION "2.1.1-r341"
+#define MM_VERSION "2.3-r531"
 
 #ifdef __linux__
 #include <sys/resource.h>
@@ -25,17 +25,22 @@ void liftrlimit() {}
 static struct option long_options[] = {
 	{ "bucket-bits",    required_argument, 0, 0 },
 	{ "mb-size",        required_argument, 0, 'K' },
-	{ "int-rname",      no_argument,       0, 0 },
+	{ "seed",           required_argument, 0, 0 },
 	{ "no-kalloc",      no_argument,       0, 0 },
 	{ "print-qname",    no_argument,       0, 0 },
 	{ "no-self",        no_argument,       0, 0 },
-	{ "print-seed",     no_argument,       0, 0 },
+	{ "print-seeds",    no_argument,       0, 0 },
 	{ "max-chain-skip", required_argument, 0, 0 },
 	{ "min-dp-len",     required_argument, 0, 0 },
 	{ "print-aln-seq",  no_argument,       0, 0 },
 	{ "splice",         no_argument,       0, 0 },
 	{ "cost-non-gt-ag", required_argument, 0, 0 },
-	{ "no-sam-sq",      no_argument,       0, 0 },
+	{ "no-long-join",   no_argument,       0, 0 },
+	{ "sr",             no_argument,       0, 0 },
+	{ "frag",           optional_argument, 0, 0 },
+	{ "secondary",      optional_argument, 0, 0 },
+	{ "cs",             optional_argument, 0, 0 },
+	{ "end-bonus",      required_argument, 0, 0 },
 	{ "help",           no_argument,       0, 'h' },
 	{ "max-intron-len", required_argument, 0, 'G' },
 	{ "version",        no_argument,       0, 'V' },
@@ -60,37 +65,50 @@ static inline int64_t mm_parse_num(const char *str)
 
 int main(int argc, char *argv[])
 {
+	const char *opt_str = "2aSw:k:K:t:r:f:Vv:g:G:I:d:XT:s:x:Hcp:M:n:z:A:B:O:E:m:N:Qu:R:hF:i:L";
 	mm_mapopt_t opt;
-	int i, c, k = 15, w = -1, bucket_bits = MM_IDX_DEF_B, n_threads = 3, keep_name = 1, is_idx, is_hpc = 0, long_idx, idx_par_set = 0, max_intron_len = 0, n_idx_part = 0;
-	int minibatch_size = 200000000;
-	uint64_t batch_size = 4000000000ULL;
-	mm_bseq_file_t *fp = 0;
+	mm_idxopt_t ipt;
+	int i, c, n_threads = 3, long_idx;
 	char *fnw = 0, *rg = 0, *s;
-	FILE *fpr = 0, *fpw = 0, *fp_help = stderr;
+	FILE *fp_help = stderr;
+	mm_idx_reader_t *idx_rdr;
+	mm_idx_t *mi;
 
+	mm_verbose = 3;
 	liftrlimit();
 	mm_realtime0 = realtime();
-	mm_mapopt_init(&opt);
+	mm_set_opt(0, &ipt, &opt);
 
-	while ((c = getopt_long(argc, argv, "aSw:k:K:t:r:f:Vv:g:G:I:d:XT:s:x:Hcp:M:n:z:A:B:O:E:m:N:Qu:R:h", long_options, &long_idx)) >= 0) {
-		if (c == 'w') w = atoi(optarg), idx_par_set = 1;
-		else if (c == 'k') k = atoi(optarg), idx_par_set = 1;
-		else if (c == 'H') is_hpc = 1, idx_par_set = 1;
+	while ((c = getopt_long(argc, argv, opt_str, long_options, &long_idx)) >= 0) // apply option -x/preset first
+		if (c == 'x') {
+			if (mm_set_opt(optarg, &ipt, &opt) < 0) {
+				fprintf(stderr, "[ERROR] unknown preset '%s'\n", optarg);
+				return 1;
+			}
+			break;
+		}
+	optreset = 1;
+
+	while ((c = getopt_long(argc, argv, opt_str, long_options, &long_idx)) >= 0) {
+		if (c == 'w') ipt.w = atoi(optarg);
+		else if (c == 'k') ipt.k = atoi(optarg);
+		else if (c == 'H') ipt.is_hpc = 1;
 		else if (c == 'd') fnw = optarg; // the above are indexing related options, except -I
 		else if (c == 'r') opt.bw = (int)mm_parse_num(optarg);
-		else if (c == 'f') opt.mid_occ_frac = atof(optarg);
 		else if (c == 't') n_threads = atoi(optarg);
 		else if (c == 'v') mm_verbose = atoi(optarg);
 		else if (c == 'g') opt.max_gap = (int)mm_parse_num(optarg);
-		else if (c == 'G') max_intron_len = (int)mm_parse_num(optarg);
+		else if (c == 'G') mm_mapopt_max_intron_len(&opt, (int)mm_parse_num(optarg));
+		else if (c == 'F') opt.max_frag_len = (int)mm_parse_num(optarg);
+		else if (c == 'i') opt.min_iden = atof(optarg);
 		else if (c == 'N') opt.best_n = atoi(optarg);
 		else if (c == 'p') opt.pri_ratio = atof(optarg);
 		else if (c == 'M') opt.mask_level = atof(optarg);
 		else if (c == 'c') opt.flag |= MM_F_OUT_CG | MM_F_CIGAR;
-		else if (c == 'S') opt.flag |= MM_F_OUT_CS | MM_F_CIGAR;
 		else if (c == 'X') opt.flag |= MM_F_AVA | MM_F_NO_SELF;
 		else if (c == 'a') opt.flag |= MM_F_OUT_SAM | MM_F_CIGAR;
 		else if (c == 'Q') opt.flag |= MM_F_NO_QUAL;
+		else if (c == 'L') opt.flag |= MM_F_LONG_CIGAR;
 		else if (c == 'T') opt.sdust_thres = atoi(optarg);
 		else if (c == 'n') opt.min_cnt = atoi(optarg);
 		else if (c == 'm') opt.min_chain_score = atoi(optarg);
@@ -98,33 +116,65 @@ int main(int argc, char *argv[])
 		else if (c == 'B') opt.b = atoi(optarg);
 		else if (c == 'z') opt.zdrop = atoi(optarg);
 		else if (c == 's') opt.min_dp_max = atoi(optarg);
-		else if (c == 'I') batch_size = mm_parse_num(optarg);
-		else if (c == 'K') minibatch_size = (int)mm_parse_num(optarg);
+		else if (c == 'I') ipt.batch_size = mm_parse_num(optarg);
+		else if (c == 'K') opt.mini_batch_size = (int)mm_parse_num(optarg);
 		else if (c == 'R') rg = optarg;
 		else if (c == 'h') fp_help = stdout;
-		else if (c == 0 && long_idx == 0) bucket_bits = atoi(optarg); // --bucket-bits
-		else if (c == 0 && long_idx == 2) keep_name = 0; // --int-rname
+		else if (c == '2') opt.flag |= MM_F_2_IO_THREADS;
+		else if (c == 0 && long_idx == 0) ipt.bucket_bits = atoi(optarg); // --bucket-bits
+		else if (c == 0 && long_idx == 2) opt.seed = atoi(optarg); // --seed
 		else if (c == 0 && long_idx == 3) mm_dbg_flag |= MM_DBG_NO_KALLOC; // --no-kalloc
 		else if (c == 0 && long_idx == 4) mm_dbg_flag |= MM_DBG_PRINT_QNAME; // --print-qname
 		else if (c == 0 && long_idx == 5) opt.flag |= MM_F_NO_SELF; // --no-self
-		else if (c == 0 && long_idx == 6) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_SEED; // --print-seed
+		else if (c == 0 && long_idx == 6) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_SEED, n_threads = 1; // --print-seed
 		else if (c == 0 && long_idx == 7) opt.max_chain_skip = atoi(optarg); // --max-chain-skip
 		else if (c == 0 && long_idx == 8) opt.min_ksw_len = atoi(optarg); // --min-dp-len
 		else if (c == 0 && long_idx == 9) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_ALN_SEQ; // --print-aln-seq
 		else if (c == 0 && long_idx ==10) opt.flag |= MM_F_SPLICE; // --splice
 		else if (c == 0 && long_idx ==11) opt.noncan = atoi(optarg); // --cost-non-gt-ag
-		else if (c == 0 && long_idx ==12) opt.flag |= MM_F_NO_SAM_SQ; // --no-sam-sq
-		else if (c == 'V') {
+		else if (c == 0 && long_idx ==12) opt.flag |= MM_F_NO_LJOIN; // --no-long-join
+		else if (c == 0 && long_idx ==13) opt.flag |= MM_F_SR; // --sr
+		else if (c == 0 && long_idx ==17) opt.end_bonus = atoi(optarg); // --end-bonus
+		else if (c == 0 && long_idx == 14) { // --frag
+			if (optarg == 0 || strcmp(optarg, "yes") == 0 || strcmp(optarg, "y") == 0)
+				opt.flag |= MM_F_FRAG_MODE;
+			else opt.flag &= ~MM_F_FRAG_MODE;
+		} else if (c == 0 && long_idx == 15) { // --secondary
+			if (optarg == 0 || strcmp(optarg, "yes") == 0 || strcmp(optarg, "y") == 0)
+				opt.flag &= ~MM_F_NO_PRINT_2ND;
+			else opt.flag |= MM_F_NO_PRINT_2ND;
+		} else if (c == 0 && long_idx == 16) { // --cs
+			opt.flag |= MM_F_OUT_CS | MM_F_CIGAR;
+			if (optarg == 0 || strcmp(optarg, "short") == 0) {
+				opt.flag &= ~MM_F_OUT_CS_LONG;
+			} else if (strcmp(optarg, "long") == 0) {
+				opt.flag |= MM_F_OUT_CS_LONG;
+			} else if (strcmp(optarg, "none") == 0) {
+				opt.flag &= ~MM_F_OUT_CS;
+			} else if (mm_verbose >= 2) {
+				fprintf(stderr, "[WARNING]\033[1;31m --cs only takes 'short' or 'long'. Invalid values are assumed to be 'short'.\033[0m\n");
+			}
+		} else if (c == 'S') {
+			opt.flag |= MM_F_OUT_CS | MM_F_CIGAR | MM_F_OUT_CS_LONG;
+			if (mm_verbose >= 2)
+				fprintf(stderr, "[WARNING]\033[1;31m option -S is deprecated and may be removed in future. Please use --cs=long instead.\033[0m\n");
+		} else if (c == 'V') {
 			puts(MM_VERSION);
 			return 0;
+		} else if (c == 'f') {
+			double x;
+			char *p;
+			x = strtod(optarg, &p);
+			if (x < 1.0) opt.mid_occ_frac = x, opt.mid_occ = 0;
+			else opt.mid_occ = (int)(x + .499);
+			if (*p == ',') opt.max_occ = (int)(strtod(p+1, &p) + .499);
 		} else if (c == 'u') {
-			if (*optarg == 'b') opt.flag |= MM_F_SPLICE_FOR|MM_F_SPLICE_REV;
-			else if (*optarg == 'B') opt.flag |= MM_F_SPLICE_BOTH;
-			else if (*optarg == 'f') opt.flag |= MM_F_SPLICE_FOR, opt.flag &= ~MM_F_SPLICE_REV;
-			else if (*optarg == 'r') opt.flag |= MM_F_SPLICE_REV, opt.flag &= ~MM_F_SPLICE_FOR;
-			else if (*optarg == 'n') opt.flag &= ~(MM_F_SPLICE_FOR|MM_F_SPLICE_REV);
+			if (*optarg == 'b') opt.flag |= MM_F_SPLICE_FOR|MM_F_SPLICE_REV; // both strands
+			else if (*optarg == 'f') opt.flag |= MM_F_SPLICE_FOR, opt.flag &= ~MM_F_SPLICE_REV; // match GT-AG
+			else if (*optarg == 'r') opt.flag |= MM_F_SPLICE_REV, opt.flag &= ~MM_F_SPLICE_FOR; // match CT-AC (reverse complement of GT-AG)
+			else if (*optarg == 'n') opt.flag &= ~(MM_F_SPLICE_FOR|MM_F_SPLICE_REV); // don't try to match the GT-AG signal
 			else {
-				fprintf(stderr, "[E::%s] unrecognized cDNA direction\n", __func__);
+				fprintf(stderr, "[ERROR]\033[1;31m unrecognized cDNA direction\033[0m\n");
 				return 1;
 			}
 		} else if (c == 'O') {
@@ -133,66 +183,34 @@ int main(int argc, char *argv[])
 		} else if (c == 'E') {
 			opt.e = opt.e2 = strtol(optarg, &s, 10);
 			if (*s == ',') opt.e2 = strtol(s + 1, &s, 10);
-		} else if (c == 'x') {
-			if (strcmp(optarg, "ava-ont") == 0) {
-				opt.flag |= MM_F_AVA | MM_F_NO_SELF;
-				opt.min_chain_score = 100, opt.pri_ratio = 0.0f, opt.max_gap = 10000, opt.max_chain_skip = 25;
-				minibatch_size = 500000000;
-				k = 15, w = 5;
-			} else if (strcmp(optarg, "ava-pb") == 0) {
-				opt.flag |= MM_F_AVA | MM_F_NO_SELF;
-				opt.min_chain_score = 100, opt.pri_ratio = 0.0f, opt.max_gap = 10000, opt.max_chain_skip = 25;
-				minibatch_size = 500000000;
-				is_hpc = 1, k = 19, w = 5;
-			} else if (strcmp(optarg, "map10k") == 0 || strcmp(optarg, "map-pb") == 0) {
-				is_hpc = 1, k = 19;
-			} else if (strcmp(optarg, "map-ont") == 0) {
-				is_hpc = 0, k = 15;
-			} else if (strcmp(optarg, "asm5") == 0) {
-				k = 19, w = 19;
-				opt.a = 1, opt.b = 19, opt.q = 39, opt.q2 = 81, opt.e = 3, opt.e2 = 1, opt.zdrop = 200;
-				opt.min_dp_max = 200;
-			} else if (strcmp(optarg, "asm10") == 0) {
-				k = 19, w = 19;
-				opt.a = 1, opt.b = 9, opt.q = 16, opt.q2 = 41, opt.e = 2, opt.e2 = 1, opt.zdrop = 200;
-				opt.min_dp_max = 200;
-			} else if (strcmp(optarg, "splice") == 0 || strcmp(optarg, "cdna") == 0) {
-				k = 15, w = 5;
-				opt.flag |= MM_F_SPLICE | MM_F_SPLICE_FOR | MM_F_SPLICE_REV;
-				opt.max_gap = 2000, opt.max_gap_ref = opt.bw = 200000;
-				opt.a = 1, opt.b = 2, opt.q = 2, opt.e = 1, opt.q2 = 32, opt.e2 = 0;
-				opt.noncan = 5;
-				opt.zdrop = 200;
-			} else {
-				fprintf(stderr, "[E::%s] unknown preset '%s'\n", __func__, optarg);
-				return 1;
-			}
 		}
 	}
-	if (w < 0) w = (int)(.6666667 * k + .499);
-	if ((opt.flag & MM_F_SPLICE) && max_intron_len > 0)
-		opt.max_gap_ref = opt.bw = max_intron_len;
+	if ((opt.flag & MM_F_SPLICE) && (opt.flag & MM_F_FRAG_MODE)) {
+		fprintf(stderr, "[ERROR]\033[1;31m --splice and --frag should not be specified at the same time.\033[0m\n");
+		return 1;
+	}
 
 	if (argc == optind || fp_help == stdout) {
 		fprintf(fp_help, "Usage: minimap2 [options] <target.fa>|<target.idx> [query.fa] [...]\n");
 		fprintf(fp_help, "Options:\n");
 		fprintf(fp_help, "  Indexing:\n");
 		fprintf(fp_help, "    -H           use homopolymer-compressed k-mer\n");
-		fprintf(fp_help, "    -k INT       k-mer size (no larger than 28) [%d]\n", k);
-		fprintf(fp_help, "    -w INT       minizer window size [{-k}*2/3]\n");
+		fprintf(fp_help, "    -k INT       k-mer size (no larger than 28) [%d]\n", ipt.k);
+		fprintf(fp_help, "    -w INT       minizer window size [%d]\n", ipt.w);
 		fprintf(fp_help, "    -I NUM       split index for every ~NUM input bases [4G]\n");
 		fprintf(fp_help, "    -d FILE      dump index to FILE []\n");
 		fprintf(fp_help, "  Mapping:\n");
 		fprintf(fp_help, "    -f FLOAT     filter out top FLOAT fraction of repetitive minimizers [%g]\n", opt.mid_occ_frac);
-		fprintf(fp_help, "    -g INT       stop chain enlongation if there are no minimizers in INT-bp [%d]\n", opt.max_gap);
-		fprintf(fp_help, "    -r INT       bandwidth used in chaining and DP-based alignment [%d]\n", opt.bw);
+		fprintf(fp_help, "    -g NUM       stop chain enlongation if there are no minimizers in INT-bp [%d]\n", opt.max_gap);
+		fprintf(fp_help, "    -G NUM       max intron length (effective with -xsplice; changing -r) [200k]\n");
+		fprintf(fp_help, "    -F NUM       max fragment length (effective with -xsr or in the fragment mode) [800]\n");
+		fprintf(fp_help, "    -r NUM       bandwidth used in chaining and DP-based alignment [%d]\n", opt.bw);
 		fprintf(fp_help, "    -n INT       minimal number of minimizers on a chain [%d]\n", opt.min_cnt);
 		fprintf(fp_help, "    -m INT       minimal chaining score (matching bases minus log gap penalty) [%d]\n", opt.min_chain_score);
 //		fprintf(fp_help, "    -T INT       SDUST threshold; 0 to disable SDUST [%d]\n", opt.sdust_thres); // TODO: this option is never used; might be buggy
 		fprintf(fp_help, "    -X           skip self and dual mappings (for the all-vs-all mode)\n");
 		fprintf(fp_help, "    -p FLOAT     min secondary-to-primary score ratio [%g]\n", opt.pri_ratio);
 		fprintf(fp_help, "    -N INT       retain at most INT secondary alignments [%d]\n", opt.best_n);
-		fprintf(fp_help, "    -G NUM       max intron length (only effective following -x splice) [200k]\n");
 		fprintf(fp_help, "  Alignment:\n");
 		fprintf(fp_help, "    -A INT       matching score [%d]\n", opt.a);
 		fprintf(fp_help, "    -B INT       mismatch penalty [%d]\n", opt.b);
@@ -201,74 +219,67 @@ int main(int argc, char *argv[])
 		fprintf(fp_help, "    -z INT       Z-drop score [%d]\n", opt.zdrop);
 		fprintf(fp_help, "    -s INT       minimal peak DP alignment score [%d]\n", opt.min_dp_max);
 		fprintf(fp_help, "    -u CHAR      how to find GT-AG. f:transcript strand, b:both strands, n:don't match GT-AG [n]\n");
+		fprintf(fp_help, "    -i FLOAT     min identity (mapQ reduced to 0 if below) [0]\n");
 		fprintf(fp_help, "  Input/Output:\n");
 		fprintf(fp_help, "    -a           output in the SAM format (PAF by default)\n");
 		fprintf(fp_help, "    -Q           don't output base quality in SAM\n");
+		fprintf(fp_help, "    -L           write CIGAR with >65535 ops at the CG tag\n");
 		fprintf(fp_help, "    -R STR       SAM read group line in a format like '@RG\\tID:foo\\tSM:bar' []\n");
 		fprintf(fp_help, "    -c           output CIGAR in PAF\n");
-		fprintf(fp_help, "    -S           output the cs tag in PAF (cs encodes both query and ref sequences)\n");
+		fprintf(fp_help, "    --cs[=STR]   output the cs tag; STR is 'short' (if absent) or 'long' [none]\n");
 		fprintf(fp_help, "    -t INT       number of threads [%d]\n", n_threads);
-		fprintf(fp_help, "    -K NUM       minibatch size [200M]\n");
+		fprintf(fp_help, "    -K NUM       minibatch size for mapping [500M]\n");
 //		fprintf(fp_help, "    -v INT       verbose level [%d]\n", mm_verbose);
 		fprintf(fp_help, "    --version    show version number\n");
 		fprintf(fp_help, "  Preset:\n");
-		fprintf(fp_help, "    -x STR       preset (recommended to be applied before other options) []\n");
-		fprintf(fp_help, "                 map10k/map-pb: -Hk19 (PacBio/ONT vs reference mapping)\n");
-		fprintf(fp_help, "                 map-ont: -k15 (slightly more sensitive than 'map10k' for ONT vs reference)\n");
+		fprintf(fp_help, "    -x STR       preset (always applied before other options) []\n");
+		fprintf(fp_help, "                 map-pb: -Hk19 (PacBio vs reference mapping)\n");
+		fprintf(fp_help, "                 map-ont: -k15 (Oxford Nanopore vs reference mapping)\n");
 		fprintf(fp_help, "                 asm5: -k19 -w19 -A1 -B19 -O39,81 -E3,1 -s200 -z200 (asm to ref mapping; break at 5%% div.)\n");
 		fprintf(fp_help, "                 asm10: -k19 -w19 -A1 -B9 -O16,41 -E2,1 -s200 -z200 (asm to ref mapping; break at 10%% div.)\n");
-		fprintf(fp_help, "                 ava-pb: -Hk19 -w5 -Xp0 -m100 -g10000 -K500m --max-chain-skip 25 (PacBio read overlap)\n");
-		fprintf(fp_help, "                 ava-ont: -k15 -w5 -Xp0 -m100 -g10000 -K500m --max-chain-skip 25 (ONT read overlap)\n");
+		fprintf(fp_help, "                 ava-pb: -Hk19 -w5 -Xp0 -m100 -g10000 --max-chain-skip 25 (PacBio read overlap)\n");
+		fprintf(fp_help, "                 ava-ont: -k15 -w5 -Xp0 -m100 -g10000 --max-chain-skip 25 (ONT read overlap)\n");
 		fprintf(fp_help, "                 splice: long-read spliced alignment (see minimap2.1 for details)\n");
+		fprintf(fp_help, "                 sr: short single-end reads without splicing (see minimap2.1 for details)\n");
 		fprintf(fp_help, "\nSee `man ./minimap2.1' for detailed description of command-line options.\n");
 		return fp_help == stdout? 0 : 1;
 	}
 
-	is_idx = mm_idx_is_idx(argv[optind]);
-	if (is_idx < 0) {
+	idx_rdr = mm_idx_reader_open(argv[optind], &ipt, fnw);
+	if (idx_rdr == 0) {
 		fprintf(stderr, "[ERROR] failed to open file '%s'\n", argv[optind]);
 		return 1;
 	}
-	if (!is_idx && fnw == 0 && argc - optind < 2) {
+	if (!idx_rdr->is_idx && fnw == 0 && argc - optind < 2) {
 		fprintf(stderr, "[ERROR] missing input: please specify a query file to map or option -d to keep the index\n");
 		return 1;
 	}
-	if (is_idx) fpr = fopen(argv[optind], "rb");
-	else fp = mm_bseq_open(argv[optind]);
-	if (fnw) fpw = fopen(fnw, "wb");
-	if (opt.flag & MM_F_OUT_SAM)
-		mm_write_sam_hdr_no_SQ(rg, MM_VERSION, argc, argv);
-	for (;;) {
-		mm_idx_t *mi;
-		if (fpr) {
-			mi = mm_idx_load(fpr);
-			if (mi == 0) break;
-			if (idx_par_set && mm_verbose >= 2 && (mi->k != k || mi->w != w || mi->is_hpc != is_hpc))
-				fprintf(stderr, "[WARNING] \033[1;31mIndexing parameters on the command line (-k/-w/-H) overridden by parameters in the prebuilt index.\033[0m\n");
-		} else {
-			mi = mm_idx_gen(fp, w, k, bucket_bits, is_hpc, minibatch_size, n_threads, batch_size, keep_name);
+	if (opt.best_n == 0 && (opt.flag&MM_F_CIGAR) && mm_verbose >= 2)
+		fprintf(stderr, "[WARNING]\033[1;31m `-N 0' reduces alignment accuracy. Please use --secondary=no to suppress secondary alignments.\033[0m\n");
+	while ((mi = mm_idx_reader_read(idx_rdr, n_threads)) != 0) {
+		if ((opt.flag & MM_F_OUT_SAM) && idx_rdr->n_parts == 1) {
+			if (mm_idx_reader_eof(idx_rdr)) {
+				mm_write_sam_hdr(mi, rg, MM_VERSION, argc, argv);
+			} else {
+				mm_write_sam_hdr(0, rg, MM_VERSION, argc, argv);
+				if (mm_verbose >= 2)
+					fprintf(stderr, "[WARNING]\033[1;31m For a multi-part index, no @SQ lines will be outputted.\033[0m\n");
+			}
 		}
-		if (mi == 0) break;
-		++n_idx_part;
-		if (mm_verbose >= 2 && n_idx_part > 1 && (opt.flag&MM_F_OUT_SAM) && !(opt.flag&MM_F_NO_SAM_SQ))
-			fprintf(stderr, "[WARNING] \033[1;31mSAM output is malformated due to internal @SQ lines. Please add option --no-sam-sq or filter afterwards.\033[0m\n");
 		if (mm_verbose >= 3)
 			fprintf(stderr, "[M::%s::%.3f*%.2f] loaded/built the index for %d target sequence(s)\n",
 					__func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), mi->n_seq);
-		if (fpw) {
-			mm_idx_dump(fpw, mi);
-			if (mm_verbose >= 3)
-				fprintf(stderr, "[M::%s::%.3f*%.2f] dumpped the (partial) index to disk\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
-		}
 		if (argc != optind + 1) mm_mapopt_update(&opt, mi);
 		if (mm_verbose >= 3) mm_idx_stat(mi);
-		for (i = optind + 1; i < argc; ++i)
-			mm_map_file(mi, argv[i], &opt, n_threads, minibatch_size);
+		if (!(opt.flag & MM_F_FRAG_MODE)) {
+			for (i = optind + 1; i < argc; ++i)
+				mm_map_file(mi, argv[i], &opt, n_threads);
+		} else {
+			mm_map_file_frag(mi, argc - (optind + 1), (const char**)&argv[optind + 1], &opt, n_threads);
+		}
 		mm_idx_destroy(mi);
 	}
-	if (fpw) fclose(fpw);
-	if (fpr) fclose(fpr);
-	if (fp)  mm_bseq_close(fp);
+	mm_idx_reader_close(idx_rdr);
 
 	fprintf(stderr, "[M::%s] Version: %s\n", __func__, MM_VERSION);
 	fprintf(stderr, "[M::%s] CMD:", __func__);

map.c

@@ -7,13 +7,14 @@
 #include "sdust.h"
 #include "mmpriv.h"
 #include "bseq.h"
+#include "khash.h"
 
 void mm_mapopt_init(mm_mapopt_t *opt)
 {
 	memset(opt, 0, sizeof(mm_mapopt_t));
-	opt->max_occ_frac = 1e-5f;
+	opt->seed = 11;
 	opt->mid_occ_frac = 2e-4f;
-	opt->sdust_thres = 0;
+	opt->sdust_thres = 0; // no SDUST masking
 
 	opt->min_cnt = 3;
 	opt->min_chain_score = 40;
@@ -32,28 +33,92 @@ void mm_mapopt_init(mm_mapopt_t *opt)
 
 	opt->a = 2, opt->b = 4, opt->q = 4, opt->e = 2, opt->q2 = 24, opt->e2 = 1;
 	opt->zdrop = 400;
+	opt->end_bonus = -1;
 	opt->min_dp_max = opt->min_chain_score * opt->a;
 	opt->min_ksw_len = 200;
+	opt->mini_batch_size = 500000000;
+
+	opt->pe_ori = 0; // FF
+	opt->pe_bonus = 33;
 }
 
 void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi)
 {
-	if (opt->flag & MM_F_SPLICE_BOTH)
-		opt->flag &= ~(MM_F_SPLICE_FOR|MM_F_SPLICE_REV);
-	opt->max_occ = mm_idx_cal_max_occ(mi, opt->max_occ_frac);
-	opt->mid_occ = mm_idx_cal_max_occ(mi, opt->mid_occ_frac);
+	if ((opt->flag & MM_F_SPLICE_FOR) && (opt->flag & MM_F_SPLICE_REV))
+		opt->flag |= MM_F_SPLICE;
+	if (opt->mid_occ <= 0)
+		opt->mid_occ = mm_idx_cal_max_occ(mi, opt->mid_occ_frac);
 	if (mm_verbose >= 3)
-		fprintf(stderr, "[M::%s::%.3f*%.2f] mid_occ = %d; max_occ = %d\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0),
-				opt->mid_occ, opt->max_occ);
+		fprintf(stderr, "[M::%s::%.3f*%.2f] mid_occ = %d\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), opt->mid_occ);
+}
+
+void mm_mapopt_max_intron_len(mm_mapopt_t *opt, int max_intron_len)
+{
+	if ((opt->flag & MM_F_SPLICE) && max_intron_len > 0)
+		opt->max_gap_ref = opt->bw = max_intron_len;
+}
+
+int mm_set_opt(const char *preset, mm_idxopt_t *io, mm_mapopt_t *mo)
+{
+	if (preset == 0) {
+		mm_idxopt_init(io);
+		mm_mapopt_init(mo);
+	} else if (strcmp(preset, "ava-ont") == 0) {
+		io->is_hpc = 0, io->k = 15, io->w = 5;
+		mo->flag |= MM_F_AVA | MM_F_NO_SELF;
+		mo->min_chain_score = 100, mo->pri_ratio = 0.0f, mo->max_gap = 10000, mo->max_chain_skip = 25;
+	} else if (strcmp(preset, "ava-pb") == 0) {
+		io->is_hpc = 1, io->k = 19, io->w = 5;
+		mo->flag |= MM_F_AVA | MM_F_NO_SELF;
+		mo->min_chain_score = 100, mo->pri_ratio = 0.0f, mo->max_gap = 10000, mo->max_chain_skip = 25;
+	} else if (strcmp(preset, "map10k") == 0 || strcmp(preset, "map-pb") == 0) {
+		io->is_hpc = 1, io->k = 19;
+	} else if (strcmp(preset, "map-ont") == 0) {
+		io->is_hpc = 0, io->k = 15;
+	} else if (strcmp(preset, "asm5") == 0) {
+		io->is_hpc = 0, io->k = 19, io->w = 19;
+		mo->a = 1, mo->b = 19, mo->q = 39, mo->q2 = 81, mo->e = 3, mo->e2 = 1, mo->zdrop = 200;
+		mo->min_dp_max = 200;
+		mo->best_n = 50;
+	} else if (strcmp(preset, "asm10") == 0) {
+		io->is_hpc = 0, io->k = 19, io->w = 19;
+		mo->a = 1, mo->b = 9, mo->q = 16, mo->q2 = 41, mo->e = 2, mo->e2 = 1, mo->zdrop = 200;
+		mo->min_dp_max = 200;
+		mo->best_n = 50;
+	} else if (strcmp(preset, "short") == 0 || strcmp(preset, "sr") == 0) {
+		io->is_hpc = 0, io->k = 21, io->w = 11;
+		mo->flag |= MM_F_SR | MM_F_FRAG_MODE | MM_F_NO_PRINT_2ND | MM_F_2_IO_THREADS;
+		mo->pe_ori = 0<<1|1; // FR
+		mo->a = 2, mo->b = 8, mo->q = 12, mo->e = 2, mo->q2 = 24, mo->e2 = 1;
+		mo->zdrop = 100;
+		mo->end_bonus = 10;
+		mo->max_frag_len = 800;
+		mo->max_gap = 100;
+		mo->bw = 100;
+		mo->pri_ratio = 0.5f;
+		mo->min_cnt = 2;
+		mo->min_chain_score = 25;
+		mo->min_dp_max = 40;
+		mo->best_n = 20;
+		mo->mid_occ = 1000;
+		mo->max_occ = 5000;
+		mo->mini_batch_size = 50000000;
+	} else if (strcmp(preset, "splice") == 0 || strcmp(preset, "cdna") == 0) {
+		io->is_hpc = 0, io->k = 15, io->w = 5;
+		mo->flag |= MM_F_SPLICE | MM_F_SPLICE_FOR | MM_F_SPLICE_REV;
+		mo->max_gap = 2000, mo->max_gap_ref = mo->bw = 200000;
+		mo->a = 1, mo->b = 2, mo->q = 2, mo->e = 1, mo->q2 = 32, mo->e2 = 0;
+		mo->noncan = 5;
+		mo->zdrop = 200;
+	} else return -1;
+	return 0;
 }
 
 typedef struct {
-	uint32_t n:31, is_alloc:1;
+	uint32_t n;
 	uint32_t qpos;
-	union {
-		const uint64_t *cr;
-		uint64_t *r;
-	} x;
+	uint32_t seg_id;
+	const uint64_t *cr;
 } mm_match_t;
 
 struct mm_tbuf_s {
@@ -80,14 +145,14 @@ void mm_tbuf_destroy(mm_tbuf_t *b)
 	free(b);
 }
 
-static void mm_dust_minier(mm128_v *mini, int l_seq, const char *seq, int sdust_thres, sdust_buf_t *sdb)
+static int mm_dust_minier(int n, mm128_t *a, int l_seq, const char *seq, int sdust_thres, sdust_buf_t *sdb)
 {
 	int n_dreg, j, k, u = 0;
 	const uint64_t *dreg;
-	if (sdust_thres <= 0 || sdb == 0) return;
+	if (sdust_thres <= 0 || sdb == 0) return n;
 	dreg = sdust_core((const uint8_t*)seq, l_seq, sdust_thres, 64, &n_dreg, sdb);
-	for (j = k = 0; j < mini->n; ++j) { // squeeze out minimizers that significantly overlap with LCRs
-		int32_t qpos = (uint32_t)mini->a[j].y>>1, span = mini->a[j].x&0xff;
+	for (j = k = 0; j < n; ++j) { // squeeze out minimizers that significantly overlap with LCRs
+		int32_t qpos = (uint32_t)a[j].y>>1, span = a[j].x&0xff;
 		int32_t s = qpos - (span - 1), e = s + span;
 		while (u < n_dreg && (uint32_t)dreg[u] <= s) ++u;
 		if (u < n_dreg && dreg[u]>>32 < e) {
@@ -97,196 +162,216 @@ static void mm_dust_minier(mm128_v *mini, int l_seq, const char *seq, int sdust_
 				int ee = e < (uint32_t)dreg[v]? e : (uint32_t)dreg[v];
 				l += ee - ss;
 			}
-			if (l <= span>>1) mini->a[k++] = mini->a[j]; // keep the minimizer if less than half of it falls in masked region
+			if (l <= span>>1) a[k++] = a[j]; // keep the minimizer if less than half of it falls in masked region
 		}
 	}
-	mini->n = k;
-}
-#if 0
-int mm_pair_thin_core(mm_tbuf_t *b, uint64_t x, int radius, int rel, int st0, int n, const uint64_t *z, uint64_v *a)
-{
-	int i, st = st0, en = n, mid = en - 1;
-	while (st < en) {
-		uint64_t y;
-		mid = st + ((en - st) >> 1);
-		y = z[mid];
-		if (y < x && (x - y)>>1 > radius) st = mid + 1;
-		else if (y >= x && (y - x)>>1 > radius) en = mid;
-		else break;
-	}
-	if (st < en) {
-		for (en = mid + 1; en < n; ++en)
-			if (z[en] > x && (z[en] - x)>>1 > radius)
-				break;
-		for (st = mid - 1; st >= st0; --st)
-			if (z[st] < x && (x - z[st])>>1 > radius)
-				break;
-		++st;
-		for (i = st; i < en; ++i) {
-			uint64_t y = z[i];
-			if (((x ^ y) & 1) == rel) {
-//				printf("* %d,%d\n", (uint32_t)x>>1, (uint32_t)y>>1);
-				kv_push(uint64_t, b->km, *a, y);
-			}
-		}
-		return en;
-	} else return st < n && z[st] < x? st + 1 : en;
+	return k; // the new size
 }
 
-void mm_pair_thin(mm_tbuf_t *b, int radius, mm_match_t *m1, mm_match_t *m2)
+static void collect_minimizers(const mm_mapopt_t *opt, const mm_idx_t *mi, int n_segs, const int *qlens, const char **seqs, mm_tbuf_t *b)
 {
-	mm_match_t *m[2];
-	const uint64_t *z[2];
-	uint64_v a[2];
-	int i, n[2], k[2], u = 0, rel = (m1->qpos ^ m2->qpos) & 1;
-
-	m[0] = m1, m[1] = m2;
-	for (i = 0; i < 2; ++i) {
-		n[i] = m[i]->n;
-		z[i] = m[i]->x.cr;
-		k[i] = 0;
-		kv_init(a[i]);
-		kv_resize(uint64_t, b->km, a[i], 256);
+	int i, j, n, sum = 0;
+	b->mini.n = 0;
+	for (i = n = 0; i < n_segs; ++i) {
+		mm_sketch(b->km, seqs[i], qlens[i], mi->w, mi->k, i, mi->is_hpc, &b->mini);
+		for (j = n; j < b->mini.n; ++j)
+			b->mini.a[j].y += sum << 1;
+		if (opt->sdust_thres > 0) // mask low-complexity minimizers
+			b->mini.n = n + mm_dust_minier(b->mini.n - n, b->mini.a + n, qlens[i], seqs[i], opt->sdust_thres, b->sdb);
+		sum += qlens[i], n = b->mini.n;
 	}
-	while (k[0] < n[0] && k[1] < n[1]) {
-		//printf("%d; %d,%d\n", u, k[0], k[1]);
-		int v = u^1, dist = (int)(m[v]->qpos>>1) - (int)(m[u]->qpos>>1);
-		uint64_t x = z[u][k[u]];
-		int uori = (x ^ m[u]->qpos) & 1, last;
-		int64_t tpos = x>>1 & 0x7fffffff;
-		tpos = uori == 0? tpos + dist : tpos - dist;
-		if (tpos < 0) tpos = 0;
-		x = x>>32<<32 | tpos<<1 | (x&1);
-		last = a[v].n;
-		k[v] = mm_pair_thin_core(b, x, radius, rel, k[v], n[v], z[v], &a[v]);
-		if (a[v].n > last) kv_push(uint64_t, b->km, a[u], z[u][k[u]]);
-		++k[u];
-		u ^= 1;
-	}
-	for (i = 0; i < 2; ++i)
-		m[i]->n = a[i].n, m[i]->x.r = a[i].a, m[i]->is_alloc = 1;
-//	printf("%d,%d; %d,%d\n", m[0]->qpos>>1, m[1]->qpos>>1, m[0]->n, m[1]->n);
 }
-#endif
-mm_reg1_t *mm_map_frag(const mm_mapopt_t *opt, const mm_idx_t *mi, mm_tbuf_t *b, uint32_t m_st, uint32_t m_en, const char *qname, int qlen, const char *seq, int *n_regs)
+
+static mm128_t *collect_seed_hits(const mm_mapopt_t *opt, int max_occ, const mm_idx_t *mi, const char *qname, int qlen, int64_t *n_a, int *rep_len, mm_tbuf_t *b)
 {
-	int i, n = m_en - m_st, j, n_u, max_gap_ref;
-	int64_t n_a;
-	uint64_t *u;
+	int rep_st = 0, rep_en = 0, i;
 	mm_match_t *m;
 	mm128_t *a;
-	mm_reg1_t *regs;
 
-	// convert to local representation
-	m = (mm_match_t*)kmalloc(b->km, n * sizeof(mm_match_t));
-	for (i = 0; i < n; ++i) {
+	m = (mm_match_t*)kmalloc(b->km, b->mini.n * sizeof(mm_match_t));
+	for (i = 0; i < b->mini.n; ++i) {
 		int t;
-		mm128_t *p = &b->mini.a[i + m_st];
-		m[i].is_alloc = 0;
+		mm128_t *p = &b->mini.a[i];
 		m[i].qpos = (uint32_t)p->y;
-		m[i].x.cr = mm_idx_get(mi, p->x>>8, &t);
+		m[i].cr = mm_idx_get(mi, p->x>>8, &t);
 		m[i].n = t;
+		m[i].seg_id = p->y >> 32;
 	}
-#if 0
-	int last = -1, last2 = -1;
-	// pair k-mer thinning
-	for (i = 0; i < n; ++i) {
-		if (m[i].n >= opt->mid_occ && m[i].n < opt->max_occ) {
-			if (last2 < 0) last2 = i;
-			if (last < 0 || m[last].n < m[i].n) last = i;
-			if (last >= 0 && (m[last].qpos>>1) + (m[last].span>>1) <= m[i].qpos>>1) {
-				mm_pair_thin(b, opt->bw, &m[last], &m[i]);
-				last2 = last = -1;
-			} else if (last2 >= 0 && (m[last2].qpos>>1) + (m[last2].span>>1) <= m[i].qpos>>1) {
-				mm_pair_thin(b, opt->bw, &m[last2], &m[i]);
-				last2 = last = -1;
-			}
-		}
-	}
-#endif
-	// fill the _a_ array
-	for (i = 0, n_a = 0; i < n; ++i) // find the length of a[]
-		if (m[i].n < opt->mid_occ) n_a += m[i].n;
-	a = (mm128_t*)kmalloc(b->km, n_a * sizeof(mm128_t));
-	for (i = j = 0; i < n; ++i) {
-		mm128_t *p = &b->mini.a[i + m_st];
+	for (i = 0, *n_a = 0; i < b->mini.n; ++i) // find the length of a[]
+		if (m[i].n < max_occ) *n_a += m[i].n;
+	a = (mm128_t*)kmalloc(b->km, *n_a * sizeof(mm128_t));
+	for (i = *rep_len = 0, *n_a = 0; i < b->mini.n; ++i) {
+		mm128_t *p = &b->mini.a[i];
 		mm_match_t *q = &m[i];
-		const uint64_t *r = q->x.cr;
+		const uint64_t *r = q->cr;
 		int k, q_span = p->x & 0xff, is_tandem = 0;
-		if (q->n >= opt->mid_occ) continue;
-		if (i > 0 && p->x>>8 == b->mini.a[m_st + i - 1].x>>8) is_tandem = 1;
-		if (i < n - 1 && p->x>>8 == b->mini.a[m_st + i + 1].x>>8) is_tandem = 1;
+		if (q->n >= max_occ) {
+			int en = (q->qpos>>1) + 1, st = en - q_span;
+			if (st > rep_en) {
+				*rep_len += rep_en - rep_st;
+				rep_st = st, rep_en = en;
+			} else rep_en = en;
+			continue;
+		}
+		if (i > 0 && p->x>>8 == b->mini.a[i - 1].x>>8) is_tandem = 1;
+		if (i < b->mini.n - 1 && p->x>>8 == b->mini.a[i + 1].x>>8) is_tandem = 1;
 		for (k = 0; k < q->n; ++k) {
-			const char *tname = mi->seq[r[k]>>32].name;
 			int32_t rpos = (uint32_t)r[k] >> 1;
 			mm128_t *p;
-			if (qname && (opt->flag&MM_F_NO_SELF) && strcmp(qname, tname) == 0 && rpos == (q->qpos>>1)) // avoid the diagonal
-				continue;
-			if (qname && (opt->flag&MM_F_AVA) && strcmp(qname, tname) > 0) // all-vs-all mode: map once
-				continue;
-			p = &a[j++];
+			if (qname && (opt->flag&(MM_F_NO_SELF|MM_F_AVA))) {
+				const char *tname = mi->seq[r[k]>>32].name;
+				int cmp;
+				cmp = strcmp(qname, tname);
+				if ((opt->flag&MM_F_NO_SELF) && cmp == 0 && rpos == (q->qpos>>1)) // avoid the diagonal
+					continue;
+				if ((opt->flag&MM_F_AVA) && cmp > 0) // all-vs-all mode: map once
+					continue;
+			}
+			p = &a[(*n_a)++];
 			if ((r[k]&1) == (q->qpos&1)) { // forward strand
-				p->x = (r[k]&0xffffffff00000000ULL) | (uint32_t)r[k]>>1;
+				p->x = (r[k]&0xffffffff00000000ULL) | rpos;
 				p->y = (uint64_t)q_span << 32 | q->qpos >> 1;
 			} else { // reverse strand
-				p->x = 1ULL<<63 | (r[k]&0xffffffff00000000ULL) | (uint32_t)r[k]>>1;
+				p->x = 1ULL<<63 | (r[k]&0xffffffff00000000ULL) | rpos;
 				p->y = (uint64_t)q_span << 32 | (qlen - ((q->qpos>>1) + 1 - q_span) - 1);
 			}
+			p->y |= (uint64_t)q->seg_id << MM_SEED_SEG_SHIFT;
 			if (is_tandem) p->y |= MM_SEED_TANDEM;
 		}
 	}
-	n_a = j;
-	radix_sort_128x(a, a + n_a);
-	for (i = 0; i < n; ++i)
-		if (m[i].is_alloc) kfree(b->km, m[i].x.r);
+	*rep_len += rep_en - rep_st;
 	kfree(b->km, m);
+	return a;
+}
 
-	if (mm_dbg_flag & MM_DBG_PRINT_SEED)
-		for (i = 0; i < n_a; ++i)
-			fprintf(stderr, "SD\t%s\t%d\t%c\t%d\t%d\t%d\n", mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
-					i == 0? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
-
-	max_gap_ref = opt->max_gap_ref >= 0? opt->max_gap_ref : opt->max_gap;
-	n_u = mm_chain_dp(max_gap_ref, opt->max_gap, opt->bw, opt->max_chain_skip, opt->min_cnt, opt->min_chain_score, !!(opt->flag&MM_F_SPLICE), n_a, a, &u, b->km);
-	regs = mm_gen_regs(b->km, qlen, n_u, u, a);
-	*n_regs = n_u;
-
-	if (mm_dbg_flag & MM_DBG_PRINT_SEED)
-		for (j = 0; j < n_u; ++j)
-			for (i = regs[j].as; i < regs[j].as + regs[j].cnt; ++i)
-				fprintf(stderr, "CN\t%d\t%s\t%d\t%c\t%d\t%d\t%d\n", j, mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
-						i == regs[j].as? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
-
+static void chain_post(const mm_mapopt_t *opt, int max_chain_gap_ref, const mm_idx_t *mi, void *km, int qlen, int n_segs, const int *qlens, int *n_regs, mm_reg1_t *regs, mm128_t *a)
+{
 	if (!(opt->flag & MM_F_AVA)) { // don't choose primary mapping(s) for read overlap
-		mm_set_parent(b->km, opt->mask_level, *n_regs, regs);
-		mm_select_sub(b->km, opt->mask_level, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
-		if (!(opt->flag & MM_F_SPLICE))
-			mm_join_long(b->km, opt, qlen, n_regs, regs, a); // TODO: this can be applied to all-vs-all in principle
+		mm_set_parent(km, opt->mask_level, *n_regs, regs, opt->a * 2 + opt->b);
+		if (n_segs <= 1) mm_select_sub(km, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
+		else mm_select_sub_multi(km, opt->pri_ratio, 0.2f, 0.7f, max_chain_gap_ref, mi->k*2, opt->best_n, n_segs, qlens, n_regs, regs);
+		if (!(opt->flag & MM_F_SPLICE) && !(opt->flag & MM_F_SR) && !(opt->flag & MM_F_NO_LJOIN))
+			mm_join_long(km, opt, qlen, n_regs, regs, a);
 	}
-	if (opt->flag & MM_F_CIGAR) {
-		regs = mm_align_skeleton(b->km, opt, mi, qlen, seq, n_regs, regs, a); // this calls mm_filter_regs()
-		if (!(opt->flag & MM_F_AVA)) {
-			mm_set_parent(b->km, opt->mask_level, *n_regs, regs);
-			mm_select_sub(b->km, opt->mask_level, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
-			mm_set_sam_pri(*n_regs, regs);
-		}
-	}
-	mm_set_mapq(*n_regs, regs, opt->min_chain_score);
+}
 
-	// free
-	kfree(b->km, a);
-	kfree(b->km, u);
+static mm_reg1_t *align_regs(const mm_mapopt_t *opt, const mm_idx_t *mi, void *km, int qlen, const char *seq, const char *qual, int *n_regs, mm_reg1_t *regs, mm128_t *a)
+{
+	if (!(opt->flag & MM_F_CIGAR)) return regs;
+	regs = mm_align_skeleton(km, opt, mi, qlen, seq, qual, n_regs, regs, a); // this calls mm_filter_regs()
+	if (!(opt->flag & MM_F_AVA)) {
+		mm_set_parent(km, opt->mask_level, *n_regs, regs, opt->a * 2 + opt->b);
+		mm_select_sub(km, opt->pri_ratio, mi->k*2, opt->best_n, n_regs, regs);
+		mm_set_sam_pri(*n_regs, regs);
+	}
 	return regs;
 }
 
-mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
+void mm_map_frag(const mm_idx_t *mi, int n_segs, const int *qlens, const char **seqs, const char **quals, int *n_regs, mm_reg1_t **regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
+{
+	int i, j, rep_len, qlen_sum, n_regs0;
+	int max_chain_gap_qry, max_chain_gap_ref, is_splice = !!(opt->flag & MM_F_SPLICE);
+	uint32_t hash;
+	int64_t n_a;
+	uint64_t *u;
+	mm128_t *a;
+	mm_reg1_t *regs0;
+
+	for (i = 0, qlen_sum = 0; i < n_segs; ++i)
+		qlen_sum += qlens[i], n_regs[i] = 0, regs[i] = 0;
+
+	if (qlen_sum == 0 || n_segs <= 0 || n_segs > MM_MAX_SEG) return;
+
+	hash  = qname? __ac_X31_hash_string(qname) : 0;
+	hash ^= __ac_Wang_hash(qlen_sum) + __ac_Wang_hash(opt->seed);
+	hash  = __ac_Wang_hash(hash);
+
+	collect_minimizers(opt, mi, n_segs, qlens, seqs, b);
+	a = collect_seed_hits(opt, opt->mid_occ, mi, qname, qlen_sum, &n_a, &rep_len, b);
+	radix_sort_128x(a, a + n_a);
+
+	if (mm_dbg_flag & MM_DBG_PRINT_SEED) {
+		fprintf(stderr, "RS\t%d\n", rep_len);
+		for (i = 0; i < n_a; ++i)
+			fprintf(stderr, "SD\t%s\t%d\t%c\t%d\t%d\t%d\n", mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
+					i == 0? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
+	}
+
+	// set max chaining gap on the query and the reference sequence
+	if (opt->flag & MM_F_SR)
+		max_chain_gap_qry = qlen_sum > opt->max_gap? qlen_sum : opt->max_gap;
+	else max_chain_gap_qry = opt->max_gap;
+	if (opt->max_gap_ref > 0) {
+		max_chain_gap_ref = opt->max_gap_ref; // always honor mm_mapopt_t::max_gap_ref if set
+	} else if (opt->max_frag_len > 0) {
+		max_chain_gap_ref = opt->max_frag_len - qlen_sum;
+		if (max_chain_gap_ref < opt->max_gap) max_chain_gap_ref = opt->max_gap;
+	} else max_chain_gap_ref = opt->max_gap;
+
+	a = mm_chain_dp(max_chain_gap_ref, max_chain_gap_qry, opt->bw, opt->max_chain_skip, opt->min_cnt, opt->min_chain_score, is_splice, n_segs, n_a, a, &n_regs0, &u, b->km);
+
+	if (opt->max_occ > opt->mid_occ && rep_len > 0) {
+		int rechain = 0;
+		if (n_regs0 > 0) { // test if the best chain has all the segments
+			int n_chained_segs = 1, max = 0, max_i = -1, max_off = -1, off = 0;
+			for (i = 0; i < n_regs0; ++i) { // find the best chain
+				if (max < u[i]>>32) max = u[i]>>32, max_i = i, max_off = off;
+				off += (uint32_t)u[i];
+			}
+			for (i = 1; i < (uint32_t)u[max_i]; ++i) // count the number of segments in the best chain
+				if ((a[max_off+i].y&MM_SEED_SEG_MASK) != (a[max_off+i-1].y&MM_SEED_SEG_MASK))
+					++n_chained_segs;
+			if (n_chained_segs < n_segs)
+				rechain = 1;
+		} else rechain = 1;
+		if (rechain) { // redo chaining with a higher max_occ threshold
+			kfree(b->km, a);
+			kfree(b->km, u);
+			a = collect_seed_hits(opt, opt->max_occ, mi, qname, qlen_sum, &n_a, &rep_len, b);
+			radix_sort_128x(a, a + n_a);
+			a = mm_chain_dp(max_chain_gap_ref, max_chain_gap_qry, opt->bw, opt->max_chain_skip, opt->min_cnt, opt->min_chain_score, is_splice, n_segs, n_a, a, &n_regs0, &u, b->km);
+		}
+	}
+
+	regs0 = mm_gen_regs(b->km, hash, qlen_sum, n_regs0, u, a);
+
+	if (mm_dbg_flag & MM_DBG_PRINT_SEED)
+		for (j = 0; j < n_regs0; ++j)
+			for (i = regs0[j].as; i < regs0[j].as + regs0[j].cnt; ++i)
+				fprintf(stderr, "CN\t%d\t%s\t%d\t%c\t%d\t%d\t%d\n", j, mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
+						i == regs0[j].as? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
+
+	chain_post(opt, max_chain_gap_ref, mi, b->km, qlen_sum, n_segs, qlens, &n_regs0, regs0, a);
+
+	if (n_segs == 1) { // uni-segment
+		regs0 = align_regs(opt, mi, b->km, qlens[0], seqs[0], quals? quals[0] : 0, &n_regs0, regs0, a);
+		mm_set_mapq(n_regs0, regs0, opt->min_chain_score, opt->a, rep_len);
+		n_regs[0] = n_regs0, regs[0] = regs0;
+	} else { // multi-segment
+		mm_seg_t *seg;
+		seg = mm_seg_gen(b->km, hash, n_segs, qlens, n_regs0, regs0, n_regs, regs, a); // split fragment chain to separate segment chains
+		free(regs0);
+		for (i = 0; i < n_segs; ++i) {
+			mm_set_parent(b->km, opt->mask_level, n_regs[i], regs[i], opt->a * 2 + opt->b); // update mm_reg1_t::parent
+			regs[i] = align_regs(opt, mi, b->km, qlens[i], seqs[i], quals? quals[i] : 0, &n_regs[i], regs[i], seg[i].a);
+			mm_set_mapq(n_regs[i], regs[i], opt->min_chain_score, opt->a, rep_len);
+		}
+		mm_seg_free(b->km, n_segs, seg);
+		if (n_segs == 2 && opt->pe_ori >= 0 && (opt->flag&MM_F_CIGAR))
+			mm_pair(b->km, max_chain_gap_ref, opt->pe_bonus, opt->a * 2 + opt->b, opt->a, qlens, n_regs, regs); // pairing
+	}
+	if (opt->min_iden > 0.0f)
+		for (i = 0; i < n_segs; ++i)
+			mm_filter_by_identity(b->km, n_regs[i], regs[i], opt->min_iden, qlens[i], quals[i]);
+
+	kfree(b->km, a);
+	kfree(b->km, u);
+}
+
+mm_reg1_t *mm_map(const mm_idx_t *mi, int qlen, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
 {
 	mm_reg1_t *regs;
-	b->mini.n = 0;
-	mm_sketch(b->km, seq, l_seq, mi->w, mi->k, 0, mi->is_hpc, &b->mini);
-	if (opt->sdust_thres > 0)
-		mm_dust_minier(&b->mini, l_seq, seq, opt->sdust_thres, b->sdb);
-	regs = mm_map_frag(opt, mi, b, 0, b->mini.n, qname, l_seq, seq, n_regs);
+	mm_map_frag(mi, 1, &qlen, &seq, 0, n_regs, &regs, b, opt, qname);
 	return regs;
 }
 
@@ -295,39 +380,69 @@ mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, m
  **************************/
 
 typedef struct {
-	int mini_batch_size, n_processed, n_threads;
+	int mini_batch_size, n_processed, n_threads, n_fp;
 	const mm_mapopt_t *opt;
-	mm_bseq_file_t *fp;
+	mm_bseq_file_t **fp;
 	const mm_idx_t *mi;
 	kstring_t str;
 } pipeline_t;
 
 typedef struct {
 	const pipeline_t *p;
-    int n_seq;
+    int n_seq, n_frag;
 	mm_bseq1_t *seq;
-	int *n_reg;
+	int *n_reg, *seg_off, *n_seg;
 	mm_reg1_t **reg;
 	mm_tbuf_t **buf;
 } step_t;
 
 static void worker_for(void *_data, long i, int tid) // kt_for() callback
 {
-    step_t *step = (step_t*)_data;
+    step_t *s = (step_t*)_data;
+	int *qlens, j, off = s->seg_off[i], pe_ori = s->p->opt->pe_ori, is_sr = !!(s->p->opt->flag & MM_F_SR);
+	const char **qseqs, **quals = 0;
+	mm_tbuf_t *b = s->buf[tid];
 	if (mm_dbg_flag & MM_DBG_PRINT_QNAME)
-		fprintf(stderr, "QR\t%s\t%d\n", step->seq[i].name, tid);
-	step->reg[i] = mm_map(step->p->mi, step->seq[i].l_seq, step->seq[i].seq, &step->n_reg[i], step->buf[tid], step->p->opt, step->seq[i].name);
+		fprintf(stderr, "QR\t%s\t%d\n", s->seq[off].name, tid);
+	qlens = (int*)kmalloc(b->km, s->n_seg[i] * sizeof(int));
+	qseqs = (const char**)kmalloc(b->km, s->n_seg[i] * sizeof(const char**));
+	quals = (const char**)kmalloc(b->km, s->n_seg[i] * sizeof(const char**));
+	for (j = 0; j < s->n_seg[i]; ++j) {
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1))))
+			mm_revcomp_bseq(&s->seq[off + j]);
+		qlens[j] = s->seq[off + j].l_seq;
+		qseqs[j] = s->seq[off + j].seq;
+		quals[j] = is_sr? s->seq[off + j].qual : 0;
+	}
+	mm_map_frag(s->p->mi, s->n_seg[i], qlens, qseqs, quals, &s->n_reg[off], &s->reg[off], b, s->p->opt, s->seq[off].name);
+	for (j = 0; j < s->n_seg[i]; ++j) // flip the query strand and coordinate to the original read strand
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1)))) {
+			int k, t;
+			mm_revcomp_bseq(&s->seq[off + j]);
+			for (k = 0; k < s->n_reg[off + j]; ++k) {
+				mm_reg1_t *r = &s->reg[off + j][k];
+				t = r->qs;
+				r->qs = qlens[j] - r->qe;
+				r->qe = qlens[j] - t;
+				r->rev = !r->rev;
+			}
+		}
+	kfree(b->km, qlens);
+	kfree(b->km, qseqs);
+	kfree(b->km, quals);
 }
 
 static void *worker_pipeline(void *shared, int step, void *in)
 {
-	int i, j;
+	int i, j, k;
     pipeline_t *p = (pipeline_t*)shared;
     if (step == 0) { // step 0: read sequences
 		int with_qual = (!!(p->opt->flag & MM_F_OUT_SAM) && !(p->opt->flag & MM_F_NO_QUAL));
+		int frag_mode = (p->n_fp > 1 || !!(p->opt->flag & MM_F_FRAG_MODE));
         step_t *s;
         s = (step_t*)calloc(1, sizeof(step_t));
-		s->seq = mm_bseq_read(p->fp, p->mini_batch_size, with_qual, &s->n_seq);
+		if (p->n_fp > 1) s->seq = mm_bseq_read_frag(p->n_fp, p->fp, p->mini_batch_size, with_qual, &s->n_seq);
+		else s->seq = mm_bseq_read2(p->fp[0], p->mini_batch_size, with_qual, frag_mode, &s->n_seq);
 		if (s->seq) {
 			s->p = p;
 			for (i = 0; i < s->n_seq; ++i)
@@ -335,12 +450,20 @@ static void *worker_pipeline(void *shared, int step, void *in)
 			s->buf = (mm_tbuf_t**)calloc(p->n_threads, sizeof(mm_tbuf_t*));
 			for (i = 0; i < p->n_threads; ++i)
 				s->buf[i] = mm_tbuf_init();
-			s->n_reg = (int*)calloc(s->n_seq, sizeof(int));
+			s->n_reg = (int*)calloc(3 * s->n_seq, sizeof(int));
+			s->seg_off = s->n_reg + s->n_seq; // seg_off and n_seg are allocated together with n_reg
+			s->n_seg = s->seg_off + s->n_seq;
 			s->reg = (mm_reg1_t**)calloc(s->n_seq, sizeof(mm_reg1_t*));
+			for (i = 1, j = 0; i <= s->n_seq; ++i)
+				if (i == s->n_seq || !frag_mode || !mm_qname_same(s->seq[i-1].name, s->seq[i].name)) {
+					s->n_seg[s->n_frag] = i - j;
+					s->seg_off[s->n_frag++] = j;
+					j = i;
+				}
 			return s;
 		} else free(s);
     } else if (step == 1) { // step 1: map
-		kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_seq);
+		kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_frag);
 		return in;
     } else if (step == 2) { // step 2: output
 		void *km = 0;
@@ -349,26 +472,34 @@ static void *worker_pipeline(void *shared, int step, void *in)
 		for (i = 0; i < p->n_threads; ++i) mm_tbuf_destroy(s->buf[i]);
 		free(s->buf);
 		if ((p->opt->flag & MM_F_OUT_CS) && !(mm_dbg_flag & MM_DBG_NO_KALLOC)) km = km_init();
-		for (i = 0; i < s->n_seq; ++i) {
-			mm_bseq1_t *t = &s->seq[i];
-			for (j = 0; j < s->n_reg[i]; ++j) {
-				mm_reg1_t *r = &s->reg[i][j];
-				if (p->opt->flag & MM_F_OUT_SAM)
-					mm_write_sam(&p->str, mi, t, r, s->n_reg[i], s->reg[i]);
-				else
-					mm_write_paf(&p->str, mi, t, r, km, p->opt->flag);
-				puts(p->str.s);
+		for (k = 0; k < s->n_frag; ++k) {
+			int seg_st = s->seg_off[k], seg_en = s->seg_off[k] + s->n_seg[k];
+			for (i = seg_st; i < seg_en; ++i) {
+				mm_bseq1_t *t = &s->seq[i];
+				for (j = 0; j < s->n_reg[i]; ++j) {
+					mm_reg1_t *r = &s->reg[i][j];
+					assert(!r->sam_pri || r->id == r->parent);
+					if ((p->opt->flag & MM_F_NO_PRINT_2ND) && r->id != r->parent)
+						continue;
+					if (p->opt->flag & MM_F_OUT_SAM)
+						mm_write_sam2(&p->str, mi, t, i - seg_st, j, s->n_seg[k], &s->n_reg[seg_st], (const mm_reg1_t*const*)&s->reg[seg_st], km, p->opt->flag);
+					else
+						mm_write_paf(&p->str, mi, t, r, km, p->opt->flag);
+					puts(p->str.s);
+				}
+				if (s->n_reg[i] == 0 && (p->opt->flag & MM_F_OUT_SAM)) {
+					mm_write_sam2(&p->str, mi, t, i - seg_st, -1, s->n_seg[k], &s->n_reg[seg_st], (const mm_reg1_t*const*)&s->reg[seg_st], km, p->opt->flag);
+					puts(p->str.s);
+				}
 			}
-			if (s->n_reg[i] == 0 && (p->opt->flag & MM_F_OUT_SAM)) {
-				mm_write_sam(&p->str, 0, t, 0, 0, 0);
-				puts(p->str.s);
+			for (i = seg_st; i < seg_en; ++i) {
+				for (j = 0; j < s->n_reg[i]; ++j) free(s->reg[i][j].p);
+				free(s->reg[i]);
+				free(s->seq[i].seq); free(s->seq[i].name);
+				if (s->seq[i].qual) free(s->seq[i].qual);
 			}
-			for (j = 0; j < s->n_reg[i]; ++j) free(s->reg[i][j].p);
-			free(s->reg[i]);
-			free(s->seq[i].seq); free(s->seq[i].name);
-			if (s->seq[i].qual) free(s->seq[i].qual);
 		}
-		free(s->reg); free(s->n_reg); free(s->seq);
+		free(s->reg); free(s->n_reg); free(s->seq); // seg_off and n_seg were allocated with reg; no memory leak here
 		km_destroy(km);
 		if (mm_verbose >= 3)
 			fprintf(stderr, "[M::%s::%.3f*%.2f] mapped %d sequences\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), s->n_seq);
@@ -377,22 +508,38 @@ static void *worker_pipeline(void *shared, int step, void *in)
     return 0;
 }
 
-int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads, int mini_batch_size)
+int mm_map_file_frag(const mm_idx_t *idx, int n_segs, const char **fn, const mm_mapopt_t *opt, int n_threads)
 {
+	int i, j, pl_threads;
 	pipeline_t pl;
+	if (n_segs < 1) return -1;
 	memset(&pl, 0, sizeof(pipeline_t));
-	pl.fp = mm_bseq_open(fn);
-	if (pl.fp == 0) {
-		if (mm_verbose >= 1)
-			fprintf(stderr, "ERROR: failed to open file '%s'\n", fn);
-		return -1;
+	pl.n_fp = n_segs;
+	pl.fp = (mm_bseq_file_t**)calloc(n_segs, sizeof(mm_bseq_file_t*));
+	for (i = 0; i < n_segs; ++i) {
+		pl.fp[i] = mm_bseq_open(fn[i]);
+		if (pl.fp[i] == 0) {
+			if (mm_verbose >= 1)
+				fprintf(stderr, "ERROR: failed to open file '%s'\n", fn[i]);
+			for (j = 0; j < i; ++j)
+				mm_bseq_close(pl.fp[j]);
+			free(pl.fp);
+			return -1;
+		}
 	}
 	pl.opt = opt, pl.mi = idx;
-	pl.n_threads = n_threads, pl.mini_batch_size = mini_batch_size;
-	if ((opt->flag & MM_F_OUT_SAM) && !(opt->flag & MM_F_NO_SAM_SQ))
-		mm_write_sam_SQ(idx);
-	kt_pipeline(n_threads == 1? 1 : 2, worker_pipeline, &pl, 3);
+	pl.n_threads = n_threads > 1? n_threads : 1;
+	pl.mini_batch_size = opt->mini_batch_size;
+	pl_threads = n_threads == 1? 1 : (opt->flag&MM_F_2_IO_THREADS)? 3 : 2;
+	kt_pipeline(pl_threads, worker_pipeline, &pl, 3);
 	free(pl.str.s);
-	mm_bseq_close(pl.fp);
+	for (i = 0; i < n_segs; ++i)
+		mm_bseq_close(pl.fp[i]);
+	free(pl.fp);
 	return 0;
 }
+
+int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads)
+{
+	return mm_map_file_frag(idx, 1, &fn, opt, n_threads);
+}

minimap.h

@@ -5,8 +5,6 @@
 #include <stdio.h>
 #include <sys/types.h>
 
-#define MM_IDX_DEF_B     14
-
 #define MM_F_NO_SELF     0x001
 #define MM_F_AVA         0x002
 #define MM_F_CIGAR       0x004
@@ -14,33 +12,30 @@
 #define MM_F_NO_QUAL     0x010
 #define MM_F_OUT_CG      0x020
 #define MM_F_OUT_CS      0x040
-#define MM_F_SPLICE      0x080
-#define MM_F_SPLICE_FOR  0x100
-#define MM_F_SPLICE_REV  0x200
-#define MM_F_SPLICE_BOTH 0x400
-#define MM_F_NO_SAM_SQ   0x800
+#define MM_F_SPLICE      0x080 // splice mode
+#define MM_F_SPLICE_FOR  0x100 // match GT-AG
+#define MM_F_SPLICE_REV  0x200 // match CT-AC, the reverse complement of GT-AG
+#define MM_F_NO_LJOIN    0x400
+#define MM_F_OUT_CS_LONG 0x800
+#define MM_F_SR          0x1000
+#define MM_F_FRAG_MODE   0x2000
+#define MM_F_NO_PRINT_2ND  0x4000
+#define MM_F_2_IO_THREADS  0x8000
+#define MM_F_LONG_CIGAR    0x10000
 
 #define MM_IDX_MAGIC   "MMI\2"
 
+#define MM_MAX_SEG       255
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct {
- 	uint64_t x, y;
-} mm128_t;
-
+// emulate 128-bit integers and arrays
+typedef struct { uint64_t x, y; } mm128_t;
 typedef struct { size_t n, m; mm128_t *a; } mm128_v;
-typedef struct { size_t n, m; uint64_t *a; } uint64_v;
-typedef struct { size_t n, m; uint32_t *a; } uint32_v;
-
-typedef struct {
-	mm128_v a;   // (minimizer, position) array
-	int32_t n;   // size of the _p_ array
-	uint64_t *p; // position array for minimizers appearing >1 times
-	void *h;     // hash table indexing _p_ and minimizers appearing once
-} mm_idx_bucket_t;
 
+// minimap2 index
 typedef struct {
 	char *name;      // name of the db sequence
 	uint64_t offset; // offset in mm_idx_t::S
@@ -49,103 +44,232 @@ typedef struct {
 
 typedef struct {
 	int32_t b, w, k, is_hpc;
-	uint32_t n_seq;     // number of reference sequences
-	mm_idx_seq_t *seq;  // sequence name, length and offset
-	uint32_t *S;        // 4-bit packed sequence
-	mm_idx_bucket_t *B; // index
+	uint32_t n_seq;            // number of reference sequences
+	mm_idx_seq_t *seq;         // sequence name, length and offset
+	uint32_t *S;               // 4-bit packed sequence
+	struct mm_idx_bucket_s *B; // index (hidden)
 	void *km;
 } mm_idx_t;
 
+// minimap2 alignment
 typedef struct {
-	uint32_t capacity;
-	int32_t dp_score, dp_max, dp_max2;
-	uint32_t blen;
-	uint32_t n_diff;
-	uint32_t n_ambi:30, trans_strand:2;
-	uint32_t n_cigar;
+	uint32_t capacity;                  // the capacity of cigar[]
+	int32_t dp_score, dp_max, dp_max2;  // DP score; score of the max-scoring segment; score of the best alternate mappings
+	uint32_t n_ambi:30, trans_strand:2; // number of ambiguous bases; transcript strand: 0 for unknown, 1 for +, 2 for -
+	uint32_t n_cigar;                   // number of cigar operations in cigar[]
+	float n_diff2;
+	uint32_t blen2;
 	uint32_t cigar[];
 } mm_extra_t;
 
 typedef struct {
-	int32_t id;
-	uint32_t cnt:31, rev:1;
-	uint32_t rid:31, inv:1;
-	int32_t score;
-	int32_t qs, qe, rs, re;
-	int32_t parent, subsc;
-	int32_t as;
-	int32_t fuzzy_mlen, fuzzy_blen;
-	uint32_t mapq:8, split:2, sam_pri:1, n_sub:21; // TODO: n_sub is not used for now
+	int32_t id;                     // ID for internal uses (see also parent below)
+	uint32_t cnt:30, rev:1, seg_split:1; // number of minimizers; if on the reverse strand
+	uint32_t rid:31, inv:1;         // reference index; if this is an alignment from inversion rescue
+	int32_t score;                  // DP alignment score
+	int32_t qs, qe, rs, re;         // query start and end; reference start and end
+	int32_t parent, subsc;          // parent==id if primary; best alternate mapping score
+	int32_t as;                     // offset in the a[] array (for internal uses only)
+	int32_t mlen, blen;             // seeded exact match length; seeded alignment block length
+	uint32_t mapq:8, split:2, n_sub:22; // mapQ; split pattern; number of suboptimal mappings
+	uint32_t sam_pri:1, proper_frag:1, iden_flt:1, pe_thru:1, dummy:29;
+	uint32_t hash;
 	mm_extra_t *p;
 } mm_reg1_t;
 
+// indexing and mapping options
 typedef struct {
-	float max_occ_frac;
-	float mid_occ_frac;
-	int sdust_thres;  // score threshold for SDUST; 0 to disable
-	int flag;    // see MM_F_* macros
+	short k, w, is_hpc, bucket_bits;
+	int mini_batch_size;
+	uint64_t batch_size;
+} mm_idxopt_t;
 
-	int bw;  // bandwidth
+typedef struct {
+	int seed;
+	int sdust_thres; // score threshold for SDUST; 0 to disable
+	int flag;        // see MM_F_* macros
+
+	int bw;          // bandwidth
 	int max_gap, max_gap_ref; // break a chain if there are no minimizers in a max_gap window
+	int max_frag_len;
 	int max_chain_skip;
-	int min_cnt;
-	int min_chain_score;
+	int min_cnt;         // min number of minimizers on each chain
+	int min_chain_score; // min chaining score
 
 	float mask_level;
 	float pri_ratio;
-	int best_n;
+	int best_n;      // top best_n chains are subjected to DP alignment
+	float min_iden;
 
 	int max_join_long, max_join_short;
 	int min_join_flank_sc;
 
 	int a, b, q, e, q2, e2; // matching score, mismatch, gap-open and gap-ext penalties
-	int noncan;
-	int zdrop;
-	int min_dp_max;
+	int noncan;      // cost of non-canonical splicing sites
+	int zdrop;       // break alignment if alignment score drops too fast along the diagonal
+	int end_bonus;
+	int min_dp_max;  // drop an alignment if the score of the max scoring segment is below this threshold
 	int min_ksw_len;
 
-	int max_occ;
-	int mid_occ;
+	int pe_ori, pe_bonus;
+
+	float mid_occ_frac;  // only used by mm_mapopt_update(); see below
+	int32_t mid_occ;     // ignore seeds with occurrences above this threshold
+	int32_t max_occ;
+	int mini_batch_size; // size of a batch of query bases to process in parallel
 } mm_mapopt_t;
 
-extern int mm_verbose, mm_dbg_flag;
-extern double mm_realtime0;
+// index reader
+typedef struct {
+	int is_idx, n_parts;
+	int64_t idx_size;
+	mm_idxopt_t opt;
+	FILE *fp_out;
+	union {
+		struct mm_bseq_file_s *seq;
+		FILE *idx;
+	} fp;
+} mm_idx_reader_t;
 
-struct mm_tbuf_s;
+// memory buffer for thread-local storage during mapping
 typedef struct mm_tbuf_s mm_tbuf_t;
 
-struct mm_bseq_file_s;
+// global variables
+extern int mm_verbose, mm_dbg_flag; // verbose level: 0 for no info, 1 for error, 2 for warning, 3 for message (default); debugging flag
+extern double mm_realtime0; // wall-clock timer
 
-#define mm_seq4_set(s, i, c) ((s)[(i)>>3] |= (uint32_t)(c) << (((i)&7)<<2))
-#define mm_seq4_get(s, i)    ((s)[(i)>>3] >> (((i)&7)<<2) & 0xf)
+/**
+ * Set default or preset parameters
+ *
+ * @param preset     NULL to set all parameters as default; otherwise apply preset to affected parameters
+ * @param io         pointer to indexing parameters
+ * @param mo         pointer to mapping parameters
+ *
+ * @return 0 if success; -1 if _present_ unknown
+ */
+int mm_set_opt(const char *preset, mm_idxopt_t *io, mm_mapopt_t *mo);
 
-// compute minimizers
-void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p);
-
-// minimizer indexing
-mm_idx_t *mm_idx_init(int w, int k, int b, int is_hpc);
-void mm_idx_destroy(mm_idx_t *mi);
-mm_idx_t *mm_idx_gen(struct mm_bseq_file_s *fp, int w, int k, int b, int is_hpc, int mini_batch_size, int n_threads, uint64_t batch_size, int keep_name);
-uint32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f);
-void mm_idx_stat(const mm_idx_t *idx);
-const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n);
-int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq);
-
-mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads);
-int mm_idx_is_idx(const char *fn);
-
-// minimizer index I/O
-void mm_idx_dump(FILE *fp, const mm_idx_t *mi);
-mm_idx_t *mm_idx_load(FILE *fp);
-
-// mapping
-void mm_mapopt_init(mm_mapopt_t *opt);
+/**
+ * Update mm_mapopt_t::mid_occ via mm_mapopt_t::mid_occ_frac
+ *
+ * If mm_mapopt_t::mid_occ is 0, this function sets it to a number such that no
+ * more than mm_mapopt_t::mid_occ_frac of minimizers in the index have a higher
+ * occurrence.
+ *
+ * @param opt        mapping parameters
+ * @param mi         minimap2 index
+ */
 void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi);
+
+void mm_mapopt_max_intron_len(mm_mapopt_t *opt, int max_intron_len);
+
+/**
+ * Initialize an index reader
+ *
+ * @param fn         index or fasta/fastq file name (this function tests the file type)
+ * @param opt        indexing parameters
+ * @param fn_out     if not NULL, write built index to this file
+ *
+ * @return an index reader on success; NULL if fail to open _fn_
+ */
+mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out);
+
+/**
+ * Read/build an index
+ *
+ * If the input file is an index file, this function reads one part of the
+ * index and returns. If the input file is a sequence file (fasta or fastq),
+ * this function constructs the index for about mm_idxopt_t::batch_size bases.
+ * Importantly, for a huge collection of sequences, this function may only
+ * return an index for part of sequences. It needs to be repeatedly called
+ * to traverse the entire index/sequence file.
+ *
+ * @param r          index reader
+ * @param n_threads  number of threads for constructing index
+ *
+ * @return an index on success; NULL if reaching the end of the input file
+ */
+mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads);
+
+/**
+ * Destroy/deallocate an index reader
+ *
+ * @param r          index reader
+ */
+void mm_idx_reader_close(mm_idx_reader_t *r);
+
+int mm_idx_reader_eof(const mm_idx_reader_t *r);
+
+/**
+ * Print index statistics to stderr
+ *
+ * @param mi         minimap2 index
+ */
+void mm_idx_stat(const mm_idx_t *idx);
+
+/**
+ * Destroy/deallocate an index
+ *
+ * @param r          minimap2 index
+ */
+void mm_idx_destroy(mm_idx_t *mi);
+
+/**
+ * Initialize a thread-local buffer for mapping
+ *
+ * Each mapping thread requires a buffer specific to the thread (see mm_map()
+ * below). The primary purpose of this buffer is to reduce frequent heap
+ * allocations across threads. A buffer shall not be used by two or more
+ * threads.
+ *
+ * @return pointer to a thread-local buffer
+ */
 mm_tbuf_t *mm_tbuf_init(void);
+
+/**
+ * Destroy/deallocate a thread-local buffer for mapping
+ *
+ * @param b          the buffer
+ */
 void mm_tbuf_destroy(mm_tbuf_t *b);
+
+/**
+ * Align a query sequence against an index
+ *
+ * This function possibly finds multiple alignments of the query sequence.
+ * The returned array and the mm_reg1_t::p field of each element are allocated
+ * with malloc().
+ *
+ * @param mi         minimap2 index
+ * @param l_seq      length of the query sequence
+ * @param seq        the query sequence
+ * @param n_regs     number of hits (out)
+ * @param b          thread-local buffer; two mm_map() calls shall not use one buffer at the same time!
+ * @param opt        mapping parameters
+ * @param name       query name, used for all-vs-all overlapping and debugging
+ *
+ * @return an array of hits which need to be deallocated with free() together
+ *         with mm_reg1_t::p of each element. The size is written to _n_regs_.
+ */
 mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *name);
 
-int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads, int tbatch_size);
+/**
+ * Align a fasta/fastq file and print alignments to stdout
+ *
+ * @param idx        minimap2 index
+ * @param fn         fasta/fastq file name
+ * @param opt        mapping parameters
+ * @param n_threads  number of threads
+ *
+ * @return 0 on success; -1 if _fn_ can't be read
+ */
+int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads);
+
+int mm_map_file_frag(const mm_idx_t *idx, int n_segs, const char **fn, const mm_mapopt_t *opt, int n_threads);
+
+// deprecated APIs for backward compatibility
+void mm_mapopt_init(mm_mapopt_t *opt);
+mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads);
 
 #ifdef __cplusplus
 }

minimap2.1

@@ -1,4 +1,4 @@
-.TH minimap2 1 "6 September 2017" "minimap2-2.1.1-r341" "Bioinformatics tools"
+.TH minimap2 1 "22 October 2017" "minimap2-2.2-dirty (r531)" "Bioinformatics tools"
 .SH NAME
 .PP
 minimap2 - mapping and alignment between collections of DNA sequences
@@ -126,7 +126,7 @@ Stop chain enlongation if there are no minimizers in
 [10000].
 .TP
 .BI -r \ INT
-Bandwidth used in chaining and DP-based alignment [1000]. This option
+Bandwidth used in chaining and DP-based alignment [500]. This option
 approximately controls the maximum gap size.
 .TP
 .BI -n \ INT
@@ -148,7 +148,7 @@ diagonal minimizer hits will also be suppressed.
 .TP
 .BI -p \ FLOAT
 Minimal secondary-to-primary score ratio to output secondary mappings [0.8].
-Between two chains overlaping over half of the shorter chain (controled by
+Between two chains overlaping over half of the shorter chain (controlled by
 .BR --mask-level ),
 the chain with a lower score is secondary to the chain with a higher score.
 If the ratio of the scores is below
@@ -163,10 +163,16 @@ secondary alignments [5]. This option has no effect when
 is applied.
 .TP
 .BI -G \ NUM
-Maximal intron length in the splice mode [200k]. This option also changes the
-bandwidth to
+Maximum gap on the reference (effective with
+.BR -xsplice / --splice ).
+This option also changes the chaining and alignment band width to
 .IR NUM .
-Increasing this option slows down spliced alignment.
+Increasing this option slows down spliced alignment. [200k]
+.TP
+.BI -F \ NUM
+Maximum fragment length (aka insert size; effective with
+.BR -xsr / --frag)
+[800]
 .TP
 .BI --max-chain-skip \ INT
 A heuristics that stops chaining early [50]. Minimap2 uses dynamic programming
@@ -175,6 +181,23 @@ option makes minimap2 exits the inner loop if it repeatedly sees seeds already
 on chains. Set
 .I INT
 to a large number to switch off this heurstics.
+.TP
+.B --no-long-join
+Disable the long gap patching heuristic. When this option is applied, the
+maximum alignment gap is mostly controlled by
+.BR -r .
+.TP
+.B --splice
+Enable the splice alignment mode.
+.TP
+.B --sr
+Enable short-read alignment heuristics. In the short-read mode, minimap2
+applies a second round of chaining with a higher minimizer occurrence threshold
+if no good chain is found. In addition, minimap2 attempts to patch gaps between
+seeds with ungapped alignment.
+.TP
+.BR --frag [= no | yes ]
+Whether to enable the fragment mode [no]
 .SS Alignment options
 .TP 10
 .BI -A \ INT
@@ -194,6 +217,7 @@ Gap extension penalty [2,1]. A gap of length
 .I k
 costs
 .RI min{ O1 + k * E1 , O2 + k * E2 }.
+In the splice mode, the second gap penalties are not used.
 .TP
 .BI -z \ INT
 Break an alignment if the running score drops too quickly along the diagonal of
@@ -217,6 +241,9 @@ no attempt to match GT-AG [n]
 .TP
 .BI --cost-non-gt-ag \ INT
 Cost of non-canonical splicing sites [0].
+.TP
+.BI --end-bonus \ INT
+Score bonus when alignment extends to the end of the query sequence [10].
 .SS Input/output options
 .TP 10
 .B -a
@@ -226,14 +253,40 @@ by default.
 .B -Q
 Ignore base quality in the input file.
 .TP
+.B -L
+Write CIGAR with >65535 operators at the CG tag. Older tools are unable to
+convert alignments with >65535 CIGAR ops to BAM. This option makes minimap2 SAM
+compatible with older tools. Newer tools recognizes this tag and reconstruct
+the real CIGAR in memory.
+.TP
 .BI -R \ STR
 SAM read group line in a format like
-.B @RG\\\\tID:foo\\\\tSM:bar
+.RB @RG\\\\tID:foo\\\\tSM:bar
 [].
 .TP
 .B -c
 Generate CIGAR. In PAF, the CIGAR is written to the `cg' custom tag.
 .TP
+.BI --cs[= STR ]
+Output the
+.B cs
+tag.
+.I STR
+can be either
+.I short
+or
+.IR long .
+If no
+.I STR
+is given,
+.I short
+is assumed. [none]
+.TP
+.BI --seed \ INT
+Integer seed for randomizing equally best hits. Minimap2 hashes
+.I INT
+and read name when choosing between equally best hits. [11]
+.TP
 .BI -t \ INT
 Number of threads [3]. Minimap2 uses at most three threads when indexing target
 sequences, and uses up to
@@ -241,27 +294,26 @@ sequences, and uses up to
 threads when mapping (the extra thread is for I/O, which is frequently idle and
 takes little CPU time).
 .TP
+.B -2
+Use two I/O threads during mapping. By default, minimap2 uses one I/O thread.
+When I/O is slow (e.g. piping to gzip, or reading from a slow pipe), the I/O
+thread may become the bottleneck. Apply this option to use one thread for input
+and another thread for output, at the cost of increased peak RAM.
+.TP
 .BI -K \ NUM
-Number of bases loaded into memory to process in a mini-batch [200M].
+Number of bases loaded into memory to process in a mini-batch [500M].
 Similar to option
 .BR -I ,
 K/M/G/k/m/g suffix is accepted. A large
 .I NUM
 helps load balancing in the multi-threading mode, at the cost of increased
-memory. Preset
-.B ava-pb
-and
-.B ava-ont
-use
-.BR -K500m .
+memory.
+.TP
+.BR --secondary [= yes | no ]
+Whether to output secondary alignments [yes]
 .TP
 .B --version
 Print version number to stdout
-.TP
-.B --no-sam-hdr
-Don't output SAM header lines. Use this option if the index consists of
-multiple parts; otherwise the SAM output is malformated due to internal header
-lines.
 .SS Preset options
 .TP 10
 .BI -x \ STR
@@ -278,11 +330,6 @@ are:
 PacBio/Oxford Nanopore read to reference mapping
 .RB ( -Hk19 )
 .TP
-.B map10k
-The same as
-.B map-pb
-.RB ( -Hk19 )
-.TP
 .B map-ont
 Slightly more sensitive for Oxford Nanopore to reference mapping
 .RB ( -k15 ).
@@ -309,13 +356,13 @@ Up to 10% sequence divergence.
 .B ava-pb
 PacBio all-vs-all overlap mapping
 .RB ( -Hk19
-.B -w5 -Xp0 -m100 -K500m -g10000 --max-chain-skip
+.B -w5 -Xp0 -m100 -g10000 --max-chain-skip
 .BR 25 ).
 .TP
 .B ava-ont
 Oxford Nanopore all-vs-all overlap mapping
 .RB ( -k15
-.B -w5 -Xp0 -m100 -K500m -g10000 --max-chain-skip
+.B -w5 -Xp0 -m100 -g10000 --max-chain-skip
 .BR 25 ).
 Similarly, the major difference from
 .B ava-pb
@@ -333,6 +380,13 @@ CIGAR operator; 2) long insertions are disabled; 3) deletion and insertion gap
 costs are different during chaining; 4) the computation of the
 .RB ` ms '
 tag ignores introns to demote hits to pseudogenes.
+.TP
+.B sr
+Short single-end reads without splicing
+.RB ( -k21
+.B -w11 --sr --frag -A2 -B8 -O12,32 -E2,1 -r50 -p.5 -N20 -f1000,5000 -n2 -m20
+.B -s40 -g200 -2K50m
+.BR --secondary=no ).
 .RE
 .SS Miscellaneous options
 .TP 10
@@ -345,7 +399,7 @@ multi-threading mode.
 .B --print-qname
 Print query names to stderr, mostly to see which query is crashing minimap2.
 .TP
-.B --print-seed
+.B --print-seeds
 Print seed positions to stderr, for debugging only.
 .SH OUTPUT FORMAT
 .PP
@@ -392,7 +446,29 @@ NM	i	Total number of mismatches and gaps in the alignment
 AS	i	DP alignment score
 ms	i	DP score of the max scoring segment in the alignment
 nn	i	Number of ambiguous bases in the alignment
+ts	A	Transcript strand (splice mode only)
 cg	Z	CIGAR string (only in PAF)
+cs	Z	Difference string
+.TE
+
+.PP
+The
+.B cs
+tag encodes difference sequences in the short form or the entire query
+.I AND
+reference sequences in the long form. It consists of a series of operations:
+.TS
+center box;
+cb | cb |cb
+r | l | l .
+Op	Regex	Description
+_
+ =	[ACGTN]+	Identical sequence (long form)
+ :	[0-9]+	Identical sequence length
+ *	[acgtn][acgtn]	Substitution: ref to query
+ +	[acgtn]+	Insertion to the reference
+ -	[acgtn]+	Deletion from the reference
+ ~	[acgtn]{2}[0-9]+[acgtn]{2}	Intron length and splice signal
 .TE
 
 .SH LIMITATIONS
@@ -403,9 +479,6 @@ where seed positions may be suboptimal. This should not be a big concern
 because even the optimal alignment may be wrong in such regions.
 .TP
 *
-Minimap2 does not work well with Illumina short reads as of now.
-.TP
-*
 Minimap2 requires SSE2 instructions to compile. It is possible to add
 non-SSE2 support, but it would make minimap2 slower by several times.
 .SH SEE ALSO

misc.c

@@ -1,6 +1,6 @@
 #include "minimap.h"
 
-int mm_verbose = 3;
+int mm_verbose = 1;
 int mm_dbg_flag = 0;
 double mm_realtime0;
 
@@ -90,7 +90,7 @@ double cputime()
 #include <sys/resource.h>
 #include <sys/time.h>
 
-double cputime()
+double cputime(void)
 {
 	struct rusage r;
 	getrusage(RUSAGE_SELF, &r);
@@ -98,7 +98,7 @@ double cputime()
 }
 #endif /* WIN32 || _WIN32 */
 
-double realtime()
+double realtime(void)
 {
 	struct timeval tp;
 	struct timezone tzp;

misc/sam2paf.js

@@ -103,7 +103,10 @@ while (file.readline(buf) >= 0) {
 	if (flag&16) qs = clip[1], qe = qlen - clip[0];
 	else qs = clip[0], qe = qlen - clip[1];
 	var ts = parseInt(t[3]) - 1, te = ts + M + D[1] + N;
-	var a = [t[0], qlen, qs, qe, flag&16? '-' : '+', t[2], tlen, ts, te, match, blen, t[4]];
+	var qname = t[0];
+	if ((flag&1) && (flag&0x40)) qname += '/1';
+	if ((flag&1) && (flag&0x80)) qname += '/2';
+	var a = [qname, qlen, qs, qe, flag&16? '-' : '+', t[2], tlen, ts, te, match, blen, t[4]];
 	print(a.join("\t"), extra.join("\t"));
 }
 

misc/sim-eval.js

@@ -142,7 +142,9 @@ while (file.readline(buf) >= 0) {
 		} else { // SAM
 			var flag = parseInt(t[1]);
 			var read_no = flag>>6&0x3;
-			var qname = read_no == 1 || read_no == 2? t[0] + '/' + read_no : t[0];
+			var qname = t[0];
+			if (!/\/[12]$/.test(qname))
+				qname = read_no == 1 || read_no == 2? t[0] + '/' + read_no : t[0];
 			if (last != qname) {
 				if (last != null) count_err(last, a, tot, err, mode);
 				a = [], last = qname;
@@ -181,11 +183,11 @@ var sum_tot = 0, sum_err = 0, q_out = -1, sum_tot2 = 0, sum_err2 = 0;
 for (var q = max_mapq; q >= 0; --q) {
 	if (tot[q] == 0) continue;
 	if (q_out < 0 || err[q] > 0) {
-		if (q_out >= 0) print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9));
+		if (q_out >= 0) print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9), sum_tot2);
 		sum_tot = sum_err = 0, q_out = q;
 	}
 	sum_tot += tot[q], sum_err += err[q];
 	sum_tot2 += tot[q], sum_err2 += err[q];
 }
-print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9));
+print('Q', q_out, sum_tot, sum_err, (sum_err2/sum_tot2).toFixed(9), sum_tot2);
 if (n_unmapped != null) print('U', n_unmapped);

mmpriv.h

@@ -17,10 +17,16 @@
 #define MM_SEED_IGNORE     (1ULL<<41)
 #define MM_SEED_TANDEM     (1ULL<<42)
 
+#define MM_SEED_SEG_SHIFT  48
+#define MM_SEED_SEG_MASK   (0xffULL<<(MM_SEED_SEG_SHIFT))
+
 #ifndef kroundup32
 #define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
 #endif
 
+#define mm_seq4_set(s, i, c) ((s)[(i)>>3] |= (uint32_t)(c) << (((i)&7)<<2))
+#define mm_seq4_get(s, i)    ((s)[(i)>>3] >> (((i)&7)<<2) & 0xf)
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -33,6 +39,12 @@ typedef struct __kstring_t {
 } kstring_t;
 #endif
 
+typedef struct {
+	int n_u, n_a;
+	uint64_t *u;
+	mm128_t *a;
+} mm_seg_t;
+
 double cputime(void);
 double realtime(void);
 
@@ -40,23 +52,37 @@ void radix_sort_128x(mm128_t *beg, mm128_t *end);
 void radix_sort_64(uint64_t *beg, uint64_t *end);
 uint32_t ks_ksmall_uint32_t(size_t n, uint32_t arr[], size_t kk);
 
-void mm_write_sam_SQ(const mm_idx_t *idx);
-void mm_write_sam_hdr_no_SQ(const char *rg, const char *ver, int argc, char *argv[]);
+void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p);
+
+void mm_write_sam_hdr(const mm_idx_t *mi, const char *rg, const char *ver, int argc, char *argv[]);
 void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag);
 void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs);
-int mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int64_t n, mm128_t *a, uint64_t **_u, void *km);
-mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, int *n_regs_, mm_reg1_t *regs, mm128_t *a);
+void mm_write_sam2(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regs, const mm_reg1_t *const* regs, void *km, int opt_flag);
 
-mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a);
+void mm_idxopt_init(mm_idxopt_t *opt);
+const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n);
+int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq);
+int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f);
+mm128_t *mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int min_cnt, int min_sc, int is_cdna, int n_segs, int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km);
+mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, const char *qual, int *n_regs_, mm_reg1_t *regs, mm128_t *a);
+
+mm_reg1_t *mm_gen_regs(void *km, uint32_t hash, int qlen, int n_u, uint64_t *u, mm128_t *a);
 void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a);
 void mm_sync_regs(void *km, int n_regs, mm_reg1_t *regs);
+int mm_squeeze_a(void *km, int n_regs, mm_reg1_t *regs, mm128_t *a);
 int mm_set_sam_pri(int n, mm_reg1_t *r);
-void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r);
-void mm_select_sub(void *km, float mask_level, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r);
+void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r, int sub_diff);
+void mm_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mm_reg1_t *r);
+void mm_select_sub_multi(void *km, float pri_ratio, float pri1, float pri2, int max_gap_ref, int min_diff, int best_n, int n_segs, const int *qlens, int *n_, mm_reg1_t *r);
 void mm_filter_regs(void *km, const mm_mapopt_t *opt, int *n_regs, mm_reg1_t *regs);
+void mm_filter_by_identity(void *km, int n_regs, mm_reg1_t *regs, float min_iden, int qlen, const char *qual);
 void mm_join_long(void *km, const mm_mapopt_t *opt, int qlen, int *n_regs, mm_reg1_t *regs, mm128_t *a);
 void mm_hit_sort_by_dp(void *km, int *n_regs, mm_reg1_t *r);
-void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc);
+void mm_set_mapq(int n_regs, mm_reg1_t *regs, int min_chain_sc, int match_sc, int rep_len);
+
+mm_seg_t *mm_seg_gen(void *km, uint32_t hash, int n_segs, const int *qlens, int n_regs0, const mm_reg1_t *regs0, int *n_regs, mm_reg1_t **regs, const mm128_t *a);
+void mm_seg_free(void *km, int n_segs, mm_seg_t *segs);
+void mm_pair(void *km, int max_gap_ref, int dp_bonus, int sub_diff, int match_sc, const int *qlens, int *n_regs, mm_reg1_t **regs);
 
 #ifdef __cplusplus
 }

pe.c

@@ -0,0 +1,177 @@
+#include <stdlib.h>
+#include <math.h>
+#include "mmpriv.h"
+#include "kvec.h"
+
+void mm_select_sub_multi(void *km, float pri_ratio, float pri1, float pri2, int max_gap_ref, int min_diff, int best_n, int n_segs, const int *qlens, int *n_, mm_reg1_t *r)
+{
+	if (pri_ratio > 0.0f && *n_ > 0) {
+		int i, k, n = *n_, n_2nd = 0;
+		int max_dist = n_segs == 2? qlens[0] + qlens[1] + max_gap_ref : 0;
+		for (i = k = 0; i < n; ++i) {
+			int to_keep = 0;
+			if (r[i].parent == i) { // primary
+				to_keep = 1;
+			} else if (r[i].score + min_diff >= r[r[i].parent].score) {
+				to_keep = 1;
+			} else {
+				mm_reg1_t *p = &r[r[i].parent], *q = &r[i];
+				if (p->rev == q->rev && p->rid == q->rid && q->re - p->rs < max_dist && p->re - q->rs < max_dist) { // child and parent are close on the ref
+					if (q->score >= p->score * pri1)
+						to_keep = 1;
+				} else {
+					int is_par_both = (n_segs == 2 && p->qs < qlens[0] && p->qe > qlens[0]);
+					int is_chi_both = (n_segs == 2 && q->qs < qlens[0] && q->qe > qlens[0]);
+					if (is_chi_both || is_chi_both == is_par_both) {
+						if (q->score >= p->score * pri_ratio)
+							to_keep = 1;
+					} else { // the remaining case: is_chi_both == 0 && is_par_both == 1
+						if (q->score >= p->score * pri2)
+							to_keep = 1;
+					}
+				}
+			}
+			if (to_keep && r[i].parent != i) {
+				if (n_2nd++ >= best_n) to_keep = 0; // don't keep if there are too many secondary hits
+			}
+			if (to_keep) r[k++] = r[i];
+			else if (r[i].p) free(r[i].p);
+		}
+		if (k != n) mm_sync_regs(km, k, r); // removing hits requires sync()
+		*n_ = k;
+	}
+}
+
+void mm_set_pe_thru(const int *qlens, int *n_regs, mm_reg1_t **regs)
+{
+	int s, i, n_pri[2], pri[2];
+	n_pri[0] = n_pri[1] = 0;
+	pri[0] = pri[1] = -1;
+	for (s = 0; s < 2; ++s)
+		for (i = 0; i < n_regs[s]; ++i)
+			if (regs[s][i].id == regs[s][i].parent)
+				++n_pri[s], pri[s] = i;
+	if (n_pri[0] == 1 && n_pri[1] == 1) {
+		mm_reg1_t *p = &regs[0][pri[0]];
+		mm_reg1_t *q = &regs[1][pri[1]];
+		if (p->rid == q->rid && p->rev == q->rev && abs(p->rs - q->rs) < 3 && abs(p->re - p->re) < 3
+			&& ((p->qs == 0 && qlens[1] - q->qe == 0) || (q->qs == 0 && qlens[0] - p->qe == 0)))
+		{
+			p->pe_thru = q->pe_thru = 1;
+		}
+	}
+}
+
+#include "ksort.h"
+
+typedef struct {
+	int s, rev;
+	uint64_t key;
+	mm_reg1_t *r;
+} pair_arr_t;
+
+#define sort_key_pair(a) ((a).key)
+KRADIX_SORT_INIT(pair, pair_arr_t, sort_key_pair, 8)
+
+void mm_pair(void *km, int max_gap_ref, int pe_bonus, int sub_diff, int match_sc, const int *qlens, int *n_regs, mm_reg1_t **regs)
+{
+	int i, j, s, n, last[2], dp_thres, segs = 0, max_idx[2];
+	int64_t max;
+	pair_arr_t *a;
+	kvec_t(uint64_t) sc = {0,0,0};
+
+	a = (pair_arr_t*)kmalloc(km, (n_regs[0] + n_regs[1]) * sizeof(pair_arr_t));
+	for (s = n = 0, dp_thres = 0; s < 2; ++s) {
+		int max = 0;
+		for (i = 0; i < n_regs[s]; ++i) {
+			a[n].s = s;
+			a[n].r = &regs[s][i];
+			a[n].rev = a[n].r->rev;
+			a[n].key = (uint64_t)a[n].r->rid << 32 | a[n].r->rs<<1 | (s^a[n].rev);
+			max = max > a[n].r->p->dp_max? max : a[n].r->p->dp_max;
+			++n;
+			segs |= 1<<s;
+		}
+		dp_thres += max;
+	}
+	if (segs != 3) {
+		kfree(km, a); // only one end is mapped
+		return;
+	}
+	dp_thres -= pe_bonus;
+	if (dp_thres < 0) dp_thres = 0;
+	radix_sort_pair(a, a + n);
+
+	max = -1;
+	max_idx[0] = max_idx[1] = -1;
+	last[0] = last[1] = -1;
+	kv_resize(uint64_t, km, sc, n);
+	for (i = 0; i < n; ++i) {
+		if (a[i].key & 1) { // reverse first read or forward second read
+			mm_reg1_t *q, *r;
+			if (last[a[i].rev] < 0) continue;
+			r = a[i].r;
+			q = a[last[a[i].rev]].r;
+			if (r->rid != q->rid || r->rs - q->re > max_gap_ref) continue;
+			for (j = last[a[i].rev]; j >= 0; --j) {
+				int64_t score;
+				if (a[j].rev != a[i].rev || a[j].s == a[i].s) continue;
+				q = a[j].r;
+				if (r->rid != q->rid || r->rs - q->re > max_gap_ref) break;
+				if (r->p->dp_max + q->p->dp_max < dp_thres) continue;
+				score = (int64_t)(r->p->dp_max + q->p->dp_max) << 32 | (r->hash + q->hash);
+				if (score > max)
+					max = score, max_idx[a[j].s] = j, max_idx[a[i].s] = i;
+				kv_push(uint64_t, km, sc, score);
+			}
+		} else { // forward first read or reverse second read
+			last[a[i].rev] = i;
+		}
+	}
+	if (sc.n > 1)
+		radix_sort_64(sc.a, sc.a + sc.n);
+
+	if (sc.n > 0 && max > 0) { // found at least one pair
+		int n_sub = 0, mapq_pe;
+		mm_reg1_t *r[2];
+		r[0] = a[max_idx[0]].r, r[1] = a[max_idx[1]].r;
+		r[0]->proper_frag = r[1]->proper_frag = 1;
+		for (s = 0; s < 2; ++s) {
+			if (r[s]->id != r[s]->parent) { // then lift to primary and update parent
+				mm_reg1_t *p = &regs[s][r[s]->parent];
+				for (i = 0; i < n_regs[s]; ++i)
+					if (regs[s][i].parent == p->id)
+						regs[s][i].parent = r[s]->id;
+				p->mapq = 0;
+			}
+			if (!r[s]->sam_pri) { // then sync sam_pri
+				for (i = 0; i < n_regs[s]; ++i)
+					regs[s][i].sam_pri = 0;
+				r[s]->sam_pri = 1;
+			}
+		}
+		mapq_pe = r[0]->mapq > r[1]->mapq? r[0]->mapq : r[1]->mapq;
+		for (i = 0; i < sc.n; ++i)
+			if ((sc.a[i]>>32) + sub_diff >= max>>32)
+				++n_sub;
+		if (sc.n > 1) {
+			int mapq_pe_alt;
+			mapq_pe_alt = (int)(6.02f * ((max>>32) - (sc.a[sc.n - 2]>>32)) / match_sc - 4.343f * logf(n_sub)); // n_sub > 0 because it counts the optimal, too
+			mapq_pe = mapq_pe < mapq_pe_alt? mapq_pe : mapq_pe_alt;
+		}
+		if (r[0]->mapq < mapq_pe) r[0]->mapq = (r[0]->mapq + mapq_pe) / 2;
+		if (r[1]->mapq < mapq_pe) r[1]->mapq = (r[1]->mapq + mapq_pe) / 2;
+		if (sc.n == 1) {
+			if (r[0]->mapq < 2) r[0]->mapq = 2;
+			if (r[1]->mapq < 2) r[1]->mapq = 2;
+		} else if (max>>32 > sc.a[sc.n - 2]>>32) {
+			if (r[0]->mapq < 1) r[0]->mapq = 1;
+			if (r[1]->mapq < 1) r[1]->mapq = 1;
+		}
+	}
+
+	kfree(km, a);
+	kfree(km, sc.a);
+
+	mm_set_pe_thru(qlens, n_regs, regs);
+}

python/README.rst

@@ -0,0 +1,145 @@
+==============================
+Mappy: Minimap2 Python Binding
+==============================
+
+Mappy provides a convenient interface to `minimap2
+<https://github.com/lh3/minimap2>`_, a fast and accurate C program to align
+genomic and transcribe nucleotide sequences.
+
+Installation
+------------
+
+Mappy depends on `zlib <http://zlib.net>`_. It can be installed with `pip
+<https://en.wikipedia.org/wiki/Pip_(package_manager)>`_:
+
+.. code:: shell
+
+	pip install --user mappy
+
+or from the minimap2 github repo (`Cython <http://cython.org>`_ required):
+
+.. code:: shell
+
+	git clone https://github.com/lh3/minimap2
+	cd minimap2
+	python setup.py install
+
+Usage
+-----
+
+The following Python script demonstrates the key functionality of mappy:
+
+.. code:: python
+
+	import mappy as mp
+	a = mp.Aligner("test/MT-human.fa")  # load or build index
+	if not a: raise Exception("ERROR: failed to load/build index")
+	for name, seq, qual in mp.fastx_read("test/MT-orang.fa"): # read a fasta/q sequence
+		for hit in a.map(seq): # traverse alignments
+			print("{}\t{}\t{}\t{}".format(hit.ctg, hit.r_st, hit.r_en, hit.cigar_str))
+
+APIs
+----
+
+Mappy implements two classes and one global function.
+
+Class mappy.Aligner
+~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+	mappy.Aligner(fn_idx_in, preset=None, ...)
+
+This constructor accepts the following arguments:
+
+* **fn_idx_in**: index or sequence file name. Minimap2 automatically tests the
+  file type. If a sequence file is provided, minimap2 builds an index. The
+  sequence file can be optionally gzip'd.
+
+* **preset**: minimap2 preset. Currently, minimap2 supports the following
+  presets: **sr** for single-end short reads; **map-pb** for PacBio
+  read-to-reference mapping; **map-ont** for Oxford Nanopore read mapping;
+  **splice** for long-read spliced alignment; **asm5** for assembly-to-assembly
+  alignment; **asm10** for full genome alignment of closely related species. Note
+  that the Python module does not support all-vs-all read overlapping.
+
+* **k**: k-mer length, no larger than 28
+
+* **w**: minimizer window size, no larger than 255
+
+* **min_cnt**: mininum number of minimizers on a chain
+
+* **min_chain_score**: minimum chaing score
+
+* **bw**: chaining and alignment band width
+
+* **best_n**: max number of alignments to return
+
+* **n_threads**: number of indexing threads; 3 by default
+
+* **fn_idx_out**: name of file to which the index is written
+
+.. code:: python
+
+	mappy.Aligner.map(seq)
+
+This method aligns :code:`seq` against the index. It is a generator, *yielding*
+a series of :code:`mappy.Alignment` objects.
+
+Class mappy.Alignment
+~~~~~~~~~~~~~~~~~~~~~
+
+This class describes an alignment. An object of this class has the following
+properties:
+
+* **ctg**: name of the reference sequence the query is mapped to
+
+* **ctg_len**: total length of the reference sequence
+
+* **r_st** and **r_en**: start and end positions on the reference
+
+* **q_st** and **q_en**: start and end positions on the query
+
+* **strand**: +1 if on the forward strand; -1 if on the reverse strand
+
+* **mapq**: mapping quality
+
+* **blen**: length of the alignment, including both alignment matches and gaps
+  but excluding ambiguous bases.
+
+* **mlen**: length of the matching bases in the alignment, excluding ambiguous
+  base matches.
+
+* **NM**: number of mismatches, gaps and ambiguous poistions in the alignment
+
+* **trans_strand**: transcript strand. +1 if on the forward strand; -1 if on the
+  reverse strand; 0 if unknown
+
+* **is_primary**: if the alignment is primary (typically the best and the first
+  to generate)
+
+* **cigar_str**: CIGAR string
+
+* **cigar**: CIGAR returned as an array of shape :code:`(n_cigar,2)`. The two
+  numbers give the length and the operator of each CIGAR operation.
+
+An :code:`Alignment` object can be converted to a string with :code:`str()` in
+the following format:
+
+::
+
+	q_st  q_en  strand  ctg  ctg_len  r_st  r_en  mlen  blen  mapq  cg:Z:cigar_str
+
+It is effectively the PAF format without the QueryName and QueryLength columns
+(the first two columns in PAF).
+
+Function mappy.fastx_read
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+	mappy.fastx_read(fn)
+
+This generator function opens a FASTA/FASTQ file and *yields* a
+:code:`(name,seq,qual)` tuple for each sequence entry. The input file may be
+optionally gzip'd.

python/cmappy.h

@@ -0,0 +1,71 @@
+#ifndef CMAPPY_H
+#define CMAPPY_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include "minimap.h"
+#include "kseq.h"
+KSEQ_DECLARE(gzFile)
+
+typedef struct {
+	const char *ctg;
+	int32_t ctg_start, ctg_end;
+	int32_t qry_start, qry_end;
+	int32_t blen, mlen, NM, ctg_len;
+	uint8_t mapq, is_primary;
+	int8_t strand, trans_strand;
+	int32_t n_cigar32;
+	uint32_t *cigar32;
+} mm_hitpy_t;
+
+static inline void mm_reg2hitpy(const mm_idx_t *mi, mm_reg1_t *r, mm_hitpy_t *h)
+{
+	h->ctg = mi->seq[r->rid].name;
+	h->ctg_len = mi->seq[r->rid].len;
+	h->ctg_start = r->rs, h->ctg_end = r->re;
+	h->qry_start = r->qs, h->qry_end = r->qe;
+	h->strand = r->rev? -1 : 1;
+	h->mapq = r->mapq;
+	h->mlen = r->mlen;
+	h->blen = r->blen;
+	h->NM = r->blen - r->mlen + r->p->n_ambi;
+	h->trans_strand = r->p->trans_strand == 1? 1 : r->p->trans_strand == 2? -1 : 0;
+	h->is_primary = (r->id == r->parent);
+	h->n_cigar32 = r->p->n_cigar;
+	h->cigar32 = r->p->cigar;
+}
+
+static inline void mm_free_reg1(mm_reg1_t *r)
+{
+	free(r->p);
+}
+
+static inline kseq_t *mm_fastx_open(const char *fn)
+{
+	gzFile fp;
+	fp = fn && strcmp(fn, "-") != 0? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+	return kseq_init(fp);
+}
+
+static inline void mm_fastx_close(kseq_t *ks)
+{
+	gzFile fp;
+	fp = ks->f->f;
+	kseq_destroy(ks);
+	gzclose(fp);
+}
+
+static inline int mm_verbose_level(int v)
+{
+	if (v >= 0) mm_verbose = v;
+	return mm_verbose;
+}
+
+static inline void mm_reset_timer(void)
+{
+	extern double realtime(void);
+	mm_realtime0 = realtime();
+}
+
+#endif

python/cmappy.pxd

@@ -0,0 +1,118 @@
+from libc.stdint cimport int8_t, uint8_t, int32_t, int64_t, uint32_t, uint64_t
+
+cdef extern from "minimap.h":
+	#
+	# Options
+	#
+	ctypedef struct mm_idxopt_t:
+		short k, w, is_hpc, bucket_bits
+		int mini_batch_size
+		uint64_t batch_size
+
+	ctypedef struct mm_mapopt_t:
+		int seed
+		int sdust_thres
+		int flag
+		int bw
+		int max_gap, max_gap_ref
+		int max_frag_len
+		int max_chain_skip
+		int min_cnt
+		int min_chain_score
+		float mask_level
+		float pri_ratio
+		int best_n
+		float min_iden
+		int max_join_long, max_join_short
+		int min_join_flank_sc
+		int a, b, q, e, q2, e2
+		int noncan
+		int zdrop
+		int end_bonus
+		int min_dp_max
+		int min_ksw_len
+		int pe_ori, pe_bonus
+		float mid_occ_frac
+		int32_t mid_occ
+		int32_t max_occ
+		int mini_batch_size
+
+	int mm_set_opt(char *preset, mm_idxopt_t *io, mm_mapopt_t *mo)
+	int mm_verbose
+
+	#
+	# Indexing
+	#
+	ctypedef struct mm_idx_seq_t:
+		char *name
+		uint64_t offset
+		uint32_t len
+
+	ctypedef struct mm_idx_bucket_t:
+		pass
+
+	ctypedef struct mm_idx_t:
+		int32_t b, w, k, is_hpc
+		uint32_t n_seq
+		mm_idx_seq_t *seq
+		uint32_t *S
+		mm_idx_bucket_t *B
+		void *km
+
+	ctypedef struct mm_idx_reader_t:
+		pass
+
+	mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out)
+	mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
+	void mm_idx_reader_close(mm_idx_reader_t *r)
+	void mm_idx_destroy(mm_idx_t *mi)
+	void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi)
+
+	#
+	# Mapping (key struct defined in cmappy.h below)
+	#
+	ctypedef struct mm_reg1_t:
+		pass
+
+	ctypedef struct mm_tbuf_t:
+		pass
+
+	mm_tbuf_t *mm_tbuf_init()
+	void mm_tbuf_destroy(mm_tbuf_t *b)
+	mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *name)
+
+#
+# Helper header (because it is hard to expose mm_reg1_t with Cython)
+#
+cdef extern from "cmappy.h":
+	ctypedef struct mm_hitpy_t:
+		const char *ctg
+		int32_t ctg_start, ctg_end
+		int32_t qry_start, qry_end
+		int32_t blen, mlen, NM, ctg_len
+		uint8_t mapq, is_primary
+		int8_t strand, trans_strand
+		int32_t n_cigar32
+		uint32_t *cigar32
+
+	void mm_reg2hitpy(const mm_idx_t *mi, mm_reg1_t *r, mm_hitpy_t *h)
+	void mm_free_reg1(mm_reg1_t *r)
+
+	ctypedef struct kstring_t:
+		unsigned l, m
+		char *s
+
+	ctypedef struct kstream_t:
+		pass
+
+	ctypedef struct kseq_t:
+		kstring_t name, comment, seq, qual
+		int last_char
+		kstream_t *f
+
+	kseq_t *mm_fastx_open(const char *fn)
+	void mm_fastx_close(kseq_t *ks)
+	int kseq_read(kseq_t *seq)
+
+	int mm_verbose_level(int v)
+	void mm_reset_timer()

python/mappy.pyx

@@ -0,0 +1,160 @@
+from libc.stdint cimport uint8_t, int8_t
+from libc.stdlib cimport free
+cimport cmappy
+
+cmappy.mm_reset_timer()
+
+cdef class Alignment:
+	cdef int _ctg_len, _r_st, _r_en
+	cdef int _q_st, _q_en
+	cdef int _NM, _mlen, _blen
+	cdef int8_t _strand, _trans_strand
+	cdef uint8_t _mapq, _is_primary
+	cdef _ctg, _cigar # these are python objects
+
+	def __cinit__(self, ctg, cl, cs, ce, strand, qs, qe, mapq, cigar, is_primary, mlen, blen, NM, trans_strand):
+		self._ctg, self._ctg_len, self._r_st, self._r_en = str(ctg), cl, cs, ce
+		self._strand, self._q_st, self._q_en = strand, qs, qe
+		self._NM, self._mlen, self._blen = NM, mlen, blen
+		self._mapq = mapq
+		self._cigar = cigar
+		self._is_primary = is_primary
+		self._trans_strand = trans_strand
+
+	@property
+	def ctg(self): return self._ctg
+
+	@property
+	def ctg_len(self): return self._ctg_len
+
+	@property
+	def r_st(self): return self._r_st
+
+	@property
+	def r_en(self): return self._r_en
+
+	@property
+	def strand(self): return self.strand
+
+	@property
+	def trans_strand(self): return self._trans_strand
+
+	@property
+	def blen(self): return self._blen
+
+	@property
+	def mlen(self): return self._mlen
+
+	@property
+	def NM(self): return self._NM
+
+	@property
+	def is_primary(self): return (self._is_primary != 0)
+
+	@property
+	def q_st(self): return self._q_st
+
+	@property
+	def q_en(self): return self._q_en
+
+	@property
+	def mapq(self): return self._mapq
+
+	@property
+	def cigar(self): return self._cigar
+
+	@property
+	def cigar_str(self):
+		return "".join(map(lambda x: str(x[0]) + 'MIDNSH'[x[1]], self._cigar))
+
+	def __str__(self):
+		if self._strand > 0: strand = '+'
+		elif self._strand < 0: strand = '-'
+		else: strand = '?'
+		if self._is_primary != 0: tp = 'tp:A:P'
+		else: tp = 'tp:A:S'
+		if self._trans_strand > 0: ts = 'ts:A:+'
+		elif self._trans_strand < 0: ts = 'ts:A:-'
+		else: ts = 'ts:A:.'
+		return "\t".join([str(self._q_st), str(self._q_en), strand, self._ctg, str(self._ctg_len), str(self._r_st), str(self._r_en),
+				str(self._mlen), str(self._blen), str(self._mapq), tp, ts, "cg:Z:" + self.cigar_str])
+
+cdef class ThreadBuffer:
+	cdef cmappy.mm_tbuf_t *_b
+
+	def __cinit__(self):
+		self._b = cmappy.mm_tbuf_init()
+
+	def __dealloc__(self):
+		cmappy.mm_tbuf_destroy(self._b)
+
+cdef class Aligner:
+	cdef cmappy.mm_idx_t *_idx
+	cdef cmappy.mm_idxopt_t idx_opt
+	cdef cmappy.mm_mapopt_t map_opt
+
+	def __cinit__(self, fn_idx_in, preset=None, k=None, w=None, min_cnt=None, min_chain_score=None, min_dp_score=None, bw=None, best_n=None, n_threads=3, fn_idx_out=None):
+		cmappy.mm_set_opt(NULL, &self.idx_opt, &self.map_opt) # set the default options
+		if preset is not None:
+			cmappy.mm_set_opt(str.encode(preset), &self.idx_opt, &self.map_opt) # apply preset
+		self.map_opt.flag |= 4 # always perform alignment
+		self.idx_opt.batch_size = 0x7fffffffffffffffL # always build a uni-part index
+		if k is not None: self.idx_opt.k = k
+		if w is not None: self.idx_opt.w = w
+		if min_cnt is not None: self.map_opt.min_cnt = min_cnt
+		if min_chain_score is not None: self.map_opt.min_chain_score = min_chain_score
+		if min_dp_score is not None: self.map_opt.min_dp_max = min_dp_score
+		if bw is not None: self.map_opt.bw = bw
+		if best_n is not None: self.best_n = best_n
+
+		cdef cmappy.mm_idx_reader_t *r;
+		if fn_idx_out is None:
+			r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, NULL)
+		else:
+			r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, fn_idx_out)
+		if r is not NULL:
+			self._idx = cmappy.mm_idx_reader_read(r, n_threads) # NB: ONLY read the first part
+			cmappy.mm_idx_reader_close(r)
+			cmappy.mm_mapopt_update(&self.map_opt, self._idx)
+
+	def __dealloc__(self):
+		if self._idx is not NULL:
+			cmappy.mm_idx_destroy(self._idx)
+
+	def __bool__(self):
+		return (self._idx != NULL)
+
+	def map(self, seq, buf=None):
+		cdef cmappy.mm_reg1_t *regs
+		cdef cmappy.mm_hitpy_t h
+		cdef ThreadBuffer b
+		cdef int n_regs
+
+		if self._idx is NULL: return None
+		if buf is None: b = ThreadBuffer()
+		else: b = buf
+		regs = cmappy.mm_map(self._idx, len(seq), str.encode(seq), &n_regs, b._b, &self.map_opt, NULL)
+
+		for i in range(n_regs):
+			cmappy.mm_reg2hitpy(self._idx, &regs[i], &h)
+			cigar = []
+			for k in range(h.n_cigar32):
+				c = h.cigar32[k]
+				cigar.append([c>>4, c&0xf])
+			yield Alignment(h.ctg, h.ctg_len, h.ctg_start, h.ctg_end, h.strand, h.qry_start, h.qry_end, h.mapq, cigar, h.is_primary, h.mlen, h.blen, h.NM, h.trans_strand)
+			cmappy.mm_free_reg1(&regs[i])
+		free(regs)
+
+def fastx_read(fn):
+	cdef cmappy.kseq_t *ks
+	ks = cmappy.mm_fastx_open(str.encode(fn))
+	if ks is NULL: return None
+	while cmappy.kseq_read(ks) >= 0:
+		if ks.qual.l > 0: qual = str(ks.qual.s)
+		else: qual = None
+		yield str(ks.name.s), str(ks.seq.s), qual
+	cmappy.mm_fastx_close(ks)
+
+def verbose(v=None):
+	if v is None: v = -1
+	return cmappy.mm_verbose_level(v)

python/minimap2.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import sys, getopt
+import mappy as mp
+
+def main(argv):
+	opts, args = getopt.getopt(argv[1:], "x:n:m:k:w:r:")
+	if len(args) < 2:
+		print("Usage: minimap2.py [options] <ref.fa>|<ref.mmi> <query.fq>")
+		print("Options:")
+		print("  -x STR      preset: sr, map-pb, map-ont, asm5, asm10 or splice")
+		print("  -n INT      mininum number of minimizers")
+		print("  -m INT      mininum chaining score")
+		print("  -k INT      k-mer length")
+		print("  -w INT      minimizer window length")
+		print("  -r INT      band width")
+		sys.exit(1)
+
+	preset, min_cnt, min_sc, k, w, bw = None, None, None, None, None, None
+	for opt, arg in opts:
+		if opt == '-x': preset = arg
+		elif opt == '-n': min_cnt = int(arg)
+		elif opt == '-m': min_chain_score = int(arg)
+		elif opt == '-r': bw = int(arg)
+		elif opt == '-k': k = int(arg)
+		elif opt == '-w': w = int(arg)
+
+	a = mp.Aligner(args[0], preset=preset, min_cnt=min_cnt, min_chain_score=min_sc, k=k, w=w, bw=bw)
+	if not a: raise Exception("ERROR: failed to load/build index file '{}'".format(args[0]))
+	for name, seq, qual in mp.fastx_read(args[1]): # read one sequence
+		for h in a.map(seq): # traverse hits
+			print('{}\t{}\t{}'.format(name, len(seq), h))
+
+if __name__ == "__main__":
+	main(sys.argv)

setup.py

@@ -0,0 +1,55 @@
+try:
+	from setuptools import setup, Extension
+except ImportError:
+	from distutils.core import setup
+	from distutils.extension import Extension
+
+cmdclass = {}
+
+try:
+	from Cython.Build import build_ext
+except ImportError: # without Cython
+	module_src = 'python/mappy.c'
+else: # with Cython
+	module_src = 'python/mappy.pyx'
+	cmdclass['build_ext'] = build_ext
+
+import sys
+sys.path.append('python')
+
+def readme():
+    with open('python/README.rst') as f:
+        return f.read()
+
+setup(
+	name = 'mappy',
+	version = '2.3',
+	url = 'https://github.com/lh3/minimap2',
+	description = 'Minimap2 python binding',
+	long_description = readme(),
+	author = 'Heng Li',
+	author_email = 'lh3@me.com',
+	license = 'MIT',
+	keywords = 'sequence-alignment',
+	scripts = ['python/minimap2.py'],
+    ext_modules = [Extension('mappy',
+		sources = [module_src, 'align.c', 'bseq.c', 'chain.c', 'format.c', 'hit.c', 'index.c', 'pe.c',
+				   'ksw2_extd2_sse.c', 'ksw2_exts2_sse.c', 'ksw2_extz2_sse.c', 'ksw2_ll_sse.c',
+				   'kalloc.c', 'kthread.c', 'map.c', 'misc.c', 'sdust.c', 'sketch.c'],
+		depends = ['minimap.h', 'bseq.h', 'kalloc.h', 'kdq.h', 'khash.h', 'kseq.h', 'ksort.h',
+				   'ksw2.h', 'kthread.h', 'kvec.h', 'mmpriv.h', 'sdust.h',
+				   'python/cmappy.h', 'python/cmappy.pxd'],
+		extra_compile_args = ['-msse4'], # WARNING: ancient x86_64 CPUs don't have SSE4
+		include_dirs = ['.'],
+		libraries = ['z', 'm', 'pthread'])],
+	classifiers = [
+		'Development Status :: 4 - Beta',
+		'License :: OSI Approved :: MIT License',
+		'Operating System :: POSIX',
+		'Programming Language :: C',
+		'Programming Language :: Cython',
+		'Programming Language :: Python :: 2.7',
+		'Programming Language :: Python :: 3',
+		'Intended Audience :: Science/Research',
+		'Topic :: Scientific/Engineering :: Bio-Informatics'],
+	cmdclass = cmdclass)

sketch.c

@@ -6,21 +6,21 @@
 #include "minimap.h"
 
 unsigned char seq_nt4_table[256] = {
-	0, 1, 2, 3,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	0, 1, 2, 3,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
-	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4
 };
 

tex/bowtie2-s3.sam.eval

@@ -0,0 +1,28 @@
+Q	42	16872292	669	0.000039651	16872292
+Q	40	835329	636	0.000073697	17707621
+Q	31	6544	2	0.000073783	17714165
+Q	30	8882	6	0.000074084	17723047
+Q	27	68499	9	0.000074305	17791546
+Q	26	132041	81	0.000078277	17923587
+Q	25	129378	96	0.000083033	18052965
+Q	24	92056	382	0.000103665	18145021
+Q	23	14341	402	0.000125720	18159362
+Q	22	132838	146	0.000132789	18292200
+Q	21	122274	124	0.000138641	18414474
+Q	18	112183	103	0.000143361	18526657
+Q	17	126981	213	0.000153804	18653638
+Q	16	16356	208	0.000164810	18669994
+Q	15	42804	782	0.000206223	18712798
+Q	14	16026	318	0.000223025	18728824
+Q	12	170250	814	0.000264087	18899074
+Q	11	48351	1409	0.000337777	18947425
+Q	8	1843	311	0.000354156	18949268
+Q	7	62266	4435	0.000586276	19011534
+Q	6	413997	50057	0.003150647	19425531
+Q	5	404	58	0.003153568	19425935
+Q	4	704	154	0.003161381	19426639
+Q	3	1473	681	0.003196193	19428112
+Q	2	17541	16462	0.004039875	19445653
+Q	1	534344	354879	0.021693547	19979997
+Q	0	11939	9917	0.022176642	19991936
+U	8064

tex/bwa-s3.sam.eval

@@ -0,0 +1,52 @@
+Q	60	18784147	3	0.000000160	18784147
+Q	52	19002	1	0.000000213	18803149
+Q	50	7152	2	0.000000319	18810301
+Q	49	6797	1	0.000000372	18817098
+Q	48	52188	2	0.000000477	18869286
+Q	47	48775	3	0.000000634	18918061
+Q	46	19447	2	0.000000739	18937508
+Q	45	25983	3	0.000000896	18963491
+Q	44	13455	1	0.000000949	18976946
+Q	43	14573	2	0.000001053	18991519
+Q	42	8697	4	0.000001263	19000216
+Q	41	8645	2	0.000001368	19008861
+Q	40	176603	75	0.000005264	19185464
+Q	38	2503	2	0.000005368	19187967
+Q	37	4117	3	0.000005523	19192084
+Q	36	2924	16	0.000006356	19195008
+Q	35	2323	8	0.000006772	19197331
+Q	34	2344	10	0.000007292	19199675
+Q	33	4279	6	0.000007603	19203954
+Q	32	2092	4	0.000007810	19206046
+Q	31	2625	11	0.000008382	19208671
+Q	30	2828	13	0.000009057	19211499
+Q	29	1581	1	0.000009108	19213080
+Q	28	1543	6	0.000009420	19214623
+Q	27	70916	223	0.000020948	19285539
+Q	26	1288	16	0.000021777	19286827
+Q	25	25551	122	0.000028065	19312378
+Q	24	14345	84	0.000032390	19326723
+Q	23	7308	87	0.000036878	19334031
+Q	22	8358	125	0.000043325	19342389
+Q	21	4836	71	0.000046983	19347225
+Q	20	5888	123	0.000053325	19353113
+Q	19	4656	83	0.000057600	19357769
+Q	18	3948	87	0.000062081	19361717
+Q	17	4418	114	0.000067954	19366135
+Q	16	4226	131	0.000074702	19370361
+Q	15	5760	164	0.000083144	19376121
+Q	14	4697	257	0.000096384	19380818
+Q	13	5246	313	0.000112503	19386064
+Q	12	4170	241	0.000124908	19390234
+Q	11	4095	304	0.000140557	19394329
+Q	10	3857	360	0.000159087	19398186
+Q	9	5300	438	0.000181617	19403486
+Q	8	4206	572	0.000211050	19407692
+Q	7	4676	787	0.000251541	19412368
+Q	6	3923	688	0.000286924	19416291
+Q	5	3294	708	0.000323333	19419585
+Q	4	2936	693	0.000358965	19422521
+Q	3	3928	816	0.000400897	19426449
+Q	2	2613	810	0.000442533	19429062
+Q	1	3515	1188	0.000503587	19432577
+Q	0	567423	376636	0.019321100	20000000

tex/hs38-simu.sh

@@ -2,3 +2,9 @@
 
 bin/mason_variator -ir hs38.fa -s 1 -ov hs38-s1.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
 bin/mason_simulator -ir hs38.fa -iv hs38-s1.vcf -n 1000000 --seed 1 -o s1_1.fq -or s1_2.fq -oa s1.sam --illumina-prob-mismatch-scale 2.5
+
+bin/mason_variator -ir hs38.fa -s 2 -ov hs38-s2.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
+bin/mason_simulator -ir hs38.fa -iv hs38-s2.vcf -n 1000000 --seed 2 -o mason-s2_1.fq -or mason-s2_2.fq -oa mason-s2.sam --illumina-prob-mismatch-scale 2.5 --illumina-read-length 150
+
+bin/mason_variator -ir hs38.fa -s 3 -ov hs38-s3.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
+bin/mason_simulator -ir hs38.fa -iv hs38-s3.vcf -n 10000000 --seed 3 -o mason-s3_1.fq -or mason-s3_2.fq -oa mason-s3.sam --illumina-prob-mismatch-scale 2.5 --illumina-read-length 150

tex/minimap2.bib

@@ -259,3 +259,25 @@
 	Title = {{Striped Smith-Waterman speeds database searches six times over other SIMD implementations}},
 	Volume = {23},
 	Year = {2007}}
+
+@techreport{Holtgrewe:2010aa,
+	Address = {Freie Universit{\"a}t Berlin},
+	Author = {Holtgrewe, M.},
+	Institution = {Institut f{\"u}r Mathematik und Informatik},
+	Number = {TR-B-10-06},
+	Title = {Mason -- a read simulator for second generation sequencing data},
+	Year = {2010}}
+
+@article{Langmead:2012fk,
+	Author = {Langmead, Ben and Salzberg, Steven L},
+	Journal = {Nat Methods},
+	Pages = {357-9},
+	Title = {Fast gapped-read alignment with Bowtie 2},
+	Volume = {9},
+	Year = {2012}}
+
+@article{Zaharia:2011aa,
+	Author = {Zaharia, Matei and others},
+	Journal = {arXiv:1111:5572},
+	Title = {Faster and More Accurate Sequence Alignment with SNAP},
+	Year = {2011}}

tex/minimap2.tex

@@ -20,22 +20,30 @@
 \begin{document}
 \firstpage{1}
 
-\title[Aligning long nucleotide sequences with minimap2]{Minimap2: fast pairwise alignment for long nucleotide sequences}
+\title[Aligning nucleotide sequences with minimap2]{Minimap2: versatile pairwise alignment for nucleotide sequences}
 \author[Li]{Heng Li}
 \address{Broad Institute, 415 Main Street, Cambridge, MA 02142, USA}
 
 \maketitle
 
 \begin{abstract}
-\section{Summary:} Minimap2 is a general-purpose mapper to align long noisy DNA
-or mRNA sequences against a large reference database. It targets query
-sequences of 1kb--100Mb in length with per-base divergence typically below
-25\%. For DNA sequence reads, minimap2 is $\sim$30 times faster than many
-mainstream long-read aligners and achieves higher accuracy on simulated data.
-It also employs concave gap cost and rescues inversions for improved alignment
-around potential structural variations. For real long RNA-seq reads, minimap2
-is $\sim$40 times faster than peers and produces alignment more consistent with
-existing gene annotations.
+
+\section{Motivation:} Recent advances in sequencing technologies promise
+ultra-long reads of $\sim$100 kilo bases (kb) in average, full-length mRNA or
+cDNA reads in high throughput and genomic contigs over 100 mega bases (Mb) in
+length. Existing alignment tools are unable or inefficient to process such data
+at scale, which presses for the development of new alignment algorithms.
+
+\section{Results:} Minimap2 is a general-purpose aligner to map DNA or long
+mRNA sequences against a large reference database. It works with accurate short
+reads of $\ge$100bp in length, $\ge$1kb genomic reads at error rate $\sim$15\%,
+full-length noisy Direct RNA or cDNA reads, and assembly contigs or closely
+related full chromosomes of hundreds of megabases in length. Minimap2 does
+split-read alignment, employs concave gap cost for long insertions and
+deletions (INDELs) and introduces new heuristics to reduce spurious alignments.
+It is 3--4 times faster than mainstream short-read mappers at comparable
+accuracy and $\ge$30 times faster at higher accuracy for both genomic and mRNA
+reads, surpassing most aligners specialized in one type of alignment.
 
 \section{Availability and implementation:}
 \href{https://github.com/lh3/minimap2}{https://github.com/lh3/minimap2}
@@ -60,7 +68,7 @@ approximate mapping 50 times faster than BWA-MEM~\citep{Li:2016aa}.
 generating base-level alignment, which in turn inspired us to develop minimap2
 towards higher accuracy and more practical functionality.
 
-Both SMRT and ONT have been applied to sequence spliced mRNAs (RNA-seq). While
+Both SMRT and ONT have been applied to the sequencing of spliced mRNAs (RNA-seq). While
 traditional mRNA aligners work~\citep{Wu:2005vn,Iwata:2012aa}, they are not
 optimized for long noisy sequence reads and are tens of times slower than
 dedicated long-read aligners. When developing minimap2 initially for aligning
@@ -103,8 +111,11 @@ distance between two anchors is too large); otherwise
 \begin{equation}\label{eq:chain-gap}
 \beta(j,i)=\gamma_c\big((y_i-y_j)-(x_i-x_j)\big)
 \end{equation}
-In implementation, a gap of length $l$ costs $\gamma_c(l)=0.01\cdot \bar{w}\cdot
-|l|+0.5\log_2|l|$, where $\bar{w}$ is the average seed length. For $m$ anchors, directly computing all $f(\cdot)$ with
+In implementation, a gap of length $l$ costs
+\[
+\gamma_c(l)=0.01\cdot \bar{w}\cdot|l|+0.5\log_2|l|
+\]
+where $\bar{w}$ is the average seed length. For $m$ anchors, directly computing all $f(\cdot)$ with
 Eq.~(\ref{eq:chain}) takes $O(m^2)$ time. Although theoretically faster
 chaining algorithms exist~\citep{Abouelhoda:2005aa}, they
 are inapplicable to generic gap cost, complex to implement and usually
@@ -126,7 +137,7 @@ find its predecessor and mark each visited $i$ as `used', until $P(i)=0$ or we
 reach an already `used' $i$. This way we find all chains with no anchors used
 in more than one chains.
 
-\subsubsection{Identifying primary chains}
+\subsubsection{Identifying primary chains}\label{sec:primary}
 In the absence of copy number changes, each query segment should not be mapped
 to two places in the reference. However, chains found at the previous step may
 have significant or complete overlaps due to repeats in the reference.
@@ -139,7 +150,7 @@ add the chain to $Q$. In the end, $Q$ contains all the primary chains. We did
 not choose a more sophisticated data structure (e.g. range tree or k-d tree)
 because this step is not the performance bottleneck.
 
-\subsection{Aligning genomic DNA}
+\subsection{Aligning genomic DNA}\label{sec:genomic}
 
 \subsubsection{Alignment with 2-piece affine gap cost}
 
@@ -352,28 +363,48 @@ minimizers and disables banded alignment. Together with the two-round DP-based
 alignment, spliced alignment is several times slower than DNA sequence
 alignment.
 
+\subsection{Aligning short paired-end reads}
+
+During chainging, minimap2 takes a pair of reads as one read with a gap of
+unknown length in the middle. It applies a normal gap cost between seeds on the
+same read but is a more permissive gap cost between seeds on different reads.
+More precisely, the gap cost during chaining is:
+\[
+\gamma_c(l)=\left\{\begin{array}{ll}
+0.01\cdot\bar{w}\cdot l+0.5\log_2 l & \mbox{if two seeds on the same read} \\
+\min\{0.01\cdot\bar{w}\cdot|l|,\log_2|l|\} & \mbox{otherwise}
+\end{array}\right.
+\]
+After identifying primary chains (Section~\ref{sec:primary}), we split each
+fragment chain into two read chains and perform alignment for each read as in
+Section~\ref{sec:genomic}.  Finally, we pair hits of each read end to find
+consistent paired-end alignments.
+
 \end{methods}
 
 \section{Results}
 
-\subsection{Aligning genomic reads}
+\subsection{Aligning long genomic reads}
 
 \begin{figure}[!tb]
 \centering
 \includegraphics[width=.5\textwidth]{roc-color.pdf}
-\caption{Evaluation on simulated SMRT reads aligned against human genome
-GRCh38.  33,088 $\ge$1000bp reads were simulated using pbsim~\citep{Ono:2013aa}
-with error profile sampled from file `m131017\_060208\_42213\_*.1.*' downloaded
-at \href{http://bit.ly/chm1p5c3}{http://bit.ly/chm1p5c3}. The N50 read length
-is 11,628. A read is considered correctly mapped if the true position overlaps
-with the best mapping position by 10\% of the read length. All aligners were
-run under the default setting for SMRT reads.  (a) ROC-like curve. Alignments
-are sorted by mapping quality in the descending order. For each mapping quality
-threshold, the fraction of alignments with mapping quality above the threshold
-and their error rate are plotted.  Kart outputted all alignments at mapping
-quality 60, so is not shown in the figure. It mapped nearly all reads with
-4.1\% of alignments being wrong, less accurate than others.  (b) Accumulative
-mapping error rate as a function of mapping quality.}\label{fig:eval}
+\caption{Evaluation on aligning simulated reads. Simulated reads were mapped 
+to the primary assembly of human genome GRCh38. A read is considered correctly
+mapped if the true position overlaps with the best mapping position by 10\% of
+the read length. Read alignments are sorted by mapping quality in the
+descending order. For each mapping quality threshold, the fraction of
+alignments with mapping quality above the threshold and their error rate are
+plotted along the curve. (a) long-read alignment evaluation. 33,088 $\ge$1000bp
+reads were simulated using pbsim~\citep{Ono:2013aa} with error profile sampled
+from file `m131017\_060208\_42213\_*.1.*' downloaded at
+\href{http://bit.ly/chm1p5c3}{http://bit.ly/chm1p5c3}. The N50 read length is
+11,628. Aligners were run under the default setting for SMRT reads.
+(b) short-read alignment evaluation. 10 million pairs of 150bp reads were
+simulated using mason2~\citep{Holtgrewe:2010aa} with option
+`\mbox{--illumina-prob-mismatch-scale 2.5}'. Short-read aligners were run under the
+default setting except for changing the maximum fragment length to
+800bp.}\label{fig:eval}
 \end{figure}
 
 As a sanity check, we evaluated minimap2 on simulated human reads along with
@@ -390,7 +421,7 @@ higher mapping accuracy (Fig.~\ref{fig:eval}a). It is still the most accurate
 even if we skip DP-based alignment (data not shown), confirming chaining alone
 is sufficient to achieve high accuracy for approximate mapping. Minimap2 and
 NGMLR provide better mapping quality estimate: they rarely give repetitive hits
-high mapping quality (Fig.~\ref{fig:eval}b).  Apparently, other aligners may
+high mapping quality.  Apparently, other aligners may
 occasionally miss close suboptimal hits and be overconfident in wrong mappings.
 On run time, minialign is slightly faster than minimap2 and Kart. They are over
 30 times faster than the rest.  Minimap2 consumed 6.1GB memory at the peak,
@@ -406,7 +437,7 @@ confirm the observation by~\citet{Sedlazeck169557} that BWA-MEM often breaks
 them into shorter gaps. The issue is much alleviated with minimap2, thanks
 to the 2-piece affine gap cost.
 
-\subsection{Aligning spliced reads}
+\subsection{Aligning long spliced reads}
 
 We evaluated minimap2 on SIRV control data~(AC:SRR5286959;
 \citealp{Byrne:2017aa}) where the truth is known. Minimap2 predicted 59\,916
@@ -427,15 +458,15 @@ sophisticated models. We have not tried this approach.
 \toprule
 & GMAP & minimap2 & SpAln & STAR\\
 \midrule
-Run time (CPU min) & 631 & 15.5 & 2\,076 & 33.9 \\
-Peak RAM (GByte) & 8.9 & 14.5 & 3.2 & 29.2\vspace{1em}\\
-\# aligned reads & 103\,669 & 103\,917 & 103\,711 & 26\,479\\
-\# chimeric alignments & 1\,904 & 1\,671 & 0 & 0\\
-\# non-spliced alignments & 15\,854 & 14\,483 & 17\,033 & 10\,545\vspace{1em}\\
-\# aligned introns & 692\,275 & 694\,237 & 692\,945 & 78\,603 \\
-\# novel introns & 11\,239 & 3\,217 & 8\,550 & 1\,214 \\
-\% exact introns & 83.8\% & 91.8\% & 87.9\% & 55.2\% \\
-\% approx. introns & 91.8\% & 96.5\% & 92.5\% & 82.4\% \\
+Run time (CPU min)        & 631      & 15.9     & 2\,076   & 33.9 \\
+Peak RAM (GByte)          & 8.9      & 14.5     & 3.2      & 29.2\vspace{1em}\\
+\# aligned reads          & 103\,669 & 104\,200 & 103\,711 & 26\,479 \\
+\# chimeric alignments    & 1\,904   & 1\,488   & 0        & 0 \\
+\# non-spliced alignments & 15\,854  & 14\,639  & 17\,033  & 10\,545\vspace{1em}\\
+\# aligned introns        & 692\,275 & 694\,103 & 692\,945 & 78\,603 \\
+\# novel introns          & 11\,239  & 3\,207   & 8\,550   & 1\,214 \\
+\% exact introns          & 83.8\%   & 91.7\%   & 87.9\%   & 55.2\% \\
+\% approx. introns        & 91.8\%   & 96.5\%   & 92.5\%   & 82.4\% \\
 \botrule
 \end{tabular}
 }{Mouse reads (AC:SRR5286960) were mapped to the primary assembly of mouse
@@ -470,21 +501,32 @@ able to improve their accuracy further.
 %{\footnotesize
 %\begin{tabular}{lrrrr}
 %\toprule
-%& GMAP & minimap2 & SpAln & STAR\\
+%                          &      GMAP &  minimap2 &     SpAln &      STAR \\ % one GMAP thread took 14 days to align a tiny fraction of reads
 %\midrule
-%Run time (CPU min) & & 243 & 2\,352 & 1\,647 \\
-%\# aligned reads & & 1\,123\,025 & 1\,094\,092 & 682\,452\\
-%\# chimeric alignments & & 33\,091 & 0 & 0\\
-%\# non-spliced alignments & & 339\,081 & 291\,447 & 272\,536\vspace{1em}\\
-%\# aligned introns & & 9\,071\,755 & 9\,208\,564 & 3\,029\,121 \\
-%\# novel introns & & 42\,773 & 82\,230 & 17\,791 \\
-%\% exact introns & & 94.9\% & 91.7\% & 84.7\% \\
-%\% approx. introns&& 96.9\% & 93.4\% & 93.8\% \\
+%Run time (CPU min)        &         - &       243 &     2,352 &     1,647 \\
+%\# aligned reads          & 1,113,502 & 1,123,025 & 1,094,092 &   682,452 \\
+%\# chimeric alignments    &    48,927 &    33,091 &         0 &         0 \\
+%\# non-spliced alignments &   334,097 &   339,081 &   291,447 &   272,536 \vspace{1em}\\
+%\# aligned introns        & 8,922,221 & 9,071,755 & 9,208,564 & 3,029,121 \\
+%\# novel introns          &    48,927 &    42,773 &    82,230 &    17,791 \\
+%\% exact introns          &    90.6\% &    94.9\% &    91.7\% &    84.7\% \\
+%\% approx. introns        &    94.0\% &    96.9\% &    93.4\% &    93.8\% \\
 %\botrule
 %\end{tabular}
 %}{}
 %\end{table}
 
+\subsection{Aligning short genomic reads}
+
+We evaluated minimap2 along with Bowtie2~\citep{Langmead:2012fk}, BWA-MEM and
+SNAP~\citep{Zaharia:2011aa}. Minimap2 is 3--4 times as fast as Bowtie2 and
+BWA-MEM, but is 1.3 times slower than SNAP. Minimap2 is more accurate on this
+simulated data set than Bowtie2 and SNAP but less accurate than BWA-MEM
+(Fig.~\ref{fig:eval}b). Closer investigation reveals that BWA-MEM achieves
+a higher accuracy partly because it tries to locally align a read in a small
+region close to its mate. If we disable this feature, BWA-MEM becomes slightly
+less accurate than minimap2. We might consider to implement a similar heuristic
+in minimap2 in future.
 
 \section{Conclusion}
 

tex/mm2-s3.sam.eval

@@ -0,0 +1,60 @@
+Q	60	18345673	8	0.000000436	18345673
+Q	59	33966	4	0.000000653	18379639
+Q	58	34178	1	0.000000706	18413817
+Q	56	49138	1	0.000000758	18462955
+Q	54	22442	4	0.000000974	18485397
+Q	53	19070	2	0.000001081	18504467
+Q	52	14169	3	0.000001242	18518636
+Q	51	13233	4	0.000001457	18531869
+Q	50	12133	2	0.000001564	18544002
+Q	49	11138	4	0.000001778	18555140
+Q	48	11174	8	0.000002208	18566314
+Q	47	17139	4	0.000002422	18583453
+Q	46	20428	10	0.000002956	18603881
+Q	45	16503	3	0.000003115	18620384
+Q	44	11933	6	0.000003435	18632317
+Q	43	25392	11	0.000004020	18657709
+Q	42	16734	9	0.000004498	18674443
+Q	41	13826	10	0.000005030	18688269
+Q	40	13023	10	0.000005561	18701292
+Q	39	12686	10	0.000006092	18713978
+Q	38	17275	4	0.000006300	18731253
+Q	37	17241	2	0.000006401	18748494
+Q	36	12458	12	0.000007036	18760952
+Q	35	11981	5	0.000007298	18772933
+Q	34	12004	11	0.000007879	18784937
+Q	33	12111	7	0.000008246	18797048
+Q	32	11782	9	0.000008719	18808830
+Q	31	11811	7	0.000009086	18820641
+Q	30	33507	32	0.000010767	18854148
+Q	29	11243	21	0.000011874	18865391
+Q	28	10779	17	0.000012767	18876170
+Q	27	15733	24	0.000014027	18891903
+Q	26	16762	40	0.000016130	18908665
+Q	25	13811	49	0.000018708	18922476
+Q	24	14141	46	0.000021123	18936617
+Q	23	13429	55	0.000024010	18950046
+Q	22	13116	26	0.000025365	18963162
+Q	21	13436	46	0.000027771	18976598
+Q	20	13441	55	0.000030648	18990039
+Q	19	12988	53	0.000033416	19003027
+Q	18	13353	51	0.000036074	19016380
+Q	17	13782	77	0.000040094	19030162
+Q	16	14065	94	0.000045001	19044227
+Q	15	14044	124	0.000051474	19058271
+Q	14	14714	140	0.000058774	19072985
+Q	13	17459	197	0.000069040	19090444
+Q	12	17339	259	0.000082532	19107783
+Q	11	17381	280	0.000097097	19125164
+Q	10	17732	295	0.000112418	19142896
+Q	9	17959	416	0.000134023	19160855
+Q	8	18234	530	0.000161530	19179089
+Q	7	19048	514	0.000188143	19198137
+Q	6	19722	656	0.000222085	19217859
+Q	5	19753	775	0.000262143	19237612
+Q	4	19818	1030	0.000315359	19257430
+Q	3	17088	1100	0.000372149	19274518
+Q	2	43045	6708	0.000718569	19317563
+Q	1	126377	25255	0.002012761	19443940
+Q	0	554357	372087	0.020562901	19998297
+U	1703

tex/mm2.eval

@@ -1,17 +1,9 @@
-Q	60	32072	0	0.000000000
-Q	43	206	1	0.000030981
-Q	27	201	1	0.000061578
-Q	15	59	1	0.000092200
-Q	12	25	1	0.000122839
-Q	11	16	1	0.000153473
-Q	10	24	1	0.000184032
-Q	9	17	2	0.000245248
-Q	8	27	3	0.000336938
-Q	7	23	1	0.000367309
-Q	6	20	1	0.000397675
-Q	5	18	4	0.000519751
-Q	4	17	1	0.000550038
-Q	3	29	5	0.000702204
-Q	2	32	4	0.000823522
-Q	1	54	6	0.001004872
-Q	0	234	106	0.004202697
+Q	60	32226	0	0.000000000	32226
+Q	20	267	1	0.000030776	32493
+Q	10	34	1	0.000061487	32527
+Q	9	118	1	0.000091898	32645
+Q	5	27	2	0.000153036	32672
+Q	4	68	2	0.000213806	32740
+Q	1	314	101	0.003267381	33054
+Q	0	31	17	0.003778147	33085
+U	3

tex/roc.gp

@@ -14,7 +14,7 @@ set size 1.59,1.04
 set multiplot layout 1,2
 
 set label "(a)" at graph -0.245,1.06 font "Helvetica-bold,40"
-set xlab "Error rate of mapped reads"
+set xlab "Error rate of mapped PacBio reads"
 set ylab "Fraction of mapped reads" off +1.8
 set ytics 0.02
 set yran [0.9:1]
@@ -34,19 +34,27 @@ unset label
 set origin 0.8,0
 set size 0.79,1
 set label "(b)" at graph -0.245,1.06 font "Helvetica-bold,40"
-unset log
-unset format
-unset key
-set log y
-set ylab "Accumulative mapping error rate" off +0
-set xlab "Mapping quality"
-set yran [1e-5:0.1]
-set ytics 1e-5,0.1
-set format y "10^{%L}"
-set xran [60:0] reverse
-plot "<./eval2roc.pl blasr-mc.eval" u 1:2 w lp ls 4, \
-     "<./eval2roc.pl bwa.eval" u 1:2 t "bwa-mem" w lp ls 2, \
-     "<./eval2roc.pl graphmap.eval" u 1:2 t "graphmap" w lp ls 3, \
-     "<./eval2roc.pl minialign.eval" u 1:2 t "minialign" w lp ls 1, \
-     "<./eval2roc.pl mm2.eval" u 1:2 t "minimap2" w lp ls 6, \
-     "<./eval2roc.pl ngmlr.eval" u 1:2 t "ngm-lr" w lp ls 5
+set xlab "Error rate of mapped short reads"
+
+set key top left
+plot "<./eval2roc.pl -n2e7 bowtie2-s3.sam.eval" u 2:3 t "bowtie2" w lp ls 5, \
+	 "<./eval2roc.pl -n2e7 bwa-s3.sam.eval" u 2:3 t "bwa-mem" w lp ls 2, \
+	 "<./eval2roc.pl -n2e7 mm2-s3.sam.eval" u 2:3 t "minimap2" w lp ls 6, \
+	 "<./eval2roc.pl -n2e7 snap-s3.sam.eval" u 2:3 t "snap" w lp ls 3
+
+#unset log
+#unset format
+#unset key
+#set log y
+#set ylab "Accumulative mapping error rate" off +0
+#set xlab "Mapping quality"
+#set yran [1e-5:0.1]
+#set ytics 1e-5,0.1
+#set format y "10^{%L}"
+#set xran [60:0] reverse
+#plot "<./eval2roc.pl blasr-mc.eval" u 1:2 w lp ls 4, \
+#     "<./eval2roc.pl bwa.eval" u 1:2 t "bwa-mem" w lp ls 2, \
+#     "<./eval2roc.pl graphmap.eval" u 1:2 t "graphmap" w lp ls 3, \
+#     "<./eval2roc.pl minialign.eval" u 1:2 t "minialign" w lp ls 1, \
+#     "<./eval2roc.pl mm2.eval" u 1:2 t "minimap2" w lp ls 6, \
+#     "<./eval2roc.pl ngmlr.eval" u 1:2 t "ngm-lr" w lp ls 5

tex/snap-s3.sam.eval

@@ -0,0 +1,62 @@
+Q	60	18993268	10320	0.000543350	18993268
+Q	59	33156	216	0.000553756	19026424
+Q	58	29982	295	0.000568365	19056406
+Q	57	9412	278	0.000582666	19065818
+Q	56	11012	228	0.000594281	19076830
+Q	55	9968	235	0.000606283	19086798
+Q	54	8602	292	0.000621301	19095400
+Q	53	6094	259	0.000634662	19101494
+Q	52	5026	257	0.000647946	19106520
+Q	51	4278	224	0.000659522	19110798
+Q	50	3682	178	0.000668708	19114480
+Q	49	2750	156	0.000676772	19117230
+Q	48	2314	112	0.000682548	19119544
+Q	47	2056	96	0.000687495	19121600
+Q	46	1658	62	0.000690677	19123258
+Q	45	1492	74	0.000694493	19124750
+Q	44	1150	56	0.000697379	19125900
+Q	43	1062	48	0.000699850	19126962
+Q	42	976	60	0.000702951	19127938
+Q	41	884	36	0.000704800	19128822
+Q	40	708	52	0.000707493	19129530
+Q	39	870	26	0.000708819	19130400
+Q	38	598	26	0.000710156	19130998
+Q	37	542	34	0.000711913	19131540
+Q	36	846	50	0.000714495	19132386
+Q	35	590	50	0.000717087	19132976
+Q	34	550	42	0.000719261	19133526
+Q	33	2174	66	0.000722628	19135700
+Q	32	876	86	0.000727089	19136576
+Q	31	638	104	0.000732500	19137214
+Q	30	1718	196	0.000742675	19138932
+Q	29	91022	968	0.000789497	19229954
+Q	28	12864	781	0.000829556	19242818
+Q	27	5806	427	0.000851489	19248624
+Q	26	25274	728	0.000888144	19273898
+Q	25	7418	680	0.000923070	19281316
+Q	24	11800	701	0.000958839	19293116
+Q	23	57328	3933	0.001159250	19350444
+Q	22	7662	846	0.001202494	19358106
+Q	21	5924	617	0.001233989	19364030
+Q	20	4623	574	0.001263330	19368653
+Q	19	4988	942	0.001311627	19373641
+Q	18	3968	793	0.001352282	19377609
+Q	17	3630	681	0.001387166	19381239
+Q	16	2921	513	0.001413422	19384160
+Q	15	2716	424	0.001435095	19386876
+Q	14	2366	365	0.001453744	19389242
+Q	13	2169	412	0.001474828	19391411
+Q	12	2077	360	0.001493233	19393488
+Q	11	2016	441	0.001515815	19395504
+Q	10	2292	738	0.001553682	19397796
+Q	9	4165	1832	0.001647772	19401961
+Q	8	3963	1862	0.001743385	19405924
+Q	7	3927	1793	0.001835408	19409851
+Q	6	3572	1639	0.001919497	19413423
+Q	5	3270	1533	0.001998126	19416693
+Q	4	3046	1610	0.002080718	19419739
+Q	3	251447	125550	0.008436553	19671186
+Q	2	24390	13537	0.009113417	19695576
+Q	1	124406	86780	0.013434624	19819982
+Q	0	171254	153874	0.021016609	19991236
+U	8764