Release minimap2-2.24 (r1122)

Resolve #852

.github/workflows/ci.yaml

@@ -0,0 +1,21 @@
+name: CI
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        compiler: [gcc, clang]
+
+    steps:
+    - name: Checkout minimap2
+      uses: actions/checkout@v2
+
+    - name: Compile with ${{ matrix.compiler }}
+      run:  make CC=${{ matrix.compiler }}

.gitignore

@@ -1,4 +1,8 @@
+.cproject
+.project
 .*.swp
 *.a
 *.o
 *.dSYM
+minimap2
+mappy.c

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/simde"]
+	path = lib/simde
+	url = https://github.com/nemequ/simde.git

.travis.yml

@@ -0,0 +1,24 @@
+matrix:
+  include:
+    - language: c
+      compiler: gcc
+      script: make
+    - language: c
+      compiler: clang
+      script: make
+    - arch: arm64
+      language: c
+      compiler: gcc
+      script: make arm_neon=1 aarch64=1
+    - language: python
+      python: "2.7"
+      before_install: pip install cython
+      script: python setup.py build_ext
+    - language: python
+      python: "3.5"
+      before_install: pip install cython
+      script: python setup.py build_ext
+    - language: python
+      python: "3.9"
+      before_install: pip install cython
+      script: python setup.py build_ext

FAQ.md

@@ -0,0 +1,46 @@
+#### 1. Alignment different with option `-a` or `-c`?
+
+Without `-a`, `-c` or `--cs`, minimap2 only finds *approximate* mapping
+locations without detailed base alignment. In particular, the start and end
+positions of the alignment are impricise. With one of those options, minimap2
+will perform base alignment, which is generally more accurate but is much
+slower.
+
+#### 2. How to map Illumina short reads to noisy long reads?
+
+No good solutions. The better approach is to assemble short reads into contigs
+and then map noisy reads to contigs.
+
+#### 3. The output SAM doesn't have a header.
+
+By default, minimap2 indexes 4 billion reference bases (4Gb) in a batch and map
+all reads against each reference batch. Given a reference longer than 4Gb,
+minimap2 is unable to see all the sequences and thus can't produce a correct
+SAM header. In this case, minimap2 doesn't output any SAM header. There are two
+solutions to this issue. First, you may increase option `-I` to, for example,
+`-I8g` to index more reference bases in a batch. This is preferred if your
+machine has enough memory. Second, if your machines doesn't have enough memory
+to hold the reference index, you can use the `--split-prefix` option in a
+command line like:
+```sh
+minimap2 -ax map-ont --split-prefix=tmp ref.fa reads.fq
+```
+This second approach uses less memory, but it is slower and requires temporary
+disk space.
+
+#### 4. The output SAM is malformatted.
+
+This typically happens when you use nohup to wrap a minimap2 command line.
+Nohup is discouraged as it breaks piping. If you have to use nohup, please
+specify an output file with option `-o`.
+
+#### 5. How to output one alignment per read?
+
+You can use `--secondary=no` to suppress secondary alignments (aka multiple
+mappings), but you can't suppress supplementary alignment (aka split or
+chimeric alignment) this way. You can use samtools to filter out these
+alignments:
+```sh
+minimap2 -ax map-out ref.fa reads.fq | samtools view -F0x900
+```
+However, this is discouraged as supplementary alignment is informative.

LICENSE.txt

@@ -1,6 +1,7 @@
 The MIT License
 
-Copyright (c) 2017 Broad Institute, Inc.
+Copyright (c) 2018-     Dana-Farber Cancer Institute
+              2017-2018 Broad Institute, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

MANIFEST.in

@@ -0,0 +1,10 @@
+include *.h
+include Makefile
+include ksw2_dispatch.c
+include main.c
+include README.md
+include sse2neon/emmintrin.h
+include python/cmappy.h
+include python/cmappy.pxd
+include python/mappy.pyx
+include python/README.rst

Makefile

@@ -1,17 +1,40 @@
-CC=			gcc
+CFLAGS=		-g -Wall -O2 -Wc++-compat #-Wextra
-CFLAGS=		-g -Wall -O2 -Wc++-compat
 CPPFLAGS=	-DHAVE_KALLOC
-INCLUDES=	-I.
+INCLUDES=
-OBJS=		kthread.o kalloc.o ksw2_extz2_sse.o ksw2_extd2_sse.o misc.o bseq.o \
+OBJS=		kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o options.o index.o \
-			sketch.o sdust.o index.o chain.o align.o hit.o map.o format.o
+			lchain.o align.o hit.o seed.o map.o format.o pe.o esterr.o splitidx.o \
+			ksw2_ll_sse.o
 PROG=		minimap2
 PROG_EXTRA=	sdust minimap2-lite
 LIBS=		-lm -lz -lpthread
 
-ifeq ($(sse2only),)
+ifeq ($(arm_neon),) # if arm_neon is not defined
-	CFLAGS+=-msse4
+ifeq ($(sse2only),) # if sse2only is not defined
+	OBJS+=ksw2_extz2_sse41.o ksw2_extd2_sse41.o ksw2_exts2_sse41.o ksw2_extz2_sse2.o ksw2_extd2_sse2.o ksw2_exts2_sse2.o ksw2_dispatch.o
+else                # if sse2only is defined
+	OBJS+=ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_exts2_sse.o
+endif
+else				# if arm_neon is defined
+	OBJS+=ksw2_extz2_neon.o ksw2_extd2_neon.o ksw2_exts2_neon.o
+    INCLUDES+=-Isse2neon
+ifeq ($(aarch64),)	#if aarch64 is not defined
+	CFLAGS+=-D_FILE_OFFSET_BITS=64 -mfpu=neon -fsigned-char
+else				#if aarch64 is defined
+	CFLAGS+=-D_FILE_OFFSET_BITS=64 -fsigned-char
+endif
 endif
 
+ifneq ($(asan),)
+	CFLAGS+=-fsanitize=address
+	LIBS+=-fsanitize=address
+endif
+
+ifneq ($(tsan),)
+	CFLAGS+=-fsanitize=thread
+	LIBS+=-fsanitize=thread
+endif
+
+.PHONY:all extra clean depend
 .SUFFIXES:.c .o
 
 .c.o:
@@ -22,7 +45,7 @@ all:$(PROG)
 extra:all $(PROG_EXTRA)
 
 minimap2:main.o libminimap2.a
-		$(CC) $(CFLAGS) $< -o $@ -L. -lminimap2 $(LIBS)
+		$(CC) $(CFLAGS) main.o -o $@ -L. -lminimap2 $(LIBS)
 
 minimap2-lite:example.o libminimap2.a
 		$(CC) $(CFLAGS) $< -o $@ -L. -lminimap2 $(LIBS)
@@ -30,29 +53,80 @@ minimap2-lite:example.o libminimap2.a
 libminimap2.a:$(OBJS)
 		$(AR) -csru $@ $(OBJS)
 
-sdust:sdust.c kalloc.o kalloc.h kdq.h kvec.h kseq.h sdust.h
+sdust:sdust.c kalloc.o kalloc.h kdq.h kvec.h kseq.h ketopt.h sdust.h
 		$(CC) -D_SDUST_MAIN $(CFLAGS) $< kalloc.o -o $@ -lz
 
+# SSE-specific targets on x86/x86_64
+
+ifeq ($(arm_neon),)   # if arm_neon is defined, compile this target with the default setting (i.e. no -msse2)
+ksw2_ll_sse.o:ksw2_ll_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse2 $(CPPFLAGS) $(INCLUDES) $< -o $@
+endif
+
+ksw2_extz2_sse41.o:ksw2_extz2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+ksw2_extz2_sse2.o:ksw2_extz2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
+
+ksw2_extd2_sse41.o:ksw2_extd2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+ksw2_extd2_sse2.o:ksw2_extd2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
+
+ksw2_exts2_sse41.o:ksw2_exts2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+ksw2_exts2_sse2.o:ksw2_exts2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
+
+ksw2_dispatch.o:ksw2_dispatch.c ksw2.h
+		$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
+
+# NEON-specific targets on ARM
+
+ksw2_extz2_neon.o:ksw2_extz2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_SSE2_ONLY -D__SSE2__ $(INCLUDES) $< -o $@
+
+ksw2_extd2_neon.o:ksw2_extd2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_SSE2_ONLY -D__SSE2__ $(INCLUDES) $< -o $@
+
+ksw2_exts2_neon.o:ksw2_exts2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_SSE2_ONLY -D__SSE2__ $(INCLUDES) $< -o $@
+
+# other non-file targets
+
 clean:
-		rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM session*
+		rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM build dist mappy*.so mappy.c python/mappy.c mappy.egg*
 
 depend:
 		(LC_ALL=C; export LC_ALL; makedepend -Y -- $(CFLAGS) $(CPPFLAGS) -- *.c)
 
 # DO NOT DELETE
 
-align.o: minimap.h mmpriv.h bseq.h ksw2.h kalloc.h
+align.o: minimap.h mmpriv.h bseq.h kseq.h ksw2.h kalloc.h
-bseq.o: bseq.h kseq.h
+bseq.o: bseq.h kvec.h kalloc.h kseq.h
-chain.o: minimap.h mmpriv.h bseq.h kalloc.h
+esterr.o: mmpriv.h minimap.h bseq.h kseq.h
 example.o: minimap.h kseq.h
-format.o: mmpriv.h minimap.h bseq.h
+format.o: kalloc.h mmpriv.h minimap.h bseq.h kseq.h
-hit.o: mmpriv.h minimap.h bseq.h kalloc.h
+hit.o: mmpriv.h minimap.h bseq.h kseq.h kalloc.h khash.h
-index.o: kthread.h bseq.h minimap.h mmpriv.h kvec.h kalloc.h khash.h
+index.o: kthread.h bseq.h minimap.h mmpriv.h kseq.h kvec.h kalloc.h khash.h
+index.o: ksort.h
 kalloc.o: kalloc.h
 ksw2_extd2_sse.o: ksw2.h kalloc.h
+ksw2_exts2_sse.o: ksw2.h kalloc.h
 ksw2_extz2_sse.o: ksw2.h kalloc.h
-main.o: bseq.h minimap.h mmpriv.h
+ksw2_ll_sse.o: ksw2.h kalloc.h
-map.o: kthread.h kvec.h kalloc.h sdust.h mmpriv.h minimap.h bseq.h
+kthread.o: kthread.h
-misc.o: minimap.h ksort.h
+lchain.o: mmpriv.h minimap.h bseq.h kseq.h kalloc.h krmq.h
+main.o: bseq.h minimap.h mmpriv.h kseq.h ketopt.h
+map.o: kthread.h kvec.h kalloc.h sdust.h mmpriv.h minimap.h bseq.h kseq.h
+map.o: khash.h ksort.h
+misc.o: mmpriv.h minimap.h bseq.h kseq.h ksort.h
+options.o: mmpriv.h minimap.h bseq.h kseq.h
+pe.o: mmpriv.h minimap.h bseq.h kseq.h kvec.h kalloc.h ksort.h
 sdust.o: kalloc.h kdq.h kvec.h sdust.h
-sketch.o: kvec.h kalloc.h minimap.h
+seed.o: mmpriv.h minimap.h bseq.h kseq.h kalloc.h ksort.h
+sketch.o: kvec.h kalloc.h mmpriv.h minimap.h bseq.h kseq.h
+splitidx.o: mmpriv.h minimap.h bseq.h kseq.h

Makefile.simde

@@ -0,0 +1,97 @@
+CFLAGS=		-g -Wall -O2 -Wc++-compat #-Wextra
+CPPFLAGS=	-DHAVE_KALLOC -DUSE_SIMDE -DSIMDE_ENABLE_NATIVE_ALIASES
+INCLUDES=	-Ilib/simde
+OBJS=		kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o options.o index.o lchain.o align.o hit.o map.o format.o pe.o seed.o esterr.o splitidx.o \
+			ksw2_extz2_simde.o ksw2_extd2_simde.o ksw2_exts2_simde.o ksw2_ll_simde.o
+PROG=		minimap2
+PROG_EXTRA=	sdust minimap2-lite
+LIBS=		-lm -lz -lpthread
+
+
+ifneq ($(arm_neon),) # if arm_neon is defined
+ifeq ($(aarch64),)   #if aarch64 is not defined
+	CFLAGS+=-D_FILE_OFFSET_BITS=64 -mfpu=neon -fsigned-char
+else                 #if aarch64 is defined
+	CFLAGS+=-D_FILE_OFFSET_BITS=64 -fsigned-char
+endif
+endif
+
+ifneq ($(asan),)
+	CFLAGS+=-fsanitize=address
+	LIBS+=-fsanitize=address
+endif
+
+ifneq ($(tsan),)
+	CFLAGS+=-fsanitize=thread
+	LIBS+=-fsanitize=thread
+endif
+
+.PHONY:all extra clean depend
+.SUFFIXES:.c .o
+
+.c.o:
+		$(CC) -c $(CFLAGS) $(CPPFLAGS) $(INCLUDES) $< -o $@
+
+all:$(PROG)
+
+extra:all $(PROG_EXTRA)
+
+minimap2:main.o libminimap2.a
+		$(CC) $(CFLAGS) main.o -o $@ -L. -lminimap2 $(LIBS)
+
+minimap2-lite:example.o libminimap2.a
+		$(CC) $(CFLAGS) $< -o $@ -L. -lminimap2 $(LIBS)
+
+libminimap2.a:$(OBJS)
+		$(AR) -csru $@ $(OBJS)
+
+sdust:sdust.c kalloc.o kalloc.h kdq.h kvec.h kseq.h ketopt.h sdust.h
+		$(CC) -D_SDUST_MAIN $(CFLAGS) $< kalloc.o -o $@ -lz
+
+ksw2_ll_simde.o:ksw2_ll_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse2 $(CPPFLAGS) $(INCLUDES) $< -o $@
+
+ksw2_extz2_simde.o:ksw2_extz2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) $(INCLUDES) $< -o $@
+
+ksw2_extd2_simde.o:ksw2_extd2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) $(INCLUDES) $< -o $@
+
+ksw2_exts2_simde.o:ksw2_exts2_sse.c ksw2.h kalloc.h
+		$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) $(INCLUDES) $< -o $@
+
+# other non-file targets
+
+clean:
+		rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM build dist mappy*.so mappy.c python/mappy.c mappy.egg*
+
+depend:
+		(LC_ALL=C; export LC_ALL; makedepend -Y -- $(CFLAGS) $(CPPFLAGS) -- *.c)
+
+# DO NOT DELETE
+
+align.o: minimap.h mmpriv.h bseq.h kseq.h ksw2.h kalloc.h
+bseq.o: bseq.h kvec.h kalloc.h kseq.h
+chain.o: minimap.h mmpriv.h bseq.h kseq.h kalloc.h
+esterr.o: mmpriv.h minimap.h bseq.h kseq.h
+example.o: minimap.h kseq.h
+format.o: kalloc.h mmpriv.h minimap.h bseq.h kseq.h
+hit.o: mmpriv.h minimap.h bseq.h kseq.h kalloc.h khash.h
+index.o: kthread.h bseq.h minimap.h mmpriv.h kseq.h kvec.h kalloc.h khash.h
+index.o: ksort.h
+kalloc.o: kalloc.h
+ksw2_extd2_sse.o: ksw2.h kalloc.h
+ksw2_exts2_sse.o: ksw2.h kalloc.h
+ksw2_extz2_sse.o: ksw2.h kalloc.h
+ksw2_ll_sse.o: ksw2.h kalloc.h
+kthread.o: kthread.h
+main.o: bseq.h minimap.h mmpriv.h kseq.h ketopt.h
+map.o: kthread.h kvec.h kalloc.h sdust.h mmpriv.h minimap.h bseq.h kseq.h
+map.o: khash.h ksort.h
+misc.o: mmpriv.h minimap.h bseq.h kseq.h ksort.h
+options.o: mmpriv.h minimap.h bseq.h kseq.h
+pe.o: mmpriv.h minimap.h bseq.h kseq.h kvec.h kalloc.h ksort.h
+sdust.o: kalloc.h kdq.h kvec.h sdust.h
+self-chain.o: minimap.h kseq.h
+sketch.o: kvec.h kalloc.h mmpriv.h minimap.h bseq.h kseq.h
+splitidx.o: mmpriv.h minimap.h bseq.h kseq.h

NEWS.md

@@ -0,0 +1,821 @@
+Release 2.24-r1122 (26 December 2021)
+-------------------------------------
+
+This release improves alignment around long poorly aligned regions. Older
+minimap2 may chain through such regions in rare cases which may result in
+missing alignments later. The issue has become worse since the the change of
+the chaining algorithm in v2.19. v2.23 implements an incomplete remedy. This
+release provides a better solution with a X-drop-like heuristic and by enabling
+two-bandwidth chaining in the assembly mode.
+
+(2.24: 26 December 2021, r1122)
+
+
+
+Release 2.23-r1111 (18 November 2021)
+-------------------------------------
+
+Notable changes:
+
+ * Bugfix: fixed missing alignments around long inversions (#806 and #816).
+   This bug affected v2.19 through v2.22.
+
+ * Improvement: avoid extremely long mapping time for pathologic reads with
+   highly repeated k-mers not in the reference (#771). Use --q-occ-frac=0
+   to disable the new heuristic.
+
+ * Change: use --cap-kalloc=1g by default.
+
+(2.23: 18 November 2021, r1111)
+
+
+
+Release 2.22-r1101 (7 August 2021)
+----------------------------------
+
+When choosing the best alignment, this release uses logarithm gap penalty and
+query-specific mismatch penalty. It improves the sensitivity to long INDELs in
+repetitive regions.
+
+Other notable changes:
+
+ * Bugfix: fixed an indirect memory leak that may waste a large amount of
+   memory given highly repetitive reference such as a 16S RNA database (#749).
+   All versions of minimap2 have this issue.
+
+ * New feature: added --cap-kalloc to reduce the peak memory. This option is
+   not enabled by default but may become the default in future releases.
+
+Known issue:
+
+ * Minimap2 may take a long time to map a read (#771). So far it is not clear
+   if this happens to v2.18 and earlier versions.
+
+(2.22: 7 August 2021, r1101)
+
+
+
+Release 2.21-r1071 (6 July 2021)
+--------------------------------
+
+This release fixed a regression in short-read mapping introduced in v2.19
+(#776). It also fixed invalid comparisons of uninitialized variables, though
+these are harmless (#752). Long-read alignment should be identical to v2.20.
+
+(2.21: 6 July 2021, r1071)
+
+
+
+Release 2.20-r1061 (27 May 2021)
+--------------------------------
+
+This release fixed a bug in the Python module and improves the command-line
+compatibiliity with v2.18. In v2.19, if `-r` is specified with an `asm*` preset,
+users would get alignments more fragmented than v2.18. This could be an issue
+for existing pipelines specifying `-r`. This release resolves this issue.
+
+(2.20: 27 May 2021, r1061)
+
+
+
+Release 2.19-r1057 (26 May 2021)
+--------------------------------
+
+This release includes a few important improvements backported from unimap:
+
+ * Improvement: more contiguous alignment through long INDELs. This is enabled
+   by the minigraph chaining algorithm. All `asm*` presets now use the new
+   algorithm. They can find INDELs up to 100kb and may be faster for
+   chromosome-long contigs. The default mode and `map*` presets use this
+   algorithm to replace the long-join heuristic.
+
+ * Improvement: better alignment in highly repetitive regions by rescuing
+   high-occurrence seeds. If the distance between two adjacent seeds is too
+   large, attempt to choose a fraction of high-occurrence seeds in-between.
+   Minimap2 now produces fewer clippings and alignment break points in long
+   satellite regions.
+
+ * Improvement: allow to specify an interval of k-mer occurrences with `-U`.
+   For repeat-rich genomes, the automatic k-mer occurrence threshold determined
+   by `-f` may be too large and makes alignment impractically slow. The new
+   option protects against such cases. Enabled for `asm*` and `map-hifi`.
+
+ * New feature: added the `map-hifi` preset for maping PacBio High-Fidelity
+   (HiFi) reads.
+
+ * Change to the default: apply `--cap-sw-mem=100m` for genomic alignment.
+
+ * Bugfix: minimap2 could not generate an index file with `-xsr` (#734).
+
+This release represents the most signficant algorithmic change since v2.1 in
+2017. With features backported from unimap, minimap2 now has similar power to
+unimap for contig alignment. Unimap will remain an experimental project and is
+no longer recommended over minimap2. Sorry for reverting the recommendation in
+short time.
+
+(2.19: 26 May 2021, r1057)
+
+
+
+Release 2.18-r1015 (9 April 2021)
+---------------------------------
+
+This release fixes multiple rare bugs in minimap2 and adds additional
+functionality to paftools.js.
+
+Changes to minimap2:
+
+ * Bugfix: a rare segfault caused by an off-by-one error (#489)
+
+ * Bugfix: minimap2 segfaulted due to an uninitilized variable (#622 and #625).
+
+ * Bugfix: minimap2 parsed spaces as field separators in BED (#721). This led
+   to issues when the BED name column contains spaces.
+
+ * Bugfix: minimap2 `--split-prefix` did not work with long reference names
+   (#394).
+
+ * Bugfix: option `--junc-bonus` didn't work (#513)
+
+ * Bugfix: minimap2 didn't return 1 on I/O errors (#532)
+
+ * Bugfix: the `de:f` tag (sequence divergence) could be negative if there were
+   ambiguous bases
+
+ * Bugfix: fixed two undefined behaviors caused by calling memcpy() on
+   zero-length blocks (#443)
+
+ * Bugfix: there were duplicated SAM @SQ lines if option `--split-prefix` is in
+   use (#400 and #527)
+
+ * Bugfix: option -K had to be smaller than 2 billion (#491). This was caused
+   by a 32-bit integer overflow.
+
+ * Improvement: optionally compile against SIMDe (#597). Minimap2 should work
+   with IBM POWER CPUs, though this has not been tested. To compile with SIMDe,
+   please use `make -f Makefile.simde`.
+
+ * Improvement: more informative error message for I/O errors (#454) and for
+   FASTQ parsing errors (#510)
+
+ * Improvement: abort given malformatted RG line (#541)
+
+ * Improvement: better formula to estimate the `dv:f` tag (approximate sequence
+   divergence). See DOI:10.1101/2021.01.15.426881.
+
+ * New feature: added the `--mask-len` option to fine control the removal of
+   redundant hits (#659). The default behavior is unchanged.
+
+Changes to mappy:
+
+ * Bugfix: mappy caused segmentation fault if the reference index is not
+   present (#413).
+
+ * Bugfix: fixed a memory leak via 238b6bb3
+
+ * Change: always require Cython to compile the mappy module (#723). Older
+   mappy packages at PyPI bundled the C source code generated by Cython such
+   that end users did not need to install Cython to compile mappy. However, as
+   Python 3.9 is breaking backward compatibility, older mappy does not work
+   with Python 3.9 anymore. We have to add this Cython dependency as a
+   workaround.
+
+Changes to paftools.js:
+
+ * Bugfix: the "part10-" line from asmgene was wrong (#581)
+
+ * Improvement: compatibility with GTF files from GenBank (#422)
+
+ * New feature: asmgene also checks missing multi-copy genes
+
+ * New feature: added the misjoin command to evaluate large-scale misjoins and
+   megabase-long inversions.
+
+Although given the many bug fixes and minor improvements, the core algorithm
+stays the same. This version of minimap2 produces nearly identical alignments
+to v2.17 except very rare corner cases.
+
+Now unimap is recommended over minimap2 for aligning long contigs against a
+reference genome. It often takes less wall-clock time and is much more
+sensitive to long insertions and deletions.
+
+(2.18: 9 April 2021, r1015)
+
+
+
+Release 2.17-r941 (4 May 2019)
+------------------------------
+
+Changes since the last release:
+
+ * Fixed flawed CIGARs like `5I6D7I` (#392).
+
+ * Bugfix: TLEN should be 0 when either end is unmapped (#373 and #365).
+
+ * Bugfix: mappy is unable to write index (#372).
+
+ * Added option `--junc-bed` to load known gene annotations in the BED12
+   format. Minimap2 prefers annotated junctions over novel junctions (#197 and
+   #348). GTF can be converted to BED12 with `paftools.js gff2bed`.
+
+ * Added option `--sam-hit-only` to suppress unmapped hits in SAM (#377).
+
+ * Added preset `splice:hq` for high-quality CCS or mRNA sequences. It applies
+   better scoring and improves the sensitivity to small exons. This preset may
+   introduce false small introns, but the overall accuracy should be higher.
+
+This version produces nearly identical alignments to v2.16, except for CIGARs
+affected by the bug mentioned above.
+
+(2.17: 5 May 2019, r941)
+
+
+
+Release 2.16-r922 (28 February 2019)
+------------------------------------
+
+This release is 50% faster for mapping ultra-long nanopore reads at comparable
+accuracy. For short-read mapping, long-read overlapping and ordinary long-read
+mapping, the performance and accuracy remain similar. This speedup is achieved
+with a new heuristic to limit the number of chaining iterations (#324). Users
+can disable the heuristic by increasing a new option `--max-chain-iter` to a
+huge number.
+
+Other changes to minimap2:
+
+ * Implemented option `--paf-no-hit` to output unmapped query sequences in PAF.
+   The strand and reference name columns are both `*` at an unmapped line. The
+   hidden option is available in earlier minimap2 but had a different 2-column
+   output format instead of PAF.
+
+ * Fixed a bug that leads to wrongly calculated `de` tags when ambiguous bases
+   are involved (#309). This bug only affects v2.15.
+
+ * Fixed a bug when parsing command-line option `--splice` (#344). This bug was
+   introduced in v2.13.
+
+ * Fixed two division-by-zero cases (#326). They don't affect final alignments
+   because the results of the divisions are not used in both case.
+
+ * Added an option `-o` to output alignments to a specified file. It is still
+   recommended to use UNIX pipes for on-the-fly conversion or compression.
+
+ * Output a new `rl` tag to give the length of query regions harboring
+   repetitive seeds.
+
+Changes to paftool.js:
+
+ * Added a new option to convert the MD tag to the long form of the cs tag.
+
+Changes to mappy:
+
+ * Added the `mappy.Aligner.seq_names` method to return sequence names (#312).
+
+For NA12878 ultra-long reads, this release changes the alignments of <0.1% of
+reads in comparison to v2.15. All these reads have highly fragmented alignments
+and are likely to be problematic anyway. For shorter or well aligned reads,
+this release should produce mostly identical alignments to v2.15.
+
+(2.16: 28 February 2019, r922)
+
+
+
+Release 2.15-r905 (10 January 2019)
+-----------------------------------
+
+Changes to minimap2:
+
+ * Fixed a rare segmentation fault when option -H is in use (#307). This may
+   happen when there are very long homopolymers towards the 5'-end of a read.
+
+ * Fixed wrong CIGARs when option --eqx is used (#266).
+
+ * Fixed a typo in the base encoding table (#264). This should have no
+   practical effect.
+
+ * Fixed a typo in the example code (#265).
+
+ * Improved the C++ compatibility by removing "register" (#261). However,
+   minimap2 still can't be compiled in the pedantic C++ mode (#306).
+
+ * Output a new "de" tag for gap-compressed sequence divergence.
+
+Changes to paftools.js:
+
+ * Added "asmgene" to evaluate the completeness of an assembly by measuring the
+   uniquely mapped single-copy genes. This command learns the idea of BUSCO.
+
+ * Added "vcfpair" to call a phased VCF from phased whole-genome assemblies. An
+   earlier version of this script is used to produce the ground truth for the
+   syndip benchmark [PMID:30013044].
+
+This release produces identical alignment coordinates and CIGARs in comparison
+to v2.14. Users are advised to upgrade due to the several bug fixes.
+
+(2.15: 10 Janurary 2019, r905)
+
+
+
+Release 2.14-r883 (5 November 2018)
+-----------------------------------
+
+Notable changes:
+
+ * Fixed two minor bugs caused by typos (#254 and #266).
+
+ * Fixed a bug that made minimap2 abort when --eqx was used together with --MD
+   or --cs (#257).
+
+ * Added --cap-sw-mem to cap the size of DP matrices (#259). Base alignment may
+   take a lot of memory in the splicing mode. This may lead to issues when we
+   run minimap2 on a cluster with a hard memory limit. The new option avoids
+   unlimited memory usage at the cost of missing a few long introns.
+
+ * Conforming to C99 and C11 when possible (#261).
+
+ * Warn about malformatted FASTA or FASTQ (#252 and #255).
+
+This release occasionally produces base alignments different from v2.13. The
+overall alignment accuracy remain similar.
+
+(2.14: 5 November 2018, r883)
+
+
+
+Release 2.13-r850 (11 October 2018)
+-----------------------------------
+
+Changes to minimap2:
+
+ * Fixed wrongly formatted SAM when -L is in use (#231 and #233).
+
+ * Fixed an integer overflow in rare cases.
+
+ * Added --hard-mask-level to fine control split alignments (#244).
+
+ * Made --MD work with spliced alignment (#139).
+
+ * Replaced musl's getopt with ketopt for portability.
+
+ * Log peak memory usage on exit.
+
+This release should produce alignments identical to v2.12 and v2.11.
+
+(2.13: 11 October 2018, r850)
+
+
+
+Release 2.12-r827 (6 August 2018)
+---------------------------------
+
+Changes to minimap2:
+
+ * Added option --split-prefix to write proper alignments (correct mapping
+   quality and clustered query sequences) given a multi-part index (#141 and
+   #189; mostly by @hasindu2008).
+
+ * Fixed a memory leak when option -y is in use.
+
+Changes to mappy:
+
+ * Support the MD/cs tag (#183 and #203).
+
+ * Allow mappy to index a single sequence, to add extra flags and to change the
+   scoring system.
+
+Minimap2 should produce alignments identical to v2.11.
+
+(2.12: 6 August 2018, r827)
+
+
+
+Release 2.11-r797 (20 June 2018)
+--------------------------------
+
+Changes to minimap2:
+
+ * Improved alignment accuracy in low-complexity regions for SV calling. Thank
+   @armintoepfer for multiple offline examples.
+
+ * Added option --eqx to encode sequence match/mismatch with the =/X CIGAR
+   operators (#156, #157 and #175).
+
+ * When compiled with VC++, minimap2 generated wrong alignments due to a
+   comparison between a signed integer and an unsigned integer (#184). Also
+   fixed warnings reported by "clang -Wextra".
+
+ * Fixed incorrect anchor filtering due to a missing 64- to 32-bit cast.
+
+ * Fixed incorrect mapping quality for inversions (#148).
+
+ * Fixed incorrect alignment involving ambiguous bases (#155).
+
+ * Fixed incorrect presets: option `-r 2000` is intended to be used with
+   ava-ont, not ava-pb. The bug was introduced in 2.10.
+
+ * Fixed a bug when --for-only/--rev-only is used together with --sr or
+   --heap-sort=yes (#166).
+
+ * Fixed option -Y that was not working in the previous releases.
+
+ * Added option --lj-min-ratio to fine control the alignment of long gaps
+   found by the "long-join" heuristic (#128).
+
+ * Exposed `mm_idx_is_idx`, `mm_idx_load` and `mm_idx_dump` C APIs (#177).
+   Also fixed a bug when indexing without reference names (this feature is not
+   exposed to the command line).
+
+Changes to mappy:
+
+ * Added `__version__` (#165).
+
+ * Exposed the maximum fragment length parameter to mappy (#174).
+
+Changes to paftools:
+
+ * Don't crash when there is no "cg" tag (#153).
+
+ * Fixed wrong coverage report by "paftools.js call" (#145).
+
+This version may produce slightly different base-level alignment. The overall
+alignment statistics should remain similar.
+
+(2.11: 20 June 2018, r797)
+
+
+
+Release 2.10-r761 (27 March 2018)
+---------------------------------
+
+Changes to minimap2:
+
+ * Optionally output the MD tag for compatibility with existing tools (#63,
+   #118 and #137).
+
+ * Use SSE compiler flags more precisely to prevent compiling errors on certain
+   machines (#127).
+
+ * Added option --min-occ-floor to set a minimum occurrence threshold. Presets
+   intended for assembly-to-reference alignment set this option to 100. This
+   option alleviates issues with regions having high copy numbers (#107).
+
+ * Exit with non-zero code on file writing errors (e.g. disk full; #103 and
+   #132).
+
+ * Added option -y to copy FASTA/FASTQ comments in query sequences to the
+   output (#136).
+
+ * Added the asm20 preset for alignments between genomes at 5-10% sequence
+   divergence.
+
+ * Changed the band-width in the ava-ont preset from 500 to 2000. Oxford
+   Nanopore reads may contain long deletion sequencing errors that break
+   chaining.
+
+Changes to mappy, the Python binding:
+
+ * Fixed a typo in Align.seq() (#126).
+
+Changes to paftools.js, the companion script:
+
+ * Command sam2paf now converts the MD tag to cs.
+
+ * Support VCF output for assembly-to-reference variant calling (#109).
+
+This version should produce identical alignment for read overlapping, RNA-seq
+read mapping, and genomic read mapping. We have also added a cook book to show
+the variety uses of minimap2 on real datasets. Please see cookbook.md in the
+minimap2 source code directory.
+
+(2.10: 27 March 2017, r761)
+
+
+
+Release 2.9-r720 (23 February 2018)
+-----------------------------------
+
+This release fixed multiple minor bugs.
+
+* Fixed two bugs that lead to incorrect inversion alignment. Also improved the
+  sensitivity to small inversions by using double Z-drop cutoff (#112).
+
+* Fixed an issue that may cause the end of a query sequence unmapped (#104).
+
+* Added a mappy API to retrieve sequences from the index (#126) and to reverse
+  complement DNA sequences. Fixed a bug where the `best_n` parameter did not
+  work (#117).
+
+* Avoided segmentation fault given incorrect FASTQ input (#111).
+
+* Combined all auxiliary javascripts to paftools.js. Fixed several bugs in
+  these scripts at the same time.
+
+(2.9: 24 February 2018, r720)
+
+
+
+Release 2.8-r672 (1 February 2018)
+----------------------------------
+
+Notable changes in this release include:
+
+ * Speed up short-read alignment by ~10%. The overall mapping accuracy stays
+   the same, but the output alignments are not always identical to v2.7 due to
+   unstable sorting employed during chaining. Long-read alignment is not
+   affected by this change as the speedup is short-read specific.
+
+ * Mappy now supports paired-end short-read alignment (#87). Please see
+   python/README.rst for details.
+
+ * Added option --for-only and --rev-only to perform alignment against the
+   forward or the reverse strand of the reference genome only (#91).
+
+ * Alleviated the issue with undesired diagonal alignment in the self mapping
+   mode (#10). Even if the output is not ideal, it should not interfere with
+   other alignments. Fully resolving the issue is intricate and may require
+   additional heuristic thresholds.
+
+ * Enhanced error checking against incorrect input (#92 and #96).
+
+For long query sequences, minimap2 should output identical alignments to v2.7.
+
+(2.8: 1 February 2018, r672)
+
+
+
+Release 2.7-r654 (9 January 2018)
+---------------------------------
+
+This release fixed a bug in the splice mode and added a few minor features:
+
+ * Fixed a bug that occasionally takes an intron as a long deletion in the
+   splice mode. This was caused by wrong backtracking at the last CIGAR
+   operator. The current fix eliminates the error, but it is not optimal in
+   that it often produces a wrong junction when the last operator is an intron.
+   A future version of minimap2 may improve upon this.
+
+ * Support high-end ARM CPUs that implement the NEON instruction set (#81).
+   This enables minimap2 to work on Raspberry Pi 3 and Odroid XU4.
+
+ * Added a C API to construct a minimizer index from a set of C strings (#80).
+
+ * Check scoring specified on the command line (#79). Due to the 8-bit limit,
+   excessively large score penalties fail minimap2.
+
+For genomic sequences, minimap2 should give identical alignments to v2.6.
+
+(2.7: 9 January 2018, r654)
+
+
+
+Release 2.6-r623 (12 December 2017)
+-----------------------------------
+
+This release adds several features and fixes two minor bugs:
+
+ * Optionally build an index without sequences. This helps to reduce the
+   peak memory for read overlapping and is automatically applied when
+   base-level alignment is not requested.
+
+ * Approximately estimate per-base sequence divergence (i.e. 1-identity)
+   without performing base-level alignment, using a MashMap-like method. The
+   estimate is written to a new dv:f tag.
+
+ * Reduced the number of tiny terminal exons in RNA-seq alignment. The current
+   setting is conservative. Increase --end-seed-pen to drop more such exons.
+
+ * Reduced the peak memory when aligning long query sequences.
+
+ * Fixed a bug that is caused by HPC minimizers longer than 256bp. This should
+   have no effect in practice, but it is recommended to rebuild HPC indices if
+   possible.
+
+ * Fixed a bug when identifying identical hits (#71). This should only affect
+   artifactual reference consisting of near identical sequences.
+
+For genomic sequences, minimap2 should give nearly identical alignments to
+v2.5, except the new dv:f tag.
+
+(2.6: 12 December 2017, r623)
+
+
+
+Release 2.5-r572 (11 November 2017)
+-----------------------------------
+
+This release fixes several bugs and brings a couple of minor improvements:
+
+ * Fixed a severe bug that leads to incorrect mapping coordinates in rare
+   corner cases.
+
+ * Fixed underestimated mapping quality for chimeric alignments when the whole
+   query sequence contain many repetitive minimizers, and for chimeric
+   alignments caused by Z-drop.
+
+ * Fixed two bugs in Python binding: incorrect strand field (#57) and incorrect
+   sequence names for Python3 (#55).
+
+ * Improved mapping accuracy for highly overlapping paired ends.
+
+ * Added option -Y to use soft clipping for supplementary alignments (#56).
+
+(2.5: 11 November 2017, r572)
+
+
+
+Release 2.4-r555 (6 November 2017)
+----------------------------------
+
+As is planned, this release focuses on fine tuning the base algorithm. Notable
+changes include
+
+ * Changed the mapping quality scale to match the scale of BWA-MEM. This makes
+   minimap2 and BWA-MEM achieve similar sensitivity-specificity balance on real
+   short-read data.
+
+ * Improved the accuracy of splice alignment by modeling one additional base
+   close to the GT-AG signal. This model is used by default with `-x splice`.
+   For SIRV control data, however, it is recommended to add `--splice-flank=no`
+   to disable this feature as the SIRV splice signals are slightly different.
+
+ * Tuned the parameters for Nanopore Direct RNA reads. The recommended command
+   line is `-axsplice -k14 -uf` (#46).
+
+ * Fixed a segmentation fault when aligning PacBio reads (#47 and #48). This
+   bug is very rare but it affects all versions of minimap2. It is also
+   recommended to re-index reference genomes created with `map-pb`. For human,
+   two minimizers in an old index are wrong.
+
+ * Changed option `-L` in sync with the final decision of hts-specs: a fake
+   CIGAR takes the form of `<readLen>S<refLen>N`. Note that `-L` only enables
+   future tools to recognize long CIGARs. It is not possible for older tools to
+   work with such alignments in BAM (#43 and #51).
+
+ * Fixed a tiny issue whereby minimap2 may waste 8 bytes per candidate
+   alignment.
+
+The minimap2 technical note hosted at arXiv has also been updated to reflect
+recent changes.
+
+(2.4: 6 November 2017, r555)
+
+
+
+Release 2.3-r531 (22 October 2017)
+----------------------------------
+
+This release come with many improvements and bug fixes:
+
+ * The **sr** preset now supports paired-end short-read alignment. Minimap2 is
+   3-4 times as fast as BWA-MEM, but is slightly less accurate on simulated
+   reads.
+
+ * Meticulous improvements to assembly-to-assembly alignment (special thanks to
+   Alexey Gurevich from the QUAST team): a) apply a small penalty to matches
+   between ambiguous bases; b) reduce missing alignments due to spurious
+   overlaps; c) introduce the short form of the `cs` tag, an improvement to the
+   SAM MD tag.
+
+ * Make sure gaps are always left-aligned.
+
+ * Recognize `U` bases from Oxford Nanopore Direct RNA-seq (#33).
+
+ * Fixed slightly wrong chaining score. Fixed slightly inaccurate coordinates
+   for split alignment.
+
+ * Fixed multiple reported bugs: 1) wrong reference name for inversion
+   alignment (#30); 2) redundant SQ lines when multiple query files are
+   specified (#39); 3) non-functioning option `-K` (#36).
+
+This release has implemented all the major features I planned five months ago,
+with the addition of spliced long-read alignment. The next couple of releases
+will focus on fine tuning of the base algorithms.
+
+(2.3: 22 October 2017, r531)
+
+
+
+Release 2.2-r409 (17 September 2017)
+------------------------------------
+
+This is a feature release. It improves single-end short-read alignment and
+comes with Python bindings. Detailed changes include:
+
+ * Added the **sr** preset for single-end short-read alignment. In this mode,
+   minimap2 runs faster than BWA-MEM, but is slightly less accurate on
+   simulated data sets. Paired-end alignment is not supported as of now.
+
+ * Improved mapping quality estimate with more accurate identification of
+   repetitive hits. This mainly helps short-read alignment.
+
+ * Implemented **mappy**, a Python binding for minimap2, which is available
+   from PyPI and can be installed with `pip install --user mappy`. Python users
+   can perform read alignment without the minimap2 executable.
+
+ * Restructured the indexing APIs and documented key minimap2 APIs in the
+   header file minimap.h. Updated example.c with the new APIs. Old APIs still
+   work but may become deprecated in future.
+
+This release may output alignments different from the previous version, though
+the overall alignment statistics, such as the number of aligned bases and long
+gaps, remain close.
+
+(2.2: 17 September 2017, r409)
+
+
+
+Release 2.1.1-r341 (6 September 2017)
+-------------------------------------
+
+This is a maintenance release that is expected to output identical alignment to
+v2.1. Detailed changes include:
+
+ * Support CPU dispatch. By default, minimap2 is compiled with both SSE2 and
+   SSE4 based implementation of alignment and automatically chooses the right
+   one at runtime. This avoids unexpected errors on older CPUs (#21).
+
+ * Improved Windows support as is requested by Oxford Nanopore (#19). Minimap2
+   now avoids variable-length stacked arrays, eliminates alloca(), ships with
+   getopt_long() and provides timing functions implemented with Windows APIs.
+
+ * Fixed a potential segmentation fault when specifying -k/-w/-H with
+   multi-part index (#23).
+
+ * Fixed two memory leaks in example.c
+
+(2.1.1: 6 September 2017, r341)
+
+
+
+Release 2.1-r311 (25 August 2017)
+---------------------------------
+
+This release adds spliced alignment for long noisy RNA-seq reads. On a SMRT
+Iso-Seq and a Oxford Nanopore data sets, minimap2 appears to outperform
+traditional mRNA aligners. For DNA alignment, this release gives almost
+identical output to v2.0. Other changes include:
+
+ * Added option `-R` to set the read group header line in SAM.
+
+ * Optionally output the `cs:Z` tag in PAF to encode both the query and the
+   reference sequences in the alignment.
+
+ * Fixed an issue where DP alignment uses excessive memory.
+
+The minimap2 technical report has been updated with more details and the
+evaluation of spliced alignment:
+
+ * Li, H. (2017). Minimap2: fast pairwise alignment for long nucleotide
+   sequences. [arXiv:1708.01492v2](https://arxiv.org/abs/1708.01492v2).
+
+(2.1: 25 August 2017, r311)
+
+
+
+Release 2.0-r275 (8 August 2017)
+--------------------------------
+
+This release is identical to version 2.0rc1, except the version number. It is
+described and evaluated in the following technical report:
+
+ * Li, H. (2017). Minimap2: fast pairwise alignment for long DNA sequences.
+   [arXiv:1708.01492v1](https://arxiv.org/abs/1708.01492v1).
+
+(2.0: 8 August 2017, r275)
+
+
+
+Release 2.0rc1-r232 (30 July 2017)
+----------------------------------
+
+This release improves the accuracy of long-read alignment and added several
+minor features.
+
+ * Improved mapping quality estimate for short alignments containing few seed
+   hits.
+
+ * Fixed a minor bug that affects the chaining accuracy towards the ends of a
+   chain. Changed the gap cost for chaining to reduce false seeding.
+
+ * Skip potentially wrong seeding and apply dynamic programming more frequently.
+   This slightly increases run time, but greatly reduces false long gaps.
+
+ * Perform local alignment at Z-drop break point to recover potential inversion
+   alignment. Output the SA tag in the SAM format. Added scripts to evaluate
+   mapping accuracy for reads simulated with pbsim.
+
+This release completes features intended for v2.0. No major features will be
+added to the master branch before the final v2.0.
+
+(2.0rc1: 30 July 2017, r232)
+
+
+
+Release r191 (19 July 2017)
+---------------------------
+
+This is the first public release of minimap2, an aligner for long reads and
+assemblies. This release has a few issues and is generally not recommended for
+production uses.
+
+(19 July 2017, r191)

README.md

@@ -1,62 +1,385 @@
-## Getting Started
+[![GitHub Downloads](https://img.shields.io/github/downloads/lh3/minimap2/total.svg?style=social&logo=github&label=Download)](https://github.com/lh3/minimap2/releases)
+[![BioConda Install](https://img.shields.io/conda/dn/bioconda/minimap2.svg?style=flag&label=BioConda%20install)](https://anaconda.org/bioconda/minimap2)
+[![PyPI](https://img.shields.io/pypi/v/mappy.svg?style=flat)](https://pypi.python.org/pypi/mappy)
+[![Build Status](https://github.com/lh3/minimap2/actions/workflows/ci.yaml/badge.svg)](https://github.com/lh3/minimap2/actions)
+## <a name="started"></a>Getting Started
 ```sh
 git clone https://github.com/lh3/minimap2
 cd minimap2 && make
-# long reads against a reference genome
+# long sequences against a reference genome
-./minimap2 -ax map10k test/MT-human.fa test/MT-orang.fa > test.sam
+./minimap2 -a test/MT-human.fa test/MT-orang.fa > test.sam
 # create an index first and then map
-./minimap2 -x map10k -d MT-human.mmi test/MT-human.fa
+./minimap2 -x map-ont -d MT-human-ont.mmi test/MT-human.fa
-./minimap2 -ax map10k MT-human.mmi test/MT-orang.fa > test.sam
+./minimap2 -a MT-human-ont.mmi test/MT-orang.fa > test.sam
-# long-read overlap (no test data)
+# use presets (no test data)
-./minimap2 -x ava-pb your-reads.fa your-reads.fa > overlaps.paf
+./minimap2 -ax map-pb ref.fa pacbio.fq.gz > aln.sam       # PacBio CLR genomic reads
-# man page
+./minimap2 -ax map-ont ref.fa ont.fq.gz > aln.sam         # Oxford Nanopore genomic reads
+./minimap2 -ax map-hifi ref.fa pacbio-ccs.fq.gz > aln.sam # PacBio HiFi/CCS genomic reads (v2.19 or later)
+./minimap2 -ax asm20 ref.fa pacbio-ccs.fq.gz > aln.sam    # PacBio HiFi/CCS genomic reads (v2.18 or earlier)
+./minimap2 -ax sr ref.fa read1.fa read2.fa > aln.sam      # short genomic paired-end reads
+./minimap2 -ax splice ref.fa rna-reads.fa > aln.sam       # spliced long reads (strand unknown)
+./minimap2 -ax splice -uf -k14 ref.fa reads.fa > aln.sam  # noisy Nanopore Direct RNA-seq
+./minimap2 -ax splice:hq -uf ref.fa query.fa > aln.sam    # Final PacBio Iso-seq or traditional cDNA
+./minimap2 -ax splice --junc-bed anno.bed12 ref.fa query.fa > aln.sam  # prioritize on annotated junctions
+./minimap2 -cx asm5 asm1.fa asm2.fa > aln.paf             # intra-species asm-to-asm alignment
+./minimap2 -x ava-pb reads.fa reads.fa > overlaps.paf     # PacBio read overlap
+./minimap2 -x ava-ont reads.fa reads.fa > overlaps.paf    # Nanopore read overlap
+# man page for detailed command line options
 man ./minimap2.1
 ```
 
-## Introduction
+## Table of Contents
 
-Minimap2 is a fast sequence mapping and alignment program that can find
+- [Getting Started](#started)
-overlaps between long noisy reads, or map long reads or their assemblies to a
+- [Users' Guide](#uguide)
-reference genome optionally with detailed alignment (i.e. CIGAR). At present,
+  - [Installation](#install)
-it works efficiently with query sequences from a few kilobases to ~100
+  - [General usage](#general)
-megabases in length at an error rate ~15%. Minimap2 outputs in the [PAF][paf] or
+  - [Use cases](#cases)
-the [SAM format][sam]. On limited test data sets, minimap2 is over 20 times
+    - [Map long noisy genomic reads](#map-long-genomic)
-faster than most other long-read aligners. It will replace BWA-MEM for long
+    - [Map long mRNA/cDNA reads](#map-long-splice)
-reads and contig alignment.
+    - [Find overlaps between long reads](#long-overlap)
+    - [Map short accurate genomic reads](#short-genomic)
+    - [Full genome/assembly alignment](#full-genome)
+  - [Advanced features](#advanced)
+    - [Working with >65535 CIGAR operations](#long-cigar)
+    - [The cs optional tag](#cs)
+    - [Working with the PAF format](#paftools)
+  - [Algorithm overview](#algo)
+  - [Getting help](#help)
+  - [Citing minimap2](#cite)
+- [Developers' Guide](#dguide)
+- [Limitations](#limit)
 
-Minimap2 is the successor of [minimap][minimap]. It uses a similar
+## <a name="uguide"></a>Users' Guide
-minimizer-based indexing and seeding algorithm, and improves the original
-minimap with homopolyer-compressed k-mers (see also [SMARTdenovo][smartdenovo]
-and [longISLND][longislnd]), better chaining and the ability to produce CIGAR
-with fast extension alignment (see also [libgaba][gaba] and [ksw2][ksw2]) and
-piece-wise affine gap cost.
 
-## Installation
+Minimap2 is a versatile sequence alignment program that aligns DNA or mRNA
+sequences against a large reference database. Typical use cases include: (1)
+mapping PacBio or Oxford Nanopore genomic reads to the human genome; (2)
+finding overlaps between long reads with error rate up to ~15%; (3)
+splice-aware alignment of PacBio Iso-Seq or Nanopore cDNA or Direct RNA reads
+against a reference genome; (4) aligning Illumina single- or paired-end reads;
+(5) assembly-to-assembly alignment; (6) full-genome alignment between two
+closely related species with divergence below ~15%.
 
-For modern x86-64 CPUs, just type `make` in the source code directory. This
+For ~10kb noisy reads sequences, minimap2 is tens of times faster than
-will compile a binary `minimap2` which you can copy to your desired location.
+mainstream long-read mappers such as BLASR, BWA-MEM, NGMLR and GMAP. It is more
-If you see compilation errors, try `make sse2only=1` to disable SSE4. Minimap2
+accurate on simulated long reads and produces biologically meaningful alignment
-will run a little slower. At present, minimap2 does not work with non-x86 CPUs
+ready for downstream analyses. For >100bp Illumina short reads, minimap2 is
-or ancient CPUs that do not support SSE2. SSE2 is critical to the performance
+three times as fast as BWA-MEM and Bowtie2, and as accurate on simulated data.
-of minimap2.
+Detailed evaluations are available from the [minimap2 paper][doi] or the
+[preprint][preprint].
 
-## Limitations
+### <a name="install"></a>Installation
 
-* At the alignment phase, minimap2 performs global alignments between minimizer
+Minimap2 is optimized for x86-64 CPUs. You can acquire precompiled binaries from
-  hits. If the positions of these minimizer hits are incorrect, the final
+the [release page][release] with:
-  alignment may be suboptimal or unnecessarily fragmented.
+```sh
+curl -L https://github.com/lh3/minimap2/releases/download/v2.24/minimap2-2.24_x64-linux.tar.bz2 | tar -jxvf -
+./minimap2-2.24_x64-linux/minimap2
+```
+If you want to compile from the source, you need to have a C compiler, GNU make
+and zlib development files installed. Then type `make` in the source code
+directory to compile. If you see compilation errors, try `make sse2only=1`
+to disable SSE4 code, which will make minimap2 slightly slower.
 
-* Minimap2 may produce poor alignments that may need post-filtering. We are
+Minimap2 also works with ARM CPUs supporting the NEON instruction sets. To
-  still exploring a reliable and consistent way to report good alignments.
+compile for 32 bit ARM architectures (such as ARMv7), use `make arm_neon=1`. To
+compile for for 64 bit ARM architectures (such as ARMv8), use `make arm_neon=1
+aarch64=1`.
 
-* Minimap2 does not work well with Illumina short reads as of now.
+Minimap2 can use [SIMD Everywhere (SIMDe)][simde] library for porting
+implementation to the different SIMD instruction sets. To compile using SIMDe,
+use `make -f Makefile.simde`. To compile for ARM CPUs, use `Makefile.simde`
+with the ARM related command lines given above.
 
-* Minimap2 requires SSE2 instructions to compile. It is possible to add
+### <a name="general"></a>General usage
-  non-SSE2 support, but it would make minimap2 slower by several times.
 
-In general, minimap2 is a young project with most code written since June,
+Without any options, minimap2 takes a reference database and a query sequence
-2017. It may have bugs and room for improvements. Bug reports and suggestions
+file as input and produce approximate mapping, without base-level alignment
-are warmly welcomed.
+(i.e. coordinates are only approximate and no CIGAR in output), in the [PAF format][paf]:
+```sh
+minimap2 ref.fa query.fq > approx-mapping.paf
+```
+You can ask minimap2 to generate CIGAR at the `cg` tag of PAF with:
+```sh
+minimap2 -c ref.fa query.fq > alignment.paf
+```
+or to output alignments in the [SAM format][sam]:
+```sh
+minimap2 -a ref.fa query.fq > alignment.sam
+```
+Minimap2 seamlessly works with gzip'd FASTA and FASTQ formats as input. You
+don't need to convert between FASTA and FASTQ or decompress gzip'd files first.
+
+For the human reference genome, minimap2 takes a few minutes to generate a
+minimizer index for the reference before mapping. To reduce indexing time, you
+can optionally save the index with option **-d** and replace the reference
+sequence file with the index file on the minimap2 command line:
+```sh
+minimap2 -d ref.mmi ref.fa                     # indexing
+minimap2 -a ref.mmi reads.fq > alignment.sam   # alignment
+```
+***Importantly***, it should be noted that once you build the index, indexing
+parameters such as **-k**, **-w**, **-H** and **-I** can't be changed during
+mapping. If you are running minimap2 for different data types, you will
+probably need to keep multiple indexes generated with different parameters.
+This makes minimap2 different from BWA which always uses the same index
+regardless of query data types.
+
+### <a name="cases"></a>Use cases
+
+Minimap2 uses the same base algorithm for all applications. However, due to the
+different data types it supports (e.g. short vs long reads; DNA vs mRNA reads),
+minimap2 needs to be tuned for optimal performance and accuracy. It is usually
+recommended to choose a preset with option **-x**, which sets multiple
+parameters at the same time. The default setting is the same as `map-ont`.
+
+#### <a name="map-long-genomic"></a>Map long noisy genomic reads
+
+```sh
+minimap2 -ax map-pb  ref.fa pacbio-reads.fq > aln.sam   # for PacBio CLR reads
+minimap2 -ax map-ont ref.fa ont-reads.fq > aln.sam      # for Oxford Nanopore reads
+```
+The difference between `map-pb` and `map-ont` is that `map-pb` uses
+homopolymer-compressed (HPC) minimizers as seeds, while `map-ont` uses ordinary
+minimizers as seeds. Emperical evaluation suggests HPC minimizers improve
+performance and sensitivity when aligning PacBio CLR reads, but hurt when aligning
+Nanopore reads.
+
+#### <a name="map-long-splice"></a>Map long mRNA/cDNA reads
+
+```sh
+minimap2 -ax splice:hq -uf ref.fa iso-seq.fq > aln.sam       # PacBio Iso-seq/traditional cDNA
+minimap2 -ax splice ref.fa nanopore-cdna.fa > aln.sam        # Nanopore 2D cDNA-seq
+minimap2 -ax splice -uf -k14 ref.fa direct-rna.fq > aln.sam  # Nanopore Direct RNA-seq
+minimap2 -ax splice --splice-flank=no SIRV.fa SIRV-seq.fa    # mapping against SIRV control
+```
+There are different long-read RNA-seq technologies, including tranditional
+full-length cDNA, EST, PacBio Iso-seq, Nanopore 2D cDNA-seq and Direct RNA-seq.
+They produce data of varying quality and properties. By default, `-x splice`
+assumes the read orientation relative to the transcript strand is unknown. It
+tries two rounds of alignment to infer the orientation and write the strand to
+the `ts` SAM/PAF tag if possible. For Iso-seq, Direct RNA-seq and tranditional
+full-length cDNAs, it would be desired to apply `-u f` to force minimap2 to
+consider the forward transcript strand only. This speeds up alignment with
+slight improvement to accuracy. For noisy Nanopore Direct RNA-seq reads, it is
+recommended to use a smaller k-mer size for increased sensitivity to the first
+or the last exons.
+
+Minimap2 rates an alignment by the score of the max-scoring sub-segment,
+*excluding* introns, and marks the best alignment as primary in SAM. When a
+spliced gene also has unspliced pseudogenes, minimap2 does not intentionally
+prefer spliced alignment, though in practice it more often marks the spliced
+alignment as the primary. By default, minimap2 outputs up to five secondary
+alignments (i.e. likely pseudogenes in the context of RNA-seq mapping). This
+can be tuned with option **-N**.
+
+For long RNA-seq reads, minimap2 may produce chimeric alignments potentially
+caused by gene fusions/structural variations or by an intron longer than the
+max intron length **-G** (200k by default). For now, it is not recommended to
+apply an excessively large **-G** as this slows down minimap2 and sometimes
+leads to false alignments.
+
+It is worth noting that by default `-x splice` prefers GT[A/G]..[C/T]AG
+over GT[C/T]..[A/G]AG, and then over other splicing signals. Considering
+one additional base improves the junction accuracy for noisy reads, but
+reduces the accuracy when aligning against the widely used SIRV control data.
+This is because SIRV does not honor the evolutionarily conservative splicing
+signal. If you are studying SIRV, you may apply `--splice-flank=no` to let
+minimap2 only model GT..AG, ignoring the additional base.
+
+Since v2.17, minimap2 can optionally take annotated genes as input and
+prioritize on annotated splice junctions. To use this feature, you can 
+```sh
+paftools.js gff2bed anno.gff > anno.bed
+minimap2 -ax splice --junc-bed anno.bed ref.fa query.fa > aln.sam
+```
+Here, `anno.gff` is the gene annotation in the GTF or GFF3 format (`gff2bed`
+automatically tests the format). The output of `gff2bed` is in the 12-column
+BED format, or the BED12 format. With the `--junc-bed` option, minimap2 adds a
+bonus score (tuned by `--junc-bonus`) if an aligned junction matches a junction
+in the annotation. Option `--junc-bed` also takes 5-column BED, including the
+strand field. In this case, each line indicates an oriented junction.
+
+#### <a name="long-overlap"></a>Find overlaps between long reads
+
+```sh
+minimap2 -x ava-pb  reads.fq reads.fq > ovlp.paf    # PacBio CLR read overlap
+minimap2 -x ava-ont reads.fq reads.fq > ovlp.paf    # Oxford Nanopore read overlap
+```
+Similarly, `ava-pb` uses HPC minimizers while `ava-ont` uses ordinary
+minimizers. It is usually not recommended to perform base-level alignment in
+the overlapping mode because it is slow and may produce false positive
+overlaps. However, if performance is not a concern, you may try to add `-a` or
+`-c` anyway.
+
+#### <a name="short-genomic"></a>Map short accurate genomic reads
+
+```sh
+minimap2 -ax sr ref.fa reads-se.fq > aln.sam           # single-end alignment
+minimap2 -ax sr ref.fa read1.fq read2.fq > aln.sam     # paired-end alignment
+minimap2 -ax sr ref.fa reads-interleaved.fq > aln.sam  # paired-end alignment
+```
+When two read files are specified, minimap2 reads from each file in turn and
+merge them into an interleaved stream internally. Two reads are considered to
+be paired if they are adjacent in the input stream and have the same name (with
+the `/[0-9]` suffix trimmed if present). Single- and paired-end reads can be
+mixed.
+
+Minimap2 does not work well with short spliced reads. There are many capable
+RNA-seq mappers for short reads.
+
+#### <a name="full-genome"></a>Full genome/assembly alignment
+
+```sh
+minimap2 -ax asm5 ref.fa asm.fa > aln.sam       # assembly to assembly/ref alignment
+```
+For cross-species full-genome alignment, the scoring system needs to be tuned
+according to the sequence divergence.
+
+### <a name="advanced"></a>Advanced features
+
+#### <a name="long-cigar"></a>Working with >65535 CIGAR operations
+
+Due to a design flaw, BAM does not work with CIGAR strings with >65535
+operations (SAM and CRAM work). However, for ultra-long nanopore reads minimap2
+may align ~1% of read bases with long CIGARs beyond the capability of BAM. If
+you convert such SAM/CRAM to BAM, Picard and recent samtools will throw an
+error and abort. Older samtools and other tools may create corrupted BAM.
+
+To avoid this issue, you can add option `-L` at the minimap2 command line.
+This option moves a long CIGAR to the `CG` tag and leaves a fully clipped CIGAR
+at the SAM CIGAR column. Current tools that don't read CIGAR (e.g. merging and
+sorting) still work with such BAM records; tools that read CIGAR will
+effectively ignore these records. It has been decided that future tools
+will seamlessly recognize long-cigar records generated by option `-L`.
+
+**TL;DR**: if you work with ultra-long reads and use tools that only process
+BAM files, please add option `-L`.
+
+#### <a name="cs"></a>The cs optional tag
+
+The `cs` SAM/PAF tag encodes bases at mismatches and INDELs. It matches regular
+expression `/(:[0-9]+|\*[a-z][a-z]|[=\+\-][A-Za-z]+)+/`. Like CIGAR, `cs`
+consists of series of operations.  Each leading character specifies the
+operation; the following sequence is the one involved in the operation.
+
+The `cs` tag is enabled by command line option `--cs`. The following alignment,
+for example:
+```txt
+CGATCGATAAATAGAGTAG---GAATAGCA
+||||||   ||||||||||   |||| |||
+CGATCG---AATAGAGTAGGTCGAATtGCA
+```
+is represented as `:6-ata:10+gtc:4*at:3`, where `:[0-9]+` represents an
+identical block, `-ata` represents a deletion, `+gtc` an insertion and `*at`
+indicates reference base `a` is substituted with a query base `t`. It is
+similar to the `MD` SAM tag but is standalone and easier to parse.
+
+If `--cs=long` is used, the `cs` string also contains identical sequences in
+the alignment. The above example will become
+`=CGATCG-ata=AATAGAGTAG+gtc=GAAT*at=GCA`. The long form of `cs` encodes both
+reference and query sequences in one string. The `cs` tag also encodes intron
+positions and splicing signals (see the [minimap2 manpage][manpage-cs] for
+details).
+
+#### <a name="paftools"></a>Working with the PAF format
+
+Minimap2 also comes with a (java)script [paftools.js](misc/paftools.js) that
+processes alignments in the PAF format. It calls variants from
+assembly-to-reference alignment, lifts over BED files based on alignment,
+converts between formats and provides utilities for various evaluations. For
+details, please see [misc/README.md](misc/README.md).
+
+### <a name="algo"></a>Algorithm overview
+
+In the following, minimap2 command line options have a dash ahead and are
+highlighted in bold. The description may help to tune minimap2 parameters.
+
+1. Read **-I** [=*4G*] reference bases, extract (**-k**,**-w**)-minimizers and
+   index them in a hash table.
+
+2. Read **-K** [=*200M*] query bases. For each query sequence, do step 3
+   through 7:
+
+3. For each (**-k**,**-w**)-minimizer on the query, check against the reference
+   index. If a reference minimizer is not among the top **-f** [=*2e-4*] most
+   frequent, collect its the occurrences in the reference, which are called
+   *seeds*.
+
+4. Sort seeds by position in the reference. Chain them with dynamic
+   programming. Each chain represents a potential mapping. For read
+   overlapping, report all chains and then go to step 8. For reference mapping,
+   do step 5 through 7:
+
+5. Let *P* be the set of primary mappings, which is an empty set initially. For
+   each chain from the best to the worst according to their chaining scores: if
+   on the query, the chain overlaps with a chain in *P* by **--mask-level**
+   [=*0.5*] or higher fraction of the shorter chain, mark the chain as
+   *secondary* to the chain in *P*; otherwise, add the chain to *P*.
+
+6. Retain all primary mappings. Also retain up to **-N** [=*5*] top secondary
+   mappings if their chaining scores are higher than **-p** [=*0.8*] of their
+   corresponding primary mappings.
+
+7. If alignment is requested, filter out an internal seed if it potentially
+   leads to both a long insertion and a long deletion. Extend from the
+   left-most seed. Perform global alignments between internal seeds.  Split the
+   chain if the accumulative score along the global alignment drops by **-z**
+   [=*400*], disregarding long gaps. Extend from the right-most seed.  Output
+   chains and their alignments.
+
+8. If there are more query sequences in the input, go to step 2 until no more
+   queries are left.
+
+9. If there are more reference sequences, reopen the query file from the start
+   and go to step 1; otherwise stop.
+
+### <a name="help"></a>Getting help
+
+Manpage [minimap2.1][manpage] provides detailed description of minimap2
+command line options and optional tags. The [FAQ](FAQ.md) page answers several
+frequently asked questions. If you encounter bugs or have further questions or
+requests, you can raise an issue at the [issue page][issue].  There is not a
+specific mailing list for the time being.
+
+### <a name="cite"></a>Citing minimap2
+
+If you use minimap2 in your work, please cite:
+
+> Li, H. (2018). Minimap2: pairwise alignment for nucleotide sequences.
+> *Bioinformatics*, **34**:3094-3100. [doi:10.1093/bioinformatics/bty191][doi]
+
+## <a name="dguide"></a>Developers' Guide
+
+Minimap2 is not only a command line tool, but also a programming library.
+It provides C APIs to build/load index and to align sequences against the
+index. File [example.c](example.c) demonstrates typical uses of C APIs. Header
+file [minimap.h](minimap.h) gives more detailed API documentation. Minimap2
+aims to keep APIs in this header stable. File [mmpriv.h](mmpriv.h) contains
+additional private APIs which may be subjected to changes frequently.
+
+This repository also provides Python bindings to a subset of C APIs. File
+[python/README.rst](python/README.rst) gives the full documentation;
+[python/minimap2.py](python/minimap2.py) shows an example. This Python
+extension, mappy, is also [available from PyPI][mappypypi] via `pip install
+mappy` or [from BioConda][mappyconda] via `conda install -c bioconda mappy`.
+
+## <a name="limit"></a>Limitations
+
+* Minimap2 may produce suboptimal alignments through long low-complexity
+  regions where seed positions may be suboptimal. This should not be a big
+  concern because even the optimal alignment may be wrong in such regions.
+
+* Minimap2 requires SSE2 instructions on x86 CPUs or NEON on ARM CPUs. It is
+  possible to add non-SIMD support, but it would make minimap2 slower by
+  several times.
+
+* Minimap2 does not work with a single query or database sequence ~2
+  billion bases or longer (2,147,483,647 to be exact). The total length of all
+  sequences can well exceed this threshold.
+
+* Minimap2 often misses small exons.
 
 
 
@@ -67,3 +390,14 @@ are warmly welcomed.
 [longislnd]: https://www.ncbi.nlm.nih.gov/pubmed/27667791
 [gaba]: https://github.com/ocxtal/libgaba
 [ksw2]: https://github.com/lh3/ksw2
+[preprint]: https://arxiv.org/abs/1708.01492
+[release]: https://github.com/lh3/minimap2/releases
+[mappypypi]: https://pypi.python.org/pypi/mappy
+[mappyconda]: https://anaconda.org/bioconda/mappy
+[issue]: https://github.com/lh3/minimap2/issues
+[k8]: https://github.com/attractivechaos/k8
+[manpage]: https://lh3.github.io/minimap2/minimap2.html
+[manpage-cs]: https://lh3.github.io/minimap2/minimap2.html#10
+[doi]: https://doi.org/10.1093/bioinformatics/bty191
+[smide]: https://github.com/nemequ/simde
+[unimap]: https://github.com/lh3/unimap

bseq.c

@@ -1,23 +1,45 @@
 #include <zlib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 #include <assert.h>
+#define __STDC_LIMIT_MACROS
 #include "bseq.h"
+#include "kvec.h"
 #include "kseq.h"
-KSEQ_INIT(gzFile, gzread)
+KSEQ_INIT2(, gzFile, gzread)
+
+unsigned char seq_comp_table[256] = {
+	  0,   1,	2,	 3,	  4,   5,	6,	 7,	  8,   9,  10,	11,	 12,  13,  14,	15,
+	 16,  17,  18,	19,	 20,  21,  22,	23,	 24,  25,  26,	27,	 28,  29,  30,	31,
+	 32,  33,  34,	35,	 36,  37,  38,	39,	 40,  41,  42,	43,	 44,  45,  46,	47,
+	 48,  49,  50,	51,	 52,  53,  54,	55,	 56,  57,  58,	59,	 60,  61,  62,	63,
+	 64, 'T', 'V', 'G', 'H', 'E', 'F', 'C', 'D', 'I', 'J', 'M', 'L', 'K', 'N', 'O',
+	'P', 'Q', 'Y', 'S', 'A', 'A', 'B', 'W', 'X', 'R', 'Z',	91,	 92,  93,  94,	95,
+	 96, 't', 'v', 'g', 'h', 'e', 'f', 'c', 'd', 'i', 'j', 'm', 'l', 'k', 'n', 'o',
+	'p', 'q', 'y', 's', 'a', 'a', 'b', 'w', 'x', 'r', 'z', 123, 124, 125, 126, 127,
+	128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
+	144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+	160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
+	176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191,
+	192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
+	208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223,
+	224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+	240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255
+};
+
+#define CHECK_PAIR_THRES 1000000
 
 struct mm_bseq_file_s {
-	int is_eof;
 	gzFile fp;
 	kseq_t *ks;
+	mm_bseq1_t s;
 };
 
 mm_bseq_file_t *mm_bseq_open(const char *fn)
 {
 	mm_bseq_file_t *fp;
 	gzFile f;
-	f = fn && strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+	f = fn && strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(0, "r");
 	if (f == 0) return 0;
 	fp = (mm_bseq_file_t*)calloc(1, sizeof(mm_bseq_file_t));
 	fp->fp = f;
@@ -32,33 +54,116 @@ void mm_bseq_close(mm_bseq_file_t *fp)
 	free(fp);
 }
 
-mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_)
+static inline char *kstrdup(const kstring_t *s)
 {
-	int size = 0, m, n;
+	char *t;
-	mm_bseq1_t *seqs;
+	t = (char*)malloc(s->l + 1);
+	memcpy(t, s->s, s->l + 1);
+	return t;
+}
+
+static inline void kseq2bseq(kseq_t *ks, mm_bseq1_t *s, int with_qual, int with_comment)
+{
+	int i;
+	if (ks->name.l == 0)
+		fprintf(stderr, "[WARNING]\033[1;31m empty sequence name in the input.\033[0m\n");
+	s->name = kstrdup(&ks->name);
+	s->seq = kstrdup(&ks->seq);
+	for (i = 0; i < (int)ks->seq.l; ++i) // convert U to T
+		if (s->seq[i] == 'u' || s->seq[i] == 'U')
+			--s->seq[i];
+	s->qual = with_qual && ks->qual.l? kstrdup(&ks->qual) : 0;
+	s->comment = with_comment && ks->comment.l? kstrdup(&ks->comment) : 0;
+	s->l_seq = ks->seq.l;
+}
+
+mm_bseq1_t *mm_bseq_read3(mm_bseq_file_t *fp, int64_t chunk_size, int with_qual, int with_comment, int frag_mode, int *n_)
+{
+	int64_t size = 0;
+	int ret;
+	kvec_t(mm_bseq1_t) a = {0,0,0};
 	kseq_t *ks = fp->ks;
-	m = n = 0; seqs = 0;
+	*n_ = 0;
-	while (kseq_read(ks) >= 0) {
+	if (fp->s.seq) {
+		kv_resize(mm_bseq1_t, 0, a, 256);
+		kv_push(mm_bseq1_t, 0, a, fp->s);
+		size = fp->s.l_seq;
+		memset(&fp->s, 0, sizeof(mm_bseq1_t));
+	}
+	while ((ret = kseq_read(ks)) >= 0) {
 		mm_bseq1_t *s;
 		assert(ks->seq.l <= INT32_MAX);
-		if (n >= m) {
+		if (a.m == 0) kv_resize(mm_bseq1_t, 0, a, 256);
-			m = m? m<<1 : 256;
+		kv_pushp(mm_bseq1_t, 0, a, &s);
-			seqs = (mm_bseq1_t*)realloc(seqs, m * sizeof(mm_bseq1_t));
+		kseq2bseq(ks, s, with_qual, with_comment);
+		size += s->l_seq;
+		if (size >= chunk_size) {
+			if (frag_mode && a.a[a.n-1].l_seq < CHECK_PAIR_THRES) {
+				while ((ret = kseq_read(ks)) >= 0) {
+					kseq2bseq(ks, &fp->s, with_qual, with_comment);
+					if (mm_qname_same(fp->s.name, a.a[a.n-1].name)) {
+						kv_push(mm_bseq1_t, 0, a, fp->s);
+						memset(&fp->s, 0, sizeof(mm_bseq1_t));
+					} else break;
+				}
+			}
+			break;
+		}
+	}
+	if (ret < -1) {
+		if (a.n) fprintf(stderr, "[WARNING]\033[1;31m failed to parse the FASTA/FASTQ record next to '%s'. Continue anyway.\033[0m\n", a.a[a.n-1].name);
+		else fprintf(stderr, "[WARNING]\033[1;31m failed to parse the first FASTA/FASTQ record. Continue anyway.\033[0m\n");
+	}
+	*n_ = a.n;
+	return a.a;
+}
+
+mm_bseq1_t *mm_bseq_read2(mm_bseq_file_t *fp, int64_t chunk_size, int with_qual, int frag_mode, int *n_)
+{
+	return mm_bseq_read3(fp, chunk_size, with_qual, 0, frag_mode, n_);
+}
+
+mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int64_t chunk_size, int with_qual, int *n_)
+{
+	return mm_bseq_read2(fp, chunk_size, with_qual, 0, n_);
+}
+
+mm_bseq1_t *mm_bseq_read_frag2(int n_fp, mm_bseq_file_t **fp, int64_t chunk_size, int with_qual, int with_comment, int *n_)
+{
+	int i;
+	int64_t size = 0;
+	kvec_t(mm_bseq1_t) a = {0,0,0};
+	*n_ = 0;
+	if (n_fp < 1) return 0;
+	while (1) {
+		int n_read = 0;
+		for (i = 0; i < n_fp; ++i)
+			if (kseq_read(fp[i]->ks) >= 0)
+				++n_read;
+		if (n_read < n_fp) {
+			if (n_read > 0)
+				fprintf(stderr, "[W::%s]\033[1;31m query files have different number of records; extra records skipped.\033[0m\n", __func__);
+			break; // some file reaches the end
+		}
+		if (a.m == 0) kv_resize(mm_bseq1_t, 0, a, 256);
+		for (i = 0; i < n_fp; ++i) {
+			mm_bseq1_t *s;
+			kv_pushp(mm_bseq1_t, 0, a, &s);
+			kseq2bseq(fp[i]->ks, s, with_qual, with_comment);
+			size += s->l_seq;
 		}
-		s = &seqs[n];
-		s->name = strdup(ks->name.s);
-		s->seq = strdup(ks->seq.s);
-		s->qual = with_qual && ks->qual.l? strdup(ks->qual.s) : 0;
-		s->l_seq = ks->seq.l;
-		size += seqs[n++].l_seq;
 		if (size >= chunk_size) break;
 	}
-	if (size < chunk_size) fp->is_eof = 1;
+	*n_ = a.n;
-	*n_ = n;
+	return a.a;
-	return seqs;
+}
+
+mm_bseq1_t *mm_bseq_read_frag(int n_fp, mm_bseq_file_t **fp, int64_t chunk_size, int with_qual, int *n_)
+{
+	return mm_bseq_read_frag2(n_fp, fp, chunk_size, with_qual, 0, n_);
 }
 
 int mm_bseq_eof(mm_bseq_file_t *fp)
 {
-	return fp->is_eof;
+	return (ks_eof(fp->ks->f) && fp->s.seq == 0);
 }

bseq.h

@@ -2,6 +2,7 @@
 #define MM_BSEQ_H
 
 #include <stdint.h>
+#include <string.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -12,15 +13,49 @@ typedef struct mm_bseq_file_s mm_bseq_file_t;
 
 typedef struct {
 	int l_seq, rid;
-	char *name, *seq, *qual;
+	char *name, *seq, *qual, *comment;
 } mm_bseq1_t;
 
 mm_bseq_file_t *mm_bseq_open(const char *fn);
 void mm_bseq_close(mm_bseq_file_t *fp);
-mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int chunk_size, int with_qual, int *n_);
+mm_bseq1_t *mm_bseq_read3(mm_bseq_file_t *fp, int64_t chunk_size, int with_qual, int with_comment, int frag_mode, int *n_);
+mm_bseq1_t *mm_bseq_read2(mm_bseq_file_t *fp, int64_t chunk_size, int with_qual, int frag_mode, int *n_);
+mm_bseq1_t *mm_bseq_read(mm_bseq_file_t *fp, int64_t chunk_size, int with_qual, int *n_);
+mm_bseq1_t *mm_bseq_read_frag2(int n_fp, mm_bseq_file_t **fp, int64_t chunk_size, int with_qual, int with_comment, int *n_);
+mm_bseq1_t *mm_bseq_read_frag(int n_fp, mm_bseq_file_t **fp, int64_t chunk_size, int with_qual, int *n_);
 int mm_bseq_eof(mm_bseq_file_t *fp);
 
 extern unsigned char seq_nt4_table[256];
+extern unsigned char seq_comp_table[256];
+
+static inline int mm_qname_len(const char *s)
+{
+	int l;
+	l = strlen(s);
+	return l >= 3 && s[l-1] >= '0' && s[l-1] <= '9' && s[l-2] == '/'? l - 2 : l;
+}
+
+static inline int mm_qname_same(const char *s1, const char *s2)
+{
+	int l1, l2;
+	l1 = mm_qname_len(s1);
+	l2 = mm_qname_len(s2);
+	return (l1 == l2 && strncmp(s1, s2, l1) == 0);
+}
+
+static inline void mm_revcomp_bseq(mm_bseq1_t *s)
+{
+	int i, t, l = s->l_seq;
+	for (i = 0; i < l>>1; ++i) {
+		t = s->seq[l - i - 1];
+		s->seq[l - i - 1] = seq_comp_table[(uint8_t)s->seq[i]];
+		s->seq[i] = seq_comp_table[t];
+	}
+	if (l&1) s->seq[l>>1] = seq_comp_table[(uint8_t)s->seq[l>>1]];
+	if (s->qual)
+		for (i = 0; i < l>>1; ++i)
+			t = s->qual[l - i - 1], s->qual[l - i - 1] = s->qual[i], s->qual[i] = t;
+}
 
 #ifdef __cplusplus
 }

chain.c

@@ -1,135 +0,0 @@
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
-#include "minimap.h"
-#include "mmpriv.h"
-#include "kalloc.h"
-
-static const char LogTable256[256] = {
-#define LT(n) n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n
-    -1, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
-    LT(4), LT(5), LT(5), LT(6), LT(6), LT(6), LT(6),
-    LT(7), LT(7), LT(7), LT(7), LT(7), LT(7), LT(7), LT(7)
-};
-
-static inline int ilog2_32(uint32_t v)
-{
-	register uint32_t t, tt;
-	if ((tt = v>>16)) return (t = tt>>8) ? 24 + LogTable256[t] : 16 + LogTable256[tt];
-	return (t = v>>8) ? 8 + LogTable256[t] : LogTable256[v];
-}
-
-int mm_chain_dp(int max_dist, int bw, int max_skip, int min_cnt, int min_sc, int64_t n, mm128_t *a, uint64_t **_u, void *km)
-{ // TODO: make sure this works when n has more than 32 bits
-	int32_t st = 0, j, k, *f, *p, *t, *v, n_u, n_v;
-	int64_t i;
-	uint64_t *u, *u2;
-	mm128_t *b, *w;
-
-	if (_u) *_u = 0;
-	f = (int32_t*)kmalloc(km, n * 4);
-	p = (int32_t*)kmalloc(km, n * 4);
-	t = (int32_t*)kmalloc(km, n * 4);
-	memset(t, 0, n * 4);
-
-	// fill the score and backtrack arrays
-	for (i = 0; i < n; ++i) {
-		uint64_t ri = a[i].x;
-		int32_t qi = (int32_t)a[i].y, q_span = a[i].y>>32&0xff; // NB: only 8 bits of span is used!!!
-		int32_t max_f = -INT32_MAX, max_j = -1, n_skip = 0, min_d;
-		while (st < i && ri - a[st].x > max_dist) ++st;
-		for (j = i - 1; j >= st; --j) {
-			int64_t dr = ri - a[j].x;
-			int32_t dq = qi - (int32_t)a[j].y, dd, sc;
-			if (dr == 0 || dq <= 0 || dq > max_dist) continue;
-			dd = dr > dq? dr - dq : dq - dr;
-			if (dd > bw) continue;
-			min_d = dq < dr? dq : dr;
-			sc = min_d > q_span? q_span : dq < dr? dq : dr;
-			sc -= dd? ilog2_32(dd) * 2 : 0;
-			if (min_d > q_span) sc -= ilog2_32(min_d) / 2;
-			sc += f[j];
-			if (sc > max_f) {
-				max_f = sc, max_j = j;
-				if (n_skip > 0) --n_skip;
-			} else if (t[j] == i) {
-				if (++n_skip > max_skip)
-					break;
-			}
-			if (p[j] >= 0) t[p[j]] = i;
-		}
-		if (max_j >= 0) f[i] = max_f, p[i] = max_j;
-		else f[i] = q_span, p[i] = -1;
-	}
-
-	// find the ending positions of chains
-	memset(t, 0, n * 4);
-	for (i = 0; i < n; ++i)
-		if (p[i] >= 0) t[p[i]] = 1;
-	for (i = n_u = 0; i < n; ++i)
-		if (t[i] == 0 && f[i] >= min_sc)
-			++n_u;
-	if (n_u == 0) {
-		kfree(km, f); kfree(km, p); kfree(km, t);
-		return 0;
-	}
-	u = (uint64_t*)kmalloc(km, n_u * 8);
-	for (i = n_u = 0; i < n; ++i)
-		if (t[i] == 0 && f[i] >= min_sc)
-			u[n_u++] = (uint64_t)f[i] << 32 | i;
-	radix_sort_64(u, u + n_u);
-	for (i = 0; i < n_u>>1; ++i) { // reverse, s.t. the highest scoring chain is the first
-		uint64_t t = u[i];
-		u[i] = u[n_u - i - 1], u[n_u - i - 1] = t;
-	}
-
-	// backtrack
-	memset(t, 0, n * 4);
-	v = (int32_t*)kmalloc(km, n * 4);
-	for (i = n_v = k = 0; i < n_u; ++i) { // starting from the highest score
-		int32_t n_v0 = n_v, k0 = k;
-		j = (int32_t)u[i];
-		do {
-			v[n_v++] = j;
-			t[j] = 1;
-			j = p[j];
-		} while (j >= 0 && t[j] == 0);
-		if (j < 0) {
-			if (n_v - n_v0 >= min_cnt) u[k++] = u[i]>>32<<32 | (n_v - n_v0);
-		} else if ((int32_t)(u[i]>>32) - f[j] >= min_sc) {
-			if (n_v - n_v0 >= min_cnt) u[k++] = ((u[i]>>32) - f[j]) << 32 | (n_v - n_v0);
-		}
-		if (k0 == k) n_v = n_v0; // no new chain added, reset
-	}
-	n_u = k, *_u = u; // NB: note that u[] may not be sorted by score here
-
-	// free
-	kfree(km, f); kfree(km, p); kfree(km, t);
-
-	// write the result to b[]
-	b = (mm128_t*)kmalloc(km, n_v * sizeof(mm128_t));
-	for (i = 0, k = 0; i < n_u; ++i) {
-		int32_t k0 = k, ni = (int32_t)u[i];
-		for (j = 0; j < ni; ++j)
-			b[k] = a[v[k0 + (ni - j - 1)]], ++k;
-	}
-	kfree(km, v);
-
-	// sort u[] and a[] by a[].x, such that adjacent chains may be joined (required by mm_join_long)
-	w = (mm128_t*)kmalloc(km, n_u * sizeof(mm128_t));
-	for (i = k = 0; i < n_u; ++i) {
-		w[i].x = b[k].x, w[i].y = (uint64_t)k<<32|i;
-		k += (int32_t)u[i];
-	}
-	radix_sort_128x(w, w + n_u);
-	u2 = (uint64_t*)kmalloc(km, n_u * 8);
-	for (i = k = 0; i < n_u; ++i) {
-		int32_t j = (int32_t)w[i].y, n = (int32_t)u[j];
-		u2[i] = u[j];
-		memcpy(&a[k], &b[w[i].y>>32], n * sizeof(mm128_t));
-		k += n;
-	}
-	memcpy(u, u2, n_u * 8);
-	kfree(km, b); kfree(km, w); kfree(km, u2);
-	return n_u;
-}

code_of_conduct.md

@@ -0,0 +1,30 @@
+## Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all
+people who contribute through reporting issues, posting feature requests,
+updating documentation, submitting pull requests or patches, and other
+activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual
+language or imagery, derogatory comments or personal attacks, trolling, public
+or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct. Project maintainers or
+contributors who do not follow the Code of Conduct may be removed from the
+project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by opening an issue or contacting the maintainer via email.
+
+This Code of Conduct is adapted from the [Contributor Covenant][cc], [version
+1.0.0][v1].
+
+[cc]: http://contributor-covenant.org/
+[v1]: http://contributor-covenant.org/version/1/0/0/

cookbook.md

@@ -0,0 +1,243 @@
+## Table of Contents
+
+- [Introduction & Installation](#intro)
+- [Mapping Genomic Reads](#map-reads)
+  * [Mapping long reads](#map-pb)
+  * [Mapping Illumina paired-end reads](#map-sr)
+  * [Evaluating mapping accuracy with simulated reads (for developers)](#mapeval)
+- [Mapping Long RNA-seq Reads](#map-rna)
+  * [Mapping Nanopore 2D cDNA reads](#map-ont-cdna-2d)
+  * [Mapping Nanopore direct-RNA reads](#map-direct-rna)
+  * [Mapping PacBio Iso-seq reads](#map-iso-seq)
+- [Full-Genome Alignment](#genome-aln)
+  * [Intra-species assembly alignment](#asm-to-ref)
+  * [Cross-species full-genome alignment](#x-species)
+  * [Eyeballing alignment](#view-aln)
+  * [Calling variants from assembly-to-reference alignment](#asm-var)
+  * [Constructing self-homology map](#hom-map)
+  * [Lift Over (for developers)](#liftover)
+- [Read Overlap](#read-overlap)
+  * [Long-read overlap](#long-read-overlap)
+  * [Evaluating overlap sensitivity (for developers)](#ov-eval)
+
+## <a name="intro"></a>Introduction & Installation
+
+This cookbook walks you through a variety of applications of minimap2 and its
+companion script `paftools.js`. All data here are freely available from the
+minimap2 release page at version tag [v2.10][v2.10]. Some examples only work
+with v2.10 or later.
+
+To acquire the data used in this cookbook and to install minimap2 and paftools,
+please follow the command lines below:
+```sh
+# install minimap2 executables
+curl -L https://github.com/lh3/minimap2/releases/download/v2.24/minimap2-2.24_x64-linux.tar.bz2 | tar jxf -
+cp minimap2-2.24_x64-linux/{minimap2,k8,paftools.js} .  # copy executables
+export PATH="$PATH:"`pwd`                               # put the current directory on PATH
+# download example datasets
+curl -L https://github.com/lh3/minimap2/releases/download/v2.10/cookbook-data.tgz | tar zxf -
+```
+
+## <a name="map-reads"></a>Mapping Genomic Reads
+
+### <a name="map-pb"></a>Mapping long reads
+```sh
+minimap2 -ax map-pb -t4 ecoli_ref.fa ecoli_p6_25x_canu.fa > mapped.sam
+```
+Alternatively, you can create a minimap2 index first and then map:
+```sh
+minimap2 -x map-pb -d ecoli-pb.mmi ecoli_ref.fa                      # create an index
+minimap2 -ax map-pb ecoli-pb.mmi ecoli_p6_25x_canu.fa > mapped.sam
+```
+This will save you a couple of minutes when you map against the human genome.
+**HOWEVER**, key algorithm parameters such as the k-mer length and window
+size can't be changed after indexing. Minimap2 will give you a warning if
+parameters used in a pre-built index doesn't match parameters on the command
+line. **Please always make sure you are using an intended pre-built index.**
+
+### <a name="map-sr"></a>Mapping Illumina paired-end reads:
+```sh
+minimap2 -ax sr -t4 ecoli_ref.fa ecoli_mason_1.fq ecoli_mason_2.fq > mapped-sr.sam
+```
+
+### <a name="mapeval"></a>Evaluating mapping accuracy with simulated reads (for developers)
+```sh
+minimap2 -ax sr ecoli_ref.fa ecoli_mason_1.fq ecoli_mason_2.fq | paftools.js mapeval -
+```
+The output is:
+```
+Q       60      19712   0       0.000000000     19712
+Q       0       282     219     0.010953286     19994
+U       6
+```
+where a `U`-line gives the number of unmapped reads (for SAM input only); a
+`Q`-line gives:
+
+1. Mapping quality (mapQ) threshold
+2. Number of mapped reads between this threshold and the previous mapQ threshold.
+3. Number of wrong mappings in the same mapQ interval
+4. Accumulative mapping error rate
+5. Accumulative number of mappings
+
+For `paftools.js mapeval` to work, you need to encode the true read positions
+in read names in the right format. For [pbsim2][pbsim] and [mason2][mason2], we
+provide scripts to generate the right format. Simulated reads in this cookbook
+were created with the following command lines:
+```sh
+# in the pbsim2 source code directory:
+src/pbsim --depth 1 --length-min 5000 --length-mean 20000 --accuracy-mean 0.95 --hmm_model data/R94.model ../ecoli_ref.fa
+paftools.js pbsim2fq ../ecoli_ref.fa.fai sd_0001.maf > ../ecoli_pbsim.fa
+
+# mason2 simulation
+mason_simulator --illumina-prob-mismatch-scale 2.5 -ir ecoli_ref.fa -n 10000 -o tmp-l.fq -or tmp-r.fq -oa tmp.sam
+paftools.js mason2fq tmp.sam | seqtk seq -1 > ecoli_mason_1.fq
+paftools.js mason2fq tmp.sam | seqtk seq -2 > ecoli_mason_2.fq
+```
+
+
+
+## <a name="map-rna"></a>Mapping Long RNA-seq Reads
+
+### <a name="map-ont-cdna-2d"></a>Mapping Nanopore 2D cDNA reads
+```sh
+minimap2 -ax splice SIRV_E2.fa SIRV_ont-cdna.fa > aln.sam
+```
+You can compare the alignment to the true annotations with:
+```sh
+paftools.js junceval SIRV_E2C.gtf aln.sam
+```
+It gives the percentage of introns found in the annotation. For SIRV data, it
+is possible to achieve higher junction accuracy with
+```sh
+minimap2 -ax splice --splice-flank=no SIRV_E2.fa SIRV_ont-cdna.fa | paftools.js junceval SIRV_E2C.gtf
+```
+This is because minimap2 models one additional evolutionarily conserved base
+around a canonical junction, but SIRV doesn't honor this signal. Option
+`--splice-flank=no` asks minimap2 no to model this additional base.
+
+In the output a tag `ts:A:+` indicates that the read strand is the same as the
+transcript strand; `ts:A:-` indicates the read strand is opposite to the
+transcript strand. This tag is inferred from the GT-AG signal and is thus only
+available to spliced reads.
+
+### <a name="map-direct-rna"></a>Mapping Nanopore direct-RNA reads
+```sh
+minimap2 -ax splice -k14 -uf SIRV_E2.fa SIRV_ont-drna.fa > aln.sam
+```
+Direct-RNA reads are noisier, so we use a shorter k-mer for improved
+sensitivity. Here, option `-uf` forces minimap2 to map reads to the forward
+transcript strand only because direct-RNA reads are stranded. Again, applying
+`--splice-flank=no` helps junction accuracy for SIRV data.
+
+### <a name="map-iso-seq"></a>Mapping PacBio Iso-seq reads
+```sh
+minimap2 -ax splice -uf -C5 SIRV_E2.fa SIRV_iso-seq.fq > aln.sam
+```
+Option `-C5` reduces the penalty on non-canonical splicing sites. It helps
+to align such sites correctly for data with low error rate such as Iso-seq
+reads and traditional cDNAs. On this example, minimap2 makes one junction
+error. Applying `--splice-flank=no` fixes this alignment error.
+
+Note that the command line above is optimized for the final Iso-seq reads.
+PacBio's Iso-seq pipeline produces intermediate sequences at varying quality.
+For example, some intermediate reads are not stranded. For these reads, option
+`-uf` will lead to more errors. Please revise the minimap2 command line
+accordingly.
+
+
+
+## <a name="genome-aln"></a>Full-Genome Alignment
+
+### <a name="asm-to-ref"></a>Intra-species assembly alignment
+```sh
+# option "--cs" is recommended as paftools.js may need it
+minimap2 -cx asm5 --cs ecoli_ref.fa ecoli_canu.fa > ecoli_canu.paf
+```
+Here `ecoli_canu.fa` is the Canu assembly of `ecoli_p6_25x_canu.fa`. This
+command line outputs alignments in the [PAF format][paf]. Use `-a` instead of
+`-c` to get output in the SAM format.
+
+### <a name="x-species"></a>Cross-species full-genome alignment
+```sh
+minimap2 -cx asm20 --cs ecoli_ref.fa ecoli_O104:H4.fa > ecoli_O104:H4.paf
+sort -k6,6 -k8,8n ecoli_O104:H4.paf | paftools.js call -f ecoli_ref.fa -L10000 -l1000 - > out.vcf
+```
+Minimap2 has three presets for full-genome alignment: "asm5" for sequence
+divergence below 1%, "asm10" for divergence around a couple of percent and
+"asm20" for divergence not more than 10%. In theory, with the right setting,
+minimap2 should work for sequence pairs with sequence divergence up to ~15%,
+but this has not been carefully evaluated.
+
+### <a name="view-aln"></a>Eyeballing alignment
+```sh
+# option "--cs" required; minimap2-r741 or higher required for the "asm20" preset
+minimap2 -cx asm20 --cs ecoli_ref.fa ecoli_O104:H4.fa | paftools.js view - | less -S
+```
+This prints the alignment in a BLAST-like format.
+
+### <a name="asm-var"></a>Calling variants from assembly-to-reference alignment
+```sh
+# don't forget the "--cs" option; otherwise it doesn't work
+minimap2 -cx asm5 --cs ecoli_ref.fa ecoli_canu.fa \
+  | sort -k6,6 -k8,8n \
+  | paftools.js call -f ecoli_ref.fa - > out.vcf
+```
+Without option `-f`, `paftools.js call` outputs in a custom format. In this
+format, lines starting with `R` give the regions covered by one contig only.
+This information is not available in the VCF output.
+
+### <a name="hom-map"></a>Constructing self-homology map
+```sh
+minimap2 -DP -k19 -w19 -m200 ecoli_ref.fa ecoli_ref.fa > out.paf
+```
+Option `-D` asks minimap2 to ignore anchors from perfect self match and `-P`
+outputs all chains. For large nomes, we don't recommend to perform base-level
+alignment (with `-c`, `-a` or `--cs`) when `-P` is applied. This is because
+base-alignment is slow and occasionally gives wrong alignments close to the
+diagonal of a dotter plot. For E. coli, though, base-alignment is still fast.
+
+### <a name="liftover"></a>Lift over (for developers)
+```sh
+minimap2 -cx asm5 --cs ecoli_ref.fa ecoli_canu.fa > ecoli_canu.paf
+echo -e 'tig00000001\t200000\t300000' | paftools.js liftover ecoli_canu.paf -
+```
+This lifts over a region on query sequences to one or multiple regions on
+reference sequences. Note that this paftools.js command may not be efficient
+enough to lift millions of regions.
+
+
+
+## <a name="read-overlap"></a>Read Overlap
+
+### <a name="long-read-overlap"></a>Long read overlap
+```sh
+# For pacbio reads:
+minimap2 -x ava-pb ecoli_p6_25x_canu.fa ecoli_p6_25x_canu.fa > overlap.paf
+# For Nanopore reads (ava-ont also works with PacBio but not as good):
+minimap2 -x ava-ont -r 10000 ecoli_p6_25x_canu.fa ecoli_p6_25x_canu.fa > overlap.paf
+# If you have miniasm installed:
+miniasm -f ecoli_p6_25x_canu.fa overlap.paf > asm.gfa
+```
+Here we explicitly applied `-r 10000`. We are considering to set this as the
+default for the `ava-ont` mode as this seems to improve the contiguity for
+nanopore read assembly (Loman, personal communication).
+
+*Minimap2 doesn't work well with short-read overlap.*
+
+### <a name="ov-eval"></a>Evaluating overlap sensitivity (for developers)
+
+```sh
+# read to reference mapping
+minimap2 -cx map-pb ecoli_ref.fa ecoli_p6_25x_canu.fa > to-ref.paf
+# evaluate overlap sensitivity
+sort -k6,6 -k8,8n to-ref.paf | paftools.js ov-eval - overlap.paf
+```
+You can see that for PacBio reads, minimap2 achieves higher overlap sensitivity
+with `-x ava-pb` (99% vs 93% with `-x ava-ont`).
+
+
+
+[pbsim]: https://github.com/yukiteruono/pbsim2
+[mason2]: https://github.com/seqan/seqan/tree/master/apps/mason2
+[paf]: https://github.com/lh3/miniasm/blob/master/PAF.md
+[v2.10]: https://github.com/lh3/minimap2/releases/tag/v2.10

esterr.c

@@ -0,0 +1,64 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "mmpriv.h"
+
+static inline int32_t get_for_qpos(int32_t qlen, const mm128_t *a)
+{
+	int32_t x = (int32_t)a->y;
+	int32_t q_span = a->y>>32 & 0xff;
+	if (a->x>>63)
+		x = qlen - 1 - (x + 1 - q_span); // revert the position to the forward strand of query
+	return x;
+}
+
+static int get_mini_idx(int qlen, const mm128_t *a, int32_t n, const uint64_t *mini_pos)
+{
+	int32_t x, L = 0, R = n - 1;
+	x = get_for_qpos(qlen, a);
+	while (L <= R) { // binary search
+		int32_t m = ((uint64_t)L + R) >> 1;
+		int32_t y = (int32_t)mini_pos[m];
+		if (y < x) L = m + 1;
+		else if (y > x) R = m - 1;
+		else return m;
+	}
+	return -1;
+}
+
+void mm_est_err(const mm_idx_t *mi, int qlen, int n_regs, mm_reg1_t *regs, const mm128_t *a, int32_t n, const uint64_t *mini_pos)
+{
+	int i;
+	uint64_t sum_k = 0;
+	float avg_k;
+
+	if (n == 0) return;
+	for (i = 0; i < n; ++i)
+		sum_k += mini_pos[i] >> 32 & 0xff;
+	avg_k = (float)sum_k / n;
+
+	for (i = 0; i < n_regs; ++i) {
+		mm_reg1_t *r = &regs[i];
+		int32_t st, en, j, k, n_match, n_tot, l_ref;
+		r->div = -1.0f;
+		if (r->cnt == 0) continue;
+		st = en = get_mini_idx(qlen, r->rev? &a[r->as + r->cnt - 1] : &a[r->as], n, mini_pos);
+		if (st < 0) {
+			if (mm_verbose >= 2)
+				fprintf(stderr, "[WARNING] logic inconsistency in mm_est_err(). Please contact the developer.\n");
+			continue;
+		}
+		l_ref = mi->seq[r->rid].len;
+		for (k = 1, j = st + 1, n_match = 1; j < n && k < r->cnt; ++j) {
+			int32_t x;
+			x = get_for_qpos(qlen, r->rev? &a[r->as + r->cnt - 1 - k] : &a[r->as + k]);
+			if (x == (int32_t)mini_pos[j])
+				++k, en = j, ++n_match;
+		}
+		n_tot = en - st + 1;
+		if (r->qs > avg_k && r->rs > avg_k) ++n_tot;
+		if (qlen - r->qs > avg_k && l_ref - r->re > avg_k) ++n_tot;
+		r->div = n_match >= n_tot? 0.0f : (float)(1.0 - pow((double)n_match / n_tot, 1.0 / avg_k));
+	}
+}

example.c

@@ -11,7 +11,13 @@ KSEQ_INIT(gzFile, gzread)
 
 int main(int argc, char *argv[])
 {
+	mm_idxopt_t iopt;
+	mm_mapopt_t mopt;
+	int n_threads = 3;
+
 	mm_verbose = 2; // disable message output to stderr
+	mm_set_opt(0, &iopt, &mopt);
+	mopt.flag |= MM_F_CIGAR; // perform alignment
 
 	if (argc < 3) {
 		fprintf(stderr, "Usage: minimap2-lite <target.fa> <query.fa>\n");
@@ -23,39 +29,35 @@ int main(int argc, char *argv[])
 	assert(f);
 	kseq_t *ks = kseq_init(f);
 
-	// create index for target; we are creating one index for all target sequence
+	// open index reader
-	int n_threads = 4, w = 10, k = 15, is_hpc = 0;
+	mm_idx_reader_t *r = mm_idx_reader_open(argv[1], &iopt, 0);
-	mm_idx_t *mi = mm_idx_build(argv[1], w, k, is_hpc, n_threads);
+	mm_idx_t *mi;
-	assert(mi);
+	while ((mi = mm_idx_reader_read(r, n_threads)) != 0) { // traverse each part of the index
-
+		mm_mapopt_update(&mopt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()!
-	// mapping
+		mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread
-	mm_mapopt_t opt;
+		gzrewind(f);
-	mm_mapopt_init(&opt); // initialize mapping parameters
+		kseq_rewind(ks);
-	mm_mapopt_update(&opt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()!
+		while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence
-	opt.flag |= MM_F_CIGAR; // perform alignment
+			mm_reg1_t *reg;
-	mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread
+			int j, i, n_reg;
-	while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence
+			reg = mm_map(mi, ks->seq.l, ks->seq.s, &n_reg, tbuf, &mopt, 0); // get all hits for the query
-		const mm_reg1_t *reg;
+			for (j = 0; j < n_reg; ++j) { // traverse hits and print them out
-		int j, i, n_reg;
+				mm_reg1_t *r = &reg[j];
-		// get all hits for the query
+				assert(r->p); // with MM_F_CIGAR, this should not be NULL
-		reg = mm_map(mi, ks->seq.l, ks->seq.s, &n_reg, tbuf, &opt, 0);
+				printf("%s\t%d\t%d\t%d\t%c\t", ks->name.s, ks->seq.l, r->qs, r->qe, "+-"[r->rev]);
-		// traverse hits and print them out
+				printf("%s\t%d\t%d\t%d\t%d\t%d\t%d\tcg:Z:", mi->seq[r->rid].name, mi->seq[r->rid].len, r->rs, r->re, r->mlen, r->blen, r->mapq);
-		for (j = 0; j < n_reg; ++j) {
+				for (i = 0; i < r->p->n_cigar; ++i) // IMPORTANT: this gives the CIGAR in the aligned regions. NO soft/hard clippings!
-			const mm_reg1_t *r = &reg[j];
+					printf("%d%c", r->p->cigar[i]>>4, MM_CIGAR_STR[r->p->cigar[i]&0xf]);
-			assert(r->p); // with MM_F_CIGAR, this should not be NULL
+				putchar('\n');
-			printf("%s\t%d\t%d\t%d\t%c\t", ks->name.s, ks->seq.l, r->qs, r->qe, "+-"[r->rev]);
+				free(r->p);
-			printf("%s\t%d\t%d\t%d\t%d\t%d\t%d\tcg:Z:", mi->seq[r->rid].name, mi->seq[r->rid].len, r->rs, r->re,
+			}
-				   r->p->blen - r->p->n_ambi - r->p->n_diff, r->p->blen, r->mapq);
+			free(reg);
-			for (i = 0; i < r->p->n_cigar; ++i) // IMPORTANT: this gives the CIGAR in the aligned regions. NO soft/hard clippings!
-				printf("%d%c", r->p->cigar[i]>>4, "MIDSHN"[r->p->cigar[i]&0xf]);
-			putchar('\n');
 		}
+		mm_tbuf_destroy(tbuf);
+		mm_idx_destroy(mi);
 	}
-	mm_tbuf_destroy(tbuf);
+	mm_idx_reader_close(r); // close the index reader
-
+	kseq_destroy(ks); // close the query file
-	// deallocate index and close the query file
-	mm_idx_destroy(mi);
-	kseq_destroy(ks);
 	gzclose(f);
 	return 0;
 }

format.c

@@ -1,9 +1,13 @@
 #include <stdarg.h>
 #include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 #include <stdio.h>
+#include "kalloc.h"
 #include "mmpriv.h"
 
+static char mm_rg_id[256];
+
 static inline void str_enlarge(kstring_t *s, int l)
 {
 	if (s->l + l + 1 > s->m) {
@@ -39,6 +43,13 @@ static void mm_sprintf_lite(kstring_t *s, const char *fmt, ...)
 				if (c < 0) buf[l++] = '-';
 				str_enlarge(s, l);
 				for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
+			} else if (*p == 'u') {
+				int i, l = 0;
+				uint32_t x;
+				x = va_arg(ap, uint32_t);
+				do { buf[l++] = x%10 + '0'; x /= 10; } while (x > 0);
+				str_enlarge(s, l);
+				for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
 			} else if (*p == 's') {
 				char *r = va_arg(ap, char*);
 				str_copy(s, r, r + strlen(r));
@@ -54,83 +65,429 @@ static void mm_sprintf_lite(kstring_t *s, const char *fmt, ...)
 	s->s[s->l] = 0;
 }
 
-static inline void write_tags(kstring_t *s, const mm_reg1_t *r)
+static char *mm_escape(char *s)
 {
-	mm_sprintf_lite(s, "\tcm:i:%d\ts1:i:%d", r->cnt, r->score);
+	char *p, *q;
-	if (r->parent == r->id) mm_sprintf_lite(s, "\ts2:i:%d", r->subsc);
+	for (p = q = s; *p; ++p) {
-	if (r->split) mm_sprintf_lite(s, "\tzd:i:%d", r->split);
+		if (*p == '\\') {
-	if (r->p) mm_sprintf_lite(s, "\tNM:i:%d\tms:i:%d\tAS:i:%d\tnn:i:%d", r->p->n_diff, r->p->dp_max, r->p->dp_score, r->p->n_ambi);
+			++p;
+			if (*p == 't') *q++ = '\t';
+			else if (*p == '\\') *q++ = '\\';
+		} else *q++ = *p;
+	}
+	*q = '\0';
+	return s;
 }
 
-void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r)
+static int sam_write_rg_line(kstring_t *str, const char *s)
+{
+	char *p, *q, *r, *rg_line = 0;
+	memset(mm_rg_id, 0, 256);
+	if (s == 0) return 0;
+	if (strstr(s, "@RG") != s) {
+		if (mm_verbose >= 1) fprintf(stderr, "[ERROR] the read group line is not started with @RG\n");
+		goto err_set_rg;
+	}
+	if (strstr(s, "\t") != NULL) {
+		if (mm_verbose >= 1) fprintf(stderr, "[ERROR] the read group line contained literal <tab> characters -- replace with escaped tabs: \\t\n");
+		goto err_set_rg;
+	}
+	rg_line = (char*)malloc(strlen(s) + 1);
+	strcpy(rg_line, s);
+	mm_escape(rg_line);
+	if ((p = strstr(rg_line, "\tID:")) == 0) {
+		if (mm_verbose >= 1) fprintf(stderr, "[ERROR] no ID within the read group line\n");
+		goto err_set_rg;
+	}
+	p += 4;
+	for (q = p; *q && *q != '\t' && *q != '\n'; ++q);
+	if (q - p + 1 > 256) {
+		if (mm_verbose >= 1) fprintf(stderr, "[ERROR] @RG:ID is longer than 255 characters\n");
+		goto err_set_rg;
+	}
+	for (q = p, r = mm_rg_id; *q && *q != '\t' && *q != '\n'; ++q)
+		*r++ = *q;
+	mm_sprintf_lite(str, "%s\n", rg_line);
+	return 0;
+
+err_set_rg:
+	free(rg_line);
+	return -1;
+}
+
+int mm_write_sam_hdr(const mm_idx_t *idx, const char *rg, const char *ver, int argc, char *argv[])
+{
+	kstring_t str = {0,0,0};
+	int ret = 0;
+	if (idx) {
+		uint32_t i;
+		for (i = 0; i < idx->n_seq; ++i)
+			mm_sprintf_lite(&str, "@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len);
+	}
+	if (rg) ret = sam_write_rg_line(&str, rg);
+	mm_sprintf_lite(&str, "@PG\tID:minimap2\tPN:minimap2");
+	if (ver) mm_sprintf_lite(&str, "\tVN:%s", ver);
+	if (argc > 1) {
+		int i;
+		mm_sprintf_lite(&str, "\tCL:minimap2");
+		for (i = 1; i < argc; ++i)
+			mm_sprintf_lite(&str, " %s", argv[i]);
+	}
+	mm_err_puts(str.s);
+	free(str.s);
+	return ret;
+}
+
+static void write_cs_core(kstring_t *s, const uint8_t *tseq, const uint8_t *qseq, const mm_reg1_t *r, char *tmp, int no_iden, int write_tag)
+{
+	int i, q_off, t_off;
+	if (write_tag) mm_sprintf_lite(s, "\tcs:Z:");
+	for (i = q_off = t_off = 0; i < (int)r->p->n_cigar; ++i) {
+		int j, op = r->p->cigar[i]&0xf, len = r->p->cigar[i]>>4;
+		assert((op >= MM_CIGAR_MATCH && op <= MM_CIGAR_N_SKIP) || op == MM_CIGAR_EQ_MATCH || op == MM_CIGAR_X_MISMATCH);
+		if (op == MM_CIGAR_MATCH || op == MM_CIGAR_EQ_MATCH || op == MM_CIGAR_X_MISMATCH) {
+			int l_tmp = 0;
+			for (j = 0; j < len; ++j) {
+				if (qseq[q_off + j] != tseq[t_off + j]) {
+					if (l_tmp > 0) {
+						if (!no_iden) {
+							tmp[l_tmp] = 0;
+							mm_sprintf_lite(s, "=%s", tmp);
+						} else mm_sprintf_lite(s, ":%d", l_tmp);
+						l_tmp = 0;
+					}
+					mm_sprintf_lite(s, "*%c%c", "acgtn"[tseq[t_off + j]], "acgtn"[qseq[q_off + j]]);
+				} else tmp[l_tmp++] = "ACGTN"[qseq[q_off + j]];
+			}
+			if (l_tmp > 0) {
+				if (!no_iden) {
+					tmp[l_tmp] = 0;
+					mm_sprintf_lite(s, "=%s", tmp);
+				} else mm_sprintf_lite(s, ":%d", l_tmp);
+			}
+			q_off += len, t_off += len;
+		} else if (op == MM_CIGAR_INS) {
+			for (j = 0, tmp[len] = 0; j < len; ++j)
+				tmp[j] = "acgtn"[qseq[q_off + j]];
+			mm_sprintf_lite(s, "+%s", tmp);
+			q_off += len;
+		} else if (op == MM_CIGAR_DEL) {
+			for (j = 0, tmp[len] = 0; j < len; ++j)
+				tmp[j] = "acgtn"[tseq[t_off + j]];
+			mm_sprintf_lite(s, "-%s", tmp);
+			t_off += len;
+		} else { // intron
+			assert(len >= 2);
+			mm_sprintf_lite(s, "~%c%c%d%c%c", "acgtn"[tseq[t_off]], "acgtn"[tseq[t_off+1]],
+				len, "acgtn"[tseq[t_off+len-2]], "acgtn"[tseq[t_off+len-1]]);
+			t_off += len;
+		}
+	}
+	assert(t_off == r->re - r->rs && q_off == r->qe - r->qs);
+}
+
+static void write_MD_core(kstring_t *s, const uint8_t *tseq, const uint8_t *qseq, const mm_reg1_t *r, char *tmp, int write_tag)
+{
+	int i, q_off, t_off, l_MD = 0;
+	if (write_tag) mm_sprintf_lite(s, "\tMD:Z:");
+	for (i = q_off = t_off = 0; i < (int)r->p->n_cigar; ++i) {
+		int j, op = r->p->cigar[i]&0xf, len = r->p->cigar[i]>>4;
+		assert((op >= MM_CIGAR_MATCH && op <= MM_CIGAR_N_SKIP) || op == MM_CIGAR_EQ_MATCH || op == MM_CIGAR_X_MISMATCH);
+		if (op == MM_CIGAR_MATCH || op == MM_CIGAR_EQ_MATCH || op == MM_CIGAR_X_MISMATCH) {
+			for (j = 0; j < len; ++j) {
+				if (qseq[q_off + j] != tseq[t_off + j]) {
+					mm_sprintf_lite(s, "%d%c", l_MD, "ACGTN"[tseq[t_off + j]]);
+					l_MD = 0;
+				} else ++l_MD;
+			}
+			q_off += len, t_off += len;
+		} else if (op == MM_CIGAR_INS) {
+			q_off += len;
+		} else if (op == MM_CIGAR_DEL) {
+			for (j = 0, tmp[len] = 0; j < len; ++j)
+				tmp[j] = "ACGTN"[tseq[t_off + j]];
+			mm_sprintf_lite(s, "%d^%s", l_MD, tmp);
+			l_MD = 0;
+			t_off += len;
+		} else if (op == MM_CIGAR_N_SKIP) {
+			t_off += len;
+		}
+	}
+	if (l_MD > 0) mm_sprintf_lite(s, "%d", l_MD);
+	assert(t_off == r->re - r->rs && q_off == r->qe - r->qs);
+}
+
+static void write_cs_or_MD(void *km, kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int no_iden, int is_MD, int write_tag, int is_qstrand)
+{
+	extern unsigned char seq_nt4_table[256];
+	int i;
+	uint8_t *qseq, *tseq;
+	char *tmp;
+	if (r->p == 0) return;
+	qseq = (uint8_t*)kmalloc(km, r->qe - r->qs);
+	tseq = (uint8_t*)kmalloc(km, r->re - r->rs);
+	tmp = (char*)kmalloc(km, r->re - r->rs > r->qe - r->qs? r->re - r->rs + 1 : r->qe - r->qs + 1);
+	if (is_qstrand) {
+		mm_idx_getseq2(mi, r->rev, r->rid, r->rs, r->re, tseq);
+		for (i = r->qs; i < r->qe; ++i)
+			qseq[i - r->qs] = seq_nt4_table[(uint8_t)t->seq[i]];
+	} else {
+		mm_idx_getseq(mi, r->rid, r->rs, r->re, tseq);
+		if (!r->rev) {
+			for (i = r->qs; i < r->qe; ++i)
+				qseq[i - r->qs] = seq_nt4_table[(uint8_t)t->seq[i]];
+		} else {
+			for (i = r->qs; i < r->qe; ++i) {
+				uint8_t c = seq_nt4_table[(uint8_t)t->seq[i]];
+				qseq[r->qe - i - 1] = c >= 4? 4 : 3 - c;
+			}
+		}
+	}
+	if (is_MD) write_MD_core(s, tseq, qseq, r, tmp, write_tag);
+	else write_cs_core(s, tseq, qseq, r, tmp, no_iden, write_tag);
+	kfree(km, qseq); kfree(km, tseq); kfree(km, tmp);
+}
+
+int mm_gen_cs_or_MD(void *km, char **buf, int *max_len, const mm_idx_t *mi, const mm_reg1_t *r, const char *seq, int is_MD, int no_iden, int is_qstrand)
+{
+	mm_bseq1_t t;
+	kstring_t str;
+	str.s = *buf, str.l = 0, str.m = *max_len;
+	t.l_seq = strlen(seq);
+	t.seq = (char*)seq;
+	write_cs_or_MD(km, &str, mi, &t, r, no_iden, is_MD, 0, is_qstrand);
+	*max_len = str.m;
+	*buf = str.s;
+	return str.l;
+}
+
+int mm_gen_cs(void *km, char **buf, int *max_len, const mm_idx_t *mi, const mm_reg1_t *r, const char *seq, int no_iden)
+{
+	return mm_gen_cs_or_MD(km, buf, max_len, mi, r, seq, 0, no_iden, 0);
+}
+
+int mm_gen_MD(void *km, char **buf, int *max_len, const mm_idx_t *mi, const mm_reg1_t *r, const char *seq)
+{
+	return mm_gen_cs_or_MD(km, buf, max_len, mi, r, seq, 1, 0, 0);
+}
+
+static inline void write_tags(kstring_t *s, const mm_reg1_t *r)
+{
+	int type;
+	if (r->id == r->parent) type = r->inv? 'I' : 'P';
+	else type = r->inv? 'i' : 'S';
+	if (r->p) {
+		mm_sprintf_lite(s, "\tNM:i:%d\tms:i:%d\tAS:i:%d\tnn:i:%d", r->blen - r->mlen + r->p->n_ambi, r->p->dp_max, r->p->dp_score, r->p->n_ambi);
+		if (r->p->trans_strand == 1 || r->p->trans_strand == 2)
+			mm_sprintf_lite(s, "\tts:A:%c", "?+-?"[r->p->trans_strand]);
+	}
+	mm_sprintf_lite(s, "\ttp:A:%c\tcm:i:%d\ts1:i:%d", type, r->cnt, r->score);
+	if (r->parent == r->id) mm_sprintf_lite(s, "\ts2:i:%d", r->subsc);
+	if (r->p) {
+		char buf[16];
+		double div;
+		div = 1.0 - mm_event_identity(r);
+		if (div == 0.0) buf[0] = '0', buf[1] = 0;
+		else snprintf(buf, 16, "%.4f", 1.0 - mm_event_identity(r));
+		mm_sprintf_lite(s, "\tde:f:%s", buf);
+	} else if (r->div >= 0.0f && r->div <= 1.0f) {
+		char buf[16];
+		if (r->div == 0.0f) buf[0] = '0', buf[1] = 0;
+		else snprintf(buf, 16, "%.4f", r->div);
+		mm_sprintf_lite(s, "\tdv:f:%s", buf);
+	}
+	if (r->split) mm_sprintf_lite(s, "\tzd:i:%d", r->split);
+}
+
+void mm_write_paf3(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int64_t opt_flag, int rep_len)
 {
 	s->l = 0;
+	if (r == 0) {
+		mm_sprintf_lite(s, "%s\t%d\t0\t0\t*\t*\t0\t0\t0\t0\t0\t0", t->name, t->l_seq);
+		if (rep_len >= 0) mm_sprintf_lite(s, "\trl:i:%d", rep_len);
+		return;
+	}
 	mm_sprintf_lite(s, "%s\t%d\t%d\t%d\t%c\t", t->name, t->l_seq, r->qs, r->qe, "+-"[r->rev]);
 	if (mi->seq[r->rid].name) mm_sprintf_lite(s, "%s", mi->seq[r->rid].name);
 	else mm_sprintf_lite(s, "%d", r->rid);
-	mm_sprintf_lite(s, "\t%d\t%d\t%d", mi->seq[r->rid].len, r->rs, r->re);
+	mm_sprintf_lite(s, "\t%d", mi->seq[r->rid].len);
-	if (r->p) mm_sprintf_lite(s, "\t%d\t%d", r->p->blen - r->p->n_ambi - r->p->n_diff, r->p->blen);
+	if ((opt_flag & MM_F_QSTRAND) && r->rev)
-	else mm_sprintf_lite(s, "\t%d\t%d", r->fuzzy_mlen, r->fuzzy_blen);
+		mm_sprintf_lite(s, "\t%d\t%d", mi->seq[r->rid].len - r->re, mi->seq[r->rid].len - r->rs);
+	else
+		mm_sprintf_lite(s, "\t%d\t%d", r->rs, r->re);
+	mm_sprintf_lite(s, "\t%d\t%d", r->mlen, r->blen);
 	mm_sprintf_lite(s, "\t%d", r->mapq);
 	write_tags(s, r);
-	if (r->p) {
+	if (rep_len >= 0) mm_sprintf_lite(s, "\trl:i:%d", rep_len);
+	if (r->p && (opt_flag & MM_F_OUT_CG)) {
 		uint32_t k;
 		mm_sprintf_lite(s, "\tcg:Z:");
 		for (k = 0; k < r->p->n_cigar; ++k)
-			mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MID"[r->p->cigar[k]&0xf]);
+			mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, MM_CIGAR_STR[r->p->cigar[k]&0xf]);
 	}
+	if (r->p && (opt_flag & (MM_F_OUT_CS|MM_F_OUT_MD)))
+		write_cs_or_MD(km, s, mi, t, r, !(opt_flag&MM_F_OUT_CS_LONG), opt_flag&MM_F_OUT_MD, 1, !!(opt_flag&MM_F_QSTRAND));
+	if ((opt_flag & MM_F_COPY_COMMENT) && t->comment)
+		mm_sprintf_lite(s, "\t%s", t->comment);
 }
 
-static char comp_tab[] = {
+void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int64_t opt_flag)
-	  0,   1,	2,	 3,	  4,   5,	6,	 7,	  8,   9,  10,	11,	 12,  13,  14,	15,
+{
-	 16,  17,  18,	19,	 20,  21,  22,	23,	 24,  25,  26,	27,	 28,  29,  30,	31,
+	mm_write_paf3(s, mi, t, r, km, opt_flag, -1);
-	 32,  33,  34,	35,	 36,  37,  38,	39,	 40,  41,  42,	43,	 44,  45,  46,	47,
+}
-	 48,  49,  50,	51,	 52,  53,  54,	55,	 56,  57,  58,	59,	 60,  61,  62,	63,
-	 64, 'T', 'V', 'G', 'H', 'E', 'F', 'C', 'D', 'I', 'J', 'M', 'L', 'K', 'N', 'O',
-	'P', 'Q', 'Y', 'S', 'A', 'A', 'B', 'W', 'X', 'R', 'Z',	91,	 92,  93,  94,	95,
-	 64, 't', 'v', 'g', 'h', 'e', 'f', 'c', 'd', 'i', 'j', 'm', 'l', 'k', 'n', 'o',
-	'p', 'q', 'y', 's', 'a', 'a', 'b', 'w', 'x', 'r', 'z', 123, 124, 125, 126, 127
-};
 
 static void sam_write_sq(kstring_t *s, char *seq, int l, int rev, int comp)
 {
+	extern unsigned char seq_comp_table[256];
 	if (rev) {
 		int i;
 		str_enlarge(s, l);
 		for (i = 0; i < l; ++i) {
 			int c = seq[l - 1 - i];
-			s->s[s->l + i] = c < 128 && comp? comp_tab[c] : c;
+			s->s[s->l + i] = c < 128 && comp? seq_comp_table[c] : c;
 		}
 		s->l += l;
 	} else str_copy(s, seq, seq + l);
 }
 
-void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r)
+static inline const mm_reg1_t *get_sam_pri(int n_regs, const mm_reg1_t *regs)
 {
-	int flag = 0;
+	int i;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].sam_pri)
+			return &regs[i];
+	assert(n_regs == 0);
+	return NULL;
+}
+
+static void write_sam_cigar(kstring_t *s, int sam_flag, int in_tag, int qlen, const mm_reg1_t *r, int64_t opt_flag)
+{
+	if (r->p == 0) {
+		mm_sprintf_lite(s, "*");
+	} else {
+		uint32_t k, clip_len[2];
+		clip_len[0] = r->rev? qlen - r->qe : r->qs;
+		clip_len[1] = r->rev? r->qs : qlen - r->qe;
+		if (in_tag) {
+			int clip_char = (sam_flag&0x800) && !(opt_flag&MM_F_SOFTCLIP)? 5 : 4;
+			mm_sprintf_lite(s, "\tCG:B:I");
+			if (clip_len[0]) mm_sprintf_lite(s, ",%u", clip_len[0]<<4|clip_char);
+			for (k = 0; k < r->p->n_cigar; ++k)
+				mm_sprintf_lite(s, ",%u", r->p->cigar[k]);
+			if (clip_len[1]) mm_sprintf_lite(s, ",%u", clip_len[1]<<4|clip_char);
+		} else {
+			int clip_char = (sam_flag&0x800) && !(opt_flag&MM_F_SOFTCLIP)? 'H' : 'S';
+			assert(clip_len[0] < qlen && clip_len[1] < qlen);
+			if (clip_len[0]) mm_sprintf_lite(s, "%d%c", clip_len[0], clip_char);
+			for (k = 0; k < r->p->n_cigar; ++k)
+				mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, MM_CIGAR_STR[r->p->cigar[k]&0xf]);
+			if (clip_len[1]) mm_sprintf_lite(s, "%d%c", clip_len[1], clip_char);
+		}
+	}
+}
+
+void mm_write_sam3(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regss, const mm_reg1_t *const* regss, void *km, int64_t opt_flag, int rep_len)
+{
+	const int max_bam_cigar_op = 65535;
+	int flag, n_regs = n_regss[seg_idx], cigar_in_tag = 0;
+	int this_rid = -1, this_pos = -1;
+	const mm_reg1_t *regs = regss[seg_idx], *r_prev = NULL, *r_next;
+	const mm_reg1_t *r = n_regs > 0 && reg_idx < n_regs && reg_idx >= 0? &regs[reg_idx] : NULL;
+
+	// find the primary of the previous and the next segments, if they are mapped
+	if (n_seg > 1) {
+		int i, next_sid = (seg_idx + 1) % n_seg;
+		r_next = get_sam_pri(n_regss[next_sid], regss[next_sid]);
+		if (n_seg > 2) {
+			for (i = 1; i <= n_seg - 1; ++i) {
+				int prev_sid = (seg_idx + n_seg - i) % n_seg;
+				if (n_regss[prev_sid] > 0) {
+					r_prev = get_sam_pri(n_regss[prev_sid], regss[prev_sid]);
+					break;
+				}
+			}
+		} else r_prev = r_next;
+	} else r_prev = r_next = NULL;
+
+	// write QNAME
 	s->l = 0;
+	mm_sprintf_lite(s, "%s", t->name);
+	if (n_seg > 1) s->l = mm_qname_len(t->name); // trim the suffix like /1 or /2
+
+	// write flag
+	flag = n_seg > 1? 0x1 : 0x0;
+	if (r == 0) {
+		flag |= 0x4;
+	} else {
+		if (r->rev) flag |= 0x10;
+		if (r->parent != r->id) flag |= 0x100;
+		else if (!r->sam_pri) flag |= 0x800;
+	}
+	if (n_seg > 1) {
+		if (r && r->proper_frag) flag |= 0x2; // TODO: this doesn't work when there are more than 2 segments
+		if (seg_idx == 0) flag |= 0x40;
+		else if (seg_idx == n_seg - 1) flag |= 0x80;
+		if (r_next == NULL) flag |= 0x8;
+		else if (r_next->rev) flag |= 0x20;
+	}
+	mm_sprintf_lite(s, "\t%d", flag);
+
+	// write coordinate, MAPQ and CIGAR
+	if (r == 0) {
+		if (r_prev) {
+			this_rid = r_prev->rid, this_pos = r_prev->rs;
+			mm_sprintf_lite(s, "\t%s\t%d\t0\t*", mi->seq[this_rid].name, this_pos+1);
+		} else mm_sprintf_lite(s, "\t*\t0\t0\t*");
+	} else {
+		this_rid = r->rid, this_pos = r->rs;
+		mm_sprintf_lite(s, "\t%s\t%d\t%d\t", mi->seq[r->rid].name, r->rs+1, r->mapq);
+		if ((opt_flag & MM_F_LONG_CIGAR) && r->p && r->p->n_cigar > max_bam_cigar_op - 2) {
+			int n_cigar = r->p->n_cigar;
+			if (r->qs != 0) ++n_cigar;
+			if (r->qe != t->l_seq) ++n_cigar;
+			if (n_cigar > max_bam_cigar_op)
+				cigar_in_tag = 1;
+		}
+		if (cigar_in_tag) {
+			int slen;
+			if ((flag & 0x900) == 0 || (opt_flag & MM_F_SOFTCLIP)) slen = t->l_seq;
+			else if (flag & 0x100) slen = 0;
+			else slen = r->qe - r->qs;
+			mm_sprintf_lite(s, "%dS%dN", slen, r->re - r->rs);
+		} else write_sam_cigar(s, flag, 0, t->l_seq, r, opt_flag);
+	}
+
+	// write mate positions
+	if (n_seg > 1) {
+		int tlen = 0;
+		if (this_rid >= 0 && r_next) {
+			if (this_rid == r_next->rid) {
+				if (r) {
+					int this_pos5 = r->rev? r->re - 1 : this_pos;
+					int next_pos5 = r_next->rev? r_next->re - 1 : r_next->rs;
+					tlen = next_pos5 - this_pos5;
+				}
+				mm_sprintf_lite(s, "\t=\t");
+			} else mm_sprintf_lite(s, "\t%s\t", mi->seq[r_next->rid].name);
+			mm_sprintf_lite(s, "%d\t", r_next->rs + 1);
+		} else if (r_next) { // && this_rid < 0
+			mm_sprintf_lite(s, "\t%s\t%d\t", mi->seq[r_next->rid].name, r_next->rs + 1);
+		} else if (this_rid >= 0) { // && r_next == NULL
+			mm_sprintf_lite(s, "\t=\t%d\t", this_pos + 1); // next segment will take r's coordinate
+		} else mm_sprintf_lite(s, "\t*\t0\t"); // neither has coordinates
+		if (tlen > 0) ++tlen;
+		else if (tlen < 0) --tlen;
+		mm_sprintf_lite(s, "%d\t", tlen);
+	} else mm_sprintf_lite(s, "\t*\t0\t0\t");
+
+	// write SEQ and QUAL
 	if (r == 0) {
-		mm_sprintf_lite(s, "%s\t4\t*\t0\t0\t*\t*\t0\t0\t", t->name);
 		sam_write_sq(s, t->seq, t->l_seq, 0, 0);
 		mm_sprintf_lite(s, "\t");
 		if (t->qual) sam_write_sq(s, t->qual, t->l_seq, 0, 0);
 		else mm_sprintf_lite(s, "*");
 	} else {
-		if (r->rev) flag |= 0x10;
+		if ((flag & 0x900) == 0 || (opt_flag & MM_F_SOFTCLIP)) {
-		if (r->parent != r->id) flag |= 0x100;
-		else if (!r->sam_pri) flag |= 0x800;
-		mm_sprintf_lite(s, "%s\t%d\t%s\t%d\t%d\t", t->name, flag, mi->seq[r->rid].name, r->rs+1, r->mapq);
-		if (r->p) { // TODO: using hard clippings
-			uint32_t k, clip_len = r->rev? t->l_seq - r->qe : r->qs;
-			int clip_char = (flag&0x800)? 'H' : 'S';
-			if (clip_len) mm_sprintf_lite(s, "%d%c", clip_len, clip_char);
-			for (k = 0; k < r->p->n_cigar; ++k)
-				mm_sprintf_lite(s, "%d%c", r->p->cigar[k]>>4, "MID"[r->p->cigar[k]&0xf]);
-			clip_len = r->rev? r->qs : t->l_seq - r->qe;
-			if (clip_len) mm_sprintf_lite(s, "%d%c", clip_len, clip_char);
-		} else mm_sprintf_lite(s, "*");
-		mm_sprintf_lite(s, "\t*\t0\t0\t");
-		if ((flag & 0x900) == 0) {
 			sam_write_sq(s, t->seq, t->l_seq, r->rev, r->rev);
 			mm_sprintf_lite(s, "\t");
 			if (t->qual) sam_write_sq(s, t->qual, t->l_seq, r->rev, 0);
@@ -143,7 +500,60 @@ void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const m
 			if (t->qual) sam_write_sq(s, t->qual + r->qs, r->qe - r->qs, r->rev, 0);
 			else mm_sprintf_lite(s, "*");
 		}
-		write_tags(s, r);
 	}
+
+	// write tags
+	if (mm_rg_id[0]) mm_sprintf_lite(s, "\tRG:Z:%s", mm_rg_id);
+	if (n_seg > 2) mm_sprintf_lite(s, "\tFI:i:%d", seg_idx);
+	if (r) {
+		write_tags(s, r);
+		if (r->parent == r->id && r->p && n_regs > 1 && regs && r >= regs && r - regs < n_regs) { // supplementary aln may exist
+			int i, n_sa = 0; // n_sa: number of SA fields
+			for (i = 0; i < n_regs; ++i)
+				if (i != r - regs && regs[i].parent == regs[i].id && regs[i].p)
+					++n_sa;
+			if (n_sa > 0) {
+				mm_sprintf_lite(s, "\tSA:Z:");
+				for (i = 0; i < n_regs; ++i) {
+					const mm_reg1_t *q = &regs[i];
+					int l_M, l_I = 0, l_D = 0, clip5 = 0, clip3 = 0;
+					if (r == q || q->parent != q->id || q->p == 0) continue;
+					if (q->qe - q->qs < q->re - q->rs) l_M = q->qe - q->qs, l_D = (q->re - q->rs) - l_M;
+					else l_M = q->re - q->rs, l_I = (q->qe - q->qs) - l_M;
+					clip5 = q->rev? t->l_seq - q->qe : q->qs;
+					clip3 = q->rev? q->qs : t->l_seq - q->qe;
+					mm_sprintf_lite(s, "%s,%d,%c,", mi->seq[q->rid].name, q->rs+1, "+-"[q->rev]);
+					if (clip5) mm_sprintf_lite(s, "%dS", clip5);
+					if (l_M) mm_sprintf_lite(s, "%dM", l_M);
+					if (l_I) mm_sprintf_lite(s, "%dI", l_I);
+					if (l_D) mm_sprintf_lite(s, "%dD", l_D);
+					if (clip3) mm_sprintf_lite(s, "%dS", clip3);
+					mm_sprintf_lite(s, ",%d,%d;", q->mapq, q->blen - q->mlen + q->p->n_ambi);
+				}
+			}
+		}
+		if (r->p && (opt_flag & (MM_F_OUT_CS|MM_F_OUT_MD)))
+			write_cs_or_MD(km, s, mi, t, r, !(opt_flag&MM_F_OUT_CS_LONG), opt_flag&MM_F_OUT_MD, 1, 0);
+		if (cigar_in_tag)
+			write_sam_cigar(s, flag, 1, t->l_seq, r, opt_flag);
+	}
+	if (rep_len >= 0) mm_sprintf_lite(s, "\trl:i:%d", rep_len);
+
+	if ((opt_flag & MM_F_COPY_COMMENT) && t->comment)
+		mm_sprintf_lite(s, "\t%s", t->comment);
+
 	s->s[s->l] = 0; // we always have room for an extra byte (see str_enlarge)
 }
+
+void mm_write_sam2(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regss, const mm_reg1_t *const* regss, void *km, int64_t opt_flag)
+{
+	mm_write_sam3(s, mi, t, seg_idx, reg_idx, n_seg, n_regss, regss, km, opt_flag, -1);
+}
+
+void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs)
+{
+	int i;
+	for (i = 0; i < n_regs; ++i)
+		if (r == &regs[i]) break;
+	mm_write_sam2(s, mi, t, 0, i, 1, &n_regs, &regs, NULL, 0);
+}

hit.c

@@ -1,31 +1,33 @@
 #include <string.h>
+#include <stdlib.h>
 #include <math.h>
 #include "mmpriv.h"
 #include "kalloc.h"
+#include "khash.h"
 
 static inline void mm_cal_fuzzy_len(mm_reg1_t *r, const mm128_t *a)
 {
 	int i;
-	r->fuzzy_mlen = r->fuzzy_blen = 0;
+	r->mlen = r->blen = 0;
 	if (r->cnt <= 0) return;
-	r->fuzzy_mlen = r->fuzzy_blen = a[r->as].y>>32&0xff;
+	r->mlen = r->blen = a[r->as].y>>32&0xff;
 	for (i = r->as + 1; i < r->as + r->cnt; ++i) {
 		int span = a[i].y>>32&0xff;
 		int tl = (int32_t)a[i].x - (int32_t)a[i-1].x;
 		int ql = (int32_t)a[i].y - (int32_t)a[i-1].y;
-		r->fuzzy_blen += tl > ql? tl : ql;
+		r->blen += tl > ql? tl : ql;
-		r->fuzzy_mlen += tl > span && ql > span? span : tl < ql? tl : ql;
+		r->mlen += tl > span && ql > span? span : tl < ql? tl : ql;
 	}
 }
 
-static inline void mm_reg_set_coor(mm_reg1_t *r, int32_t qlen, const mm128_t *a)
+static inline void mm_reg_set_coor(mm_reg1_t *r, int32_t qlen, const mm128_t *a, int is_qstrand)
 { // NB: r->as and r->cnt MUST BE set correctly for this function to work
 	int32_t k = r->as, q_span = (int32_t)(a[k].y>>32&0xff);
 	r->rev = a[k].x>>63;
 	r->rid = a[k].x<<1>>33;
 	r->rs = (int32_t)a[k].x + 1 > q_span? (int32_t)a[k].x + 1 - q_span : 0; // NB: target span may be shorter, so this test is necessary
 	r->re = (int32_t)a[k + r->cnt - 1].x + 1;
-	if (!r->rev) {
+	if (!r->rev || is_qstrand) {
 		r->qs = (int32_t)a[k].y + 1 - q_span;
 		r->qe = (int32_t)a[k + r->cnt - 1].y + 1;
 	} else {
@@ -35,7 +37,19 @@ static inline void mm_reg_set_coor(mm_reg1_t *r, int32_t qlen, const mm128_t *a)
 	mm_cal_fuzzy_len(r, a);
 }
 
-mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) // convert chains to hits
+static inline uint64_t hash64(uint64_t key)
+{
+	key = (~key + (key << 21));
+	key = key ^ key >> 24;
+	key = ((key + (key << 3)) + (key << 8));
+	key = key ^ key >> 14;
+	key = ((key + (key << 2)) + (key << 4));
+	key = key ^ key >> 28;
+	key = (key + (key << 31));
+	return key;
+}
+
+mm_reg1_t *mm_gen_regs(void *km, uint32_t hash, int qlen, int n_u, uint64_t *u, mm128_t *a, int is_qstrand) // convert chains to hits
 {
 	mm128_t *z, tmp;
 	mm_reg1_t *r;
@@ -46,7 +60,9 @@ mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) //
 	// sort by score
 	z = (mm128_t*)kmalloc(km, n_u * 16);
 	for (i = k = 0; i < n_u; ++i) {
-		z[i].x = u[i] >> 32;
+		uint32_t h;
+		h = (uint32_t)hash64((hash64(a[k].x) + hash64(a[k].y)) ^ hash);
+		z[i].x = u[i] ^ h; // u[i] -- higher 32 bits: chain score; lower 32 bits: number of seeds in the chain
 		z[i].y = (uint64_t)k << 32 | (int32_t)u[i];
 		k += (int32_t)u[i];
 	}
@@ -60,82 +76,141 @@ mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a) //
 		mm_reg1_t *ri = &r[i];
 		ri->id = i;
 		ri->parent = MM_PARENT_UNSET;
-		ri->score = z[i].x;
+		ri->score = ri->score0 = z[i].x >> 32;
+		ri->hash = (uint32_t)z[i].x;
 		ri->cnt = (int32_t)z[i].y;
 		ri->as = z[i].y >> 32;
-		mm_reg_set_coor(ri, qlen, a);
+		ri->div = -1.0f;
+		mm_reg_set_coor(ri, qlen, a, is_qstrand);
 	}
 	kfree(km, z);
 	return r;
 }
 
-void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a)
+void mm_mark_alt(const mm_idx_t *mi, int n, mm_reg1_t *r)
+{
+	int i;
+	if (mi->n_alt == 0) return;
+	for (i = 0; i < n; ++i)
+		if (mi->seq[r[i].rid].is_alt)
+			r[i].is_alt = 1;
+}
+
+static inline int mm_alt_score(int score, float alt_diff_frac)
+{
+	if (score < 0) return score;
+	score = (int)(score * (1.0 - alt_diff_frac) + .499);
+	return score > 0? score : 1;
+}
+
+void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a, int is_qstrand)
 {
 	if (n <= 0 || n >= r->cnt) return;
 	*r2 = *r;
 	r2->id = -1;
 	r2->sam_pri = 0;
 	r2->p = 0;
+	r2->split_inv = 0;
 	r2->cnt = r->cnt - n;
 	r2->score = (int32_t)(r->score * ((float)r2->cnt / r->cnt) + .499);
 	r2->as = r->as + n;
 	if (r->parent == r->id) r2->parent = MM_PARENT_TMP_PRI;
-	mm_reg_set_coor(r2, qlen, a);
+	mm_reg_set_coor(r2, qlen, a, is_qstrand);
 	r->cnt -= r2->cnt;
 	r->score -= r2->score;
-	mm_reg_set_coor(r, qlen, a);
+	mm_reg_set_coor(r, qlen, a, is_qstrand);
 	r->split |= 1, r2->split |= 2;
 }
 
-void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r) // and compute mm_reg1_t::subsc
+void mm_set_parent(void *km, float mask_level, int mask_len, int n, mm_reg1_t *r, int sub_diff, int hard_mask_level, float alt_diff_frac) // and compute mm_reg1_t::subsc
 {
 	int i, j, k, *w;
+	uint64_t *cov;
 	if (n <= 0) return;
 	for (i = 0; i < n; ++i) r[i].id = i;
+	cov = (uint64_t*)kmalloc(km, n * sizeof(uint64_t));
 	w = (int*)kmalloc(km, n * sizeof(int));
 	w[0] = 0, r[0].parent = 0;
 	for (i = 1, k = 1; i < n; ++i) {
 		mm_reg1_t *ri = &r[i];
-		int si = ri->qs, ei = ri->qe;
+		int si = ri->qs, ei = ri->qe, n_cov = 0, uncov_len = 0;
-		for (j = 0; j < k; ++j) {
+		if (hard_mask_level) goto skip_uncov;
+		for (j = 0; j < k; ++j) { // traverse existing primary hits to find overlapping hits
 			mm_reg1_t *rp = &r[w[j]];
 			int sj = rp->qs, ej = rp->qe;
-			int min = ej - sj < ei - si? ej - sj : ei - si;
+			if (ej <= si || sj >= ei) continue;
-			int ol = si < sj? (ei < sj? 0 : ei < ej? ei - sj : ej - sj) : (ej < si? 0 : ej < ei? ej - si : ei - si);
+			if (sj < si) sj = si;
-			if (ol > mask_level * min) {
+			if (ej > ei) ej = ei;
+			cov[n_cov++] = (uint64_t)sj<<32 | ej;
+		}
+		if (n_cov == 0) {
+			goto set_parent_test; // no overlapping primary hits; then i is a new primary hit
+		} else if (n_cov > 0) { // there are overlapping primary hits; find the length not covered by existing primary hits
+			int j, x = si;
+			radix_sort_64(cov, cov + n_cov);
+			for (j = 0; j < n_cov; ++j) {
+				if ((int)(cov[j]>>32) > x) uncov_len += (cov[j]>>32) - x;
+				x = (int32_t)cov[j] > x? (int32_t)cov[j] : x;
+			}
+			if (ei > x) uncov_len += ei - x;
+		}
+skip_uncov:
+		for (j = 0; j < k; ++j) { // traverse existing primary hits again
+			mm_reg1_t *rp = &r[w[j]];
+			int sj = rp->qs, ej = rp->qe, min, max, ol;
+			if (ej <= si || sj >= ei) continue; // no overlap
+			min = ej - sj < ei - si? ej - sj : ei - si;
+			max = ej - sj > ei - si? ej - sj : ei - si;
+			ol = si < sj? (ei < sj? 0 : ei < ej? ei - sj : ej - sj) : (ej < si? 0 : ej < ei? ej - si : ei - si); // overlap length; TODO: this can be simplified
+			if ((float)ol / min - (float)uncov_len / max > mask_level && uncov_len <= mask_len) { // then this is a secondary hit
+				int cnt_sub = 0, sci = ri->score;
 				ri->parent = rp->parent;
-				rp->subsc = rp->subsc > ri->score? rp->subsc : ri->score;
+				if (!rp->is_alt && ri->is_alt) sci = mm_alt_score(sci, alt_diff_frac);
-				if (rp->p && ri->p)
+				rp->subsc = rp->subsc > sci? rp->subsc : sci;
-					rp->p->dp_max2 = rp->p->dp_max2 > ri->p->dp_max? rp->p->dp_max2 : ri->p->dp_max;
+				if (ri->cnt >= rp->cnt) cnt_sub = 1;
+				if (rp->p && ri->p && (rp->rid != ri->rid || rp->rs != ri->rs || rp->re != ri->re || ol != min)) { // the last condition excludes identical hits after DP
+					sci = ri->p->dp_max;
+					if (!rp->is_alt && ri->is_alt) sci = mm_alt_score(sci, alt_diff_frac);
+					rp->p->dp_max2 = rp->p->dp_max2 > sci? rp->p->dp_max2 : sci;
+					if (rp->p->dp_max - ri->p->dp_max <= sub_diff) cnt_sub = 1;
+				}
+				if (cnt_sub) ++rp->n_sub;
 				break;
 			}
 		}
-		if (j == k) w[k++] = i, ri->parent = i;
+set_parent_test:
+		if (j == k) w[k++] = i, ri->parent = i, ri->n_sub = 0;
 	}
+	kfree(km, cov);
 	kfree(km, w);
 }
 
-void mm_hit_sort_by_dp(void *km, int *n_regs, mm_reg1_t *r)
+void mm_hit_sort(void *km, int *n_regs, mm_reg1_t *r, float alt_diff_frac)
 {
-	int32_t i, n_aux, n = *n_regs;
+	int32_t i, n_aux, n = *n_regs, has_cigar = 0, no_cigar = 0;
-	uint64_t *aux;
+	mm128_t *aux;
 	mm_reg1_t *t;
 
 	if (n <= 1) return;
-	aux = (uint64_t*)kmalloc(km, n * 8);
+	aux = (mm128_t*)kmalloc(km, n * 16);
 	t = (mm_reg1_t*)kmalloc(km, n * sizeof(mm_reg1_t));
 	for (i = n_aux = 0; i < n; ++i) {
-		if (r[i].cnt > 0) { // squeeze out elements with cnt==0 (soft deleted)
+		if (r[i].inv || r[i].cnt > 0) { // squeeze out elements with cnt==0 (soft deleted)
-			assert(r[i].p);
+			int score;
-			aux[n_aux++] = (uint64_t)r[i].p->dp_max << 32 | i;
+			if (r[i].p) score = r[i].p->dp_max, has_cigar = 1;
+			else score = r[i].score, no_cigar = 1;
+			if (r[i].is_alt) score = mm_alt_score(score, alt_diff_frac);
+			aux[n_aux].x = (uint64_t)score << 32 | r[i].hash;
+			aux[n_aux++].y = i;
 		} else if (r[i].p) {
-			kfree(km, r[i].p);
+			free(r[i].p);
 			r[i].p = 0;
 		}
 	}
-	radix_sort_64(aux, aux + n_aux);
+	assert(has_cigar + no_cigar == 1);
+	radix_sort_128x(aux, aux + n_aux);
 	for (i = n_aux - 1; i >= 0; --i)
-		t[n_aux - 1 - i] = r[(int32_t)aux[i]];
+		t[n_aux - 1 - i] = r[aux[i].y];
 	memcpy(r, t, sizeof(mm_reg1_t) * n_aux);
 	*n_regs = n_aux;
 	kfree(km, aux);
@@ -177,29 +252,52 @@ void mm_sync_regs(void *km, int n_regs, mm_reg1_t *regs) // keep mm_reg1_t::{id,
 	mm_set_sam_pri(n_regs, regs);
 }
 
-void mm_select_sub(void *km, float mask_level, float pri_ratio, int best_n, int *n_, mm_reg1_t *r)
+void mm_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int check_strand, int min_strand_sc, int *n_, mm_reg1_t *r)
 {
 	if (pri_ratio > 0.0f && *n_ > 0) {
 		int i, k, n = *n_, n_2nd = 0;
-		for (i = k = 0; i < n; ++i)
+		for (i = k = 0; i < n; ++i) {
-			if (r[i].parent == i) r[k++] = r[i];
+			int p = r[i].parent;
-			else if (r[i].score >= r[r[i].parent].score * pri_ratio && n_2nd++ < best_n) r[k++] = r[i];
+			if (p == i || r[i].inv) { // primary or inversion
-			else if (r[i].p) free(r[i].p);
+				r[k++] = r[i];
+			} else if ((r[i].score >= r[p].score * pri_ratio || r[i].score + min_diff >= r[p].score) && n_2nd < best_n) {
+				if (!(r[i].qs == r[p].qs && r[i].qe == r[p].qe && r[i].rid == r[p].rid && r[i].rs == r[p].rs && r[i].re == r[p].re)) // not identical hits
+					r[k++] = r[i], ++n_2nd;
+				else if (r[i].p) free(r[i].p);
+			} else if (check_strand && n_2nd < best_n && r[i].score > min_strand_sc && r[i].rev != r[p].rev) {
+				r[i].strand_retained = 1;
+				r[k++] = r[i], ++n_2nd;
+			} else if (r[i].p) free(r[i].p);
+		}
 		if (k != n) mm_sync_regs(km, k, r); // removing hits requires sync()
 		*n_ = k;
 	}
 }
 
-void mm_filter_regs(void *km, const mm_mapopt_t *opt, int *n_regs, mm_reg1_t *regs)
+int mm_filter_strand_retained(int n_regs, mm_reg1_t *r)
+{
+	int i, k;
+	for (i = k = 0; i < n_regs; ++i) {
+		int p = r[i].parent;
+		if (!r[i].strand_retained || r[i].div < r[p].div * 5.0f || r[i].div < 0.01f) {
+			if (k < i) r[k++] = r[i];
+			else ++k;
+		}
+	}
+	return k;
+}
+
+void mm_filter_regs(const mm_mapopt_t *opt, int qlen, int *n_regs, mm_reg1_t *regs)
 { // NB: after this call, mm_reg1_t::parent can be -1 if its parent filtered out
 	int i, k;
 	for (i = k = 0; i < *n_regs; ++i) {
 		mm_reg1_t *r = &regs[i];
 		int flt = 0;
-		if (r->cnt < opt->min_cnt) flt = 1;
+		if (!r->inv && !r->seg_split && r->cnt < opt->min_cnt) flt = 1;
-		if (r->p) {
+		if (r->p) { // these filters are only applied when base-alignment is available
-			if (r->p->blen - r->p->n_ambi - r->p->n_diff < opt->min_chain_score) flt = 1;
+			if (r->mlen < opt->min_chain_score) flt = 1;
 			else if (r->p->dp_max < opt->min_dp_max) flt = 1;
+			else if (r->qs > qlen * opt->max_clip_ratio && qlen - r->qe > qlen * opt->max_clip_ratio) flt = 1;
 			if (flt) free(r->p);
 		}
 		if (!flt) {
@@ -230,76 +328,139 @@ int mm_squeeze_a(void *km, int n_regs, mm_reg1_t *regs, mm128_t *a)
 	return as;
 }
 
-void mm_join_long(void *km, const mm_mapopt_t *opt, int qlen, int *n_regs_, mm_reg1_t *regs, mm128_t *a)
+mm_seg_t *mm_seg_gen(void *km, uint32_t hash, int n_segs, const int *qlens, int n_regs0, const mm_reg1_t *regs0, int *n_regs, mm_reg1_t **regs, const mm128_t *a)
 {
-	int i, n_aux, n_regs = *n_regs_, n_drop = 0;
+	int s, i, j, acc_qlen[MM_MAX_SEG+1], qlen_sum = 0;
-	uint64_t *aux;
+	mm_seg_t *seg;
 
-	if (n_regs < 2) return; // nothing to join
+	assert(n_segs <= MM_MAX_SEG);
-	mm_squeeze_a(km, n_regs, regs, a);
+	for (s = 1, acc_qlen[0] = 0; s < n_segs; ++s)
+		acc_qlen[s] = acc_qlen[s-1] + qlens[s-1];
+	qlen_sum = acc_qlen[n_segs - 1] + qlens[n_segs - 1];
 
-	aux = (uint64_t*)kmalloc(km, n_regs * 8);
+	seg = (mm_seg_t*)kcalloc(km, n_segs, sizeof(mm_seg_t));
-	for (i = n_aux = 0; i < n_regs; ++i)
+	for (s = 0; s < n_segs; ++s) {
-		if (regs[i].parent == i || regs[i].parent < 0)
+		seg[s].u = (uint64_t*)kmalloc(km, n_regs0 * 8);
-			aux[n_aux++] = (uint64_t)regs[i].as << 32 | i;
+		for (i = 0; i < n_regs0; ++i)
-	radix_sort_64(aux, aux + n_aux);
+			seg[s].u[i] = (uint64_t)regs0[i].score << 32;
-
-	for (i = n_aux - 1; i >= 1; --i) {
-		mm_reg1_t *r0 = &regs[(int32_t)aux[i-1]], *r1 = &regs[(int32_t)aux[i]];
-		mm128_t *a0e, *a1s;
-		int max_gap, min_gap, sc_thres;
-
-		// test
-		if (r0->as + r0->cnt != r1->as) continue; // not adjacent in a[]
-		if (r0->rid != r1->rid || r0->rev != r1->rev) continue; // make sure on the same target and strand
-		a0e = &a[r0->as + r0->cnt - 1];
-		a1s = &a[r1->as];
-		if (a1s->x <= a0e->x || (int32_t)a1s->y <= (int32_t)a0e->y) continue; // keep colinearity
-		max_gap = min_gap = (int32_t)a1s->y - (int32_t)a0e->y;
-		max_gap = max_gap > a1s->x - a0e->x? max_gap : a1s->x - a0e->x;
-		min_gap = min_gap < a1s->x - a0e->x? min_gap : a1s->x - a0e->x;
-		if (max_gap > opt->max_join_long || min_gap > opt->max_join_short) continue;
-		sc_thres = (int)((float)opt->min_join_flank_sc / opt->max_join_long * max_gap + .499);
-		if (r0->score < sc_thres || r1->score < sc_thres) continue; // require good flanking chains
-		if (r0->re - r0->rs < max_gap>>1 || r0->qe - r0->qs < max_gap>>1) continue; // require enough flanking length
-		if (r1->re - r1->rs < max_gap>>1 || r1->qe - r1->qs < max_gap>>1) continue;
-
-		// all conditions satisfied; join
-		a[r1->as].y |= 1ULL<<40;
-		r0->cnt += r1->cnt, r0->score += r1->score;
-		mm_reg_set_coor(r0, qlen, a);
-		r1->cnt = 0;
-		r1->parent = r0->id;
-		++n_drop;
 	}
-	kfree(km, aux);
+	for (i = 0; i < n_regs0; ++i) {
-
+		const mm_reg1_t *r = &regs0[i];
-	if (n_drop > 0) { // then fix the hits hierarchy
+		for (j = 0; j < r->cnt; ++j) {
-		for (i = 0; i < n_regs; ++i) { // adjust the mm_reg1_t::parent
+			int sid = (a[r->as + j].y&MM_SEED_SEG_MASK)>>MM_SEED_SEG_SHIFT;
-			mm_reg1_t *r = &regs[i];
+			++seg[sid].u[i];
-			if (r->parent >= 0 && r->id != r->parent) { // fix for secondary hits only
+			++seg[sid].n_a;
-				if (regs[r->parent].parent >= 0 && regs[r->parent].parent != r->parent)
-					r->parent = regs[r->parent].parent;
-			}
 		}
-		mm_filter_regs(km, opt, n_regs_, regs);
-		mm_sync_regs(km, *n_regs_, regs);
 	}
+	for (s = 0; s < n_segs; ++s) {
+		mm_seg_t *sr = &seg[s];
+		for (i = 0, sr->n_u = 0; i < n_regs0; ++i) // squeeze out zero-length per-segment chains
+			if ((int32_t)sr->u[i] != 0)
+				sr->u[sr->n_u++] = sr->u[i];
+		sr->a = (mm128_t*)kmalloc(km, sr->n_a * sizeof(mm128_t));
+		sr->n_a = 0;
+	}
+
+	for (i = 0; i < n_regs0; ++i) {
+		const mm_reg1_t *r = &regs0[i];
+		for (j = 0; j < r->cnt; ++j) {
+			int sid = (a[r->as + j].y&MM_SEED_SEG_MASK)>>MM_SEED_SEG_SHIFT;
+			mm128_t a1 = a[r->as + j];
+			// on reverse strand, the segment position is:
+			//   x_for_cat = qlen_sum - 1 - (int32_t)a1.y - 1 + q_span
+			//   (int32_t)new_a1.y = qlens[sid] - (x_for_cat - acc_qlen[sid] + 1 - q_span) - 1 = (int32_t)a1.y - (qlen_sum - (qlens[sid] + acc_qlen[sid]))
+			a1.y -= a1.x>>63? qlen_sum - (qlens[sid] + acc_qlen[sid]) : acc_qlen[sid];
+			seg[sid].a[seg[sid].n_a++] = a1;
+		}
+	}
+	for (s = 0; s < n_segs; ++s) {
+		regs[s] = mm_gen_regs(km, hash, qlens[s], seg[s].n_u, seg[s].u, seg[s].a, 0);
+		n_regs[s] = seg[s].n_u;
+		for (i = 0; i < n_regs[s]; ++i) {
+			regs[s][i].seg_split = 1;
+			regs[s][i].seg_id = s;
+		}
+	}
+	return seg;
 }
 
-void mm_set_mapq(int n_regs, mm_reg1_t *regs)
+void mm_seg_free(void *km, int n_segs, mm_seg_t *segs)
 {
 	int i;
+	for (i = 0; i < n_segs; ++i) kfree(km, segs[i].u);
+	for (i = 0; i < n_segs; ++i) kfree(km, segs[i].a);
+	kfree(km, segs);
+}
+
+static void mm_set_inv_mapq(void *km, int n_regs, mm_reg1_t *regs)
+{
+	int i, n_aux;
+	mm128_t *aux;
+	if (n_regs < 3) return;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].inv) break;
+	if (i == n_regs) return; // no inversion hits
+
+	aux = (mm128_t*)kmalloc(km, n_regs * 16);
+	for (i = n_aux = 0; i < n_regs; ++i)
+		if (regs[i].parent == i || regs[i].parent < 0)
+			aux[n_aux].y = i, aux[n_aux++].x = (uint64_t)regs[i].rid << 32 | regs[i].rs;
+	radix_sort_128x(aux, aux + n_aux);
+
+	for (i = 1; i < n_aux - 1; ++i) {
+		mm_reg1_t *inv = &regs[aux[i].y];
+		if (inv->inv) {
+			mm_reg1_t *l = &regs[aux[i-1].y];
+			mm_reg1_t *r = &regs[aux[i+1].y];
+			inv->mapq = l->mapq < r->mapq? l->mapq : r->mapq;
+		}
+	}
+	kfree(km, aux);
+}
+
+void mm_set_mapq(void *km, int n_regs, mm_reg1_t *regs, int min_chain_sc, int match_sc, int rep_len, int is_sr)
+{
+	static const float q_coef = 40.0f;
+	int64_t sum_sc = 0;
+	float uniq_ratio;
+	int i;
+	if (n_regs == 0) return;
+	for (i = 0; i < n_regs; ++i)
+		if (regs[i].parent == regs[i].id)
+			sum_sc += regs[i].score;
+	uniq_ratio = (float)sum_sc / (sum_sc + rep_len);
 	for (i = 0; i < n_regs; ++i) {
 		mm_reg1_t *r = &regs[i];
-		if (r->parent == r->id) {
+		if (r->inv) {
-			int mapq;
+			r->mapq = 0;
+		} else if (r->parent == r->id) {
+			int mapq, subsc;
+			float pen_s1 = (r->score > 100? 1.0f : 0.01f * r->score) * uniq_ratio;
+			float pen_cm = r->cnt > 10? 1.0f : 0.1f * r->cnt;
+			pen_cm = pen_s1 < pen_cm? pen_s1 : pen_cm;
+			subsc = r->subsc > min_chain_sc? r->subsc : min_chain_sc;
 			if (r->p && r->p->dp_max2 > 0 && r->p->dp_max > 0) {
-				float identity = (float)(r->p->blen - r->p->n_diff - r->p->n_ambi) / (r->p->blen - r->p->n_ambi);
+				float identity = (float)r->mlen / r->blen;
-				mapq = (int)(identity * 30.0 * (1. - (float)r->p->dp_max2 * r->subsc / r->p->dp_max / r->score) * logf(r->score));
+				float x = (float)r->p->dp_max2 * subsc / r->p->dp_max / r->score0;
-			} else mapq = (int)(30.0 * (1. - (float)r->subsc / r->score) * logf(r->score));
+				mapq = (int)(identity * pen_cm * q_coef * (1.0f - x * x) * logf((float)r->p->dp_max / match_sc));
+				if (!is_sr) {
+					int mapq_alt = (int)(6.02f * identity * identity * (r->p->dp_max - r->p->dp_max2) / match_sc + .499f); // BWA-MEM like mapQ, mostly for short reads
+					mapq = mapq < mapq_alt? mapq : mapq_alt; // in case the long-read heuristic fails
+				}
+			} else {
+				float x = (float)subsc / r->score0;
+				if (r->p) {
+					float identity = (float)r->mlen / r->blen;
+					mapq = (int)(identity * pen_cm * q_coef * (1.0f - x) * logf((float)r->p->dp_max / match_sc));
+				} else {
+					mapq = (int)(pen_cm * q_coef * (1.0f - x) * logf(r->score));
+				}
+			}
+			mapq -= (int)(4.343f * logf(r->n_sub + 1) + .499f);
 			mapq = mapq > 0? mapq : 0;
 			r->mapq = mapq < 60? mapq : 60;
+			if (r->p && r->p->dp_max > r->p->dp_max2 && r->mapq == 0) r->mapq = 1;
 		} else r->mapq = 0;
 	}
+	mm_set_inv_mapq(km, n_regs, regs);
 }

index.c

@@ -1,8 +1,13 @@
 #include <stdlib.h>
 #include <assert.h>
+#if defined(WIN32) || defined(_WIN32)
+#include <io.h> // for open(2)
+#else
 #include <unistd.h>
+#endif
 #include <fcntl.h>
 #include <stdio.h>
+#define __STDC_LIMIT_MACROS
 #include "kthread.h"
 #include "bseq.h"
 #include "minimap.h"
@@ -15,31 +20,62 @@
 KHASH_INIT(idx, uint64_t, uint64_t, 1, idx_hash, idx_eq)
 typedef khash_t(idx) idxhash_t;
 
+KHASH_MAP_INIT_STR(str, uint32_t)
+
 #define kroundup64(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, (x)|=(x)>>32, ++(x))
 
-mm_idx_t *mm_idx_init(int w, int k, int b, int is_hpc)
+typedef struct mm_idx_bucket_s {
+	mm128_v a;   // (minimizer, position) array
+	int32_t n;   // size of the _p_ array
+	uint64_t *p; // position array for minimizers appearing >1 times
+	void *h;     // hash table indexing _p_ and minimizers appearing once
+} mm_idx_bucket_t;
+
+typedef struct {
+	int32_t st, en, max; // max is not used for now
+	int32_t score:30, strand:2;
+} mm_idx_intv1_t;
+
+typedef struct mm_idx_intv_s {
+	int32_t n, m;
+	mm_idx_intv1_t *a;
+} mm_idx_intv_t;
+
+mm_idx_t *mm_idx_init(int w, int k, int b, int flag)
 {
 	mm_idx_t *mi;
 	if (k*2 < b) b = k * 2;
 	if (w < 1) w = 1;
 	mi = (mm_idx_t*)calloc(1, sizeof(mm_idx_t));
-	mi->w = w, mi->k = k, mi->b = b, mi->is_hpc = is_hpc;
+	mi->w = w, mi->k = k, mi->b = b, mi->flag = flag;
 	mi->B = (mm_idx_bucket_t*)calloc(1<<b, sizeof(mm_idx_bucket_t));
+	if (!(mm_dbg_flag & 1)) mi->km = km_init();
 	return mi;
 }
 
 void mm_idx_destroy(mm_idx_t *mi)
 {
-	int i;
+	uint32_t i;
 	if (mi == 0) return;
-	for (i = 0; i < 1<<mi->b; ++i) {
+	if (mi->h) kh_destroy(str, (khash_t(str)*)mi->h);
-		free(mi->B[i].p);
+	if (mi->B) {
-		free(mi->B[i].a.a);
+		for (i = 0; i < 1U<<mi->b; ++i) {
-		kh_destroy(idx, (idxhash_t*)mi->B[i].h);
+			free(mi->B[i].p);
+			free(mi->B[i].a.a);
+			kh_destroy(idx, (idxhash_t*)mi->B[i].h);
+		}
 	}
-	for (i = 0; i < mi->n_seq; ++i)
+	if (mi->I) {
-		free(mi->seq[i].name);
+		for (i = 0; i < mi->n_seq; ++i)
-	free(mi->seq); free(mi->B); free(mi->S); free(mi);
+			free(mi->I[i].a);
+		free(mi->I);
+	}
+	if (!mi->km) {
+		for (i = 0; i < mi->n_seq; ++i)
+			free(mi->seq[i].name);
+		free(mi->seq);
+	} else km_destroy(mi->km);
+	free(mi->B); free(mi->S); free(mi);
 }
 
 const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n)
@@ -63,14 +99,15 @@ const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n)
 
 void mm_idx_stat(const mm_idx_t *mi)
 {
-	int i, n = 0, n1 = 0;
+	int n = 0, n1 = 0;
+	uint32_t i;
 	uint64_t sum = 0, len = 0;
-	fprintf(stderr, "[M::%s] kmer size: %d; skip: %d; is_HPC: %d; #seq: %d\n", __func__, mi->k, mi->w, mi->is_hpc, mi->n_seq);
+	fprintf(stderr, "[M::%s] kmer size: %d; skip: %d; is_hpc: %d; #seq: %d\n", __func__, mi->k, mi->w, mi->flag&MM_I_HPC, mi->n_seq);
 	for (i = 0; i < mi->n_seq; ++i)
 		len += mi->seq[i].len;
-	for (i = 0; i < 1<<mi->b; ++i)
+	for (i = 0; i < 1U<<mi->b; ++i)
 		if (mi->B[i].h) n += kh_size((idxhash_t*)mi->B[i].h);
-	for (i = 0; i < 1<<mi->b; ++i) {
+	for (i = 0; i < 1U<<mi->b; ++i) {
 		idxhash_t *h = (idxhash_t*)mi->B[i].h;
 		khint_t k;
 		if (h == 0) continue;
@@ -80,8 +117,36 @@ void mm_idx_stat(const mm_idx_t *mi)
 				if (kh_key(h, k)&1) ++n1;
 			}
 	}
-	fprintf(stderr, "[M::%s::%.3f*%.2f] distinct minimizers: %d (%.2f%% are singletons); average occurrences: %.3lf; average spacing: %.3lf\n",
+	fprintf(stderr, "[M::%s::%.3f*%.2f] distinct minimizers: %d (%.2f%% are singletons); average occurrences: %.3lf; average spacing: %.3lf; total length: %ld\n",
-			__func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), n, 100.0*n1/n, (double)sum / n, (double)len / sum);
+			__func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), n, 100.0*n1/n, (double)sum / n, (double)len / sum, (long)len);
+}
+
+int mm_idx_index_name(mm_idx_t *mi)
+{
+	khash_t(str) *h;
+	uint32_t i;
+	int has_dup = 0, absent;
+	if (mi->h) return 0;
+	h = kh_init(str);
+	for (i = 0; i < mi->n_seq; ++i) {
+		khint_t k;
+		k = kh_put(str, h, mi->seq[i].name, &absent);
+		if (absent) kh_val(h, k) = i;
+		else has_dup = 1;
+	}
+	mi->h = h;
+	if (has_dup && mm_verbose >= 2)
+		fprintf(stderr, "[WARNING] some database sequences have identical sequence names\n");
+	return has_dup;
+}
+
+int mm_idx_name2id(const mm_idx_t *mi, const char *name)
+{
+	khash_t(str) *h = (khash_t(str)*)mi->h;
+	khint_t k;
+	if (h == 0) return -2;
+	k = kh_get(str, h, name);
+	return k == kh_end(h)? -1 : kh_val(h, k);
 }
 
 int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq)
@@ -96,13 +161,35 @@ int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, ui
 	return en - st;
 }
 
-uint32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
+int mm_idx_getseq_rev(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq)
+{
+	uint64_t i, st1, en1;
+	const mm_idx_seq_t *s;
+	if (rid >= mi->n_seq || st >= mi->seq[rid].len) return -1;
+	s = &mi->seq[rid];
+	if (en > s->len) en = s->len;
+	st1 = s->offset + (s->len - en);
+	en1 = s->offset + (s->len - st);
+	for (i = st1; i < en1; ++i) {
+		uint8_t c = mm_seq4_get(mi->S, i);
+		seq[en1 - i - 1] = c < 4? 3 - c : c;
+	}
+	return en - st;
+}
+
+int mm_idx_getseq2(const mm_idx_t *mi, int is_rev, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq)
+{
+	if (is_rev) return mm_idx_getseq_rev(mi, rid, st, en, seq);
+	else return mm_idx_getseq(mi, rid, st, en, seq);
+}
+
+int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
 {
 	int i;
 	size_t n = 0;
 	uint32_t thres;
 	khint_t *a, k;
-	if (f <= 0.) return UINT32_MAX;
+	if (f <= 0.) return INT32_MAX;
 	for (i = 0; i < 1<<mi->b; ++i)
 		if (mi->B[i].h) n += kh_size((idxhash_t*)mi->B[i].h);
 	a = (uint32_t*)malloc(n * 4);
@@ -125,7 +212,8 @@ uint32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
 
 static void worker_post(void *g, long i, int tid)
 {
-	int j, start_a, start_p, n, n_keys;
+	int n, n_keys;
+	size_t j, start_a, start_p;
 	idxhash_t *h;
 	mm_idx_t *mi = (mm_idx_t*)g;
 	mm_idx_bucket_t *b = &mi->B[i];
@@ -153,7 +241,7 @@ static void worker_post(void *g, long i, int tid)
 			int absent;
 			mm128_t *p = &b->a.a[j-1];
 			itr = kh_put(idx, h, p->x>>8>>mi->b<<1, &absent);
-			assert(absent && j - start_a == n);
+			assert(absent && j == start_a + n);
 			if (n == 1) {
 				kh_key(h, itr) |= 1;
 				kh_val(h, itr) = p->y;
@@ -169,10 +257,10 @@ static void worker_post(void *g, long i, int tid)
 		} else ++n;
 	}
 	b->h = h;
-	assert(b->n == start_p);
+	assert(b->n == (int32_t)start_p);
 
 	// deallocate and clear b->a
-	free(b->a.a);
+	kfree(0, b->a.a);
 	b->a.n = b->a.m = 0, b->a.a = 0;
 }
  
@@ -190,7 +278,7 @@ static void mm_idx_post(mm_idx_t *mi, int n_threads)
 #include "bseq.h"
 
 typedef struct {
-	int mini_batch_size, keep_name;
+	int mini_batch_size;
 	uint64_t batch_size, sum_len;
 	mm_bseq_file_t *fp;
 	mm_idx_t *mi;
@@ -222,37 +310,42 @@ static void *worker_pipeline(void *shared, int step, void *in)
 		s->seq = mm_bseq_read(p->fp, p->mini_batch_size, 0, &s->n_seq); // read a mini-batch
 		if (s->seq) {
 			uint32_t old_m, m;
-			uint64_t sum_len, old_max_len, max_len;
 			assert((uint64_t)p->mi->n_seq + s->n_seq <= UINT32_MAX); // to prevent integer overflow
 			// make room for p->mi->seq
 			old_m = p->mi->n_seq, m = p->mi->n_seq + s->n_seq;
 			kroundup32(m); kroundup32(old_m);
 			if (old_m != m)
-				p->mi->seq = (mm_idx_seq_t*)realloc(p->mi->seq, m * sizeof(mm_idx_seq_t));
+				p->mi->seq = (mm_idx_seq_t*)krealloc(p->mi->km, p->mi->seq, m * sizeof(mm_idx_seq_t));
 			// make room for p->mi->S
-			for (i = 0, sum_len = 0; i < s->n_seq; ++i) sum_len += s->seq[i].l_seq;
+			if (!(p->mi->flag & MM_I_NO_SEQ)) {
-			old_max_len = (p->sum_len + 7) / 8;
+				uint64_t sum_len, old_max_len, max_len;
-			max_len = (p->sum_len + sum_len + 7) / 8;
+				for (i = 0, sum_len = 0; i < s->n_seq; ++i) sum_len += s->seq[i].l_seq;
-			kroundup64(old_max_len); kroundup64(max_len);
+				old_max_len = (p->sum_len + 7) / 8;
-			if (old_max_len != max_len) {
+				max_len = (p->sum_len + sum_len + 7) / 8;
-				p->mi->S = (uint32_t*)realloc(p->mi->S, max_len * 4);
+				kroundup64(old_max_len); kroundup64(max_len);
-				memset(&p->mi->S[old_max_len], 0, 4 * (max_len - old_max_len));
+				if (old_max_len != max_len) {
+					p->mi->S = (uint32_t*)realloc(p->mi->S, max_len * 4);
+					memset(&p->mi->S[old_max_len], 0, 4 * (max_len - old_max_len));
+				}
 			}
 			// populate p->mi->seq
 			for (i = 0; i < s->n_seq; ++i) {
 				mm_idx_seq_t *seq = &p->mi->seq[p->mi->n_seq];
 				uint32_t j;
-				if (p->keep_name) {
+				if (!(p->mi->flag & MM_I_NO_NAME)) {
-					assert(strlen(s->seq[i].name) <= 254); // a long query name breaks BAM
+					seq->name = (char*)kmalloc(p->mi->km, strlen(s->seq[i].name) + 1);
-					seq->name = strdup(s->seq[i].name);
+					strcpy(seq->name, s->seq[i].name);
 				} else seq->name = 0;
 				seq->len = s->seq[i].l_seq;
 				seq->offset = p->sum_len;
+				seq->is_alt = 0;
 				// copy the sequence
-				for (j = 0; j < seq->len; ++j) { // TODO: this is not the fastest way, but let's first see if speed matters here
+				if (!(p->mi->flag & MM_I_NO_SEQ)) {
-					uint64_t o = p->sum_len + j;
+					for (j = 0; j < seq->len; ++j) { // TODO: this is not the fastest way, but let's first see if speed matters here
-					int c = seq_nt4_table[(uint8_t)s->seq[i].seq[j]];
+						uint64_t o = p->sum_len + j;
-					mm_seq4_set(p->mi->S, o, c);
+						int c = seq_nt4_table[(uint8_t)s->seq[i].seq[j]];
+						mm_seq4_set(p->mi->S, o, c);
+					}
 				}
 				// update p->sum_len and p->mi->n_seq
 				p->sum_len += seq->len;
@@ -264,7 +357,10 @@ static void *worker_pipeline(void *shared, int step, void *in)
         step_t *s = (step_t*)in;
 		for (i = 0; i < s->n_seq; ++i) {
 			mm_bseq1_t *t = &s->seq[i];
-			mm_sketch(0, t->seq, t->l_seq, p->mi->w, p->mi->k, t->rid, p->mi->is_hpc, &s->a);
+			if (t->l_seq > 0)
+				mm_sketch(0, t->seq, t->l_seq, p->mi->w, p->mi->k, t->rid, p->mi->flag&MM_I_HPC, &s->a);
+			else if (mm_verbose >= 2)
+				fprintf(stderr, "[WARNING] the length database sequence '%s' is 0\n", t->name);
 			free(t->seq); free(t->name);
 		}
 		free(s->seq); s->seq = 0;
@@ -272,21 +368,20 @@ static void *worker_pipeline(void *shared, int step, void *in)
     } else if (step == 2) { // dispatch sketch to buckets
         step_t *s = (step_t*)in;
 		mm_idx_add(p->mi, s->a.n, s->a.a);
-		free(s->a.a); free(s);
+		kfree(0, s->a.a); free(s);
 	}
     return 0;
 }
 
-mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int is_hpc, int mini_batch_size, int n_threads, uint64_t batch_size, int keep_name)
+mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size)
 {
 	pipeline_t pl;
+	if (fp == 0 || mm_bseq_eof(fp)) return 0;
 	memset(&pl, 0, sizeof(pipeline_t));
-	pl.mini_batch_size = mini_batch_size < batch_size? mini_batch_size : batch_size;
+	pl.mini_batch_size = (uint64_t)mini_batch_size < batch_size? mini_batch_size : batch_size;
-	pl.keep_name = keep_name;
 	pl.batch_size = batch_size;
 	pl.fp = fp;
-	if (pl.fp == 0) return 0;
+	pl.mi = mm_idx_init(w, k, b, flag);
-	pl.mi = mm_idx_init(w, k, b, is_hpc);
 
 	kt_pipeline(n_threads < 3? n_threads : 3, worker_pipeline, &pl, 3);
 	if (mm_verbose >= 3)
@@ -299,17 +394,67 @@ mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int is_hpc, int mi
 	return pl.mi;
 }
 
-mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads) // a simpler interface
+mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads) // a simpler interface; deprecated
 {
 	mm_bseq_file_t *fp;
 	mm_idx_t *mi;
 	fp = mm_bseq_open(fn);
 	if (fp == 0) return 0;
-	mi = mm_idx_gen(fp, w, k, MM_IDX_DEF_B, is_hpc, 1<<18, n_threads, UINT64_MAX, 1);
+	mi = mm_idx_gen(fp, w, k, 14, flag, 1<<18, n_threads, UINT64_MAX);
 	mm_bseq_close(fp);
 	return mi;
 }
 
+mm_idx_t *mm_idx_str(int w, int k, int is_hpc, int bucket_bits, int n, const char **seq, const char **name)
+{
+	uint64_t sum_len = 0;
+	mm128_v a = {0,0,0};
+	mm_idx_t *mi;
+	khash_t(str) *h;
+	int i, flag = 0;
+
+	if (n <= 0) return 0;
+	for (i = 0; i < n; ++i) // get the total length
+		sum_len += strlen(seq[i]);
+	if (is_hpc) flag |= MM_I_HPC;
+	if (name == 0) flag |= MM_I_NO_NAME;
+	if (bucket_bits < 0) bucket_bits = 14;
+	mi = mm_idx_init(w, k, bucket_bits, flag);
+	mi->n_seq = n;
+	mi->seq = (mm_idx_seq_t*)kcalloc(mi->km, n, sizeof(mm_idx_seq_t)); // ->seq is allocated from km
+	mi->S = (uint32_t*)calloc((sum_len + 7) / 8, 4);
+	mi->h = h = kh_init(str);
+	for (i = 0, sum_len = 0; i < n; ++i) {
+		const char *s = seq[i];
+		mm_idx_seq_t *p = &mi->seq[i];
+		uint32_t j;
+		if (name && name[i]) {
+			int absent;
+			p->name = (char*)kmalloc(mi->km, strlen(name[i]) + 1);
+			strcpy(p->name, name[i]);
+			kh_put(str, h, p->name, &absent);
+			assert(absent);
+		}
+		p->offset = sum_len;
+		p->len = strlen(s);
+		p->is_alt = 0;
+		for (j = 0; j < p->len; ++j) {
+			int c = seq_nt4_table[(uint8_t)s[j]];
+			uint64_t o = sum_len + j;
+			mm_seq4_set(mi->S, o, c);
+		}
+		sum_len += p->len;
+		if (p->len > 0) {
+			a.n = 0;
+			mm_sketch(0, s, p->len, w, k, i, is_hpc, &a);
+			mm_idx_add(mi, a.n, a.a);
+		}
+	}
+	free(a.a);
+	mm_idx_post(mi, 1);
+	return mi;
+}
+
 /*************
  * index I/O *
  *************/
@@ -317,17 +462,20 @@ mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads)
 void mm_idx_dump(FILE *fp, const mm_idx_t *mi)
 {
 	uint64_t sum_len = 0;
-	uint32_t x[5];
+	uint32_t x[5], i;
-	int i;
 
-	x[0] = mi->w, x[1] = mi->k, x[2] = mi->b, x[3] = mi->n_seq, x[4] = mi->is_hpc;
+	x[0] = mi->w, x[1] = mi->k, x[2] = mi->b, x[3] = mi->n_seq, x[4] = mi->flag;
 	fwrite(MM_IDX_MAGIC, 1, 4, fp);
 	fwrite(x, 4, 5, fp);
 	for (i = 0; i < mi->n_seq; ++i) {
-		uint8_t l;
+		if (mi->seq[i].name) {
-		l = strlen(mi->seq[i].name);
+			uint8_t l = strlen(mi->seq[i].name);
-		fwrite(&l, 1, 1, fp);
+			fwrite(&l, 1, 1, fp);
-		fwrite(mi->seq[i].name, 1, l, fp);
+			fwrite(mi->seq[i].name, 1, l, fp);
+		} else {
+			uint8_t l = 0;
+			fwrite(&l, 1, 1, fp);
+		}
 		fwrite(&mi->seq[i].len, 4, 1, fp);
 		sum_len += mi->seq[i].len;
 	}
@@ -347,15 +495,15 @@ void mm_idx_dump(FILE *fp, const mm_idx_t *mi)
 			fwrite(x, 8, 2, fp);
 		}
 	}
-	fwrite(mi->S, 4, (sum_len + 7) / 8, fp);
+	if (!(mi->flag & MM_I_NO_SEQ))
+		fwrite(mi->S, 4, (sum_len + 7) / 8, fp);
 	fflush(fp);
 }
 
 mm_idx_t *mm_idx_load(FILE *fp)
 {
-	int i;
 	char magic[4];
-	uint32_t x[5];
+	uint32_t x[5], i;
 	uint64_t sum_len = 0;
 	mm_idx_t *mi;
 
@@ -364,16 +512,19 @@ mm_idx_t *mm_idx_load(FILE *fp)
 	if (fread(x, 4, 5, fp) != 5) return 0;
 	mi = mm_idx_init(x[0], x[1], x[2], x[4]);
 	mi->n_seq = x[3];
-	mi->seq = (mm_idx_seq_t*)calloc(mi->n_seq, sizeof(mm_idx_seq_t));
+	mi->seq = (mm_idx_seq_t*)kcalloc(mi->km, mi->n_seq, sizeof(mm_idx_seq_t));
 	for (i = 0; i < mi->n_seq; ++i) {
 		uint8_t l;
 		mm_idx_seq_t *s = &mi->seq[i];
 		fread(&l, 1, 1, fp);
-		s->name = (char*)malloc(l + 1);
+		if (l) {
-		fread(s->name, 1, l, fp);
+			s->name = (char*)kmalloc(mi->km, l + 1);
-		s->name[l] = 0;
+			fread(s->name, 1, l, fp);
+			s->name[l] = 0;
+		}
 		fread(&s->len, 4, 1, fp);
 		s->offset = sum_len;
+		s->is_alt = 0;
 		sum_len += s->len;
 	}
 	for (i = 0; i < 1<<mi->b; ++i) {
@@ -397,26 +548,228 @@ mm_idx_t *mm_idx_load(FILE *fp)
 			kh_val(h, k) = x[1];
 		}
 	}
-	mi->S = (uint32_t*)malloc((sum_len + 7) / 8 * 4);
+	if (!(mi->flag & MM_I_NO_SEQ)) {
-	fread(mi->S, 4, (sum_len + 7) / 8, fp);
+		mi->S = (uint32_t*)malloc((sum_len + 7) / 8 * 4);
+		fread(mi->S, 4, (sum_len + 7) / 8, fp);
+	}
 	return mi;
 }
 
-int mm_idx_is_idx(const char *fn)
+int64_t mm_idx_is_idx(const char *fn)
 {
 	int fd, is_idx = 0;
-	off_t ret;
+	int64_t ret, off_end;
 	char magic[4];
 
 	if (strcmp(fn, "-") == 0) return 0; // read from pipe; not an index
 	fd = open(fn, O_RDONLY);
 	if (fd < 0) return -1; // error
-	if ((ret = lseek(fd, 0, SEEK_END)) >= 4) {
+#ifdef WIN32
+	if ((off_end = _lseeki64(fd, 0, SEEK_END)) >= 4) {
+		_lseeki64(fd, 0, SEEK_SET);
+#else
+	if ((off_end = lseek(fd, 0, SEEK_END)) >= 4) {
 		lseek(fd, 0, SEEK_SET);
+#endif // WIN32
 		ret = read(fd, magic, 4);
 		if (ret == 4 && strncmp(magic, MM_IDX_MAGIC, 4) == 0)
 			is_idx = 1;
 	}
 	close(fd);
-	return is_idx;
+	return is_idx? off_end : 0;
+}
+
+mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out)
+{
+	int64_t is_idx;
+	mm_idx_reader_t *r;
+	is_idx = mm_idx_is_idx(fn);
+	if (is_idx < 0) return 0; // failed to open the index
+	r = (mm_idx_reader_t*)calloc(1, sizeof(mm_idx_reader_t));
+	r->is_idx = is_idx;
+	if (opt) r->opt = *opt;
+	else mm_idxopt_init(&r->opt);
+	if (r->is_idx) {
+		r->fp.idx = fopen(fn, "rb");
+		r->idx_size = is_idx;
+	} else r->fp.seq = mm_bseq_open(fn);
+	if (fn_out) r->fp_out = fopen(fn_out, "wb");
+	return r;
+}
+
+void mm_idx_reader_close(mm_idx_reader_t *r)
+{
+	if (r->is_idx) fclose(r->fp.idx);
+	else mm_bseq_close(r->fp.seq);
+	if (r->fp_out) fclose(r->fp_out);
+	free(r);
+}
+
+mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
+{
+	mm_idx_t *mi;
+	if (r->is_idx) {
+		mi = mm_idx_load(r->fp.idx);
+		if (mi && mm_verbose >= 2 && (mi->k != r->opt.k || mi->w != r->opt.w || (mi->flag&MM_I_HPC) != (r->opt.flag&MM_I_HPC)))
+			fprintf(stderr, "[WARNING]\033[1;31m Indexing parameters (-k, -w or -H) overridden by parameters used in the prebuilt index.\033[0m\n");
+	} else
+		mi = mm_idx_gen(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.flag, r->opt.mini_batch_size, n_threads, r->opt.batch_size);
+	if (mi) {
+		if (r->fp_out) mm_idx_dump(r->fp_out, mi);
+		mi->index = r->n_parts++;
+	}
+	return mi;
+}
+
+int mm_idx_reader_eof(const mm_idx_reader_t *r) // TODO: in extremely rare cases, mm_bseq_eof() might not work
+{
+	return r->is_idx? (feof(r->fp.idx) || ftell(r->fp.idx) == r->idx_size) : mm_bseq_eof(r->fp.seq);
+}
+
+#include <ctype.h>
+#include <zlib.h>
+#include "ksort.h"
+#include "kseq.h"
+KSTREAM_DECLARE(gzFile, gzread)
+
+int mm_idx_alt_read(mm_idx_t *mi, const char *fn)
+{
+	int n_alt = 0;
+	gzFile fp;
+	kstream_t *ks;
+	kstring_t str = {0,0,0};
+	fp = fn && strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+	if (fp == 0) return -1;
+	ks = ks_init(fp);
+	if (mi->h == 0) mm_idx_index_name(mi);
+	while (ks_getuntil(ks, KS_SEP_LINE, &str, 0) >= 0) {
+		char *p;
+		int id;
+		for (p = str.s; *p && !isspace(*p); ++p) { }
+		*p = 0;
+		id = mm_idx_name2id(mi, str.s);
+		if (id >= 0) mi->seq[id].is_alt = 1, ++n_alt;
+	}
+	mi->n_alt = n_alt;
+	if (mm_verbose >= 3)
+		fprintf(stderr, "[M::%s] found %d ALT contigs\n", __func__, n_alt);
+	return n_alt;
+}
+
+#define sort_key_bed(a) ((a).st)
+KRADIX_SORT_INIT(bed, mm_idx_intv1_t, sort_key_bed, 4)
+
+mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn, int read_junc)
+{
+	gzFile fp;
+	kstream_t *ks;
+	kstring_t str = {0,0,0};
+	mm_idx_intv_t *I;
+
+	fp = fn && strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+	if (fp == 0) return 0;
+	I = (mm_idx_intv_t*)calloc(mi->n_seq, sizeof(*I));
+	ks = ks_init(fp);
+	while (ks_getuntil(ks, KS_SEP_LINE, &str, 0) >= 0) {
+		mm_idx_intv_t *r;
+		mm_idx_intv1_t t = {-1,-1,-1,-1,0};
+		char *p, *q, *bl, *bs;
+		int32_t i, id = -1, n_blk = 0;
+		for (p = q = str.s, i = 0;; ++p) {
+			if (*p == 0 || *p == '\t') {
+				int32_t c = *p;
+				*p = 0;
+				if (i == 0) { // chr
+					id = mm_idx_name2id(mi, q);
+					if (id < 0) break; // unknown name; TODO: throw a warning
+				} else if (i == 1) { // start
+					t.st = atol(q); // TODO: watch out integer overflow!
+					if (t.st < 0) break;
+				} else if (i == 2) { // end
+					t.en = atol(q);
+					if (t.en < 0) break;
+				} else if (i == 4) { // BED score
+					t.score = atol(q);
+				} else if (i == 5) { // strand
+					t.strand = *q == '+'? 1 : *q == '-'? -1 : 0;
+				} else if (i == 9) {
+					if (!isdigit(*q)) break;
+					n_blk = atol(q);
+				} else if (i == 10) {
+					bl = q;
+				} else if (i == 11) {
+					bs = q;
+					break;
+				}
+				if (c == 0) break;
+				++i, q = p + 1;
+			}
+		}
+		if (id < 0 || t.st < 0 || t.st >= t.en) continue;
+		r = &I[id];
+		if (i >= 11 && read_junc) { // BED12
+			int32_t st, sz, en;
+			st = strtol(bs, &bs, 10); ++bs;
+			sz = strtol(bl, &bl, 10); ++bl;
+			en = t.st + st + sz;
+			for (i = 1; i < n_blk; ++i) {
+				mm_idx_intv1_t s = t;
+				if (r->n == r->m) {
+					r->m = r->m? r->m + (r->m>>1) : 16;
+					r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a) * r->m);
+				}
+				st = strtol(bs, &bs, 10); ++bs;
+				sz = strtol(bl, &bl, 10); ++bl;
+				s.st = en, s.en = t.st + st;
+				en = t.st + st + sz;
+				if (s.en > s.st) r->a[r->n++] = s;
+			}
+		} else {
+			if (r->n == r->m) {
+				r->m = r->m? r->m + (r->m>>1) : 16;
+				r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a) * r->m);
+			}
+			r->a[r->n++] = t;
+		}
+	}
+	free(str.s);
+	ks_destroy(ks);
+	gzclose(fp);
+	return I;
+}
+
+int mm_idx_bed_read(mm_idx_t *mi, const char *fn, int read_junc)
+{
+	int32_t i;
+	if (mi->h == 0) mm_idx_index_name(mi);
+	mi->I = mm_idx_read_bed(mi, fn, read_junc);
+	if (mi->I == 0) return -1;
+	for (i = 0; i < mi->n_seq; ++i) // TODO: eliminate redundant intervals
+		radix_sort_bed(mi->I[i].a, mi->I[i].a + mi->I[i].n);
+	return 0;
+}
+
+int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uint8_t *s)
+{
+	int32_t i, left, right;
+	mm_idx_intv_t *r;
+	memset(s, 0, en - st);
+	if (mi->I == 0 || ctg < 0 || ctg >= mi->n_seq) return -1;
+	r = &mi->I[ctg];
+	left = 0, right = r->n;
+	while (right > left) {
+		int32_t mid = left + ((right - left) >> 1);
+		if (r->a[mid].st >= st) right = mid;
+		else left = mid + 1;
+	}
+	for (i = left; i < r->n; ++i) {
+		if (st <= r->a[i].st && en >= r->a[i].en && r->a[i].strand != 0) {
+			if (r->a[i].strand > 0) {
+				s[r->a[i].st - st] |= 1, s[r->a[i].en - 1 - st] |= 2;
+			} else {
+				s[r->a[i].st - st] |= 8, s[r->a[i].en - 1 - st] |= 4;
+			}
+		}
+	}
+	return left;
 }

kalloc.c

@@ -1,175 +1,151 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <limits.h>
 #include "kalloc.h"
 
-/* The whole thing is: ("@" for the kheader_t of the block, "-" for free
+/* In kalloc, a *core* is a large chunk of contiguous memory. Each core is
- * memory, and "+" for allocated memory. One char for one unit.)
+ * associated with a master header, which keeps the size of the current core
+ * and the pointer to next core. Kalloc allocates small *blocks* of memory from
+ * the cores and organizes free memory blocks in a circular single-linked list.
  *
- *           This region is core 1.                             This region is core 2.
+ * In the following diagram, "@" stands for the header of a free block (of type
+ * header_t), "#" for the header of an allocated block (of type size_t), "-"
+ * for free memory, and "+" for allocated memory.
  *
- *   @-------@++++++@++++++++++++@------------           @----------@++++++++++++@+++++++@------------
+ * master        This region is core 1.          master           This region is core 2.
- *   |                           |                       |                               |
+ *      |                                             |
- *   p=p->ptr->ptr->ptr->ptr     p->ptr             p->ptr->ptr                p->ptr->ptr->ptr
+ *      *@-------#++++++#++++++++++++@--------        *@----------#++++++++++++#+++++++@------------
+ *       |                           |                 |                               |
+ *       p=p->ptr->ptr->ptr->ptr     p->ptr            p->ptr->ptr                     p->ptr->ptr->ptr
  */
-
+typedef struct header_t {
-#define PTR(p) ((size_t*)((size_t*)p)[1])
+	size_t size;
-
+	struct header_t *ptr;
-typedef struct _allocated_t {
+} header_t;
-	struct _allocated_t *next;
-	size_t *ptr;
-} allocated_t;
 
 typedef struct {
-	size_t base[2], *loop_head;
+	void *par;
-	allocated_t list_head, *list_tail;
+	size_t min_core_size;
-	size_t total_allocated;
+	header_t base, *loop_head, *core_head; /* base is a zero-sized block always kept in the loop */
 } kmem_t;
 
-void *km_init()
+static void panic(const char *s)
-{
-	return calloc(1, sizeof(kmem_t));
-}
-
-static void kerror(const char *s)
 {
 	fprintf(stderr, "%s\n", s);
-	exit(1);
+	abort();
 }
 
-static size_t *morecore(kmem_t *km, size_t nu)
+void *km_init2(void *km_par, size_t min_core_size)
 {
-	size_t rnu, *up;
+	kmem_t *km;
-
+	km = (kmem_t*)kcalloc(km_par, 1, sizeof(kmem_t));
-	rnu = (nu + 0xfffff) & (~(size_t)0xfffff);
+	km->par = km_par;
-	up = (size_t*)malloc(rnu * sizeof(size_t));
+	km->min_core_size = min_core_size > 0? min_core_size : 0x80000;
-	if (!up) { /* fail to allocate memory */
+	return (void*)km;
-		km_stat(km);
-		fprintf(stderr, "[morecore] %lu bytes requested but not available.\n", rnu * sizeof(size_t));
-		exit(1);
-	}
-	/* put the pointer in km->list_head */
-	if (km->list_tail == 0) km->list_tail = &km->list_head;
-	km->list_tail->ptr = up;
-	km->list_tail->next = (allocated_t*)calloc(1, sizeof(allocated_t));
-	km->list_tail = km->list_tail->next;
-
-	km->total_allocated += rnu * sizeof(size_t);
-	*up = rnu; /* the size of the current block, and in this case the block is the same as the new core */
-	kfree(km, up + 1); /* initialize the new "core" */
-	return km->loop_head;
 }
 
+void *km_init(void) { return km_init2(0, 0); }
+
 void km_destroy(void *_km)
 {
 	kmem_t *km = (kmem_t*)_km;
-	allocated_t *p, *q;
+	void *km_par;
-	if (km == 0) return;
+	header_t *p, *q;
-	p = &km->list_head;
+	if (km == NULL) return;
-	do {
+	km_par = km->par;
-		q = p->next;
+	for (p = km->core_head; p != NULL;) {
-		free(p->ptr);
+		q = p->ptr;
-		if (p != &km->list_head) free(p);
+		kfree(km_par, p);
 		p = q;
-	} while (p && p->next);
+	}
-	if (p != &km->list_head) free(p);
+	kfree(km_par, km);
-	free(km);
 }
 
-void kfree(void *_km, void *ap)
+static header_t *morecore(kmem_t *km, size_t nu)
 {
-	size_t *p, *q;
+	header_t *q;
+	size_t bytes, *p;
+	nu = (nu + 1 + (km->min_core_size - 1)) / km->min_core_size * km->min_core_size; /* the first +1 for core header */
+	bytes = nu * sizeof(header_t);
+	q = (header_t*)kmalloc(km->par, bytes);
+	if (!q) panic("[morecore] insufficient memory");
+	q->ptr = km->core_head, q->size = nu, km->core_head = q;
+	p = (size_t*)(q + 1);
+	*p = nu - 1; /* the size of the free block; -1 because the first unit is used for the core header */
+	kfree(km, p + 1); /* initialize the new "core"; NB: the core header is not looped. */
+	return km->loop_head;
+}
+
+void kfree(void *_km, void *ap) /* kfree() also adds a new core to the circular list */
+{
+	header_t *p, *q;
 	kmem_t *km = (kmem_t*)_km;
 	
 	if (!ap) return;
-	if (km == 0) {
+	if (km == NULL) {
 		free(ap);
 		return;
 	}
-	p = (size_t*)ap - 1; /* *p is the size of the current block */
+	p = (header_t*)((size_t*)ap - 1);
+	p->size = *((size_t*)ap - 1);
 	/* Find the pointer that points to the block to be freed. The following loop can stop on two conditions:
 	 *
-	 * a) "p>q && p<q->ptr": @------@++++++++@+++++++@-------    @---------------@+++++++@-------
+	 * a) "p>q && p<q->ptr": @------#++++++++#+++++++@-------    @---------------#+++++++@-------
 	 *    (can also be in    |      |                |        -> |                       |
 	 *     two cores)        q      p           q->ptr           q                  q->ptr
 	 *
-	 *                       @--------    @+++++++++@--------    @--------    @------------------
+	 *                       @--------    #+++++++++@--------    @--------    @------------------
 	 *                       |            |         |         -> |            |
 	 *                       q            p    q->ptr            q       q->ptr
 	 *
-	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------@+++++   @--------@+++++++     @-------@+++++   @----------------
+	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------#+++++   @--------#+++++++     @-------#+++++   @----------------
 	 *                                       |                |        |         -> |                |
 	 *                                  q->ptr                q        p       q->ptr                q
 	 *
-	 *                                       @+++++++@-----   @++++++++@-------     @-------------   @++++++++@-------
+	 *                                       #+++++++@-----   #++++++++@-------     @-------------   #++++++++@-------
 	 *                                       |       |                 |         -> |                         |
 	 *                                       p  q->ptr                 q       q->ptr                         q
 	 */
-	for (q = km->loop_head; !(p > q && p < PTR(q)); q = PTR(q))
+	for (q = km->loop_head; !(p > q && p < q->ptr); q = q->ptr)
-		if (q >= PTR(q) && (p > q || p < PTR(q))) break;
+		if (q >= q->ptr && (p > q || p < q->ptr)) break;
-	if (p + (*p) == PTR(q)) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
+	if (p + p->size == q->ptr) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
-		*p += *PTR(q); /* this is the new q->ptr size */
+		p->size += q->ptr->size;
-		p[1] = (size_t)PTR(PTR(q)); /* this is the new q->ptr->ptr */
+		p->ptr = q->ptr->ptr;
-		/* p is actually the new q->ptr. The actual change happens a few lines below. */
+	} else if (p + p->size > q->ptr && q->ptr >= p) {
-	} else if (p + (*p) > PTR(q) && PTR(q) >= p) { /* the end of the allocated block is in the next free block */
+		panic("[kfree] The end of the allocated block enters a free block.");
-		kerror("[kfree] The end of the allocated block enters a free block.");
+	} else p->ptr = q->ptr; /* backup q->ptr */
-	} else p[1] = (size_t)PTR(q); /* backup q->ptr */
 
-	if (q + (*q) == p) { /* two adjacent blocks, merge q and p (the other two cases) */
+	if (q + q->size == p) { /* two adjacent blocks, merge q and p (the other two cases) */
-		*q += *p;
+		q->size += p->size;
-		q[1] = (size_t)PTR(p);
+		q->ptr = p->ptr;
 		km->loop_head = q;
-	} else if (q + (*q) > p && p >= q) { /* the end of a free block in the allocated block */
+	} else if (q + q->size > p && p >= q) {
-		kerror("[kfree] The end of a free block enters the allocated block.");
+		panic("[kfree] The end of a free block enters the allocated block.");
-	} else km->loop_head = p, q[1] = (size_t)p; /* in two cores, cannot be merged */
+	} else km->loop_head = p, q->ptr = p; /* in two cores, cannot be merged; create a new block in the list */
-}
-
-void *krealloc(void *_km, void *ap, size_t n_bytes)
-{
-	kmem_t *km = (kmem_t*)_km;
-	size_t n_units, *p, *q;
-
-	if (n_bytes == 0) {
-		kfree(km, ap); return 0;
-	}
-	if (km == 0) return realloc(ap, n_bytes);
-	if (!ap) return kmalloc(km, n_bytes);
-	n_units = 1 + (n_bytes + sizeof(size_t) - 1) / sizeof(size_t);
-	p = (size_t*)ap - 1;
-	if (*p >= n_units) return ap; /* TODO: this prevents shrinking */
-	q = (size_t*)kmalloc(km, n_bytes);
-	memcpy(q, ap, (*p - 1) * sizeof(size_t));
-	kfree(km, ap);
-	return q;
 }
 
 void *kmalloc(void *_km, size_t n_bytes)
 {
 	kmem_t *km = (kmem_t*)_km;
-	size_t n_units, *p, *q;
+	size_t n_units;
+	header_t *p, *q;
 
 	if (n_bytes == 0) return 0;
-	if (km == 0) return malloc(n_bytes);
+	if (km == NULL) return malloc(n_bytes);
-	/* "n_units" means the number of units. The size of one unit equals to sizeof(kheader_t).
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t); /* header+n_bytes requires at least this number of units */
-	 * "1" is the kheader_t of a block, which is always required. */
-	n_units = 1 + (n_bytes + sizeof(size_t) - 1) / sizeof(size_t);
-	if (n_units&1) ++n_units; /* make n_units an even number, or it will segfault if only one unit remains */
 
-	if (!(q = km->loop_head)) { /* the first time when kmalloc() is called, intialization */
+	if (!(q = km->loop_head)) /* the first time when kmalloc() is called, intialize it */
-		km->base[1] = (size_t)(km->loop_head = q = km->base); *q = 0;
+		q = km->loop_head = km->base.ptr = &km->base;
-	}
+	for (p = q->ptr;; q = p, p = p->ptr) { /* search for a suitable block */
-	for (p = PTR(q);; q = p, p = PTR(p)) { /* search for a suitable block */
+		if (p->size >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
-		if (*p >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
+			if (p->size == n_units) q->ptr = p->ptr; /* no need to split the block */
-			if (*p == n_units) q[1] = (size_t)PTR(p); /* no need to split the block */
+			else { /* split the block. NB: memory is allocated at the end of the block! */
-			else { /* split the block */
+				p->size -= n_units; /* reduce the size of the free block */
-				/* memory is allocated at the end of the block */
+				p += p->size; /* p points to the allocated block */
-				*p -= n_units; /* reduce the size of the free block */
+				*(size_t*)p = n_units; /* set the size */
-				p += *p; /* skip to the kheader_t of the allocated block */
-				*p = n_units; /* set the size */
 			}
 			km->loop_head = q; /* set the end of chain */
-			return p + 1; /* skip the kheader_t */
+			return (size_t*)p + 1;
 		}
 		if (p == km->loop_head) { /* then ask for more "cores" */
 			if ((p = morecore(km, n_units)) == 0) return 0;
@@ -182,33 +158,48 @@ void *kcalloc(void *_km, size_t count, size_t size)
 	kmem_t *km = (kmem_t*)_km;
 	void *p;
 	if (size == 0 || count == 0) return 0;
-	if (km == 0) return calloc(count, size);
+	if (km == NULL) return calloc(count, size);
 	p = kmalloc(km, count * size);
 	memset(p, 0, count * size);
 	return p;
 }
 
-void km_stat(const void *_km)
+void *krealloc(void *_km, void *ap, size_t n_bytes) // TODO: this can be made more efficient in principle
 {
 	kmem_t *km = (kmem_t*)_km;
-	unsigned n_blocks, n_units;
+	size_t cap, *p, *q;
-	size_t max_block = 0, *p, *q;
-	float frag;
 
-	if (km == 0 || !(p = km->loop_head)) return;
+	if (n_bytes == 0) {
-	n_blocks = n_units = 0;
+		kfree(km, ap); return 0;
-	do {
+	}
-		q = PTR(p);
+	if (km == NULL) return realloc(ap, n_bytes);
-		if (*p > max_block) max_block = *p;
+	if (ap == NULL) return kmalloc(km, n_bytes);
-		n_units += *p;
+	p = (size_t*)ap - 1;
-		if (p + (*p) > q && q > p)
+	cap = (*p) * sizeof(header_t) - sizeof(size_t);
-			kerror("[kr_stat] The end of a free block enters another free block.");
+	if (cap >= n_bytes) return ap; /* TODO: this prevents shrinking */
-		p = q;
+	q = (size_t*)kmalloc(km, n_bytes);
-		++n_blocks;
+	memcpy(q, ap, cap);
-	} while (p != km->loop_head);
+	kfree(km, ap);
+	return q;
+}
 
-	--n_blocks;
+void km_stat(const void *_km, km_stat_t *s)
-	frag = 1.0/1024.0 * n_units * sizeof(size_t) / n_blocks;
+{
-	fprintf(stderr, "[kr_stat] tot=%lu, free=%lu, n_block=%u, max_block=%lu, frag_len=%.3fK\n",
+	kmem_t *km = (kmem_t*)_km;
-			km->total_allocated, n_units * sizeof(size_t), n_blocks, max_block * sizeof(size_t), frag);
+	header_t *p;
+	memset(s, 0, sizeof(km_stat_t));
+	if (km == NULL || km->loop_head == NULL) return;
+	for (p = km->loop_head;; p = p->ptr) {
+		s->available += p->size * sizeof(header_t);
+		if (p->size != 0) ++s->n_blocks; /* &kmem_t::base is always one of the cores. It is zero-sized. */
+		if (p->ptr > p && p + p->size > p->ptr)
+			panic("[km_stat] The end of a free block enters another free block.");
+		if (p->ptr == km->loop_head) break;
+	}
+	for (p = km->core_head; p != NULL; p = p->ptr) {
+		size_t size = p->size * sizeof(header_t);
+		++s->n_cores;
+		s->capacity += size;
+		s->largest = s->largest > size? s->largest : size;
+	}
 }

kalloc.h

@@ -1,26 +1,76 @@
 #ifndef _KALLOC_H_
 #define _KALLOC_H_
 
-#include <stdlib.h>
+#include <stddef.h> /* for size_t */
-
-#define km_size(x) (*(((size_t*)(x))-1) * sizeof(size_t))
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct {
+	size_t capacity, available, n_blocks, n_cores, largest;
+} km_stat_t;
+
 void *kmalloc(void *km, size_t size);
 void *krealloc(void *km, void *ptr, size_t size);
 void *kcalloc(void *km, size_t count, size_t size);
 void kfree(void *km, void *ptr);
 
 void *km_init(void);
+void *km_init2(void *km_par, size_t min_core_size);
 void km_destroy(void *km);
-
+void km_stat(const void *_km, km_stat_t *s);
-void km_stat(const void *km); // TODO: return numbers instead of print to stderr
 
 #ifdef __cplusplus
 }
 #endif
 
+#define KMALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kmalloc((km), (len) * sizeof(*(ptr))))
+#define KCALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kcalloc((km), (len), sizeof(*(ptr))))
+#define KREALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))krealloc((km), (ptr), (len) * sizeof(*(ptr))))
+
+#define KEXPAND(km, a, m) do { \
+		(m) = (m) >= 4? (m) + ((m)>>1) : 16; \
+		KREALLOC((km), (a), (m)); \
+	} while (0)
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#define KALLOC_POOL_INIT2(SCOPE, name, kmptype_t) \
+	typedef struct { \
+		size_t cnt, n, max; \
+		kmptype_t **buf; \
+		void *km; \
+	} kmp_##name##_t; \
+	SCOPE kmp_##name##_t *kmp_init_##name(void *km) { \
+		kmp_##name##_t *mp; \
+		KCALLOC(km, mp, 1); \
+		mp->km = km; \
+		return mp; \
+	} \
+	SCOPE void kmp_destroy_##name(kmp_##name##_t *mp) { \
+		size_t k; \
+		for (k = 0; k < mp->n; ++k) kfree(mp->km, mp->buf[k]); \
+		kfree(mp->km, mp->buf); kfree(mp->km, mp); \
+	} \
+	SCOPE kmptype_t *kmp_alloc_##name(kmp_##name##_t *mp) { \
+		++mp->cnt; \
+		if (mp->n == 0) return (kmptype_t*)kcalloc(mp->km, 1, sizeof(kmptype_t)); \
+		return mp->buf[--mp->n]; \
+	} \
+	SCOPE void kmp_free_##name(kmp_##name##_t *mp, kmptype_t *p) { \
+		--mp->cnt; \
+		if (mp->n == mp->max) KEXPAND(mp->km, mp->buf, mp->max); \
+		mp->buf[mp->n++] = p; \
+	}
+
+#define KALLOC_POOL_INIT(name, kmptype_t) \
+	KALLOC_POOL_INIT2(static inline klib_unused, name, kmptype_t)
+
 #endif

kdq.h

@@ -3,11 +3,12 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <stdint.h>
 #include "kalloc.h"
 
 #define __KDQ_TYPE(type) \
 	typedef struct { \
-		size_t front:58, bits:6, count, mask; \
+		uint64_t front:58, bits:6, count, mask; \
 		type *a; \
 		void *km; \
 	} kdq_##type##_t;

ketopt.h

@@ -0,0 +1,120 @@
+#ifndef KETOPT_H
+#define KETOPT_H
+
+#include <string.h> /* for strchr() and strncmp() */
+
+#define ko_no_argument       0
+#define ko_required_argument 1
+#define ko_optional_argument 2
+
+typedef struct {
+	int ind;   /* equivalent to optind */
+	int opt;   /* equivalent to optopt */
+	char *arg; /* equivalent to optarg */
+	int longidx; /* index of a long option; or -1 if short */
+	/* private variables not intended for external uses */
+	int i, pos, n_args;
+} ketopt_t;
+
+typedef struct {
+	char *name;
+	int has_arg;
+	int val;
+} ko_longopt_t;
+
+static ketopt_t KETOPT_INIT = { 1, 0, 0, -1, 1, 0, 0 };
+
+static void ketopt_permute(char *argv[], int j, int n) /* move argv[j] over n elements to the left */
+{
+	int k;
+	char *p = argv[j];
+	for (k = 0; k < n; ++k)
+		argv[j - k] = argv[j - k - 1];
+	argv[j - k] = p;
+}
+
+/**
+ * Parse command-line options and arguments
+ *
+ * This fuction has a similar interface to GNU's getopt_long(). Each call
+ * parses one option and returns the option name.  s->arg points to the option
+ * argument if present. The function returns -1 when all command-line arguments
+ * are parsed. In this case, s->ind is the index of the first non-option
+ * argument.
+ *
+ * @param s         status; shall be initialized to KETOPT_INIT on the first call
+ * @param argc      length of argv[]
+ * @param argv      list of command-line arguments; argv[0] is ignored
+ * @param permute   non-zero to move options ahead of non-option arguments
+ * @param ostr      option string
+ * @param longopts  long options
+ *
+ * @return ASCII for a short option; ko_longopt_t::val for a long option; -1 if
+ *         argv[] is fully processed; '?' for an unknown option or an ambiguous
+ *         long option; ':' if an option argument is missing
+ */
+static int ketopt(ketopt_t *s, int argc, char *argv[], int permute, const char *ostr, const ko_longopt_t *longopts)
+{
+	int opt = -1, i0, j;
+	if (permute) {
+		while (s->i < argc && (argv[s->i][0] != '-' || argv[s->i][1] == '\0'))
+			++s->i, ++s->n_args;
+	}
+	s->arg = 0, s->longidx = -1, i0 = s->i;
+	if (s->i >= argc || argv[s->i][0] != '-' || argv[s->i][1] == '\0') {
+		s->ind = s->i - s->n_args;
+		return -1;
+	}
+	if (argv[s->i][0] == '-' && argv[s->i][1] == '-') { /* "--" or a long option */
+		if (argv[s->i][2] == '\0') { /* a bare "--" */
+			ketopt_permute(argv, s->i, s->n_args);
+			++s->i, s->ind = s->i - s->n_args;
+			return -1;
+		}
+		s->opt = 0, opt = '?', s->pos = -1;
+		if (longopts) { /* parse long options */
+			int k, n_exact = 0, n_partial = 0;
+			const ko_longopt_t *o = 0, *o_exact = 0, *o_partial = 0;
+			for (j = 2; argv[s->i][j] != '\0' && argv[s->i][j] != '='; ++j) {} /* find the end of the option name */
+			for (k = 0; longopts[k].name != 0; ++k)
+				if (strncmp(&argv[s->i][2], longopts[k].name, j - 2) == 0) {
+					if (longopts[k].name[j - 2] == 0) ++n_exact, o_exact = &longopts[k];
+					else ++n_partial, o_partial = &longopts[k];
+				}
+			if (n_exact > 1 || (n_exact == 0 && n_partial > 1)) return '?';
+			o = n_exact == 1? o_exact : n_partial == 1? o_partial : 0;
+			if (o) {
+				s->opt = opt = o->val, s->longidx = o - longopts;
+				if (argv[s->i][j] == '=') s->arg = &argv[s->i][j + 1];
+				if (o->has_arg == 1 && argv[s->i][j] == '\0') {
+					if (s->i < argc - 1) s->arg = argv[++s->i];
+					else opt = ':'; /* missing option argument */
+				}
+			}
+		}
+	} else { /* a short option */
+		char *p;
+		if (s->pos == 0) s->pos = 1;
+		opt = s->opt = argv[s->i][s->pos++];
+		p = strchr((char*)ostr, opt);
+		if (p == 0) {
+			opt = '?'; /* unknown option */
+		} else if (p[1] == ':') {
+			if (argv[s->i][s->pos] == 0) {
+				if (s->i < argc - 1) s->arg = argv[++s->i];
+				else opt = ':'; /* missing option argument */
+			} else s->arg = &argv[s->i][s->pos];
+			s->pos = -1;
+		}
+	}
+	if (s->pos < 0 || argv[s->i][s->pos] == 0) {
+		++s->i, s->pos = 0;
+		if (s->n_args > 0) /* permute */
+			for (j = i0; j < s->i; ++j)
+				ketopt_permute(argv, j, s->n_args);
+	}
+	s->ind = s->i - s->n_args;
+	return opt;
+}
+
+#endif

krmq.h

@@ -0,0 +1,474 @@
+/* The MIT License
+
+   Copyright (c) 2019 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* An example:
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "krmq.h"
+
+struct my_node {
+  char key;
+  KRMQ_HEAD(struct my_node) head;
+};
+#define my_cmp(p, q) (((q)->key < (p)->key) - ((p)->key < (q)->key))
+KRMQ_INIT(my, struct my_node, head, my_cmp)
+
+int main(void) {
+  const char *str = "MNOLKQOPHIA"; // from wiki, except a duplicate
+  struct my_node *root = 0;
+  int i, l = strlen(str);
+  for (i = 0; i < l; ++i) {        // insert in the input order
+    struct my_node *q, *p = malloc(sizeof(*p));
+    p->key = str[i];
+    q = krmq_insert(my, &root, p, 0);
+    if (p != q) free(p);           // if already present, free
+  }
+  krmq_itr_t(my) itr;
+  krmq_itr_first(my, root, &itr);  // place at first
+  do {                             // traverse
+    const struct my_node *p = krmq_at(&itr);
+    putchar(p->key);
+    free((void*)p);                // free node
+  } while (krmq_itr_next(my, &itr));
+  putchar('\n');
+  return 0;
+}
+*/
+
+#ifndef KRMQ_H
+#define KRMQ_H
+
+#ifdef __STRICT_ANSI__
+#define inline __inline__
+#endif
+
+#define KRMQ_MAX_DEPTH 64
+
+#define krmq_size(head, p) ((p)? (p)->head.size : 0)
+#define krmq_size_child(head, q, i) ((q)->head.p[(i)]? (q)->head.p[(i)]->head.size : 0)
+
+#define KRMQ_HEAD(__type) \
+	struct { \
+		__type *p[2], *s; \
+		signed char balance; /* balance factor */ \
+		unsigned size; /* #elements in subtree */ \
+	}
+
+#define __KRMQ_FIND(suf, __scope, __type, __head,  __cmp) \
+	__scope __type *krmq_find_##suf(const __type *root, const __type *x, unsigned *cnt_) { \
+		const __type *p = root; \
+		unsigned cnt = 0; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += krmq_size_child(__head, p, 0) + 1; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		return (__type*)p; \
+	} \
+	__scope __type *krmq_interval_##suf(const __type *root, const __type *x, __type **lower, __type **upper) { \
+		const __type *p = root, *l = 0, *u = 0; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) u = p, p = p->__head.p[0]; \
+			else if (cmp > 0) l = p, p = p->__head.p[1]; \
+			else { l = u = p; break; } \
+		} \
+		if (lower) *lower = (__type*)l; \
+		if (upper) *upper = (__type*)u; \
+		return (__type*)p; \
+	}
+
+#define __KRMQ_RMQ(suf, __scope, __type, __head,  __cmp, __lt2) \
+	__scope __type *krmq_rmq_##suf(const __type *root, const __type *lo, const __type *up) { /* CLOSED interval */ \
+		const __type *p = root, *path[2][KRMQ_MAX_DEPTH], *min; \
+		int plen[2] = {0, 0}, pcmp[2][KRMQ_MAX_DEPTH], i, cmp, lca; \
+		if (root == 0) return 0; \
+		while (p) { \
+			cmp = __cmp(lo, p); \
+			path[0][plen[0]] = p, pcmp[0][plen[0]++] = cmp; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		p = root; \
+		while (p) { \
+			cmp = __cmp(up, p); \
+			path[1][plen[1]] = p, pcmp[1][plen[1]++] = cmp; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		for (i = 0; i < plen[0] && i < plen[1]; ++i) /* find the LCA */ \
+			if (path[0][i] == path[1][i] && pcmp[0][i] <= 0 && pcmp[1][i] >= 0) \
+				break; \
+		if (i == plen[0] || i == plen[1]) return 0; /* no elements in the closed interval */ \
+		lca = i, min = path[0][lca]; \
+		for (i = lca + 1; i < plen[0]; ++i) { \
+			if (pcmp[0][i] <= 0) { \
+				if (__lt2(path[0][i], min)) min = path[0][i]; \
+				if (path[0][i]->__head.p[1] && __lt2(path[0][i]->__head.p[1]->__head.s, min)) \
+					min = path[0][i]->__head.p[1]->__head.s; \
+			} \
+		} \
+		for (i = lca + 1; i < plen[1]; ++i) { \
+			if (pcmp[1][i] >= 0) { \
+				if (__lt2(path[1][i], min)) min = path[1][i]; \
+				if (path[1][i]->__head.p[0] && __lt2(path[1][i]->__head.p[0]->__head.s, min)) \
+					min = path[1][i]->__head.p[0]->__head.s; \
+			} \
+		} \
+		return (__type*)min; \
+	}
+
+#define __KRMQ_ROTATE(suf, __type, __head, __lt2) \
+	/* */ \
+	static inline void krmq_update_min_##suf(__type *p, const __type *q, const __type *r) { \
+		p->__head.s = !q || __lt2(p, q->__head.s)? p : q->__head.s; \
+		p->__head.s = !r || __lt2(p->__head.s, r->__head.s)? p->__head.s : r->__head.s; \
+	} \
+	/* one rotation: (a,(b,c)q)p => ((a,b)p,c)q */ \
+	static inline __type *krmq_rotate1_##suf(__type *p, int dir) { /* dir=0 to left; dir=1 to right */ \
+		int opp = 1 - dir; /* opposite direction */ \
+		__type *q = p->__head.p[opp], *s = p->__head.s; \
+		unsigned size_p = p->__head.size; \
+		p->__head.size -= q->__head.size - krmq_size_child(__head, q, dir); \
+		q->__head.size = size_p; \
+		krmq_update_min_##suf(p, p->__head.p[dir], q->__head.p[dir]); \
+		q->__head.s = s; \
+		p->__head.p[opp] = q->__head.p[dir]; \
+		q->__head.p[dir] = p; \
+		return q; \
+	} \
+	/* two consecutive rotations: (a,((b,c)r,d)q)p => ((a,b)p,(c,d)q)r */ \
+	static inline __type *krmq_rotate2_##suf(__type *p, int dir) { \
+		int b1, opp = 1 - dir; \
+		__type *q = p->__head.p[opp], *r = q->__head.p[dir], *s = p->__head.s; \
+		unsigned size_x_dir = krmq_size_child(__head, r, dir); \
+		r->__head.size = p->__head.size; \
+		p->__head.size -= q->__head.size - size_x_dir; \
+		q->__head.size -= size_x_dir + 1; \
+		krmq_update_min_##suf(p, p->__head.p[dir], r->__head.p[dir]); \
+		krmq_update_min_##suf(q, q->__head.p[opp], r->__head.p[opp]); \
+		r->__head.s = s; \
+		p->__head.p[opp] = r->__head.p[dir]; \
+		r->__head.p[dir] = p; \
+		q->__head.p[dir] = r->__head.p[opp]; \
+		r->__head.p[opp] = q; \
+		b1 = dir == 0? +1 : -1; \
+		if (r->__head.balance == b1) q->__head.balance = 0, p->__head.balance = -b1; \
+		else if (r->__head.balance == 0) q->__head.balance = p->__head.balance = 0; \
+		else q->__head.balance = b1, p->__head.balance = 0; \
+		r->__head.balance = 0; \
+		return r; \
+	}
+
+#define __KRMQ_INSERT(suf, __scope, __type, __head, __cmp, __lt2) \
+	__scope __type *krmq_insert_##suf(__type **root_, __type *x, unsigned *cnt_) { \
+		unsigned char stack[KRMQ_MAX_DEPTH]; \
+		__type *path[KRMQ_MAX_DEPTH]; \
+		__type *bp, *bq; \
+		__type *p, *q, *r = 0; /* _r_ is potentially the new root */ \
+		int i, which = 0, top, b1, path_len; \
+		unsigned cnt = 0; \
+		bp = *root_, bq = 0; \
+		/* find the insertion location */ \
+		for (p = bp, q = bq, top = path_len = 0; p; q = p, p = p->__head.p[which]) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += krmq_size_child(__head, p, 0) + 1; \
+			if (cmp == 0) { \
+				if (cnt_) *cnt_ = cnt; \
+				return p; \
+			} \
+			if (p->__head.balance != 0) \
+				bq = q, bp = p, top = 0; \
+			stack[top++] = which = (cmp > 0); \
+			path[path_len++] = p; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		x->__head.balance = 0, x->__head.size = 1, x->__head.p[0] = x->__head.p[1] = 0, x->__head.s = x; \
+		if (q == 0) *root_ = x; \
+		else q->__head.p[which] = x; \
+		if (bp == 0) return x; \
+		for (i = 0; i < path_len; ++i) ++path[i]->__head.size; \
+		for (i = path_len - 1; i >= 0; --i) { \
+			krmq_update_min_##suf(path[i], path[i]->__head.p[0], path[i]->__head.p[1]); \
+			if (path[i]->__head.s != x) break; \
+		} \
+		for (p = bp, top = 0; p != x; p = p->__head.p[stack[top]], ++top) /* update balance factors */ \
+			if (stack[top] == 0) --p->__head.balance; \
+			else ++p->__head.balance; \
+		if (bp->__head.balance > -2 && bp->__head.balance < 2) return x; /* no re-balance needed */ \
+		/* re-balance */ \
+		which = (bp->__head.balance < 0); \
+		b1 = which == 0? +1 : -1; \
+		q = bp->__head.p[1 - which]; \
+		if (q->__head.balance == b1) { \
+			r = krmq_rotate1_##suf(bp, which); \
+			q->__head.balance = bp->__head.balance = 0; \
+		} else r = krmq_rotate2_##suf(bp, which); \
+		if (bq == 0) *root_ = r; \
+		else bq->__head.p[bp != bq->__head.p[0]] = r; \
+		return x; \
+	}
+
+#define __KRMQ_ERASE(suf, __scope, __type, __head, __cmp, __lt2) \
+	__scope __type *krmq_erase_##suf(__type **root_, const __type *x, unsigned *cnt_) { \
+		__type *p, *path[KRMQ_MAX_DEPTH], fake; \
+		unsigned char dir[KRMQ_MAX_DEPTH]; \
+		int i, d = 0, cmp; \
+		unsigned cnt = 0; \
+		fake = **root_, fake.__head.p[0] = *root_, fake.__head.p[1] = 0; \
+		if (cnt_) *cnt_ = 0; \
+		if (x) { \
+			for (cmp = -1, p = &fake; cmp; cmp = __cmp(x, p)) { \
+				int which = (cmp > 0); \
+				if (cmp > 0) cnt += krmq_size_child(__head, p, 0) + 1; \
+				dir[d] = which; \
+				path[d++] = p; \
+				p = p->__head.p[which]; \
+				if (p == 0) { \
+					if (cnt_) *cnt_ = 0; \
+					return 0; \
+				} \
+			} \
+			cnt += krmq_size_child(__head, p, 0) + 1; /* because p==x is not counted */ \
+		} else { \
+			for (p = &fake, cnt = 1; p; p = p->__head.p[0]) \
+				dir[d] = 0, path[d++] = p; \
+			p = path[--d]; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		for (i = 1; i < d; ++i) --path[i]->__head.size; \
+		if (p->__head.p[1] == 0) { /* ((1,.)2,3)4 => (1,3)4; p=2 */ \
+			path[d-1]->__head.p[dir[d-1]] = p->__head.p[0]; \
+		} else { \
+			__type *q = p->__head.p[1]; \
+			if (q->__head.p[0] == 0) { /* ((1,2)3,4)5 => ((1)2,4)5; p=3,q=2 */ \
+				q->__head.p[0] = p->__head.p[0]; \
+				q->__head.balance = p->__head.balance; \
+				path[d-1]->__head.p[dir[d-1]] = q; \
+				path[d] = q, dir[d++] = 1; \
+				q->__head.size = p->__head.size - 1; \
+			} else { /* ((1,((.,2)3,4)5)6,7)8 => ((1,(2,4)5)3,7)8; p=6 */ \
+				__type *r; \
+				int e = d++; /* backup _d_ */\
+				for (;;) { \
+					dir[d] = 0; \
+					path[d++] = q; \
+					r = q->__head.p[0]; \
+					if (r->__head.p[0] == 0) break; \
+					q = r; \
+				} \
+				r->__head.p[0] = p->__head.p[0]; \
+				q->__head.p[0] = r->__head.p[1]; \
+				r->__head.p[1] = p->__head.p[1]; \
+				r->__head.balance = p->__head.balance; \
+				path[e-1]->__head.p[dir[e-1]] = r; \
+				path[e] = r, dir[e] = 1; \
+				for (i = e + 1; i < d; ++i) --path[i]->__head.size; \
+				r->__head.size = p->__head.size - 1; \
+			} \
+		} \
+		for (i = d - 1; i >= 0; --i) /* not sure why adding condition "path[i]->__head.s==p" doesn't work */ \
+			krmq_update_min_##suf(path[i], path[i]->__head.p[0], path[i]->__head.p[1]); \
+		while (--d > 0) { \
+			__type *q = path[d]; \
+			int which, other, b1 = 1, b2 = 2; \
+			which = dir[d], other = 1 - which; \
+			if (which) b1 = -b1, b2 = -b2; \
+			q->__head.balance += b1; \
+			if (q->__head.balance == b1) break; \
+			else if (q->__head.balance == b2) { \
+				__type *r = q->__head.p[other]; \
+				if (r->__head.balance == -b1) { \
+					path[d-1]->__head.p[dir[d-1]] = krmq_rotate2_##suf(q, which); \
+				} else { \
+					path[d-1]->__head.p[dir[d-1]] = krmq_rotate1_##suf(q, which); \
+					if (r->__head.balance == 0) { \
+						r->__head.balance = -b1; \
+						q->__head.balance = b1; \
+						break; \
+					} else r->__head.balance = q->__head.balance = 0; \
+				} \
+			} \
+		} \
+		*root_ = fake.__head.p[0]; \
+		return p; \
+	}
+
+#define krmq_free(__type, __head, __root, __free) do { \
+		__type *_p, *_q; \
+		for (_p = __root; _p; _p = _q) { \
+			if (_p->__head.p[0] == 0) { \
+				_q = _p->__head.p[1]; \
+				__free(_p); \
+			} else { \
+				_q = _p->__head.p[0]; \
+				_p->__head.p[0] = _q->__head.p[1]; \
+				_q->__head.p[1] = _p; \
+			} \
+		} \
+	} while (0)
+
+#define __KRMQ_ITR(suf, __scope, __type, __head, __cmp) \
+	struct krmq_itr_##suf { \
+		const __type *stack[KRMQ_MAX_DEPTH], **top; \
+	}; \
+	__scope void krmq_itr_first_##suf(const __type *root, struct krmq_itr_##suf *itr) { \
+		const __type *p; \
+		for (itr->top = itr->stack - 1, p = root; p; p = p->__head.p[0]) \
+			*++itr->top = p; \
+	} \
+	__scope int krmq_itr_find_##suf(const __type *root, const __type *x, struct krmq_itr_##suf *itr) { \
+		const __type *p = root; \
+		itr->top = itr->stack - 1; \
+		while (p != 0) { \
+			int cmp; \
+			*++itr->top = p; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		return p? 1 : 0; \
+	} \
+	__scope int krmq_itr_next_bidir_##suf(struct krmq_itr_##suf *itr, int dir) { \
+		const __type *p; \
+		if (itr->top < itr->stack) return 0; \
+		dir = !!dir; \
+		p = (*itr->top)->__head.p[dir]; \
+		if (p) { /* go down */ \
+			for (; p; p = p->__head.p[!dir]) \
+				*++itr->top = p; \
+			return 1; \
+		} else { /* go up */ \
+			do { \
+				p = *itr->top--; \
+			} while (itr->top >= itr->stack && p == (*itr->top)->__head.p[dir]); \
+			return itr->top < itr->stack? 0 : 1; \
+		} \
+	} \
+
+/**
+ * Insert a node to the tree
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node to insert (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return _x_ if not present in the tree, or the node equal to x.
+ */
+#define krmq_insert(suf, proot, x, cnt) krmq_insert_##suf(proot, x, cnt)
+
+/**
+ * Find a node in the tree
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param root    root of the tree
+ * @param x       node value to find (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return node equal to _x_ if present, or NULL if absent
+ */
+#define krmq_find(suf, root, x, cnt) krmq_find_##suf(root, x, cnt)
+#define krmq_interval(suf, root, x, lower, upper) krmq_interval_##suf(root, x, lower, upper)
+#define krmq_rmq(suf, root, lo, up) krmq_rmq_##suf(root, lo, up)
+
+/**
+ * Delete a node from the tree
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node value to delete; if NULL, delete the first node (in)
+ *
+ * @return node removed from the tree if present, or NULL if absent
+ */
+#define krmq_erase(suf, proot, x, cnt) krmq_erase_##suf(proot, x, cnt)
+#define krmq_erase_first(suf, proot) krmq_erase_##suf(proot, 0, 0)
+
+#define krmq_itr_t(suf) struct krmq_itr_##suf
+
+/**
+ * Place the iterator at the smallest object
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param root    root of the tree
+ * @param itr     iterator
+ */
+#define krmq_itr_first(suf, root, itr) krmq_itr_first_##suf(root, itr)
+
+/**
+ * Place the iterator at the object equal to or greater than the query
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param root    root of the tree
+ * @param x       query (in)
+ * @param itr     iterator (out)
+ *
+ * @return 1 if find; 0 otherwise. krmq_at(itr) is NULL if and only if query is
+ *         larger than all objects in the tree
+ */
+#define krmq_itr_find(suf, root, x, itr) krmq_itr_find_##suf(root, x, itr)
+
+/**
+ * Move to the next object in order
+ *
+ * @param itr     iterator (modified)
+ *
+ * @return 1 if there is a next object; 0 otherwise
+ */
+#define krmq_itr_next(suf, itr) krmq_itr_next_bidir_##suf(itr, 1)
+#define krmq_itr_prev(suf, itr) krmq_itr_next_bidir_##suf(itr, 0)
+
+/**
+ * Return the pointer at the iterator
+ *
+ * @param itr     iterator
+ *
+ * @return pointer if present; NULL otherwise
+ */
+#define krmq_at(itr) ((itr)->top < (itr)->stack? 0 : *(itr)->top)
+
+#define KRMQ_INIT2(suf, __scope, __type, __head, __cmp, __lt2) \
+	__KRMQ_FIND(suf, __scope, __type, __head,  __cmp) \
+	__KRMQ_RMQ(suf, __scope, __type, __head,  __cmp, __lt2) \
+	__KRMQ_ROTATE(suf, __type, __head, __lt2) \
+	__KRMQ_INSERT(suf, __scope, __type, __head, __cmp, __lt2) \
+	__KRMQ_ERASE(suf, __scope, __type, __head, __cmp, __lt2) \
+	__KRMQ_ITR(suf, __scope, __type, __head, __cmp)
+
+#define KRMQ_INIT(suf, __type, __head, __cmp, __lt2) \
+	KRMQ_INIT2(suf,, __type, __head, __cmp, __lt2)
+
+#endif

kseq.h

@@ -37,6 +37,14 @@
 #define KS_SEP_LINE  2 // line separator: "\n" (Unix) or "\r\n" (Windows)
 #define KS_SEP_MAX   2
 
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
 #define __KS_TYPE(type_t) \
 	typedef struct __kstream_t { \
 		int begin, end; \
@@ -64,7 +72,7 @@
 	}
 
 #define __KS_INLINED(__read) \
-	static inline int ks_getc(kstream_t *ks) \
+	static inline klib_unused int ks_getc(kstream_t *ks) \
 	{ \
 		if (ks->is_eof && ks->begin >= ks->end) return -1; \
 		if (ks->begin >= ks->end) { \
@@ -81,7 +89,7 @@
 #ifndef KSTRING_T
 #define KSTRING_T kstring_t
 typedef struct __kstring_t {
-	unsigned l, m;
+	size_t l, m;
 	char *s;
 } kstring_t;
 #endif

ksort.h

@@ -30,15 +30,33 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 
 typedef struct {
 	void *left, *right;
 	int depth;
 } ks_isort_stack_t;
 
-#define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; }
+#define KSORT_SWAP(type_t, a, b) { type_t t=(a); (a)=(b); (b)=t; }
 
-#define KSORT_INIT(name, type_t, __sort_lt)								\
+#define KSORT_INIT(name, type_t, __sort_lt) \
+	void ks_heapdown_##name(size_t i, size_t n, type_t l[]) \
+	{ \
+		size_t k = i; \
+		type_t tmp = l[i]; \
+		while ((k = (k << 1) + 1) < n) { \
+			if (k != n - 1 && __sort_lt(l[k], l[k+1])) ++k; \
+			if (__sort_lt(l[k], tmp)) break; \
+			l[i] = l[k]; i = k; \
+		} \
+		l[i] = tmp; \
+	} \
+	void ks_heapmake_##name(size_t lsize, type_t l[]) \
+	{ \
+		size_t i; \
+		for (i = (lsize >> 1) - 1; i != (size_t)(-1); --i) \
+			ks_heapdown_##name(i, lsize, l); \
+	} \
 	type_t ks_ksmall_##name(size_t n, type_t arr[], size_t kk)			\
 	{																	\
 		type_t *low, *high, *k, *ll, *hh, *mid;							\
@@ -78,6 +96,7 @@ typedef const char *ksstr_t;
 #define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str)
 
 #define RS_MIN_SIZE 64
+#define RS_MAX_BITS 8
 
 #define KRADIX_SORT_INIT(name, rstype_t, rskey, sizeof_key) \
 	typedef struct { \
@@ -98,7 +117,8 @@ typedef const char *ksstr_t;
 	{ \
 		rstype_t *i; \
 		int size = 1<<n_bits, m = size - 1; \
-		rsbucket_##name##_t *k, b[size], *be = b + size; \
+		rsbucket_##name##_t *k, b[1<<RS_MAX_BITS], *be = b + size; \
+		assert(n_bits <= RS_MAX_BITS); \
 		for (k = b; k != be; ++k) k->b = k->e = beg; \
 		for (i = beg; i != end; ++i) ++b[rskey(*i)>>s&m].e; \
 		for (k = b + 1; k != be; ++k) \
@@ -127,7 +147,7 @@ typedef const char *ksstr_t;
 	void radix_sort_##name(rstype_t *beg, rstype_t *end) \
 	{ \
 		if (end - beg <= RS_MIN_SIZE) rs_insertsort_##name(beg, end); \
-		else rs_sort_##name(beg, end, 8, sizeof_key * 8 - 8); \
+		else rs_sort_##name(beg, end, RS_MAX_BITS, (sizeof_key - 1) * RS_MAX_BITS); \
 	}
 
 #endif

ksw2.h

@@ -12,6 +12,16 @@
 #define KSW_EZ_APPROX_DROP 0x10 // approximate Z-drop; faster with sse
 #define KSW_EZ_EXTZ_ONLY   0x40 // only perform extension
 #define KSW_EZ_REV_CIGAR   0x80 // reverse CIGAR in the output
+#define KSW_EZ_SPLICE_FOR  0x100
+#define KSW_EZ_SPLICE_REV  0x200
+#define KSW_EZ_SPLICE_FLANK 0x400
+
+// The subset of CIGAR operators used by ksw code.
+// Use MM_CIGAR_* from minimap.h if you need the full list.
+#define KSW_CIGAR_MATCH  0
+#define KSW_CIGAR_INS    1
+#define KSW_CIGAR_DEL    2
+#define KSW_CIGAR_N_SKIP 3
 
 #ifdef __cplusplus
 extern "C" {
@@ -24,6 +34,7 @@ typedef struct {
 	int mte, mte_q;        // max score when reaching the end of target
 	int score;             // max score reaching both ends; may be KSW_NEG_INF
 	int m_cigar, n_cigar;
+	int reach_end;
 	uint32_t *cigar;
 } ksw_extz_t;
 
@@ -39,19 +50,27 @@ typedef struct {
  * @param mat       m*m scoring mattrix in one-dimension array
  * @param gapo      gap open penalty; a gap of length l cost "-(gapo+l*gape)"
  * @param gape      gap extension penalty
- * @param w         band width
+ * @param w         band width (<0 to disable)
- * @param zdrop     off-diagonal drop-off to stop extension (positive)
+ * @param zdrop     off-diagonal drop-off to stop extension (positive; <0 to disable)
  * @param flag      flag (see KSW_EZ_* macros)
  * @param ez        (out) scores and cigar
  */
-void ksw_extz(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+void ksw_extz(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+			  int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez);
+
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
 
 void ksw_extd(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
 			  int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int flag, ksw_extz_t *ez);
 
 void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int flag, ksw_extz_t *ez);
+				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+
+void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t gapo, int8_t gape, int8_t gapo2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez);
+
+void ksw_extf2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t mch, int8_t mis, int8_t e, int w, int xdrop, ksw_extz_t *ez);
 
 /**
  * Global alignment
@@ -67,6 +86,9 @@ int ksw_gg(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *ta
 int ksw_gg2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t gapo, int8_t gape, int w, int *m_cigar_, int *n_cigar_, uint32_t **cigar_);
 int ksw_gg2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t gapo, int8_t gape, int w, int *m_cigar_, int *n_cigar_, uint32_t **cigar_);
 
+void *ksw_ll_qinit(void *km, int size, int qlen, const uint8_t *query, int m, const int8_t *mat);
+int ksw_ll_i16(void *q, int tlen, const uint8_t *target, int gapo, int gape, int *qe, int *te);
+
 #ifdef __cplusplus
 }
 #endif
@@ -101,22 +123,34 @@ static inline uint32_t *ksw_push_cigar(void *km, int *n_cigar, int *m_cigar, uin
 //   bit 0-2: which type gets the max - 0 for H, 1 for E, 2 for F, 3 for \tilde{E} and 4 for \tilde{F}
 //   bit 3/0x08: 1 if a continuation on the E state (bit 5/0x20 for a continuation on \tilde{E})
 //   bit 4/0x10: 1 if a continuation on the F state (bit 6/0x40 for a continuation on \tilde{F})
-static inline void ksw_backtrack(void *km, int is_rot, int is_rev, const uint8_t *p, const int *off, int n_col, int i0, int j0, int *m_cigar_, int *n_cigar_, uint32_t **cigar_)
+static inline void ksw_backtrack(void *km, int is_rot, int is_rev, int min_intron_len, const uint8_t *p, const int *off, const int *off_end, int n_col, int i0, int j0,
+								 int *m_cigar_, int *n_cigar_, uint32_t **cigar_)
 { // p[] - lower 3 bits: which type gets the max; bit
 	int n_cigar = 0, m_cigar = *m_cigar_, i = i0, j = j0, r, state = 0;
 	uint32_t *cigar = *cigar_, tmp;
 	while (i >= 0 && j >= 0) { // at the beginning of the loop, _state_ tells us which state to check
-		if (is_rot) r = i + j, tmp = p[r * n_col + i - off[r]];
+		int force_state = -1;
-		else tmp = p[i * n_col + j - off[i]];
+		if (is_rot) {
+			r = i + j;
+			if (i < off[r]) force_state = 2;
+			if (off_end && i > off_end[r]) force_state = 1;
+			tmp = force_state < 0? p[(size_t)r * n_col + i - off[r]] : 0;
+		} else {
+			if (j < off[i]) force_state = 2;
+			if (off_end && j > off_end[i]) force_state = 1;
+			tmp = force_state < 0? p[(size_t)i * n_col + j - off[i]] : 0;
+		}
 		if (state == 0) state = tmp & 7; // if requesting the H state, find state one maximizes it.
 		else if (!(tmp >> (state + 2) & 1)) state = 0; // if requesting other states, _state_ stays the same if it is a continuation; otherwise, set to H
 		if (state == 0) state = tmp & 7; // TODO: probably this line can be merged into the "else if" line right above; not 100% sure
-		if (state == 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 0, 1), --i, --j; // match
+		if (force_state >= 0) state = force_state;
-		else if (state == 1 || state == 3) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 2, 1), --i; // deletion
+		if (state == 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, KSW_CIGAR_MATCH, 1), --i, --j;
-		else cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 1, 1), --j; // insertion
+		else if (state == 1 || (state == 3 && min_intron_len <= 0)) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, KSW_CIGAR_DEL, 1), --i;
+		else if (state == 3 && min_intron_len > 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, KSW_CIGAR_N_SKIP, 1), --i;
+		else cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, KSW_CIGAR_INS, 1), --j;
 	}
-	if (i >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 2, i + 1); // first deletion
+	if (i >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, min_intron_len > 0 && i >= min_intron_len? KSW_CIGAR_N_SKIP : KSW_CIGAR_DEL, i + 1); // first deletion
-	if (j >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 1, j + 1); // first insertion
+	if (j >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, KSW_CIGAR_INS, j + 1); // first insertion
 	if (!is_rev)
 		for (i = 0; i < n_cigar>>1; ++i) // reverse CIGAR
 			tmp = cigar[i], cigar[i] = cigar[n_cigar-1-i], cigar[n_cigar-1-i] = tmp;
@@ -127,7 +161,7 @@ static inline void ksw_reset_extz(ksw_extz_t *ez)
 {
 	ez->max_q = ez->max_t = ez->mqe_t = ez->mte_q = -1;
 	ez->max = 0, ez->score = ez->mqe = ez->mte = KSW_NEG_INF;
-	ez->n_cigar = 0, ez->zdropped = 0;
+	ez->n_cigar = 0, ez->zdropped = 0, ez->reach_end = 0;
 }
 
 static inline int ksw_apply_zdrop(ksw_extz_t *ez, int is_rot, int32_t H, int a, int b, int zdrop, int8_t e)
@@ -147,5 +181,4 @@ static inline int ksw_apply_zdrop(ksw_extz_t *ez, int is_rot, int32_t H, int a,
 	}
 	return 0;
 }
-
 #endif

ksw2_dispatch.c

@@ -0,0 +1,96 @@
+#ifdef KSW_CPU_DISPATCH
+#include <stdlib.h>
+#include "ksw2.h"
+
+#define SIMD_SSE     0x1
+#define SIMD_SSE2    0x2
+#define SIMD_SSE3    0x4
+#define SIMD_SSSE3   0x8
+#define SIMD_SSE4_1  0x10
+#define SIMD_SSE4_2  0x20
+#define SIMD_AVX     0x40
+#define SIMD_AVX2    0x80
+#define SIMD_AVX512F 0x100
+
+#ifndef _MSC_VER
+// adapted from https://github.com/01org/linux-sgx/blob/master/common/inc/internal/linux/cpuid_gnu.h
+void __cpuidex(int cpuid[4], int func_id, int subfunc_id)
+{
+#if defined(__x86_64__)
+	__asm__ volatile ("cpuid"
+			: "=a" (cpuid[0]), "=b" (cpuid[1]), "=c" (cpuid[2]), "=d" (cpuid[3])
+			: "0" (func_id), "2" (subfunc_id));
+#else // on 32bit, ebx can NOT be used as PIC code
+	__asm__ volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
+			: "=a" (cpuid[0]), "=r" (cpuid[1]), "=c" (cpuid[2]), "=d" (cpuid[3])
+			: "0" (func_id), "2" (subfunc_id));
+#endif
+}
+#endif
+
+static int ksw_simd = -1;
+
+static int x86_simd(void)
+{
+	int flag = 0, cpuid[4], max_id;
+	__cpuidex(cpuid, 0, 0);
+	max_id = cpuid[0];
+	if (max_id == 0) return 0;
+	__cpuidex(cpuid, 1, 0);
+	if (cpuid[3]>>25&1) flag |= SIMD_SSE;
+	if (cpuid[3]>>26&1) flag |= SIMD_SSE2;
+	if (cpuid[2]>>0 &1) flag |= SIMD_SSE3;
+	if (cpuid[2]>>9 &1) flag |= SIMD_SSSE3;
+	if (cpuid[2]>>19&1) flag |= SIMD_SSE4_1;
+	if (cpuid[2]>>20&1) flag |= SIMD_SSE4_2;
+	if (cpuid[2]>>28&1) flag |= SIMD_AVX;
+	if (max_id >= 7) {
+		__cpuidex(cpuid, 7, 0);
+		if (cpuid[1]>>5 &1) flag |= SIMD_AVX2;
+		if (cpuid[1]>>16&1) flag |= SIMD_AVX512F;
+	}
+	return flag;
+}
+
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+{
+	extern void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	extern void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	if (ksw_simd < 0) ksw_simd = x86_simd();
+	if (ksw_simd & SIMD_SSE4_1)
+		ksw_extz2_sse41(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, end_bonus, flag, ez);
+	else if (ksw_simd & SIMD_SSE2)
+		ksw_extz2_sse2(km, qlen, query, tlen, target, m, mat, q, e, w, zdrop, end_bonus, flag, ez);
+	else abort();
+}
+
+void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+{
+	extern void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	extern void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
+	if (ksw_simd < 0) ksw_simd = x86_simd();
+	if (ksw_simd & SIMD_SSE4_1)
+		ksw_extd2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
+	else if (ksw_simd & SIMD_SSE2)
+		ksw_extd2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
+	else abort();
+}
+
+void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez)
+{
+	extern void ksw_exts2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez);
+	extern void ksw_exts2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez);
+	if (ksw_simd < 0) ksw_simd = x86_simd();
+	if (ksw_simd & SIMD_SSE4_1)
+		ksw_exts2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, junc_bonus, flag, junc, ez);
+	else if (ksw_simd & SIMD_SSE2)
+		ksw_exts2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, junc_bonus, flag, junc, ez);
+	else abort();
+}
+#endif

ksw2_extd2_sse.c

@@ -1,16 +1,39 @@
 #include <string.h>
 #include <stdio.h>
+#include <assert.h>
 #include "ksw2.h"
 
 #ifdef __SSE2__
+#ifdef USE_SIMDE
+#include <simde/x86/sse2.h>
+#else
 #include <emmintrin.h>
-
-#ifdef __SSE4_1__
-#include <smmintrin.h>
 #endif
 
+#ifdef KSW_SSE2_ONLY
+#undef __SSE4_1__
+#endif
+
+#ifdef __SSE4_1__
+#ifdef USE_SIMDE
+#include <simde/x86/sse4.1.h>
+#else
+#include <smmintrin.h>
+#endif
+#endif
+
+#ifdef KSW_CPU_DISPATCH
+#ifdef __SSE4_1__
+void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#else
+void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif
+#else
 void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
-				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez)
+				   int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif // ~KSW_CPU_DISPATCH
 {
 #define __dp_code_block1 \
 	z = _mm_load_si128(&s[t]); \
@@ -42,11 +65,11 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	a2= _mm_sub_epi8(a2, tmp); \
 	b2= _mm_sub_epi8(b2, tmp);
 
-	int r, t, qe = q + e, n_col_, *off = 0, tlen_, qlen_, last_st, last_en, wl, wr, max_sc, min_sc, long_thres, long_diff;
+	int r, t, qe = q + e, n_col_, *off = 0, *off_end = 0, tlen_, qlen_, last_st, last_en, wl, wr, max_sc, min_sc, long_thres, long_diff;
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_;
+	__m128i q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_, sc_N_;
 	__m128i *u, *v, *x, *y, *x2, *y2, *s, *p = 0;
 
 	ksw_reset_extz(ez);
@@ -61,12 +84,14 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	qe2_    = _mm_set1_epi8(q2 + e2);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = mat[m*m-1] == 0? _mm_set1_epi8(-e2) : _mm_set1_epi8(mat[m*m-1]);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 
 	if (w < 0) w = tlen > qlen? tlen : qlen;
 	wl = wr = w;
 	tlen_ = (tlen + 15) / 16;
-	n_col_ = ((w + 1 < tlen? (w + 1 < qlen? w + 1 : qlen): tlen) + 15) / 16 + 1;
+	n_col_ = qlen < tlen? qlen : tlen;
+	n_col_ = ((n_col_ < w + 1? n_col_ : w + 1) + 15) / 16 + 1;
 	qlen_ = (qlen + 15) / 16;
 	for (t = 1, max_sc = mat[0], min_sc = mat[1]; t < m * m; ++t) {
 		max_sc = max_sc > mat[t]? max_sc : mat[t];
@@ -94,9 +119,10 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 		for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF;
 	}
 	if (with_cigar) {
-		mem2 = (uint8_t*)kmalloc(km, ((qlen + tlen - 1) * n_col_ + 1) * 16);
+		mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * 16);
 		p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4);
-		off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int));
+		off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2);
+		off_end = off + qlen + tlen - 1;
 	}
 
 	for (t = 0; t < qlen; ++t) qr[t] = query[qlen - 1 - t];
@@ -105,7 +131,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	for (r = 0, last_st = last_en = -1; r < qlen + tlen - 1; ++r) {
 		int st = 0, en = tlen - 1, st0, en0, st_, en_;
 		int8_t x1, x21, v1;
-		uint8_t *qrr = qr + (qlen - 1 - r), p_en0 = 0;
+		uint8_t *qrr = qr + (qlen - 1 - r);
 		int8_t *u8 = (int8_t*)u, *v8 = (int8_t*)v, *x8 = (int8_t*)x, *x28 = (int8_t*)x2;
 		__m128i x1_, x21_, v1_;
 		// find the boundaries
@@ -145,10 +171,11 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp);
 			}
 		} else {
@@ -160,6 +187,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 		x21_ = _mm_cvtsi32_si128((uint8_t)x21);
 		v1_  = _mm_cvtsi32_si128((uint8_t)v1);
 		st_ = st / 16, en_ = en / 16;
+		assert(en_ - st_ + 1 <= n_col_);
 		if (!with_cigar) { // score only
 			for (t = st_; t <= en_; ++t) {
 				__m128i z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp;
@@ -198,19 +226,8 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 #endif
 			}
 		} else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment
-			__m128i *pr = p + r * n_col_ - st_;
+			__m128i *pr = p + (size_t)r * n_col_ - st_;
-			off[r] = st;
+			off[r] = st, off_end[r] = en;
-			if (en0 < r && en0 < tlen - 1) { // to avoid backtracking out of the band; this assumes a fixed band
-				int8_t a, a2, z = ((uint8_t*)s)[en0];
-				a = x8[en0-1] + v8[en0-1];
-				p_en0 = a > z? 1 : 0;
-				z = a > z? a : z;
-				p_en0 |= a - (z - q) > 0? 1<<4 : 0;
-				a2 = x28[en0-1] + v8[en0-1];
-				p_en0 = a2 > z? 3 : p_en0;
-				z = a2 > z? a2 : z;
-				p_en0 |= a2 - (z - q2) > 0? 1<<6 : 0;
-			}
 			for (t = st_; t <= en_; ++t) {
 				__m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp;
 				__dp_code_block1;
@@ -256,19 +273,8 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				_mm_store_si128(&pr[t], d);
 			}
 		} else { // gap right-alignment
-			__m128i *pr = p + r * n_col_ - st_;
+			__m128i *pr = p + (size_t)r * n_col_ - st_;
-			off[r] = st;
+			off[r] = st, off_end[r] = en;
-			if (en0 < r && en0 < tlen - 1) { // to avoid backtracking out of the band; this assumes a fixed band
-				int8_t a, a2, z = ((uint8_t*)s)[en0];
-				a = x8[en0-1] + v8[en0-1];
-				p_en0 = a >= z? 1 : 0;
-				z = a >= z? a : z;
-				p_en0 |= a - (z - q) >= 0? 1<<4 : 0;
-				a2 = x28[en0-1] + v8[en0-1];
-				p_en0 = a2 >= z? 3 : p_en0;
-				z = a2 >= z? a2 : z;
-				p_en0 |= a2 - (z - q2) >= 0? 1<<6 : 0;
-			}
 			for (t = st_; t <= en_; ++t) {
 				__m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp;
 				__dp_code_block1;
@@ -314,7 +320,6 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				_mm_store_si128(&pr[t], d);
 			}
 		}
-		if (with_cigar && en0 < r && en0 < tlen - 1) ((uint8_t*)(p + r * n_col_))[en0 - st] = p_en0;
 		if (!approx_max) { // find the exact max with a 32-bit score array
 			int32_t max_H, max_t;
 			// compute H[], max_H and max_t
@@ -383,10 +388,14 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	if (!approx_max) kfree(km, H);
 	if (with_cigar) { // backtrack
 		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
-		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
+		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) {
-			ksw_backtrack(km, 1, rev_cigar, (uint8_t*)p, off, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
-		else if (ez->max_t >= 0 && ez->max_q >= 0)
+		} else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > (int)ez->max) {
-			ksw_backtrack(km, 1, rev_cigar, (uint8_t*)p, off, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+			ez->reach_end = 1;
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		} else if (ez->max_t >= 0 && ez->max_q >= 0) {
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		}
 		kfree(km, mem2); kfree(km, off);
 	}
 }

ksw2_exts2_sse.c

@@ -0,0 +1,416 @@
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include "ksw2.h"
+
+#ifdef __SSE2__
+#ifdef USE_SIMDE
+#include <simde/x86/sse2.h>
+#else
+#include <emmintrin.h>
+#endif
+#ifdef KSW_SSE2_ONLY
+#undef __SSE4_1__
+#endif
+
+#ifdef __SSE4_1__
+#ifdef USE_SIMDE
+#include <simde/x86/sse4.1.h>
+#else
+#include <smmintrin.h>
+#endif
+#endif
+
+#ifdef KSW_CPU_DISPATCH
+#ifdef __SSE4_1__
+void ksw_exts2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez)
+#else
+void ksw_exts2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez)
+#endif
+#else
+void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
+				   int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez)
+#endif // ~KSW_CPU_DISPATCH
+{
+#define __dp_code_block1 \
+	z = _mm_load_si128(&s[t]); \
+	xt1 = _mm_load_si128(&x[t]);                     /* xt1 <- x[r-1][t..t+15] */ \
+	tmp = _mm_srli_si128(xt1, 15);                   /* tmp <- x[r-1][t+15] */ \
+	xt1 = _mm_or_si128(_mm_slli_si128(xt1, 1), x1_); /* xt1 <- x[r-1][t-1..t+14] */ \
+	x1_ = tmp; \
+	vt1 = _mm_load_si128(&v[t]);                     /* vt1 <- v[r-1][t..t+15] */ \
+	tmp = _mm_srli_si128(vt1, 15);                   /* tmp <- v[r-1][t+15] */ \
+	vt1 = _mm_or_si128(_mm_slli_si128(vt1, 1), v1_); /* vt1 <- v[r-1][t-1..t+14] */ \
+	v1_ = tmp; \
+	a = _mm_add_epi8(xt1, vt1);                      /* a <- x[r-1][t-1..t+14] + v[r-1][t-1..t+14] */ \
+	ut = _mm_load_si128(&u[t]);                      /* ut <- u[t..t+15] */ \
+	b = _mm_add_epi8(_mm_load_si128(&y[t]), ut);     /* b <- y[r-1][t..t+15] + u[r-1][t..t+15] */ \
+	x2t1= _mm_load_si128(&x2[t]); \
+	tmp = _mm_srli_si128(x2t1, 15); \
+	x2t1= _mm_or_si128(_mm_slli_si128(x2t1, 1), x21_); \
+	x21_= tmp; \
+	a2  = _mm_add_epi8(x2t1, vt1); \
+	a2a = _mm_add_epi8(a2, _mm_load_si128(&acceptor[t]));
+
+#define __dp_code_block2 \
+	_mm_store_si128(&u[t], _mm_sub_epi8(z, vt1));    /* u[r][t..t+15] <- z - v[r-1][t-1..t+14] */ \
+	_mm_store_si128(&v[t], _mm_sub_epi8(z, ut));     /* v[r][t..t+15] <- z - u[r-1][t..t+15] */ \
+	tmp = _mm_sub_epi8(z, q_); \
+	a = _mm_sub_epi8(a, tmp); \
+	b = _mm_sub_epi8(b, tmp); \
+	a2= _mm_sub_epi8(a2, _mm_sub_epi8(z, q2_));
+
+	int r, t, qe = q + e, n_col_, *off = 0, *off_end = 0, tlen_, qlen_, last_st, last_en, max_sc, min_sc, long_thres, long_diff;
+	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
+	int32_t *H = 0, H0 = 0, last_H0_t = 0;
+	uint8_t *qr, *sf, *mem, *mem2 = 0;
+	__m128i q_, q2_, qe_, zero_, sc_mch_, sc_mis_, sc_N_, m1_;
+	__m128i *u, *v, *x, *y, *x2, *s, *p = 0, *donor, *acceptor;
+
+	ksw_reset_extz(ez);
+	if (m <= 1 || qlen <= 0 || tlen <= 0 || q2 <= q + e) return;
+
+	zero_   = _mm_set1_epi8(0);
+	q_      = _mm_set1_epi8(q);
+	q2_     = _mm_set1_epi8(q2);
+	qe_     = _mm_set1_epi8(q + e);
+	sc_mch_ = _mm_set1_epi8(mat[0]);
+	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = mat[m*m-1] == 0? _mm_set1_epi8(-e) : _mm_set1_epi8(mat[m*m-1]);
+	m1_     = _mm_set1_epi8(m - 1); // wildcard
+
+	tlen_ = (tlen + 15) / 16;
+	n_col_ = ((qlen < tlen? qlen : tlen) + 15) / 16 + 1;
+	qlen_ = (qlen + 15) / 16;
+	for (t = 1, max_sc = mat[0], min_sc = mat[1]; t < m * m; ++t) {
+		max_sc = max_sc > mat[t]? max_sc : mat[t];
+		min_sc = min_sc < mat[t]? min_sc : mat[t];
+	}
+	if (-min_sc > 2 * (q + e)) return; // otherwise, we won't see any mismatches
+
+	long_thres = (q2 - q) / e - 1;
+	if (q2 > q + e + long_thres * e)
+		++long_thres;
+	long_diff = long_thres * e - (q2 - q);
+
+	mem = (uint8_t*)kcalloc(km, tlen_ * 9 + qlen_ + 1, 16);
+	u = (__m128i*)(((size_t)mem + 15) >> 4 << 4); // 16-byte aligned
+	v = u + tlen_, x = v + tlen_, y = x + tlen_, x2 = y + tlen_;
+	donor = x2 + tlen_, acceptor = donor + tlen_;
+	s = acceptor + tlen_, sf = (uint8_t*)(s + tlen_), qr = sf + tlen_ * 16;
+	memset(u,  -q - e,  tlen_ * 16 * 4); // this set u, v, x, y (because they are in the same array)
+	memset(x2, -q2,     tlen_ * 16);
+	if (!approx_max) {
+		H = (int32_t*)kmalloc(km, tlen_ * 16 * 4);
+		for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF;
+	}
+	if (with_cigar) {
+		mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * 16);
+		p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4);
+		off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2);
+		off_end = off + qlen + tlen - 1;
+	}
+
+	for (t = 0; t < qlen; ++t) qr[t] = query[qlen - 1 - t];
+	memcpy(sf, target, tlen);
+
+	// set the donor and acceptor arrays. TODO: this assumes 0/1/2/3 encoding!
+	if (flag & (KSW_EZ_SPLICE_FOR|KSW_EZ_SPLICE_REV)) {
+		int semi_cost = flag&KSW_EZ_SPLICE_FLANK? -noncan/2 : 0; // GTr or yAG is worth 0.5 bit; see PMID:18688272
+		memset(donor, -noncan, tlen_ * 16);
+		memset(acceptor, -noncan, tlen_ * 16);
+		if (!(flag & KSW_EZ_REV_CIGAR)) {
+			for (t = 0; t < tlen - 4; ++t) {
+				int can_type = 0; // type of canonical site: 0=none, 1=GT/AG only, 2=GTr/yAG
+				if ((flag & KSW_EZ_SPLICE_FOR) && target[t+1] == 2 && target[t+2] == 3) can_type = 1; // GTr...
+				if ((flag & KSW_EZ_SPLICE_REV) && target[t+1] == 1 && target[t+2] == 3) can_type = 1; // CTr...
+				if (can_type && (target[t+3] == 0 || target[t+3] == 2)) can_type = 2;
+				if (can_type) ((int8_t*)donor)[t] = can_type == 2? 0 : semi_cost;
+			}
+			if (junc)
+				for (t = 0; t < tlen - 1; ++t)
+					if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t+1]&1)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t+1]&8)))
+						((int8_t*)donor)[t] += junc_bonus;
+			for (t = 2; t < tlen; ++t) {
+				int can_type = 0;
+				if ((flag & KSW_EZ_SPLICE_FOR) && target[t-1] == 0 && target[t] == 2) can_type = 1; // ...yAG
+				if ((flag & KSW_EZ_SPLICE_REV) && target[t-1] == 0 && target[t] == 1) can_type = 1; // ...yAC
+				if (can_type && (target[t-2] == 1 || target[t-2] == 3)) can_type = 2;
+				if (can_type) ((int8_t*)acceptor)[t] = can_type == 2? 0 : semi_cost;
+			}
+			if (junc)
+				for (t = 0; t < tlen; ++t)
+					if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t]&2)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t]&4)))
+						((int8_t*)acceptor)[t] += junc_bonus;
+		} else {
+			for (t = 0; t < tlen - 4; ++t) {
+				int can_type = 0; // type of canonical site: 0=none, 1=GT/AG only, 2=GTr/yAG
+				if ((flag & KSW_EZ_SPLICE_FOR) && target[t+1] == 2 && target[t+2] == 0) can_type = 1; // GAy...
+				if ((flag & KSW_EZ_SPLICE_REV) && target[t+1] == 1 && target[t+2] == 0) can_type = 1; // CAy...
+				if (can_type && (target[t+3] == 1 || target[t+3] == 3)) can_type = 2;
+				if (can_type) ((int8_t*)donor)[t] = can_type == 2? 0 : semi_cost;
+			}
+			if (junc)
+				for (t = 0; t < tlen - 1; ++t)
+					if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t+1]&2)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t+1]&4)))
+						((int8_t*)donor)[t] += junc_bonus;
+			for (t = 2; t < tlen; ++t) {
+				int can_type = 0;
+				if ((flag & KSW_EZ_SPLICE_FOR) && target[t-1] == 3 && target[t] == 2) can_type = 1; // ...rTG
+				if ((flag & KSW_EZ_SPLICE_REV) && target[t-1] == 3 && target[t] == 1) can_type = 1; // ...rTC
+				if (can_type && (target[t-2] == 0 || target[t-2] == 2)) can_type = 2;
+				if (can_type) ((int8_t*)acceptor)[t] = can_type == 2? 0 : semi_cost;
+			}
+			if (junc)
+				for (t = 0; t < tlen; ++t)
+					if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t]&1)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t]&8)))
+						((int8_t*)acceptor)[t] += junc_bonus;
+		}
+	}
+
+	for (r = 0, last_st = last_en = -1; r < qlen + tlen - 1; ++r) {
+		int st = 0, en = tlen - 1, st0, en0, st_, en_;
+		int8_t x1, x21, v1, *u8 = (int8_t*)u, *v8 = (int8_t*)v;
+		uint8_t *qrr = qr + (qlen - 1 - r);
+		__m128i x1_, x21_, v1_;
+		// find the boundaries
+		if (st < r - qlen + 1) st = r - qlen + 1;
+		if (en > r) en = r;
+		st0 = st, en0 = en;
+		st = st / 16 * 16, en = (en + 16) / 16 * 16 - 1;
+		// set boundary conditions
+		if (st > 0) {
+			if (st - 1 >= last_st && st - 1 <= last_en)
+				x1 = ((int8_t*)x)[st - 1], x21 = ((int8_t*)x2)[st - 1], v1 = v8[st - 1]; // (r-1,s-1) calculated in the last round
+			else x1 = -q - e, x21 = -q2, v1 = -q - e;
+		} else {
+			x1 = -q - e, x21 = -q2;
+			v1 = r == 0? -q - e : r < long_thres? -e : r == long_thres? long_diff : 0;
+		}
+		if (en >= r) {
+			((int8_t*)y)[r] = -q - e;
+			u8[r] = r == 0? -q - e : r < long_thres? -e : r == long_thres? long_diff : 0;
+		}
+		// loop fission: set scores first
+		if (!(flag & KSW_EZ_GENERIC_SC)) {
+			for (t = st0; t <= en0; t += 16) {
+				__m128i sq, st, tmp, mask;
+				sq = _mm_loadu_si128((__m128i*)&sf[t]);
+				st = _mm_loadu_si128((__m128i*)&qrr[t]);
+				mask = _mm_or_si128(_mm_cmpeq_epi8(sq, m1_), _mm_cmpeq_epi8(st, m1_));
+				tmp = _mm_cmpeq_epi8(sq, st);
+#ifdef __SSE4_1__
+				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
+#else
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
+#endif
+				_mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp);
+			}
+		} else {
+			for (t = st0; t <= en0; ++t)
+				((uint8_t*)s)[t] = mat[sf[t] * m + qrr[t]];
+		}
+		// core loop
+		x1_  = _mm_cvtsi32_si128((uint8_t)x1);
+		x21_ = _mm_cvtsi32_si128((uint8_t)x21);
+		v1_  = _mm_cvtsi32_si128((uint8_t)v1);
+		st_ = st / 16, en_ = en / 16;
+		assert(en_ - st_ + 1 <= n_col_);
+		if (!with_cigar) { // score only
+			for (t = st_; t <= en_; ++t) {
+				__m128i z, a, b, a2, a2a, xt1, x2t1, vt1, ut, tmp;
+				__dp_code_block1;
+#ifdef __SSE4_1__
+				z = _mm_max_epi8(z, a);
+				z = _mm_max_epi8(z, b);
+				z = _mm_max_epi8(z, a2a);
+				__dp_code_block2; // save u[] and v[]; update a, b and a2
+				_mm_store_si128(&x[t],  _mm_sub_epi8(_mm_max_epi8(a,  zero_), qe_));
+				_mm_store_si128(&y[t],  _mm_sub_epi8(_mm_max_epi8(b,  zero_), qe_));
+				tmp = _mm_load_si128(&donor[t]);
+				_mm_store_si128(&x2[t], _mm_sub_epi8(_mm_max_epi8(a2, tmp), q2_));
+#else
+				tmp = _mm_cmpgt_epi8(a,  z);
+				z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a));
+				tmp = _mm_cmpgt_epi8(b,  z);
+				z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, b));
+				tmp = _mm_cmpgt_epi8(a2a, z);
+				z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a2a));
+				__dp_code_block2;
+				tmp = _mm_cmpgt_epi8(a, zero_);
+				_mm_store_si128(&x[t],  _mm_sub_epi8(_mm_and_si128(tmp, a),  qe_));
+				tmp = _mm_cmpgt_epi8(b, zero_);
+				_mm_store_si128(&y[t],  _mm_sub_epi8(_mm_and_si128(tmp, b),  qe_));
+				tmp = _mm_load_si128(&donor[t]); // TODO: check if this is correct
+				tmp = _mm_cmpgt_epi8(a2, tmp);
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp, tmp), _mm_and_si128(tmp, a2));
+				_mm_store_si128(&x2[t], _mm_sub_epi8(tmp, q2_));
+#endif
+			}
+		} else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment
+			__m128i *pr = p + r * n_col_ - st_;
+			off[r] = st, off_end[r] = en;
+			for (t = st_; t <= en_; ++t) {
+				__m128i d, z, a, b, a2, a2a, xt1, x2t1, vt1, ut, tmp, tmp2;
+				__dp_code_block1;
+#ifdef __SSE4_1__
+				d = _mm_and_si128(_mm_cmpgt_epi8(a, z), _mm_set1_epi8(1));       // d = a  > z? 1 : 0
+				z = _mm_max_epi8(z, a);
+				d = _mm_blendv_epi8(d, _mm_set1_epi8(2), _mm_cmpgt_epi8(b,  z)); // d = b  > z? 2 : d
+				z = _mm_max_epi8(z, b);
+				d = _mm_blendv_epi8(d, _mm_set1_epi8(3), _mm_cmpgt_epi8(a2a, z)); // d = a2 > z? 3 : d
+				z = _mm_max_epi8(z, a2a);
+#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8()
+				tmp = _mm_cmpgt_epi8(a,  z);
+				d = _mm_and_si128(tmp, _mm_set1_epi8(1));
+				z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a));
+				tmp = _mm_cmpgt_epi8(b,  z);
+				d = _mm_or_si128(_mm_andnot_si128(tmp, d), _mm_and_si128(tmp, _mm_set1_epi8(2)));
+				z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, b));
+				tmp = _mm_cmpgt_epi8(a2a, z);
+				d = _mm_or_si128(_mm_andnot_si128(tmp, d), _mm_and_si128(tmp, _mm_set1_epi8(3)));
+				z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a2a));
+#endif
+				__dp_code_block2;
+				tmp = _mm_cmpgt_epi8(a, zero_);
+				_mm_store_si128(&x[t],  _mm_sub_epi8(_mm_and_si128(tmp, a),  qe_));
+				d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0
+				tmp = _mm_cmpgt_epi8(b, zero_);
+				_mm_store_si128(&y[t],  _mm_sub_epi8(_mm_and_si128(tmp, b),  qe_));
+				d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0
+
+				tmp2 = _mm_load_si128(&donor[t]);
+				tmp = _mm_cmpgt_epi8(a2, tmp2);
+#ifdef __SSE4_1__
+				tmp2 = _mm_max_epi8(a2, tmp2);
+#else
+				tmp2 = _mm_or_si128(_mm_andnot_si128(tmp, tmp2), _mm_and_si128(tmp, a2));
+#endif
+				_mm_store_si128(&x2[t], _mm_sub_epi8(tmp2, q2_));
+				d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x20)));
+				_mm_store_si128(&pr[t], d);
+			}
+		} else { // gap right-alignment
+			__m128i *pr = p + r * n_col_ - st_;
+			off[r] = st, off_end[r] = en;
+			for (t = st_; t <= en_; ++t) {
+				__m128i d, z, a, b, a2, a2a, xt1, x2t1, vt1, ut, tmp, tmp2;
+				__dp_code_block1;
+#ifdef __SSE4_1__
+				d = _mm_andnot_si128(_mm_cmpgt_epi8(z, a), _mm_set1_epi8(1));    // d = z > a?  0 : 1
+				z = _mm_max_epi8(z, a);
+				d = _mm_blendv_epi8(_mm_set1_epi8(2), d, _mm_cmpgt_epi8(z, b));  // d = z > b?  d : 2
+				z = _mm_max_epi8(z, b);
+				d = _mm_blendv_epi8(_mm_set1_epi8(3), d, _mm_cmpgt_epi8(z, a2a)); // d = z > a2? d : 3
+				z = _mm_max_epi8(z, a2a);
+#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8()
+				tmp = _mm_cmpgt_epi8(z, a);
+				d = _mm_andnot_si128(tmp, _mm_set1_epi8(1));
+				z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a));
+				tmp = _mm_cmpgt_epi8(z, b);
+				d = _mm_or_si128(_mm_and_si128(tmp, d), _mm_andnot_si128(tmp, _mm_set1_epi8(2)));
+				z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, b));
+				tmp = _mm_cmpgt_epi8(z, a2a);
+				d = _mm_or_si128(_mm_and_si128(tmp, d), _mm_andnot_si128(tmp, _mm_set1_epi8(3)));
+				z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a2a));
+#endif
+				__dp_code_block2;
+				tmp = _mm_cmpgt_epi8(zero_, a);
+				_mm_store_si128(&x[t],  _mm_sub_epi8(_mm_andnot_si128(tmp, a),  qe_));
+				d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0
+				tmp = _mm_cmpgt_epi8(zero_, b);
+				_mm_store_si128(&y[t],  _mm_sub_epi8(_mm_andnot_si128(tmp, b),  qe_));
+				d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0
+
+				tmp2 = _mm_load_si128(&donor[t]);
+				tmp = _mm_cmpgt_epi8(tmp2, a2);
+#ifdef __SSE4_1__
+				tmp2 = _mm_max_epi8(tmp2, a2);
+#else
+				tmp2 = _mm_or_si128(_mm_andnot_si128(tmp, a2), _mm_and_si128(tmp, tmp2));
+#endif
+				_mm_store_si128(&x2[t], _mm_sub_epi8(tmp2, q2_));
+				d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0
+				_mm_store_si128(&pr[t], d);
+			}
+		}
+		if (!approx_max) { // find the exact max with a 32-bit score array
+			int32_t max_H, max_t;
+			// compute H[], max_H and max_t
+			if (r > 0) {
+				int32_t HH[4], tt[4], en1 = st0 + (en0 - st0) / 4 * 4, i;
+				__m128i max_H_, max_t_;
+				max_H = H[en0] = en0 > 0? H[en0-1] + u8[en0] : H[en0] + v8[en0]; // special casing the last element
+				max_t = en0;
+				max_H_ = _mm_set1_epi32(max_H);
+				max_t_ = _mm_set1_epi32(max_t);
+				for (t = st0; t < en1; t += 4) { // this implements: H[t]+=v8[t]-qe; if(H[t]>max_H) max_H=H[t],max_t=t;
+					__m128i H1, tmp, t_;
+					H1 = _mm_loadu_si128((__m128i*)&H[t]);
+					t_ = _mm_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3]);
+					H1 = _mm_add_epi32(H1, t_);
+					_mm_storeu_si128((__m128i*)&H[t], H1);
+					t_ = _mm_set1_epi32(t);
+					tmp = _mm_cmpgt_epi32(H1, max_H_);
+#ifdef __SSE4_1__
+					max_H_ = _mm_blendv_epi8(max_H_, H1, tmp);
+					max_t_ = _mm_blendv_epi8(max_t_, t_, tmp);
+#else
+					max_H_ = _mm_or_si128(_mm_and_si128(tmp, H1), _mm_andnot_si128(tmp, max_H_));
+					max_t_ = _mm_or_si128(_mm_and_si128(tmp, t_), _mm_andnot_si128(tmp, max_t_));
+#endif
+				}
+				_mm_storeu_si128((__m128i*)HH, max_H_);
+				_mm_storeu_si128((__m128i*)tt, max_t_);
+				for (i = 0; i < 4; ++i)
+					if (max_H < HH[i]) max_H = HH[i], max_t = tt[i] + i;
+				for (; t < en0; ++t) { // for the rest of values that haven't been computed with SSE
+					H[t] += (int32_t)v8[t];
+					if (H[t] > max_H)
+						max_H = H[t], max_t = t;
+				}
+			} else H[0] = v8[0] - qe, max_H = H[0], max_t = 0; // special casing r==0
+			// update ez
+			if (en0 == tlen - 1 && H[en0] > ez->mte)
+				ez->mte = H[en0], ez->mte_q = r - en;
+			if (r - st0 == qlen - 1 && H[st0] > ez->mqe)
+				ez->mqe = H[st0], ez->mqe_t = st0;
+			if (ksw_apply_zdrop(ez, 1, max_H, r, max_t, zdrop, 0)) break;
+			if (r == qlen + tlen - 2 && en0 == tlen - 1)
+				ez->score = H[tlen - 1];
+		} else { // find approximate max; Z-drop might be inaccurate, too.
+			if (r > 0) {
+				if (last_H0_t >= st0 && last_H0_t <= en0 && last_H0_t + 1 >= st0 && last_H0_t + 1 <= en0) {
+					int32_t d0 = v8[last_H0_t];
+					int32_t d1 = u8[last_H0_t + 1];
+					if (d0 > d1) H0 += d0;
+					else H0 += d1, ++last_H0_t;
+				} else if (last_H0_t >= st0 && last_H0_t <= en0) {
+					H0 += v8[last_H0_t];
+				} else {
+					++last_H0_t, H0 += u8[last_H0_t];
+				}
+			} else H0 = v8[0] - qe, last_H0_t = 0;
+			if ((flag & KSW_EZ_APPROX_DROP) && ksw_apply_zdrop(ez, 1, H0, r, last_H0_t, zdrop, 0)) break;
+			if (r == qlen + tlen - 2 && en0 == tlen - 1)
+				ez->score = H0;
+		}
+		last_st = st, last_en = en;
+		//for (t = st0; t <= en0; ++t) printf("(%d,%d)\t(%d,%d,%d,%d)\t%d\n", r, t, ((int8_t*)u)[t], ((int8_t*)v)[t], ((int8_t*)x)[t], ((int8_t*)y)[t], H[t]); // for debugging
+	}
+	kfree(km, mem);
+	if (!approx_max) kfree(km, H);
+	if (with_cigar) { // backtrack
+		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
+		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
+			ksw_backtrack(km, 1, rev_cigar, long_thres, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		else if (ez->max_t >= 0 && ez->max_q >= 0)
+			ksw_backtrack(km, 1, rev_cigar, long_thres, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		kfree(km, mem2); kfree(km, off);
+	}
+}
+#endif // __SSE2__

ksw2_extz2_sse.c

@@ -1,14 +1,35 @@
 #include <string.h>
+#include <assert.h>
 #include "ksw2.h"
 
 #ifdef __SSE2__
+#ifdef USE_SIMDE
+#include <simde/x86/sse2.h>
+#else
 #include <emmintrin.h>
-
-#ifdef __SSE4_1__
-#include <smmintrin.h>
 #endif
 
-void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez)
+#ifdef KSW_SSE2_ONLY
+#undef __SSE4_1__
+#endif
+
+#ifdef __SSE4_1__
+#ifdef USE_SIMDE
+#include <simde/x86/sse4.1.h>
+#else
+#include <smmintrin.h>
+#endif
+#endif
+
+#ifdef KSW_CPU_DISPATCH
+#ifdef __SSE4_1__
+void ksw_extz2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#else
+void ksw_extz2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif
+#else
+void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
+#endif // ~KSW_CPU_DISPATCH
 {
 #define __dp_code_block1 \
 	z = _mm_add_epi8(_mm_load_si128(&s[t]), qe2_); \
@@ -33,11 +54,11 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	a = _mm_sub_epi8(a, z); \
 	b = _mm_sub_epi8(b, z);
 
-	int r, t, qe = q + e, n_col_, *off = 0, tlen_, qlen_, last_st, last_en, wl, wr, max_sc, min_sc;
+	int r, t, qe = q + e, n_col_, *off = 0, *off_end = 0, tlen_, qlen_, last_st, last_en, wl, wr, max_sc, min_sc;
 	int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
 	int32_t *H = 0, H0 = 0, last_H0_t = 0;
 	uint8_t *qr, *sf, *mem, *mem2 = 0;
-	__m128i q_, qe2_, zero_, flag1_, flag2_, flag8_, flag16_, sc_mch_, sc_mis_, m1_, max_sc_;
+	__m128i q_, qe2_, zero_, flag1_, flag2_, flag8_, flag16_, sc_mch_, sc_mis_, sc_N_, m1_, max_sc_;
 	__m128i *u, *v, *x, *y, *s, *p = 0;
 
 	ksw_reset_extz(ez);
@@ -52,13 +73,15 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	flag16_ = _mm_set1_epi8(0x10);
 	sc_mch_ = _mm_set1_epi8(mat[0]);
 	sc_mis_ = _mm_set1_epi8(mat[1]);
+	sc_N_   = mat[m*m-1] == 0? _mm_set1_epi8(-e) : _mm_set1_epi8(mat[m*m-1]);
 	m1_     = _mm_set1_epi8(m - 1); // wildcard
 	max_sc_ = _mm_set1_epi8(mat[0] + (q + e) * 2);
 
 	if (w < 0) w = tlen > qlen? tlen : qlen;
 	wl = wr = w;
 	tlen_ = (tlen + 15) / 16;
-	n_col_ = ((w + 1 < tlen? (w + 1 < qlen? w + 1 : qlen): tlen) + 15) / 16 + 1;
+	n_col_ = qlen < tlen? qlen : tlen;
+	n_col_ = ((n_col_ < w + 1? n_col_ : w + 1) + 15) / 16 + 1;
 	qlen_ = (qlen + 15) / 16;
 	for (t = 1, max_sc = mat[0], min_sc = mat[1]; t < m * m; ++t) {
 		max_sc = max_sc > mat[t]? max_sc : mat[t];
@@ -74,9 +97,10 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 		for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF;
 	}
 	if (with_cigar) {
-		mem2 = (uint8_t*)kmalloc(km, ((qlen + tlen - 1) * n_col_ + 1) * 16);
+		mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * 16);
 		p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4);
-		off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int));
+		off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2);
+		off_end = off + qlen + tlen - 1;
 	}
 
 	for (t = 0; t < qlen; ++t) qr[t] = query[qlen - 1 - t];
@@ -85,7 +109,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	for (r = 0, last_st = last_en = -1; r < qlen + tlen - 1; ++r) {
 		int st = 0, en = tlen - 1, st0, en0, st_, en_;
 		int8_t x1, v1;
-		uint8_t *qrr = qr + (qlen - 1 - r), *u8 = (uint8_t*)u, *v8 = (uint8_t*)v, *x8 = (uint8_t*)x, p_en0 = 0;
+		uint8_t *qrr = qr + (qlen - 1 - r), *u8 = (uint8_t*)u, *v8 = (uint8_t*)v;
 		__m128i x1_, v1_;
 		// find the boundaries
 		if (st < r - qlen + 1) st = r - qlen + 1;
@@ -115,10 +139,11 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				tmp = _mm_cmpeq_epi8(sq, st);
 #ifdef __SSE4_1__
 				tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp);
+				tmp = _mm_blendv_epi8(tmp,     sc_N_,   mask);
 #else
-				tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(tmp,  sc_mis_), _mm_and_si128(tmp,  sc_mch_));
+				tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp),     _mm_and_si128(mask, sc_N_));
 #endif
-				tmp = _mm_andnot_si128(mask, tmp);
 				_mm_storeu_si128((__m128i*)((uint8_t*)s + t), tmp);
 			}
 		} else {
@@ -129,6 +154,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 		x1_ = _mm_cvtsi32_si128(x1);
 		v1_ = _mm_cvtsi32_si128(v1);
 		st_ = st / 16, en_ = en / 16;
+		assert(en_ - st_ + 1 <= n_col_);
 		if (!with_cigar) { // score only
 			for (t = st_; t <= en_; ++t) {
 				__m128i z, a, b, xt1, vt1, ut, tmp;
@@ -151,15 +177,8 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 #endif
 			}
 		} else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment
-			__m128i *pr = p + r * n_col_ - st_;
+			__m128i *pr = p + (size_t)r * n_col_ - st_;
-			off[r] = st;
+			off[r] = st, off_end[r] = en;
-			if (en0 < r && en0 < tlen - 1) { // to avoid backtracking out of the band; this assumes a fixed band
-				int8_t a, z = ((uint8_t*)s)[en0] + 2 * qe;
-				a = x8[en0-1] + v8[en0-1];
-				p_en0 = a > z? 1 : 0;
-				z = a > z? a : z;
-				p_en0 |= a - (z - q) > 0? 0x08 : 0;
-			}
 			for (t = st_; t <= en_; ++t) {
 				__m128i d, z, a, b, xt1, vt1, ut, tmp;
 				__dp_code_block1;
@@ -184,15 +203,8 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				_mm_store_si128(&pr[t], d);
 			}
 		} else { // gap right-alignment
-			__m128i *pr = p + r * n_col_ - st_;
+			__m128i *pr = p + (size_t)r * n_col_ - st_;
-			off[r] = st;
+			off[r] = st, off_end[r] = en;
-			if (en0 < r && en0 < tlen - 1) {
-				int8_t a, z = ((uint8_t*)s)[en0] + 2 * qe;
-				a = x8[en0-1] + v8[en0-1];
-				p_en0 = a >= z? 1 : 0;
-				z = a >= z? a : z;
-				p_en0 |= a - (z - q) >= 0? 0x08 : 0;
-			}
 			for (t = st_; t <= en_; ++t) {
 				__m128i d, z, a, b, xt1, vt1, ut, tmp;
 				__dp_code_block1;
@@ -217,7 +229,6 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 				_mm_store_si128(&pr[t], d);
 			}
 		}
-		if (with_cigar && en0 < r && en0 < tlen - 1) ((uint8_t*)(p + r * n_col_))[en0 - st] = p_en0;
 		if (!approx_max) { // find the exact max with a 32-bit score array
 			int32_t max_H, max_t;
 			// compute H[], max_H and max_t
@@ -288,10 +299,14 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
 	if (!approx_max) kfree(km, H);
 	if (with_cigar) { // backtrack
 		int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
-		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY))
+		if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) {
-			ksw_backtrack(km, 1, rev_cigar, (uint8_t*)p, off, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
-		else if (ez->max_t >= 0 && ez->max_q >= 0)
+		} else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > (int)ez->max) {
-			ksw_backtrack(km, 1, rev_cigar, (uint8_t*)p, off, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+			ez->reach_end = 1;
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		} else if (ez->max_t >= 0 && ez->max_q >= 0) {
+			ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
+		}
 		kfree(km, mem2); kfree(km, off);
 	}
 }

ksw2_ll_sse.c

@@ -0,0 +1,152 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include "ksw2.h"
+
+#ifdef USE_SIMDE
+#include <simde/x86/sse2.h>
+#else
+#include <emmintrin.h>
+#endif
+
+#ifdef __GNUC__
+#define LIKELY(x) __builtin_expect((x),1)
+#define UNLIKELY(x) __builtin_expect((x),0)
+#else
+#define LIKELY(x) (x)
+#define UNLIKELY(x) (x)
+#endif
+
+typedef struct {
+	int qlen, slen;
+	uint8_t shift, mdiff, max, size;
+	__m128i *qp, *H0, *H1, *E, *Hmax;
+} kswq_t;
+
+/**
+ * Initialize the query data structure
+ *
+ * @param size   Number of bytes used to store a score; valid valures are 1 or 2
+ * @param qlen   Length of the query sequence
+ * @param query  Query sequence
+ * @param m      Size of the alphabet
+ * @param mat    Scoring matrix in a one-dimension array
+ *
+ * @return       Query data structure
+ */
+void *ksw_ll_qinit(void *km, int size, int qlen, const uint8_t *query, int m, const int8_t *mat)
+{
+	kswq_t *q;
+	int slen, a, tmp, p;
+
+	size = size > 1? 2 : 1;
+	p = 8 * (3 - size); // # values per __m128i
+	slen = (qlen + p - 1) / p; // segmented length
+	q = (kswq_t*)kmalloc(km, sizeof(kswq_t) + 256 + 16 * slen * (m + 4)); // a single block of memory
+	q->qp = (__m128i*)(((size_t)q + sizeof(kswq_t) + 15) >> 4 << 4); // align memory
+	q->H0 = q->qp + slen * m;
+	q->H1 = q->H0 + slen;
+	q->E  = q->H1 + slen;
+	q->Hmax = q->E + slen;
+	q->slen = slen; q->qlen = qlen; q->size = size;
+	// compute shift
+	tmp = m * m;
+	for (a = 0, q->shift = 127, q->mdiff = 0; a < tmp; ++a) { // find the minimum and maximum score
+		if (mat[a] < (int8_t)q->shift) q->shift = mat[a];
+		if (mat[a] > (int8_t)q->mdiff) q->mdiff = mat[a];
+	}
+	q->max = q->mdiff;
+	q->shift = 256 - q->shift; // NB: q->shift is uint8_t
+	q->mdiff += q->shift; // this is the difference between the min and max scores
+	// An example: p=8, qlen=19, slen=3 and segmentation:
+	//  {{0,3,6,9,12,15,18,-1},{1,4,7,10,13,16,-1,-1},{2,5,8,11,14,17,-1,-1}}
+	if (size == 1) {
+		int8_t *t = (int8_t*)q->qp;
+		for (a = 0; a < m; ++a) {
+			int i, k, nlen = slen * p;
+			const int8_t *ma = mat + a * m;
+			for (i = 0; i < slen; ++i)
+				for (k = i; k < nlen; k += slen) // p iterations
+					*t++ = (k >= qlen? 0 : ma[query[k]]) + q->shift;
+		}
+	} else {
+		int16_t *t = (int16_t*)q->qp;
+		for (a = 0; a < m; ++a) {
+			int i, k, nlen = slen * p;
+			const int8_t *ma = mat + a * m;
+			for (i = 0; i < slen; ++i)
+				for (k = i; k < nlen; k += slen) // p iterations
+					*t++ = (k >= qlen? 0 : ma[query[k]]);
+		}
+	}
+	return q;
+}
+
+int ksw_ll_i16(void *q_, int tlen, const uint8_t *target, int _gapo, int _gape, int *qe, int *te)
+{
+	kswq_t *q = (kswq_t*)q_;
+	int slen, i, gmax = 0, qlen8;
+	__m128i zero, gapoe, gape, *H0, *H1, *E, *Hmax;
+	uint16_t *H8;
+
+#define __max_8(ret, xx) do { \
+		(xx) = _mm_max_epi16((xx), _mm_srli_si128((xx), 8)); \
+		(xx) = _mm_max_epi16((xx), _mm_srli_si128((xx), 4)); \
+		(xx) = _mm_max_epi16((xx), _mm_srli_si128((xx), 2)); \
+		(ret) = _mm_extract_epi16((xx), 0); \
+	} while (0)
+
+	// initialization
+	*qe = *te = -1;
+	zero = _mm_set1_epi32(0);
+	gapoe = _mm_set1_epi16(_gapo + _gape);
+	gape = _mm_set1_epi16(_gape);
+	H0 = q->H0; H1 = q->H1; E = q->E; Hmax = q->Hmax;
+	slen = q->slen, qlen8 = slen * 8;
+	memset(E,    0, slen * sizeof(__m128i));
+	memset(H0,   0, slen * sizeof(__m128i));
+	memset(Hmax, 0, slen * sizeof(__m128i));
+	// the core loop
+	for (i = 0; i < tlen; ++i) {
+		int j, k, imax;
+		__m128i e, h, f = zero, max = zero, *S = q->qp + target[i] * slen; // s is the 1st score vector
+		h = _mm_load_si128(H0 + slen - 1); // h={2,5,8,11,14,17,-1,-1} in the above example
+		h = _mm_slli_si128(h, 2);
+		for (j = 0; LIKELY(j < slen); ++j) {
+			h = _mm_adds_epi16(h, *S++);
+			e = _mm_load_si128(E + j);
+			h = _mm_max_epi16(h, e);
+			h = _mm_max_epi16(h, f);
+			max = _mm_max_epi16(max, h);
+			_mm_store_si128(H1 + j, h);
+			h = _mm_subs_epu16(h, gapoe);
+			e = _mm_subs_epu16(e, gape);
+			e = _mm_max_epi16(e, h);
+			_mm_store_si128(E + j, e);
+			f = _mm_subs_epu16(f, gape);
+			f = _mm_max_epi16(f, h);
+			h = _mm_load_si128(H0 + j);
+		}
+		for (k = 0; LIKELY(k < 8); ++k) {
+			f = _mm_slli_si128(f, 2);
+			for (j = 0; LIKELY(j < slen); ++j) {
+				h = _mm_load_si128(H1 + j);
+				h = _mm_max_epi16(h, f);
+				_mm_store_si128(H1 + j, h);
+				h = _mm_subs_epu16(h, gapoe);
+				f = _mm_subs_epu16(f, gape);
+				if(UNLIKELY(!_mm_movemask_epi8(_mm_cmpgt_epi16(f, h)))) goto end_loop_i16;
+			}
+		}
+end_loop_i16:
+		__max_8(imax, max);
+		if (imax >= gmax) {
+			gmax = imax; *te = i;
+			memcpy(Hmax, H1, slen * sizeof(__m128i));
+		}
+		S = H1; H1 = H0; H0 = S;
+	}
+	for (i = 0, H8 = (uint16_t*)Hmax; i < qlen8; ++i)
+		if ((int)H8[i] == gmax) *qe = i / 8 + i % 8 * slen;
+	return gmax;
+}

kthread.c

@@ -1,6 +1,12 @@
 #include <pthread.h>
 #include <stdlib.h>
 #include <limits.h>
+#include <stdint.h>
+#include "kthread.h"
+
+#if (defined(WIN32) || defined(_WIN32)) && defined(_MSC_VER)
+#define __sync_fetch_and_add(ptr, addend)     _InterlockedExchangeAdd((void*)ptr, addend)
+#endif
 
 /************
  * kt_for() *
@@ -52,12 +58,13 @@ void kt_for(int n_threads, void (*func)(void*,long,int), void *data, long n)
 		kt_for_t t;
 		pthread_t *tid;
 		t.func = func, t.data = data, t.n_threads = n_threads, t.n = n;
-		t.w = (ktf_worker_t*)alloca(n_threads * sizeof(ktf_worker_t));
+		t.w = (ktf_worker_t*)calloc(n_threads, sizeof(ktf_worker_t));
-		tid = (pthread_t*)alloca(n_threads * sizeof(pthread_t));
+		tid = (pthread_t*)calloc(n_threads, sizeof(pthread_t));
 		for (i = 0; i < n_threads; ++i)
 			t.w[i].t = &t, t.w[i].i = i;
 		for (i = 0; i < n_threads; ++i) pthread_create(&tid[i], 0, ktf_worker, &t.w[i]);
 		for (i = 0; i < n_threads; ++i) pthread_join(tid[i], 0);
+		free(tid); free(t.w);
 	} else {
 		long j;
 		for (j = 0; j < n; ++j) func(data, j, 0);
@@ -135,16 +142,17 @@ void kt_pipeline(int n_threads, void *(*func)(void*, int, void*), void *shared_d
 	pthread_mutex_init(&aux.mutex, 0);
 	pthread_cond_init(&aux.cv, 0);
 
-	aux.workers = (ktp_worker_t*)alloca(n_threads * sizeof(ktp_worker_t));
+	aux.workers = (ktp_worker_t*)calloc(n_threads, sizeof(ktp_worker_t));
 	for (i = 0; i < n_threads; ++i) {
 		ktp_worker_t *w = &aux.workers[i];
 		w->step = 0; w->pl = &aux; w->data = 0;
 		w->index = aux.index++;
 	}
 
-	tid = (pthread_t*)alloca(n_threads * sizeof(pthread_t));
+	tid = (pthread_t*)calloc(n_threads, sizeof(pthread_t));
 	for (i = 0; i < n_threads; ++i) pthread_create(&tid[i], 0, ktp_worker, &aux.workers[i]);
 	for (i = 0; i < n_threads; ++i) pthread_join(tid[i], 0);
+	free(tid); free(aux.workers);
 
 	pthread_mutex_destroy(&aux.mutex);
 	pthread_cond_destroy(&aux.cv);

lchain.c

@@ -0,0 +1,369 @@
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include "mmpriv.h"
+#include "kalloc.h"
+#include "krmq.h"
+
+static int64_t mg_chain_bk_end(int32_t max_drop, const mm128_t *z, const int32_t *f, const int64_t *p, int32_t *t, int64_t k)
+{
+	int64_t i = z[k].y, end_i = -1, max_i = i;
+	int32_t max_s = 0;
+	if (i < 0 || t[i] != 0) return i;
+	do {
+		int32_t s;
+		t[i] = 2;
+		end_i = i = p[i];
+		s = i < 0? z[k].x : (int32_t)z[k].x - f[i];
+		if (s > max_s) max_s = s, max_i = i;
+		else if (max_s - s > max_drop) break;
+	} while (i >= 0 && t[i] == 0);
+	for (i = z[k].y; i >= 0 && i != end_i; i = p[i]) // reset modified t[]
+		t[i] = 0;
+	return max_i;
+}
+
+uint64_t *mg_chain_backtrack(void *km, int64_t n, const int32_t *f, const int64_t *p, int32_t *v, int32_t *t, int32_t min_cnt, int32_t min_sc, int32_t max_drop, int32_t *n_u_, int32_t *n_v_)
+{
+	mm128_t *z;
+	uint64_t *u;
+	int64_t i, k, n_z, n_v;
+	int32_t n_u;
+
+	*n_u_ = *n_v_ = 0;
+	for (i = 0, n_z = 0; i < n; ++i) // precompute n_z
+		if (f[i] >= min_sc) ++n_z;
+	if (n_z == 0) return 0;
+	KMALLOC(km, z, n_z);
+	for (i = 0, k = 0; i < n; ++i) // populate z[]
+		if (f[i] >= min_sc) z[k].x = f[i], z[k++].y = i;
+	radix_sort_128x(z, z + n_z);
+
+	memset(t, 0, n * 4);
+	for (k = n_z - 1, n_v = n_u = 0; k >= 0; --k) { // precompute n_u
+		if (t[z[k].y] == 0) {
+			int64_t n_v0 = n_v, end_i;
+			int32_t sc;
+			end_i = mg_chain_bk_end(max_drop, z, f, p, t, k);
+			for (i = z[k].y; i != end_i; i = p[i])
+				++n_v, t[i] = 1;
+			sc = i < 0? z[k].x : (int32_t)z[k].x - f[i];
+			if (sc >= min_sc && n_v > n_v0 && n_v - n_v0 >= min_cnt)
+				++n_u;
+			else n_v = n_v0;
+		}
+	}
+	KMALLOC(km, u, n_u);
+	memset(t, 0, n * 4);
+	for (k = n_z - 1, n_v = n_u = 0; k >= 0; --k) { // populate u[]
+		if (t[z[k].y] == 0) {
+			int64_t n_v0 = n_v, end_i;
+			int32_t sc;
+			end_i = mg_chain_bk_end(max_drop, z, f, p, t, k);
+			for (i = z[k].y; i != end_i; i = p[i])
+				v[n_v++] = i, t[i] = 1;
+			sc = i < 0? z[k].x : (int32_t)z[k].x - f[i];
+			if (sc >= min_sc && n_v > n_v0 && n_v - n_v0 >= min_cnt)
+				u[n_u++] = (uint64_t)sc << 32 | (n_v - n_v0);
+			else n_v = n_v0;
+		}
+	}
+	kfree(km, z);
+	assert(n_v < INT32_MAX);
+	*n_u_ = n_u, *n_v_ = n_v;
+	return u;
+}
+
+static mm128_t *compact_a(void *km, int32_t n_u, uint64_t *u, int32_t n_v, int32_t *v, mm128_t *a)
+{
+	mm128_t *b, *w;
+	uint64_t *u2;
+	int64_t i, j, k;
+
+	// write the result to b[]
+	KMALLOC(km, b, n_v);
+	for (i = 0, k = 0; i < n_u; ++i) {
+		int32_t k0 = k, ni = (int32_t)u[i];
+		for (j = 0; j < ni; ++j)
+			b[k++] = a[v[k0 + (ni - j - 1)]];
+	}
+	kfree(km, v);
+
+	// sort u[] and a[] by the target position, such that adjacent chains may be joined
+	KMALLOC(km, w, n_u);
+	for (i = k = 0; i < n_u; ++i) {
+		w[i].x = b[k].x, w[i].y = (uint64_t)k<<32|i;
+		k += (int32_t)u[i];
+	}
+	radix_sort_128x(w, w + n_u);
+	KMALLOC(km, u2, n_u);
+	for (i = k = 0; i < n_u; ++i) {
+		int32_t j = (int32_t)w[i].y, n = (int32_t)u[j];
+		u2[i] = u[j];
+		memcpy(&a[k], &b[w[i].y>>32], n * sizeof(mm128_t));
+		k += n;
+	}
+	memcpy(u, u2, n_u * 8);
+	memcpy(b, a, k * sizeof(mm128_t)); // write _a_ to _b_ and deallocate _a_ because _a_ is oversized, sometimes a lot
+	kfree(km, a); kfree(km, w); kfree(km, u2);
+	return b;
+}
+
+static inline int32_t comput_sc(const mm128_t *ai, const mm128_t *aj, int32_t max_dist_x, int32_t max_dist_y, int32_t bw, float chn_pen_gap, float chn_pen_skip, int is_cdna, int n_seg)
+{
+	int32_t dq = (int32_t)ai->y - (int32_t)aj->y, dr, dd, dg, q_span, sc;
+	int32_t sidi = (ai->y & MM_SEED_SEG_MASK) >> MM_SEED_SEG_SHIFT;
+	int32_t sidj = (aj->y & MM_SEED_SEG_MASK) >> MM_SEED_SEG_SHIFT;
+	if (dq <= 0 || dq > max_dist_x) return INT32_MIN;
+	dr = (int32_t)(ai->x - aj->x);
+	if (sidi == sidj && (dr == 0 || dq > max_dist_y)) return INT32_MIN;
+	dd = dr > dq? dr - dq : dq - dr;
+	if (sidi == sidj && dd > bw) return INT32_MIN;
+	if (n_seg > 1 && !is_cdna && sidi == sidj && dr > max_dist_y) return INT32_MIN;
+	dg = dr < dq? dr : dq;
+	q_span = aj->y>>32&0xff;
+	sc = q_span < dg? q_span : dg;
+	if (dd || dg > q_span) {
+		float lin_pen, log_pen;
+		lin_pen = chn_pen_gap * (float)dd + chn_pen_skip * (float)dg;
+		log_pen = dd >= 1? mg_log2(dd + 1) : 0.0f; // mg_log2() only works for dd>=2
+		if (is_cdna || sidi != sidj) {
+			if (sidi != sidj && dr == 0) ++sc; // possibly due to overlapping paired ends; give a minor bonus
+			else if (dr > dq || sidi != sidj) sc -= (int)(lin_pen < log_pen? lin_pen : log_pen); // deletion or jump between paired ends
+			else sc -= (int)(lin_pen + .5f * log_pen);
+		} else sc -= (int)(lin_pen + .5f * log_pen);
+	}
+	return sc;
+}
+
+/* Input:
+ *   a[].x: tid<<33 | rev<<32 | tpos
+ *   a[].y: flags<<40 | q_span<<32 | q_pos
+ * Output:
+ *   n_u: #chains
+ *   u[]: score<<32 | #anchors (sum of lower 32 bits of u[] is the returned length of a[])
+ * input a[] is deallocated on return
+ */
+mm128_t *mg_lchain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int max_iter, int min_cnt, int min_sc, float chn_pen_gap, float chn_pen_skip,
+					  int is_cdna, int n_seg, int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km)
+{ // TODO: make sure this works when n has more than 32 bits
+	int32_t *f, *t, *v, n_u, n_v, mmax_f = 0, max_drop = bw;
+	int64_t *p, i, j, max_ii, st = 0, n_iter = 0;
+	uint64_t *u;
+
+	if (_u) *_u = 0, *n_u_ = 0;
+	if (n == 0 || a == 0) {
+		kfree(km, a);
+		return 0;
+	}
+	if (max_dist_x < bw) max_dist_x = bw;
+	if (max_dist_y < bw && !is_cdna) max_dist_y = bw;
+	if (is_cdna) max_drop = INT32_MAX;
+	KMALLOC(km, p, n);
+	KMALLOC(km, f, n);
+	KMALLOC(km, v, n);
+	KCALLOC(km, t, n);
+
+	// fill the score and backtrack arrays
+	for (i = 0, max_ii = -1; i < n; ++i) {
+		int64_t max_j = -1, end_j;
+		int32_t max_f = a[i].y>>32&0xff, n_skip = 0;
+		while (st < i && (a[i].x>>32 != a[st].x>>32 || a[i].x > a[st].x + max_dist_x)) ++st;
+		if (i - st > max_iter) st = i - max_iter;
+		for (j = i - 1; j >= st; --j) {
+			int32_t sc;
+			sc = comput_sc(&a[i], &a[j], max_dist_x, max_dist_y, bw, chn_pen_gap, chn_pen_skip, is_cdna, n_seg);
+			++n_iter;
+			if (sc == INT32_MIN) continue;
+			sc += f[j];
+			if (sc > max_f) {
+				max_f = sc, max_j = j;
+				if (n_skip > 0) --n_skip;
+			} else if (t[j] == (int32_t)i) {
+				if (++n_skip > max_skip)
+					break;
+			}
+			if (p[j] >= 0) t[p[j]] = i;
+		}
+		end_j = j;
+		if (max_ii < 0 || a[i].x - a[max_ii].x > (int64_t)max_dist_x) {
+			int32_t max = INT32_MIN;
+			max_ii = -1;
+			for (j = i - 1; j >= st; --j)
+				if (max < f[j]) max = f[j], max_ii = j;
+		}
+		if (max_ii >= 0 && max_ii < end_j) {
+			int32_t tmp;
+			tmp = comput_sc(&a[i], &a[max_ii], max_dist_x, max_dist_y, bw, chn_pen_gap, chn_pen_skip, is_cdna, n_seg);
+			if (tmp != INT32_MIN && max_f < tmp + f[max_ii])
+				max_f = tmp + f[max_ii], max_j = max_ii;
+		}
+		f[i] = max_f, p[i] = max_j;
+		v[i] = max_j >= 0 && v[max_j] > max_f? v[max_j] : max_f; // v[] keeps the peak score up to i; f[] is the score ending at i, not always the peak
+		if (max_ii < 0 || (a[i].x - a[max_ii].x <= (int64_t)max_dist_x && f[max_ii] < f[i]))
+			max_ii = i;
+		if (mmax_f < max_f) mmax_f = max_f;
+	}
+
+	u = mg_chain_backtrack(km, n, f, p, v, t, min_cnt, min_sc, max_drop, &n_u, &n_v);
+	*n_u_ = n_u, *_u = u; // NB: note that u[] may not be sorted by score here
+	kfree(km, p); kfree(km, f); kfree(km, t);
+	if (n_u == 0) {
+		kfree(km, a); kfree(km, v);
+		return 0;
+	}
+	return compact_a(km, n_u, u, n_v, v, a);
+}
+
+typedef struct lc_elem_s {
+	int32_t y;
+	int64_t i;
+	double pri;
+	KRMQ_HEAD(struct lc_elem_s) head;
+} lc_elem_t;
+
+#define lc_elem_cmp(a, b) ((a)->y < (b)->y? -1 : (a)->y > (b)->y? 1 : ((a)->i > (b)->i) - ((a)->i < (b)->i))
+#define lc_elem_lt2(a, b) ((a)->pri < (b)->pri)
+KRMQ_INIT(lc_elem, lc_elem_t, head, lc_elem_cmp, lc_elem_lt2)
+
+KALLOC_POOL_INIT(rmq, lc_elem_t)
+
+static inline int32_t comput_sc_simple(const mm128_t *ai, const mm128_t *aj, float chn_pen_gap, float chn_pen_skip, int32_t *exact, int32_t *width)
+{
+	int32_t dq = (int32_t)ai->y - (int32_t)aj->y, dr, dd, dg, q_span, sc;
+	dr = (int32_t)(ai->x - aj->x);
+	*width = dd = dr > dq? dr - dq : dq - dr;
+	dg = dr < dq? dr : dq;
+	q_span = aj->y>>32&0xff;
+	sc = q_span < dg? q_span : dg;
+	if (exact) *exact = (dd == 0 && dg <= q_span);
+	if (dd || dq > q_span) {
+		float lin_pen, log_pen;
+		lin_pen = chn_pen_gap * (float)dd + chn_pen_skip * (float)dg;
+		log_pen = dd >= 1? mg_log2(dd + 1) : 0.0f; // mg_log2() only works for dd>=2
+		sc -= (int)(lin_pen + .5f * log_pen);
+	}
+	return sc;
+}
+
+mm128_t *mg_lchain_rmq(int max_dist, int max_dist_inner, int bw, int max_chn_skip, int cap_rmq_size, int min_cnt, int min_sc, float chn_pen_gap, float chn_pen_skip,
+					   int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km)
+{
+	int32_t *f,*t, *v, n_u, n_v, mmax_f = 0, max_rmq_size = 0, max_drop = bw;
+	int64_t *p, i, i0, st = 0, st_inner = 0, n_iter = 0;
+	uint64_t *u;
+	lc_elem_t *root = 0, *root_inner = 0;
+	void *mem_mp = 0;
+	kmp_rmq_t *mp;
+
+	if (_u) *_u = 0, *n_u_ = 0;
+	if (n == 0 || a == 0) {
+		kfree(km, a);
+		return 0;
+	}
+	if (max_dist < bw) max_dist = bw;
+	if (max_dist_inner <= 0 || max_dist_inner >= max_dist) max_dist_inner = 0;
+	KMALLOC(km, p, n);
+	KMALLOC(km, f, n);
+	KCALLOC(km, t, n);
+	KMALLOC(km, v, n);
+	mem_mp = km_init2(km, 0x10000);
+	mp = kmp_init_rmq(mem_mp);
+
+	// fill the score and backtrack arrays
+	for (i = i0 = 0; i < n; ++i) {
+		int64_t max_j = -1;
+		int32_t q_span = a[i].y>>32&0xff, max_f = q_span;
+		lc_elem_t s, *q, *r, lo, hi;
+		// add in-range anchors
+		if (i0 < i && a[i0].x != a[i].x) {
+			int64_t j;
+			for (j = i0; j < i; ++j) {
+				q = kmp_alloc_rmq(mp);
+				q->y = (int32_t)a[j].y, q->i = j, q->pri = -(f[j] + 0.5 * chn_pen_gap * ((int32_t)a[j].x + (int32_t)a[j].y));
+				krmq_insert(lc_elem, &root, q, 0);
+				if (max_dist_inner > 0) {
+					r = kmp_alloc_rmq(mp);
+					*r = *q;
+					krmq_insert(lc_elem, &root_inner, r, 0);
+				}
+			}
+			i0 = i;
+		}
+		// get rid of active chains out of range
+		while (st < i && (a[i].x>>32 != a[st].x>>32 || a[i].x > a[st].x + max_dist || krmq_size(head, root) > cap_rmq_size)) {
+			s.y = (int32_t)a[st].y, s.i = st;
+			if ((q = krmq_find(lc_elem, root, &s, 0)) != 0) {
+				q = krmq_erase(lc_elem, &root, q, 0);
+				kmp_free_rmq(mp, q);
+			}
+			++st;
+		}
+		if (max_dist_inner > 0)  { // similar to the block above, but applied to the inner tree
+			while (st_inner < i && (a[i].x>>32 != a[st_inner].x>>32 || a[i].x > a[st_inner].x + max_dist_inner || krmq_size(head, root_inner) > cap_rmq_size)) {
+				s.y = (int32_t)a[st_inner].y, s.i = st_inner;
+				if ((q = krmq_find(lc_elem, root_inner, &s, 0)) != 0) {
+					q = krmq_erase(lc_elem, &root_inner, q, 0);
+					kmp_free_rmq(mp, q);
+				}
+				++st_inner;
+			}
+		}
+		// RMQ
+		lo.i = INT32_MAX, lo.y = (int32_t)a[i].y - max_dist;
+		hi.i = 0, hi.y = (int32_t)a[i].y;
+		if ((q = krmq_rmq(lc_elem, root, &lo, &hi)) != 0) {
+			int32_t sc, exact, width, n_skip = 0;
+			int64_t j = q->i;
+			assert(q->y >= lo.y && q->y <= hi.y);
+			sc = f[j] + comput_sc_simple(&a[i], &a[j], chn_pen_gap, chn_pen_skip, &exact, &width);
+			if (width <= bw && sc > max_f) max_f = sc, max_j = j;
+			if (!exact && root_inner && (int32_t)a[i].y > 0) {
+				lc_elem_t *lo, *hi;
+				s.y = (int32_t)a[i].y - 1, s.i = n;
+				krmq_interval(lc_elem, root_inner, &s, &lo, &hi);
+				if (lo) {
+					const lc_elem_t *q;
+					int32_t width, n_rmq_iter = 0;
+					krmq_itr_t(lc_elem) itr;
+					krmq_itr_find(lc_elem, root_inner, lo, &itr);
+					while ((q = krmq_at(&itr)) != 0) {
+						if (q->y < (int32_t)a[i].y - max_dist_inner) break;
+						++n_rmq_iter;
+						j = q->i;
+						sc = f[j] + comput_sc_simple(&a[i], &a[j], chn_pen_gap, chn_pen_skip, 0, &width);
+						if (width <= bw) {
+							if (sc > max_f) {
+								max_f = sc, max_j = j;
+								if (n_skip > 0) --n_skip;
+							} else if (t[j] == (int32_t)i) {
+								if (++n_skip > max_chn_skip)
+									break;
+							}
+							if (p[j] >= 0) t[p[j]] = i;
+						}
+						if (!krmq_itr_prev(lc_elem, &itr)) break;
+					}
+					n_iter += n_rmq_iter;
+				}
+			}
+		}
+		// set max
+		assert(max_j < 0 || (a[max_j].x < a[i].x && (int32_t)a[max_j].y < (int32_t)a[i].y));
+		f[i] = max_f, p[i] = max_j;
+		v[i] = max_j >= 0 && v[max_j] > max_f? v[max_j] : max_f; // v[] keeps the peak score up to i; f[] is the score ending at i, not always the peak
+		if (mmax_f < max_f) mmax_f = max_f;
+		if (max_rmq_size < krmq_size(head, root)) max_rmq_size = krmq_size(head, root);
+	}
+	km_destroy(mem_mp);
+
+	u = mg_chain_backtrack(km, n, f, p, v, t, min_cnt, min_sc, max_drop, &n_u, &n_v);
+	*n_u_ = n_u, *_u = u; // NB: note that u[] may not be sorted by score here
+	kfree(km, p); kfree(km, f); kfree(km, t);
+	if (n_u == 0) {
+		kfree(km, a); kfree(km, v);
+		return 0;
+	}
+	return compact_a(km, n_u, u, n_v, v, a);
+}

main.c

@@ -1,226 +1,459 @@
-#include <getopt.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <sys/resource.h>
+#include <errno.h>
-#include <sys/time.h>
 #include "bseq.h"
 #include "minimap.h"
 #include "mmpriv.h"
+#include "ketopt.h"
 
-#define MM_VERSION "2.0-r191-dirty"
+#define MM_VERSION "2.24-r1122"
 
+#ifdef __linux__
+#include <sys/resource.h>
+#include <sys/time.h>
 void liftrlimit()
 {
-#ifdef __linux__
 	struct rlimit r;
 	getrlimit(RLIMIT_AS, &r);
 	r.rlim_cur = r.rlim_max;
 	setrlimit(RLIMIT_AS, &r);
+}
+#else
+void liftrlimit() {}
 #endif
+
+static ko_longopt_t long_options[] = {
+	{ "bucket-bits",    ko_required_argument, 300 },
+	{ "mb-size",        ko_required_argument, 'K' },
+	{ "seed",           ko_required_argument, 302 },
+	{ "no-kalloc",      ko_no_argument,       303 },
+	{ "print-qname",    ko_no_argument,       304 },
+	{ "no-self",        ko_no_argument,       'D' },
+	{ "print-seeds",    ko_no_argument,       306 },
+	{ "max-chain-skip", ko_required_argument, 307 },
+	{ "min-dp-len",     ko_required_argument, 308 },
+	{ "print-aln-seq",  ko_no_argument,       309 },
+	{ "splice",         ko_no_argument,       310 },
+	{ "cost-non-gt-ag", ko_required_argument, 'C' },
+	{ "no-long-join",   ko_no_argument,       312 },
+	{ "sr",             ko_no_argument,       313 },
+	{ "frag",           ko_required_argument, 314 },
+	{ "secondary",      ko_required_argument, 315 },
+	{ "cs",             ko_optional_argument, 316 },
+	{ "end-bonus",      ko_required_argument, 317 },
+	{ "no-pairing",     ko_no_argument,       318 },
+	{ "splice-flank",   ko_required_argument, 319 },
+	{ "idx-no-seq",     ko_no_argument,       320 },
+	{ "end-seed-pen",   ko_required_argument, 321 },
+	{ "for-only",       ko_no_argument,       322 },
+	{ "rev-only",       ko_no_argument,       323 },
+	{ "heap-sort",      ko_required_argument, 324 },
+	{ "all-chain",      ko_no_argument,       'P' },
+	{ "dual",           ko_required_argument, 326 },
+	{ "max-clip-ratio", ko_required_argument, 327 },
+	{ "min-occ-floor",  ko_required_argument, 328 },
+	{ "MD",             ko_no_argument,       329 },
+	{ "lj-min-ratio",   ko_required_argument, 330 },
+	{ "score-N",        ko_required_argument, 331 },
+	{ "eqx",            ko_no_argument,       332 },
+	{ "paf-no-hit",     ko_no_argument,       333 },
+	{ "split-prefix",   ko_required_argument, 334 },
+	{ "no-end-flt",     ko_no_argument,       335 },
+	{ "hard-mask-level",ko_no_argument,       336 },
+	{ "cap-sw-mem",     ko_required_argument, 337 },
+	{ "max-qlen",       ko_required_argument, 338 },
+	{ "max-chain-iter", ko_required_argument, 339 },
+	{ "junc-bed",       ko_required_argument, 340 },
+	{ "junc-bonus",     ko_required_argument, 341 },
+	{ "sam-hit-only",   ko_no_argument,       342 },
+	{ "chain-gap-scale",ko_required_argument, 343 },
+	{ "alt",            ko_required_argument, 344 },
+	{ "alt-drop",       ko_required_argument, 345 },
+	{ "mask-len",       ko_required_argument, 346 },
+	{ "rmq",            ko_optional_argument, 347 },
+	{ "qstrand",        ko_no_argument,       348 },
+	{ "cap-kalloc",     ko_required_argument, 349 },
+	{ "q-occ-frac",     ko_required_argument, 350 },
+	{ "chain-skip-scale",ko_required_argument,351 },
+	{ "print-chains",   ko_no_argument,       352 },
+	{ "no-hash-name",   ko_no_argument,       353 },
+	{ "help",           ko_no_argument,       'h' },
+	{ "max-intron-len", ko_required_argument, 'G' },
+	{ "version",        ko_no_argument,       'V' },
+	{ "min-count",      ko_required_argument, 'n' },
+	{ "min-chain-score",ko_required_argument, 'm' },
+	{ "mask-level",     ko_required_argument, 'M' },
+	{ "min-dp-score",   ko_required_argument, 's' },
+	{ "sam",            ko_no_argument,       'a' },
+	{ 0, 0, 0 }
+};
+
+static inline int64_t mm_parse_num2(const char *str, char **q)
+{
+	double x;
+	char *p;
+	x = strtod(str, &p);
+	if (*p == 'G' || *p == 'g') x *= 1e9, ++p;
+	else if (*p == 'M' || *p == 'm') x *= 1e6, ++p;
+	else if (*p == 'K' || *p == 'k') x *= 1e3, ++p;
+	if (q) *q = p;
+	return (int64_t)(x + .499);
 }
 
-static struct option long_options[] = {
+static inline int64_t mm_parse_num(const char *str)
-	{ "bucket-bits",    required_argument, 0, 0 },
+{
-	{ "mb-size",        required_argument, 0, 'K' },
+	return mm_parse_num2(str, 0);
-	{ "int-rname",      no_argument,       0, 0 },
+}
-	{ "no-kalloc",      no_argument,       0, 0 },
+
-	{ "print-qname",    no_argument,       0, 0 },
+static inline void yes_or_no(mm_mapopt_t *opt, int64_t flag, int long_idx, const char *arg, int yes_to_set)
-	{ "no-self",        no_argument,       0, 0 },
+{
-	{ "print-seed",     no_argument,       0, 0 },
+	if (yes_to_set) {
-	{ "max-chain-skip", required_argument, 0, 0 },
+		if (strcmp(arg, "yes") == 0 || strcmp(arg, "y") == 0) opt->flag |= flag;
-	{ "min-dp-len",     required_argument, 0, 0 },
+		else if (strcmp(arg, "no") == 0 || strcmp(arg, "n") == 0) opt->flag &= ~flag;
-	{ "version",        no_argument,       0, 'V' },
+		else fprintf(stderr, "[WARNING]\033[1;31m option '--%s' only accepts 'yes' or 'no'.\033[0m\n", long_options[long_idx].name);
-	{ "min-count",      required_argument, 0, 'n' },
+	} else {
-	{ "min-chain-score",required_argument, 0, 'm' },
+		if (strcmp(arg, "yes") == 0 || strcmp(arg, "y") == 0) opt->flag &= ~flag;
-	{ "mask-level",     required_argument, 0, 'M' },
+		else if (strcmp(arg, "no") == 0 || strcmp(arg, "n") == 0) opt->flag |= flag;
-	{ "min-dp-score",   required_argument, 0, 's' },
+		else fprintf(stderr, "[WARNING]\033[1;31m option '--%s' only accepts 'yes' or 'no'.\033[0m\n", long_options[long_idx].name);
-	{ "sam",            no_argument,       0, 'a' },
+	}
-	{ 0, 0, 0, 0}
+}
-};
 
 int main(int argc, char *argv[])
 {
+	const char *opt_str = "2aSDw:k:K:t:r:f:Vv:g:G:I:d:XT:s:x:Hcp:M:n:z:A:B:O:E:m:N:Qu:R:hF:LC:yYPo:e:U:";
+	ketopt_t o = KETOPT_INIT;
 	mm_mapopt_t opt;
-	int i, c, k = 15, w = -1, bucket_bits = MM_IDX_DEF_B, n_threads = 3, keep_name = 1, is_idx, is_hpc = 0, long_idx, idx_par_set = 0;
+	mm_idxopt_t ipt;
-	int minibatch_size = 200000000;
+	int i, c, n_threads = 3, n_parts, old_best_n = -1;
-	uint64_t batch_size = 4000000000ULL;
+	char *fnw = 0, *rg = 0, *junc_bed = 0, *s, *alt_list = 0;
-	mm_bseq_file_t *fp = 0;
+	FILE *fp_help = stderr;
-	char *fnw = 0, *s;
+	mm_idx_reader_t *idx_rdr;
-	FILE *fpr = 0, *fpw = 0;
+	mm_idx_t *mi;
 
+	mm_verbose = 3;
 	liftrlimit();
 	mm_realtime0 = realtime();
-	mm_mapopt_init(&opt);
+	mm_set_opt(0, &ipt, &opt);
 
-	while ((c = getopt_long(argc, argv, "aw:k:K:t:r:f:Vv:g:I:d:XT:s:x:Hcp:M:n:z:A:B:O:E:m:N:Q", long_options, &long_idx)) >= 0) {
+	while ((c = ketopt(&o, argc, argv, 1, opt_str, long_options)) >= 0) { // test command line options and apply option -x/preset first
-		if (c == 'w') w = atoi(optarg), idx_par_set = 1;
+		if (c == 'x') {
-		else if (c == 'k') k = atoi(optarg), idx_par_set = 1;
+			if (mm_set_opt(o.arg, &ipt, &opt) < 0) {
-		else if (c == 'H') is_hpc = 1, idx_par_set = 1;
+				fprintf(stderr, "[ERROR] unknown preset '%s'\n", o.arg);
-		else if (c == 'd') fnw = optarg; // the above are indexing related options, except -I
+				return 1;
-		else if (c == 'r') opt.bw = atoi(optarg);
+			}
-		else if (c == 'f') opt.mid_occ_frac = atof(optarg);
+		} else if (c == ':') {
-		else if (c == 't') n_threads = atoi(optarg);
+			fprintf(stderr, "[ERROR] missing option argument\n");
-		else if (c == 'v') mm_verbose = atoi(optarg);
+			return 1;
-		else if (c == 'g') opt.max_gap = atoi(optarg);
+		} else if (c == '?') {
-		else if (c == 'N') opt.best_n = atoi(optarg);
+			fprintf(stderr, "[ERROR] unknown option in \"%s\"\n", argv[o.i - 1]);
-		else if (c == 'p') opt.pri_ratio = atof(optarg);
+			return 1;
-		else if (c == 'M') opt.mask_level = atof(optarg);
+		}
-		else if (c == 'c') opt.flag |= MM_F_CIGAR;
+	}
-		else if (c == 'X') opt.flag |= MM_F_AVA | MM_F_NO_SELF;
+	o = KETOPT_INIT;
+
+	while ((c = ketopt(&o, argc, argv, 1, opt_str, long_options)) >= 0) {
+		if (c == 'w') ipt.w = atoi(o.arg);
+		else if (c == 'k') ipt.k = atoi(o.arg);
+		else if (c == 'H') ipt.flag |= MM_I_HPC;
+		else if (c == 'd') fnw = o.arg; // the above are indexing related options, except -I
+		else if (c == 't') n_threads = atoi(o.arg);
+		else if (c == 'v') mm_verbose = atoi(o.arg);
+		else if (c == 'g') opt.max_gap = (int)mm_parse_num(o.arg);
+		else if (c == 'G') mm_mapopt_max_intron_len(&opt, (int)mm_parse_num(o.arg));
+		else if (c == 'F') opt.max_frag_len = (int)mm_parse_num(o.arg);
+		else if (c == 'N') old_best_n = opt.best_n, opt.best_n = atoi(o.arg);
+		else if (c == 'p') opt.pri_ratio = atof(o.arg);
+		else if (c == 'M') opt.mask_level = atof(o.arg);
+		else if (c == 'c') opt.flag |= MM_F_OUT_CG | MM_F_CIGAR;
+		else if (c == 'D') opt.flag |= MM_F_NO_DIAG;
+		else if (c == 'P') opt.flag |= MM_F_ALL_CHAINS;
+		else if (c == 'X') opt.flag |= MM_F_ALL_CHAINS | MM_F_NO_DIAG | MM_F_NO_DUAL | MM_F_NO_LJOIN; // -D -P --no-long-join --dual=no
 		else if (c == 'a') opt.flag |= MM_F_OUT_SAM | MM_F_CIGAR;
 		else if (c == 'Q') opt.flag |= MM_F_NO_QUAL;
-		else if (c == 'T') opt.sdust_thres = atoi(optarg);
+		else if (c == 'Y') opt.flag |= MM_F_SOFTCLIP;
-		else if (c == 'n') opt.min_cnt = atoi(optarg);
+		else if (c == 'L') opt.flag |= MM_F_LONG_CIGAR;
-		else if (c == 'm') opt.min_chain_score = atoi(optarg);
+		else if (c == 'y') opt.flag |= MM_F_COPY_COMMENT;
-		else if (c == 'A') opt.a = atoi(optarg);
+		else if (c == 'T') opt.sdust_thres = atoi(o.arg);
-		else if (c == 'B') opt.b = atoi(optarg);
+		else if (c == 'n') opt.min_cnt = atoi(o.arg);
-		else if (c == 'z') opt.zdrop = atoi(optarg);
+		else if (c == 'm') opt.min_chain_score = atoi(o.arg);
-		else if (c == 's') opt.min_dp_max = atoi(optarg);
+		else if (c == 'A') opt.a = atoi(o.arg);
-		else if (c == 0 && long_idx == 0) bucket_bits = atoi(optarg); // --bucket-bits
+		else if (c == 'B') opt.b = atoi(o.arg);
-		else if (c == 0 && long_idx == 2) keep_name = 0; // --int-rname
+		else if (c == 's') opt.min_dp_max = atoi(o.arg);
-		else if (c == 0 && long_idx == 3) mm_dbg_flag |= MM_DBG_NO_KALLOC; // --no-kalloc
+		else if (c == 'C') opt.noncan = atoi(o.arg);
-		else if (c == 0 && long_idx == 4) mm_dbg_flag |= MM_DBG_PRINT_QNAME; // --print-qname
+		else if (c == 'I') ipt.batch_size = mm_parse_num(o.arg);
-		else if (c == 0 && long_idx == 5) opt.flag |= MM_F_NO_SELF; // --no-self
+		else if (c == 'K') opt.mini_batch_size = mm_parse_num(o.arg);
-		else if (c == 0 && long_idx == 6) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_SEED; // --print-seed
+		else if (c == 'e') opt.occ_dist = mm_parse_num(o.arg);
-		else if (c == 0 && long_idx == 7) opt.max_chain_skip = atoi(optarg); // --max-chain-skip
+		else if (c == 'R') rg = o.arg;
-		else if (c == 0 && long_idx == 8) opt.min_ksw_len = atoi(optarg); // --min-dp-len
+		else if (c == 'h') fp_help = stdout;
-		else if (c == 'V') {
+		else if (c == '2') opt.flag |= MM_F_2_IO_THREADS;
+		else if (c == 'o') {
+			if (strcmp(o.arg, "-") != 0) {
+				if (freopen(o.arg, "wb", stdout) == NULL) {
+					fprintf(stderr, "[ERROR]\033[1;31m failed to write the output to file '%s'\033[0m: %s\n", o.arg, strerror(errno));
+					exit(1);
+				}
+			}
+		}
+		else if (c == 300) ipt.bucket_bits = atoi(o.arg); // --bucket-bits
+		else if (c == 302) opt.seed = atoi(o.arg); // --seed
+		else if (c == 303) mm_dbg_flag |= MM_DBG_NO_KALLOC; // --no-kalloc
+		else if (c == 304) mm_dbg_flag |= MM_DBG_PRINT_QNAME; // --print-qname
+		else if (c == 306) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_SEED, n_threads = 1; // --print-seed
+		else if (c == 307) opt.max_chain_skip = atoi(o.arg); // --max-chain-skip
+		else if (c == 339) opt.max_chain_iter = atoi(o.arg); // --max-chain-iter
+		else if (c == 308) opt.min_ksw_len = atoi(o.arg); // --min-dp-len
+		else if (c == 309) mm_dbg_flag |= MM_DBG_PRINT_QNAME | MM_DBG_PRINT_ALN_SEQ, n_threads = 1; // --print-aln-seq
+		else if (c == 310) opt.flag |= MM_F_SPLICE; // --splice
+		else if (c == 312) opt.flag |= MM_F_NO_LJOIN; // --no-long-join
+		else if (c == 313) opt.flag |= MM_F_SR; // --sr
+		else if (c == 317) opt.end_bonus = atoi(o.arg); // --end-bonus
+		else if (c == 318) opt.flag |= MM_F_INDEPEND_SEG; // --no-pairing
+		else if (c == 320) ipt.flag |= MM_I_NO_SEQ; // --idx-no-seq
+		else if (c == 321) opt.anchor_ext_shift = atoi(o.arg); // --end-seed-pen
+		else if (c == 322) opt.flag |= MM_F_FOR_ONLY; // --for-only
+		else if (c == 323) opt.flag |= MM_F_REV_ONLY; // --rev-only
+		else if (c == 327) opt.max_clip_ratio = atof(o.arg); // --max-clip-ratio
+		else if (c == 328) opt.min_mid_occ = atoi(o.arg); // --min-occ-floor
+		else if (c == 329) opt.flag |= MM_F_OUT_MD; // --MD
+		else if (c == 331) opt.sc_ambi = atoi(o.arg); // --score-N
+		else if (c == 332) opt.flag |= MM_F_EQX; // --eqx
+		else if (c == 333) opt.flag |= MM_F_PAF_NO_HIT; // --paf-no-hit
+		else if (c == 334) opt.split_prefix = o.arg; // --split-prefix
+		else if (c == 335) opt.flag |= MM_F_NO_END_FLT; // --no-end-flt
+		else if (c == 336) opt.flag |= MM_F_HARD_MLEVEL; // --hard-mask-level
+		else if (c == 337) opt.max_sw_mat = mm_parse_num(o.arg); // --cap-sw-mat
+		else if (c == 338) opt.max_qlen = mm_parse_num(o.arg); // --max-qlen
+		else if (c == 340) junc_bed = o.arg; // --junc-bed
+		else if (c == 341) opt.junc_bonus = atoi(o.arg); // --junc-bonus
+		else if (c == 342) opt.flag |= MM_F_SAM_HIT_ONLY; // --sam-hit-only
+		else if (c == 343) opt.chain_gap_scale = atof(o.arg); // --chain-gap-scale
+		else if (c == 351) opt.chain_skip_scale = atof(o.arg); // --chain-skip-scale
+		else if (c == 344) alt_list = o.arg; // --alt
+		else if (c == 345) opt.alt_drop = atof(o.arg); // --alt-drop
+		else if (c == 346) opt.mask_len = mm_parse_num(o.arg); // --mask-len
+		else if (c == 348) opt.flag |= MM_F_QSTRAND | MM_F_NO_INV; // --qstrand
+		else if (c == 349) opt.cap_kalloc = mm_parse_num(o.arg); // --cap-kalloc
+		else if (c == 350) opt.q_occ_frac = atof(o.arg); // --q-occ-frac
+		else if (c == 352) mm_dbg_flag |= MM_DBG_PRINT_CHAIN; // --print-chains
+		else if (c == 353) opt.flag |= MM_F_NO_HASH_NAME; // --no-hash-name
+		else if (c == 330) {
+			fprintf(stderr, "[WARNING] \033[1;31m --lj-min-ratio has been deprecated.\033[0m\n");
+		} else if (c == 314) { // --frag
+			yes_or_no(&opt, MM_F_FRAG_MODE, o.longidx, o.arg, 1);
+		} else if (c == 315) { // --secondary
+			yes_or_no(&opt, MM_F_NO_PRINT_2ND, o.longidx, o.arg, 0);
+		} else if (c == 316) { // --cs
+			opt.flag |= MM_F_OUT_CS | MM_F_CIGAR;
+			if (o.arg == 0 || strcmp(o.arg, "short") == 0) {
+				opt.flag &= ~MM_F_OUT_CS_LONG;
+			} else if (strcmp(o.arg, "long") == 0) {
+				opt.flag |= MM_F_OUT_CS_LONG;
+			} else if (strcmp(o.arg, "none") == 0) {
+				opt.flag &= ~MM_F_OUT_CS;
+			} else if (mm_verbose >= 2) {
+				fprintf(stderr, "[WARNING]\033[1;31m --cs only takes 'short' or 'long'. Invalid values are assumed to be 'short'.\033[0m\n");
+			}
+		} else if (c == 319) { // --splice-flank
+			yes_or_no(&opt, MM_F_SPLICE_FLANK, o.longidx, o.arg, 1);
+		} else if (c == 324) { // --heap-sort
+			yes_or_no(&opt, MM_F_HEAP_SORT, o.longidx, o.arg, 1);
+		} else if (c == 326) { // --dual
+			yes_or_no(&opt, MM_F_NO_DUAL, o.longidx, o.arg, 0);
+		} else if (c == 347) { // --rmq
+			yes_or_no(&opt, MM_F_RMQ, o.longidx, o.arg, 1);
+		} else if (c == 'S') {
+			opt.flag |= MM_F_OUT_CS | MM_F_CIGAR | MM_F_OUT_CS_LONG;
+			if (mm_verbose >= 2)
+				fprintf(stderr, "[WARNING]\033[1;31m option -S is deprecated and may be removed in future. Please use --cs=long instead.\033[0m\n");
+		} else if (c == 'V') {
 			puts(MM_VERSION);
 			return 0;
-		} else if (c == 'O') {
+		} else if (c == 'r') {
-			opt.q = opt.q2 = strtol(optarg, &s, 10);
+			opt.bw = (int)mm_parse_num2(o.arg, &s);
-			if (*s == ',') opt.q2 = strtol(s + 1, &s, 10);
+			if (*s == ',') opt.bw_long = (int)mm_parse_num2(s + 1, &s);
-		} else if (c == 'E') {
+		} else if (c == 'U') {
-			opt.e = opt.e2 = strtol(optarg, &s, 10);
+			opt.min_mid_occ = strtol(o.arg, &s, 10);
-			if (*s == ',') opt.e2 = strtol(s + 1, &s, 10);
+			if (*s == ',') opt.max_mid_occ = strtol(s + 1, &s, 10);
-		} else if (c == 'I' || c == 'K') {
+		} else if (c == 'f') {
 			double x;
 			char *p;
-			x = strtod(optarg, &p);
+			x = strtod(o.arg, &p);
-			if (*p == 'G' || *p == 'g') x *= 1e9;
+			if (x < 1.0) opt.mid_occ_frac = x, opt.mid_occ = 0;
-			else if (*p == 'M' || *p == 'm') x *= 1e6;
+			else opt.mid_occ = (int)(x + .499);
-			else if (*p == 'K' || *p == 'k') x *= 1e3;
+			if (*p == ',') opt.max_occ = (int)(strtod(p+1, &p) + .499);
-			if (c == 'I') batch_size = (uint64_t)(x + .499);
+		} else if (c == 'u') {
-			else minibatch_size = (uint64_t)(x + .499);
+			if (*o.arg == 'b') opt.flag |= MM_F_SPLICE_FOR|MM_F_SPLICE_REV; // both strands
-		} else if (c == 'x') {
+			else if (*o.arg == 'f') opt.flag |= MM_F_SPLICE_FOR, opt.flag &= ~MM_F_SPLICE_REV; // match GT-AG
-			if (strcmp(optarg, "ava-ont") == 0) {
+			else if (*o.arg == 'r') opt.flag |= MM_F_SPLICE_REV, opt.flag &= ~MM_F_SPLICE_FOR; // match CT-AC (reverse complement of GT-AG)
-				opt.flag |= MM_F_AVA | MM_F_NO_SELF;
+			else if (*o.arg == 'n') opt.flag &= ~(MM_F_SPLICE_FOR|MM_F_SPLICE_REV); // don't try to match the GT-AG signal
-				opt.min_chain_score = 100, opt.pri_ratio = 0.0f, opt.max_gap = 10000, opt.max_chain_skip = 25;
+			else {
-				minibatch_size = 500000000;
+				fprintf(stderr, "[ERROR]\033[1;31m unrecognized cDNA direction\033[0m\n");
-				k = 15, w = 5;
+				return 1;
-			} else if (strcmp(optarg, "ava-pb") == 0) {
+			}
-				opt.flag |= MM_F_AVA | MM_F_NO_SELF;
+		} else if (c == 'z') {
-				opt.min_chain_score = 100, opt.pri_ratio = 0.0f, opt.max_gap = 10000, opt.max_chain_skip = 25;
+			opt.zdrop = opt.zdrop_inv = strtol(o.arg, &s, 10);
-				minibatch_size = 500000000;
+			if (*s == ',') opt.zdrop_inv = strtol(s + 1, &s, 10);
-				is_hpc = 1, k = 19, w = 5;
+		} else if (c == 'O') {
-			} else if (strcmp(optarg, "map10k") == 0 || strcmp(optarg, "map-pb") == 0) {
+			opt.q = opt.q2 = strtol(o.arg, &s, 10);
-				is_hpc = 1, k = 19;
+			if (*s == ',') opt.q2 = strtol(s + 1, &s, 10);
-			} else if (strcmp(optarg, "map-ont") == 0) {
+		} else if (c == 'E') {
-				is_hpc = 0, k = 15;
+			opt.e = opt.e2 = strtol(o.arg, &s, 10);
-			} else if (strcmp(optarg, "asm5") == 0) {
+			if (*s == ',') opt.e2 = strtol(s + 1, &s, 10);
-				k = 19, w = 19;
+		}
-				opt.a = 1, opt.b = 19, opt.q = 39, opt.q2 = 81, opt.e = 3, opt.e2 = 1, opt.zdrop = 200;
+	}
-				opt.min_dp_max = 200;
+	if ((opt.flag & MM_F_SPLICE) && (opt.flag & MM_F_FRAG_MODE)) {
-			} else if (strcmp(optarg, "asm10") == 0) {
+		fprintf(stderr, "[ERROR]\033[1;31m --splice and --frag should not be specified at the same time.\033[0m\n");
-				k = 19, w = 19;
+		return 1;
-				opt.a = 1, opt.b = 9, opt.q = 16, opt.q2 = 41, opt.e = 2, opt.e2 = 1, opt.zdrop = 200;
+	}
-				opt.min_dp_max = 200;
+	if (!fnw && !(opt.flag&MM_F_CIGAR))
+		ipt.flag |= MM_I_NO_SEQ;
+	if (mm_check_opt(&ipt, &opt) < 0)
+		return 1;
+	if (opt.best_n == 0) {
+		fprintf(stderr, "[WARNING]\033[1;31m changed '-N 0' to '-N %d --secondary=no'.\033[0m\n", old_best_n);
+		opt.best_n = old_best_n, opt.flag |= MM_F_NO_PRINT_2ND;
+	}
+
+	if (argc == o.ind || fp_help == stdout) {
+		fprintf(fp_help, "Usage: minimap2 [options] <target.fa>|<target.idx> [query.fa] [...]\n");
+		fprintf(fp_help, "Options:\n");
+		fprintf(fp_help, "  Indexing:\n");
+		fprintf(fp_help, "    -H           use homopolymer-compressed k-mer (preferrable for PacBio)\n");
+		fprintf(fp_help, "    -k INT       k-mer size (no larger than 28) [%d]\n", ipt.k);
+		fprintf(fp_help, "    -w INT       minimizer window size [%d]\n", ipt.w);
+		fprintf(fp_help, "    -I NUM       split index for every ~NUM input bases [4G]\n");
+		fprintf(fp_help, "    -d FILE      dump index to FILE []\n");
+		fprintf(fp_help, "  Mapping:\n");
+		fprintf(fp_help, "    -f FLOAT     filter out top FLOAT fraction of repetitive minimizers [%g]\n", opt.mid_occ_frac);
+		fprintf(fp_help, "    -g NUM       stop chain enlongation if there are no minimizers in INT-bp [%d]\n", opt.max_gap);
+		fprintf(fp_help, "    -G NUM       max intron length (effective with -xsplice; changing -r) [200k]\n");
+		fprintf(fp_help, "    -F NUM       max fragment length (effective with -xsr or in the fragment mode) [800]\n");
+		fprintf(fp_help, "    -r NUM[,NUM] chaining/alignment bandwidth and long-join bandwidth [%d,%d]\n", opt.bw, opt.bw_long);
+		fprintf(fp_help, "    -n INT       minimal number of minimizers on a chain [%d]\n", opt.min_cnt);
+		fprintf(fp_help, "    -m INT       minimal chaining score (matching bases minus log gap penalty) [%d]\n", opt.min_chain_score);
+//		fprintf(fp_help, "    -T INT       SDUST threshold; 0 to disable SDUST [%d]\n", opt.sdust_thres); // TODO: this option is never used; might be buggy
+		fprintf(fp_help, "    -X           skip self and dual mappings (for the all-vs-all mode)\n");
+		fprintf(fp_help, "    -p FLOAT     min secondary-to-primary score ratio [%g]\n", opt.pri_ratio);
+		fprintf(fp_help, "    -N INT       retain at most INT secondary alignments [%d]\n", opt.best_n);
+		fprintf(fp_help, "  Alignment:\n");
+		fprintf(fp_help, "    -A INT       matching score [%d]\n", opt.a);
+		fprintf(fp_help, "    -B INT       mismatch penalty (larger value for lower divergence) [%d]\n", opt.b);
+		fprintf(fp_help, "    -O INT[,INT] gap open penalty [%d,%d]\n", opt.q, opt.q2);
+		fprintf(fp_help, "    -E INT[,INT] gap extension penalty; a k-long gap costs min{O1+k*E1,O2+k*E2} [%d,%d]\n", opt.e, opt.e2);
+		fprintf(fp_help, "    -z INT[,INT] Z-drop score and inversion Z-drop score [%d,%d]\n", opt.zdrop, opt.zdrop_inv);
+		fprintf(fp_help, "    -s INT       minimal peak DP alignment score [%d]\n", opt.min_dp_max);
+		fprintf(fp_help, "    -u CHAR      how to find GT-AG. f:transcript strand, b:both strands, n:don't match GT-AG [n]\n");
+		fprintf(fp_help, "  Input/Output:\n");
+		fprintf(fp_help, "    -a           output in the SAM format (PAF by default)\n");
+		fprintf(fp_help, "    -o FILE      output alignments to FILE [stdout]\n");
+		fprintf(fp_help, "    -L           write CIGAR with >65535 ops at the CG tag\n");
+		fprintf(fp_help, "    -R STR       SAM read group line in a format like '@RG\\tID:foo\\tSM:bar' []\n");
+		fprintf(fp_help, "    -c           output CIGAR in PAF\n");
+		fprintf(fp_help, "    --cs[=STR]   output the cs tag; STR is 'short' (if absent) or 'long' [none]\n");
+		fprintf(fp_help, "    --MD         output the MD tag\n");
+		fprintf(fp_help, "    --eqx        write =/X CIGAR operators\n");
+		fprintf(fp_help, "    -Y           use soft clipping for supplementary alignments\n");
+		fprintf(fp_help, "    -t INT       number of threads [%d]\n", n_threads);
+		fprintf(fp_help, "    -K NUM       minibatch size for mapping [500M]\n");
+//		fprintf(fp_help, "    -v INT       verbose level [%d]\n", mm_verbose);
+		fprintf(fp_help, "    --version    show version number\n");
+		fprintf(fp_help, "  Preset:\n");
+		fprintf(fp_help, "    -x STR       preset (always applied before other options; see minimap2.1 for details) []\n");
+		fprintf(fp_help, "                 - map-pb/map-ont - PacBio CLR/Nanopore vs reference mapping\n");
+		fprintf(fp_help, "                 - map-hifi - PacBio HiFi reads vs reference mapping\n");
+		fprintf(fp_help, "                 - ava-pb/ava-ont - PacBio/Nanopore read overlap\n");
+		fprintf(fp_help, "                 - asm5/asm10/asm20 - asm-to-ref mapping, for ~0.1/1/5%% sequence divergence\n");
+		fprintf(fp_help, "                 - splice/splice:hq - long-read/Pacbio-CCS spliced alignment\n");
+		fprintf(fp_help, "                 - sr - genomic short-read mapping\n");
+		fprintf(fp_help, "\nSee `man ./minimap2.1' for detailed description of these and other advanced command-line options.\n");
+		return fp_help == stdout? 0 : 1;
+	}
+
+	if ((opt.flag & MM_F_SR) && argc - o.ind > 3) {
+		fprintf(stderr, "[ERROR] incorrect input: in the sr mode, please specify no more than two query files.\n");
+		return 1;
+	}
+	idx_rdr = mm_idx_reader_open(argv[o.ind], &ipt, fnw);
+	if (idx_rdr == 0) {
+		fprintf(stderr, "[ERROR] failed to open file '%s': %s\n", argv[o.ind], strerror(errno));
+		return 1;
+	}
+	if (!idx_rdr->is_idx && fnw == 0 && argc - o.ind < 2) {
+		fprintf(stderr, "[ERROR] missing input: please specify a query file to map or option -d to keep the index\n");
+		mm_idx_reader_close(idx_rdr);
+		return 1;
+	}
+	if (opt.best_n == 0 && (opt.flag&MM_F_CIGAR) && mm_verbose >= 2)
+		fprintf(stderr, "[WARNING]\033[1;31m `-N 0' reduces alignment accuracy. Please use --secondary=no to suppress secondary alignments.\033[0m\n");
+	while ((mi = mm_idx_reader_read(idx_rdr, n_threads)) != 0) {
+		int ret;
+		if ((opt.flag & MM_F_CIGAR) && (mi->flag & MM_I_NO_SEQ)) {
+			fprintf(stderr, "[ERROR] the prebuilt index doesn't contain sequences.\n");
+			mm_idx_destroy(mi);
+			mm_idx_reader_close(idx_rdr);
+			return 1;
+		}
+		if ((opt.flag & MM_F_OUT_SAM) && idx_rdr->n_parts == 1) {
+			if (mm_idx_reader_eof(idx_rdr)) {
+				if (opt.split_prefix == 0)
+					ret = mm_write_sam_hdr(mi, rg, MM_VERSION, argc, argv);
+				else
+					ret = mm_write_sam_hdr(0, rg, MM_VERSION, argc, argv);
 			} else {
-				fprintf(stderr, "[E::%s] unknown preset '%s'\n", __func__, optarg);
+				ret = mm_write_sam_hdr(0, rg, MM_VERSION, argc, argv);
+				if (opt.split_prefix == 0 && mm_verbose >= 2)
+					fprintf(stderr, "[WARNING]\033[1;31m For a multi-part index, no @SQ lines will be outputted. Please use --split-prefix.\033[0m\n");
+			}
+			if (ret != 0) {
+				mm_idx_destroy(mi);
+				mm_idx_reader_close(idx_rdr);
 				return 1;
 			}
 		}
-	}
-	if (w < 0) w = (int)(.6666667 * k + .499);
-
-	if (argc == optind) {
-		fprintf(stderr, "Usage: minimap2 [options] <target.fa>|<target.idx> [query.fa] [...]\n");
-		fprintf(stderr, "Options:\n");
-		fprintf(stderr, "  Indexing:\n");
-		fprintf(stderr, "    -H           use homopolymer-compressed k-mer\n");
-		fprintf(stderr, "    -k INT       k-mer size (no larger than 28) [%d]\n", k);
-		fprintf(stderr, "    -w INT       minizer window size [{-k}*2/3]\n");
-		fprintf(stderr, "    -I NUM       split index for every ~NUM input bases [4G]\n");
-		fprintf(stderr, "    -d FILE      dump index to FILE []\n");
-		fprintf(stderr, "  Mapping:\n");
-		fprintf(stderr, "    -f FLOAT     filter out top FLOAT fraction of repetitive minimizers [%g]\n", opt.mid_occ_frac);
-		fprintf(stderr, "    -g INT       stop chain enlongation if there are no minimizers in INT-bp [%d]\n", opt.max_gap);
-		fprintf(stderr, "    -r INT       bandwidth used in chaining and DP-based alignment [%d]\n", opt.bw);
-		fprintf(stderr, "    -n INT       minimal number of minimizers on a chain [%d]\n", opt.min_cnt);
-		fprintf(stderr, "    -m INT       minimal chaining score (matching bases minus log gap penalty) [%d]\n", opt.min_chain_score);
-//		fprintf(stderr, "    -T INT       SDUST threshold; 0 to disable SDUST [%d]\n", opt.sdust_thres); // TODO: this option is never used; might be buggy
-		fprintf(stderr, "    -X           skip self and dual mappings (for the all-vs-all mode)\n");
-		fprintf(stderr, "    -p FLOAT     min secondary-to-primary score ratio [%g]\n", opt.pri_ratio);
-		fprintf(stderr, "    -N INT       retain at most INT secondary alignments [%d]\n", opt.best_n);
-		fprintf(stderr, "  Alignment:\n");
-		fprintf(stderr, "    -A INT       matching score [%d]\n", opt.a);
-		fprintf(stderr, "    -B INT       mismatch penalty [%d]\n", opt.b);
-		fprintf(stderr, "    -O INT[,INT] gap open penalty [%d,%d]\n", opt.q, opt.q2);
-		fprintf(stderr, "    -E INT[,INT] gap extension penalty; a k-long gap costs min{O1+k*E1,O2+k*E2} [%d,%d]\n", opt.e, opt.e2);
-		fprintf(stderr, "    -z INT       Z-drop score [%d]\n", opt.zdrop);
-		fprintf(stderr, "    -s INT       minimal peak DP alignment score [%d]\n", opt.min_dp_max);
-		fprintf(stderr, "  Input/Output:\n");
-		fprintf(stderr, "    -Q           ignore base quality in the input\n");
-		fprintf(stderr, "    -a           output in the SAM format (PAF by default)\n");
-		fprintf(stderr, "    -c           output CIGAR in PAF\n");
-		fprintf(stderr, "    -t INT       number of threads [%d]\n", n_threads);
-		fprintf(stderr, "    -K NUM       minibatch size [200M]\n");
-//		fprintf(stderr, "    -v INT       verbose level [%d]\n", mm_verbose);
-		fprintf(stderr, "    -V           show version number\n");
-		fprintf(stderr, "  Preset:\n");
-		fprintf(stderr, "    -x STR       preset (recommended to be applied before other options) []\n");
-		fprintf(stderr, "                 map10k/map-pb: -Hk19 (PacBio/ONT vs reference mapping)\n");
-		fprintf(stderr, "                 map-ont: -k15 (slightly more sensitive than 'map10k' for ONT vs reference)\n");
-		fprintf(stderr, "                 asm5: -k19 -w19 -A1 -B19 -O39,81 -E3,1 -s200 -z200 (asm to ref mapping; break at 5%% div.)\n");
-		fprintf(stderr, "                 asm10: -k19 -w19 -A1 -B9 -O16,41 -E2,1 -s200 -z200 (asm to ref mapping; break at 10%% div.)\n");
-		fprintf(stderr, "                 ava-pb: -Hk19 -w5 -Xp0 -m100 -g10000 -K500m --max-chain-skip 25 (PacBio read overlap)\n");
-		fprintf(stderr, "                 ava-ont: -k15 -w5 -Xp0 -m100 -g10000 -K500m --max-chain-skip 25 (ONT read overlap)\n");
-		fprintf(stderr, "\nSee `man ./minimap2.1' for detailed description of command-line options.\n");
-		return 1;
-	}
-
-	is_idx = mm_idx_is_idx(argv[optind]);
-	if (is_idx < 0) {
-		fprintf(stderr, "[E::%s] failed to open file '%s'\n", __func__, argv[optind]);
-		return 1;
-	}
-	if (is_idx) fpr = fopen(argv[optind], "rb");
-	else fp = mm_bseq_open(argv[optind]);
-	if (fnw) fpw = fopen(fnw, "wb");
-	for (;;) {
-		mm_idx_t *mi = 0;
-		if (fpr) {
-			mi = mm_idx_load(fpr);
-			if (idx_par_set && mm_verbose >= 2 && (mi->k != k || mi->w != w || mi->is_hpc != mi->is_hpc))
-				fprintf(stderr, "[W::%s::%.3f*%.2f] Indexing parameters on the command line (-k/-w/-H) overridden by parameters in the prebuilt index.\n",
-						__func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
-		} else if (!mm_bseq_eof(fp)) {
-			mi = mm_idx_gen(fp, w, k, bucket_bits, is_hpc, minibatch_size, n_threads, batch_size, keep_name);
-		}
-		if (mi == 0) break;
 		if (mm_verbose >= 3)
 			fprintf(stderr, "[M::%s::%.3f*%.2f] loaded/built the index for %d target sequence(s)\n",
 					__func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), mi->n_seq);
-		if (fpw) {
+		if (argc != o.ind + 1) mm_mapopt_update(&opt, mi);
-			mm_idx_dump(fpw, mi);
-			if (mm_verbose >= 3)
-				fprintf(stderr, "[M::%s::%.3f*%.2f] dumpped the (partial) index to disk\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
-		}
-		if (argc != optind + 1) mm_mapopt_update(&opt, mi);
 		if (mm_verbose >= 3) mm_idx_stat(mi);
-		for (i = optind + 1; i < argc; ++i)
+		if (junc_bed) mm_idx_bed_read(mi, junc_bed, 1);
-			mm_map_file(mi, argv[i], &opt, n_threads, minibatch_size);
+		if (alt_list) mm_idx_alt_read(mi, alt_list);
+		if (argc - (o.ind + 1) == 0) {
+			mm_idx_destroy(mi);
+			continue; // no query files
+		}
+		ret = 0;
+		if (!(opt.flag & MM_F_FRAG_MODE)) {
+			for (i = o.ind + 1; i < argc; ++i) {
+				ret = mm_map_file(mi, argv[i], &opt, n_threads);
+				if (ret < 0) break;
+			}
+		} else {
+			ret = mm_map_file_frag(mi, argc - (o.ind + 1), (const char**)&argv[o.ind + 1], &opt, n_threads);
+		}
 		mm_idx_destroy(mi);
+		if (ret < 0) {
+			fprintf(stderr, "ERROR: failed to map the query file\n");
+			exit(EXIT_FAILURE);
+		}
 	}
-	if (fpw) fclose(fpw);
+	n_parts = idx_rdr->n_parts;
-	if (fpr) fclose(fpr);
+	mm_idx_reader_close(idx_rdr);
-	if (fp)  mm_bseq_close(fp);
 
-	fprintf(stderr, "[M::%s] Version: %s\n", __func__, MM_VERSION);
+	if (opt.split_prefix)
-	fprintf(stderr, "[M::%s] CMD:", __func__);
+		mm_split_merge(argc - (o.ind + 1), (const char**)&argv[o.ind + 1], &opt, n_parts);
-	for (i = 0; i < argc; ++i)
+
-		fprintf(stderr, " %s", argv[i]);
+	if (fflush(stdout) == EOF) {
-	fprintf(stderr, "\n[M::%s] Real time: %.3f sec; CPU: %.3f sec\n", __func__, realtime() - mm_realtime0, cputime());
+		perror("[ERROR] failed to write the results");
+		exit(EXIT_FAILURE);
+	}
+
+	if (mm_verbose >= 3) {
+		fprintf(stderr, "[M::%s] Version: %s\n", __func__, MM_VERSION);
+		fprintf(stderr, "[M::%s] CMD:", __func__);
+		for (i = 0; i < argc; ++i)
+			fprintf(stderr, " %s", argv[i]);
+		fprintf(stderr, "\n[M::%s] Real time: %.3f sec; CPU: %.3f sec; Peak RSS: %.3f GB\n", __func__, realtime() - mm_realtime0, cputime(), peakrss() / 1024.0 / 1024.0 / 1024.0);
+	}
 	return 0;
 }

map.c

@@ -1,62 +1,18 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include <errno.h>
 #include "kthread.h"
 #include "kvec.h"
 #include "kalloc.h"
 #include "sdust.h"
 #include "mmpriv.h"
 #include "bseq.h"
-
+#include "khash.h"
-void mm_mapopt_init(mm_mapopt_t *opt)
-{
-	memset(opt, 0, sizeof(mm_mapopt_t));
-	opt->max_occ_frac = 1e-5f;
-	opt->mid_occ_frac = 2e-4f;
-	opt->sdust_thres = 0;
-
-	opt->min_cnt = 3;
-	opt->min_chain_score = 40;
-	opt->bw = 500;
-	opt->max_gap = 5000;
-	opt->max_chain_skip = 25;
-
-	opt->mask_level = 0.5f;
-	opt->pri_ratio = 0.8f;
-	opt->best_n = 5;
-
-	opt->max_join_long = 20000;
-	opt->max_join_short = 2000;
-	opt->min_join_flank_sc = 1000;
-
-	opt->a = 2, opt->b = 4, opt->q = 4, opt->e = 2, opt->q2 = 24, opt->e2 = 1;
-	opt->zdrop = 400;
-	opt->min_dp_max = opt->min_chain_score;
-	opt->min_ksw_len = 200;
-}
-
-void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi)
-{
-	opt->max_occ = mm_idx_cal_max_occ(mi, opt->max_occ_frac);
-	opt->mid_occ = mm_idx_cal_max_occ(mi, opt->mid_occ_frac);
-	if (mm_verbose >= 3)
-		fprintf(stderr, "[M::%s::%.3f*%.2f] mid_occ = %d; max_occ = %d\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0),
-				opt->mid_occ, opt->max_occ);
-}
-
-typedef struct {
-	uint32_t n:31, is_alloc:1;
-	uint32_t qpos;
-	union {
-		const uint64_t *cr;
-		uint64_t *r;
-	} x;
-} mm_match_t;
 
 struct mm_tbuf_s {
-	sdust_buf_t *sdb;
-	mm128_v mini;
 	void *km;
+	int rep_len, frag_gap;
 };
 
 mm_tbuf_t *mm_tbuf_init(void)
@@ -64,213 +20,368 @@ mm_tbuf_t *mm_tbuf_init(void)
 	mm_tbuf_t *b;
 	b = (mm_tbuf_t*)calloc(1, sizeof(mm_tbuf_t));
 	if (!(mm_dbg_flag & 1)) b->km = km_init();
-	b->sdb = sdust_buf_init(b->km);
 	return b;
 }
 
 void mm_tbuf_destroy(mm_tbuf_t *b)
 {
 	if (b == 0) return;
-	kfree(b->km, b->mini.a);
-	sdust_buf_destroy(b->sdb);
 	km_destroy(b->km);
 	free(b);
 }
 
-static void mm_dust_minier(mm128_v *mini, int l_seq, const char *seq, int sdust_thres, sdust_buf_t *sdb)
+void *mm_tbuf_get_km(mm_tbuf_t *b)
+{
+	return b->km;
+}
+
+static int mm_dust_minier(void *km, int n, mm128_t *a, int l_seq, const char *seq, int sdust_thres)
 {
 	int n_dreg, j, k, u = 0;
 	const uint64_t *dreg;
-	if (sdust_thres <= 0 || sdb == 0) return;
+	sdust_buf_t *sdb;
+	if (sdust_thres <= 0) return n;
+	sdb = sdust_buf_init(km);
 	dreg = sdust_core((const uint8_t*)seq, l_seq, sdust_thres, 64, &n_dreg, sdb);
-	for (j = k = 0; j < mini->n; ++j) { // squeeze out minimizers that significantly overlap with LCRs
+	for (j = k = 0; j < n; ++j) { // squeeze out minimizers that significantly overlap with LCRs
-		int32_t qpos = (uint32_t)mini->a[j].y>>1, span = mini->a[j].x&0xff;
+		int32_t qpos = (uint32_t)a[j].y>>1, span = a[j].x&0xff;
 		int32_t s = qpos - (span - 1), e = s + span;
-		while (u < n_dreg && (uint32_t)dreg[u] <= s) ++u;
+		while (u < n_dreg && (int32_t)dreg[u] <= s) ++u;
-		if (u < n_dreg && dreg[u]>>32 < e) {
+		if (u < n_dreg && (int32_t)(dreg[u]>>32) < e) {
 			int v, l = 0;
-			for (v = u; v < n_dreg && dreg[v]>>32 < e; ++v) { // iterate over LCRs overlapping this minimizer
+			for (v = u; v < n_dreg && (int32_t)(dreg[v]>>32) < e; ++v) { // iterate over LCRs overlapping this minimizer
-				int ss = s > dreg[v]>>32? s : dreg[v]>>32;
+				int ss = s > (int32_t)(dreg[v]>>32)? s : dreg[v]>>32;
-				int ee = e < (uint32_t)dreg[v]? e : (uint32_t)dreg[v];
+				int ee = e < (int32_t)dreg[v]? e : (uint32_t)dreg[v];
 				l += ee - ss;
 			}
-			if (l <= span>>1) mini->a[k++] = mini->a[j]; // keep the minimizer if less than half of it falls in masked region
+			if (l <= span>>1) a[k++] = a[j]; // keep the minimizer if less than half of it falls in masked region
-		}
+		} else a[k++] = a[j];
 	}
-	mini->n = k;
+	sdust_buf_destroy(sdb);
-}
+	return k; // the new size
-#if 0
-int mm_pair_thin_core(mm_tbuf_t *b, uint64_t x, int radius, int rel, int st0, int n, const uint64_t *z, uint64_v *a)
-{
-	int i, st = st0, en = n, mid = en - 1;
-	while (st < en) {
-		uint64_t y;
-		mid = st + ((en - st) >> 1);
-		y = z[mid];
-		if (y < x && (x - y)>>1 > radius) st = mid + 1;
-		else if (y >= x && (y - x)>>1 > radius) en = mid;
-		else break;
-	}
-	if (st < en) {
-		for (en = mid + 1; en < n; ++en)
-			if (z[en] > x && (z[en] - x)>>1 > radius)
-				break;
-		for (st = mid - 1; st >= st0; --st)
-			if (z[st] < x && (x - z[st])>>1 > radius)
-				break;
-		++st;
-		for (i = st; i < en; ++i) {
-			uint64_t y = z[i];
-			if (((x ^ y) & 1) == rel) {
-//				printf("* %d,%d\n", (uint32_t)x>>1, (uint32_t)y>>1);
-				kv_push(uint64_t, b->km, *a, y);
-			}
-		}
-		return en;
-	} else return st < n && z[st] < x? st + 1 : en;
 }
 
-void mm_pair_thin(mm_tbuf_t *b, int radius, mm_match_t *m1, mm_match_t *m2)
+static void collect_minimizers(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int n_segs, const int *qlens, const char **seqs, mm128_v *mv)
 {
-	mm_match_t *m[2];
+	int i, n, sum = 0;
-	const uint64_t *z[2];
+	mv->n = 0;
-	uint64_v a[2];
+	for (i = n = 0; i < n_segs; ++i) {
-	int i, n[2], k[2], u = 0, rel = (m1->qpos ^ m2->qpos) & 1;
+		size_t j;
-
+		mm_sketch(km, seqs[i], qlens[i], mi->w, mi->k, i, mi->flag&MM_I_HPC, mv);
-	m[0] = m1, m[1] = m2;
+		for (j = n; j < mv->n; ++j)
-	for (i = 0; i < 2; ++i) {
+			mv->a[j].y += sum << 1;
-		n[i] = m[i]->n;
+		if (opt->sdust_thres > 0) // mask low-complexity minimizers
-		z[i] = m[i]->x.cr;
+			mv->n = n + mm_dust_minier(km, mv->n - n, mv->a + n, qlens[i], seqs[i], opt->sdust_thres);
-		k[i] = 0;
+		sum += qlens[i], n = mv->n;
-		kv_init(a[i]);
-		kv_resize(uint64_t, b->km, a[i], 256);
 	}
-	while (k[0] < n[0] && k[1] < n[1]) {
-		//printf("%d; %d,%d\n", u, k[0], k[1]);
-		int v = u^1, dist = (int)(m[v]->qpos>>1) - (int)(m[u]->qpos>>1);
-		uint64_t x = z[u][k[u]];
-		int uori = (x ^ m[u]->qpos) & 1, last;
-		int64_t tpos = x>>1 & 0x7fffffff;
-		tpos = uori == 0? tpos + dist : tpos - dist;
-		if (tpos < 0) tpos = 0;
-		x = x>>32<<32 | tpos<<1 | (x&1);
-		last = a[v].n;
-		k[v] = mm_pair_thin_core(b, x, radius, rel, k[v], n[v], z[v], &a[v]);
-		if (a[v].n > last) kv_push(uint64_t, b->km, a[u], z[u][k[u]]);
-		++k[u];
-		u ^= 1;
-	}
-	for (i = 0; i < 2; ++i)
-		m[i]->n = a[i].n, m[i]->x.r = a[i].a, m[i]->is_alloc = 1;
-//	printf("%d,%d; %d,%d\n", m[0]->qpos>>1, m[1]->qpos>>1, m[0]->n, m[1]->n);
 }
-#endif
-mm_reg1_t *mm_map_frag(const mm_mapopt_t *opt, const mm_idx_t *mi, mm_tbuf_t *b, uint32_t m_st, uint32_t m_en, const char *qname, int qlen, const char *seq, int *n_regs)
-{
-	int i, n = m_en - m_st, j, n_u;
-	int64_t n_a;
-	uint64_t *u;
-	mm_match_t *m;
-	mm128_t *a;
-	mm_reg1_t *regs;
 
-	// convert to local representation
+#include "ksort.h"
-	m = (mm_match_t*)kmalloc(b->km, n * sizeof(mm_match_t));
+#define heap_lt(a, b) ((a).x > (b).x)
-	for (i = 0; i < n; ++i) {
+KSORT_INIT(heap, mm128_t, heap_lt)
-		int t;
+
-		mm128_t *p = &b->mini.a[i + m_st];
+static inline int skip_seed(int flag, uint64_t r, const mm_seed_t *q, const char *qname, int qlen, const mm_idx_t *mi, int *is_self)
-		m[i].is_alloc = 0;
+{
-		m[i].qpos = (uint32_t)p->y;
+	*is_self = 0;
-		m[i].x.cr = mm_idx_get(mi, p->x>>8, &t);
+	if (qname && (flag & (MM_F_NO_DIAG|MM_F_NO_DUAL))) {
-		m[i].n = t;
+		const mm_idx_seq_t *s = &mi->seq[r>>32];
+		int cmp;
+		cmp = strcmp(qname, s->name);
+		if ((flag&MM_F_NO_DIAG) && cmp == 0 && (int)s->len == qlen) {
+			if ((uint32_t)r>>1 == (q->q_pos>>1)) return 1; // avoid the diagnonal anchors
+			if ((r&1) == (q->q_pos&1)) *is_self = 1; // this flag is used to avoid spurious extension on self chain
+		}
+		if ((flag&MM_F_NO_DUAL) && cmp > 0) // all-vs-all mode: map once
+			return 1;
 	}
-#if 0
+	if (flag & (MM_F_FOR_ONLY|MM_F_REV_ONLY)) {
-	int last = -1, last2 = -1;
+		if ((r&1) == (q->q_pos&1)) { // forward strand
-	// pair k-mer thinning
+			if (flag & MM_F_REV_ONLY) return 1;
-	for (i = 0; i < n; ++i) {
+		} else {
-		if (m[i].n >= opt->mid_occ && m[i].n < opt->max_occ) {
+			if (flag & MM_F_FOR_ONLY) return 1;
-			if (last2 < 0) last2 = i;
-			if (last < 0 || m[last].n < m[i].n) last = i;
-			if (last >= 0 && (m[last].qpos>>1) + (m[last].span>>1) <= m[i].qpos>>1) {
-				mm_pair_thin(b, opt->bw, &m[last], &m[i]);
-				last2 = last = -1;
-			} else if (last2 >= 0 && (m[last2].qpos>>1) + (m[last2].span>>1) <= m[i].qpos>>1) {
-				mm_pair_thin(b, opt->bw, &m[last2], &m[i]);
-				last2 = last = -1;
-			}
 		}
 	}
-#endif
+	return 0;
-	// fill the _a_ array
+}
-	for (i = 0, n_a = 0; i < n; ++i) // find the length of a[]
+
-		if (m[i].n < opt->mid_occ) n_a += m[i].n;
+static mm128_t *collect_seed_hits_heap(void *km, const mm_mapopt_t *opt, int max_occ, const mm_idx_t *mi, const char *qname, const mm128_v *mv, int qlen, int64_t *n_a, int *rep_len,
-	a = (mm128_t*)kmalloc(b->km, n_a * sizeof(mm128_t));
+								  int *n_mini_pos, uint64_t **mini_pos)
-	for (i = j = 0; i < n; ++i) {
+{
-		mm128_t *p = &b->mini.a[i + m_st];
+	int i, n_m, heap_size = 0;
-		mm_match_t *q = &m[i];
+	int64_t j, n_for = 0, n_rev = 0;
-		const uint64_t *r = q->x.cr;
+	mm_seed_t *m;
-		int k, q_span = p->x & 0xff;
+	mm128_t *a, *heap;
-		if (q->n >= opt->mid_occ) continue;
+
-		for (k = 0; k < q->n; ++k) {
+	m = mm_collect_matches(km, &n_m, qlen, max_occ, opt->max_max_occ, opt->occ_dist, mi, mv, n_a, rep_len, n_mini_pos, mini_pos);
-			const char *tname = mi->seq[r[k]>>32].name;
+
-			int32_t rpos = (uint32_t)r[k] >> 1;
+	heap = (mm128_t*)kmalloc(km, n_m * sizeof(mm128_t));
-			mm128_t *p;
+	a = (mm128_t*)kmalloc(km, *n_a * sizeof(mm128_t));
-			if (qname && (opt->flag&MM_F_NO_SELF) && strcmp(qname, tname) == 0 && rpos == (q->qpos>>1)) // avoid the diagonal
+
-				continue;
+	for (i = 0, heap_size = 0; i < n_m; ++i) {
-			if (qname && (opt->flag&MM_F_AVA) && strcmp(qname, tname) > 0) // all-vs-all mode: map once
+		if (m[i].n > 0) {
-				continue;
+			heap[heap_size].x = m[i].cr[0];
-			p = &a[j++];
+			heap[heap_size].y = (uint64_t)i<<32;
-			if ((r[k]&1) == (q->qpos&1)) { // forward strand
+			++heap_size;
-				p->x = (r[k]&0xffffffff00000000ULL) | (uint32_t)r[k]>>1;
+		}
-				p->y = (uint64_t)q_span << 32 | q->qpos >> 1;
+	}
+	ks_heapmake_heap(heap_size, heap);
+	while (heap_size > 0) {
+		mm_seed_t *q = &m[heap->y>>32];
+		mm128_t *p;
+		uint64_t r = heap->x;
+		int32_t is_self, rpos = (uint32_t)r >> 1;
+		if (!skip_seed(opt->flag, r, q, qname, qlen, mi, &is_self)) {
+			if ((r&1) == (q->q_pos&1)) { // forward strand
+				p = &a[n_for++];
+				p->x = (r&0xffffffff00000000ULL) | rpos;
+				p->y = (uint64_t)q->q_span << 32 | q->q_pos >> 1;
 			} else { // reverse strand
-				p->x = 1ULL<<63 | (r[k]&0xffffffff00000000ULL) | (uint32_t)r[k]>>1;
+				p = &a[(*n_a) - (++n_rev)];
-				p->y = (uint64_t)q_span << 32 | (qlen - ((q->qpos>>1) + 1 - q_span) - 1);
+				p->x = 1ULL<<63 | (r&0xffffffff00000000ULL) | rpos;
+				p->y = (uint64_t)q->q_span << 32 | (qlen - ((q->q_pos>>1) + 1 - q->q_span) - 1);
 			}
+			p->y |= (uint64_t)q->seg_id << MM_SEED_SEG_SHIFT;
+			if (q->is_tandem) p->y |= MM_SEED_TANDEM;
+			if (is_self) p->y |= MM_SEED_SELF;
+		}
+		// update the heap
+		if ((uint32_t)heap->y < q->n - 1) {
+			++heap[0].y;
+			heap[0].x = m[heap[0].y>>32].cr[(uint32_t)heap[0].y];
+		} else {
+			heap[0] = heap[heap_size - 1];
+			--heap_size;
+		}
+		ks_heapdown_heap(0, heap_size, heap);
+	}
+	kfree(km, m);
+	kfree(km, heap);
+
+	// reverse anchors on the reverse strand, as they are in the descending order
+	for (j = 0; j < n_rev>>1; ++j) {
+		mm128_t t = a[(*n_a) - 1 - j];
+		a[(*n_a) - 1 - j] = a[(*n_a) - (n_rev - j)];
+		a[(*n_a) - (n_rev - j)] = t;
+	}
+	if (*n_a > n_for + n_rev) {
+		memmove(a + n_for, a + (*n_a) - n_rev, n_rev * sizeof(mm128_t));
+		*n_a = n_for + n_rev;
+	}
+	return a;
+}
+
+static mm128_t *collect_seed_hits(void *km, const mm_mapopt_t *opt, int max_occ, const mm_idx_t *mi, const char *qname, const mm128_v *mv, int qlen, int64_t *n_a, int *rep_len,
+								  int *n_mini_pos, uint64_t **mini_pos)
+{
+	int i, n_m;
+	mm_seed_t *m;
+	mm128_t *a;
+	m = mm_collect_matches(km, &n_m, qlen, max_occ, opt->max_max_occ, opt->occ_dist, mi, mv, n_a, rep_len, n_mini_pos, mini_pos);
+	a = (mm128_t*)kmalloc(km, *n_a * sizeof(mm128_t));
+	for (i = 0, *n_a = 0; i < n_m; ++i) {
+		mm_seed_t *q = &m[i];
+		const uint64_t *r = q->cr;
+		uint32_t k;
+		for (k = 0; k < q->n; ++k) {
+			int32_t is_self, rpos = (uint32_t)r[k] >> 1;
+			mm128_t *p;
+			if (skip_seed(opt->flag, r[k], q, qname, qlen, mi, &is_self)) continue;
+			p = &a[(*n_a)++];
+			if ((r[k]&1) == (q->q_pos&1)) { // forward strand
+				p->x = (r[k]&0xffffffff00000000ULL) | rpos;
+				p->y = (uint64_t)q->q_span << 32 | q->q_pos >> 1;
+			} else if (!(opt->flag & MM_F_QSTRAND)) { // reverse strand and not in the query-strand mode
+				p->x = 1ULL<<63 | (r[k]&0xffffffff00000000ULL) | rpos;
+				p->y = (uint64_t)q->q_span << 32 | (qlen - ((q->q_pos>>1) + 1 - q->q_span) - 1);
+			} else { // reverse strand; query-strand
+				int32_t len = mi->seq[r[k]>>32].len;
+				p->x = 1ULL<<63 | (r[k]&0xffffffff00000000ULL) | (len - (rpos + 1 - q->q_span) - 1); // coordinate only accurate for non-HPC seeds
+				p->y = (uint64_t)q->q_span << 32 | q->q_pos >> 1;
+			}
+			p->y |= (uint64_t)q->seg_id << MM_SEED_SEG_SHIFT;
+			if (q->is_tandem) p->y |= MM_SEED_TANDEM;
+			if (is_self) p->y |= MM_SEED_SELF;
 		}
 	}
-	n_a = j;
+	kfree(km, m);
-	radix_sort_128x(a, a + n_a);
+	radix_sort_128x(a, a + (*n_a));
-	for (i = 0; i < n; ++i)
+	return a;
-		if (m[i].is_alloc) kfree(b->km, m[i].x.r);
+}
-	kfree(b->km, m);
 
-	if (mm_dbg_flag & MM_DBG_PRINT_SEED)
+static void chain_post(const mm_mapopt_t *opt, int max_chain_gap_ref, const mm_idx_t *mi, void *km, int qlen, int n_segs, const int *qlens, int *n_regs, mm_reg1_t *regs, mm128_t *a)
-		for (i = 0; i < n_a; ++i)
+{
-			fprintf(stderr, "SD\t%s\t%d\t%c\t%d\t%d\n", mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff));
+	if (!(opt->flag & MM_F_ALL_CHAINS)) { // don't choose primary mapping(s)
-
+		mm_set_parent(km, opt->mask_level, opt->mask_len, *n_regs, regs, opt->a * 2 + opt->b, opt->flag&MM_F_HARD_MLEVEL, opt->alt_drop);
-	n_u = mm_chain_dp(opt->max_gap, opt->bw, opt->max_chain_skip, opt->min_cnt, opt->min_chain_score, n_a, a, &u, b->km);
+		if (n_segs <= 1) mm_select_sub(km, opt->pri_ratio, mi->k*2, opt->best_n, 1, opt->max_gap * 0.8, n_regs, regs);
-	regs = mm_gen_regs(b->km, qlen, n_u, u, a);
+		else mm_select_sub_multi(km, opt->pri_ratio, 0.2f, 0.7f, max_chain_gap_ref, mi->k*2, opt->best_n, n_segs, qlens, n_regs, regs);
-	*n_regs = n_u;
-	if (!(opt->flag & MM_F_AVA)) { // don't choose primary mapping(s) for read overlap
-		mm_set_parent(b->km, opt->mask_level, *n_regs, regs);
-		mm_select_sub(b->km, opt->mask_level, opt->pri_ratio, opt->best_n, n_regs, regs);
-		mm_join_long(b->km, opt, qlen, n_regs, regs, a); // TODO: this can be applied to all-vs-all in principle
 	}
-	if (opt->flag & MM_F_CIGAR) {
+}
-		regs = mm_align_skeleton(b->km, opt, mi, qlen, seq, n_regs, regs, a); // this calls mm_filter_regs()
-		if (!(opt->flag & MM_F_AVA)) {
-			mm_set_parent(b->km, opt->mask_level, *n_regs, regs);
-			mm_select_sub(b->km, opt->mask_level, opt->pri_ratio, opt->best_n, n_regs, regs);
-			mm_set_sam_pri(*n_regs, regs);
-		}
-	}
-	mm_set_mapq(*n_regs, regs);
 
-	// free
+static mm_reg1_t *align_regs(const mm_mapopt_t *opt, const mm_idx_t *mi, void *km, int qlen, const char *seq, int *n_regs, mm_reg1_t *regs, mm128_t *a)
-	kfree(b->km, a);
+{
-	kfree(b->km, u);
+	if (!(opt->flag & MM_F_CIGAR)) return regs;
+	regs = mm_align_skeleton(km, opt, mi, qlen, seq, n_regs, regs, a); // this calls mm_filter_regs()
+	if (!(opt->flag & MM_F_ALL_CHAINS)) { // don't choose primary mapping(s)
+		mm_set_parent(km, opt->mask_level, opt->mask_len, *n_regs, regs, opt->a * 2 + opt->b, opt->flag&MM_F_HARD_MLEVEL, opt->alt_drop);
+		mm_select_sub(km, opt->pri_ratio, mi->k*2, opt->best_n, 0, opt->max_gap * 0.8, n_regs, regs);
+		mm_set_sam_pri(*n_regs, regs);
+	}
 	return regs;
 }
 
-mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
+void mm_map_frag(const mm_idx_t *mi, int n_segs, const int *qlens, const char **seqs, int *n_regs, mm_reg1_t **regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
+{
+	int i, j, rep_len, qlen_sum, n_regs0, n_mini_pos;
+	int max_chain_gap_qry, max_chain_gap_ref, is_splice = !!(opt->flag & MM_F_SPLICE), is_sr = !!(opt->flag & MM_F_SR);
+	uint32_t hash;
+	int64_t n_a;
+	uint64_t *u, *mini_pos;
+	mm128_t *a;
+	mm128_v mv = {0,0,0};
+	mm_reg1_t *regs0;
+	km_stat_t kmst;
+	float chn_pen_gap, chn_pen_skip;
+
+	for (i = 0, qlen_sum = 0; i < n_segs; ++i)
+		qlen_sum += qlens[i], n_regs[i] = 0, regs[i] = 0;
+
+	if (qlen_sum == 0 || n_segs <= 0 || n_segs > MM_MAX_SEG) return;
+	if (opt->max_qlen > 0 && qlen_sum > opt->max_qlen) return;
+
+	hash  = qname && !(opt->flag & MM_F_NO_HASH_NAME)? __ac_X31_hash_string(qname) : 0;
+	hash ^= __ac_Wang_hash(qlen_sum) + __ac_Wang_hash(opt->seed);
+	hash  = __ac_Wang_hash(hash);
+
+	collect_minimizers(b->km, opt, mi, n_segs, qlens, seqs, &mv);
+	if (opt->q_occ_frac > 0.0f) mm_seed_mz_flt(b->km, &mv, opt->mid_occ, opt->q_occ_frac);
+	if (opt->flag & MM_F_HEAP_SORT) a = collect_seed_hits_heap(b->km, opt, opt->mid_occ, mi, qname, &mv, qlen_sum, &n_a, &rep_len, &n_mini_pos, &mini_pos);
+	else a = collect_seed_hits(b->km, opt, opt->mid_occ, mi, qname, &mv, qlen_sum, &n_a, &rep_len, &n_mini_pos, &mini_pos);
+
+	if (mm_dbg_flag & MM_DBG_PRINT_SEED) {
+		fprintf(stderr, "RS\t%d\n", rep_len);
+		for (i = 0; i < n_a; ++i)
+			fprintf(stderr, "SD\t%s\t%d\t%c\t%d\t%d\t%d\n", mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
+					i == 0? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
+	}
+
+	// set max chaining gap on the query and the reference sequence
+	if (is_sr)
+		max_chain_gap_qry = qlen_sum > opt->max_gap? qlen_sum : opt->max_gap;
+	else max_chain_gap_qry = opt->max_gap;
+	if (opt->max_gap_ref > 0) {
+		max_chain_gap_ref = opt->max_gap_ref; // always honor mm_mapopt_t::max_gap_ref if set
+	} else if (opt->max_frag_len > 0) {
+		max_chain_gap_ref = opt->max_frag_len - qlen_sum;
+		if (max_chain_gap_ref < opt->max_gap) max_chain_gap_ref = opt->max_gap;
+	} else max_chain_gap_ref = opt->max_gap;
+
+	chn_pen_gap  = opt->chain_gap_scale * 0.01 * mi->k;
+	chn_pen_skip = opt->chain_skip_scale * 0.01 * mi->k;
+	if (opt->flag & MM_F_RMQ) {
+		a = mg_lchain_rmq(opt->max_gap, opt->rmq_inner_dist, opt->bw, opt->max_chain_skip, opt->rmq_size_cap, opt->min_cnt, opt->min_chain_score,
+						  chn_pen_gap, chn_pen_skip, n_a, a, &n_regs0, &u, b->km);
+	} else {
+		a = mg_lchain_dp(max_chain_gap_ref, max_chain_gap_qry, opt->bw, opt->max_chain_skip, opt->max_chain_iter, opt->min_cnt, opt->min_chain_score,
+						 chn_pen_gap, chn_pen_skip, is_splice, n_segs, n_a, a, &n_regs0, &u, b->km);
+	}
+
+	if (opt->bw_long > opt->bw && (opt->flag & (MM_F_SPLICE|MM_F_SR|MM_F_NO_LJOIN)) == 0 && n_segs == 1 && n_regs0 > 1) { // re-chain/long-join for long sequences
+		int32_t st = (int32_t)a[0].y, en = (int32_t)a[(int32_t)u[0] - 1].y;
+		if (qlen_sum - (en - st) > opt->rmq_rescue_size || en - st > qlen_sum * opt->rmq_rescue_ratio) {
+			int32_t i;
+			for (i = 0, n_a = 0; i < n_regs0; ++i) n_a += (int32_t)u[i];
+			kfree(b->km, u);
+			radix_sort_128x(a, a + n_a);
+			a = mg_lchain_rmq(opt->max_gap, opt->rmq_inner_dist, opt->bw_long, opt->max_chain_skip, opt->rmq_size_cap, opt->min_cnt, opt->min_chain_score,
+							  chn_pen_gap, chn_pen_skip, n_a, a, &n_regs0, &u, b->km);
+		}
+	} else if (opt->max_occ > opt->mid_occ && rep_len > 0 && !(opt->flag & MM_F_RMQ)) { // re-chain, mostly for short reads
+		int rechain = 0;
+		if (n_regs0 > 0) { // test if the best chain has all the segments
+			int n_chained_segs = 1, max = 0, max_i = -1, max_off = -1, off = 0;
+			for (i = 0; i < n_regs0; ++i) { // find the best chain
+				if (max < (int)(u[i]>>32)) max = u[i]>>32, max_i = i, max_off = off;
+				off += (uint32_t)u[i];
+			}
+			for (i = 1; i < (int32_t)u[max_i]; ++i) // count the number of segments in the best chain
+				if ((a[max_off+i].y&MM_SEED_SEG_MASK) != (a[max_off+i-1].y&MM_SEED_SEG_MASK))
+					++n_chained_segs;
+			if (n_chained_segs < n_segs)
+				rechain = 1;
+		} else rechain = 1;
+		if (rechain) { // redo chaining with a higher max_occ threshold
+			kfree(b->km, a);
+			kfree(b->km, u);
+			kfree(b->km, mini_pos);
+			if (opt->flag & MM_F_HEAP_SORT) a = collect_seed_hits_heap(b->km, opt, opt->max_occ, mi, qname, &mv, qlen_sum, &n_a, &rep_len, &n_mini_pos, &mini_pos);
+			else a = collect_seed_hits(b->km, opt, opt->max_occ, mi, qname, &mv, qlen_sum, &n_a, &rep_len, &n_mini_pos, &mini_pos);
+			a = mg_lchain_dp(max_chain_gap_ref, max_chain_gap_qry, opt->bw, opt->max_chain_skip, opt->max_chain_iter, opt->min_cnt, opt->min_chain_score,
+							 chn_pen_gap, chn_pen_skip, is_splice, n_segs, n_a, a, &n_regs0, &u, b->km);
+		}
+	}
+	b->frag_gap = max_chain_gap_ref;
+	b->rep_len = rep_len;
+
+	regs0 = mm_gen_regs(b->km, hash, qlen_sum, n_regs0, u, a, !!(opt->flag&MM_F_QSTRAND));
+	if (mi->n_alt) {
+		mm_mark_alt(mi, n_regs0, regs0);
+		mm_hit_sort(b->km, &n_regs0, regs0, opt->alt_drop); // this step can be merged into mm_gen_regs(); will do if this shows up in profile
+	}
+
+	if (mm_dbg_flag & (MM_DBG_PRINT_SEED|MM_DBG_PRINT_CHAIN))
+		for (j = 0; j < n_regs0; ++j)
+			for (i = regs0[j].as; i < regs0[j].as + regs0[j].cnt; ++i)
+				fprintf(stderr, "CN\t%d\t%s\t%d\t%c\t%d\t%d\t%d\n", j, mi->seq[a[i].x<<1>>33].name, (int32_t)a[i].x, "+-"[a[i].x>>63], (int32_t)a[i].y, (int32_t)(a[i].y>>32&0xff),
+						i == regs0[j].as? 0 : ((int32_t)a[i].y - (int32_t)a[i-1].y) - ((int32_t)a[i].x - (int32_t)a[i-1].x));
+
+	chain_post(opt, max_chain_gap_ref, mi, b->km, qlen_sum, n_segs, qlens, &n_regs0, regs0, a);
+	if (!is_sr && !(opt->flag&MM_F_QSTRAND)) {
+		mm_est_err(mi, qlen_sum, n_regs0, regs0, a, n_mini_pos, mini_pos);
+		n_regs0 = mm_filter_strand_retained(n_regs0, regs0);
+	}
+
+	if (n_segs == 1) { // uni-segment
+		regs0 = align_regs(opt, mi, b->km, qlens[0], seqs[0], &n_regs0, regs0, a);
+		regs0 = (mm_reg1_t*)realloc(regs0, sizeof(*regs0) * n_regs0);
+		mm_set_mapq(b->km, n_regs0, regs0, opt->min_chain_score, opt->a, rep_len, is_sr);
+		n_regs[0] = n_regs0, regs[0] = regs0;
+	} else { // multi-segment
+		mm_seg_t *seg;
+		seg = mm_seg_gen(b->km, hash, n_segs, qlens, n_regs0, regs0, n_regs, regs, a); // split fragment chain to separate segment chains
+		free(regs0);
+		for (i = 0; i < n_segs; ++i) {
+			mm_set_parent(b->km, opt->mask_level, opt->mask_len, n_regs[i], regs[i], opt->a * 2 + opt->b, opt->flag&MM_F_HARD_MLEVEL, opt->alt_drop); // update mm_reg1_t::parent
+			regs[i] = align_regs(opt, mi, b->km, qlens[i], seqs[i], &n_regs[i], regs[i], seg[i].a);
+			mm_set_mapq(b->km, n_regs[i], regs[i], opt->min_chain_score, opt->a, rep_len, is_sr);
+		}
+		mm_seg_free(b->km, n_segs, seg);
+		if (n_segs == 2 && opt->pe_ori >= 0 && (opt->flag&MM_F_CIGAR))
+			mm_pair(b->km, max_chain_gap_ref, opt->pe_bonus, opt->a * 2 + opt->b, opt->a, qlens, n_regs, regs); // pairing
+	}
+
+	kfree(b->km, mv.a);
+	kfree(b->km, a);
+	kfree(b->km, u);
+	kfree(b->km, mini_pos);
+
+	if (b->km) {
+		km_stat(b->km, &kmst);
+		if (mm_dbg_flag & MM_DBG_PRINT_QNAME)
+			fprintf(stderr, "QM\t%s\t%d\tcap=%ld,nCore=%ld,largest=%ld\n", qname, qlen_sum, kmst.capacity, kmst.n_cores, kmst.largest);
+		assert(kmst.n_blocks == kmst.n_cores); // otherwise, there is a memory leak
+		if (kmst.largest > 1U<<28 || (opt->cap_kalloc > 0 && kmst.capacity > opt->cap_kalloc)) {
+			if (mm_dbg_flag & MM_DBG_PRINT_QNAME)
+				fprintf(stderr, "[W::%s] reset thread-local memory after read %s\n", __func__, qname);
+			km_destroy(b->km);
+			b->km = km_init();
+		}
+	}
+}
+
+mm_reg1_t *mm_map(const mm_idx_t *mi, int qlen, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname)
 {
 	mm_reg1_t *regs;
-	b->mini.n = 0;
+	mm_map_frag(mi, 1, &qlen, &seq, n_regs, &regs, b, opt, qname);
-	mm_sketch(b->km, seq, l_seq, mi->w, mi->k, 0, mi->is_hpc, &b->mini);
-	if (opt->sdust_thres > 0)
-		mm_dust_minier(&b->mini, l_seq, seq, opt->sdust_thres, b->sdb);
-	regs = mm_map_frag(opt, mi, b, 0, b->mini.n, qname, l_seq, seq, n_regs);
 	return regs;
 }
 
@@ -279,39 +390,151 @@ mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, m
  **************************/
 
 typedef struct {
-	int mini_batch_size, n_processed, n_threads;
+	int n_processed, n_threads, n_fp;
+	int64_t mini_batch_size;
 	const mm_mapopt_t *opt;
-	mm_bseq_file_t *fp;
+	mm_bseq_file_t **fp;
 	const mm_idx_t *mi;
 	kstring_t str;
+
+	int n_parts;
+	uint32_t *rid_shift;
+	FILE *fp_split, **fp_parts;
 } pipeline_t;
 
 typedef struct {
 	const pipeline_t *p;
-    int n_seq;
+    int n_seq, n_frag;
 	mm_bseq1_t *seq;
-	int *n_reg;
+	int *n_reg, *seg_off, *n_seg, *rep_len, *frag_gap;
 	mm_reg1_t **reg;
 	mm_tbuf_t **buf;
 } step_t;
 
 static void worker_for(void *_data, long i, int tid) // kt_for() callback
 {
-    step_t *step = (step_t*)_data;
+    step_t *s = (step_t*)_data;
+	int qlens[MM_MAX_SEG], j, off = s->seg_off[i], pe_ori = s->p->opt->pe_ori;
+	const char *qseqs[MM_MAX_SEG];
+	double t = 0.0;
+	mm_tbuf_t *b = s->buf[tid];
+	assert(s->n_seg[i] <= MM_MAX_SEG);
+	if (mm_dbg_flag & MM_DBG_PRINT_QNAME) {
+		fprintf(stderr, "QR\t%s\t%d\t%d\n", s->seq[off].name, tid, s->seq[off].l_seq);
+		t = realtime();
+	}
+	for (j = 0; j < s->n_seg[i]; ++j) {
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1))))
+			mm_revcomp_bseq(&s->seq[off + j]);
+		qlens[j] = s->seq[off + j].l_seq;
+		qseqs[j] = s->seq[off + j].seq;
+	}
+	if (s->p->opt->flag & MM_F_INDEPEND_SEG) {
+		for (j = 0; j < s->n_seg[i]; ++j) {
+			mm_map_frag(s->p->mi, 1, &qlens[j], &qseqs[j], &s->n_reg[off+j], &s->reg[off+j], b, s->p->opt, s->seq[off+j].name);
+			s->rep_len[off + j] = b->rep_len;
+			s->frag_gap[off + j] = b->frag_gap;
+		}
+	} else {
+		mm_map_frag(s->p->mi, s->n_seg[i], qlens, qseqs, &s->n_reg[off], &s->reg[off], b, s->p->opt, s->seq[off].name);
+		for (j = 0; j < s->n_seg[i]; ++j) {
+			s->rep_len[off + j] = b->rep_len;
+			s->frag_gap[off + j] = b->frag_gap;
+		}
+	}
+	for (j = 0; j < s->n_seg[i]; ++j) // flip the query strand and coordinate to the original read strand
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1)))) {
+			int k, t;
+			mm_revcomp_bseq(&s->seq[off + j]);
+			for (k = 0; k < s->n_reg[off + j]; ++k) {
+				mm_reg1_t *r = &s->reg[off + j][k];
+				t = r->qs;
+				r->qs = qlens[j] - r->qe;
+				r->qe = qlens[j] - t;
+				r->rev = !r->rev;
+			}
+		}
 	if (mm_dbg_flag & MM_DBG_PRINT_QNAME)
-		fprintf(stderr, "QR\t%s\t%d\n", step->seq[i].name, tid);
+		fprintf(stderr, "QT\t%s\t%d\t%.6f\n", s->seq[off].name, tid, realtime() - t);
-	step->reg[i] = mm_map(step->p->mi, step->seq[i].l_seq, step->seq[i].seq, &step->n_reg[i], step->buf[tid], step->p->opt, step->seq[i].name);
+}
+
+static void merge_hits(step_t *s)
+{
+	int f, i, k0, k, max_seg = 0, *n_reg_part, *rep_len_part, *frag_gap_part, *qlens;
+	void *km;
+	FILE **fp = s->p->fp_parts;
+	const mm_mapopt_t *opt = s->p->opt;
+
+	km = km_init();
+	for (f = 0; f < s->n_frag; ++f)
+		max_seg = max_seg > s->n_seg[f]? max_seg : s->n_seg[f];
+	qlens = CALLOC(int, max_seg + s->p->n_parts * 3);
+	n_reg_part = qlens + max_seg;
+	rep_len_part = n_reg_part + s->p->n_parts;
+	frag_gap_part = rep_len_part + s->p->n_parts;
+	for (f = 0, k = k0 = 0; f < s->n_frag; ++f) {
+		k0 = k;
+		for (i = 0; i < s->n_seg[f]; ++i, ++k) {
+			int j, l, t, rep_len = 0;
+			qlens[i] = s->seq[k].l_seq;
+			for (j = 0, s->n_reg[k] = 0; j < s->p->n_parts; ++j) {
+				mm_err_fread(&n_reg_part[j],    sizeof(int), 1, fp[j]);
+				mm_err_fread(&rep_len_part[j],  sizeof(int), 1, fp[j]);
+				mm_err_fread(&frag_gap_part[j], sizeof(int), 1, fp[j]);
+				s->n_reg[k] += n_reg_part[j];
+				if (rep_len < rep_len_part[j])
+					rep_len = rep_len_part[j];
+			}
+			s->reg[k] = CALLOC(mm_reg1_t, s->n_reg[k]);
+			for (j = 0, l = 0; j < s->p->n_parts; ++j) {
+				for (t = 0; t < n_reg_part[j]; ++t, ++l) {
+					mm_reg1_t *r = &s->reg[k][l];
+					uint32_t capacity;
+					mm_err_fread(r, sizeof(mm_reg1_t), 1, fp[j]);
+					r->rid += s->p->rid_shift[j];
+					if (opt->flag & MM_F_CIGAR) {
+						mm_err_fread(&capacity, 4, 1, fp[j]);
+						r->p = (mm_extra_t*)calloc(capacity, 4);
+						r->p->capacity = capacity;
+						mm_err_fread(r->p, r->p->capacity, 4, fp[j]);
+					}
+				}
+			}
+			if (!(opt->flag&MM_F_SR) && s->seq[k].l_seq >= opt->rank_min_len)
+				mm_update_dp_max(s->seq[k].l_seq, s->n_reg[k], s->reg[k], opt->rank_frac, opt->a, opt->b);
+			for (j = 0; j < s->n_reg[k]; ++j) {
+				mm_reg1_t *r = &s->reg[k][j];
+				if (r->p) r->p->dp_max2 = 0; // reset ->dp_max2 as mm_set_parent() doesn't clear it; necessary with mm_update_dp_max()
+				r->subsc = 0; // this may not be necessary
+				r->n_sub = 0; // n_sub will be an underestimate as we don't see all the chains now, but it can't be accurate anyway
+			}
+			mm_hit_sort(km, &s->n_reg[k], s->reg[k], opt->alt_drop);
+			mm_set_parent(km, opt->mask_level, opt->mask_len, s->n_reg[k], s->reg[k], opt->a * 2 + opt->b, opt->flag&MM_F_HARD_MLEVEL, opt->alt_drop);
+			if (!(opt->flag & MM_F_ALL_CHAINS)) {
+				mm_select_sub(km, opt->pri_ratio, s->p->mi->k*2, opt->best_n, 0, opt->max_gap * 0.8, &s->n_reg[k], s->reg[k]);
+				mm_set_sam_pri(s->n_reg[k], s->reg[k]);
+			}
+			mm_set_mapq(km, s->n_reg[k], s->reg[k], opt->min_chain_score, opt->a, rep_len, !!(opt->flag & MM_F_SR));
+		}
+		if (s->n_seg[f] == 2 && opt->pe_ori >= 0 && (opt->flag&MM_F_CIGAR))
+			mm_pair(km, frag_gap_part[0], opt->pe_bonus, opt->a * 2 + opt->b, opt->a, qlens, &s->n_reg[k0], &s->reg[k0]);
+	}
+	free(qlens);
+	km_destroy(km);
 }
 
 static void *worker_pipeline(void *shared, int step, void *in)
 {
-	int i, j;
+	int i, j, k;
     pipeline_t *p = (pipeline_t*)shared;
     if (step == 0) { // step 0: read sequences
 		int with_qual = (!!(p->opt->flag & MM_F_OUT_SAM) && !(p->opt->flag & MM_F_NO_QUAL));
+		int with_comment = !!(p->opt->flag & MM_F_COPY_COMMENT);
+		int frag_mode = (p->n_fp > 1 || !!(p->opt->flag & MM_F_FRAG_MODE));
         step_t *s;
         s = (step_t*)calloc(1, sizeof(step_t));
-		s->seq = mm_bseq_read(p->fp, p->mini_batch_size, with_qual, &s->n_seq);
+		if (p->n_fp > 1) s->seq = mm_bseq_read_frag2(p->n_fp, p->fp, p->mini_batch_size, with_qual, with_comment, &s->n_seq);
+		else s->seq = mm_bseq_read3(p->fp[0], p->mini_batch_size, with_qual, with_comment, frag_mode, &s->n_seq);
 		if (s->seq) {
 			s->p = p;
 			for (i = 0; i < s->n_seq; ++i)
@@ -319,36 +542,77 @@ static void *worker_pipeline(void *shared, int step, void *in)
 			s->buf = (mm_tbuf_t**)calloc(p->n_threads, sizeof(mm_tbuf_t*));
 			for (i = 0; i < p->n_threads; ++i)
 				s->buf[i] = mm_tbuf_init();
-			s->n_reg = (int*)calloc(s->n_seq, sizeof(int));
+			s->n_reg = (int*)calloc(5 * s->n_seq, sizeof(int));
+			s->seg_off = s->n_reg + s->n_seq; // seg_off, n_seg, rep_len and frag_gap are allocated together with n_reg
+			s->n_seg = s->seg_off + s->n_seq;
+			s->rep_len = s->n_seg + s->n_seq;
+			s->frag_gap = s->rep_len + s->n_seq;
 			s->reg = (mm_reg1_t**)calloc(s->n_seq, sizeof(mm_reg1_t*));
+			for (i = 1, j = 0; i <= s->n_seq; ++i)
+				if (i == s->n_seq || !frag_mode || !mm_qname_same(s->seq[i-1].name, s->seq[i].name)) {
+					s->n_seg[s->n_frag] = i - j;
+					s->seg_off[s->n_frag++] = j;
+					j = i;
+				}
 			return s;
 		} else free(s);
     } else if (step == 1) { // step 1: map
-		kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_seq);
+		if (p->n_parts > 0) merge_hits((step_t*)in);
+		else kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_frag);
 		return in;
     } else if (step == 2) { // step 2: output
+		void *km = 0;
         step_t *s = (step_t*)in;
 		const mm_idx_t *mi = p->mi;
 		for (i = 0; i < p->n_threads; ++i) mm_tbuf_destroy(s->buf[i]);
 		free(s->buf);
-		for (i = 0; i < s->n_seq; ++i) {
+		if ((p->opt->flag & MM_F_OUT_CS) && !(mm_dbg_flag & MM_DBG_NO_KALLOC)) km = km_init();
-			mm_bseq1_t *t = &s->seq[i];
+		for (k = 0; k < s->n_frag; ++k) {
-			for (j = 0; j < s->n_reg[i]; ++j) {
+			int seg_st = s->seg_off[k], seg_en = s->seg_off[k] + s->n_seg[k];
-				mm_reg1_t *r = &s->reg[i][j];
+			for (i = seg_st; i < seg_en; ++i) {
-				if (p->opt->flag & MM_F_OUT_SAM) mm_write_sam(&p->str, mi, t, r);
+				mm_bseq1_t *t = &s->seq[i];
-				else mm_write_paf(&p->str, mi, t, r);
+				if (p->opt->split_prefix && p->n_parts == 0) { // then write to temporary files
-				puts(p->str.s);
+					mm_err_fwrite(&s->n_reg[i],    sizeof(int), 1, p->fp_split);
-				free(r->p);
+					mm_err_fwrite(&s->rep_len[i],  sizeof(int), 1, p->fp_split);
+					mm_err_fwrite(&s->frag_gap[i], sizeof(int), 1, p->fp_split);
+					for (j = 0; j < s->n_reg[i]; ++j) {
+						mm_reg1_t *r = &s->reg[i][j];
+						mm_err_fwrite(r, sizeof(mm_reg1_t), 1, p->fp_split);
+						if (p->opt->flag & MM_F_CIGAR) {
+							mm_err_fwrite(&r->p->capacity, 4, 1, p->fp_split);
+							mm_err_fwrite(r->p, r->p->capacity, 4, p->fp_split);
+						}
+					}
+				} else if (s->n_reg[i] > 0) { // the query has at least one hit
+					for (j = 0; j < s->n_reg[i]; ++j) {
+						mm_reg1_t *r = &s->reg[i][j];
+						assert(!r->sam_pri || r->id == r->parent);
+						if ((p->opt->flag & MM_F_NO_PRINT_2ND) && r->id != r->parent)
+							continue;
+						if (p->opt->flag & MM_F_OUT_SAM)
+							mm_write_sam3(&p->str, mi, t, i - seg_st, j, s->n_seg[k], &s->n_reg[seg_st], (const mm_reg1_t*const*)&s->reg[seg_st], km, p->opt->flag, s->rep_len[i]);
+						else
+							mm_write_paf3(&p->str, mi, t, r, km, p->opt->flag, s->rep_len[i]);
+						mm_err_puts(p->str.s);
+					}
+				} else if ((p->opt->flag & MM_F_PAF_NO_HIT) || ((p->opt->flag & MM_F_OUT_SAM) && !(p->opt->flag & MM_F_SAM_HIT_ONLY))) { // output an empty hit, if requested
+					if (p->opt->flag & MM_F_OUT_SAM)
+						mm_write_sam3(&p->str, mi, t, i - seg_st, -1, s->n_seg[k], &s->n_reg[seg_st], (const mm_reg1_t*const*)&s->reg[seg_st], km, p->opt->flag, s->rep_len[i]);
+					else
+						mm_write_paf3(&p->str, mi, t, 0, 0, p->opt->flag, s->rep_len[i]);
+					mm_err_puts(p->str.s);
+				}
 			}
-			if (s->n_reg[i] == 0 && (p->opt->flag & MM_F_OUT_SAM)) {
+			for (i = seg_st; i < seg_en; ++i) {
-				mm_write_sam(&p->str, 0, t, 0);
+				for (j = 0; j < s->n_reg[i]; ++j) free(s->reg[i][j].p);
-				puts(p->str.s);
+				free(s->reg[i]);
+				free(s->seq[i].seq); free(s->seq[i].name);
+				if (s->seq[i].qual) free(s->seq[i].qual);
+				if (s->seq[i].comment) free(s->seq[i].comment);
 			}
-			free(s->reg[i]);
-			free(s->seq[i].seq); free(s->seq[i].name);
-			if (s->seq[i].qual) free(s->seq[i].qual);
 		}
-		free(s->reg); free(s->n_reg); free(s->seq);
+		free(s->reg); free(s->n_reg); free(s->seq); // seg_off, n_seg, rep_len and frag_gap were allocated with reg; no memory leak here
+		km_destroy(km);
 		if (mm_verbose >= 3)
 			fprintf(stderr, "[M::%s::%.3f*%.2f] mapped %d sequences\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), s->n_seq);
 		free(s);
@@ -356,21 +620,95 @@ static void *worker_pipeline(void *shared, int step, void *in)
     return 0;
 }
 
-int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads, int mini_batch_size)
+static mm_bseq_file_t **open_bseqs(int n, const char **fn)
 {
+	mm_bseq_file_t **fp;
+	int i, j;
+	fp = (mm_bseq_file_t**)calloc(n, sizeof(mm_bseq_file_t*));
+	for (i = 0; i < n; ++i) {
+		if ((fp[i] = mm_bseq_open(fn[i])) == 0) {
+			if (mm_verbose >= 1)
+				fprintf(stderr, "ERROR: failed to open file '%s': %s\n", fn[i], strerror(errno));
+			for (j = 0; j < i; ++j)
+				mm_bseq_close(fp[j]);
+			free(fp);
+			return 0;
+		}
+	}
+	return fp;
+}
+
+int mm_map_file_frag(const mm_idx_t *idx, int n_segs, const char **fn, const mm_mapopt_t *opt, int n_threads)
+{
+	int i, pl_threads;
 	pipeline_t pl;
+	if (n_segs < 1) return -1;
 	memset(&pl, 0, sizeof(pipeline_t));
-	pl.fp = mm_bseq_open(fn);
+	pl.n_fp = n_segs;
+	pl.fp = open_bseqs(pl.n_fp, fn);
 	if (pl.fp == 0) return -1;
 	pl.opt = opt, pl.mi = idx;
-	pl.n_threads = n_threads, pl.mini_batch_size = mini_batch_size;
+	pl.n_threads = n_threads > 1? n_threads : 1;
-	if (opt->flag & MM_F_OUT_SAM) {
+	pl.mini_batch_size = opt->mini_batch_size;
-		uint32_t i;
+	if (opt->split_prefix)
-		for (i = 0; i < idx->n_seq; ++i)
+		pl.fp_split = mm_split_init(opt->split_prefix, idx);
-			printf("@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len);
+	pl_threads = n_threads == 1? 1 : (opt->flag&MM_F_2_IO_THREADS)? 3 : 2;
-	}
+	kt_pipeline(pl_threads, worker_pipeline, &pl, 3);
-	kt_pipeline(n_threads == 1? 1 : 2, worker_pipeline, &pl, 3);
+
 	free(pl.str.s);
-	mm_bseq_close(pl.fp);
+	if (pl.fp_split) fclose(pl.fp_split);
+	for (i = 0; i < pl.n_fp; ++i)
+		mm_bseq_close(pl.fp[i]);
+	free(pl.fp);
+	return 0;
+}
+
+int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads)
+{
+	return mm_map_file_frag(idx, 1, &fn, opt, n_threads);
+}
+
+int mm_split_merge(int n_segs, const char **fn, const mm_mapopt_t *opt, int n_split_idx)
+{
+	int i;
+	pipeline_t pl;
+	mm_idx_t *mi;
+	if (n_segs < 1 || n_split_idx < 1) return -1;
+	memset(&pl, 0, sizeof(pipeline_t));
+	pl.n_fp = n_segs;
+	pl.fp = open_bseqs(pl.n_fp, fn);
+	if (pl.fp == 0) return -1;
+	pl.opt = opt;
+	pl.mini_batch_size = opt->mini_batch_size;
+
+	pl.n_parts = n_split_idx;
+	pl.fp_parts  = CALLOC(FILE*, pl.n_parts);
+	pl.rid_shift = CALLOC(uint32_t, pl.n_parts);
+	pl.mi = mi = mm_split_merge_prep(opt->split_prefix, n_split_idx, pl.fp_parts, pl.rid_shift);
+	if (pl.mi == 0) {
+		free(pl.fp_parts);
+		free(pl.rid_shift);
+		return -1;
+	}
+	for (i = n_split_idx - 1; i > 0; --i)
+		pl.rid_shift[i] = pl.rid_shift[i - 1];
+	for (pl.rid_shift[0] = 0, i = 1; i < n_split_idx; ++i)
+		pl.rid_shift[i] += pl.rid_shift[i - 1];
+	if (opt->flag & MM_F_OUT_SAM)
+		for (i = 0; i < (int32_t)pl.mi->n_seq; ++i)
+			printf("@SQ\tSN:%s\tLN:%d\n", pl.mi->seq[i].name, pl.mi->seq[i].len);
+
+	kt_pipeline(2, worker_pipeline, &pl, 3);
+
+	free(pl.str.s);
+	mm_idx_destroy(mi);
+	free(pl.rid_shift);
+	for (i = 0; i < n_split_idx; ++i)
+		fclose(pl.fp_parts[i]);
+	free(pl.fp_parts);
+	for (i = 0; i < pl.n_fp; ++i)
+		mm_bseq_close(pl.fp[i]);
+	free(pl.fp);
+	mm_split_rm_tmp(opt->split_prefix, n_split_idx);
 	return 0;
 }

minimap.h

@@ -5,137 +5,403 @@
 #include <stdio.h>
 #include <sys/types.h>
 
-#define MM_IDX_DEF_B    14
+#define MM_F_NO_DIAG       0x001 // no exact diagonal hit
+#define MM_F_NO_DUAL       0x002 // skip pairs where query name is lexicographically larger than target name
+#define MM_F_CIGAR         0x004
+#define MM_F_OUT_SAM       0x008
+#define MM_F_NO_QUAL       0x010
+#define MM_F_OUT_CG        0x020
+#define MM_F_OUT_CS        0x040
+#define MM_F_SPLICE        0x080 // splice mode
+#define MM_F_SPLICE_FOR    0x100 // match GT-AG
+#define MM_F_SPLICE_REV    0x200 // match CT-AC, the reverse complement of GT-AG
+#define MM_F_NO_LJOIN      0x400
+#define MM_F_OUT_CS_LONG   0x800
+#define MM_F_SR            0x1000
+#define MM_F_FRAG_MODE     0x2000
+#define MM_F_NO_PRINT_2ND  0x4000
+#define MM_F_2_IO_THREADS  0x8000
+#define MM_F_LONG_CIGAR    0x10000
+#define MM_F_INDEPEND_SEG  0x20000
+#define MM_F_SPLICE_FLANK  0x40000
+#define MM_F_SOFTCLIP      0x80000
+#define MM_F_FOR_ONLY      0x100000
+#define MM_F_REV_ONLY      0x200000
+#define MM_F_HEAP_SORT     0x400000
+#define MM_F_ALL_CHAINS    0x800000
+#define MM_F_OUT_MD        0x1000000
+#define MM_F_COPY_COMMENT  0x2000000
+#define MM_F_EQX           0x4000000 // use =/X instead of M
+#define MM_F_PAF_NO_HIT    0x8000000 // output unmapped reads to PAF
+#define MM_F_NO_END_FLT    0x10000000
+#define MM_F_HARD_MLEVEL   0x20000000
+#define MM_F_SAM_HIT_ONLY  0x40000000
+#define MM_F_RMQ           (0x80000000LL)
+#define MM_F_QSTRAND       (0x100000000LL)
+#define MM_F_NO_INV        (0x200000000LL)
+#define MM_F_NO_HASH_NAME  (0x400000000LL)
 
-#define MM_F_NO_SELF   0x01
+#define MM_I_HPC          0x1
-#define MM_F_AVA       0x02
+#define MM_I_NO_SEQ       0x2
-#define MM_F_CIGAR     0x04
+#define MM_I_NO_NAME      0x4
-#define MM_F_OUT_SAM   0x08
-#define MM_F_NO_QUAL   0x10
 
 #define MM_IDX_MAGIC   "MMI\2"
 
+#define MM_MAX_SEG       255
+
+#define MM_CIGAR_MATCH      0
+#define MM_CIGAR_INS        1
+#define MM_CIGAR_DEL        2
+#define MM_CIGAR_N_SKIP     3
+#define MM_CIGAR_SOFTCLIP   4
+#define MM_CIGAR_HARDCLIP   5
+#define MM_CIGAR_PADDING    6
+#define MM_CIGAR_EQ_MATCH   7
+#define MM_CIGAR_X_MISMATCH 8
+
+#define MM_CIGAR_STR  "MIDNSHP=XB"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct {
+// emulate 128-bit integers and arrays
- 	uint64_t x, y;
+typedef struct { uint64_t x, y; } mm128_t;
-} mm128_t;
-
 typedef struct { size_t n, m; mm128_t *a; } mm128_v;
-typedef struct { size_t n, m; uint64_t *a; } uint64_v;
-typedef struct { size_t n, m; uint32_t *a; } uint32_v;
-
-typedef struct {
-	mm128_v a;   // (minimizer, position) array
-	int32_t n;   // size of the _p_ array
-	uint64_t *p; // position array for minimizers appearing >1 times
-	void *h;     // hash table indexing _p_ and minimizers appearing once
-} mm_idx_bucket_t;
 
+// minimap2 index
 typedef struct {
 	char *name;      // name of the db sequence
 	uint64_t offset; // offset in mm_idx_t::S
 	uint32_t len;    // length
+	uint32_t is_alt;
 } mm_idx_seq_t;
 
 typedef struct {
-	int32_t b, w, k, is_hpc;
+	int32_t b, w, k, flag;
-	uint32_t n_seq;     // number of reference sequences
+	uint32_t n_seq;            // number of reference sequences
-	mm_idx_seq_t *seq;  // sequence name, length and offset
+	int32_t index;
-	uint32_t *S;        // 4-bit packed sequence
+	int32_t n_alt;
-	mm_idx_bucket_t *B; // index
+	mm_idx_seq_t *seq;         // sequence name, length and offset
+	uint32_t *S;               // 4-bit packed sequence
+	struct mm_idx_bucket_s *B; // index (hidden)
+	struct mm_idx_intv_s *I;   // intervals (hidden)
+	void *km, *h;
 } mm_idx_t;
 
+// minimap2 alignment
 typedef struct {
-	uint32_t capacity;
+	uint32_t capacity;                  // the capacity of cigar[]
-	int32_t dp_score, dp_max, dp_max2;
+	int32_t dp_score, dp_max, dp_max2;  // DP score; score of the max-scoring segment; score of the best alternate mappings
-	uint32_t blen;
+	uint32_t n_ambi:30, trans_strand:2; // number of ambiguous bases; transcript strand: 0 for unknown, 1 for +, 2 for -
-	uint32_t n_diff, n_ambi;
+	uint32_t n_cigar;                   // number of cigar operations in cigar[]
-	uint32_t n_cigar;
 	uint32_t cigar[];
 } mm_extra_t;
 
 typedef struct {
-	int32_t id;
+	int32_t id;             // ID for internal uses (see also parent below)
-	uint32_t cnt:31, rev:1;
+	int32_t cnt;            // number of minimizers; if on the reverse strand
-	uint32_t rid:31, rep:1;
+	int32_t rid;            // reference index; if this is an alignment from inversion rescue
-	int32_t score;
+	int32_t score;          // DP alignment score
-	int32_t qs, qe, rs, re;
+	int32_t qs, qe, rs, re; // query start and end; reference start and end
-	int32_t parent, subsc;
+	int32_t parent, subsc;  // parent==id if primary; best alternate mapping score
-	int32_t as;
+	int32_t as;             // offset in the a[] array (for internal uses only)
-	int32_t fuzzy_mlen, fuzzy_blen;
+	int32_t mlen, blen;     // seeded exact match length; seeded alignment block length
-	uint32_t mapq:8, split:2, sam_pri:1, n_sub:21; // TODO: n_sub is not used for now
+	int32_t n_sub;          // number of suboptimal mappings
+	int32_t score0;         // initial chaining score (before chain merging/spliting)
+	uint32_t mapq:8, split:2, rev:1, inv:1, sam_pri:1, proper_frag:1, pe_thru:1, seg_split:1, seg_id:8, split_inv:1, is_alt:1, strand_retained:1, dummy:5;
+	uint32_t hash;
+	float div;
 	mm_extra_t *p;
 } mm_reg1_t;
 
+// indexing and mapping options
 typedef struct {
-	float max_occ_frac;
+	short k, w, flag, bucket_bits;
-	float mid_occ_frac;
+	int64_t mini_batch_size;
-	int sdust_thres;  // score threshold for SDUST; 0 to disable
+	uint64_t batch_size;
-	int flag;    // see MM_F_* macros
+} mm_idxopt_t;
 
-	int bw;  // bandwidth
+typedef struct {
-	int max_gap; // break a chain if there are no minimizers in a max_gap window
+	int64_t flag;    // see MM_F_* macros
-	int max_chain_skip;
+	int seed;
-	int min_cnt;
+	int sdust_thres; // score threshold for SDUST; 0 to disable
-	int min_chain_score;
+
+	int max_qlen;    // max query length
+
+	int bw, bw_long; // bandwidth
+	int max_gap, max_gap_ref; // break a chain if there are no minimizers in a max_gap window
+	int max_frag_len;
+	int max_chain_skip, max_chain_iter;
+	int min_cnt;         // min number of minimizers on each chain
+	int min_chain_score; // min chaining score
+	float chain_gap_scale;
+	float chain_skip_scale;
+	int rmq_size_cap, rmq_inner_dist;
+	int rmq_rescue_size;
+	float rmq_rescue_ratio;
 
 	float mask_level;
+	int mask_len;
 	float pri_ratio;
-	int best_n;
+	int best_n;      // top best_n chains are subjected to DP alignment
 
-	int max_join_long, max_join_short;
+	float alt_drop;
-	int min_join_flank_sc;
 
 	int a, b, q, e, q2, e2; // matching score, mismatch, gap-open and gap-ext penalties
-	int zdrop;
+	int sc_ambi; // score when one or both bases are "N"
-	int min_dp_max;
+	int noncan;      // cost of non-canonical splicing sites
+	int junc_bonus;
+	int zdrop, zdrop_inv;   // break alignment if alignment score drops too fast along the diagonal
+	int end_bonus;
+	int min_dp_max;  // drop an alignment if the score of the max scoring segment is below this threshold
 	int min_ksw_len;
+	int anchor_ext_len, anchor_ext_shift;
+	float max_clip_ratio; // drop an alignment if BOTH ends are clipped above this ratio
 
-	int max_occ;
+	int rank_min_len;
-	int mid_occ;
+	float rank_frac;
+
+	int pe_ori, pe_bonus;
+
+	float mid_occ_frac;  // only used by mm_mapopt_update(); see below
+	float q_occ_frac;
+	int32_t min_mid_occ, max_mid_occ;
+	int32_t mid_occ;     // ignore seeds with occurrences above this threshold
+	int32_t max_occ, max_max_occ, occ_dist;
+	int64_t mini_batch_size; // size of a batch of query bases to process in parallel
+	int64_t max_sw_mat;
+	int64_t cap_kalloc;
+
+	const char *split_prefix;
 } mm_mapopt_t;
 
-extern int mm_verbose, mm_dbg_flag;
+// index reader
-extern double mm_realtime0;
+typedef struct {
+	int is_idx, n_parts;
+	int64_t idx_size;
+	mm_idxopt_t opt;
+	FILE *fp_out;
+	union {
+		struct mm_bseq_file_s *seq;
+		FILE *idx;
+	} fp;
+} mm_idx_reader_t;
 
-struct mm_tbuf_s;
+// memory buffer for thread-local storage during mapping
 typedef struct mm_tbuf_s mm_tbuf_t;
 
-struct mm_bseq_file_s;
+// global variables
+extern int mm_verbose, mm_dbg_flag; // verbose level: 0 for no info, 1 for error, 2 for warning, 3 for message (default); debugging flag
+extern double mm_realtime0; // wall-clock timer
 
-#define mm_seq4_set(s, i, c) ((s)[(i)>>3] |= (uint32_t)(c) << (((i)&7)<<2))
+/**
-#define mm_seq4_get(s, i)    ((s)[(i)>>3] >> (((i)&7)<<2) & 0xf)
+ * Set default or preset parameters
+ *
+ * @param preset     NULL to set all parameters as default; otherwise apply preset to affected parameters
+ * @param io         pointer to indexing parameters
+ * @param mo         pointer to mapping parameters
+ *
+ * @return 0 if success; -1 if _present_ unknown
+ */
+int mm_set_opt(const char *preset, mm_idxopt_t *io, mm_mapopt_t *mo);
+int mm_check_opt(const mm_idxopt_t *io, const mm_mapopt_t *mo);
 
-// compute minimizers
+/**
-void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p);
+ * Update mm_mapopt_t::mid_occ via mm_mapopt_t::mid_occ_frac
+ *
+ * If mm_mapopt_t::mid_occ is 0, this function sets it to a number such that no
+ * more than mm_mapopt_t::mid_occ_frac of minimizers in the index have a higher
+ * occurrence.
+ *
+ * @param opt        mapping parameters
+ * @param mi         minimap2 index
+ */
+void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi);
 
-// minimizer indexing
+void mm_mapopt_max_intron_len(mm_mapopt_t *opt, int max_intron_len);
-mm_idx_t *mm_idx_init(int w, int k, int b, int is_hpc);
-void mm_idx_destroy(mm_idx_t *mi);
-mm_idx_t *mm_idx_gen(struct mm_bseq_file_s *fp, int w, int k, int b, int is_hpc, int mini_batch_size, int n_threads, uint64_t batch_size, int keep_name);
-uint32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f);
-void mm_idx_stat(const mm_idx_t *idx);
-const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n);
-int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq);
 
-mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads);
+/**
-int mm_idx_is_idx(const char *fn);
+ * Initialize an index reader
+ *
+ * @param fn         index or fasta/fastq file name (this function tests the file type)
+ * @param opt        indexing parameters
+ * @param fn_out     if not NULL, write built index to this file
+ *
+ * @return an index reader on success; NULL if fail to open _fn_
+ */
+mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out);
 
-// minimizer index I/O
+/**
-void mm_idx_dump(FILE *fp, const mm_idx_t *mi);
+ * Read/build an index
+ *
+ * If the input file is an index file, this function reads one part of the
+ * index and returns. If the input file is a sequence file (fasta or fastq),
+ * this function constructs the index for about mm_idxopt_t::batch_size bases.
+ * Importantly, for a huge collection of sequences, this function may only
+ * return an index for part of sequences. It needs to be repeatedly called
+ * to traverse the entire index/sequence file.
+ *
+ * @param r          index reader
+ * @param n_threads  number of threads for constructing index
+ *
+ * @return an index on success; NULL if reaching the end of the input file
+ */
+mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads);
+
+/**
+ * Destroy/deallocate an index reader
+ *
+ * @param r          index reader
+ */
+void mm_idx_reader_close(mm_idx_reader_t *r);
+
+int mm_idx_reader_eof(const mm_idx_reader_t *r);
+
+/**
+ * Check whether the file contains a minimap2 index
+ *
+ * @param fn         file name
+ *
+ * @return the file size if fn is an index file; 0 if fn is not.
+ */
+int64_t mm_idx_is_idx(const char *fn);
+
+/**
+ * Load a part of an index
+ *
+ * Given a uni-part index, this function loads the entire index into memory.
+ * Given a multi-part index, it loads one part only and places the file pointer
+ * at the end of that part.
+ *
+ * @param fp         pointer to FILE object
+ *
+ * @return minimap2 index read from fp
+ */
 mm_idx_t *mm_idx_load(FILE *fp);
 
-// mapping
+/**
-void mm_mapopt_init(mm_mapopt_t *opt);
+ * Append an index (or one part of a full index) to file
-void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi);
+ *
+ * @param fp         pointer to FILE object
+ * @param mi         minimap2 index
+ */
+void mm_idx_dump(FILE *fp, const mm_idx_t *mi);
+
+/**
+ * Create an index from strings in memory
+ *
+ * @param w            minimizer window size
+ * @param k            minimizer k-mer size
+ * @param is_hpc       use HPC k-mer if true
+ * @param bucket_bits  number of bits for the first level of the hash table
+ * @param n            number of sequences
+ * @param seq          sequences in A/C/G/T
+ * @param name         sequence names; could be NULL
+ *
+ * @return minimap2 index
+ */
+mm_idx_t *mm_idx_str(int w, int k, int is_hpc, int bucket_bits, int n, const char **seq, const char **name);
+
+/**
+ * Print index statistics to stderr
+ *
+ * @param mi         minimap2 index
+ */
+void mm_idx_stat(const mm_idx_t *idx);
+
+/**
+ * Destroy/deallocate an index
+ *
+ * @param r          minimap2 index
+ */
+void mm_idx_destroy(mm_idx_t *mi);
+
+/**
+ * Initialize a thread-local buffer for mapping
+ *
+ * Each mapping thread requires a buffer specific to the thread (see mm_map()
+ * below). The primary purpose of this buffer is to reduce frequent heap
+ * allocations across threads. A buffer shall not be used by two or more
+ * threads.
+ *
+ * @return pointer to a thread-local buffer
+ */
 mm_tbuf_t *mm_tbuf_init(void);
+
+/**
+ * Destroy/deallocate a thread-local buffer for mapping
+ *
+ * @param b          the buffer
+ */
 void mm_tbuf_destroy(mm_tbuf_t *b);
+
+void *mm_tbuf_get_km(mm_tbuf_t *b);
+
+/**
+ * Align a query sequence against an index
+ *
+ * This function possibly finds multiple alignments of the query sequence.
+ * The returned array and the mm_reg1_t::p field of each element are allocated
+ * with malloc().
+ *
+ * @param mi         minimap2 index
+ * @param l_seq      length of the query sequence
+ * @param seq        the query sequence
+ * @param n_regs     number of hits (out)
+ * @param b          thread-local buffer; two mm_map() calls shall not use one buffer at the same time!
+ * @param opt        mapping parameters
+ * @param name       query name, used for all-vs-all overlapping and debugging
+ *
+ * @return an array of hits which need to be deallocated with free() together
+ *         with mm_reg1_t::p of each element. The size is written to _n_regs_.
+ */
 mm_reg1_t *mm_map(const mm_idx_t *mi, int l_seq, const char *seq, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *name);
 
-int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads, int tbatch_size);
+void mm_map_frag(const mm_idx_t *mi, int n_segs, const int *qlens, const char **seqs, int *n_regs, mm_reg1_t **regs, mm_tbuf_t *b, const mm_mapopt_t *opt, const char *qname);
+
+/**
+ * Align a fasta/fastq file and print alignments to stdout
+ *
+ * @param idx        minimap2 index
+ * @param fn         fasta/fastq file name
+ * @param opt        mapping parameters
+ * @param n_threads  number of threads
+ *
+ * @return 0 on success; -1 if _fn_ can't be read
+ */
+int mm_map_file(const mm_idx_t *idx, const char *fn, const mm_mapopt_t *opt, int n_threads);
+
+int mm_map_file_frag(const mm_idx_t *idx, int n_segs, const char **fn, const mm_mapopt_t *opt, int n_threads);
+
+/**
+ * Generate the cs tag (new in 2.12)
+ *
+ * @param km         memory blocks; set to NULL if unsure
+ * @param buf        buffer to write the cs/MD tag; typicall NULL on the first call
+ * @param max_len    max length of the buffer; typically set to 0 on the first call
+ * @param mi         index
+ * @param r          alignment
+ * @param seq        query sequence
+ * @param no_iden    true to use : instead of =
+ *
+ * @return the length of cs
+ */
+int mm_gen_cs(void *km, char **buf, int *max_len, const mm_idx_t *mi, const mm_reg1_t *r, const char *seq, int no_iden);
+int mm_gen_MD(void *km, char **buf, int *max_len, const mm_idx_t *mi, const mm_reg1_t *r, const char *seq);
+
+// query sequence name and sequence in the minimap2 index
+int mm_idx_index_name(mm_idx_t *mi);
+int mm_idx_name2id(const mm_idx_t *mi, const char *name);
+int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq);
+
+int mm_idx_alt_read(mm_idx_t *mi, const char *fn);
+int mm_idx_bed_read(mm_idx_t *mi, const char *fn, int read_junc);
+int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uint8_t *s);
+
+// deprecated APIs for backward compatibility
+void mm_mapopt_init(mm_mapopt_t *opt);
+mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads);
 
 #ifdef __cplusplus
 }

minimap2.1

@@ -1,4 +1,4 @@
-.TH minimap2 1 "19 July 2017" "minimap2-2.0-r190-dirty" "Bioinformatics tools"
+.TH minimap2 1 "18 December 2021" "minimap2-2.24 (r1122)" "Bioinformatics tools"
 .SH NAME
 .PP
 minimap2 - mapping and alignment between collections of DNA sequences
@@ -77,7 +77,7 @@ SAM format.
 Minimizer k-mer length [15]
 .TP
 .BI -w \ INT
-Minimizer window size [2/3 of k-mer length]. A minimizer is the smallest k-mer
+Minimizer window size [10]. A minimizer is the smallest k-mer
 in a window of w consecutive k-mers.
 .TP
 .B -H
@@ -99,6 +99,14 @@ multiple times to map it against each batch of target sequences.
 may be ending with k/K/m/M/g/G. NB: mapping quality is incorrect given a
 multi-part index.
 .TP
+.B --idx-no-seq
+Don't store target sequences in the index. It saves disk space and memory but
+the index generated with this option will not work with
+.B -a
+or
+.BR -c .
+When base-level alignment is not requested, this option is automatically applied.
+.TP
 .BI -d \ FILE
 Save the minimizer index of
 .I target.fa
@@ -113,21 +121,62 @@ provided as the target sequences, options
 .BR -w ,
 .B -I
 will be effectively overridden by the options stored in the index file.
+.TP
+.BI --alt \ FILE
+List of ALT contigs [null]
+.TP
+.BI --alt-drop \ FLOAT
+Drop ALT hits by
+.I FLOAT
+fraction when ranking and computing mapping quality [0.15]
 .SS Mapping options
 .TP 10
-.BI -f \ FLOAT
+.BI -f \ FLOAT | INT1 [, INT2 ]
-Ignore top
+If fraction, ignore top
 .I FLOAT
-fraction of most frequent minimizers [0.0002]
+fraction of most frequent minimizers [0.0002]. If integer,
+ignore minimizers occuring more than
+.I INT1
+times.
+.I INT2
+is only effective in the
+.B --sr
+or
+.B -xsr
+mode, which sets the threshold for a second round of seeding.
 .TP
-.BI -g \ INT
+.BI -U \ INT1 [, INT2 ]
-Stop chain enlongation if there are no minimizers in
+Lower and upper bounds of k-mer occurrences [10,1000000]. The final k-mer occurrence threshold is
-.IR INT -bp
+.RI max{ INT1 ,\ min{ INT2 ,
-[10000].
+.BR -f }}.
+This option prevents excessively small or large
+.B -f
+estimated from the input reference. Available since r1034 and deprecating
+.B --min-occ-floor
+in earlier versions of minimap2.
 .TP
-.BI -r \ INT
+.BI --q-occ-frac \ FLOAT
-Bandwidth used in chaining and DP-based alignment [1000]. This option
+Discard a query minimizer if its occurrence is higher than
-approximately controls the maximum gap size.
+.I FLOAT
+fraction of query minimizers and than the reference occurrence threshold
+[0.01]. Set 0 to disable. Available since r1105.
+.TP
+.BI -e \ INT
+Sample a high-frequency minimizer every
+.I INT
+basepairs [500].
+.TP
+.BI -g \ NUM
+Stop chain enlongation if there are no minimizers within
+.IR NUM -bp
+[10k].
+.TP
+.BI -r \ NUM1 [, NUM2 ]
+Bandwidth for chaining and base alignment [500,20k].
+.I NUM1
+is used for initial chaining and alignment extension;
+.I NUM2
+for RMQ-based re-chaining and closing gaps in alignments.
 .TP
 .BI -n \ INT
 Discard chains consisting of
@@ -137,25 +186,45 @@ number of minimizers [3]
 .BI -m \ INT
 Discard chains with chaining score
 .RI < INT
-[40]. Chaining score equals the approximate number of matching bases (exact if
+[40]. Chaining score equals the approximate number of matching bases minus a
-not using
+concave gap penalty. It is computed with dynamic programming.
-.BR -H )
+.TP
-minus base-2 logarithm gap penalty. It is computed with dynamic programming.
+.B -D
+If query sequence name/length are identical to the target name/length, ignore
+diagonal anchors. This option also reduces DP-based extension along the
+diagonal.
+.TP
+.B -P
+Retain all chains and don't attempt to set primary chains. Options
+.B -p
+and
+.B -N
+have no effect when this option is in use.
+.TP
+.BR --dual = yes | no
+If
+.BR no ,
+skip query-target pairs wherein the query name is lexicographically greater
+than the target name [yes]
 .TP
 .B -X
-Perform all-vs-all mapping. In this mode, if the query sequence name is
+Equivalent to
-lexicographically larger than the target sequence name, the hits between them
+.RB ' -DP
-will be suppressed; if the query sequence name is the same as the target name,
+.BR --dual = no
-diagonal minimizer hits will also be suppressed.
+.BR --no-long-join '.
+Primarily used for all-vs-all read overlapping.
 .TP
 .BI -p \ FLOAT
 Minimal secondary-to-primary score ratio to output secondary mappings [0.8].
-Between two chains overlaping over half of the shorter chain (controled by
+Between two chains overlaping over half of the shorter chain (controlled by
-.BR --mask-level ),
+.BR -M ),
 the chain with a lower score is secondary to the chain with a higher score.
 If the ratio of the scores is below
 .IR FLOAT ,
 the secondary chain will not be outputted or extended with DP alignment later.
+This option has no effect when
+.B -X
+is applied.
 .TP
 .BI -N \ INT
 Output at most
@@ -164,13 +233,94 @@ secondary alignments [5]. This option has no effect when
 .B -X
 is applied.
 .TP
+.BI -G \ NUM
+Maximum gap on the reference (effective with
+.BR -xsplice / --splice ).
+This option also changes the chaining and alignment band width to
+.IR NUM .
+Increasing this option slows down spliced alignment. [200k]
+.TP
+.BI -F \ NUM
+Maximum fragment length (aka insert size; effective with
+.BR -xsr / --frag = yes )
+[800]
+.TP
+.BI -M \ FLOAT
+Mark as secondary a chain that overlaps with a better chain by
+.I FLOAT
+or more of the shorter chain [0.5]
+.TP
+.BR --rmq = no | yes
+Use the minigraph chaining algorithm [no]. The minigraph algorithm is better
+for aligning contigs through long INDELs.
+.TP
+.B --hard-mask-level
+Honor option
+.B -M
+and disable a heurstic to save unmapped subsequences and disables
+.BR --mask-len .
+.TP
+.BI --mask-len \ NUM
+Keep an alignment if dropping it leaves an unaligned region on query longer than
+.IR INT
+[inf]. Effective without
+.BR --hard-mask-level .
+.TP
 .BI --max-chain-skip \ INT
-A heuristics that stops chaining early [50]. Minimap2 uses dynamic programming
+A heuristics that stops chaining early [25]. Minimap2 uses dynamic programming
 for chaining. The time complexity is quadratic in the number of seeds. This
 option makes minimap2 exits the inner loop if it repeatedly sees seeds already
 on chains. Set
 .I INT
 to a large number to switch off this heurstics.
+.TP
+.BI --max-chain-iter \ INT
+Check up to
+.I INT
+partial chains during chaining [5000]. This is a heuristic to avoid quadratic
+time complexity in the worst case.
+.TP
+.BI --chain-gap-scale \ FLOAT
+Scale of gap cost during chaining [1.0]
+.TP
+.B --no-long-join
+Disable the long gap patching heuristic. When this option is applied, the
+maximum alignment gap is mostly controlled by
+.BR -r .
+.TP
+.B --splice
+Enable the splice alignment mode.
+.TP
+.B --sr
+Enable short-read alignment heuristics. In the short-read mode, minimap2
+applies a second round of chaining with a higher minimizer occurrence threshold
+if no good chain is found. In addition, minimap2 attempts to patch gaps between
+seeds with ungapped alignment.
+.TP
+.BI --split-prefix \ STR
+Prefix to create temporary files. Typically used for a multi-part index.
+.TP
+.BR --frag = no | yes
+Whether to enable the fragment mode [no]
+.TP
+.B --for-only
+Only map to the forward strand of the reference sequences. For paired-end
+reads in the forward-reverse orientation, the first read is mapped to forward
+strand of the reference and the second read to the reverse stand.
+.TP
+.B --rev-only
+Only map to the reverse complement strand of the reference sequences.
+.TP
+.BR --heap-sort = no | yes
+If yes, sort anchors with heap merge, instead of radix sort. Heap merge is
+faster for short reads, but slower for long reads. [no]
+.TP
+.B --no-pairing
+Treat two reads in a pair as independent reads. The mate related fields in SAM
+are still properly populated.
+.TP
+.B --no-hash-name
+Produce the same alignment for identical sequences regardless of their sequence names.
 .SS Alignment options
 .TP 10
 .BI -A \ INT
@@ -190,29 +340,163 @@ Gap extension penalty [2,1]. A gap of length
 .I k
 costs
 .RI min{ O1 + k * E1 , O2 + k * E2 }.
+In the splice mode, the second gap penalties are not used.
 .TP
-.BI -z \ INT
+.BI -C \ INT
-Break an alignment if the running score drops too quickly along the diagonal of
+Cost for a non-canonical GT-AG splicing (effective with
-the DP matrix (diagonal X-drop, or Z-drop) [400]. Increasing the value improves
+.BR --splice )
-the contiguity of the alignment at the cost of poor alignment in the middle
+[0]
-(e.g. caused by a long inversion).
+.TP
+.BI -z \ INT1[,INT2]
+Truncate an alignment if the running alignment score drops too quickly along
+the diagonal of the DP matrix (diagonal X-drop, or Z-drop) [400,200]. If the
+drop of score is above
+.IR INT2 ,
+minimap2 will reverse complement the query in the related region and align
+again to test small inversions. Minimap2 truncates alignment if there is an
+inversion or the drop of score is greater than
+.IR INT1 .
+Decrease
+.I INT2
+to find small inversions at the cost of performance and false positives.
+Increase
+.I INT1
+to improves the contiguity of alignment at the cost of poor alignment in the
+middle.
 .TP
 .BI -s \ INT
 Minimal peak DP alignment score to output [40]. The peak score is computed from
 the final CIGAR. It is the score of the max scoring segment in the alignment
 and may be different from the total alignment score.
+.TP
+.BI -u \ CHAR
+How to find canonical splicing sites GT-AG -
+.BR f :
+transcript strand;
+.BR b :
+both strands;
+.BR n :
+no attempt to match GT-AG [n]
+.TP
+.BI --end-bonus \ INT
+Score bonus when alignment extends to the end of the query sequence [0].
+.TP
+.BI --score-N \ INT
+Score of a mismatch involving ambiguous bases [1].
+.TP
+.BR --splice-flank = yes | no
+Assume the next base to a
+.B GT
+donor site tends to be A/G (91% in human and 92% in mouse) and the preceding
+base to a
+.B AG
+acceptor tends to be C/T [no].
+This trend is evolutionarily conservative, all the way to S. cerevisiae
+(PMID:18688272). Specifying this option generally leads to higher junction
+accuracy by several percents, so it is applied by default with
+.BR --splice .
+However, the SIRV control does not honor this trend
+(only ~60%). This option reduces accuracy. If you are benchmarking minimap2
+on SIRV data, please add
+.B --splice-flank=no
+to the command line.
+.TP
+.BR --junc-bed \ FILE
+Gene annotations in the BED12 format (aka 12-column BED), or intron positions
+in 5-column BED. With this option, minimap2 prefers splicing in annotations.
+BED12 file can be converted from GTF/GFF3 with `paftools.js gff2bed anno.gtf'
+[].
+.TP
+.BR --junc-bonus \ INT
+Score bonus for a splice donor or acceptor found in annotation (effective with
+.BR --junc-bed )
+[9].
+.TP
+.BI --end-seed-pen \ INT
+Drop a terminal anchor if
+.IR s <log( g )+ INT ,
+where
+.I s
+is the local alignment score around the anchor and
+.I g
+the length of the terminal gap in the chain. This option is only effective
+with
+.BR --splice .
+It helps to avoid tiny terminal exons. [6]
+.TP
+.B --no-end-flt
+Don't filter seeds towards the ends of chains before performing base-level
+alignment.
+.TP
+.BI --cap-sw-mem \ NUM
+Skip alignment if the DP matrix size is above
+.IR NUM .
+Set 0 to disable [100m].
+.TP
+.BI --cap-kalloc \ NUM
+Free thread-local kalloc memory reservoir if after the alignment the size of the reservoir above
+.IR NUM .
+Set 0 to disable [0].
 .SS Input/output options
 .TP 10
-.B -Q
-Ignore base quality in the input file.
-.TP
 .B -a
 Generate CIGAR and output alignments in the SAM format. Minimap2 outputs in PAF
 by default.
 .TP
+.BI -o \ FILE
+Output alignments to
+.I FILE
+[stdout].
+.TP
+.B -Q
+Ignore base quality in the input file.
+.TP
+.B -L
+Write CIGAR with >65535 operators at the CG tag. Older tools are unable to
+convert alignments with >65535 CIGAR ops to BAM. This option makes minimap2 SAM
+compatible with older tools. Newer tools recognizes this tag and reconstruct
+the real CIGAR in memory.
+.TP
+.BI -R \ STR
+SAM read group line in a format like
+.B @RG\\\\tID:foo\\\\tSM:bar
+[].
+.TP
+.B -y
+Copy input FASTA/Q comments to output.
+.TP
 .B -c
 Generate CIGAR. In PAF, the CIGAR is written to the `cg' custom tag.
 .TP
+.BI --cs[= STR ]
+Output the
+.B cs
+tag.
+.I STR
+can be either
+.I short
+or
+.IR long .
+If no
+.I STR
+is given,
+.I short
+is assumed. [none]
+.TP
+.B --MD
+Output the MD tag (see the SAM spec).
+.TP
+.B --eqx
+Output =/X CIGAR operators for sequence match/mismatch.
+.TP
+.B -Y
+In SAM output, use soft clipping for supplementary alignments.
+.TP
+.BI --seed \ INT
+Integer seed for randomizing equally best hits. Minimap2 hashes
+.I INT
+and read name when choosing between equally best hits. [11]
+.TP
 .BI -t \ INT
 Number of threads [3]. Minimap2 uses at most three threads when indexing target
 sequences, and uses up to
@@ -220,21 +504,37 @@ sequences, and uses up to
 threads when mapping (the extra thread is for I/O, which is frequently idle and
 takes little CPU time).
 .TP
+.B -2
+Use two I/O threads during mapping. By default, minimap2 uses one I/O thread.
+When I/O is slow (e.g. piping to gzip, or reading from a slow pipe), the I/O
+thread may become the bottleneck. Apply this option to use one thread for input
+and another thread for output, at the cost of increased peak RAM.
+.TP
 .BI -K \ NUM
-Number of bases loaded into memory to process in a mini-batch [200M].
+Number of bases loaded into memory to process in a mini-batch [500M].
 Similar to option
 .BR -I ,
 K/M/G/k/m/g suffix is accepted. A large
 .I NUM
 helps load balancing in the multi-threading mode, at the cost of increased
-memory. Preset
+memory.
-.B ava-pb
-and
-.B ava-ont
-use
-.BR -K500m .
 .TP
-.B -V
+.BR --secondary = yes | no
+Whether to output secondary alignments [yes]
+.TP
+.BI --max-qlen \ NUM
+Filter out query sequences longer than
+.IR NUM .
+.TP
+.B --paf-no-hit
+In PAF, output unmapped queries; the strand and the reference name fields are
+set to `*'. Warning: some paftools.js commands may not work with such output
+for the moment.
+.TP
+.B --sam-hit-only
+In SAM, don't output unmapped reads.
+.TP
+.B --version
 Print version number to stdout
 .SS Preset options
 .TP 10
@@ -247,39 +547,80 @@ Available
 .I STR
 are:
 .RS
-.TP 8
+.TP 10
-.B map-pb
-PacBio/Oxford Nanopore read to reference mapping (-Hk19)
-.TP
-.B map10k
-The same as
-.B map-pb
-(-Hk19)
-.TP
 .B map-ont
-Slightly more sensitive for Oxford Nanopore to reference mapping (-k15). For
+Align noisy long reads of ~10% error rate to a reference genome. This is the
-PacBio reads, HPC minimizers consistently leads to faster performance and more
+default mode.
-sensitive results in comparison to normal minimizers. For Oxford Nanopore data,
+.TP
-normal minimizers are better, though not much. The effectiveness of HPC is
+.B map-hifi
-determined by the sequencing error mode.
+Align PacBio high-fidelity (HiFi) reads to a reference genome
+.RB ( -k19
+.B -w19 -U50,500 -g10k -A1 -B4 -O6,26 -E2,1
+.BR -s200 ).
+.TP
+.B map-pb
+Align older PacBio continuous long (CLR) reads to a reference genome
+.RB ( -Hk19 ).
 .TP
 .B asm5
-Long assembly to reference mapping (-k19 -w19 -A1 -B19 -O39,81 -E3,1 -s200 -z200).
+Long assembly to reference mapping
+.RB ( -k19
+.B -w19 -U50,500 --rmq -r1k,100k -g10k -A1 -B19 -O39,81 -E3,1 -s200 -z200
+.BR -N50 ).
 Typically, the alignment will not extend to regions with 5% or higher sequence
 divergence. Only use this preset if the average divergence is far below 5%.
 .TP
 .B asm10
-Long assembly to reference mapping (-k19 -w19 -A1 -B9 -O16,41 -E2,1 -s200 -z200). Up
+Long assembly to reference mapping
-to 10% sequence divergence.
+.RB ( -k19
-.TP 8
+.B -w19 -U50,500 --rmq -r1k,100k -g10k -A1 -B9 -O16,41 -E2,1 -s200 -z200
+.BR -N50 ).
+Up to 10% sequence divergence.
+.TP
+.B asm20
+Long assembly to reference mapping
+.RB ( -k19
+.B -w10 -U50,500 --rmq -r1k,100k -g10k -A1 -B4 -O6,26 -E2,1 -s200 -z200
+.BR -N50 ).
+Up to 20% sequence divergence.
+.TP
+.B splice
+Long-read spliced alignment
+.RB ( -k15
+.B -w5 --splice -g2k -G200k -A1 -B2 -O2,32 -E1,0 -b0 -C9 -z200 -ub --junc-bonus=9 --cap-sw-mem=0
+.BR --splice-flank=yes ).
+In the splice mode, 1) long deletions are taken as introns and represented as
+the
+.RB ` N '
+CIGAR operator; 2) long insertions are disabled; 3) deletion and insertion gap
+costs are different during chaining; 4) the computation of the
+.RB ` ms '
+tag ignores introns to demote hits to pseudogenes.
+.TP
+.B splice:hq
+Long-read splice alignment for PacBio CCS reads
+.RB ( -xsplice
+.B -C5 -O6,24
+.BR -B4 ).
+.TP
+.B sr
+Short single-end reads without splicing
+.RB ( -k21
+.B -w11 --sr --frag=yes -A2 -B8 -O12,32 -E2,1 -b0 -r100 -p.5 -N20 -f1000,5000 -n2 -m25
+.B -s40 -g100 -2K50m --heap-sort=yes
+.BR --secondary=no ).
+.TP
 .B ava-pb
-PacBio all-vs-all overlap mapping (-Hk19 -w5 -Xp0 -m100 -K500m -g10000 --max-chain-skip 25)
+PacBio CLR all-vs-all overlap mapping
-.TP 8
+.RB ( -Hk19
+.B -Xw5 -e0
+.BR -m100 ).
+.TP
 .B ava-ont
-Oxford Nanopore all-vs-all overlap mapping (-k15 -w5 -Xp0 -m100 -K500m -g10000
+Oxford Nanopore all-vs-all overlap mapping
---max-chain-skip 25). Similarly, the major difference from
+.RB ( -k15
-.B ava-pb
+.B -Xw5 -e0 -m100
-is that this preset is not using HPC minimizers.
+.BR -r2k ).
 .RE
 .SS Miscellaneous options
 .TP 10
@@ -292,7 +633,7 @@ multi-threading mode.
 .B --print-qname
 Print query names to stderr, mostly to see which query is crashing minimap2.
 .TP
-.B --print-seed
+.B --print-seeds
 Print seed positions to stderr, for debugging only.
 .SH OUTPUT FORMAT
 .PP
@@ -331,33 +672,54 @@ cb | cb | cb
 r | c | l .
 Tag	Type	Description
 _
+tp	A	Type of aln: P/primary, S/secondary and I,i/inversion
 cm	i	Number of minimizers on the chain
 s1	i	Chaining score
 s2	i	Chaining score of the best secondary chain
 NM	i	Total number of mismatches and gaps in the alignment
+MD	Z	To generate the ref sequence in the alignment
 AS	i	DP alignment score
+SA	Z	List of other supplementary alignments
 ms	i	DP score of the max scoring segment in the alignment
 nn	i	Number of ambiguous bases in the alignment
+ts	A	Transcript strand (splice mode only)
 cg	Z	CIGAR string (only in PAF)
+cs	Z	Difference string
+dv	f	Approximate per-base sequence divergence
+de	f	Gap-compressed per-base sequence divergence
+rl	i	Length of query regions harboring repetitive seeds
+.TE
+
+.PP
+The
+.B cs
+tag encodes difference sequences in the short form or the entire query
+.I AND
+reference sequences in the long form. It consists of a series of operations:
+.TS
+center box;
+cb | cb |cb
+r | l | l .
+Op	Regex	Description
+_
+ =	[ACGTN]+	Identical sequence (long form)
+ :	[0-9]+	Identical sequence length
+ *	[acgtn][acgtn]	Substitution: ref to query
+ +	[acgtn]+	Insertion to the reference
+ -	[acgtn]+	Deletion from the reference
+ ~	[acgtn]{2}[0-9]+[acgtn]{2}	Intron length and splice signal
 .TE
 
 .SH LIMITATIONS
 .TP 2
 *
-At the alignment phase, minimap2 performs global alignments between minimizer
+Minimap2 may produce suboptimal alignments through long low-complexity regions
-hits. If the positions of these minimizer hits are incorrect, the final
+where seed positions may be suboptimal. This should not be a big concern
-alignment may be suboptimal or unnecessarily fragmented.
+because even the optimal alignment may be wrong in such regions.
 .TP
 *
-Minimap2 may produce poor alignments that may need post-filtering. We are still
+Minimap2 requires SSE2 or NEON instructions to compile. It is possible to add
-exploring a reliable and consistent way to report good alignments.
+non-SSE2/NEON support, but it would make minimap2 slower by several times.
-.TP
-*
-Minimap2 does not work well with Illumina short reads as of now.
-.TP
-*
-Minimap2 requires SSE2 instructions to compile. It is possible to add
-non-SSE2 support, but it would make minimap2 slower by several times.
 .SH SEE ALSO
 .PP
 miniasm(1), minimap(1), bwa(1).

misc.c

@@ -1,26 +1,155 @@
-#include <sys/resource.h>
+#include <stdlib.h>
-#include <sys/time.h>
+#include "mmpriv.h"
-#include "minimap.h"
 
-int mm_verbose = 3;
+int mm_verbose = 1;
 int mm_dbg_flag = 0;
 double mm_realtime0;
 
+#if defined(WIN32) || defined(_WIN32)
+#include <windows.h>
+
+struct timezone
+{
+  __int32  tz_minuteswest; /* minutes W of Greenwich */
+  int  tz_dsttime;     /* type of dst correction */
+};
+
+/*
+ * gettimeofday.c
+ *    Win32 gettimeofday() replacement
+ *    taken from PostgreSQL, according to
+ *    https://stackoverflow.com/questions/1676036/what-should-i-use-to-replace-gettimeofday-on-windows
+ *
+ * src/port/gettimeofday.c
+ *
+ * Copyright (c) 2003 SRA, Inc.
+ * Copyright (c) 2003 SKC, Inc.
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose, without fee, and without a
+ * written agreement is hereby granted, provided that the above
+ * copyright notice and this paragraph and the following two
+ * paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT,
+ * INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS
+ * IS" BASIS, AND THE AUTHOR HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE,
+ * SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+/* FILETIME of Jan 1 1970 00:00:00. */
+static const unsigned __int64 epoch = ((unsigned __int64) 116444736000000000ULL);
+
+/*
+ * timezone information is stored outside the kernel so tzp isn't used anymore.
+ *
+ * Note: this function is not for Win32 high precision timing purpose. See
+ * elapsed_time().
+ */
+int gettimeofday(struct timeval * tp, struct timezone *tzp)
+{
+    FILETIME    file_time;
+    SYSTEMTIME  system_time;
+    ULARGE_INTEGER ularge;
+
+    GetSystemTime(&system_time);
+    SystemTimeToFileTime(&system_time, &file_time);
+    ularge.LowPart = file_time.dwLowDateTime;
+    ularge.HighPart = file_time.dwHighDateTime;
+
+    tp->tv_sec = (long) ((ularge.QuadPart - epoch) / 10000000L);
+    tp->tv_usec = (long) (system_time.wMilliseconds * 1000);
+
+    return 0;
+}
+
+// taken from https://stackoverflow.com/questions/5272470/c-get-cpu-usage-on-linux-and-windows
 double cputime()
+{
+	HANDLE hProcess = GetCurrentProcess();
+	FILETIME ftCreation, ftExit, ftKernel, ftUser;
+	SYSTEMTIME stKernel;
+	SYSTEMTIME stUser;
+
+	GetProcessTimes(hProcess, &ftCreation, &ftExit, &ftKernel, &ftUser);
+	FileTimeToSystemTime(&ftKernel, &stKernel);
+	FileTimeToSystemTime(&ftUser, &stUser);
+
+	double kernelModeTime = ((stKernel.wHour * 60.) + stKernel.wMinute * 60.) + stKernel.wSecond * 1. + stKernel.wMilliseconds / 1000.;
+	double userModeTime = ((stUser.wHour * 60.) + stUser.wMinute * 60.) + stUser.wSecond * 1. + stUser.wMilliseconds / 1000.;
+
+	return kernelModeTime + userModeTime;
+}
+
+long peakrss(void) { return 0; }
+#else
+#include <sys/resource.h>
+#include <sys/time.h>
+
+double cputime(void)
 {
 	struct rusage r;
 	getrusage(RUSAGE_SELF, &r);
 	return r.ru_utime.tv_sec + r.ru_stime.tv_sec + 1e-6 * (r.ru_utime.tv_usec + r.ru_stime.tv_usec);
 }
 
-double realtime()
+long peakrss(void)
+{
+	struct rusage r;
+	getrusage(RUSAGE_SELF, &r);
+#ifdef __linux__
+	return r.ru_maxrss * 1024;
+#else
+	return r.ru_maxrss;
+#endif
+}
+
+#endif /* WIN32 || _WIN32 */
+
+double realtime(void)
 {
 	struct timeval tp;
-	struct timezone tzp;
+	gettimeofday(&tp, NULL);
-	gettimeofday(&tp, &tzp);
 	return tp.tv_sec + tp.tv_usec * 1e-6;
 }
 
+void mm_err_puts(const char *str)
+{
+	int ret;
+	ret = puts(str);
+	if (ret == EOF) {
+		perror("[ERROR] failed to write the results");
+		exit(EXIT_FAILURE);
+	}
+}
+
+void mm_err_fwrite(const void *p, size_t size, size_t nitems, FILE *fp)
+{
+	int ret;
+	ret = fwrite(p, size, nitems, fp);
+	if (ret == EOF) {
+		perror("[ERROR] failed to write data");
+		exit(EXIT_FAILURE);
+	}
+}
+
+void mm_err_fread(void *p, size_t size, size_t nitems, FILE *fp)
+{
+	int ret;
+	ret = fread(p, size, nitems, fp);
+	if (ret == EOF) {
+		perror("[ERROR] failed to read data");
+		exit(EXIT_FAILURE);
+	}
+}
+
 #include "ksort.h"
 
 #define sort_key_128x(a) ((a).x)
@@ -30,3 +159,4 @@ KRADIX_SORT_INIT(128x, mm128_t, sort_key_128x, 8)
 KRADIX_SORT_INIT(64, uint64_t, sort_key_64, 8)
 
 KSORT_INIT_GENERIC(uint32_t)
+KSORT_INIT_GENERIC(uint64_t)

misc/README.md

@@ -0,0 +1,179 @@
+## <a name="started"></a>Getting Started
+
+```sh
+# install minimap2
+git clone https://github.com/lh3/minimap2
+cd minimap2 && make
+# install the k8 javascript shell
+curl -L https://github.com/attractivechaos/k8/releases/download/v0.2.4/k8-0.2.4.tar.bz2 | tar -jxf -
+cp k8-0.2.4/k8-`uname -s` k8              # or copy it to a directory on your $PATH
+# export PATH="$PATH:`pwd`:`pwd`/misc"    # run this if k8, minimap2 or paftools.js not on your $PATH
+minimap2 --cs test/MT-human.fa test/MT-orang.fa | paftools.js view -     # view alignment
+minimap2 -c test/MT-human.fa test/MT-orang.fa | paftools.js stat -       # basic alignment statistics
+minimap2 -c --cs test/MT-human.fa test/MT-orang.fa \
+  | sort -k6,6 -k8,8n | paftools.js call -L15000 -     # calling variants from asm-to-ref alignment
+minimap2 -c test/MT-human.fa test/MT-orang.fa \
+  | paftools.js liftover -l10000 - <(echo -e "MT_orang\t2000\t5000")     # liftOver
+# no test data for the following examples
+paftools.js junceval -e anno.gtf splice.sam > out.txt  # compare splice junctions to annotations
+paftools.js splice2bed anno.gtf > anno.bed             # convert GTF/GFF3 to BED12
+```
+
+## Table of Contents
+
+- [Getting Started](#started)
+- [Introduction](#intro)
+- [Evaluation](#eval)
+  - [Evaluating mapping accuracy with simulated reads](#mapeval)
+  - [Evaluating read overlap sensitivity](#oveval)
+- [Calling Variants from Assemblies](#asmvar)
+
+## <a name="intro"></a>Introduction
+
+paftools.js is a script that processes alignments in the [PAF format][paf],
+such as converting between formats, evaluating mapping accuracy, lifting over
+BED files based on alignment, and calling variants from assembly-to-assembly
+alignment. This script *requires* the [k8 Javascript shell][k8] to run. On
+Linux or Mac, you can download the precompiled k8 binary with:
+
+```sh
+curl -L https://github.com/attractivechaos/k8/releases/download/v0.2.4/k8-0.2.4.tar.bz2 | tar -jxf -
+cp k8-0.2.4/k8-`uname -s` $HOME/bin/k8  # assuming $HOME/bin in your $PATH
+```
+
+It is highly recommended to copy the executable `k8` to a directory on your
+`$PATH` such as `/usr/bin/env` can find it. Like python scripts, once you
+install `k8`, you can launch paftools.js in one of the two ways:
+
+```sh
+path/to/paftools.js             # only if k8 is on your $PATH
+k8 path/to/paftools.js
+```
+
+In a nutshell, paftools.js has the following commands:
+
+```
+Usage: paftools.js <command> [arguments]
+Commands:
+  view       convert PAF to BLAST-like (for eyeballing) or MAF
+  splice2bed convert spliced alignment in PAF/SAM to BED12
+  sam2paf    convert SAM to PAF
+  delta2paf  convert MUMmer's delta to PAF
+  gff2bed    convert GTF/GFF3 to BED12
+
+  stat       collect basic mapping information in PAF/SAM
+  liftover   simplistic liftOver
+  call       call variants from asm-to-ref alignment with the cs tag
+  bedcov     compute the number of bases covered
+
+  mapeval    evaluate mapping accuracy using mason2/PBSIM-simulated FASTQ
+  mason2fq   convert mason2-simulated SAM to FASTQ
+  pbsim2fq   convert PBSIM-simulated MAF to FASTQ
+  junceval   evaluate splice junction consistency with known annotations
+  ov-eval    evaluate read overlap sensitivity using read-to-ref mapping
+```
+
+paftools.js seamlessly reads both plain text files and gzip'd text files.
+
+## <a name="eval"></a>Evaluation
+
+### <a name="mapeval"></a>Evaluating mapping accuracy with simulated reads
+
+The **pbsim2fq** command of paftools.js converts the MAF output of [pbsim][pbsim]
+to FASTQ and encodes the true mapping position in the read name in a format like
+`S1_33!chr1!225258409!225267761!-`. Similarly, the **mason2fq** command
+converts [mason2][mason2] simulated SAM to FASTQ.
+
+Command **mapeval** evaluates mapped SAM/PAF. Here is example output:
+
+```
+Q       60      32478   0       0.000000000     32478
+Q       22      16      1       0.000030775     32494
+Q       21      43      1       0.000061468     32537
+Q       19      73      1       0.000091996     32610
+Q       14      66      1       0.000122414     32676
+Q       10      27      3       0.000214048     32703
+Q       8       14      1       0.000244521     32717
+Q       7       13      2       0.000305530     32730
+Q       6       46      1       0.000335611     32776
+Q       3       10      1       0.000366010     32786
+Q       2       20      2       0.000426751     32806
+Q       1       248     94      0.003267381     33054
+Q       0       31      17      0.003778147     33085
+U       3
+```
+
+where each Q-line gives the quality threshold, the number of reads mapped with
+mapping quality equal to or greater than the threshold, number of wrong
+mappings, accumulative mapping error rate and the accumulative number of
+mapped reads. The U-line, if present, gives the number of unmapped reads if
+they are present in the SAM file.
+
+Suppose the reported mapping coordinate overlap with the true coordinate like
+the following:
+
+```
+truth:   --------------------
+mapper:           ----------------------
+         |<- l1 ->|<-- o -->|<-- l2 -->|
+```
+
+Let `r=o/(l1+o+l2)`. The reported mapping is considered correct if `r>0.1` by
+default.
+
+### <a name="oveval"></a>Evaluating read overlap sensitivity
+
+Command **ov-eval** takes *sorted* read-to-reference alignment and read
+overlaps in PAF as input, and evaluates the sensitivity. For example:
+
+```sh
+minimap2 -cx map-pb ref.fa reads.fq.gz | sort -k6,6 -k8,8n > reads-to-ref.paf
+minimap2 -x ava-pb reads.fq.gz reads.fq.gz > ovlp.paf
+k8 ov-eval.js reads-to-ref.paf ovlp.paf
+```
+
+## <a name="asmvar"></a>Calling Variants from Haploid Assemblies
+
+The **call** command of paftools.js calls variants from coordinate-sorted
+assembly-to-reference alignment. It calls variants from the [cs tag][cs] and
+identifies confident/callable regions as those covered by exactly one contig.
+Here are example command lines:
+
+```sh
+minimap2 -cx asm5 -t8 --cs ref.fa asm.fa > asm.paf  # keeping this file is recommended; --cs required!
+sort -k6,6 -k8,8n asm.paf > asm.srt.paf             # sort by reference start coordinate
+k8 paftools.js call asm.srt.paf > asm.var.txt
+```
+
+Here is sample output:
+
+```
+V   chr1    2276040 2276041 1   60  c   g   LJII01000171.1  1217409 1217410 +
+V   chr1    2280409 2280410 1   60  a   g   LJII01000171.1  1221778 1221779 +
+V   chr1    2280504 2280505 1   60  a   g   LJII01000171.1  1221873 1221874 +
+R   chr1    2325140 2436340
+V   chr1    2325287 2325287 1   60  -   ct  LJII01000171.1  1272894 1272896 +
+V   chr1    2325642 2325644 1   60  tt  -   LJII01000171.1  1273251 1273251 +
+V   chr1    2326051 2326052 1   60  c   t   LJII01000171.1  1273658 1273659 +
+V   chr1    2326287 2326288 1   60  c   t   LJII01000171.1  1273894 1273895 +
+```
+
+where a line starting with `R` gives regions covered by one query contig, and a
+V-line encodes a variant in the following format: chr, start, end, query depth,
+mapping quality, REF allele, ALT allele, query name, query start, end and the
+query orientation. Generally, you should only look at variants where column 5
+is one.
+
+By default, when calling variants, "paftools.js call" ignores alignments 50kb
+or shorter; when deriving callable regions, it ignores alignments 10kb or
+shorter.  It uses two thresholds to avoid edge effects. These defaults are
+designed for long-read assemblies. For short reads, both should be reduced.
+
+
+
+[paf]: https://github.com/lh3/miniasm/blob/master/PAF.md
+[cs]: https://github.com/lh3/minimap2#cs
+[k8]: https://github.com/attractivechaos/k8
+[maf]: https://genome.ucsc.edu/FAQ/FAQformat#format5
+[pbsim]: https://github.com/pfaucon/PBSIM-PacBio-Simulator
+[mason2]: https://github.com/seqan/seqan/tree/master/apps/mason2

misc/mapstat.js

@@ -1,183 +0,0 @@
-var getopt = function(args, ostr) {
-	var oli; // option letter list index
-	if (typeof(getopt.place) == 'undefined')
-		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
-	if (getopt.place == -1) { // update scanning pointer
-		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
-			getopt.place = -1;
-			return null;
-		}
-		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
-			++getopt.ind;
-			getopt.place = -1;
-			return null;
-		}
-	}
-	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
-	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
-		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
-		if (getopt.place < 0) ++getopt.ind;
-		return '?';
-	}
-	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
-		getopt.arg = null;
-		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
-	} else { // need an argument
-		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
-			getopt.arg = args[getopt.ind].substr(getopt.place);
-		else if (args.length <= ++getopt.ind) { // no arg
-			getopt.place = -1;
-			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
-			return '?';
-		} else getopt.arg = args[getopt.ind]; // white space
-		getopt.place = -1;
-		++getopt.ind;
-	}
-	return optopt;
-}
-
-var c, gap_out_len = null;
-while ((c = getopt(arguments, "l:")) != null)
-	if (c == 'l') gap_out_len = parseInt(getopt.arg);
-
-if (getopt.ind == arguments.length) {
-	print("Usage: k8 mapstat.js [-l gapOutLen] <in.sam>|<in.paf>");
-	exit(1);
-}
-
-var buf = new Bytes();
-var file = new File(arguments[getopt.ind]);
-var re = /(\d+)([MIDSHNX=])/g;
-
-var lineno = 0, n_pri = 0, n_2nd = 0, n_seq = 0, n_cigar_64k = 0, l_tot = 0, l_cov = 0;
-var n_gap = [[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]];
-
-function cov_len(regs)
-{
-	regs.sort(function(a,b) {return a[0]-b[0]});
-	var st = regs[0][0], en = regs[0][1], l = 0;
-	for (var i = 1; i < regs.length; ++i) {
-		if (regs[i][0] < en)
-			en = en > regs[i][1]? en : regs[i][1];
-		else l += en - st, st = regs[i][0], en = regs[i][1];
-	}
-	l += en - st;
-	return l;
-}
-
-var last = null, last_qlen = null, regs = [];
-while (file.readline(buf) >= 0) {
-	var line = buf.toString();
-	++lineno;
-	if (line.charAt(0) != '@') {
-		var t = line.split("\t", 12);
-		var m, rs, cigar = null, is_pri = false, is_sam = false, is_rev = false, tname = null;
-		var atlen = null, aqlen, qs, qe, mapq, ori_qlen;
-		if (t[4] == '+' || t[4] == '-') { // PAF
-			if (!/\ts2:i:\d+/.test(line)) {
-				++n_2nd;
-				continue;
-			}
-			if ((m = /\tcg:Z:(\S+)/.exec(line)) != null)
-				cigar = m[1];
-			if (cigar == null) {
-				warn("WARNING: no CIGAR at line " + lineno);
-				continue;
-			}
-			tname = t[5];
-			qs = parseInt(t[2]), qe = parseInt(t[3]);
-			aqlen = qe - qs;
-			is_rev = t[4] == '+'? false : true;
-			rs = parseInt(t[7]);
-			atlen = parseInt(t[8]) - rs;
-			mapq = parseInt(t[11]);
-			ori_qlen = parseInt(t[1]);
-		} else { // SAM
-			var flag = parseInt(t[1]);
-			if (flag & 4) continue;
-			if (flag & 0x100) {
-				++n_2nd;
-				continue;
-			}
-			cigar = t[5];
-			tname = t[2];
-			rs = parseInt(t[3]) - 1;
-			mapq = parseInt(t[4]);
-			aqlen = t[9].length;
-			is_sam = true;
-			is_rev = !!(flag&0x10);
-		}
-		++n_pri;
-		if (last != t[0]) {
-			if (last != null) {
-				l_tot += last_qlen;
-				l_cov += cov_len(regs);
-			}
-			regs = [];
-			++n_seq, last = t[0];
-		}
-		var M = 0, tl = 0, ql = 0, clip = [0, 0], n_cigar = 0, sclip = 0;
-		while ((m = re.exec(cigar)) != null) {
-			var l = parseInt(m[1]);
-			++n_cigar;
-			if (m[2] == 'M' || m[2] == '=' || m[2] == 'X') {
-				tl += l, ql += l, M += l;
-			} else if (m[2] == 'I' || m[2] == 'D') {
-				var type;
-				if (l < 50) type = 0;
-				else if (l < 100) type = 1;
-				else if (l < 300) type = 2;
-				else if (l < 400) type = 3;
-				else if (l < 1000) type = 4;
-				else type = 5;
-				if (m[2] == 'I') ql += l, ++n_gap[0][type];
-				else tl += l, ++n_gap[1][type];
-				if (gap_out_len != null && l >= gap_out_len)
-					print(t[0], ql, is_rev? '-' : '+', tname, rs + tl, m[2], l);
-			} else if (m[2] == 'N') {
-				tl += l;
-			} else if (m[2] == 'S') {
-				clip[M == 0? 0 : 1] = l, sclip += l;
-			} else if (m[2] == 'H') {
-				clip[M == 0? 0 : 1] = l;
-			}
-		}
-		if (n_cigar > 65535) ++n_cigar_64k;
-		if (ql + sclip != aqlen)
-			warn("WARNING: aligned query length is inconsistent with CIGAR at line " + lineno);
-		if (atlen != null && atlen != tl)
-			warn("WARNING: aligned reference length is inconsistent with CIGAR at line " + lineno);
-		if (is_sam) {
-			qs = clip[is_rev? 1 : 0], qe = qs + ql;
-			ori_qlen = clip[0] + ql + clip[1];
-		}
-		regs.push([qs, qe]);
-		last_qlen = ori_qlen;
-	}
-}
-l_tot += last_qlen;
-l_cov += cov_len(regs);
-
-file.close();
-buf.destroy();
-
-if (gap_out_len == null) {
-	print("Number of mapped sequences: " + n_seq);
-	print("Number of primary alignments: " + n_pri);
-	print("Number of secondary alignments: " + n_2nd);
-	print("Number of primary alignments with >65535 CIGAR operations: " + n_cigar_64k);
-	print("Number of bases in mapped sequences: " + l_tot);
-	print("Number of mapped bases: " + l_cov);
-	print("Number of insertions in [0,50): " + n_gap[0][0]);
-	print("Number of insertions in [50,100): " + n_gap[0][1]);
-	print("Number of insertions in [100,300): " + n_gap[0][2]);
-	print("Number of insertions in [300,400): " + n_gap[0][3]);
-	print("Number of insertions in [400,1000): " + n_gap[0][4]);
-	print("Number of insertions in [1000,inf): " + n_gap[0][5]);
-	print("Number of deletions in [0,50): " + n_gap[1][0]);
-	print("Number of deletions in [50,100): " + n_gap[1][1]);
-	print("Number of deletions in [100,300): " + n_gap[1][2]);
-	print("Number of deletions in [300,400): " + n_gap[1][3]);
-	print("Number of deletions in [400,1000): " + n_gap[1][4]);
-	print("Number of deletions in [1000,inf): " + n_gap[1][5]);
-}

misc/mmphase.js

@@ -0,0 +1,335 @@
+#!/usr/bin/env k8
+
+var getopt = function(args, ostr) {
+	var oli; // option letter list index
+	if (typeof(getopt.place) == 'undefined')
+		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
+	if (getopt.place == -1) { // update scanning pointer
+		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
+			getopt.place = -1;
+			return null;
+		}
+		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
+			++getopt.ind;
+			getopt.place = -1;
+			return null;
+		}
+	}
+	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
+	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
+		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
+		if (getopt.place < 0) ++getopt.ind;
+		return '?';
+	}
+	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
+		getopt.arg = null;
+		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
+	} else { // need an argument
+		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
+			getopt.arg = args[getopt.ind].substr(getopt.place);
+		else if (args.length <= ++getopt.ind) { // no arg
+			getopt.place = -1;
+			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
+			return '?';
+		} else getopt.arg = args[getopt.ind]; // white space
+		getopt.place = -1;
+		++getopt.ind;
+	}
+	return optopt;
+}
+
+function read_fastx(file, buf)
+{
+	if (file.readline(buf) < 0) return null;
+	var m, line = buf.toString();
+	if ((m = /^([>@])(\S+)/.exec(line)) == null)
+		throw Error("wrong fastx format");
+	var is_fq = (m[1] == '@');
+	var name = m[2];
+	if (file.readline(buf) < 0)
+		throw Error("missing sequence line");
+	var seq = buf.toString();
+	if (is_fq) { // skip quality
+		file.readline(buf);
+		file.readline(buf);
+	}
+	return [name, seq];
+}
+
+function filter_paf(a, opt)
+{
+	if (a.length == 0) return;
+	var k = 0;
+	for (var i = 0; i < a.length; ++i) {
+		var ai = a[i];
+		if (ai[10] < opt.min_blen) continue;
+		if (ai[9] < ai[10] * opt.min_iden) continue;
+		var clip = [0, 0];
+		if (ai[4] == '+') {
+			clip[0] = ai[2] < ai[7]? ai[2] : ai[7];
+			clip[1] = ai[1] - ai[3] < ai[6] - ai[8]? ai[1] - ai[3] : ai[6] - ai[8];
+		} else {
+			clip[0] = ai[2] < ai[6] - ai[8]? ai[2] : ai[6] - ai[8];
+			clip[1] = ai[1] - ai[3] < ai[7]? ai[1] - ai[3] : ai[7];
+		}
+		if (clip[0] > opt.max_clip_len || clip[1] > opt.max_clip_len) continue;
+		a[k++] = ai;
+	}
+	a.length = k;
+}
+
+function parse_events(t, ev, id, buf)
+{
+	var re = /(:(\d+))|(([\+\-\*])([a-z]+))/g;
+	var m, cs = null;
+	for (var j = 12; j < t.length; ++j) {
+		if ((m = /^cs:Z:(\S+)/.exec(t[j])) != null) {
+			cs = m[1].toLowerCase();
+			break;
+		}
+	}
+	if (cs == null) {
+		warn("Warning: no cs tag for read '" + t[0] + "'");
+		return;
+	}
+	var st = t[2], en = t[3];
+	var x = st;
+	while ((m = re.exec(cs)) != null) {
+		var l;
+		if (m[2] != null) { // an identitcal match ":\d+"
+			l = parseInt(m[2]);
+			// [start, end, type, index, changed_base]
+			ev.push([x, x + l, 0, id]);
+		} else {
+			if (m[4] == '*') {
+				l = 1;
+				ev.push([x, x + 1, 1, id, m[5][0]]);
+			} else if (m[4] == '+') {
+				l = m[5].length;
+				ev.push([x, x + l, 2, id]);
+			} else if (m[4] == '-') {
+				l = 0;
+				ev.push([x, x, -1, id, m[5]]);
+			}
+		}
+		x += l;
+	}
+	if (x != en)
+		throw Error("inconsistent cs for read '" + t[0] + "'");
+}
+
+function find_het_sub(ev, a, opt)
+{
+	var n = a.length, last0_i = -1, h = [], d = [];
+	for (var i = 0; i < n; ++i) h[i] = [], d[i] = [];
+	for (var i = 0; i < ev.length; ++i) {
+		if (ev[i][2] == 0) {
+			if (last0_i < 0 || ev[i][0] != ev[last0_i][0]) last0_i = i;
+			else if (ev[i][1] > ev[last0_i][1])
+				last0_i = i;
+		} else if (ev[i][2] == 1 && last0_i >= 0 && ev[i][0] < ev[last0_i][1]) {
+			if (ev[last0_i][1] - ev[last0_i][0] >= opt.min_mlen) {
+				if (opt.dbg_ev) print("EV", ev[last0_i].join("\t"), "|", ev[i].join("\t"));
+				var e0 = ev[last0_i], hl = h[e0[3]];
+				if (hl.length == 0 || hl[hl.length-1][0] != e0[0])
+					hl.push([e0[0], e0[1]]);
+				d[ev[i][3]].push([ev[i][0], e0[1] - e0[0]]);
+			}
+		}
+	}
+	var b = [];
+	for (var i = 0; i < n; ++i) {
+		var sh = 0, dh = 0;
+		for (var j = 0; j < h[i].length; ++j)
+			sh += h[i][j][1] - h[i][j][0];
+		for (var j = 0; j < d[i].length; ++j)
+			dh += d[i][j][1];
+		// [start, end, index, #consistent, lenConsistent, #conflictive, lenConflictive, identity, mlen]
+		b[i] = [a[i][2], a[i][3], i, h[i].length, sh, d[i].length, dh, a[i][9] / a[i][10], a[i][9]];
+	}
+	return b;
+}
+
+function flt_utg_for_ec(b, opt)
+{
+	var k = 0;
+	for (var i = 0; i < b.length; ++i) {
+		var bi = b[i];
+		if (bi[4] == 0 && bi[6] == 0) b[k++] = bi; // entirely ambiguous
+		else if (bi[6] < (bi[4] + bi[6]) * opt.max_ratio0) b[k++] = bi;
+	}
+	b.length = k;
+	if (b.length == 0) return;
+	// find the longest contiguous segment
+	b.sort(function(x,y) { return x[0]-y[0] });
+	var st = b[0][0], en = b[0][1], max_st = 0, max_en = 0, max_max_en = en;
+	for (var i = 1; i < b.length; ++i) {
+		if (b[i][0] > en) {
+			if (en - st > max_en - max_st)
+				max_st = st, max_en = en;
+			st = b[i][0], en = b[i][1];
+		} else {
+			en = en > b[i][1]? en : b[i][1];
+		}
+		max_max_en = max_max_en > b[i][1]? max_max_en : b[i][1];
+	}
+	if (en - st > max_en - max_st)
+		max_st = st, max_en = en;
+	if (max_max_en != en || st != b[0][0]) {
+		var k = 0;
+		for (var i = 0; i < b.length; ++i)
+			if (b[i][0] < max_en && b[i][1] > max_st)
+				b[k++] = b[i];
+		b.length = k;
+	}
+}
+
+function flt_utg_for_bin(b, opt) // filter out alignments clearly on the wrong phase
+{
+	var k = 0;
+	for (var i = 0; i < b.length; ++i) {
+		var bi = b[i];
+		if (bi[4] + bi[6] == 0 || bi[4] >= (bi[4] + bi[6]) * opt.max_ratio0) b[k++] = bi;
+	}
+	b.length = k;
+}
+
+function ec_core(b, n_a, ev, buf, ecb) // error correction
+{
+	var intv = [];
+	for (var i = 0; i < n_a; ++i)
+		intv[i] = null;
+	intv[b[0][2]] = [b[0][0], b[0][1]];
+	var en = b[0][1];
+	for (var i = 1; i < b.length; ++i) {
+		if (b[i][1] <= en) continue;
+		intv[b[i][2]] = [en, b[i][1]];
+		en = b[i][1];
+	}
+	var k = 0;
+	ecb.capacity = buf.capacity;
+	ecb.length = 0;
+	for (var i = 0; i < ev.length; ++i) {
+		var e = ev[i], I = intv[e[3]];
+		if (I == null) continue;
+		if (e[0] >= I[0] && e[0] < I[1]) { // this is to reduce duplicated events around junctions
+			//print("X", e.join("\t"));
+			if (e[2] == 0) {
+				ecb.length += e[1] - e[0];
+				for (var j = e[0]; j < e[1]; ++j)
+					ecb[k++] = buf[j];
+			} else if (e[2] == 1) {
+				++ecb.length;
+				ecb[k++] = e[4].charCodeAt(0);
+			} else if (e[2] < 0) {
+				ecb.length += e[4].length;
+				for (var j = 0; j < e[4].length; ++j)
+					ecb[k++] = e[4].charCodeAt(j);
+			} // else, skip e[2] == 2
+		}
+	}
+	if (ecb.length != k) throw Error("BUG!");
+}
+
+function process_paf(a, opt, fp_seq, buf, ecb)
+{
+	if (a.length == 0) return;
+	var len = a[0][1], name = a[0][0], seq = null;
+	if (len < opt.min_rlen) return;
+	if (fp_seq) {
+		var ret;
+		while ((ret = read_fastx(fp_seq, buf)) != null)
+			if (ret[0] == a[0][0])
+				break;
+		if (ret == null)
+			throw Error("failed to find sequence for read '" + a[0][0] + "'");
+		name = ret[0], seq = ret[1];
+		if (seq.length != len)
+			throw Error("inconsistent length for read '" + name + "'");
+	}
+	filter_paf(a, opt);
+	if (a.length == 0) return;
+	var ev = [];
+	for (var i = 0; i < a.length; ++i)
+		parse_events(a[i], ev, i, buf);
+	ev.sort(function(x,y) { return x[0]!=y[0]? x[0]-y[0] : x[2]-y[2] });
+	if (seq == null) print("SQ", name, a[0][1], a.length);
+	var b = find_het_sub(ev, a, opt);
+	if (opt.ec) flt_utg_for_ec(b, opt);
+	else flt_utg_for_bin(b, opt);
+	if (seq == null) {
+		for (var i = 0; i < b.length; ++i) {
+			var m, ai = a[b[i][2]], score = 0;
+			for (var j = 10; j < ai.length; ++j)
+				if ((m = /^AS:i:(\d+)/.exec(ai[j])) != null)
+					score = m[1];
+			print("TS", b[i][2], b[i][0], b[i][1], ai.slice(5, 9).join("\t"), b[i].slice(3, 7).join("\t"), score);
+		}
+		print("//");
+	} else { // error correction
+		if (b.length == 0) return;
+		buf.set(seq, 0);
+		ec_core(b, a.length, ev, buf, ecb);
+		print(">" + name);
+		print(ecb);
+	}
+}
+
+function main(args)
+{
+	var c, opt = { min_rlen:5000, min_blen:5000, min_iden:0.8, min_mlen:5, max_clip_len:500, max_ratio0:0.25, dbg_ev:false };
+	while ((c = getopt(args, "l:b:d:m:c:r:E")) != null) {
+		if (c == 'l') opt.min_rlen = parseInt(getopt.arg);
+		else if (c == 'b') opt.min_blen = parseInt(getopt.arg);
+		else if (c == 'd') opt.min_iden = parseFloat(getopt.arg);
+		else if (c == 'm') opt.min_slen = parseInt(getopt.arg);
+		else if (c == 'c') opt.max_clip_len = parseInt(getopt.arg);
+		else if (c == 'r') opt.max_ratio0 = parseFloat(getopt.arg);
+		else if (c == 'E') opt.dbg_ev = true;
+	}
+	if (args.length - getopt.ind < 1) {
+		print("Usage: mmphase.js [options] <map-with-cs.paf> [reads.fa]");
+		print("Options:");
+		print("  -l INT    min read length [" + opt.min_rlen + "]");
+		print("  -b INT    min alignment length [" + opt.min_blen + "]");
+		print("  -d FLOAT  min identity [" + opt.min_iden + "]");
+		print("  -s INT    min match length [" + opt.min_mlen + "]");
+		print("  -c INT    max clip length [" + opt.max_clip_len + "]");
+		print("  -r FLOAT  initial ratio for haplotype filtering [" + opt.max_ratio0 + "]");
+		return 0;
+	}
+
+	opt.ec = args.length - getopt.ind < 2? false : true;
+	if (!opt.ec) {
+		print("CC");
+		print("CC", "SQ  qName  qLen    nHits");
+		print("CC", "TS  index  qStart  qEnd  tName  tLen  tStart  tEnd  nConsistent  lCons  nConflictive  lConf  score");
+		print("CC");
+	}
+
+	var buf = new Bytes(), ecb = new Bytes();
+	var fp_paf = new File(args[getopt.ind]);
+	var fp_seq = args.length - getopt.ind >= 2? new File(args[getopt.ind+1]) : null;
+	var a = [];
+	while (fp_paf.readline(buf) >= 0) {
+		var t = buf.toString().split("\t");
+		if (a.length > 0 && a[0][0] != t[0]) {
+			process_paf(a, opt, fp_seq, buf, ecb);
+			a.length = 0;
+		}
+		for (var i = 1; i <= 3; ++i) t[i] = parseInt(t[i]);
+		if (t[1] < opt.min_rlen) continue;
+		for (var i = 6; i <= 10; ++i) t[i] = parseInt(t[i]);
+		if (t[10] < opt.min_blen) continue;
+		a.push(t);
+	}
+	if (a.length >= 0)
+		process_paf(a, opt, fp_seq, buf, ecb);
+	if (fp_seq) fp_seq.close();
+	fp_paf.close();
+	ecb.destroy();
+	buf.destroy();
+}
+
+var ret = main(arguments)
+exit(ret)

mmpriv.h

@@ -4,52 +4,126 @@
 #include <assert.h>
 #include "minimap.h"
 #include "bseq.h"
+#include "kseq.h"
 
 #define MM_PARENT_UNSET   (-1)
 #define MM_PARENT_TMP_PRI (-2)
 
-#define MM_DBG_NO_KALLOC   0x1
+#define MM_DBG_NO_KALLOC     0x1
-#define MM_DBG_PRINT_QNAME 0x2
+#define MM_DBG_PRINT_QNAME   0x2
-#define MM_DBG_PRINT_SEED  0x4
+#define MM_DBG_PRINT_SEED    0x4
+#define MM_DBG_PRINT_ALN_SEQ 0x8
+#define MM_DBG_PRINT_CHAIN   0x10
+
+#define MM_SEED_LONG_JOIN  (1ULL<<40)
+#define MM_SEED_IGNORE     (1ULL<<41)
+#define MM_SEED_TANDEM     (1ULL<<42)
+#define MM_SEED_SELF       (1ULL<<43)
+
+#define MM_SEED_SEG_SHIFT  48
+#define MM_SEED_SEG_MASK   (0xffULL<<(MM_SEED_SEG_SHIFT))
 
 #ifndef kroundup32
 #define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
 #endif
 
+#define mm_seq4_set(s, i, c) ((s)[(i)>>3] |= (uint32_t)(c) << (((i)&7)<<2))
+#define mm_seq4_get(s, i)    ((s)[(i)>>3] >> (((i)&7)<<2) & 0xf)
+
+#define MALLOC(type, len) ((type*)malloc((len) * sizeof(type)))
+#define CALLOC(type, len) ((type*)calloc((len), sizeof(type)))
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#ifndef KSTRING_T
+typedef struct {
-#define KSTRING_T kstring_t
+	uint32_t n;
-typedef struct __kstring_t {
+	uint32_t q_pos;
-	unsigned l, m;
+	uint32_t q_span:31, flt:1;
-	char *s;
+	uint32_t seg_id:31, is_tandem:1;
-} kstring_t;
+	const uint64_t *cr;
-#endif
+} mm_seed_t;
+
+typedef struct {
+	int n_u, n_a;
+	uint64_t *u;
+	mm128_t *a;
+} mm_seg_t;
 
 double cputime(void);
 double realtime(void);
+long peakrss(void);
 
 void radix_sort_128x(mm128_t *beg, mm128_t *end);
 void radix_sort_64(uint64_t *beg, uint64_t *end);
 uint32_t ks_ksmall_uint32_t(size_t n, uint32_t arr[], size_t kk);
 
-void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r);
+void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p);
-void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r);
-int mm_chain_dp(int max_dist, int bw, int max_skip, int min_cnt, int min_sc, int64_t n, mm128_t *a, uint64_t **_u, void *km);
-mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, int *n_regs_, mm_reg1_t *regs, mm128_t *a);
 
-mm_reg1_t *mm_gen_regs(void *km, int qlen, int n_u, uint64_t *u, mm128_t *a);
+mm_seed_t *mm_collect_matches(void *km, int *_n_m, int qlen, int max_occ, int max_max_occ, int dist, const mm_idx_t *mi, const mm128_v *mv, int64_t *n_a, int *rep_len, int *n_mini_pos, uint64_t **mini_pos);
-void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a);
+void mm_seed_mz_flt(void *km, mm128_v *mv, int32_t q_occ_max, float q_occ_frac);
+
+double mm_event_identity(const mm_reg1_t *r);
+int mm_write_sam_hdr(const mm_idx_t *mi, const char *rg, const char *ver, int argc, char *argv[]);
+void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int64_t opt_flag);
+void mm_write_paf3(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int64_t opt_flag, int rep_len);
+void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs);
+void mm_write_sam2(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regs, const mm_reg1_t *const* regs, void *km, int64_t opt_flag);
+void mm_write_sam3(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int seg_idx, int reg_idx, int n_seg, const int *n_regss, const mm_reg1_t *const* regss, void *km, int64_t opt_flag, int rep_len);
+
+void mm_idxopt_init(mm_idxopt_t *opt);
+const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n);
+int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f);
+int mm_idx_getseq2(const mm_idx_t *mi, int is_rev, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq);
+mm_reg1_t *mm_align_skeleton(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, const char *qstr, int *n_regs_, mm_reg1_t *regs, mm128_t *a);
+mm_reg1_t *mm_gen_regs(void *km, uint32_t hash, int qlen, int n_u, uint64_t *u, mm128_t *a, int is_qstrand);
+
+mm128_t *mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int max_iter, int min_cnt, int min_sc, float gap_scale,
+					 int is_cdna, int n_segs, int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km);
+mm128_t *mg_lchain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int max_iter, int min_cnt, int min_sc, float chn_pen_gap, float chn_pen_skip,
+					  int is_cdna, int n_segs, int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km);
+mm128_t *mg_lchain_rmq(int max_dist, int max_dist_inner, int bw, int max_chn_skip, int cap_rmq_size, int min_cnt, int min_sc, float chn_pen_gap, float chn_pen_skip,
+					   int64_t n, mm128_t *a, int *n_u_, uint64_t **_u, void *km);
+
+void mm_mark_alt(const mm_idx_t *mi, int n, mm_reg1_t *r);
+void mm_split_reg(mm_reg1_t *r, mm_reg1_t *r2, int n, int qlen, mm128_t *a, int is_qstrand);
 void mm_sync_regs(void *km, int n_regs, mm_reg1_t *regs);
+int mm_squeeze_a(void *km, int n_regs, mm_reg1_t *regs, mm128_t *a);
 int mm_set_sam_pri(int n, mm_reg1_t *r);
-void mm_set_parent(void *km, float mask_level, int n, mm_reg1_t *r);
+void mm_set_parent(void *km, float mask_level, int mask_len, int n, mm_reg1_t *r, int sub_diff, int hard_mask_level, float alt_diff_frac);
-void mm_select_sub(void *km, float mask_level, float pri_ratio, int best_n, int *n_, mm_reg1_t *r);
+void mm_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int check_strand, int min_strand_sc, int *n_, mm_reg1_t *r);
-void mm_filter_regs(void *km, const mm_mapopt_t *opt, int *n_regs, mm_reg1_t *regs);
+void mm_select_sub_multi(void *km, float pri_ratio, float pri1, float pri2, int max_gap_ref, int min_diff, int best_n, int n_segs, const int *qlens, int *n_, mm_reg1_t *r);
-void mm_join_long(void *km, const mm_mapopt_t *opt, int qlen, int *n_regs, mm_reg1_t *regs, mm128_t *a);
+int mm_filter_strand_retained(int n_regs, mm_reg1_t *r);
-void mm_hit_sort_by_dp(void *km, int *n_regs, mm_reg1_t *r);
+void mm_filter_regs(const mm_mapopt_t *opt, int qlen, int *n_regs, mm_reg1_t *regs);
-void mm_set_mapq(int n_regs, mm_reg1_t *regs);
+void mm_hit_sort(void *km, int *n_regs, mm_reg1_t *r, float alt_diff_frac);
+void mm_set_mapq(void *km, int n_regs, mm_reg1_t *regs, int min_chain_sc, int match_sc, int rep_len, int is_sr);
+void mm_update_dp_max(int qlen, int n_regs, mm_reg1_t *regs, float frac, int a, int b);
+
+void mm_est_err(const mm_idx_t *mi, int qlen, int n_regs, mm_reg1_t *regs, const mm128_t *a, int32_t n, const uint64_t *mini_pos);
+
+mm_seg_t *mm_seg_gen(void *km, uint32_t hash, int n_segs, const int *qlens, int n_regs0, const mm_reg1_t *regs0, int *n_regs, mm_reg1_t **regs, const mm128_t *a);
+void mm_seg_free(void *km, int n_segs, mm_seg_t *segs);
+void mm_pair(void *km, int max_gap_ref, int dp_bonus, int sub_diff, int match_sc, const int *qlens, int *n_regs, mm_reg1_t **regs);
+
+FILE *mm_split_init(const char *prefix, const mm_idx_t *mi);
+mm_idx_t *mm_split_merge_prep(const char *prefix, int n_splits, FILE **fp, uint32_t *n_seq_part);
+int mm_split_merge(int n_segs, const char **fn, const mm_mapopt_t *opt, int n_split_idx);
+void mm_split_rm_tmp(const char *prefix, int n_splits);
+
+void mm_err_puts(const char *str);
+void mm_err_fwrite(const void *p, size_t size, size_t nitems, FILE *fp);
+void mm_err_fread(void *p, size_t size, size_t nitems, FILE *fp);
+
+static inline float mg_log2(float x) // NB: this doesn't work when x<2
+{
+	union { float f; uint32_t i; } z = { x };
+	float log_2 = ((z.i >> 23) & 255) - 128;
+	z.i &= ~(255 << 23);
+	z.i += 127 << 23;
+	log_2 += (-0.34484843f * z.f + 2.02466578f) * z.f - 0.67487759f;
+	return log_2;
+}
 
 #ifdef __cplusplus
 }

options.c

@@ -0,0 +1,234 @@
+#include <stdio.h>
+#include <limits.h>
+#include "mmpriv.h"
+
+void mm_idxopt_init(mm_idxopt_t *opt)
+{
+	memset(opt, 0, sizeof(mm_idxopt_t));
+	opt->k = 15, opt->w = 10, opt->flag = 0;
+	opt->bucket_bits = 14;
+	opt->mini_batch_size = 50000000;
+	opt->batch_size = 4000000000ULL;
+}
+
+void mm_mapopt_init(mm_mapopt_t *opt)
+{
+	memset(opt, 0, sizeof(mm_mapopt_t));
+	opt->seed = 11;
+	opt->mid_occ_frac = 2e-4f;
+	opt->min_mid_occ = 10;
+	opt->max_mid_occ = 1000000;
+	opt->sdust_thres = 0; // no SDUST masking
+	opt->q_occ_frac = 0.01f;
+
+	opt->min_cnt = 3;
+	opt->min_chain_score = 40;
+	opt->bw = 500, opt->bw_long = 20000;
+	opt->max_gap = 5000;
+	opt->max_gap_ref = -1;
+	opt->max_chain_skip = 25;
+	opt->max_chain_iter = 5000;
+	opt->rmq_inner_dist = 1000;
+	opt->rmq_size_cap = 100000;
+	opt->rmq_rescue_size = 1000;
+	opt->rmq_rescue_ratio = 0.1f;
+	opt->chain_gap_scale = 0.8f;
+	opt->chain_skip_scale = 0.0f;
+	opt->max_max_occ = 4095;
+	opt->occ_dist = 500;
+
+	opt->mask_level = 0.5f;
+	opt->mask_len = INT_MAX;
+	opt->pri_ratio = 0.8f;
+	opt->best_n = 5;
+
+	opt->alt_drop = 0.15f;
+
+	opt->a = 2, opt->b = 4, opt->q = 4, opt->e = 2, opt->q2 = 24, opt->e2 = 1;
+	opt->sc_ambi = 1;
+	opt->zdrop = 400, opt->zdrop_inv = 200;
+	opt->end_bonus = -1;
+	opt->min_dp_max = opt->min_chain_score * opt->a;
+	opt->min_ksw_len = 200;
+	opt->anchor_ext_len = 20, opt->anchor_ext_shift = 6;
+	opt->max_clip_ratio = 1.0f;
+	opt->mini_batch_size = 500000000;
+	opt->max_sw_mat = 100000000;
+	opt->cap_kalloc = 1000000000;
+
+	opt->rank_min_len = 500;
+	opt->rank_frac = 0.9f;
+
+	opt->pe_ori = 0; // FF
+	opt->pe_bonus = 33;
+}
+
+void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi)
+{
+	if ((opt->flag & MM_F_SPLICE_FOR) || (opt->flag & MM_F_SPLICE_REV))
+		opt->flag |= MM_F_SPLICE;
+	if (opt->mid_occ <= 0) {
+		opt->mid_occ = mm_idx_cal_max_occ(mi, opt->mid_occ_frac);
+		if (opt->mid_occ < opt->min_mid_occ)
+			opt->mid_occ = opt->min_mid_occ;
+		if (opt->max_mid_occ > opt->min_mid_occ && opt->mid_occ > opt->max_mid_occ)
+			opt->mid_occ = opt->max_mid_occ;
+	}
+	if (opt->bw_long < opt->bw) opt->bw_long = opt->bw;
+	if (mm_verbose >= 3)
+		fprintf(stderr, "[M::%s::%.3f*%.2f] mid_occ = %d\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), opt->mid_occ);
+}
+
+void mm_mapopt_max_intron_len(mm_mapopt_t *opt, int max_intron_len)
+{
+	if ((opt->flag & MM_F_SPLICE) && max_intron_len > 0)
+		opt->max_gap_ref = opt->bw = opt->bw_long = max_intron_len;
+}
+
+int mm_set_opt(const char *preset, mm_idxopt_t *io, mm_mapopt_t *mo)
+{
+	if (preset == 0) {
+		mm_idxopt_init(io);
+		mm_mapopt_init(mo);
+	} else if (strcmp(preset, "map-ont") == 0) { // this is the same as the default
+	} else if (strcmp(preset, "ava-ont") == 0) {
+		io->flag = 0, io->k = 15, io->w = 5;
+		mo->flag |= MM_F_ALL_CHAINS | MM_F_NO_DIAG | MM_F_NO_DUAL | MM_F_NO_LJOIN;
+		mo->min_chain_score = 100, mo->pri_ratio = 0.0f, mo->max_chain_skip = 25;
+		mo->bw = mo->bw_long = 2000;
+		mo->occ_dist = 0;
+	} else if (strcmp(preset, "map10k") == 0 || strcmp(preset, "map-pb") == 0) {
+		io->flag |= MM_I_HPC, io->k = 19;
+	} else if (strcmp(preset, "ava-pb") == 0) {
+		io->flag |= MM_I_HPC, io->k = 19, io->w = 5;
+		mo->flag |= MM_F_ALL_CHAINS | MM_F_NO_DIAG | MM_F_NO_DUAL | MM_F_NO_LJOIN;
+		mo->min_chain_score = 100, mo->pri_ratio = 0.0f, mo->max_chain_skip = 25;
+		mo->bw_long = mo->bw;
+		mo->occ_dist = 0;
+	} else if (strcmp(preset, "map-hifi") == 0 || strcmp(preset, "map-ccs") == 0) {
+		io->flag = 0, io->k = 19, io->w = 19;
+		mo->max_gap = 10000;
+		mo->a = 1, mo->b = 4, mo->q = 6, mo->q2 = 26, mo->e = 2, mo->e2 = 1;
+		mo->occ_dist = 500;
+		mo->min_mid_occ = 50, mo->max_mid_occ = 500;
+		mo->min_dp_max = 200;
+	} else if (strncmp(preset, "asm", 3) == 0) {
+		io->flag = 0, io->k = 19, io->w = 19;
+		mo->bw = 1000, mo->bw_long = 100000;
+		mo->max_gap = 10000;
+		mo->flag |= MM_F_RMQ;
+		mo->min_mid_occ = 50, mo->max_mid_occ = 500;
+		mo->min_dp_max = 200;
+		mo->best_n = 50;
+		if (strcmp(preset, "asm5") == 0) {
+			mo->a = 1, mo->b = 19, mo->q = 39, mo->q2 = 81, mo->e = 3, mo->e2 = 1, mo->zdrop = mo->zdrop_inv = 200;
+		} else if (strcmp(preset, "asm10") == 0) {
+			mo->a = 1, mo->b = 9, mo->q = 16, mo->q2 = 41, mo->e = 2, mo->e2 = 1, mo->zdrop = mo->zdrop_inv = 200;
+		} else if (strcmp(preset, "asm20") == 0) {
+			mo->a = 1, mo->b = 4, mo->q = 6, mo->q2 = 26, mo->e = 2, mo->e2 = 1, mo->zdrop = mo->zdrop_inv = 200;
+			io->w = 10;
+		} else return -1;
+	} else if (strcmp(preset, "short") == 0 || strcmp(preset, "sr") == 0) {
+		io->flag = 0, io->k = 21, io->w = 11;
+		mo->flag |= MM_F_SR | MM_F_FRAG_MODE | MM_F_NO_PRINT_2ND | MM_F_2_IO_THREADS | MM_F_HEAP_SORT;
+		mo->pe_ori = 0<<1|1; // FR
+		mo->a = 2, mo->b = 8, mo->q = 12, mo->e = 2, mo->q2 = 24, mo->e2 = 1;
+		mo->zdrop = mo->zdrop_inv = 100;
+		mo->end_bonus = 10;
+		mo->max_frag_len = 800;
+		mo->max_gap = 100;
+		mo->bw = mo->bw_long = 100;
+		mo->pri_ratio = 0.5f;
+		mo->min_cnt = 2;
+		mo->min_chain_score = 25;
+		mo->min_dp_max = 40;
+		mo->best_n = 20;
+		mo->mid_occ = 1000;
+		mo->max_occ = 5000;
+		mo->mini_batch_size = 50000000;
+	} else if (strncmp(preset, "splice", 6) == 0 || strcmp(preset, "cdna") == 0) {
+		io->flag = 0, io->k = 15, io->w = 5;
+		mo->flag |= MM_F_SPLICE | MM_F_SPLICE_FOR | MM_F_SPLICE_REV | MM_F_SPLICE_FLANK;
+		mo->max_sw_mat = 0;
+		mo->max_gap = 2000, mo->max_gap_ref = mo->bw = mo->bw_long = 200000;
+		mo->a = 1, mo->b = 2, mo->q = 2, mo->e = 1, mo->q2 = 32, mo->e2 = 0;
+		mo->noncan = 9;
+		mo->junc_bonus = 9;
+		mo->zdrop = 200, mo->zdrop_inv = 100; // because mo->a is halved
+		if (strcmp(preset, "splice:hq") == 0)
+			mo->junc_bonus = 5, mo->b = 4, mo->q = 6, mo->q2 = 24;
+	} else return -1;
+	return 0;
+}
+
+int mm_check_opt(const mm_idxopt_t *io, const mm_mapopt_t *mo)
+{
+	if (mo->bw > mo->bw_long) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m with '-rNUM1,NUM2', NUM1 (%d) can't be larger than NUM2 (%d)\033[0m\n", mo->bw, mo->bw_long);
+		return -8;
+	}
+	if ((mo->flag & MM_F_RMQ) && (mo->flag & (MM_F_SR|MM_F_SPLICE))) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m --rmq doesn't work with --sr or --splice\033[0m\n");
+		return -7;
+	}
+	if (mo->split_prefix && (mo->flag & (MM_F_OUT_CS|MM_F_OUT_MD))) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m --cs or --MD doesn't work with --split-prefix\033[0m\n");
+		return -6;
+	}
+	if (io->k <= 0 || io->w <= 0) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m -k and -w must be positive\033[0m\n");
+		return -5;
+	}
+	if (mo->best_n < 0) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m -N must be no less than 0\033[0m\n");
+		return -4;
+	}
+	if (mo->best_n == 0 && mm_verbose >= 2)
+		fprintf(stderr, "[WARNING]\033[1;31m '-N 0' reduces mapping accuracy. Please use '--secondary=no' instead.\033[0m\n");
+	if (mo->pri_ratio < 0.0f || mo->pri_ratio > 1.0f) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m -p must be within 0 and 1 (including 0 and 1)\033[0m\n");
+		return -4;
+	}
+	if ((mo->flag & MM_F_FOR_ONLY) && (mo->flag & MM_F_REV_ONLY)) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m --for-only and --rev-only can't be applied at the same time\033[0m\n");
+		return -3;
+	}
+	if (mo->e <= 0 || mo->q <= 0) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m -O and -E must be positive\033[0m\n");
+		return -1;
+	}
+	if ((mo->q != mo->q2 || mo->e != mo->e2) && !(mo->e > mo->e2 && mo->q + mo->e < mo->q2 + mo->e2)) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m dual gap penalties violating E1>E2 and O1+E1<O2+E2\033[0m\n");
+		return -2;
+	}
+	if ((mo->q + mo->e) + (mo->q2 + mo->e2) > 127) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m scoring system violating ({-O}+{-E})+({-O2}+{-E2}) <= 127\033[0m\n");
+		return -1;
+	}
+	if (mo->zdrop < mo->zdrop_inv) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m Z-drop should not be less than inversion-Z-drop\033[0m\n");
+		return -5;
+	}
+	if ((mo->flag & MM_F_NO_PRINT_2ND) && (mo->flag & MM_F_ALL_CHAINS)) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m -X/-P and --secondary=no can't be applied at the same time\033[0m\n");
+		return -5;
+	}
+	if ((mo->flag & MM_F_QSTRAND) && ((mo->flag & (MM_F_OUT_SAM|MM_F_SPLICE|MM_F_FRAG_MODE)) || (io->flag & MM_I_HPC))) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m --qstrand doesn't work with -a, -H, --frag or --splice\033[0m\n");
+		return -5;
+	}
+	return 0;
+}

pe.c

@@ -0,0 +1,177 @@
+#include <stdlib.h>
+#include <math.h>
+#include "mmpriv.h"
+#include "kvec.h"
+
+void mm_select_sub_multi(void *km, float pri_ratio, float pri1, float pri2, int max_gap_ref, int min_diff, int best_n, int n_segs, const int *qlens, int *n_, mm_reg1_t *r)
+{
+	if (pri_ratio > 0.0f && *n_ > 0) {
+		int i, k, n = *n_, n_2nd = 0;
+		int max_dist = n_segs == 2? qlens[0] + qlens[1] + max_gap_ref : 0;
+		for (i = k = 0; i < n; ++i) {
+			int to_keep = 0;
+			if (r[i].parent == i) { // primary
+				to_keep = 1;
+			} else if (r[i].score + min_diff >= r[r[i].parent].score) {
+				to_keep = 1;
+			} else {
+				mm_reg1_t *p = &r[r[i].parent], *q = &r[i];
+				if (p->rev == q->rev && p->rid == q->rid && q->re - p->rs < max_dist && p->re - q->rs < max_dist) { // child and parent are close on the ref
+					if (q->score >= p->score * pri1)
+						to_keep = 1;
+				} else {
+					int is_par_both = (n_segs == 2 && p->qs < qlens[0] && p->qe > qlens[0]);
+					int is_chi_both = (n_segs == 2 && q->qs < qlens[0] && q->qe > qlens[0]);
+					if (is_chi_both || is_chi_both == is_par_both) {
+						if (q->score >= p->score * pri_ratio)
+							to_keep = 1;
+					} else { // the remaining case: is_chi_both == 0 && is_par_both == 1
+						if (q->score >= p->score * pri2)
+							to_keep = 1;
+					}
+				}
+			}
+			if (to_keep && r[i].parent != i) {
+				if (n_2nd++ >= best_n) to_keep = 0; // don't keep if there are too many secondary hits
+			}
+			if (to_keep) r[k++] = r[i];
+			else if (r[i].p) free(r[i].p);
+		}
+		if (k != n) mm_sync_regs(km, k, r); // removing hits requires sync()
+		*n_ = k;
+	}
+}
+
+void mm_set_pe_thru(const int *qlens, int *n_regs, mm_reg1_t **regs)
+{
+	int s, i, n_pri[2], pri[2];
+	n_pri[0] = n_pri[1] = 0;
+	pri[0] = pri[1] = -1;
+	for (s = 0; s < 2; ++s)
+		for (i = 0; i < n_regs[s]; ++i)
+			if (regs[s][i].id == regs[s][i].parent)
+				++n_pri[s], pri[s] = i;
+	if (n_pri[0] == 1 && n_pri[1] == 1) {
+		mm_reg1_t *p = &regs[0][pri[0]];
+		mm_reg1_t *q = &regs[1][pri[1]];
+		if (p->rid == q->rid && p->rev == q->rev && abs(p->rs - q->rs) < 3 && abs(p->re - q->re) < 3
+			&& ((p->qs == 0 && qlens[1] - q->qe == 0) || (q->qs == 0 && qlens[0] - p->qe == 0)))
+		{
+			p->pe_thru = q->pe_thru = 1;
+		}
+	}
+}
+
+#include "ksort.h"
+
+typedef struct {
+	int s, rev;
+	uint64_t key;
+	mm_reg1_t *r;
+} pair_arr_t;
+
+#define sort_key_pair(a) ((a).key)
+KRADIX_SORT_INIT(pair, pair_arr_t, sort_key_pair, 8)
+
+void mm_pair(void *km, int max_gap_ref, int pe_bonus, int sub_diff, int match_sc, const int *qlens, int *n_regs, mm_reg1_t **regs)
+{
+	int i, j, s, n, last[2], dp_thres, segs = 0, max_idx[2];
+	int64_t max;
+	pair_arr_t *a;
+	kvec_t(uint64_t) sc = {0,0,0};
+
+	a = (pair_arr_t*)kmalloc(km, (n_regs[0] + n_regs[1]) * sizeof(pair_arr_t));
+	for (s = n = 0, dp_thres = 0; s < 2; ++s) {
+		int max = 0;
+		for (i = 0; i < n_regs[s]; ++i) {
+			a[n].s = s;
+			a[n].r = &regs[s][i];
+			a[n].rev = a[n].r->rev;
+			a[n].key = (uint64_t)a[n].r->rid << 32 | a[n].r->rs<<1 | (s^a[n].rev);
+			max = max > a[n].r->p->dp_max? max : a[n].r->p->dp_max;
+			++n;
+			segs |= 1<<s;
+		}
+		dp_thres += max;
+	}
+	if (segs != 3) {
+		kfree(km, a); // only one end is mapped
+		return;
+	}
+	dp_thres -= pe_bonus;
+	if (dp_thres < 0) dp_thres = 0;
+	radix_sort_pair(a, a + n);
+
+	max = -1;
+	max_idx[0] = max_idx[1] = -1;
+	last[0] = last[1] = -1;
+	kv_resize(uint64_t, km, sc, (size_t)n);
+	for (i = 0; i < n; ++i) {
+		if (a[i].key & 1) { // reverse first read or forward second read
+			mm_reg1_t *q, *r;
+			if (last[a[i].rev] < 0) continue;
+			r = a[i].r;
+			q = a[last[a[i].rev]].r;
+			if (r->rid != q->rid || r->rs - q->re > max_gap_ref) continue;
+			for (j = last[a[i].rev]; j >= 0; --j) {
+				int64_t score;
+				if (a[j].rev != a[i].rev || a[j].s == a[i].s) continue;
+				q = a[j].r;
+				if (r->rid != q->rid || r->rs - q->re > max_gap_ref) break;
+				if (r->p->dp_max + q->p->dp_max < dp_thres) continue;
+				score = (int64_t)(r->p->dp_max + q->p->dp_max) << 32 | (r->hash + q->hash);
+				if (score > max)
+					max = score, max_idx[a[j].s] = j, max_idx[a[i].s] = i;
+				kv_push(uint64_t, km, sc, score);
+			}
+		} else { // forward first read or reverse second read
+			last[a[i].rev] = i;
+		}
+	}
+	if (sc.n > 1)
+		radix_sort_64(sc.a, sc.a + sc.n);
+
+	if (sc.n > 0 && max > 0) { // found at least one pair
+		int n_sub = 0, mapq_pe;
+		mm_reg1_t *r[2];
+		r[0] = a[max_idx[0]].r, r[1] = a[max_idx[1]].r;
+		r[0]->proper_frag = r[1]->proper_frag = 1;
+		for (s = 0; s < 2; ++s) {
+			if (r[s]->id != r[s]->parent) { // then lift to primary and update parent
+				mm_reg1_t *p = &regs[s][r[s]->parent];
+				for (i = 0; i < n_regs[s]; ++i)
+					if (regs[s][i].parent == p->id)
+						regs[s][i].parent = r[s]->id;
+				p->mapq = 0;
+			}
+			if (!r[s]->sam_pri) { // then sync sam_pri
+				for (i = 0; i < n_regs[s]; ++i)
+					regs[s][i].sam_pri = 0;
+				r[s]->sam_pri = 1;
+			}
+		}
+		mapq_pe = r[0]->mapq > r[1]->mapq? r[0]->mapq : r[1]->mapq;
+		for (i = 0; i < (int)sc.n; ++i)
+			if ((sc.a[i]>>32) + sub_diff >= (uint64_t)max>>32)
+				++n_sub;
+		if (sc.n > 1) {
+			int mapq_pe_alt;
+			mapq_pe_alt = (int)(6.02f * ((max>>32) - (sc.a[sc.n - 2]>>32)) / match_sc - 4.343f * logf(n_sub)); // n_sub > 0 because it counts the optimal, too
+			mapq_pe = mapq_pe < mapq_pe_alt? mapq_pe : mapq_pe_alt;
+		}
+		if (r[0]->mapq < mapq_pe) r[0]->mapq = (int)(.2f * r[0]->mapq + .8f * mapq_pe + .499f);
+		if (r[1]->mapq < mapq_pe) r[1]->mapq = (int)(.2f * r[1]->mapq + .8f * mapq_pe + .499f);
+		if (sc.n == 1) {
+			if (r[0]->mapq < 2) r[0]->mapq = 2;
+			if (r[1]->mapq < 2) r[1]->mapq = 2;
+		} else if ((uint64_t)max>>32 > sc.a[sc.n - 2]>>32) {
+			if (r[0]->mapq < 1) r[0]->mapq = 1;
+			if (r[1]->mapq < 1) r[1]->mapq = 1;
+		}
+	}
+
+	kfree(km, a);
+	kfree(km, sc.a);
+
+	mm_set_pe_thru(qlens, n_regs, regs);
+}

python/README.rst

@@ -0,0 +1,196 @@
+==============================
+Mappy: Minimap2 Python Binding
+==============================
+
+Mappy provides a convenient interface to `minimap2
+<https://github.com/lh3/minimap2>`_, a fast and accurate C program to align
+genomic and transcribe nucleotide sequences.
+
+Installation
+------------
+
+Mappy depends on `zlib <http://zlib.net>`_. It can be installed with `pip
+<https://en.wikipedia.org/wiki/Pip_(package_manager)>`_:
+
+.. code:: shell
+
+	pip install --user mappy
+
+or from the minimap2 github repo (`Cython <http://cython.org>`_ required):
+
+.. code:: shell
+
+	git clone https://github.com/lh3/minimap2
+	cd minimap2
+	python setup.py install
+
+Usage
+-----
+
+The following Python script demonstrates the key functionality of mappy:
+
+.. code:: python
+
+	import mappy as mp
+	a = mp.Aligner("test/MT-human.fa")  # load or build index
+	if not a: raise Exception("ERROR: failed to load/build index")
+	s = a.seq("MT_human", 100, 200)     # retrieve a subsequence from the index
+	print(mp.revcomp(s))                # reverse complement
+	for name, seq, qual in mp.fastx_read("test/MT-orang.fa"): # read a fasta/q sequence
+		for hit in a.map(seq): # traverse alignments
+			print("{}\t{}\t{}\t{}".format(hit.ctg, hit.r_st, hit.r_en, hit.cigar_str))
+
+APIs
+----
+
+Mappy implements two classes and two global function.
+
+Class mappy.Aligner
+~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+	mappy.Aligner(fn_idx_in=None, preset=None, ...)
+
+This constructor accepts the following arguments:
+
+* **fn_idx_in**: index or sequence file name. Minimap2 automatically tests the
+  file type. If a sequence file is provided, minimap2 builds an index. The
+  sequence file can be optionally gzip'd. This option has no effect if **seq**
+  is set.
+
+* **seq**: a single sequence to index. The sequence name will be set to
+  :code:`N/A`.
+
+* **preset**: minimap2 preset. Currently, minimap2 supports the following
+  presets: **sr** for single-end short reads; **map-pb** for PacBio
+  read-to-reference mapping; **map-ont** for Oxford Nanopore read mapping;
+  **splice** for long-read spliced alignment; **asm5** for assembly-to-assembly
+  alignment; **asm10** for full genome alignment of closely related species. Note
+  that the Python module does not support all-vs-all read overlapping.
+
+* **k**: k-mer length, no larger than 28
+
+* **w**: minimizer window size, no larger than 255
+
+* **min_cnt**: mininum number of minimizers on a chain
+
+* **min_chain_score**: minimum chaing score
+
+* **bw**: chaining and alignment band width
+
+* **best_n**: max number of alignments to return
+
+* **n_threads**: number of indexing threads; 3 by default
+
+* **extra_flags**: additional flags defined in minimap.h
+
+* **fn_idx_out**: name of file to which the index is written. This parameter
+  has no effect if **seq** is set.
+
+* **scoring**: scoring system. It is a tuple/list consisting of 4, 6 or 7
+  positive integers. The first 4 elements specify match scoring, mismatch
+  penalty, gap open and gap extension penalty. The 5th and 6th elements, if
+  present, set long-gap open and long-gap extension penalty. The 7th sets a
+  mismatch penalty involving ambiguous bases.
+
+.. code:: python
+
+	mappy.Aligner.map(seq, seq2=None, cs=False, MD=False)
+
+This method aligns :code:`seq` against the index. It is a generator, *yielding*
+a series of :code:`mappy.Alignment` objects. If :code:`seq2` is present, mappy
+performs paired-end alignment, assuming the two ends are in the FR orientation.
+Alignments of the two ends can be distinguished by the :code:`read_num` field
+(see Class mappy.Alignment below). Argument :code:`cs` asks mappy to generate
+the :code:`cs` tag; :code:`MD` is similar. These two arguments might slightly
+degrade performance and are not enabled by default.
+
+.. code:: python
+
+	mappy.Aligner.seq(name, start=0, end=0x7fffffff)
+
+This method retrieves a (sub)sequence from the index and returns it as a Python
+string. :code:`None` is returned if :code:`name` is not present in the index or
+the start/end coordinates are invalid.
+
+.. code:: python
+
+	mappy.Aligner.seq_names
+
+This property gives the array of sequence names in the index.
+
+Class mappy.Alignment
+~~~~~~~~~~~~~~~~~~~~~
+
+This class describes an alignment. An object of this class has the following
+properties:
+
+* **ctg**: name of the reference sequence the query is mapped to
+
+* **ctg_len**: total length of the reference sequence
+
+* **r_st** and **r_en**: start and end positions on the reference
+
+* **q_st** and **q_en**: start and end positions on the query
+
+* **strand**: +1 if on the forward strand; -1 if on the reverse strand
+
+* **mapq**: mapping quality
+
+* **blen**: length of the alignment, including both alignment matches and gaps
+  but excluding ambiguous bases.
+
+* **mlen**: length of the matching bases in the alignment, excluding ambiguous
+  base matches.
+
+* **NM**: number of mismatches, gaps and ambiguous positions in the alignment
+
+* **trans_strand**: transcript strand. +1 if on the forward strand; -1 if on the
+  reverse strand; 0 if unknown
+
+* **is_primary**: if the alignment is primary (typically the best and the first
+  to generate)
+
+* **read_num**: read number that the alignment corresponds to; 1 for the first
+  read and 2 for the second read
+
+* **cigar_str**: CIGAR string
+
+* **cigar**: CIGAR returned as an array of shape :code:`(n_cigar,2)`. The two
+  numbers give the length and the operator of each CIGAR operation.
+
+* **MD**: the :code:`MD` tag as in the SAM format. It is an empty string unless
+  the :code:`MD` argument is applied when calling :code:`mappy.Aligner.map()`.
+
+* **cs**: the :code:`cs` tag.
+
+An :code:`Alignment` object can be converted to a string with :code:`str()` in
+the following format:
+
+::
+
+	q_st  q_en  strand  ctg  ctg_len  r_st  r_en  mlen  blen  mapq  cg:Z:cigar_str
+
+It is effectively the PAF format without the QueryName and QueryLength columns
+(the first two columns in PAF).
+
+Miscellaneous Functions
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+	mappy.fastx_read(fn, read_comment=False)
+
+This generator function opens a FASTA/FASTQ file and *yields* a
+:code:`(name,seq,qual)` tuple for each sequence entry. The input file may be
+optionally gzip'd. If :code:`read_comment` is True, this generator yields
+a :code:`(name,seq,qual,comment)` tuple instead.
+
+.. code:: python
+
+	mappy.revcomp(seq)
+
+Return the reverse complement of DNA string :code:`seq`. This function
+recognizes IUB code and preserves the letter cases. Uracil :code:`U` is
+complemented to :code:`A`.

python/cmappy.h

@@ -0,0 +1,152 @@
+#ifndef CMAPPY_H
+#define CMAPPY_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include "minimap.h"
+#include "kseq.h"
+KSEQ_DECLARE(gzFile)
+
+typedef struct {
+	const char *ctg;
+	int32_t ctg_start, ctg_end;
+	int32_t qry_start, qry_end;
+	int32_t blen, mlen, NM, ctg_len;
+	uint8_t mapq, is_primary;
+	int8_t strand, trans_strand;
+	int32_t seg_id;
+	int32_t n_cigar32;
+	uint32_t *cigar32;
+} mm_hitpy_t;
+
+static inline void mm_reg2hitpy(const mm_idx_t *mi, mm_reg1_t *r, mm_hitpy_t *h)
+{
+	h->ctg = mi->seq[r->rid].name;
+	h->ctg_len = mi->seq[r->rid].len;
+	h->ctg_start = r->rs, h->ctg_end = r->re;
+	h->qry_start = r->qs, h->qry_end = r->qe;
+	h->strand = r->rev? -1 : 1;
+	h->mapq = r->mapq;
+	h->mlen = r->mlen;
+	h->blen = r->blen;
+	h->NM = r->blen - r->mlen + r->p->n_ambi;
+	h->trans_strand = r->p->trans_strand == 1? 1 : r->p->trans_strand == 2? -1 : 0;
+	h->is_primary = (r->id == r->parent);
+	h->seg_id = r->seg_id;
+	h->n_cigar32 = r->p->n_cigar;
+	h->cigar32 = r->p->cigar;
+}
+
+static inline void mm_free_reg1(mm_reg1_t *r)
+{
+	free(r->p);
+}
+
+static inline kseq_t *mm_fastx_open(const char *fn)
+{
+	gzFile fp;
+	fp = fn && strcmp(fn, "-") != 0? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+	return kseq_init(fp);
+}
+
+static inline void mm_fastx_close(kseq_t *ks)
+{
+	gzFile fp;
+	fp = ks->f->f;
+	kseq_destroy(ks);
+	gzclose(fp);
+}
+
+static inline int mm_verbose_level(int v)
+{
+	if (v >= 0) mm_verbose = v;
+	return mm_verbose;
+}
+
+static inline void mm_reset_timer(void)
+{
+	extern double realtime(void);
+	mm_realtime0 = realtime();
+}
+
+extern unsigned char seq_comp_table[256];
+static inline mm_reg1_t *mm_map_aux(const mm_idx_t *mi, const char *seq1, const char *seq2, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt)
+{
+	mm_reg1_t *r;
+
+	Py_BEGIN_ALLOW_THREADS
+	if (seq2 == 0) {
+		r = mm_map(mi, strlen(seq1), seq1, n_regs, b, opt, NULL);
+	} else {
+		int _n_regs[2];
+		mm_reg1_t *regs[2];
+		char *seq[2];
+		int i, len[2];
+
+		len[0] = strlen(seq1);
+		len[1] = strlen(seq2);
+		seq[0] = (char*)seq1;
+		seq[1] = strdup(seq2);
+		for (i = 0; i < len[1]>>1; ++i) {
+			int t = seq[1][len[1] - i - 1];
+			seq[1][len[1] - i - 1] = seq_comp_table[(uint8_t)seq[1][i]];
+			seq[1][i] = seq_comp_table[t];
+		}
+		if (len[1]&1) seq[1][len[1]>>1] = seq_comp_table[(uint8_t)seq[1][len[1]>>1]];
+		mm_map_frag(mi, 2, len, (const char**)seq, _n_regs, regs, b, opt, NULL);
+		for (i = 0; i < _n_regs[1]; ++i)
+			regs[1][i].rev = !regs[1][i].rev;
+		*n_regs = _n_regs[0] + _n_regs[1];
+		regs[0] = (mm_reg1_t*)realloc(regs[0], sizeof(mm_reg1_t) * (*n_regs));
+		memcpy(&regs[0][_n_regs[0]], regs[1], _n_regs[1] * sizeof(mm_reg1_t));
+		free(regs[1]);
+		r = regs[0];
+	}
+	Py_END_ALLOW_THREADS
+
+	return r;
+}
+
+static inline char *mappy_revcomp(int len, const uint8_t *seq)
+{
+	int i;
+	char *rev;
+	rev = (char*)malloc(len + 1);
+	for (i = 0; i < len; ++i)
+		rev[len - i - 1] = seq_comp_table[seq[i]];
+	rev[len] = 0;
+	return rev;
+}
+
+static char *mappy_fetch_seq(const mm_idx_t *mi, const char *name, int st, int en, int *len)
+{
+	int i, rid;
+	char *s;
+	*len = 0;
+	rid = mm_idx_name2id(mi, name);
+	if (rid < 0) return 0;
+	if ((uint32_t)st >= mi->seq[rid].len || st >= en) return 0;
+	if (en < 0 || (uint32_t)en > mi->seq[rid].len)
+		en = mi->seq[rid].len;
+	s = (char*)malloc(en - st + 1);
+	*len = mm_idx_getseq(mi, rid, st, en, (uint8_t*)s);
+	for (i = 0; i < *len; ++i)
+		s[i] = "ACGTN"[(uint8_t)s[i]];
+	s[*len] = 0;
+	return s;
+}
+
+static mm_idx_t *mappy_idx_seq(int w, int k, int is_hpc, int bucket_bits, const char *seq, int len)
+{
+	const char *fake_name = "N/A";
+	char *s;
+	mm_idx_t *mi;
+	s = (char*)calloc(len + 1, 1);
+	memcpy(s, seq, len);
+	mi = mm_idx_str(w, k, is_hpc, bucket_bits, 1, (const char**)&s, (const char**)&fake_name);
+	free(s);
+	return mi;
+}
+
+#endif

python/cmappy.pxd

@@ -0,0 +1,153 @@
+from libc.stdint cimport int8_t, uint8_t, int32_t, int64_t, uint32_t, uint64_t
+
+cdef extern from "minimap.h":
+	#
+	# Options
+	#
+	ctypedef struct mm_idxopt_t:
+		short k, w, flag, bucket_bits
+		int64_t mini_batch_size
+		uint64_t batch_size
+
+	ctypedef struct mm_mapopt_t:
+		int64_t flag
+		int seed
+		int sdust_thres
+
+		int max_qlen
+
+		int bw, bw_long
+		int max_gap, max_gap_ref
+		int max_frag_len
+		int max_chain_skip, max_chain_iter
+		int min_cnt
+		int min_chain_score
+		float chain_gap_scale
+		float chain_skip_scale
+		int rmq_size_cap, rmq_inner_dist
+		int rmq_rescue_size
+		float rmq_rescue_ratio
+
+		float mask_level
+		int mask_len
+		float pri_ratio
+		int best_n
+
+		float alt_drop
+
+		int a, b, q, e, q2, e2
+		int sc_ambi
+		int noncan
+		int junc_bonus
+		int zdrop, zdrop_inv
+		int end_bonus
+		int min_dp_max
+		int min_ksw_len
+		int anchor_ext_len, anchor_ext_shift
+		float max_clip_ratio
+
+		int rank_min_len
+		float rank_frac
+
+		int pe_ori, pe_bonus
+
+		float mid_occ_frac
+		float q_occ_frac
+		int32_t min_mid_occ
+		int32_t mid_occ
+		int32_t max_occ
+		int64_t mini_batch_size
+		int64_t max_sw_mat
+		int64_t cap_kalloc
+
+		const char *split_prefix
+
+	int mm_set_opt(char *preset, mm_idxopt_t *io, mm_mapopt_t *mo)
+	int mm_verbose
+
+	#
+	# Indexing
+	#
+	ctypedef struct mm_idx_seq_t:
+		char *name
+		uint64_t offset
+		uint32_t len
+
+	ctypedef struct mm_idx_bucket_t:
+		pass
+
+	ctypedef struct mm_idx_t:
+		int32_t b, w, k, flag
+		uint32_t n_seq
+		mm_idx_seq_t *seq
+		uint32_t *S
+		mm_idx_bucket_t *B
+		void *km
+		void *h
+
+	ctypedef struct mm_idx_reader_t:
+		pass
+
+	mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out)
+	mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
+	void mm_idx_reader_close(mm_idx_reader_t *r)
+	void mm_idx_destroy(mm_idx_t *mi)
+	void mm_mapopt_update(mm_mapopt_t *opt, const mm_idx_t *mi)
+
+	int mm_idx_index_name(mm_idx_t *mi)
+
+	#
+	# Mapping (key struct defined in cmappy.h below)
+	#
+	ctypedef struct mm_reg1_t:
+		pass
+
+	ctypedef struct mm_tbuf_t:
+		pass
+
+	mm_tbuf_t *mm_tbuf_init()
+	void mm_tbuf_destroy(mm_tbuf_t *b)
+	void *mm_tbuf_get_km(mm_tbuf_t *b)
+	int mm_gen_cs(void *km, char **buf, int *max_len, const mm_idx_t *mi, const mm_reg1_t *r, const char *seq, int no_iden)
+	int mm_gen_MD(void *km, char **buf, int *max_len, const mm_idx_t *mi, const mm_reg1_t *r, const char *seq)
+
+#
+# Helper header (because it is hard to expose mm_reg1_t with Cython)
+#
+cdef extern from "cmappy.h":
+	ctypedef struct mm_hitpy_t:
+		const char *ctg
+		int32_t ctg_start, ctg_end
+		int32_t qry_start, qry_end
+		int32_t blen, mlen, NM, ctg_len
+		uint8_t mapq, is_primary
+		int8_t strand, trans_strand
+		int32_t seg_id
+		int32_t n_cigar32
+		uint32_t *cigar32
+
+	void mm_reg2hitpy(const mm_idx_t *mi, mm_reg1_t *r, mm_hitpy_t *h)
+	void mm_free_reg1(mm_reg1_t *r)
+	mm_reg1_t *mm_map_aux(const mm_idx_t *mi, const char *seq1, const char *seq2, int *n_regs, mm_tbuf_t *b, const mm_mapopt_t *opt)
+	char *mappy_fetch_seq(const mm_idx_t *mi, const char *name, int st, int en, int *l)
+	mm_idx_t *mappy_idx_seq(int w, int k, int is_hpc, int bucket_bits, const char *seq, int l)
+
+	ctypedef struct kstring_t:
+		unsigned l, m
+		char *s
+
+	ctypedef struct kstream_t:
+		pass
+
+	ctypedef struct kseq_t:
+		kstring_t name, comment, seq, qual
+		int last_char
+		kstream_t *f
+
+	kseq_t *mm_fastx_open(const char *fn)
+	void mm_fastx_close(kseq_t *ks)
+	int kseq_read(kseq_t *seq)
+
+	char *mappy_revcomp(int l, const uint8_t *seq)
+	int mm_verbose_level(int v)
+	void mm_reset_timer()

python/mappy.pyx

@@ -0,0 +1,273 @@
+from libc.stdint cimport uint8_t, int8_t
+from libc.stdlib cimport free
+cimport cmappy
+import sys
+
+__version__ = '2.24'
+
+cmappy.mm_reset_timer()
+
+cdef class Alignment:
+	cdef int _ctg_len, _r_st, _r_en
+	cdef int _q_st, _q_en
+	cdef int _NM, _mlen, _blen
+	cdef int8_t _strand, _trans_strand
+	cdef uint8_t _mapq, _is_primary
+	cdef int _seg_id
+	cdef _ctg, _cigar, _cs, _MD # these are python objects
+
+	def __cinit__(self, ctg, cl, cs, ce, strand, qs, qe, mapq, cigar, is_primary, mlen, blen, NM, trans_strand, seg_id, cs_str, MD_str):
+		self._ctg = ctg if isinstance(ctg, str) else ctg.decode()
+		self._ctg_len, self._r_st, self._r_en = cl, cs, ce
+		self._strand, self._q_st, self._q_en = strand, qs, qe
+		self._NM, self._mlen, self._blen = NM, mlen, blen
+		self._mapq = mapq
+		self._cigar = cigar
+		self._is_primary = is_primary
+		self._trans_strand = trans_strand
+		self._seg_id = seg_id
+		self._cs = cs_str
+		self._MD = MD_str
+
+	@property
+	def ctg(self): return self._ctg
+
+	@property
+	def ctg_len(self): return self._ctg_len
+
+	@property
+	def r_st(self): return self._r_st
+
+	@property
+	def r_en(self): return self._r_en
+
+	@property
+	def strand(self): return self._strand
+
+	@property
+	def trans_strand(self): return self._trans_strand
+
+	@property
+	def blen(self): return self._blen
+
+	@property
+	def mlen(self): return self._mlen
+
+	@property
+	def NM(self): return self._NM
+
+	@property
+	def is_primary(self): return (self._is_primary != 0)
+
+	@property
+	def q_st(self): return self._q_st
+
+	@property
+	def q_en(self): return self._q_en
+
+	@property
+	def mapq(self): return self._mapq
+
+	@property
+	def cigar(self): return self._cigar
+
+	@property
+	def read_num(self): return self._seg_id + 1
+
+	@property
+	def cs(self): return self._cs
+
+	@property
+	def MD(self): return self._MD
+
+	@property
+	def cigar_str(self):
+		return "".join(map(lambda x: str(x[0]) + 'MIDNSHP=XB'[x[1]], self._cigar))
+
+	def __str__(self):
+		if self._strand > 0: strand = '+'
+		elif self._strand < 0: strand = '-'
+		else: strand = '?'
+		if self._is_primary != 0: tp = 'tp:A:P'
+		else: tp = 'tp:A:S'
+		if self._trans_strand > 0: ts = 'ts:A:+'
+		elif self._trans_strand < 0: ts = 'ts:A:-'
+		else: ts = 'ts:A:.'
+		a = [str(self._q_st), str(self._q_en), strand, self._ctg, str(self._ctg_len), str(self._r_st), str(self._r_en),
+			str(self._mlen), str(self._blen), str(self._mapq), tp, ts, "cg:Z:" + self.cigar_str]
+		if self._cs != "": a.append("cs:Z:" + self._cs)
+		return "\t".join(a)
+
+cdef class ThreadBuffer:
+	cdef cmappy.mm_tbuf_t *_b
+
+	def __cinit__(self):
+		self._b = cmappy.mm_tbuf_init()
+
+	def __dealloc__(self):
+		cmappy.mm_tbuf_destroy(self._b)
+
+cdef class Aligner:
+	cdef cmappy.mm_idx_t *_idx
+	cdef cmappy.mm_idxopt_t idx_opt
+	cdef cmappy.mm_mapopt_t map_opt
+
+	def __cinit__(self, fn_idx_in=None, preset=None, k=None, w=None, min_cnt=None, min_chain_score=None, min_dp_score=None, bw=None, best_n=None, n_threads=3, fn_idx_out=None, max_frag_len=None, extra_flags=None, seq=None, scoring=None):
+		self._idx = NULL
+		cmappy.mm_set_opt(NULL, &self.idx_opt, &self.map_opt) # set the default options
+		if preset is not None:
+			cmappy.mm_set_opt(str.encode(preset), &self.idx_opt, &self.map_opt) # apply preset
+		self.map_opt.flag |= 4 # always perform alignment
+		self.idx_opt.batch_size = 0x7fffffffffffffffL # always build a uni-part index
+		if k is not None: self.idx_opt.k = k
+		if w is not None: self.idx_opt.w = w
+		if min_cnt is not None: self.map_opt.min_cnt = min_cnt
+		if min_chain_score is not None: self.map_opt.min_chain_score = min_chain_score
+		if min_dp_score is not None: self.map_opt.min_dp_max = min_dp_score
+		if bw is not None: self.map_opt.bw = bw
+		if best_n is not None: self.map_opt.best_n = best_n
+		if max_frag_len is not None: self.map_opt.max_frag_len = max_frag_len
+		if extra_flags is not None: self.map_opt.flag |= extra_flags
+		if scoring is not None and len(scoring) >= 4:
+			self.map_opt.a, self.map_opt.b = scoring[0], scoring[1]
+			self.map_opt.q, self.map_opt.e = scoring[2], scoring[3]
+			self.map_opt.q2, self.map_opt.e2 = self.map_opt.q, self.map_opt.e
+			if len(scoring) >= 6:
+				self.map_opt.q2, self.map_opt.e2 = scoring[4], scoring[5]
+				if len(scoring) >= 7:
+					self.map_opt.sc_ambi = scoring[6]
+
+		cdef cmappy.mm_idx_reader_t *r;
+
+		if seq is None:
+			if fn_idx_out is None:
+				r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, NULL)
+			else:
+				r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, str.encode(fn_idx_out))
+			if r is not NULL:
+				self._idx = cmappy.mm_idx_reader_read(r, n_threads) # NB: ONLY read the first part
+				cmappy.mm_idx_reader_close(r)
+				cmappy.mm_mapopt_update(&self.map_opt, self._idx)
+				cmappy.mm_idx_index_name(self._idx)
+		else:
+			self._idx = cmappy.mappy_idx_seq(self.idx_opt.w, self.idx_opt.k, self.idx_opt.flag&1, self.idx_opt.bucket_bits, str.encode(seq), len(seq))
+			cmappy.mm_mapopt_update(&self.map_opt, self._idx)
+			self.map_opt.mid_occ = 1000 # don't filter high-occ seeds
+
+	def __dealloc__(self):
+		if self._idx is not NULL:
+			cmappy.mm_idx_destroy(self._idx)
+
+	def __bool__(self):
+		return (self._idx != NULL)
+
+	def map(self, seq, seq2=None, buf=None, cs=False, MD=False, max_frag_len=None, extra_flags=None):
+		cdef cmappy.mm_reg1_t *regs
+		cdef cmappy.mm_hitpy_t h
+		cdef ThreadBuffer b
+		cdef int n_regs
+		cdef char *cs_str = NULL
+		cdef int l_cs_str, m_cs_str = 0
+		cdef void *km
+		cdef cmappy.mm_mapopt_t map_opt
+
+		if self._idx == NULL: return
+		map_opt = self.map_opt
+		if max_frag_len is not None: map_opt.max_frag_len = max_frag_len
+		if extra_flags is not None: map_opt.flag |= extra_flags
+
+		if self._idx is NULL: return None
+		if buf is None: b = ThreadBuffer()
+		else: b = buf
+		km = cmappy.mm_tbuf_get_km(b._b)
+
+		_seq = seq if isinstance(seq, bytes) else seq.encode()
+		if seq2 is None:
+			regs = cmappy.mm_map_aux(self._idx, _seq, NULL,  &n_regs, b._b, &map_opt)
+		else:
+			_seq2 = seq2 if isinstance(seq2, bytes) else seq2.encode()
+			regs = cmappy.mm_map_aux(self._idx, _seq, _seq2, &n_regs, b._b, &map_opt)
+
+		try:
+			i = 0
+			while i < n_regs:
+				cmappy.mm_reg2hitpy(self._idx, &regs[i], &h)
+				cigar, _cs, _MD = [], '', ''
+				for k in range(h.n_cigar32): # convert the 32-bit CIGAR encoding to Python array
+					c = h.cigar32[k]
+					cigar.append([c>>4, c&0xf])
+				if cs or MD: # generate the cs and/or the MD tag, if requested
+					if cs:
+						l_cs_str = cmappy.mm_gen_cs(km, &cs_str, &m_cs_str, self._idx, &regs[i], _seq, 1)
+						_cs = cs_str[:l_cs_str] if isinstance(cs_str, str) else cs_str[:l_cs_str].decode()
+					if MD:
+						l_cs_str = cmappy.mm_gen_MD(km, &cs_str, &m_cs_str, self._idx, &regs[i], _seq)
+						_MD = cs_str[:l_cs_str] if isinstance(cs_str, str) else cs_str[:l_cs_str].decode()
+				yield Alignment(h.ctg, h.ctg_len, h.ctg_start, h.ctg_end, h.strand, h.qry_start, h.qry_end, h.mapq, cigar, h.is_primary, h.mlen, h.blen, h.NM, h.trans_strand, h.seg_id, _cs, _MD)
+				cmappy.mm_free_reg1(&regs[i])
+				i += 1
+		finally:
+			while i < n_regs:
+				cmappy.mm_free_reg1(&regs[i])
+				i += 1
+			free(regs)
+			free(cs_str)
+
+	def seq(self, str name, int start=0, int end=0x7fffffff):
+		cdef int l
+		cdef char *s
+		if self._idx == NULL: return
+		s = cmappy.mappy_fetch_seq(self._idx, name.encode(), start, end, &l)
+		if l == 0: return None
+		r = s[:l] if isinstance(s, str) else s[:l].decode()
+		free(s)
+		return r
+
+	@property
+	def k(self): return self._idx.k
+
+	@property
+	def w(self): return self._idx.w
+
+	@property
+	def n_seq(self): return self._idx.n_seq
+
+	@property
+	def seq_names(self):
+		cdef char *p
+		if self._idx == NULL: return
+		sn = []
+		for i in range(self._idx.n_seq):
+			p = self._idx.seq[i].name
+			s = p if isinstance(p, str) else p.decode()
+			sn.append(s)
+		return sn
+
+def fastx_read(fn, read_comment=False):
+	cdef cmappy.kseq_t *ks
+	ks = cmappy.mm_fastx_open(str.encode(fn))
+	if ks is NULL: return None
+	while cmappy.kseq_read(ks) >= 0:
+		if ks.qual.l > 0: qual = ks.qual.s if isinstance(ks.qual.s, str) else ks.qual.s.decode()
+		else: qual = None
+		name = ks.name.s if isinstance(ks.name.s, str) else ks.name.s.decode()
+		seq = ks.seq.s if isinstance(ks.seq.s, str) else ks.seq.s.decode()
+		if read_comment:
+			if ks.comment.l > 0: comment = ks.comment.s if isinstance(ks.comment.s, str) else ks.comment.s.decode()
+			else: comment = None
+			yield name, seq, qual, comment
+		else:
+			yield name, seq, qual
+	cmappy.mm_fastx_close(ks)
+
+def revcomp(seq):
+	l = len(seq)
+	bseq = seq if isinstance(seq, bytes) else seq.encode()
+	cdef char *s = cmappy.mappy_revcomp(l, bseq)
+	r = s[:l] if isinstance(s, str) else s[:l].decode()
+	free(s)
+	return r
+
+def verbose(v=None):
+	if v is None: v = -1
+	return cmappy.mm_verbose_level(v)

python/minimap2.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import sys
+import getopt
+import mappy as mp
+
+def main(argv):
+	opts, args = getopt.getopt(argv[1:], "x:n:m:k:w:r:c")
+	if len(args) < 2:
+		print("Usage: minimap2.py [options] <ref.fa>|<ref.mmi> <query.fq>")
+		print("Options:")
+		print("  -x STR      preset: sr, map-pb, map-ont, asm5, asm10 or splice")
+		print("  -n INT      mininum number of minimizers")
+		print("  -m INT      mininum chaining score")
+		print("  -k INT      k-mer length")
+		print("  -w INT      minimizer window length")
+		print("  -r INT      band width")
+		print("  -c          output the cs tag")
+		sys.exit(1)
+
+	preset = min_cnt = min_sc = k = w = bw = None
+	out_cs = False
+	for opt, arg in opts:
+		if opt == '-x': preset = arg
+		elif opt == '-n': min_cnt = int(arg)
+		elif opt == '-m': min_chain_score = int(arg)
+		elif opt == '-r': bw = int(arg)
+		elif opt == '-k': k = int(arg)
+		elif opt == '-w': w = int(arg)
+		elif opt == '-c': out_cs = True
+
+	a = mp.Aligner(args[0], preset=preset, min_cnt=min_cnt, min_chain_score=min_sc, k=k, w=w, bw=bw)
+	if not a: raise Exception("ERROR: failed to load/build index file '{}'".format(args[0]))
+	for name, seq, qual in mp.fastx_read(args[1]): # read one sequence
+		for h in a.map(seq, cs=out_cs): # traverse hits
+			print('{}\t{}\t{}'.format(name, len(seq), h))
+
+if __name__ == "__main__":
+	main(sys.argv)

sdust.c

@@ -56,6 +56,7 @@ sdust_buf_t *sdust_buf_init(void *km)
 	buf = (sdust_buf_t*)kcalloc(km, 1, sizeof(sdust_buf_t));
 	buf->km = km;
 	buf->w = kdq_init(int, buf->km);
+	kdq_resize(int, buf->w, 8);
 	return buf;
 }
 
@@ -69,10 +70,10 @@ void sdust_buf_destroy(sdust_buf_t *buf)
 static inline void shift_window(int t, kdq_t(int) *w, int T, int W, int *L, int *rw, int *rv, int *cw, int *cv)
 {
 	int s;
-	if (kdq_size(w) >= W - SD_WLEN + 1) { // TODO: is this right for SD_WLEN!=3?
+	if ((int)kdq_size(w) >= W - SD_WLEN + 1) { // TODO: is this right for SD_WLEN!=3?
 		s = *kdq_shift(int, w);
 		*rw -= --cw[s];
-		if (*L > kdq_size(w))
+		if (*L > (int)kdq_size(w))
 			--*L, *rv -= --cv[s];
 	}
 	kdq_push(int, w, t);
@@ -113,7 +114,7 @@ static void find_perfect(void *km, perf_intv_v *P, const kdq_t(int) *w, int T, i
 		r += c[t]++;
 		new_r = r, new_l = kdq_size(w) - i - 1;
 		if (new_r * 10 > T * new_l) {
-			for (j = 0; j < P->n && P->a[j].start >= i + start; ++j) { // find insertion position
+			for (j = 0; j < (int)P->n && P->a[j].start >= i + start; ++j) { // find insertion position
 				perf_intv_t *p = &P->a[j];
 				if (max_r == 0 || p->r * max_l > max_r * p->l)
 					max_r = p->r, max_l = p->l;
@@ -176,7 +177,7 @@ uint64_t *sdust(void *km, const uint8_t *seq, int l_seq, int T, int W, int *n)
 #ifdef _SDUST_MAIN
 #include <zlib.h>
 #include <stdio.h>
-#include <unistd.h>
+#include "ketopt.h"
 #include "kseq.h"
 KSEQ_INIT(gzFile, gzread)
 
@@ -185,16 +186,17 @@ int main(int argc, char *argv[])
 	gzFile fp;
 	kseq_t *ks;
 	int W = 64, T = 20, c;
+	ketopt_t o = KETOPT_INIT;
 
-	while ((c = getopt(argc, argv, "w:t:")) >= 0) {
+	while ((c = ketopt(&o, argc, argv, 1, "w:t:", 0)) >= 0) {
-		if (c == 'w') W = atoi(optarg);
+		if (c == 'w') W = atoi(o.arg);
-		else if (c == 't') T = atoi(optarg);
+		else if (c == 't') T = atoi(o.arg);
 	}
-	if (optind == argc) {
+	if (o.ind == argc) {
 		fprintf(stderr, "Usage: sdust [-w %d] [-t %d] <in.fa>\n", W, T);
 		return 1;
 	}
-	fp = strcmp(argv[optind], "-")? gzopen(argv[optind], "r") : gzdopen(fileno(stdin), "r");
+	fp = strcmp(argv[o.ind], "-")? gzopen(argv[o.ind], "r") : gzdopen(fileno(stdin), "r");
 	ks = kseq_init(fp);
 	while (kseq_read(ks) >= 0) {
 		uint64_t *r;

seed.c

@@ -0,0 +1,131 @@
+#include "mmpriv.h"
+#include "kalloc.h"
+#include "ksort.h"
+
+void mm_seed_mz_flt(void *km, mm128_v *mv, int32_t q_occ_max, float q_occ_frac)
+{
+	mm128_t *a;
+	size_t i, j, st;
+	if (mv->n <= q_occ_max || q_occ_frac <= 0.0f || q_occ_max <= 0) return;
+	KMALLOC(km, a, mv->n);
+	for (i = 0; i < mv->n; ++i)
+		a[i].x = mv->a[i].x, a[i].y = i;
+	radix_sort_128x(a, a + mv->n);
+	for (st = 0, i = 1; i <= mv->n; ++i) {
+		if (i == mv->n || a[i].x != a[st].x) {
+			int32_t cnt = i - st;
+			if (cnt > q_occ_max && cnt > mv->n * q_occ_frac)
+				for (j = st; j < i; ++j)
+					mv->a[a[j].y].x = 0;
+			st = i;
+		}
+	}
+	kfree(km, a);
+	for (i = j = 0; i < mv->n; ++i)
+		if (mv->a[i].x != 0)
+			mv->a[j++] = mv->a[i];
+	mv->n = j;
+}
+
+mm_seed_t *mm_seed_collect_all(void *km, const mm_idx_t *mi, const mm128_v *mv, int32_t *n_m_)
+{
+	mm_seed_t *m;
+	size_t i;
+	int32_t k;
+	m = (mm_seed_t*)kmalloc(km, mv->n * sizeof(mm_seed_t));
+	for (i = k = 0; i < mv->n; ++i) {
+		const uint64_t *cr;
+		mm_seed_t *q;
+		mm128_t *p = &mv->a[i];
+		uint32_t q_pos = (uint32_t)p->y, q_span = p->x & 0xff;
+		int t;
+		cr = mm_idx_get(mi, p->x>>8, &t);
+		if (t == 0) continue;
+		q = &m[k++];
+		q->q_pos = q_pos, q->q_span = q_span, q->cr = cr, q->n = t, q->seg_id = p->y >> 32;
+		q->is_tandem = q->flt = 0;
+		if (i > 0 && p->x>>8 == mv->a[i - 1].x>>8) q->is_tandem = 1;
+		if (i < mv->n - 1 && p->x>>8 == mv->a[i + 1].x>>8) q->is_tandem = 1;
+	}
+	*n_m_ = k;
+	return m;
+}
+
+#define MAX_MAX_HIGH_OCC 128
+
+void mm_seed_select(int32_t n, mm_seed_t *a, int len, int max_occ, int max_max_occ, int dist)
+{ // for high-occ minimizers, choose up to max_high_occ in each high-occ streak
+	extern void ks_heapdown_uint64_t(size_t i, size_t n, uint64_t*);
+	extern void ks_heapmake_uint64_t(size_t n, uint64_t*);
+	int32_t i, last0, m;
+	uint64_t b[MAX_MAX_HIGH_OCC]; // this is to avoid a heap allocation
+
+	if (n == 0 || n == 1) return;
+	for (i = m = 0; i < n; ++i)
+		if (a[i].n > max_occ) ++m;
+	if (m == 0) return; // no high-frequency k-mers; do nothing
+	for (i = 0, last0 = -1; i <= n; ++i) {
+		if (i == n || a[i].n <= max_occ) {
+			if (i - last0 > 1) {
+				int32_t ps = last0 < 0? 0 : (uint32_t)a[last0].q_pos>>1;
+				int32_t pe = i == n? len : (uint32_t)a[i].q_pos>>1;
+				int32_t j, k, st = last0 + 1, en = i;
+				int32_t max_high_occ = (int32_t)((double)(pe - ps) / dist + .499);
+				if (max_high_occ > 0) {
+					if (max_high_occ > MAX_MAX_HIGH_OCC)
+						max_high_occ = MAX_MAX_HIGH_OCC;
+					for (j = st, k = 0; j < en && k < max_high_occ; ++j, ++k)
+						b[k] = (uint64_t)a[j].n<<32 | j;
+					ks_heapmake_uint64_t(k, b); // initialize the binomial heap
+					for (; j < en; ++j) { // if there are more, choose top max_high_occ
+						if (a[j].n < (int32_t)(b[0]>>32)) { // then update the heap
+							b[0] = (uint64_t)a[j].n<<32 | j;
+							ks_heapdown_uint64_t(0, k, b);
+						}
+					}
+					for (j = 0; j < k; ++j) a[(uint32_t)b[j]].flt = 1;
+				}
+				for (j = st; j < en; ++j) a[j].flt ^= 1;
+				for (j = st; j < en; ++j)
+					if (a[j].n > max_max_occ)
+						a[j].flt = 1;
+			}
+			last0 = i;
+		}
+	}
+}
+
+mm_seed_t *mm_collect_matches(void *km, int *_n_m, int qlen, int max_occ, int max_max_occ, int dist, const mm_idx_t *mi, const mm128_v *mv, int64_t *n_a, int *rep_len, int *n_mini_pos, uint64_t **mini_pos)
+{
+	int rep_st = 0, rep_en = 0, n_m, n_m0;
+	size_t i;
+	mm_seed_t *m;
+	*n_mini_pos = 0;
+	*mini_pos = (uint64_t*)kmalloc(km, mv->n * sizeof(uint64_t));
+	m = mm_seed_collect_all(km, mi, mv, &n_m0);
+	if (dist > 0 && max_max_occ > max_occ) {
+		mm_seed_select(n_m0, m, qlen, max_occ, max_max_occ, dist);
+	} else {
+		for (i = 0; i < n_m0; ++i)
+			if (m[i].n > max_occ)
+				m[i].flt = 1;
+	}
+	for (i = 0, n_m = 0, *rep_len = 0, *n_a = 0; i < n_m0; ++i) {
+		mm_seed_t *q = &m[i];
+		//fprintf(stderr, "X\t%d\t%d\t%d\n", q->q_pos>>1, q->n, q->flt);
+		if (q->flt) {
+			int en = (q->q_pos >> 1) + 1, st = en - q->q_span;
+			if (st > rep_en) {
+				*rep_len += rep_en - rep_st;
+				rep_st = st, rep_en = en;
+			} else rep_en = en;
+		} else {
+			*n_a += q->n;
+			(*mini_pos)[(*n_mini_pos)++] = (uint64_t)q->q_span<<32 | q->q_pos>>1;
+			m[n_m++] = *q;
+		}
+	}
+	*rep_len += rep_en - rep_st;
+	*_n_m = n_m;
+	return m;
+}

setup.py

@@ -0,0 +1,55 @@
+try:
+	from setuptools import setup, Extension
+except ImportError:
+	from distutils.core import setup
+	from distutils.extension import Extension
+
+import sys, platform
+
+sys.path.append('python')
+
+extra_compile_args = ['-DHAVE_KALLOC']
+include_dirs = ["."]
+
+if platform.machine() in ["aarch64", "arm64"]:
+	include_dirs.append("sse2neon/")
+	extra_compile_args.extend(['-ftree-vectorize', '-DKSW_SSE2_ONLY', '-D__SSE2__'])
+else:
+	extra_compile_args.append('-msse4.1') # WARNING: ancient x86_64 CPUs don't have SSE4
+
+def readme():
+	with open('python/README.rst') as f:
+		return f.read()
+
+setup(
+	name = 'mappy',
+	version = '2.24',
+	url = 'https://github.com/lh3/minimap2',
+	description = 'Minimap2 python binding',
+	long_description = readme(),
+	author = 'Heng Li',
+	author_email = 'lh3@me.com',
+	license = 'MIT',
+	keywords = 'sequence-alignment',
+	scripts = ['python/minimap2.py'],
+	ext_modules = [Extension('mappy',
+		sources = ['python/mappy.pyx', 'align.c', 'bseq.c', 'lchain.c', 'seed.c', 'format.c', 'hit.c', 'index.c', 'pe.c', 'options.c',
+				   'ksw2_extd2_sse.c', 'ksw2_exts2_sse.c', 'ksw2_extz2_sse.c', 'ksw2_ll_sse.c',
+				   'kalloc.c', 'kthread.c', 'map.c', 'misc.c', 'sdust.c', 'sketch.c', 'esterr.c', 'splitidx.c'],
+		depends = ['minimap.h', 'bseq.h', 'kalloc.h', 'kdq.h', 'khash.h', 'kseq.h', 'ksort.h',
+				   'ksw2.h', 'kthread.h', 'kvec.h', 'mmpriv.h', 'sdust.h',
+				   'python/cmappy.h', 'python/cmappy.pxd'],
+		extra_compile_args = extra_compile_args,
+		include_dirs = include_dirs,
+		libraries = ['z', 'm', 'pthread'])],
+	classifiers = [
+		'Development Status :: 5 - Production/Stable',
+		'License :: OSI Approved :: MIT License',
+		'Operating System :: POSIX',
+		'Programming Language :: C',
+		'Programming Language :: Cython',
+		'Programming Language :: Python :: 2.7',
+		'Programming Language :: Python :: 3',
+		'Intended Audience :: Science/Research',
+		'Topic :: Scientific/Engineering :: Bio-Informatics'],
+	setup_requires=["cython"])

sketch.c

@@ -2,8 +2,9 @@
 #include <stdlib.h>
 #include <assert.h>
 #include <string.h>
+#define __STDC_LIMIT_MACROS
 #include "kvec.h"
-#include "minimap.h"
+#include "mmpriv.h"
 
 unsigned char seq_nt4_table[256] = {
 	0, 1, 2, 3,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
@@ -11,9 +12,9 @@ unsigned char seq_nt4_table[256] = {
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
-	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
-	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
 	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
@@ -77,11 +78,10 @@ void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, i
 {
 	uint64_t shift1 = 2 * (k - 1), mask = (1ULL<<2*k) - 1, kmer[2] = {0,0};
 	int i, j, l, buf_pos, min_pos, kmer_span = 0;
-	mm128_t *buf, min = { UINT64_MAX, UINT64_MAX };
+	mm128_t buf[256], min = { UINT64_MAX, UINT64_MAX };
 	tiny_queue_t tq;
 
-	assert(len > 0 && w > 0 && k > 0 && k <= 28); // 56 bits for k-mer; could use long k-mers, but 28 enough in practice
+	assert(len > 0 && (w > 0 && w < 256) && (k > 0 && k <= 28)); // 56 bits for k-mer; could use long k-mers, but 28 enough in practice
-	buf = (mm128_t*)alloca(w * 16);
 	memset(buf, 0xff, w * 16);
 	memset(&tq, 0, sizeof(tiny_queue_t));
 	kv_resize(mm128_t, km, *p, p->n + len/w);
@@ -102,34 +102,34 @@ void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, i
 				tq_push(&tq, skip_len);
 				kmer_span += skip_len;
 				if (tq.count > k) kmer_span -= tq_shift(&tq);
-				if (kmer_span >= 256) continue; // make sure $kmer_span does not take more than 8 bits
 			} else kmer_span = l + 1 < k? l + 1 : k;
 			kmer[0] = (kmer[0] << 2 | c) & mask;           // forward k-mer
 			kmer[1] = (kmer[1] >> 2) | (3ULL^c) << shift1; // reverse k-mer
 			if (kmer[0] == kmer[1]) continue; // skip "symmetric k-mers" as we don't know it strand
 			z = kmer[0] < kmer[1]? 0 : 1; // strand
-			if (++l >= k) {
+			++l;
+			if (l >= k && kmer_span < 256) {
 				info.x = hash64(kmer[z], mask) << 8 | kmer_span;
 				info.y = (uint64_t)rid<<32 | (uint32_t)i<<1 | z;
 			}
 		} else l = 0, tq.count = tq.front = 0, kmer_span = 0;
 		buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
-		if (l == w + k - 1) { // special case for the first window - because identical k-mers are not stored yet
+		if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
 			for (j = buf_pos + 1; j < w; ++j)
 				if (min.x == buf[j].x && buf[j].y != min.y) kv_push(mm128_t, km, *p, buf[j]);
 			for (j = 0; j < buf_pos; ++j)
 				if (min.x == buf[j].x && buf[j].y != min.y) kv_push(mm128_t, km, *p, buf[j]);
 		}
 		if (info.x <= min.x) { // a new minimum; then write the old min
-			if (l >= w + k) kv_push(mm128_t, km, *p, min);
+			if (l >= w + k && min.x != UINT64_MAX) kv_push(mm128_t, km, *p, min);
 			min = info, min_pos = buf_pos;
 		} else if (buf_pos == min_pos) { // old min has moved outside the window
-			if (l >= w + k - 1) kv_push(mm128_t, km, *p, min);
+			if (l >= w + k - 1 && min.x != UINT64_MAX) kv_push(mm128_t, km, *p, min);
 			for (j = buf_pos + 1, min.x = UINT64_MAX; j < w; ++j) // the two loops are necessary when there are identical k-mers
 				if (min.x >= buf[j].x) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
 			for (j = 0; j <= buf_pos; ++j)
 				if (min.x >= buf[j].x) min = buf[j], min_pos = j;
-			if (l >= w + k - 1) { // write identical k-mers
+			if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
 				for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
 					if (min.x == buf[j].x && min.y != buf[j].y) kv_push(mm128_t, km, *p, buf[j]);
 				for (j = 0; j <= buf_pos; ++j)

splitidx.c

@@ -0,0 +1,84 @@
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include "mmpriv.h"
+
+FILE *mm_split_init(const char *prefix, const mm_idx_t *mi)
+{
+	char *fn;
+	FILE *fp;
+	uint32_t i, k = mi->k;
+	fn = (char*)calloc(strlen(prefix) + 10, 1);
+	sprintf(fn, "%s.%.4d.tmp", prefix, mi->index);
+	if ((fp = fopen(fn, "wb")) == NULL) {
+		if (mm_verbose >= 1)
+			fprintf(stderr, "[ERROR]\033[1;31m failed to write to temporary file '%s'\033[0m: %s\n", fn, strerror(errno));
+		exit(1);
+	}
+	mm_err_fwrite(&k, 4, 1, fp);
+	mm_err_fwrite(&mi->n_seq, 4, 1, fp);
+	for (i = 0; i < mi->n_seq; ++i) {
+		uint32_t l;
+		l = strlen(mi->seq[i].name);
+		mm_err_fwrite(&l, 1, 4, fp);
+		mm_err_fwrite(mi->seq[i].name, 1, l, fp);
+		mm_err_fwrite(&mi->seq[i].len, 4, 1, fp);
+	}
+	free(fn);
+	return fp;
+}
+
+mm_idx_t *mm_split_merge_prep(const char *prefix, int n_splits, FILE **fp, uint32_t *n_seq_part)
+{
+	mm_idx_t *mi = 0;
+	char *fn;
+	int i, j;
+
+	if (n_splits < 1) return 0;
+	fn = CALLOC(char, strlen(prefix) + 10);
+	for (i = 0; i < n_splits; ++i) {
+		sprintf(fn, "%s.%.4d.tmp", prefix, i);
+		if ((fp[i] = fopen(fn, "rb")) == 0) {
+			if (mm_verbose >= 1)
+				fprintf(stderr, "ERROR: failed to open temporary file '%s': %s\n", fn, strerror(errno));
+			for (j = 0; j < i; ++j)
+				fclose(fp[j]);
+			free(fn);
+			return 0;
+		}
+	}
+	free(fn);
+
+	mi = CALLOC(mm_idx_t, 1);
+	for (i = 0; i < n_splits; ++i) {
+		mm_err_fread(&mi->k, 4, 1, fp[i]); // TODO: check if k is all the same
+		mm_err_fread(&n_seq_part[i], 4, 1, fp[i]);
+		mi->n_seq += n_seq_part[i];
+	}
+	mi->seq = CALLOC(mm_idx_seq_t, mi->n_seq);
+	for (i = j = 0; i < n_splits; ++i) {
+		uint32_t k;
+		for (k = 0; k < n_seq_part[i]; ++k, ++j) {
+			uint32_t l;
+			mm_err_fread(&l, 1, 4, fp[i]);
+			mi->seq[j].name = (char*)calloc(l + 1, 1);
+			mm_err_fread(mi->seq[j].name, 1, l, fp[i]);
+			mm_err_fread(&mi->seq[j].len, 4, 1, fp[i]);
+		}
+	}
+	return mi;
+}
+
+void mm_split_rm_tmp(const char *prefix, int n_splits)
+{
+	int i;
+	char *fn;
+	fn = CALLOC(char, strlen(prefix) + 10);
+	for (i = 0; i < n_splits; ++i) {
+		sprintf(fn, "%s.%.4d.tmp", prefix, i);
+		remove(fn);
+	}
+	free(fn);
+}

test/MT-orang.fa

@@ -1,4 +1,4 @@
->MT_orang
+>MT_orang co:Z:comment
 GTTTATGTAGCTTATTCTATCCAAAGCAATGCACTGAAAATGTCTCGACGGGCCCACACG
 CCCCATAAACAAATAGGTTTGGTCCTAGCCTTTCTATTAGCTCTTAGTGAGGTTACACAT
 GCAAGCATCCCCGCCCCAGTGAGTCGCCCTCCAAGTCACTCTGACTAAGAGGAGCAAGCA

test/q2.fa

@@ -0,0 +1,2 @@
+>q2
+GGACATCCCGATGGTGCAGTCCTACCTGTACGAAAGGAC

test/t-inv.fa

@@ -0,0 +1,127 @@
+>ref
+TGCGGAGGCTGAAGCAACTCCATCTTGGAAGCTAATCTACCATGTTGGCTTCTGATTAAC
+ATCAGTTCTGGGAAGGCTTGTAAGATTTCCTGTTTGTCTATTATTTCCTAGGTAAGAGCA
+GATACTTACTGTAAATCCTGCCCCTAGATTAAACAACCTTGGTGTTATCGTACTTCCATT
+GTCCTATACATCCCTTCGGAATCCCCCTTTCCCTATGGTCCTCAAGCCCTTGGTCTGGGG
+AGTAACAGCATAGGGATCAACCATCTCGTCTTGCCACTGCCCGAAATACAGACATGGCTT
+CTGTTCCTAAGTCCCTATTCAACTTTTCTTTCTAAGAAACTGGATTTGTCAGCCTCTTTC
+TTCACCTCTCAGCTTCCTTGGACTTTGGGGGTAGGTTTGCGTAGACATGCTCACCACAGA
+CACAATATCAGCTTCATTCTACAGATGAGGAAGGCAAGCCTTGGGGAGCTTAACCAACTT
+GTCGAGACTCATGTATATACCAACACTGAAAAGCAGATATTCCAGACTCCCAGTCATGCC
+ACAGGCACACCCCTCAGTGAGAGGTGGGGTTTGTAGTTGAGGCTATTTCCTGCCCAGGGA
+GCAGGGAGGCACTCTAGCTTCCCTGAGCTAACGTGGTTCTGCTTGTGTCTGACTTCCAGG
+TCTCTGCCCTTTCCAAGCTCACTAGGATGGGCTTCGGGTGTGTCAAATGCCTCAGACAGT
+ACAGATCCACACAGAATGGGCATATGCAACCAATCAGTGTCATAAAAAAGAAGGAAATGA
+CTCGGGCCCCCTGTGTGTTCAACATGTCGAAGGTATCTGTGCAGCAGAAGAAAGAGGGGC
+AAAAGCCCCCAGTGCCACAGGCCAGAGGCAGCAGCTTGGGCCCATGTGGGAGGGTTTGCT
+TTCCCCTGCCAAAGTGATGGGCTGCTGCAGCCTGGGGCTTGTGGGAATCCTTCCTGGGCC
+TGTGTGGGAAGTGTAGGCAGGGAGAGTGCTGCTTTCCCAAGCTCATCCCAGCTACAGCTA
+CCTTTGTGCTCTGGGATTCAGGACCCCCGAGGGGGCTGGCAGGAGAGTCTCTGTTCTCGG
+ATGGGTTGTCACCAGGGCATACATGGGAAGTGGGCTCTCTGGAGTCACCCTCCAGGGGAC
+AATGCCAATTCCAGACACATTTACTGGAACCCCTACACTGATGACCTTTTGTTGAGGGTT
+GAATTATGTCCCCAAAAAAGATACATTGAAGTCCAAACCTCTGGTGTCTATAAATGTGAT
+TTTATTTGAAAATGAGGTTTCTATGGACTAAATTGTGTCCCTCCCAAATTCATATTTTGA
+AGCCCTAGCCCCCAGTGTGACTATACCTAGAGACAGAGATCTTTAGGAGGTAATTAAGGT
+TCAATGAGGTCAGGTGGGTGGGGCCCTAAACCAACAGGAAGGACTGTGGCCTTACTAGAA
+AAGGAAGAAAAAGCATTTCCTCTCTTCTAGTATAAAAGGACACAGAAAGAAGGCAGATAT
+CTACAAGCCACGAAGAGAGACGTCACTGAGAACTGAATTTGTGTACATTGATCTGGAACT
+TCCAGCCTCCAGAACTTGAGAAATACATTTCTGTTGTTTATTTTTTTTTCATGTAATCAA
+TTCATTTATCATATATTTATTGAGTGCCTACTATGTGCCAGAGGATACAGCAGTAACAAA
+ACTAGGCAAAAATTGTGCCTAAAAGAGGGAAGATGACTTTTCTTAAAGTGTGGAATAAAG
+AAAAGTAAGATAGCGGATAGAAGCTTGAAGTGAAAGCAGGTTCACAGGAAGTTTCTTTGG
+TCATTTGTTTTGTTTTTAAATAGTGGAAAGATGTATATGTTTATGGAGAAAGATTGCCTT
+GAAGATGCAAGAGGAAGAGATGATCAAAATTCAAGAAGAAGCAGAAAGTGATAGAATAAA
+GAGCACAAGTGGAGAATTAGTGTTAATGAAAAGAAGGATGCTTCCTTTGATATGAAGTGA
+AGGAAGAGAGAATGAGTAAAGACCAAGACTTGAAGTCCCTAGTTTAATAGAGGGAGATTT
+CTTCTTTTGATAGCAACAATGGTATTCTGAATTATTTGAAGACATGTCATATTTCTCTTG
+TGCCATTTTCCTCCCAGTTTAAACATTCTCATAACCTCTATTCCTCACATGATGTTTTTC
+CAGGTCCTTTATTCTTTGGCACTCTCTTCTCTGGACACATTGTATTCTGTCATTGGTCCT
+AAAATTTAGATACCCACAATTGAACATACTCCTCTAGATATGGTCTAGCTAATGCAAAAG
+AACTGCTGCCTTCCAACTTGTTCAGACATCATATGTTTGTTGTCAAACGCTAAGTTGAGT
+TGTTATCTTTTAAGTTTTGTTTTTGTTTTTTTTTTTTTTTTTTAATTCCAAGAGGTGCCC
+ACGTTGGCTAAGTACCAAACAGGGTACTAGGGAATTTTACTTCTGAGTTAAATGCCATTC
+TAGTTGTTTTTTCTTCATCTCCAGTAAGGTTATCTTTATTCACCAGTTGTTACAATAGCT
+GTGGGTCTTGCTTCTCACAGTTTTATGCTGTCTGTGCTATTTTCTCTACTGATCATCACC
+ACAATCATTATTGCTTATCATAATTGTTATCTTTATTTTCTCCTTTAATCAAGAATCAGT
+CTTCCTTTATCTCATTATTCTCTTTTGCAGGCTTCAGGATAATTATGGTTGGAGTGCACT
+GGGGGAACCAGTGCAGCTAAGCTCTGACATCTTTGCATCCCTTTTCCATCTGCTGTTTTG
+GCACTCTGGTAGAATAGATAACCTAAAAACGACTTTAAAACATCTAGAAATTTTGGATAA
+AATATAACAAACATCCCTTTAAATGCACAACTGATCTTCCATGGAAGTCACAGAAATATA
+TAACGCCAAAAAGAAGGGAAGCTGAAACCCAGGGCTGTAAACATGAACATCATCTTCTCT
+CCCTTTTTCTTGTGACTTATCTTGTTTTTCTCAGCTTTGGTGCTACCAAGGCTTGACTTT
+AATAGGCATTTCCAATCAATGAGAGAATTTCTTTTGCTTTCATCAACAATTCAGTTATTG
+ATGTTAACATATATATCATTTGAGTACTTTTCTTTTTTTTATTATTATTATACTTTAAGT
+TTTAGGGTCCATGTGCACAATGTGCAGGTTAGTTACGTATGTATACATGTGCCATGCTGG
+TGTGCTGCACCCATTAACTCATCATTTAGCATTAGGTATATCTCCTAATGCTATCCCTTC
+CCCCTCTCCCCACCCCACAACAGTCCCCAGAGTGTTCCCCTTCCTGTGTCCATGTGTTCT
+CATTGTTCAATCCCCATCTATGAGTGAGAACATGCGGTGTTTGGTTTTTTGTCCTTGCAA
+TAGTTTACTGAGAATGATGATTTCTAATTTCATCCATGTCCCTAAAGAGCTTCTGCACAG
+CAAAAGAAACTACCATCAGAGTGAACAGGCAACCTACAAAATGGGAGAAAATTTTCACAA
+CCTGCTCATCTGACAAAGGGCTAATATCCAGAATCTACAATGAACTCAAACAAATTTACA
+AGAAAAAAACAAACAACCCCATCAAAAAGTGGGCAAAGGATATGAACAGACACTTCTCAA
+AAGAAGACATTTATGCAGCCAAAAGACACATGAAAAAATGCTCATCATCACTGGCCATCA
+GAGAAATGCAAACCAAAACCACAATGAGATACCATCTCACACCAGTTAAAATGGCAATCA
+TTAAAAAGTCAGGAAACAACAGGTGCTGGAGAGGATGTGGAGAAACAGGAACACTTTTAC
+ACTGTTGGTGGGACTGTAAACTAGTTCAACCATTGTGGAAGTCAGTGTGCTGATTCCTCA
+GGGATCTAGAACTAGAAATACCATTTGACCCAGCCATCCCATTACTGGGTATATACCCAA
+AGGACTATAAATCATGCTGCTATAAAGACACATGCACACGTATGTTTATTGCGGCACTAT
+TCACAATAGCAAAGACTTGGAACCAACCCAAATGTCCAACAATGATAGACTGGATTAAGA
+AAATGTGGCACATATACACCACGGAATACTGTGCAGCCATAAAAAATGATGAGTTCATGT
+CCTTTGTAGGGACACGGATGAAATTGGAAATCATTTCTGTTGTTTAAACCACGAAGTCTA
+TGGTATCTGGTTATGACAACCTGAGAATACTAACTCAAGGGTCTTTCGCAGATGTCATTA
+AGTTGTTAAAGTGAGGTCATTATGGTGGGTCCTAATCCAAGAGAAGAGATGCATGGACAG
+ACGTGCACAACGGGAGGACCAAGCCAAGACACACAGGGAGAATGGCCATGGGAAGATGGA
+GGCAGAGATCAAAGTGAGGCACCCACAAGCCAAGAAATGGCAGGAGCTACCAGCAGCTGG
+AAGATGCAGAGAAGCATTCCTTCTTAGAGGTTTCAGAGAGAGTATGGTGCTACTGACACC
+TTGATTTTGAACTTCTAGTCTCCAGAACTATGAGAGAATAAATTTCTGTTGGTTAAGCCA
+TCGAGTTTGTGTAAGTTTGTTATAAGAGCCCTAGGAAATAAACATATCCATTTATTCAGG
+AAAGCCTGCTAGAGTGCAAATATTTGGAAAAGATACTACTATGCAAATGTTTGAAAAAGA
+TATTGCTCTTGATTCTGCCTTATGGGTTTTTCATTTCTGTAAGCTATTCTCAAAGTTTTG
+TTCTTGGACTACTATTGGTAATTAAGACTGCAACATGTTTGGCAACATCAGTTGAGAACT
+GTTGCTCTGGGAACGTTTTCGGCAAGCCTCAGCCCTTCTTTTCCCTTGGCTTGCATTGAG
+GAGTTAGGTGATACTCTGCTGCTCAGGCCCAGCACCTTTATGGACCGTATTCCCCTGGTG
+GAATGACCATCTCTGCTTGCTCTGATTGGCTGTTGGGGTTTTCTAGCATGCCCTATTTAA
+TATGTATGATTTATCTCTTACTTCAGTTGGAAGGTACAGTTGCTCTGTAGTTGGCATGCA
+GTCATGGTGACTATGAAAATATAAAATAATGTTTTGGTTTACAGACACTTAGAAATAAGT
+TGTGTCTCAAAATTGGGTGACTATTCTAGTTATCTGCTACTCAATATCCTTGTGCGAGCC
+CTCTTTACCCAGAATCAAACTAAACCATGAGGGGCACTATAGAATGTCACCCCTGGGTCC
+AGGATACTATGGGGACTCAGAAGCCAAGCTCCCACTGGGGGATCTAGGGCATGCCCCCAA
+GGTAAGATTCCCACCTCTTTGTTCAGCAGGAAGCACCCATCACACAAGGAGGTAGGAATA
+AACAAGCATTCGTCAAGAACAAAAGATACAGATGTTCTGCTGGAGCTTGGATACATAGCA
+TAAGAGGGAACAGTTCTCACAGGTAAGAGTAAGTTTTCCTCTGGTGGTGACAGTGGGACC
+TGTGGGGGAGAGAATTGGGAGTACTGACAGGAAGGCAGAGTGGCTGTCCAAATGAACGGA
+TTGTTTGCACATGGCCTTTAGGGCACGTTGTGTTAGCCTTCCATTGCTGCTTATATTAGT
+CTGTTTTCACACTGCCCATAAATGCATACCTGAGACTGGATAATTTATAAAGAAAAAGAG
+CCTTAATGTACTCATAGTTGCATGTGGCTGGGGAGGCCTCACAATCATGGCAGAAGGTGA
+AAGGCACATCTTACATGGAAGCAGACAAGAGAGAATTGAGGACCAAGTGAAAGGGGTTTC
+CCCTTATAAAACCATCAGATCACATGAGACTTTTTCACCACCATGAGAACAGTAAGGGGA
+AAACTATGCTCATGATTCAATTGTCTCCCACTGGATTCCTCCCACAACACATAGGAATTA
+TGGGAGCTAAAATTCAAGATGAGATTTGGGTGAGGACACAGCCAAACCCTATCACTGCTG
+TAATCAATTCCCACCAACTTAGTGGCTCGAAACATCACAGATTTATGATCTTATGACGGT
+GGAGGTCCCCAAATGGATCTTCTAGGTCTAGAATCAAGGTATCAGCAGACCACTTCTTTT
+GGAGGCTCTGGTGGAGAAACCATTTCCTCGCCTTTTCCAGCTTCTAGAGGCTGCCCTTCT
+CATTCCTTGGTTCACGGCCACACTCATTTCCATCTCTGCTTCCACTGTGACAACTTCTCT
+GCCTCAGACCCTCCTGCTTTGCCTTTGTAAGGACCCTTGTGATGAGATCAGGCCCATCCA
+GGATTATCCCTCATCTCAAGACCTTTACCTTAATCACATTTGCAAGGTCTCTTCCACTGT
+GTCAGGTAACATTTTCACAGGTTCCAGGGATTAGGGTGTGGACATCTTGGGGAGCTGGAG
+GATATTATTTCATCTACCACACACATCTCTACCTTGTACAGGCAAGCACTTGCAAAGTGC
+AATGTGATCCTCTGGAGCCACTGTCCTCCCAGAGCTTATATATACTCTGAAAGTCAACTC
+TCAGACCACAGCCTCCTGTCCATGCACCACTCTCATCAACACCCCCACCCGAAACACTTT
+CACTCCACCCTCTTTGTCCCCTAACTCATGGAGAAGAAAATCTAATTAGTAGGAGTGGAA
+TTTGGCTTTCATCTTTACCAGTACTAGAAATATGGTGTGTGTCTTTTTGTAAAAATTCTC
+TCAACTAAATTGTTTTTATTAATTTCTGCAAAATGTGAACATCAACTCCCTTCATGTGAA
+TGTCAATAAGATTAAATGAGCTGTCTCAGCTCCTAGCCTGTGCAAGCTAACAGCTCAGGA
+GATGTTTATTTCTTTCCCTCTTCTTTCCTTAATGAAGCCCTCTCCTTTGACATCTTCAAT
+TCTGGAGCGCTTCTTTTCTGAGGCCTTGGCTCCCCCACATTGCCCACCCTTTTCCTGCTC
+GTCCACATTTCTGGCTTCTATTCTCTTGTCTTTACCATCTCCCTGAACAATGTTATCCGT
+TCCAATGACTTCAACAGTCTCTCCGCTTACATATGATGCCTCTCAAACTCTGATCTCCAA
+CTCTTCCAAAGAGCTCTGGACCTTTGTTCCAATTACCTGAAAAACATCTTCTTGGATGTC
+CCATTAGCACTGTTAAATCAAACAAGAATTTCCCTCCCTCCTGCCTTGCTGTAGTTCCCC
+TAGGGATTCGGTTGTGTGGGAAGATGTGTGGAGAGCTCTTAGTTGACTCCCTTCTCTGCA
+GTTCTACCTCTCTAGAGACTTGGAGGACCCACTGTTTCCGCCTCGCTTTTTCAGGCCTAG
+AGATTGCTCGCTCCTGGGCTGGCTGCTTCATAATTCCTTATTAGTAGTTTCCCAAGCTTA
+CATATCTGTAAATATTTACTTTAGTTAAATTCTCCCCAATTTCCACAATATGTTGGCTGC
+ACATGCTTTCTACTAGGAGTCACACAACTATGATAAGAACCAAGAAATATTAGTAAACGT
+TTTTTACCATTATTGGCCTATACCCTGGAATAGCCAACAATAACCTAGAACCTATGCAAC
+AAGAATATCCAACAAGAACCTAGAGACCTGTCAGTCTATAGGTGGGAACTACAGGATGAG
+A

test/t2.fa

@@ -0,0 +1,2 @@
+>t2
+GGACATCCCGATGGTGCAGgtGCTATTAAAGGTTCGTTTGTTCAACGATTAAagTCCTACCTGTACGAAAGGAC

tex/Makefile

@@ -0,0 +1,21 @@
+.SUFFIXES: .gp .tex .eps .pdf .eps.gz
+
+.eps.pdf:
+		epstopdf --outfile $@ $<
+
+.eps.gz.pdf:
+		gzip -dc $< | epstopdf --filter > $@
+
+.pdf.eps:
+		pdftops -eps $< $@
+
+all:minimap2.pdf
+
+roc-color.eps:roc.gp
+		gnuplot roc.gp
+
+minimap2.pdf:minimap2.tex minimap2.bib roc-color.pdf
+		pdflatex minimap2; bibtex minimap2; pdflatex minimap2; pdflatex minimap2;
+
+clean:
+		rm -fr *.toc *.aux *.bbl *.blg *.idx *.log *.out *~ minimap2.pdf

tex/bioinfo.cls

@@ -0,0 +1,930 @@
+\newcommand\classname{bioinfo}
+\newcommand\lastmodifieddate{2003/02/08}
+\newcommand\versionnumber{0.1}
+
+% Are we printing crop marks?
+\newif\if@cropmarkson \@cropmarksontrue
+
+\NeedsTeXFormat{LaTeX2e}[2001/06/01]
+\ProvidesClass{\classname}[\lastmodifieddate\space\versionnumber]
+
+\setlength{\paperheight}{11truein}
+\setlength{\paperwidth}{8.5truein}
+
+\newif\if@final
+
+\DeclareOption{draft}{\PassOptionsToPackage{draft}{graphicx}}
+\DeclareOption{a4paper}{\PassOptionsToPackage{a4}{crop}}
+\DeclareOption{centre}{\PassOptionsToPackage{center}{crop}}
+\DeclareOption{crop}{\PassOptionsToPackage{cam}{crop}\global\@cropmarksontrue}
+\DeclareOption{nocrop}{\PassOptionsToPackage{off}{crop}\global\@cropmarksonfalse}
+\DeclareOption{info}{\PassOptionsToPackage{info}{crop}}
+\DeclareOption{noinfo}{\PassOptionsToPackage{noinfo}{crop}}
+\DeclareOption{final}{\global\@finaltrue}
+
+\ExecuteOptions{a4paper,nocrop,centre,info}
+
+\ProcessOptions
+
+% Load all necessary packages
+\RequirePackage{inputenc,crop,graphicx,amsmath,array,color,amssymb,flushend,stfloats,amsthm,chngpage,times}
+%\RequirePackage[LY1]{fontenc}
+%\RequirePackage[LY1,mtbold]{mathtime}
+\def\authoraffliate{\fontfamily{phv}\selectfont}
+\def\helvetica{\fontfamily{phv}\selectfont}
+\def\helveticaitalic{\fontfamily{phv}\itshape\selectfont}
+\def\helveticabold{\fontfamily{phv}\bfseries\selectfont}
+\def\helveticabolditalic{\fontfamily{phv}\bfseries\itshape\selectfont}
+
+% Not sure if needed.
+\newcommand\@ptsize{0}
+
+% Set twoside printing
+\@twosidetrue
+
+% Marginal notes are on the outside edge
+\@mparswitchfalse
+
+\reversemarginpar
+
+\renewcommand\normalsize{%
+   \@setfontsize\normalsize{9}{11}%
+   \abovedisplayskip 10\p@ \@plus2\p@ \@minus5\p@
+   \abovedisplayshortskip \z@ \@plus3\p@
+   \belowdisplayshortskip 6\p@ \@plus3\p@ \@minus3\p@
+   \belowdisplayskip \abovedisplayskip
+   \let\@listi\@listI}
+\normalsize
+\let\@bls\baselineskip
+
+\newcommand\small{%
+    \@setfontsize\small{9}{11}%
+    \abovedisplayskip 11\p@ minus 3\p@
+    \belowdisplayskip \abovedisplayskip
+    \abovedisplayshortskip \z@ plus 2\p@
+    \belowdisplayshortskip 4\p@ plus 2\p@ minus2\p@
+    \def\@listi{\topsep 4.5\p@ plus 2\p@ minus 1\p@
+       \itemsep \parsep
+       \topsep 4\p@ plus 2\p@ minus 2\p@}}
+
+\newcommand\footnotesize{%
+    \@setfontsize\footnotesize{8}{10}%
+    \abovedisplayskip 6\p@ minus 3\p@
+    \belowdisplayskip\abovedisplayskip
+    \abovedisplayshortskip \z@ plus 3\p@
+    \belowdisplayshortskip 6\p@ plus 3\p@ minus 3\p@
+    \def\@listi{\topsep 3\p@ plus 1\p@ minus 1\p@
+       \parsep 2\p@ plus 1\p@ minus 1\p@\itemsep \parsep}}
+
+\def\scriptsize{\@setfontsize\scriptsize{7pt}{9pt}}
+\def\tiny{\@setfontsize\tiny{5pt}{7pt}}
+\def\large{\@setfontsize\large{11.5pt}{12pt}}
+\def\Large{\@setfontsize\Large{14pt}{16}}
+\def\LARGE{\@setfontsize\LARGE{15pt}{17pt}}
+\def\huge{\@setfontsize\huge{22pt}{22pt}}
+\def\Huge{\@setfontsize\Huge{30pt}{30pt}}
+
+\DeclareOldFontCommand{\rm}{\normalfont\rmfamily}{\mathrm}
+\DeclareOldFontCommand{\sf}{\normalfont\sffamily}{\mathsf}
+\DeclareOldFontCommand{\tt}{\normalfont\ttfamily}{\mathtt}
+\DeclareOldFontCommand{\bf}{\normalfont\bfseries}{\mathbf}
+\DeclareOldFontCommand{\it}{\normalfont\itshape}{\mathit}
+\DeclareOldFontCommand{\sl}{\normalfont\slshape}{\@nomath\sl}
+\DeclareOldFontCommand{\sc}{\normalfont\scshape}{\@nomath\sc}
+
+% Line spacing
+\setlength\lineskip{1\p@}
+\setlength\normallineskip{1\p@}
+\renewcommand\baselinestretch{}
+
+% Paragraph dimensions and inter-para spacing
+\setlength\parskip{0\p@}
+\setlength\parindent{3mm}
+
+% Set inter-para skips
+\setlength\smallskipamount{3\p@ \@plus 1\p@ \@minus 1\p@}
+\setlength\medskipamount{6\p@ \@plus 2\p@}
+\setlength\bigskipamount{12\p@ \@plus 4\p@ \@minus 4\p@}
+
+% Page break penalties
+\@lowpenalty   51
+\@medpenalty  151
+\@highpenalty 301
+
+% Disallow widows and orphans
+\clubpenalty 10000
+\widowpenalty 10000
+
+% Disable page breaks before equations, allow pagebreaks after
+% equations and discourage widow lines before equations.
+\displaywidowpenalty 100
+\predisplaypenalty   10000
+\postdisplaypenalty  2500
+
+% Allow breaking the page in the middle of a paragraph
+\interlinepenalty 0
+
+% Disallow breaking the page after a hyphenated line
+\brokenpenalty 10000
+
+% Hyphenation; don't split words into less than three characters
+\lefthyphenmin=3
+\righthyphenmin=3
+
+%
+% Set page layout dimensions
+%
+\setlength\headheight{16\p@}        % height of running head
+\setlength\topmargin{2.9pc}         % head margin
+\addtolength\topmargin{-1in}        % subtract out the 1 inch driver margin
+
+\setlength\topskip{10\p@}           % height of first line of text
+\setlength\headsep{19\p@}        %   space below running head --
+
+\setlength\footskip{34\p@}          % space above footer line
+\setlength\maxdepth{.5\topskip}     % pages can be short or deep by half a line?
+
+\setlength\textwidth{42pc}          % text measure excluding margins
+
+\setlength\textheight{58\baselineskip} % 54 lines on a full page,
+\addtolength\textheight{\topskip}      %   including the first
+                                       %   line on the page
+
+% Set the margins
+\setlength\marginparsep{3\p@}
+\setlength\marginparpush{3\p@}
+\setlength\marginparwidth{35\p@}
+
+\setlength\oddsidemargin{4.5pc}
+\addtolength\oddsidemargin{-1in}     % subtract out the 1 inch driver margin
+\setlength\@tempdima{\paperwidth}
+\addtolength\@tempdima{-\textwidth}
+\addtolength\@tempdima{-4.5pc}
+\setlength\evensidemargin{\@tempdima}
+\addtolength\evensidemargin{-1in}
+
+\setlength\columnsep{1.5pc}          % space between columns for double-column text
+\setlength\columnseprule{0\p@}       % width of rule between two columns
+
+% Footnotes
+\setlength\footnotesep{9\p@}      % space between footnotes
+% space between text and footnote
+\setlength{\skip\footins}{12\p@ \@plus 6\p@ \@minus 1\p@}
+
+% Float placement parameters
+
+% The total number of floats that can be allowed on a page.
+\setcounter{totalnumber}{10}
+% The maximum number of floats at the top and bottom of a page.
+\setcounter{topnumber}{5}
+\setcounter{bottomnumber}{5}
+% The maximum part of the top or bottom of a text page that can be
+% occupied by floats. This is set so that at least four lines of text
+% fit on the page.
+\renewcommand\topfraction{.9}
+\renewcommand\bottomfraction{.9}
+% The minimum amount of a text page that must be occupied by text.
+% This should accomodate four lines of text.
+\renewcommand\textfraction{.06}
+% The minimum amount of a float page that must be occupied by floats.
+\renewcommand\floatpagefraction{.94}
+
+% The same parameters repeated for double column output
+\renewcommand\dbltopfraction{.9}
+\renewcommand\dblfloatpagefraction{.9}
+
+% Space between floats
+\setlength\floatsep    {12\p@ \@plus 2\p@ \@minus 2\p@}
+% Space between floats and text
+\setlength\textfloatsep{20\p@ \@plus 2\p@ \@minus 4\p@}
+% Space above and below an inline figure
+\setlength\intextsep   {18\p@ \@plus 2\p@ \@minus 2\p@}
+
+% For double column floats
+\setlength\dblfloatsep    {12\p@ \@plus 2\p@ \@minus 2\p@}
+\setlength\dbltextfloatsep{20\p@ \@plus 2\p@ \@minus 4\p@}
+
+% Space left at top, bottom and inbetween floats on a float page.
+\setlength\@fptop{0\p@}         % no space above float page figures
+\setlength\@fpsep{12\p@ \@plus 1fil}
+\setlength\@fpbot{0\p@}
+
+% The same for double column
+\setlength\@dblfptop{0\p@}
+\setlength\@dblfpsep{12\p@ \@plus 1fil}
+\setlength\@dblfpbot{0\p@}
+
+% Override settings in mathtime back to TeX defaults
+\DeclareMathSizes{5}     {5}   {5}  {5}
+\DeclareMathSizes{6}     {6}   {5}  {5}
+\DeclareMathSizes{7}     {7}   {5}  {5}
+\DeclareMathSizes{8}     {8}   {6}  {5}
+\DeclareMathSizes{9}     {9}   {6.5}  {5}
+\DeclareMathSizes{10}   {10}   {7.5}  {5}
+\DeclareMathSizes{12}   {12}   {9}  {7}
+
+% Page styles
+\def\ps@headings
+  {%
+  \def\@oddfoot{\vbox to 12.5\p@{\hbox{\rule{\textwidth}{0.5\p@}}\vss
+        \hbox to \textwidth{\hfill\helveticabold\small\thepage}%
+        }}%
+  \def\@evenfoot{\vbox to 12.5\p@{\rule{\textwidth}{0.5\p@}\vss
+        \hbox to \textwidth{\helveticabold\small\thepage\hfill}%
+        }}%
+  \def\@evenhead{\vbox{\hbox to \textwidth{\fontsize{8}{10}\selectfont
+        \helveticabold{\fontshape{it}\selectfont
+        \strut\leftmark}\hfill}\vspace{6.5\p@}\rule{\textwidth}{0.5\p@}}}%
+  \def\@oddhead{\vbox{\hbox to \textwidth{\hfill\fontsize{8}{10}\selectfont
+        \helveticabold{\fontshape{it}\selectfont\strut\rightmark}}%
+        \vspace{6.5\p@}\rule{\textwidth}{0.5\p@}}}%
+  \def\titlemark##1{\markboth{##1}{##1}}%
+  \def\authormark##1{\gdef\leftmark{##1}}%
+  }
+
+\def\ps@opening
+  {%
+  \def\@oddfoot{\vbox to 13\p@{\hbox{\rule{\textwidth}{1\p@}}\vss
+        \hbox to \textwidth{\helvetica
+        \fontsize{7}{9}\fontshape{n}\selectfont%
+          \hfill\small\helveticabold\thepage}%
+        }}%
+  \def\@evenfoot{\vbox to 13\p@{\rule{\textwidth}\vss
+        \hbox to \textwidth{\helvetica\thepage\hfill
+          \fontsize{7}{9}\fontshape{n}\selectfont}%
+        }}%
+  \let\@evenhead\relax
+  \let\@oddhead\relax}
+
+% Page range
+\newif\iflastpagegiven   \lastpagegivenfalse
+\newcommand\firstpage[1]{%
+  \gdef\@firstpage{#1}%
+  \ifnum\@firstpage>\c@page
+    \setcounter{page}{#1}%
+    \ClassWarning{BIO}{Increasing pagenumber to \@firstpage}%
+  \else \ifnum\@firstpage<\c@page
+    \ClassWarning{BIO}{Firstpage lower than pagenumber}\fi\fi
+    \xdef\@firstpage{\the\c@page}%
+    }
+\def\@firstpage{1}
+\def\pagenumbering#1{%
+    \global\c@page \@ne
+    \gdef\thepage{\csname @#1\endcsname \c@page}%
+    \gdef\thefirstpage{%
+            \csname @#1\endcsname \@firstpage}%
+    \gdef\thelastpage{%
+            \csname @#1\endcsname \@lastpage}%
+    }
+
+\newcommand\lastpage[1]{\xdef\@lastpage{#1}%
+  \global\lastpagegiventrue}
+\def\@lastpage{0}
+\def\setlastpage{\iflastpagegiven\else
+    \edef\@tempa{@lastpage@}%
+    \expandafter
+    \ifx \csname \@tempa \endcsname \relax
+        \gdef\@lastpage{0}%
+    \else
+        \xdef\@lastpage{\@nameuse{@lastpage@}}%
+        \fi
+    \fi }
+\def\writelastpage{%
+    \iflastpagegiven \else
+    \immediate\write\@auxout%
+    {\string\global\string\@namedef{@lastpage@}{\the\c@page}}%
+    \fi
+    }
+\def\thepagerange{%
+  \ifnum\@lastpage =0 {\ \bf ???} \else
+  \ifnum\@lastpage = \@firstpage \ \thefirstpage\else
+  \thefirstpage--\thelastpage \fi\fi}
+
+\AtBeginDocument{\setlastpage
+     \pagenumbering{arabic}%
+   }
+\AtEndDocument{%
+    \writelastpage
+    \if@final
+    \clearemptydoublepage
+    \else
+    \clearpage
+    \fi}
+
+%
+% Sectional units
+%
+
+% Counters
+\newcounter{section}
+\newcounter{subsection}[section]
+\newcounter{subsubsection}[subsection]
+\newcounter{paragraph}[subsubsection]
+\newcounter{subparagraph}[paragraph]
+\newcounter{figure}
+\newcounter{table}
+
+% Form of the numbers
+\newcommand\thepage{\arabic{page}}
+\renewcommand\thesection{\arabic{section}}
+\renewcommand\thesubsection{{\thesection.\arabic{subsection}}}
+\renewcommand\thesubsubsection{{\thesubsection.\arabic{subsubsection}}}
+\renewcommand\theparagraph{\thesubsubsection.\arabic{paragraph}}
+\renewcommand\thesubparagraph{\theparagraph.\arabic{subparagraph}}
+\renewcommand\theequation{\arabic{equation}}
+
+% Form of the words
+\newcommand\contentsname{Contents}
+\newcommand\listfigurename{List of Figures}
+\newcommand\listtablename{List of Tables}
+\newcommand\partname{Part}
+\newcommand\appendixname{Appendix}
+\newcommand\abstractname{Abstract}
+\newcommand\refname{References}
+\newcommand\bibname{References}
+\newcommand\indexname{Index}
+\newcommand\figurename{Fig.}
+\newcommand\tablename{Table}
+
+% Clearemptydoublepage should really clear the running heads too
+\newcommand{\clearemptydoublepage}{\newpage{\pagestyle{empty}\cleardoublepage}}
+
+% Frontmatter, mainmatter and backmatter
+
+\newif\if@mainmatter \@mainmattertrue
+
+\newcommand\frontmatter{%
+  \clearpage
+  \@mainmatterfalse
+  \pagenumbering{roman}}
+
+\newcommand\mainmatter{%
+  \clearpage
+  \@mainmattertrue
+  \pagenumbering{arabic}}
+
+\newcommand\backmatter{%
+  \clearpage
+  \@mainmatterfalse}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% TITLE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newlength{\dropfromtop}
+\setlength{\dropfromtop}{\z@}
+
+% Application Notes
+\newif\if@appnotes
+\newcommand{\application}{%
+%  \setlength{\dropfromtop}{-2.25pc}%
+  \global\@appnotestrue}
+
+\long\def\title{\@ifnextchar[{\short@title}{\@@title}}
+\def\short@title[#1]{\titlemark{#1}\@@@title}
+\def\@@title#1{\authormark{#1}\@@@title{#1}}
+\long\def\@@@title#1{\gdef\@title{#1}}
+
+\long\def\author{\@ifnextchar[{\short@uthor}{\@uthor}}
+\def\short@uthor[#1]{\authormark{#1}\@@author}
+\def\@uthor#1{\authormark{#1}\@@author{#1}}
+\long\def\@@author#1{\gdef\@author{#1}}
+
+\def\vol#1{\global\def\@vol{#1}}
+\def\issue#1{\global\def\@issue{#1}}
+\def\address#1{\global\def\@issue{#1}}
+\def\history#1{\global\def\@history{#1}}
+\def\editor#1{\global\def\@editor{#1}}
+\def\pubyear#1{\global\def\@pubyear{#1}}
+\def\copyrightyear#1{\global\def\@copyrightyear{#1}}
+\def\address#1{\global\def\@address{#1}}
+\def\DOI#1{\global\def\@DOI{#1}}
+
+\definecolor{gray}{cmyk}{0, 0, 0, 0.15}
+\newlength{\extraspace}
+\setlength{\extraspace}{\z@}
+
+\newcommand\maketitle{\par
+  \begingroup
+    \renewcommand\thefootnote{\@fnsymbol\c@footnote}%
+    \def\@makefnmark{\rlap{\@textsuperscript{\normalfont\@thefnmark}}}%
+    \long\def\@makefntext##1{\parindent 3mm\noindent
+%      \@textsuperscript{\normalfont\@thefnmark}\raggedright##1}%
+      \@textsuperscript{\normalfont\@thefnmark}##1}%
+    \if@twocolumn
+      \ifnum \col@number=\@ne
+        \@maketitle
+      \else
+        \twocolumn[\@maketitle]%
+      \fi
+    \else
+      \newpage
+      \global\@topnum\z@   % Prevents figures from going at top of page.
+      \@maketitle
+    \fi
+    \thispagestyle{opening}\@thanks
+  \endgroup
+  \setcounter{footnote}{0}%
+  \global\let\thanks\relax
+  \global\let\maketitle\relax
+  \global\let\@maketitle\relax
+  \global\let\@address\@empty
+  \global\let\@history\@empty
+  \global\let\@editor\@empty
+  \global\let\@thanks\@empty
+  \global\let\@author\@empty
+  \global\let\@date\@empty
+  \global\let\@title\@empty
+  \global\let\@pubyear\@empty
+  \global\let\address\relax
+  \global\let\history\relax
+  \global\let\editor\relax
+  \global\let\title\relax
+  \global\let\author\relax
+  \global\let\date\relax
+  \global\let\pubyear\relax
+  \global\let\@copyrightline\@empty
+  \global\let\and\relax
+  \@afterindentfalse\@afterheading
+}
+
+\newlength{\aboveskipchk}%for checking oddpage or evenpage top skip
+\setlength{\aboveskipchk}{\z@}%
+
+\def\@maketitle{%
+  \let\footnote\thanks
+  \clearemptydoublepage
+    \checkoddpage\ifcpoddpage\setlength{\aboveskipchk}{-3pc}\else\setlength{\aboveskipchk}{-5pc}\fi%for checking oddpage or evenpage top skip%%
+  \vspace*{\aboveskipchk}%
+  \vspace{\dropfromtop}%
+  \hbox to \textwidth{%
+  {\helvetica\itshape\bfseries\fontsize{19}{12}\selectfont {\color{gray}TECHNICAL REPORT}
+    \hfil
+    \if@appnotes APPLICATIONS NOTE\hfil\fi
+    }%
+\enskip  \parbox[b]{11.3pc}{%
+    \helvetica
+    \flushright\fontsize{8}{10}\fontshape{it}\selectfont
+        \hfill
+  }}
+  \rule{\textwidth}{1\p@}\par%
+  \helvetica
+  \hbox to \textwidth{%
+  \parbox[t]{41pc}{%
+    \vspace*{1sp}
+    {\helveticabold\fontsize{16}{21}\selectfont\raggedright \@title \par}%
+    \vspace{4.5\p@}
+    {\authoraffliate\fontsize{11}{13}\selectfont\raggedright \@author \par}%
+        \vspace{4\p@}
+    {\authoraffliate\fontsize{9}{11}\selectfont\raggedright \@address \par}%
+    \vspace{4\p@}
+    %{\helvetica\fontsize{8}{10}\selectfont\raggedright \@history \par}
+    %\vspace{24\p@}
+    %{\helvetica\fontsize{10}{12}\selectfont\raggedright \@editor \par}
+    %\vspace{20\p@}
+    }%
+  }
+  \vspace{4.5\p@}%
+  \rule{\textwidth}{1\p@}%
+  \vspace{12\p@ plus 6\p@ minus 6\p@}%
+  \vspace{\extraspace}
+  }
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%% Abstract %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\absection}[1]{%
+  \par\noindent{\bfseries #1}\space\ignorespaces}
+
+\newenvironment{abstract}{%
+  \begingroup
+  \let\section\absection
+  \fontfamily{\sfdefault}\fontsize{8}{11}\sffamily\selectfont
+  {\fontseries{b}\selectfont ABSTRACT}\par}
+{\endgroup\bigskip\@afterheading\@afterindentfalse\vskip 12pt plus 3pt minus 1pt}
+
+% Section macros
+
+% Lowest level heading that takes a number by default
+\setcounter{secnumdepth}{3}
+
+\renewcommand{\@seccntformat}[1]{\csname the#1\endcsname\quad}
+
+\def\section{%
+  \@startsection{section}{1}{\z@}
+  {-22\p@ plus -3\p@}{3\p@}
+  {\reset@font\raggedright\helveticabold\fontsize{10}{12}\selectfont\MakeUppercase}}
+
+\def\subsection{%
+  \@startsection{subsection}{2}{\z@}
+  {-11\p@ plus -2\p@}{3\p@}
+  {\reset@font\raggedright\mathversion{bold}\fontseries{b}\fontsize{10}{12}\selectfont}}
+
+\def\subsubsection{%
+  \@startsection{subsubsection}{3}{\z@}
+  %{-11\p@ plus -1\p@}{-1em}
+  {-11\p@ plus -1\p@}{0.001em}
+  {\reset@font\normalfont\normalsize\itshape}}
+
+\def\textcolon{\text{\rm :}}
+
+ \def\paragraph{%
+   \@startsection{paragraph}{4}{\z@}
+   {-6\p@}
+   {-.4em}
+   {\reset@font\itshape}}
+
+% ********************
+% Figures and tables *
+% ********************
+
+% Table and array parameters
+\setlength\arraycolsep{.5em}
+\setlength\tabcolsep{.5em}
+\setlength\arrayrulewidth{.5pt}
+\setlength\doublerulesep{2.5pt}
+\setlength\extrarowheight{\z@}
+\renewcommand\arraystretch{1}
+
+\newlength{\abovecaptionskip}
+\newlength{\belowcaptionskip}
+\setlength{\abovecaptionskip}{13pt}
+\setlength{\belowcaptionskip}{10.5pt}
+
+\long\def\@makecaption#1#2{\vspace{\abovecaptionskip}%
+  \begingroup
+  \footnotesize
+  \textbf{#1.}\enskip{#2}\par
+  \endgroup}
+
+\long\def\@tablecaption#1#2{%
+  \begingroup
+  \footnotesize
+  \textbf{#1.}\enskip{#2\strut\par}
+  \endgroup\vspace{\belowcaptionskip}}
+
+% Table rules
+\def\toprule{\noalign{\ifnum0=`}\fi\hrule \@height 0.5pt \hrule \@height 6pt \@width 0pt \futurelet
+   \@tempa\@xhline}
+\def\midrule{\noalign{\ifnum0=`}\fi \hrule \@height 6.75pt \@width 0pt \hrule \@height 0.5pt
+    \hrule \@height 6pt \@width 0pt \futurelet \@tempa\@xhline}
+\def\botrule{\noalign{\ifnum0=`}\fi \hrule \@height 5.75pt \@width 0pt \hrule \@height 0.5pt \futurelet
+   \@tempa\@xhline}
+\def\hrulefill{\leavevmode\leaders\hrule height .5pt\hfill\kern\z@}
+
+\def\thefigure{\@arabic\c@figure}
+\def\fps@figure{tbp}
+\def\ftype@figure{1}
+\def\ext@figure{lof}
+\def\fnum@figure{\figurename~\thefigure}
+\def\figure{\@float{figure}}
+\let\endfigure\end@float
+\@namedef{figure*}{\@dblfloat{figure}}
+\@namedef{endfigure*}{\end@dblfloat}
+\def\thetable{\@arabic\c@table}
+\def\fps@table{tbp}
+\def\ftype@table{2}
+\def\ext@table{lot}
+\def\fnum@table{Table~\thetable}
+\def\table{\let\@makecaption\@tablecaption\let\source\tablesource\@float{table}}
+\def\endtable{\end@float}
+\@namedef{table*}{\let\@makecaption\@tablecaption\@dblfloat{table}}
+\@namedef{endtable*}{\end@dblfloat}
+
+\newif\if@rotate \@rotatefalse
+\newif\if@rotatecenter \@rotatecenterfalse
+\def\rotatecenter{\global\@rotatecentertrue}
+\def\rotateendcenter{\global\@rotatecenterfalse}
+\def\rotate{\global\@rotatetrue}
+\def\endrotate{\global\@rotatefalse}
+\newdimen\rotdimen
+\def\rotstart#1{\special{ps: gsave currentpoint currentpoint translate
+    #1 neg exch neg exch translate}}
+\def\rotfinish{\special{ps: currentpoint grestore moveto}}
+\def\rotl#1{\rotdimen=\ht#1\advance\rotdimen by \dp#1
+    \hbox to \rotdimen{\vbox to\wd#1{\vskip \wd#1
+    \rotstart{270 rotate}\box #1\vss}\hss}\rotfinish}
+\def\rotr#1{\rotdimen=\ht #1\advance\rotdimen by \dp#1
+    \hbox to \rotdimen{\vbox to \wd#1{\vskip \wd#1
+    \rotstart{90 rotate}\box #1\vss}\hss}\rotfinish}
+
+\newdimen\tempdime
+\newbox\temptbox
+
+% From ifmtarg.sty
+% Copyright Peter Wilson and Donald Arseneau, 2000
+\begingroup
+\catcode`\Q=3
+\long\gdef\@ifmtarg#1{\@xifmtarg#1QQ\@secondoftwo\@firstoftwo\@nil}
+\long\gdef\@xifmtarg#1#2Q#3#4#5\@nil{#4}
+\long\gdef\@ifnotmtarg#1{\@xifmtarg#1QQ\@firstofone\@gobble\@nil}
+\endgroup
+
+\def\tablesize{\@setfontsize\tablesize{8\p@}{10\p@}}
+
+\newenvironment{processtable}[3]{\setbox\temptbox=\hbox{{\tablesize #2}}%
+\tempdime\wd\temptbox\@processtable{#1}{#2}{#3}{\tempdime}}
+{\relax}
+
+\newcommand{\@processtable}[4]{%
+\if@rotate
+\setbox4=\vbox to \hsize{\vss\hbox to \textheight{%
+\begin{minipage}{#4}%
+\@ifmtarg{#1}{}{\caption{#1}}{\tablesize #2}%
+\vskip7\p@\noindent
+\parbox{#4}{\fontsize{7}{9}\selectfont #3\par}%
+\end{minipage}}\vss}%
+\rotr{4}
+\else
+\hbox to \hsize{\hss\begin{minipage}[t]{#4}%
+\vskip2.9pt
+\@ifmtarg{#1}{}{\caption{#1}}{\tablesize #2}%
+\vskip6\p@\noindent
+\parbox{#4}{\fontsize{7}{9}\selectfont #3\par}%
+\end{minipage}\hss}\fi}%
+
+\newcolumntype{P}[1]{>{\raggedright\let\\\@arraycr\hangindent1em}p{#1}}
+
+% ******************************
+% List numbering and lettering *
+% ******************************
+\def\labelenumi{{\rm\arabic{enumi}.}}
+\def\theenumi{\arabic{enumi}}
+\def\labelenumii{{\rm\alph{enumii}.}}
+\def\theenumii{\alph{enumii}}
+\def\p@enumii{\theenumi}
+\def\labelenumiii{{\rm(\arabic{enumiii})}}
+\def\theenumiii{\roman{enumiii}}
+\def\p@enumiii{\theenumi(\theenumii)}
+\def\labelenumiv{{\rm(\arabic{enumiv})}}
+\def\theenumiv{\Alph{enumiv}}
+\def\p@enumiv{\p@enumiii\theenumiii}
+\def\labelitemi{{\small$\bullet$}}
+\def\labelitemii{{\small$\bullet$}}
+\def\labelitemiii{{\small$\bullet$}}
+\def\labelitemiv{{\small$\bullet$}}
+
+\def\@listI{\leftmargin\leftmargini \topsep\medskipamount}
+\let\@listi\@listI
+\@listi
+\def\@listii{\topsep\z@\leftmargin\leftmarginii}
+\def\@listiii{\leftmargin\leftmarginiii \topsep\z@}
+\def\@listiv{\leftmargin\leftmarginiv \topsep\z@}
+\def\@listv{\leftmargin\leftmarginv \topsep\z@}
+\def\@listvi{\leftmargin\leftmarginvi \topsep\z@}
+
+\setlength{\leftmargini}{3mm}
+\setlength{\leftmarginii}{\z@}
+\setlength{\leftmarginiii}{\z@}
+\setlength{\leftmarginiv}{\z@}
+
+% Changes to the list parameters for enumerate
+\def\enumargs{%
+   \partopsep     \z@
+   \itemsep       3\p@
+   \parsep        \z@
+   \labelsep      0.5em
+   \listparindent \parindent
+   \itemindent    \z@
+   \topsep 11\p@
+}
+
+\def\enumerate{%
+    \@ifnextchar[{\@numerate}{\@numerate[0]}}
+
+\def\@numerate[#1]{%
+     \ifnum \@enumdepth >3 \@toodeep\else
+     \advance\@enumdepth \@ne
+     \edef\@enumctr{enum\romannumeral\the\@enumdepth}
+     \list{\csname label\@enumctr\endcsname}{%
+       \enumargs
+       \setlength{\leftmargin}{\csname leftmargin\romannumeral\the\@enumdepth\endcsname}
+       \usecounter{\@enumctr}
+       \settowidth\labelwidth{#1}
+       \addtolength{\leftmargin}{\labelwidth}
+       \addtolength{\leftmargin}{\labelsep}
+       \def\makelabel##1{\hss \llap{##1}}}%
+     \fi
+   }
+\let\endenumerate\endlist
+
+% Changes to the list parameters for itemize
+\def\itemargs{%
+   \partopsep     \z@
+   \itemsep       3\p@
+   \parsep        \z@
+   \labelsep      0.5em
+   \rightmargin   \z@
+   \listparindent \parindent
+   \itemindent    \z@
+   \topsep11\p@
+}
+
+\def\itemize{%
+    \@ifnextchar[{\@itemize}{\@itemize[$\bullet$]}}
+
+\def\@itemize[#1]{%
+     \ifnum \@itemdepth >3 \@toodeep\else
+     \advance\@itemdepth \@ne
+     \edef\@itemctr{item\romannumeral\the\@itemdepth}
+     \list{\csname label\@itemctr\endcsname}{%
+       \itemargs
+       \setlength{\leftmargin}{\csname leftmargin\romannumeral\the\@itemdepth\endcsname}
+       \settowidth\labelwidth{#1}
+       \addtolength{\leftmargin}{\labelwidth}
+       \addtolength{\leftmargin}{\labelsep}
+       \def\makelabel##1{\hss \llap{##1}}}%
+     \fi
+   }
+\let\enditemize\endlist
+
+\newenvironment{unlist}{%
+  \begin{list}{}%
+    {\setlength{\labelwidth}{\z@}%
+     \setlength{\labelsep}{\z@}%
+     \setlength{\topsep}{\medskipamount}%
+     \setlength{\itemsep}{3\p@}%
+     \setlength{\leftmargin}{2em}%
+     \setlength{\itemindent}{-2em}}}
+{\end{list}}
+
+
+% ***********************
+% Quotes and Quotations *
+% ***********************
+\def\quotation{\par\begin{list}{}{
+      \setlength{\topsep}{\medskipamount}
+      \setlength{\leftmargin}{2em}%
+      \setlength{\rightmargin}{\z@}%
+      \setlength\labelwidth{0pt}%
+      \setlength\labelsep{0pt}%
+      \listparindent\parindent}%
+  \item[]}
+\def\endquotation{\end{list}}
+\let\quote\quotation
+\let\endquote\endquotation
+
+\skip\@mpfootins = \skip\footins
+\fboxsep=6\p@
+\fboxrule=1\p@
+
+% *******************
+% Table of contents *
+% *******************
+\newcommand\@pnumwidth{4em}
+\newcommand\@tocrmarg{2.55em plus 1fil}
+\newcommand\@dotsep{1000}
+\setcounter{tocdepth}{4}
+
+\def\numberline#1{\hbox to \@tempdima{{#1}}}
+
+\def\@authortocline#1#2#3#4#5{%
+  \vskip 1.5\p@
+  \ifnum #1>\c@tocdepth \else
+    {\leftskip #2\relax \rightskip \@tocrmarg \parfillskip -\rightskip
+     \parindent #2\relax\@afterindenttrue
+     \interlinepenalty\@M
+     \leavevmode
+     \@tempdima #3\relax
+     \advance\leftskip \@tempdima \null\nobreak\hskip -\leftskip
+     {\itshape #4}\nobreak
+     \leaders\hbox{$\m@th
+        \mkern \@dotsep mu\hbox{.}\mkern \@dotsep
+        mu$}\hfill
+     \nobreak
+     \hb@xt@\@pnumwidth{\hfil}%
+     \par}%
+  \fi}
+
+\newcommand*\l@author{\@authortocline{2}{0pt}{30pt}}
+\newcommand*\l@section{\@dottedtocline{3}{11pt}{20pt}}
+\newcommand*\l@subsection{\@dottedtocline{4}{31pt}{29pt}}
+\newcommand*\l@subsubsection[2]{}
+
+
+
+% ***********
+% Footnotes *
+% ***********
+
+\def\footnoterule{\noindent\rule{\columnwidth}{0.5pt}}
+\def\@makefnmark{\@textsuperscript{\normalfont\@thefnmark}}%
+\newcommand\@makefntext[1]{\noindent{\@makefnmark}\enskip#1}
+
+% ***********
+% References *
+% ***********
+
+\providecommand{\newblock}{}
+\newenvironment{thebibliography}{%
+  \section{\bibname}%
+  \begingroup
+  \small
+  \begin{list}{}{%
+      \setlength{\topsep}{\z@}%
+      \setlength{\labelsep}{\z@}%
+      \settowidth{\labelwidth}{\z@}%
+      \setlength{\leftmargin}{4mm}%
+      \setlength{\itemindent}{-4mm}}\small}
+{\end{list}\endgroup}
+
+\RequirePackage{natbib}
+
+% **********
+% Appendix *
+% **********
+\newif\ifappend % Are we in the Appendix?
+\def\appendix{\par
+        \setcounter{section}{0}
+        \setcounter{subsection}{0}
+        \appendtrue
+}
+
+%Math parameters
+
+\setlength{\jot}{5\p@}
+\mathchardef\@m=1500   % adapted value
+
+\def\frenchspacing{\sfcode`\.\@m \sfcode`\?\@m \sfcode`\!\@m
+  \sfcode`\:\@m \sfcode`\;\@m \sfcode`\,\@m}
+
+% Theorems
+\def\th@plain{%
+%%  \let\thm@indent\noindent % no indent
+\thm@headfont{\quad\scshape}% heading font is bold
+\thm@notefont{\upshape\mdseries}% same as heading font
+\thm@headpunct{.}% no period after heading
+\thm@headsep 5\p@ plus\p@ minus\p@\relax
+%%  \let\thm@swap\@gobble
+%%  \thm@preskip\topsep
+%%  \thm@postskip\theorempreskipamount
+\itshape % body font
+}
+
+\vbadness=9999
+\tolerance=9999
+\doublehyphendemerits=10000
+\doublehyphendemerits 640000   % corresponds to badness 800
+\finalhyphendemerits  1000000  % corresponds to badness 1000
+
+\flushbottom
+\frenchspacing
+\ps@headings
+\twocolumn
+
+% Screen PDF compatability
+\newcommand{\medline}[1]{%
+  \unskip\unskip\ignorespaces}
+
+
+%%%%for smaller size text
+\newenvironment{methods}{%
+  \begingroup
+\def\section{%
+  \@startsection{section}{1}{\z@}
+  {-24\p@ plus -3\p@}{4\p@}
+  {\reset@font\raggedright\helveticabold\fontsize{10}{12}\selectfont\MakeUppercase}}
+ \def\subsection{%
+  \@startsection{subsection}{2}{\z@}
+  {-5\p@ plus -2\p@}{4\p@}
+  {\reset@font\raggedright\mathversion{bold}\fontseries{b}\fontsize{10}{12}\selectfont}}
+ \def\subsubsection{%
+  \@startsection{subsubsection}{3}{\z@}
+%  {-6\p@ plus -1\p@}{-1em}
+  {-6\p@ plus -1\p@}{0.001em}
+  {\reset@font\normalfont\normalsize\itshape}}
+\footnotesize
+  \par}
+{\par\endgroup\bigskip\@afterheading\@afterindentfalse}
+
+
+
+\graphicspath{{g:/artwork/oup/bioinfo/}}
+
+\language=2
+
+\hyphenation{Figure Table Figures Tables}
+
+\newcommand{\href}[2]{#2}
+
+\renewenvironment{proof}[1][\proofname]{\par
+  \normalfont \topsep6\p@\@plus6\p@\relax
+  \labelsep 0.5em
+  \trivlist
+  \item[\hskip\labelsep\hskip1em\textsc{#1}.]\ignorespaces
+}{\endtrivlist\@endpefalse}
+
+%%Different Bonds
+
+\def\sbond{\ensuremath{\raise.25ex\hbox{${-}\!\!\!\!{-}$}}\kern -.9pt}
+\def\dbond{\ensuremath{\raise.25ex\hbox{=$\!$=}}}
+\def\tbond{\ensuremath{\raise.20ex\hbox{${\equiv}\!\!\!{\equiv}$}}}
+
+% Author queries
+%\fboxsep=4\p@
+%\fboxrule=0.5\p@
+\newcommand{\query}[2][0pt]{}%
+%  \marginpar{\vspace*{#1}%
+%    {\parbox{\marginparwidth}{%
+%     \raggedright\fontsize{6}{8}\selectfont
+%         #2}}}}
+
+\renewcommand{\dag}{{\mathversion{normal}$^{\dagger}$}}
+
+\endinput

tex/blasr-mc.eval

@@ -0,0 +1,17 @@
+Q	60	32681	57	0.001744133
+Q	39	3	1	0.001774569
+Q	38	3	1	0.001804999
+Q	35	5	1	0.001835311
+Q	34	31	2	0.001894692
+Q	20	11	2	0.001955154
+Q	19	4	1	0.001985460
+Q	15	29	5	0.002136296
+Q	14	6	1	0.002166417
+Q	10	11	1	0.002196193
+Q	6	11	2	0.002256442
+Q	5	1	1	0.002286864
+Q	4	1	1	0.002317285
+Q	3	36	15	0.002771602
+Q	2	5	2	0.002832085
+Q	1	12	9	0.003105023
+Q	0	220	83	0.005594194

tex/bowtie2-s3.sam.eval

@@ -0,0 +1,28 @@
+Q	42	16872292	669	0.000039651	16872292
+Q	40	835329	636	0.000073697	17707621
+Q	31	6544	2	0.000073783	17714165
+Q	30	8882	6	0.000074084	17723047
+Q	27	68499	9	0.000074305	17791546
+Q	26	132041	81	0.000078277	17923587
+Q	25	129378	96	0.000083033	18052965
+Q	24	92056	382	0.000103665	18145021
+Q	23	14341	402	0.000125720	18159362
+Q	22	132838	146	0.000132789	18292200
+Q	21	122274	124	0.000138641	18414474
+Q	18	112183	103	0.000143361	18526657
+Q	17	126981	213	0.000153804	18653638
+Q	16	16356	208	0.000164810	18669994
+Q	15	42804	782	0.000206223	18712798
+Q	14	16026	318	0.000223025	18728824
+Q	12	170250	814	0.000264087	18899074
+Q	11	48351	1409	0.000337777	18947425
+Q	8	1843	311	0.000354156	18949268
+Q	7	62266	4435	0.000586276	19011534
+Q	6	413997	50057	0.003150647	19425531
+Q	5	404	58	0.003153568	19425935
+Q	4	704	154	0.003161381	19426639
+Q	3	1473	681	0.003196193	19428112
+Q	2	17541	16462	0.004039875	19445653
+Q	1	534344	354879	0.021693547	19979997
+Q	0	11939	9917	0.022176642	19991936
+U	8064

tex/bwa-s3.sam.eval

@@ -0,0 +1,52 @@
+Q	60	18784147	3	0.000000160	18784147
+Q	52	19002	1	0.000000213	18803149
+Q	50	7152	2	0.000000319	18810301
+Q	49	6797	1	0.000000372	18817098
+Q	48	52188	2	0.000000477	18869286
+Q	47	48775	3	0.000000634	18918061
+Q	46	19447	2	0.000000739	18937508
+Q	45	25983	3	0.000000896	18963491
+Q	44	13455	1	0.000000949	18976946
+Q	43	14573	2	0.000001053	18991519
+Q	42	8697	4	0.000001263	19000216
+Q	41	8645	2	0.000001368	19008861
+Q	40	176603	75	0.000005264	19185464
+Q	38	2503	2	0.000005368	19187967
+Q	37	4117	3	0.000005523	19192084
+Q	36	2924	16	0.000006356	19195008
+Q	35	2323	8	0.000006772	19197331
+Q	34	2344	10	0.000007292	19199675
+Q	33	4279	6	0.000007603	19203954
+Q	32	2092	4	0.000007810	19206046
+Q	31	2625	11	0.000008382	19208671
+Q	30	2828	13	0.000009057	19211499
+Q	29	1581	1	0.000009108	19213080
+Q	28	1543	6	0.000009420	19214623
+Q	27	70916	223	0.000020948	19285539
+Q	26	1288	16	0.000021777	19286827
+Q	25	25551	122	0.000028065	19312378
+Q	24	14345	84	0.000032390	19326723
+Q	23	7308	87	0.000036878	19334031
+Q	22	8358	125	0.000043325	19342389
+Q	21	4836	71	0.000046983	19347225
+Q	20	5888	123	0.000053325	19353113
+Q	19	4656	83	0.000057600	19357769
+Q	18	3948	87	0.000062081	19361717
+Q	17	4418	114	0.000067954	19366135
+Q	16	4226	131	0.000074702	19370361
+Q	15	5760	164	0.000083144	19376121
+Q	14	4697	257	0.000096384	19380818
+Q	13	5246	313	0.000112503	19386064
+Q	12	4170	241	0.000124908	19390234
+Q	11	4095	304	0.000140557	19394329
+Q	10	3857	360	0.000159087	19398186
+Q	9	5300	438	0.000181617	19403486
+Q	8	4206	572	0.000211050	19407692
+Q	7	4676	787	0.000251541	19412368
+Q	6	3923	688	0.000286924	19416291
+Q	5	3294	708	0.000323333	19419585
+Q	4	2936	693	0.000358965	19422521
+Q	3	3928	816	0.000400897	19426449
+Q	2	2613	810	0.000442533	19429062
+Q	1	3515	1188	0.000503587	19432577
+Q	0	567423	376636	0.019321100	20000000

tex/bwa.eval

@@ -0,0 +1,55 @@
+Q	60	31721	27	0.000851171
+Q	59	54	4	0.000975610
+Q	58	29	5	0.001131933
+Q	57	21	2	0.001194030
+Q	56	14	4	0.001319137
+Q	55	22	6	0.001506544
+Q	54	12	4	0.001631475
+Q	53	16	3	0.001724733
+Q	51	10	1	0.001755541
+Q	50	10	1	0.001786330
+Q	49	11	3	0.001879699
+Q	47	8	2	0.001941869
+Q	46	17	1	0.001972140
+Q	44	8	3	0.002065534
+Q	43	10	1	0.002096174
+Q	42	13	1	0.002126595
+Q	41	14	3	0.002219444
+Q	40	13	2	0.002281036
+Q	38	17	4	0.002404747
+Q	37	15	4	0.002528484
+Q	36	12	1	0.002558742
+Q	35	19	3	0.002650783
+Q	34	12	3	0.002743313
+Q	33	7	1	0.002773882
+Q	32	21	3	0.002865508
+Q	31	11	2	0.002926799
+Q	30	14	3	0.003018891
+Q	29	17	1	0.003048401
+Q	28	11	2	0.003109549
+Q	27	20	5	0.003262998
+Q	26	11	1	0.003292948
+Q	25	14	4	0.003415725
+Q	24	16	5	0.003569212
+Q	23	43	6	0.003750426
+Q	21	15	1	0.003779664
+Q	20	29	7	0.003992943
+Q	19	22	2	0.004052089
+Q	18	28	4	0.004172204
+Q	16	25	5	0.004323390
+Q	15	24	5	0.004474480
+Q	14	25	5	0.004625204
+Q	13	23	3	0.004714365
+Q	12	22	1	0.004741963
+Q	11	32	11	0.005075674
+Q	10	35	7	0.005285315
+Q	9	32	12	0.005648503
+Q	8	33	8	0.005888126
+Q	7	39	7	0.006095506
+Q	6	42	14	0.006515953
+Q	5	38	15	0.006966725
+Q	4	37	12	0.007325113
+Q	3	49	18	0.007862737
+Q	2	63	21	0.008486434
+Q	1	55	27	0.009292156
+Q	0	153	77	0.011576593

tex/eval2roc.pl

@@ -0,0 +1,33 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use Getopt::Std;
+
+my %opts = (n=>33088, s=>100);
+getopts('n:', \%opts);
+
+my $pseudo = .5;
+my $tot = $pseudo;
+my $err = $pseudo;
+my $tot_last_out = -$opts{s};
+my $state = 0;
+my $mapq = 0;
+while (<>) {
+	chomp;
+	if (/^Q\t(\d+)\t(\d+)\t(\d+)/) {
+		$tot += $2;
+		$err += $3;
+		if ($tot - $tot_last_out >= $opts{s}) {
+			print join("\t", $1, $err/$tot, $tot / $opts{n}), "\n";
+			$tot_last_out = $tot;
+			$state = 0;
+		} else {
+			$state = 1;
+			$mapq = $1;
+		}
+	}
+}
+if ($state) {
+	print join("\t", $mapq, $err/$tot, $tot / $opts{n}), "\n";
+}

tex/graphmap.eval

@@ -0,0 +1,4 @@
+Q	40	31897	63	0.001975107
+Q	3	423	267	0.010210396
+Q	2	162	120	0.013853827
+Q	1	188	172	0.019038874

tex/hs38-simu.sh

@@ -0,0 +1,10 @@
+./pbsim --prefix pb-1 --depth 0.1 --sample-fastq m131017_060208_42213_c100579642550000001823095604021496_s1_p0.1.subreads.fastq --length-min 1000 --length-max 30000 --seed 11 hs38.fa
+
+bin/mason_variator -ir hs38.fa -s 1 -ov hs38-s1.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
+bin/mason_simulator -ir hs38.fa -iv hs38-s1.vcf -n 1000000 --seed 1 -o s1_1.fq -or s1_2.fq -oa s1.sam --illumina-prob-mismatch-scale 2.5
+
+bin/mason_variator -ir hs38.fa -s 2 -ov hs38-s2.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
+bin/mason_simulator -ir hs38.fa -iv hs38-s2.vcf -n 1000000 --seed 2 -o mason-s2_1.fq -or mason-s2_2.fq -oa mason-s2.sam --illumina-prob-mismatch-scale 2.5 --illumina-read-length 150
+
+bin/mason_variator -ir hs38.fa -s 3 -ov hs38-s3.vcf --snp-rate 1e-3 --small-indel-rate 2e-4 --sv-indel-rate 0 --sv-inversion-rate 0 --sv-translocation-rate 0 --sv-duplication-rate 0 --max-small-indel-size 10
+bin/mason_simulator -ir hs38.fa -iv hs38-s3.vcf -n 10000000 --seed 3 -o mason-s3_1.fq -or mason-s3_2.fq -oa mason-s3.sam --illumina-prob-mismatch-scale 2.5 --illumina-read-length 150

tex/minialign.eval

@@ -0,0 +1,49 @@
+Q	60	32070	190	0.005924540
+Q	59	62	2	0.005975352
+Q	58	37	5	0.006123908
+Q	57	40	7	0.006333633
+Q	56	39	6	0.006512032
+Q	55	32	2	0.006567534
+Q	54	54	2	0.006618420
+Q	53	33	4	0.006735255
+Q	52	39	2	0.006788866
+Q	51	48	3	0.006871264
+Q	50	34	2	0.006925634
+Q	49	32	3	0.007011070
+Q	48	35	2	0.007064967
+Q	47	36	4	0.007179896
+Q	46	23	1	0.007205495
+Q	45	25	1	0.007230614
+Q	44	17	3	0.007318716
+Q	43	17	2	0.007376121
+Q	42	31	5	0.007522016
+Q	41	25	4	0.007638486
+Q	40	26	4	0.007754541
+Q	39	35	2	0.007807258
+Q	37	18	4	0.007924896
+Q	36	13	3	0.008013162
+Q	35	15	2	0.008070411
+Q	34	20	3	0.008156805
+Q	33	11	1	0.008184501
+Q	32	15	3	0.008272003
+Q	31	25	1	0.008296107
+Q	29	8	1	0.008324472
+Q	28	7	2	0.008383452
+Q	27	9	2	0.008441894
+Q	26	30	2	0.008494888
+Q	23	2	1	0.008524710
+Q	22	11	3	0.008612846
+Q	20	23	3	0.008697760
+Q	19	6	1	0.008726479
+Q	18	8	1	0.008754658
+Q	16	6	1	0.008783354
+Q	13	2	1	0.008813108
+Q	12	4	2	0.008872604
+Q	11	7	2	0.008931275
+Q	10	4	3	0.009021009
+Q	9	6	4	0.009140436
+Q	8	6	3	0.009229559
+Q	7	5	1	0.009258419
+Q	6	8	3	0.009346925
+Q	4	8	5	0.009495872
+Q	3	17	8	0.009732801

tex/minimap2.bib

@@ -0,0 +1,460 @@
+@article{Chaisson:2012aa,
+	Author = {Chaisson, Mark J and Tesler, Glenn},
+	Journal = {BMC Bioinformatics},
+	Pages = {238},
+	Title = {{Mapping single molecule sequencing reads using basic local alignment with successive refinement (BLASR): application and theory}},
+	Volume = {13},
+	Year = {2012}}
+
+@article{Liu:2016ab,
+	Author = {Liu, Bo and others},
+	Journal = {Bioinformatics},
+	Pages = {1625-31},
+	Title = {{rHAT}: fast alignment of noisy long reads with regional hashing},
+	Volume = {32},
+	Year = {2016}}
+
+@article{Liu:2017aa,
+	Author = {Liu, Bo and others},
+	Journal = {Bioinformatics},
+	Pages = {192-201},
+	Title = {{LAMSA}: fast split read alignment with long approximate matches},
+	Volume = {33},
+	Year = {2017}}
+
+@article{Lin:2017aa,
+	Author = {Lin, Hsin-Nan and Hsu, Wen-Lian},
+	Journal = {Bioinformatics},
+	Title = {Kart: a divide-and-conquer algorithm for {NGS} read alignment},
+	Year = {2017}}
+
+@article{Li:2013aa,
+	Author = {Li, Heng},
+	Journal = {arXiv:1303.3997},
+	Title = {Aligning sequence reads, clone sequences and assembly contigs with {BWA-MEM}},
+	archivePrefix = "arXiv",
+	eprint = {1303.3997},
+	primaryClass = "q-bio",
+	Year = {2013}}
+
+@article{Sovic:2016aa,
+	Author = {Sovi{\'c}, Ivan and others},
+	Journal = {Nat Commun},
+	Pages = {11307},
+	Title = {Fast and sensitive mapping of nanopore sequencing reads with {GraphMap}},
+	Volume = {7},
+	Year = {2016}}
+
+@article{Langmead:2012fk,
+	Author = {Langmead, Ben and Salzberg, Steven L},
+	Journal = {Nat Methods},
+	Pages = {357-9},
+	Title = {Fast gapped-read alignment with {Bowtie} 2},
+	Volume = {9},
+	Year = {2012}}
+
+@article{Li:2016aa,
+	Author = {Li, Heng},
+	Journal = {Bioinformatics},
+	Pages = {2103-10},
+	Title = {Minimap and miniasm: fast mapping and de novo assembly for noisy long sequences},
+	Volume = {32},
+	Year = {2016}}
+
+@misc{Ruan:2016,
+	title = {Ultra-fast de novo assembler using long noisy reads},
+	author = {Jue Ruan},
+	journal = {Unpulished},
+	howpublished = {\href{https://github.com/ruanjue/smartdenovo}{https://github.com/ruanjue/smartdenovo}},
+	year = {2016}}
+
+@article{Miller:1988aa,
+	Author = {Miller, W and Myers, E W},
+	Journal = {Bull Math Biol},
+	Number = {2},
+	Pages = {97-120},
+	Title = {Sequence comparison with concave weighting functions},
+	Volume = {50},
+	Year = {1988}}
+
+@article{Gotoh:1990aa,
+	Author = {Gotoh, O},
+	Journal = {Bull Math Biol},
+	Pages = {359-73},
+	Title = {Optimal sequence alignment allowing for long gaps},
+	Volume = {52},
+	Year = {1990}}
+
+@article{Wu:1996aa,
+	Author = {Wu, Sun and others},
+	Journal = {Algorithmica},
+	Pages = {50-67},
+	Title = {A subquadratic algorithm for approximate limited expression matching},
+	Volume = {15},
+	Year = {1996}}
+
+@article{Daily:2016aa,
+	Author = {Daily, Jeff},
+	Journal = {BMC Bioinformatics},
+	Month = {Feb},
+	Pages = {81},
+	Title = {Parasail: {SIMD C} library for global, semi-global, and local pairwise sequence alignments},
+	Volume = {17},
+	Year = {2016}}
+
+@article{Sedlazeck169557,
+	author = {Sedlazeck, Fritz J and others},
+	title = {Accurate detection of complex structural variations using single molecule sequencing},
+	note = {doi:10.1101/169557},
+	journal = {bioRxiv},
+	year = {2017}}
+
+@article{Altschul:1997vn,
+	Author = {Altschul, S F and others},
+	Journal = {Nucleic Acids Res},
+	Pages = {3389-402},
+	Title = {Gapped {BLAST} and {PSI-BLAST}: a new generation of protein database search programs},
+	Volume = {25},
+	Year = {1997}}
+
+@article{Sosic:2017aa,
+	Author = {{\v S}o{\v s}i\'{c}, Martin and {\v S}ikic, Mile},
+	Journal = {Bioinformatics},
+	Pages = {1394-1395},
+	Title = {Edlib: a {C/C++} library for fast, exact sequence alignment using edit distance},
+	Volume = {33},
+	Year = {2017}}
+
+@article{Abouelhoda:2005aa,
+	Author = {Mohamed Ibrahim Abouelhoda and Enno Ohlebusch},
+	Journal = {J. Discrete Algorithms},
+	Pages = {321-41},
+	Title = {Chaining algorithms for multiple genome comparison},
+	Volume = {3},
+	Year = {2005}}
+
+@article{Ono:2013aa,
+	Author = {Ono, Yukiteru and others},
+	Journal = {Bioinformatics},
+	Pages = {119-21},
+	Title = {{PBSIM}: {PacBio} reads simulator--toward accurate genome assembly},
+	Volume = {29},
+	Year = {2013}}
+
+@article {Jain128835,
+	author = {Jain, Miten and others},
+	title = {Nanopore sequencing and assembly of a human genome with ultra-long reads},
+	year = {2017},
+	note = {doi:10.1101/128835},
+	publisher = {Cold Spring Harbor Labs Journals},
+	journal = {bioRxiv}}
+
+@article{Lau:2016aa,
+	Author = {Lau, Bayo and others},
+	Journal = {Bioinformatics},
+	Pages = {3829-3832},
+	Title = {{LongISLND}: in silico sequencing of lengthy and noisy datatypes},
+	Volume = {32},
+	Year = {2016}}
+
+@article{Robinson:2011aa,
+	Author = {Robinson, James T and others},
+	Journal = {Nat Biotechnol},
+	Pages = {24-6},
+	Title = {Integrative genomics viewer},
+	Volume = {29},
+	Year = {2011}}
+
+@article{Gotoh:1982aa,
+	Author = {Gotoh, O},
+	Journal = {J Mol Biol},
+	Pages = {705-8},
+	Title = {An improved algorithm for matching biological sequences},
+	Volume = {162},
+	Year = {1982}}
+
+@article{Altschul:1986aa,
+	Author = {Altschul, S F and Erickson, B W},
+	Journal = {Bull Math Biol},
+	Pages = {603-16},
+	Title = {Optimal sequence alignment using affine gap costs},
+	Volume = {48},
+	Year = {1986}}
+
+@article{Wu:2005vn,
+	Author = {Wu, Thomas D and Watanabe, Colin K},
+	Journal = {Bioinformatics},
+	Pages = {1859-75},
+	Title = {{GMAP}: a genomic mapping and alignment program for {mRNA} and {EST} sequences},
+	Volume = {21},
+	Year = {2005}}
+
+@article{Iwata:2012aa,
+	Author = {Iwata, Hiroaki and Gotoh, Osamu},
+	Journal = {Nucleic Acids Res},
+	Pages = {e161},
+	Title = {Benchmarking spliced alignment programs including {Spaln2}, an extended version of {Spaln} that incorporates additional species-specific features},
+	Volume = {40},
+	Year = {2012}}
+
+@article{Dobin:2013kx,
+	Author = {Dobin, Alexander and others},
+	Journal = {Bioinformatics},
+	Pages = {15-21},
+	Title = {{STAR}: ultrafast universal {RNA-seq} aligner},
+	Volume = {29},
+	Year = {2013}}
+
+@article{Byrne:2017aa,
+	Author = {Byrne, Ashley and others},
+	Journal = {Nat Commun},
+	Pages = {16027},
+	Title = {Nanopore long-read {RNAseq} reveals widespread transcriptional variation among the surface receptors of individual {B} cells},
+	Volume = {8},
+	Year = {2017}}
+
+@article{Roberts:2004fv,
+	Author = {Roberts, Michael and others},
+	Journal = {Bioinformatics},
+	Pages = {3363-9},
+	Title = {Reducing storage requirements for biological sequence comparison},
+	Volume = {20},
+	Year = {2004}}
+
+@article{Zhang:2006aa,
+	Author = {Zhang, Miao and Gish, Warren},
+	Journal = {Bioinformatics},
+	Pages = {13-20},
+	Title = {Improved spliced alignment from an information theoretic approach},
+	Volume = {22},
+	Year = {2006}}
+
+@article{Li:2007aa,
+	Author = {Li, Heng and others},
+	Journal = {BMC Bioinformatics},
+	Pages = {349},
+	Title = {A cross-species alignment tool {(CAT)}},
+	Volume = {8},
+	Year = {2007}}
+
+@article{Farrar:2007hs,
+	Author = {Farrar, Michael},
+	Journal = {Bioinformatics},
+	Pages = {156-61},
+	Title = {{Striped Smith-Waterman speeds database searches six times over other SIMD implementations}},
+	Volume = {23},
+	Year = {2007}}
+
+@techreport{Holtgrewe:2010aa,
+	Address = {Freie Universit{\"a}t Berlin},
+	Author = {Holtgrewe, M.},
+	Institution = {Institut f{\"u}r Mathematik und Informatik},
+	Number = {TR-B-10-06},
+	Title = {Mason -- a read simulator for second generation sequencing data},
+	Year = {2010}}
+
+@article{Zaharia:2011aa,
+	Author = {Zaharia, Matei and others},
+	Journal = {arXiv:1111:5572},
+	Title = {Faster and More Accurate Sequence Alignment with {SNAP}},
+	Year = {2011}}
+
+@article{Irimia:2008aa,
+	Author = {Irimia, Manuel and Roy, Scott William},
+	Journal = {PLoS Genet},
+	Pages = {e1000148},
+	Title = {Evolutionary convergence on highly-conserved 3' intron structures in intron-poor eukaryotes and insights into the ancestral eukaryotic genome},
+	Volume = {4},
+	Year = {2008}}
+
+@article{Depristo:2011vn,
+	Author = {Depristo, Mark A and others},
+	Journal = {Nat Genet},
+	Pages = {491-8},
+	Title = {A framework for variation discovery and genotyping using next-generation {DNA} sequencing data},
+	Volume = {43},
+	Year = {2011}}
+
+@article{Kurtz:2004zr,
+	Author = {Kurtz, Stefan and others},
+	Journal = {Genome Biol},
+	Pages = {R12},
+	Title = {Versatile and open software for comparing large genomes},
+	Volume = {5},
+	Year = {2004}}
+
+@article {Li223297,
+	author = {Li, Heng and others},
+	title = {New synthetic-diploid benchmark for accurate variant calling evaluation},
+	year = {2017},
+	note = {doi:10.1101/223297},
+	journal = {bioRxiv}
+}
+
+@article{Berlin:2015xy,
+	Author = {Berlin, Konstantin and others},
+	Journal = {Nat Biotechnol},
+	Pages = {623-30},
+	Title = {Assembling large genomes with single-molecule sequencing and locality-sensitive hashing},
+	Volume = {33},
+	Year = {2015}}
+
+@article{Gurevich:2013aa,
+	Author = {Gurevich, Alexey and others},
+	Journal = {Bioinformatics},
+	Pages = {1072-5},
+	Title = {{QUAST}: quality assessment tool for genome assemblies},
+	Volume = {29},
+	Year = {2013}}
+
+@article{Li:2010fk,
+	Author = {Li, Heng and Durbin, Richard},
+	Journal = {Bioinformatics},
+	Pages = {589-95},
+	Title = {Fast and accurate long-read alignment with {Burrows-Wheeler} transform},
+	Volume = {26},
+	Year = {2010}}
+
+@article{Marcais:2018aa,
+	Author = {Mar{\c c}ais, Guillaume and others},
+	Journal = {PLoS Comput Biol},
+	Pages = {e1005944},
+	Title = {{MUMmer4}: A fast and versatile genome alignment system},
+	Volume = {14},
+	Year = {2018}}
+
+@article{Li:2009ys,
+	Author = {Li, Heng and others},
+	Journal = {Bioinformatics},
+	Pages = {2078-9},
+	Title = {The {Sequence Alignment/Map format and SAMtools}},
+	Volume = {25},
+	Year = {2009}}
+
+@article{Suzuki:2018aa,
+	Author = {Suzuki, Hajime and Kasahara, Masahiro},
+	Journal = {BMC Bioinformatics},
+	Pages = {45},
+	Title = {Introducing difference recurrence relations for faster semi-global alignment of long sequences},
+	Volume = {19},
+	Year = {2018}}
+
+@article{Li:2018ab,
+	Author = {Li, Heng},
+	Journal = {Bioinformatics},
+	Pages = {3094-3100},
+	Title = {Minimap2: pairwise alignment for nucleotide sequences},
+	Volume = {34},
+	Year = {2018}}
+
+@article{Jain:2020aa,
+	Author = {Jain, Chirag and others},
+	Journal = {Bioinformatics},
+	Pages = {i111-i118},
+	Title = {Weighted minimizer sampling improves long read mapping},
+	Volume = {36},
+	Year = {2020}}
+
+@article{Miga:2020aa,
+	Author = {Miga, Karen H and others},
+	Journal = {Nature},
+	Pages = {79-84},
+	Title = {Telomere-to-telomere assembly of a complete human {X} chromosome},
+	Volume = {585},
+	Year = {2020}}
+
+@article {Jain2020.11.01.363887,
+	author = {Jain, Chirag and others},
+	title = {A long read mapping method for highly repetitive reference sequences},
+	elocation-id = {2020.11.01.363887},
+	year = {2020},
+	doi = {10.1101/2020.11.01.363887},
+	publisher = {Cold Spring Harbor Laboratory},
+	URL = {https://www.biorxiv.org/content/early/2020/11/02/2020.11.01.363887},
+	eprint = {https://www.biorxiv.org/content/early/2020/11/02/2020.11.01.363887.full.pdf},
+	journal = {bioRxiv}
+}
+
+@article{Li:2020aa,
+	Author = {Li, Heng and others},
+	Journal = {Genome Biol},
+	Pages = {265},
+	Title = {The design and construction of reference pangenome graphs with minigraph},
+	Volume = {21},
+	Year = {2020}}
+
+@article{Ren:2021aa,
+	Author = {Ren, Jingwen and Chaisson, Mark J P},
+	Journal = {PLoS Comput Biol},
+	Pages = {e1009078},
+	Title = {lra: A long read aligner for sequences and contigs},
+	Volume = {17},
+	Year = {2021}}
+
+@inproceedings{DBLP:conf/wabi/AbouelhodaO03,
+	Author = {Mohamed Ibrahim Abouelhoda and Enno Ohlebusch},
+	Booktitle = {Algorithms in Bioinformatics, Third International Workshop, {WABI} 2003, Budapest, Hungary, September 15-20, 2003, Proceedings},
+	Crossref = {DBLP:conf/wabi/2003},
+	Pages = {1--16},
+	Title = {A Local Chaining Algorithm and Its Applications in Comparative Genomics},
+	Year = {2003}}
+
+@article{Ono:2021aa,
+	Author = {Ono, Yukiteru and others},
+	Journal = {Bioinformatics},
+	Pages = {589-595},
+	Title = {{PBSIM2}: a simulator for long-read sequencers with a novel generative model of quality scores},
+	Volume = {37},
+	Year = {2021}}
+
+@article{Sedlazeck:2018ab,
+	Author = {Sedlazeck, Fritz J and others},
+	Journal = {Nat Methods},
+	Pages = {461-468},
+	Title = {Accurate detection of complex structural variations using single-molecule sequencing},
+	Volume = {15},
+	Year = {2018}}
+
+@article{Jeffares:2017aa,
+	Author = {Jeffares, Daniel C and others},
+	Journal = {Nat Commun},
+	Pages = {14061},
+	Title = {Transient structural variations have strong effects on quantitative traits and reproductive isolation in fission yeast},
+	Volume = {8},
+	Year = {2017}}
+
+@article{Zook:2020aa,
+	Author = {Zook, Justin M and others},
+	Journal = {Nat Biotechnol},
+	Pages = {1347-1355},
+	Title = {A robust benchmark for detection of germline large deletions and insertions},
+	Volume = {38},
+	Year = {2020}}
+
+@article{Harpak:2017aa,
+	Author = {Harpak, Arbel and others},
+	Journal = {Proc Natl Acad Sci U S A},
+	Pages = {12779-12784},
+	Title = {Frequent nonallelic gene conversion on the human lineage and its effect on the divergence of gene duplicates},
+	Volume = {114},
+	Year = {2017}}
+
+@article{Li:2018aa,
+	Author = {Li, Heng and others},
+	Journal = {Nat Methods},
+	Month = {Aug},
+	Number = {8},
+	Pages = {595-597},
+	Title = {A synthetic-diploid benchmark for accurate variant-calling evaluation},
+	Volume = {15},
+	Year = {2018}}
+
+@article{Gu:1995wt,
+	author = {Gu, X and Li, W H},
+	journal = {J Mol Evol},
+	month = {Apr},
+	number = {4},
+	pages = {464-73},
+	title = {The size distribution of insertions and deletions in human and rodent pseudogenes suggests the logarithmic gap penalty for sequence alignment},
+	volume = {40},
+	year = {1995}}

tex/minimap2.tex

@@ -0,0 +1,724 @@
+\documentclass{bioinfo}
+\copyrightyear{2018}
+\pubyear{2018}
+
+\usepackage{graphicx}
+\usepackage{hyperref}
+\usepackage{url}
+\usepackage{amsmath}
+\usepackage[ruled,vlined]{algorithm2e}
+\newcommand\mycommfont[1]{\footnotesize\rmfamily{\it #1}}
+\SetCommentSty{mycommfont}
+\SetKwComment{Comment}{$\triangleright$\ }{}
+
+\usepackage{natbib}
+\bibliographystyle{apalike}
+
+\DeclareMathOperator*{\argmax}{argmax}
+
+\begin{document}
+\firstpage{1}
+
+\title[Aligning nucleotide sequences with minimap2]{Minimap2: pairwise alignment for nucleotide sequences}
+\author[Li]{Heng Li}
+\address{Broad Institute, 415 Main Street, Cambridge, MA 02142, USA}
+
+\maketitle
+
+\begin{abstract}
+
+\section{Motivation:} Recent advances in sequencing technologies promise
+ultra-long reads of $\sim$100 kilo bases (kb) in average, full-length mRNA or
+cDNA reads in high throughput and genomic contigs over 100 mega bases (Mb) in
+length. Existing alignment programs are unable or inefficient to process such data
+at scale, which presses for the development of new alignment algorithms.
+
+\section{Results:} Minimap2 is a general-purpose alignment program to map DNA or long
+mRNA sequences against a large reference database. It works with accurate short
+reads of $\ge$100bp in length, $\ge$1kb genomic reads at error rate $\sim$15\%,
+full-length noisy Direct RNA or cDNA reads, and assembly contigs or closely
+related full chromosomes of hundreds of megabases in length. Minimap2 does
+split-read alignment, employs concave gap cost for long insertions and
+deletions (INDELs) and introduces new heuristics to reduce spurious alignments.
+It is 3--4 times as fast as mainstream short-read mappers at comparable
+accuracy, and is $\ge$30 times faster than long-read genomic or cDNA
+mappers at higher accuracy, surpassing most aligners specialized in one type of
+alignment.
+
+\section{Availability and implementation:}
+\href{https://github.com/lh3/minimap2}{https://github.com/lh3/minimap2}
+
+\section{Contact:} hengli@broadinstitute.org
+\end{abstract}
+
+\section{Introduction}
+
+Single Molecule Real-Time (SMRT) sequencing technology and Oxford Nanopore
+technologies (ONT) produce reads over 10kbp in length at an error rate
+$\sim$15\%. Several aligners have been developed for such
+data~\citep{Chaisson:2012aa,Li:2013aa,Liu:2016ab,Sovic:2016aa,Liu:2017aa,Lin:2017aa,Sedlazeck169557}.
+Most of them were five times as slow as mainstream short-read
+aligners~\citep{Langmead:2012fk,Li:2013aa} in terms of the number of bases
+mapped per second. We speculated there could be substantial room for speedup on
+the thought that 10kb long sequences should be easier to map than 100bp reads
+because we can more effectively skip repetitive regions, which are often the
+bottleneck of short-read alignment. We confirmed our speculation by achieving
+approximate mapping 50 times faster than BWA-MEM~\citep{Li:2016aa}.
+\citet{Suzuki:2018aa} extended our work with a fast and novel algorithm on
+generating base-level alignment, which in turn inspired us to develop minimap2
+with added functionality.
+
+Both SMRT and ONT have been applied to the sequencing of spliced mRNAs (RNA-seq). While
+traditional mRNA aligners work~\citep{Wu:2005vn,Iwata:2012aa}, they are not
+optimized for long noisy sequence reads and are tens of times slower than
+dedicated long-read aligners. When developing minimap2 initially for aligning
+genomic DNA only, we realized minor modifications could enable the base
+algorithm to map mRNAs as well. Minimap2 becomes a first RNA-seq aligner
+specifically designed for long noisy reads. We have also extended the original
+algorithm to map short reads at a speed faster than several mainstream
+short-read mappers.
+
+In this article, we will describe the minimap2 algorithm and its applications
+to different types of input sequences. We will evaluate the performance and
+accuracy of minimap2 on several simulated and real data sets and demonstrate
+the versatility of minimap2.
+
+\begin{methods}
+\section{Methods}
+
+Minimap2 follows a typical seed-chain-align procedure as is used by most
+full-genome aligners. It collects minimizers~\citep{Roberts:2004fv} of the
+reference sequences and indexes them in a hash table, with the key being the
+hash of a minimizer and the value being a list of locations of the minimizer
+copies. Then for each query
+sequence, minimap2 takes query minimizers as \emph{seeds}, finds exact matches
+(i.e. \emph{anchors}) to the reference, and identifies sets of colinear anchors as
+\emph{chains}. If base-level alignment is requested, minimap2 applies dynamic
+programming (DP) to extend from the ends of chains and to close
+regions between adjacent anchors in chains.
+
+Minimap2 uses indexing and seeding algorithms similar to
+minimap~\citep{Li:2016aa}, and furthers the predecessor with more accurate
+chaining, the ability to produce base-level alignment and the support of
+spliced alignment.
+
+\subsection{Chaining}
+
+\subsubsection{Chaining}
+An \emph{anchor} is a 3-tuple $(x,y,w)$, indicating interval $[x-w+1,x]$ on the
+reference matching interval $[y-w+1,y]$ on the query. Given a list of anchors
+sorted by ending reference position $x$, let $f(i)$ be the maximal chaining
+score up to the $i$-th anchor in the list. $f(i)$ can be calculated with
+dynamic programming:
+\begin{equation}\label{eq:chain}
+f(i)=\max\big\{\max_{i>j\ge 1} \{ f(j)+\alpha(j,i)-\beta(j,i) \},w_i\big\}
+\end{equation}
+where $\alpha(j,i)=\min\big\{\min\{y_i-y_j,x_i-x_j\},w_i\big\}$ is the number of
+matching bases between the two anchors. $\beta(j,i)>0$ is the gap cost. It
+equals $\infty$ if $y_j\ge y_i$ or $\max\{y_i-y_j,x_i-x_j\}>G$ (i.e. the
+distance between two anchors is too large); otherwise
+\begin{equation}\label{eq:chain-gap}
+\beta(j,i)=\gamma_c\big((y_i-y_j)-(x_i-x_j)\big)
+\end{equation}
+In implementation, a gap of length $l$ costs
+\[
+\gamma_c(l)=\left\{\begin{array}{ll}
+0.01\cdot \bar{w}\cdot|l|+0.5\log_2|l| & (l\not=0) \\
+0 & (l=0)
+\end{array}\right.
+\]
+where $\bar{w}$ is the average seed length. For $N$ anchors, directly computing all $f(\cdot)$ with
+Eq.~(\ref{eq:chain}) takes $O(N^2)$ time. Although theoretically faster
+chaining algorithms exist~\citep{Abouelhoda:2005aa}, they
+are inapplicable to generic gap cost, complex to implement and usually
+associated with a large constant. We introduced a simple heuristic to
+accelerate chaining.
+
+We note that if anchor $i$ is chained to $j$, chaining $i$ to a predecessor
+of $j$ is likely to yield a lower score. When evaluating Eq.~(\ref{eq:chain}),
+we start from anchor $i-1$ and stop the process if we cannot find a better
+score after up to $h$ iterations. This approach reduces the average time to
+$O(hN)$. In practice, we can almost always find the optimal chain with
+$h=50$; even if the heuristic fails, the optimal chain is often close.
+
+\subsubsection{Backtracking}
+Let $P(i)$ be the index of the best predecessor of anchor $i$. It equals 0 if
+$f(i)=w_i$ or $\argmax_j\{f(j)+\alpha(j,i)-\beta(j,i)\}$ otherwise. For each
+anchor $i$ in the descending order of $f(i)$, we apply $P(\cdot)$ repeatedly to
+find its predecessor and mark each visited $i$ as `used', until $P(i)=0$ or we
+reach an already `used' $i$. This way we find all chains with no anchors used
+in more than one chains.
+
+\subsubsection{Identifying primary chains}\label{sec:primary}
+In the absence of copy number changes, each query segment should not be mapped
+to two places in the reference. However, chains found at the previous step may
+have significant or complete overlaps due to repeats in the reference~\citep{Li:2010fk}.
+Minimap2 used the following procedure to identify \emph{primary chains} that do
+not greatly overlap on the query.
+
+Let $Q$ be an empty set initially. For each
+chain from the best to the worst according to their chaining scores: if on the
+query, the chain overlaps with a chain in $Q$ by 50\% or higher percentage of
+the shorter chain, mark the chain as secondary to the chain in $Q$; otherwise,
+add the chain to $Q$. In the end, $Q$ contains all the primary chains. We did
+not choose a more sophisticated data structure (e.g. range tree or k-d tree)
+because this step is not the performance bottleneck.
+
+For each primary chain, minimap2 estimates its mapping quality with an
+empirical formula:
+\[
+{\rm mapQ}=40\cdot (1-f_2/f_1)\cdot\min\{1,m/10\}\cdot\log f_1
+\]
+where $\log$ denotes natural logarithm, $m$ is the number of anchors on the primary chain, $f_1$ is the chaining
+score, and $f_2\le f_1$ is the score of the best chain that is secondary to the
+primary chain. Intuitively, a chain is assigned to a higher mapping quality if
+it is long and its best secondary chain is weak.
+
+\subsubsection{Estimating per-base sequence divergence}
+Suppose a query sequence harbors $n$ seeds of length $k$, $m$ of which are
+present in a chain. We want to estimate the sequence divergence $\epsilon$
+between the query and the reference sequences in the chain. This is useful
+when base-level alignment is too expensive to perform.
+
+If we model substitutions with a homogeneous Poisson process along the query
+sequence, the probablity of seeing $k$ consecutive bases without substitutions
+is $e^{-k\epsilon}$. On the assumption that all $k$-mers are independent of
+each other, the likelihood function of $\epsilon$ is
+\[
+\mathcal{L}(\epsilon|n,m,k)=e^{-m\cdot k\epsilon}(1-e^{-k\epsilon})^{n-m}
+\]
+The maximum likelihood estimate of $\epsilon$ is
+\[
+\hat{\epsilon}=\frac{1}{k}\log\frac{n}{m}
+\]
+In reality, sequencing errors are sometimes clustered and $k$-mers are not
+independent of each other, especially when we take minimizers as seeds. These
+violate the assumptions in the derivation above. As a result, $\hat{\epsilon}$
+is only approximate and can be biased. It also ignores long deletions from the
+reference sequence. In practice, fortunately, $\hat{\epsilon}$ is often close
+to and strongly correlated with the sequence divergence estimated from
+base-level alignments. On the several datasets used in
+Section~\ref{sec:long-genomic}, the Spearman correlation coefficient is around
+$0.9$.
+
+\subsubsection{Indexing with homopolymer compressed $k$-mers}
+SmartDenovo
+(\href{https://github.com/ruanjue/smartdenovo}{https://github.com/ruanjue/smartdenovo};
+J. Ruan, personal communication) indexes reads with homopolymer-compressed (HPC)
+$k$-mers and finds the strategy improves overlap sensitivity for SMRT reads.
+Minimap2 adopts the same heuristic.
+
+The HPC string of a string $s$, denoted by ${\rm HPC}(s)$, is constructed by
+contracting homopolymers in $s$ to a single base.  An HPC $k$-mer of $s$ is a
+$k$-long substring of ${\rm HPC}(s)$. For example, suppose $s={\tt GGATTTTCCA}$,
+${\rm HPC}(s)={\tt GATCA}$ and the first HPC 4-mer is ${\tt GATC}$.
+
+To demonstrate the effectiveness of HPC $k$-mers, we performed read overlapping
+for the example {\it E. coli} SMRT reads from PBcR~\citep{Berlin:2015xy}, using
+different types of $k$-mers. With normal 15bp minimizers per 5bp window,
+minimap2 finds 90.9\% of $\ge$2kb overlaps inferred from the read-to-reference
+alignment. With HPC 19-mers per 5bp window, minimap2 finds 97.4\% of overlaps. It achieves this
+higher sensitivity by indexing 1/3 fewer minimizers, which further helps
+performance. HPC-based indexing reduces the sensitivity for current ONT reads, though.
+
+\subsection{Aligning genomic DNA}\label{sec:genomic}
+
+\subsubsection{Alignment with 2-piece affine gap cost}
+
+Minimap2 performs DP-based global alignment between adjacent anchors in a
+chain. It uses a 2-piece affine gap cost~\citep{Gotoh:1990aa}:
+\begin{equation}\label{eq:2-piece}
+\gamma_a(l)=\min\{q+|l|\cdot e,\tilde{q}+|l|\cdot\tilde{e}\}
+\end{equation}
+Without losing generality, we always assume $q+e<\tilde{q}+\tilde{e}$.
+On the condition that $e>\tilde{e}$, it applies cost $q+|l|\cdot e$ to gaps
+shorter than $\lceil(\tilde{q}-q)/(e-\tilde{e})\rceil$ and applies
+$\tilde{q}+|l|\cdot\tilde{e}$ to longer gaps.  This scheme helps to recover
+longer insertions and deletions~(INDELs).
+
+The equation to compute the optimal alignment under $\gamma_a(\cdot)$ is
+\begin{equation}\label{eq:ae86}
+\left\{\begin{array}{l}
+H_{ij} = \max\{H_{i-1,j-1}+s(i,j),E_{ij},F_{ij},\tilde{E}_{ij},\tilde{F}_{ij}\}\\
+E_{i+1,j}= \max\{H_{ij}-q,E_{ij}\}-e\\
+F_{i,j+1}= \max\{H_{ij}-q,F_{ij}\}-e\\
+\tilde{E}_{i+1,j}= \max\{H_{ij}-\tilde{q},\tilde{E}_{ij}\}-\tilde{e}\\
+\tilde{F}_{i,j+1}= \max\{H_{ij}-\tilde{q},\tilde{F}_{ij}\}-\tilde{e}
+\end{array}\right.
+\end{equation}
+where $s(i,j)$ is the score between the $i$-th reference base and $j$-th query
+base. Eq.~(\ref{eq:ae86}) is a natural extension to the equation under affine
+gap cost~\citep{Gotoh:1982aa,Altschul:1986aa}.
+
+\subsubsection{The Suzuki-Kasahara formulation}
+
+When we allow gaps longer than several hundred base pairs, nucleotide-level
+alignment is much slower than chaining. SSE acceleration is critical to the
+performance of minimap2. Traditional SSE implementations~\citep{Farrar:2007hs}
+based on Eq.~(\ref{eq:ae86}) can achieve 16-way parallelization for short
+sequences, but only 4-way parallelization when the peak alignment score reaches
+32767. Long sequence alignment may exceed this threshold. Inspired by
+\citet{Wu:1996aa} and the following work, \citet{Suzuki:2018aa} proposed a
+difference-based formulation that lifted this limitation.
+In case of 2-piece gap cost, define
+\[
+\left\{\begin{array}{ll}
+u_{ij}\triangleq H_{ij}-H_{i-1,j} & v_{ij}\triangleq H_{ij}-H_{i,j-1} \\
+x_{ij}\triangleq E_{i+1,j}-H_{ij} & \tilde{x}_{ij}\triangleq \tilde{E}_{i+1,j}-H_{ij} \\
+y_{ij}\triangleq F_{i,j+1}-H_{ij} & \tilde{y}_{ij}\triangleq \tilde{F}_{i,j+1}-H_{ij}
+\end{array}\right.
+\]
+We can transform Eq.~(\ref{eq:ae86}) to
+\begin{equation}\label{eq:suzuki}
+\left\{\begin{array}{lll}
+z_{ij}&=&\max\{s(i,j),x_{i-1,j}+v_{i-1,j},y_{i,j-1}+u_{i,j-1},\\
+&&\tilde{x}_{i-1,j}+v_{i-1,j},\tilde{y}_{i,j-1}+u_{i,j-1}\}\\
+u_{ij}&=&z_{ij}-v_{i-1,j}\\
+v_{ij}&=&z_{ij}-u_{i,j-1}\\
+x_{ij}&=&\max\{0,x_{i-1,j}+v_{i-1,j}-z_{ij}+q\}-q-e\\
+y_{ij}&=&\max\{0,y_{i,j-1}+u_{i,j-1}-z_{ij}+q\}-q-e\\
+\tilde{x}_{ij}&=&\max\{0,\tilde{x}_{i-1,j}+v_{i-1,j}-z_{ij}+\tilde{q}\}-\tilde{q}-\tilde{e}\\
+\tilde{y}_{ij}&=&\max\{0,\tilde{y}_{i,j-1}+u_{i,j-1}-z_{ij}+\tilde{q}\}-\tilde{q}-\tilde{e}
+\end{array}\right.
+\end{equation}
+where $z_{ij}$ is a temporary variable that does not need to be stored.
+
+An important property of Eq.~(\ref{eq:suzuki}) is that all values are bounded
+by scoring parameters. To see that,
+\[
+x_{ij}=E_{i+1,j}-H_{ij}=\max\{-q,E_{ij}-H_{ij}\}-e
+\]
+With $E_{ij}\le H_{ij}$, we have
+\[
+-q-e\le x_{ij}\le\max\{-q,0\}-e=-e
+\]
+and similar inequations for $y_{ij}$, $\tilde{x}_{ij}$ and $\tilde{y}_{ij}$.
+In addition,
+\[
+u_{ij}=z_{ij}-v_{i-1,j}\ge\max\{x_{i-1,j},\tilde{x}_{i-1,j}\}\ge-q-e
+\]
+As the maximum value of $z_{ij}=H_{ij}-H_{i-1,j-1}$ is $M$, the maximal
+matching score, we can derive
+\[
+u_{ij}\le M-v_{i-1,j}\le M+q+e
+\]
+In conclusion, in Eq.~(\ref{eq:suzuki}), $x$ and $y$ are bounded by $[-q-e,-e]$,
+$\tilde{x}$ and $\tilde{y}$ by $[-\tilde{q}-\tilde{e},-\tilde{e}]$, and $u$ and
+$v$ by $[-q-e,M+q+e]$. When $-128\le-q-e<M+q+e\le127$, each of them can be stored as
+a 8-bit integer. This enables 16-way SSE vectorization regardless of the peak
+score of the alignment.
+
+For a more efficient SSE implementation, we transform the row-column coordinate
+to the diagonal-antidiagonal coordinate by letting $r\gets i+j$ and $t\gets i$.
+Eq.~(\ref{eq:suzuki}) becomes:
+\begin{equation*}
+\left\{\begin{array}{lll}
+z_{rt}&=&\max\{s(t,r-t),x_{r-1,t-1}+v_{r-1,t-1},y_{r-1,t}\\
+&&+u_{r-1,t},\tilde{x}_{r-1,t-1}+v_{r-1,t-1},\tilde{y}_{r-1,t}+u_{r-1,t}\}\\
+u_{rt}&=&z_{rt}-v_{r-1,t-1}\\
+v_{rt}&=&z_{rt}-u_{r-1,t}\\
+x_{rt}&=&\max\{0,x_{r-1,t-1}+v_{r-1,t-1}-z_{rt}+q\}-q-e\\
+y_{rt}&=&\max\{0,y_{r-1,t}+u_{r-1,t}-z_{rt}+q\}-q-e\\
+\tilde{x}_{rt}&=&\max\{0,\tilde{x}_{r-1,t-1}+v_{r-1,t-1}-z_{rt}+\tilde{q}\}-\tilde{q}-\tilde{e}\\
+\tilde{y}_{rt}&=&\max\{0,\tilde{y}_{r-1,t}+u_{r-1,t}-z_{rt}+\tilde{q}\}-\tilde{q}-\tilde{e}
+\end{array}\right.
+\end{equation*}
+In this formulation, cells with the same diagonal index $r$ are independent of
+each other. This allows us to fully vectorize the computation of all cells on
+the same anti-diagonal in one inner loop. It also simplifies banded alignment (500bp band width by default),
+which would be difficult with striped vectorization~\citep{Farrar:2007hs}.
+
+On the condition that $q+e<\tilde{q}+\tilde{e}$ and $e>\tilde{e}$, the initial
+values in the diagonal-antidiagonal formuation are
+\[
+\left\{\begin{array}{l}
+x_{r-1,-1}=y_{r-1,r}=-q-e\\
+\tilde{x}_{r-1,-1}=\tilde{y}_{r-1,r}=-\tilde{q}-\tilde{e}\\
+u_{r-1,r}=v_{r-1,-1}=\eta(r)\\
+\end{array}\right.
+\]
+where
+\[
+\eta(r)=\left\{\begin{array}{ll}
+-q-e & (r=0) \\
+-e & (r<\lceil\frac{\tilde{q}-q}{e-\tilde{e}}-1\rceil) \\
+r\cdot(e-\tilde{e})-(\tilde{q}-q)-\tilde{e} & (r=\lceil\frac{\tilde{q}-q}{e-\tilde{e}}-1\rceil) \\
+-\tilde{e} & (r>\lceil\frac{\tilde{q}-q}{e-\tilde{e}}-1\rceil)
+\end{array}\right.
+\]
+These can be derived from the initial values for Eq.~(\ref{eq:ae86}).
+
+When performing global alignment, we do not need to compute $H_{rt}$ in each cell.
+We use 16-way vectorization throughout the alignment process. When extending
+alignments from ends of chains, we need to find the cell $(r,t)$ where $H_{rt}$
+reaches the maximum. We resort to 4-way vectorization to compute
+$H_{rt}=H_{r-1,t}+u_{rt}$. Because this computation is simple,
+Eq.~(\ref{eq:suzuki}) is still the dominant performance bottleneck.
+
+In practice, our 16-way vectorized implementation of global alignment is three
+times as fast as Parasail's 4-way vectorization~\citep{Daily:2016aa}.  Without
+banding, our implementation is slower than Edlib~\citep{Sosic:2017aa}, but with
+a 1000bp band, it is considerably faster. When performing global alignment
+between anchors, we expect the alignment to stay close to the diagonal of the
+DP matrix. Banding is applicable most of the time.
+
+\subsubsection{The Z-drop heuristic}
+
+With global alignment, minimap2 may force to align unrelated sequences between
+two adjacent anchors. To avoid such an artifact, we compute accumulative
+alignment score along the alignment path and break the alignment where the
+score drops too fast in the diagonal direction. More precisely, let $S(i,j)$ be
+the alignment score along the alignment path ending at cell $(i,j)$ in the DP
+matrix. We break the alignment if there exist $(i',j')$ and $(i,j)$, $i'<i$ and
+$j'<j$, such that
+\[
+S(i',j')-S(i,j)>Z+e\cdot|(i-i')-(j-j')|
+\]
+where $e$ is the gap extension cost and $Z$ is an arbitrary threshold.
+This strategy is first used in BWA-MEM. It is similar to X-drop employed in
+BLAST~\citep{Altschul:1997vn}, but unlike X-drop, it would not break the
+alignment in the presence of a single long gap.
+
+When minimap2 breaks a global alignment between two anchors, it performs local
+alignment between the two subsequences involved in the global alignment, but
+this time with the one subsequence reverse complemented. This additional
+alignment step may identify short inversions that are missed during chaining.
+
+\subsubsection{Filtering out misplaced anchors}
+Due to sequencing errors and local homology, some anchors in a chain may be
+wrong. If we blindly align regions between two misplaced anchors, we will
+produce a suboptimal alignment. To reduce this artifact, we filter out
+anchors that lead to a $>$10bp insertion and a $>$10bp deletion at the same
+time, and filter out terminal anchors that lead to a long gap towards the ends
+of a chain. These heuristics greatly alleviate the issues with misplaced
+anchors, but they are unable to fix all such errors. Local misalignment is a
+limitation of minimap2 which we hope to address in future.
+
+\subsection{Aligning spliced sequences}
+
+The algorithm described above can be adapted to spliced alignment. In this
+mode, the chaining gap cost distinguishes insertions to and deletions from the
+reference: $\gamma_c(l)$ in Eq.~(\ref{eq:chain-gap}) takes the form of
+\[
+\gamma_c(l)=\left\{\begin{array}{ll}
+0.01\cdot\bar{w}\cdot l+0.5\log_2 l & (l>0) \\
+\min\{0.01\cdot\bar{w}\cdot|l|,\log_2|l|\} & (l<0)
+\end{array}\right.
+\]
+Similarly, the gap cost function used for DP-based alignment is changed to
+\[
+\gamma_a(l)=\left\{\begin{array}{ll}
+q+l\cdot e & (l>0) \\
+\min\{q+|l|\cdot e,\tilde{q}\} & (l<0)
+\end{array}\right.
+\]
+In alignment, a deletion no shorter than $\lceil(\tilde{q}-q)/e\rceil$ is
+regarded as an intron, which pays no cost to gap extensions.
+
+To pinpoint precise splicing junctions, minimap2 introduces reference-dependent
+cost to penalize non-canonical splicing:
+\begin{equation}\label{eq:splice}
+\left\{\begin{array}{l}
+H_{ij} = \max\{H_{i-1,j-1}+s(i,j),E_{ij},F_{ij},\tilde{E}_{ij}-a(i)\}\\
+E_{i+1,j}= \max\{H_{ij}-q,E_{ij}\}-e\\
+F_{i,j+1}= \max\{H_{ij}-q,F_{ij}\}-e\\
+\tilde{E}_{i+1,j}= \max\{H_{ij}-d(i)-\tilde{q},\tilde{E}_{ij}\}\\
+\end{array}\right.
+\end{equation}
+Let $T$ be the reference sequence. $d(i)$ is computed as
+\[d(i)=\left\{\begin{array}{ll}
+0 & \mbox{if $T[i+1,i+3]$ is ${\tt GTA}$ or ${\tt GTG}$} \\
+p/2 & \mbox{if $T[i+1,i+3]$ is ${\tt GTC}$ or ${\tt GTT}$} \\
+p & \mbox{otherwise}
+\end{array}\right.\]
+where $T[i,j]$ extracts a substring of $T$ between $i$ and $j$ inclusively.
+$d(i)$ penalizes non-canonical donor sites with $p$ and less frequent Eukaryotic
+splicing signal ${\tt GT[C/T]}$ with $p/2$~\citep{Irimia:2008aa}. Similarly,
+\[a(i)=\left\{\begin{array}{ll}
+0 & \mbox{if $T[i-2,i]$ is ${\tt CAG}$ or ${\tt TAG}$} \\
+p/2 & \mbox{if $T[i-2,i]$ is ${\tt AAG}$ or ${\tt GAG}$} \\
+p & \mbox{otherwise}
+\end{array}\right.\]
+models the acceptor signal. Eq.~(\ref{eq:splice}) is close to an equation in
+\citet{Zhang:2006aa} except that we allow insertions immediately followed by
+deletions and vice versa; in addition, we use the Suzuki-Kasahara diagonal
+formulation in actual implementation.
+
+If RNA-seq reads are not sequenced from stranded libraries, the read strand
+relative to the underlying transcript is unknown. By default, minimap2 aligns
+each chain twice, first assuming ${\tt GT}$--${\tt AG}$ as the splicing signal
+and then assuming ${\tt CT}$--${\tt AC}$, the reverse complement of ${\tt
+GT}$--${\tt AG}$, as the splicing signal. The alignment with a higher score is
+taken as the final alignment. This procedure also infers the relative strand of
+reads that span canonical splicing sites.
+
+In the spliced alignment mode, minimap2 further increases the density of
+minimizers and disables banded alignment. Together with the two-round DP-based
+alignment, spliced alignment is several times slower than genomic DNA
+alignment.
+
+\subsection{Aligning short paired-end reads}
+
+During chaining, minimap2 takes a pair of reads as one fragment with a gap of
+unknown length in the middle. It applies a normal gap cost between seeds on the
+same read but is a more permissive gap cost between seeds on different reads.
+More precisely, the gap cost during chaining is ($l\not=0$):
+\[
+\gamma_c(l)=\left\{\begin{array}{ll}
+0.01\cdot\bar{w}\cdot |l|+0.5\log_2 |l| & \mbox{if two seeds on the same read} \\
+\min\{0.01\cdot\bar{w}\cdot|l|,\log_2|l|\} & \mbox{otherwise}
+\end{array}\right.
+\]
+After identifying primary chains (Section~\ref{sec:primary}), we split each
+fragment chain into two read chains and perform alignment for each read as in
+Section~\ref{sec:genomic}.  Finally, we pair hits of each read end to find
+consistent paired-end alignments.
+
+\end{methods}
+
+\section{Results}
+
+Minimap2 is implemented in the C programming language and comes with APIs in
+both C and Python. It is distributed under the MIT license, free to both
+commercial and academic uses. Minimap2 uses the same base algorithm for all
+applications, but it has to apply different sets of parameters depending on
+input data types. Similar to BWA-MEM, minimap2 introduces `presets' that
+modify multiple parameters with a simple invocation. Detailed settings
+and command-line options can be found in the minimap2 manpage. In addition to
+the applications evaluated in the following sections, minimap2 also retains
+minimap's functionality to find overlaps between long reads and to search
+against large multi-species databases such as \emph{nt} from NCBI.
+
+\subsection{Aligning long genomic reads}\label{sec:long-genomic}
+
+\begin{figure}[!tb]
+\centering
+\includegraphics[width=.5\textwidth]{roc-color.pdf}
+\caption{Evaluation on aligning simulated reads. Simulated reads were mapped 
+to the primary assembly of human genome GRCh38. A read is considered correctly
+mapped if its longest alignment overlaps with the true interval, and the
+overlap length is $\ge$10\% of the true interval length.  Read alignments are
+sorted by mapping quality in the descending order. For each mapping quality
+threshold, the fraction of alignments (out of the number of input reads) with
+mapping quality above the threshold and their error rate are
+plotted along the curve. (a) long-read alignment evaluation. 33,088 $\ge$1000bp
+reads were simulated using pbsim~\citep{Ono:2013aa} with error profile sampled
+from file `m131017\_060208\_42213\_*.1.*' downloaded at
+\href{http://bit.ly/chm1p5c3}{http://bit.ly/chm1p5c3}. The N50 read length is
+11,628. Aligners were run under the default setting for SMRT reads.
+Kart outputted all alignments at mapping quality 60, so is not shown in the
+figure. It mapped nearly all reads with 4.1\% of alignments being wrong, less
+accurate than others.  (b) short-read alignment evaluation. 10 million pairs of
+150bp reads were simulated using mason2~\citep{Holtgrewe:2010aa} with option
+`\mbox{--illumina-prob-mismatch-scale 2.5}'. Short-read aligners were run under
+the default setting except for changing the maximum fragment length to
+800bp.}\label{fig:eval}
+\end{figure}
+
+As a sanity check, we evaluated minimap2 on simulated human reads along with
+BLASR~(v1.MC.rc64; \citealp{Chaisson:2012aa}),
+BWA-MEM~(v0.7.15; \citealp{Li:2013aa}),
+GraphMap~(v0.5.2; \citealp{Sovic:2016aa}),
+Kart~(v2.2.5; \citealp{Lin:2017aa}),
+minialign~(v0.5.3; \href{https://github.com/ocxtal/minialign}{https://github.com/ocxtal/minialign}) and
+NGMLR~(v0.2.5; \citealp{Sedlazeck169557}). We excluded rHAT~\citep{Liu:2016ab}
+and LAMSA~\citep{Liu:2017aa} because they either
+crashed or produced malformatted output. In this evaluation, minimap2 has
+higher power to distinguish unique and repetitive hits, and achieves overall
+higher mapping accuracy (Fig.~\ref{fig:eval}a). Minimap2 and
+NGMLR provide better mapping quality estimate: they rarely give repetitive hits
+high mapping quality.  Apparently, other aligners may
+occasionally miss close suboptimal hits and be overconfident in wrong mappings.
+On run time, minimap2 took 200 CPU seconds, comparable to minialign and Kart, and is over
+30 times faster than the rest.  Minimap2 consumed 6.8GB memory at the peak,
+more than BWA-MEM (5.4GB), similar to NGMLR and less than others.
+
+On real human SMRT reads, the relative performance and fraction of mapped reads reported by
+these aligners are broadly similar to the metrics on simulated data. We are
+unable to provide a good estimate of mapping error rate due to the lack of the
+truth.  On ONT $\sim$100kb human reads~\citep{Jain128835}, BWA-MEM failed.
+Kart, minialign and minimap2 are over 70 times faster than others. We have also
+examined tens of $\ge$100bp INDELs in IGV~\citep{Robinson:2011aa} and can
+confirm the observation by~\citet{Sedlazeck169557} that BWA-MEM often breaks
+them into shorter gaps. The issue is much alleviated with minimap2, thanks
+to the 2-piece affine gap cost.
+
+\subsection{Aligning long spliced reads}
+
+We evaluated minimap2 on SIRV control data~(AC:SRR5286959;
+\citealp{Byrne:2017aa}) where the truth is known. Minimap2 predicted 59\,918
+introns from 11\,018 reads. 93.8\% of splice juctions are precise. We examined
+wrongly predicted junctions and found the majority were caused by clustered
+splicing signals (e.g. two adjacent ${\tt GT}$ sites). When INDEL sequencing
+errors are frequent, it is difficult to find precise splicing sites in this
+case. If we allow up to 10bp distance from true splicing sites, 98.4\% of
+aligned introns are approximately correct. It is worth noting that for SIRV, we
+asked minimap2 to model the ${\tt GT..AG}$ splicing signal only without extra
+bases. This is because SIRV does not honor the evolutionarily prevalent signal
+${\tt GT[A/G]..[C/T]AG}$~\citep{Irimia:2008aa}.
+
+\begin{table}[!tb]
+\processtable{Evaluation of junction accuracy on 2D ONT reads}
+{\footnotesize\label{tab:intron}
+\begin{tabular}{p{3.1cm}rrrr}
+\toprule
+& GMAP & minimap2 & SpAln & STAR\\
+\midrule
+Run time (CPU min)        & 631      & 15.9     & 2\,076   & 33.9 \\
+Peak RAM (GByte)          & 8.9      & 14.5     & 3.2      & 29.2\vspace{1em}\\
+\# aligned reads          & 103\,669 & 104\,199 & 103\,711 & 26\,479 \\
+\# chimeric alignments    & 1\,904   & 1\,488   & 0        & 0 \\
+\# non-spliced alignments & 15\,854  & 14\,798  & 17\,033  & 10\,545\vspace{1em}\\
+\# aligned introns        & 692\,275 & 693\,553 & 692\,945 & 78\,603 \\
+\# novel introns          & 11\,239  & 3\,113   & 8\,550   & 1\,214 \\
+\% exact introns          & 83.8\%   & 94.0\%   & 87.9\%   & 55.2\% \\
+\% approx. introns        & 91.8\%   & 96.9\%   & 92.5\%   & 82.4\% \\
+\botrule
+\end{tabular}
+}{Mouse cDNA reads (AC:SRR5286960; R9.4 chemistry) were mapped to the primary assembly of mouse
+genome GRCm38 with the following tools and command options: minimap2 (`-ax
+splice'); GMAP (`-n 0 --min-intronlength 30 --cross-species'); SpAln (`-Q7 -LS
+-S3'); STARlong (according to
+\href{http://bit.ly/star-pb}{http://bit.ly/star-pb}). The alignments were
+compared to the EnsEMBL gene annotation, release 89. A predicted intron
+is \emph{novel} if it has no overlaps with any annotated introns. An intron
+is \emph{exact} if it is identical to an annotated intron. An intron is
+\emph{approximate} if both its 5'- and 3'-end are within 10bp around the ends
+of an annotated intron. Chimeric alignments are defined in the SAM spec~\citep{Li:2009ys}.}
+\end{table}
+
+We next aligned real mouse reads~\citep{Byrne:2017aa} with GMAP~(v2017-06-20;
+\citealp{Wu:2005vn}), minimap2, SpAln~(v2.3.1; \citealp{Iwata:2012aa}) and
+STAR~(v2.5.3a; \citealp{Dobin:2013kx}). In general, minimap2 is more
+consistent with existing annotations (Table~\ref{tab:intron}): it finds
+more junctions with a higher percentage being exactly or approximately correct.
+Minimap2 is over 40 times faster than GMAP and SpAln. While STAR is close to
+minimap2 in speed, it does not work well with noisy reads.
+
+We have also evaluated spliced aligners on a human Nanopore Direct RNA-seq
+dataset (\href{http://bit.ly/na12878ont}{http://bit.ly/na12878ont}). Minimap2
+aligned 10 million reads in $<$1 wall-clock hour using 16 CPU cores. 94.2\% of
+aligned splice junctions consistent with gene annotations. In comparison,
+GMAP under option `-k 14 -n 0 --min-intronlength 30 --cross-species' is 160
+times slower; 68.7\% of GMAP junctions are found in known gene annotations. The
+percentage increases to 84.1\% if an aligned junction within 10bp from an
+annotated junction is considered to be correct. On a public Iso-Seq dataset
+(human Alzheimer brain from
+\href{http://bit.ly/isoseqpub}{http://bit.ly/isoseqpub}), minimap2 is also
+faster at higher junction accuracy in comparison to other aligners in
+Table~\ref{tab:intron}.
+
+We noted that GMAP and SpAln have not been optimized for noisy reads. We are
+showing the best setting we have experimented, but their developers should be
+able to improve their accuracy further.
+
+%\begin{table}[!tb]
+%\processtable{Evaluation of junction accuracy on SMRT Iso-Seq reads}
+%{\footnotesize
+%\begin{tabular}{lrrrr}
+%\toprule
+%                          &      GMAP &  minimap2 &     SpAln &      STAR \\ % one GMAP thread took 14 days to align a tiny fraction of reads
+%\midrule
+%Run time (CPU min)        &         - &       243 &     2,352 &     1,647 \\
+%\# aligned reads          & 1,113,502 & 1,123,025 & 1,094,092 &   682,452 \\
+%\# chimeric alignments    &    48,927 &    33,091 &         0 &         0 \\
+%\# non-spliced alignments &   334,097 &   339,081 &   291,447 &   272,536 \vspace{1em}\\
+%\# aligned introns        & 8,922,221 & 9,071,755 & 9,208,564 & 3,029,121 \\
+%\# novel introns          &    48,927 &    42,773 &    82,230 &    17,791 \\
+%\% exact introns          &    90.6\% &    94.9\% &    91.7\% &    84.7\% \\
+%\% approx. introns        &    94.0\% &    96.9\% &    93.4\% &    93.8\% \\
+%\botrule
+%\end{tabular}
+%}{}
+%\end{table}
+
+\subsection{Aligning short genomic reads}
+
+We evaluated minimap2 along with Bowtie2~(v2.3.3; \citealt{Langmead:2012fk}), BWA-MEM and
+SNAP (v1.0beta23; \citealt{Zaharia:2011aa}). Minimap2 is 3--4 times as fast as Bowtie2 and
+BWA-MEM, but is 1.3 times slower than SNAP. Minimap2 is more accurate on this
+simulated data set than Bowtie2 and SNAP but less accurate than BWA-MEM
+(Fig.~\ref{fig:eval}b). Closer investigation reveals that BWA-MEM achieves
+a higher accuracy partly because it tries to locally align a read in a small
+region close to its mate. If we disable this feature, BWA-MEM becomes slightly
+less accurate than minimap2. We might implement a similar heuristic
+in minimap2 in future.
+
+To evaluate the accuracy of minimap2 on real data, we aligned human reads
+(AC:ERR1341796) with BWA-MEM and minimap2, and called SNPs and small INDELs
+with GATK HaplotypeCaller v3.5~\citep{Depristo:2011vn}. This run was sequenced
+from experimentally mixed CHM1 and CHM13 cell lines. Both of them are homozygous
+across the whole genome and have been \emph{de novo} assembled with SMRT reads
+to high quality. This allowed us to construct an independent truth variant
+dataset~\citep{Li223297} for
+ERR1341796. In this evaluation, minimap2 has higher SNP false negative rate
+(FNR; 2.6\% of minimap2 vs 2.3\% of BWA-MEM), but fewer false positive SNPs per
+million bases (FPPM; 7.0 vs 8.8), similar INDEL FNR (11.2\% vs 11.3\%) and
+similar INDEL FPPM (6.4 vs 6.5). Minimap2 is broadly comparable to BWA-MEM in the
+context of small variant calling.
+
+\subsection{Aligning long-read assemblies}
+
+Minimap2 can align a SMRT assembly (AC:GCA\_001297185.1) against GRCh38 in 7
+minutes using 8 CPU cores, over 20 times faster than nucmer from
+MUMmer4~\citep{Marcais:2018aa}. With the paftools.js script from the minimap2
+package, we called 2.67 million single-base substitutions out of 2.78Gbp
+genomic regions. The transition-to-transversion ratio (ts/tv) is 2.01. In
+comparison, using MUMmer4's dnadiff pipeline, we called 2.86 million
+substitutions in 2.83Gbp at ts/tv=1.87. Given that ts/tv averaged across the
+human genome is about 2 but ts/tv averaged over random errors is 0.5, the
+minimap2 callset arguably has higher precision at lower sensitivity.
+
+The sample being assembled is a female. Minimap2 still called 201 substitutions
+on the Y chromosome. These substitutions all come from one contig aligned at
+96.8\% sequence identity. The contig could be a segmental duplication
+absent from GRCh38. In constrast, dnadiff called 9070 substitutions on the Y
+chromosome across 73 SMRT contigs. This again implies our minimap2-based
+pipeline has higher precision.
+
+\section{Discussions}
+
+Minimap2 is a versatile mapper and pairwise aligner for nucleotide sequences.
+It works with short reads, assembly contigs and long noisy genomic and RNA-seq
+reads, and can be used as a read mapper, long-read overlapper or a full-genome
+aligner. Minimap2 is also accurate and efficient, often outperforming other
+domain-specific alignment tools in terms of both speed and accuracy.
+
+The capability of minimap2 comes from a fast base-level alignment algorithm and
+an accurate chaining algorithm. When aligning long query sequences, base-level
+alignment is often the performance bottleneck. The Suzuki-Kasahara algorithm
+greatly alleviates the bottleneck and enables DP-based splice alignment
+involving $>$100kb introns, which was impractically slow ten years ago.  The
+minimap2 chaining algorithm is fast and highly accurate by itself.  In fact,
+chaining alone is more accurate than all the other long-read mappers in
+Fig.~\ref{fig:eval}a (data not shown). This accuracy helps to reduce downstream
+base-level alignment of candidate chains, which is still several times slower than
+chaining even with the Suzuki-Kasahara improvement. In addition, taking a
+general form, minimap2 chaining can be adapted to non-typical data types such as
+spliced reads and multiple reads per fragment. This gives us the opportunity to
+extend the same base algorithm to a variety of use cases.
+
+Modern mainstream aligners often use a full-text index, such as suffix array or
+FM-index, to index reference sequences. An advantage of this approach is that
+we can use exact seeds of arbitrary lengths, which helps to increase seed
+uniqueness and reduce unsuccessful extensions. Minimap2 indexes reference
+k-mers with a hash table instead. Such fixed-length seeds are inferior to
+variable-length seeds in theory, but can be computed much more efficiently in
+practice. When a query sequence has multiple seed hits, we can afford to skip
+highly repetitive seeds without affecting the final accuracy. This further
+alleviates the concern with the seeding uniqueness. At the same time, at low
+sequence identity, it is rare to see long seeds anyway. Hash table is the ideal
+data structure for mapping long noisy sequences.
+
+\section*{Acknowledgements}
+We owe a debt of gratitude to H. Suzuki and M. Kasahara for releasing their
+masterpiece and insightful notes before formal publication. We thank M.
+Schatz, P. Rescheneder and F. Sedlazeck for pointing out the limitation of
+BWA-MEM. We are also grateful to minimap2 users who have greatly helped to
+suggest features and to fix various issues.
+
+\paragraph{Funding\textcolon} NHGRI 1R01HG010040-01
+
+\bibliography{minimap2}
+
+\end{document}

tex/mm2-s3.sam.eval

@@ -0,0 +1,62 @@
+Q	60	18579866	27	0.000001453	18579866
+Q	59	27087	4	0.000001666	18606953
+Q	58	21435	1	0.000001718	18628388
+Q	57	45663	3	0.000001874	18674051
+Q	56	36031	2	0.000001978	18710082
+Q	55	18499	2	0.000002082	18728581
+Q	54	14754	2	0.000002187	18743335
+Q	53	25541	2	0.000002291	18768876
+Q	52	26397	5	0.000002554	18795273
+Q	51	15090	3	0.000002711	18810363
+Q	50	13425	11	0.000003294	18823788
+Q	49	15175	2	0.000003397	18838963
+Q	48	19407	4	0.000003606	18858370
+Q	47	11538	16	0.000004452	18869908
+Q	46	12558	17	0.000005349	18882466
+Q	45	40362	28	0.000006817	18922828
+Q	44	10465	13	0.000007500	18933293
+Q	43	10098	20	0.000008552	18943391
+Q	42	10682	19	0.000009549	18954073
+Q	41	9823	11	0.000010125	18963896
+Q	40	9685	16	0.000010963	18973581
+Q	39	10273	18	0.000011905	18983854
+Q	38	9515	18	0.000012847	18993369
+Q	37	9474	27	0.000014261	19002843
+Q	36	10430	25	0.000015568	19013273
+Q	35	9241	34	0.000017348	19022514
+Q	34	9162	31	0.000018968	19031676
+Q	33	10164	49	0.000021532	19041840
+Q	32	9152	55	0.000024408	19050992
+Q	31	9252	35	0.000026233	19060244
+Q	30	9872	55	0.000029103	19070116
+Q	29	8938	65	0.000032496	19079054
+Q	28	8951	73	0.000036306	19088005
+Q	27	9949	95	0.000041261	19097954
+Q	26	9784	97	0.000046316	19107738
+Q	25	10126	97	0.000051366	19117864
+Q	24	11260	123	0.000057765	19129124
+Q	23	10047	114	0.000063691	19139171
+Q	22	9661	123	0.000070083	19148832
+Q	21	10339	168	0.000078813	19159171
+Q	20	17928	193	0.000088804	19177099
+Q	19	9842	193	0.000098817	19186941
+Q	18	14737	247	0.000111605	19201678
+Q	17	10218	238	0.000123934	19211896
+Q	16	10271	242	0.000136457	19222167
+Q	15	12241	333	0.000153683	19234408
+Q	14	9189	336	0.000171070	19243597
+Q	13	9493	515	0.000197734	19253090
+Q	12	11502	743	0.000236185	19264592
+Q	11	8211	507	0.000262390	19272803
+Q	10	9133	606	0.000293695	19281936
+Q	9	10014	931	0.000341801	19291950
+Q	8	8436	698	0.000377816	19300386
+Q	7	8443	705	0.000414163	19308829
+Q	6	10203	944	0.000462808	19319032
+Q	5	6936	756	0.000501760	19325968
+Q	4	6732	843	0.000545190	19332700
+Q	3	8215	1104	0.000602040	19340915
+Q	2	21201	5440	0.000882342	19362116
+Q	1	82328	22186	0.002019600	19444444
+Q	0	553853	371953	0.020562901	19998297
+U	1703

tex/mm2-update.tex

@@ -0,0 +1,240 @@
+\documentclass{bioinfo}
+\copyrightyear{2021}
+\pubyear{2021}
+
+\usepackage{graphicx}
+\usepackage{hyperref}
+\usepackage{url}
+\usepackage{amsmath}
+\usepackage[ruled,vlined]{algorithm2e}
+\newcommand\mycommfont[1]{\footnotesize\rmfamily{\it #1}}
+\SetCommentSty{mycommfont}
+\SetKwComment{Comment}{$\triangleright$\ }{}
+
+\usepackage{natbib}
+\bibliographystyle{apalike}
+
+\DeclareMathOperator*{\argmax}{argmax}
+
+\begin{document}
+\firstpage{1}
+
+\title[Improvements to minimap2]{New strategies to improve minimap2 alignment accuracy}
+\author[Li]{Heng Li$^{1,2}$}
+\address{$^1$Dana-Farber Cancer Institute, 450 Brookline Ave, Boston, MA 02215, USA,
+$^2$Harvard Medical School, 10 Shattuck St, Boston, MA 02215, USA}
+
+\maketitle
+
+\begin{abstract}
+
+\section{Summary:} We present several recent improvements to minimap2, a
+versatile pairwise aligner for nucleotide sequences. Now minimap2 v2.22 can
+more accurately map long reads to highly repetitive regions and align through
+insertions or deletions up to 100kb by default, addressing major weakness in
+minimap2 v2.18 or earlier.
+
+\section{Availability and implementation:}
+\href{https://github.com/lh3/minimap2}{https://github.com/lh3/minimap2}
+
+\section{Contact:} hli@ds.dfci.harvard.edu
+\end{abstract}
+
+\section{Introduction}
+Minimap2~\citep{Li:2018ab} is widely used for maping long sequence
+reads and assembly contigs. \citet{Jain:2020aa} found minimap2 v2.18 or earlier occasionally
+misaligned reads from highly repetitive regions as minimap2 ignored seeds of
+high occurrence. They also noticed minimap2 may misplace reads with structural
+variations (SVs) in such regions~\citep{Jain2020.11.01.363887}. These
+misalignments have become a pressing issue in the advent of
+temolere-to-telomore human assembly~\citep{Miga:2020aa}. Meanwhile, old minimap2
+was unable to efficiently align long insertions/deletions (INDELs) and often
+breaks an alignment around variable-number tandem repeats (VNTRs). This has
+inspired new chaining algorithms~\citep{Li:2020aa,Ren:2021aa} which are not
+integrated into minimap2. Here we will describe recent efforts implemented
+in v2.19 through v2.22 to improve mapping results.
+
+\begin{methods}
+\section{Methods}
+
+\subsection{Rescuing high-occurrence $k$-mers}\label{sec:high-occ}
+Minimap2 keeps all $k$-mer minimizers~\citep{Roberts:2004fv} during indexing. Its original
+implementation only selected low-occurrence minimizers during mapping. The
+cutoff is a few hundred for mapping long reads against a human genome. If a
+read habors only a few or even no low-occurrence minimizers, it will fail
+chaining due to insufficient anchors.
+
+To resolve this issue, we implemented a new heuristic to add additional
+minimizers. Suppose we are looking at two adjacent low-occurence $k$-mers
+located at position $x_1$ and $x_2$, respectively. If $|x_1-x_2|\ge L$,
+minimap2 v2.22 additionally selects $\lfloor|x_1-x_2|/L\rfloor$ minimizers
+of the lowest occurrence among minimizers between $x_1$ and $x_2$. Here
+parameter $L$ controls the frequency of sampling. It defaults to 500.
+This strategy adds necessary anchors at the cost of increasing total alignment
+time by a few percent on real data.
+
+\subsection{Aligning through longer INDELs}
+The original minimap2 may fail to align long INDELs due to its chaining
+heuristics. Briefly, minimap2 applies dynamic programming (DP) to chain
+minimizer anchors. This is a quadratic algorithm, slow for chaining
+contigs. For acceptable performance, the original minimap2 uses a 500bp band by
+default, which means a gap longer than 500bp will stop chaining.
+To align through longer gaps, older minimap2 implemented a long-join heurstic as follows.
+If there is an INDEL longer than 500bp and the two chains around the INDEL
+have no overlaps on either the query or the reference sequence, minimap2 may
+join the two short chains later.
+This heuristic may fail around VNTRs because short chains
+often have overlaps in VNTRs. More subtly, minimap2 may escape the inner DP
+loop early, again for performance, if the chaining result is not improved for
+50 iterations. When there is a copy number change in a long segmental
+duplication, the early escape may break around the event even if users
+specify a large band.
+
+In minigraph~\citep{Li:2020aa}, we developed a new chaining algorithm that
+finds up to 1kb INDELs with DP-based chaining and goes through longer INDELs with a
+subquadratic algorithm~\citep{DBLP:conf/wabi/AbouelhodaO03}. We ported the same
+algorithm to minimap2 for contig mapping. For long-read mapping, the minigraph
+algorithm is slower. Minimap2 v2.22 still uses the DP-based algorithm to
+find short chains and then invokes the minigraph algorithm to rechain anchors in
+these short chains. The rechaining step achieves the same goal as long-join
+but is more reliable because it can resolve overlaps between short chains. The old
+long-join heuristic has since been removed.
+
+\subsection{Properly mapping long reads with SVs}
+The original minimap2 ranks an alignment by its Smith-Waterman score and
+outputs the best scoring alignment. However, when there are SVs on the read,
+the best scoring alignment is sometimes not the correct alignment.
+\citet{Jain2020.11.01.363887} resolved this dilemma by altering the mapping
+algorithm.
+
+In our view, this problem is rooted in inapropriate scoring: affine-gap penalty
+over-penalizes a long INDEL that was often evolutionarily created in one event.
+We should not penalize a SV by a function linear in the SV length. Minimap2 v2.22 instead rescores
+an alignment with the following scoring function. Suppose an alignment consists
+of $M$ matching bases, $N$ substitutions and $G$ gap opens, we empirically
+score the alignment with
+$$
+S=M-\frac{N+G}{2d}-\sum_{i=1}^G\log_2(1+g_i)
+$$
+where $g_i\ge1$ is the length of the $i$-th gap and
+$$
+d=\max\left\{\frac{N+G}{M+N+G},0.02\right\}
+$$
+It approximates per-base sequence divergence except with the smallest value set
+to 2\%. As an analogy to affine-gap scoring, the matching score in our scheme
+is 1, the mismatch and gap open penalties are both $1/2d$ and the gap extension
+penalty is a logarithm function of the gap length~\citep{Gu:1995wt}. Our scoring gives a long SV
+a much milder penalty. In terms of time complexity, scoring an alignment is
+linear in the length of the alignment. The time spent on rescoring is negligible in
+practice.
+
+%If we assume sequences evolve under a duplication-mutation model, we may have a
+%better way to choose the best alignment. If a long read can be mapped to $n$
+%loci, we can take the read as the template and build a
+%pseudo-multi-sequence-alignment (pMSA) of $n+1$ sequences. In this pMSA, we say
+%a site on the read is informative if the $n$ reference subsequences differ at
+%the position.
+
+\end{methods}
+
+\section{Results}
+
+\begin{table}
+\processtable{Evaluation of minimap2 v2.22}
+{\footnotesize\label{tab:1}\begin{tabular}{p{4.2cm}rrrr}
+\toprule
+$[$Benchmark$]$ Metric & v2.22 & v2.18 & Winno & lra \\
+\midrule
+$[$sim-map$]$ \% mapped reads at Q10      & 97.9       & 97.6      & {\bf 99.0}& 97.3 \\
+$[$sim-map$]$ err. rate at Q10 (phredQ)   & {\bf 52}   & {\bf 52}  & 38        & 24 \\
+$[$winno-cmp$]$ rate of diff. (phredQ)    & {\bf 41}   & 37        & truth     & 18 \\
+$[$winno-cmp$]$ CPU time (hour)           & {\bf 5.0}  & 5.3       & 71.8      & 13.1 \\
+$[$winno-cmp$]$ peak RAM (Gb)             & 17.1       & 14.4      & {\bf 9.6} & 12.4 \\
+$[$sim-sv$]$  \% false negative rate      & {\bf 0.5}  & 2.0       & {\bf 0.5} & 1.4  \\
+$[$sim-sv$]$  \% false discovery rate     & {\bf 0.0}  & 0.1       & {\bf 0.0} & 0.1  \\
+$[$real-sv-1k$]$ \% false negative rate   & {\bf 7.3}  & 20.0      & 13.0      & N/A \\
+$[$real-sv-1k$]$ \% false discovery rate  & 2.7        & {\bf 2.4} & 2.7       & N/A \\
+\botrule
+\end{tabular}}
+{In $[$sim-map$]$, 152,713 reads were simulated from the CHM13 telomere-to-telomere assembly v1.1
+(AC: GCA\_009914755.3) with pbsim2~\citep{Ono:2021aa}: ``pbsim2 -{}-hmm\_model R94.model -{}-length-min
+5000 -{}-length-mean 20000 -{}-accuracy-mean 0.95''. Alignments of mapping quality
+10 or higher were evaluated by ``paftools.js mapeval''. The mapping error rate
+is measured in the phred scale: if the error rate is $e$, $-10\log_{10}e$ is
+reported in the table. In $[$winno-cmp$]$, 1.39 million CHM13 HiFi reads from
+SRR11292121 were mapped against the same CHM13 assembly. 99.3\% of them were mapped by Winnowmap2
+at mapping quality 10 or higher and were taken as ground truth to evaluate
+minimap2 and lra with ``paftools.js pafcmp''. $[$sim-sv$]$ simulated 1,000
+50bp to 1000bp INDELs from chr8 in CHM13 using SURVIVOR~\citep{Jeffares:2017aa} and simulated Nanopore
+reads at 30-fold coverage with the same pbsim2 command line. SVs were called with
+``sniffles -q 10''~\citep{Sedlazeck:2018ab} and compared to the simulated truth with ``SURVIVOR eval
+call.vcf truth.bed 50''. In $[$real-sv-1k$]$, small and long variants were
+called by dipcall-0.3~\citep{Li:2018aa} for HG002 assemblies (AC: GCA\_018852605.1 and
+GCA\_018852615.1) and compared to the GIAB truth~\citep{Zook:2020aa} using ``truvari -r 2000 -s
+1000 -S 400 -{}-multimatch -{}-passonly'' which sets the minimum INDEL size to 1kb in evaluation. }
+\end{table}
+
+We evaluated minimap2 v2.22 along with v2.18, Winnowmap2 v2.03 and lra v1.3.2
+(Table~\ref{tab:1}), using the default setting of each mapper according to the input data types.
+Both versions of minimap2 achieved high mapping accuracy on
+simulated Nanopore reads (sim-map). Winnowmap2 aligned more reads at mapping
+quality 10 or higher (mapQ10). However, it may occasionally assign a high mapping
+quality to a read with multiple identical best alignments. This reduced its
+mapping accuracy.
+
+In lack of groud truth for real data, we took Winnowmap2 mapping as ground
+truth to evaluate other mappers (winno-cmp in Table~\ref{tab:1}). Out of 1,378,092 reads with mapQ10
+alignments by Winnowmap2, minimap2 v2.22 could map all of them. 118 reads, less
+than 0.01\% of all reads, were mapped differently by v2.22. 51 of them have
+multiple identical best alignments. We believe these are more likely to be
+Winnowmap2 errors. Most of the remaining 67 (=118-51) reads have multiple
+highly similar but not identical alignments.
+Minimap2 v2.18 is less consistent with 275 differences including 30 unmapped
+reads mappable by both Winnowmap2 and v2.22.
+
+For the minimizer rescuing parameter $L$ in Section~\ref{sec:high-occ},
+we set its default to 500 such that v2.22 has comparable performance to v2.18 given simulated PacBio and Nanopore human reads.
+To see the effect of this parameter on real data, we tried several different $L$ values.
+v2.22 gave 99 mapping differences at $L=200$,
+118 at $L=500$ (default), 167 at $L=750$ and 224 differences at $L=1000$ in comparison to Winnowmap2.
+$L=200$ is 28\% slower than the default while $L=1000$ is 9\% faster.
+Changing the default minimizer window size (option ``-w'')
+and the initial minimizer occurrence cutoff (option ``-f'')
+also affects performance and accuracy to a similar magnitude.
+
+The two benchmarks above only evaluate read mappings when there are no variations between the reads and the reference.
+To measure the mapping accuracy in the presence of SVs (sim-sv), we reproduced
+the results by~\citep{Jain2020.11.01.363887}. Minimap2 v2.22 is as good as
+Winnowmap2 now. Note that we were setting the Sniffles mapping quality
+threshold to 10 in consistent with the benchmarks above. If we used the
+default threshold 20, v2.22 would miss additional five SVs (accounting for
+0.5\% of simulated SVs). For four out of these five missing SVs, minimap2 v2.22
+mapped more variant reads than Winnowmap2. Sniffles did not call these SVs
+because minimap2 tended to give them conservative mapping quality. It is worth
+noting that the simulation here only considers a simple scenario in evolution.
+Non-allelic gene conversions, which happen often in segmental
+duplications~\citep{Harpak:2017aa}, would obscure the optimal mapping
+strategies. How much such simple SV simulation informs real-world SV calling
+remains a question.
+
+To see if minimap2 v2.22 could improve long INDEL alignment, we ran dipcall on
+contig-to-reference alignments and focused on INDELs longer than 1kb
+(real-sv-1k). v2.22 is more sensitive at comparable specificity, confirming its
+advantage in more contiguous alignment. We could not get dipcall to work well with lra,
+so did not report the numbers.
+
+Minimap2 spends most computing time on base alignment. As recent improvements
+in v2.22 incur little additional computing and do not change the base alignment
+algorithm, the new version has similar performance to older versions. It is
+consistently faster than Winnowmap2 by several times. Sometimes simple
+heuristics can be as effective as more sophisticated yet slower solutions.
+
+\section*{Acknowledgements}
+We thank Arang Rhie and Chirag Jain for providing motivating examples for which
+older minimap2 underperforms.
+
+\paragraph{Funding\textcolon} This work is funded by NHGRI grant R01HG010040.
+
+\bibliography{minimap2}
+
+\end{document}

tex/mm2.approx.eval

@@ -0,0 +1,12 @@
+Q	60	32084	0	0.000000000	32084
+Q	24	318	2	0.000061725	32402
+Q	11	98	2	0.000123077	32500
+Q	8	37	2	0.000184405	32537
+Q	7	37	3	0.000276294	32574
+Q	6	40	3	0.000367940	32614
+Q	5	34	2	0.000428816	32648
+Q	4	37	5	0.000581306	32685
+Q	3	28	6	0.000764222	32713
+Q	2	38	6	0.000946536	32751
+Q	1	50	21	0.001585318	32801
+Q	0	286	150	0.006105117	33087

tex/mm2.eval

@@ -0,0 +1,13 @@
+Q	60	32477	0	0.000000000	32477
+Q	22	16	1	0.000030776	32493
+Q	21	44	1	0.000061468	32537
+Q	19	73	1	0.000091996	32610
+Q	14	66	1	0.000122414	32676
+Q	10	26	3	0.000214054	32702
+Q	8	14	1	0.000244529	32716
+Q	7	13	2	0.000305539	32729
+Q	6	47	1	0.000335611	32776
+Q	3	10	1	0.000366010	32786
+Q	2	20	2	0.000426751	32806
+Q	1	248	94	0.003267381	33054
+Q	0	31	17	0.003778147	33085

tex/natbib.sty

@@ -0,0 +1,803 @@
+%%
+%% This is file `natbib.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% natbib.dtx  (with options: `package,all')
+%% =============================================
+%% IMPORTANT NOTICE:
+%% 
+%% This program can be redistributed and/or modified under the terms
+%% of the LaTeX Project Public License Distributed from CTAN
+%% archives in directory macros/latex/base/lppl.txt; either
+%% version 1 of the License, or any later version.
+%% 
+%% This is a generated file.
+%% It may not be distributed without the original source file natbib.dtx.
+%% 
+%% Full documentation can be obtained by LaTeXing that original file.
+%% Only a few abbreviated comments remain here to describe the usage.
+%% =============================================
+%% Copyright 1993-2000 Patrick W Daly
+%% Max-Planck-Institut f\"ur Aeronomie
+%% Max-Planck-Str. 2
+%% D-37191 Katlenburg-Lindau
+%% Germany
+%% E-mail: daly@linmpi.mpg.de
+\NeedsTeXFormat{LaTeX2e}[1995/06/01]
+\ProvidesPackage{natbib}
+        [2000/07/24 7.0a (PWD)]
+ % This package reimplements the LaTeX \cite command to be used for various
+ % citation styles, both author-year and numerical. It accepts BibTeX
+ % output intended for many other packages, and therefore acts as a
+ % general, all-purpose citation-style interface.
+ %
+ % With standard numerical .bst files, only numerical citations are
+ % possible. With an author-year .bst file, both numerical and
+ % author-year citations are possible.
+ %
+ % If author-year citations are selected, \bibitem must have one of the
+ %   following forms:
+ %   \bibitem[Jones et al.(1990)]{key}...
+ %   \bibitem[Jones et al.(1990)Jones, Baker, and Williams]{key}...
+ %   \bibitem[Jones et al., 1990]{key}...
+ %   \bibitem[\protect\citeauthoryear{Jones, Baker, and Williams}{Jones
+ %       et al.}{1990}]{key}...
+ %   \bibitem[\protect\citeauthoryear{Jones et al.}{1990}]{key}...
+ %   \bibitem[\protect\astroncite{Jones et al.}{1990}]{key}...
+ %   \bibitem[\protect\citename{Jones et al., }1990]{key}...
+ %   \harvarditem[Jones et al.]{Jones, Baker, and Williams}{1990}{key}...
+ %
+ % This is either to be made up manually, or to be generated by an
+ % appropriate .bst file with BibTeX.
+ %                            Author-year mode     ||   Numerical mode
+ % Then, \citet{key}  ==>>  Jones et al. (1990)    ||   Jones et al. [21]
+ %       \citep{key}  ==>> (Jones et al., 1990)    ||   [21]
+ % Multiple citations as normal:
+ % \citep{key1,key2}  ==>> (Jones et al., 1990; Smith, 1989) || [21,24]
+ %                           or  (Jones et al., 1990, 1991)  || [21,24]
+ %                           or  (Jones et al., 1990a,b)     || [21,24]
+ % \cite{key} is the equivalent of \citet{key} in author-year mode
+ %                         and  of \citep{key} in numerical mode
+ % Full author lists may be forced with \citet* or \citep*, e.g.
+ %       \citep*{key}      ==>> (Jones, Baker, and Williams, 1990)
+ % Optional notes as:
+ %   \citep[chap. 2]{key}    ==>> (Jones et al., 1990, chap. 2)
+ %   \citep[e.g.,][]{key}    ==>> (e.g., Jones et al., 1990)
+ %   \citep[see][pg. 34]{key}==>> (see Jones et al., 1990, pg. 34)
+ %  (Note: in standard LaTeX, only one note is allowed, after the ref.
+ %   Here, one note is like the standard, two make pre- and post-notes.)
+ %   \citealt{key}          ==>> Jones et al. 1990
+ %   \citealt*{key}         ==>> Jones, Baker, and Williams 1990
+ %   \citealp{key}          ==>> Jones et al., 1990
+ %   \citealp*{key}         ==>> Jones, Baker, and Williams, 1990
+ % Additional citation possibilities (both author-year and numerical modes)
+ %   \citeauthor{key}       ==>> Jones et al.
+ %   \citeauthor*{key}      ==>> Jones, Baker, and Williams
+ %   \citeyear{key}         ==>> 1990
+ %   \citeyearpar{key}      ==>> (1990)
+ %   \citetext{priv. comm.} ==>> (priv. comm.)
+ % Note: full author lists depends on whether the bib style supports them;
+ %       if not, the abbreviated list is printed even when full requested.
+ %
+ % For names like della Robbia at the start of a sentence, use
+ %   \Citet{dRob98}         ==>> Della Robbia (1998)
+ %   \Citep{dRob98}         ==>> (Della Robbia, 1998)
+ %   \Citeauthor{dRob98}    ==>> Della Robbia
+ %
+ %
+ % Citation aliasing is achieved with
+ %   \defcitealias{key}{text}
+ %   \citetalias{key}  ==>> text
+ %   \citepalias{key}  ==>> (text)
+ %
+ % Defining the citation style of a given bib style:
+ % Use \bibpunct (in the preamble only) with 6 mandatory arguments:
+ %    1. opening bracket for citation
+ %    2. closing bracket
+ %    3. citation separator (for multiple citations in one \cite)
+ %    4. the letter n for numerical styles, s for superscripts
+ %        else anything for author-year
+ %    5. punctuation between authors and date
+ %    6. punctuation between years (or numbers) when common authors missing
+ % One optional argument is the character coming before post-notes. It
+ %   appears in square braces before all other arguments. May be left off.
+ % Example (and default) \bibpunct[, ]{(}{)}{;}{a}{,}{,}
+ %
+ % To make this automatic for a given bib style, named newbib, say, make
+ % a local configuration file, natbib.cfg, with the definition
+ %   \newcommand{\bibstyle@newbib}{\bibpunct...}
+ % Then the \bibliographystyle{newbib} will cause \bibstyle@newbib to
+ % be called on THE NEXT LATEX RUN (via the aux file).
+ %
+ % Such preprogrammed definitions may be invoked in the text (preamble only)
+ %  by calling \citestyle{newbib}. This is only useful if the style specified
+ %  differs from that in \bibliographystyle.
+ %
+ % With \citeindextrue and \citeindexfalse, one can control whether the
+ % \cite commands make an automatic entry of the citation in the .idx
+ % indexing file. For this, \makeindex must also be given in the preamble.
+ %
+ % LaTeX2e Options: (for selecting punctuation)
+ %   round  -  round parentheses are used (default)
+ %   square -  square brackets are used   [option]
+ %   curly  -  curly braces are used      {option}
+ %   angle  -  angle brackets are used    <option>
+ %   colon  -  multiple citations separated by colon (default)
+ %   comma  -  separated by comma
+ %   authoryear - selects author-year citations (default)
+ %   numbers-  selects numerical citations
+ %   super  -  numerical citations as superscripts
+ %   sort   -  sorts multiple citations according to order in ref. list
+ %   sort&compress   -  like sort, but also compresses numerical citations
+ %   longnamesfirst  -  makes first citation full author list
+ %   sectionbib - puts bibliography in a \section* instead of \chapter*
+ % Punctuation so selected dominates over any predefined ones.
+ % LaTeX2e options are called as, e.g.
+ %        \usepackage[square,comma]{natbib}
+ % LaTeX the source file natbib.dtx to obtain more details
+ % or the file natnotes.tex for a brief reference sheet.
+ %-----------------------------------------------------------
+\@ifclassloaded{aguplus}{\PackageError{natbib}
+  {The aguplus class already includes natbib coding,\MessageBreak
+   so you should not add it explicitly}
+  {Type <Return> for now, but then later remove\MessageBreak
+   the command \protect\usepackage{natbib} from the document}
+  \endinput}{}
+\@ifclassloaded{nlinproc}{\PackageError{natbib}
+  {The nlinproc class already includes natbib coding,\MessageBreak
+   so you should not add it explicitly}
+  {Type <Return> for now, but then later remove\MessageBreak
+   the command \protect\usepackage{natbib} from the document}
+  \endinput}{}
+\@ifclassloaded{egs}{\PackageError{natbib}
+  {The egs class already includes natbib coding,\MessageBreak
+   so you should not add it explicitly}
+  {Type <Return> for now, but then later remove\MessageBreak
+   the command \protect\usepackage{natbib} from the document}
+  \endinput}{}
+ % Define citation punctuation for some author-year styles
+ % One may add and delete at this point
+ % Or put additions into local configuration file natbib.cfg
+\newcommand\bibstyle@chicago{\bibpunct{(}{)}{;}{a}{,}{,}}
+\newcommand\bibstyle@named{\bibpunct{[}{]}{;}{a}{,}{,}}
+\newcommand\bibstyle@agu{\bibpunct{[}{]}{;}{a}{,}{,~}}%Amer. Geophys. Union
+\newcommand\bibstyle@egs{\bibpunct{(}{)}{;}{a}{,}{,}}%Eur. Geophys. Soc.
+\newcommand\bibstyle@agsm{\bibpunct{(}{)}{,}{a}{}{,}\gdef\harvardand{\&}}
+\newcommand\bibstyle@kluwer{\bibpunct{(}{)}{,}{a}{}{,}\gdef\harvardand{\&}}
+\newcommand\bibstyle@dcu{\bibpunct{(}{)}{;}{a}{;}{,}\gdef\harvardand{and}}
+\newcommand\bibstyle@aa{\bibpunct{(}{)}{;}{a}{}{,}} %Astronomy & Astrophysics
+\newcommand\bibstyle@pass{\bibpunct{(}{)}{;}{a}{,}{,}}%Planet. & Space Sci
+\newcommand\bibstyle@anngeo{\bibpunct{(}{)}{;}{a}{,}{,}}%Annales Geophysicae
+\newcommand\bibstyle@nlinproc{\bibpunct{(}{)}{;}{a}{,}{,}}%Nonlin.Proc.Geophys.
+ % Define citation punctuation for some numerical styles
+\newcommand\bibstyle@cospar{\bibpunct{/}{/}{,}{n}{}{}%
+     \gdef\NAT@biblabelnum##1{##1.}}
+\newcommand\bibstyle@esa{\bibpunct{(Ref.~}{)}{,}{n}{}{}%
+     \gdef\NAT@biblabelnum##1{##1.\hspace{1em}}}
+\newcommand\bibstyle@nature{\bibpunct{}{}{,}{s}{}{\textsuperscript{,}}%
+     \gdef\NAT@biblabelnum##1{##1.}}
+ % The standard LaTeX styles
+\newcommand\bibstyle@plain{\bibpunct{[}{]}{,}{n}{}{,}}
+\let\bibstyle@alpha=\bibstyle@plain
+\let\bibstyle@abbrv=\bibstyle@plain
+\let\bibstyle@unsrt=\bibstyle@plain
+ % The author-year modifications of the standard styles
+\newcommand\bibstyle@plainnat{\bibpunct{[}{]}{,}{a}{,}{,}}
+\let\bibstyle@abbrvnat=\bibstyle@plainnat
+\let\bibstyle@unsrtnat=\bibstyle@plainnat
+\newif\ifNAT@numbers \NAT@numbersfalse
+\newif\ifNAT@super \NAT@superfalse
+\DeclareOption{numbers}{\NAT@numberstrue
+   \ExecuteOptions{square,comma,nobibstyle}}
+\DeclareOption{super}{\NAT@supertrue\NAT@numberstrue
+   \renewcommand\NAT@open{}\renewcommand\NAT@close{}
+   \ExecuteOptions{nobibstyle}}
+\DeclareOption{authoryear}{\NAT@numbersfalse
+   \ExecuteOptions{round,colon,bibstyle}}
+\DeclareOption{round}{%
+      \renewcommand\NAT@open{(} \renewcommand\NAT@close{)}
+   \ExecuteOptions{nobibstyle}}
+\DeclareOption{square}{%
+      \renewcommand\NAT@open{[} \renewcommand\NAT@close{]}
+   \ExecuteOptions{nobibstyle}}
+\DeclareOption{angle}{%
+      \renewcommand\NAT@open{$<$} \renewcommand\NAT@close{$>$}
+   \ExecuteOptions{nobibstyle}}
+\DeclareOption{curly}{%
+      \renewcommand\NAT@open{\{} \renewcommand\NAT@close{\}}
+   \ExecuteOptions{nobibstyle}}
+\DeclareOption{comma}{\renewcommand\NAT@sep{,}
+   \ExecuteOptions{nobibstyle}}
+\DeclareOption{colon}{\renewcommand\NAT@sep{;}
+   \ExecuteOptions{nobibstyle}}
+\DeclareOption{nobibstyle}{\let\bibstyle=\@gobble}
+\DeclareOption{bibstyle}{\let\bibstyle=\@citestyle}
+\newif\ifNAT@openbib \NAT@openbibfalse
+\DeclareOption{openbib}{\NAT@openbibtrue}
+\DeclareOption{sectionbib}{\def\NAT@sectionbib{on}}
+\def\NAT@sort{0}
+\DeclareOption{sort}{\def\NAT@sort{1}}
+\DeclareOption{sort&compress}{\def\NAT@sort{2}}
+\@ifpackageloaded{cite}{\PackageWarningNoLine{natbib}
+  {The `cite' package should not be used\MessageBreak
+   with natbib. Use option `sort' instead}\ExecuteOptions{sort}}{}
+\newif\ifNAT@longnames\NAT@longnamesfalse
+\DeclareOption{longnamesfirst}{\NAT@longnamestrue}
+\DeclareOption{nonamebreak}{\def\NAT@nmfmt#1{\mbox{\NAT@up#1}}}
+\def\NAT@nmfmt#1{{\NAT@up#1}}
+\renewcommand\bibstyle[1]{\@ifundefined{bibstyle@#1}{\relax}
+     {\csname bibstyle@#1\endcsname}}
+\AtBeginDocument{\global\let\bibstyle=\@gobble}
+\let\@citestyle\bibstyle
+\newcommand\citestyle[1]{\@citestyle{#1}\let\bibstyle\@gobble}
+\@onlypreamble{\citestyle}\@onlypreamble{\@citestyle}
+\newcommand\bibpunct[7][, ]%
+  {\gdef\NAT@open{#2}\gdef\NAT@close{#3}\gdef
+   \NAT@sep{#4}\global\NAT@numbersfalse\ifx #5n\global\NAT@numberstrue
+   \else
+   \ifx #5s\global\NAT@numberstrue\global\NAT@supertrue
+   \fi\fi
+   \gdef\NAT@aysep{#6}\gdef\NAT@yrsep{#7}%
+   \gdef\NAT@cmt{#1}%
+   \global\let\bibstyle\@gobble
+  }
+\@onlypreamble{\bibpunct}
+\newcommand\NAT@open{(} \newcommand\NAT@close{)}
+\newcommand\NAT@sep{;}
+\ProcessOptions
+\newcommand\NAT@aysep{,} \newcommand\NAT@yrsep{,}
+\newcommand\NAT@cmt{, }
+\newcommand\NAT@cite%
+    [3]{\ifNAT@swa\NAT@@open\if*#2*\else#2\ \fi
+        #1\if*#3*\else\NAT@cmt#3\fi\NAT@@close\else#1\fi\endgroup}
+\newcommand\NAT@citenum%
+    [3]{\ifNAT@swa\NAT@@open\if*#2*\else#2\ \fi
+        #1\if*#3*\else\NAT@cmt#3\fi\NAT@@close\else#1\fi\endgroup}
+\newcommand\NAT@citesuper[3]{\ifNAT@swa
+\unskip\hspace{1\p@}\textsuperscript{#1}%
+   \if*#3*\else\ (#3)\fi\else #1\fi\endgroup}
+\providecommand
+  \textsuperscript[1]{\mbox{$^{\mbox{\scriptsize#1}}$}}
+\providecommand\@firstofone[1]{#1}
+\newcommand\NAT@citexnum{}
+\def\NAT@citexnum[#1][#2]#3{%
+ \NAT@sort@cites{#3}%
+ \let\@citea\@empty
+  \@cite{\def\NAT@num{-1}\let\NAT@last@yr\relax\let\NAT@nm\@empty
+    \@for\@citeb:=\NAT@cite@list\do
+    {\edef\@citeb{\expandafter\@firstofone\@citeb}%
+     \if@filesw\immediate\write\@auxout{\string\citation{\@citeb}}\fi
+     \@ifundefined{b@\@citeb\@extra@b@citeb}{%
+       {\reset@font\bfseries?}
+        \NAT@citeundefined\PackageWarning{natbib}%
+       {Citation `\@citeb' on page \thepage \space undefined}}%
+     {\let\NAT@last@num\NAT@num\let\NAT@last@nm\NAT@nm
+      \NAT@parse{\@citeb}%
+      \ifNAT@longnames\@ifundefined{bv@\@citeb\@extra@b@citeb}{%
+        \let\NAT@name=\NAT@all@names
+        \global\@namedef{bv@\@citeb\@extra@b@citeb}{}}{}%
+      \fi
+      \ifNAT@full\let\NAT@nm\NAT@all@names\else
+        \let\NAT@nm\NAT@name\fi
+      \ifNAT@swa
+       \ifnum\NAT@ctype>1\relax\@citea
+        \hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+            \ifnum\NAT@ctype=2\relax\NAT@test{\NAT@ctype}%
+            \else\NAT@alias
+            \fi\hyper@natlinkend\else
+       \ifnum\NAT@sort>1
+         \begingroup\catcode`\_=8
+            \ifcat _\ifnum\z@<0\NAT@num _\else A\fi
+              \global\let\NAT@nm=\NAT@num \else \gdef\NAT@nm{-2}\fi
+            \ifcat _\ifnum\z@<0\NAT@last@num _\else A\fi
+              \global\@tempcnta=\NAT@last@num \global\advance\@tempcnta by\@ne
+              \else \global\@tempcnta\m@ne\fi
+         \endgroup
+         \ifnum\NAT@nm=\@tempcnta
+           \ifx\NAT@last@yr\relax
+             \edef\NAT@last@yr{\@citea \mbox{\noexpand\citenumfont{\NAT@num}}}%
+           \else
+             \edef\NAT@last@yr{--\penalty\@m\mbox{\noexpand\citenumfont{\NAT@num}}}%
+           \fi
+         \else
+           \NAT@last@yr \@citea \mbox{\citenumfont{\NAT@num}}%
+           \let\NAT@last@yr\relax
+         \fi
+       \else
+         \@citea \mbox{\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+           {\citenumfont{\NAT@num}}\hyper@natlinkend}%
+       \fi
+       \fi
+       \def\@citea{\NAT@sep\penalty\@m\NAT@space}%
+      \else
+        \ifcase\NAT@ctype\relax
+          \ifx\NAT@last@nm\NAT@nm \NAT@yrsep\penalty\@m\NAT@space\else
+          \@citea \NAT@test{1}\ \NAT@@open
+          \if*#1*\else#1\ \fi\fi \NAT@mbox{%
+          \hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+          {\citenumfont{\NAT@num}}\hyper@natlinkend}%
+          \def\@citea{\NAT@@close\NAT@sep\penalty\@m\ }%
+        \or\@citea
+          \hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+           \NAT@test{\NAT@ctype}\hyper@natlinkend
+          \def\@citea{\NAT@sep\penalty\@m\ }%
+        \or\@citea
+          \hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+           \NAT@test{\NAT@ctype}\hyper@natlinkend
+          \def\@citea{\NAT@sep\penalty\@m\ }%
+        \or\@citea
+          \hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+           \NAT@alias\hyper@natlinkend
+          \def\@citea{\NAT@sep\penalty\@m\ }%
+        \fi
+      \fi
+      }}%
+      \ifnum\NAT@sort>1\relax\NAT@last@yr\fi
+      \ifNAT@swa\else\ifnum\NAT@ctype=0\if*#2*\else
+      \NAT@cmt#2\fi \NAT@@close\fi\fi}{#1}{#2}}
+\newcommand\NAT@test[1]{\ifnum#1=1 \ifx\NAT@nm\NAT@noname
+  {\reset@font\bfseries(author?)}\PackageWarning{natbib}
+  {Author undefined for citation`\@citeb'
+   \MessageBreak
+   on page \thepage}\else \NAT@nm \fi
+  \else \if\relax\NAT@date\relax
+  {\reset@font\bfseries(year?)}\PackageWarning{natbib}
+  {Year undefined for citation`\@citeb'
+   \MessageBreak
+   on page \thepage}\else \NAT@date \fi \fi}
+\let\citenumfont=\relax
+\newcommand\NAT@citex{}
+\def\NAT@citex%
+  [#1][#2]#3{%
+  \NAT@sort@cites{#3}%
+  \let\@citea\@empty
+  \@cite{\let\NAT@nm\@empty\let\NAT@year\@empty
+    \@for\@citeb:=\NAT@cite@list\do
+    {\edef\@citeb{\expandafter\@firstofone\@citeb}%
+     \if@filesw\immediate\write\@auxout{\string\citation{\@citeb}}\fi
+     \@ifundefined{b@\@citeb\@extra@b@citeb}{\@citea%
+       {\reset@font\bfseries ?}\NAT@citeundefined
+                 \PackageWarning{natbib}%
+       {Citation `\@citeb' on page \thepage \space undefined}\def\NAT@date{}}%
+     {\let\NAT@last@nm=\NAT@nm\let\NAT@last@yr=\NAT@year
+     \NAT@parse{\@citeb}%
+      \ifNAT@longnames\@ifundefined{bv@\@citeb\@extra@b@citeb}{%
+        \let\NAT@name=\NAT@all@names
+        \global\@namedef{bv@\@citeb\@extra@b@citeb}{}}{}%
+      \fi
+     \ifNAT@full\let\NAT@nm\NAT@all@names\else
+       \let\NAT@nm\NAT@name\fi
+     \ifNAT@swa\ifcase\NAT@ctype
+       \if\relax\NAT@date\relax
+         \@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+         \NAT@nmfmt{\NAT@nm}\NAT@date\hyper@natlinkend
+       \else
+         \ifx\NAT@last@nm\NAT@nm\NAT@yrsep
+            \ifx\NAT@last@yr\NAT@year
+              \hyper@natlinkstart{\@citeb\@extra@b@citeb}\NAT@exlab
+              \hyper@natlinkend
+            \else\unskip\
+              \hyper@natlinkstart{\@citeb\@extra@b@citeb}\NAT@date
+              \hyper@natlinkend
+            \fi
+         \else\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+           \NAT@nmfmt{\NAT@nm}%
+           \hyper@natlinkbreak{\NAT@aysep\ }{\@citeb\@extra@b@citeb}%
+           \NAT@date\hyper@natlinkend
+         \fi
+       \fi
+     \or\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+         \NAT@nmfmt{\NAT@nm}\hyper@natlinkend
+     \or\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+         \NAT@date\hyper@natlinkend
+     \or\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+         \NAT@alias\hyper@natlinkend
+     \fi \def\@citea{\NAT@sep\ }%
+     \else\ifcase\NAT@ctype
+        \if\relax\NAT@date\relax
+          \@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+          \NAT@nmfmt{\NAT@nm}\hyper@natlinkend
+        \else
+         \ifx\NAT@last@nm\NAT@nm\NAT@yrsep
+            \ifx\NAT@last@yr\NAT@year
+              \hyper@natlinkstart{\@citeb\@extra@b@citeb}\NAT@exlab
+              \hyper@natlinkend
+            \else\unskip\
+              \hyper@natlinkstart{\@citeb\@extra@b@citeb}\NAT@date
+              \hyper@natlinkend
+            \fi
+         \else\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+           \NAT@nmfmt{\NAT@nm}%
+           \hyper@natlinkbreak{\ \NAT@@open\if*#1*\else#1\ \fi}%
+              {\@citeb\@extra@b@citeb}%
+           \NAT@date\hyper@natlinkend\fi
+        \fi
+       \or\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+         \NAT@nmfmt{\NAT@nm}\hyper@natlinkend
+       \or\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+         \NAT@date\hyper@natlinkend
+       \or\@citea\hyper@natlinkstart{\@citeb\@extra@b@citeb}%
+         \NAT@alias\hyper@natlinkend
+       \fi \if\relax\NAT@date\relax\def\@citea{\NAT@sep\ }%
+           \else\def\@citea{\NAT@@close\NAT@sep\ }\fi
+     \fi
+     }}\ifNAT@swa\else\if*#2*\else\NAT@cmt#2\fi
+     \if\relax\NAT@date\relax\else\NAT@@close\fi\fi}{#1}{#2}}
+\newif\ifNAT@par \NAT@partrue
+\newcommand\NAT@@open{\ifNAT@par\NAT@open\fi}
+\newcommand\NAT@@close{\ifNAT@par\NAT@close\fi}
+\newcommand\NAT@alias{\@ifundefined{al@\@citeb\@extra@b@citeb}{%
+  {\reset@font\bfseries(alias?)}\PackageWarning{natbib}
+  {Alias undefined for citation `\@citeb'
+  \MessageBreak on page \thepage}}{\@nameuse{al@\@citeb\@extra@b@citeb}}}
+\let\NAT@up\relax
+\newcommand\NAT@Up[1]{{\let\protect\@unexpandable@protect\let~\relax
+  \expandafter\NAT@deftemp#1}\expandafter\NAT@UP\NAT@temp}
+\newcommand\NAT@deftemp[1]{\xdef\NAT@temp{#1}}
+\newcommand\NAT@UP[1]{\let\@tempa\NAT@UP\ifcat a#1\MakeUppercase{#1}%
+  \let\@tempa\relax\else#1\fi\@tempa}
+\newcommand\shortcites[1]{%
+  \@bsphack\@for\@citeb:=#1\do
+  {\edef\@citeb{\expandafter\@firstofone\@citeb}%
+   \global\@namedef{bv@\@citeb\@extra@b@citeb}{}}\@esphack}
+\newcommand\NAT@biblabel[1]{\hfill}
+\newcommand\NAT@biblabelnum[1]{\bibnumfmt{#1}}
+\newcommand\bibnumfmt[1]{[#1]}
+\def\@tempa#1{[#1]}
+\ifx\@tempa\@biblabel\let\@biblabel\@empty\fi
+\newcommand\NAT@bibsetnum[1]{\settowidth\labelwidth{\@biblabel{#1}}%
+   \setlength{\leftmargin}{\labelwidth}\addtolength{\leftmargin}{\labelsep}%
+   \setlength{\itemsep}{\bibsep}\setlength{\parsep}{\z@}%
+   \ifNAT@openbib
+     \addtolength{\leftmargin}{4mm}%
+     \setlength{\itemindent}{-4mm}%
+     \setlength{\listparindent}{\itemindent}%
+     \setlength{\parsep}{0pt}%
+   \fi
+}
+\newlength{\bibhang}
+\setlength{\bibhang}{1em}
+\newlength{\bibsep}
+{\@listi \global\bibsep\itemsep \global\advance\bibsep by\parsep}
+
+\newcommand\NAT@bibsetup%
+   [1]{\setlength{\leftmargin}{\bibhang}\setlength{\itemindent}{-\leftmargin}%
+       \setlength{\itemsep}{\bibsep}\setlength{\parsep}{\z@}}
+\newcommand\NAT@set@cites{\ifNAT@numbers
+  \ifNAT@super \let\@cite\NAT@citesuper
+     \def\NAT@mbox##1{\unskip\nobreak\hspace{1\p@}\textsuperscript{##1}}%
+     \let\citeyearpar=\citeyear
+     \let\NAT@space\relax\else
+     \let\NAT@mbox=\mbox
+     \let\@cite\NAT@citenum \def\NAT@space{ }\fi
+  \let\@citex\NAT@citexnum
+  \ifx\@biblabel\@empty\let\@biblabel\NAT@biblabelnum\fi
+  \let\@bibsetup\NAT@bibsetnum
+  \def\natexlab##1{}%
+ \else
+  \let\@cite\NAT@cite
+  \let\@citex\NAT@citex
+  \let\@biblabel\NAT@biblabel
+  \let\@bibsetup\NAT@bibsetup
+  \def\natexlab##1{##1}%
+ \fi}
+\AtBeginDocument{\NAT@set@cites}
+\AtBeginDocument{\ifx\SK@def\@undefined\else
+\ifx\SK@cite\@empty\else
+  \SK@def\@citex[#1][#2]#3{\SK@\SK@@ref{#3}\SK@@citex[#1][#2]{#3}}\fi
+\ifx\SK@citeauthor\@undefined\def\HAR@checkdef{}\else
+  \let\citeauthor\SK@citeauthor
+  \let\citefullauthor\SK@citefullauthor
+  \let\citeyear\SK@citeyear\fi
+\fi}
+\AtBeginDocument{\@ifpackageloaded{hyperref}{%
+  \ifnum\NAT@sort=2\def\NAT@sort{1}\fi}{}}
+\newif\ifNAT@full\NAT@fullfalse
+\newif\ifNAT@swa
+\DeclareRobustCommand\citet
+   {\begingroup\NAT@swafalse\def\NAT@ctype{0}\NAT@partrue
+     \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\newcommand\NAT@citetp{\@ifnextchar[{\NAT@@citetp}{\NAT@@citetp[]}}
+\newcommand\NAT@@citetp{}
+\def\NAT@@citetp[#1]{\@ifnextchar[{\@citex[#1]}{\@citex[][#1]}}
+\DeclareRobustCommand\citep
+   {\begingroup\NAT@swatrue\def\NAT@ctype{0}\NAT@partrue
+         \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\cite
+    {\begingroup\def\NAT@ctype{0}\NAT@partrue\NAT@swatrue
+      \@ifstar{\NAT@fulltrue\NAT@cites}{\NAT@fullfalse\NAT@cites}}
+\newcommand\NAT@cites{\@ifnextchar [{\NAT@@citetp}{%
+     \ifNAT@numbers\else
+     \NAT@swafalse
+     \fi
+    \NAT@@citetp[]}}
+\DeclareRobustCommand\citealt
+   {\begingroup\NAT@swafalse\def\NAT@ctype{0}\NAT@parfalse
+         \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\citealp
+   {\begingroup\NAT@swatrue\def\NAT@ctype{0}\NAT@parfalse
+         \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\citeauthor
+   {\begingroup\NAT@swafalse\def\NAT@ctype{1}\NAT@parfalse
+    \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\Citet
+   {\begingroup\NAT@swafalse\def\NAT@ctype{0}\NAT@partrue
+     \let\NAT@up\NAT@Up
+     \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\Citep
+   {\begingroup\NAT@swatrue\def\NAT@ctype{0}\NAT@partrue
+     \let\NAT@up\NAT@Up
+         \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\Citealt
+   {\begingroup\NAT@swafalse\def\NAT@ctype{0}\NAT@parfalse
+     \let\NAT@up\NAT@Up
+         \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\Citealp
+   {\begingroup\NAT@swatrue\def\NAT@ctype{0}\NAT@parfalse
+     \let\NAT@up\NAT@Up
+         \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\Citeauthor
+   {\begingroup\NAT@swafalse\def\NAT@ctype{1}\NAT@parfalse
+     \let\NAT@up\NAT@Up
+    \@ifstar{\NAT@fulltrue\NAT@citetp}{\NAT@fullfalse\NAT@citetp}}
+\DeclareRobustCommand\citeyear
+   {\begingroup\NAT@swafalse\def\NAT@ctype{2}\NAT@parfalse\NAT@citetp}
+\DeclareRobustCommand\citeyearpar
+   {\begingroup\NAT@swatrue\def\NAT@ctype{2}\NAT@partrue\NAT@citetp}
+\newcommand\citetext[1]{\NAT@open#1\NAT@close}
+\DeclareRobustCommand\citefullauthor
+   {\citeauthor*}
+\newcommand\defcitealias[2]{%
+   \@ifundefined{al@#1\@extra@b@citeb}{}
+   {\PackageWarning{natbib}{Overwriting existing alias for citation #1}}
+   \@namedef{al@#1\@extra@b@citeb}{#2}}
+\DeclareRobustCommand\citetalias{\begingroup
+   \NAT@swafalse\def\NAT@ctype{3}\NAT@parfalse\NAT@citetp}
+\DeclareRobustCommand\citepalias{\begingroup
+   \NAT@swatrue\def\NAT@ctype{3}\NAT@partrue\NAT@citetp}
+\renewcommand\nocite[1]{\@bsphack
+  \@for\@citeb:=#1\do{%
+    \edef\@citeb{\expandafter\@firstofone\@citeb}%
+    \if@filesw\immediate\write\@auxout{\string\citation{\@citeb}}\fi
+    \if*\@citeb\else
+    \@ifundefined{b@\@citeb\@extra@b@citeb}{%
+       \NAT@citeundefined \PackageWarning{natbib}%
+       {Citation `\@citeb' undefined}}{}\fi}%
+  \@esphack}
+\newcommand\NAT@parse[1]{{%
+     \let\protect=\@unexpandable@protect\let~\relax
+     \let\active@prefix=\@gobble
+     \xdef\NAT@temp{\csname b@#1\@extra@b@citeb\endcsname}}%
+     \expandafter\NAT@split\NAT@temp
+     \expandafter\NAT@parse@date\NAT@date??????@@%
+     \ifciteindex\NAT@index\fi
+}
+\newcommand\NAT@split[4]{%
+  \gdef\NAT@num{#1}\gdef\NAT@name{#3}\gdef\NAT@date{#2}%
+  \gdef\NAT@all@names{#4}%
+  \ifx\NAT@noname\NAT@all@names \gdef\NAT@all@names{#3}\fi}
+\newcommand\NAT@parse@date{}
+\def\NAT@parse@date#1#2#3#4#5#6@@{%
+  \ifnum\the\catcode`#1=11\def\NAT@year{}\def\NAT@exlab{#1}\else
+  \ifnum\the\catcode`#2=11\def\NAT@year{#1}\def\NAT@exlab{#2}\else
+  \ifnum\the\catcode`#3=11\def\NAT@year{#1#2}\def\NAT@exlab{#3}\else
+  \ifnum\the\catcode`#4=11\def\NAT@year{#1#2#3}\def\NAT@exlab{#4}\else
+    \def\NAT@year{#1#2#3#4}\def\NAT@exlab{{#5}}\fi\fi\fi\fi}
+\newcommand\NAT@index{}
+\let\NAT@makeindex=\makeindex
+\renewcommand\makeindex{\NAT@makeindex
+  \renewcommand\NAT@index{\@bsphack\begingroup
+     \def~{\string~}\@wrindex{\NAT@idxtxt}}}
+\newcommand\NAT@idxtxt{\NAT@name\ \NAT@open\NAT@date\NAT@close}
+\@ifundefined{@indexfile}{}{\let\NAT@makeindex\relax\makeindex}
+\newif\ifciteindex \citeindexfalse
+\newcommand\citeindextype{default}
+\newcommand\NAT@index@alt{{\let\protect=\noexpand\let~\relax
+  \xdef\NAT@temp{\NAT@idxtxt}}\expandafter\NAT@exp\NAT@temp\@nil}
+\newcommand\NAT@exp{}
+\def\NAT@exp#1\@nil{\mbox{}\index[\citeindextype]{#1}}
+
+\AtBeginDocument{%
+\@ifpackageloaded{index}{\let\NAT@index=\NAT@index@alt}{}}
+\newcommand\NAT@ifcmd{\futurelet\NAT@temp\NAT@ifxcmd}
+\newcommand\NAT@ifxcmd{\ifx\NAT@temp\relax\else\expandafter\NAT@bare\fi}
+\def\NAT@bare#1(#2)#3(@)#4\@nil#5{%
+  \if @#2
+  \expandafter\NAT@apalk#1, , \@nil{#5}\else
+  \stepcounter{NAT@ctr}%
+  \NAT@wrout{\arabic {NAT@ctr}}{#2}{#1}{#3}{#5}
+\fi
+}
+\newcommand\NAT@wrout[5]{%
+\if@filesw
+      {\let\protect\noexpand\let~\relax
+       \immediate
+       \write\@auxout{\string\bibcite{#5}{{#1}{#2}{{#3}}{{#4}}}}}\fi
+\ignorespaces}
+\def\NAT@noname{{}}
+\renewcommand\bibitem{%
+  \@ifnextchar[{\@lbibitem}{%
+    \global\NAT@stdbsttrue
+    \stepcounter{NAT@ctr}\@lbibitem[\arabic{NAT@ctr}]}}
+\def\@lbibitem[#1]#2{%
+  \if\relax\@extra@b@citeb\relax\else
+    \@ifundefined{br@#2\@extra@b@citeb}{}{%
+     \@namedef{br@#2}{\@nameuse{br@#2\@extra@b@citeb}}}\fi
+   \@ifundefined{b@#2\@extra@b@citeb}{\def\NAT@num{}}{\NAT@parse{#2}}%
+   \item[\hfil\hyper@natanchorstart{#2\@extra@b@citeb}\@biblabel{\NAT@num}%
+    \hyper@natanchorend]%
+    \NAT@ifcmd#1(@)(@)\@nil{#2}}
+\ifx\SK@lbibitem\@undefined\else
+   \let\SK@lbibitem\@lbibitem
+   \def\@lbibitem[#1]#2{%
+     \SK@lbibitem[#1]{#2}\SK@\SK@@label{#2}\ignorespaces}\fi
+\newif\ifNAT@stdbst \NAT@stdbstfalse
+
+\AtEndDocument
+  {\ifNAT@stdbst\if@filesw\immediate\write\@auxout{\string
+   \global\string\NAT@numberstrue}\fi\fi
+  }
+\providecommand\bibcite{}
+\renewcommand\bibcite[2]{\@ifundefined{b@#1\@extra@binfo}\relax
+     {\NAT@citemultiple
+      \PackageWarningNoLine{natbib}{Citation `#1' multiply defined}}%
+  \global\@namedef{b@#1\@extra@binfo}{#2}}
+\AtEndDocument{\NAT@swatrue\let\bibcite\NAT@testdef}
+\newcommand\NAT@testdef[2]{%
+  \def\NAT@temp{#2}\expandafter \ifx \csname b@#1\@extra@binfo\endcsname
+    \NAT@temp \else \ifNAT@swa \NAT@swafalse
+       \PackageWarningNoLine{natbib}{Citation(s) may have
+          changed.\MessageBreak
+          Rerun to get citations correct}\fi\fi}
+\newcommand\NAT@apalk{}
+\def\NAT@apalk#1, #2, #3\@nil#4{\if\relax#2\relax
+  \global\NAT@stdbsttrue
+  \NAT@wrout{#1}{}{}{}{#4}\else
+  \stepcounter{NAT@ctr}%
+  \NAT@wrout{\arabic {NAT@ctr}}{#2}{#1}{}{#4}\fi}
+\newcommand\citeauthoryear{}
+\def\citeauthoryear#1#2#3(@)(@)\@nil#4{\stepcounter{NAT@ctr}\if\relax#3\relax
+   \NAT@wrout{\arabic {NAT@ctr}}{#2}{#1}{}{#4}\else
+   \NAT@wrout{\arabic {NAT@ctr}}{#3}{#2}{#1}{#4}\fi}
+\newcommand\citestarts{\NAT@open}
+\newcommand\citeends{\NAT@close}
+\newcommand\betweenauthors{and}
+\newcommand\astroncite{}
+\def\astroncite#1#2(@)(@)\@nil#3{\stepcounter{NAT@ctr}\NAT@wrout{\arabic
+{NAT@ctr}}{#2}{#1}{}{#3}}
+\newcommand\citename{}
+\def\citename#1#2(@)(@)\@nil#3{\expandafter\NAT@apalk#1#2, \@nil{#3}}
+\newcommand\harvarditem[4][]%
+    {\if\relax#1\relax\bibitem[#2(#3)]{#4}\else
+        \bibitem[#1(#3)#2]{#4}\fi }
+\newcommand\harvardleft{\NAT@open}
+\newcommand\harvardright{\NAT@close}
+\newcommand\harvardyearleft{\NAT@open}
+\newcommand\harvardyearright{\NAT@close}
+\AtBeginDocument{\providecommand{\harvardand}{and}}
+\newcommand\harvardurl[1]{\textbf{URL:} \textit{#1}}
+\providecommand\bibsection{}
+\@ifundefined{chapter}%
+  {\renewcommand\bibsection{\section*{\refname
+    \@mkboth{\MakeUppercase{\refname}}{\MakeUppercase{\refname}}}}}
+  {\@ifundefined{NAT@sectionbib}%
+    {\renewcommand\bibsection{\chapter*{\bibname
+     \@mkboth{\MakeUppercase{\bibname}}{\MakeUppercase{\bibname}}}}}
+    {\renewcommand\bibsection{\section*{\bibname
+     \ifx\@mkboth\@gobbletwo\else\markright{\MakeUppercase{\bibname}}\fi}}}}
+\@ifclassloaded{amsart}%
+  {\renewcommand\bibsection{\section*{\refname}}}{}
+\@ifclassloaded{amsbook}%
+  {\renewcommand\bibsection{\chapter*{\bibname}}}{}
+\@ifundefined{bib@heading}{}{\let\bibsection\bib@heading}
+\newcounter{NAT@ctr}
+\renewenvironment{thebibliography}[1]{%
+ \bibsection
+ \vspace{1\p@}\parindent \z@\bibpreamble\bibfont\list
+   {\@biblabel{\arabic{NAT@ctr}}}{\@bibsetup{#1}%
+    \setcounter{NAT@ctr}{0}}%
+    \ifNAT@openbib
+      \renewcommand\newblock{\par}
+    \else
+      \renewcommand\newblock{\hskip .11em \@plus.33em \@minus.07em}%
+    \fi
+    \sloppy\clubpenalty4000\widowpenalty4000
+    \sfcode`\.=1000\relax
+    \let\citeN\cite \let\shortcite\cite
+    \let\citeasnoun\cite\fontsize{7}{9}\selectfont
+ }{\def\@noitemerr{%
+  \PackageWarning{natbib}
+     {Empty `thebibliography' environment}}%
+  \endlist\vskip-\lastskip}
+\let\bibfont\relax
+\let\bibpreamble\relax
+\providecommand\reset@font{\relax}
+\providecommand\bibname{Bibliography}
+\providecommand\refname{References}
+\newcommand\NAT@citeundefined{\gdef \NAT@undefined {%
+    \PackageWarningNoLine{natbib}{There were undefined citations}}}
+\let \NAT@undefined \relax
+\newcommand\NAT@citemultiple{\gdef \NAT@multiple {%
+    \PackageWarningNoLine{natbib}{There were multiply defined citations}}}
+\let \NAT@multiple \relax
+\AtEndDocument{\NAT@undefined\NAT@multiple}
+\providecommand\@mkboth[2]{}
+\providecommand\MakeUppercase{\uppercase}
+\providecommand{\@extra@b@citeb}{}
+\gdef\@extra@binfo{}
+\providecommand\hyper@natanchorstart[1]{}
+\providecommand\hyper@natanchorend{}
+\providecommand\hyper@natlinkstart[1]{}
+\providecommand\hyper@natlinkend{}
+\providecommand\hyper@natlinkbreak[2]{#1}
+\@ifundefined{bbl@redefine}{}{%
+ \bbl@redefine\nocite#1{%
+  \@safe@activestrue\org@nocite{#1}\@safe@activesfalse}%
+\bbl@redefine\@lbibitem[#1]#2{%
+  \@safe@activestrue\org@@lbibitem[#1]{#2}\@safe@activesfalse}%
+}
+\AtBeginDocument{\@ifundefined{bbl@redefine}{}{%
+\bbl@redefine\@citex[#1][#2]#3{%
+  \@safe@activestrue\org@@citex[#1][#2]{#3}\@safe@activesfalse}%
+\bbl@redefine\NAT@testdef#1#2{%
+  \@safe@activestrue\org@NAT@testdef{#1}{#2}\@safe@activesfalse}%
+\@ifundefined{org@@lbibitem}{%
+\bbl@redefine\@lbibitem[#1]#2{%
+  \@safe@activestrue\org@@lbibitem[#1]{#2}\@safe@activesfalse}}{}%
+}}
+\ifnum\NAT@sort>0
+\newcommand\NAT@sort@cites[1]{%
+\@tempcntb\m@ne
+\let\@celt\delimiter
+\def\NAT@num@list{}%
+\def\NAT@cite@list{}%
+\def\NAT@nonsort@list{}%
+\@for \@citeb:=#1\do{\NAT@make@cite@list}%
+\edef\NAT@cite@list{\NAT@cite@list\NAT@nonsort@list}%
+\edef\NAT@cite@list{\expandafter\NAT@xcom\NAT@cite@list @@}}
+\begingroup \catcode`\_=8
+\gdef\NAT@make@cite@list{%
+     \edef\@citeb{\expandafter\@firstofone\@citeb}%
+    \@ifundefined{b@\@citeb\@extra@b@citeb}{\def\NAT@num{A}}%
+    {\NAT@parse{\@citeb}}%
+      \ifcat _\ifnum\z@<0\NAT@num _\else A\fi
+       \@tempcnta\NAT@num \relax
+       \ifnum \@tempcnta>\@tempcntb
+          \edef\NAT@num@list{\NAT@num@list \@celt{\NAT@num}}%
+          \edef\NAT@cite@list{\NAT@cite@list\@citeb,}%
+          \@tempcntb\@tempcnta
+       \else
+          \let\NAT@@cite@list=\NAT@cite@list \def\NAT@cite@list{}%
+          \edef\NAT@num@list{\expandafter\NAT@num@celt \NAT@num@list \@gobble @}%
+          {\let\@celt=\NAT@celt\NAT@num@list}%
+       \fi
+    \else
+       \edef\NAT@nonsort@list{\NAT@nonsort@list\@citeb,}%
+ \fi}
+\endgroup
+\def\NAT@celt#1{\ifnum #1<\@tempcnta
+  \xdef\NAT@cite@list{\NAT@cite@list\expandafter\NAT@nextc\NAT@@cite@list @@}%
+  \xdef\NAT@@cite@list{\expandafter\NAT@restc\NAT@@cite@list}%
+ \else
+  \xdef\NAT@cite@list{\NAT@cite@list\@citeb,\NAT@@cite@list}\let\@celt\@gobble%
+ \fi}
+\def\NAT@num@celt#1#2{\ifx \@celt #1%
+     \ifnum #2<\@tempcnta
+        \@celt{#2}%
+        \expandafter\expandafter\expandafter\NAT@num@celt
+     \else
+        \@celt{\number\@tempcnta}\@celt{#2}%
+  \fi\fi}
+\def\NAT@nextc#1,#2@@{#1,}
+\def\NAT@restc#1,#2{#2}
+\def\NAT@xcom#1,@@{#1}
+\else
+ \newcommand\NAT@sort@cites[1]{\edef\NAT@cite@list{#1}}\fi
+\InputIfFileExists{natbib.cfg}
+       {\typeout{Local config file natbib.cfg used}}{}
+%% 
+%% <<<<< End of generated file <<<<<<
+%%
+%% End of file `natbib.sty'.

tex/ngmlr.eval

@@ -0,0 +1,38 @@
+Q	60	23616	0	0.000000000
+Q	45	3520	1	0.000036851
+Q	41	1840	1	0.000069023
+Q	37	328	2	0.000136500
+Q	36	276	1	0.000169033
+Q	35	480	1	0.000199601
+Q	33	375	2	0.000262855
+Q	31	178	2	0.000326659
+Q	30	153	5	0.000487551
+Q	29	200	1	0.000516696
+Q	27	100	3	0.000611601
+Q	26	93	3	0.000706056
+Q	25	75	2	0.000768393
+Q	24	82	1	0.000798314
+Q	23	80	6	0.000987387
+Q	22	71	6	0.001175835
+Q	21	76	7	0.001394921
+Q	20	63	9	0.001676897
+Q	19	55	4	0.001800322
+Q	18	62	8	0.002048987
+Q	17	55	7	0.002265718
+Q	16	60	10	0.002575539
+Q	15	82	9	0.002850877
+Q	14	67	7	0.003063745
+Q	13	62	11	0.003401042
+Q	12	64	13	0.003799084
+Q	11	56	5	0.003947900
+Q	10	58	17	0.004468303
+Q	9	70	22	0.005139796
+Q	8	23	9	0.005414604
+Q	7	41	17	0.005933068
+Q	6	42	18	0.006480881
+Q	5	33	9	0.006751757
+Q	4	29	9	0.007022948
+Q	3	27	15	0.007478764
+Q	2	23	10	0.007781024
+Q	1	9	2	0.007840364
+Q	0	13	8	0.008083105

tex/roc.gp

@@ -0,0 +1,60 @@
+set t po eps enh co so "Helvetica,26"
+
+set style line 1 lt 1 pt 1 lc rgb "#e41a1c" lw 2;
+set style line 2 lt 1 pt 2 lc rgb "#377eb8" lw 2;
+set style line 3 lt 1 pt 3 lc rgb "#4daf4a" lw 2;
+set style line 4 lt 1 pt 4 lc rgb "#984ea3" lw 2;
+set style line 5 lt 1 pt 6 lc rgb "#ff7f00" lw 2;
+set style line 6 lt 1 pt 8 lc rgb "#f781bf" lw 2;
+
+set out "roc-color.eps"
+
+set pointsize 2.0
+set size 1.59,1.04
+set multiplot layout 1,2
+
+set label "(a)" at graph -0.245,1.06 font "Helvetica-bold,40"
+set xlab "Error rate of mapped PacBio reads"
+set ylab "Fraction of mapped reads" off +1.8
+set ytics 0.02
+set yran [0.9:1]
+
+set size 0.8,1
+set log x
+set format x "10^{%L}"
+set key bot right
+plot "<./eval2roc.pl blasr-mc.eval" u 2:3 t "blasr-mc" w lp ls 4, \
+     "<./eval2roc.pl bwa.eval" u 2:3 t "bwa-mem" w lp ls 2, \
+     "<./eval2roc.pl graphmap.eval" u 2:3 t "graphmap" w lp ls 3, \
+     "<./eval2roc.pl minialign.eval" u 2:3 t "minialign" w lp ls 1, \
+     "<./eval2roc.pl mm2.eval" u 2:3 t "minimap2" w lp ls 6, \
+     "<./eval2roc.pl ngmlr.eval" u 2:3 t "ngm-lr" w lp ls 5
+unset label
+
+set origin 0.8,0
+set size 0.79,1
+set label "(b)" at graph -0.245,1.06 font "Helvetica-bold,40"
+set xlab "Error rate of mapped short reads"
+
+set key top left
+plot "<./eval2roc.pl -n2e7 bowtie2-s3.sam.eval" u 2:3 t "bowtie2" w lp ls 5, \
+	 "<./eval2roc.pl -n2e7 bwa-s3.sam.eval" u 2:3 t "bwa-mem" w lp ls 2, \
+	 "<./eval2roc.pl -n2e7 mm2-s3.sam.eval" u 2:3 t "minimap2" w lp ls 6, \
+	 "<./eval2roc.pl -n2e7 snap-s3.sam.eval" u 2:3 t "snap" w lp ls 3
+
+#unset log
+#unset format
+#unset key
+#set log y
+#set ylab "Accumulative mapping error rate" off +0
+#set xlab "Mapping quality"
+#set yran [1e-5:0.1]
+#set ytics 1e-5,0.1
+#set format y "10^{%L}"
+#set xran [60:0] reverse
+#plot "<./eval2roc.pl blasr-mc.eval" u 1:2 w lp ls 4, \
+#     "<./eval2roc.pl bwa.eval" u 1:2 t "bwa-mem" w lp ls 2, \
+#     "<./eval2roc.pl graphmap.eval" u 1:2 t "graphmap" w lp ls 3, \
+#     "<./eval2roc.pl minialign.eval" u 1:2 t "minialign" w lp ls 1, \
+#     "<./eval2roc.pl mm2.eval" u 1:2 t "minimap2" w lp ls 6, \
+#     "<./eval2roc.pl ngmlr.eval" u 1:2 t "ngm-lr" w lp ls 5

tex/snap-s3.sam.eval

@@ -0,0 +1,62 @@
+Q	60	18993268	10320	0.000543350	18993268
+Q	59	33156	216	0.000553756	19026424
+Q	58	29982	295	0.000568365	19056406
+Q	57	9412	278	0.000582666	19065818
+Q	56	11012	228	0.000594281	19076830
+Q	55	9968	235	0.000606283	19086798
+Q	54	8602	292	0.000621301	19095400
+Q	53	6094	259	0.000634662	19101494
+Q	52	5026	257	0.000647946	19106520
+Q	51	4278	224	0.000659522	19110798
+Q	50	3682	178	0.000668708	19114480
+Q	49	2750	156	0.000676772	19117230
+Q	48	2314	112	0.000682548	19119544
+Q	47	2056	96	0.000687495	19121600
+Q	46	1658	62	0.000690677	19123258
+Q	45	1492	74	0.000694493	19124750
+Q	44	1150	56	0.000697379	19125900
+Q	43	1062	48	0.000699850	19126962
+Q	42	976	60	0.000702951	19127938
+Q	41	884	36	0.000704800	19128822
+Q	40	708	52	0.000707493	19129530
+Q	39	870	26	0.000708819	19130400
+Q	38	598	26	0.000710156	19130998
+Q	37	542	34	0.000711913	19131540
+Q	36	846	50	0.000714495	19132386
+Q	35	590	50	0.000717087	19132976
+Q	34	550	42	0.000719261	19133526
+Q	33	2174	66	0.000722628	19135700
+Q	32	876	86	0.000727089	19136576
+Q	31	638	104	0.000732500	19137214
+Q	30	1718	196	0.000742675	19138932
+Q	29	91022	968	0.000789497	19229954
+Q	28	12864	781	0.000829556	19242818
+Q	27	5806	427	0.000851489	19248624
+Q	26	25274	728	0.000888144	19273898
+Q	25	7418	680	0.000923070	19281316
+Q	24	11800	701	0.000958839	19293116
+Q	23	57328	3933	0.001159250	19350444
+Q	22	7662	846	0.001202494	19358106
+Q	21	5924	617	0.001233989	19364030
+Q	20	4623	574	0.001263330	19368653
+Q	19	4988	942	0.001311627	19373641
+Q	18	3968	793	0.001352282	19377609
+Q	17	3630	681	0.001387166	19381239
+Q	16	2921	513	0.001413422	19384160
+Q	15	2716	424	0.001435095	19386876
+Q	14	2366	365	0.001453744	19389242
+Q	13	2169	412	0.001474828	19391411
+Q	12	2077	360	0.001493233	19393488
+Q	11	2016	441	0.001515815	19395504
+Q	10	2292	738	0.001553682	19397796
+Q	9	4165	1832	0.001647772	19401961
+Q	8	3963	1862	0.001743385	19405924
+Q	7	3927	1793	0.001835408	19409851
+Q	6	3572	1639	0.001919497	19413423
+Q	5	3270	1533	0.001998126	19416693
+Q	4	3046	1610	0.002080718	19419739
+Q	3	251447	125550	0.008436553	19671186
+Q	2	24390	13537	0.009113417	19695576
+Q	1	124406	86780	0.013434624	19819982
+Q	0	171254	153874	0.021016609	19991236
+U	8764