added two more filters

Its use in minimap2 didn't trigger the bug, so the older minimap2 is still ok.

FAQ.md

@@ -0,0 +1,46 @@
+#### 1. Alignment different with option `-a` or `-c`?
+
+Without `-a`, `-c` or `--cs`, minimap2 only finds *approximate* mapping
+locations without detailed base alignment. In particular, the start and end
+positions of the alignment are impricise. With one of those options, minimap2
+will perform base alignment, which is generally more accurate but is much
+slower.
+
+#### 2. How to map Illumina short reads to noisy long reads?
+
+No good solutions. The better approach is to assemble short reads into contigs
+and then map noisy reads to contigs.
+
+#### 3. The output SAM doesn't have a header.
+
+By default, minimap2 indexes 4 billion reference bases (4Gb) in a batch and map
+all reads against each reference batch. Given a reference longer than 4Gb,
+minimap2 is unable to see all the sequences and thus can't produce a correct
+SAM header. In this case, minimap2 doesn't output any SAM header. There are two
+solutions to this issue. First, you may increase option `-I` to, for example,
+`-I8g` to index more reference bases in a batch. This is preferred if your
+machine has enough memory. Second, if your machines doesn't have enough memory
+to hold the reference index, you can use the `--split-prefix` option in a
+command line like:
+```sh
+minimap2 -ax map-ont --split-prefix=tmp ref.fa reads.fq
+```
+This second approach uses less memory, but it is slower and requires temporary
+disk space.
+
+#### 4. The output SAM is malformatted.
+
+This typically happens when you use nohup to wrap a minimap2 command line.
+Nohup is discouraged as it breaks piping. If you have to use nohup, please
+specify an output file with option `-o`.
+
+#### 5. How to output one alignment per read?
+
+You can use `--secondary=no` to suppress secondary alignments (aka multiple
+mappings), but you can't suppress supplementary alignment (aka split or
+chimeric alignment) this way. You can use samtools to filter out these
+alignments:
+```sh
+minimap2 -ax map-out ref.fa reads.fq | samtools view -F0x900
+```
+However, this is discouraged as supplementary alignment is informative.

README.md

@@ -315,9 +315,10 @@ highlighted in bold. The description may help to tune minimap2 parameters.
 ### <a name="help"></a>Getting help
 
 Manpage [minimap2.1][manpage] provides detailed description of minimap2
-command line options and optional tags. If you encounter bugs or have further
+command line options and optional tags. The [FAQ](FAQ.md) page answers several
-questions or requests, you can raise an issue at the [issue page][issue].
+frequently asked questions. If you encounter bugs or have further questions or
-There is not a specific mailing list for the time being.
+requests, you can raise an issue at the [issue page][issue].  There is not a
+specific mailing list for the time being.
 
 ### <a name="cite"></a>Citing minimap2
 

chain.c

@@ -155,8 +155,8 @@ mm128_t *mm_chain_dp(int max_dist_x, int max_dist_y, int bw, int max_skip, int m
 		memcpy(&a[k], &b[w[i].y>>32], n * sizeof(mm128_t));
 		k += n;
 	}
-	memcpy(u, u2, n_u * 8);
+	if (n_u) memcpy(u, u2, n_u * 8);
-	memcpy(b, a, k * sizeof(mm128_t)); // write _a_ to _b_ and deallocate _a_ because _a_ is oversized, sometimes a lot
+	if (k) memcpy(b, a, k * sizeof(mm128_t)); // write _a_ to _b_ and deallocate _a_ because _a_ is oversized, sometimes a lot
 	kfree(km, a); kfree(km, w); kfree(km, u2);
 	return b;
 }

index.c

@@ -1,4 +1,5 @@
 #include <stdlib.h>
+#include <limits.h>
 #include <assert.h>
 #if defined(WIN32) || defined(_WIN32)
 #include <io.h> // for open(2)
@@ -188,12 +189,18 @@ int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
  * Sort and generate hash tables *
  *********************************/
 
+typedef struct {
+	mm_idx_t *mi;
+	int min_occ, max_occ;
+} idx_post_t;
+
 static void worker_post(void *g, long i, int tid)
 {
 	int n, n_keys;
 	size_t j, start_a, start_p;
 	idxhash_t *h;
-	mm_idx_t *mi = (mm_idx_t*)g;
+	idx_post_t *o = (idx_post_t*)g;
+	mm_idx_t *mi = o->mi;
 	mm_idx_bucket_t *b = &mi->B[i];
 	if (b->a.n == 0) return;
 
@@ -203,8 +210,10 @@ static void worker_post(void *g, long i, int tid)
 	// count and preallocate
 	for (j = 1, n = 1, n_keys = 0, b->n = 0; j <= b->a.n; ++j) {
 		if (j == b->a.n || b->a.a[j].x>>8 != b->a.a[j-1].x>>8) {
-			++n_keys;
+			if (n >= o->min_occ && n <= o->max_occ) {
-			if (n > 1) b->n += n;
+				++n_keys;
+				if (n > 1) b->n += n;
+			}
 			n = 1;
 		} else ++n;
 	}
@@ -218,18 +227,20 @@ static void worker_post(void *g, long i, int tid)
 			khint_t itr;
 			int absent;
 			mm128_t *p = &b->a.a[j-1];
-			itr = kh_put(idx, h, p->x>>8>>mi->b<<1, &absent);
+			if (n >= o->min_occ && n <= o->max_occ) {
-			assert(absent && j == start_a + n);
+				itr = kh_put(idx, h, p->x>>8>>mi->b<<1, &absent);
-			if (n == 1) {
+				assert(absent && j == start_a + n);
-				kh_key(h, itr) |= 1;
+				if (n == 1) {
-				kh_val(h, itr) = p->y;
+					kh_key(h, itr) |= 1;
-			} else {
+					kh_val(h, itr) = p->y;
-				int k;
+				} else {
-				for (k = 0; k < n; ++k)
+					int k;
-					b->p[start_p + k] = b->a.a[start_a + k].y;
+					for (k = 0; k < n; ++k)
-				radix_sort_64(&b->p[start_p], &b->p[start_p + n]); // sort by position; needed as in-place radix_sort_128x() is not stable
+						b->p[start_p + k] = b->a.a[start_a + k].y;
-				kh_val(h, itr) = (uint64_t)start_p<<32 | n;
+					radix_sort_64(&b->p[start_p], &b->p[start_p + n]); // sort by position; needed as in-place radix_sort_128x() is not stable
-				start_p += n;
+					kh_val(h, itr) = (uint64_t)start_p<<32 | n;
+					start_p += n;
+				}
 			}
 			start_a = j, n = 1;
 		} else ++n;
@@ -242,9 +253,12 @@ static void worker_post(void *g, long i, int tid)
 	b->a.n = b->a.m = 0, b->a.a = 0;
 }
  
-static void mm_idx_post(mm_idx_t *mi, int n_threads)
+static void mm_idx_post(mm_idx_t *mi, int n_threads, int min_occ, int max_occ)
 {
-	kt_for(n_threads, worker_post, mi, 1<<mi->b);
+	idx_post_t t;
+	if (max_occ <= 0 || max_occ < min_occ) max_occ = INT_MAX;
+	t.mi = mi, t.min_occ = min_occ, t.max_occ = max_occ;
+	kt_for(n_threads, worker_post, &t, 1<<mi->b);
 }
 
 /******************
@@ -350,7 +364,7 @@ static void *worker_pipeline(void *shared, int step, void *in)
     return 0;
 }
 
-mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size)
+mm_idx_t *mm_idx_gen2(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size, int min_occ, int max_occ)
 {
 	pipeline_t pl;
 	if (fp == 0 || mm_bseq_eof(fp)) return 0;
@@ -364,13 +378,18 @@ mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini
 	if (mm_verbose >= 3)
 		fprintf(stderr, "[M::%s::%.3f*%.2f] collected minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
 
-	mm_idx_post(pl.mi, n_threads);
+	mm_idx_post(pl.mi, n_threads, min_occ, max_occ);
 	if (mm_verbose >= 3)
 		fprintf(stderr, "[M::%s::%.3f*%.2f] sorted minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
 
 	return pl.mi;
 }
 
+mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size)
+{
+	return mm_idx_gen2(fp, w, k, b, flag, mini_batch_size, n_threads, batch_size, 0, INT_MAX);
+}
+
 mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads) // a simpler interface; deprecated
 {
 	mm_bseq_file_t *fp;
@@ -427,7 +446,7 @@ mm_idx_t *mm_idx_str(int w, int k, int is_hpc, int bucket_bits, int n, const cha
 		}
 	}
 	free(a.a);
-	mm_idx_post(mi, 1);
+	mm_idx_post(mi, 1, 0, 0);
 	return mi;
 }
 
@@ -588,7 +607,7 @@ mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
 		if (mi && mm_verbose >= 2 && (mi->k != r->opt.k || mi->w != r->opt.w || (mi->flag&MM_I_HPC) != (r->opt.flag&MM_I_HPC)))
 			fprintf(stderr, "[WARNING]\033[1;31m Indexing parameters (-k, -w or -H) overridden by parameters used in the prebuilt index.\033[0m\n");
 	} else
-		mi = mm_idx_gen(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.flag, r->opt.mini_batch_size, n_threads, r->opt.batch_size);
+		mi = mm_idx_gen2(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.flag, r->opt.mini_batch_size, n_threads, r->opt.batch_size, r->opt.min_occ, r->opt.max_occ);
 	if (mi) {
 		if (r->fp_out) mm_idx_dump(r->fp_out, mi);
 		mi->index = r->n_parts++;

kalloc.c

@@ -18,15 +18,14 @@
  *       |                           |                 |                               |
  *       p=p->ptr->ptr->ptr->ptr     p->ptr            p->ptr->ptr                     p->ptr->ptr->ptr
  */
-
-#define MIN_CORE_SIZE 0x80000
-
 typedef struct header_t {
 	size_t size;
 	struct header_t *ptr;
 } header_t;
 
 typedef struct {
+	void *par;
+	size_t min_core_size;
 	header_t base, *loop_head, *core_head; /* base is a zero-sized block always kept in the loop */
 } kmem_t;
 
@@ -36,31 +35,39 @@ static void panic(const char *s)
 	abort();
 }
 
-void *km_init(void)
+void *km_init2(void *km_par, size_t min_core_size)
 {
-	return calloc(1, sizeof(kmem_t));
+	kmem_t *km;
+	km = (kmem_t*)kcalloc(km_par, 1, sizeof(kmem_t));
+	km->par = km_par;
+	km->min_core_size = min_core_size > 0? min_core_size : 0x80000;
+	return (void*)km;
 }
 
+void *km_init(void) { return km_init2(0, 0); }
+
 void km_destroy(void *_km)
 {
 	kmem_t *km = (kmem_t*)_km;
+	void *km_par;
 	header_t *p, *q;
 	if (km == NULL) return;
+	km_par = km->par;
 	for (p = km->core_head; p != NULL;) {
 		q = p->ptr;
-		free(p);
+		kfree(km_par, p);
 		p = q;
 	}
-	free(km);
+	kfree(km_par, km);
 }
 
 static header_t *morecore(kmem_t *km, size_t nu)
 {
 	header_t *q;
 	size_t bytes, *p;
-	nu = (nu + 1 + (MIN_CORE_SIZE - 1)) / MIN_CORE_SIZE * MIN_CORE_SIZE; /* the first +1 for core header */
+	nu = (nu + 1 + (km->min_core_size - 1)) / km->min_core_size * km->min_core_size; /* the first +1 for core header */
 	bytes = nu * sizeof(header_t);
-	q = (header_t*)malloc(bytes);
+	q = (header_t*)kmalloc(km->par, bytes);
 	if (!q) panic("[morecore] insufficient memory");
 	q->ptr = km->core_head, q->size = nu, km->core_head = q;
 	p = (size_t*)(q + 1);
@@ -125,7 +132,7 @@ void *kmalloc(void *_km, size_t n_bytes)
 
 	if (n_bytes == 0) return 0;
 	if (km == NULL) return malloc(n_bytes);
-	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t) + 1;
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t); /* header+n_bytes requires at least this number of units */
 
 	if (!(q = km->loop_head)) /* the first time when kmalloc() is called, intialize it */
 		q = km->loop_head = km->base.ptr = &km->base;
@@ -160,18 +167,18 @@ void *kcalloc(void *_km, size_t count, size_t size)
 void *krealloc(void *_km, void *ap, size_t n_bytes) // TODO: this can be made more efficient in principle
 {
 	kmem_t *km = (kmem_t*)_km;
-	size_t n_units, *p, *q;
+	size_t cap, *p, *q;
 
 	if (n_bytes == 0) {
 		kfree(km, ap); return 0;
 	}
 	if (km == NULL) return realloc(ap, n_bytes);
 	if (ap == NULL) return kmalloc(km, n_bytes);
-	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t);
 	p = (size_t*)ap - 1;
-	if (*p >= n_units) return ap; /* TODO: this prevents shrinking */
+	cap = (*p) * sizeof(header_t) - sizeof(size_t);
+	if (cap >= n_bytes) return ap; /* TODO: this prevents shrinking */
 	q = (size_t*)kmalloc(km, n_bytes);
-	memcpy(q, ap, (*p - 1) * sizeof(header_t));
+	memcpy(q, ap, cap);
 	kfree(km, ap);
 	return q;
 }

kalloc.h

@@ -17,6 +17,7 @@ void *kcalloc(void *km, size_t count, size_t size);
 void kfree(void *km, void *ptr);
 
 void *km_init(void);
+void *km_init2(void *km_par, size_t min_core_size);
 void km_destroy(void *km);
 void km_stat(const void *_km, km_stat_t *s);
 
@@ -24,4 +25,13 @@ void km_stat(const void *_km, km_stat_t *s);
 }
 #endif
 
+#define KMALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kmalloc((km), (len) * sizeof(*(ptr))))
+#define KCALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kcalloc((km), (len), sizeof(*(ptr))))
+#define KREALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))krealloc((km), (ptr), (len) * sizeof(*(ptr))))
+
+#define KEXPAND(km, a, m) do { \
+		(m) = (m) >= 4? (m) + ((m)>>1) : 16; \
+		KREALLOC((km), (a), (m)); \
+	} while (0)
+
 #endif

main.c

@@ -1,12 +1,13 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <errno.h>
 #include "bseq.h"
 #include "minimap.h"
 #include "mmpriv.h"
 #include "ketopt.h"
 
-#define MM_VERSION "2.17-r941"
+#define MM_VERSION "2.17-r954-dirty"
 
 #ifdef __linux__
 #include <sys/resource.h>
@@ -66,6 +67,10 @@ static ko_longopt_t long_options[] = {
 	{ "junc-bed",       ko_required_argument, 340 },
 	{ "junc-bonus",     ko_required_argument, 341 },
 	{ "sam-hit-only",   ko_no_argument,       342 },
+	{ "idx-min-occ",    ko_required_argument, 343 },
+	{ "idx-max-occ",    ko_required_argument, 344 },
+	{ "flt-max-dv",     ko_required_argument, 345 },
+	{ "flt-min-blen",   ko_required_argument, 346 },
 	{ "help",           ko_no_argument,       'h' },
 	{ "max-intron-len", ko_required_argument, 'G' },
 	{ "version",        ko_no_argument,       'V' },
@@ -172,7 +177,7 @@ int main(int argc, char *argv[])
 		else if (c == 'o') {
 			if (strcmp(o.arg, "-") != 0) {
 				if (freopen(o.arg, "wb", stdout) == NULL) {
-					fprintf(stderr, "[ERROR]\033[1;31m failed to write the output to file '%s'\033[0m\n", o.arg);
+					fprintf(stderr, "[ERROR]\033[1;31m failed to write the output to file '%s'\033[0m: %s\n", o.arg, strerror(errno));
 					exit(1);
 				}
 			}
@@ -209,6 +214,10 @@ int main(int argc, char *argv[])
 		else if (c == 338) opt.max_qlen = mm_parse_num(o.arg); // --max-qlen
 		else if (c == 340) junc_bed = o.arg; // --junc-bed
 		else if (c == 342) opt.flag |= MM_F_SAM_HIT_ONLY; // --sam-hit-only
+		else if (c == 343) ipt.min_occ = mm_parse_num(o.arg); // --idx-min-occ
+		else if (c == 344) ipt.max_occ = mm_parse_num(o.arg); // --idx-max-occ
+		else if (c == 345) opt.flt_max_dv = atof(o.arg); // --flt-max-dv
+		else if (c == 346) opt.flt_min_blen = mm_parse_num(o.arg); // --flt-min-blen
 		else if (c == 314) { // --frag
 			yes_or_no(&opt, MM_F_FRAG_MODE, o.longidx, o.arg, 1);
 		} else if (c == 315) { // --secondary
@@ -337,7 +346,7 @@ int main(int argc, char *argv[])
 	}
 	idx_rdr = mm_idx_reader_open(argv[o.ind], &ipt, fnw);
 	if (idx_rdr == 0) {
-		fprintf(stderr, "[ERROR] failed to open file '%s'\n", argv[o.ind]);
+		fprintf(stderr, "[ERROR] failed to open file '%s': %s\n", argv[o.ind], strerror(errno));
 		return 1;
 	}
 	if (!idx_rdr->is_idx && fnw == 0 && argc - o.ind < 2) {
@@ -384,7 +393,7 @@ int main(int argc, char *argv[])
 		mm_split_merge(argc - (o.ind + 1), (const char**)&argv[o.ind + 1], &opt, n_parts);
 
 	if (fflush(stdout) == EOF) {
-		fprintf(stderr, "[ERROR] failed to write the results\n");
+		perror("[ERROR] failed to write the results");
 		exit(EXIT_FAILURE);
 	}
 

map.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include <errno.h>
 #include "kthread.h"
 #include "kvec.h"
 #include "kalloc.h"
@@ -350,6 +351,15 @@ void mm_map_frag(const mm_idx_t *mi, int n_segs, const int *qlens, const char **
 
 	chain_post(opt, max_chain_gap_ref, mi, b->km, qlen_sum, n_segs, qlens, &n_regs0, regs0, a);
 	if (!is_sr) mm_est_err(mi, qlen_sum, n_regs0, regs0, a, n_mini_pos, mini_pos);
+	if (!is_sr && n_segs == 1) {
+		for (i = j = 0; i < n_regs0; ++i) {
+			mm_reg1_t *r = &regs0[i];
+			if (r->div > opt->flt_max_dv) continue;
+			if (r->blen < opt->flt_min_blen) continue;
+			regs0[j++] = regs0[i];
+		}
+		n_regs0 = j;
+	}
 
 	if (n_segs == 1) { // uni-segment
 		regs0 = align_regs(opt, mi, b->km, qlens[0], seqs[0], &n_regs0, regs0, a);
@@ -622,7 +632,7 @@ static mm_bseq_file_t **open_bseqs(int n, const char **fn)
 	for (i = 0; i < n; ++i) {
 		if ((fp[i] = mm_bseq_open(fn[i])) == 0) {
 			if (mm_verbose >= 1)
-				fprintf(stderr, "ERROR: failed to open file '%s'\n", fn[i]);
+				fprintf(stderr, "ERROR: failed to open file '%s': %s\n", fn[i], strerror(errno));
 			for (j = 0; j < i; ++j)
 				mm_bseq_close(fp[j]);
 			free(fp);

minimap.h

@@ -101,6 +101,7 @@ typedef struct {
 typedef struct {
 	short k, w, flag, bucket_bits;
 	int mini_batch_size;
+	int min_occ, max_occ;
 	uint64_t batch_size;
 } mm_idxopt_t;
 
@@ -118,6 +119,9 @@ typedef struct {
 	int min_cnt;         // min number of minimizers on each chain
 	int min_chain_score; // min chaining score
 
+	float flt_max_dv;
+	int flt_min_blen;
+
 	float mask_level;
 	float pri_ratio;
 	int best_n;      // top best_n chains are subjected to DP alignment

minimap2.1

@@ -638,6 +638,7 @@ s2	i	Chaining score of the best secondary chain
 NM	i	Total number of mismatches and gaps in the alignment
 MD	Z	To generate the ref sequence in the alignment
 AS	i	DP alignment score
+SA	Z	List of other supplementary alignments
 ms	i	DP score of the max scoring segment in the alignment
 nn	i	Number of ambiguous bases in the alignment
 ts	A	Transcript strand (splice mode only)

misc.c

@@ -125,7 +125,7 @@ void mm_err_puts(const char *str)
 	int ret;
 	ret = puts(str);
 	if (ret == EOF) {
-		fprintf(stderr, "[ERROR] failed to write the results\n");
+		perror("[ERROR] failed to write the results");
 		exit(EXIT_FAILURE);
 	}
 }
@@ -135,7 +135,7 @@ void mm_err_fwrite(const void *p, size_t size, size_t nitems, FILE *fp)
 	int ret;
 	ret = fwrite(p, size, nitems, fp);
 	if (ret == EOF) {
-		fprintf(stderr, "[ERROR] failed to write data\n");
+		perror("[ERROR] failed to write data");
 		exit(EXIT_FAILURE);
 	}
 }
@@ -145,7 +145,7 @@ void mm_err_fread(void *p, size_t size, size_t nitems, FILE *fp)
 	int ret;
 	ret = fread(p, size, nitems, fp);
 	if (ret == EOF) {
-		fprintf(stderr, "[ERROR] failed to read data\n");
+		perror("[ERROR] failed to read data");
 		exit(EXIT_FAILURE);
 	}
 }

misc/paftools.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env k8
 
-var paftools_version = '2.17-r941';
+var paftools_version = '2.17-r949-dirty';
 
 /*****************************
  ***** Library functions *****
@@ -1509,6 +1509,7 @@ function paf_gff2bed(args)
 
 	var colors = {
 		'protein_coding':'0,128,255',
+		'mRNA':'0,128,255',
 		'lincRNA':'0,192,0',
 		'snRNA':'0,192,0',
 		'miRNA':'0,192,0',
@@ -1541,8 +1542,8 @@ function paf_gff2bed(args)
 		print(a[0][0], st, en, name, 1000, a[0][3], cds_st, cds_en, color, a.length, sizes.join(",") + ",", starts.join(",") + ",");
 	}
 
-	var re_gtf = /(transcript_id|transcript_type|transcript_biotype|gene_name|transcript_name) "([^"]+)";/g;
+	var re_gtf = /\b(transcript_id|transcript_type|transcript_biotype|gene_name|gene_id|gbkey|transcript_name) "([^"]+)";/g;
-	var re_gff3 = /(transcript_id|transcript_type|transcript_biotype|gene_name|transcript_name)=([^;]+)/g;
+	var re_gff3 = /\b(transcript_id|transcript_type|transcript_biotype|gene_name|gene_id|gbkey|transcript_name)=([^;]+)/g;
 	var buf = new Bytes();
 	var file = args[getopt.ind] == '-'? new File() : new File(args[getopt.ind]);
 
@@ -1559,19 +1560,19 @@ function paf_gff2bed(args)
 		if (t[2] != "CDS" && t[2] != "exon") continue;
 		t[3] = parseInt(t[3]) - 1;
 		t[4] = parseInt(t[4]);
-		var id = null, type = "", gname = "N/A", biotype = "", m, tname = "N/A";
+		var id = null, type = "", name = "N/A", biotype = "", m, tname = "N/A";
 		while ((m = re_gtf.exec(t[8])) != null) {
 			if (m[1] == "transcript_id") id = m[2];
 			else if (m[1] == "transcript_type") type = m[2];
-			else if (m[1] == "transcript_biotype") biotype = m[2];
+			else if (m[1] == "transcript_biotype" || m[1] == "gbkey") biotype = m[2];
-			else if (m[1] == "gene_name") name = m[2];
+			else if (m[1] == "gene_name" || m[1] == "gene_id") name = m[2];
 			else if (m[1] == "transcript_name") tname = m[2];
 		}
 		while ((m = re_gff3.exec(t[8])) != null) {
 			if (m[1] == "transcript_id") id = m[2];
 			else if (m[1] == "transcript_type") type = m[2];
-			else if (m[1] == "transcript_biotype") biotype = m[2];
+			else if (m[1] == "transcript_biotype" || m[1] == "gbkey") biotype = m[2];
-			else if (m[1] == "gene_name") name = m[2];
+			else if (m[1] == "gene_name" || m[1] == "gene_id") name = m[2];
 			else if (m[1] == "transcript_name") tname = m[2];
 		}
 		if (type == "" && biotype != "") type = biotype;

options.c

@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <limits.h>
 #include "mmpriv.h"
 
 void mm_idxopt_init(mm_idxopt_t *opt)
@@ -8,6 +9,7 @@ void mm_idxopt_init(mm_idxopt_t *opt)
 	opt->bucket_bits = 14;
 	opt->mini_batch_size = 50000000;
 	opt->batch_size = 4000000000ULL;
+	opt->min_occ = 0, opt->max_occ = INT_MAX;
 }
 
 void mm_mapopt_init(mm_mapopt_t *opt)
@@ -25,6 +27,9 @@ void mm_mapopt_init(mm_mapopt_t *opt)
 	opt->max_chain_skip = 25;
 	opt->max_chain_iter = 5000;
 
+	opt->flt_max_dv = 1.0f;
+	opt->flt_min_blen = 0;
+
 	opt->mask_level = 0.5f;
 	opt->pri_ratio = 0.8f;
 	opt->best_n = 5;

python/mappy.pyx

@@ -113,6 +113,7 @@ cdef class Aligner:
 	cdef cmappy.mm_mapopt_t map_opt
 
 	def __cinit__(self, fn_idx_in=None, preset=None, k=None, w=None, min_cnt=None, min_chain_score=None, min_dp_score=None, bw=None, best_n=None, n_threads=3, fn_idx_out=None, max_frag_len=None, extra_flags=None, seq=None, scoring=None):
+		self._idx = NULL
 		cmappy.mm_set_opt(NULL, &self.idx_opt, &self.map_opt) # set the default options
 		if preset is not None:
 			cmappy.mm_set_opt(str.encode(preset), &self.idx_opt, &self.map_opt) # apply preset
@@ -170,6 +171,7 @@ cdef class Aligner:
 		cdef void *km
 		cdef cmappy.mm_mapopt_t map_opt
 
+		if self._idx == NULL: return
 		map_opt = self.map_opt
 		if max_frag_len is not None: map_opt.max_frag_len = max_frag_len
 		if extra_flags is not None: map_opt.flag |= extra_flags
@@ -186,27 +188,36 @@ cdef class Aligner:
 			_seq2 = seq2 if isinstance(seq2, bytes) else seq2.encode()
 			regs = cmappy.mm_map_aux(self._idx, _seq, _seq2, &n_regs, b._b, &map_opt)
 
-		for i in range(n_regs):
+		try:
-			cmappy.mm_reg2hitpy(self._idx, &regs[i], &h)
+			i = 0
-			cigar, _cs, _MD = [], '', ''
+			while i < n_regs:
-			for k in range(h.n_cigar32): # convert the 32-bit CIGAR encoding to Python array
+				cmappy.mm_reg2hitpy(self._idx, &regs[i], &h)
-				c = h.cigar32[k]
+				cigar, _cs, _MD = [], '', ''
-				cigar.append([c>>4, c&0xf])
+				for k in range(h.n_cigar32): # convert the 32-bit CIGAR encoding to Python array
-			if cs or MD: # generate the cs and/or the MD tag, if requested
+					c = h.cigar32[k]
-				if cs:
+					cigar.append([c>>4, c&0xf])
-					l_cs_str = cmappy.mm_gen_cs(km, &cs_str, &m_cs_str, self._idx, &regs[i], _seq, 1)
+				if cs or MD: # generate the cs and/or the MD tag, if requested
-					_cs = cs_str[:l_cs_str] if isinstance(cs_str, str) else cs_str[:l_cs_str].decode()
+					if cs:
-				if MD:
+						l_cs_str = cmappy.mm_gen_cs(km, &cs_str, &m_cs_str, self._idx, &regs[i], _seq, 1)
-					l_cs_str = cmappy.mm_gen_MD(km, &cs_str, &m_cs_str, self._idx, &regs[i], _seq)
+						_cs = cs_str[:l_cs_str] if isinstance(cs_str, str) else cs_str[:l_cs_str].decode()
-					_MD = cs_str[:l_cs_str] if isinstance(cs_str, str) else cs_str[:l_cs_str].decode()
+					if MD:
-			yield Alignment(h.ctg, h.ctg_len, h.ctg_start, h.ctg_end, h.strand, h.qry_start, h.qry_end, h.mapq, cigar, h.is_primary, h.mlen, h.blen, h.NM, h.trans_strand, h.seg_id, _cs, _MD)
+						l_cs_str = cmappy.mm_gen_MD(km, &cs_str, &m_cs_str, self._idx, &regs[i], _seq)
-			cmappy.mm_free_reg1(&regs[i])
+						_MD = cs_str[:l_cs_str] if isinstance(cs_str, str) else cs_str[:l_cs_str].decode()
-		free(regs)
+				yield Alignment(h.ctg, h.ctg_len, h.ctg_start, h.ctg_end, h.strand, h.qry_start, h.qry_end, h.mapq, cigar, h.is_primary, h.mlen, h.blen, h.NM, h.trans_strand, h.seg_id, _cs, _MD)
-		free(cs_str)
+				cmappy.mm_free_reg1(&regs[i])
+				i += 1
+		finally:
+			while i < n_regs:
+				cmappy.mm_free_reg1(&regs[i])
+				i += 1
+			free(regs)
+			free(cs_str)
 
 	def seq(self, str name, int start=0, int end=0x7fffffff):
 		cdef int l
-		cdef char *s = cmappy.mappy_fetch_seq(self._idx, name.encode(), start, end, &l)
+		cdef char *s
+		if self._idx == NULL: return
+		s = cmappy.mappy_fetch_seq(self._idx, name.encode(), start, end, &l)
 		if l == 0: return None
 		r = s[:l] if isinstance(s, str) else s[:l].decode()
 		free(s)
@@ -224,6 +235,7 @@ cdef class Aligner:
 	@property
 	def seq_names(self):
 		cdef char *p
+		if self._idx == NULL: return
 		sn = []
 		for i in range(self._idx.n_seq):
 			p = self._idx.seq[i].name

splitidx.c

@@ -2,6 +2,7 @@
 #include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <errno.h>
 #include "mmpriv.h"
 
 FILE *mm_split_init(const char *prefix, const mm_idx_t *mi)
@@ -13,15 +14,15 @@ FILE *mm_split_init(const char *prefix, const mm_idx_t *mi)
 	sprintf(fn, "%s.%.4d.tmp", prefix, mi->index);
 	if ((fp = fopen(fn, "wb")) == NULL) {
 		if (mm_verbose >= 1)
-			fprintf(stderr, "[ERROR]\033[1;31m failed to write to temporary file '%s'\033[0m\n", fn);
+			fprintf(stderr, "[ERROR]\033[1;31m failed to write to temporary file '%s'\033[0m: %s\n", fn, strerror(errno));
 		exit(1);
 	}
 	mm_err_fwrite(&k, 4, 1, fp);
 	mm_err_fwrite(&mi->n_seq, 4, 1, fp);
 	for (i = 0; i < mi->n_seq; ++i) {
-		uint8_t l;
+		uint32_t l;
 		l = strlen(mi->seq[i].name);
-		mm_err_fwrite(&l, 1, 1, fp);
+		mm_err_fwrite(&l, 1, 4, fp);
 		mm_err_fwrite(mi->seq[i].name, 1, l, fp);
 		mm_err_fwrite(&mi->seq[i].len, 4, 1, fp);
 	}
@@ -41,7 +42,7 @@ mm_idx_t *mm_split_merge_prep(const char *prefix, int n_splits, FILE **fp, uint3
 		sprintf(fn, "%s.%.4d.tmp", prefix, i);
 		if ((fp[i] = fopen(fn, "rb")) == 0) {
 			if (mm_verbose >= 1)
-				fprintf(stderr, "ERROR: failed to open temporary file '%s'\n", fn);
+				fprintf(stderr, "ERROR: failed to open temporary file '%s': %s\n", fn, strerror(errno));
 			for (j = 0; j < i; ++j)
 				fclose(fp[j]);
 			free(fn);
@@ -60,8 +61,8 @@ mm_idx_t *mm_split_merge_prep(const char *prefix, int n_splits, FILE **fp, uint3
 	for (i = j = 0; i < n_splits; ++i) {
 		uint32_t k;
 		for (k = 0; k < n_seq_part[i]; ++k, ++j) {
-			uint8_t l;
+			uint32_t l;
-			mm_err_fread(&l, 1, 1, fp[i]);
+			mm_err_fread(&l, 1, 4, fp[i]);
 			mi->seq[j].name = (char*)calloc(l + 1, 1);
 			mm_err_fread(mi->seq[j].name, 1, l, fp[i]);
 			mm_err_fread(&mi->seq[j].len, 4, 1, fp[i]);