From 371bc9513a543951aab114e2cd1cbd89bd5e985e Mon Sep 17 00:00:00 2001 From: John Marshall Date: Thu, 4 Apr 2019 11:14:07 +0100 Subject: [PATCH 01/13] SAM TLEN should be 0 when either read is unmapped this_rid/this_pos will be copied from r_prev(=r_next)'s values when this read is unmapped (i.e., r is NULL). In this case, we can write RNEXT as '=' but should not calculate TLEN from these placeholder values. Similarly when the mate is unmapped (i.e., r_next is NULL). Fixes #365. --- format.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/format.c b/format.c index 29138d1..6d9e7ab 100644 --- a/format.c +++ b/format.c @@ -460,17 +460,17 @@ void mm_write_sam3(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, int se int tlen = 0; if (this_rid >= 0 && r_next) { if (this_rid == r_next->rid) { - int this_pos5 = r && r->rev? r->re - 1 : this_pos; - int next_pos5 = r_next->rev? r_next->re - 1 : r_next->rs; - tlen = next_pos5 - this_pos5; + if (r) { + int this_pos5 = r->rev? r->re - 1 : this_pos; + int next_pos5 = r_next->rev? r_next->re - 1 : r_next->rs; + tlen = next_pos5 - this_pos5; + } mm_sprintf_lite(s, "\t=\t"); } else mm_sprintf_lite(s, "\t%s\t", mi->seq[r_next->rid].name); mm_sprintf_lite(s, "%d\t", r_next->rs + 1); } else if (r_next) { // && this_rid < 0 mm_sprintf_lite(s, "\t%s\t%d\t", mi->seq[r_next->rid].name, r_next->rs + 1); } else if (this_rid >= 0) { // && r_next == NULL - int this_pos5 = this_rev? r->re - 1 : this_pos; // this_rev is only true when r != NULL - tlen = this_pos - this_pos5; // next_pos5 will be this_pos mm_sprintf_lite(s, "\t=\t%d\t", this_pos + 1); // next segment will take r's coordinate } else mm_sprintf_lite(s, "\t*\t0\t"); // neither has coordinates if (tlen > 0) ++tlen; From 6420acca6d898123b5e86b0f56e54d9a1a94953b Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sat, 27 Apr 2019 21:50:02 -0400 Subject: [PATCH 02/13] BED I/O --- index.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ kseq.h | 10 ++++- main.c | 6 ++- minimap.h | 4 ++ 4 files changed, 125 insertions(+), 2 deletions(-) diff --git a/index.c b/index.c index 05420d8..d842ec4 100644 --- a/index.c +++ b/index.c @@ -585,3 +585,110 @@ int mm_idx_reader_eof(const mm_idx_reader_t *r) // TODO: in extremely rare cases { return r->is_idx? (feof(r->fp.idx) || ftell(r->fp.idx) == r->idx_size) : mm_bseq_eof(r->fp.seq); } + +#include +#include +#include "ksort.h" +#include "kseq.h" +KSTREAM_DECLARE(gzFile, gzread) + +typedef struct { + int32_t st, en, max; // max is not used for now + int32_t score:30, strand:2; +} mm_idx_intv1_t; + +typedef struct mm_idx_intv_s { + int32_t n, m; + mm_idx_intv1_t *a; +} mm_idx_intv_t; + +#define sort_key_bed(a) ((a).st) +KRADIX_SORT_INIT(bed, mm_idx_intv1_t, sort_key_bed, 4) + +mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn) +{ + gzFile fp; + kstream_t *ks; + kstring_t str = {0,0,0}; + mm_idx_intv_t *I; + + fp = fn && strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r"); + if (fp == 0) return 0; + I = (mm_idx_intv_t*)calloc(mi->n_seq, sizeof(*I)); + ks = ks_init(fp); + while (ks_getuntil(ks, KS_SEP_LINE, &str, 0) >= 0) { + mm_idx_intv_t *r; + mm_idx_intv1_t t = {-1,-1,-1,-1,0}; + char *p, *q; + int32_t i, id = -1; + for (p = q = str.s, i = 0;; ++p) { + if (*p == 0 || isspace(*p)) { + int32_t c = *p; + *p = 0; + if (i == 0) { // chr + id = mm_idx_name2id(mi, q); + if (id < 0) break; // unknown name; TODO: throw a warning + } else if (i == 1) { // start + t.st = atol(q); // TODO: watch out integer overflow! + if (t.st < 0) break; + } else if (i == 2) { // end + t.en = atol(q); + if (t.en < 0) break; + } else if (i == 3) { // name; do nothing + } else if (i == 4) { // BED score + t.score = atol(q); + } else if (i == 5) { // strand + t.strand = *q == '+'? 1 : *q == '-'? -1 : 0; + } else break; + if (c == 0) break; + ++i, q = p + 1; + } + } + if (id < 0 || t.st < 0 || t.st >= t.en) continue; + r = &I[id]; + if (r->n == r->m) { + r->m = r->m? r->m + (r->m>>1) : 16; + r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a)); + } + r->a[r->n++] = t; + } + ks_destroy(ks); + gzclose(fp); + return I; +} + +int mm_idx_bed_read(mm_idx_t *mi, const char *fn) +{ + int32_t i; + if (mi->h == 0) mm_idx_index_name(mi); + mi->I = mm_idx_read_bed(mi, fn); + if (mi->I == 0) return -1; + for (i = 0; i < mi->n_seq; ++i) + radix_sort_bed(mi->I[i].a, mi->I[i].a + mi->I[i].n); + return mi->I? 0 : -1; +} + +int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, int32_t strand, int8_t *s) +{ + int32_t i, left, right; + mm_idx_intv_t *r; + memset(s, 0, en - st); + if (mi->I == 0 || ctg < 0 || ctg >= mi->n_seq) return -1; + r = &mi->I[ctg]; + left = 0, right = r->n; + while (right > left) { + int32_t mid = left + ((right - left) >> 1); + if (r->a[mid].st >= st) right = mid; + else left = mid + 1; + } + for (i = left; i < r->n; ++i) { + if (st <= r->a[i].st && en >= r->a[i].en && r->a[i].strand != 0 && strand * r->a[i].strand >= 0) { + if (r->a[i].strand > 0) { + s[r->a[i].st] |= 1, s[r->a[i].en - 1] |= 2; + } else { + s[r->a[i].st] |= 2, s[r->a[i].en - 1] |= 1; + } + } + } + return left; +} diff --git a/kseq.h b/kseq.h index d301ddc..8021e56 100644 --- a/kseq.h +++ b/kseq.h @@ -37,6 +37,14 @@ #define KS_SEP_LINE 2 // line separator: "\n" (Unix) or "\r\n" (Windows) #define KS_SEP_MAX 2 +#ifndef klib_unused +#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3) +#define klib_unused __attribute__ ((__unused__)) +#else +#define klib_unused +#endif +#endif /* klib_unused */ + #define __KS_TYPE(type_t) \ typedef struct __kstream_t { \ int begin, end; \ @@ -64,7 +72,7 @@ } #define __KS_INLINED(__read) \ - static inline int ks_getc(kstream_t *ks) \ + static inline klib_unused int ks_getc(kstream_t *ks) \ { \ if (ks->is_eof && ks->begin >= ks->end) return -1; \ if (ks->begin >= ks->end) { \ diff --git a/main.c b/main.c index aa0cf91..f7b529b 100644 --- a/main.c +++ b/main.c @@ -63,6 +63,8 @@ static ko_longopt_t long_options[] = { { "cap-sw-mem", ko_required_argument, 337 }, { "max-qlen", ko_required_argument, 338 }, { "max-chain-iter", ko_required_argument, 339 }, + { "junc-bed", ko_required_argument, 340 }, + { "junc-bonus", ko_required_argument, 341 }, { "help", ko_no_argument, 'h' }, { "max-intron-len", ko_required_argument, 'G' }, { "version", ko_no_argument, 'V' }, @@ -105,7 +107,7 @@ int main(int argc, char *argv[]) mm_mapopt_t opt; mm_idxopt_t ipt; int i, c, n_threads = 3, n_parts, old_best_n = -1; - char *fnw = 0, *rg = 0, *s; + char *fnw = 0, *rg = 0, *junc_bed = 0, *s; FILE *fp_help = stderr; mm_idx_reader_t *idx_rdr; mm_idx_t *mi; @@ -204,6 +206,7 @@ int main(int argc, char *argv[]) else if (c == 336) opt.flag |= MM_F_HARD_MLEVEL; // --hard-mask-level else if (c == 337) opt.max_sw_mat = mm_parse_num(o.arg); // --cap-sw-mat else if (c == 338) opt.max_qlen = mm_parse_num(o.arg); // --max-qlen + else if (c == 339) junc_bed = o.arg; // --junc-bed else if (c == 314) { // --frag yes_or_no(&opt, MM_F_FRAG_MODE, o.longidx, o.arg, 1); } else if (c == 315) { // --secondary @@ -343,6 +346,7 @@ int main(int argc, char *argv[]) if (opt.best_n == 0 && (opt.flag&MM_F_CIGAR) && mm_verbose >= 2) fprintf(stderr, "[WARNING]\033[1;31m `-N 0' reduces alignment accuracy. Please use --secondary=no to suppress secondary alignments.\033[0m\n"); while ((mi = mm_idx_reader_read(idx_rdr, n_threads)) != 0) { + if (junc_bed) mm_idx_bed_read(mi, junc_bed); if ((opt.flag & MM_F_CIGAR) && (mi->flag & MM_I_NO_SEQ)) { fprintf(stderr, "[ERROR] the prebuilt index doesn't contain sequences.\n"); mm_idx_destroy(mi); diff --git a/minimap.h b/minimap.h index 176342b..947efe1 100644 --- a/minimap.h +++ b/minimap.h @@ -66,6 +66,7 @@ typedef struct { mm_idx_seq_t *seq; // sequence name, length and offset uint32_t *S; // 4-bit packed sequence struct mm_idx_bucket_s *B; // index (hidden) + struct mm_idx_intv_s *I; // intervals (hidden) void *km, *h; } mm_idx_t; @@ -365,6 +366,9 @@ int mm_idx_index_name(mm_idx_t *mi); int mm_idx_name2id(const mm_idx_t *mi, const char *name); int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq); +int mm_idx_bed_read(mm_idx_t *mi, const char *fn); +int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, int32_t strand, int8_t *s); + // deprecated APIs for backward compatibility void mm_mapopt_init(mm_mapopt_t *opt); mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads); From cdc730d5736e12fbfb577b72efbc14a29a715a5c Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sat, 27 Apr 2019 22:15:40 -0400 Subject: [PATCH 03/13] gff2bed to output junction BED --- misc/paftools.js | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/misc/paftools.js b/misc/paftools.js index 459eebd..e92114c 100755 --- a/misc/paftools.js +++ b/misc/paftools.js @@ -1,6 +1,6 @@ #!/usr/bin/env k8 -var paftools_version = '2.16-r922'; +var paftools_version = '2.16-dirty-r928'; /***************************** ***** Library functions ***** @@ -1469,15 +1469,21 @@ function paf_view(args) function paf_gff2bed(args) { - var c, fn_ucsc_fai = null, is_short = false, keep_gff = false; - while ((c = getopt(args, "u:sg")) != null) { + var c, fn_ucsc_fai = null, is_short = false, keep_gff = false, print_junc = false; + while ((c = getopt(args, "u:sgj")) != null) { if (c == 'u') fn_ucsc_fai = getopt.arg; else if (c == 's') is_short = true; else if (c == 'g') keep_gff = true; + else if (c == 'j') print_junc = true; } if (getopt.ind == args.length) { - print("Usage: paftools.js gff2bed [-g] [-u ucsc-genome.fa.fai] "); + print("Usage: paftools.js gff2bed [options] "); + print("Options:"); + print(" -j Output junction BED"); + print(" -s Print names in the short form"); + print(" -u FILE hg38.fa.fai for chr name conversion"); + print(" -g Output GFF (used with -u)"); exit(1); } @@ -1509,11 +1515,16 @@ function paf_gff2bed(args) 'misc_RNA':'0,192,0' }; - function print_bed12(exons, cds_st, cds_en, is_short) + function print_bed12(exons, cds_st, cds_en, is_short, print_junc) { if (exons.length == 0) return; var name = is_short? exons[0][7] + "|" + exons[0][5] : exons[0].slice(4, 7).join("|"); var a = exons.sort(function(a,b) {return a[1]-b[1]}); + if (print_junc) { + for (var i = 1; i < a.length; ++i) + print(a[i][0], a[i-1][2], a[i][1], name, 1000, a[i][3]); + return; + } var sizes = [], starts = [], st, en; st = a[0][1]; en = a[a.length - 1][2]; @@ -1566,7 +1577,7 @@ function paf_gff2bed(args) if (type == "" && biotype != "") type = biotype; if (id == null) throw Error("No transcript_id"); if (id != last_id) { - print_bed12(exons, cds_st, cds_en, is_short); + print_bed12(exons, cds_st, cds_en, is_short, print_junc); exons = [], cds_st = 1<<30, cds_en = 0; last_id = id; } @@ -1584,7 +1595,7 @@ function paf_gff2bed(args) } } if (last_id != null) - print_bed12(exons, cds_st, cds_en, is_short); + print_bed12(exons, cds_st, cds_en, is_short, print_junc); file.close(); buf.destroy(); From be171aa2dc22cc287725df2d74a94e202417c944 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sat, 27 Apr 2019 22:39:26 -0400 Subject: [PATCH 04/13] implemented in exts; testing is the next --- align.c | 22 ++++++++++++++-------- index.c | 4 ++-- ksw2.h | 2 +- ksw2_dispatch.c | 10 +++++----- ksw2_exts2_sse.c | 14 +++++++++++--- minimap.h | 3 ++- options.c | 1 + python/cmappy.pxd | 1 + 8 files changed, 37 insertions(+), 20 deletions(-) diff --git a/align.c b/align.c index c3b04d3..60bc08a 100644 --- a/align.c +++ b/align.c @@ -291,7 +291,7 @@ static void mm_append_cigar(mm_reg1_t *r, uint32_t n_cigar, uint32_t *cigar) // } } -static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint8_t *qseq, int tlen, const uint8_t *tseq, const int8_t *mat, int w, int end_bonus, int zdrop, int flag, ksw_extz_t *ez) +static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint8_t *qseq, int tlen, const uint8_t *tseq, const uint8_t *junc, const int8_t *mat, int w, int end_bonus, int zdrop, int flag, ksw_extz_t *ez) { if (mm_dbg_flag & MM_DBG_PRINT_ALN_SEQ) { int i; @@ -305,7 +305,7 @@ static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint ksw_reset_extz(ez); ez->zdropped = 1; } else if (opt->flag & MM_F_SPLICE) - ksw_exts2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->noncan, zdrop, flag, ez); + ksw_exts2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->noncan, zdrop, opt->junc_bonus, flag, junc, ez); else if (opt->q == opt->q2 && opt->e == opt->e2) ksw_extz2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, w, zdrop, end_bonus, flag, ez); else @@ -547,7 +547,7 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int { int is_sr = !!(opt->flag & MM_F_SR), is_splice = !!(opt->flag & MM_F_SPLICE); int32_t rid = a[r->as].x<<1>>33, rev = a[r->as].x>>63, as1, cnt1; - uint8_t *tseq, *qseq; + uint8_t *tseq, *qseq, *junc; int32_t i, l, bw, dropped = 0, extra_flag = 0, rs0, re0, qs0, qe0; int32_t rs, re, qs, qe; int32_t rs1, qs1, re1, qe1; @@ -666,13 +666,16 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int assert(re0 > rs0); tseq = (uint8_t*)kmalloc(km, re0 - rs0); + junc = (uint8_t*)kmalloc(km, re0 - rs0); if (qs > 0 && rs > 0) { // left extension; probably the condition can be changed to "qs > qs0 && rs > rs0" qseq = &qseq0[rev][qs0]; mm_idx_getseq(mi, rid, rs0, rs, tseq); + mm_idx_bed_junc(mi, rid, rs0, rs, junc); mm_seq_rev(qs - qs0, qseq); mm_seq_rev(rs - rs0, tseq); - mm_align_pair(km, opt, qs - qs0, qseq, rs - rs0, tseq, mat, bw, opt->end_bonus, r->split_inv? opt->zdrop_inv : opt->zdrop, extra_flag|KSW_EZ_EXTZ_ONLY|KSW_EZ_RIGHT|KSW_EZ_REV_CIGAR, ez); + mm_seq_rev(rs - rs0, junc); + mm_align_pair(km, opt, qs - qs0, qseq, rs - rs0, tseq, junc, mat, bw, opt->end_bonus, r->split_inv? opt->zdrop_inv : opt->zdrop, extra_flag|KSW_EZ_EXTZ_ONLY|KSW_EZ_RIGHT|KSW_EZ_REV_CIGAR, ez); if (ez->n_cigar > 0) { mm_append_cigar(r, ez->n_cigar, ez->cigar); r->p->dp_score += ez->max; @@ -698,6 +701,7 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int // perform alignment qseq = &qseq0[rev][qs]; mm_idx_getseq(mi, rid, rs, re, tseq); + mm_idx_bed_junc(mi, rid, rs, re, junc); if (is_sr) { // perform ungapped alignment assert(qe - qs == re - rs); ksw_reset_extz(ez); @@ -707,11 +711,11 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int } ez->cigar = ksw_push_cigar(km, &ez->n_cigar, &ez->m_cigar, ez->cigar, 0, qe - qs); } else { // perform normal gapped alignment - mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, -1, opt->zdrop, extra_flag|KSW_EZ_APPROX_MAX, ez); // first pass: with approximate Z-drop + mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, junc, mat, bw1, -1, opt->zdrop, extra_flag|KSW_EZ_APPROX_MAX, ez); // first pass: with approximate Z-drop } // test Z-drop and inversion Z-drop if ((zdrop_code = mm_test_zdrop(km, opt, qseq, tseq, ez->n_cigar, ez->cigar, mat)) != 0) - mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, mat, bw1, -1, zdrop_code == 2? opt->zdrop_inv : opt->zdrop, extra_flag, ez); // second pass: lift approximate + mm_align_pair(km, opt, qe - qs, qseq, re - rs, tseq, junc, mat, bw1, -1, zdrop_code == 2? opt->zdrop_inv : opt->zdrop, extra_flag, ez); // second pass: lift approximate // update CIGAR if (ez->n_cigar > 0) mm_append_cigar(r, ez->n_cigar, ez->cigar); @@ -737,7 +741,8 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int if (!dropped && qe < qe0 && re < re0) { // right extension qseq = &qseq0[rev][qe]; mm_idx_getseq(mi, rid, re, re0, tseq); - mm_align_pair(km, opt, qe0 - qe, qseq, re0 - re, tseq, mat, bw, opt->end_bonus, opt->zdrop, extra_flag|KSW_EZ_EXTZ_ONLY, ez); + mm_idx_bed_junc(mi, rid, re, re0, junc); + mm_align_pair(km, opt, qe0 - qe, qseq, re0 - re, tseq, junc, mat, bw, opt->end_bonus, opt->zdrop, extra_flag|KSW_EZ_EXTZ_ONLY, ez); if (ez->n_cigar > 0) { mm_append_cigar(r, ez->n_cigar, ez->cigar); r->p->dp_score += ez->max; @@ -760,6 +765,7 @@ static void mm_align1(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int } kfree(km, tseq); + kfree(km, junc); } static int mm_align1_inv(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, int qlen, uint8_t *qseq0[2], const mm_reg1_t *r1, const mm_reg1_t *r2, mm_reg1_t *r_inv, ksw_extz_t *ez) @@ -793,7 +799,7 @@ static int mm_align1_inv(void *km, const mm_mapopt_t *opt, const mm_idx_t *mi, i mm_seq_rev(tl, tseq); if (score < opt->min_dp_max) goto end_align1_inv; q_off = ql - (q_off + 1), t_off = tl - (t_off + 1); - mm_align_pair(km, opt, ql - q_off, qseq + q_off, tl - t_off, tseq + t_off, mat, (int)(opt->bw * 1.5), -1, opt->zdrop, KSW_EZ_EXTZ_ONLY, ez); + mm_align_pair(km, opt, ql - q_off, qseq + q_off, tl - t_off, tseq + t_off, 0, mat, (int)(opt->bw * 1.5), -1, opt->zdrop, KSW_EZ_EXTZ_ONLY, ez); if (ez->n_cigar == 0) goto end_align1_inv; // should never be here mm_append_cigar(r_inv, ez->n_cigar, ez->cigar); r_inv->p->dp_score = ez->max; diff --git a/index.c b/index.c index d842ec4..4b92fe6 100644 --- a/index.c +++ b/index.c @@ -668,7 +668,7 @@ int mm_idx_bed_read(mm_idx_t *mi, const char *fn) return mi->I? 0 : -1; } -int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, int32_t strand, int8_t *s) +int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uint8_t *s) { int32_t i, left, right; mm_idx_intv_t *r; @@ -682,7 +682,7 @@ int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, int else left = mid + 1; } for (i = left; i < r->n; ++i) { - if (st <= r->a[i].st && en >= r->a[i].en && r->a[i].strand != 0 && strand * r->a[i].strand >= 0) { + if (st <= r->a[i].st && en >= r->a[i].en && r->a[i].strand != 0) { if (r->a[i].strand > 0) { s[r->a[i].st] |= 1, s[r->a[i].en - 1] |= 2; } else { diff --git a/ksw2.h b/ksw2.h index 213c27f..c700136 100644 --- a/ksw2.h +++ b/ksw2.h @@ -61,7 +61,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez); void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t gapo, int8_t gape, int8_t gapo2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez); + int8_t gapo, int8_t gape, int8_t gapo2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez); void ksw_extf2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t mch, int8_t mis, int8_t e, int w, int xdrop, ksw_extz_t *ez); diff --git a/ksw2_dispatch.c b/ksw2_dispatch.c index 22ca9bc..792b267 100644 --- a/ksw2_dispatch.c +++ b/ksw2_dispatch.c @@ -80,17 +80,17 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin } void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez) + int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez) { extern void ksw_exts2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez); + int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez); extern void ksw_exts2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez); + int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez); if (ksw_simd < 0) ksw_simd = x86_simd(); if (ksw_simd & SIMD_SSE4_1) - ksw_exts2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, flag, ez); + ksw_exts2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, junc_bonus, flag, junc, ez); else if (ksw_simd & SIMD_SSE2) - ksw_exts2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, flag, ez); + ksw_exts2_sse2(km, qlen, query, tlen, target, m, mat, q, e, q2, noncan, zdrop, junc_bonus, flag, junc, ez); else abort(); } #endif diff --git a/ksw2_exts2_sse.c b/ksw2_exts2_sse.c index 308f491..f025838 100644 --- a/ksw2_exts2_sse.c +++ b/ksw2_exts2_sse.c @@ -17,14 +17,14 @@ #ifdef KSW_CPU_DISPATCH #ifdef __SSE4_1__ void ksw_exts2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez) + int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez) #else void ksw_exts2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez) + int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez) #endif #else void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int flag, ksw_extz_t *ez) + int8_t q, int8_t e, int8_t q2, int8_t noncan, int zdrop, int8_t junc_bonus, int flag, const uint8_t *junc, ksw_extz_t *ez) #endif // ~KSW_CPU_DISPATCH { #define __dp_code_block1 \ @@ -120,6 +120,10 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin if (can_type && (target[t+3] == 0 || target[t+3] == 2)) can_type = 2; if (can_type) ((int8_t*)donor)[t] = can_type == 2? 0 : semi_cost; } + if (junc) + for (t = 0; t < tlen - 1; ++t) + if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t+1]&1)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t+1]&2))) + ((int8_t*)donor)[t] += junc_bonus; memset(acceptor, -noncan, tlen_ * 16); for (t = 2; t < tlen; ++t) { int can_type = 0; @@ -128,6 +132,10 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin if (can_type && (target[t-2] == 1 || target[t-2] == 3)) can_type = 2; if (can_type) ((int8_t*)acceptor)[t] = can_type == 2? 0 : semi_cost; } + if (junc) + for (t = 0; t < tlen; ++t) + if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t]&2)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t]&1))) + ((int8_t*)acceptor)[t] += junc_bonus; } for (r = 0, last_st = last_en = -1; r < qlen + tlen - 1; ++r) { diff --git a/minimap.h b/minimap.h index 947efe1..358cad7 100644 --- a/minimap.h +++ b/minimap.h @@ -128,6 +128,7 @@ typedef struct { int a, b, q, e, q2, e2; // matching score, mismatch, gap-open and gap-ext penalties int sc_ambi; // score when one or both bases are "N" int noncan; // cost of non-canonical splicing sites + int junc_bonus; int zdrop, zdrop_inv; // break alignment if alignment score drops too fast along the diagonal int end_bonus; int min_dp_max; // drop an alignment if the score of the max scoring segment is below this threshold @@ -367,7 +368,7 @@ int mm_idx_name2id(const mm_idx_t *mi, const char *name); int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq); int mm_idx_bed_read(mm_idx_t *mi, const char *fn); -int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, int32_t strand, int8_t *s); +int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uint8_t *s); // deprecated APIs for backward compatibility void mm_mapopt_init(mm_mapopt_t *opt); diff --git a/options.c b/options.c index eac73e3..ac1e5e1 100644 --- a/options.c +++ b/options.c @@ -126,6 +126,7 @@ int mm_set_opt(const char *preset, mm_idxopt_t *io, mm_mapopt_t *mo) mo->max_gap = 2000, mo->max_gap_ref = mo->bw = 200000; mo->a = 1, mo->b = 2, mo->q = 2, mo->e = 1, mo->q2 = 32, mo->e2 = 0; mo->noncan = 9; + mo->junc_bonus = 9; mo->zdrop = 200, mo->zdrop_inv = 100; // because mo->a is halved } else return -1; return 0; diff --git a/python/cmappy.pxd b/python/cmappy.pxd index 1bbe749..ba1871d 100644 --- a/python/cmappy.pxd +++ b/python/cmappy.pxd @@ -29,6 +29,7 @@ cdef extern from "minimap.h": int a, b, q, e, q2, e2 int sc_ambi int noncan + int junc_bonus int zdrop, zdrop_inv int end_bonus int min_dp_max From f4c844b1436e3b290be4e0a03aa8d1aa82fbae4b Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sun, 28 Apr 2019 14:52:47 -0400 Subject: [PATCH 05/13] fixed a few simple bugs and leaks --- index.c | 33 ++++++++++++++++++++------------- main.c | 4 ++-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/index.c b/index.c index 4b92fe6..533394c 100644 --- a/index.c +++ b/index.c @@ -31,6 +31,16 @@ typedef struct mm_idx_bucket_s { void *h; // hash table indexing _p_ and minimizers appearing once } mm_idx_bucket_t; +typedef struct { + int32_t st, en, max; // max is not used for now + int32_t score:30, strand:2; +} mm_idx_intv1_t; + +typedef struct mm_idx_intv_s { + int32_t n, m; + mm_idx_intv1_t *a; +} mm_idx_intv_t; + mm_idx_t *mm_idx_init(int w, int k, int b, int flag) { mm_idx_t *mi; @@ -55,6 +65,11 @@ void mm_idx_destroy(mm_idx_t *mi) kh_destroy(idx, (idxhash_t*)mi->B[i].h); } } + if (mi->I) { + for (i = 0; i < mi->n_seq; ++i) + free(mi->I[i].a); + free(mi->I); + } if (!mi->km) { for (i = 0; i < mi->n_seq; ++i) free(mi->seq[i].name); @@ -592,16 +607,6 @@ int mm_idx_reader_eof(const mm_idx_reader_t *r) // TODO: in extremely rare cases #include "kseq.h" KSTREAM_DECLARE(gzFile, gzread) -typedef struct { - int32_t st, en, max; // max is not used for now - int32_t score:30, strand:2; -} mm_idx_intv1_t; - -typedef struct mm_idx_intv_s { - int32_t n, m; - mm_idx_intv1_t *a; -} mm_idx_intv_t; - #define sort_key_bed(a) ((a).st) KRADIX_SORT_INIT(bed, mm_idx_intv1_t, sort_key_bed, 4) @@ -648,10 +653,11 @@ mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn) r = &I[id]; if (r->n == r->m) { r->m = r->m? r->m + (r->m>>1) : 16; - r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a)); + r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a) * r->m); } r->a[r->n++] = t; } + free(str.s); ks_destroy(ks); gzclose(fp); return I; @@ -663,9 +669,9 @@ int mm_idx_bed_read(mm_idx_t *mi, const char *fn) if (mi->h == 0) mm_idx_index_name(mi); mi->I = mm_idx_read_bed(mi, fn); if (mi->I == 0) return -1; - for (i = 0; i < mi->n_seq; ++i) + for (i = 0; i < mi->n_seq; ++i) // TODO: eliminate redundant intervals radix_sort_bed(mi->I[i].a, mi->I[i].a + mi->I[i].n); - return mi->I? 0 : -1; + return 0; } int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uint8_t *s) @@ -683,6 +689,7 @@ int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uin } for (i = left; i < r->n; ++i) { if (st <= r->a[i].st && en >= r->a[i].en && r->a[i].strand != 0) { + //fprintf(stderr, "[2] %d\t%d\t%c\n", r->a[i].st, r->a[i].en, r->a[i].strand > 0? '+' : r->a[i].strand < 0? '-' : '.'); if (r->a[i].strand > 0) { s[r->a[i].st] |= 1, s[r->a[i].en - 1] |= 2; } else { diff --git a/main.c b/main.c index f7b529b..e2e7837 100644 --- a/main.c +++ b/main.c @@ -206,7 +206,7 @@ int main(int argc, char *argv[]) else if (c == 336) opt.flag |= MM_F_HARD_MLEVEL; // --hard-mask-level else if (c == 337) opt.max_sw_mat = mm_parse_num(o.arg); // --cap-sw-mat else if (c == 338) opt.max_qlen = mm_parse_num(o.arg); // --max-qlen - else if (c == 339) junc_bed = o.arg; // --junc-bed + else if (c == 340) junc_bed = o.arg; // --junc-bed else if (c == 314) { // --frag yes_or_no(&opt, MM_F_FRAG_MODE, o.longidx, o.arg, 1); } else if (c == 315) { // --secondary @@ -346,7 +346,6 @@ int main(int argc, char *argv[]) if (opt.best_n == 0 && (opt.flag&MM_F_CIGAR) && mm_verbose >= 2) fprintf(stderr, "[WARNING]\033[1;31m `-N 0' reduces alignment accuracy. Please use --secondary=no to suppress secondary alignments.\033[0m\n"); while ((mi = mm_idx_reader_read(idx_rdr, n_threads)) != 0) { - if (junc_bed) mm_idx_bed_read(mi, junc_bed); if ((opt.flag & MM_F_CIGAR) && (mi->flag & MM_I_NO_SEQ)) { fprintf(stderr, "[ERROR] the prebuilt index doesn't contain sequences.\n"); mm_idx_destroy(mi); @@ -367,6 +366,7 @@ int main(int argc, char *argv[]) __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), mi->n_seq); if (argc != o.ind + 1) mm_mapopt_update(&opt, mi); if (mm_verbose >= 3) mm_idx_stat(mi); + if (junc_bed) mm_idx_bed_read(mi, junc_bed); if (!(opt.flag & MM_F_FRAG_MODE)) { for (i = o.ind + 1; i < argc; ++i) mm_map_file(mi, argv[i], &opt, n_threads); From e80759c97a205158cbea7b1e04782a8dfb6f79ec Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sun, 28 Apr 2019 16:44:30 -0400 Subject: [PATCH 06/13] --junc-bed apparently working Also fixed an issue with splice alignment in the reverse strand, though this should have a very minor effect in practice. --- index.c | 5 ++-- ksw2_exts2_sse.c | 67 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/index.c b/index.c index 533394c..0ba6fde 100644 --- a/index.c +++ b/index.c @@ -689,11 +689,10 @@ int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uin } for (i = left; i < r->n; ++i) { if (st <= r->a[i].st && en >= r->a[i].en && r->a[i].strand != 0) { - //fprintf(stderr, "[2] %d\t%d\t%c\n", r->a[i].st, r->a[i].en, r->a[i].strand > 0? '+' : r->a[i].strand < 0? '-' : '.'); if (r->a[i].strand > 0) { - s[r->a[i].st] |= 1, s[r->a[i].en - 1] |= 2; + s[r->a[i].st - st] |= 1, s[r->a[i].en - 1 - st] |= 2; } else { - s[r->a[i].st] |= 2, s[r->a[i].en - 1] |= 1; + s[r->a[i].st - st] |= 8, s[r->a[i].en - 1 - st] |= 4; } } } diff --git a/ksw2_exts2_sse.c b/ksw2_exts2_sse.c index f025838..e7984c6 100644 --- a/ksw2_exts2_sse.c +++ b/ksw2_exts2_sse.c @@ -113,29 +113,54 @@ void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin if (flag & (KSW_EZ_SPLICE_FOR|KSW_EZ_SPLICE_REV)) { int semi_cost = flag&KSW_EZ_SPLICE_FLANK? -noncan/2 : 0; // GTr or yAG is worth 0.5 bit; see PMID:18688272 memset(donor, -noncan, tlen_ * 16); - for (t = 0; t < tlen - 4; ++t) { - int can_type = 0; // type of canonical site: 0=none, 1=GT/AG only, 2=GTr/yAG - if ((flag & KSW_EZ_SPLICE_FOR) && target[t+1] == 2 && target[t+2] == 3) can_type = 1; // GTr... - if ((flag & KSW_EZ_SPLICE_REV) && target[t+1] == 1 && target[t+2] == 3) can_type = 1; // CTr... - if (can_type && (target[t+3] == 0 || target[t+3] == 2)) can_type = 2; - if (can_type) ((int8_t*)donor)[t] = can_type == 2? 0 : semi_cost; - } - if (junc) - for (t = 0; t < tlen - 1; ++t) - if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t+1]&1)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t+1]&2))) - ((int8_t*)donor)[t] += junc_bonus; memset(acceptor, -noncan, tlen_ * 16); - for (t = 2; t < tlen; ++t) { - int can_type = 0; - if ((flag & KSW_EZ_SPLICE_FOR) && target[t-1] == 0 && target[t] == 2) can_type = 1; // ...yAG - if ((flag & KSW_EZ_SPLICE_REV) && target[t-1] == 0 && target[t] == 1) can_type = 1; // ...yAC - if (can_type && (target[t-2] == 1 || target[t-2] == 3)) can_type = 2; - if (can_type) ((int8_t*)acceptor)[t] = can_type == 2? 0 : semi_cost; + if (!(flag & KSW_EZ_REV_CIGAR)) { + for (t = 0; t < tlen - 4; ++t) { + int can_type = 0; // type of canonical site: 0=none, 1=GT/AG only, 2=GTr/yAG + if ((flag & KSW_EZ_SPLICE_FOR) && target[t+1] == 2 && target[t+2] == 3) can_type = 1; // GTr... + if ((flag & KSW_EZ_SPLICE_REV) && target[t+1] == 1 && target[t+2] == 3) can_type = 1; // CTr... + if (can_type && (target[t+3] == 0 || target[t+3] == 2)) can_type = 2; + if (can_type) ((int8_t*)donor)[t] = can_type == 2? 0 : semi_cost; + } + if (junc) + for (t = 0; t < tlen - 1; ++t) + if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t+1]&1)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t+1]&8))) + ((int8_t*)donor)[t] += junc_bonus; + for (t = 2; t < tlen; ++t) { + int can_type = 0; + if ((flag & KSW_EZ_SPLICE_FOR) && target[t-1] == 0 && target[t] == 2) can_type = 1; // ...yAG + if ((flag & KSW_EZ_SPLICE_REV) && target[t-1] == 0 && target[t] == 1) can_type = 1; // ...yAC + if (can_type && (target[t-2] == 1 || target[t-2] == 3)) can_type = 2; + if (can_type) ((int8_t*)acceptor)[t] = can_type == 2? 0 : semi_cost; + } + if (junc) + for (t = 0; t < tlen; ++t) + if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t]&2)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t]&4))) + ((int8_t*)acceptor)[t] += junc_bonus; + } else { + for (t = 0; t < tlen - 4; ++t) { + int can_type = 0; // type of canonical site: 0=none, 1=GT/AG only, 2=GTr/yAG + if ((flag & KSW_EZ_SPLICE_FOR) && target[t+1] == 2 && target[t+2] == 0) can_type = 1; // GAy... + if ((flag & KSW_EZ_SPLICE_REV) && target[t+1] == 1 && target[t+2] == 0) can_type = 1; // CAy... + if (can_type && (target[t+3] == 1 || target[t+3] == 3)) can_type = 2; + if (can_type) ((int8_t*)donor)[t] = can_type == 2? 0 : semi_cost; + } + if (junc) + for (t = 0; t < tlen - 1; ++t) + if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t+1]&2)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t+1]&4))) + ((int8_t*)donor)[t] += junc_bonus; + for (t = 2; t < tlen; ++t) { + int can_type = 0; + if ((flag & KSW_EZ_SPLICE_FOR) && target[t-1] == 3 && target[t] == 2) can_type = 1; // ...rTG + if ((flag & KSW_EZ_SPLICE_REV) && target[t-1] == 3 && target[t] == 1) can_type = 1; // ...rTC + if (can_type && (target[t-2] == 0 || target[t-2] == 2)) can_type = 2; + if (can_type) ((int8_t*)acceptor)[t] = can_type == 2? 0 : semi_cost; + } + if (junc) + for (t = 0; t < tlen; ++t) + if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t]&1)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t]&8))) + ((int8_t*)acceptor)[t] += junc_bonus; } - if (junc) - for (t = 0; t < tlen; ++t) - if (((flag & KSW_EZ_SPLICE_FOR) && (junc[t]&2)) || ((flag & KSW_EZ_SPLICE_REV) && (junc[t]&1))) - ((int8_t*)acceptor)[t] += junc_bonus; } for (r = 0, last_st = last_en = -1; r < qlen + tlen - 1; ++r) { From 2bb8cbbeef2aa5902cd9cfd830a4a9c7e3f9a7cb Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sun, 28 Apr 2019 17:02:49 -0400 Subject: [PATCH 07/13] updated manpage --- minimap2.1 | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/minimap2.1 b/minimap2.1 index be1f9ed..12fa80d 100644 --- a/minimap2.1 +++ b/minimap2.1 @@ -364,6 +364,17 @@ on SIRV data, please add .B --splice-flank=no to the command line. .TP +.BR --junc-bed \ FILE +BED file consisting of annotated introns and their strands. With this option, +minimap2 prefers splicing in annotations. +.I FILE +can be generated with `paftools.js gff2bed -j ann.gtf' []. +.TP +.BR --junc-bonus \ INT +Score bonus for a splice donor or acceptor found in annotation (effective with +.BR --junc-bed ) +[0]. +.TP .BI --end-seed-pen \ INT Drop a terminal anchor if .IR s Date: Sun, 28 Apr 2019 17:05:37 -0400 Subject: [PATCH 08/13] r933: resume versioning --- main.c | 2 +- minimap2.1 | 2 +- misc/paftools.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.c b/main.c index e2e7837..bf179c0 100644 --- a/main.c +++ b/main.c @@ -6,7 +6,7 @@ #include "mmpriv.h" #include "ketopt.h" -#define MM_VERSION "2.16-r922" +#define MM_VERSION "2.16-r933-dirty" #ifdef __linux__ #include diff --git a/minimap2.1 b/minimap2.1 index 12fa80d..d13e3ee 100644 --- a/minimap2.1 +++ b/minimap2.1 @@ -1,4 +1,4 @@ -.TH minimap2 1 "28 Feburary 2019" "minimap2-2.16 (r922)" "Bioinformatics tools" +.TH minimap2 1 "28 Feburary 2019" "minimap2-2.16-dirty (r933)" "Bioinformatics tools" .SH NAME .PP minimap2 - mapping and alignment between collections of DNA sequences diff --git a/misc/paftools.js b/misc/paftools.js index e92114c..6e60bc4 100755 --- a/misc/paftools.js +++ b/misc/paftools.js @@ -1,6 +1,6 @@ #!/usr/bin/env k8 -var paftools_version = '2.16-dirty-r928'; +var paftools_version = '2.16-r933-dirty'; /***************************** ***** Library functions ***** From 49c6d83a8ec6cb0f37b7e5c55be70c9377c4eac3 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sun, 28 Apr 2019 20:12:28 -0400 Subject: [PATCH 09/13] r934: --junc-bed to read BED12 --- index.c | 48 +++++++++++++++++++++++++++++++++++++----------- main.c | 4 ++-- minimap.h | 2 +- minimap2.1 | 10 +++++----- 4 files changed, 45 insertions(+), 19 deletions(-) diff --git a/index.c b/index.c index 0ba6fde..164e8e5 100644 --- a/index.c +++ b/index.c @@ -610,7 +610,7 @@ KSTREAM_DECLARE(gzFile, gzread) #define sort_key_bed(a) ((a).st) KRADIX_SORT_INIT(bed, mm_idx_intv1_t, sort_key_bed, 4) -mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn) +mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn, int read_junc) { gzFile fp; kstream_t *ks; @@ -624,8 +624,8 @@ mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn) while (ks_getuntil(ks, KS_SEP_LINE, &str, 0) >= 0) { mm_idx_intv_t *r; mm_idx_intv1_t t = {-1,-1,-1,-1,0}; - char *p, *q; - int32_t i, id = -1; + char *p, *q, *bl, *bs; + int32_t i, id = -1, n_blk = 0; for (p = q = str.s, i = 0;; ++p) { if (*p == 0 || isspace(*p)) { int32_t c = *p; @@ -639,23 +639,49 @@ mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn) } else if (i == 2) { // end t.en = atol(q); if (t.en < 0) break; - } else if (i == 3) { // name; do nothing } else if (i == 4) { // BED score t.score = atol(q); } else if (i == 5) { // strand t.strand = *q == '+'? 1 : *q == '-'? -1 : 0; - } else break; + } else if (i == 9) { + if (!isdigit(*q)) break; + n_blk = atol(q); + } else if (i == 10) { + bl = q; + } else if (i == 11) { + bs = q; + break; + } if (c == 0) break; ++i, q = p + 1; } } if (id < 0 || t.st < 0 || t.st >= t.en) continue; r = &I[id]; - if (r->n == r->m) { - r->m = r->m? r->m + (r->m>>1) : 16; - r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a) * r->m); + if (i >= 11 && read_junc) { // BED12 + int32_t st, sz, en; + st = strtol(bs, &bs, 10); ++bs; + sz = strtol(bl, &bl, 10); ++bl; + en = t.st + st + sz; + for (i = 1; i < n_blk; ++i) { + mm_idx_intv1_t s = t; + if (r->n == r->m) { + r->m = r->m? r->m + (r->m>>1) : 16; + r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a) * r->m); + } + st = strtol(bs, &bs, 10); ++bs; + sz = strtol(bl, &bl, 10); ++bl; + s.st = en, s.en = t.st + st; + en = t.st + st + sz; + if (s.en > s.st) r->a[r->n++] = s; + } + } else { + if (r->n == r->m) { + r->m = r->m? r->m + (r->m>>1) : 16; + r->a = (mm_idx_intv1_t*)realloc(r->a, sizeof(*r->a) * r->m); + } + r->a[r->n++] = t; } - r->a[r->n++] = t; } free(str.s); ks_destroy(ks); @@ -663,11 +689,11 @@ mm_idx_intv_t *mm_idx_read_bed(const mm_idx_t *mi, const char *fn) return I; } -int mm_idx_bed_read(mm_idx_t *mi, const char *fn) +int mm_idx_bed_read(mm_idx_t *mi, const char *fn, int read_junc) { int32_t i; if (mi->h == 0) mm_idx_index_name(mi); - mi->I = mm_idx_read_bed(mi, fn); + mi->I = mm_idx_read_bed(mi, fn, read_junc); if (mi->I == 0) return -1; for (i = 0; i < mi->n_seq; ++i) // TODO: eliminate redundant intervals radix_sort_bed(mi->I[i].a, mi->I[i].a + mi->I[i].n); diff --git a/main.c b/main.c index bf179c0..9d11102 100644 --- a/main.c +++ b/main.c @@ -6,7 +6,7 @@ #include "mmpriv.h" #include "ketopt.h" -#define MM_VERSION "2.16-r933-dirty" +#define MM_VERSION "2.16-r934-dirty" #ifdef __linux__ #include @@ -366,7 +366,7 @@ int main(int argc, char *argv[]) __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), mi->n_seq); if (argc != o.ind + 1) mm_mapopt_update(&opt, mi); if (mm_verbose >= 3) mm_idx_stat(mi); - if (junc_bed) mm_idx_bed_read(mi, junc_bed); + if (junc_bed) mm_idx_bed_read(mi, junc_bed, 1); if (!(opt.flag & MM_F_FRAG_MODE)) { for (i = o.ind + 1; i < argc; ++i) mm_map_file(mi, argv[i], &opt, n_threads); diff --git a/minimap.h b/minimap.h index 358cad7..993b008 100644 --- a/minimap.h +++ b/minimap.h @@ -367,7 +367,7 @@ int mm_idx_index_name(mm_idx_t *mi); int mm_idx_name2id(const mm_idx_t *mi, const char *name); int mm_idx_getseq(const mm_idx_t *mi, uint32_t rid, uint32_t st, uint32_t en, uint8_t *seq); -int mm_idx_bed_read(mm_idx_t *mi, const char *fn); +int mm_idx_bed_read(mm_idx_t *mi, const char *fn, int read_junc); int mm_idx_bed_junc(const mm_idx_t *mi, int32_t ctg, int32_t st, int32_t en, uint8_t *s); // deprecated APIs for backward compatibility diff --git a/minimap2.1 b/minimap2.1 index d13e3ee..99cab5c 100644 --- a/minimap2.1 +++ b/minimap2.1 @@ -1,4 +1,4 @@ -.TH minimap2 1 "28 Feburary 2019" "minimap2-2.16-dirty (r933)" "Bioinformatics tools" +.TH minimap2 1 "28 Feburary 2019" "minimap2-2.16-dirty (r934)" "Bioinformatics tools" .SH NAME .PP minimap2 - mapping and alignment between collections of DNA sequences @@ -365,10 +365,10 @@ on SIRV data, please add to the command line. .TP .BR --junc-bed \ FILE -BED file consisting of annotated introns and their strands. With this option, -minimap2 prefers splicing in annotations. -.I FILE -can be generated with `paftools.js gff2bed -j ann.gtf' []. +Gene annotations in the BED12 format (aka 12-column BED), or intron positions +in 5-column BED. With this option, minimap2 prefers splicing in annotations. +BED12 file can be converted from GTF/GFF3 with `paftools.js gff2bed anno.gtf' +[]. .TP .BR --junc-bonus \ INT Score bonus for a splice donor or acceptor found in annotation (effective with From 69af86657e9db4b213efe44a97c9902c38c5b380 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Tue, 30 Apr 2019 21:35:24 -0400 Subject: [PATCH 10/13] r935: fixed a cigar like 5I6D7I; resolved #392 --- align.c | 19 +++++++++++++++++++ main.c | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/align.c b/align.c index 60bc08a..5fe73ff 100644 --- a/align.c +++ b/align.c @@ -123,6 +123,25 @@ static void mm_fix_cigar(mm_reg1_t *r, const uint8_t *qseq, const uint8_t *tseq, } } assert(qoff == r->qe - r->qs && toff == r->re - r->rs); + for (k = 0; k < p->n_cigar - 2; ++k) { // fix CIGAR like 5I6D7I + if ((p->cigar[k]&0xf) > 0 && (p->cigar[k]&0xf) + (p->cigar[k+1]&0xf) == 3) { + uint32_t l, s[3] = {0,0,0}; + for (l = k; l < p->n_cigar; ++l) { // count number of adjacent I and D + uint32_t op = p->cigar[l]&0xf; + if (op == 1 || op == 2 || p->cigar[l]>>4 == 0) + s[op] += p->cigar[l] >> 4; + else break; + } + if (s[1] > 0 && s[2] > 0 && l - k > 2) { // turn to a single I and a single D + p->cigar[k] = s[1]<<4|1; + p->cigar[k+1] = s[2]<<4|2; + for (k += 2; k < l; ++k) + p->cigar[k] &= 0xf; + to_shrink = 1; + } + k = l; + } + } if (to_shrink) { // squeeze out zero-length operations int32_t l = 0; for (k = 0; k < p->n_cigar; ++k) // squeeze out zero-length operations diff --git a/main.c b/main.c index 9d11102..3ce15b7 100644 --- a/main.c +++ b/main.c @@ -6,7 +6,7 @@ #include "mmpriv.h" #include "ketopt.h" -#define MM_VERSION "2.16-r934-dirty" +#define MM_VERSION "2.16-r935-dirty" #ifdef __linux__ #include From 189555503a2035a95dcb6510d2c687ade5e0695c Mon Sep 17 00:00:00 2001 From: Heng Li Date: Tue, 30 Apr 2019 21:49:51 -0400 Subject: [PATCH 11/13] potentially fix issue #372 Needs someone to confirm --- python/mappy.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mappy.pyx b/python/mappy.pyx index 2528d9d..8c68920 100644 --- a/python/mappy.pyx +++ b/python/mappy.pyx @@ -142,7 +142,7 @@ cdef class Aligner: if fn_idx_out is None: r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, NULL) else: - r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, fn_idx_out) + r = cmappy.mm_idx_reader_open(str.encode(fn_idx_in), &self.idx_opt, str.encode(fn_idx_out)) if r is not NULL: self._idx = cmappy.mm_idx_reader_read(r, n_threads) # NB: ONLY read the first part cmappy.mm_idx_reader_close(r) From 97f67a2a0a001f3f1f9342e561290d4f5ee697e4 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Tue, 30 Apr 2019 22:30:32 -0400 Subject: [PATCH 12/13] r937: enlarge mm_mapopt_t::flag to 64 bits --- main.c | 2 +- minimap.h | 2 +- python/cmappy.pxd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.c b/main.c index 3ce15b7..d60c43e 100644 --- a/main.c +++ b/main.c @@ -6,7 +6,7 @@ #include "mmpriv.h" #include "ketopt.h" -#define MM_VERSION "2.16-r935-dirty" +#define MM_VERSION "2.16-r937-dirty" #ifdef __linux__ #include diff --git a/minimap.h b/minimap.h index 993b008..3aa52c4 100644 --- a/minimap.h +++ b/minimap.h @@ -104,9 +104,9 @@ typedef struct { } mm_idxopt_t; typedef struct { + int64_t flag; // see MM_F_* macros int seed; int sdust_thres; // score threshold for SDUST; 0 to disable - int flag; // see MM_F_* macros int max_qlen; // max query length diff --git a/python/cmappy.pxd b/python/cmappy.pxd index ba1871d..7545cb8 100644 --- a/python/cmappy.pxd +++ b/python/cmappy.pxd @@ -10,9 +10,9 @@ cdef extern from "minimap.h": uint64_t batch_size ctypedef struct mm_mapopt_t: + int64_t flag int seed int sdust_thres - int flag int max_qlen int bw int max_gap, max_gap_ref From c2aec88b84ad515622fdcbf2a1e9b23f8786a4a8 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Tue, 30 Apr 2019 22:40:36 -0400 Subject: [PATCH 13/13] r938: added --sam-hit-only; resolved #377 --- main.c | 2 ++ map.c | 2 +- minimap.h | 1 + minimap2.1 | 5 ++++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/main.c b/main.c index d60c43e..49b6ab1 100644 --- a/main.c +++ b/main.c @@ -65,6 +65,7 @@ static ko_longopt_t long_options[] = { { "max-chain-iter", ko_required_argument, 339 }, { "junc-bed", ko_required_argument, 340 }, { "junc-bonus", ko_required_argument, 341 }, + { "sam-hit-only", ko_no_argument, 342 }, { "help", ko_no_argument, 'h' }, { "max-intron-len", ko_required_argument, 'G' }, { "version", ko_no_argument, 'V' }, @@ -207,6 +208,7 @@ int main(int argc, char *argv[]) else if (c == 337) opt.max_sw_mat = mm_parse_num(o.arg); // --cap-sw-mat else if (c == 338) opt.max_qlen = mm_parse_num(o.arg); // --max-qlen else if (c == 340) junc_bed = o.arg; // --junc-bed + else if (c == 342) opt.flag |= MM_F_SAM_HIT_ONLY; // --sam-hit-only else if (c == 314) { // --frag yes_or_no(&opt, MM_F_FRAG_MODE, o.longidx, o.arg, 1); } else if (c == 315) { // --secondary diff --git a/map.c b/map.c index 41de91e..d33405c 100644 --- a/map.c +++ b/map.c @@ -589,7 +589,7 @@ static void *worker_pipeline(void *shared, int step, void *in) mm_write_paf3(&p->str, mi, t, r, km, p->opt->flag, s->rep_len[i]); mm_err_puts(p->str.s); } - } else if (p->opt->flag & (MM_F_OUT_SAM|MM_F_PAF_NO_HIT)) { // output an empty hit, if requested + } else if ((p->opt->flag & MM_F_PAF_NO_HIT) || ((p->opt->flag & MM_F_OUT_SAM) && !(p->opt->flag & MM_F_SAM_HIT_ONLY))) { // output an empty hit, if requested if (p->opt->flag & MM_F_OUT_SAM) mm_write_sam3(&p->str, mi, t, i - seg_st, -1, s->n_seg[k], &s->n_reg[seg_st], (const mm_reg1_t*const*)&s->reg[seg_st], km, p->opt->flag, s->rep_len[i]); else diff --git a/minimap.h b/minimap.h index 3aa52c4..cdf955f 100644 --- a/minimap.h +++ b/minimap.h @@ -35,6 +35,7 @@ #define MM_F_PAF_NO_HIT 0x8000000 // output unmapped reads to PAF #define MM_F_NO_END_FLT 0x10000000 #define MM_F_HARD_MLEVEL 0x20000000 +#define MM_F_SAM_HIT_ONLY 0x40000000 #define MM_I_HPC 0x1 #define MM_I_NO_SEQ 0x2 diff --git a/minimap2.1 b/minimap2.1 index 99cab5c..73e1154 100644 --- a/minimap2.1 +++ b/minimap2.1 @@ -1,4 +1,4 @@ -.TH minimap2 1 "28 Feburary 2019" "minimap2-2.16-dirty (r934)" "Bioinformatics tools" +.TH minimap2 1 "30 April 2019" "minimap2-2.16-dirty (r938)" "Bioinformatics tools" .SH NAME .PP minimap2 - mapping and alignment between collections of DNA sequences @@ -489,6 +489,9 @@ In PAF, output unmapped queries; the strand and the reference name fields are set to `*'. Warning: some paftools.js commands may not work with such output for the moment. .TP +.B --sam-hit-only +In SAM, don't output unmapped reads. +.TP .B --version Print version number to stdout .SS Preset options