debug mbg

This commit is contained in:
chhylp123
2021-05-30 09:02:25 -04:00
parent a39f01f4d8
commit e774a83be2
18 changed files with 2497 additions and 207 deletions
+508 -2
View File
@@ -489,8 +489,8 @@ ma_ug_t *ug, uint32_t flag, double score, const char* cmd)
if(kh->tue <= kh->tus) continue;
kv_pushp(u_trans_t, o->k_trans, &kt);
kt->f = flag; kt->rev = ((kh->qn ^ kh->tn) & 1); kt->del = 0;
kt->qn = kh->qn>>1; kt->qs = kh->qus; kt->qe = kh->que; kt->qo = kh->qn&1;
kt->tn = kh->tn>>1; kt->ts = kh->tus; kt->te = kh->tue; kt->to = kh->tn&1;
kt->qn = kh->qn>>1; kt->qs = kh->qus; kt->qe = kh->que; ///kt->qo = kh->qn&1;
kt->tn = kh->tn>>1; kt->ts = kh->tus; kt->te = kh->tue; ///kt->to = kh->tn&1;
if(score < 0)
{
kt->nw = (MIN((kt->qe - kt->qs), (kt->te - kt->ts)))*CHAIN_MATCH;
@@ -1438,4 +1438,510 @@ R_to_U* ruIndex, utg_trans_t *o)
free(path_p);
free(path_q);
return n_reduced;
}
typedef struct {
uint64_t *idx;
kvec_t(uint64_t) pos;
} mz_ds_t;
typedef struct {
uint64_t x, y;
} pt128_t;
typedef struct {
uint64_t x;
uint64_t rid:32, span:32;
uint64_t pos:63, rev:1;
} pt_mz1_t;
typedef struct {
///cnt1: how many unique minimizers
///cnt2: how many non-unique minimizers
uint32_t cnt2, cnt1;
uint32_t m[2];
int8_t s;
} pt_uinfo_t;
typedef struct {
uint32_t n_seq; // number of segments; same as gfa_t::n_seg
pt_uinfo_t *info; // of size n_seg
kv_u_trans_t *ma;
} pt_match_t;
#define mz_key(z) ((z).x)
KRADIX_SORT_INIT(mz, pt_mz1_t, mz_key, 8)
#define pt128x_key(z) ((z).x)
KRADIX_SORT_INIT(pt128x, pt128_t, pt128x_key, 8)
#define generic_key(x) (x)
KRADIX_SORT_INIT(tb64, uint64_t, generic_key, 8)
typedef struct { uint32_t n, m; pt128_t *a; } pt128_v;
typedef struct { uint32_t n, m; pt_mz1_t *a; } pt_mz1_v;
static inline int mzcmp(const pt_mz1_t *a, const pt_mz1_t *b)
{
return (a->x > b->x) - (a->x < b->x);
}
void pt_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, pt_mz1_v *p)
{
static const pt_mz1_t dummy = { UINT64_MAX, (1<<28) - 1, 0, 0 };
uint64_t shift1 = k - 1, mask = (1ULL<<k) - 1, kmer[4] = {0,0,0,0};
int i, j, l, buf_pos, min_pos, kmer_span = 0;
pt_mz1_t buf[256], min = dummy;
tiny_queue_t tq;
assert(len > 0 && rid < (uint64_t)1<<32 && (w > 0 && w < 256) && (k > 0 && k <= 63));
memset(buf, 0xff, w * sizeof(pt_mz1_t));
memset(&tq, 0, sizeof(tiny_queue_t));
kv_resize(pt_mz1_t, *p, p->n + len/w);
for (i = l = buf_pos = min_pos = 0; i < len; ++i) {
int c = seq_nt4_table[(uint8_t)str[i]];
pt_mz1_t info = dummy;
if (c < 4) { // not an ambiguous base
int z;
if (is_hpc) {
int skip_len = 1;
if (i + 1 < len && seq_nt4_table[(uint8_t)str[i + 1]] == c) {
for (skip_len = 2; i + skip_len < len; ++skip_len)
if (seq_nt4_table[(uint8_t)str[i + skip_len]] != c)
break;
i += skip_len - 1; // put $i at the end of the current homopolymer run
}
tq_push(&tq, skip_len);
kmer_span += skip_len;
if (tq.count > k) kmer_span -= tq_shift(&tq);
} else kmer_span = l + 1 < k? l + 1 : k;
kmer[0] = (kmer[0] << 1 | (c&1)) & mask; // forward k-mer
kmer[1] = (kmer[1] << 1 | (c>>1)) & mask;
kmer[2] = kmer[2] >> 1 | (uint64_t)(1 - (c&1)) << shift1; // reverse k-mer
kmer[3] = kmer[3] >> 1 | (uint64_t)(1 - (c>>1)) << shift1;
if (kmer[1] == kmer[3]) continue; // skip "symmetric k-mers" as we don't know its strand
z = kmer[1] < kmer[3]? 0 : 1; // strand
++l;
if (l >= k && kmer_span < 256) {
uint64_t y;
y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]);
info.x = y, info.rid = rid, info.pos = i, info.rev = z, info.span = kmer_span; // initially pt_mz1_t::rid keeps the k-mer count
}
} else l = 0, tq.count = tq.front = 0, kmer_span = 0;
buf[buf_pos] = info; // need to do this here as appropriate buf_pos and buf[buf_pos] are needed below
if (l == w + k - 1 && min.x != UINT64_MAX) { // special case for the first window - because identical k-mers are not stored yet
for (j = buf_pos + 1; j < w; ++j)
if (mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos) kv_push(pt_mz1_t, *p, buf[j]);
for (j = 0; j < buf_pos; ++j)
if (mzcmp(&min, &buf[j]) == 0 && buf[j].pos != min.pos) kv_push(pt_mz1_t, *p, buf[j]);
}
///three cases: 1.
if (info.x <= min.x) { // a new minimum; then write the old min
if (l >= w + k && min.x != UINT64_MAX) kv_push(pt_mz1_t, *p, min);
min = info, min_pos = buf_pos;
} else if (buf_pos == min_pos) { // old min has moved outside the window
if (l >= w + k - 1 && min.x != UINT64_MAX) kv_push(pt_mz1_t, *p, min);
for (j = buf_pos + 1, min.x = UINT64_MAX; j < w; ++j) // the two loops are necessary when there are identical k-mers
if (mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j; // >= is important s.t. min is always the closest k-mer
for (j = 0; j <= buf_pos; ++j)
if (mzcmp(&min, &buf[j]) >= 0) min = buf[j], min_pos = j;
if (l >= w + k - 1 && min.x != UINT64_MAX) { // write identical k-mers
for (j = buf_pos + 1; j < w; ++j) // these two loops make sure the output is sorted
if (mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos) kv_push(pt_mz1_t, *p, buf[j]);
for (j = 0; j <= buf_pos; ++j)
if (mzcmp(&min, &buf[j]) == 0 && min.pos != buf[j].pos) kv_push(pt_mz1_t, *p, buf[j]);
}
}
if (++buf_pos == w) buf_pos = 0;
}
if (min.x != UINT64_MAX)
kv_push(pt_mz1_t, *p, min);
}
pt_mz1_v *pt_collect_minimizers(ma_ug_t *ug, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources,
kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp)
{
uint32_t i;
pt_mz1_v *mz = NULL; CALLOC(mz, 1);
for (i = 0; i < ug->u.n; i++)
{
if(!ug->u.a[i].len) continue;
pt_sketch(ug->u.a[i].s, ug->u.a[i].len, asm_opt.mz_win, asm_opt.k_mer_length, i, 0, mz);
}
radix_sort_mz(mz->a, mz->a + mz->n);
return mz;
}
pt128_v *pt_collect_anchors(ma_ug_t *ug, pt_mz1_v *mz, mz_ds_t *mz_idx, uint32_t max_occ)
{
uint32_t st, j;
pt128_v *pa = NULL; CALLOC(pa, 1);
pt128_t *p = NULL;
mz_idx->pos.n = 0;
///all minimizers
for (j = 1, st = 0; j <= mz->n; ++j) {
if (j == mz->n || mz->a[j].x != mz->a[st].x) {
uint32_t k, l;
// if (j - st == 1) ++info[mz->a[st].rid].cnt1; ///of size n_seg
///max_occ is the frquency threshold of minimizer
///if (j - st) == 1, means minimizer only occurs in one read, it is not useful
if (j - st == 1 || j - st > max_occ) goto end_anchor;
for (k = st; k < j; ++k) {
// ++info[mz->a[k].rid].cnt2;
kv_push(uint64_t, mz_idx->pos, (uint64_t)(mz->a[k].rid)<<32|(uint64_t)(mz->a[k].pos));
for (l = k + 1; l < j; ++l) {
///k is current minimizer
uint32_t span, rev = (mz->a[k].rev != mz->a[l].rev);
int32_t lk = ug->u.a[mz->a[k].rid].len, ll = ug->u.a[mz->a[l].rid].len;
kv_pushp(pt128_t, *pa, &p);
span = mz->a[l].span;
p->x = (uint64_t)mz->a[k].rid << 33 | mz->a[l].rid << 1 | rev;
p->y = (uint64_t)mz->a[k].pos << 32 | (rev? ll - (mz->a[l].pos + 1 - span) - 1 : mz->a[l].pos);
kv_pushp(pt128_t, *pa, &p);
span = mz->a[k].span;
p->x = (uint64_t)mz->a[l].rid << 33 | mz->a[k].rid << 1 | rev;
p->y = (uint64_t)mz->a[l].pos << 32 | (rev? lk - (mz->a[k].pos + 1 - span) - 1 : mz->a[k].pos);
}
}
end_anchor: st = j;
}
}
radix_sort_pt128x(pa->a, pa->a + pa->n);
radix_sort_tb64(mz_idx->pos.a, mz_idx->pos.a + mz_idx->pos.n);
CALLOC(mz_idx->idx, ug->u.n);
for (st = 0, j = 1; j <= mz_idx->pos.n; ++j)
{
if (j == mz_idx->pos.n || (mz_idx->pos.a[j]>>32) != (mz_idx->pos.a[st]>>32))
{
mz_idx->idx[mz_idx->pos.a[st]>>32] = (uint64_t)st << 32 | (j - st), st = j;
}
}
return pa;
}
int32_t pt_lis_64(int32_t n, const uint64_t *a_idx, int32_t *b, int32_t *M)
{
int32_t i, k, L = 0, *P = b;
// MALLOC(M, n+1);
for (i = 0; i < n; ++i) {
int32_t lo = 1, hi = L, newL;
while (lo <= hi) {
int32_t mid = (lo + hi + 1) >> 1;
if ((uint32_t)a_idx[M[mid]] < (uint32_t)a_idx[i]) lo = mid + 1;
else hi = mid - 1;
}
newL = lo, P[i] = M[newL - 1], M[newL] = i;
if (newL > L) L = newL;
}
k = M[L];
memcpy(M, P, n * sizeof(int32_t));
for (i = L - 1; i >= 0; --i) b[i] = k, k = M[k];
// free(M);
return L;
}
uint32_t debug_lis_64(const uint64_t *a, int32_t *b, uint64_t n)
{
uint32_t i;
if(n <= 1) return 1;
for (i = 0; i+1 < n; i++)
{
if(((a[b[i]]>>32) > (a[b[i+1]]>>32)) || (((uint32_t)a[b[i]]) > ((uint32_t)a[b[i+1]])))
{
i = (uint32_t)-1;
break;
}
}
if(i == (uint32_t)-1)
{
fprintf(stderr, "\nERROR-chain\n");
for (i = 0; i < n; i++)
{
fprintf(stderr, "x-%lu, y-%lu\n", (a[b[i]]>>32), (uint64_t)((uint32_t)a[b[i]]));
}
}
return 1;
}
void update_mz_ovlp(uint32_t* n_x_beg, uint32_t* n_x_end, int64_t xLen,
uint32_t* n_y_beg, uint32_t* n_y_end, int64_t yLen, uint32_t rev)
{
int64_t x_beg = (*n_x_beg), x_end = (*n_x_end);
int64_t y_beg = (*n_y_beg), y_end = (*n_y_end);
if(x_beg <= y_beg)
{
y_beg = y_beg - x_beg;
x_beg = 0;
}
else
{
x_beg = x_beg - y_beg;
y_beg = 0;
}
long long x_right_length = xLen - x_end - 1;
long long y_right_length = yLen - y_end - 1;
if(x_right_length <= y_right_length)
{
x_end = xLen - 1;
y_end = y_end + x_right_length;
}
else
{
x_end = x_end + y_right_length;
y_end = yLen - 1;
}
if(rev == 0)
{
(*n_y_beg) = y_beg;
(*n_y_end) = y_end + 1;
}
else
{
(*n_y_beg) = yLen - y_end - 1;
(*n_y_end) = yLen - y_beg - 1 + 1;
}
(*n_x_beg) = x_beg;
(*n_x_end) = x_end + 1;
}
int64_t get_insert_pos(uint64_t *a, int64_t n, int64_t target)
{
int64_t left, right, ans, mid;
left = 0; right = n - 1; ans = n;
while (left <= right) {
mid = ((right - left) >> 1) + left;
if (target <= (uint32_t)a[mid]) {
ans = mid;
right = mid - 1;
} else {
left = mid + 1;
}
}
return ans;
}
uint32_t get_mz_occ(mz_ds_t *mz_idx, uint64_t uid, uint64_t s, uint64_t e)
{
uint64_t *a = mz_idx->pos.a + (mz_idx->idx[uid]>>32), n = (uint32_t)(mz_idx->idx[uid]);
if(n == 0) return 0;
int64_t sid, eid;
sid = get_insert_pos(a, n, s);
eid = get_insert_pos(a, n, e);
return eid + 1 - sid;
}
kv_u_trans_t *pt_cal_sim(pt128_v *pa, mz_ds_t *mz_idx, ma_ug_t *ug, uint32_t min_cnt, double min_sim)
{
int64_t st, i, j;
kvec_t(uint64_t) a; kv_init(a);
kvec_t(int32_t) b; kv_init(b);
kvec_t(int32_t) M; kv_init(M);
kv_u_trans_t *ma = NULL; CALLOC(ma, 1);
u_trans_t m;
///x = (uint64_t)mz[l].rid << 33 | mz[k].rid << 1 | rev;
for (st = 0, i = 1; i <= pa->n; ++i) {
if (i == pa->n || pa->a[i].x != pa->a[st].x) {///minimizers between a pair of unitigs
if((pa->a[st].x>>33) == (((uint32_t)pa->a[st].x)>>1)) goto end_chain;
uint32_t nn[2], nn_min;
a.n = 0; memset(&m, 0, sizeof(m));
if (i - st < min_cnt) goto end_chain;
//(uint64_t)mz[l].pos << 32 | (rev? lk - (mz[k].pos + 1 - span) - 1 : mz[k].pos);
for (j = st; j < i; ++j) kv_push(uint64_t, a, pa->a[j].y);
radix_sort_tb64(a.a, a.a + a.n);///sort by query pos + target pos
// for (l = 0; l < a.n; ++l) a.a[l] = (uint32_t)a.a[l];///only need target pos to do LIS
kv_resize(int32_t, b, a.n); kv_resize(int32_t, M, a.n+1);
m.occ = pt_lis_64(a.n, a.a, b.a, M.a);
/*******************************for debug************************************/
// debug_lis_64(a.a, b.a, m.occ);
/*******************************for debug************************************/
if (m.occ == 0 || m.occ < min_cnt) goto end_chain;//chain occ
m.qn = pa->a[st].x >> 33;///query id
m.tn = ((uint32_t)pa->a[st].x) >> 1;///target id
if (m.qn == m.tn) goto end_chain;
m.qs = a.a[b.a[0]]>>32; m.qe = a.a[b.a[m.occ-1]]>>32;
m.ts = (uint32_t)(a.a[b.a[0]]); m.te = (uint32_t)(a.a[b.a[m.occ-1]]);
m.rev = (pa->a[st].x>>32&1) ^ (pa->a[st].x&1);
update_mz_ovlp(&(m.qs), &(m.qe), ug->u.a[m.qn].len, &(m.ts), &(m.te), ug->u.a[m.tn].len, m.rev);
nn[0] = get_mz_occ(mz_idx, m.qn, m.qs, m.qe-1);
nn[1] = get_mz_occ(mz_idx, m.tn, m.ts, m.te-1);
nn_min = MIN(nn[0], nn[1]);
nn_min = MAX(nn_min, m.occ);
// m.sim = pow(2.0 * m.m / (nn[0] + nn[1]), 1.0 / k);
if(m.occ >= nn_min*min_sim){
m.nw = (double)(m.occ) - (double)(nn_min-m.occ)*0.2;
if(m.nw > 0) kv_push(u_trans_t, *ma, m);
}
end_chain: st = i;
}
}
kv_destroy(b); kv_destroy(a); kv_destroy(M);
return ma;
}
void clean_mz_ovlp(kv_u_trans_t *ta, ma_ug_t *ug)
{
u_trans_t *a = NULL;
asg_arc_t *as = NULL;
uint32_t k, i, n, v, ns;
pdq pq; init_pdq(&pq, ug->g->n_seq<<1);
uint8_t *vis = NULL; CALLOC(vis, ug->g->n_seq);
kvec_t_u32_warp p; kv_init(p.a);
for (k = 0; k < ta->idx.n; k++)
{
p.a.n = 0;
a = u_trans_a(*ta, k);
n = u_trans_n(*ta, k);
if(n == 0) continue;
v = k<<1;
as = asg_arc_a(ug->g, v); ns = asg_arc_n(ug->g, v);
if(ns > 0)
{
for (i = 0; i < ns; i++)
{
if(as[i].del) continue;
kv_push(uint32_t, p.a, as[i].v);
}
set_utg_by_dis(v, &pq, ug->g, &p, ug->g->seq[v>>1].len);
}
v = (k<<1) + 1;
as = asg_arc_a(ug->g, v); ns = asg_arc_n(ug->g, v);
if(ns > 0)
{
for (i = 0; i < ns; i++)
{
if(as[i].del) continue;
kv_push(uint32_t, p.a, as[i].v);
}
set_utg_by_dis(v, &pq, ug->g, &p, ug->g->seq[v>>1].len);
}
for (i = 0; i < p.a.n; i++) vis[p.a.a[i]>>1] = 1;
for (i = 0; i < n; i++)
{
if(vis[a[i].tn]) a[i].del = 1;
}
for (i = 0; i < p.a.n; i++) vis[p.a.a[i]>>1] = 0;
}
destory_pdq(&pq);
kv_destroy(p.a);
free(vis);
for (k = n = 0; k < ta->n; ++k)
{
if(ta->a[k].del) continue;
ta->a[n] = ta->a[k];
n++;
}
ta->n = n;
kt_u_trans_t_idx(ta, ug->g->n_seq);
kt_u_trans_t_simple_symm(ta, ug->g->n_seq, 0);
}
int cmp_u_trans_nw(const void * a, const void * b)
{
if((*(u_trans_t*)a).nw == (*(u_trans_t*)b).nw) return 0;
return (*(u_trans_t*)a).nw < (*(u_trans_t*)b).nw ? 1 : -1;
}
/**
void flat_mz_ovlp(kv_u_trans_t *ta, ma_ug_t *ug, asg_t *read_g, ma_sub_t* coverage_cut,
ma_hit_t_alloc* sources, R_to_U* ruIndex, uint32_t min_cnt, uint32_t cov_thres,
double cov_pass_rate, double nw_pass_rate)
{
u_trans_t *a = NULL, *p = NULL;
uint8_t *vis = NULL; CALLOC(vis, ug->g->n_seq);
uint32_t *cov = NULL; CALLOC(cov, ug->g->n_seq);
kvec_t(uint32_t) tc; kv_init(tc);
uint32_t k, i, j, n, v, ns, qs, qe, occ, hapN, pass;
double w;
for (i = 0; i < ug->u.n; i++)
{
cov[i] = get_utg_cov(ug, i, read_g, coverage_cut, sources, ruIndex, vis);
}
for (k = 0; k < ta->idx.n; k++)
{
a = u_trans_a(*ta, k);
n = u_trans_n(*ta, k);
if(n == 0) continue;
qsort(a, n, sizeof(u_trans_t), cmp_u_trans_nw);
kv_resize(uint32_t, tc, ug->u.a[k].len);
tc.n = ug->u.a[k].len;
memset(tc.a, 0, tc.n*sizeof(uint32_t));
for (i = 0, p = NULL; i < n; i++)
{
qs = a[i].qs; qe = a[i].qe; w = a[i].nw; occ = a[i].occ; pass = 0;
for (j = qs; j < qe; j++)
{
if(tc.a[j] + cov[k] >= cov_thres) pass++;
}
if(pass > cov_pass_rate*(qe - qs))
{
if(p && nw_pass_rate*) a[i].del = 1;
}
else
{
for (j = qs; j < qe; j++) tc.a[j] += cov[k];
}
}
}
free(vis); free(cov); kv_destroy(tc);
}
**/
void print_u_trans(kv_u_trans_t *ta)
{
uint32_t i;
u_trans_t *p = NULL;
for (i = 0; i < ta->n; i++)
{
p = &(ta->a[i]);
fprintf(stderr, "q-utg%.6ul\tqs(%u)\tqe(%u)\tt-utg%.6ul\tts(%u)\tte(%u)\trev(%u)\tw(%f)\tf(%u)\n",
p->qn+1, p->qs, p->qe, p->tn+1, p->ts, p->te, p->rev, p->nw, p->f);
}
fprintf(stderr, "[M::%s::] \n", __func__);
}
kv_u_trans_t *pt_pdist(ma_ug_t *ug, asg_t *read_g, ma_sub_t *coverage_cut, ma_hit_t_alloc* sources,
kvec_asg_arc_t_warp* edge, int max_hang, int min_ovlp, uint32_t min_chain_cnt)
{
pt_mz1_v *mz = NULL;
mz_ds_t mz_idx; memset(&mz_idx, 0, sizeof(mz_idx));
pt128_v *an = NULL;
kv_u_trans_t *ma = NULL; CALLOC(ma, 1);
mz = pt_collect_minimizers(ug, read_g, coverage_cut, sources, edge, max_hang, min_ovlp);
an = pt_collect_anchors(ug, mz, &mz_idx, asm_opt.polyploidy*10);
kv_destroy(*mz); free(mz);
ma = pt_cal_sim(an, &mz_idx, ug, min_chain_cnt, asm_opt.purge_simi_thres);
kv_destroy(*an); free(an);
kv_destroy(mz_idx.pos); free(mz_idx.idx);
kt_u_trans_t_idx(ma, ug->g->n_seq);
kt_u_trans_t_simple_symm(ma, ug->g->n_seq, 1);
clean_mz_ovlp(ma, ug);
// print_u_trans(ma);
// exit(1);
return ma;
}