mirror of
https://github.com/lh3/minimap2.git
synced 2026-10-04 18:48:12 +08:00
mm2-fast-v2.22 init
This commit is contained in:
Saurabh
@@ -14,6 +14,21 @@
|
||||
#include "mmpriv.h"
|
||||
#include "kvec.h"
|
||||
#include "khash.h"
|
||||
#include <map>
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <x86intrin.h>
|
||||
using namespace std;
|
||||
|
||||
|
||||
|
||||
#ifdef LISA_HASH
|
||||
#include "lisa_hash.h"
|
||||
extern lisa_hash<uint64_t, uint64_t> *lh;
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
#define idx_hash(a) ((a)>>1)
|
||||
#define idx_eq(a, b) ((a)>>1 == (b)>>1)
|
||||
@@ -52,9 +67,43 @@ mm_idx_t *mm_idx_init(int w, int k, int b, int flag)
|
||||
if (!(mm_dbg_flag & 1)) mi->km = km_init();
|
||||
return mi;
|
||||
}
|
||||
void mm_idx_destroy_mm_hash(mm_idx_t *mi)
|
||||
{
|
||||
//fprintf(stderr, "mm_destroy_hash\n");
|
||||
uint32_t i;
|
||||
if (mi == 0) return;
|
||||
if (mi->h) kh_destroy(str, (khash_t(str)*)mi->h);
|
||||
if (mi->B) {
|
||||
for (i = 0; i < 1U<<mi->b; ++i) {
|
||||
free(mi->B[i].p);
|
||||
free(mi->B[i].a.a);
|
||||
kh_destroy(idx, (idxhash_t*)mi->B[i].h);
|
||||
}
|
||||
}
|
||||
}
|
||||
void mm_idx_destroy_seq(mm_idx_t *mi)
|
||||
{
|
||||
//fprintf(stderr, "mm_destroy_seq\n");
|
||||
|
||||
uint32_t i;
|
||||
if (mi == 0) return;
|
||||
if (mi->I) {
|
||||
for (i = 0; i < mi->n_seq; ++i)
|
||||
free(mi->I[i].a);
|
||||
free(mi->I);
|
||||
}
|
||||
if (!mi->km) {
|
||||
for (i = 0; i < mi->n_seq; ++i)
|
||||
free(mi->seq[i].name);
|
||||
free(mi->seq);
|
||||
} else km_destroy(mi->km);
|
||||
free(mi->B); free(mi->S); free(mi);
|
||||
}
|
||||
|
||||
|
||||
void mm_idx_destroy(mm_idx_t *mi)
|
||||
{
|
||||
|
||||
uint32_t i;
|
||||
if (mi == 0) return;
|
||||
if (mi->h) kh_destroy(str, (khash_t(str)*)mi->h);
|
||||
@@ -96,6 +145,317 @@ const uint64_t *mm_idx_get(const mm_idx_t *mi, uint64_t minier, int *n)
|
||||
return &b->p[kh_val(h, k)>>32];
|
||||
}
|
||||
}
|
||||
//Output minimap2's hash table entries
|
||||
class hash_entry {
|
||||
public:
|
||||
uint64_t key;
|
||||
uint64_t n;
|
||||
uint64_t *p;
|
||||
hash_entry(uint64_t k, uint64_t n_, uint64_t *p_){
|
||||
key = k;
|
||||
n = n_;
|
||||
p = p_;
|
||||
}
|
||||
|
||||
};
|
||||
bool key_sort( hash_entry i1, hash_entry i2)
|
||||
{
|
||||
return (i1.key < i2.key);
|
||||
}
|
||||
|
||||
#if 0
|
||||
void mm_idx_load_key_value_lisa(const char* f_name, const mm_idx_t *mi)
|
||||
{
|
||||
uint64_t tic = __rdtsc();
|
||||
std::vector<hash_entry> v_hash;
|
||||
|
||||
//ofstream f(f_name);
|
||||
fprintf(stderr, "Building sorted key-val map\n");
|
||||
|
||||
uint32_t i,j;
|
||||
uint64_t num_values = 0;
|
||||
for (i = 0; i < 1U<<mi->b; ++i) {
|
||||
|
||||
|
||||
//fprintf(stderr, "BucketID %lu \n", i);
|
||||
idxhash_t *h = (idxhash_t*)mi->B[i].h;
|
||||
khint_t k;
|
||||
if (h == 0) continue;
|
||||
for (k = 0; k < kh_end(h); ++k){
|
||||
if (kh_exist(h, k)) {
|
||||
uint64_t key = kh_key(h, k), bucket_id = i;
|
||||
key = key>>1;
|
||||
|
||||
key = key<<mi->b | bucket_id;
|
||||
|
||||
if(kh_key(h, k)&1)
|
||||
{
|
||||
//print key value
|
||||
//fprintf(stderr, "%llu %llu %llu\n", key, kh_val(h, k), 0);
|
||||
v_hash.push_back(hash_entry(key, kh_val(h, k), NULL));
|
||||
}
|
||||
else
|
||||
{ // print key
|
||||
uint32_t n = (uint32_t)kh_val(h, k);
|
||||
//fprintf(stderr, "%llu %llu %llu ", key, kh_val(h, k), n);
|
||||
// for 0 to lsb 32 val
|
||||
// print b->p[msb 32 of val]
|
||||
|
||||
v_hash.push_back(hash_entry(key, n, &mi->B[i].p[(kh_val(h, k)>>32) + 0]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort(v_hash.begin(), v_hash.end(), key_sort);
|
||||
fprintf(stderr, "Sorted map building time = %lld \n", __rdtsc() - tic);
|
||||
fprintf(stderr, "Storing hash to %s \n", f_name);
|
||||
tic = __rdtsc();
|
||||
|
||||
int64_t itr_p = 0;
|
||||
for( int i = 0; i < v_hash.size(); i++){
|
||||
|
||||
if(v_hash[i].p == NULL){
|
||||
//f<<v_hash[i].key << " "<<1<<"\n"<<v_hash[i].n<<" \n";
|
||||
|
||||
lh->p[itr_p++] = v_hash[i].n;
|
||||
continue;
|
||||
}
|
||||
|
||||
//f<<v_hash[i].key << " "<<v_hash[i].n<<endl;
|
||||
|
||||
for(int j = 0; j < v_hash[i].n; j++){
|
||||
// f<<v_hash[i].p[j]<<" ";
|
||||
lh->p[itr_p++] = v_hash[i].p[j];
|
||||
num_values++;
|
||||
}
|
||||
//f<<endl;
|
||||
|
||||
}
|
||||
|
||||
//f.close();
|
||||
|
||||
string size_file_name = (string) f_name + "_size";
|
||||
ofstream size_f(size_file_name);
|
||||
size_f<<v_hash.size()<<" "<<num_values;
|
||||
size_f.close();
|
||||
|
||||
string prefix = (string)f_name + "_keys";
|
||||
string keys_bin_file_name = prefix + ".uint64";
|
||||
ofstream wf(keys_bin_file_name, ios::out | ios::binary);
|
||||
wf.write((char*)&key_list[0], (key_list.size())*sizeof(uint64_t));
|
||||
wf.close();
|
||||
|
||||
key_list.clear();
|
||||
|
||||
m.clear();
|
||||
v_hash.clear();
|
||||
|
||||
fprintf(stderr, "Index store File IO time %lld \n", __rdtsc() - tic);
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
void mm_idx_dump_hash(const char* f_name, const mm_idx_t *mi)
|
||||
{
|
||||
uint64_t tic = __rdtsc();
|
||||
//std::map<uint64_t, vector<uint64_t>> m;
|
||||
std::vector<hash_entry> v_hash;
|
||||
|
||||
//ofstream f(f_name);
|
||||
fprintf(stderr, "Building sorted key-val map\n");
|
||||
|
||||
uint32_t i,j;
|
||||
uint64_t num_values = 0;
|
||||
for (i = 0; i < 1U<<mi->b; ++i) {
|
||||
|
||||
|
||||
//fprintf(stderr, "BucketID %lu \n", i);
|
||||
idxhash_t *h = (idxhash_t*)mi->B[i].h;
|
||||
khint_t k;
|
||||
if (h == 0) continue;
|
||||
for (k = 0; k < kh_end(h); ++k){
|
||||
if (kh_exist(h, k)) {
|
||||
uint64_t key = kh_key(h, k), bucket_id = i;
|
||||
key = key>>1;
|
||||
|
||||
key = key<<mi->b | bucket_id;
|
||||
|
||||
if(kh_key(h, k)&1)
|
||||
{
|
||||
//print key value
|
||||
//fprintf(stderr, "%llu %llu %llu\n", key, kh_val(h, k), 0);
|
||||
//m[key].push_back(kh_val(h, k));
|
||||
v_hash.push_back(hash_entry(key, kh_val(h, k), NULL));
|
||||
}
|
||||
else
|
||||
{ // print key
|
||||
uint32_t n = (uint32_t)kh_val(h, k);
|
||||
//fprintf(stderr, "%llu %llu %llu ", key, kh_val(h, k), n);
|
||||
// for 0 to lsb 32 val
|
||||
// print b->p[msb 32 of val]
|
||||
|
||||
v_hash.push_back(hash_entry(key, n, &mi->B[i].p[(kh_val(h, k)>>32) + 0]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort(v_hash.begin(), v_hash.end(), key_sort);
|
||||
fprintf(stderr, "Sorted map building time = %lld \n", __rdtsc() - tic);
|
||||
fprintf(stderr, "Storing hash to %s \n", f_name);
|
||||
tic = __rdtsc();
|
||||
|
||||
vector<uint64_t> key_list;
|
||||
vector<uint64_t> val_list;
|
||||
vector<uint64_t> p_list;
|
||||
/*
|
||||
key_list.push_back(m.size());
|
||||
for(auto k : m){
|
||||
key_list.push_back(k.first);
|
||||
f<<k.first << " "<<k.second.size()<<endl;
|
||||
for(int j = 0; j < k.second.size(); j++){
|
||||
f<<k.second[j]<<" ";
|
||||
num_values++;
|
||||
}
|
||||
f<<endl;
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
key_list.push_back(v_hash.size());
|
||||
int64_t itr_p = 0;
|
||||
uint64_t sum_pos = 0;
|
||||
string f1_name = (string)f_name + "_pos_bin";
|
||||
string f2_name = (string)f_name + "_val_bin";
|
||||
ofstream f1(f1_name, ios::out | ios::binary);
|
||||
ofstream f2(f2_name, ios::out | ios::binary);
|
||||
for( int i = 0; i < v_hash.size(); i++){
|
||||
key_list.push_back(v_hash[i].key);
|
||||
if(v_hash[i].p == NULL){
|
||||
//f<<v_hash[i].key << " "<<1<<"\n"<<v_hash[i].n<<" \n";
|
||||
val_list.push_back(sum_pos<<32|(uint64_t)1);
|
||||
sum_pos+=1;
|
||||
p_list.push_back(v_hash[i].n);
|
||||
num_values++;
|
||||
continue;
|
||||
}
|
||||
|
||||
//f<<v_hash[i].key << " "<<v_hash[i].n<<endl;
|
||||
val_list.push_back(sum_pos<<32|(uint64_t)v_hash[i].n);
|
||||
sum_pos+=v_hash[i].n;
|
||||
|
||||
num_values+=v_hash[i].n;
|
||||
|
||||
|
||||
for(int j = 0; j < v_hash[i].n; j++){
|
||||
//f<<v_hash[i].p[j]<<" ";
|
||||
p_list.push_back(v_hash[i].p[j]);
|
||||
}
|
||||
// f<<endl;
|
||||
|
||||
}
|
||||
f1.write((char*)&val_list[0], (val_list.size())*sizeof(uint64_t));
|
||||
f2.write((char*)&p_list[0], (p_list.size())*sizeof(uint64_t));
|
||||
f1.close();
|
||||
f2.close();
|
||||
fprintf(stderr, "Index sorted SoA time %lld \n", __rdtsc() - tic);
|
||||
|
||||
//f.close();
|
||||
|
||||
string size_file_name = (string) f_name + "_size";
|
||||
ofstream size_f(size_file_name);
|
||||
size_f<<v_hash.size()<<" "<<num_values;
|
||||
size_f.close();
|
||||
|
||||
string prefix = (string)f_name + "_keys";
|
||||
string keys_bin_file_name = prefix + ".uint64";
|
||||
ofstream wf(keys_bin_file_name, ios::out | ios::binary);
|
||||
wf.write((char*)&key_list[0], (key_list.size())*sizeof(uint64_t));
|
||||
wf.close();
|
||||
|
||||
key_list.clear();
|
||||
|
||||
//m.clear();
|
||||
v_hash.clear();
|
||||
|
||||
fprintf(stderr, "Index store File IO time %lld \n", __rdtsc() - tic);
|
||||
|
||||
}
|
||||
void mm_idx_dump_hash_1(const char* f_name, const mm_idx_t *mi)
|
||||
{
|
||||
uint64_t tic = __rdtsc();
|
||||
std::map<uint64_t, vector<uint64_t>> m;
|
||||
|
||||
ofstream f(f_name);
|
||||
fprintf(stderr, "Building sorted key-val map\n");
|
||||
|
||||
uint32_t i,j;
|
||||
uint64_t num_values = 0;
|
||||
for (i = 0; i < 1U<<mi->b; ++i) {
|
||||
|
||||
|
||||
//fprintf(stderr, "BucketID %lu \n", i);
|
||||
idxhash_t *h = (idxhash_t*)mi->B[i].h;
|
||||
khint_t k;
|
||||
if (h == 0) continue;
|
||||
for (k = 0; k < kh_end(h); ++k){
|
||||
if (kh_exist(h, k)) {
|
||||
uint64_t key = kh_key(h, k), bucket_id = i;
|
||||
key = key>>1;
|
||||
|
||||
key = key<<mi->b | bucket_id;
|
||||
|
||||
if(kh_key(h, k)&1)
|
||||
{
|
||||
//print key value
|
||||
//fprintf(stderr, "%llu %llu %llu\n", key, kh_val(h, k), 0);
|
||||
m[key].push_back(kh_val(h, k));
|
||||
}
|
||||
else
|
||||
{ // print key
|
||||
uint32_t n = (uint32_t)kh_val(h, k);
|
||||
//fprintf(stderr, "%llu %llu %llu ", key, kh_val(h, k), n);
|
||||
// for 0 to lsb 32 val
|
||||
// print b->p[msb 32 of val]
|
||||
for(j = 0; j < n; j++)
|
||||
{
|
||||
//fprintf(stderr, "%llu ", mi->B[i].p[(kh_val(h, k)>>32) + j]);
|
||||
m[key].push_back(mi->B[i].p[(kh_val(h, k)>>32) + j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fprintf(stderr, "Sorted map building time = %lld \n", __rdtsc() - tic);
|
||||
fprintf(stderr, "Storing hash to %s \n", f_name);
|
||||
tic = __rdtsc();
|
||||
vector<uint64_t> key_list;
|
||||
key_list.push_back(m.size());
|
||||
for(auto k : m){
|
||||
key_list.push_back(k.first);
|
||||
f<<k.first << " "<<k.second.size()<<endl;
|
||||
for(int j = 0; j < k.second.size(); j++){
|
||||
f<<k.second[j]<<" ";
|
||||
num_values++;
|
||||
}
|
||||
f<<endl;
|
||||
}
|
||||
f.close();
|
||||
string size_file_name = (string) f_name + "_size";
|
||||
ofstream size_f(size_file_name);
|
||||
size_f<<m.size()<<" "<<num_values;
|
||||
size_f.close();
|
||||
|
||||
string prefix = (string)f_name + "_keys";
|
||||
string keys_bin_file_name = prefix + ".uint64";
|
||||
ofstream wf(keys_bin_file_name, ios::out | ios::binary);
|
||||
wf.write((char*)&key_list[0], (key_list.size())*sizeof(uint64_t));
|
||||
wf.close();
|
||||
|
||||
key_list.clear();
|
||||
m.clear();
|
||||
fprintf(stderr, "Index store File IO time %lld \n", __rdtsc() - tic);
|
||||
|
||||
}
|
||||
|
||||
void mm_idx_stat(const mm_idx_t *mi)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user