xxay/hifiasm
1
0
Fork 0
mirror of https://github.com/chhylp123/hifiasm.git synced 2026-09-15 12:47:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

removed hifiasm hash table index

Browse Source
This commit is contained in:
Heng Li
2020-03-26 16:10:01 -04:00
parent 387d6336d8
commit 79bc553da6
5 changed files with 69 additions and 1137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

425
Hash_Table.cpp
View File

@@ -779,195 +779,6 @@ void init_Pos_Table(Pos_Table** table)
*table = ha_pt_init();
}
void init_Total_Count_Table(int k, Total_Count_Table* TCB)
{
if(k>64)
{
fprintf(stderr, "k-mer is too long. The length of k-mer must <= 64.");
fflush(stderr);
exit(0);
}
int total_bits = k * 2;
TCB->prefix_bits = PREFIX_BITS;
TCB->suffix_bits = total_bits - TCB->prefix_bits;
if (TCB->suffix_bits > MAX_SUFFIX_BITS)
{
TCB->suffix_bits = MAX_SUFFIX_BITS;
TCB->prefix_bits = total_bits - TCB->suffix_bits;
}
///TCB->suffix_mode = (1ULL<<TCB->suffix_bits) - 1;
///right shift is safe, since TCB->suffix_bits cannot be 0
TCB->suffix_mode = ALL >> (64 - TCB->suffix_bits);
///number of small hash table
TCB->size = (1ULL<<TCB->prefix_bits);
TCB->sub_h = (Count_Table**)malloc(sizeof(Count_Table*)*TCB->size);
TCB->sub_h_lock = (Hash_table_spin_lock*)malloc(sizeof(Hash_table_spin_lock)*TCB->size);
memset(TCB->sub_h_lock, 0, sizeof(Hash_table_spin_lock)*TCB->size);
int i = 0;
for (i = 0; i < TCB->size; i++)
{
init_Count_Table(&(TCB->sub_h[i]));
TCB->sub_h_lock[i].lock = 0;
}
TCB->non_unique_k_mer = 0;
}
void init_Total_Pos_Table(Total_Pos_Table* TCB, Total_Count_Table* pre_TCB)
{
TCB->prefix_bits = pre_TCB->prefix_bits;
TCB->suffix_bits = pre_TCB->suffix_bits;
TCB->suffix_mode = pre_TCB->suffix_mode;
TCB->size = pre_TCB->size;
TCB->useful_k_mer = 0;
TCB->total_occ = 0;
TCB->k_mer_index = NULL;
TCB->sub_h_lock = (Hash_table_spin_lock*)malloc(sizeof(Hash_table_spin_lock)*TCB->size);
memset(TCB->sub_h_lock, 0, sizeof(Hash_table_spin_lock)*TCB->size);
TCB->sub_h = (Pos_Table**)malloc(sizeof(Pos_Table*)*TCB->size);
TCB->pos = NULL;
int i = 0;
for (i = 0; i < TCB->size; i++)
{
init_Pos_Table(&(TCB->sub_h[i]));
TCB->sub_h_lock[i].lock = 0;
}
}
void destory_Total_Count_Table(Total_Count_Table* TCB)
{
int i;
for (i = 0; i < TCB->size; i++)
{
ha_ct_destroy(TCB->sub_h[i]);
}
free(TCB->sub_h);
free(TCB->sub_h_lock);
}
void destory_Total_Pos_Table(Total_Pos_Table* TCB)
{
free(TCB->k_mer_index);
free(TCB->sub_h_lock);
free(TCB->pos);
int i;
for (i = 0; i < TCB->size; i++)
{
ha_pt_destroy(TCB->sub_h[i]);
}
free(TCB->sub_h);
}
/*
void write_Total_Pos_Table(Total_Pos_Table* TCB, char* read_file_name)
{
fprintf(stderr, "Writing index to disk... \n");
char* index_name = (char*)malloc(strlen(read_file_name)+5);
sprintf(index_name, "%s.idx", read_file_name);
FILE* fp = fopen(index_name, "w");
fwrite(&asm_opt.adapterLen, sizeof(asm_opt.adapterLen), 1, fp);
fwrite(&asm_opt.k_mer_min_freq, sizeof(asm_opt.k_mer_min_freq), 1, fp);
fwrite(&asm_opt.k_mer_max_freq, sizeof(asm_opt.k_mer_max_freq), 1, fp);
fwrite(&TCB->prefix_bits, sizeof(TCB->prefix_bits), 1, fp);
fwrite(&TCB->suffix_bits, sizeof(TCB->suffix_bits), 1, fp);
fwrite(&TCB->suffix_mode, sizeof(TCB->suffix_mode), 1, fp);
fwrite(&TCB->size, sizeof(TCB->size), 1, fp);
fwrite(&TCB->useful_k_mer, sizeof(TCB->useful_k_mer), 1, fp);
fwrite(&TCB->total_occ, sizeof(TCB->total_occ), 1, fp);
fwrite(TCB->k_mer_index, sizeof(uint64_t), TCB->useful_k_mer+1, fp);
fwrite(TCB->pos, sizeof(k_mer_pos), TCB->total_occ, fp);
int i;
for (i = 0; i < TCB->size; i++)
{
kh_write(POS64, TCB->sub_h[i], fp);
}
free(index_name);
fclose(fp);
fprintf(stderr, "Index has been written.\n");
}
int load_Total_Pos_Table(Total_Pos_Table* TCB, char* read_file_name)
{
fprintf(stderr, "Loading index from disk... \n");
char* index_name = (char*)malloc(strlen(read_file_name)+5);
sprintf(index_name, "%s.idx", read_file_name);
FILE* fp = fopen(index_name, "r");
if (!fp)
{
return 0;
}
int f_flag;
int local_adapterLen;
f_flag = fread(&local_adapterLen, sizeof(local_adapterLen), 1, fp);
if(local_adapterLen != asm_opt.adapterLen)
{
fprintf(stderr, "the adapterLen of index is: %d, but the adapterLen set by user is: %d\n",
local_adapterLen, asm_opt.adapterLen);
exit(1);
}
f_flag += fread(&asm_opt.k_mer_min_freq, sizeof(asm_opt.k_mer_min_freq), 1, fp);
f_flag += fread(&asm_opt.k_mer_max_freq, sizeof(asm_opt.k_mer_max_freq), 1, fp);
f_flag += fread(&TCB->prefix_bits, sizeof(TCB->prefix_bits), 1, fp);
f_flag += fread(&TCB->suffix_bits, sizeof(TCB->suffix_bits), 1, fp);
f_flag += fread(&TCB->suffix_mode, sizeof(TCB->suffix_mode), 1, fp);
f_flag += fread(&TCB->size, sizeof(TCB->size), 1, fp);
f_flag += fread(&TCB->useful_k_mer, sizeof(TCB->useful_k_mer), 1, fp);
f_flag += fread(&TCB->total_occ, sizeof(TCB->total_occ), 1, fp);
if (TCB->useful_k_mer+1)
{
TCB->k_mer_index = (uint64_t*)malloc(sizeof(uint64_t)*(TCB->useful_k_mer+1));
f_flag += fread(TCB->k_mer_index, sizeof(uint64_t), TCB->useful_k_mer+1, fp);
}
else
{
TCB->k_mer_index = NULL;
}
if (TCB->total_occ)
{
TCB->pos = (k_mer_pos*)malloc(sizeof(k_mer_pos)*TCB->total_occ);
f_flag += fread(TCB->pos, sizeof(k_mer_pos), TCB->total_occ, fp);
}
else
{
TCB->pos = NULL;
}
TCB->sub_h_lock = (Hash_table_spin_lock*)malloc(sizeof(Hash_table_spin_lock)*TCB->size);
memset(TCB->sub_h_lock, 0, sizeof(Hash_table_spin_lock)*TCB->size);
TCB->sub_h = (Pos_Table**)malloc(sizeof(Pos_Table*)*TCB->size);
int i;
for (i = 0; i < TCB->size; i++)
{
init_Pos_Table(&(TCB->sub_h[i]));
TCB->sub_h_lock[i].lock = 0;
kh_load(POS64, TCB->sub_h[i], fp);
}
free(index_name);
fclose(fp);
fprintf(stderr, "Index has been loaded.\n");
return 1;
}
*/
typedef struct
{
@@ -988,242 +799,6 @@ void insert_H_peaks(H_peaks* h, long long index, long long value)
h->list[index] += value;
}
///1: a > b; -1: a < b; 0: a=b
int cmp_Hash_code(Hash_code* a, Hash_code* b)
{
if(a->x[1] > b->x[1])
{
return 1;
}
if(a->x[1] < b->x[1])
{
return -1;
}
///a->x[1] == b->x[1]
if(a->x[0] > b->x[0])
{
return 1;
}
if(a->x[0] < b->x[0])
{
return -1;
}
return 0;
}
void get_total_freq(Total_Count_Table* TCB, uint64_t sub_ID, uint64_t sub_key, long long* T_count)
{
khint_t t = ha_ct_get(TCB->sub_h[sub_ID], sub_key);
*T_count = kh_val(TCB->sub_h[sub_ID], t);
}
void get_peak(Total_Count_Table* TCB, long long* min, long long* max, long long* up_boundary)
{
int i;
Count_Table* h;
khint_t k;
long long count;
H_peaks LH;
LH.list = NULL;
LH.length = 0;
uint64_t sub_ID;
uint64_t sub_key;
for (i = 0; i < TCB->size; i++)
{
h = TCB->sub_h[i];
for (k = 0; k != kh_end(h); ++k)
{
if (kh_exist(h, k)) // test if a bucket contains data
{
sub_ID = i;
sub_key = kh_key(h, k);
get_total_freq(TCB, sub_ID, sub_key, &count);
insert_H_peaks(&LH, count, count);
}
}
}
(*max) = -1;
(*min) = -1;
long long max_value = -1;
//// seed with freq 1 is useless
for (i = 2; i < (long long)LH.length; i++)
{
if(LH.list[i] >= max_value)
{
max_value = LH.list[i];
(*max) = i;
}
}
long long opt = 4;
(*up_boundary) = -1;
for (i = (*max) + opt; i < (long long)LH.length; i++)
{
if(LH.list[i] > LH.list[i-opt])
{
long long j = i-opt;
for (; j < i; j++)
{
if(LH.list[j] < LH.list[j+1])
{
(*up_boundary) = j;
goto end_opt;
}
}
(*up_boundary) = i;
goto end_opt;
}
}
end_opt:
if((*up_boundary) == -1 || (*up_boundary) > (*max) * 10)
{
(*up_boundary) = (*max) * 10;
}
long long min_value = max_value;
//// seed with freq 1 is useless
for (i = 2; i < (long long)LH.length && i < (*max); i++)
{
if(LH.list[i] < min_value && LH.list[i] != 0)
{
min_value = LH.list[i];
(*min) = i;
}
}
free(LH.list);
}
void Traverse_Counting_Table(Total_Count_Table* TCB, Total_Pos_Table* PCB, int k_mer_min_freq, int k_mer_max_freq)
{
int i;
Count_Table* h;
khint_t k;
uint64_t sub_key;
uint64_t sub_ID;
PCB->useful_k_mer = 0;
PCB->total_occ = 0;
long long freq_min, max, freq_up;
///get_peak_debug(TCB, &freq_min, &freq_max);
get_peak(TCB, &freq_min, &max, &freq_up);
// fprintf(stdout, "freq_min: %d, freq_max: %d, freq_up:%d\n",
// freq_min, max, freq_up);
if(freq_min < k_mer_min_freq)
{
k_mer_min_freq = freq_min;
}
if(freq_up > k_mer_max_freq)
{
k_mer_max_freq = freq_up;
}
// fprintf(stdout, "k_mer_min_freq: %d, k_mer_max_freq: %d\n",
// k_mer_min_freq, k_mer_max_freq);
khint_t t;
int absent;
long long count;
/********************************************
hash_table(key) ----> PCB->k_mer_index ------> PCB->pos
********************************************/
for (i = 0; i < TCB->size; i++)
{
h = TCB->sub_h[i];
for (k = 0; k != kh_end(h); ++k)
{
if (kh_exist(h, k)) // test if a bucket contains data
{
sub_ID = i;
sub_key = kh_key(h, k);
get_total_freq(TCB, sub_ID, sub_key, &count);
if (count>=k_mer_min_freq && count<=k_mer_max_freq)
{
t = ha_pt_put(PCB->sub_h[sub_ID], sub_key, &absent);
if (absent)
{
///kh_val(PCB->sub_h[sub_ID], t) = useful_k_mer + total_occ;
kh_val(PCB->sub_h[sub_ID], t) = PCB->useful_k_mer;
}
else
{
///kh_val(PCB->sub_h[sub_ID], t)++;
fprintf(stderr, "ERROR\n");
}
PCB->useful_k_mer++;
PCB->total_occ = PCB->total_occ + kh_val(h, k);
}
}
}
}
// fprintf(stdout, "useful_k_mer: %lld\n",PCB->useful_k_mer);
// fprintf(stdout, "total_occ: %lld\n",PCB->total_occ);
PCB->k_mer_index = (uint64_t*)malloc(sizeof(uint64_t)*(PCB->useful_k_mer+1));
PCB->k_mer_index[0] = 0;
PCB->total_occ = 0;
PCB->useful_k_mer = 0;
for (i = 0; i < TCB->size; i++)
{
h = TCB->sub_h[i];
for (k = 0; k != kh_end(h); ++k)
{
if (kh_exist(h, k)) // test if a bucket contains data
{
sub_ID = i;
sub_key = kh_key(h, k);
get_total_freq(TCB, sub_ID, sub_key, &count);
///if (kh_val(h, k)>=k_mer_min_freq && kh_val(h, k)<=k_mer_max_freq)
if (count>=k_mer_min_freq && count<=k_mer_max_freq)
{
PCB->useful_k_mer++;
PCB->total_occ = PCB->total_occ + kh_val(h, k);
PCB->k_mer_index[PCB->useful_k_mer] = PCB->total_occ;
}
}
}
}
PCB->pos = (k_mer_pos*)malloc(sizeof(k_mer_pos)*PCB->total_occ);
memset(PCB->pos, 0, sizeof(k_mer_pos)*PCB->total_occ);
}
int cmp_k_mer_pos(const void * a, const void * b)
{
if ((*(k_mer_pos*)a).readID != (*(k_mer_pos*)b).readID)
{
return (*(k_mer_pos*)a).readID > (*(k_mer_pos*)b).readID ? 1 : -1;
}
else
{
if ((*(k_mer_pos*)a).offset != (*(k_mer_pos*)b).offset)
{
return (*(k_mer_pos*)a).offset > (*(k_mer_pos*)b).offset ? 1 : -1;
}
else
{
return 0;
}
}
}
void init_Chain_Data(Chain_Data* x)
{
x->length = 0;