diff --git a/lisa_hash.h b/lisa_hash.h deleted file mode 100644 index 5d1b89a..0000000 --- a/lisa_hash.h +++ /dev/null @@ -1,307 +0,0 @@ -/************************************************************************************* -MIT License - -Copyright (c) 2020 Intel Labs - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -Authors: Saurabh Kalikar ; Sanchit Misra ; -*****************************************************************************************/ - -#include -#include -#include -#include -#include -#include "RMI.h" -using namespace std; - -template -class lisa_hash{ - public: - - rmi_val_t* p; - rmi_val_t* p_bin; - - private: - - rmi_key_t* keys; - - uint64_t* values_enc; - uint64_t* values_enc_bin; - - uint64_t keys_size; - uint64_t p_size; - - RMI *rmi; - - void mem_alloc(uint64_t hash_size, uint64_t p_size){ - - // Removed for memory optimization - #if 0 - keys = (rmi_key_t*)malloc((1+hash_size)*sizeof(rmi_key_t)); - keys = keys + 1; // keys[-1] stores total number of keys - #endif - values_enc = (uint64_t*)malloc(hash_size*sizeof(uint64_t)); - - p = (rmi_val_t*)malloc(p_size*sizeof(rmi_val_t)); - - //values_enc_bin = (uint64_t*)malloc(hash_size*sizeof(uint64_t)); - - //p_bin = (rmi_val_t*)malloc(p_size*sizeof(rmi_val_t)); - - keys_size = hash_size; - // keys[-1] = keys_size; - this->p_size = p_size; - - fprintf(stderr, "Memory allocated %lld \n", p_size); - - } - //This function is no longer used. - void store_sorted_keys(string binFileName){ - - - ofstream wf(binFileName, ios::out | ios::binary); - wf.write((char*)&keys[-1], (1+keys_size)*sizeof(rmi_key_t)); - - wf.close(); - - rmi_key_t *temp_keys = (rmi_key_t*)malloc((1 + keys_size)*sizeof(rmi_key_t)); - ifstream rf(binFileName, ios::out | ios::binary); - rf.read((char*)&temp_keys[0], (1 + keys_size)*sizeof(rmi_key_t)); - - if(temp_keys[0] == keys[-1]){ - for(int i = 0; i < keys_size; i++){ - if(temp_keys[i+1] != keys[i]){ - cout<<"Error: "<> 32; - p_size = (uint32_t) values_enc[i]; - } - - - - void load_bin(string inputFile){ - fprintf(stderr, "Loading from bin\n"); - - string f1_name = (string) inputFile + "_pos_bin"; - string f2_name = (string) inputFile + "_val_bin"; - ifstream instream_f1(f1_name, ifstream::binary); - ifstream instream_f2(f2_name, ifstream::binary); - instream_f1.seekg(0); - instream_f1.read((char*)&values_enc[0], keys_size*sizeof(uint64_t)); - instream_f1.close(); - - instream_f2.seekg(0); - instream_f2.read((char*)&p[0], p_size*sizeof(uint64_t)); - instream_f2.close(); - /* - for(uint64_t i = 0; i < p_size; i++){ - if(p[i] != p_bin[i]){ - fprintf(stderr, "Error!! %lld %lld\n",p[i], p_bin[i] ); - } - } - for(uint64_t i = 0; i < keys_size; i++){ - if(values_enc[i] != values_enc_bin[i]){ - fprintf(stderr, "Error!! %lld %lld\n", values_enc[i], values_enc_bin[i]); - } - } - */ - } - - void load(string inputFile){ - - ifstream f(inputFile); - - rmi_key_t key; - rmi_val_t val; - uint64_t n; - int64_t offset = 0; - int64_t i = 0; - while(f>>key){ - f>>n; - //values_enc[++key_size] = val_size << 32 | n; - for(int j = 0; j < n; j++){ - f>>val; - p[offset + j] = val; - } - set_key_value(i, key, offset, n); - offset +=n; - i++; - } - } - - - - public: - lisa_hash(string inputFile, char* rmi_prefix, long leaf = 0){ - uint64_t start_time, load_time, rmi_building_time, rmi_object; - start_time = __rdtsc(); - - uint64_t val_count = 0; - - ifstream f_size(inputFile+"_size"); - f_size>>keys_size; - f_size>>p_size; - mem_alloc(keys_size, p_size); - - fprintf(stderr, "Num_keys: %lld, num_values = %lld", keys_size, p_size); - //load(inputFile); - load_bin(inputFile); - - string prefix = inputFile + "_keys"; -#ifdef UINT64 - string keys_bin_file_name = prefix + ".uint64"; -#else - string keys_bin_file_name = prefix + ".f64"; -#endif - - ifstream rf(keys_bin_file_name, ios::out | ios::binary); - if(!rf.good()){ - cout<<"Error: Binary file with keys not found!!\n"; - //call store keys - //store_sorted_keys(keys_bin_file_name); - exit(0); - } - - - string keys_rmi_file = prefix + ".rmi_PARAMETERS"; - ifstream rmi_f(keys_rmi_file, ios::out | ios::binary); - if(!rmi_f.good() || leaf != 0){ - cout<<"rmi file not found: "<< keys_rmi_file <(&prefix[0]); - - - } - - - rmi_val_t* get_hash_value(rmi_key_t key, int*n){ - uint64_t index = rmi->lookup(key); - if(index == -1){ - cout<<"Key not found\n"; - return NULL; - } - uint64_t pos, p_size; - - get_val(index, pos, p_size); - *n = p_size; - return &p[pos]; - } - - rmi_val_t* get_hash_values_batched(rmi_key_t *keys, uint64_t num_keys, int* &num_values){ - - int64_t *pos = (int64_t*) malloc(num_keys*sizeof(int64_t)); - - rmi->lookup_batched(keys, num_keys, &pos[0]); - - - num_values = (int*) malloc(num_keys*sizeof(int)); - rmi_val_t **p_ptrs = (rmi_val_t**) malloc(num_keys*sizeof(rmi_val_t*)); - uint64_t total_num_values = 0; - for(int i = 0; i < num_keys; i++){ - if(pos[i] == -1){ - num_values[i] = 0; - continue; - } - num_values[i] = (uint32_t) values_enc[pos[i]]; - total_num_values+= num_values[i]; - p_ptrs[i] = &p[values_enc[pos[i]] >> 32]; - } - free(pos); - rmi_val_t* ret_values = (rmi_val_t*) malloc(total_num_values*sizeof(rmi_val_t)); - - uint64_t cnt= 0; - for(int i = 0; i < num_keys; i++){ - - rmi_val_t *value_start_ptr = p_ptrs[i]; - uint64_t numhit = num_values[i]; - - - for(int j = 0; j < numhit; j++){ - ret_values[cnt++] = value_start_ptr[j]; - } - } - free(p_ptrs); - return ret_values; - } - - - void mm_idx_get_batched(uint64_t* &minimizers, uint64_t num_minimizers, int64_t* &pos, uint64_t** &p_ptrs, int* &num_hits){ - - rmi->lookup_batched(minimizers, num_minimizers, &pos[0]); - - for(int i = 0; i < num_minimizers; i++){ - int64_t p_i = pos[i]; - - if(p_i < 0 || p_i > keys_size ) - num_hits[i] = 0; - else - num_hits[i] = (uint32_t) values_enc[p_i]; - p_ptrs[i] = p + (values_enc[p_i] >> 32); - } - } - ~lisa_hash(){ - delete rmi; - free(values_enc); - free(p); - } - -};