From ada03ea388d55bfc285182e602b7cc63bec37200 Mon Sep 17 00:00:00 2001 From: Haoyu Cheng Date: Wed, 13 Nov 2019 20:18:11 -0500 Subject: [PATCH] final DP-based assembly --- .vscode/settings.json | 11 +- Assembly.cpp | 1849 +++++++--- CommandLines.cpp | 21 +- CommandLines.h | 6 + Correct.cpp | 6136 +++++++++++++++++++++++++++++--- Correct.h | 849 ++++- Hash_Table.cpp | 1540 +++++++- Hash_Table.h | 138 +- Makefile | 2 +- Overlaps.cpp | 7709 +++++++++++++++++++++++++++++++++++++++++ Overlaps.h | 322 ++ POA.cpp | 428 ++- POA.h | 788 ++++- Process_Read.cpp | 96 +- Process_Read.h | 66 +- kdq.h | 128 + khash.h | 4 +- kmer.cpp | 180 +- kmer.h | 19 + ksort.h | 187 + kvec.h | 110 + main.cpp | 21 +- 22 files changed, 19432 insertions(+), 1178 deletions(-) create mode 100644 Overlaps.cpp create mode 100644 Overlaps.h create mode 100644 kdq.h create mode 100644 ksort.h create mode 100644 kvec.h diff --git a/.vscode/settings.json b/.vscode/settings.json index 90b3c25..4084c7f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,6 +16,15 @@ "string": "cpp", "array": "cpp", "utility": "cpp", - "cmath": "cpp" + "cmath": "cpp", + "unordered_map": "cpp", + "system_error": "cpp", + "set": "cpp", + "sstream": "cpp", + "istream": "cpp", + "ostream": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "typeinfo": "cpp" } } \ No newline at end of file diff --git a/Assembly.cpp b/Assembly.cpp index 25773c4..017c7dc 100644 --- a/Assembly.cpp +++ b/Assembly.cpp @@ -13,6 +13,7 @@ Total_Count_Table TCB; Total_Pos_Table PCB; All_reads R_INF; +Assembly_Graph assembly; pthread_mutex_t statistics; long long total_matched_overlap_0 = 0; @@ -21,6 +22,7 @@ long long total_potiental_matched_overlap_0 = 0; long long total_potiental_matched_overlap_1 = 0; long long total_num_read_base = 0; long long total_num_correct_base = 0; +long long total_second_num_correct_base = 0; int roundID = 0; @@ -726,6 +728,9 @@ void debug_merge_result(Candidates_list* x, Candidates_list* y) + + + void* Overlap_calculate(void* arg) { @@ -883,69 +888,6 @@ Output_buffer_sub_block* current_sub_buffer) } -int get_required_read(const char *required_name, long long RID, All_reads* R_INF) -{ - - - - int required_name_length = strlen(required_name); - char* debug_name = Get_NAME((*R_INF), RID); - int debug_name_length = Get_NAME_LENGTH((*R_INF), RID); - int i; - - - if (required_name_length == debug_name_length) - { - for (i = 0; i < debug_name_length; i++) - { - if (required_name[i] != debug_name[i]) - { - break; - } - } - - if (i == debug_name_length) - { - fprintf(stderr, "required_name: %s\n", required_name); - - return 1; - ///fprintf(stderr, "read_length: %d\n", R_INF->g_read->length); - - /** - int aviable_overlap_name = 0; - for (i = 0; i < overlap_list->length; i++) - { - long long Len_x = overlap_list->list[i].x_pos_e - overlap_list->list[i].x_pos_s + 1; - - if (Len_x * OVERLAP_THRESHOLD <= overlap_list->list[i].align_length) - { - fprintf(stderr, "a_i: %d\n", aviable_overlap_name); - fprintf(stderr, "x_pos_s: %d, x_pos_e: %d\n", - overlap_list->list[i].x_pos_s, overlap_list->list[i].x_pos_e); - aviable_overlap_name++; - - debug_name = Get_NAME((*R_INF), overlap_list->list[i].y_id); - debug_name_length = Get_NAME_LENGTH((*R_INF), overlap_list->list[i].y_id); - - int j = 0; - for (j = 0; j < debug_name_length; j++) - { - fprintf(stderr, "%c", debug_name[j]); - } - fprintf(stderr, "\n"); - - } - } - **/ - } - - } - - - return 0; - -} - void get_corrected_read_from_cigar(Cigar_record* cigar, char* pre_read, int pre_length, char* new_read, int* new_length) @@ -964,28 +906,26 @@ char* new_read, int* new_length) if (operation == 0) { + ///fprintf(stderr, "0 new_i: %d\n", new_i); memcpy(new_read + new_i, pre_read + pre_i, operation_length); pre_i = pre_i + operation_length; new_i = new_i + operation_length; + ///fprintf(stderr, "0 new_i: %d\n", new_i); } else if (operation == 1) { for (j = 0; j < operation_length; j++) { + // fprintf(stderr, "1 new_i: %d, diff_char_i: %d, lost_base_length: %d, lost_base: %d\n", + // new_i, diff_char_i, cigar->lost_base_length, cigar->lost_base[diff_char_i]); + new_read[new_i] = Get_MisMatch_Base(cigar->lost_base[diff_char_i]); new_i++; diff_char_i++; + ///fprintf(stderr, "1 new_i: %d\n", new_i); } pre_i = pre_i + operation_length; - - /** - pre_i = pre_i + operation_length; - memcpy(new_read + new_i, cigar->lost_base + diff_char_i, operation_length); - new_i = new_i + operation_length; - diff_char_i = diff_char_i + operation_length; - **/ - } else if (operation == 3) { @@ -994,12 +934,20 @@ char* new_read, int* new_length) } else if (operation == 2) { + ///fprintf(stderr, "2 new_i: %d\n", new_i); memcpy(new_read + new_i, cigar->lost_base + diff_char_i, operation_length); new_i = new_i + operation_length; diff_char_i = diff_char_i + operation_length; + ///fprintf(stderr, "2 new_i: %d\n", new_i); } } *new_length = new_i; + + // if(pre_i != pre_length) + // { + // fprintf(stderr, "error\n"); + // } + ///0xffffffff; } @@ -1051,6 +999,22 @@ void get_uncorrected_read_from_cigar(Cigar_record* cigar, char* new_read, int ne *pre_length = pre_i; } + +inline int get_cigar_errors(Cigar_record* cigar) +{ + int i; + int total_errors = 0; + for (i = 0; i < cigar->length; i++) + { + if (Get_Cigar_Type(cigar->record[i]) > 0) + { + total_errors = total_errors + Get_Cigar_Length(cigar->record[i]); + } + } + + return total_errors; +} + int debug_cigar(Cigar_record* cigar, char* pre_read, int pre_length, char* new_read, int new_length, int correct_base) { @@ -1069,6 +1033,8 @@ char* new_read, int new_length, int correct_base) fprintf(stderr, "total_errors: %d, correct_base: %d\n", total_errors, correct_base); } + + int pre_i, new_i; int operation, operation_length; pre_i = new_i = 0; @@ -1103,17 +1069,22 @@ char* new_read, int new_length, int correct_base) } - /** - fprintf(stderr, "total_errors: %d, correct_base: %d, length: %d, lost_base_length: %d\n", - total_errors, correct_base, cigar->length, cigar->lost_base_length); - **/ - if (pre_i != pre_length || new_i != new_length) + + if (pre_i != pre_length) { fprintf(stderr, "pre_i: %d, pre_length: %d\n", pre_i, pre_length); + } + + + if(new_i != new_length) + { fprintf(stderr, "new_i: %d, new_length: %d\n", new_i, new_length); } + return 1; + + char* tmp_seq = (char*)malloc(new_length + pre_length); int tmp_length; @@ -1184,59 +1155,678 @@ inline void push_cigar(Compressed_Cigar_record* records, long long ID, Cigar_rec } -void just_debug(overlap_region_alloc* overlap_list, All_reads* R_INF) + +void push_overlaps(ma_hit_t_alloc* paf, overlap_region_alloc* overlap_list, int flag) { - int j; - int i; - int y_id; - int y_strand; - int y_readLen; - int overlap_length; - int high_quality_overlaps = 0; - UC_Read g_read; - init_UC_Read(&g_read); - - for (j = 0; j < overlap_list->length; j++) + long long i = 0; + ma_hit_t tmp; + clear_ma_hit_t_alloc(paf); + for (i = 0; i < overlap_list->length; i++) { - y_id = overlap_list->list[j].y_id; - y_strand = overlap_list->list[j].y_pos_strand; - y_readLen = Get_READ_LENGTH((*R_INF), y_id); - overlap_length = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; - if (overlap_length * OVERLAP_THRESHOLD <= overlap_list->list[j].align_length) + if (overlap_list->list[i].is_match == flag) { - high_quality_overlaps++; + tmp.qns = overlap_list->list[i].x_id; + tmp.qns = tmp.qns << 32; + tmp.qns = tmp.qns | (uint64_t)(overlap_list->list[i].x_pos_s); - fprintf(stderr, "i: %d, overlap_length: %d, align_length: %d, y_strand: %d\n", - high_quality_overlaps, overlap_length, overlap_list->list[j].align_length, y_strand); - fprintf(stderr, "x_pos_s: %d, x_pos_e: %d\n", - overlap_list->list[j].x_pos_s, overlap_list->list[j].x_pos_e); + tmp.qe = overlap_list->list[i].x_pos_e; - fprintf(stderr, "%.*s\n\n", Get_NAME_LENGTH((*R_INF), y_id), Get_NAME((*R_INF), y_id)); + tmp.tn = overlap_list->list[i].y_id; + tmp.ts = overlap_list->list[i].y_pos_s; + tmp.te = overlap_list->list[i].y_pos_e; + ///for overlap_list, the x_strand of all overlaps are 0, so the tmp.rev is the same as the y_strand + tmp.rev = overlap_list->list[i].y_pos_strand; + + tmp.bl = R_INF.read_length[overlap_list->list[i].y_id]; + tmp.ml = overlap_list->list[i].strong; + + add_ma_hit_t_alloc(paf, &tmp); + } + } + +} + +int check_weak_overlap(ma_hit_t_alloc* reverse_paf_list, +overlap_region_alloc* overlap_list, long long weakID) +{ + long long i = 0; + long long strongID, index; + for (i = 0; i < overlap_list->length; i++) + { + ///if this is a matched strong overlap + if (overlap_list->list[i].is_match == 1 && overlap_list->list[i].strong == 1) + { + strongID = overlap_list->list[i].y_id; + index = get_specific_overlap(&(reverse_paf_list[strongID]), strongID, weakID); + if(index != -1) + { + return 0; + } + } + } + + return 1; +} + +int if_exact_match(char* x, long long xLen, char* y, long long yLen, +long long xBeg, long long xEnd, long long yBeg, long long yEnd) +{ + long long overlapLen = xEnd - xBeg + 1; + + if(yEnd - yBeg + 1 == overlapLen) + { + long long i; + + for (i = 0; i < overlapLen; i++) + { + if(x[xBeg + i] != y[yBeg + i]) + { + break; + } + } + + if(i == overlapLen) + { + return 1; + } + } + + return 0; +} + +long long push_final_overlaps(ma_hit_t_alloc* paf, ma_hit_t_alloc* reverse_paf_list, +overlap_region_alloc* overlap_list, UC_Read* x_read, UC_Read* y_read) +{ + long long i = 0; + long long available_overlaps = 0; + ma_hit_t tmp; + clear_ma_hit_t_alloc(paf); + for (i = 0; i < overlap_list->length; i++) + { + if (overlap_list->list[i].is_match == 1) + { + available_overlaps++; + /**********************query***************************/ + //the interval of overlap is half-open [start, end) + tmp.qns = overlap_list->list[i].x_id; + tmp.qns = tmp.qns << 32; + tmp.qns = tmp.qns | (uint64_t)(overlap_list->list[i].x_pos_s); + ///the end pos is open + tmp.qe = overlap_list->list[i].x_pos_e + 1; + /**********************query***************************/ + + + + ///for overlap_list, the x_strand of all overlaps are 0, so the tmp.rev is the same as the y_strand + tmp.rev = overlap_list->list[i].y_pos_strand; + + + /**********************target***************************/ + tmp.tn = overlap_list->list[i].y_id; + if(tmp.rev == 1) + { + long long y_readLen = R_INF.read_length[overlap_list->list[i].y_id]; + tmp.ts = y_readLen - overlap_list->list[i].y_pos_e - 1; + tmp.te = y_readLen - overlap_list->list[i].y_pos_s - 1; + } + else + { + tmp.ts = overlap_list->list[i].y_pos_s; + tmp.te = overlap_list->list[i].y_pos_e; + } + ///the end pos is open + tmp.te++; + /**********************target***************************/ + + tmp.bl = R_INF.read_length[overlap_list->list[i].y_id]; + tmp.ml = overlap_list->list[i].strong; + + + + + + + if(overlap_list->list[i].y_pos_strand == 0) + { + recover_UC_Read(y_read, &R_INF, overlap_list->list[i].y_id); + } + else + { + recover_UC_Read_RC(y_read, &R_INF, overlap_list->list[i].y_id); + } + + tmp.el = if_exact_match(x_read->seq, x_read->length, y_read->seq, y_read->length, + overlap_list->list[i].x_pos_s, overlap_list->list[i].x_pos_e, + overlap_list->list[i].y_pos_s, overlap_list->list[i].y_pos_e); + + + + + + add_ma_hit_t_alloc(paf, &tmp); } } - recover_UC_Read_RC(&g_read, R_INF, overlap_list->list[0].x_id); + return available_overlaps; + +} - for (i = 0; i < g_read.length && i < 81; i++) +int fix_overlap_region_by_cigar(long long* r_beg, long long* r_end, Cigar_record* cigar, long long new_read_length, +long long pre_read_length, long long y_ID) +{ + long long pre_r_beg = (*r_beg); + long long pre_r_end = (*r_end); + (*r_beg) = (*r_end) = -1; + long long pre_i, new_i, cigar_i; + pre_i = new_i = cigar_i = 0; + int operation, operationLen; + + for (cigar_i = 0; cigar_i < cigar->length; cigar_i++) { - fprintf(stderr, "%c", g_read.seq[i]); + operation = Get_Cigar_Type(cigar->record[cigar_i]); + operationLen = Get_Cigar_Length(cigar->record[cigar_i]); + if (operation == 0 || operation == 1) + { + if(pre_r_beg >= pre_i && pre_r_beg < pre_i + operationLen) + { + (*r_beg) = new_i + (pre_r_beg - pre_i); + } + + + if(pre_r_end >= pre_i && pre_r_end < pre_i + operationLen) + { + (*r_end) = new_i + (pre_r_end - pre_i); + } + + if((*r_beg) != -1 && (*r_end) != -1) + { + return 1; + } + + + new_i += operationLen; + pre_i += operationLen; + } + else if (operation == 2) ///2是x缺字符（y多字符） + { + new_i += operationLen; + }///3是y缺字符（x多字符） + else if (operation == 3) + { + + if(pre_r_beg >= pre_i && pre_r_beg < pre_i + operationLen) + { + (*r_beg) = new_i - 1; + if((*r_beg) < 0) + { + (*r_beg) = 0; + } + } + + + if(pre_r_end >= pre_i && pre_r_end < pre_i + operationLen) + { + (*r_end) = new_i; + if((*r_end) >= new_read_length) + { + (*r_end) = new_read_length - 1; + } + } + + if((*r_beg) != -1 && (*r_end) != -1) + { + return 1; + } + + pre_i += operationLen; + } } - fprintf(stderr, "\n"); + if(pre_i != pre_read_length) + { + if(memcmp("m64013_190412_043951/172426111/ccs", Get_NAME(R_INF, y_ID), + Get_NAME_LENGTH(R_INF, y_ID)) == 0) + { + fprintf(stderr, "error, pre_i: %d, new_i:%d, pre_read_length: %d, new_read_length:%d\n", + pre_i, new_i, pre_read_length, new_read_length); + for (cigar_i = 0; cigar_i < cigar->length; cigar_i++) + { + operation = Get_Cigar_Type(cigar->record[cigar_i]); + operationLen = Get_Cigar_Length(cigar->record[cigar_i]); + fprintf(stderr, "operation: %d, operationLen: %d\n", operation, operationLen); + } + + } + ///fprintf(stderr, "error, pre_i: %d, pre_read_length: %d\n", pre_i, pre_read_length); + return 0; + } - fprintf(stderr, "x_length: %d\n", g_read.length); + ///return; + if((*r_end) == -1 && pre_r_end >= new_read_length) + { + (*r_end) = new_read_length - 1; + } + + if((*r_beg) == -1 && pre_r_beg >= new_read_length) + { + (*r_beg) = new_read_length - 1; + } + + return 1; + + // if((*r_beg) == -1 || (*r_end) == -1) + // { + // fprintf(stderr, "pre_r_beg: %d, pre_r_end: %d, pre_read_length: %d, new_read_length: %d\n", + // pre_r_beg, pre_r_end, pre_read_length, new_read_length); + // for (cigar_i = 0; cigar_i < cigar->length; cigar_i++) + // { + // operation = Get_Cigar_Type(cigar->record[cigar_i]); + // operationLen = Get_Cigar_Length(cigar->record[cigar_i]); + // fprintf(stderr, "operation: %d, operationLen: %d\n", operation, operationLen); + // } + // } +} + +void convert_kmer(k_v* kv, Hash_code* k_code, uint64_t end_pos) +{ + kv->key.x[0] = k_code->x[0]; + kv->key.x[1] = k_code->x[1]; + kv->value = end_pos; +} + + +void debug_sort_small_hash(small_hash_table* Table) +{ + long long i; + for (i = 1; i < Table->length; i++) + { + if(compare_k_mer(&Table->buffer[i], &Table->buffer[i-1]) < 0) + { + fprintf(stderr, "error\n"); + } + + if(compare_k_mer(&Table->buffer[i], &Table->buffer[i-1]) == 0) + { + if(Table->buffer[i].value < Table->buffer[i-1].value) + { + fprintf(stderr, "error\n"); + } + } + } + + for (i = 0; i < Table->length; i++) + { + long long left, right; + + if(query_small_hash_table(Table, &Table->buffer[i], &left, &right) == 0) + { + fprintf(stderr, "Table->length: %d, i: %d, x[1]: %llu, x[0]: %llu, value: %llu\n", + Table->length, i, + Table->buffer[i].key.x[1], Table->buffer[i].key.x[0], + Table->buffer[i].value); + } + + long long single_count = 0; + long long first_i = -1; + + for (long long j = 0; j < Table->length; j++) + { + if(compare_k_mer(&Table->buffer[i], &Table->buffer[j]) == 0) + { + single_count++; + if(first_i == -1) + { + first_i = j; + } + } + else if(compare_k_mer(&Table->buffer[i], &Table->buffer[j]) < 0) + { + break; + } + + } + + if(left != first_i) + { + fprintf(stderr, "error left\n"); + } + + if(right - left + 1 != single_count) + { + fprintf(stderr, "error right\n"); + } + + } +} + +void get_candidates_from_existing_overlaps(long long readID, UC_Read* g_read, UC_Read* overlap_read, +overlap_region_alloc* overlap_list, k_mer_pos_list_alloc* array_list, +HeapSq* heap, Candidates_list* l, small_hash_table* forward, small_hash_table* reverse) +{ + HPC_seq HPC_read; + Hash_code k_code; + long long avalible_k; + uint64_t code; + uint64_t end_pos; + k_mer_pos* list; + uint64_t list_length; + uint64_t sub_ID; + k_v k_mer_kv; + + clear_Heap(heap); + clear_Candidates_list(l); + + clear_k_mer_pos_list_alloc(array_list); + clear_overlap_region_alloc(overlap_list); + + clear_small_hash_table(forward); + clear_small_hash_table(reverse); + + recover_UC_Read(g_read, &R_INF, readID); + + ///forward strand + init_HPC_seq(&HPC_read, g_read->seq, g_read->length); + init_Hash_code(&k_code); + avalible_k = 0; + + while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) + { + if(code < 4) + { + k_mer_append(&k_code,code, k_mer_length); + avalible_k++; + if (avalible_k>= k_mer_length) + { + if(if_k_mer_available(&k_code, k_mer_length)) + { + convert_kmer(&k_mer_kv, &k_code, end_pos); + add_small_hash_table(forward, &k_mer_kv); + } + } + } + else + { + avalible_k = 0; + init_Hash_code(&k_code); + } + } + + sort_small_hash_table(forward); + + ///debug_sort_small_hash(forward); + + + + /** + ///reverse complement strand + reverse_complement(g_read->seq, g_read->length); + init_HPC_seq(&HPC_read, g_read->seq, g_read->length); + init_Hash_code(&k_code); + avalible_k = 0; + + while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) + { + if(code < 4) + { + k_mer_append(&k_code,code, k_mer_length); + avalible_k++; + if (avalible_k>= k_mer_length) + { + if(if_k_mer_available(&k_code, k_mer_length)) + { + convert_kmer(&k_mer_kv, &k_code, end_pos); + add_small_hash_table(reverse, &k_mer_kv); + } + } + } + else + { + avalible_k = 0; + init_Hash_code(&k_code); + } + } + + sort_small_hash_table(reverse); + + ///debug_sort_small_hash(reverse); + **/ + + long long i; + long long y_start, y_end, y_strand, y_ID, y_overlapLen, extraLen, y_readLen, tmp; + char* new_y_string; + long long new_y_length; + long long result_left, result_right, result_occ; + long long total_result_occ = 0; + Cigar_record cigar; + for (i = 0; i < R_INF.paf[readID].length; i++) + { + y_ID = R_INF.paf[readID].buffer[i].tn; + y_start = R_INF.paf[readID].buffer[i].ts; + y_end = R_INF.paf[readID].buffer[i].te; + y_strand = R_INF.paf[readID].buffer[i].rev; + y_readLen = R_INF.read_length[y_ID]; + + + + y_overlapLen = y_end - y_start + 1; + extraLen = y_overlapLen * 0.1; + y_start = y_start - extraLen; + if(y_start < 0) + { + y_start = 0; + } + y_end = y_end + extraLen; + if(extraLen >= y_readLen) + { + y_end = y_readLen - 1; + } + + + y_start = y_readLen - y_start - 1; + y_end = y_readLen - y_end - 1; + tmp = y_start; + y_start = y_end; + y_end = tmp; + + + + + if(y_strand == 0) + { + recover_UC_Read(overlap_read, &R_INF, y_ID); + } + else + { + recover_UC_Read_RC(overlap_read, &R_INF, y_ID); + } + + new_y_string = overlap_read->seq + y_start; + new_y_length = y_end - y_start + 1; + + /** + new_y_string = overlap_read->seq; + new_y_length = overlap_read->length; + **/ + total_result_occ = 0; + clear_Candidates_list(l); + + + init_HPC_seq(&HPC_read, new_y_string, new_y_length); + init_Hash_code(&k_code); + avalible_k = 0; + while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) + { + if(code < 4) + { + k_mer_append(&k_code,code, k_mer_length); + avalible_k++; + if (avalible_k>= k_mer_length) + { + if(if_k_mer_available(&k_code, k_mer_length)) + { + convert_kmer(&k_mer_kv, &k_code, end_pos); + result_occ = query_small_hash_table(forward, &k_mer_kv, &result_left, &result_right); + if(result_occ > 0) + { + total_result_occ += result_occ; + + insert_kv_list_to_candidates(forward->buffer + result_left, result_occ, y_ID, + end_pos + y_start, y_strand, l); + + /** + insert_kv_list_to_candidates(forward->buffer + result_left, result_occ, y_ID, + end_pos, y_strand, l); + **/ + } + + } + } + } + else + { + avalible_k = 0; + init_Hash_code(&k_code); + } + } + + ///要在这里整理出一个candidate位置，然后插入进overlap_list + sort_candidates(l, readID, overlap_list, &R_INF); + + // fprintf(stderr, "x_pos_s: (pre: %d), x_pos_e: (pre: %d), y_pos_s: (pre: %d), y_pos_e: (pre: %d), y_strand: (pre: %d)\n\n", + // (uint32_t)(R_INF.paf[readID].buffer[i].qns), R_INF.paf[readID].buffer[i].qe, R_INF.paf[readID].buffer[i].ts, R_INF.paf[readID].buffer[i].te, + // R_INF.paf[readID].buffer[i].rev); + + // fprintf(stderr, "x_id: %d, y_ID:%d, x_len: %d, y_len: %d, overlap_list->length: %d\n", readID, y_ID, + // R_INF.read_length[readID], R_INF.read_length[y_ID], overlap_list->length); + // fprintf(stderr, "x_pos_s: %d (pre: %d), x_pos_e: %d (pre: %d), y_pos_s: %d (pre: %d), y_pos_e: %d (pre: %d), x_strand: %d, y_strand: %d (pre: %d)\n\n\n", + // overlap_list->list[overlap_list->length - 1].x_pos_s, (uint32_t)(R_INF.paf[readID].buffer[i].qns), + // overlap_list->list[overlap_list->length - 1].x_pos_e, R_INF.paf[readID].buffer[i].qe, + // overlap_list->list[overlap_list->length - 1].y_pos_s, R_INF.paf[readID].buffer[i].ts, + // overlap_list->list[overlap_list->length - 1].y_pos_e, R_INF.paf[readID].buffer[i].te, + // overlap_list->list[overlap_list->length - 1].x_pos_strand, overlap_list->list[overlap_list->length - 1].y_pos_strand, + // R_INF.paf[readID].buffer[i].rev); + + + + // if(l->length != total_result_occ) + // { + // fprintf(stderr, "error\n"); + // } + ///fprintf(stderr, "i:%d, total_result_occ: %d, y_strand: %d, new_y_length: %d\n", i, total_result_occ, y_strand, new_y_length); + } + + + qsort(overlap_list->list, overlap_list->length, sizeof(overlap_region), cmp_by_x_pos_s); + + reverse_complement(g_read->seq, g_read->length); + + + // for (long long i = 0; i < overlap_list->length; i++) + // { + // fprintf(stderr, "****x_pos_s: %d, x_pos_e: %d, y_pos_s: %d, y_pos_e: %d, x_strand: %d, y_strand: %d, x_id: %d, y_id: %d\n", + // overlap_list->list[i].x_pos_s, overlap_list->list[i].x_pos_e, + // overlap_list->list[i].y_pos_s, overlap_list->list[i].y_pos_e, + // overlap_list->list[i].x_pos_strand, overlap_list->list[i].y_pos_strand, + // overlap_list->list[i].x_id, overlap_list->list[i].y_id); + // } + +} + +void get_new_candidates(long long readID, UC_Read* g_read, overlap_region_alloc* overlap_list, k_mer_pos_list_alloc* array_list, +HeapSq* heap, Candidates_list* l, double band_width_threshold) +{ + HPC_seq HPC_read; + Hash_code k_code; + long long avalible_k; + uint64_t code; + uint64_t end_pos; + k_mer_pos* list; + uint64_t list_length; + uint64_t sub_ID; + + clear_Heap(heap); + clear_Candidates_list(l); + + clear_k_mer_pos_list_alloc(array_list); + clear_overlap_region_alloc(overlap_list); + + recover_UC_Read(g_read, &R_INF, readID); + + ///forward strand + init_HPC_seq(&HPC_read, g_read->seq, g_read->length); + init_Hash_code(&k_code); + avalible_k = 0; + + + + while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) + { + if(code < 4) + { + k_mer_append(&k_code,code,k_mer_length); + avalible_k++; + if (avalible_k>=k_mer_length) + { + list_length = locate_Total_Pos_Table(&PCB, &k_code, &list, k_mer_length, &sub_ID); + + if (list_length != 0) + { + append_k_mer_pos_list_alloc(array_list, list, list_length, end_pos, 0); + } + } + } + else + { + avalible_k = 0; + init_Hash_code(&k_code); + } + } + + + + ///reverse complement strand + reverse_complement(g_read->seq, g_read->length); + init_HPC_seq(&HPC_read, g_read->seq, g_read->length); + init_Hash_code(&k_code); + avalible_k = 0; + + while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) + { + if(code < 4) + { + k_mer_append(&k_code,code,k_mer_length); + avalible_k++; + if (avalible_k>=k_mer_length) + { + list_length = locate_Total_Pos_Table(&PCB, &k_code, &list, k_mer_length, &sub_ID); + if (list_length != 0) + { + append_k_mer_pos_list_alloc(array_list, list, list_length, end_pos, 1); + } + } + } + else + { + avalible_k = 0; + init_Hash_code(&k_code); + } + } - fprintf(stderr, "high_quality_overlaps: %d, overlap_list->length: %d\n", high_quality_overlaps,overlap_list->length); - - destory_UC_Read(&g_read); - + merge_k_mer_pos_list_alloc_heap_sort(array_list, l, heap); + + ///以x_pos_e，即结束位置为主元排序 + ///calculate_overlap_region(l, overlap_list, readID, g_read->length, &R_INF); + calculate_overlap_region_by_chaining(l, overlap_list, readID, g_read->length, &R_INF, band_width_threshold); } + + + void* Overlap_calculate_heap_merge(void* arg) { /************需要注释掉**********/ @@ -1250,7 +1840,9 @@ void* Overlap_calculate_heap_merge(void* arg) long long potiental_matched_overlap_1 = 0; long long num_read_base = 0; long long num_correct_base = 0; + long long num_second_correct_base = 0; long long j; + int fully_cov; int thr_ID = *((int*)arg); uint64_t POA_i; @@ -1277,8 +1869,11 @@ void* Overlap_calculate_heap_merge(void* arg) Candidates_list l; //Candidates_list debug_l; Graph POA_Graph; - + Graph DAGCon; + init_Graph(&DAGCon); init_Graph(&POA_Graph); + + init_Candidates_list(&l); //init_Candidates_list(&debug_l); @@ -1310,264 +1905,58 @@ void* Overlap_calculate_heap_merge(void* arg) Round2_alignment second_round; init_Round2_alignment(&second_round); + small_hash_table forward, reverse; + init_small_hash_table(&forward); + init_small_hash_table(&reverse); + + for (i = thr_ID; i < R_INF.total_reads; i = i + thread_num) { + get_new_candidates(i, &g_read, &overlap_list, &array_list, &heap, &l, 0.02); - - - clear_Cigar_record(¤t_cigar); clear_Round2_alignment(&second_round); - - clear_Heap(&heap); - clear_Candidates_list(&l); - ///clear_Candidates_list(&debug_l); - - clear_k_mer_pos_list_alloc(&array_list); - clear_overlap_region_alloc(&overlap_list); - - recover_UC_Read(&g_read, &R_INF, i); - - - ///forward strand - init_HPC_seq(&HPC_read, g_read.seq, g_read.length); - init_Hash_code(&k_code); - avalible_k = 0; - - while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) - { - if(code < 4) - { - k_mer_append(&k_code,code,k_mer_length); - avalible_k++; - if (avalible_k>=k_mer_length) - { - list_length = locate_Total_Pos_Table(&PCB, &k_code, &list, k_mer_length, &sub_ID); - - if (list_length != 0) - { - /************需要注释掉**********/ - // debug_overlap = debug_overlap + list_length; - /************需要注释掉**********/ - append_k_mer_pos_list_alloc(&array_list, list, list_length, end_pos, 0); - ///append_k_mer_pos_list_alloc_prefilter(&array_list, list, list_length, end_pos, 0, &g_read, &R_INF, &correct); - } - ///merge_Candidates_list(&l, list, list_length, end_pos, 0); - ///merge_Candidates_list_version(&debug_l, list, list_length, end_pos, 0); - } - } - else - { - avalible_k = 0; - init_Hash_code(&k_code); - } - - ///HPC_base++; - } - - - - - - - - - ///reverse complement strand - reverse_complement(g_read.seq, g_read.length); - /** - UC_Read rc_read; - init_UC_Read(&rc_read); - recover_UC_Read_RC(&rc_read, &R_INF, i); - uint64_t j = 0; - for (j = 0; j < g_read.length; j++) - { - if (g_read.seq[j] != rc_read.seq[j]) - { - fprintf(stderr, "j error: %llu, i: %llu\n", j, i); - fprintf(stderr, "g_read.seq[j]: %c\n", g_read.seq[j]); - fprintf(stderr, "rc_read.seq[j]: %c\n", rc_read.seq[j]); - } - - } - destory_UC_Read(&rc_read); - **/ - init_HPC_seq(&HPC_read, g_read.seq, g_read.length); - init_Hash_code(&k_code); - avalible_k = 0; - - while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) - { - if(code < 4) - { - k_mer_append(&k_code,code,k_mer_length); - avalible_k++; - if (avalible_k>=k_mer_length) - { - list_length = locate_Total_Pos_Table(&PCB, &k_code, &list, k_mer_length, &sub_ID); - if (list_length != 0) - { - /************需要注释掉**********/ - // debug_overlap = debug_overlap + list_length; - /************需要注释掉**********/ - append_k_mer_pos_list_alloc(&array_list, list, list_length, end_pos, 1); - ///append_k_mer_pos_list_alloc_prefilter(&array_list, list, list_length, end_pos, 1, &g_read, &R_INF, &correct); - } - ///merge_Candidates_list(&l, list, list_length, end_pos, 1); - //merge_Candidates_list_version(&debug_l, list, list_length, end_pos, 1); - } - } - else - { - avalible_k = 0; - init_Hash_code(&k_code); - } - - ///HPC_base++; - } - - - - - /************需要注释掉**********/ - // for (int ijk = 0; ijk < array_list.length; ijk++) - // { - // filtered_debug_overlap = filtered_debug_overlap + array_list.list[ijk].length; - // } - /************需要注释掉**********/ - - ///merge_k_mer_pos_list_alloc(&array_list, &l); - merge_k_mer_pos_list_alloc_heap_sort(&array_list, &l, &heap); - - // if (array_list.length < 3) - // { - // merge_k_mer_pos_list_alloc(&array_list, &l); - // } - // else - // { - // merge_k_mer_pos_list_alloc_heap_sort_advance(&array_list, &l, &heap); - // } - - - ///以x_pos_e，即结束位置为主元排序 - calculate_overlap_region(&l, &overlap_list, i, g_read.length, &R_INF); - - ///clear_Graph(&POA_Graph); - - correct_overlap(&overlap_list, &R_INF, &g_read, &correct, &overlap_read, &POA_Graph, + correct_overlap(&overlap_list, &R_INF, &g_read, &correct, &overlap_read, &POA_Graph, &DAGCon, &matched_overlap_0, &matched_overlap_1, &potiental_matched_overlap_0, &potiental_matched_overlap_1, - ¤t_cigar, &hap, &second_round); + ¤t_cigar, &hap, &second_round, 0, 1, &fully_cov); num_read_base = num_read_base + g_read.length; num_correct_base = num_correct_base + correct.corrected_base; + num_second_correct_base = num_second_correct_base + second_round.dumy.corrected_base; push_cigar(R_INF.cigars, i, ¤t_cigar); push_cigar(R_INF.second_round_cigar, i, &(second_round.cigar)); - /** - fprintf(stderr, "current_cigar.new_read_length: %d, second_round.cigar.new_read_length: %d\n", - current_cigar.new_read_length, second_round.cigar.new_read_length); - **/ - /** - if(memcmp("m54334_180926_225337/39780640/ccs", Get_NAME(R_INF, i), Get_NAME_LENGTH(R_INF, i)) == 0) + + + R_INF.paf[i].is_fully_corrected = 0; + if(fully_cov) { - fprintf(stderr, "i: %d\n", i); - ///fprintf(stderr, "overlap_list.length: %d\n", overlap_list.length); - just_debug(&overlap_list, &R_INF); - } - **/ - - - - - - ///output_read_to_buffer(i, &R_INF, correct.corrected_read, correct.corrected_read_length, ¤t_sub_buffer); - /** - debug_cigar(¤t_cigar, g_read.seq, g_read.length, correct.corrected_read, correct.corrected_read_length, - correct.corrected_base); - **/ - - - - - - - - /** - POA_i = 0; - fprintf(stderr, "\n\n**************\ni: %u\n", i); - - for (POA_i = 0; POA_i < overlap_list.length; POA_i++) - { - fprintf(stderr, "x_id: %u, y_id: %u\n", overlap_list.list[POA_i].x_id, overlap_list.list[POA_i].y_id); - fprintf(stderr, "x_strand: %u, y_strand: %u\n", overlap_list.list[POA_i].x_pos_strand, overlap_list.list[POA_i].y_pos_strand); - fprintf(stderr, "x_pos_s: %u\n", overlap_list.list[POA_i].x_pos_s); - - } - **/ - - /** - POA_i = 0; - - for (POA_i = 1; POA_i < overlap_list.length; POA_i++) - { - if(overlap_list.list[POA_i].x_pos_s < overlap_list.list[POA_i - 1].x_pos_s) + /** + if(current_cigar.length == 1 && second_round.cigar.length == 1) { - fprintf(stderr, "1 sbsbsbsbs\n"); + if(Get_Cigar_Type(current_cigar.record[0]) == 0 && + Get_Cigar_Length(current_cigar.record[0]) == g_read.length && + Get_Cigar_Type(second_round.cigar.record[0]) == 0 && + Get_Cigar_Length(second_round.cigar.record[0]) == g_read.length) + { + R_INF.paf[i].is_fully_corrected = 1; + } } - else if(overlap_list.list[POA_i].x_pos_s == overlap_list.list[POA_i - 1].x_pos_s && - overlap_list.list[POA_i].x_pos_e > overlap_list.list[POA_i - 1].x_pos_e) + **/ + if( + get_cigar_errors(¤t_cigar) == 0 + && + get_cigar_errors(&second_round.cigar) == 0) { - fprintf(stderr, "2 sbsbsbsbs\n"); - } - - + R_INF.paf[i].is_fully_corrected = 1; + } } - **/ - - - - - - ///merge_k_mer_pos_list_alloc_heap_sort_advance(&array_list, &l, &heap); - - - - ///debug_merge_result(&l, &debug_l); - - /** - clear_Graph(&POA_Graph); - - - Perform_POA(&POA_Graph, &overlap_list, &R_INF, &g_read); - **/ - - ///fprintf(stderr, "i: %u\n", i); - - - /** - POA_i = 0; - - if (overlap_list.length > 0) - { - - } - - - for (POA_i = 1; POA_i < overlap_list.length; POA_i++) - { - if(overlap_list.list[POA_i].x_pos_strand == 1) - { - fprintf(stderr, "sbsbsbsbs\n"); - } - - } - **/ - - - + push_overlaps(&(R_INF.paf[i]), &overlap_list, 1); + push_overlaps(&(R_INF.reverse_paf[i]), &overlap_list, 2); } @@ -1595,6 +1984,9 @@ void* Overlap_calculate_heap_merge(void* arg) ///destory_k_mer_pos_list_alloc_prefilter(&array_list); destory_Graph(&POA_Graph); + destory_Graph(&DAGCon); + + destory_UC_Read(&g_read); destory_UC_Read(&overlap_read); destory_Cigar_record(¤t_cigar); @@ -1605,6 +1997,10 @@ void* Overlap_calculate_heap_merge(void* arg) destory_Round2_alignment(&second_round); + destory_small_hash_table(&forward); + destory_small_hash_table(&reverse); + + pthread_mutex_lock(&statistics); total_matched_overlap_0 += matched_overlap_0; @@ -1613,6 +2009,7 @@ void* Overlap_calculate_heap_merge(void* arg) total_potiental_matched_overlap_1 += potiental_matched_overlap_1; total_num_read_base += num_read_base; total_num_correct_base += num_correct_base; + total_second_num_correct_base +=num_second_correct_base; complete_threads++; if(complete_threads == thread_num) @@ -1623,6 +2020,8 @@ void* Overlap_calculate_heap_merge(void* arg) fprintf(stderr, "total_potiental_matched_overlap_1: %llu\n", total_potiental_matched_overlap_1); fprintf(stderr, "total_num_read_base: %llu\n", total_num_read_base); fprintf(stderr, "total_num_correct_base: %llu\n", total_num_correct_base); + fprintf(stderr, "total_second_num_correct_base: %llu\n", total_second_num_correct_base); + } pthread_mutex_unlock(&statistics); @@ -1631,6 +2030,170 @@ void* Overlap_calculate_heap_merge(void* arg) +void* Output_related_reads(void* arg) +{ + /************需要注释掉**********/ + // long long debug_overlap = 0; + // long long filtered_debug_overlap = 0; + /************需要注释掉**********/ + + long long matched_overlap_0 = 0; + long long matched_overlap_1 = 0; + long long potiental_matched_overlap_0 = 0; + long long potiental_matched_overlap_1 = 0; + long long num_read_base = 0; + long long num_correct_base = 0; + long long num_second_correct_base = 0; + long long j; + int fully_cov; + + int thr_ID = *((int*)arg); + uint64_t POA_i; + long long i = 0; + int avalible_k = 0; + + UC_Read g_read; + init_UC_Read(&g_read); + + UC_Read overlap_read; + init_UC_Read(&overlap_read); + + HPC_seq HPC_read; + Hash_code k_code; + uint64_t code; + uint64_t end_pos; + k_mer_pos* list; + uint64_t list_length; + uint64_t sub_ID; + long long total_shared_seed = 0; + long long candidate_overlap_reads = 0; + + + Candidates_list l; + //Candidates_list debug_l; + Graph POA_Graph; + Graph DAGCon; + init_Graph(&DAGCon); + init_Graph(&POA_Graph); + + + init_Candidates_list(&l); + //init_Candidates_list(&debug_l); + + k_mer_pos_list_alloc array_list; + init_k_mer_pos_list_alloc(&array_list); + + overlap_region_alloc overlap_list; + init_overlap_region_alloc(&overlap_list); + + HeapSq heap; + + Init_Heap(&heap); + + Correct_dumy correct; + init_Correct_dumy(&correct); + + + Output_buffer_sub_block current_sub_buffer; + + init_buffer_sub_block(¤t_sub_buffer); + + Cigar_record current_cigar; + init_Cigar_record(¤t_cigar); + + haplotype_evdience_alloc hap; + InitHaplotypeEvdience(&hap); + + + Round2_alignment second_round; + init_Round2_alignment(&second_round); + + small_hash_table forward, reverse; + init_small_hash_table(&forward); + init_small_hash_table(&reverse); + + long long required_read_name_length = strlen(required_read_name); + for (i = thr_ID; i < R_INF.total_reads; i = i + thread_num) + { + + if(required_read_name_length == Get_NAME_LENGTH((R_INF),i) + && + memcmp(required_read_name, Get_NAME((R_INF), i), Get_NAME_LENGTH((R_INF),i)) == 0) + { + get_new_candidates(i, &g_read, &overlap_list, &array_list, &heap, &l, 0.02); + + + fprintf(stderr, ">%.*s\n", Get_NAME_LENGTH((R_INF), i), + Get_NAME((R_INF), i)); + recover_UC_Read(&g_read, &R_INF, i); + fprintf(stderr, "%.*s\n", g_read.length, g_read.seq); + + + long long k; + for (k = 0; k < overlap_list.length; k++) + { + fprintf(stderr, ">%.*s\n", Get_NAME_LENGTH((R_INF),overlap_list.list[k].y_id), + Get_NAME((R_INF),overlap_list.list[k].y_id)); + recover_UC_Read(&g_read, &R_INF, overlap_list.list[k].y_id); + fprintf(stderr, "%.*s\n", g_read.length, g_read.seq); + } + + } + } + + finish_output_buffer(); + + destory_buffer_sub_block(¤t_sub_buffer); + + + + + destory_Candidates_list(&l); + destory_overlap_region_alloc(&overlap_list); + //destory_Candidates_list(&debug_l); + + destory_Heap(&heap); + destory_k_mer_pos_list_alloc(&array_list); + ///destory_k_mer_pos_list_alloc_prefilter(&array_list); + + destory_Graph(&POA_Graph); + destory_Graph(&DAGCon); + + + destory_UC_Read(&g_read); + destory_UC_Read(&overlap_read); + destory_Cigar_record(¤t_cigar); + + destory_Correct_dumy(&correct); + + destoryHaplotypeEvdience(&hap); + + destory_Round2_alignment(&second_round); + + destory_small_hash_table(&forward); + destory_small_hash_table(&reverse); + + free(arg); +} + + + +inline long long get_N_occ(char* seq, long long length) +{ + long long N_occ = 0; + long long j; + for (j = 0; j < length; j++) + { + if(seq_nt6_table[seq[j]] >= 4) + { + ///fprintf(stderr, "seq[%d]: %c\n", j, seq[j]); + N_occ++; + } + } + + return N_occ; +} + void* Save_corrected_reads(void* arg) { @@ -1666,18 +2229,27 @@ void* Save_corrected_reads(void* arg) first_round_read = (char*)realloc(first_round_read, first_round_read_size); } + + + + + cigar.length = R_INF.cigars[i].length; cigar.lost_base_length = R_INF.cigars[i].lost_base_length; cigar.record = R_INF.cigars[i].record; cigar.lost_base = R_INF.cigars[i].lost_base; get_corrected_read_from_cigar(&cigar, g_read.seq, g_read.length, first_round_read, &first_round_read_length); - /** - if(first_round_read_length != R_INF.cigars[i].new_length) - { - fprintf(stderr, "error\n"); - } - **/ + + + + + + /********************debug********************/ + // N_occ = get_N_occ(first_round_read, first_round_read_length); + // fprintf(stderr, "i: %d, first N_occ: %d\n", i, N_occ); + // fflush(stderr); + /********************debug********************/ /********************************1 round******************************/ /********************************2 round******************************/ @@ -1692,35 +2264,36 @@ void* Save_corrected_reads(void* arg) cigar.lost_base = R_INF.second_round_cigar[i].lost_base; get_corrected_read_from_cigar(&cigar, first_round_read, first_round_read_length, second_round_read, &second_round_read_length); - /** - if(second_round_read_length != R_INF.second_round_cigar[i].new_length) - { - fprintf(stderr, "error\n"); - } - **/ + + + /********************debug********************/ + // N_occ = get_N_occ(second_round_read, second_round_read_length); + // fprintf(stderr, "i: %d, second N_occ: %d\n\n\n", i, N_occ); + // fflush(stderr); + /********************debug********************/ /********************************2 round******************************/ + - - ///new_read = first_round_read; - ///new_read_length = first_round_read_length; new_read = second_round_read; new_read_length = second_round_read_length; - - ///need modification - reverse_complement(new_read, new_read_length); - - - N_occ = 0; - for (j = 0; j < new_read_length; j++) + + if (roundID != number_of_round - 1) { - if(new_read[j] == 'N') - { - N_occ++; - } + ///need modification + reverse_complement(new_read, new_read_length); } + else if(number_of_round % 2 == 0) + { + ///need modification + reverse_complement(new_read, new_read_length); + } + + + N_occ = get_N_occ(new_read, new_read_length); + if(R_INF.read_size[i] < new_read_length) { @@ -1734,8 +2307,6 @@ void* Save_corrected_reads(void* arg) compress_base(Get_READ(R_INF, i), new_read, new_read_length, &R_INF.N_site[i], N_occ); - - } destory_UC_Read(&g_read); @@ -1753,6 +2324,17 @@ void Output_corrected_reads() FILE* output_file = fopen(output_file_name, "w"); + for (i = 0; i < R_INF.total_reads; i++) + { + recover_UC_Read(&g_read, &R_INF, i); + fwrite(">", 1, 1, output_file); + fwrite(Get_NAME(R_INF, i), 1, Get_NAME_LENGTH(R_INF, i), output_file); + fwrite("\n", 1, 1, output_file); + fwrite(g_read.seq, 1, g_read.length, output_file); + fwrite("\n", 1, 1, output_file); + } + + /** if(number_of_round % 2 == 0) { for (i = 0; i < R_INF.total_reads; i++) @@ -1777,6 +2359,7 @@ void Output_corrected_reads() fwrite("\n", 1, 1, output_file); } } + **/ @@ -1788,8 +2371,6 @@ void Output_corrected_reads() - - void Overlap_calculate_multipe_thr() { @@ -1797,16 +2378,7 @@ void Overlap_calculate_multipe_thr() double start_time = Get_T(); pthread_t outputResultSinkHandle; - /** - if (roundID == number_of_round - 1) - { - init_output_buffer(thread_num); - pthread_create(&outputResultSinkHandle, NULL, pop_buffer, NULL); - } - **/ - - fprintf(stdout, "Begin Overlap Calculate ...... \n"); pthread_t *_r_threads; @@ -1819,27 +2391,30 @@ void Overlap_calculate_multipe_thr() { int *arg = (int*)malloc(sizeof(*arg)); *arg = i; - - pthread_create(_r_threads + i, NULL, Overlap_calculate_heap_merge, (void*)arg); - + if(!required_read_name) + { + pthread_create(_r_threads + i, NULL, Overlap_calculate_heap_merge, (void*)arg); + } + else + { + pthread_create(_r_threads + i, NULL, Output_related_reads, (void*)arg); + } + + } for (i = 0; iseq.s, seq->seq.l); @@ -1987,7 +2565,7 @@ void verify_Position_hash_table() fprintf(stdout, "Start Verifying Position Table...\n"); - while (get_read(seq)) + while (get_read(seq, adapterLen)) { @@ -2122,8 +2700,464 @@ void verify_Position_hash_table() } + +void get_inexact_overlaps(long long readID, UC_Read* g_read, overlap_region_alloc* overlap_list, k_mer_pos_list_alloc* array_list, +HeapSq* heap, Candidates_list* l) +{ + HPC_seq HPC_read; + Hash_code k_code; + long long avalible_k; + uint64_t code; + uint64_t end_pos; + k_mer_pos* list; + uint64_t list_length; + uint64_t sub_ID; + + clear_Heap(heap); + clear_Candidates_list(l); + + clear_k_mer_pos_list_alloc(array_list); + clear_overlap_region_alloc(overlap_list); + + recover_UC_Read(g_read, &R_INF, readID); + + ///forward strand + init_HPC_seq(&HPC_read, g_read->seq, g_read->length); + init_Hash_code(&k_code); + avalible_k = 0; + + // if(g_read->length != R_INF.read_length[readID]) + // { + // fprintf(stderr, "error\n"); + // } + + while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) + { + if(code < 4) + { + k_mer_append(&k_code,code,k_mer_length); + avalible_k++; + if (avalible_k>=k_mer_length) + { + list_length = locate_Total_Pos_Table(&PCB, &k_code, &list, k_mer_length, &sub_ID); + + if (list_length != 0) + { + append_k_mer_pos_list_alloc(array_list, list, list_length, end_pos, 0); + } + } + } + else + { + avalible_k = 0; + init_Hash_code(&k_code); + } + } + + + + ///reverse complement strand + reverse_complement(g_read->seq, g_read->length); + init_HPC_seq(&HPC_read, g_read->seq, g_read->length); + init_Hash_code(&k_code); + avalible_k = 0; + + while ((code = get_HPC_code(&HPC_read, &end_pos)) != 6) + { + if(code < 4) + { + k_mer_append(&k_code,code,k_mer_length); + avalible_k++; + if (avalible_k>=k_mer_length) + { + list_length = locate_Total_Pos_Table(&PCB, &k_code, &list, k_mer_length, &sub_ID); + if (list_length != 0) + { + append_k_mer_pos_list_alloc(array_list, list, list_length, end_pos, 1); + } + } + } + else + { + avalible_k = 0; + init_Hash_code(&k_code); + } + } + + + + merge_k_mer_pos_list_alloc_heap_sort(array_list, l, heap); + calculate_inexact_overlap_region(l, overlap_list, readID, g_read->length, &R_INF); + ///calculate_overlap_region(l, overlap_list, readID, g_read->length, &R_INF); + + + + + overlap_region_sort_y_id(overlap_list->list, overlap_list->length); + ma_hit_sort_tn(R_INF.paf[readID].buffer, R_INF.paf[readID].length); + overlap_list->mapped_overlaps_length = 0; + long long inner_j = 0; + long long j = 0; + while (j < overlap_list->length && inner_j < R_INF.paf[readID].length) + { + if(overlap_list->list[j].y_id < R_INF.paf[readID].buffer[inner_j].tn) + { + j++; + } + else if(overlap_list->list[j].y_id > R_INF.paf[readID].buffer[inner_j].tn) + { + inner_j++; + } + else + { + if(overlap_list->list[j].y_pos_strand == R_INF.paf[readID].buffer[inner_j].rev) + { + overlap_list->list[j].is_match = 1; + overlap_list->mapped_overlaps_length++; + } + j++; + inner_j++; + } + } +} + + +void* Final_overlap_calculate_heap_merge(void* arg) +{ + long long matched_overlap_0 = 0; + long long matched_overlap_1 = 0; + long long potiental_matched_overlap_0 = 0; + long long potiental_matched_overlap_1 = 0; + long long num_read_base = 0; + long long num_correct_base = 0; + long long num_second_correct_base = 0; + long long j, inner_j; + + int thr_ID = *((int*)arg); + uint64_t POA_i; + long long i = 0; + int avalible_k = 0; + + UC_Read g_read; + init_UC_Read(&g_read); + + UC_Read overlap_read; + init_UC_Read(&overlap_read); + + HPC_seq HPC_read; + Hash_code k_code; + uint64_t code; + uint64_t end_pos; + k_mer_pos* list; + uint64_t list_length; + uint64_t sub_ID; + long long total_shared_seed = 0; + long long candidate_overlap_reads = 0; + + + Candidates_list l; + //Candidates_list debug_l; + Graph POA_Graph; + Graph DAGCon; + init_Graph(&DAGCon); + init_Graph(&POA_Graph); + + + init_Candidates_list(&l); + //init_Candidates_list(&debug_l); + + k_mer_pos_list_alloc array_list; + init_k_mer_pos_list_alloc(&array_list); + + overlap_region_alloc overlap_list; + init_overlap_region_alloc(&overlap_list); + + HeapSq heap; + + Init_Heap(&heap); + + Correct_dumy correct; + init_Correct_dumy(&correct); + + + Output_buffer_sub_block current_sub_buffer; + + init_buffer_sub_block(¤t_sub_buffer); + + Cigar_record current_cigar; + init_Cigar_record(¤t_cigar); + + haplotype_evdience_alloc hap; + InitHaplotypeEvdience(&hap); + + + Round2_alignment second_round; + init_Round2_alignment(&second_round); + + small_hash_table forward, reverse; + init_small_hash_table(&forward); + init_small_hash_table(&reverse); + + + for (i = thr_ID; i < R_INF.total_reads; i = i + thread_num) + { + ///0.1% + get_new_candidates(i, &g_read, &overlap_list, &array_list, &heap, &l, 0.001); + /** + correct_overlap(&overlap_list, &R_INF, &g_read, &correct, &overlap_read, &POA_Graph, &DAGCon, + &matched_overlap_0, &matched_overlap_1, &potiental_matched_overlap_0, &potiental_matched_overlap_1, + ¤t_cigar, &hap, &second_round, 0, 0); + + push_final_overlaps(&(R_INF.paf[i]), &overlap_list); + **/ + + overlap_region_sort_y_id(overlap_list.list, overlap_list.length); + ma_hit_sort_tn(R_INF.paf[i].buffer, R_INF.paf[i].length); + + + + + overlap_list.mapped_overlaps_length = 0; + inner_j = 0; + j = 0; + while (j < overlap_list.length && inner_j < R_INF.paf[i].length) + { + if(overlap_list.list[j].y_id < R_INF.paf[i].buffer[inner_j].tn) + { + j++; + } + else if(overlap_list.list[j].y_id > R_INF.paf[i].buffer[inner_j].tn) + { + inner_j++; + } + else + { + if(overlap_list.list[j].y_pos_strand == R_INF.paf[i].buffer[inner_j].rev) + { + + overlap_list.list[j].is_match = 1; + overlap_list.list[j].strong = R_INF.paf[i].buffer[inner_j].ml; + overlap_list.mapped_overlaps_length++; + + if(overlap_list.list[j].strong == 1) + { + matched_overlap_1++; + } + else if(overlap_list.list[j].strong == 0) + { + matched_overlap_0++; + } + else + { + fprintf(stderr, "error\n"); + } + } + j++; + inner_j++; + } + } + + + reverse_complement(g_read.seq, g_read.length); + for (j = 0; j < overlap_list.length; j++) + { + if (overlap_list.list[j].is_match != 1) + { + if(overlap_list.list[j].y_pos_strand == 0) + { + recover_UC_Read(&overlap_read, &R_INF, overlap_list.list[j].y_id); + } + else + { + recover_UC_Read_RC(&overlap_read, &R_INF, overlap_list.list[j].y_id); + } + + if(if_exact_match(g_read.seq, g_read.length, overlap_read.seq, overlap_read.length, + overlap_list.list[j].x_pos_s, overlap_list.list[j].x_pos_e, + overlap_list.list[j].y_pos_s, overlap_list.list[j].y_pos_e)) + { + overlap_list.list[j].is_match = 1; + overlap_list.list[j].strong = 0; + overlap_list.mapped_overlaps_length++; + potiental_matched_overlap_0++; + } + + } + } + + + if(R_INF.paf[i].is_fully_corrected) + { + potiental_matched_overlap_1++; + } + + + + + num_correct_base += + push_final_overlaps(&(R_INF.paf[i]), R_INF.reverse_paf, &overlap_list, &g_read, &overlap_read); + } + + finish_output_buffer(); + + destory_buffer_sub_block(¤t_sub_buffer); + destory_Candidates_list(&l); + destory_overlap_region_alloc(&overlap_list); + destory_Heap(&heap); + destory_k_mer_pos_list_alloc(&array_list); + destory_Graph(&POA_Graph); + destory_Graph(&DAGCon); + destory_UC_Read(&g_read); + destory_UC_Read(&overlap_read); + destory_Cigar_record(¤t_cigar); + destory_Correct_dumy(&correct); + destoryHaplotypeEvdience(&hap); + destory_Round2_alignment(&second_round); + destory_small_hash_table(&forward); + destory_small_hash_table(&reverse); + + + + pthread_mutex_lock(&statistics); + total_matched_overlap_0 += matched_overlap_0; + total_matched_overlap_1 += matched_overlap_1; + total_potiental_matched_overlap_0 += potiental_matched_overlap_0; + total_potiental_matched_overlap_1 += potiental_matched_overlap_1; + total_num_correct_base += num_correct_base; + + + complete_threads++; + if(complete_threads == thread_num) + { + fprintf(stderr, "weak overlaps: %llu\n", total_matched_overlap_0); + fprintf(stderr, "strong overlaps: %llu\n", total_matched_overlap_1); + fprintf(stderr, "recover weak overlaps: %llu\n", total_potiental_matched_overlap_0); + fprintf(stderr, "final available overlaps: %llu\n", total_num_correct_base); + + fprintf(stderr, "fully corrected reads: %llu\n", total_potiental_matched_overlap_1); + + + + } + pthread_mutex_unlock(&statistics); + + free(arg); +} + + +void Output_PAF() +{ + + fprintf(stdout, "Writing PAF to disk ...... \n"); + char* paf_name = (char*)malloc(strlen(output_file_name)+5); + sprintf(paf_name, "%s.paf", output_file_name); + FILE* output_file = fopen(paf_name, "w"); + long long i, j; + ma_hit_t_alloc* sources = R_INF.paf; + + + + for (i = 0; i < R_INF.total_reads; i++) + { + for (j = 0; j < sources[i].length; j++) + { + fwrite(Get_NAME(R_INF, Get_qn(sources[i].buffer[j])), 1, + Get_NAME_LENGTH(R_INF, Get_qn(sources[i].buffer[j])), output_file); + fwrite("\t", 1, 1, output_file); + fprintf(output_file, "%d\t", Get_READ_LENGTH(R_INF, Get_qn(sources[i].buffer[j]))); + fprintf(output_file, "%d\t", Get_qs(sources[i].buffer[j])); + fprintf(output_file, "%d\t", Get_qe(sources[i].buffer[j])); + if(sources[i].buffer[j].rev) + { + fprintf(output_file, "-\t"); + } + else + { + fprintf(output_file, "+\t"); + } + fwrite(Get_NAME(R_INF, Get_tn(sources[i].buffer[j])), 1, + Get_NAME_LENGTH(R_INF, Get_tn(sources[i].buffer[j])), output_file); + fwrite("\t", 1, 1, output_file); + fprintf(output_file, "%d\t", Get_READ_LENGTH(R_INF, Get_tn(sources[i].buffer[j]))); + fprintf(output_file, "%d\t", Get_ts(sources[i].buffer[j])); + fprintf(output_file, "%d\t", Get_te(sources[i].buffer[j])); + fprintf(output_file, "%d\t", sources[i].buffer[j].ml); + fprintf(output_file, "%d\t", sources[i].buffer[j].bl); + fprintf(output_file, "255\n"); + + } + } + + free(paf_name); + fclose(output_file); +} + + + +void generate_overlaps(int last_round) +{ + double start_time = Get_T(); + roundID = number_of_round - last_round; + fprintf(stdout, "Calculting final overlaps ...\n"); + + Counting_multiple_thr(); + Build_hash_table_multiple_thr(); + + ///thread_num = 1; + pthread_t *_r_threads; + + _r_threads = (pthread_t *)malloc(sizeof(pthread_t)*thread_num); + + int i = 0; + + for (i = 0; i < thread_num; i++) + { + int *arg = (int*)malloc(sizeof(*arg)); + *arg = i; + pthread_create(_r_threads + i, NULL, Final_overlap_calculate_heap_merge, (void*)arg); + } + + + for (i = 0; i= 0) { + while ((c = ketopt(&opt, argc, argv, 1, "ht:o:q:k:lwm:n:r:c:a:b:z:", longopts)) >= 0) { if (c == 100 || c == 'h') Print_H(); else if (c == 103 || c == 't') thread_num = atoi(opt.arg); else if (c == 102 || c == 'o') output_file_name = opt.arg; @@ -61,6 +67,10 @@ int CommandLine_process (int argc, char *argv[]) else if (c == 'k') k_mer_length = atoi(opt.arg); else if (c == 'l') load_index_from_disk = 1; else if (c == 'w') write_index_to_disk = 1; + else if (c == 'c') coverage = atoi(opt.arg); + else if (c == 'a') c_round = atoi(opt.arg); + else if (c == 'z') adapterLen = atoi(opt.arg); + else if (c == 'b') required_read_name = opt.arg; else if (c == '?') printf("unknown opt: -%c\n", opt.opt? opt.opt : ':'); else if (c == ':') printf("missing arg: -%c\n", opt.opt? opt.opt : ':'); } @@ -72,6 +82,13 @@ int CommandLine_process (int argc, char *argv[]) return 0; } + if(coverage == -1) + { + fprintf(stdout, "Please set -c!\n"); + Print_H(); + return 0; + } + return 1; } \ No newline at end of file diff --git a/CommandLines.h b/CommandLines.h index dd0525d..b433ab9 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -13,6 +13,12 @@ extern int k_mer_max_freq; extern int load_index_from_disk; extern int write_index_to_disk; extern int number_of_round; +extern int roundID; +extern int coverage; +extern int read_graph; +extern int c_round; +extern char* required_read_name; +extern int adapterLen; int CommandLine_process (int argc, char *argv[]); diff --git a/Correct.cpp b/Correct.cpp index f355933..cf573ee 100644 --- a/Correct.cpp +++ b/Correct.cpp @@ -6,6 +6,7 @@ #include "Levenshtein_distance.h" #include "edlib.h" #include "Assembly.h" +#include "CommandLines.h" long long T_total_match=0; long long T_total_unmatch=0; @@ -222,6 +223,9 @@ inline int get_interval(long long window_start, long long window_end, overlap_re int flag = 0; long long Begin, End, Len; + // fprintf(stderr, "overlap_list->length: %d, dumy->size: %d, dumy->start_i: %d\n", + // overlap_list->length, dumy->size, dumy->start_i); + // fflush(stderr); for (i = dumy->start_i; i < overlap_list->length; i++) { @@ -231,6 +235,7 @@ inline int get_interval(long long window_start, long long window_end, overlap_re { dumy->start_i = 0; dumy->length = 0; + dumy->lengthNT = 0; return 0; } else ///只要window_end >= overlap_list->list[i].x_pos_s，就有可能重叠 @@ -240,12 +245,15 @@ inline int get_interval(long long window_start, long long window_end, overlap_re } } + + ///只会发生在这个window比list里所有元素都大的情况 ///这种情况下一个window也无需遍历了 if (i >= overlap_list->length) { dumy->start_i = overlap_list->length; dumy->length = 0; + dumy->lengthNT = 0; return -2; } @@ -254,10 +262,19 @@ inline int get_interval(long long window_start, long long window_end, overlap_re for (; i < overlap_list->length; i++) { + // fprintf(stderr, "inner i: %d, x_pos_s: %d, x_pos_e: %d, window_start: %d, window_end: %d\n", + // i, overlap_list->list[i].x_pos_s, overlap_list->list[i].x_pos_e, + // window_start, window_end); + // fflush(stderr); + + // fprintf(stderr, "dumy->length: %d, dumy->lengthNT: %d, dumy->size: %d\n", + // dumy->length, dumy->lengthNT, dumy->size); + // fflush(stderr); if((Len = OVERLAP(window_start, window_end, overlap_list->list[i].x_pos_s, overlap_list->list[i].x_pos_e)) > 0) { - if (Len == WINDOW) + ///sometimes the length of window > WINDOW, but overlap length == WINDOW + if (Len == WINDOW && window_end - window_start + 1 == WINDOW) { dumy->overlapID[dumy->length] = i; dumy->length++; @@ -339,7 +356,7 @@ inline int get_available_interval(long long window_start, long long window_end, ///重叠是否有效 overlap_length = overlap_list->list[i].x_pos_e - overlap_list->list[i].x_pos_s + 1; - if (overlap_length * OVERLAP_THRESHOLD <= overlap_list->list[i].align_length) + if (overlap_list->list[i].is_match == 1) { dumy->overlapID[dumy->length] = i; dumy->length++; @@ -477,7 +494,7 @@ int extra_begin, int extra_end) memset(r+extra_begin+length, 'N', extra_end); } -void determine_overlap_region(int threshold, long long y_start, long long y_ID, long long Window_Len, All_reads* R_INF, +int determine_overlap_region(int threshold, long long y_start, long long y_ID, long long Window_Len, All_reads* R_INF, int* r_extra_begin, int* r_extra_end, long long* r_y_start, long long* r_y_length) { int extra_begin; @@ -485,11 +502,26 @@ int* r_extra_begin, int* r_extra_end, long long* r_y_start, long long* r_y_lengt long long currentIDLen; long long o_len; + ///the length of y + currentIDLen = Get_READ_LENGTH((*R_INF), y_ID); + + ///since Window_Len == x_len + (threshold << 1) + if(currentIDLen <= y_start || + currentIDLen - y_start + 2 * threshold + THRESHOLD_MAX_SIZE < Window_Len) + { + return 0; + } + + /** + if(currentIDLen <= y_start) + { + return 0; + } + **/ + extra_begin = extra_end = 0; ///y maybe less than 0 y_start = y_start - threshold; - ///the length of y - currentIDLen = Get_READ_LENGTH((*R_INF), y_ID); o_len = MIN(Window_Len, currentIDLen - y_start); extra_end = Window_Len - o_len; @@ -504,8 +536,12 @@ int* r_extra_begin, int* r_extra_end, long long* r_y_start, long long* r_y_lengt (*r_extra_end) = extra_end; (*r_y_start) = y_start; (*r_y_length) = o_len; + + return 1; } + + void verify_window(long long window_start, long long window_end, overlap_region_alloc* overlap_list,Correct_dumy* dumy, All_reads* R_INF, char* r_string) { @@ -526,6 +562,7 @@ char* r_string) uint64_t y_startGroup[GROUP_SIZE]; int y_extra_begin[GROUP_SIZE]; int y_extra_end[GROUP_SIZE]; + int error_threshold[GROUP_SIZE]; int extra_begin; int extra_end; @@ -539,19 +576,30 @@ char* r_string) x_start = window_start; ///y上的相对位置 y_start = (x_start - overlap_list->list[currentID].x_pos_s) + overlap_list->list[currentID].y_pos_s; - - + /****************************may have bugs********************************/ + y_start += y_start_offset(x_start, &(overlap_list->list[currentID].f_cigar)); + /****************************may have bugs********************************/ + - determine_overlap_region(THRESHOLD, y_start, overlap_list->list[currentID].y_id, Window_Len, R_INF, - &extra_begin, &extra_end, &y_start, &o_len); + if(!determine_overlap_region(THRESHOLD, y_start, overlap_list->list[currentID].y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len)) + { + append_window_list(&overlap_list->list[currentID], window_start, window_end, + -1, -1, -1, -1, -1, -1); + continue; + } fill_subregion(dumy->overlap_region_group[groupLen], y_start, o_len, overlap_list->list[currentID].y_pos_strand, R_INF, overlap_list->list[currentID].y_id, extra_begin, extra_end); + // fprintf(stderr, "!i: %d\n", i); + // fflush(stderr); + y_extra_begin[groupLen] = extra_begin; y_extra_end[groupLen] = extra_end; overlapID[groupLen] = currentID; y_startGroup[groupLen] = y_start; + error_threshold[groupLen] = THRESHOLD; x_string = r_string + x_start; groupLen++; @@ -570,12 +618,12 @@ char* r_string) append_window_list(&overlap_list->list[overlapID[0]], window_start, window_end, y_startGroup[0], y_startGroup[0] + return_sites[0], (int)return_sites_error[0], - y_extra_begin[0], y_extra_end[0]); + y_extra_begin[0], y_extra_end[0], error_threshold[0]); } else { append_window_list(&overlap_list->list[overlapID[0]], window_start, window_end, y_startGroup[0], -1, -1, - y_extra_begin[0], y_extra_end[0]); + y_extra_begin[0], y_extra_end[0], error_threshold[0]); } @@ -585,12 +633,12 @@ char* r_string) append_window_list(&overlap_list->list[overlapID[1]], window_start, window_end, y_startGroup[1], y_startGroup[1] + return_sites[1], (int)return_sites_error[1], - y_extra_begin[1], y_extra_end[1]); + y_extra_begin[1], y_extra_end[1], error_threshold[1]); } else { append_window_list(&overlap_list->list[overlapID[1]], window_start, window_end, y_startGroup[1], -1, -1, - y_extra_begin[1], y_extra_end[1]); + y_extra_begin[1], y_extra_end[1], error_threshold[1]); } @@ -600,12 +648,12 @@ char* r_string) append_window_list(&overlap_list->list[overlapID[2]], window_start, window_end, y_startGroup[2], y_startGroup[2] + return_sites[2], (int)return_sites_error[2], - y_extra_begin[2], y_extra_end[2]); + y_extra_begin[2], y_extra_end[2], error_threshold[2]); } else { append_window_list(&overlap_list->list[overlapID[2]], window_start, window_end, y_startGroup[2], -1, -1, - y_extra_begin[2], y_extra_end[2]); + y_extra_begin[2], y_extra_end[2], error_threshold[2]); } @@ -615,17 +663,18 @@ char* r_string) append_window_list(&overlap_list->list[overlapID[3]], window_start, window_end, y_startGroup[3], y_startGroup[3] + return_sites[3], (int)return_sites_error[3], - y_extra_begin[3], y_extra_end[3]); + y_extra_begin[3], y_extra_end[3], error_threshold[3]); } else { append_window_list(&overlap_list->list[overlapID[3]], window_start, window_end, y_startGroup[3], -1, -1, - y_extra_begin[3], y_extra_end[3]); + y_extra_begin[3], y_extra_end[3], error_threshold[3]); } } } - + // fprintf(stderr, "(1) dumy->size: %d\n", dumy->size); + // fflush(stderr); if (groupLen == 1) @@ -640,12 +689,12 @@ char* r_string) append_window_list(&overlap_list->list[overlapID[0]], window_start, window_end, y_startGroup[0], y_startGroup[0] + end_site, (int)error, - y_extra_begin[0], y_extra_end[0]); + y_extra_begin[0], y_extra_end[0], error_threshold[0]); } else { append_window_list(&overlap_list->list[overlapID[0]], window_start, window_end, y_startGroup[0], -1, -1, - y_extra_begin[0], y_extra_end[0]); + y_extra_begin[0], y_extra_end[0], error_threshold[0]); } } else if (groupLen > 1) @@ -661,18 +710,22 @@ char* r_string) overlap_list->list[overlapID[i]].align_length += x_len; append_window_list(&overlap_list->list[overlapID[i]], window_start, window_end, y_startGroup[i], y_startGroup[i] + return_sites[i], (int)return_sites_error[i], - y_extra_begin[i], y_extra_end[i]); + y_extra_begin[i], y_extra_end[i], error_threshold[i]); } else { append_window_list(&overlap_list->list[overlapID[i]], window_start, window_end, y_startGroup[i], -1, -1, - y_extra_begin[i], y_extra_end[i]); + y_extra_begin[i], y_extra_end[i], error_threshold[i]); } } groupLen = 0; } + + + // fprintf(stderr, "(2) dumy->size: %d\n", dumy->size); + // fflush(stderr); long long reverse_i = dumy->size - 1; @@ -681,25 +734,99 @@ char* r_string) ///这些是整个window被部分覆盖的 for (i = 0; i < dumy->lengthNT; i++) { + extra_begin = extra_end = 0; currentID = dumy->overlapID[reverse_i--]; x_start = MAX(window_start, overlap_list->list[currentID].x_pos_s); x_end = MIN(window_end, overlap_list->list[currentID].x_pos_e); + ///这个是和当前窗口重叠的长度 x_len = x_end - x_start + 1; threshold = x_len * THRESHOLD_RATE; + /****************************may have bugs********************************/ + threshold = Adjust_Threshold(threshold, x_len); + /****************************may have bugs********************************/ + ///y上的相对位置 y_start = (x_start - overlap_list->list[currentID].x_pos_s) + overlap_list->list[currentID].y_pos_s; + /****************************may have bugs********************************/ + y_start += y_start_offset(x_start, &(overlap_list->list[currentID].f_cigar)); + /****************************may have bugs********************************/ + - Window_Len = x_len + (threshold << 1); - determine_overlap_region(threshold, y_start, overlap_list->list[currentID].y_id, Window_Len, R_INF, - &extra_begin, &extra_end, &y_start, &o_len); + + + // fprintf(stderr, "lengthNT: %d, i: %d, window_start: %d, window_end: %d, x_start: %d, x_end: %d\n", + // dumy->lengthNT, i, window_start, window_end, x_start, x_end); + // fflush(stderr); + + // fprintf(stderr, "x_pos_s: %d, x_pos_e: %d, y_pos_s: %d, y_pos_e: %d\n", + // overlap_list->list[currentID].x_pos_s, + // overlap_list->list[currentID].x_pos_e, + // overlap_list->list[currentID].y_pos_s, + // overlap_list->list[currentID].y_pos_e); + + // fprintf(stderr, "x_id: %d, x_length: %d, y_id: %d, y_length: %d, y_pos_strand: %d\n", + // overlap_list->list[currentID].x_id, + // Get_READ_LENGTH((*R_INF), overlap_list->list[currentID].x_id), + // overlap_list->list[currentID].y_id, + // Get_READ_LENGTH((*R_INF), overlap_list->list[currentID].y_id), + // overlap_list->list[currentID].y_pos_strand); + // fflush(stderr); + + // fprintf(stderr, "y_start: %d, y_start_offset: %d\n", y_start, + // y_start_offset(x_start, &(overlap_list->list[currentID].f_cigar))); + // print_fake_gap(&(overlap_list->list[currentID].f_cigar)); + // fflush(stderr); + + // fprintf(stderr, "(31) i: %d, dumy->size: %d, y_start: %d, yLen: %d\n", i, dumy->size, y_start, + // Get_READ_LENGTH((*R_INF), overlap_list->list[currentID].y_id)); + // fflush(stderr); + + Window_Len = x_len + (threshold << 1); + + if(!determine_overlap_region(threshold, y_start, overlap_list->list[currentID].y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len)) + { + append_window_list(&overlap_list->list[currentID], x_start, x_end, + -1, -1, -1, -1, -1, -1); + continue; + } + + /** + if(overlap_list->list[currentID].x_id == 18390 + && overlap_list->list[currentID].y_id == 18419) + { + fprintf(stderr, "x_start: %d, x_len: %d, y_start: %d, y_offset: %d, extra_begin: %d, extra_end: %d, o_len: %d\n", + x_start, x_len, y_start, + y_start_offset(x_start, &(overlap_list->list[currentID].f_cigar)), + extra_begin, extra_end, o_len); + + } + **/ + + + // fprintf(stderr, "(32) i: %d, dumy->size: %d, y_start: %d, o_len: %d, extra_begin: %d, extra_end: %d\n", + // i, dumy->size, y_start, o_len, extra_begin, extra_end); + // if(o_len == -13) + // { + // print_fake_gap(&(overlap_list->list[currentID].f_cigar)); + // } + // fflush(stderr); + + fill_subregion(dumy->overlap_region, y_start, o_len, overlap_list->list[currentID].y_pos_strand, R_INF, overlap_list->list[currentID].y_id, extra_begin, extra_end); + + + + // fprintf(stderr, "(333332) i: %d, dumy->size: %d, y_start: %d, yLen: %d, threshold: %d, Window_Len: %d\n", i, dumy->size, y_start, + // Get_READ_LENGTH((*R_INF), overlap_list->list[currentID].y_id), threshold, Window_Len); + // fflush(stderr); x_string = r_string + x_start; @@ -709,19 +836,28 @@ char* r_string) end_site = Reserve_Banded_BPM(y_string, Window_Len, x_string, x_len, threshold, &error); + // fprintf(stderr, "(33) dumy->size: %d\n", dumy->size); + // fflush(stderr); + if (error!=(unsigned int)-1) { overlap_list->list[currentID].align_length += x_len; append_window_list(&overlap_list->list[currentID], x_start, x_end, y_start, y_start + end_site, (int)error, - extra_begin, extra_end); + extra_begin, extra_end, threshold); } else { append_window_list(&overlap_list->list[currentID], x_start, x_end, y_start, -1, -1, - extra_begin, extra_end); + extra_begin, extra_end, threshold); } } + // fprintf(stderr, "(3) dumy->size: %d\n", dumy->size); + // fflush(stderr); + + + // fprintf(stderr, "************groupLen: %d\n", groupLen); + // fflush(stderr); } void debug_stats(overlap_region_alloc* overlap_list, All_reads* R_INF, @@ -738,7 +874,7 @@ void debug_stats(overlap_region_alloc* overlap_list, All_reads* R_INF, { Len_x = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; - if (Len_x * OVERLAP_THRESHOLD <= overlap_list->list[j].align_length) + if (overlap_list->list[j].is_match == 1) { if (overlap_list->list[j].y_pos_strand == 0) { @@ -752,181 +888,292 @@ void debug_stats(overlap_region_alloc* overlap_list, All_reads* R_INF, ///(*matched_overlap_1) = (*matched_overlap_1) + overlap_list->list[j].align_length; ///(*matched_overlap_1) = (*matched_overlap_1) + Len_x; } + } + } - /** - threshold = Len_x * 0.04; + - y_start = overlap_list->list[j].y_pos_s - threshold; - if (y_start < 0) + +} + +inline double trim_error_rate(overlap_region_alloc* overlap_list, long long ID) +{ + long long tLen, tError,i, subWinLen, subWinNum; + + tLen = 0; + tError = 0; + + subWinNum = overlap_list->list[ID].w_list_length; + + if(subWinNum < 5) + { + for (i = 0; i < subWinNum; i++) + { + subWinLen = overlap_list->list[ID].w_list[i].x_end - overlap_list->list[ID].w_list[i].x_start + 1; + tLen += subWinLen; + + if(overlap_list->list[ID].w_list[i].y_end != -1) { - y_start = 0; - } - - - ///当前y的长度 - currentIDLen = Get_READ_LENGTH((*R_INF), overlap_list->list[j].y_id); - ///不能超过y的剩余长度 - Len_y = MIN(Len_x + 2 * threshold, currentIDLen - y_start); - - - if (overlap_list->list[j].y_pos_strand == 0) - { - recover_UC_Read(overlap_read, R_INF, overlap_list->list[j].y_id); - (*matched_overlap_0)++; + tError += overlap_list->list[ID].w_list[i].error; } else { - recover_UC_Read_RC(overlap_read, R_INF, overlap_list->list[j].y_id); - (*matched_overlap_1)++; + ///tError += (Adjust_Threshold(subWinLen*THRESHOLD_RATE, subWinLen) * 2); + tError += (Adjust_Threshold(subWinLen*THRESHOLD_RATE, subWinLen) * 3); } - - - - - EdlibAlignResult result = edlibAlign(g_read->seq + overlap_list->list[j].x_pos_s, Len_x, - overlap_read->seq + y_start, Len_y, - edlibNewAlignConfig(threshold, EDLIB_MODE_HW, EDLIB_TASK_PATH, NULL, 0)); - - - if (result.status == EDLIB_STATUS_OK && result.editDistance != -1) { - char* cigar = edlibAlignmentToCigar(result.alignment, result.alignmentLength, EDLIB_CIGAR_STANDARD); - int cigar_length = strlen(cigar); - free(cigar); - - - if (overlap_list->list[j].y_pos_strand == 0) - { - (*potiental_matched_overlap_0)++; - } - else - { - (*potiental_matched_overlap_1)++; - } - - // if (overlap_list->list[j].shared_seed == 1 && Len_x >= 1000) - // { - // (*matched_overlap_0)++; - // } - - // if (overlap_list->list[j].shared_seed == 2 && Len_x >= 1000) - // { - // (*matched_overlap_1)++; - // } - - - } - - edlibFreeAlignResult(result); - **/ } } + else + { + for (i = 1; i < subWinNum - 1; i++) + { + subWinLen = overlap_list->list[ID].w_list[i].x_end - overlap_list->list[ID].w_list[i].x_start + 1; + tLen += subWinLen; + + if(overlap_list->list[ID].w_list[i].y_end != -1) + { + tError += overlap_list->list[ID].w_list[i].error; + } + else + { + ///tError += (Adjust_Threshold(subWinLen*THRESHOLD_RATE, subWinLen) * 2); + tError += (Adjust_Threshold(subWinLen*THRESHOLD_RATE, subWinLen) * 3); + } + } + } + + + + + double error_rate = (double)(tError)/(double)(tLen); + + return error_rate; +} + + + +void mark_duplicate(overlap_region_alloc* overlap_list, All_reads* R_INF, + UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read) +{ + long long j, overlapLen; + double rate; + + + for (j = 0; j < overlap_list->length; j++) + { + if(overlap_list->list[j].is_match == 1) + { + rate = trim_error_rate(overlap_list, j); + + if(rate > 0.01) + { + overlapLen = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; + overlap_list->mapped_overlaps_length -= overlapLen; + overlap_list->list[j].is_match = 0; + } + } + } + } -void output_stats(overlap_region_alloc* overlap_list, All_reads* R_INF, - UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read) +int calculate_hpm_errors(char* x, int x_len, char* y, int y_len, CIGAR* cigar, int error) +{ + int x_i, y_i, cigar_i; + x_i = 0; + y_i = 0; + cigar_i = 0; + int operation; + int operationLen; + int i; + int cigar_error = 0; + int hpm_error = 0; + + ///0 is match, 1 is mismatch, 2 is up, 3 is left + ///2是x缺字符（y多字符），而3是y缺字符（x多字符） + ///while (x_i < x_len && y_i < y_len && cigar_i < cigar->length) + while (cigar_i < cigar->length) + { + operation = cigar->C_C[cigar_i]; + operationLen = cigar->C_L[cigar_i]; + + if (operation == 0) + { + x_i = x_i + operationLen; + y_i = y_i + operationLen; + } + else if (operation == 1) + { + cigar_error += operationLen; + for (i = 0; i < operationLen; i++) + { + if(if_is_homopolymer_repeat(x_i, x, x_len) || if_is_homopolymer_repeat(y_i, y, y_len)) + { + hpm_error++; + } + + x_i++; + y_i++; + } + }///2是x缺字符（y多字符） + else if (operation == 2) + { + + if(if_is_homopolymer_repeat(x_i, x, x_len) || if_is_homopolymer_repeat(y_i, y, y_len)) + { + hpm_error++; + }/** + else + { + if(x_i - 5 > 0 && x_i + 5 <= x_len + && + y_i - 5 > 0 && y_i + 5 <= y_len) + { + fprintf(stderr, "x: %.*s\ny: %.*s\n\n", 10, x + x_i - 5, 10, y + y_i - 5); + } + + } + **/ + + + cigar_error += operationLen; + y_i += operationLen; + }///3是y缺字符（x多字符） + else if (operation == 3) + { + + if(if_is_homopolymer_repeat(x_i, x, x_len) || if_is_homopolymer_repeat(y_i, y, y_len)) + { + hpm_error++; + } + + cigar_error += operationLen; + x_i += operationLen; + } + + cigar_i++; + } + + + + return hpm_error; + +} + + + +void count_no_HPM_errors(overlap_region_alloc* overlap_list, All_reads* R_INF, + UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, + long long* total_errors, long long* total_hpm_errors) +{ + + + long long j, i; + long long y_id, y_strand, y_readLen; + long long x_start, x_end, x_len, y_start, y_end, y_len, error; + char* x_string; + char* y_string; + CIGAR* cigar; + int hpm_error; + (*total_errors) = 0; + (*total_hpm_errors) = 0; + + for (j = 0; j < overlap_list->length; j++) + { + y_id = overlap_list->list[j].y_id; + y_strand = overlap_list->list[j].y_pos_strand; + y_readLen = Get_READ_LENGTH((*R_INF), y_id); + + if (overlap_list->list[j].is_match == 1) + { + ///for (i = 0; i < overlap_list->list[j].w_list_length; i++) + for (i = 1; i < overlap_list->list[j].w_list_length - 1; i++) + { + if(overlap_list->list[j].w_list[i].y_end != -1) + { + x_start = overlap_list->list[j].w_list[i].x_start; + x_end = overlap_list->list[j].w_list[i].x_end; + x_len = x_end - x_start + 1; + + x_string = g_read->seq + x_start; + + y_start = overlap_list->list[j].w_list[i].y_start; + y_end = overlap_list->list[j].w_list[i].y_end; + y_len = y_end - y_start + 1; + + recover_UC_Read_sub_region(dumy->overlap_region, y_start, y_len, y_strand, R_INF, y_id); + y_string = dumy->overlap_region; + + + cigar = &overlap_list->list[j].w_list[i].cigar; + error = overlap_list->list[j].w_list[i].error; + + hpm_error = calculate_hpm_errors(x_string, x_len, y_string, y_len, cigar, error); + + ///fprintf(stderr, "hpm_error: %d, error: %d\n", hpm_error, error); + (*total_errors) += error; + (*total_hpm_errors) += hpm_error; + } + } + + } + } + + + +} + + +void debug_output_overlaps(overlap_region_alloc* overlap_list, All_reads* R_INF, + UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, + long long* matched_overlap_0, long long* matched_overlap_1) { long long j; long long Len_x; int threshold; long long y_start; long long Len_y; - long long currentIDLen; - long long whole; + long long currentIDLen = 0; + fprintf(stderr, "overlap_list->length: %d\n", overlap_list->length); for (j = 0; j < overlap_list->length; j++) { + if(memcmp("m64013_190412_043951/108332093/ccs", Get_NAME((*R_INF),overlap_list->list[j].y_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id)) == 0) + { + fprintf(stderr, "******************************x_id: %d, y_id: %d, y_name: %.*s, error_rate: %f, is_match: %d*******************************\n", + overlap_list->list[j].x_id, + overlap_list->list[j].y_id, + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id), Get_NAME((*R_INF),overlap_list->list[j].y_id), + trim_error_rate(overlap_list, j), overlap_list->list[j].is_match); + + print_fake_gap(&overlap_list->list[j].f_cigar); + } + Len_x = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; - if (Len_x * OVERLAP_THRESHOLD <= overlap_list->list[j].align_length) + if (overlap_list->list[j].is_match == 1) { - ; + currentIDLen++; + fprintf(stderr, "y_name: %.*s\n", + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id), Get_NAME((*R_INF),overlap_list->list[j].y_id)); } + // else + // { + // fprintf(stderr, "not match, y_name: %.*s\n", + // Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id), Get_NAME((*R_INF),overlap_list->list[j].y_id)); + // } + } + fprintf(stderr, "currentIDLen: %d\n\n", currentIDLen); + } -inline void generate_cigar( - char* path, int path_length, window_list* result, int* start, int* end, int error) -{ - - if (error == 0) - { - result->cigar.C_L[0] = result->x_end - result->x_start + 1; - result->cigar.C_C[0] = 0; - result->cigar.length = 1; - - return; - } - - - int i = 0; - result->cigar.length = 0; - ///0 is match, 1 is mismatch, 2 is up, 3 is left - char pre_ciga = 5; - int pre_ciga_length = 0; - - - for (i = 0; i < path_length; i++) - { - if(path[i] == 1) - { - path[i] = 3; - (*end)--; - } - else - { - break; - } - } - - for (i = path_length - 1; i >= 0; i--) - { - if(path[i] == 1) - { - path[i] = 3; - (*start)++; - } - else - { - break; - } - } - - - for (i = path_length - 1; i >= 0; i--) - { - - if (pre_ciga != path[i]) - { - if (pre_ciga_length != 0) - { - result->cigar.C_L[result->cigar.length] = pre_ciga_length; - result->cigar.C_C[result->cigar.length] = pre_ciga; - result->cigar.length++; - } - - pre_ciga = path[i]; - pre_ciga_length = 1; - } - else - { - pre_ciga_length++; - } - } - - if (pre_ciga_length != 0) - { - result->cigar.C_L[result->cigar.length] = pre_ciga_length; - result->cigar.C_C[result->cigar.length] = pre_ciga; - result->cigar.length++; - } -} - int verify_cigar(char* x, int x_len, char* y, int y_len, CIGAR* cigar, int error) { @@ -955,7 +1202,7 @@ int verify_cigar(char* x, int x_len, char* y, int y_len, CIGAR* cigar, int error if (x[x_i]!=y[y_i]) { - fprintf(stderr, "error match\n"); + ///fprintf(stderr, "error match\n"); flag_error = 1; } x_i++; @@ -970,7 +1217,7 @@ int verify_cigar(char* x, int x_len, char* y, int y_len, CIGAR* cigar, int error if (x[x_i]==y[y_i]) { - fprintf(stderr, "error mismatch, cigar_i: %d, x_i: %d, y_i: %d\n",cigar_i, x_i, y_i); + ///fprintf(stderr, "error mismatch, cigar_i: %d, x_i: %d, y_i: %d\n",cigar_i, x_i, y_i); flag_error = 1; } x_i++; @@ -997,17 +1244,22 @@ int verify_cigar(char* x, int x_len, char* y, int y_len, CIGAR* cigar, int error if (cigar_error != error) { + /** fprintf(stderr, "error cigar_error: cigar_error: %d, error: %d\n", cigar_error, error); for (i = 0; i < cigar->length; i++) { fprintf(stderr, "%u: %u\n", cigar->C_L[i], cigar->C_C[i]); } + **/ + + flag_error = 1; } if (flag_error == 1) { + /** print_string(x, x_len); print_string(y, y_len); fprintf(stderr, "x_len: %d, y_len: %d, cigar_len: %d, error: %d\n", x_len, y_len, cigar->length, error); @@ -1015,6 +1267,8 @@ int verify_cigar(char* x, int x_len, char* y, int y_len, CIGAR* cigar, int error { fprintf(stderr, "%u: %u\n", cigar->C_L[i], cigar->C_C[i]); } + **/ + } @@ -1022,6 +1276,342 @@ int verify_cigar(char* x, int x_len, char* y, int y_len, CIGAR* cigar, int error } +inline int move_gap_greedy(char* path, int path_i, int path_length, char* x, int x_i, char* y, int y_i, unsigned int* new_error) +{ + if(path[path_i] < 2) + { + return 0; + } + + /** + * + GGCG-TGTGCCTGT + * + GGCAATGTGCCTGT + * + 00013000000000 + **/ + + int flag = 0; + + char oper = path[path_i]; + + + if(oper == 3) + { + path_i++; + y_i--; + for (; path_i < path_length && x_i >= 0 && y_i >= 0; path_i++, x_i--, y_i--) + { + if(path[path_i] == 2 || path[path_i] == 3 || (path[path_i] == 0 && x[x_i] != y[y_i])) + { + break; + } + else + { + if(path[path_i] == 1 && x[x_i] == y[y_i]) + { + path[path_i - 1] = 0; + (*new_error)--; + } + else + { + path[path_i - 1] = path[path_i]; + } + + path[path_i] = oper; + + + flag = 1; + } + + } + } + else if(oper == 2) + { + path_i++; + x_i--; + for (; path_i < path_length && x_i >= 0 && y_i >= 0; path_i++, x_i--, y_i--) + { + if(path[path_i] == 2 || path[path_i] == 3 || (path[path_i] == 0 && x[x_i] != y[y_i])) + { + break; + } + else + { + + if(path[path_i] == 1 && x[x_i] == y[y_i]) + { + path[path_i - 1] = 0; + (*new_error)--; + } + else + { + path[path_i - 1] = path[path_i]; + } + + + path[path_i] = oper; + flag = 1; + } + + } + } + + return flag; +} + +inline void generate_cigar( + char* path, int path_length, window_list* result, int* start, int* end, unsigned int* old_error, + char* x, int x_len, char* y) +{ + + if ((*old_error) == 0) + { + result->cigar.C_L[0] = result->x_end - result->x_start + 1; + result->cigar.C_C[0] = 0; + result->cigar.length = 1; + + return; + } + + + int i = 0; + result->cigar.length = 0; + ///0 is match, 1 is mismatch, 2 is up, 3 is left + char pre_ciga = 5; + int pre_ciga_length = 0; + + int terminate_site = -1; + + for (i = 0; i < path_length; i++) + { + if(path[i] == 1) + { + path[i] = 3; + (*end)--; + terminate_site = i; + } + else + { + break; + } + } + + for (i = path_length - 1; i >= 0; i--) + { + if(path[i] == 1) + { + path[i] = 3; + (*start)++; + } + else + { + break; + } + } + + + for (i = path_length - 1; i >= 0; i--) + { + + if (pre_ciga != path[i]) + { + if (pre_ciga_length != 0) + { + result->cigar.C_L[result->cigar.length] = pre_ciga_length; + result->cigar.C_C[result->cigar.length] = pre_ciga; + result->cigar.length++; + } + + pre_ciga = path[i]; + pre_ciga_length = 1; + } + else + { + pre_ciga_length++; + } + } + + if (pre_ciga_length != 0) + { + result->cigar.C_L[result->cigar.length] = pre_ciga_length; + result->cigar.C_C[result->cigar.length] = pre_ciga; + result->cigar.length++; + } + + ///verify_cigar(x, x_len, y + (*start), (*end) - (*start) + 1, &(result->cigar), error); + + + y = y + (*start); + + int x_i, y_i; + x_i = 0; + y_i = 0; + ///terminate_site = -1; + for (i = path_length - 1; i > terminate_site; i--) + { + if(path[i] == 0) + { + x_i++; + y_i++; + } + else if(path[i] == 1) + { + x_i++; + y_i++; + } + else if(path[i] == 2) + { + move_gap_greedy(path, i, path_length, x, x_i, y, y_i, old_error); + y_i++; + } + else if(path[i] == 3) + { + move_gap_greedy(path, i, path_length, x, x_i, y, y_i, old_error); + x_i++; + } + } + + + + pre_ciga = 5; + pre_ciga_length = 0; + result->cigar.length = 0; + for (i = path_length - 1; i >= 0; i--) + { + + if (pre_ciga != path[i]) + { + if (pre_ciga_length != 0) + { + result->cigar.C_L[result->cigar.length] = pre_ciga_length; + result->cigar.C_C[result->cigar.length] = pre_ciga; + result->cigar.length++; + } + + pre_ciga = path[i]; + pre_ciga_length = 1; + } + else + { + pre_ciga_length++; + } + } + + if (pre_ciga_length != 0) + { + result->cigar.C_L[result->cigar.length] = pre_ciga_length; + result->cigar.C_C[result->cigar.length] = pre_ciga; + result->cigar.length++; + } + + + // if(verify_cigar(x, x_len, y, (*end) - (*start) + 1, &(result->cigar), *old_error)) + // { + // fprintf(stderr, "error\n"); + // } + + + + /** + int x_i, y_i; + x_i = 0; + y_i = 0; + int new_error = error; + ///0 is match, 1 is mismatch, 2 is up, 3 is left + ///2是x缺字符（y多字符），而3是y缺字符（x多字符） + ///while (x_i < x_len && y_i < y_len && cigar_i < cigar->length) + for (i = path_length - 1; i >= 0; i--) + { + if(path[i] == 0) + { + x_i++; + y_i++; + } + else if(path[i] == 1) + { + x_i++; + y_i++; + } + else if(path[i] == 2) + { + move_gap_greedy(path, i, path_length, x, x_i, y, y_i, &new_error); + y_i++; + } + else if(path[i] == 3) + { + move_gap_greedy(path, i, path_length, x, x_i, y, y_i, &new_error); + x_i++; + } + } + + + + + + + + CIGAR new_cigar; + + new_cigar.length = 0; + pre_ciga = 5; + pre_ciga_length = 0; + + + for (i = path_length - 1; i >= 0; i--) + { + + if (pre_ciga != path[i]) + { + if (pre_ciga_length != 0) + { + new_cigar.C_L[new_cigar.length] = pre_ciga_length; + new_cigar.C_C[new_cigar.length] = pre_ciga; + new_cigar.length++; + } + + pre_ciga = path[i]; + pre_ciga_length = 1; + } + else + { + pre_ciga_length++; + } + } + + if (pre_ciga_length != 0) + { + new_cigar.C_L[new_cigar.length] = pre_ciga_length; + new_cigar.C_C[new_cigar.length] = pre_ciga; + new_cigar.length++; + } + + + + + + if(verify_cigar(x, x_len, y, (*end) - (*start) + 1, &new_cigar, new_error)) + { + + fprintf(stderr, "x_string: %.*s\n", x_len, x); + fprintf(stderr, "y_string: %.*s\n\n", (*end) - (*start) + 1 , y); + + for (int j = 0; j < result->cigar.length; j++) + { + fprintf(stderr, "oper: %d, len: %d\n", result->cigar.C_C[j], result->cigar.C_L[j]); + } + + for (int j = 0; j < new_cigar.length; j++) + { + fprintf(stderr, "new_cigar.oper: %d, new_cigar.len: %d\n", new_cigar.C_C[j], new_cigar.C_L[j]); + } + + } + **/ +} + + int verify_cigar_2(char* x, int x_len, char* y, int y_len, Cigar_record* cigar, int error) { int x_i, y_i, cigar_i; @@ -1033,6 +1623,7 @@ int verify_cigar_2(char* x, int x_len, char* y, int y_len, Cigar_record* cigar, int i; int cigar_error = 0; int flag_error = 0; + int diff_i = 0; ///0 is match, 1 is mismatch, 2 is up, 3 is left ///2是x缺字符（y多字符），而3是y缺字符（x多字符） @@ -1067,19 +1658,58 @@ int verify_cigar_2(char* x, int x_len, char* y, int y_len, Cigar_record* cigar, fprintf(stderr, "error mismatch, cigar_i: %d, x_i: %d, y_i: %d\n",cigar_i, x_i, y_i); flag_error = 1; } + + if(Get_MisMatch_Base(cigar->lost_base[diff_i]) != y[y_i]) + { + fprintf(stderr, "mismatch x: %c, y: %c, mis[%d]: %c\n", x[x_i],y[y_i],diff_i, + Get_MisMatch_Base(cigar->lost_base[diff_i])); + } + + if(Get_Match_Base(cigar->lost_base[diff_i]) != x[x_i]) + { + fprintf(stderr, "match x: %c, y: %c, deletion[%d]: %c\n", + x[x_i],y[y_i],diff_i, + Get_Match_Base(cigar->lost_base[diff_i])); + } + + + + x_i++; y_i++; + diff_i++; } }///2是x缺字符（y多字符） else if (operation == 2) { cigar_error += operationLen; - y_i += operationLen; + for (i = 0; i < operationLen; i++) + { + if(cigar->lost_base[diff_i] != y[y_i]) + { + fprintf(stderr, "insertion x: %c, y: %c, insertion[%d]: %c\n", x[x_i],y[y_i],diff_i, + cigar->lost_base[diff_i]); + } + y_i++; + diff_i++; + } }///3是y缺字符（x多字符） else if (operation == 3) { cigar_error += operationLen; - x_i += operationLen; + + for (i = 0; i < operationLen; i++) + { + if(cigar->lost_base[diff_i] != x[x_i]) + { + fprintf(stderr, "deletion x: %c, y: %c, deletion[%d]: %c\n", x[x_i],y[y_i],diff_i, + cigar->lost_base[diff_i]); + } + x_i++; + diff_i++; + } + // x_i += operationLen; + // diff_i += operationLen; } cigar_i++; @@ -1122,13 +1752,146 @@ int verify_cigar_2(char* x, int x_len, char* y, int y_len, Cigar_record* cigar, } -/** -///记住等于0也要重算，虽然意义不那么大就是了 -#define NEED_START_POS(x) (x.error<=0) -#define GET_ERROR(x) ((x.error>=0)? x.error : x.error*-1) -**/ +inline int fix_boundary(char* x_string, long long x_len, int threshold, +long long total_y_start, long long local_y_start, long long local_y_end, +long long old_extra_begin, long long old_extra_end, +long long y_ID, long long Window_Len, All_reads* R_INF, +Correct_dumy* dumy, int y_strand, unsigned int old_error, +long long* r_total_y_start, int* r_start_site, int* r_end_site, +int* r_extra_begin, int* r_extra_end, unsigned int* r_error) +{ -inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_INF, + + int new_extra_begin, new_extra_end; + long long new_y_start, new_y_length; + int new_end_site, new_start_site; + unsigned int new_error; + char* y_string; + + + int path_length; + + if(local_y_start == 0) + { + total_y_start = total_y_start + local_y_start; + ///if local_y_start == 0 and old_extra_begin != 0 + ///this means total_y_start == 0, so shift to the left cannot get a new start pos + if(old_extra_begin != 0) + { + return 0; + } + + ///if the begining of alignment is 0, we should try to shift the window to find a better result + ///shift to the left by threshold-1 bases + if(!determine_overlap_region(threshold, total_y_start, y_ID, Window_Len, R_INF, + &new_extra_begin, &new_extra_end, &new_y_start, &new_y_length)) + { + return 0; + } + + ///if new_y_start is equal to total_y_start, recalculate makes no sense + if(new_y_start == total_y_start) + { + return 0; + } + + fill_subregion(dumy->overlap_region_fix, new_y_start, new_y_length, y_strand, R_INF, y_ID, + new_extra_begin, new_extra_end); + + y_string = dumy->overlap_region_fix; + + new_end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &new_error, &new_start_site, + &path_length, dumy->matrix_bit, dumy->path_fix, -1, -1); + + if (new_error != (unsigned int)-1 && new_error < old_error) + { + (*r_total_y_start) = new_y_start; + (*r_start_site) = new_start_site; + (*r_end_site) = new_end_site; + (*r_extra_begin) = new_extra_begin; + (*r_extra_end) = new_extra_end; + (*r_error) = new_error; + + dumy->path_length = path_length; + memcpy(dumy->path, dumy->path_fix, path_length); + memcpy(dumy->overlap_region, dumy->overlap_region_fix, Window_Len); + return 1; + } + } + else if(local_y_end == Window_Len - 1) + { + ///if local_y_end == Window_Len - 1 and old_extra_end > 0 + ///this means local_y_end is the end of the y + ///so shit to the right makes no sense + if(old_extra_end != 0) + { + return 0; + } + long long total_y_end = total_y_start + local_y_end; + + total_y_start = total_y_end - x_len + 1; + + if(!determine_overlap_region(threshold, total_y_start, y_ID, Window_Len, R_INF, + &new_extra_begin, &new_extra_end, &new_y_start, &new_y_length)) + { + return 0; + } + + if(new_y_start == total_y_end - local_y_end) + { + return 0; + } + + fill_subregion(dumy->overlap_region_fix, new_y_start, new_y_length, y_strand, R_INF, y_ID, + new_extra_begin, new_extra_end); + + y_string = dumy->overlap_region_fix; + + new_end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &new_error, &new_start_site, + &path_length, dumy->matrix_bit, dumy->path_fix, -1, -1); + + if (new_error != (unsigned int)-1 && new_error < old_error) + { + (*r_total_y_start) = new_y_start; + (*r_start_site) = new_start_site; + (*r_end_site) = new_end_site; + (*r_extra_begin) = new_extra_begin; + (*r_extra_end) = new_extra_end; + (*r_error) = new_error; + + dumy->path_length = path_length; + memcpy(dumy->path, dumy->path_fix, path_length); + memcpy(dumy->overlap_region, dumy->overlap_region_fix, Window_Len); + return 1; + } + + + } + return 0; +} + + +inline int double_error_threshold(int pre_threshold, int x_len) +{ + + pre_threshold = Adjust_Threshold(pre_threshold, x_len); + int threshold = pre_threshold * 2; + ///may have some bugs + if(x_len >= 300 && threshold < THRESHOLD_MAX_SIZE) + { + threshold = THRESHOLD_MAX_SIZE; + } + + if(threshold > THRESHOLD_MAX_SIZE) + { + threshold = THRESHOLD_MAX_SIZE; + } + + return threshold; +} + + +inline void recalcate_window_back(overlap_region_alloc* overlap_list, All_reads* R_INF, UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read) { long long j, k, i; @@ -1166,6 +1929,8 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN y_strand = overlap_list->list[j].y_pos_strand; y_readLen = Get_READ_LENGTH((*R_INF), y_id); + + //i负责每个overlap里面的window @@ -1181,13 +1946,30 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN ///this is the actual end postion in ystring total_y_start = overlap_list->list[j].w_list[i].y_end - overlap_list->list[j].w_list[i].extra_begin + 1; - + ///k遍历匹配window右侧所有不匹配的window ///如果i匹配，则k从i+1开始 ///知道第一个匹配的window结束 for (k = i + 1; k < overlap_list->list[j].w_list_length && overlap_list->list[j].w_list[k].y_end == -1; k++) { + /** + if(memcmp("m64013_190324_024932/92733922/ccs", Get_NAME((*R_INF),overlap_list->list[j].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].x_id)) == 0) + { + if(memcmp("m64013_190324_024932/123996697/ccs", Get_NAME((*R_INF),overlap_list->list[j].y_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id)) == 0) + { + fprintf(stderr, "total_y_start: %d, x_start: %lld, x_end: %lld, y_start: %lld, y_end: %lld, match: %d, cigar.length: %d, error: %d\n", + total_y_start, overlap_list->list[j].w_list[i].x_start, overlap_list->list[j].w_list[i].x_end, + overlap_list->list[j].w_list[i].y_start, overlap_list->list[j].w_list[i].y_end, + overlap_list->list[j].w_list[i].y_end - overlap_list->list[j].w_list[i].y_start + 1, + overlap_list->list[j].w_list[i].cigar.length, overlap_list->list[j].w_list[i].error); + } + } + **/ + + extra_begin = extra_end = 0; ///y_start有可能大于y_readLen @@ -1202,8 +1984,15 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN x_start = overlap_list->list[j].w_list[k].x_start; x_end = overlap_list->list[j].w_list[k].x_end; x_len = x_end - x_start + 1; - threshold = x_len * THRESHOLD_RATE; - + // /****************************may have bugs********************************/ + // ///threshold = x_len * THRESHOLD_RATE; + // threshold = overlap_list->list[j].w_list[k].error_threshold; + // /****************************may have bugs********************************/ + // /****************************may have bugs********************************/ + // threshold = Adjust_Threshold(threshold, x_len); + // /****************************may have bugs********************************/ + threshold = double_error_threshold(overlap_list->list[j].w_list[k].error_threshold, x_len); + y_start = total_y_start; Window_Len = x_len + (threshold << 1); @@ -1234,6 +2023,8 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN ///note!!! need notification overlap_list->list[j].w_list[k].extra_begin = extra_begin; overlap_list->list[j].w_list[k].extra_end = extra_end; + overlap_list->list[j].w_list[k].error_threshold = threshold; + overlap_list->list[j].align_length += x_len; } else @@ -1275,7 +2066,13 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN x_start = overlap_list->list[j].w_list[i].x_start; x_end = overlap_list->list[j].w_list[i].x_end; x_len = x_end - x_start + 1; - threshold = x_len * THRESHOLD_RATE; + /****************************may have bugs********************************/ + ///threshold = x_len * THRESHOLD_RATE; + threshold = overlap_list->list[j].w_list[i].error_threshold; + /****************************may have bugs********************************/ + /****************************may have bugs********************************/ + threshold = Adjust_Threshold(threshold, x_len); + /****************************may have bugs********************************/ Window_Len = x_len + (threshold << 1); @@ -1294,6 +2091,9 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN &(dumy->path_length), dumy->matrix_bit, dumy->path, overlap_list->list[j].w_list[i].error, overlap_list->list[j].w_list[i].y_end - y_start); + + + @@ -1301,11 +2101,27 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN if (error != (unsigned int)-1) { - + if (end_site == Window_Len - 1 || real_y_start == 0) + { + + if(fix_boundary(x_string, x_len, threshold, y_start, real_y_start, end_site, + extra_begin, extra_end, y_id, Window_Len, R_INF, dumy, y_strand, error, + &y_start, &real_y_start, &end_site, + &extra_begin, &extra_end, &error)) + { + ///fprintf(stderr, "old_error: %d, new_error: %d\n", overlap_list->list[j].w_list[i].error, error); + overlap_list->list[j].w_list[i].error = error; + overlap_list->list[j].w_list[i].extra_begin = extra_begin; + overlap_list->list[j].w_list[i].extra_end = extra_end; + } + + + } + generate_cigar(dumy->path, dumy->path_length, &(overlap_list->list[j].w_list[i]), - &real_y_start, &end_site, error); + &real_y_start, &end_site, &error, x_string, x_len, y_string); if(real_y_start < extra_begin || end_site >= Window_Len - extra_end) { @@ -1321,7 +2137,8 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN ///real_y_start = y_start + real_y_start; real_y_start = y_start + real_y_start - extra_begin; overlap_list->list[j].w_list[i].y_start = real_y_start; - overlap_list->list[j].w_list[i].y_end = y_start + end_site; + overlap_list->list[j].w_list[i].y_end = y_start + end_site; + overlap_list->list[j].w_list[i].error = error; } else { @@ -1349,7 +2166,15 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN x_start = overlap_list->list[j].w_list[k].x_start; x_end = overlap_list->list[j].w_list[k].x_end; x_len = x_end - x_start + 1; - threshold = x_len * THRESHOLD_RATE; + // /****************************may have bugs********************************/ + // ///threshold = x_len * THRESHOLD_RATE; + // threshold = overlap_list->list[j].w_list[k].error_threshold; + // /****************************may have bugs********************************/ + // /****************************may have bugs********************************/ + // threshold = Adjust_Threshold(threshold, x_len); + // /****************************may have bugs********************************/ + threshold = double_error_threshold(overlap_list->list[j].w_list[k].error_threshold, x_len); + Window_Len = x_len + (threshold << 1); if(total_y_end <= 0) @@ -1381,8 +2206,16 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN if (error!=(unsigned int)-1) { + if (end_site == Window_Len - 1 || real_y_start == 0) + { + fix_boundary(x_string, x_len, threshold, y_start, real_y_start, end_site, + extra_begin, extra_end, y_id, Window_Len, R_INF, dumy, y_strand, error, + &y_start, &real_y_start, &end_site, + &extra_begin, &extra_end, &error); + } + generate_cigar(dumy->path, dumy->path_length, &(overlap_list->list[j].w_list[k]), - &real_y_start, &end_site, error); + &real_y_start, &end_site, &error, x_string, x_len, y_string); if(real_y_start < extra_begin || end_site >= Window_Len - extra_end) { @@ -1401,6 +2234,7 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN ///note!!! need notification overlap_list->list[j].w_list[k].extra_begin = extra_begin; overlap_list->list[j].w_list[k].extra_end = extra_end; + overlap_list->list[j].w_list[k].error_threshold = threshold; } else { @@ -1418,8 +2252,11 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN - + overlap_list->mapped_overlaps_length = 0; + int pre_threshold; + long long tLen, tError; + double error_rate; ///j负责遍历整个overlap list for (j = 0; j < overlap_list->length; j++) { @@ -1427,10 +2264,38 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN y_strand = overlap_list->list[j].y_pos_strand; y_readLen = Get_READ_LENGTH((*R_INF), y_id); overlap_length = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; + overlap_list->list[j].is_match = 0; + + /** + if(memcmp("m64013_190324_024932/92733922/ccs", Get_NAME((*R_INF),overlap_list->list[j].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].x_id)) == 0) + { + if(memcmp("m64013_190324_024932/123996697/ccs", Get_NAME((*R_INF),overlap_list->list[j].y_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id)) == 0) + { + fprintf(stderr, "##############y_name: %.*s, error_rate: %f, is_match: %d, overlap_length: %d, align_length: %d##################\n", + Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id), Get_NAME((*R_INF),overlap_list->list[j].y_id), + trim_error_rate(overlap_list, j), overlap_list->list[j].is_match, overlap_length, + overlap_list->list[j].align_length); + for (i = 0; i < overlap_list->list[j].w_list_length; i++) + { + fprintf(stderr, "x_start: %lld, x_end: %lld, y_start: %lld, y_end: %lld, match: %d, cigar.length: %d, error: %d\n", + overlap_list->list[j].w_list[i].x_start, overlap_list->list[j].w_list[i].x_end, + overlap_list->list[j].w_list[i].y_start, overlap_list->list[j].w_list[i].y_end, + overlap_list->list[j].w_list[i].y_end - overlap_list->list[j].w_list[i].y_start + 1, + overlap_list->list[j].w_list[i].cigar.length, overlap_list->list[j].w_list[i].error); + + } + } + } + **/ + + ///only calculate cigar for high quality overlaps - if (overlap_length * OVERLAP_THRESHOLD <= overlap_list->list[j].align_length) + if (overlap_length * OVERLAP_THRESHOLD_FILTER <= overlap_list->list[j].align_length) { + for (i = 0; i < overlap_list->list[j].w_list_length; i++) { ///判断cigar是否被计算 @@ -1444,7 +2309,13 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN x_start = overlap_list->list[j].w_list[i].x_start; x_end = overlap_list->list[j].w_list[i].x_end; x_len = x_end - x_start + 1; - threshold = x_len * THRESHOLD_RATE; + /****************************may have bugs********************************/ + ///threshold = x_len * THRESHOLD_RATE; + threshold = overlap_list->list[j].w_list[i].error_threshold; + /****************************may have bugs********************************/ + /****************************may have bugs********************************/ + threshold = Adjust_Threshold(threshold, x_len); + /****************************may have bugs********************************/ Window_Len = x_len + (threshold << 1); @@ -1469,8 +2340,24 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN if (error != (unsigned int)-1) { + if (end_site == Window_Len - 1 || real_y_start == 0) + { + + if(fix_boundary(x_string, x_len, threshold, y_start, real_y_start, end_site, + extra_begin, extra_end, y_id, Window_Len, R_INF, dumy, y_strand, error, + &y_start, &real_y_start, &end_site, + &extra_begin, &extra_end, &error)) + { + ///fprintf(stderr, "old_error: %d, new_error: %d\n", overlap_list->list[j].w_list[i].error, error); + overlap_list->list[j].w_list[i].error = error; + overlap_list->list[j].w_list[i].extra_begin = extra_begin; + overlap_list->list[j].w_list[i].extra_end = extra_end; + } + + } + generate_cigar(dumy->path, dumy->path_length, &(overlap_list->list[j].w_list[i]), - &real_y_start, &end_site, error); + &real_y_start, &end_site, &error, x_string, x_len, y_string); if(real_y_start < extra_begin || end_site >= Window_Len - extra_end) @@ -1488,7 +2375,8 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN ///real_y_start = y_start + real_y_start; real_y_start = y_start + real_y_start - extra_begin; overlap_list->list[j].w_list[i].y_start = real_y_start; - overlap_list->list[j].w_list[i].y_end = y_start + end_site - extra_begin; + overlap_list->list[j].w_list[i].y_end = y_start + end_site - extra_begin; + overlap_list->list[j].w_list[i].error = error; } else { @@ -1499,9 +2387,119 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN { overlap_list->list[j].w_list[i].y_end -= overlap_list->list[j].w_list[i].extra_begin; } + + } - - + else ///try to calculate using higher threshold + { + ///there is no problem for x + ///there is no problem for x + x_start = overlap_list->list[j].w_list[i].x_start; + x_end = overlap_list->list[j].w_list[i].x_end; + x_len = x_end - x_start + 1; + ///double the threshold + pre_threshold = overlap_list->list[j].w_list[i].error_threshold; + threshold = double_error_threshold(pre_threshold, x_len); + // /****************************may have bugs********************************/ + // ///pre_threshold = x_len * THRESHOLD_RATE; + // pre_threshold = overlap_list->list[j].w_list[i].error_threshold; + // /****************************may have bugs********************************/ + // /****************************may have bugs********************************/ + // pre_threshold = Adjust_Threshold(pre_threshold, x_len); + // /****************************may have bugs********************************/ + // threshold = pre_threshold * 2; + // ///may have some bugs + // if(x_len >= 300 && threshold < THRESHOLD_MAX_SIZE) + // { + // threshold = THRESHOLD_MAX_SIZE; + // } + // if(threshold > THRESHOLD_MAX_SIZE) + // { + // threshold = THRESHOLD_MAX_SIZE; + // } + Window_Len = x_len + (threshold << 1); + + + + ///if the previous window is mapped + if(i > 0 && overlap_list->list[j].w_list[i - 1].y_end != -1) + { + y_start = overlap_list->list[j].w_list[i - 1].y_end + 1; + determine_overlap_region(threshold, y_start, y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len); + }///if the next window is mapped + else if(i < overlap_list->list[j].w_list_length - 1 && overlap_list->list[j].w_list[i + 1].y_end != -1) + { + y_start = overlap_list->list[j].w_list[i + 1].y_start - 1 - x_len + 1; + determine_overlap_region(threshold, y_start, y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len); + } + else///if the previous window and next window are not mapped, using the y_start itself + { + ///y_start is the real y_start + y_start = overlap_list->list[j].w_list[i].y_start; + /// since y_start has already substacted pre_threshold + ///here we just need to substact threshold - pre_threshold + determine_overlap_region(threshold - pre_threshold, y_start, y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len); + } + + + fill_subregion(dumy->overlap_region, y_start, o_len, y_strand, R_INF, y_id, extra_begin, extra_end); + + x_string = g_read->seq + x_start; + y_string = dumy->overlap_region; + + ///note!!! need notification + end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &error, &real_y_start, + &(dumy->path_length), dumy->matrix_bit, dumy->path, -1, -1); + + if (error!=(unsigned int)-1) + { + + if (end_site == Window_Len - 1 || real_y_start == 0) + { + fix_boundary(x_string, x_len, threshold, y_start, real_y_start, end_site, + extra_begin, extra_end, y_id, Window_Len, R_INF, dumy, y_strand, error, + &y_start, &real_y_start, &end_site, + &extra_begin, &extra_end, &error); + } + + generate_cigar(dumy->path, dumy->path_length, &(overlap_list->list[j].w_list[i]), + &real_y_start, &end_site, &error, x_string, x_len, y_string); + + + if(real_y_start < extra_begin || end_site >= Window_Len - extra_end) + { + fprintf(stderr, "\nreal_y_start: %d, extra_begin: %d, error: %d, Window_Len: %d, x_len: %d\n", + real_y_start, extra_begin, error, Window_Len, x_len); + + fprintf(stderr, "end_site: %d, Window_Len: %d, extra_end: %d\n", + end_site, Window_Len, extra_end); + } + + real_y_start = y_start + real_y_start - extra_begin; + overlap_list->list[j].w_list[i].y_start = real_y_start; + overlap_list->list[j].w_list[i].y_end = y_start + end_site - extra_begin; + overlap_list->list[j].w_list[i].error = error; + overlap_list->list[j].align_length += x_len; + overlap_list->list[j].w_list[i].extra_begin = extra_begin; + overlap_list->list[j].w_list[i].extra_end = extra_end; + } + } + } + + + error_rate = trim_error_rate(overlap_list, j); + + + + ///if(error_rate <= 0.015) + if(error_rate <= 0.025) + { + ///overlap_list->mapped_overlaps++; + overlap_list->mapped_overlaps_length += overlap_length; + overlap_list->list[j].is_match = 1; } } } @@ -1520,7 +2518,7 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN overlap_length = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; ///only calculate cigar for high quality overlaps - if (overlap_length * OVERLAP_THRESHOLD <= overlap_list->list[j].align_length) + if (overlap_length * OVERLAP_THRESHOLD_FILTER <= overlap_list->list[j].align_length) { for (i = 0; i < overlap_list->list[j].w_list_length; i++) { @@ -1550,12 +2548,590 @@ inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_IN } } + + + // for (i = 0; i < overlap_list->list[j].w_list_length; i++) + // { + + // if(overlap_list->list[j].w_list[i].x_end - overlap_list->list[j].w_list[i].x_start + 1 != WINDOW && + // overlap_list->list[j].w_list[i].x_end != overlap_list->list[j].x_pos_e && + // overlap_list->list[j].w_list[i].x_start != overlap_list->list[j].x_pos_s) + // { + // fprintf(stderr, "x_start:%d, x_end: %d, g_read->length: %d, x_pos_s: %d, x_pos_e: %d\n", + // overlap_list->list[j].w_list[i].x_start, + // overlap_list->list[j].w_list[i].x_end, + // g_read->length, + // overlap_list->list[j].x_pos_s, + // overlap_list->list[j].x_pos_e); + // } + // } } **/ + + + + } +inline void recalcate_window(overlap_region_alloc* overlap_list, All_reads* R_INF, + UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read) +{ + long long j, k, i; + long long Len_x; + int threshold; + long long y_len; + long long currentIDLen; + long long matches; + long long y_id; + int y_strand; + long long y_readLen; + long long x_start; + long long x_end; + long long x_len; + long long total_y_start; + long long total_y_end; + long long y_start; + long long y_end; + long long Window_Len; + char* x_string; + char* y_string; + int end_site; + unsigned int error; + int real_y_start; + long long overlap_length; + int extra_begin, extra_end; + long long o_len; + + + ///j负责遍历整个overlap list + for (j = 0; j < overlap_list->length; j++) + { + + y_id = overlap_list->list[j].y_id; + y_strand = overlap_list->list[j].y_pos_strand; + y_readLen = Get_READ_LENGTH((*R_INF), y_id); + + + + + + //i负责每个overlap里面的window + //倒着找 + //倒着用结束位置矫正 + for (i = overlap_list->list[j].w_list_length - 1; i >= 0; i--) + { + ///找到第一个匹配的window + if(overlap_list->list[j].w_list[i].y_end != -1) + { + ///note!!! need notification + ///total_y_start = overlap_list->list[j].w_list[i].y_end + 1; + ///this is the actual end postion in ystring + total_y_start = overlap_list->list[j].w_list[i].y_end - overlap_list->list[j].w_list[i].extra_begin + 1; + + + + ///k遍历匹配window右侧所有不匹配的window + ///如果i匹配，则k从i+1开始 + ///知道第一个匹配的window结束 + for (k = i + 1; k < overlap_list->list[j].w_list_length && overlap_list->list[j].w_list[k].y_end == -1; k++) + { + extra_begin = extra_end = 0; + + ///y_start有可能大于y_readLen + ///这多发于最后一个window长度仅为几，而前面一个window的结束位置也超过了y_readLen-1 + ///这个时候做动态规划会给超过的部分补N + if (total_y_start >= y_readLen) + { + break; + } + + ///there is no problem for x + x_start = overlap_list->list[j].w_list[k].x_start; + x_end = overlap_list->list[j].w_list[k].x_end; + x_len = x_end - x_start + 1; + // /****************************may have bugs********************************/ + // ///threshold = x_len * THRESHOLD_RATE; + // threshold = overlap_list->list[j].w_list[k].error_threshold; + // /****************************may have bugs********************************/ + // /****************************may have bugs********************************/ + // threshold = Adjust_Threshold(threshold, x_len); + // /****************************may have bugs********************************/ + threshold = double_error_threshold(overlap_list->list[j].w_list[k].error_threshold, x_len); + + + y_start = total_y_start; + Window_Len = x_len + (threshold << 1); + if(!determine_overlap_region(threshold, y_start, y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len)) + { + break; + } + + if(o_len + threshold < x_len) + { + break; + } + + fill_subregion(dumy->overlap_region, y_start, o_len, y_strand, + R_INF, y_id, extra_begin, extra_end); + + x_string = g_read->seq + x_start; + y_string = dumy->overlap_region; + + ///note!!! need notification + end_site = Reserve_Banded_BPM(y_string, Window_Len, x_string, x_len, threshold, &error); + + ///error等于-1说明没匹配 + if (error!=(unsigned int)-1) + { + overlap_list->list[j].w_list[k].cigar.length = -1; + overlap_list->list[j].w_list[k].y_start = y_start; + overlap_list->list[j].w_list[k].y_end = y_start + end_site; + overlap_list->list[j].w_list[k].error = (int)error; + ///note!!! need notification + overlap_list->list[j].w_list[k].extra_begin = extra_begin; + overlap_list->list[j].w_list[k].extra_end = extra_end; + overlap_list->list[j].w_list[k].error_threshold = threshold; + + overlap_list->list[j].align_length += x_len; + } + else + { + break; + } + ///note!!! need notification + ///total_y_start = y_start + end_site + 1; + total_y_start = y_start + end_site - extra_begin + 1; + + } + + } + + } + + + + + + ///continue; + ///i负责每个overlap里面的window + ///正着找 + ///用起始位置矫正 + for (i = 0; i < overlap_list->list[j].w_list_length; i++) + { + ///找到第一个匹配的window + ///首先这个window要匹配 + ///其次不要是第一个window，这没意义 + ///最后他之前的那个window必须是不匹配，如果之前那个window匹配，也没意义 + if(overlap_list->list[j].w_list[i].y_end != -1 && i != 0 && overlap_list->list[j].w_list[i - 1].y_end == -1) + { + + ///判断这个匹配的window的起始位置有没有被计算出来 + ///如果没有，就需要重新计算 + if(overlap_list->list[j].w_list[i].cigar.length == -1) + { + ///there is no problem for x + x_start = overlap_list->list[j].w_list[i].x_start; + x_end = overlap_list->list[j].w_list[i].x_end; + x_len = x_end - x_start + 1; + /****************************may have bugs********************************/ + ///threshold = x_len * THRESHOLD_RATE; + threshold = overlap_list->list[j].w_list[i].error_threshold; + /****************************may have bugs********************************/ + /****************************may have bugs********************************/ + threshold = Adjust_Threshold(threshold, x_len); + /****************************may have bugs********************************/ + Window_Len = x_len + (threshold << 1); + + + ///y_start is the real y_start + y_start = overlap_list->list[j].w_list[i].y_start; + extra_begin = overlap_list->list[j].w_list[i].extra_begin; + extra_end = overlap_list->list[j].w_list[i].extra_end; + o_len = Window_Len - extra_end - extra_begin; + fill_subregion(dumy->overlap_region, y_start, o_len, y_strand, + R_INF, y_id, extra_begin, extra_end); + x_string = g_read->seq + x_start; + y_string = dumy->overlap_region; + + ///note!!! need notification + end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &error, &real_y_start, + &(dumy->path_length), dumy->matrix_bit, dumy->path, + overlap_list->list[j].w_list[i].error, overlap_list->list[j].w_list[i].y_end - y_start); + + + + + + + + ///到这里y_start已经被正确计算出来了 + if (error != (unsigned int)-1) + { + + if (end_site == Window_Len - 1 || real_y_start == 0) + { + + if(fix_boundary(x_string, x_len, threshold, y_start, real_y_start, end_site, + extra_begin, extra_end, y_id, Window_Len, R_INF, dumy, y_strand, error, + &y_start, &real_y_start, &end_site, + &extra_begin, &extra_end, &error)) + { + ///fprintf(stderr, "old_error: %d, new_error: %d\n", overlap_list->list[j].w_list[i].error, error); + overlap_list->list[j].w_list[i].error = error; + overlap_list->list[j].w_list[i].extra_begin = extra_begin; + overlap_list->list[j].w_list[i].extra_end = extra_end; + } + + + } + + + + generate_cigar(dumy->path, dumy->path_length, &(overlap_list->list[j].w_list[i]), + &real_y_start, &end_site, &error, x_string, x_len, y_string); + + if(real_y_start < extra_begin || end_site >= Window_Len - extra_end) + { + fprintf(stderr, "\nreal_y_start: %d, extra_begin: %d\n", + real_y_start, extra_begin); + + fprintf(stderr, "end_site: %d, Window_Len: %d, extra_end: %d\n", + end_site, Window_Len, extra_end); + } + + + ///note!!! need notification + ///real_y_start = y_start + real_y_start; + real_y_start = y_start + real_y_start - extra_begin; + overlap_list->list[j].w_list[i].y_start = real_y_start; + overlap_list->list[j].w_list[i].y_end = y_start + end_site; + overlap_list->list[j].w_list[i].error = error; + } + else + { + fprintf(stderr, "error\n"); + } + + + } + else + { + real_y_start = overlap_list->list[j].w_list[i].y_start; + } + + + + ///再次矫正的基础位置就是real_y_start + total_y_end = real_y_start - 1; + ///k遍历匹配window左侧所有不匹配的window + ///如果i匹配，则k从i-1开始 + ///直到第一个匹配的window结束 + ///因为i!=0，所以k的大小不用担心 + for (k = i - 1; k >= 0 && overlap_list->list[j].w_list[k].y_end == -1; k--) + { + ///there is no problem in x + x_start = overlap_list->list[j].w_list[k].x_start; + x_end = overlap_list->list[j].w_list[k].x_end; + x_len = x_end - x_start + 1; + // /****************************may have bugs********************************/ + // ///threshold = x_len * THRESHOLD_RATE; + // threshold = overlap_list->list[j].w_list[k].error_threshold; + // /****************************may have bugs********************************/ + // /****************************may have bugs********************************/ + // threshold = Adjust_Threshold(threshold, x_len); + // /****************************may have bugs********************************/ + threshold = double_error_threshold(overlap_list->list[j].w_list[k].error_threshold, x_len); + + Window_Len = x_len + (threshold << 1); + + if(total_y_end <= 0) + { + break; + } + + ///y_start may less than 0 + y_start = total_y_end - x_len + 1; + if(!determine_overlap_region(threshold, y_start, y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len)) + { + break; + } + + if(o_len + threshold < x_len) + { + break; + } + + fill_subregion(dumy->overlap_region, y_start, o_len, y_strand, + R_INF, y_id, extra_begin, extra_end); + x_string = g_read->seq + x_start; + y_string = dumy->overlap_region; + + ///note!!! need notification + end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &error, &real_y_start, + &(dumy->path_length), dumy->matrix_bit, dumy->path, -1, -1); + + + ///error等于-1说明没匹配 + if (error!=(unsigned int)-1) + { + + if (end_site == Window_Len - 1 || real_y_start == 0) + { + fix_boundary(x_string, x_len, threshold, y_start, real_y_start, end_site, + extra_begin, extra_end, y_id, Window_Len, R_INF, dumy, y_strand, error, + &y_start, &real_y_start, &end_site, + &extra_begin, &extra_end, &error); + } + + generate_cigar(dumy->path, dumy->path_length, &(overlap_list->list[j].w_list[k]), + &real_y_start, &end_site, &error, x_string, x_len, y_string); + + if(real_y_start < extra_begin || end_site >= Window_Len - extra_end) + { + fprintf(stderr, "\nreal_y_start: %d, extra_begin: %d\n", + real_y_start, extra_begin); + + fprintf(stderr, "end_site: %d, Window_Len: %d, extra_end: %d\n", + end_site, Window_Len, extra_end); + } + + ///y_start has no shift, but y_end has shift + overlap_list->list[j].w_list[k].y_start = y_start + real_y_start - extra_begin; + overlap_list->list[j].w_list[k].y_end = y_start + end_site; + overlap_list->list[j].w_list[k].error = error; + overlap_list->list[j].align_length += x_len; + ///note!!! need notification + overlap_list->list[j].w_list[k].extra_begin = extra_begin; + overlap_list->list[j].w_list[k].extra_end = extra_end; + overlap_list->list[j].w_list[k].error_threshold = threshold; + } + else + { + break; + } + + total_y_end = y_start + real_y_start - 1 - extra_begin; + } + + } + } + + + } + + + + overlap_list->mapped_overlaps_length = 0; + + int pre_threshold; + long long tLen, tError; + double error_rate; + ///j负责遍历整个overlap list + for (j = 0; j < overlap_list->length; j++) + { + y_id = overlap_list->list[j].y_id; + y_strand = overlap_list->list[j].y_pos_strand; + y_readLen = Get_READ_LENGTH((*R_INF), y_id); + overlap_length = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; + overlap_list->list[j].is_match = 0; + + + ///only calculate cigar for high quality overlaps + if (overlap_length * OVERLAP_THRESHOLD_FILTER <= overlap_list->list[j].align_length) + { + + for (i = 0; i < overlap_list->list[j].w_list_length; i++) + { + ///判断cigar是否被计算 + ///没被计算过就重算 + ///第一个条件是判断这个窗口是否匹配 + if(overlap_list->list[j].w_list[i].y_end != -1) + { + if(overlap_list->list[j].w_list[i].cigar.length == -1) + { + ///there is no problem for x + x_start = overlap_list->list[j].w_list[i].x_start; + x_end = overlap_list->list[j].w_list[i].x_end; + x_len = x_end - x_start + 1; + /****************************may have bugs********************************/ + ///threshold = x_len * THRESHOLD_RATE; + threshold = overlap_list->list[j].w_list[i].error_threshold; + /****************************may have bugs********************************/ + /****************************may have bugs********************************/ + threshold = Adjust_Threshold(threshold, x_len); + /****************************may have bugs********************************/ + Window_Len = x_len + (threshold << 1); + + + ///y_start is the real y_start + y_start = overlap_list->list[j].w_list[i].y_start; + extra_begin = overlap_list->list[j].w_list[i].extra_begin; + extra_end = overlap_list->list[j].w_list[i].extra_end; + o_len = Window_Len - extra_end - extra_begin; + fill_subregion(dumy->overlap_region, y_start, o_len, y_strand, + R_INF, y_id, extra_begin, extra_end); + x_string = g_read->seq + x_start; + y_string = dumy->overlap_region; + + + ///note!!! need notification + end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &error, &real_y_start, + &(dumy->path_length), dumy->matrix_bit, dumy->path, + overlap_list->list[j].w_list[i].error, overlap_list->list[j].w_list[i].y_end - y_start); + + // if(error != overlap_list->list[j].w_list[i].error) + // { + // fprintf(stderr, "error\n"); + // } + + ///到这里y_start已经被正确计算出来了 + if (error != (unsigned int)-1) + { + + if (end_site == Window_Len - 1 || real_y_start == 0) + { + + if(fix_boundary(x_string, x_len, threshold, y_start, real_y_start, end_site, + extra_begin, extra_end, y_id, Window_Len, R_INF, dumy, y_strand, error, + &y_start, &real_y_start, &end_site, + &extra_begin, &extra_end, &error)) + { + ///fprintf(stderr, "old_error: %d, new_error: %d\n", overlap_list->list[j].w_list[i].error, error); + overlap_list->list[j].w_list[i].error = error; + overlap_list->list[j].w_list[i].extra_begin = extra_begin; + overlap_list->list[j].w_list[i].extra_end = extra_end; + } + + } + + generate_cigar(dumy->path, dumy->path_length, &(overlap_list->list[j].w_list[i]), + &real_y_start, &end_site, &error, x_string, x_len, y_string); + + + if(real_y_start < extra_begin || end_site >= Window_Len - extra_end) + { + fprintf(stderr, "\nreal_y_start: %d, extra_begin: %d\n", + real_y_start, extra_begin); + + fprintf(stderr, "end_site: %d, Window_Len: %d, extra_end: %d\n", + end_site, Window_Len, extra_end); + } + + + + ///note!!! need notification + ///real_y_start = y_start + real_y_start; + real_y_start = y_start + real_y_start - extra_begin; + overlap_list->list[j].w_list[i].y_start = real_y_start; + overlap_list->list[j].w_list[i].y_end = y_start + end_site - extra_begin; + overlap_list->list[j].w_list[i].error = error; + } + else + { + fprintf(stderr, "error\n"); + } + } + else + { + overlap_list->list[j].w_list[i].y_end -= overlap_list->list[j].w_list[i].extra_begin; + } + + + } + } + + error_rate = trim_error_rate(overlap_list, j); + + ///if(error_rate <= 0.015) + if(error_rate <= 0.03) + { + ///overlap_list->mapped_overlaps++; + overlap_list->mapped_overlaps_length += overlap_length; + overlap_list->list[j].is_match = 1; + } + else if(error_rate <= 0.045) + { + overlap_list->list[j].is_match = 3; + } + } + } + + + + + + /** + ///j负责遍历整个overlap list + for (j = 0; j < overlap_list->length; j++) + { + y_id = overlap_list->list[j].y_id; + y_strand = overlap_list->list[j].y_pos_strand; + y_readLen = Get_READ_LENGTH((*R_INF), y_id); + overlap_length = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; + + ///only calculate cigar for high quality overlaps + if (overlap_length * OVERLAP_THRESHOLD_FILTER <= overlap_list->list[j].align_length) + { + for (i = 0; i < overlap_list->list[j].w_list_length; i++) + { + if(overlap_list->list[j].w_list[i].y_end != -1) + { + ///there is no problem for x + x_start = overlap_list->list[j].w_list[i].x_start; + x_end = overlap_list->list[j].w_list[i].x_end; + x_len = x_end - x_start + 1; + + x_string = g_read->seq + x_start; + + y_start = overlap_list->list[j].w_list[i].y_start; + y_end = overlap_list->list[j].w_list[i].y_end; + y_len = y_end - y_start + 1; + + recover_UC_Read_sub_region(dumy->overlap_region, y_start, y_len, y_strand, R_INF, y_id); + y_string = dumy->overlap_region; + + + if(verify_cigar(x_string, x_len, y_string, y_len, &overlap_list->list[j].w_list[i].cigar, + overlap_list->list[j].w_list[i].error)) + { + fprintf(stderr, "j: %d, i: %d, y_id: %d, y_start: %d, y_end: %d\n", j, i, y_id, y_start, y_end); + } + } + + } + } + + + // for (i = 0; i < overlap_list->list[j].w_list_length; i++) + // { + + // if(overlap_list->list[j].w_list[i].x_end - overlap_list->list[j].w_list[i].x_start + 1 != WINDOW && + // overlap_list->list[j].w_list[i].x_end != overlap_list->list[j].x_pos_e && + // overlap_list->list[j].w_list[i].x_start != overlap_list->list[j].x_pos_s) + // { + // fprintf(stderr, "x_start:%d, x_end: %d, g_read->length: %d, x_pos_s: %d, x_pos_e: %d\n", + // overlap_list->list[j].w_list[i].x_start, + // overlap_list->list[j].w_list[i].x_end, + // g_read->length, + // overlap_list->list[j].x_pos_s, + // overlap_list->list[j].x_pos_e); + // } + // } + } + **/ + + + + + +} + + + inline void add_base_to_correct_read_directly(Correct_dumy* dumy, char base) { @@ -1613,7 +3189,6 @@ inline void add_segment_to_correct_read(Correct_dumy* dumy, char* segment, long - ///返回下一个backbone节点上的ID long long inline add_path_to_correct_read(Graph* backbone, Correct_dumy* dumy, long long currentNodeID, long long type, long long edgeID, Cigar_record* current_cigar, char* self_string) @@ -1683,6 +3258,12 @@ long long type, long long edgeID, Cigar_record* current_cigar, char* self_string { nodeID = backbone->g_nodes.list[currentNodeID].deletion_edges.list[edgeID].out_node; dumy->corrected_base += nodeID - currentNodeID; + + // if(nodeID - currentNodeID != 1) + // { + // fprintf(stderr, "error\n"); + // } + ///currentNodeID = i means self_string[i - 1] add_cigar_record(self_string + currentNodeID, nodeID - currentNodeID, current_cigar, DELETION); @@ -1738,8 +3319,1707 @@ long long type, long long edgeID, Cigar_record* current_cigar, char* self_string } -void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy, Cigar_record* current_cigar, char* self_string) + +///返回下一个backbone节点上的ID +long long inline add_path_to_correct_read_new(Graph* backbone, Graph* DAGCon, Correct_dumy* dumy, long long currentNodeID, +long long type, long long edgeID, Cigar_record* current_cigar, char* self_string) { + //long long i; + long long nodeID; + + ///Note: currentNodeID must be a backbone node + ///currentNodeID = 0 means a fake node + ///currentNodeID = i means self_string[i - 1] + ///包括匹配和误配两种情况 + if (type == MISMATCH) + { + ///这是match的情况 + if(backbone->g_nodes.list[currentNodeID].mismatch_edges.list[edgeID].length == 0) + { + nodeID = backbone->g_nodes.list[currentNodeID].mismatch_edges.list[edgeID].out_node; + add_base_to_correct_read_directly(dumy, backbone->g_nodes.list[nodeID].base); + ///match所以dumy->corrected_base不要+1 + + ///nodeID = i means self_string[i - 1] + ///add_cigar_record(self_string+nodeID-1, 1, current_cigar, 0); + add_cigar_record(&(backbone->g_nodes.list[nodeID].base), 1, current_cigar, 0); + + /***********需要注释掉********* */ + if (nodeID != currentNodeID + 1) + { + fprintf(stderr, "error match\n"); + } + /***********需要注释掉********* */ + + return nodeID; + } + else ///这是mismatch的情况 + { + + + nodeID = backbone->g_nodes.list[currentNodeID].mismatch_edges.list[edgeID].out_node; + add_base_to_correct_read_directly(dumy, backbone->g_nodes.list[nodeID].base); + dumy->corrected_base++; + + + char merge_base = 0; + merge_base = seq_nt6_table[(uint8_t)backbone->g_nodes.list[nodeID].base]; + merge_base = merge_base << 3; + ///这种中间节点只有一个元素，所以直接list[0] + nodeID = backbone->g_nodes.list[nodeID].mismatch_edges.list[0].out_node; + merge_base = merge_base | seq_nt6_table[(uint8_t)backbone->g_nodes.list[nodeID].base]; + add_cigar_record(&merge_base, 1, current_cigar, 1); + + /** + add_cigar_record(&(backbone->g_nodes.list[nodeID].base), 1, current_cigar, 1); + nodeID = backbone->g_nodes.list[nodeID].mismatch_edges.list[0].out_node; + **/ + + /***********需要注释掉********* */ + if (nodeID != currentNodeID + 1) + { + fprintf(stderr, "error mismatch\n"); + } + /***********需要注释掉********* */ + + return nodeID; + } + } + else if (type == DELETION) + { + nodeID = backbone->g_nodes.list[currentNodeID].deletion_edges.list[edgeID].out_node; + dumy->corrected_base += nodeID - currentNodeID; + + // if(nodeID - currentNodeID != 1) + // { + // fprintf(stderr, "error\n"); + // } + + ///currentNodeID = i means self_string[i - 1] + add_cigar_record(self_string + currentNodeID, nodeID - currentNodeID, current_cigar, DELETION); + + + + /***********需要注释掉********* */ + if (!(nodeID >= backbone->s_start_nodeID && nodeID <= backbone->s_end_nodeID)) + { + fprintf(stderr, "error deletion 1\n"); + } + if (nodeID <= currentNodeID) + { + fprintf(stderr, "error deletion 2\n"); + } + + /***********需要注释掉********* */ + + return nodeID; + } + else if (type == INSERTION) + { + ///这个一定要变成0 + backbone->g_nodes.list[currentNodeID].num_insertions = 0; + long long str; + char str_c; + while (pop_from_Queue(&(DAGCon->node_q), &str)) + { + str_c = (char)str; + add_base_to_correct_read_directly(dumy, str_c); + add_cigar_record(&str_c, 1, current_cigar, INSERTION); + dumy->corrected_base++; + } + + return currentNodeID; + + // nodeID = backbone->g_nodes.list[currentNodeID].insertion_edges.list[edgeID].out_node; + // long long step = backbone->g_nodes.list[currentNodeID].insertion_edges.list[edgeID].length; + // long long i; + // for (i = 0; i < step; i++) + // { + // add_base_to_correct_read_directly(dumy, backbone->g_nodes.list[nodeID].base); + // add_cigar_record(&backbone->g_nodes.list[nodeID].base, 1, current_cigar, INSERTION); + // ///只有一条边 + // nodeID = backbone->g_nodes.list[nodeID].insertion_edges.list[0].out_node; + // } + // dumy->corrected_base += step; + // /***********需要注释掉********* */ + // if (nodeID != currentNodeID) + // { + // fprintf(stderr, "error insertion\n"); + // } + // /***********需要注释掉********* */ + // return nodeID; + } + else + { + fprintf(stderr, "error type\n"); + } + +} + + + +void test_single_path(Graph* DAGCon, Graph* backbone, int debug_node_in_backbone) +{ + char forward[1000]; + char reverse[1000]; + char pre[1000]; + long long i, j, outNode, inputNode, preNode, string_i, path_weight, step; + + if((Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID)).length != Input_Edges(G_Node(*DAGCon, DAGCon->s_end_nodeID)).length) + || (G_Node(*backbone, debug_node_in_backbone).insertion_edges.length != Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID)).length)) + { + fprintf(stderr, "s_out: %d, s_end: %d\n", + Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID)).length, + Input_Edges(G_Node(*DAGCon, DAGCon->s_end_nodeID)).length); + } + + + + + + for (i = 0; i < Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID)).length; i++) + { + outNode = Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID)).list[i].out_node; + string_i = 0; + path_weight = Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID)).list[i].weight; + + while(outNode != DAGCon->s_end_nodeID) + { + forward[string_i++] = G_Node(*DAGCon, outNode).base; + if(Output_Edges(G_Node(*DAGCon, outNode)).list[0].weight != path_weight) + { + fprintf(stderr, "error1\n"); + } + outNode = Output_Edges(G_Node(*DAGCon, outNode)).list[0].out_node; + } + forward[string_i] = '\0'; + + + inputNode = Input_Edges(G_Node(*DAGCon, DAGCon->s_end_nodeID)).list[i].in_node; + string_i = 0; + while(inputNode != DAGCon->s_start_nodeID) + { + reverse[string_i++] = G_Node(*DAGCon, inputNode).base; + + + + if(Input_Edges(G_Node(*DAGCon, inputNode)).list[0].weight != path_weight) + { + fprintf(stderr, "error2\n"); + } + inputNode = Input_Edges(G_Node(*DAGCon, inputNode)).list[0].in_node; + } + reverse[string_i] = '\0'; + + for(j = 0; j < string_i/2; j ++) + { + char k = reverse[j]; + reverse[j] = reverse[string_i - j - 1]; + reverse[string_i - j - 1] = k; + } + + if(strcmp(forward, reverse)) + { + fprintf(stderr, "f: %s\n, r: %s\n\n", forward, reverse); + } + + + + + if(G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].weight != path_weight) + { + fprintf(stderr, "error3\n"); + } + + if(G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].length != string_i) + { + fprintf(stderr, "error4\n"); + } + + step = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].length; + if(step != 0) + { + string_i = 0; + preNode = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].out_node; + + for (j = 0; j < step; j++) + { + pre[string_i++] = G_Node(*backbone, preNode).base; + preNode = G_Node(*backbone, preNode).insertion_edges.list[0].out_node; + } + } + + pre[string_i] = '\0'; + + + if(strcmp(forward, pre)) + { + fprintf(stderr, "f: %s, r: %s, p: %s\n\n", forward, reverse, pre); + } + + } +} + + + + +void test_single_path_new(Graph* DAGCon, Graph* backbone, int debug_node_in_backbone) +{ + char forward[1000]; + char reverse[1000]; + char pre[1000]; + long long i, j, preNode, string_i, path_weight, step; + Node* outNode; + Node* inputNode; + Node* currentStartNode; + Node* currentEndNode; + + if( + (Real_Length(Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID))) + != + Real_Length(Input_Edges(G_Node(*DAGCon, DAGCon->s_end_nodeID)))) + || + (G_Node(*backbone, debug_node_in_backbone).insertion_edges.length + != + Real_Length(Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID))))) + { + fprintf(stderr, "s_out: %d, s_end: %d, insertion_edges.length: %d\n", + Real_Length(Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID))), + Real_Length(Input_Edges(G_Node(*DAGCon, DAGCon->s_end_nodeID))), + G_Node(*backbone, debug_node_in_backbone).insertion_edges.length); + } + + RSet iter_out, iter_input; + clear_RSet(&iter_out); + clear_RSet(&iter_input); + currentStartNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + currentEndNode = &(G_Node(*DAGCon, DAGCon->s_end_nodeID)); + + i = 0; + + + + while(getOutputNodes(&iter_out, DAGCon, currentStartNode, &outNode)) + { + string_i = 0; + path_weight = Input_Edges((*outNode)).list[0].weight; + + while(outNode != &(G_Node(*DAGCon, DAGCon->s_end_nodeID))) + { + RSet inner; + Edge* e; + forward[string_i++] = (*outNode).base; + + clear_RSet(&inner); + while (getOutputEdges(&inner, DAGCon, outNode, &e)) + { + if(e->weight != path_weight) + { + fprintf(stderr, "error1\n"); + } + } + + + clear_RSet(&inner); + while (getInputEdges(&inner, DAGCon, outNode, &e)) + { + if(e->weight != path_weight) + { + fprintf(stderr, "error1\n"); + } + } + + clear_RSet(&inner); + while(getOutputNodes(&inner, DAGCon, outNode, &outNode)) + { + ; + } + } + forward[string_i] = '\0'; + + + string_i = 0; + if(!getInputNodes(&iter_input, DAGCon, currentEndNode, &inputNode)) + { + fprintf(stderr, "sbsbsb\n"); + } + + while(inputNode != &(G_Node(*DAGCon,DAGCon->s_start_nodeID))) + { + reverse[string_i++] = (*inputNode).base; + if(Input_Edges((*inputNode)).list[0].weight != path_weight) + { + fprintf(stderr, "error2\n"); + } + RSet inner; + Edge* e; + clear_RSet(&inner); + while (getOutputEdges(&inner, DAGCon, inputNode, &e)) + { + if(e->weight != path_weight) + { + fprintf(stderr, "error1\n"); + } + } + clear_RSet(&inner); + while (getInputEdges(&inner, DAGCon, inputNode, &e)) + { + if(e->weight != path_weight) + { + fprintf(stderr, "error1\n"); + } + } + + clear_RSet(&inner); + while(getInputNodes(&inner, DAGCon, inputNode, &inputNode)) + { + ; + } + + ///inputNode = &(G_Node(*DAGCon, Input_Edges((*inputNode)).list[0].in_node)); + } + reverse[string_i] = '\0'; + + for(j = 0; j < string_i/2; j ++) + { + char k = reverse[j]; + reverse[j] = reverse[string_i - j - 1]; + reverse[string_i - j - 1] = k; + } + + if(strcmp(forward, reverse)!=0) + { + fprintf(stderr, "f: %s, r: %s\n\n", forward, reverse); + } + + + + + + if(G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].weight != path_weight) + { + fprintf(stderr, "error3\n"); + } + + if(G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].length != string_i) + { + fprintf(stderr, "error4\n"); + } + + /** + step = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].length; + if(step != 0) + { + string_i = 0; + preNode = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].out_node; + + for (j = 0; j < step; j++) + { + pre[string_i++] = G_Node(*backbone, preNode).base; + preNode = G_Node(*backbone, preNode).insertion_edges.list[0].out_node; + } + } + + pre[string_i] = '\0'; + **/ + + extract_path(backbone, debug_node_in_backbone, i, pre); + + + if(strcmp(forward, pre)!=0) + { + fprintf(stderr, "f: %s, r: %s, p: %s\n\n", forward, reverse, pre); + } + + + i++; + } + + int pre_weight = 0; + for (i = 0; i < G_Node(*backbone, debug_node_in_backbone).insertion_edges.length; i++) + { + if(i == 0) + { + extract_path(backbone, debug_node_in_backbone, i, pre); + pre_weight = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].weight; + } + else if(i > 0) + { + extract_path(backbone, debug_node_in_backbone, i, forward); + + if(strcmp(forward, pre)==0) + { + fprintf(stderr, "f: %s, f_weight: %d, p: %s, p_weight: %d\n\n", + forward, G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[i].weight, + pre, pre_weight); + } + + memcpy(pre, forward, strlen(forward) + 1); + + } + } + + + + for (i = 0; i < DAGCon->g_nodes.length; i++) + { + currentStartNode = &(G_Node(*DAGCon, i)); + + if(If_Node_Exist(*currentStartNode)) + { + clear_RSet(&iter_out); + Edge* e; + Edge* e_self; + Edge* e_reverse; + while(getOutputEdges(&iter_out, DAGCon, currentStartNode, &e_self)) + { + ///e_reverse = &(Input_Edges(G_Node(*DAGCon, e_self->out_node)).list[e_self->reverse_edge_ID]); + e = e_self; + + get_bi_direction_edges(DAGCon, e_self, &e_self, &e_reverse); + + if(e != e_self) + { + fprintf(stderr, "error0\n"); + } + + if(e_self->in_node != e_reverse->in_node || e_self->out_node != e_reverse->out_node || + e_self->weight != e_reverse->weight) + { + fprintf(stderr, "error1\n"); + } + + + if(e_self != &(Output_Edges(G_Node(*DAGCon, e_self->in_node)).list[e_self->self_edge_ID])) + { + fprintf(stderr, "error2\n"); + } + + if(Visit(*e_self) == 1 || Visit(*e_reverse) == 1) + { + fprintf(stderr, "error visit flag\n"); + } + + + + + + + + get_bi_direction_edges(DAGCon, e_reverse, &e_self, &e_reverse); + + if(e != e_self) + { + fprintf(stderr, "error0\n"); + } + + if(e_self->in_node != e_reverse->in_node || e_self->out_node != e_reverse->out_node || + e_self->weight != e_reverse->weight) + { + fprintf(stderr, "error1\n"); + } + + + if(e_self != &(Output_Edges(G_Node(*DAGCon, e_self->in_node)).list[e_self->self_edge_ID])) + { + fprintf(stderr, "error2\n"); + } + } + + + + clear_RSet(&iter_input); + while(getInputEdges(&iter_input, DAGCon, currentStartNode, &e_self)) + { + + e = e_self; + + get_bi_direction_edges(DAGCon, e_self, &e_self, &e_reverse); + + if(e != e_reverse) + { + fprintf(stderr, "error0\n"); + } + + if(e_self->in_node != e_reverse->in_node || e_self->out_node != e_reverse->out_node || + e_self->weight != e_reverse->weight) + { + fprintf(stderr, "error1\n"); + } + + + if(e_self != &(Output_Edges(G_Node(*DAGCon, e_self->in_node)).list[e_self->self_edge_ID])) + { + fprintf(stderr, "error2\n"); + } + + + + + + + + get_bi_direction_edges(DAGCon, e_reverse, &e_self, &e_reverse); + + if(e != e_reverse) + { + fprintf(stderr, "error0\n"); + } + + if(e_self->in_node != e_reverse->in_node || e_self->out_node != e_reverse->out_node || + e_self->weight != e_reverse->weight) + { + fprintf(stderr, "error1\n"); + } + + + if(e_self != &(Output_Edges(G_Node(*DAGCon, e_self->in_node)).list[e_self->self_edge_ID])) + { + fprintf(stderr, "error2\n"); + } + } + } + // else + // { + // fprintf(stderr, "node does not exist\n"); + // } + + } +} + + +void debug_Queue(Graph* DAGCon) +{ + long long* input; + long long* output; + + srand((unsigned)time(0)); + long long array_length = rand() % 100; + input = (long long*)malloc(sizeof(long long) * (array_length + 1)); + output = (long long*)malloc(sizeof(long long) * (array_length + 1)); + long long i; + + for (i = 0; i < array_length; i++) + { + input[i] = rand() % 1000000; + push_to_Queue(&(DAGCon->node_q), input[i]); + } + + i = 0; + while (pop_from_Queue(&(DAGCon->node_q), &output[i])) + { + i++; + } + + if(i != array_length) + { + fprintf(stderr, "array_length: %d\n", array_length); + } + else + { + for (i = 0; i < array_length; i++) + { + if(input[i] != output[i]) + { + fprintf(stderr, "input[%d]:%d, output[%d]: %d\n", + i, input[i], i, output[i]); + } + + } + } + + + + + + + + + + srand((unsigned)time(0)); + if(array_length != 0) + { + array_length = rand() % array_length; + } + + + for (i = 0; i < array_length; i++) + { + input[i] = rand() % 1000000; + push_to_Queue(&(DAGCon->node_q), input[i]); + } + + i = 0; + while (pop_from_Queue(&(DAGCon->node_q), &output[i])) + { + i++; + } + + if(i != array_length) + { + fprintf(stderr, "array_length: %d\n", array_length); + } + else + { + for (i = 0; i < array_length; i++) + { + if(input[i] != output[i]) + { + fprintf(stderr, "input[%d]:%d, output[%d]: %d\n", + i, input[i], i, output[i]); + } + + } + } + + + free(input); + free(output); +} + + +///return the in-edge ID of outNode +long long get_In_Edge_ID(Graph* DAGCon, long long inNode, long long outNode) +{ + long long i; + + for (i = 0; i < Input_Edges(G_Node(*DAGCon, outNode)).length; i++) + { + if (Input_Edges(G_Node(*DAGCon, outNode)).list[i].in_node == inNode) + { + return i; + } + } + + return -1; +} + + +///return the out-edge ID of inNode +long long get_Out_Edge_ID(Graph* DAGCon, long long inNode, long long outNode) +{ + long long i; + for (i = 0; i < Output_Edges(G_Node(*DAGCon, inNode)).length; i++) + { + if(Output_Edges(G_Node(*DAGCon, inNode)).list[i].out_node == outNode) + { + return i; + } + } + + return -1; +} + + + + +void Merge_Out_Nodes(Graph* DAGCon, Node* currentNode) +{ + ///if this node does not have any output, directly return + if(Real_Length(Output_Edges((*currentNode))) == 0) + { + return; + } + + RSet buf, out_buf; + char Bases[4] = {'A', 'C', 'G', 'T'}; + char base; + long long base_i, i, weight; + int flag = 0; + Node* get_node_1; + Node* out_node_of_get_node_1; + Node* consensus_node_1; + Edge* e_forward_1; + Edge* e_backward_1; + + ///merge all base for each base + for (base_i = 0; base_i < 4; base_i++) + { + base = Bases[base_i]; + + clear_RSet(&buf); + flag = 0; + weight = 0; + ///should use getOutputEdges, instead of getOutputNodes + ///check all out-nodes of currentNode + while(getOutputNodes(&buf, DAGCon, currentNode, &get_node_1)) + { + ///check the corresponding node, this node must only have one in-node + ///note this is the Real_Length, instead of the Input_Edges.length + if((*get_node_1).base == base && Real_Length(Input_Edges(*get_node_1)) == 1) + { + if(flag == 0) + { + flag = 1; + ///add a new node to merge all out-node + consensus_node_1 = get_node_1; + ///link consensus_node to currentNode + ///set the new edge to be visited + if(get_bi_Edge(DAGCon, currentNode, consensus_node_1, &e_forward_1, &e_backward_1)) + { + Visit(*e_forward_1) = 1; + Visit(*e_backward_1) = 1; + } + else + { + fprintf(stderr, "error\n"); + } + + weight = (*e_forward_1).weight; + } + else + { + flag++; + ///add the weight of get_node->currentNode + if(get_bi_Edge(DAGCon, currentNode, get_node_1, &e_forward_1, &e_backward_1)) + { + weight = weight + (*e_forward_1).weight; + } + else + { + fprintf(stderr, "error\n"); + } + + ///process the out-nodes of get_node + clear_RSet(&out_buf); + while(getOutputNodes(&out_buf, DAGCon, get_node_1, &out_node_of_get_node_1)) + { + ///link consensus_node to the out-nodes of get_node + if(get_bi_Edge(DAGCon, consensus_node_1, out_node_of_get_node_1, &e_forward_1, &e_backward_1)) + { + Visit(*e_forward_1) = 1; + Visit(*e_backward_1) = 1; + (*e_forward_1).weight += get_Edge_Weight(DAGCon, get_node_1, out_node_of_get_node_1); + (*e_backward_1).weight = (*e_forward_1).weight; + } + else + { + add_bi_direction_edge(DAGCon, consensus_node_1, out_node_of_get_node_1, + get_Edge_Weight(DAGCon, get_node_1, out_node_of_get_node_1), 1); + } + } + + delete_Node_DAGCon(DAGCon, get_node_1); + } + } + } + + if(flag > 1) + { + get_bi_Edge(DAGCon, currentNode, consensus_node_1, &e_forward_1, &e_backward_1); + (*e_forward_1).weight = weight; + (*e_backward_1).weight = (*e_forward_1).weight; + } + + if(flag > 0) + { + Merge_Out_Nodes(DAGCon, consensus_node_1); + } + } +} + + + + + + +void Merge_In_Nodes(Graph* DAGCon, Node* currentNode) +{ + ///if this node does not have any input, directly return + if(Real_Length(Input_Edges((*currentNode))) == 0) + { + return; + } + + RSet buf, in_buf; + char Bases[4] = {'A', 'C', 'G', 'T'}; + char base; + long long base_i, i, weight; + int flag = 0; + Node* get_node; + Node* in_node_of_get_node; + Node* consensus_node; + Edge* e_forward; + Edge* e_backward; + + ///merge all base for each base + for (base_i = 0; base_i < 4; base_i++) + { + base = Bases[base_i]; + + clear_RSet(&buf); + flag = 0; + weight = 0; + ///should use getInputEdges, instead of getInputNodes + ///check all in-nodes of currentNode + while(getInputNodes(&buf, DAGCon, currentNode, &get_node)) + { + ///check the corresponding node, this node must only have one out-node + ///note this is the Real_Length, instead of the Output_Edges.length + if((*get_node).base == base && Real_Length(Output_Edges(*get_node)) == 1) + { + if(flag == 0) + { + flag = 1; + ///add a new node to merge all in-node + consensus_node = get_node; + ///link consensus_node to currentNode + ///set the new edge to be visited + if(get_bi_Edge(DAGCon, consensus_node, currentNode, &e_forward, &e_backward)) + { + Visit(*e_forward) = 1; + Visit(*e_backward) = 1; + } + else + { + fprintf(stderr, "error\n"); + } + + weight = (*e_forward).weight; + } + else + { + flag++; + ///add the weight of get_node->currentNode + if(get_bi_Edge(DAGCon, get_node, currentNode, &e_forward, &e_backward)) + { + weight = weight + (*e_forward).weight; + } + else + { + fprintf(stderr, "error\n"); + } + + ///process the in-nodes of get_node + clear_RSet(&in_buf); + while(getInputNodes(&in_buf, DAGCon, get_node, &in_node_of_get_node)) + { + ///link in-nodes of get_node to consensus_node + if(get_bi_Edge(DAGCon, in_node_of_get_node, consensus_node, &e_forward, &e_backward)) + { + Visit(*e_forward) = 1; + Visit(*e_backward) = 1; + (*e_forward).weight += get_Edge_Weight(DAGCon, in_node_of_get_node, get_node); + (*e_backward).weight = (*e_forward).weight; + } + else + { + add_bi_direction_edge(DAGCon, in_node_of_get_node, consensus_node, + get_Edge_Weight(DAGCon, in_node_of_get_node, get_node), 1); + } + } + + delete_Node_DAGCon(DAGCon, get_node); + } + } + } + + if(flag > 1) + { + get_bi_Edge(DAGCon, consensus_node, currentNode, &e_forward, &e_backward); + (*e_forward).weight = weight; + (*e_backward).weight = (*e_forward).weight; + } + + if(flag > 0) + { + Merge_In_Nodes(DAGCon, consensus_node); + } + } +} + + + + + +void print_graph(Graph* DAGCon) +{ + long long i; + for (long long i = 0; i < DAGCon->g_nodes.length; i++) + { + Node* currentStartNode = &(G_Node(*DAGCon, i)); + RSet iter_out; + + if(If_Node_Exist(*currentStartNode)) + { + fprintf(stderr, "ID: %d (%c) (w: %d)\n", (*currentStartNode).ID, (*currentStartNode).base, (*currentStartNode).weight); + clear_RSet(&iter_out); + Edge* e; + fprintf(stderr, "****Out-node: "); + while(getOutputEdges(&iter_out, DAGCon, currentStartNode, &e)) + { + //fprintf(stderr, "%d[%c], ", G_Node(*DAGCon, e->out_node).ID, G_Node(*DAGCon, e->out_node).base); + fprintf(stderr, "%d(w: %d), ", G_Node(*DAGCon, e->out_node).ID, e->weight); + } + fprintf(stderr, "\n"); + + + // clear_RSet(&iter_out); + // fprintf(stderr, "In-node: "); + // while(getInputEdges(&iter_out, DAGCon, currentStartNode, &e)) + // { + // fprintf(stderr, "%d[%c], ", G_Node(*DAGCon, e->in_node).ID, G_Node(*DAGCon, e->in_node).base); + // } + } + } + + fprintf(stderr, "*******\n"); + +} + + + + +void debug_DAGCon(Graph* DAGCon) +{ + for (long long i = 0; i < DAGCon->g_nodes.length; i++) + { + Node* currentStartNode = &(G_Node(*DAGCon, i)); + RSet iter_out; + + if(If_Node_Exist(*currentStartNode)) + { + clear_RSet(&iter_out); + Edge* e; + Edge* e_self; + Edge* e_reverse; + while(getOutputEdges(&iter_out, DAGCon, currentStartNode, &e_self)) + { + + get_bi_direction_edges(DAGCon, e_self, &e_self, &e_reverse); + + + if(Visit(*e_self) == 0) + { + fprintf(stderr, "Visit(*e_self): %d, error visit flag: in_node: %d, out_node: %d\n", + Visit(*e_self), (*e_self).in_node, (*e_self).out_node); + } + + + if(Visit(*e_reverse) == 0) + { + fprintf(stderr, "Visit(*e_reverse): %d, error visit flag: in_node: %d, out_node: %d\n", + Visit(*e_reverse), (*e_reverse).in_node, (*e_reverse).out_node); + } + + + if(e_self->in_node != e_reverse->in_node) + { + fprintf(stderr, "different in-node\n"); + } + if(e_self->out_node != e_reverse->out_node) + { + fprintf(stderr, "different out-node\n"); + } + if(e_self->weight != e_reverse->weight) + { + fprintf(stderr, "different weight\n"); + } + } + + clear_RSet(&iter_out); + while(getInputEdges(&iter_out, DAGCon, currentStartNode, &e_self)) + { + + get_bi_direction_edges(DAGCon, e_self, &e_self, &e_reverse); + + + if(Visit(*e_self) == 0) + { + fprintf(stderr, "Visit(*e_self): %d, error visit flag: in_node: %d, out_node: %d\n", + Visit(*e_self), (*e_self).in_node, (*e_self).out_node); + } + + + if(Visit(*e_reverse) == 0) + { + fprintf(stderr, "Visit(*e_reverse): %d, error visit flag: in_node: %d, out_node: %d\n", + Visit(*e_reverse), (*e_reverse).in_node, (*e_reverse).out_node); + } + + + if(e_self->in_node != e_reverse->in_node) + { + fprintf(stderr, "different in-node\n"); + } + if(e_self->out_node != e_reverse->out_node) + { + fprintf(stderr, "different out-node\n"); + } + if(e_self->weight != e_reverse->weight) + { + fprintf(stderr, "different weight\n"); + } + } + } + } +} + +void Merge_DAGCon(Graph* DAGCon) +{ + ///using the length of edge representing if it has been visited + ///in default, the length of edge is 0 + RSet iter_node, iter_edge; + long long flag; + + Node* currentNode; + Node* outNode; + Edge* edge; + Edge* e_forward; + Edge* e_backward; + + // int num_way = Real_Length(Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID))); + // if(num_way > 2) + // { + // print_graph(DAGCon); + // } + + + + ///at begining, only the start node has no in-node + currentNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + Push_Node(DAGCon, ¤tNode); + + + + while (Pop_Node(DAGCon, ¤tNode)) + { + ///merge in-node + Merge_In_Nodes(DAGCon, currentNode); + ///merge out-node + Merge_Out_Nodes(DAGCon, currentNode); + + clear_RSet(&iter_edge); + ///for all out-edges of currentNode, set as visited + while (getOutputEdges(&iter_edge, DAGCon, currentNode, &edge)) + { + get_bi_direction_edges(DAGCon, edge, &e_forward, &e_backward); + Visit(*e_forward) = 1; + Visit(*e_backward) = 1; + } + + + + ///check all out-node of currentNode + clear_RSet(&iter_node); + while(getOutputNodes(&iter_node, DAGCon, currentNode, &outNode)) + { + + ///for each outNode, check if all in-edges have been visited + flag = 0; + clear_RSet(&iter_edge); + while (getInputEdges(&iter_edge, DAGCon, outNode, &edge)) + { + if(Visit(*edge) == 0) + { + flag = 1; + break; + } + } + //if all in-edges of Out_node have already been visited, push it to queue + if(flag == 0) + { + Push_Node(DAGCon, &outNode); + } + } + } + + + + // if(num_way > 2) + // { + // print_graph(DAGCon); + // fprintf(stderr, "****************************note*****************\n\n"); + // } + + + ///debug_DAGCon(DAGCon); +} + + +inline void generate_seq_from_path(Graph* DAGCon, Node* node, int direction) +{ + clear_Queue(&(DAGCon->node_q)); + RSet iter; + Edge* e; + long long max; + Node* max_node; + + + if(direction == 0) + { + while (node->ID != DAGCon->s_end_nodeID) + { + push_to_Queue(&(DAGCon->node_q), node->base); + clear_RSet(&iter); + max = 0; + while(getOutputEdges(&iter, DAGCon, node, &e)) + { + if(e->weight > max) + { + max = e->weight; + max_node = &(G_Node(*DAGCon, e->out_node)); + } + } + node = max_node; + } + } + else + { + while (node->ID != DAGCon->s_start_nodeID) + { + push_to_Queue(&(DAGCon->node_q), node->base); + clear_RSet(&iter); + max = 0; + while(getInputEdges(&iter, DAGCon, node, &e)) + { + if(e->weight > max) + { + max = e->weight; + max_node = &(G_Node(*DAGCon, e->in_node)); + } + } + node = max_node; + } + + long long i, k; + long long length = (DAGCon->node_q.end - DAGCon->node_q.beg); + long long length_ex = length/2; + long long* array = DAGCon->node_q.buffer + DAGCon->node_q.beg; + for (i = 0; i < length_ex; i++) + { + k = array[i]; + array[i] = array[length - i - 1]; + array[length - i - 1] = k; + } + } +} + + +long long generate_best_seq_from_edges(Graph* DAGCon) +{ + long long max_start, max_end, max_start_edge, max_end_edge; + RSet iter; + Edge* e; + Node* newNode; + long long max_count; + + + ///check the out-edges of start node + ///must to be 0 + max_start = 0; + newNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + clear_RSet(&iter); + while(getOutputEdges(&iter, DAGCon, newNode, &e)) + { + if(e->weight > max_start) + { + max_start = e->weight; + max_start_edge = iter.index - 1; + } + } + + ///check the in-edges of end node + ///must to be 0 + max_end = 0; + newNode = &(G_Node(*DAGCon, DAGCon->s_end_nodeID)); + clear_RSet(&iter); + while(getInputEdges(&iter, DAGCon, newNode, &e)) + { + if(e->weight > max_end) + { + max_end = e->weight; + max_end_edge = iter.index - 1; + } + } + + if(max_start >= max_end) + { + max_count = max_start; + generate_seq_from_path(DAGCon, + &G_Node(*DAGCon, Output_Edges(G_Node(*DAGCon, DAGCon->s_start_nodeID)).list[max_start_edge].out_node), 0); + } + else + { + max_count = max_end; + generate_seq_from_path(DAGCon, + &G_Node(*DAGCon, Input_Edges(G_Node(*DAGCon, DAGCon->s_end_nodeID)).list[max_end_edge].in_node), 1); + } + + + + ///if((*direction) == 1) + ///if(max_start < max_end && DAGCon->g_nodes.length > 5) + ///if(DAGCon->g_nodes.length > 5) + // if(max_start < max_end && DAGCon->node_q.end - DAGCon->node_q.beg > 1) + // { + // print_graph(DAGCon); + // long long str; + // while (pop_from_Queue(&(DAGCon->node_q), &str)) + // { + // fprintf(stderr, "%c", (char)str); + // } + // fprintf(stderr, "\n"); + // fprintf(stderr, "###################(*max_count): %d, (*max_edge): %d, (*direction): %d###################\n\n", + // (*max_count), max_start >= max_end? max_start_edge:max_end_edge, max_start >= max_end? 0:1); + // } + // if(max_start >= max_end) + // { + // newNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + // if(Output_Edges(*newNode).list[max_start_edge].weight != (*max_count)) + // { + // fprintf(stderr, "ERROR\n"); + // } + // } + // else + // { + // newNode = &(G_Node(*DAGCon, DAGCon->s_end_nodeID)); + // if(Input_Edges(*newNode).list[max_end_edge].weight != (*max_count)) + // { + // fprintf(stderr, "ERROR\n"); + // } + // } + + return max_count; + + +} + + +inline void generate_seq_from_node(Graph* DAGCon, Node* node, int direction) +{ + clear_Queue(&(DAGCon->node_q)); + RSet iter; + long long max; + Node* max_node; + Node* getNodes; + + + if(direction == 0) + { + while (node->ID != DAGCon->s_end_nodeID) + { + push_to_Queue(&(DAGCon->node_q), node->base); + clear_RSet(&iter); + max = 0; + while(getOutputNodes(&iter, DAGCon, node, &getNodes)) + { + if(getNodes->weight > max) + { + max = getNodes->weight; + max_node = getNodes; + } + } + node = max_node; + } + } + else + { + while (node->ID != DAGCon->s_start_nodeID) + { + push_to_Queue(&(DAGCon->node_q), node->base); + clear_RSet(&iter); + max = 0; + while(getInputNodes(&iter, DAGCon, node, &getNodes)) + { + if(getNodes->weight > max) + { + max = getNodes->weight; + max_node = getNodes; + } + } + node = max_node; + } + + long long i, k; + long long length = (DAGCon->node_q.end - DAGCon->node_q.beg); + long long length_ex = length/2; + long long* array = DAGCon->node_q.buffer + DAGCon->node_q.beg; + for (i = 0; i < length_ex; i++) + { + k = array[i]; + array[i] = array[length - i - 1]; + array[length - i - 1] = k; + } + } +} + + +long long generate_best_seq_from_nodes(Graph* DAGCon) +{ + long long max_start, max_end; + RSet iter; + Edge* e; + Node* newNode; + Node* getNode; + Node* max_start_node; + Node* max_end_node; + long long max_count, i; + + for (i = 0; i < DAGCon->g_nodes.length; i++) + { + newNode = &(G_Node(*DAGCon, i)); + if(If_Node_Exist(*newNode)) + { + newNode->weight = 0; + clear_RSet(&iter); + while(getOutputEdges(&iter, DAGCon, newNode, &e)) + { + newNode->weight += e->weight; + } + } + } + + newNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + newNode->weight = 0; + clear_RSet(&iter); + while(getOutputEdges(&iter, DAGCon, newNode, &e)) + { + newNode->weight += e->weight; + } + + + newNode = &(G_Node(*DAGCon, DAGCon->s_end_nodeID)); + newNode->weight = 0; + clear_RSet(&iter); + while(getInputEdges(&iter, DAGCon, newNode, &e)) + { + newNode->weight += e->weight; + } + + + + + ///check the out-edges of start node + ///must to be 0 + max_start = 0; + newNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + clear_RSet(&iter); + while(getOutputNodes(&iter, DAGCon, newNode, &getNode)) + { + if(getNode->weight > max_start) + { + max_start = getNode->weight; + max_start_node = getNode; + } + } + + ///check the in-edges of end node + ///must to be 0 + max_end = 0; + newNode = &(G_Node(*DAGCon, DAGCon->s_end_nodeID)); + clear_RSet(&iter); + while(getInputNodes(&iter, DAGCon, newNode, &getNode)) + { + if(getNode->weight > max_end) + { + max_end = getNode->weight; + max_end_node = getNode; + } + } + + if(max_start >= max_end) + { + max_count = max_start; + generate_seq_from_node(DAGCon, max_start_node, 0); + } + else + { + max_count = max_end; + generate_seq_from_node(DAGCon, max_end_node, 1); + } + + + /** + if(DAGCon->g_nodes.length > 5) + ///if(max_start < max_end && DAGCon->node_q.end - DAGCon->node_q.beg > 1) + ///if(max_start < max_end) + { + print_graph(DAGCon); + long long str; + while (pop_from_Queue(&(DAGCon->node_q), &str)) + { + fprintf(stderr, "%c", (char)str); + } + fprintf(stderr, "\n"); + fprintf(stderr, "###################(*max_count): %d, (*max_node): %d, (*direction): %d###################\n\n", + max_count, max_start >= max_end? max_start_node->ID:max_end_node->ID, max_start >= max_end? 0:1); + } + if(max_start >= max_end) + { + newNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + if(max_start_node->weight != max_count) + { + fprintf(stderr, "ERROR\n"); + } + } + else + { + newNode = &(G_Node(*DAGCon, DAGCon->s_end_nodeID)); + if(max_end_node->weight != max_count) + { + fprintf(stderr, "ERROR\n"); + } + } + **/ + + return max_count; + + +} + + +void build_DAGCon(Graph* DAGCon, Graph* backbone, long long currentNodeID, long long* max_count) +{ + long long i, j, path_weight, nodeID, step; + char base; + clear_Graph(DAGCon); + Node* newNode; + Node* lastNode; + + ///add the start node and the end node + newNode = add_Node_DAGCon(DAGCon, 'S'); + DAGCon->s_start_nodeID = newNode->ID; + + newNode = add_Node_DAGCon(DAGCon, 'E'); + DAGCon->s_end_nodeID = newNode->ID; + + + for (i = 0; i < G_Node(*backbone, currentNodeID).insertion_edges.length; i++) + { + path_weight = G_Node(*backbone, currentNodeID).insertion_edges.list[i].weight; + + lastNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + + + /*****************************debug***************************************/ + // newNode = add_Node_DAGCon(DAGCon, 'F'); + // add_bi_direction_edge(DAGCon, lastNode, newNode, path_weight, 0); + // add_bi_direction_edge(DAGCon, newNode, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)), path_weight, 0); + // delete_Node_DAGCon(DAGCon, newNode); + // if(remove_and_check_bi_direction_edge_from_nodes(DAGCon, &(G_Node(*DAGCon, DAGCon->s_start_nodeID)), newNode)) + // { + // fprintf(stderr, "step: %d, j: %d\n", step, j); + // } + // if(remove_and_check_bi_direction_edge_from_nodes(DAGCon, newNode, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)))) + // { + // fprintf(stderr, "step: %d, j: %d\n", step, j); + // } + + + // Node* node0; + // Node* node1; + // Node* node2; + // newNode = add_Node_DAGCon(DAGCon, 'T'); + // add_bi_direction_edge(DAGCon, lastNode, newNode, path_weight, 0); + // node0 = newNode; + // lastNode = newNode; + // newNode = add_Node_DAGCon(DAGCon, 'T'); + // add_bi_direction_edge(DAGCon, lastNode, newNode, path_weight, 0); + // node1 = newNode; + // lastNode = newNode; + // newNode = add_Node_DAGCon(DAGCon, 'T'); + // add_bi_direction_edge(DAGCon, lastNode, newNode, path_weight, 0); + // node2 = newNode; + + + // lastNode = &(G_Node(*DAGCon, DAGCon->s_start_nodeID)); + // delete_Node_DAGCon(DAGCon, node1); + + // Edge* e_forward; + // Edge* e_backward; + // if(get_bi_Edge(DAGCon, node2, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)), &e_forward, &e_backward)) + // { + // remove_and_check_bi_direction_edge_from_edge(DAGCon, e_forward); + // } + + // if(get_bi_Edge(DAGCon, node2, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)), &e_forward, &e_backward)) + // { + // fprintf(stderr, "edge remove error\n"); + // } + + // remove_and_check_bi_direction_edge_from_nodes(DAGCon, lastNode, node0); + // remove_and_check_bi_direction_edge_from_nodes(DAGCon, node0, node1); + // remove_and_check_bi_direction_edge_from_nodes(DAGCon, node1, node2); + // if(remove_and_check_bi_direction_edge_from_nodes(DAGCon, node2, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)))) + // { + // fprintf(stderr, "edge remove error\n"); + // } + /*****************************debug***************************************/ + + + step = G_Node(*backbone, currentNodeID).insertion_edges.list[i].length; + if(step != 0) + { + nodeID = G_Node(*backbone, currentNodeID).insertion_edges.list[i].out_node; + + for (j = 0; j < step; j++) + { + base = G_Node(*backbone, nodeID).base; + newNode = add_Node_DAGCon(DAGCon, base); + add_bi_direction_edge(DAGCon, lastNode, newNode, path_weight, 0); + + /*****************************debug***************************************/ + // if(!add_and_check_bi_direction_edge(DAGCon, lastNode, newNode, path_weight, 0)) + // { + // fprintf(stderr, "haha\n"); + // } + + // if(add_and_check_bi_direction_edge(DAGCon, lastNode, newNode, path_weight, 0)) + // { + // fprintf(stderr, "haha\n"); + // } + // if(j != 0) + // { + // add_bi_direction_edge(DAGCon, &(G_Node(*DAGCon, DAGCon->s_start_nodeID)), newNode, path_weight, 0); + // if(!remove_and_check_bi_direction_edge_from_nodes(DAGCon, &(G_Node(*DAGCon, DAGCon->s_start_nodeID)), + // newNode)) + // { + // fprintf(stderr, "step: %d, j: %d\n", step, j); + // } + // } + /*****************************debug***************************************/ + + nodeID = G_Node(*backbone, nodeID).insertion_edges.list[0].out_node; + + lastNode = newNode; + } + + if(lastNode->ID != DAGCon->s_start_nodeID) + { + add_bi_direction_edge(DAGCon, lastNode, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)), path_weight, 0); + + /*****************************debug***************************************/ + // if(!add_and_check_bi_direction_edge(DAGCon, lastNode, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)), path_weight, 0)) + // { + // fprintf(stderr, "haha\n"); + // } + + // if(add_and_check_bi_direction_edge(DAGCon, lastNode, &(G_Node(*DAGCon, DAGCon->s_end_nodeID)), path_weight, 0)) + // { + // fprintf(stderr, "haha\n"); + // } + // add_bi_direction_edge(DAGCon, &(G_Node(*DAGCon, DAGCon->s_start_nodeID)), + // &(G_Node(*DAGCon, DAGCon->s_end_nodeID)), path_weight, 0); + // if(!remove_and_check_bi_direction_edge_from_nodes(DAGCon, &(G_Node(*DAGCon, DAGCon->s_start_nodeID)), + // &(G_Node(*DAGCon, DAGCon->s_end_nodeID)))) + // { + // fprintf(stderr, "step: %d, j: %d\n", step, j); + // } + /*****************************debug***************************************/ + } + } + + + + } + + ///test_single_path_new(DAGCon, backbone, currentNodeID); + + + Merge_DAGCon(DAGCon); + + ///(*max_count) = generate_best_seq_from_edges(DAGCon); + (*max_count) = generate_best_seq_from_nodes(DAGCon); + + // long long k = 0; + // for (i = 0; i < DAGCon->g_nodes.length; i++) + // { + // if(i != DAGCon->s_start_nodeID && i != DAGCon->s_end_nodeID) + // { + // if(If_Node_Exist(G_Node(*DAGCon, i)) && G_Node(*DAGCon, i).weight > k) + // { + // k = G_Node(*DAGCon, i).weight; + // } + // } + // } + + // if(k != (*max_count)) + // { + // fprintf(stderr, "k: %d, (*max_count): %d\n", k, (*max_count)); + // } + + + ///very important + backbone->g_nodes.list[currentNodeID].num_insertions = 0; + +} + +void debug_whole_graph(Graph* g) +{ + long long i, j, k; + for (i = 0; i < g->g_nodes.length; i++) + { + if(g->g_nodes.list[i].deletion_edges.length!= 0 && + g->g_nodes.list[i].deletion_edges.length!= 1) + { + fprintf(stderr, "g->g_nodes.list[i].deletion_edges.length: %d\n", + g->g_nodes.list[i].deletion_edges.length); + } + } + + for (i = g->s_start_nodeID; i < g->s_end_nodeID; i++) + { + if(g->g_nodes.list[i].mismatch_edges.length > 4 + || + g->g_nodes.list[i].mismatch_edges.length < 1) + { + fprintf(stderr, "g->s_end_nodeID: %d, g->g_nodes.list[%d].mismatch_edges.length: %d\n", + g->s_end_nodeID, i, g->g_nodes.list[i].mismatch_edges.length); + } + } + + char current[1000]; + char compare[1000]; + long long total_weight = 0; + for (i = g->s_start_nodeID; i < g->s_end_nodeID; i++) + { + total_weight = 0; + for (j = 0; j < G_Node(*g, i).insertion_edges.length; j++) + { + + total_weight = total_weight + G_Node(*g, i).insertion_edges.list[j].weight; + + extract_path(g, i, j, current); + + for (k = j + 1; k < G_Node(*g, i).insertion_edges.length; k++) + { + extract_path(g, i, k, compare); + if(strcmp(current, compare)==0) + { + fprintf(stderr,"error\n"); + } + } + } + + if(total_weight != G_Node(*g, i).num_insertions) + { + fprintf(stderr,"error\n"); + } + } + + + +} + +void get_seq_from_Graph(Graph* backbone, Graph* DAGCon, Correct_dumy* dumy, Cigar_record* current_cigar, char* self_string, +char* r_string, long long r_string_length, long long r_string_site) +{ + ///debug_whole_graph(backbone); + + // double threshold; + // if(roundID > 0) + // { + // threshold = CORRECT_THRESHOLD_SECOND; + // } + // else + // { + // threshold = CORRECT_THRESHOLD; + // } + + + + long long new_seq_length = 0; long long currentNodeID; long long i; @@ -1754,8 +5034,12 @@ void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy, Cigar_record* curre char current_base; long long current_weight; + long long max_insertion_count; + currentNodeID = backbone->s_start_nodeID; + ///fprintf(stderr, "currentNodeID: %d\n", currentNodeID); + while (currentNodeID != backbone->s_end_nodeID) { total_count = 0; @@ -1769,6 +5053,10 @@ void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy, Cigar_record* curre ///1. mismatch_edges 2. insertion_edges 3. deletion_edges if (currentNodeID >= backbone->s_start_nodeID && currentNodeID <= backbone->s_end_nodeID) { + + + + ///mismatch_edges for (i = 0; i < backbone->g_nodes.list[currentNodeID].mismatch_edges.length; i++) @@ -1798,6 +5086,170 @@ void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy, Cigar_record* curre } } + ///insertion_edges + if (backbone->g_nodes.list[currentNodeID].num_insertions != 0) + { + ///this line must be prior than the next line + ///since build_DAGCon will set backbone->g_nodes.list[currentNodeID].num_insertions to be 0 + total_count = total_count + backbone->g_nodes.list[currentNodeID].num_insertions; + + build_DAGCon(DAGCon, backbone, currentNodeID, &max_insertion_count); + + if(max_insertion_count > max_count) + { + max_count = max_insertion_count; + max_type = INSERTION; + } + // for (i = 0; i < backbone->g_nodes.list[currentNodeID].insertion_edges.length; i++) + // { + // total_count = total_count + backbone->g_nodes.list[currentNodeID].insertion_edges.list[i].weight; + + // if (backbone->g_nodes.list[currentNodeID].insertion_edges.list[i].weight > max_count) + // { + // max_count = backbone->g_nodes.list[currentNodeID].insertion_edges.list[i].weight; + // max_edge = i; + // max_type = INSERTION; + // } + // } + } + + + + + + ///deletion_edges + for (i = 0; i < backbone->g_nodes.list[currentNodeID].deletion_edges.length; i++) + { + total_count = total_count + backbone->g_nodes.list[currentNodeID].deletion_edges.list[i].weight; + + if (backbone->g_nodes.list[currentNodeID].deletion_edges.list[i].weight > max_count) + { + max_count = backbone->g_nodes.list[currentNodeID].deletion_edges.list[i].weight; + max_edge = i; + max_type = DELETION; + } + } + + + + + + ///这种情况下矫正 + if(max_count >= total_count*(CORRECT_THRESHOLD)) + ///if(max_count >= total_count * threshold) + { + currentNodeID = add_path_to_correct_read_new(backbone, DAGCon, dumy, currentNodeID, max_type, max_edge, current_cigar, + self_string); + } + else + { + ///NOTE: currentNodeID = 0 is a tmp node without any sense + if(currentNodeID > 0 && if_is_homopolymer_strict(r_string_site + currentNodeID - 1, r_string, r_string_length) + && max_count >= total_count*CORRECT_THRESHOLD_HOMOPOLYMER) + { + currentNodeID = add_path_to_correct_read_new(backbone, DAGCon, dumy, currentNodeID, max_type, max_edge, current_cigar, + self_string); + } + else///不矫正, 直接取下一个backbone节点 + { + currentNodeID++; + add_base_to_correct_read_directly(dumy, backbone->g_nodes.list[currentNodeID].base); + + add_cigar_record(&(backbone->g_nodes.list[currentNodeID].base), 1, current_cigar, 0); + } + } + + ///fprintf(stderr, "currentNodeID: %d, max_type: %d\n", currentNodeID, max_type); + } + else ///非backbone节点就会出错了 + { + fprintf(stderr, "error\n"); + } + + } + +} + + +void get_seq_from_Graph_print(Graph* backbone, Correct_dumy* dumy, Cigar_record* current_cigar, char* self_string, +char* r_string, long long r_string_length, long long r_string_site) +{ + long long new_seq_length = 0; + long long currentNodeID; + long long i; + // 总共有以下几种情况: + // 1. match 2. mismatch (A, C, G, T, N) 3. deletion 4. insertion (A, C, G, T) + // 其实就是 1. 自己本身的weight 2. alignToNode的weight 3. insertion节点的weight + long long max_count; + int max_type; + long long max_edge; + long long total_count; + long long nodeID; + char current_base; + long long current_weight; + + currentNodeID = backbone->s_start_nodeID; + + ///fprintf(stderr, "currentNodeID: %d\n", currentNodeID); + + while (currentNodeID != backbone->s_end_nodeID) + { + total_count = 0; + max_count = -1; + max_type = -1; + max_edge = -1; + + + ///假如这是个backbone节点 + ///有三类出边 + ///1. mismatch_edges 2. insertion_edges 3. deletion_edges + if (currentNodeID >= backbone->s_start_nodeID && currentNodeID <= backbone->s_end_nodeID) + { + + + + + + ///mismatch_edges + for (i = 0; i < backbone->g_nodes.list[currentNodeID].mismatch_edges.length; i++) + { + if(currentNodeID == 187) + { + fprintf(stderr, "backbone->g_nodes.list[currentNodeID].num_insertions: %d\n", + backbone->g_nodes.list[currentNodeID].num_insertions); + } + + if (backbone->g_nodes.list[currentNodeID].num_insertions != 0) + { + current_weight = backbone->g_nodes.list[currentNodeID].mismatch_edges.list[i].weight - + backbone->g_nodes.list[currentNodeID].mismatch_edges.list[i].num_insertions; + } + else + { + current_weight = backbone->g_nodes.list[currentNodeID].mismatch_edges.list[i].weight; + } + + if(currentNodeID == 187) + { + fprintf(stderr, "current_weight: %d\n", + current_weight); + } + + + total_count = total_count + current_weight; + + ///match + ///match要处理插入的情况 + ///如果这里有insertion, 这个节点会过两遍 + ///第一遍num_insertions > 0, 第二遍num_insertions=0 + if (current_weight > max_count) + { + max_count = current_weight; + max_edge = i; + max_type = MISMATCH; + } + } + ///insertion_edges if (backbone->g_nodes.list[currentNodeID].num_insertions != 0) { @@ -1811,6 +5263,16 @@ void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy, Cigar_record* curre max_edge = i; max_type = INSERTION; } + + if(currentNodeID == 187) + { + fprintf(stderr, "backbone->g_nodes.list[currentNodeID].insertion_edges.list[i].weight: %d, length: %d\n", + backbone->g_nodes.list[currentNodeID].insertion_edges.list[i].weight, + backbone->g_nodes.list[currentNodeID].insertion_edges.list[i].length); + + nodeID = backbone->g_nodes.list[currentNodeID].insertion_edges.list[i].out_node; + fprintf(stderr, "%c\n", backbone->g_nodes.list[nodeID].base); + } } } @@ -1827,26 +5289,61 @@ void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy, Cigar_record* curre max_edge = i; max_type = DELETION; } + + if(currentNodeID == 187) + { + fprintf(stderr, "backbone->g_nodes.list[currentNodeID].deletion_edges.list[i].weight: %d\n", + backbone->g_nodes.list[currentNodeID].deletion_edges.list[i].weight); + } } + + /** + if(currentNodeID > 0 && backbone->g_nodes.list[currentNodeID].base != r_string[r_string_site + currentNodeID - 1]) + { + fprintf(stderr, "currentNodeID: %d\n", currentNodeID); + } + **/ + fprintf(stderr, "currentNodeID: %d, max_count: %d, max_type: %d, total_count: %d\n", + currentNodeID, max_count, max_type, total_count); + + + + if(currentNodeID == 187) + { + nodeID = backbone->g_nodes.list[currentNodeID].insertion_edges.list[max_edge].out_node; + fprintf(stderr, "%c", backbone->g_nodes.list[nodeID].base); + nodeID = backbone->g_nodes.list[nodeID].insertion_edges.list[0].out_node; + fprintf(stderr, "%c\n", backbone->g_nodes.list[nodeID].base); + } + + + ///这种情况下矫正 if(max_count >= total_count*CORRECT_THRESHOLD) { currentNodeID = add_path_to_correct_read(backbone, dumy, currentNodeID, max_type, max_edge, current_cigar, self_string); } - else ///不矫正, 直接取下一个backbone节点 + else { - currentNodeID++; - add_base_to_correct_read_directly(dumy, backbone->g_nodes.list[currentNodeID].base); + ///NOTE: currentNodeID = 0 is a tmp node without any sense + if(currentNodeID > 0 && if_is_homopolymer_strict(r_string_site + currentNodeID - 1, r_string, r_string_length) + && max_count >= total_count*CORRECT_THRESHOLD_HOMOPOLYMER/** && max_type != MISMATCH**/) + { + currentNodeID = add_path_to_correct_read(backbone, dumy, currentNodeID, max_type, max_edge, current_cigar, + self_string); + } + else///不矫正, 直接取下一个backbone节点 + { + currentNodeID++; + add_base_to_correct_read_directly(dumy, backbone->g_nodes.list[currentNodeID].base); - add_cigar_record(&(backbone->g_nodes.list[currentNodeID].base), 1, current_cigar, 0); + add_cigar_record(&(backbone->g_nodes.list[currentNodeID].base), 1, current_cigar, 0); + } } - ///fprintf(stderr, "currentNodeID: %d, max_type: %d\n", currentNodeID, max_type); - - } else ///非backbone节点就会出错了 { @@ -1861,6 +5358,7 @@ void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy, Cigar_record* curre + /** ///从backbone_start遍历到backbone_end节点，生成出来的seq要接着放到dumy->corrected_read中 void get_seq_from_Graph(Graph* backbone, long long backbone_start, long long backbone_end, Correct_dumy* dumy) @@ -2292,10 +5790,11 @@ void get_seq_from_Graph_Len2(Graph* backbone, long long backbone_start, long lon **/ -void window_consensus(char* r_string, long long window_start, long long window_end, -overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, Graph* g, Cigar_record* current_cigar) +void window_consensus(char* r_string, long long r_total_length, long long window_start, long long window_end, +overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, Graph* g, Graph* DAGCon, Cigar_record* current_cigar) { clear_Graph(g); + clear_Graph(DAGCon); long long x_start; long long x_length; @@ -2342,12 +5841,6 @@ overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, Graph* y_start = overlap_list->list[overlapID].w_list[windowID].y_start; y_length = overlap_list->list[overlapID].w_list[windowID].y_end - overlap_list->list[overlapID].w_list[windowID].y_start + 1; - /** - fprintf(stderr, "****window_start: %d, x_start: %d, x_end: %d, x_length: %d, y_start: %d, y_length: %d, dumy->last_boundary_length: %d\n", - window_start, x_start, overlap_list->list[overlapID].w_list[windowID].x_end , x_length, y_start, y_length, - dumy->last_boundary_length); - **/ - recover_UC_Read_sub_region(dumy->overlap_region, y_start, y_length, overlap_list->list[overlapID].y_pos_strand, R_INF, overlap_list->list[overlapID].y_id); @@ -2358,12 +5851,89 @@ overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, Graph* currentNodeID = x_start - window_start; ///这个是要用的cigar: overlap_list->list[overlapID].w_list[windowID].cigar; + + /** + if(memcmp("m54238_180909_174539/6947324/ccs", Get_NAME((*R_INF),overlap_list->list[overlapID].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].x_id)) == 0 && window_start == 12750 && window_end == 13124) + { + fprintf(stderr, "********x_start: %d, window_start: %d, window_end: %d, dumy->length: %d, y_name: %.*s\n", + x_start, window_start, window_end, dumy->length, + Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].y_id), Get_NAME((*R_INF),overlap_list->list[overlapID].y_id)); + + + for (int ijk = 0; ijk < overlap_list->list[overlapID].w_list[windowID].cigar.length; ijk++) + { + fprintf(stderr, "###### Oper: %d, Len: %d\n", + overlap_list->list[overlapID].w_list[windowID].cigar.C_C[ijk], + overlap_list->list[overlapID].w_list[windowID].cigar.C_L[ijk]); + } + } + + + if(memcmp("m64011_190326_191011/163906371/ccs", Get_NAME((*R_INF),overlap_list->list[0].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id)) == 0 && window_start == 9375 && window_end == 9749) + { + fprintf(stderr, "y_name: %.*s\n", Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].y_id), + Get_NAME((*R_INF),overlap_list->list[overlapID].y_id)); + + reverse_complement(x_string, x_length); + reverse_complement(y_string, y_length); + + fprintf(stderr, "x: %.*s\n", x_length, x_string); + fprintf(stderr, "y: %.*s\n", y_length, y_string); + for (int ijk = 0; ijk < overlap_list->list[overlapID].w_list[windowID].cigar.length; ijk++) + { + fprintf(stderr, "###### Oper: %d, Len: %d\n", + overlap_list->list[overlapID].w_list[windowID].cigar.C_C[ijk], + overlap_list->list[overlapID].w_list[windowID].cigar.C_L[ijk]); + } + fprintf(stderr,"\n"); + + reverse_complement(x_string, x_length); + reverse_complement(y_string, y_length); + + + } + **/ + addmatchedSeqToGraph(g, currentNodeID, x_string, x_length, y_string, y_length, &(overlap_list->list[overlapID].w_list[windowID].cigar), startNodeID, endNodeID); + + } - get_seq_from_Graph(g, dumy, current_cigar, backbone); + /** + if(memcmp("m54238_180922_175520/52363405/ccs", Get_NAME((*R_INF),overlap_list->list[overlapID].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].x_id)) == 0 && window_start == 6750 && window_end == 7124) + { + fprintf(stderr, "dumy->corrected_read_length: %d\n", dumy->corrected_read_length); + } + + if(memcmp("m54238_180922_175520/52363405/ccs", Get_NAME((*R_INF),overlap_list->list[overlapID].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].x_id)) == 0 && window_start == 6750 && window_end == 7124) + { + get_seq_from_Graph_print(g, dumy, current_cigar, backbone, r_string, r_total_length, window_start); + } + else + { + get_seq_from_Graph(g, dumy, current_cigar, backbone, r_string, r_total_length, window_start); + } + + + + + + + + if(memcmp("m54238_180922_175520/52363405/ccs", Get_NAME((*R_INF),overlap_list->list[overlapID].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].x_id)) == 0 && window_start == 6750 && window_end == 7124) + { + fprintf(stderr, "dumy->corrected_read_length: %d\n", dumy->corrected_read_length); + } + **/ + + get_seq_from_Graph(g, DAGCon, dumy, current_cigar, backbone, r_string, r_total_length, window_start); ///get_seq_from_Graph(g, startNodeID, endNodeID, dumy); ///get_seq_from_Graph_Len2(g, startNodeID, endNodeID, dumy); @@ -2492,9 +6062,11 @@ long long new_start, long long new_length) { for (cigar_i = 0; cigar_i < operationLen; cigar_i++) { - merge_base = x_string[x_i]; + merge_base = 0; + merge_base = seq_nt6_table[(uint8_t)y_string[y_i]]; merge_base = merge_base << 3; - merge_base = merge_base | y_string[y_i]; + merge_base = merge_base | seq_nt6_table[(uint8_t)x_string[x_i]]; + add_cigar_record(&merge_base, 1, result_cigar, 1); x_i++; y_i++; @@ -2704,14 +6276,6 @@ long long total_window_start, long long total_window_end) new_cigar->length = end_cigar - start_cigar + 1; ///可以优化 memmove(new_cigar->record, new_cigar->record + start_cigar, new_cigar->length*sizeof(uint32_t)); - /** - fprintf(stderr, "start_cigar: %d, end_cigar: %d\n", start_cigar, end_cigar); - for (int ijk = 0; ijk < new_cigar->length; ijk++) - { - fprintf(stderr, "new:Oper: %d, Len: %d\n", Get_Cigar_Type(new_cigar->record[ijk]), - Get_Cigar_Length(new_cigar->record[ijk])); - } - **/ long long total_x_start = total_window_start + get_x_start; long long x_length = get_x_end -get_x_start + 1; @@ -2724,10 +6288,11 @@ long long total_window_start, long long total_window_end) } -int process_boundary(overlap_region_alloc* overlap_list, All_reads* R_INF, Correct_dumy* dumy, Graph* g, +int process_boundary(overlap_region_alloc* overlap_list, All_reads* R_INF, Correct_dumy* dumy, Graph* g, Graph* DAGCon, Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignment* second_round) { char* r_string = dumy->corrected_read; + long long r_total_length = current_cigar->new_read_length; long long corrected_window_start, corrected_window_end; int extra_begin; int extra_end; @@ -2751,6 +6316,7 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen } clear_Graph(g); + clear_Graph(DAGCon); long long x_start, x_end; long long x_length, x_len, o_len; @@ -2774,6 +6340,8 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen backbone_length = corrected_window_end - corrected_window_start + 1; addUnmatchedSeqToGraph(g, backbone, backbone_length, &startNodeID, &endNodeID); + ///fprintf(stderr, "startNodeID: %d, endNodeID: %d\n", startNodeID, endNodeID); + long long correct_x_pos_s; long long matched_coverage = 0; for (i = 0; i < dumy->length; i++) @@ -2791,6 +6359,8 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen x_start = overlap_list->list[overlapID].w_list[windowID].x_start; y_start = overlap_list->list[overlapID].w_list[windowID].y_start; + + /** * There are total 3 cases: @@ -2823,6 +6393,9 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen x_end = corrected_window_end; x_len = x_end - x_start + 1; threshold = x_len * THRESHOLD_RATE; + /****************************may have bugs********************************/ + threshold = Adjust_Threshold(threshold, x_len); + /****************************may have bugs********************************/ ///y_start may less than 0 y_start = y_start - WINDOW_BOUNDARY/2; @@ -2833,26 +6406,81 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen } Window_Len = x_len + (threshold << 1); - determine_overlap_region(threshold, y_start, overlap_list->list[overlapID].y_id, Window_Len, R_INF, - &extra_begin, &extra_end, &y_start, &o_len); - fill_subregion(dumy->overlap_region, y_start, o_len, overlap_list->list[overlapID].y_pos_strand, - R_INF, overlap_list->list[overlapID].y_id, extra_begin, extra_end); - x_string = r_string + x_start; - y_string = dumy->overlap_region; + error =(unsigned int)-1; + if(determine_overlap_region(threshold, y_start, overlap_list->list[overlapID].y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len)) + { + fill_subregion(dumy->overlap_region, y_start, o_len, overlap_list->list[overlapID].y_pos_strand, + R_INF, overlap_list->list[overlapID].y_id, extra_begin, extra_end); - ///both end site and real_y_start have extra_begin - ///有很多是完全匹配，可以先快速判断是不是完全匹配 - end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &error, &real_y_start, - &(dumy->path_length), dumy->matrix_bit, dumy->path, -1, -1); + x_string = r_string + x_start; + y_string = dumy->overlap_region; + + ///both end site and real_y_start have extra_begin + ///有很多是完全匹配，可以先快速判断是不是完全匹配 + end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &error, &real_y_start, + &(dumy->path_length), dumy->matrix_bit, dumy->path, -1, -1); + } + + + + ///try to calculate using higher threshold + if(error==(unsigned int)-1) + { + extra_begin = extra_end = 0; + x_start = corrected_window_start; + x_end = corrected_window_end; + x_len = x_end - x_start + 1; + threshold = threshold * 2; + /****************************may have bugs********************************/ + threshold = Adjust_Threshold(threshold, x_len); + /****************************may have bugs********************************/ + if(x_len >= 300 && threshold < THRESHOLD_MAX_SIZE) + { + threshold = THRESHOLD_MAX_SIZE; + } + if(threshold > THRESHOLD_MAX_SIZE) + { + threshold = THRESHOLD_MAX_SIZE; + } + Window_Len = x_len + (threshold << 1); + y_start = overlap_list->list[overlapID].w_list[windowID].y_start - WINDOW_BOUNDARY/2; + + ///其实可以不加...怕出bug + if(y_start < 0) + { + continue; + } + + error =(unsigned int)-1; + if(determine_overlap_region(threshold, y_start, overlap_list->list[overlapID].y_id, Window_Len, R_INF, + &extra_begin, &extra_end, &y_start, &o_len)) + { + fill_subregion(dumy->overlap_region, y_start, o_len, overlap_list->list[overlapID].y_pos_strand, + R_INF, overlap_list->list[overlapID].y_id, extra_begin, extra_end); + + x_string = r_string + x_start; + y_string = dumy->overlap_region; + + ///both end site and real_y_start have extra_begin + ///有很多是完全匹配，可以先快速判断是不是完全匹配 + end_site = Reserve_Banded_BPM_PATH(y_string, Window_Len, x_string, x_len, threshold, &error, &real_y_start, + &(dumy->path_length), dumy->matrix_bit, dumy->path, -1, -1); + } + } if (error!=(unsigned int)-1) { + + + total_error = total_error + error; matched_coverage++; tmp_cigar.x_start = x_start; tmp_cigar.x_end = x_end; - generate_cigar(dumy->path, dumy->path_length, &tmp_cigar, &real_y_start, &end_site, error); + generate_cigar(dumy->path, dumy->path_length, &tmp_cigar, &real_y_start, &end_site, &error, + x_string, x_len, y_string); ///both end site and real_y_start have extra_begin real_y_start -= extra_begin; end_site -= extra_begin; @@ -2870,39 +6498,39 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen x_string = r_string + x_start; y_string = dumy->overlap_region; - /** - if(memcmp("m54238_180914_183539/66650355/ccs", Get_NAME((*R_INF), overlap_list->list[overlapID].x_id), - Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].x_id)) == 0 && uncorrected_window_start == 375) - { - fprintf(stderr, "###### %.*s, x_string[124]: %c, y_string[124]: %c\n", Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].y_id), - Get_NAME((*R_INF), overlap_list->list[overlapID].y_id), x_string[124], y_string[124]); - fprintf(stderr, "######error: %d****\n", error); - for (int ijk = 0; ijk < tmp_cigar.cigar.length; ijk++) - { - fprintf(stderr, "###### Oper: %d, Len: %d\n", tmp_cigar.cigar.C_C[ijk], - tmp_cigar.cigar.C_L[ijk]); - } + // if(memcmp("m54238_180922_175520/52363405/ccs", Get_NAME((*R_INF),overlap_list->list[overlapID].x_id), + // Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].x_id)) == 0 && uncorrected_window_start == 6750) + // { + // fprintf(stderr, "********x_start: %d, uncorrected_window_start: %d, dumy->length: %d, y_name: %.*s\n", + // x_start, uncorrected_window_start, dumy->length, + // Get_NAME_LENGTH((*R_INF), overlap_list->list[overlapID].y_id), + // Get_NAME((*R_INF),overlap_list->list[overlapID].y_id)); + + + // fprintf(stderr, "*******error: %d****\n", error); + // for (int ijk = 0; ijk < tmp_cigar.cigar.length; ijk++) + // { + // fprintf(stderr, "Oper: %d, Len: %d\n", tmp_cigar.cigar.C_C[ijk], + // tmp_cigar.cigar.C_L[ijk]); + // } + + // } + + + // if(verify_cigar(x_string, x_length, y_string, y_length, &tmp_cigar.cigar, + // error)) + // { + // fprintf(stderr, "*******error: %d****\n", error); + // for (int ijk = 0; ijk < tmp_cigar.cigar.length; ijk++) + // { + // fprintf(stderr, "Oper: %d, Len: %d\n", tmp_cigar.cigar.C_C[ijk], + // tmp_cigar.cigar.C_L[ijk]); + // } - fprintf(stderr, "######dumy->path_length: %d\n****\n", dumy->path_length); - } - **/ - - - /** - if(verify_cigar(x_string, x_length, y_string, y_length, &tmp_cigar.cigar, - error)) - { - fprintf(stderr, "*******error: %d****\n", error); - for (int ijk = 0; ijk < tmp_cigar.cigar.length; ijk++) - { - fprintf(stderr, "Oper: %d, Len: %d\n", tmp_cigar.cigar.C_C[ijk], - tmp_cigar.cigar.C_L[ijk]); - } - - fprintf(stderr, "*******dumy->path_length: %d\n****\n", dumy->path_length); - } - **/ + // fprintf(stderr, "*******dumy->path_length: %d\n****\n", dumy->path_length); + // } + currentNodeID = x_start - corrected_window_start; @@ -2946,7 +6574,9 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen ///and start from i-th node, we can correct (i+1)-th base /// so the condition when traversing graph is ///(node >= start_base - corrected_window_start && node <= end_base - corrected_window_start) - get_seq_from_Graph(g, &(second_round->dumy), &(second_round->tmp_cigar), backbone); + get_seq_from_Graph(g, DAGCon, &(second_round->dumy), &(second_round->tmp_cigar), backbone, + r_string, r_total_length, corrected_window_start); + /** if(verify_cigar_2(backbone, backbone_length, second_round->dumy.corrected_read, second_round->dumy.corrected_read_length, &(second_round->tmp_cigar), -1)) @@ -2954,6 +6584,7 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen fprintf(stderr, "hahah\n"); } **/ + merge_cigars(dumy, current_cigar, second_round, start_base, end_base, corrected_window_start, corrected_window_end); @@ -2977,31 +6608,30 @@ Cigar_record* current_cigar, long long uncorrected_window_start, Round2_alignmen void generate_consensus(overlap_region_alloc* overlap_list, All_reads* R_INF, - UC_Read* g_read, Correct_dumy* dumy, Graph* g, Cigar_record* current_cigar, + UC_Read* g_read, Correct_dumy* dumy, Graph* g, Graph* DAGCon, Cigar_record* current_cigar, Round2_alignment* second_round) { clear_Cigar_record(current_cigar); - long long window_num = (g_read->length + WINDOW - 1) / WINDOW; - long long i, j, overlap_length; + long long overlap_length; long long window_start, window_end; long long num_availiable_win = 0; - window_start = 0; - window_end = WINDOW - 1; - if (window_end >= g_read->length) - { - window_end = g_read->length - 1; - } - int flag; + Window_Pool w_inf; + init_Window_Pool(&w_inf, g_read->length, WINDOW, TAIL_LENGTH); + + int flag = 0; ///for last window dumy->last_boundary_length = 0; - for (i = 0; i < window_num; i++) + while(get_Window(&w_inf, &window_start, &window_end) && flag != -2) { + + + dumy->length = 0; dumy->lengthNT = 0; ///flag返回的是重叠数量 @@ -3016,7 +6646,6 @@ void generate_consensus(overlap_region_alloc* overlap_list, All_reads* R_INF, case 0: ///没找到匹配 break; case -2: ///下一个window也不会存在匹配, 直接跳出 - i = window_num; break; } @@ -3024,41 +6653,18 @@ void generate_consensus(overlap_region_alloc* overlap_list, All_reads* R_INF, ///这个是available overlap里所有window的数量... ///num_availiable_win = num_availiable_win + dumy->length + dumy->lengthNT; num_availiable_win = num_availiable_win + dumy->length; - + ///重叠窗口数，也就是coverage大小 if(dumy->length >= MIN_COVERAGE_THRESHOLD) { - window_consensus(g_read->seq, window_start, window_end, overlap_list, dumy, R_INF, g, current_cigar); + window_consensus(g_read->seq, g_read->length, window_start, window_end, overlap_list, + dumy, R_INF, g, DAGCon, current_cigar); if(dumy->last_boundary_length != 0) { - process_boundary(overlap_list, R_INF, dumy, g, current_cigar, window_start, second_round); - /** - if(memcmp("m54238_180914_183539/66650355/ccs", Get_NAME((*R_INF), overlap_list->list[0].x_id), - Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id)) == 0) - { - - ///fprintf(stderr, "%.*s\n\n", Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id), Get_NAME((*R_INF), overlap_list->list[0].x_id)); - - fprintf(stderr, "window_start: %d, window_end: %d, dumy->last_boundary_length: %d\n", - window_start, window_end, dumy->last_boundary_length); - fprintf(stderr, "new_read_length: %d\n",second_round->cigar.new_read_length); - fprintf(stderr, "cigar.length: %d\n",second_round->cigar.length); - fprintf(stderr, "obtained_cigar_length: %d\n",second_round->obtained_cigar_length); - - fprintf(stderr, "*******\n"); - for (size_t ijk = 0; ijk < second_round->cigar.length; ijk++) - { - int operation = Get_Cigar_Type(second_round->cigar.record[ijk]); - int operation_length = Get_Cigar_Length(second_round->cigar.record[ijk]); - fprintf(stderr, "oper: %d, oper_len: %d\n", operation, operation_length); - } - - fprintf(stderr, "*******\n"); - - } - **/ + process_boundary(overlap_list, R_INF, dumy, g, DAGCon, current_cigar, + window_start, second_round); } } @@ -3069,16 +6675,7 @@ void generate_consensus(overlap_region_alloc* overlap_list, All_reads* R_INF, } - - dumy->last_boundary_length = current_cigar->new_read_length; - - window_start = window_start + WINDOW; - window_end = window_end + WINDOW; - if (window_end >= g_read->length) - { - window_end = g_read->length - 1; - } } if (window_start < g_read->length) @@ -3094,12 +6691,125 @@ void generate_consensus(overlap_region_alloc* overlap_list, All_reads* R_INF, add_cigar_record(dumy->corrected_read, current_cigar->new_read_length - second_round->obtained_cigar_length, &(second_round->cigar), 0); } - - - } +inline int get_available_fully_covered_interval(long long window_start, long long window_end, +overlap_region_alloc* overlap_list, Correct_dumy* dumy, long long* real_length) +{ + long long i; + int flag = 0; + long long Begin, End, Len; + long long overlap_length; + + + for (i = dumy->start_i; i < overlap_list->length; i++) + { + ///只会发生在这个interval比list里所有元素都小的情况 + ///这种情况下一个interval需要从0开始 + if (window_end < overlap_list->list[i].x_pos_s) + { + dumy->start_i = 0; + return 0; + } + else ///只要window_end >= overlap_list->list[i].x_pos_s，就有可能重叠 + { + dumy->start_i = i; + break; + } + } + + ///只会发生在这个window比list里所有元素都大的情况 + ///这种情况下一个window也无需遍历了 + if (i >= overlap_list->length) + { + dumy->start_i = overlap_list->length; + return -2; + } + + + + + + long long fake_length = 0; + overlap_length = window_end - window_start + 1; + (*real_length) = 0; + + for (; i < overlap_list->length; i++) + { + ///是否重叠 + if((Len = OVERLAP(window_start, window_end, overlap_list->list[i].x_pos_s, overlap_list->list[i].x_pos_e)) > 0) + { + ///重叠数量 + fake_length++; + + if (overlap_length == Len && overlap_list->list[i].is_match == 1) + { + //dumy->length++; + (*real_length)++; + } + } + + if(overlap_list->list[i].x_pos_s > window_end) + { + break; + } + } + + ///fake_length是重叠的数量，而不是有效重叠的数量 + if (fake_length == 0) + { + return 0; + } + else + { + return 1; + } +} + +int check_if_fully_covered(overlap_region_alloc* overlap_list, +All_reads* R_INF, UC_Read* g_read, Correct_dumy* dumy, Graph* g) +{ + + long long overlap_length; + long long window_start, window_end; + + Window_Pool w_inf; + init_Window_Pool(&w_inf, g_read->length, WINDOW, TAIL_LENGTH); + + int flag = 0; + long long realLen; + while(get_Window(&w_inf, &window_start, &window_end) && flag != -2) + { + dumy->length = 0; + dumy->lengthNT = 0; + ///flag返回的是重叠数量 + ///dumy->length返回的是有效完全重叠的数量 + ///dumy->lengthNT返回的是有效不完全重叠的数量 + ///return overlaps that is overlaped with [window_start, window_end] + flag = get_available_fully_covered_interval(window_start, window_end, + overlap_list, dumy, &realLen); + + + switch (flag) + { + case 1: ///找到匹配 + break; + case 0: ///没找到匹配 + break; + case -2: ///下一个window也不会存在匹配, 直接跳出 + break; + } + + if(realLen < MIN_COVERAGE_THRESHOLD * 2) + { + return 0; + } + } + + + return 1; +} @@ -3142,13 +6852,11 @@ CIGAR* cigar, haplotype_evdience_alloc* hap) { for (i = 0; i < operationLen; i++) { - /** - if(inner_offset + x_i >= WINDOW) + if(hap->flag[inner_offset + x_i] < 127) { - fprintf(stderr, "error\n"); + hap->flag[inner_offset + x_i]++; } - **/ - hap->flag[inner_offset + x_i]++; + x_i++; y_i++; } @@ -3164,24 +6872,7 @@ CIGAR* cigar, haplotype_evdience_alloc* hap) cigar_i++; } - /** - if(x_i != x_length || y_i != y_length) - { - fprintf(stderr, "x_i: %d, x_length: %d\n", x_i, x_length); - fprintf(stderr, "y_i: %d, y_length: %d\n", y_i, y_length); - } - - int no_zero = 0; - for (i = 0; i < WINDOW; i++) - { - if(hap->flag[i] != 0) - { - no_zero++; - } - } - fprintf(stderr, "no_zero: %d\n", no_zero); - **/ } @@ -3192,7 +6883,7 @@ void addSNPtohaplotype( long long window_offset, int overlapID, char* x_string, long long x_total_start, long long x_length, char* y_string, long long y_total_start, long long y_length, -CIGAR* cigar, haplotype_evdience_alloc* hap) +CIGAR* cigar, haplotype_evdience_alloc* hap, int snp_threshold) { int x_i, y_i, cigar_i; @@ -3218,7 +6909,7 @@ CIGAR* cigar, haplotype_evdience_alloc* hap) { for (i = 0; i < operationLen; i++) { - if(hap->flag[inner_offset] > FLAG_THRE) + if(hap->flag[inner_offset] > snp_threshold) { ev.misBase = y_string[y_i]; ev.overlapID = overlapID; @@ -3240,7 +6931,7 @@ CIGAR* cigar, haplotype_evdience_alloc* hap) for (i = 0; i < operationLen; i++) { - if(hap->flag[inner_offset] > FLAG_THRE) + if(hap->flag[inner_offset] > snp_threshold) { ev.misBase = y_string[y_i]; ev.overlapID = overlapID; @@ -3264,7 +6955,7 @@ CIGAR* cigar, haplotype_evdience_alloc* hap) for (i = 0; i < operationLen; i++) { - if(hap->flag[inner_offset] > FLAG_THRE) + if(hap->flag[inner_offset] > snp_threshold) { ev.misBase = 'N'; ev.overlapID = overlapID; @@ -3297,6 +6988,7 @@ CIGAR* cigar, haplotype_evdience_alloc* hap) void cluster(char* r_string, long long window_start, long long window_end, overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, haplotype_evdience_alloc* hap) { + long long useful_length = window_end - window_start + 1; long long x_start; long long x_length; char* x_string; @@ -3306,10 +6998,19 @@ overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, haplot long long overlapID, windowID; long long startNodeID, endNodeID, currentNodeID; - RsetInitHaplotypeEvdienceFlag(hap); - long long correct_x_pos_s; long long inner_window_offset; + int snp_threshold; + /** + if(overlap_list->mapped_overlaps > Coverage_Threshold(coverage)) + { + snp_threshold = 0; + } + else**/ + { + snp_threshold = 1; + } + ///与当前window重叠的所有overlap @@ -3339,19 +7040,36 @@ overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, haplot y_length = overlap_list->list[overlapID].w_list[windowID].y_end - overlap_list->list[overlapID].w_list[windowID].y_start + 1; - + markSNP(window_start, x_start, x_length, y_start, y_length, &(overlap_list->list[overlapID].w_list[windowID].cigar), hap); } - for (i = 0; i < WINDOW; i++) + /****************************may have bugs********************************/ + long long last_snp = useful_length - 1; + long long first_snp = -1; + for (i = 0; i < useful_length; i++) { - if(hap->flag[i] > FLAG_THRE) + if(hap->flag[i] != 0) + { + last_snp = i; + if(first_snp != -1) + { + first_snp = i; + } + } + + if(hap->flag[i] > snp_threshold) { hap->snp++; } } + if(first_snp == -1) + { + first_snp = 0; + } + /****************************may have bugs********************************/ @@ -3391,11 +7109,10 @@ overlap_region_alloc* overlap_list, Correct_dumy* dumy, All_reads* R_INF, haplot addSNPtohaplotype(window_start, overlapID, x_string, x_start, x_length, y_string, y_start, y_length, &(overlap_list->list[overlapID].w_list[windowID].cigar), - hap); + hap, snp_threshold); } - - + RsetInitHaplotypeEvdienceFlag(hap + first_snp, last_snp + 1); } int cmp_haplotype_evdience(const void * a, const void * b) @@ -3448,6 +7165,19 @@ int cmp_snp_stats(const void * a, const void * b) } } + +int cmp_max_DP(const void * a, const void * b) +{ + if(Get_Max_DP_Value((*(uint64_t*)a))!=Get_Max_DP_Value((*(uint64_t*)b))) + { + return Get_Max_DP_Value((*(uint64_t*)a)) < Get_Max_DP_Value((*(uint64_t*)b))? 1 : -1; + } + else + { + return 0; + } +} + void debug_hap_information(overlap_region_alloc* overlap_list, All_reads* R_INF, UC_Read* g_read, haplotype_evdience_alloc* hap, Correct_dumy* dumy) @@ -3494,7 +7224,7 @@ void debug_hap_information(overlap_region_alloc* overlap_list, All_reads* R_INF, } } - else if(hap->list[i].type == 0) + else if(hap->list[i].type == 1) { if(x_string[0] == y_string[0]) { @@ -3504,9 +7234,15 @@ void debug_hap_information(overlap_region_alloc* overlap_list, All_reads* R_INF, } } + } - + for (i = 0; i < hap->length; i++) + { + if(i != 0 && hap->list[i].site < hap->list[i-1].site) + { + fprintf(stderr, "wrong order\n"); + } } } @@ -3838,7 +7574,7 @@ int debug_snp_matrix(haplotype_evdience_alloc* hap) int split_sub_list(haplotype_evdience_alloc* hap, haplotype_evdience* sub_list, long long sub_length, long long num_haplotype, -overlap_region_alloc* overlap_list, All_reads* R_INF) +overlap_region_alloc* overlap_list, All_reads* R_INF, UC_Read* g_read) { long long i = 0; long long occ_0 = 0; @@ -3865,16 +7601,22 @@ overlap_region_alloc* overlap_list, All_reads* R_INF) } } + + /** 1. if occ_0 = 0, that means all overlaps are different with this read at this site 2. it is not possible that occ_1 = 0, 3. if occ_1 = 1, there are only one difference. It must be a sequencing error. + (for repeat, it maybe a snp at repeat. but ...) **/ - if(occ_0 == 0 || occ_1 <= 1) + ///if(occ_0 == 0 || occ_1 <= 1) + if(occ_0 == 0 || occ_1 == 0) { return 0; } + + ///note: if the max value except type0 is type2 ///that means this is no snp hapolyte long long max = occ_2; @@ -3895,12 +7637,13 @@ overlap_region_alloc* overlap_list, All_reads* R_INF) { return 0; } - + if(max <= 1) { return 0; } + ///if we have two max for (i = 0; i < 5; i++) { @@ -3921,59 +7664,22 @@ overlap_region_alloc* overlap_list, All_reads* R_INF) return 0; } - ///if we just have one snp, we need to phase it carefully - if(num_haplotype == 1) - { - ///we must have just 1 match and 1 mismatch - ///any other types are not good - if(new_0 + max != new_total) - { - return 0; - } - - if(filter_snp(new_0, max, new_total) == 0) - { - return 0; - } - } - - /** - if(filter_snp(new_0, max, new_total) == 0) + ///new_total is the number of errors here + new_total = new_total - new_0; + ///available is the number of selected errors here + available = max; + threshold = 0.70; + available = available/((double)(new_total)); + if(available < threshold) { return 0; } - **/ - + - - - InsertSNPVector(hap, sub_list, sub_length, s_H[max_i]); - - - - /** - fprintf(stderr, "new_0: %d, occ_0: %d, max: %d, max_i: %d, sub_length: %d, new_total: %d, available: %lf\n", - new_0, occ_0, max, max_i, sub_length, new_total, available); - for (i = 0; i < sub_length; i++) - { - - fprintf(stderr, "i: %d, site: %d, type: %d, char: %c, ID: %d, name: %.*s\n", - i, sub_list[i].site, sub_list[i].type, sub_list[i].misBase, sub_list[i].overlapID, - Get_NAME_LENGTH((*R_INF), overlap_list->list[sub_list[i].overlapID].y_id), - Get_NAME((*R_INF),overlap_list->list[sub_list[i].overlapID].y_id)); - - } - fprintf(stderr, "\n"); - **/ - - - - - - + InsertSNPVector(hap, sub_list, sub_length, s_H[max_i], g_read); } @@ -4139,15 +7845,16 @@ int debug_add_to_result_snp_vector(haplotype_evdience_alloc* hap, int8_t *new_ve } else ///can debug here { - if((new_vector[j] != -1 && new_vector[j] != 2 && new_vector[j] != r_vector[j])) + ///if((new_vector[j] != -1 && new_vector[j] != 2 && new_vector[j] != r_vector[j])) + if((new_vector[j] == 0 || new_vector[j] == 1) && new_vector[j] != r_vector[j]) { - fprintf(stderr, "j: %d\n", j); - return 0; + ///fprintf(stderr, "j: %d\n", j); + return j; } } } - return 1; + return -1; } @@ -4220,11 +7927,18 @@ int generate_haplotypes(haplotype_evdience_alloc* hap) return 0; } + ///sort by weight qsort(hap->snp_stat, hap->available_snp, sizeof(SnpStats), cmp_snp_stats); - ///the hap->core_snp is used to find centriod - + // for (j = 0; j < hap->available_snp; j++) + // { + // fprintf(stderr, "j: %d, score: %d\n", j, hap->snp_stat[j].score); + // } + // fprintf(stderr, "\n\n"); + + ///the hap->core_snp is used to find centriod + ///if there are <5 vectors in core_snp, we didn't allow different vector if (hap->core_snp < 5) { @@ -4290,11 +8004,13 @@ int generate_haplotypes(haplotype_evdience_alloc* hap) } + + void print_snp_in_line(haplotype_evdience_alloc* hap) { int j, i; uint32_t* column; - fprintf(stderr, "\n\n\n###########hap->available_snp: %d###########\n", hap->available_snp); + fprintf(stderr, "###########hap->available_snp: %d###########\n", hap->available_snp); for (j = 0; j < hap->available_snp; j++) { fprintf(stderr, "*********j: %d, site: %d, id: %d*********\n", j, hap->snp_stat[j].site, hap->snp_stat[j].id); @@ -4326,7 +8042,9 @@ void print_snp_in_line(haplotype_evdience_alloc* hap) fprintf(stderr, "***********************\n"); for (i = 0; i < hap->dp.snp_num; i++) { - fprintf(stderr, "hap->dp.max[%d]: %d, hap->dp.backtrack_length: %d\n", i, hap->dp.max[i], hap->dp.backtrack_length[i]); + fprintf(stderr, "hap->dp.max[%d]: %d, hap->dp.backtrack_length: %d\n", + i, hap->dp.max[i], hap->dp.backtrack_length[i]); + if(hap->dp.backtrack_length[i] != 0) { column = Get_DP_Backtrack_Column(hap->dp, i); @@ -4338,9 +8056,808 @@ void print_snp_in_line(haplotype_evdience_alloc* hap) } } + + fprintf(stderr, "#########################\n\n\n"); } -int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* overlap_list, All_reads* R_INF) +///if j == -1, print result vector +void print_single_snp(haplotype_evdience_alloc* hap, int j) +{ + + + + int i, vectorID; + int8_t* vector; + if(j != -1) + { + fprintf(stderr, "*********site: %d, j: %d, id: %d*********\n", + hap->snp_stat[j].site, j, hap->snp_stat[j].id); + vectorID = hap->snp_stat[j].id; + vector = Get_SNP_Vector((*hap), vectorID); + } + else + { + fprintf(stderr, "*********result snp*********\n"); + vector = Get_Result_SNP_Vector((*hap)); + } + + + + fprintf(stderr, "type(0):\n"); + for (i = 0; i < hap->overlap; i++) + { + if(vector[i] == 0) + { + fprintf(stderr, "%3d, ", i); + } + } + fprintf(stderr, "\n"); + + fprintf(stderr, "type(1):\n"); + for (i = 0; i < hap->overlap; i++) + { + if(vector[i] == 1) + { + fprintf(stderr, "%3d, ", i); + } + } + fprintf(stderr, "\n"); + fprintf(stderr, "###############\n\n"); +} + + + +void Preorder_Merge(uint32_t snpID, haplotype_evdience_alloc* hap, int is_merge) +{ + int vectorID = hap->snp_stat[snpID].id; + int8_t* vector = Get_SNP_Vector((*hap), vectorID); + hap->dp.visit[snpID] = 1; + + + if(is_merge) + { + if(hap->snp_stat[snpID].is_homopolymer) + { + hap->result_stat.homopolymer_num++; + } + else + { + hap->result_stat.non_homopolymer_num++; + } + + hap->result_stat.score++; + int flag; + if((flag = debug_add_to_result_snp_vector(hap, vector, Get_SNP_Vector_Length((*hap))))!= -1) + { + fprintf(stderr, "incompatible snp vector....\n"); + exit(0); + } + } + + uint32_t* column; + int j; + + if(hap->dp.backtrack_length[snpID] != 0) + { + column = Get_DP_Backtrack_Column(hap->dp, snpID); + + if(is_merge) + { + int add_ID = 0; + for (j = 0; j < hap->dp.backtrack_length[snpID]; j++) + { + if(hap->snp_stat[column[j]].is_homopolymer == 0) + { + add_ID = j; + } + } + + for (j = 0; j < hap->dp.backtrack_length[snpID]; j++) + { + if(j == add_ID) + { + Preorder_Merge(column[j], hap, 1); + } + else + { + Preorder_Merge(column[j], hap, 0); + } + } + } + else + { + for (j = 0; j < hap->dp.backtrack_length[snpID]; j++) + { + Preorder_Merge(column[j], hap, 0); + } + } + } +} + + +void generate_result_vector_repeat(haplotype_evdience_alloc* hap, int pathLen) +{ + if(pathLen != hap->dp.current_snp_num) + { + fprintf(stderr, "hahah\n"); + } + + + int8_t* vector = Get_Result_SNP_Vector((*hap)); + memset(vector, -1, Get_SNP_Vector_Length((*hap))); + hap->result_stat.occ_0 = 0; + hap->result_stat.occ_1 = 0; + hap->result_stat.occ_2 = 0; + hap->result_stat.score = pathLen; + hap->result_stat.homopolymer_num = 0; + hap->result_stat.non_homopolymer_num = 0; + + long long snpID1; + long long j = 0; + int flag, vectorID; + int current_score; + for (j = 0; j < pathLen; j++) + { + snpID1 = hap->dp.buffer[j]; + vectorID = hap->snp_stat[snpID1].id; + vector = Get_SNP_Vector((*hap), vectorID); + if(hap->snp_stat[snpID1].is_homopolymer) + { + hap->result_stat.homopolymer_num++; + } + else + { + hap->result_stat.non_homopolymer_num++; + } + + if((flag = debug_add_to_result_snp_vector(hap, vector, Get_SNP_Vector_Length((*hap))))!= -1) + { + fprintf(stderr, "incompatible snp vector....\n"); + exit(0); + } + } + hap->result_stat.overlap_num = hap->result_stat.occ_0 + hap->result_stat.occ_1; + + + insert_SNP_IDs_addition(&(hap->dp.SNP_IDs), hap->dp.buffer, pathLen, hap->result_stat.occ_0, hap->result_stat.occ_1, + hap->result_stat.homopolymer_num, hap->result_stat.non_homopolymer_num); +} + + + +void Preorder_Merge_Advance_Repeat(uint32_t snpID, haplotype_evdience_alloc* hap, int pathLen) +{ + hap->dp.visit[snpID] = 1; + hap->dp.buffer[pathLen] = snpID; + pathLen++; + + if(hap->dp.backtrack_length[snpID] == 0) + { + ///generate_result_vector_repeat(hap, pathLen); + insert_SNP_IDs_addition(&(hap->dp.SNP_IDs), hap->dp.buffer, pathLen); + return; + } + else + { + uint32_t* column; + int j; + + column = Get_DP_Backtrack_Column(hap->dp, snpID); + for (j = 0; j < hap->dp.backtrack_length[snpID]; j++) + { + Preorder_Merge_Advance_Repeat(column[j], hap, pathLen); + } + } +} + + + +void generate_result_vector(haplotype_evdience_alloc* hap, int pathLen) +{ + if(pathLen != hap->dp.current_snp_num) + { + fprintf(stderr, "hahah\n"); + } + + + int8_t* vector = Get_Result_SNP_Vector((*hap)); + memset(vector, -1, Get_SNP_Vector_Length((*hap))); + hap->result_stat.occ_0 = 0; + hap->result_stat.occ_1 = 0; + hap->result_stat.occ_2 = 0; + hap->result_stat.score = pathLen; + hap->result_stat.homopolymer_num = 0; + hap->result_stat.non_homopolymer_num = 0; + + long long snpID1; + long long j = 0; + int flag, vectorID; + int current_score; + for (j = 0; j < pathLen; j++) + { + snpID1 = hap->dp.buffer[j]; + vectorID = hap->snp_stat[snpID1].id; + vector = Get_SNP_Vector((*hap), vectorID); + if(hap->snp_stat[snpID1].is_homopolymer) + { + hap->result_stat.homopolymer_num++; + } + else + { + hap->result_stat.non_homopolymer_num++; + } + + if((flag = debug_add_to_result_snp_vector(hap, vector, Get_SNP_Vector_Length((*hap))))!= -1) + { + fprintf(stderr, "incompatible snp vector....\n"); + exit(0); + } + } + hap->result_stat.overlap_num = hap->result_stat.occ_0 + hap->result_stat.occ_1; + + + + + ///check if this is a useful snp vector + if(hap->result_stat.overlap_num !=0 && filter_one_snp_advance_nearby(hap, hap->result_stat.occ_0 + 1, + hap->result_stat.occ_1, hap->result_stat.overlap_num + 1, + hap->result_stat.homopolymer_num, hap->result_stat.non_homopolymer_num, + hap->dp.buffer, pathLen)) + { + current_score = calculate_score(hap->result_stat.occ_0 + 1, hap->result_stat.occ_1); + ///first useful snp vector + if(hap->dp.max_snp_num < pathLen) + { + hap->dp.max_snp_num = pathLen; + hap->dp.max_score = current_score; + memcpy(hap->dp.max_buffer, hap->dp.buffer, sizeof(uint32_t) * pathLen); + }///if we have multiple single best snp vector, select the vector with max score + else if(hap->dp.max_snp_num == pathLen) + { + if(current_score > hap->dp.max_score) + { + hap->dp.max_score = current_score; + memcpy(hap->dp.max_buffer, hap->dp.buffer, sizeof(uint32_t) * pathLen); + } + } + } + +} + + +void Preorder_Merge_Advance(uint32_t snpID, haplotype_evdience_alloc* hap, int pathLen) +{ + hap->dp.visit[snpID] = 1; + hap->dp.buffer[pathLen] = snpID; + pathLen++; + + if(hap->dp.backtrack_length[snpID] == 0) + { + generate_result_vector(hap, pathLen); + return; + } + else + { + uint32_t* column; + int j; + + column = Get_DP_Backtrack_Column(hap->dp, snpID); + for (j = 0; j < hap->dp.backtrack_length[snpID]; j++) + { + Preorder_Merge_Advance(column[j], hap, pathLen); + } + } +} + + + +int if_snp_vector_useful(haplotype_evdience_alloc* hap, +long long occ_0, long long occ_1, long long occ_1_low, +long long coverage, uint32_t* SNPs, long long SNPsLen, int roundID) +{ + double occ_1_coverage_low = coverage * 0.3; + + + if(occ_1 >= occ_1_low && occ_0 >= occ_1_low) + { + if(occ_1 >= occ_1_coverage_low && occ_0 >= occ_1_coverage_low) + { + return 1; + } + else if(occ_1 >= 5 && occ_0 >= 5) + { + return 1; + } + else if(occ_1 >= 2 && occ_0 >= 2 && SNPsLen >= 2) + { + int nearsnp; + int non_nearsnps; + count_nearby_snps(hap, SNPs, SNPsLen, &nearsnp, &non_nearsnps); + if(non_nearsnps > 0) + { + return 1; + } + } + else if(roundID == 1 && occ_0 >= occ_1_coverage_low) + { + return 1; + } + + } + + return 0; +} + + +void merge_SNP_Vectors(haplotype_evdience_alloc* hap, uint32_t* SNPs, long long SNPLen) +{ + + int8_t* vector = Get_Result_SNP_Vector((*hap)); + memset(vector, -1, Get_SNP_Vector_Length((*hap))); + hap->result_stat.occ_0 = 0; + hap->result_stat.occ_1 = 0; + hap->result_stat.occ_2 = 0; + hap->result_stat.score = SNPLen; + hap->result_stat.homopolymer_num = 0; + hap->result_stat.non_homopolymer_num = 0; + + long long snpID1; + long long j = 0; + int flag, vectorID; + int current_score; + for (j = 0; j < SNPLen; j++) + { + snpID1 = SNPs[j]; + vectorID = hap->snp_stat[snpID1].id; + vector = Get_SNP_Vector((*hap), vectorID); + if(hap->snp_stat[snpID1].is_homopolymer) + { + hap->result_stat.homopolymer_num++; + } + else + { + hap->result_stat.non_homopolymer_num++; + } + + if((flag = debug_add_to_result_snp_vector(hap, vector, Get_SNP_Vector_Length((*hap))))!= -1) + { + fprintf(stderr, "incompatible snp vector....\n"); + exit(0); + } + } + hap->result_stat.overlap_num = hap->result_stat.occ_0 + hap->result_stat.occ_1; +} + + +void remove_reads(haplotype_evdience_alloc* hap, uint32_t* SNPs, long long SNPsLen, overlap_region_alloc* overlap_list) +{ + long long i, j, snpID, vectorID, overlapLen; + int8_t *vector; + + for (i = 0; i < SNPsLen; i++) + { + snpID = SNPs[i]; + vectorID = hap->snp_stat[snpID].id; + vector = Get_SNP_Vector((*hap), vectorID); + ///hap->snp_stat[snpID].site; + + for (j = 0; j < Get_SNP_Vector_Length((*hap)); j++) + { + + if(vector[j] == 1 && overlap_list->list[j].is_match == 1) + { + //overlap_list->list[j].is_match = 0; + overlap_list->list[j].is_match = 2; + overlapLen = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; + ///overlap_list->mapped_overlaps--; + overlap_list->mapped_overlaps_length -= overlapLen; + } + + /****************************may have bugs********************************/ + if( hap->snp_stat[snpID].site >= overlap_list->list[j].x_pos_s + && + hap->snp_stat[snpID].site <= overlap_list->list[j].x_pos_e) + { + overlap_list->list[j].strong = 1; + } + /****************************may have bugs********************************/ + + } + } +} + + +void output_reads_phase(haplotype_evdience_alloc* hap, uint32_t* SNPs, long long SNPsLen, +overlap_region_alloc* overlap_list, All_reads* R_INF) +{ + long long i, j, snpID, vectorID, overlapLen; + int8_t *vector; + + for (i = 0; i < SNPsLen; i++) + { + snpID = SNPs[i]; + vectorID = hap->snp_stat[snpID].id; + vector = Get_SNP_Vector((*hap), vectorID); + + fprintf(stderr, "i: %d, site: %d, Get_SNP_Vector_Length((*hap)): %d\n", + i, hap->snp_stat[snpID].site, Get_SNP_Vector_Length((*hap))); + + fprintf(stderr, "flag 1\n"); + for (j = 0; j < Get_SNP_Vector_Length((*hap)); j++) + { + if(vector[j] == 1) + { + + fprintf(stderr, "%.*s\n", Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id), + Get_NAME((*R_INF), overlap_list->list[j].y_id)); + + } + } + + + fprintf(stderr, "flag 0\n"); + for (j = 0; j < Get_SNP_Vector_Length((*hap)); j++) + { + + if(vector[j] == 0) + { + + fprintf(stderr, "%.*s\n", Get_NAME_LENGTH((*R_INF), overlap_list->list[j].y_id), + Get_NAME((*R_INF), overlap_list->list[j].y_id)); + + } + } + + } +} + +void remove_reads_debug(haplotype_evdience_alloc* hap, uint32_t* SNPs, long long SNPsLen, overlap_region_alloc* overlap_list) +{ + fprintf(stderr, "SNPsLen: %d\n", SNPsLen); + long long i, j, snpID, vectorID, overlapLen; + int8_t *vector; + + for (i = 0; i < SNPsLen; i++) + { + snpID = SNPs[i]; + vectorID = hap->snp_stat[snpID].id; + vector = Get_SNP_Vector((*hap), vectorID); + fprintf(stderr, "i: %d, snpID:%d, SNPsLen: %d, available_snp: %d, snp_stat[snpID].site: %d\n", + i, snpID, SNPsLen, hap->available_snp, hap->snp_stat[snpID].site); + + for (j = 0; j < Get_SNP_Vector_Length((*hap)); j++) + { + + + if(vector[j] == 1 && overlap_list->list[j].is_match == 1) + { + //overlap_list->list[j].is_match = 0; + overlap_list->list[j].is_match = 2; + overlapLen = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; + ///overlap_list->mapped_overlaps--; + overlap_list->mapped_overlaps_length -= overlapLen; + } + + /****************************may have bugs********************************/ + if( hap->snp_stat[snpID].site >= overlap_list->list[j].x_pos_s + && + hap->snp_stat[snpID].site <= overlap_list->list[j].x_pos_e) + { + overlap_list->list[j].strong = 1; + } + + fprintf(stderr, "j: %d, x_pos_s: %d, x_pos_e: %d, strong: %d, is_match: %d", j, overlap_list->list[j].x_pos_s, + overlap_list->list[j].x_pos_e, overlap_list->list[j].strong, + overlap_list->list[j].is_match); + fprintf(stderr, "****************y: %.*s****************\n", + Get_NAME_LENGTH(R_INF, overlap_list->list[j].y_id), + Get_NAME(R_INF, overlap_list->list[j].y_id)); + /****************************may have bugs********************************/ + + } + } +} + +void try_to_remove_reads(int8_t* vector, long long vectorLen, overlap_region_alloc* overlap_list, +uint32_t* SNPs, long long SNPLen, haplotype_evdience_alloc* hap) +{ + long long i, overlapLen; + long long removed_num = 0; + + for (i = 0; i < vectorLen; i++) + { + if(vector[i] == 1 && overlap_list->list[i].is_match == 1) + { + + ///overlap_list->list[i].is_match = 0; + overlap_list->list[i].is_match = 2; + overlapLen = overlap_list->list[i].x_pos_e - overlap_list->list[i].x_pos_s + 1; + ///overlap_list->mapped_overlaps--; + overlap_list->mapped_overlaps_length -= overlapLen; + removed_num++; + } + } + + + long long snpID, j; + for (i = 0; i < SNPLen; i++) + { + snpID = SNPs[i]; + + ///check all overlaps + for (j = 0; j < Get_SNP_Vector_Length((*hap)); j++) + { + /****************************may have bugs********************************/ + if( hap->snp_stat[snpID].site >= overlap_list->list[j].x_pos_s + && + hap->snp_stat[snpID].site <= overlap_list->list[j].x_pos_e) + { + overlap_list->list[j].strong = 1; + } + /****************************may have bugs********************************/ + } + } +} + + +void process_repeat_snps(haplotype_evdience_alloc* hap, int coverage, overlap_region_alloc* overlap_list) +{ + int i, snpID, vectorID, flag; + int8_t *vector; + long long occ_1_threshold_low; + long long occ_1_threshold_up; + occ_1_threshold_low = 0; + + + uint32_t* snp_ids; + long long length; + + + + for (i = 0; i < hap->dp.SNP_IDs.IDs_length; i++) + { + snp_ids = hap->dp.SNP_IDs.buffer + hap->dp.SNP_IDs.IDs[i].beg; + length = hap->dp.SNP_IDs.IDs[i].end -hap->dp.SNP_IDs.IDs[i].beg + 1; + + merge_SNP_Vectors(hap, snp_ids, length); + + + /** + if(if_snp_vector_useful(hap, hap->dp.SNP_IDs.IDs[i].occ_0, hap->dp.SNP_IDs.IDs[i].occ_1, + occ_1_threshold_low, coverage, snp_ids, length, 0))**/ + if(if_snp_vector_useful(hap, hap->result_stat.occ_0, hap->result_stat.occ_1, + occ_1_threshold_low, coverage, snp_ids, length, 0)) + { + //fprintf(stderr, "i: %d \n", i); + ///remove_reads(hap, snp_ids, length, overlap_list); + try_to_remove_reads(Get_Result_SNP_Vector((*hap)), Get_SNP_Vector_Length((*hap)), + overlap_list, snp_ids, length, hap); + + hap->dp.SNP_IDs.IDs[i].is_remove = 1; + } + else + { + hap->dp.SNP_IDs.IDs[i].is_remove = 0; + } + + } + + + + ///print_snp_in_line(hap); + // fprintf(stderr, "-:overlap_list->mapped_overlaps: %d\n", overlap_list->mapped_overlaps); + /** + if(overlap_list->mapped_overlaps > coverage * 1.6) + { + for (i = 0; i < hap->dp.SNP_IDs.IDs_length; i++) + { + snp_ids = hap->dp.SNP_IDs.buffer + hap->dp.SNP_IDs.IDs[i].beg; + length = hap->dp.SNP_IDs.IDs[i].end -hap->dp.SNP_IDs.IDs[i].beg + 1; + if(hap->dp.SNP_IDs.IDs[i].is_remove == 0 && + if_snp_vector_useful(hap, hap->dp.SNP_IDs.IDs[i].occ_0, hap->dp.SNP_IDs.IDs[i].occ_1, + occ_1_threshold_low, coverage, snp_ids, length, 1)) + { + //fprintf(stderr, "i: %d \n", i); + remove_reads(hap, snp_ids, length, overlap_list); + + } + } + } + **/ + + // fprintf(stderr, "-:overlap_list->mapped_overlaps: %d\n\n\n", overlap_list->mapped_overlaps); + + + + +} + +void process_repeat_snps_debug(haplotype_evdience_alloc* hap, int coverage, +overlap_region_alloc* overlap_list, All_reads* R_INF) +{ + int i, snpID, vectorID, flag; + int8_t *vector; + long long occ_1_threshold_low; + long long occ_1_threshold_up; + occ_1_threshold_low = 0; + + + uint32_t* snp_ids; + long long length; + + + + for (i = 0; i < hap->dp.SNP_IDs.IDs_length; i++) + { + snp_ids = hap->dp.SNP_IDs.buffer + hap->dp.SNP_IDs.IDs[i].beg; + length = hap->dp.SNP_IDs.IDs[i].end -hap->dp.SNP_IDs.IDs[i].beg + 1; + + merge_SNP_Vectors(hap, snp_ids, length); + + if(if_snp_vector_useful(hap, hap->result_stat.occ_0, hap->result_stat.occ_1, + occ_1_threshold_low, coverage, snp_ids, length, 0)) + { + + + + if(overlap_list->list[0].x_id == 5405) + { + fprintf(stderr, "snpid length: %d, occ_0: %d, occ_1: %d\n", + length, + hap->result_stat.occ_0, + hap->result_stat.occ_1); + int k; + for (k = 0; k < length; k++) + { + fprintf(stderr, "i: %d, site: %d\n", + i, hap->snp_stat[snp_ids[k]].site); + } + + for (k = 0; k < Get_SNP_Vector_Length((*hap)); k++) + { + fprintf(stderr, "flag 0\n"); + if(Get_Result_SNP_Vector((*hap))[k] == 0) + { + fprintf(stderr, "%.*s\n", + Get_NAME_LENGTH((*R_INF),overlap_list->list[k].y_id), + Get_NAME((*R_INF),overlap_list->list[k].y_id)); + } + + fprintf(stderr, "flag 1\n"); + if(Get_Result_SNP_Vector((*hap))[k] == 1) + { + fprintf(stderr, "%.*s\n", + Get_NAME_LENGTH((*R_INF),overlap_list->list[k].y_id), + Get_NAME((*R_INF),overlap_list->list[k].y_id)); + } + } + + + + } + + try_to_remove_reads(Get_Result_SNP_Vector((*hap)), Get_SNP_Vector_Length((*hap)), + overlap_list, snp_ids, length, hap); + + hap->dp.SNP_IDs.IDs[i].is_remove = 1; + } + else + { + hap->dp.SNP_IDs.IDs[i].is_remove = 0; + } + + } +} + + + +void debug_repeat_vector(haplotype_evdience_alloc* hap) +{ + int j, i, snpID, vectorID, flag; + int8_t *vector; + + + // if(memcmp(hap->dp.SNP_IDs.buffer + hap->dp.SNP_IDs.IDs[hap->dp.SNP_IDs.max_snp_id].beg, + // hap->dp.max_buffer, + // sizeof(uint32_t) *(hap->dp.SNP_IDs.IDs[hap->dp.SNP_IDs.max_snp_id].end - + // hap->dp.SNP_IDs.IDs[hap->dp.SNP_IDs.max_snp_id].beg + 1))) + // { + // fprintf(stderr, "error1\n"); + // } + + // if(hap->dp.SNP_IDs.IDs[hap->dp.SNP_IDs.max_snp_id].end - + // hap->dp.SNP_IDs.IDs[hap->dp.SNP_IDs.max_snp_id].beg + 1 != + // hap->dp.max_snp_num) + // { + // fprintf(stderr, "error2\n"); + // } + + + + + + uint32_t* snp_ids; + long long length; + for (i = 0; i < hap->dp.SNP_IDs.IDs_length; i++) + { + snp_ids = hap->dp.SNP_IDs.buffer + hap->dp.SNP_IDs.IDs[i].beg; + length = hap->dp.SNP_IDs.IDs[i].end -hap->dp.SNP_IDs.IDs[i].beg + 1; + + ////first clear result snp + vector = Get_Result_SNP_Vector((*hap)); + memset(vector, -1, Get_SNP_Vector_Length((*hap))); + + long long non_hom = 0; + long long hom = 0; + for (j = 0; j < length; j++) + { + ///note here is hap->dp.max_buffer instead of hap->dp.buffer + snpID = snp_ids[j]; + vectorID = hap->snp_stat[snpID].id; + vector = Get_SNP_Vector((*hap), vectorID); + + + if(hap->snp_stat[snpID].is_homopolymer) + { + hom++; + } + else + { + non_hom++; + } + + + if((flag = debug_add_to_result_snp_vector(hap, vector, Get_SNP_Vector_Length((*hap))))!= -1) + { + fprintf(stderr, "incompatible snp vector....\n"); + exit(0); + } + } + + vector = Get_Result_SNP_Vector((*hap)); + long long occ_0 = 0; + long long occ_1 = 0; + for (j = 0; j < Get_SNP_Vector_Length((*hap)); j++) + { + if(vector[j] == 0) + { + occ_0++; + } + + if(vector[j] == 1) + { + occ_1++; + } + } + + if(hom != hap->dp.SNP_IDs.IDs[i].homopolymer_num) + { + fprintf(stderr, "error\n"); + } + + if(non_hom != hap->dp.SNP_IDs.IDs[i].non_homopolymer_num) + { + fprintf(stderr, "error\n"); + } + + if(occ_0 != hap->dp.SNP_IDs.IDs[i].occ_0) + { + fprintf(stderr, "error\n"); + } + + if(occ_1 != hap->dp.SNP_IDs.IDs[i].occ_1) + { + fprintf(stderr, "error\n"); + } + + + + } +} + + +int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* overlap_list, All_reads* R_INF, long long rLen, +int force_repeat) { int j, i; @@ -4355,15 +8872,7 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* return 0; } - - - /** - fprintf(stderr, "*******\nhap->available_snp: %d\n", hap->available_snp); - for (j = 0; j < hap->available_snp; j++) - { - fprintf(stderr, "site: %d, id: %d\n", hap->snp_stat[j].site, hap->snp_stat[j].id); - } - **/ + ///if hap->available_snp == 1, the following codes would have bugs ///filter snps that are highly likly false @@ -4373,7 +8882,10 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* ///if a snp is very near to others, it should not be a real snp for (j = 0; j < hap->available_snp; j++) { - + // if(hap->snp_stat[j].occ_1 == 1) + // { + // fprintf(stderr, "***\n"); + // } if(j > 0 && j < hap->available_snp - 1) { if(hap->snp_stat[j].site != hap->snp_stat[j - 1].site + 1 @@ -4405,14 +8917,19 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* hap->available_snp = i; } + + + + + int flag; - long long overlap_length, total_read, unuseful_read; + long long overlap_length, total_read, unuseful_read, last_j, last_j_ID, last_j_flag; total_read = unuseful_read = 0; ///check if any read may be conflict with others for (i = 0; i < overlap_list->length; i++) { overlap_length = overlap_list->list[i].x_pos_e - overlap_list->list[i].x_pos_s + 1; - if (overlap_length * OVERLAP_THRESHOLD <= overlap_list->list[i].align_length) + if (overlap_list->list[i].is_match == 1) { total_read++; flag = -1; @@ -4421,35 +8938,77 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* vectorID = hap->snp_stat[j].id; vector = Get_SNP_Vector((*hap), vectorID); - if(vector[i] != 2 && flag == 2) + ///flag == -1 means there are no useful signals yet + if (flag == -1) { - unuseful_read++; - flag = 3; - break; - } - - if(vector[i] == 2) + if((vector[i] == 0 || vector[i] == 1 )) + { + flag = 0; + } + }///flag == 0 means there is at least one useful signal yet + else if (flag == 0) { - flag = 2; + if(vector[i] != 0 && vector[i] != 1) + { + flag = 2; + last_j = hap->snp_stat[j].site; + last_j_ID = j; + last_j_flag = vector[i]; + } + }///flag == 0 means there is at least one useful signal first, and another unuseful signal after that + else if(flag == 2) + { + if((vector[i] == 0 || vector[i] == 1 )) + { + flag = 3; + break; + } } } if(flag == 3) { + unuseful_read++; for (j = 0; j < hap->available_snp; j++) { vectorID = hap->snp_stat[j].id; vector = Get_SNP_Vector((*hap), vectorID); + + + + if(vector[i] == 0) + { + hap->snp_stat[j].occ_0--; + hap->snp_stat[j].occ_2++; + } + else if(vector[i] == 1) + { + hap->snp_stat[j].occ_1--; + hap->snp_stat[j].occ_2++; + } + else if(vector[i] != 2) + { + hap->snp_stat[j].occ_2++; + } + + vector[i] = 2; } + // if(overlap_list->list[i].is_match == 0) + // { + // fprintf(stderr, "error\n"); + // } + ///this read may be unuseful + ///overlap_list->list[i].is_match = 0; + ///overlap_list->list[i].is_match = 2; + overlap_list->list[i].is_match = 4; + ///overlap_list->mapped_overlaps--; + overlap_list->mapped_overlaps_length -= overlap_length; } } } - - - - + /*******************************DP********************************/ init_DP_matrix(&(hap->dp), hap->available_snp); @@ -4465,6 +9024,7 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* ///vector of snp i vectorID = hap->snp_stat[i].id; vector = Get_SNP_Vector((*hap), vectorID); + hap->dp.visit[i] = 0; hap->dp.max[i] = 1; hap->dp.backtrack_length[i] = 0; equal_best = 0; @@ -4502,7 +9062,183 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* } /*******************************DP********************************/ + + + + + + uint64_t tmp_mode = 0; + + for (i = 0; i < hap->available_snp; i++) + { + tmp_mode = hap->dp.max[i]; + tmp_mode = tmp_mode << 32; + tmp_mode = tmp_mode | (uint64_t)(i); + hap->dp.max_for_sort[i] = tmp_mode; + } + + qsort(hap->dp.max_for_sort, hap->available_snp, sizeof(uint64_t), cmp_max_DP); + + + int snpID; + int group_num = 0; + ///the minmum snp_num is 1 + hap->dp.max_snp_num = 0; + hap->dp.max_score = -2; + + + + + + //repeat + if(overlap_list->mapped_overlaps_length > Coverage_Threshold(coverage, rLen) || force_repeat) + { + for (i = 0; i < hap->available_snp; i++) + { + snpID = Get_Max_DP_ID(hap->dp.max_for_sort[i]); + if(hap->dp.visit[snpID] == 0) + { + hap->dp.current_snp_num = Get_Max_DP_Value(hap->dp.max_for_sort[i]); + Preorder_Merge_Advance_Repeat(snpID, hap, 0); + } + } + } + else //non-repeat + { + for (i = 0; i < hap->available_snp; i++) + { + snpID = Get_Max_DP_ID(hap->dp.max_for_sort[i]); + if(hap->dp.visit[snpID] == 0) + { + hap->dp.current_snp_num = Get_Max_DP_Value(hap->dp.max_for_sort[i]); + Preorder_Merge_Advance(snpID, hap, 0); + } + } + } + /** + if(memcmp("m64011_190329_072846/59507330/ccs", + Get_NAME((*R_INF), overlap_list->list[0].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id)) == 0) + { + fprintf(stderr, "x_id: %d, mapped_overlaps_length: %d\n", + overlap_list->list[0].x_id, + overlap_list->mapped_overlaps_length); + fprintf(stderr, "coverage: %d\n", + coverage); + fprintf(stderr, "rLen: %d\n", + rLen); + fprintf(stderr, "Coverage_Threshold(coverage, rLen): %f\n", + Coverage_Threshold(coverage, rLen)); + fprintf(stderr, "max_snp_num: %d\n", + hap->dp.max_snp_num); + } + **/ + + + + + + if(overlap_list->mapped_overlaps_length > Coverage_Threshold(coverage, rLen) || force_repeat) + { + ///debug_repeat_vector(hap); + ///process_repeat_snps_debug(hap, coverage, overlap_list, R_INF); + process_repeat_snps(hap, coverage, overlap_list); + + return 1; + + } + else if(hap->dp.max_snp_num > 0) + { + /** + if(memcmp("m64011_190329_072846/59507330/ccs", + Get_NAME((*R_INF), overlap_list->list[0].x_id), + Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id)) == 0) + { + + output_reads_phase(hap, hap->dp.max_buffer, hap->dp.max_snp_num, + overlap_list, R_INF); + } + **/ + + + + + remove_reads(hap, hap->dp.max_buffer, hap->dp.max_snp_num, overlap_list); + return 1; + } + else + { + return 0; + } + + + + + /** + vector = Get_Result_SNP_Vector((*hap)); + memset(vector, -1, Get_SNP_Vector_Length((*hap))); + vector = Get_Result_SNP_Vector((*hap)); + for (i = 0; i < hap->available_snp; i++) + { + int debug_i = Get_Max_DP_ID(hap->dp.max_for_sort[i]); + int round = hap->dp.max[debug_i]; + ///fprintf(stderr, "round: %d\n", round); + if(round > 1) + { + vector = Get_Result_SNP_Vector((*hap)); + memset(vector, -1, Get_SNP_Vector_Length((*hap))); + + + if(hap->dp.backtrack_length[debug_i] < 1) + { + fprintf(stderr, "error\n"); + } + + + + while (round > 0) + { + + + + vectorID2 = hap->snp_stat[debug_i].id; + vector2 = Get_SNP_Vector((*hap), vectorID2); + int flag; + if((flag = debug_add_to_result_snp_vector(hap, vector2, Get_SNP_Vector_Length((*hap))))!= -1) + { + fprintf(stderr, "flag: %d, debug_i: %d, i: %d, x_name: %.*s\n", + flag, debug_i, i, Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id), Get_NAME((*R_INF),overlap_list->list[0].x_id)); + print_snp_in_line(hap); + } + + if(round != hap->dp.max[debug_i]) + { + fprintf(stderr, "error: %d\n", round); + } + + + if(hap->dp.backtrack_length[debug_i] != 0) + { + column = Get_DP_Backtrack_Column(hap->dp, debug_i); + debug_i = column[0]; + } + else if(round != 1) + { + fprintf(stderr, "round: %d\n", round); + } + + + + round--; + } + + } + } + vector = Get_Result_SNP_Vector((*hap)); + memset(vector, -1, Get_SNP_Vector_Length((*hap))); + + vector = Get_Result_SNP_Vector((*hap)); memset(vector, -1, Get_SNP_Vector_Length((*hap))); vector = Get_Result_SNP_Vector((*hap)); @@ -4518,42 +9254,66 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* fprintf(stderr, "error\n"); } + int debug_i = i; int round = hap->dp.max[i]; - while (hap->dp.max[debug_i] != 1) + while (round > 0) { - + + vectorID2 = hap->snp_stat[debug_i].id; vector2 = Get_SNP_Vector((*hap), vectorID2); - if(debug_add_to_result_snp_vector(hap, vector2, Get_SNP_Vector_Length((*hap))) == 0) + int flag; + if((flag = debug_add_to_result_snp_vector(hap, vector2, Get_SNP_Vector_Length((*hap))))!= -1) { - fprintf(stderr, "debug_i: %d\n", debug_i); + fprintf(stderr, "flag: %d, debug_i: %d, i: %d, x_name: %.*s\n", + flag, debug_i, i, Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id), Get_NAME((*R_INF),overlap_list->list[0].x_id)); print_snp_in_line(hap); - } + if(round != hap->dp.max[debug_i]) + { + fprintf(stderr, "error: %d\n", round); + } + + + if(hap->dp.backtrack_length[debug_i] != 0) + { + column = Get_DP_Backtrack_Column(hap->dp, debug_i); + debug_i = column[0]; + } + else if(round != 1) + { + fprintf(stderr, "round: %d\n", round); + } - if(hap->dp.max[debug_i] != round) - { - fprintf(stderr, "hahah\n"); - } - debug_i = hap->dp.backtrack_length[0]; + round--; } + } } vector = Get_Result_SNP_Vector((*hap)); memset(vector, -1, Get_SNP_Vector_Length((*hap))); - - + **/ + /** if(hap->available_snp > 4) + // if(memcmp("m54334_180924_221206/48759269/ccs", Get_NAME((*R_INF),overlap_list->list[0].x_id), + // Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id)) == 0) + // if(memcmp("m54328_180922_235017/65536381/ccs", Get_NAME((*R_INF),overlap_list->list[0].x_id), + // Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id)) == 0) { fprintf(stderr, "\n\n\n###########hap->available_snp: %d###########\n", hap->available_snp); + + fprintf(stderr, "x_name: %.*s\n", + Get_NAME_LENGTH((*R_INF), overlap_list->list[0].x_id), Get_NAME((*R_INF),overlap_list->list[0].x_id)); + for (j = 0; j < hap->available_snp; j++) { - fprintf(stderr, "*********site: %d, id: %d*********\n", hap->snp_stat[j].site, hap->snp_stat[j].id); + fprintf(stderr, "*********site: %d, j: %d, id: %d*********\n", + hap->snp_stat[j].site, j, hap->snp_stat[j].id); fprintf(stderr, "type(0):\n"); @@ -4577,6 +9337,18 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* } } fprintf(stderr, "\n"); + + ///if(j == 21 || j == 23) + // { + // for (i = 0; i < hap->overlap; i++) + // { + // if(vector[i] == 1) + // { + // fprintf(stderr, "1: i: %d, %.*s\n", i, Get_NAME_LENGTH((*R_INF), overlap_list->list[i].y_id), + // Get_NAME((*R_INF),overlap_list->list[i].y_id)); + // } + // } + // } } fprintf(stderr, "***********************\n"); @@ -4595,6 +9367,7 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* } } + **/ @@ -4631,9 +9404,206 @@ int generate_haplotypes_DP(haplotype_evdience_alloc* hap, overlap_region_alloc* +} + + +inline int check_informative_site(haplotype_evdience_alloc* hap, SnpStats* snp) +{ + long long vectorID = snp->id; + int8_t *vector = Get_SNP_Vector((*hap), vectorID); + snp->occ_0 = 0; + snp->occ_1 = 0; + snp->occ_2 = 0; + long long i; + for (i = 0; i < Get_SNP_Vector_Length((*hap)); i++) + { + if(vector[i] == 0) + { + snp->occ_0++; + } + else if(vector[i] == 1) + { + snp->occ_1++; + } + else if(vector[i] == 2) + { + snp->occ_2++; + } + } + + if(snp->occ_0 >= 2 || snp->occ_1 >= 2) + { + return 1; + } + return 0; } +inline long long snp_occ_in_one_read(haplotype_evdience_alloc* hap, overlap_region_alloc* overlap_list, +long long readID) +{ + long long i; + long long vectorID; + int8_t *vector; + + long long snp_occ = 0; + for (i = 0; i < hap->available_snp; i++) + { + vectorID = hap->snp_stat[i].id; + vector = Get_SNP_Vector((*hap), vectorID); + if(vector[readID] == 1) + { + snp_occ++; + } + } + + return snp_occ; +} + +inline void remove_read_from_snps(haplotype_evdience_alloc* hap, overlap_region_alloc* overlap_list, +long long readID) +{ + long long i; + long long vectorID; + int8_t *vector; + + for (i = 0; i < hap->available_snp; i++) + { + vectorID = hap->snp_stat[i].id; + vector = Get_SNP_Vector((*hap), vectorID); + vector[readID] = 2; + } + overlap_list->list[readID].is_match = 4; +} + +int generate_haplotypes_naive(haplotype_evdience_alloc* hap, overlap_region_alloc* overlap_list, All_reads* R_INF, long long rLen, +int force_repeat) +{ + int j, i; + + int vectorID, vectorID2; + int diff_core_vector = 0; + int diff_vector_ID = -1; + int8_t *vector, *vector2; + + + if(hap->available_snp == 0) + { + return 0; + } + + + + ///if hap->available_snp == 1, the following codes would have bugs + ///filter snps that are highly likly false + if(hap->available_snp > 1) + { + i = 0; + ///if a snp is very close to others, it should not be a real snp + for (j = 0; j < hap->available_snp; j++) + { + + if(j > 0 && j < hap->available_snp - 1) + { + if(hap->snp_stat[j].site != hap->snp_stat[j - 1].site + 1 + && + hap->snp_stat[j].site + 1 != hap->snp_stat[j + 1].site) + { + hap->snp_stat[i] = hap->snp_stat[j]; + i++; + } + + } + else if(j == 0) + { + if(hap->snp_stat[j].site + 1 != hap->snp_stat[j + 1].site) + { + hap->snp_stat[i] = hap->snp_stat[j]; + i++; + } + } + else + { + if(hap->snp_stat[j].site != hap->snp_stat[j - 1].site + 1) + { + hap->snp_stat[i] = hap->snp_stat[j]; + i++; + } + } + } + hap->available_snp = i; + } + + + long long m, snp_occ; + /** + while (1) + { + m = 0; + for (i = 0; i < hap->available_snp; i++) + { + if(check_informative_site(hap, &(hap->snp_stat[i]))) + { + hap->snp_stat[m] = hap->snp_stat[i]; + m++; + } + } + if(m == hap->available_snp) + { + break; + } + hap->available_snp = m; + + + for (i = 0; i < overlap_list->length; i++) + { + if (overlap_list->list[i].is_match == 1) + { + snp_occ = snp_occ_in_one_read(hap, overlap_list, i); + if(snp_occ >= 1) + { + remove_read_from_snps(hap, overlap_list, i); + } + } + } + } + **/ + if(hap->available_snp > 0) + { + ///************************debug**************************/// + m = 0; + for (i = 0; i < hap->available_snp; i++) + { + if(check_informative_site(hap, &(hap->snp_stat[i]))) + { + hap->snp_stat[m] = hap->snp_stat[i]; + m++; + } + } + hap->available_snp = m; + ///************************debug**************************/// + + + + init_DP_matrix(&(hap->dp), hap->available_snp); + + for (i = 0; i < hap->available_snp; i++) + { + hap->dp.max_buffer[i] = i; + } + hap->dp.max_snp_num = hap->available_snp; + remove_reads(hap, hap->dp.max_buffer, hap->dp.max_snp_num, overlap_list); + + return 1; + + } + else + { + return 0; + } + +} + void print_Haplotype(haplotype_evdience_alloc* hap, overlap_region_alloc* overlap_list, All_reads* R_INF) { @@ -4652,7 +9622,7 @@ void print_Haplotype(haplotype_evdience_alloc* hap, overlap_region_alloc* overla { Len_x = overlap_list->list[j].x_pos_e - overlap_list->list[j].x_pos_s + 1; - if (Len_x * OVERLAP_THRESHOLD <= overlap_list->list[j].align_length) + if (overlap_list->list[j].is_match == 1) { matched_overlap++; } @@ -4792,7 +9762,7 @@ void debug_near_snp(overlap_region_alloc* overlap_list, All_reads* R_INF, { int flag = -1; overlap_length = overlap_list->list[i].x_pos_e - overlap_list->list[i].x_pos_s + 1; - if (overlap_length * OVERLAP_THRESHOLD <= overlap_list->list[i].align_length) + if (overlap_list->list[i].is_match == 1) { total_read++; for (j = 0; j < hap->available_snp; j++) @@ -4830,9 +9800,9 @@ void debug_near_snp(overlap_region_alloc* overlap_list, All_reads* R_INF, { fprintf(stderr, "\n\nsite: %d, score: %d\n", hap->snp_stat[j].site, hap->snp_stat[j].score ); - if((j>0 && hap->snp_stat[j].site == hap->snp_stat[j - 1].site + 1) - || - (j < hap->available_snp - 1 && hap->snp_stat[j].site + 1 == hap->snp_stat[j + 1].site)) + // if((j>0 && hap->snp_stat[j].site == hap->snp_stat[j - 1].site + 1) + // || + // (j < hap->available_snp - 1 && hap->snp_stat[j].site + 1 == hap->snp_stat[j + 1].site)) { fprintf(stderr, "x_name: %.*s\n", @@ -4978,25 +9948,21 @@ void debug_near_snp(overlap_region_alloc* overlap_list, All_reads* R_INF, void partition_overlaps(overlap_region_alloc* overlap_list, All_reads* R_INF, - UC_Read* g_read, Correct_dumy* dumy, haplotype_evdience_alloc* hap) + UC_Read* g_read, Correct_dumy* dumy, haplotype_evdience_alloc* hap, + int force_repeat) { ResizeInitHaplotypeEvdience(hap); - long long window_num = (g_read->length + WINDOW - 1) / WINDOW; long long i, j, overlap_length; long long window_start, window_end; long long num_availiable_win = 0; + Window_Pool w_inf; + init_Window_Pool(&w_inf, g_read->length, WINDOW, TAIL_LENGTH); - window_start = 0; - window_end = WINDOW - 1; - if (window_end >= g_read->length) - { - window_end = g_read->length - 1; - } - int flag; - for (i = 0; i < window_num; i++) + int flag = 0; + while(get_Window(&w_inf, &window_start, &window_end) && flag != -2) { dumy->length = 0; dumy->lengthNT = 0; @@ -5012,34 +9978,24 @@ void partition_overlaps(overlap_region_alloc* overlap_list, All_reads* R_INF, case 0: ///没找到匹配 break; case -2: ///下一个window也不会存在匹配, 直接跳出 - i = window_num; break; } - - + ///这个是available overlap里所有window的数量... ///num_availiable_win = num_availiable_win + dumy->length + dumy->lengthNT; num_availiable_win = num_availiable_win + dumy->length; - cluster(g_read->seq, window_start, window_end, overlap_list, dumy, R_INF, hap); - - - window_start = window_start + WINDOW; - window_end = window_end + WINDOW; - if (window_end >= g_read->length) - { - window_end = g_read->length - 1; - } - } ///very time-consuming - ///it seems that hap->list has alread been sorted by site qsort(hap->list, hap->length, sizeof(haplotype_evdience), cmp_haplotype_evdience); + + + ///debug_hap_information(overlap_list, R_INF, g_read, hap, dumy); SetSnpMatrix(hap, hap->snp, overlap_list->length); @@ -5063,7 +10019,7 @@ void partition_overlaps(overlap_region_alloc* overlap_list, All_reads* R_INF, sub_list = hap->list + pre_i; sub_length = i - pre_i; ///debug_total_length = debug_total_length + sub_length; - split_sub_list(hap, sub_list, sub_length, hap->snp, overlap_list, R_INF); + split_sub_list(hap, sub_list, sub_length, hap->snp, overlap_list, R_INF, g_read); } num_of_snps++; pre_site = hap->list[i].site; @@ -5076,33 +10032,15 @@ void partition_overlaps(overlap_region_alloc* overlap_list, All_reads* R_INF, sub_list = hap->list + pre_i; sub_length = i - pre_i; ///debug_total_length = debug_total_length + sub_length; - split_sub_list(hap, sub_list, sub_length, hap->snp, overlap_list, R_INF); + split_sub_list(hap, sub_list, sub_length, hap->snp, overlap_list, R_INF, g_read); } ///debug_snp_matrix(hap); + ///generate_haplotypes_DP(hap, overlap_list, R_INF, g_read->length, force_repeat); + generate_haplotypes_naive(hap, overlap_list, R_INF, g_read->length, force_repeat); - generate_haplotypes_DP(hap, overlap_list, R_INF); - - - ///debug_near_snp(overlap_list, R_INF, g_read, dumy, hap); ///debug_snp_matrix(hap); - - if(generate_haplotypes(hap)) - { - //print_Haplotype(hap, overlap_list, R_INF); - ///if we can phase, we need to exclude the reads of different haplotype - int8_t* vector = Get_Result_SNP_Vector((*hap)); - for (j = 0; j < Get_SNP_Vector_Length((*hap)); j++) - { - if(vector[j] == 1) - { - overlap_list->list[j].align_length = 0; - } - } - } - - } @@ -5127,13 +10065,15 @@ void partition_overlaps(overlap_region_alloc* overlap_list, All_reads* R_INF, -void correct_overlap(overlap_region_alloc* overlap_list, All_reads* R_INF, - UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, Graph* g, +void correct_overlap_back(overlap_region_alloc* overlap_list, All_reads* R_INF, + UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, Graph* g, Graph* DAGCon, long long* matched_overlap_0, long long* matched_overlap_1, long long* potiental_matched_overlap_0, long long* potiental_matched_overlap_1, Cigar_record* current_cigar, haplotype_evdience_alloc* hap, Round2_alignment* second_round) { + + reverse_complement(g_read->seq, g_read->length); clear_Correct_dumy(dumy, overlap_list); @@ -5151,6 +10091,8 @@ void correct_overlap(overlap_region_alloc* overlap_list, All_reads* R_INF, window_end = g_read->length - 1; } + + int flag; for (i = 0; i < window_num; i++) @@ -5195,61 +10137,161 @@ void correct_overlap(overlap_region_alloc* overlap_list, All_reads* R_INF, debug_stats(overlap_list, R_INF, g_read, dumy, overlap_read, matched_overlap_0, matched_overlap_1); - partition_overlaps(overlap_list, R_INF, g_read, dumy, hap); + partition_overlaps(overlap_list, R_INF, g_read, dumy, hap, 0); - generate_consensus(overlap_list, R_INF, g_read, dumy, g, current_cigar, second_round); + generate_consensus(overlap_list, R_INF, g_read, dumy, g, DAGCon, current_cigar, second_round); +} - /** - ///fprintf(stderr, "length: %lld, corrected_base: %lld\n", g_read->length, dumy->corrected_base); - - EdlibAlignResult result = edlibAlign(g_read->seq, g_read->length, dumy->corrected_read, dumy->corrected_read_length, - edlibNewAlignConfig(-1, EDLIB_MODE_NW, EDLIB_TASK_PATH, NULL, 0)); - - - - if (result.status == EDLIB_STATUS_OK) { - if (dumy->corrected_base == 0 && result.editDistance!= 0) - { - fprintf(stderr, "error 0\n"); - } - - - if (dumy->corrected_base < result.editDistance) - { - fprintf(stderr, "error 1\n"); - } - - - // fprintf(stderr, "****\n distance: %d, alignmentLength: %d, startLocations: %d, endLocations: %d, corrected_base: %d\n", - // result.editDistance, result.alignmentLength, result.startLocations[0], result.endLocations[0], dumy->corrected_base); - - - // char* cigar = edlibAlignmentToCigar(result.alignment, result.alignmentLength, EDLIB_CIGAR_STANDARD); - // fprintf(stderr,"%s\n", cigar); - // free(cigar); - - } - else +void print_overlap(char* name, long long readID, +overlap_region_alloc* overlap_list, All_reads* R_INF) +{ + if(memcmp(name, Get_NAME((*R_INF), readID), + Get_NAME_LENGTH((*R_INF),readID)) == 0) { - fprintf(stderr, "error\n"); - } - - edlibFreeAlignResult(result); + long long i, j; + fprintf(stderr, "\n\n****************ref_read: %.*s****************\n", + Get_NAME_LENGTH((*R_INF),readID), Get_NAME((*R_INF),readID)); - for (i = 0; i < dumy->corrected_read_length; i++) - { - if (dumy->corrected_read[i] != 'A' && dumy->corrected_read[i] != 'C' && dumy->corrected_read[i] != 'G' && dumy->corrected_read[i] != 'T') + fprintf(stderr, "\n###flag: 1\n"); + + for (i = 0; i < overlap_list->length; i++) { - fprintf(stderr, "error\n"); + if(overlap_list->list[i].is_match == 1) + { + fprintf(stderr, "%.*s\n", Get_NAME_LENGTH((*R_INF),overlap_list->list[i].y_id), + Get_NAME((*R_INF),overlap_list->list[i].y_id)); + + fprintf(stderr, "alignLen: %d, x_s: %d, x_e: %d, y_s: %d, y_e: %d, y_dir: %d, strong: %d\n", + overlap_list->list[i].align_length, + overlap_list->list[i].x_pos_s, + overlap_list->list[i].x_pos_e, + overlap_list->list[i].y_pos_s, + overlap_list->list[i].y_pos_e, + overlap_list->list[i].y_pos_strand, + overlap_list->list[i].strong); + } + } + + fprintf(stderr, "\n###flag: 2\n"); + + for (i = 0; i < overlap_list->length; i++) + { + if(overlap_list->list[i].is_match == 2) + { + fprintf(stderr, "%.*s\n", Get_NAME_LENGTH((*R_INF),overlap_list->list[i].y_id), + Get_NAME((*R_INF),overlap_list->list[i].y_id)); + + fprintf(stderr, "alignLen: %d, x_s: %d, x_e: %d, y_s: %d, y_e: %d, y_dir: %d, strong: %d\n", + overlap_list->list[i].align_length, + overlap_list->list[i].x_pos_s, + overlap_list->list[i].x_pos_e, + overlap_list->list[i].y_pos_s, + overlap_list->list[i].y_pos_e, + overlap_list->list[i].y_pos_strand, + overlap_list->list[i].strong); + } + } + + fprintf(stderr, "\n###flag: 4\n"); + + for (i = 0; i < overlap_list->length; i++) + { + if(overlap_list->list[i].is_match == 4) + { + fprintf(stderr, "%.*s\n", Get_NAME_LENGTH((*R_INF),overlap_list->list[i].y_id), + Get_NAME((*R_INF),overlap_list->list[i].y_id)); + fprintf(stderr, "alignLen: %d, x_s: %d, x_e: %d, y_s: %d, y_e: %d, y_dir: %d, strong: %d\n", + overlap_list->list[i].align_length, + overlap_list->list[i].x_pos_s, + overlap_list->list[i].x_pos_e, + overlap_list->list[i].y_pos_s, + overlap_list->list[i].y_pos_e, + overlap_list->list[i].y_pos_strand, + overlap_list->list[i].strong); + } } } - **/ } +void correct_overlap(overlap_region_alloc* overlap_list, All_reads* R_INF, + UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, Graph* g, Graph* DAGCon, + long long* matched_overlap_0, long long* matched_overlap_1, + long long* potiental_matched_overlap_0, long long* potiental_matched_overlap_1, + Cigar_record* current_cigar, haplotype_evdience_alloc* hap, + Round2_alignment* second_round, int force_repeat, int is_consensus, + int* fully_cov) +{ + reverse_complement(g_read->seq, g_read->length); + + clear_Correct_dumy(dumy, overlap_list); + + long long window_start, window_end; + + Window_Pool w_inf; + + init_Window_Pool(&w_inf, g_read->length, WINDOW, TAIL_LENGTH); + + int flag = 0; + + while(get_Window(&w_inf, &window_start, &window_end) && flag != -2) + { + dumy->length = 0; + dumy->lengthNT = 0; + flag = get_interval(window_start, window_end, overlap_list, dumy); + + switch (flag) + { + case 1: ///no match here + break; + case 0: ///no match here + break; + case -2: ///if flag == -2, loop would be terminated + break; + } + + if(dumy->length + dumy->lengthNT>overlap_list->length) + { + fprintf(stderr, "error length\n"); + } + + ///dumy->lengthNT represent how many overlaps that the length of them is not equal to WINDOW; may larger or less than WINDOW + ///dumy->length represent how many overlaps that the length of them is WINDOW + /****************************may improve**************************/ + ///now the windows which are larger than WINDOW are verified one-by-one, to improve it, we can do it group-bygroup + verify_window(window_start, window_end, overlap_list, dumy, R_INF, g_read->seq); + } + + recalcate_window(overlap_list, R_INF, g_read, dumy, overlap_read); + + + // print_overlap("m64011_190329_072846/59507330/ccs", + // overlap_list->list[0].x_id, overlap_list, R_INF); + + + partition_overlaps(overlap_list, R_INF, g_read, dumy, hap, force_repeat); + + + // print_overlap("m64011_190329_072846/59507330/ccs", + // overlap_list->list[0].x_id, overlap_list, R_INF); + + + + + if(is_consensus) + { + generate_consensus(overlap_list, R_INF, g_read, dumy, g, DAGCon, current_cigar, second_round); + } + + + (*fully_cov) = check_if_fully_covered(overlap_list, R_INF, g_read, dumy, g); + +} + + void init_Cigar_record(Cigar_record* dummy) { diff --git a/Correct.h b/Correct.h index a39f04b..c7daaad 100644 --- a/Correct.h +++ b/Correct.h @@ -6,14 +6,17 @@ #include "POA.h" #include "Process_Read.h" -#define CORRECT_THRESHOLD 0.70 -#define MIN_COVERAGE_THRESHOLD 4 +//#define CORRECT_THRESHOLD 0.70 +#define CORRECT_THRESHOLD 0.60 +///#define CORRECT_THRESHOLD_SECOND 0.55 +#define CORRECT_THRESHOLD_HOMOPOLYMER 0.515 +#define MIN_COVERAGE_THRESHOLD 3 #define CORRECT_INDEL_LENGTH 2 #define MISMATCH 1 #define INSERTION 2 #define DELETION 3 -#define FLAG_THRE 1 +///#define FLAG_THRE 0 #define MAX(x, y) ((x >= y)?x:y) #define MIN(x, y) ((x <= y)?x:y) @@ -22,6 +25,89 @@ #define Get_MisMatch_Base(RECORD) (s_H[(RECORD>>3)]) #define Get_Match_Base(RECORD) (s_H[(RECORD&7)]) +#define Coverage_Threshold(coverage, r_len) (coverage*r_len*1.1) + + + +#define Get_Max_DP_Value(RECORD) (RECORD>>32) +#define Get_Max_DP_ID(RECORD) (RECORD&(uint64_t)0xffffffff) + +#define Adjust_Threshold(threshold, x_len) ((threshold == 0 && x_len >= 4)? 1: threshold) + + + +typedef struct +{ + long long read_length; + long long window_length; + long long window_num; + long long window_start; + long long window_end; + long long tail_length; + int terminal; +}Window_Pool; + +inline void init_Window_Pool(Window_Pool* dumy, long long read_length, long long window_length, long long tail_length) +{ + dumy->terminal = 0; + dumy->read_length = read_length; + dumy->window_length = window_length; + dumy->tail_length = tail_length; + + dumy->window_start = 0; + dumy->window_end = dumy->window_length - 1; + if (dumy->window_end >= dumy->read_length) + { + dumy->window_end = dumy->read_length - 1; + } + + dumy->window_num = (dumy->read_length + dumy->window_length - 1) / dumy->window_length; +} + + +inline int get_Window(Window_Pool* dumy, long long* w_beg, long long* w_end) +{ + (*w_beg) = dumy->window_start; + (*w_end) = dumy->window_end; + + if(dumy->window_end == dumy->read_length - 1) + { + if(dumy->terminal == 1 || dumy->read_length == 0) + { + return 0; + } + else if(dumy->terminal == 0) + { + dumy->terminal = 1; + } + } + + + dumy->window_start = dumy->window_start + dumy->window_length; + dumy->window_end = dumy->window_end + dumy->window_length; + if (dumy->window_end >= dumy->read_length) + { + dumy->window_end = dumy->read_length - 1; + } + // else if (dumy->read_length - dumy->window_end - 1 <= dumy->tail_length) + // { + // dumy->window_end = dumy->read_length - 1; + // } + + + /** + if((*w_end) - (*w_beg) + 1 < 375 && (*w_end) + 1 != dumy->read_length) + { + fprintf(stderr, "(*w_beg):%d, (*w_end): %d, dumy->read_length: %d\n", + (*w_beg), (*w_end), dumy->read_length); + } + **/ + + + return 1; +} + + @@ -69,16 +155,47 @@ typedef struct uint32_t occ_0; uint32_t occ_1; uint32_t occ_2; + uint32_t homopolymer_num; + uint32_t non_homopolymer_num; int score; ////the position of snp in read itself uint32_t site; + uint8_t is_homopolymer; } SnpStats; + +typedef struct +{ + uint32_t beg; + uint32_t end; + uint32_t occ_0; + uint32_t occ_1; + uint32_t homopolymer_num; + uint32_t non_homopolymer_num; + uint32_t is_remove; +} +Snp_ID_Vector; + + +typedef struct +{ + long long IDs_size; + long long IDs_length; + long long max_snp_id; + Snp_ID_Vector* IDs; + + long long buffer_size; + long long buffer_length; + uint32_t* buffer; +} +Snp_ID_Vector_Alloc; + #define Get_DP_Backtrack_Column(matrix, i) (matrix.backtrack + matrix.snp_num * i) #define Get_DP_Backtrack_Column_Length(matrix, i) (matrix.snp_num) + typedef struct { // uint32_t snp_size; @@ -90,12 +207,30 @@ typedef struct uint32_t snp_num; + uint8_t* visit; uint32_t* max; + uint64_t* max_for_sort; + + + uint32_t snp_size; uint32_t* backtrack_length; uint32_t* backtrack; uint32_t backtrack_size; + + + uint32_t* buffer; + uint32_t* max_buffer; + + + int max_snp_num; + ///int max_snp_ID; + int max_score; + int current_snp_num; + + + Snp_ID_Vector_Alloc SNP_IDs; } DP_matrix; @@ -113,7 +248,9 @@ typedef struct uint32_t length; uint32_t size; - uint32_t flag[WINDOW]; + /****************************may have bugs********************************/ + uint8_t flag[WINDOW_MAX_SIZE]; + /****************************may have bugs********************************/ uint32_t available_snp; uint32_t core_snp; @@ -171,13 +308,497 @@ inline int filter_one_snp(int occ_0, int occ_1, int total) if(available < threshold || occ_0 < MIN_COVERAGE_THRESHOLD + 1 || total < 10) + ///if(available < threshold || total < 10) { return 0; } return 1; } -inline void InsertSNPVector(haplotype_evdience_alloc* h, haplotype_evdience* sub_list, long long sub_length, char misBase) + +inline int filter_one_snp_advance_back(int occ_0, int occ_1, int total, int group_size, +long long homopolymer_num, long long non_homopolymer_num) +{ + + + double available; + + if(occ_0 <= occ_1) + { + available = occ_0; + } + else + { + available = occ_1; + } + int min = available; + + double threshold1 = 0.35; + double threshold2 = 0.24; + available = available/((double)(total)); + + + //if(non_homopolymer_num > 0 && min >= 5 && group_size > 1) + if(non_homopolymer_num > 0 && min >= 5) + { + if(available < threshold2 || total < 10) + { + return 0; + } + } + else if(available < threshold1 || occ_0 < MIN_COVERAGE_THRESHOLD + 1 || total < 10) + { + return 0; + } + return 1; +} + + +inline void count_nearby_snps(haplotype_evdience_alloc* hap, uint32_t* SNPs, int SNPsLen, int* nearsnp, int* non_nearsnps) +{ + long long i, current_id, large_id, small_id; + long long distance = 10; + + + + if(SNPsLen == 1) + { + (*non_nearsnps) = 1; + (*nearsnp) = 0; + return; + } + + if(SNPsLen == 0) + { + (*nearsnp) = 0; + (*non_nearsnps) = 0; + return; + } + + (*nearsnp) = 0; + (*non_nearsnps) = 0; + + for (i = 0; i < SNPsLen; i++) + { + if(i > 0 && i < SNPsLen - 1) + { + current_id= SNPs[i]; + + ///since SNPs[i - 1].site is larger than SNPs[i] + large_id = SNPs[i - 1]; + small_id = SNPs[i + 1]; + + + if(hap->snp_stat[large_id].site - hap->snp_stat[current_id].site < distance + || + hap->snp_stat[current_id].site - hap->snp_stat[small_id].site < distance) + { + (*nearsnp)++; + } + else + { + (*non_nearsnps)++; + } + + if(hap->snp_stat[current_id].site > hap->snp_stat[large_id].site || + hap->snp_stat[current_id].site < hap->snp_stat[small_id].site) + { + fprintf(stderr, "error\n"); + } + } + else if(i == 0) + { + current_id= SNPs[i]; + small_id = SNPs[i + 1]; + if(hap->snp_stat[current_id].site - hap->snp_stat[small_id].site < distance) + { + (*nearsnp)++; + } + else + { + (*non_nearsnps)++; + } + + if(hap->snp_stat[current_id].site < hap->snp_stat[small_id].site) + { + fprintf(stderr, "error\n"); + } + } + else + { + large_id = SNPs[i - 1]; + current_id= SNPs[i]; + if(hap->snp_stat[large_id].site - hap->snp_stat[current_id].site < distance) + { + (*nearsnp)++; + } + else + { + (*non_nearsnps)++; + } + + if(hap->snp_stat[current_id].site > hap->snp_stat[large_id].site) + { + fprintf(stderr, "error\n"); + } + + } + + } + + if((*nearsnp) + (*non_nearsnps) != SNPsLen) + { + fprintf(stderr, "(*nearsnp): %d, (*non_nearsnps): %d, SNPsLen: %d\n", (*nearsnp), (*non_nearsnps), SNPsLen); + } +} + +inline int filter_one_snp_advance_nearby(haplotype_evdience_alloc* hap, int occ_0, int occ_1, int total, +long long homopolymer_num, long long non_homopolymer_num, +uint32_t* SNPs, int SNPsLen) +{ + + + double available; + + if(occ_0 <= occ_1) + { + available = occ_0; + } + else + { + available = occ_1; + } + int min = available; + + double threshold1 = 0.35; + double threshold2 = 0.24; + available = available/((double)(total)); + + + + ///if((non_homopolymer_num > 0 && min >= 5) || (min >= 6)) + if(min >= 5) + { + if(available < threshold2 || total < 10) + { + return 0; + } + } + else if(available < threshold1 || occ_0 < MIN_COVERAGE_THRESHOLD + 1 || total < 10) + { + return 0; + } + return 1; +} + +/** +inline int if_is_homopolymer(long long site, char* read, long long read_length) +{ + long long beg, end, i; + + beg = site - 10; + if(beg < 0) + { + beg = 0; + } + + end = site + 10; + + if(end >= read_length) + { + end = read_length - 1; + } + + char f_homopolymer_ch = 0; + long long f_homopolymer_len = 0; + + for (i = site + 1; i <= end; i++) + { + if(f_homopolymer_ch == 0) + { + f_homopolymer_ch = read[i]; + f_homopolymer_len = 1; + } + else + { + if(read[i] != f_homopolymer_ch) + { + break; + } + else + { + f_homopolymer_len++; + } + } + } + + char b_homopolymer_ch = 0; + long long b_homopolymer_len = 0; + + for (i = site - 1; i >= beg; i--) + { + if(b_homopolymer_ch == 0) + { + b_homopolymer_ch = read[i]; + b_homopolymer_len = 1; + } + else + { + if(read[i] != b_homopolymer_ch) + { + break; + } + else + { + b_homopolymer_len++; + } + } + } + + if(f_homopolymer_ch == read[site]) + { + f_homopolymer_len++; + } + else if(b_homopolymer_ch == read[site]) + { + b_homopolymer_len++; + } + + + if(f_homopolymer_len >= 5 || b_homopolymer_len >= 5) + { + return 1; + } + + if (b_homopolymer_ch == f_homopolymer_ch + && + (f_homopolymer_len + b_homopolymer_len >= 5)) + { + return 1; + } + + + return 0; +} +**/ + +inline int if_is_homopolymer_strict(long long site, char* read, long long read_length) +{ + long long beg, end, i; + long long threshold = 3; + + beg = site - threshold; + if(beg < 0) + { + beg = 0; + } + + end = site + threshold; + + if(end >= read_length) + { + end = read_length - 1; + } + + char f_homopolymer_ch = 0; + long long f_homopolymer_len = 0; + + for (i = site + 1; i <= end; i++) + { + if(f_homopolymer_ch == 0) + { + f_homopolymer_ch = read[i]; + f_homopolymer_len = 1; + } + else + { + if(read[i] != f_homopolymer_ch) + { + break; + } + else + { + f_homopolymer_len++; + } + } + } + + char b_homopolymer_ch = 0; + long long b_homopolymer_len = 0; + + for (i = site - 1; i >= beg; i--) + { + if(b_homopolymer_ch == 0) + { + b_homopolymer_ch = read[i]; + b_homopolymer_len = 1; + } + else + { + if(read[i] != b_homopolymer_ch) + { + break; + } + else + { + b_homopolymer_len++; + } + } + } + + if(f_homopolymer_ch == read[site]) + { + f_homopolymer_len++; + } + else if(b_homopolymer_ch == read[site]) + { + b_homopolymer_len++; + } + + /** + fprintf(stderr, "site: %d, beg: %d, end: %d\n", site, beg, end); + for (i = beg; i <= end; i++) + { + if (i == site) + { + fprintf(stderr, "|%c|", read[i]); + } + else + { + fprintf(stderr, "%c", read[i]); + } + } + fprintf(stderr, "\n"); + **/ + + if(f_homopolymer_len >= threshold || b_homopolymer_len >= threshold) + { + return 1; + } + + if (read[site] == f_homopolymer_ch + && + b_homopolymer_ch == f_homopolymer_ch + && + (f_homopolymer_len + b_homopolymer_len >= threshold)) + { + return 1; + } + + + return 0; +} + + +inline int if_is_homopolymer_repeat(long long site, char* read, long long read_length) +{ + long long beg, end, i; + long long threshold = 3; + + beg = site - threshold; + if(beg < 0) + { + beg = 0; + } + + end = site + threshold; + + if(end >= read_length) + { + end = read_length - 1; + } + + char f_homopolymer_ch = 0; + long long f_homopolymer_len = 0; + + for (i = site + 1; i <= end; i++) + { + if(f_homopolymer_ch == 0) + { + f_homopolymer_ch = read[i]; + f_homopolymer_len = 1; + } + else + { + if(read[i] != f_homopolymer_ch) + { + break; + } + else + { + f_homopolymer_len++; + } + } + } + + char b_homopolymer_ch = 0; + long long b_homopolymer_len = 0; + + for (i = site - 1; i >= beg; i--) + { + if(b_homopolymer_ch == 0) + { + b_homopolymer_ch = read[i]; + b_homopolymer_len = 1; + } + else + { + if(read[i] != b_homopolymer_ch) + { + break; + } + else + { + b_homopolymer_len++; + } + } + } + + if(f_homopolymer_ch == read[site]) + { + f_homopolymer_len++; + } + else if(b_homopolymer_ch == read[site]) + { + b_homopolymer_len++; + } + + /** + fprintf(stderr, "site: %d, beg: %d, end: %d\n", site, beg, end); + for (i = beg; i <= end; i++) + { + if (i == site) + { + fprintf(stderr, "|%c|", read[i]); + } + else + { + fprintf(stderr, "%c", read[i]); + } + } + fprintf(stderr, "\n"); + **/ + + if(f_homopolymer_len >= threshold || b_homopolymer_len >= threshold) + { + return 1; + } + + if (read[site] == f_homopolymer_ch + && + b_homopolymer_ch == f_homopolymer_ch + && + (f_homopolymer_len + b_homopolymer_len >= threshold)) + { + return 1; + } + + + return 0; +} + +inline void InsertSNPVector(haplotype_evdience_alloc* h, haplotype_evdience* sub_list, long long sub_length, char misBase, +UC_Read* g_read) { if(sub_length <= 0) return; @@ -190,6 +811,11 @@ inline void InsertSNPVector(haplotype_evdience_alloc* h, haplotype_evdience* sub h->snp_stat[h->available_snp].site = sub_list[0].site; + h->snp_stat[h->available_snp].is_homopolymer = + if_is_homopolymer_strict(h->snp_stat[h->available_snp].site, g_read->seq, g_read->length); + + ///fprintf(stderr, "is_homopolymer: %d\n", h->snp_stat[h->available_snp].is_homopolymer); + int8_t* vector = Get_SNP_Vector((*h), h->available_snp); for (i = 0; i < sub_length; i++) { @@ -261,6 +887,51 @@ inline void InsertSNPVector(haplotype_evdience_alloc* h, haplotype_evdience* sub h->available_snp++; } + +inline int calculate_score(int new_occ_0, int new_occ_1) +{ + if(new_occ_0 + new_occ_1 == 0) + { + return -1; + } + + if(filter_snp(new_occ_0, new_occ_1, new_occ_0 + new_occ_1) == 0) + { + return -1; + } + + double consensus = new_occ_0 + new_occ_1 - abs(new_occ_0 - new_occ_1); + + consensus = consensus /((double)(new_occ_0 + new_occ_1)); + + ///50% vs 50% + if(new_occ_0 == new_occ_1) + { + consensus = consensus + 0.25; + } + else if(consensus >= 0.8) + { + consensus = consensus + 0.2; + } + else if(consensus >= 0.6) + { + consensus = consensus + 0.15; + } + else if(consensus >= 0.4) + { + consensus = consensus + 0.1; + } + else if(consensus >= 0.2) + { + consensus = consensus + 0.05; + } + + + consensus= consensus*((double)(new_occ_0 + new_occ_1)); + + return consensus; +} + inline void SetSnpMatrix(haplotype_evdience_alloc* h, long long snp_num, long long overlap_num) { long long new_size = (snp_num + 1)* overlap_num; @@ -290,30 +961,101 @@ inline void SetSnpMatrix(haplotype_evdience_alloc* h, long long snp_num, long lo } +inline void init_SNP_IDs(Snp_ID_Vector_Alloc* SNP_IDs) +{ + SNP_IDs->max_snp_id = -1; + SNP_IDs->buffer_length = 0; + SNP_IDs->buffer_size = 1000; + SNP_IDs->buffer = (uint32_t*)malloc(sizeof(uint32_t) * SNP_IDs->buffer_size); + + SNP_IDs->IDs_length = 0; + SNP_IDs->IDs_size = 10; + SNP_IDs->IDs = (Snp_ID_Vector*)calloc(SNP_IDs->IDs_size, sizeof(Snp_ID_Vector)); + +} + +inline void clear_SNP_IDs(Snp_ID_Vector_Alloc* SNP_IDs) +{ + SNP_IDs->IDs_length = 0; + SNP_IDs->buffer_length = 0; + SNP_IDs->max_snp_id = -1; +} + +inline void destory_SNP_IDs(Snp_ID_Vector_Alloc* SNP_IDs) +{ + free(SNP_IDs->buffer); + free(SNP_IDs->IDs); +} + +inline void insert_SNP_IDs_addition(Snp_ID_Vector_Alloc* SNP_IDs, uint32_t* IDs_vec, int IDs_vec_length, + uint32_t occ_0, uint32_t occ_1, uint32_t homopolymer_num, uint32_t non_homopolymer_num) +{ + if(SNP_IDs->IDs_length + 1 > SNP_IDs->IDs_size) + { + SNP_IDs->IDs_size = SNP_IDs->IDs_size * 2; + SNP_IDs->IDs = (Snp_ID_Vector*)realloc(SNP_IDs->IDs, SNP_IDs->IDs_size * sizeof(Snp_ID_Vector)); + } + SNP_IDs->IDs[SNP_IDs->IDs_length].beg = SNP_IDs->buffer_length; + SNP_IDs->IDs[SNP_IDs->IDs_length].end = SNP_IDs->buffer_length + IDs_vec_length - 1; + + SNP_IDs->IDs[SNP_IDs->IDs_length].occ_0 = occ_0; + SNP_IDs->IDs[SNP_IDs->IDs_length].occ_1 = occ_1; + SNP_IDs->IDs[SNP_IDs->IDs_length].homopolymer_num = homopolymer_num; + SNP_IDs->IDs[SNP_IDs->IDs_length].non_homopolymer_num = non_homopolymer_num; + + + + if(SNP_IDs->buffer_length + IDs_vec_length > SNP_IDs->buffer_size) + { + SNP_IDs->buffer_size = (SNP_IDs->buffer_length + IDs_vec_length) * 2; + SNP_IDs->buffer = (uint32_t*)realloc(SNP_IDs->buffer, SNP_IDs->buffer_size * sizeof(uint32_t)); + } + memcpy(SNP_IDs->buffer + SNP_IDs->buffer_length, IDs_vec, IDs_vec_length*sizeof(uint32_t)); + + + SNP_IDs->IDs_length++; + SNP_IDs->buffer_length += IDs_vec_length; +} + + +inline void insert_SNP_IDs_addition(Snp_ID_Vector_Alloc* SNP_IDs, uint32_t* IDs_vec, int IDs_vec_length) +{ + if(SNP_IDs->IDs_length + 1 > SNP_IDs->IDs_size) + { + SNP_IDs->IDs_size = SNP_IDs->IDs_size * 2; + SNP_IDs->IDs = (Snp_ID_Vector*)realloc(SNP_IDs->IDs, SNP_IDs->IDs_size * sizeof(Snp_ID_Vector)); + } + SNP_IDs->IDs[SNP_IDs->IDs_length].beg = SNP_IDs->buffer_length; + SNP_IDs->IDs[SNP_IDs->IDs_length].end = SNP_IDs->buffer_length + IDs_vec_length - 1; + + + if(SNP_IDs->buffer_length + IDs_vec_length > SNP_IDs->buffer_size) + { + SNP_IDs->buffer_size = (SNP_IDs->buffer_length + IDs_vec_length) * 2; + SNP_IDs->buffer = (uint32_t*)realloc(SNP_IDs->buffer, SNP_IDs->buffer_size * sizeof(uint32_t)); + } + memcpy(SNP_IDs->buffer + SNP_IDs->buffer_length, IDs_vec, IDs_vec_length*sizeof(uint32_t)); + + + SNP_IDs->IDs_length++; + SNP_IDs->buffer_length += IDs_vec_length; +} + inline void init_DP_matrix(DP_matrix* dp, int32_t snp_num) { - // if(snp_num + 1 > dp->snp_size) - // { - // dp->snp_size = snp_num + 1; - // dp->colum_len = (uint32_t*)realloc(dp->colum_len, dp->snp_size); - // dp->max = (uint32_t*)realloc(dp->max, dp->snp_size); - // } - - // dp->snp_num = snp_num; - - // if(snp_num > 0) - // { - // dp->colum_len[0] = 0; - // } - - // dp->matrix_size = 0; - if(snp_num > dp->snp_size) { dp->snp_size = snp_num; dp->max = (uint32_t*)realloc(dp->max, dp->snp_size * sizeof(uint32_t)); + dp->max_for_sort = (uint64_t*)realloc(dp->max_for_sort, dp->snp_size * sizeof(uint64_t)); + dp->visit = (uint8_t*)realloc(dp->visit, dp->snp_size * sizeof(uint8_t)); + dp->buffer = (uint32_t*)realloc(dp->buffer, dp->snp_size * sizeof(uint32_t)); + dp->max_buffer = (uint32_t*)realloc(dp->max_buffer, dp->snp_size * sizeof(uint32_t)); + + + dp->backtrack_length = (uint32_t*)realloc(dp->backtrack_length, dp->snp_size * sizeof(uint32_t)); dp->backtrack_size = snp_num*snp_num; @@ -321,6 +1063,10 @@ inline void init_DP_matrix(DP_matrix* dp, int32_t snp_num) } dp->snp_num = snp_num; + + clear_SNP_IDs(&(dp->SNP_IDs)); + + } inline void InitHaplotypeEvdience(haplotype_evdience_alloc* h) @@ -339,7 +1085,10 @@ inline void InitHaplotypeEvdience(haplotype_evdience_alloc* h) h->length = 0; h->size = 100; h->list = (haplotype_evdience*)calloc(h->size, sizeof(haplotype_evdience)); - memset(h->flag, 0, WINDOW * sizeof(uint32_t)); + + /****************************may have bugs********************************/ + memset(h->flag, 0, WINDOW_MAX_SIZE * sizeof(uint8_t)); + /****************************may have bugs********************************/ // h->dp.max = NULL; @@ -350,10 +1099,16 @@ inline void InitHaplotypeEvdience(haplotype_evdience_alloc* h) // h->dp.snp_size = 0; h->dp.snp_num = 0; h->dp.max = NULL; + h->dp.max_for_sort = NULL; + h->dp.visit = NULL; h->dp.snp_size = 0; h->dp.backtrack = NULL; h->dp.backtrack_size = 0; h->dp.backtrack_length = NULL; + h->dp.buffer = NULL; + h->dp.max_buffer = NULL; + + init_SNP_IDs(&(h->dp.SNP_IDs)); } @@ -373,6 +1128,17 @@ inline void destoryHaplotypeEvdience(haplotype_evdience_alloc* h) free(h->list); free(h->snp_stat); free(h->snp_matrix); + free(h->dp.backtrack); + free(h->dp.max); + free(h->dp.max_for_sort); + free(h->dp.visit); + free(h->dp.backtrack_length); + free(h->dp.buffer); + free(h->dp.max_buffer); + destory_SNP_IDs(&(h->dp.SNP_IDs)); + + + } inline void ResizeInitHaplotypeEvdience(haplotype_evdience_alloc* h) @@ -381,12 +1147,16 @@ inline void ResizeInitHaplotypeEvdience(haplotype_evdience_alloc* h) h->length = 0; h->sub_list_start = 0; h->sub_list_length = 0; - memset(h->flag, 0, WINDOW * sizeof(uint32_t)); + /****************************may have bugs********************************/ + memset(h->flag, 0, WINDOW_MAX_SIZE * sizeof(uint8_t)); + /****************************may have bugs********************************/ } -inline void RsetInitHaplotypeEvdienceFlag(haplotype_evdience_alloc* h) +inline void RsetInitHaplotypeEvdienceFlag(haplotype_evdience_alloc* h, long long useful_length) { - memset(h->flag, 0, WINDOW * sizeof(uint32_t)); + /****************************may have bugs********************************/ + memset(h->flag, 0, useful_length * sizeof(uint8_t)); + /****************************may have bugs********************************/ } inline void addHaplotypeEvdience(haplotype_evdience_alloc* h, haplotype_evdience* ev) @@ -419,11 +1189,23 @@ typedef struct uint64_t lengthNT; uint64_t size; uint64_t start_i; - char overlap_region[WINDOW + THRESHOLD*2 + 10]; - char overlap_region_group[GROUP_SIZE][WINDOW + THRESHOLD*2 + 10]; - char path[WINDOW + THRESHOLD*2 + 10]; + + /****************************may have bugs********************************/ + // char overlap_region[WINDOW + THRESHOLD*2 + 10]; + // char overlap_region_group[GROUP_SIZE][WINDOW + THRESHOLD*2 + 10]; + // char path[WINDOW + THRESHOLD*2 + 10]; + // Word matrix_bit[((WINDOW + 10)<<3)]; + + char overlap_region[WINDOW_MAX_SIZE + THRESHOLD_MAX_SIZE*2 + 10]; + char overlap_region_group[GROUP_SIZE][WINDOW_MAX_SIZE + THRESHOLD_MAX_SIZE*2 + 10]; + char path[WINDOW_MAX_SIZE + THRESHOLD_MAX_SIZE*2 + 10]; + + char path_fix[WINDOW_MAX_SIZE + THRESHOLD_MAX_SIZE*2 + 10]; + char overlap_region_fix[WINDOW_MAX_SIZE + THRESHOLD_MAX_SIZE*2 + 10]; + Word matrix_bit[((WINDOW_MAX_SIZE + 10)<<3)]; + /****************************may have bugs********************************/ + int path_length; - Word matrix_bit[((WINDOW + 10)<<3)]; __m128i Peq_SSE[256]; } Correct_dumy; @@ -443,11 +1225,12 @@ void clear_Round2_alignment(Round2_alignment* h); void correct_overlap(overlap_region_alloc* overlap_list, All_reads* R_INF, - UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, Graph* g, + UC_Read* g_read, Correct_dumy* dumy, UC_Read* overlap_read, Graph* g, Graph* DAGCon, long long* matched_overlap_0, long long* matched_overlap_1, long long* potiental_matched_overlap_0, long long* potiental_matched_overlap_1, Cigar_record* current_cigar, haplotype_evdience_alloc* hap, - Round2_alignment* second_round); + Round2_alignment* second_round, int force_repeat, int is_consensus, + int* fully_cov); void init_Correct_dumy(Correct_dumy* list); void destory_Correct_dumy(Correct_dumy* list); void clear_Correct_dumy(Correct_dumy* list, overlap_region_alloc* overlap_list); @@ -456,7 +1239,8 @@ void pre_filter_by_nearby(k_mer_pos* new_n_list, k_mer_pos* old_n_list, uint64_t All_reads* R_INF, Correct_dumy* dumy, uint64_t* new_n_length); void pre_filter_by_nearby_single(k_mer_pos* new_n_list, k_mer_pos* old_n_list, uint64_t n_length, uint64_t n_end_pos, UC_Read* g_read, All_reads* R_INF, Correct_dumy* dumy, uint64_t* new_n_length); -void get_seq_from_Graph(Graph* backbone, Correct_dumy* dumy); +void get_seq_from_Graph(Graph* backbone, Graph* DAGCon, Correct_dumy* dumy, Cigar_record* current_cigar, char* self_string, +char* r_string, long long r_string_length, long long r_string_site); void init_Cigar_record(Cigar_record* dummy); void destory_Cigar_record(Cigar_record* dummy); @@ -644,6 +1428,7 @@ inline void add_cigar_record(char* seq, uint32_t len, Cigar_record* dummy, uint3 } +int verify_cigar_2(char* x, int x_len, char* y, int y_len, Cigar_record* cigar, int error); /**********************for prefilter************************ */ void destory_k_mer_pos_list_alloc_prefilter(k_mer_pos_list_alloc* list); diff --git a/Hash_Table.cpp b/Hash_Table.cpp index 9209e8c..1b2f7e1 100644 --- a/Hash_Table.cpp +++ b/Hash_Table.cpp @@ -5,9 +5,18 @@ #include "Process_Read.h" #include "Correct.h" #include "CommandLines.h" +#include "kmer.h" #include +#include "ksort.h" pthread_mutex_t output_mutex; +#define overlap_region_key(a) ((a).y_id) +KRADIX_SORT_INIT(overlap_region_sort, overlap_region, overlap_region_key, member_size(overlap_region, y_id)) + +void overlap_region_sort_y_id(overlap_region *a, long long n) +{ + radix_sort_overlap_region_sort(a, a + n); +} void Init_Heap(HeapSq* HBT) { @@ -28,58 +37,9 @@ void clear_Heap(HeapSq* HBT) HBT->len = 0; } -inline int cmp_ElemType_back(ElemType* x, ElemType* y) -{ - if (x->node.strand < y->node.strand) - { - return 1; - } - else if (x->node.strand > y->node.strand) - { - return 2; - } - else - { - if (x->node.readID < y->node.readID) - { - return 1; - } - else if (x->node.readID > y->node.readID) - { - return 2; - } - else - { - if (x->node.offset < y->node.offset) - { - return 1; - } - else if (x->node.offset > y->node.offset) - { - return 2; - } - else - { - if (x->node.self_offset < y->node.self_offset) - { - return 1; - } - else if (x->node.self_offset > y->node.self_offset) - { - return 2; - } - else - { - return 0; - } - - } - } - } -} -inline int cmp_ElemType(ElemType* x, ElemType* y) +inline int cmp_ElemType_back_back(ElemType* x, ElemType* y) { long long r_pos_x, r_pos_y; if (x->node.strand < y->node.strand) @@ -141,6 +101,118 @@ inline int cmp_ElemType(ElemType* x, ElemType* y) } +inline int cmp_ElemType_back(ElemType* x, ElemType* y) +{ + long long r_pos_x, r_pos_y; + + if (x->node.readID < y->node.readID) + { + return 1; + } + else if (x->node.readID > y->node.readID) + { + return 2; + } + else + { if (x->node.strand < y->node.strand) + { + return 1; + } + else if (x->node.strand > y->node.strand) + { + return 2; + } + else + { + + r_pos_x = x->node.offset - x->node.self_offset; + r_pos_y = y->node.offset - y->node.self_offset; + + ///if (x->node.offset < y->node.offset) + if (r_pos_x < r_pos_y) + { + return 1; + } + ///else if (x->node.offset > y->node.offset) + else if (r_pos_x > r_pos_y) + { + return 2; + } + else + { + if (x->node.self_offset < y->node.self_offset) + { + return 1; + } + else if (x->node.self_offset > y->node.self_offset) + { + return 2; + } + else ///如果r_pos_x和self_offset都相等，那么offset肯定也相等，也没必要再比了 + { + return 0; + } + + } + } + } +} + + + + +inline int cmp_ElemType(ElemType* x, ElemType* y) +{ + + if (x->node.readID < y->node.readID) + { + return 1; + } + else if (x->node.readID > y->node.readID) + { + return 2; + } + else + { if (x->node.strand < y->node.strand) + { + return 1; + } + else if (x->node.strand > y->node.strand) + { + return 2; + } + else + { + + if(x->node.offset < y->node.offset) + { + return 1; + } + else if(x->node.offset > y->node.offset) + { + return 2; + } + else + { + if (x->node.self_offset < y->node.self_offset) + { + return 1; + } + else if (x->node.self_offset > y->node.self_offset) + { + return 2; + } + else ///如果r_pos_x和self_offset都相等，那么offset肯定也相等，也没必要再比了 + { + return 0; + } + + } + } + } +} + + inline void Insert_Heap(HeapSq* HBT, ElemType* x) { long long i, j; @@ -215,14 +287,21 @@ void init_overlap_region_alloc(overlap_region_alloc* list) list->length = 0; ///list->list = (overlap_region*)malloc(sizeof(overlap_region)*list->size); list->list = (overlap_region*)calloc(list->size, sizeof(overlap_region)); + long long i; + for (i = 0; i < list->size; i++) + { + init_fake_cigar(&(list->list[i].f_cigar)); + } } void clear_overlap_region_alloc(overlap_region_alloc* list) { list->length = 0; + list->mapped_overlaps_length = 0; int i = 0; for (i = 0; i < list->size; i++) { list->list[i].w_list_length = 0; + clear_fake_cigar(&(list->list[i].f_cigar)); } } @@ -235,11 +314,37 @@ void destory_overlap_region_alloc(overlap_region_alloc* list) { free(list->list[i].w_list); } + destory_fake_cigar(&(list->list[i].f_cigar)); } free(list->list); } +int get_fake_gap_pos(Fake_Cigar* x, int index) +{ + return (x->buffer[index]>>32); +} + +int get_fake_gap_shift(Fake_Cigar* x, int index) +{ + uint32_t tmp = ((uint32_t)(x->buffer[index])); + int result; + if(tmp & ((uint32_t)1)) + { + tmp = tmp >> 1; + result = tmp; + result = result * -1; + } + else + { + tmp = tmp >> 1; + result = tmp; + } + + return result; +} + + ///r->length = Get_READ_LENGTH((*R_INF), ID); @@ -269,6 +374,8 @@ void append_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp } + + @@ -302,13 +409,14 @@ void append_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp + ///y的区间方向永远是0 ///x有可能是1 ///如果x是1，那我们把它逆过来 if (tmp->x_pos_strand == 1) { - list->list[list->length].x_id = tmp->x_id; + list->list[list->length].x_id = tmp->x_id; list->list[list->length].x_pos_e = Get_READ_LENGTH((*R_INF), tmp->x_id) - tmp->x_pos_s - 1; list->list[list->length].x_pos_s = Get_READ_LENGTH((*R_INF), tmp->x_id) - tmp->x_pos_e - 1; list->list[list->length].x_pos_strand = 0; @@ -317,6 +425,35 @@ void append_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp list->list[list->length].y_pos_e = Get_READ_LENGTH((*R_INF), tmp->y_id) - tmp->y_pos_s - 1; list->list[list->length].y_pos_s = Get_READ_LENGTH((*R_INF), tmp->y_id) - tmp->y_pos_e - 1; list->list[list->length].y_pos_strand = 1; + + + + + // resize_fake_cigar(&(list->list[list->length].f_cigar), (tmp->f_cigar.length + 2)); + // add_fake_cigar(&(list->list[list->length].f_cigar), list->list[list->length].x_pos_s, 0); + // long long distance_gap; + // long long pre_distance_gap = 0; + // long long i = 0; + // for (i = 0; i < tmp->f_cigar.length; i++) + // { + // distance_gap = get_fake_gap_shift(&(tmp->f_cigar), i); + // if(distance_gap != pre_distance_gap) + // { + // pre_distance_gap = distance_gap; + // add_fake_cigar(&(list->list[list->length].f_cigar), + // Get_READ_LENGTH((*R_INF), tmp->x_id) - get_fake_gap_pos(&(tmp->f_cigar), i) - 1, + // pre_distance_gap); + // } + // } + + // if(get_fake_gap_pos(&(list->list[list->length].f_cigar), + // list->list[list->length].f_cigar.length - 1) != list->list[list->length].x_pos_e) + // { + // add_fake_cigar(&(list->list[list->length].f_cigar), + // list->list[list->length].x_pos_e, + // get_fake_gap_shift(&(list->list[list->length].f_cigar), + // list->list[list->length].f_cigar.length - 1)); + // } } else { @@ -329,16 +466,392 @@ void append_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp list->list[list->length].y_pos_e = tmp->y_pos_e; list->list[list->length].y_pos_s = tmp->y_pos_s; list->list[list->length].y_pos_strand = tmp->y_pos_strand; + + + + + // resize_fake_cigar(&(list->list[list->length].f_cigar), (tmp->f_cigar.length + 2)); + // add_fake_cigar(&(list->list[list->length].f_cigar), list->list[list->length].x_pos_s, 0); + // long long distance_self_pos = tmp->x_pos_e - tmp->x_pos_s; + // long long distance_pos = tmp->y_pos_e - tmp->y_pos_s; + // long long init_distance_gap = distance_pos - distance_self_pos; + // long long pre_distance_gap = init_distance_gap; + // long long distance_gap; + // long long i = 0; + // for (i = tmp->f_cigar.length - 1; i >= 0; i--) + // { + // distance_gap = get_fake_gap_shift(&(tmp->f_cigar), i); + // if(distance_gap != pre_distance_gap) + // { + // pre_distance_gap = distance_gap; + + // add_fake_cigar(&(list->list[list->length].f_cigar), + // get_fake_gap_pos(&(tmp->f_cigar), i), init_distance_gap - pre_distance_gap); + // } + // } + + // if(get_fake_gap_pos(&(list->list[list->length].f_cigar), + // list->list[list->length].f_cigar.length - 1) != list->list[list->length].x_pos_e) + // { + // add_fake_cigar(&(list->list[list->length].f_cigar), + // list->list[list->length].x_pos_e, + // get_fake_gap_shift(&(list->list[list->length].f_cigar), + // list->list[list->length].f_cigar.length - 1)); + // } + + + + /******************************for debug********************************/ + // distance_self_pos = tmp->x_pos_e - tmp->x_pos_s; + // distance_pos = tmp->y_pos_e - tmp->y_pos_s; + // distance_gap = distance_pos - distance_self_pos; + // fprintf(stderr, "\nx_s: %d, x_e: %d, y_s: %d, y_e: %d, distance_gap: %d\n", + // list->list[list->length].x_pos_s, list->list[list->length].x_pos_e, + // list->list[list->length].y_pos_s, list->list[list->length].y_pos_e, distance_gap); + // for (i = 0; i < list->list[list->length].f_cigar.length; i++) + // { + // fprintf(stderr, "##i: %d, gap_pos_in_x: %d, gap_shift: %d\n", + // i, get_fake_gap_pos(&(list->list[list->length].f_cigar), i), + // get_fake_gap_shift(&(list->list[list->length].f_cigar), i)); + // } + // for (i = 0; i < tmp->f_cigar.length; i++) + // { + // fprintf(stderr, "**i: %d, gap_pos_in_x: %d, gap_shift: %d\n", + // i, get_fake_gap_pos(&(tmp->f_cigar), i), + // get_fake_gap_shift(&(tmp->f_cigar), i)); + // } + /******************************for debug********************************/ + + + } list->list[list->length].shared_seed = tmp->shared_seed; list->list[list->length].align_length = 0; + list->list[list->length].is_match = 0; + list->list[list->length].non_homopolymer_errors = 0; list->length++; } +void append_overlap_region_alloc_from_existing(overlap_region_alloc* list, overlap_region* tmp, All_reads* R_INF) +{ + + if (list->length + 1 > list->size) + { + list->size = list->size * 2; + list->list = (overlap_region*)realloc(list->list, sizeof(overlap_region)*list->size); + ///新分配空间要初始化 + memset(list->list + (list->size/2), 0, sizeof(overlap_region)*(list->size/2)); + } + + if (list->length!=0 && + list->list[list->length - 1].y_id==tmp->y_id + ) + { + if(list->list[list->length - 1].shared_seed >= tmp->shared_seed) + { + return; + } + else + { + list->length--; + } + + + } + + + + if(tmp->x_pos_s <= tmp->y_pos_s) + { + tmp->y_pos_s = tmp->y_pos_s - tmp->x_pos_s; + tmp->x_pos_s = 0; + } + else + { + tmp->x_pos_s = tmp->x_pos_s - tmp->y_pos_s; + tmp->y_pos_s = 0; + } + + + + tmp->x_pos_e = Get_READ_LENGTH((*R_INF), tmp->x_id) - tmp->x_pos_s - 1; + tmp->y_pos_e = Get_READ_LENGTH((*R_INF), tmp->y_id) - tmp->y_pos_s - 1; + + + if(tmp->x_pos_e <= tmp->y_pos_e) + { + tmp->y_pos_e = tmp->y_pos_s + tmp->x_pos_e; + tmp->x_pos_e = tmp->x_pos_s + tmp->x_pos_e; + } + else + { + tmp->x_pos_e = tmp->x_pos_s + tmp->y_pos_e; + tmp->y_pos_e = tmp->y_pos_s + tmp->y_pos_e; + } + + + + + + // if(tmp->x_pos_strand != 0) + // { + // fprintf(stderr, "error\n"); + // } + + list->list[list->length].x_id = tmp->x_id; + list->list[list->length].x_pos_e = tmp->x_pos_e; + list->list[list->length].x_pos_s = tmp->x_pos_s; + list->list[list->length].x_pos_strand = tmp->x_pos_strand; + + list->list[list->length].y_id = tmp->y_id; + list->list[list->length].y_pos_e = tmp->y_pos_e; + list->list[list->length].y_pos_s = tmp->y_pos_s; + list->list[list->length].y_pos_strand = tmp->y_pos_strand; + + + + list->list[list->length].shared_seed = tmp->shared_seed; + list->list[list->length].align_length = 0; + list->list[list->length].is_match = 0; + list->list[list->length].non_homopolymer_errors = 0; + + list->length++; + + + // fprintf(stderr, "shared_seed: %d, x_pos_s: %d, x_pos_e: %d, y_pos_s: %d, y_pos_e: %d, list->length: %d\n", + // tmp->shared_seed, tmp->x_pos_s, tmp->x_pos_e, tmp->y_pos_s, tmp->y_pos_e, list->length); + + // fprintf(stderr, "*****x_pos_s: %d, x_pos_e: %d, y_pos_s: %d, y_pos_e: %d, list->length: %d\n", + // list->list[list->length - 1].x_pos_s, list->list[list->length - 1].x_pos_e, + // list->list[list->length - 1].y_pos_s, list->list[list->length - 1].y_pos_e, + // list->length); +} + + +int append_inexact_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp, All_reads* R_INF) +{ + + if (list->length + 1 > list->size) + { + list->size = list->size * 2; + list->list = (overlap_region*)realloc(list->list, sizeof(overlap_region)*list->size); + ///新分配空间要初始化 + memset(list->list + (list->size/2), 0, sizeof(overlap_region)*(list->size/2)); + } + + if (list->length!=0 && + list->list[list->length - 1].y_id==tmp->y_id + ) + { + if(list->list[list->length - 1].shared_seed >= tmp->shared_seed) + { + return 0; + } + else + { + list->length--; + } + } + + if(tmp->x_pos_s <= tmp->y_pos_s) + { + tmp->y_pos_s = tmp->y_pos_s - tmp->x_pos_s; + tmp->x_pos_s = 0; + } + else + { + tmp->x_pos_s = tmp->x_pos_s - tmp->y_pos_s; + tmp->y_pos_s = 0; + } + + + long long x_right_length = Get_READ_LENGTH((*R_INF), tmp->x_id) - tmp->x_pos_e - 1; + long long y_right_length = Get_READ_LENGTH((*R_INF), tmp->y_id) - tmp->y_pos_e - 1; + + + + if(x_right_length <= y_right_length) + { + tmp->x_pos_e = Get_READ_LENGTH((*R_INF), tmp->x_id) - 1; + tmp->y_pos_e = tmp->y_pos_e + x_right_length; + } + else + { + tmp->x_pos_e = tmp->x_pos_e + y_right_length; + tmp->y_pos_e = Get_READ_LENGTH((*R_INF), tmp->y_id) - 1; + } + + + if (tmp->x_pos_strand == 1) + { + list->list[list->length].x_id = tmp->x_id; + list->list[list->length].x_pos_e = Get_READ_LENGTH((*R_INF), tmp->x_id) - tmp->x_pos_s - 1; + list->list[list->length].x_pos_s = Get_READ_LENGTH((*R_INF), tmp->x_id) - tmp->x_pos_e - 1; + list->list[list->length].x_pos_strand = 0; + + list->list[list->length].y_id = tmp->y_id; + list->list[list->length].y_pos_e = Get_READ_LENGTH((*R_INF), tmp->y_id) - tmp->y_pos_s - 1; + list->list[list->length].y_pos_s = Get_READ_LENGTH((*R_INF), tmp->y_id) - tmp->y_pos_e - 1; + list->list[list->length].y_pos_strand = 1; + + + + + resize_fake_cigar(&(list->list[list->length].f_cigar), (tmp->f_cigar.length + 2)); + add_fake_cigar(&(list->list[list->length].f_cigar), list->list[list->length].x_pos_s, 0); + long long distance_gap; + long long pre_distance_gap = 0; + long long i = 0; + for (i = 0; i < tmp->f_cigar.length; i++) + { + distance_gap = get_fake_gap_shift(&(tmp->f_cigar), i); + if(distance_gap != pre_distance_gap) + { + pre_distance_gap = distance_gap; + add_fake_cigar(&(list->list[list->length].f_cigar), + Get_READ_LENGTH((*R_INF), tmp->x_id) - get_fake_gap_pos(&(tmp->f_cigar), i) - 1, + pre_distance_gap); + } + } + + if(get_fake_gap_pos(&(list->list[list->length].f_cigar), + list->list[list->length].f_cigar.length - 1) != list->list[list->length].x_pos_e) + { + add_fake_cigar(&(list->list[list->length].f_cigar), + list->list[list->length].x_pos_e, + get_fake_gap_shift(&(list->list[list->length].f_cigar), + list->list[list->length].f_cigar.length - 1)); + } + + + /******************************for debug********************************/ + // long long distance_self_pos = tmp->x_pos_e - tmp->x_pos_s; + // long long distance_pos = tmp->y_pos_e - tmp->y_pos_s; + // distance_gap = distance_pos - distance_self_pos; + + // if(distance_gap != + // get_fake_gap_shift(&(list->list[list->length].f_cigar), + // list->list[list->length].f_cigar.length - 1)) + // { + // fprintf(stderr, "error\n"); + // } + // long long distance_self_pos = tmp->x_pos_e - tmp->x_pos_s; + // long long distance_pos = tmp->y_pos_e - tmp->y_pos_s; + // distance_gap = distance_pos - distance_self_pos; + // fprintf(stderr, "\nx_s: %d, x_e: %d, y_s: %d, y_e: %d, distance_gap: %d, xLen: %d\n", + // list->list[list->length].x_pos_s, list->list[list->length].x_pos_e, + // list->list[list->length].y_pos_s, list->list[list->length].y_pos_e, distance_gap, + // Get_READ_LENGTH((*R_INF), tmp->x_id)); + // for (i = 0; i < list->list[list->length].f_cigar.length; i++) + // { + // fprintf(stderr, "##i: %d, gap_pos_in_x: %d, gap_shift: %d\n", + // i, get_fake_gap_pos(&(list->list[list->length].f_cigar), i), + // get_fake_gap_shift(&(list->list[list->length].f_cigar), i)); + // } + // for (i = 0; i < tmp->f_cigar.length; i++) + // { + // fprintf(stderr, "**i: %d, gap_pos_in_x: %d, gap_shift: %d\n", + // i, get_fake_gap_pos(&(tmp->f_cigar), i), + // get_fake_gap_shift(&(tmp->f_cigar), i)); + // } + /******************************for debug********************************/ + } + else + { + list->list[list->length].x_id = tmp->x_id; + list->list[list->length].x_pos_e = tmp->x_pos_e; + list->list[list->length].x_pos_s = tmp->x_pos_s; + list->list[list->length].x_pos_strand = tmp->x_pos_strand; + + list->list[list->length].y_id = tmp->y_id; + list->list[list->length].y_pos_e = tmp->y_pos_e; + list->list[list->length].y_pos_s = tmp->y_pos_s; + list->list[list->length].y_pos_strand = tmp->y_pos_strand; + + + + resize_fake_cigar(&(list->list[list->length].f_cigar), (tmp->f_cigar.length + 2)); + add_fake_cigar(&(list->list[list->length].f_cigar), list->list[list->length].x_pos_s, 0); + long long distance_self_pos = tmp->x_pos_e - tmp->x_pos_s; + long long distance_pos = tmp->y_pos_e - tmp->y_pos_s; + long long init_distance_gap = distance_pos - distance_self_pos; + long long pre_distance_gap = init_distance_gap; + long long distance_gap; + long long i = 0; + for (i = tmp->f_cigar.length - 1; i >= 0; i--) + { + distance_gap = get_fake_gap_shift(&(tmp->f_cigar), i); + if(distance_gap != pre_distance_gap) + { + pre_distance_gap = distance_gap; + + add_fake_cigar(&(list->list[list->length].f_cigar), + get_fake_gap_pos(&(tmp->f_cigar), i), init_distance_gap - pre_distance_gap); + } + } + + if(get_fake_gap_pos(&(list->list[list->length].f_cigar), + list->list[list->length].f_cigar.length - 1) != list->list[list->length].x_pos_e) + { + add_fake_cigar(&(list->list[list->length].f_cigar), + list->list[list->length].x_pos_e, + get_fake_gap_shift(&(list->list[list->length].f_cigar), + list->list[list->length].f_cigar.length - 1)); + } + + + + + /******************************for debug********************************/ + // distance_self_pos = tmp->x_pos_e - tmp->x_pos_s; + // distance_pos = tmp->y_pos_e - tmp->y_pos_s; + // distance_gap = distance_pos - distance_self_pos; + + // if(distance_gap != + // get_fake_gap_shift(&(list->list[list->length].f_cigar), + // list->list[list->length].f_cigar.length - 1)) + // { + // fprintf(stderr, "error\n"); + // } + // distance_self_pos = tmp->x_pos_e - tmp->x_pos_s; + // distance_pos = tmp->y_pos_e - tmp->y_pos_s; + // distance_gap = distance_pos - distance_self_pos; + // fprintf(stderr, "\nx_s: %d, x_e: %d, y_s: %d, y_e: %d, distance_gap: %d\n", + // list->list[list->length].x_pos_s, list->list[list->length].x_pos_e, + // list->list[list->length].y_pos_s, list->list[list->length].y_pos_e, distance_gap); + // for (i = 0; i < list->list[list->length].f_cigar.length; i++) + // { + // fprintf(stderr, "##i: %d, gap_pos_in_x: %d, gap_shift: %d\n", + // i, get_fake_gap_pos(&(list->list[list->length].f_cigar), i), + // get_fake_gap_shift(&(list->list[list->length].f_cigar), i)); + // } + // for (i = 0; i < tmp->f_cigar.length; i++) + // { + // fprintf(stderr, "**i: %d, gap_pos_in_x: %d, gap_shift: %d\n", + // i, get_fake_gap_pos(&(tmp->f_cigar), i), + // get_fake_gap_shift(&(tmp->f_cigar), i)); + // } + /******************************for debug********************************/ + } + + + list->list[list->length].shared_seed = tmp->shared_seed; + list->list[list->length].align_length = 0; + list->list[list->length].is_match = 0; + list->list[list->length].non_homopolymer_errors = 0; + list->list[list->length].strong = 0; + + list->length++; + + return 1; +} + + + void append_overlap_region_alloc_debug(overlap_region_alloc* list, overlap_region* tmp) { @@ -582,7 +1095,8 @@ uint64_t readID, uint64_t readLength, All_reads* R_INF) { return; } - + + i = 0; while (i < candidates->length) @@ -605,8 +1119,9 @@ uint64_t readID, uint64_t readLength, All_reads* R_INF) tmp_region.y_pos_e = candidates->list[i].offset; tmp_region.y_pos_strand = 0; ///永远是0 - i++; + + i++; while (i < candidates->length) { if (current_ID == candidates->list[i].readID && @@ -616,20 +1131,56 @@ uint64_t readID, uint64_t readLength, All_reads* R_INF) tmp_pos_distance = candidates->list[i].offset - candidates->list[i].self_offset - current_pos_diff; ///这个不一定是正值 tmp_self_pos_distance = candidates->list[i].self_offset - current_self_pos; - if (tmp_self_pos_distance >= 0) + + if(tmp_self_pos_distance < 0) + { + ///fprintf(stderr, "tmp_self_pos_distance: %d\n", tmp_self_pos_distance); + tmp_self_pos_distance = tmp_self_pos_distance * -1; + ///fprintf(stderr, "tmp_self_pos_distance: %d\n", tmp_self_pos_distance); + } + + ///if (tmp_self_pos_distance >= 0) { tmp_self_pos_distance = tmp_self_pos_distance * error_rate + constant_distance; if (tmp_pos_distance < tmp_self_pos_distance) { - i++; + /****************************may have bugs********************************/ + ///i++; + // tmp_region.x_pos_e = candidates->list[i].self_offset; + // tmp_region.y_pos_e = candidates->list[i].offset; + /****************************may have bugs********************************/ tmp_region.shared_seed++; - tmp_region.x_pos_e = candidates->list[i].self_offset; - tmp_region.y_pos_e = candidates->list[i].offset; + + if(candidates->list[i].self_offset < tmp_region.x_pos_s) + { + tmp_region.x_pos_s = candidates->list[i].self_offset; + } + + if(candidates->list[i].self_offset > tmp_region.x_pos_e) + { + tmp_region.x_pos_e = candidates->list[i].self_offset; + } + + if(candidates->list[i].offset < tmp_region.y_pos_s) + { + tmp_region.y_pos_s = candidates->list[i].offset; + } + + + if(candidates->list[i].offset > tmp_region.y_pos_e) + { + tmp_region.y_pos_e = candidates->list[i].offset; + } + + + /****************************may have bugs********************************/ + i++; + /****************************may have bugs********************************/ continue; } } } - ///i++; + break; } @@ -637,7 +1188,6 @@ uint64_t readID, uint64_t readLength, All_reads* R_INF) ///if (tmp_region.x_id != tmp_region.y_id && tmp_region.shared_seed > 1) if (tmp_region.x_id != tmp_region.y_id) { - append_overlap_region_alloc(overlap_list, &tmp_region, R_INF); } @@ -653,17 +1203,550 @@ uint64_t readID, uint64_t readLength, All_reads* R_INF) } +void debug_chain(k_mer_hit* a, long long a_n, Chain_Data* dp) +{ + long long i, j, current_j; + long long selfLen, indels; + long long distance_self_pos, distance_pos, distance_gap; + for (i = 0; i < a_n; ++i) + { + selfLen = indels = 0; + j = i; + while (j >= 0) + { + current_j = j; + + j = dp->pre[j]; + + if(j != -1) + { + distance_self_pos = a[current_j].self_offset - a[j].self_offset; + distance_pos = a[current_j].offset - a[j].offset; + distance_gap = distance_pos > distance_self_pos? distance_pos - distance_self_pos : distance_self_pos - distance_pos; + + indels += distance_gap; + selfLen += distance_self_pos; + } + } + + if(indels != dp->indels[i]) + { + fprintf(stderr, "indels: %d, dp->indels[i]: %d\n", + indels, dp->indels[i]); + } + + if(selfLen != dp->self_length[i]) + { + fprintf(stderr, "selfLen: %d, dp->self_length[i]: %d\n", + selfLen, dp->self_length[i]); + } + + } +} +///double band_width_threshold = 0.05; +void chain_DP(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, +double band_width_threshold) +{ + long long i, j; + long long self_pos, pos, max_j, max_i, max_score, score, n_skip; + long long distance_pos, distance_self_pos, distance_gap, log_distance_gap, distance_min; + ///double band_width_threshold = 0.05; + double band_width_penalty = 1 / band_width_threshold; + long long min_score = k_mer_length; + long long max_indels, max_self_length; + double gap_rate; + long long total_indels, total_self_length; + + resize_Chain_Data(dp, a_n); + // fill the score and backtrack arrays + for (i = 0; i < a_n; ++i) + { + pos = a[i].offset; + self_pos = a[i].self_offset; + max_j = -1; + max_score = min_score; + n_skip = 0; + max_indels = 0; + max_self_length = 0; + + + ///may have a pre-cut condition for j + for (j = i - 1; j >= 0; --j) + { + distance_pos = pos - a[j].offset; + distance_self_pos = self_pos - a[j].self_offset; + ///a has been sorted by a[].offset + ///note for a, we do not have any two elements that have both equal offsets and self_offsets + ///but there maybe two elements that have equal offsets or equal self_offsets + if(distance_pos == 0 || distance_self_pos <= 0) + { + continue; + } + + distance_gap = distance_pos > distance_self_pos? distance_pos - distance_self_pos : distance_self_pos - distance_pos; + + total_indels = dp->indels[j] + distance_gap; + total_self_length = dp->self_length[j] + distance_self_pos; + if(total_indels > band_width_threshold * total_self_length) + { + continue; + } + + distance_min = distance_pos < distance_self_pos? distance_pos:distance_self_pos; + score = distance_min < min_score? distance_min : min_score; + + /** + log_distance_gap = distance_gap? ilog2_32(distance_gap) : 0; + + score -= (long long)(distance_gap * 0.01 * min_score) + (log_distance_gap/2); + **/ + gap_rate = (double)((double)(total_indels)/(double)(total_self_length)); + ///if the gap rate > 0.05, score will be negative + score -= (long long)(gap_rate * score * band_width_penalty); + + score += dp->score[j]; + + if(score > max_score) + { + max_score = score; + max_j = j; + max_indels = total_indels; + max_self_length = total_self_length; + if (n_skip > 0) + { + n_skip--; + } + } + } + + dp->score[i] = max_score; + dp->pre[i] = max_j; + dp->indels[i] = max_indels; + dp->self_length[i] = max_self_length; + } + + + ///debug_chain(a, a_n, dp); + + + + max_score = -1; + max_i = -1; + for (i = 0; i < a_n; ++i) + { + if(dp->score[i] > max_score) + { + max_score = dp->score[i]; + max_i = i; + } + } + + + clear_fake_cigar(&(result->f_cigar)); + + i = max_i; + result->x_pos_e = a[i].self_offset; + result->y_pos_e = a[i].offset; + result->shared_seed = max_score; + + distance_self_pos = result->x_pos_e - a[i].self_offset; + distance_pos = result->y_pos_e - a[i].offset; + long long pre_distance_gap = distance_pos - distance_self_pos; + ///record first site + ///the length of f_cigar should be at least 1 + add_fake_cigar(&(result->f_cigar), a[i].self_offset, pre_distance_gap); + long long chainLen = 0; + if(result->x_pos_strand == 1) + { + while (i >= 0) + { + distance_self_pos = result->x_pos_e - a[i].self_offset; + distance_pos = result->y_pos_e - a[i].offset; + distance_gap = distance_pos - distance_self_pos; + if(distance_gap != pre_distance_gap) + { + pre_distance_gap = distance_gap; + ///record this site + add_fake_cigar(&(result->f_cigar), a[i].self_offset, pre_distance_gap); + } + + chainLen++; + result->x_pos_s = a[i].self_offset; + result->y_pos_s = a[i].offset; + i = dp->pre[i]; + } + } + else + { + + while (i >= 0) + { + distance_self_pos = result->x_pos_e - a[i].self_offset; + distance_pos = result->y_pos_e - a[i].offset; + distance_gap = distance_pos - distance_self_pos; + if(distance_gap == pre_distance_gap) + { + result->f_cigar.length--; + add_fake_cigar(&(result->f_cigar), a[i].self_offset, pre_distance_gap); + } + else + { + pre_distance_gap = distance_gap; + add_fake_cigar(&(result->f_cigar), a[i].self_offset, pre_distance_gap); + } + + chainLen++; + result->x_pos_s = a[i].self_offset; + result->y_pos_s = a[i].offset; + i = dp->pre[i]; + } + } +} + +void debug_seed_offset(k_mer_hit* a, long long a_n, Chain_Data* dp, overlap_region* result, +All_reads* R_INF) +{ + long long i; + long long max_score = -1; + long long max_i = -1; + for (i = 0; i < a_n; i++) + { + if(max_score < dp->score[i]) + { + max_score = dp->score[i]; + max_i = i; + } + } + + if(max_score != result->shared_seed) + { + fprintf(stderr, "max_score: %d, result->shared_seed: %d, a_n: %d, dp->length: %d\n", + max_score, result->shared_seed, a_n, dp->length); + } + + long long chainLen = 0; + i = max_i; + while (i >= 0) + { + chainLen++; + i = dp->pre[i]; + } + + + long long* chain_x_pos = (long long*)malloc(sizeof(long long)*chainLen); + long long* chain_y_pos = (long long*)malloc(sizeof(long long)*chainLen); + long long chain_i; + i = max_i; + if(result->y_pos_strand == 0) + { + chain_i = chainLen - 1; + while (i >= 0) + { + chain_x_pos[chain_i] = a[i].self_offset; + chain_y_pos[chain_i] = a[i].offset; + chain_i--; + i = dp->pre[i]; + } + } + else + { + chain_i = 0; + while (i >= 0) + { + chain_x_pos[chain_i] = + Get_READ_LENGTH((*R_INF), result->x_id) - a[i].self_offset - 1; + + chain_y_pos[chain_i] = + Get_READ_LENGTH((*R_INF), result->y_id) - a[i].offset - 1; + chain_i++; + i = dp->pre[i]; + } + } + + + chain_i = 0; + + long long distance_self_pos; + long long distance_pos; + long long distance_gap; + + + for (i = 0; i < chainLen; i++) + { + distance_self_pos = chain_x_pos[i] - result->x_pos_s; + distance_pos = chain_y_pos[i] - result->y_pos_s; + distance_gap = distance_pos - distance_self_pos; + + if(distance_gap != y_start_offset(chain_x_pos[i], &(result->f_cigar))) + { + fprintf(stderr, "distance_gap: %d, y_offset: %d\n", distance_gap, + y_start_offset(chain_x_pos[i], &(result->f_cigar))); + } + } + + + + + + + + + + + + + free(chain_x_pos); + free(chain_y_pos); + +} + + +void calculate_overlap_region_by_chaining(Candidates_list* candidates, overlap_region_alloc* overlap_list, +uint64_t readID, uint64_t readLength, All_reads* R_INF, double band_width_threshold) +{ + overlap_region tmp_region; + uint64_t i = 0; + long long current_pos_diff; + long long current_self_pos; + uint64_t current_ID; + uint64_t current_stand; + long long tmp_pos_distance; + long long tmp_self_pos_distance; + long long constant_distance = 5; + double error_rate = 0.05; + + + if (candidates->length == 0) + { + return; + } + + long long sub_region_beg; + long long sub_region_end; + + init_fake_cigar(&(tmp_region.f_cigar)); + + i = 0; + while (i < candidates->length) + { + current_ID = candidates->list[i].readID; + current_stand = candidates->list[i].strand; + + ///这个是查询read的信息 + tmp_region.x_id = readID; + tmp_region.x_pos_strand = current_stand; + ///这个是被查询的read的信息 + tmp_region.y_id = current_ID; + tmp_region.y_pos_strand = 0; ///永远是0 + + + + sub_region_beg = i; + sub_region_end = i; + i++; + + while (i < candidates->length + && + current_ID == candidates->list[i].readID + && + current_stand == candidates->list[i].strand) + { + sub_region_end = i; + i++; + } + + if (tmp_region.x_id == tmp_region.y_id) + { + continue; + } + + + chain_DP(candidates->list + sub_region_beg, + sub_region_end - sub_region_beg + 1, &(candidates->chainDP), &tmp_region, band_width_threshold); + + ///自己和自己重叠的要排除 + ///if (tmp_region.x_id != tmp_region.y_id && tmp_region.shared_seed > 1) + if (tmp_region.x_id != tmp_region.y_id) + { + ///append_overlap_region_alloc(overlap_list, &tmp_region, R_INF); + /** + if(append_inexact_overlap_region_alloc(overlap_list, &tmp_region, R_INF)) + { + debug_seed_offset(candidates->list + sub_region_beg, + sub_region_end - sub_region_beg + 1, + &(candidates->chainDP), &(overlap_list->list[overlap_list->length - 1]), R_INF); + } + **/ + append_inexact_overlap_region_alloc(overlap_list, &tmp_region, R_INF); + } + } + + destory_fake_cigar(&(tmp_region.f_cigar)); + + qsort(overlap_list->list, overlap_list->length, sizeof(overlap_region), cmp_by_x_pos_s); +} + + + + +void calculate_inexact_overlap_region(Candidates_list* candidates, overlap_region_alloc* overlap_list, +uint64_t readID, uint64_t readLength, All_reads* R_INF) +{ + overlap_region tmp_region; + uint64_t i = 0; + long long current_pos_diff; + long long current_self_pos; + uint64_t current_ID; + uint64_t current_stand; + long long tmp_pos_distance; + long long tmp_self_pos_distance; + long long constant_distance = 5; + double error_rate = 0.05; + + + if (candidates->length == 0) + { + return; + } + + + + i = 0; + while (i < candidates->length) + { + current_pos_diff = candidates->list[i].offset - candidates->list[i].self_offset; + current_self_pos = candidates->list[i].self_offset; + current_ID = candidates->list[i].readID; + current_stand = candidates->list[i].strand; + + tmp_region.shared_seed = 1; + ///这个是查询read的信息 + tmp_region.x_id = readID; + tmp_region.x_pos_s = candidates->list[i].self_offset; + tmp_region.x_pos_e = candidates->list[i].self_offset; + tmp_region.x_pos_strand = current_stand; + + ///这个是被查询的read的信息 + tmp_region.y_id = current_ID; + tmp_region.y_pos_s = candidates->list[i].offset; + tmp_region.y_pos_e = candidates->list[i].offset; + tmp_region.y_pos_strand = 0; ///永远是0 + + // if(memcmp("m64013_190412_043951/159056664/ccs", Get_NAME((*R_INF), readID), + // Get_NAME_LENGTH((*R_INF), readID)) == 0 + // && + // memcmp("m64013_190322_203854/42402522/ccs", Get_NAME((*R_INF), current_ID), + // Get_NAME_LENGTH((*R_INF), current_ID)) == 0) + // { + // fprintf(stderr, "***********i: %d, x_pos_s: %d, x_pos_e: %d, y_pos_s: %d, y_pos_e: %d\n", + // i, tmp_region.x_pos_s, tmp_region.x_pos_e, tmp_region.y_pos_s, tmp_region.y_pos_e); + // fprintf(stderr, "***********i: %d, self_offset: %d, offset: %d\n", + // i, candidates->list[i].self_offset, candidates->list[i].offset); + // } + + + + i++; + while (i < candidates->length) + { + if (current_ID == candidates->list[i].readID && + current_stand == candidates->list[i].strand) + { + ///这个一定是正值 + tmp_pos_distance = candidates->list[i].offset - candidates->list[i].self_offset - current_pos_diff; + ///这个不一定是正值 + tmp_self_pos_distance = candidates->list[i].self_offset - current_self_pos; + + if(tmp_self_pos_distance < 0) + { + ///fprintf(stderr, "tmp_self_pos_distance: %d\n", tmp_self_pos_distance); + tmp_self_pos_distance = tmp_self_pos_distance * -1; + ///fprintf(stderr, "tmp_self_pos_distance: %d\n", tmp_self_pos_distance); + } + + ///if (tmp_self_pos_distance >= 0) + { + tmp_self_pos_distance = tmp_self_pos_distance * error_rate + constant_distance; + if (tmp_pos_distance < tmp_self_pos_distance) + { + /****************************may have bugs********************************/ + ///i++; + // tmp_region.x_pos_e = candidates->list[i].self_offset; + // tmp_region.y_pos_e = candidates->list[i].offset; + /****************************may have bugs********************************/ + tmp_region.shared_seed++; + + if(candidates->list[i].self_offset < tmp_region.x_pos_s) + { + tmp_region.x_pos_s = candidates->list[i].self_offset; + } + + if(candidates->list[i].self_offset > tmp_region.x_pos_e) + { + tmp_region.x_pos_e = candidates->list[i].self_offset; + } + + if(candidates->list[i].offset < tmp_region.y_pos_s) + { + tmp_region.y_pos_s = candidates->list[i].offset; + } + + + if(candidates->list[i].offset > tmp_region.y_pos_e) + { + tmp_region.y_pos_e = candidates->list[i].offset; + } + + // if(memcmp("m64013_190412_043951/159056664/ccs", Get_NAME((*R_INF), readID), + // Get_NAME_LENGTH((*R_INF), readID)) == 0 + // && + // memcmp("m64013_190322_203854/42402522/ccs", Get_NAME((*R_INF), current_ID), + // Get_NAME_LENGTH((*R_INF), current_ID)) == 0) + // { + // fprintf(stderr, "i: %d, x_pos_s: %d, x_pos_e: %d, y_pos_s: %d, y_pos_e: %d\n", + // i, tmp_region.x_pos_s, tmp_region.x_pos_e, tmp_region.y_pos_s, tmp_region.y_pos_e); + + // fprintf(stderr, "i: %d, self_offset: %d, offset: %d, tmp_pos_distance: %d, tmp_self_pos_distance: %d\n", + // i, candidates->list[i].self_offset, candidates->list[i].offset, tmp_pos_distance, + // tmp_self_pos_distance); + // } + + + /****************************may have bugs********************************/ + i++; + /****************************may have bugs********************************/ + continue; + } + } + } + + break; + } + + ///自己和自己重叠的要排除 + ///if (tmp_region.x_id != tmp_region.y_id && tmp_region.shared_seed > 1) + if (tmp_region.x_id != tmp_region.y_id) + { + append_inexact_overlap_region_alloc(overlap_list, &tmp_region, R_INF); + } + + + } +} void append_window_list(overlap_region* region, uint64_t x_start, uint64_t x_end, int y_start, int y_end, int error, -int extra_begin, int extra_end) +int extra_begin, int extra_end, int error_threshold) { long long length = region->x_pos_e - region->x_pos_s + 1; - long long num_windows = length / WINDOW + 2; + ///the length of window may large or small than WINDOW + /****************************may have bugs********************************/ + long long num_windows = length / WINDOW + 4; + /****************************may have bugs********************************/ ///w_list_length会在clear_overlap_region_alloc中定时清空 if (num_windows > region->w_list_size) @@ -681,6 +1764,7 @@ int extra_begin, int extra_end) region->w_list[region->w_list_length].cigar.length = -1; region->w_list[region->w_list_length].extra_begin = extra_begin; region->w_list[region->w_list_length].extra_end = extra_end; + region->w_list[region->w_list_length].error_threshold = error_threshold; region->w_list_length++; } @@ -1029,6 +2113,29 @@ void merge_k_mer_pos_list_alloc_heap_sort(k_mer_pos_list_alloc* list, Candidates } +void insert_kv_list_to_candidates(k_v* list, long long occ, long long y_id, long long y_offset, long long y_strand, +Candidates_list* candidates) +{ + if(candidates->length + occ > candidates->size) + { + candidates->size = (candidates->length + occ) * 2; + candidates->list = (k_mer_hit*)realloc(candidates->list, sizeof(k_mer_hit)*candidates->size); + candidates->tmp = (k_mer_hit*)realloc(candidates->tmp, sizeof(k_mer_hit)*candidates->size); + } + + long long i; + ElemType x; + for (i = 0; i < occ; i++) + { + x.node.offset = y_offset; + x.node.readID = y_id; + x.node.self_offset = list[i].value; + x.node.strand = y_strand; + append_pos_to_Candidates_list(candidates, &x); + } +} + + ///有bug，找时间排一下 void merge_k_mer_pos_list_alloc_heap_sort_advance(k_mer_pos_list_alloc* list, Candidates_list* candidates, HeapSq* HBT) { @@ -1308,6 +2415,7 @@ void write_Total_Pos_Table(Total_Pos_Table* TCB, char* read_file_name) char* index_name = (char*)malloc(strlen(read_file_name)+5); sprintf(index_name, "%s.idx", read_file_name); FILE* fp = fopen(index_name, "w"); + fwrite(&adapterLen, sizeof(adapterLen), 1, fp); fwrite(&k_mer_min_freq, sizeof(k_mer_min_freq), 1, fp); fwrite(&k_mer_max_freq, sizeof(k_mer_max_freq), 1, fp); fwrite(&TCB->prefix_bits, sizeof(TCB->prefix_bits), 1, fp); @@ -1342,8 +2450,14 @@ int load_Total_Pos_Table(Total_Pos_Table* TCB, char* read_file_name) { return 0; } - - + int local_adapterLen; + fread(&local_adapterLen, sizeof(local_adapterLen), 1, fp); + if(local_adapterLen != adapterLen) + { + fprintf(stdout, "the adapterLen of index is: %d, but the adapterLen set by user is: %d\n", + local_adapterLen, adapterLen); + exit(1); + } fread(&k_mer_min_freq, sizeof(k_mer_min_freq), 1, fp); fread(&k_mer_max_freq, sizeof(k_mer_max_freq), 1, fp); fread(&TCB->prefix_bits, sizeof(TCB->prefix_bits), 1, fp); @@ -1506,6 +2620,44 @@ int cmp_k_mer_pos(const void * a, const void * b) } +void init_Chain_Data(Chain_Data* x) +{ + x->length = 0; + x->size = 0; + x->score = NULL; + x->pre = NULL; + x->indels = NULL; + x->self_length = NULL; +} + +void clear_Chain_Data(Chain_Data* x) +{ + x->length = 0; +} + + +void destory_Chain_Data(Chain_Data* x) +{ + free(x->score); + free(x->pre); + free(x->indels); + free(x->self_length); +} + + +void resize_Chain_Data(Chain_Data* x, long long size) +{ + if(size > x->size) + { + x->size = size; + x->score = (long long*)realloc(x->score, x->size*sizeof(long long)); + x->pre = (long long*)realloc(x->pre, x->size*sizeof(long long)); + x->indels = (long long*)realloc(x->indels, x->size*sizeof(long long)); + x->self_length = (long long*)realloc(x->self_length, x->size*sizeof(long long)); + } +} + + void init_Candidates_list(Candidates_list* l) { l->length = 0; @@ -1514,6 +2666,7 @@ void init_Candidates_list(Candidates_list* l) l->tmp = NULL; l->foward_pos = 0; l->rc_pos = 0; + init_Chain_Data(&(l->chainDP)); } void clear_Candidates_list(Candidates_list* l) @@ -1521,12 +2674,14 @@ void clear_Candidates_list(Candidates_list* l) l->length = 0; l->foward_pos = 0; l->rc_pos = 0; + clear_Chain_Data(&(l->chainDP)); } void destory_Candidates_list(Candidates_list* l) { free(l->list); free(l->tmp); + destory_Chain_Data(&(l->chainDP)); } ///1是x小，2是y小，0是相等 @@ -1961,4 +3116,263 @@ void test_COUNT64() kh_destroy(COUNT64, h); +} + + +int cmp_candidates_list(const void * a, const void * b) +{ + long long r_pos_a, r_pos_b; + + if((*((k_mer_hit*)a)).strand != (*((k_mer_hit*)b)).strand) + { + return (*((k_mer_hit*)a)).strand > (*((k_mer_hit*)b)).strand ? 1: -1; + } + else + { + if((*((k_mer_hit*)a)).readID != (*((k_mer_hit*)b)).readID) + { + return (*((k_mer_hit*)a)).readID > (*((k_mer_hit*)b)).readID ? 1: -1; + } + else + { + r_pos_a = (*((k_mer_hit*)a)).offset - (*((k_mer_hit*)a)).self_offset; + r_pos_b = (*((k_mer_hit*)b)).offset - (*((k_mer_hit*)b)).self_offset; + + if(r_pos_a != r_pos_b) + { + return r_pos_a > r_pos_b ? 1: -1; + } + else + { + if((*((k_mer_hit*)a)).self_offset != (*((k_mer_hit*)b)).self_offset) + { + return (*((k_mer_hit*)a)).self_offset > (*((k_mer_hit*)b)).self_offset ? 1: -1; + } + else + { + return 0; + } + + } + + } + + } +} + + +void sort_candidates(Candidates_list* candidates, long long readID, +overlap_region_alloc* overlap_list, All_reads* R_INF) +{ + qsort(candidates->list, candidates->length, sizeof(k_mer_hit), cmp_candidates_list); + + + overlap_region tmp_region; + uint64_t i = 0; + long long current_pos_diff; + long long current_self_pos; + uint64_t current_ID; + uint64_t current_stand; + long long tmp_pos_distance; + long long tmp_self_pos_distance; + long long constant_distance = 5; + double error_rate = 0.05; + + if (candidates->length == 0) + { + return; + } + + + // i = 0; + // for (i = 0; i < candidates->length; i++) + // { + // fprintf(stderr, "i: %d, ada_offset: %d, offset: %d, self_offset: %d, strand: %d\n", + // i, candidates->list[i].offset - candidates->list[i].self_offset, + // candidates->list[i].offset, candidates->list[i].self_offset, + // candidates->list[i].strand); + // } + + + + + i = 0; + while (i < candidates->length) + { + + current_pos_diff = candidates->list[i].offset - candidates->list[i].self_offset; + current_self_pos = candidates->list[i].self_offset; + current_ID = candidates->list[i].readID; + current_stand = candidates->list[i].strand; + + tmp_region.shared_seed = 1; + ///这个是查询read的信息 + tmp_region.x_id = readID; + tmp_region.x_pos_s = candidates->list[i].self_offset; + tmp_region.x_pos_e = candidates->list[i].self_offset; + tmp_region.x_pos_strand = 0; ///永远是0 + + ///这个是被查询的read的信息 + tmp_region.y_id = current_ID; + tmp_region.y_pos_s = candidates->list[i].offset; + tmp_region.y_pos_e = candidates->list[i].offset; + tmp_region.y_pos_strand = candidates->list[i].strand; + + i++; + + + + while (i < candidates->length) + { + if (current_ID == candidates->list[i].readID && + current_stand == candidates->list[i].strand) + { + ///这个一定是正值 + tmp_pos_distance = candidates->list[i].offset - candidates->list[i].self_offset - current_pos_diff; + ///这个不一定是正值 + tmp_self_pos_distance = candidates->list[i].self_offset - current_self_pos; + + if(tmp_self_pos_distance < 0) + { + ///fprintf(stderr, "tmp_self_pos_distance: %d\n", tmp_self_pos_distance); + tmp_self_pos_distance = tmp_self_pos_distance * -1; + ///fprintf(stderr, "tmp_self_pos_distance: %d\n", tmp_self_pos_distance); + } + + ///if (tmp_self_pos_distance >= 0) + { + tmp_self_pos_distance = tmp_self_pos_distance * error_rate + constant_distance; + if (tmp_pos_distance < tmp_self_pos_distance) + { + /****************************may have bugs********************************/ + ///i++; + // tmp_region.x_pos_e = candidates->list[i].self_offset; + // tmp_region.y_pos_e = candidates->list[i].offset; + /****************************may have bugs********************************/ + tmp_region.shared_seed++; + + if(candidates->list[i].self_offset < tmp_region.x_pos_s) + { + tmp_region.x_pos_s = candidates->list[i].self_offset; + } + + if(candidates->list[i].self_offset > tmp_region.x_pos_e) + { + tmp_region.x_pos_e = candidates->list[i].self_offset; + } + + if(candidates->list[i].offset < tmp_region.y_pos_s) + { + tmp_region.y_pos_s = candidates->list[i].offset; + } + + + if(candidates->list[i].offset > tmp_region.y_pos_e) + { + tmp_region.y_pos_e = candidates->list[i].offset; + } + + + /****************************may have bugs********************************/ + i++; + /****************************may have bugs********************************/ + continue; + } + } + } + + break; + } + + + + + + + + + + ///自己和自己重叠的要排除 + ///if (tmp_region.x_id != tmp_region.y_id && tmp_region.shared_seed > 1) + if (tmp_region.x_id != tmp_region.y_id) + { + // fprintf(stderr, "i: %lld, candidates->length: %lld, tmp_region->shared_seed: %lld, x_pos_s: %lld, x_pos_e: %lld, y_pos_s: %lld, y_pos_e: %lld\n", + // i, candidates->length, tmp_region.shared_seed, tmp_region.x_pos_s, tmp_region.x_pos_e, tmp_region.y_pos_s, tmp_region.y_pos_e); + // fprintf(stderr, "\n"); + + // fprintf(stderr, "shared_seed: %d, x_pos_s: %d, x_pos_e: %d, y_pos_s: %d, y_pos_e: %d\n", + // tmp_region.shared_seed, tmp_region.x_pos_s, tmp_region.x_pos_e, tmp_region.y_pos_s, tmp_region.y_pos_e); + ///append_overlap_region_alloc(overlap_list, &tmp_region, R_INF); + append_overlap_region_alloc_from_existing(overlap_list, &tmp_region, R_INF); + + } + } + + ///以x_pos_e，即结束位置为主元排序 + ///qsort(overlap_list->list, overlap_list->length, sizeof(overlap_region), cmp_by_x_pos_e); + ///qsort(overlap_list->list, overlap_list->length, sizeof(overlap_region), cmp_by_x_pos_s); + + clear_Candidates_list(candidates); +} + + +void init_fake_cigar(Fake_Cigar* x) +{ + x->buffer = NULL; + x->length = 0; + x->size = 0; +} + +void destory_fake_cigar(Fake_Cigar* x) +{ + if(x->size > 0) + { + free(x->buffer); + } +} + +void clear_fake_cigar(Fake_Cigar* x) +{ + x->length = 0; +} + +void add_fake_cigar(Fake_Cigar* x, uint32_t gap_site, int32_t gap_shift) +{ + if(x->length + 1 > x->size) + { + x->size = (x->length + 1) * 2; + x->buffer = (uint64_t*)realloc(x->buffer, sizeof(uint64_t) * x->size); + } + + x->buffer[x->length] = gap_site; + x->buffer[x->length] = x->buffer[x->length] << 32; + + if(gap_shift < 0) + { + gap_shift = gap_shift * -1; + gap_site = gap_shift; + gap_site = gap_site << 1; + gap_site = gap_site | ((uint32_t)1); + } + else + { + gap_site = gap_shift; + gap_site = gap_site << 1; + } + + x->buffer[x->length] = x->buffer[x->length] | ((uint32_t)gap_site); + + x->length++; +} + + +void resize_fake_cigar(Fake_Cigar* x, long long size) +{ + if(size > x->size) + { + x->size = size; + x->buffer = (uint64_t*)realloc(x->buffer, sizeof(uint64_t) * x->size); + } + + x->length = 0; } \ No newline at end of file diff --git a/Hash_Table.h b/Hash_Table.h index f279cf2..88e5d5a 100644 --- a/Hash_Table.h +++ b/Hash_Table.h @@ -17,23 +17,23 @@ typedef khash_t(POS64) Pos_Table; ///#define THRESHOLD 14 #define WINDOW 375 -#define WINDOW_BOUNDARY 150 +//#define WINDOW_BOUNDARY 150 +#define WINDOW_BOUNDARY 375 ///for one side, the first or last WINDOW_UNCORRECT_SINGLE_SIDE_BOUNDARY bases should not be corrected #define WINDOW_UNCORRECT_SINGLE_SIDE_BOUNDARY 25 #define THRESHOLD 15 #define THRESHOLD_RATE 0.04 -#define OVERLAP_THRESHOLD 0.9 +#define TAIL_LENGTH int(1/THRESHOLD_RATE) +///#define OVERLAP_THRESHOLD 0.9 +#define OVERLAP_THRESHOLD_FILTER 0.9 +#define WINDOW_MAX_SIZE WINDOW + TAIL_LENGTH + 3 +#define THRESHOLD_MAX_SIZE 31 -/** -#define WINDOW 500 -#define THRESHOLD 15 -#define THRESHOLD_RATE 0.03 -#define OVERLAP_THRESHOLD 0.95 -**/ #define GROUP_SIZE 4 ///最长是10M10D10M10D10M这种 -#define CIGAR_MAX_LENGTH THRESHOLD*2+2 +///#define CIGAR_MAX_LENGTH THRESHOLD*2+2 +#define CIGAR_MAX_LENGTH 31*2+4 typedef struct { @@ -92,12 +92,21 @@ typedef struct int y_start; int extra_begin; int extra_end; + int error_threshold; ///int y_pre_start; ///error小于等于0都要重新算 int error; CIGAR cigar; } window_list; + +typedef struct +{ + uint64_t* buffer; + uint64_t length; + uint64_t size; +}Fake_Cigar; + typedef struct { uint64_t x_id; @@ -113,10 +122,15 @@ typedef struct uint64_t shared_seed; uint64_t align_length; + ///uint64_t total_errors; + uint8_t is_match; + uint64_t non_homopolymer_errors; window_list* w_list; uint64_t w_list_size; uint64_t w_list_length; + int8_t strong; + Fake_Cigar f_cigar; } overlap_region; @@ -125,6 +139,8 @@ typedef struct overlap_region* list; uint64_t size; uint64_t length; + ///uint64_t mapped_overlaps_length; + long long mapped_overlaps_length; } overlap_region_alloc; typedef struct @@ -153,7 +169,15 @@ typedef struct int MaxSize; } HeapSq; - +typedef struct +{ + long long* score; + long long* pre; + long long* indels; + long long* self_length; + long long length; + long long size; +} Chain_Data; typedef struct { @@ -163,6 +187,7 @@ typedef struct long long size; uint64_t foward_pos; uint64_t rc_pos; + Chain_Data chainDP; } Candidates_list; typedef struct @@ -206,6 +231,23 @@ inline uint64_t mod_d(uint64_t h_key, uint64_t low_key, uint64_t d) return result; } +inline int if_k_mer_available(Hash_code* code, int k) +{ + uint64_t h_key, low_key; + ///k有可能是64，所以可能会有问题 + ///low_key = code->x[0] | (code->x[1] << k); + low_key = code->x[0] | (code->x[1] << SAFE_SHIFT(k)); + //k不可能为0, 所以这个右移不会有问题 + h_key = code->x[1] >> (64 - k); + + if(mod_d(h_key, low_key, MODE_VALUE) > 3) + { + return 0; + } + + return 1; +} + ///inline int get_sub_table(uint64_t* get_sub_ID, uint64_t* get_sub_key, Total_Count_Table* TCB, Hash_code* code, int k) @@ -458,16 +500,84 @@ void append_overlap_region_alloc(overlap_region_alloc* list, overlap_region* tmp void calculate_overlap_region(Candidates_list* candidates, overlap_region_alloc* overlap_list, uint64_t readID, uint64_t readLength, All_reads* R_INF); void append_window_list(overlap_region* region, uint64_t x_start, uint64_t x_end, int y_start, int y_end, int error, -int extra_begin, int extra_end); +int extra_begin, int extra_end, int error_threshold); + + +void insert_kv_list_to_candidates(k_v* list, long long occ, long long y_id, long long y_offset, long long y_strand, +Candidates_list* candidates); + +void overlap_region_sort_y_id(overlap_region *a, long long n); + +void calculate_inexact_overlap_region(Candidates_list* candidates, overlap_region_alloc* overlap_list, +uint64_t readID, uint64_t readLength, All_reads* R_INF); + +void calculate_overlap_region_by_chaining(Candidates_list* candidates, overlap_region_alloc* overlap_list, +uint64_t readID, uint64_t readLength, All_reads* R_INF, double band_width_threshold); +static const char LogTable256[256] = { +#define LT(n) n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n + -1, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + LT(4), LT(5), LT(5), LT(6), LT(6), LT(6), LT(6), + LT(7), LT(7), LT(7), LT(7), LT(7), LT(7), LT(7), LT(7) +}; +static inline int ilog2_32(uint32_t v) +{ + uint32_t t, tt; + if ((tt = v>>16)) return (t = tt>>8) ? 24 + LogTable256[t] : 16 + LogTable256[tt]; + return (t = v>>8) ? 8 + LogTable256[t] : LogTable256[v]; +} +void init_fake_cigar(Fake_Cigar* x); +void destory_fake_cigar(Fake_Cigar* x); +void clear_fake_cigar(Fake_Cigar* x); +void add_fake_cigar(Fake_Cigar* x, uint32_t gap_site, int32_t gap_shift); +void resize_fake_cigar(Fake_Cigar* x, long long size); +int get_fake_gap_pos(Fake_Cigar* x, int index); +int get_fake_gap_shift(Fake_Cigar* x, int index); +inline long long y_start_offset(long long x_start, Fake_Cigar* o) +{ + if(x_start == get_fake_gap_pos(o, o->length - 1)) + { + return get_fake_gap_shift(o, o->length - 1); + } + + + long long i; + for (i = 0; i < o->length; i++) + { + if(x_start < get_fake_gap_pos(o, i)) + { + break; + } + } + if(i == 0 || i == o->length) + { + fprintf(stderr, "ERROR\n"); + exit(0); + } + + ///note here return i - 1 + return get_fake_gap_shift(o, i - 1); +} + +inline void print_fake_gap(Fake_Cigar* o) +{ + long long i; + for (i = 0; i < o->length; i++) + { + fprintf(stderr, "**i: %d, gap_pos_in_x: %d, gap_shift: %d\n", + i, get_fake_gap_pos(o, i), + get_fake_gap_shift(o, i)); + } + +} @@ -585,4 +695,10 @@ void debug_mode(uint64_t d, uint64_t thread_ID, uint64_t thread_num); /********************************for debug***************************************/ void merge_Candidates_list_version(Candidates_list* l, k_mer_pos* n_list, uint64_t n_lengh, uint64_t end_pos, int strand); +void sort_candidates(Candidates_list* candidates, long long readID, +overlap_region_alloc* overlap_list, All_reads* R_INF); +void append_overlap_region_alloc_from_existing(overlap_region_alloc* list, overlap_region* tmp, All_reads* R_INF); +int cmp_by_x_pos_s(const void * a, const void * b); +void resize_Chain_Data(Chain_Data* x, long long size); + #endif \ No newline at end of file diff --git a/Makefile b/Makefile index d2e34d8..22ddde0 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ CC=g++ CFLAGS = -w -c -msse4.2 -mpopcnt -fomit-frame-pointer -Winline -O3 -lz LDFLAGS = -lm -lz -lpthread -O3 -mpopcnt -msse4.2 -lz -w -SOURCES = main.cpp Output.cpp CommandLines.cpp Process_Read.cpp Assembly.cpp kmer.cpp Hash_Table.cpp POA.cpp Correct.cpp Levenshtein_distance.cpp edlib.cpp +SOURCES = main.cpp Output.cpp CommandLines.cpp Process_Read.cpp Assembly.cpp kmer.cpp Hash_Table.cpp POA.cpp Correct.cpp Levenshtein_distance.cpp edlib.cpp Overlaps.cpp OBJECTS = $(SOURCES:.c=.o) EXECUTABLE = ccs_assembly diff --git a/Overlaps.cpp b/Overlaps.cpp new file mode 100644 index 0000000..1753736 --- /dev/null +++ b/Overlaps.cpp @@ -0,0 +1,7709 @@ +#include +#include +#include +#include "Overlaps.h" +#include "ksort.h" +#include "Process_Read.h" +#include "CommandLines.h" + +KDQ_INIT(uint64_t) + +#define ma_hit_key_tn(a) ((a).tn) +KRADIX_SORT_INIT(hit_tn, ma_hit_t, ma_hit_key_tn, member_size(ma_hit_t, tn)) + +#define ma_hit_key_qns(a) ((a).qns) +KRADIX_SORT_INIT(hit_qns, ma_hit_t, ma_hit_key_qns, member_size(ma_hit_t, qns)) + +#define asg_arc_key(a) ((a).ul) +KRADIX_SORT_INIT(asg, asg_arc_t, asg_arc_key, 8) + +#define generic_key(x) (x) +KRADIX_SORT_INIT(arch64, uint64_t, generic_key, 8) + +KSORT_INIT_GENERIC(uint32_t) + +///actually min_thres = MAX_SHORT_TIPS + 1 there are MAX_SHORT_TIPS reads +long long min_thres = MAX_SHORT_TIPS + 1; + +void ma_hit_sort_tn(ma_hit_t *a, long long n) +{ + radix_sort_hit_tn(a, a + n); +} + +void ma_hit_sort_qns(ma_hit_t *a, long long n) +{ + radix_sort_hit_qns(a, a + n); +} + + + + +asg_t *asg_init(void) +{ + return (asg_t*)calloc(1, sizeof(asg_t)); +} + +void asg_destroy(asg_t *g) +{ + if (g == 0) return; + free(g->seq); free(g->idx); free(g->arc); free(g->seq_vis); free(g); +} + +void asg_arc_sort(asg_t *g) +{ + radix_sort_asg(g->arc, g->arc + g->n_arc); +} + +void ma_ug_destroy(ma_ug_t *ug) +{ + uint32_t i; + if (ug == 0) return; + for (i = 0; i < ug->u.n; ++i) { + free(ug->u.a[i].a); + free(ug->u.a[i].s); + } + free(ug->u.a); + asg_destroy(ug->g); + free(ug); +} + +uint64_t *asg_arc_index_core(size_t max_seq, size_t n, const asg_arc_t *a) +{ + size_t i, last; + uint64_t *idx; + idx = (uint64_t*)calloc(max_seq * 2, 8); + + + /** + * ul: |____________31__________|__________1___________|______________32_____________| + qns direction of overlap length of this node (not overlap length) + **/ + ///so if we use high 32-bit, we store the index of each qn with two direction + for (i = 1, last = 0; i <= n; ++i) + if (i == n || a[i-1].ul>>32 != a[i].ul>>32) + idx[a[i-1].ul>>32] = (uint64_t)last<<32 | (i - last), last = i; + + + return idx; +} + +void asg_arc_index(asg_t *g) +{ + if (g->idx) free(g->idx); + g->idx = asg_arc_index_core(g->n_seq, g->n_arc, g->arc); +} + +void asg_seq_set(asg_t *g, int sid, int len, int del) +{ + ///just malloc size + if (sid >= g->m_seq) { + g->m_seq = sid + 1; + kv_roundup32(g->m_seq); + g->seq = (asg_seq_t*)realloc(g->seq, g->m_seq * sizeof(asg_seq_t)); + } + + + if (sid >= g->n_seq) g->n_seq = sid + 1; + + g->seq[sid].del = !!del; + g->seq[sid].len = len; + // if(g->seq[sid].del) + // { + // g->seq[sid].len = 0; + // } +} + + +// hard remove arcs marked as "del" +void asg_arc_rm(asg_t *g) +{ + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qns direction of overlap length of this node (not overlap length) + p->v : |___________31___________|__________1___________| + tns relative strand between query and target + p->ol: overlap length + **/ + uint32_t e, n; + ///just clean arc requiring: 1. arc it self must be available 2. both the query and target are available + for (e = n = 0; e < g->n_arc; ++e) { + //u and v is the read id + uint32_t u = g->arc[e].ul>>32, v = g->arc[e].v; + if (!g->arc[e].del && !g->seq[u>>1].del && !g->seq[v>>1].del) + g->arc[n++] = g->arc[e]; + } + if (n < g->n_arc) { // arc index is out of sync + if (g->idx) free(g->idx); + g->idx = 0; + } + g->n_arc = n; +} + +void asg_cleanup(asg_t *g) +{ + ///remove overlaps, instead of reads + asg_arc_rm(g); + if (!g->is_srt) { + /** + * sort by ul, that is, sort by qns + direction + * ul: |____________31__________|__________1___________|______________32_____________| + qns direction of overlap length of this node (not overlap length) + **/ + asg_arc_sort(g); + g->is_srt = 1; + } + ///index the overlaps in graph with query id + if (g->idx == 0) asg_arc_index(g); +} + + +// delete multi-arcs +/** + * remove edges like: v has two out-edges to w +**/ +int asg_arc_del_multi(asg_t *g) +{ + //the number of nodes are number of read times 2 + uint32_t *cnt, n_vtx = g->n_seq * 2, n_multi = 0, v; + cnt = (uint32_t*)calloc(n_vtx, 4); + for (v = 0; v < n_vtx; ++v) { + ///out-nodes of v + asg_arc_t *av = asg_arc_a(g, v); + int32_t i, nv = asg_arc_n(g, v); + ///if v just have one out-node, there is no muti-edge + if (nv < 2) continue; + for (i = nv - 1; i >= 0; --i) ++cnt[av[i].v]; + for (i = nv - 1; i >= 0; --i) + if (--cnt[av[i].v] != 0) + av[i].del = 1, ++n_multi; + } + free(cnt); + if (n_multi) asg_cleanup(g); + fprintf(stderr, "[M::%s] removed %d multi-arcs\n", __func__, n_multi); + return n_multi; +} + +// remove asymmetric arcs: u->v is present, but v'->u' not +int asg_arc_del_asymm(asg_t *g) +{ + uint32_t e, n_asymm = 0; + ///g->n_arc is the number of overlaps + for (e = 0; e < g->n_arc; ++e) { + uint32_t v = g->arc[e].v^1, u = g->arc[e].ul>>32^1; + uint32_t i, nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + for (i = 0; i < nv; ++i) + if (av[i].v == u) break; + if (i == nv) g->arc[e].del = 1, ++n_asymm; + } + if (n_asymm) asg_cleanup(g); + fprintf(stderr, "[M::%s] removed %d asymmetric arcs\n", __func__, n_asymm); + return n_asymm; +} + + +void asg_symm(asg_t *g) +{ + asg_arc_del_multi(g); + asg_arc_del_asymm(g); + g->is_symm = 1; +} + +void init_ma_hit_t_alloc(ma_hit_t_alloc* x) +{ + x->size = 0; + x->buffer = NULL; + x->length = 0; +} + +void clear_ma_hit_t_alloc(ma_hit_t_alloc* x) +{ + x->length = 0; +} + +void resize_ma_hit_t_alloc(ma_hit_t_alloc* x, uint64_t size) +{ + if(size > x->size) + { + x->size = size; + x->buffer = (ma_hit_t*)realloc(x->buffer, x->size*sizeof(ma_hit_t)); + } +} + +void destory_ma_hit_t_alloc(ma_hit_t_alloc* x) +{ + free(x->buffer); +} + + +void add_ma_hit_t_alloc(ma_hit_t_alloc* x, ma_hit_t* element) +{ + if(x->length + 1 > x->size) + { + x->size = (x->length + 1) * 2; + x->buffer = (ma_hit_t*)realloc(x->buffer, x->size*sizeof(ma_hit_t)); + } + + x->buffer[x->length] = (*element); + x->length++; +} + + + +void init_Assembly_Graph(Assembly_Graph* x) +{ + init_ma_hit_t_alloc(&(x->overlaps)); +} + +void destory_Assembly_Graph(Assembly_Graph* x) +{ + destory_ma_hit_t_alloc(&(x->overlaps)); +} + +long long get_specific_overlap(ma_hit_t_alloc* x, uint32_t qn, uint32_t tn) +{ + long long i; + for (i = 0; i < x->length; i++) + { + if(x->buffer[i].tn == tn + && + ((uint32_t)(x->buffer[i].qns>>32)) == qn) + { + return i; + } + } + + return -1; +} + + + +void collect_ma_hit_t(ma_hit_t_alloc* dest, ma_hit_t_alloc* sources, long long num_sources) +{ + long long bi_overlaps = 0; + long long si_overlaps = 0; + long long i, j, index; + uint32_t qn, tn; + ma_hit_t new_element; + for (i = 0; i < num_sources; i++) + { + resize_ma_hit_t_alloc(dest, dest->length + sources[i].length); + for (j = 0; j < sources[i].length; j++) + { + qn = sources[i].buffer[j].qns>>32; + tn = sources[i].buffer[j].tn; + + index = get_specific_overlap(&(sources[tn]), tn, qn); + + + if(index != -1) + { + + // fprintf(stderr, "\n+qn: %d, tn: %d, qs: %d, qe: %d, ts: %d, te: %d\n", qn, tn, + // (uint32_t)(sources[i].buffer[j].qns), sources[i].buffer[j].qe, + // sources[i].buffer[j].ts, sources[i].buffer[j].te); + + // fprintf(stderr, "-qn: %d, tn: %d, qs: %d, qe: %d, ts: %d, te: %d\n\n", + // sources[tn].buffer[index].qns>>32, sources[tn].buffer[index].tn, + // (uint32_t)(sources[tn].buffer[index].qns), sources[tn].buffer[index].qe, + // sources[tn].buffer[index].ts, sources[tn].buffer[index].te); + + if(qn <= tn) + { + + } + + + bi_overlaps++; + } + else + { + si_overlaps++; + } + } + } + + + fprintf(stderr, "bi_overlaps: %d, si_overlaps: %d\n", bi_overlaps, si_overlaps); + +} + + +inline void set_reverse_overlap(ma_hit_t* dest, ma_hit_t* source) +{ + dest->qns = Get_tn(*source); + dest->qns = dest->qns << 32; + dest->qns = dest->qns | Get_ts(*source); + dest->qe = Get_te(*source); + + + dest->tn = Get_qn(*source); + dest->ts = Get_qs(*source); + dest->te = Get_qe(*source); + + dest->rev = source->rev; + dest->el = source->el; + + + /****************************may have bugs********************************/ + if(dest->ml == 0 || source->ml == 0) + { + dest->ml = source->ml = 0; + } + else + { + dest->ml = source->ml = 1; + } + /****************************may have bugs********************************/ + dest->bl = Get_qe(*dest) - Get_qs(*dest); +} + +void normalize_ma_hit_t(ma_hit_t_alloc* sources, long long num_sources) +{ + long long bi_overlaps = 0; + long long si_overlaps = 0; + long long i, j, index; + uint32_t qn, tn; + ma_hit_t new_element; + long long qLen_0, qLen_1; + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + qn = Get_qn(sources[i].buffer[j]); + tn = Get_tn(sources[i].buffer[j]); + + sources[i].buffer[j].bl = Get_qe(sources[i].buffer[j]) - Get_qs(sources[i].buffer[j]); + + index = get_specific_overlap(&(sources[tn]), tn, qn); + + + if(index != -1) + { + qLen_0 = Get_qe(sources[i].buffer[j]) - Get_qs(sources[i].buffer[j]); + qLen_1 = Get_qe(sources[tn].buffer[index]) - Get_qs(sources[tn].buffer[index]); + + if(qLen_0 == qLen_1) + { + ///qn must be not equal to tn + ///make sources[qn] = sources[tn] if qn > tn + if(qn < tn) + { + set_reverse_overlap(&(sources[tn].buffer[index]), &(sources[i].buffer[j])); + } + } + else if(qLen_0 > qLen_1) + { + + set_reverse_overlap(&(sources[tn].buffer[index]), &(sources[i].buffer[j])); + } + + + bi_overlaps++; + } + else + { + set_reverse_overlap(&new_element, &(sources[i].buffer[j])); + add_ma_hit_t_alloc(&(sources[tn]), &new_element); + si_overlaps++; + } + } + } + + + /** + for (i = 0; i < num_sources; i++) + { + ma_hit_sort_qns(sources[i].buffer, sources[i].length); + } + **/ +} + + +void normalize_ma_hit_t_single_side(ma_hit_t_alloc* sources, long long num_sources) +{ + double startTime = Get_T(); + + long long bi_overlaps = 0; + long long si_overlaps = 0; + long long i, j, index; + uint32_t qn, tn; + ma_hit_t new_element; + long long qLen_0, qLen_1, m; + for (i = 0; i < num_sources; i++) + { + m = 0; + for (j = 0; j < sources[i].length; j++) + { + qn = Get_qn(sources[i].buffer[j]); + tn = Get_tn(sources[i].buffer[j]); + + sources[i].buffer[j].bl = Get_qe(sources[i].buffer[j]) - Get_qs(sources[i].buffer[j]); + + index = get_specific_overlap(&(sources[tn]), tn, qn); + + + if(index != -1) + { + qLen_0 = Get_qe(sources[i].buffer[j]) - Get_qs(sources[i].buffer[j]); + qLen_1 = Get_qe(sources[tn].buffer[index]) - Get_qs(sources[tn].buffer[index]); + + if(qLen_0 == qLen_1) + { + ///qn must be not equal to tn + ///make sources[qn] = sources[tn] if qn > tn + if(qn < tn) + { + set_reverse_overlap(&(sources[tn].buffer[index]), &(sources[i].buffer[j])); + } + } + else if(qLen_0 > qLen_1) + { + set_reverse_overlap(&(sources[tn].buffer[index]), &(sources[i].buffer[j])); + } + + sources[i].buffer[m] = sources[i].buffer[j]; + m++; + bi_overlaps++; + } + } + + sources[i].length = m; + } + + + fprintf(stderr, "[M::%s] takes %0.2fs\n\n", __func__, Get_T()-startTime); +} + + + + +void ma_hit_contained(ma_hit_t_alloc* sources, long long n_read, ma_sub_t *coverage_cut, int max_hang, int min_ovlp) +{ + double startTime = Get_T(); + int32_t r; + size_t i, j, m; + asg_arc_t t; + for (i = 0; i < n_read; ++i) + { + for (j = 0; j < sources[i].length; j++) + { + ma_hit_t *h = &(sources[i].buffer[j]); + //check the corresponding two reads + ma_sub_t *sq = &(coverage_cut[Get_qn(*h)]); + ma_sub_t *st = &(coverage_cut[Get_tn(*h)]); + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, MAX_HANG_PRE, min_ovlp, &t); + ///r could not be MA_HT_SHORT_OVLP or MA_HT_INT + if (r == MA_HT_QCONT) + { + sq->del = 1; + } + else if (r == MA_HT_TCONT) + { + st->del = 1; + } + } + } + + + + for (i = 0; i < n_read; ++i) + { + m = 0; + for (j = 0; j < sources[i].length; j++) + { + ma_hit_t *h = &(sources[i].buffer[j]); + ///both the qn and tn have not been deleted + if(coverage_cut[Get_qn(*h)].del != 1 && coverage_cut[Get_tn(*h)].del != 1) + { + sources[i].buffer[m] = *h; + m++; + } + } + sources[i].length = m; + ///may have bugs here + ///if sources[i].length == 0, that means all overlapped reads with read i are the contained reads + if(sources[i].length == 0) + { + coverage_cut[i].del = 1; + } + } + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + +} + + + +void ma_hit_flt(ma_hit_t_alloc* sources, long long n_read, ma_sub_t *coverage_cut, int max_hang, int min_ovlp) +{ + double startTime = Get_T(); + size_t i, j, m; + asg_arc_t t; + uint64_t tot_dp = 0, tot_len = 0; + + for (i = 0; i < n_read; ++i) + { + m = 0; + for (j = 0; j < sources[i].length; j++) + { + ma_hit_t *h = &(sources[i].buffer[j]); + //check the corresponding two reads + const ma_sub_t *sq = &(coverage_cut[Get_qn(*h)]); + const ma_sub_t *st = &(coverage_cut[Get_tn(*h)]); + int r; + if (sq->del || st->del) continue; + ///[sq->s, sq->e) and [st->s, st->e) are the high coverage region in query and target + ///here just exculde the overhang? + ///in miniasm the 5-th option is 0.5, instead of 0.8 + /**note!!! h->qn and h->qs have been normalized by sq->s + * h->ts and h->tn have been normalized by sq->e + **/ + ///here the max_hang = 1000, MAX_HANG_PRE = 0.8, min_ovlp = 500 + ///for me, there should not have any overhang..so r cannot be equal to MA_HT_INT + r = ma_hit2arc(h, sq->e - sq->s, st->e - st->s, max_hang, MAX_HANG_PRE, min_ovlp, &t); + + + ///for me, there should not have any overhang..so r cannot be equal to MA_HT_INT + ///and I think if we use same min_ovlp in all functions, r also cannot be MA_HT_SHORT_OVLP + ///so it does not matter we have ma_hit2arc or not + if (r >= 0 || r == MA_HT_QCONT || r == MA_HT_TCONT) + { + sources[i].buffer[m] = *h; + m++; + }/** + else + { + fprintf(stderr, "shit\n"); + } + **/ + + } + sources[i].length = m; + if(sources[i].length == 0) + { + (coverage_cut)[i].del = 1; + } + } + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); +} + + + +///a is the overlap vector, n is the length of overlap vector +///min_dp is used for coverage droping +///select reads with coverage >= min_dp +void ma_hit_sub_back(int min_dp, ma_hit_t_alloc* sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, ma_sub_t** coverage_cut) +{ + (*coverage_cut) = (ma_sub_t*)malloc(sizeof(ma_sub_t)*n_read); + + size_t i, j, n_remained = 0; + kvec_t(uint32_t) b = {0,0,0}; + + ///all overlaps in vector a has been sorted by qns + ///so for overlaps of one reads, it must be contiguous + for (i = 0; i < n_read; ++i) + { + // fprintf(stderr, "i: %d, n_read: %d\n", i, n_read); + + kv_resize(uint32_t, b, sources[i].length); + b.n = 0; + for (j = 0; j < sources[i].length; j++) + { + uint32_t qs, qe; + qs = Get_qs(sources[i].buffer[j]); + qe = Get_qe(sources[i].buffer[j]); + kv_push(uint32_t, b, qs<<1); + kv_push(uint32_t, b, qe<<1|1); + } + + ///we can identify the qs and qe by the 0-th bit + ks_introsort_uint32_t(b.n, b.a); + ma_sub_t max, max2; + max.s = max.e = max.del = max2.s = max2.e = max2.del = 0; + int dp, start; + ///max is the longest subregion, max2 is the second longest subregion + for (j = 0, dp = 0; j < b.n; ++j) + { + int old_dp = dp; + ///if a[j] is qe + if (b.a[j]&1) + { + --dp; + } + else + { + ++dp; + } + + ///min_dp is the coverage drop threshold + ///there are two cases: 1. old_dp = dp + 1 (b.a[j] is qe); 2. old_dp = dp - 1 (b.a[j] is qs); + ///if one read has multiple separate sub-regions with coverage > 3, does miniasm only select the longest one? + if (old_dp < min_dp && dp >= min_dp) ///old_dp < dp, b.a[j] is qs + { ///case 2, a[j] is qs + start = b.a[j]>>1; + } + else if (old_dp >= min_dp && dp < min_dp) ///old_dp > min_dp, b.a[j] is qe + { + int len = (b.a[j]>>1) - start; + if (len > max.e - max.s) + { + max2 = max; + max.s = start; + max.e = b.a[j]>>1; + } + else if (len > max2.e - max2.s) + { + max2.s = start; + max2.e = b.a[j]>>1; + } + } + } + + + + if (max.e - max.s > 0) + { + (*coverage_cut)[i].s = max.s; + (*coverage_cut)[i].e = max.e; + (*coverage_cut)[i].del = 0; + ++n_remained; + } + else + { + (*coverage_cut)[i].del = 1; + } + } + + + + + ma_hit_t* p; + ma_sub_t* rq; + ma_sub_t* rt; + long long m = 0; + for (i = 0; i < n_read; ++i) + { + m = 0; + for (j = 0; j < sources[i].length; j++) + { + ///this is a overlap + p = &(sources[i].buffer[j]); + + rq = &((*coverage_cut)[Get_qn(*p)]); + rt = &((*coverage_cut)[Get_tn(*p)]); + ///if any of target read and the query read has no enough coverage + if (rq->del || rt->del) continue; + int qs, qe, ts, te; + + + + + ///target and query in different strand + if (p->rev) + { + qs = p->te < rt->e? Get_qs(*p): Get_qs(*p) + (p->te - rt->e); + qe = p->ts > rt->s? p->qe : p->qe - (rt->s - p->ts); + ts = p->qe < rq->e? p->ts : p->ts + (p->qe - rq->e); + te = Get_qs(*p) > rq->s? p->te : p->te - (rq->s - Get_qs(*p)); + } + else ///target and query in same strand + { + ///note: ts is the targe start in this overlap, + ///while rt->s is the high coverage start in the whole target (not only in this overlap) + ///so this line is to normalize the qs in quey to high coverage region + qs = p->ts > rt->s? Get_qs(*p): Get_qs(*p) + (rt->s - p->ts); //(rt->s - p->ts) is the offset + qe = p->te < rt->e? p->qe : p->qe - (p->te - rt->e);//(p->te - rt->e) is the offset + ts = Get_qs(*p) > rq->s? p->ts : p->ts + (rq->s - Get_qs(*p));//(rq->s - Get_qs(*p) is the offset + te = p->qe < rq->e? p->te : p->te - (p->qe - rq->e);//(p->qe - rq->e) is the offset + } + + + + //cut by self coverage + qs = (qs > rq->s? qs : rq->s) - rq->s; + qe = (qe < rq->e? qe : rq->e) - rq->s; + ts = (ts > rt->s? ts : rt->s) - rt->s; + te = (te < rt->e? te : rt->e) - rt->s; + + if (qe - qs >= mini_overlap_length && te - ts >= mini_overlap_length) + { + ///p->qns = p->qns>>32<<32 | qs; + p->qns = p->qns>>32; + p->qns = p->qns << 32; + p->qns = p->qns | qs; + + p->qe = qe; + p->ts = ts; + p->te = te; + sources[i].buffer[m] = *p; + m++; + } + } + sources[i].length = m; + } + + + + + + + + + + + + free(b.a); + ///free((*coverage_cut)); +} + + +///a is the overlap vector, n is the length of overlap vector +///min_dp is used for coverage droping +///select reads with coverage >= min_dp +void ma_hit_sub(int min_dp, ma_hit_t_alloc* sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, ma_sub_t** coverage_cut) +{ + double startTime = Get_T(); + + (*coverage_cut) = (ma_sub_t*)malloc(sizeof(ma_sub_t)*n_read); + + size_t i, j, n_remained = 0; + kvec_t(uint32_t) b = {0,0,0}; + + ///all overlaps in vector a has been sorted by qns + ///so for overlaps of one reads, it must be contiguous + for (i = 0; i < n_read; ++i) + { + if(min_dp <= 1) + { + (*coverage_cut)[i].s = 0; + (*coverage_cut)[i].e = readLen[i]; + (*coverage_cut)[i].del = 0; + ++n_remained; + continue; + } + + + kv_resize(uint32_t, b, sources[i].length); + b.n = 0; + for (j = 0; j < sources[i].length; j++) + { + uint32_t qs, qe; + qs = Get_qs(sources[i].buffer[j]); + qe = Get_qe(sources[i].buffer[j]); + kv_push(uint32_t, b, qs<<1); + kv_push(uint32_t, b, qe<<1|1); + } + + ///we can identify the qs and qe by the 0-th bit + ks_introsort_uint32_t(b.n, b.a); + ma_sub_t max, max2; + max.s = max.e = max.del = max2.s = max2.e = max2.del = 0; + int dp, start; + ///max is the longest subregion, max2 is the second longest subregion + for (j = 0, dp = 0; j < b.n; ++j) + { + int old_dp = dp; + ///if a[j] is qe + if (b.a[j]&1) + { + --dp; + } + else + { + ++dp; + } + + /** + min_dp is the coverage drop threshold + there are two cases: + 1. old_dp = dp + 1 (b.a[j] is qe); 2. old_dp = dp - 1 (b.a[j] is qs); + if one read has multiple separate sub-regions with coverage >= min_dp, + does miniasm only select the longest one? + **/ + if (old_dp < min_dp && dp >= min_dp) ///old_dp < dp, b.a[j] is qs + { + ///case 2, a[j] is qs + start = b.a[j]>>1; + } + else if (old_dp >= min_dp && dp < min_dp) ///old_dp > min_dp, b.a[j] is qe + { + int len = (b.a[j]>>1) - start; + if (len > max.e - max.s) + { + max2 = max; + max.s = start; + max.e = b.a[j]>>1; + } + else if (len > max2.e - max2.s) + { + max2.s = start; + max2.e = b.a[j]>>1; + } + } + } + + + ///max.e - max.s is the + if (max.e - max.s > 0) + { + (*coverage_cut)[i].s = max.s; + (*coverage_cut)[i].e = max.e; + (*coverage_cut)[i].del = 0; + ++n_remained; + } + else + { + (*coverage_cut)[i].s = (*coverage_cut)[i].e = 0; + + (*coverage_cut)[i].del = 1; + } + } + + free(b.a); + + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); +} + + + + +void ma_hit_cut(int min_dp, ma_hit_t_alloc* sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, ma_sub_t** coverage_cut) +{ + double startTime = Get_T(); + size_t i, j; + ma_hit_t* p; + ma_sub_t* rq; + ma_sub_t* rt; + long long m = 0; + for (i = 0; i < n_read; ++i) + { + m = 0; + for (j = 0; j < sources[i].length; j++) + { + ///this is a overlap + p = &(sources[i].buffer[j]); + + rq = &((*coverage_cut)[Get_qn(*p)]); + rt = &((*coverage_cut)[Get_tn(*p)]); + ///if any of target read and the query read has no enough coverage + if (rq->del || rt->del) continue; + int qs, qe, ts, te; + + + + + ///target and query in different strand + if (p->rev) + { + /** + here is an example in different strand: + + (te) (rt->e) (rt->s) (ts) + | | | | + target ---------------------------------------------------- + ------------------------------------------- query + qs qe + **/ + qs = p->te < rt->e? Get_qs(*p): Get_qs(*p) + (p->te - rt->e); + qe = p->ts > rt->s? p->qe : p->qe - (rt->s - p->ts); + ts = p->qe < rq->e? p->ts : p->ts + (p->qe - rq->e); + te = Get_qs(*p) > rq->s? p->te : p->te - (rq->s - Get_qs(*p)); + } + else ///target and query in same strand + { + /** + note: ts is the targe start in this overlap, + while rt->s is the high coverage start in the whole target (not only in this overlap) + so this line is to normalize the qs in quey to high coverage region + **/ + //(rt->s - p->ts) is the offset + qs = p->ts > rt->s? Get_qs(*p): Get_qs(*p) + (rt->s - p->ts); + //(p->te - rt->e) is the offset + qe = p->te < rt->e? p->qe : p->qe - (p->te - rt->e); + //(rq->s - Get_qs(*p) is the offset + ts = Get_qs(*p) > rq->s? p->ts : p->ts + (rq->s - Get_qs(*p)); + //(p->qe - rq->e) is the offset + te = p->qe < rq->e? p->te : p->te - (p->qe - rq->e); + } + + + + //cut by self coverage + //and normalize the qs, qe, ts, te by rq->s and rt->e + qs = (qs > rq->s? qs : rq->s) - rq->s; + qe = (qe < rq->e? qe : rq->e) - rq->s; + ts = (ts > rt->s? ts : rt->s) - rt->s; + te = (te < rt->e? te : rt->e) - rt->s; + + if (qe - qs >= mini_overlap_length && te - ts >= mini_overlap_length) + { + ///p->qns = p->qns>>32<<32 | qs; + p->qns = p->qns>>32; + p->qns = p->qns << 32; + p->qns = p->qns | qs; + + p->qe = qe; + p->ts = ts; + p->te = te; + sources[i].buffer[m] = *p; + ///fprintf(stderr, "p->del: %d\n", p->del); + m++; + } + } + sources[i].length = m; + if(sources[i].length == 0) + { + (*coverage_cut)[i].del = 1; + } + } + + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); +} + + + +void debug_normalize_ma_hit_t(ma_hit_t_alloc* sources, long long num_sources) +{ + long long i, j, index; + uint32_t qn, tn; + long long total_overlaps = 0; + long long total_reads = 0; + + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + if(Get_qn(sources[i].buffer[j]) != i) + { + fprintf(stderr, "1 error 2\n"); + } + total_overlaps += sources[i].length; + + } + + if(sources[i].length != 0) + { + total_reads++; + } + } + + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + qn = sources[i].buffer[j].qns>>32; + tn = sources[i].buffer[j].tn; + + long long k; + for (k = 0; k < sources[i].length; k++) + { + //here can be improved, since ma_hit_t_alloc has been sorted by tn + if(sources[i].buffer[k].tn == tn + && + ((uint32_t)(sources[i].buffer[k].qns>>32)) == qn) + { + if(k != j) + { + fprintf(stderr, "2 ERROR\n"); + } + } + } + } + } + + + + + + + + + + + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + + + + if(sources[i].buffer[j].bl != Get_qe(sources[i].buffer[j]) - Get_qs(sources[i].buffer[j])) + { + fprintf(stderr, "3 error2, bl: %d, qs: %d, qe: %d\n", + sources[i].buffer[j].bl, Get_qs(sources[i].buffer[j]), + Get_qe(sources[i].buffer[j])); + } + + + qn = sources[i].buffer[j].qns>>32; + tn = sources[i].buffer[j].tn; + + index = get_specific_overlap(&(sources[tn]), tn, qn); + + + if(index == -1) + { + fprintf(stderr, "4 error3\n"); + } + else + { + + if(sources[i].buffer[j].rev != sources[tn].buffer[index].rev) + { + fprintf(stderr, "5 hahaha\n"); + } + + if(sources[i].buffer[j].el != sources[tn].buffer[index].el) + { + fprintf(stderr, "el hahaha\n"); + } + + if(Get_qn(sources[i].buffer[j]) != Get_tn(sources[tn].buffer[index])) + { + fprintf(stderr, "6 error4\n"); + } + + if(Get_tn(sources[i].buffer[j]) != Get_qn(sources[tn].buffer[index])) + { + fprintf(stderr, "7 error5\n"); + } + + if(Get_ts(sources[i].buffer[j]) != Get_qs(sources[tn].buffer[index])) + { + fprintf(stderr, "error6, Get_ts(%d, %d)=%d, Get_qs(%d, %d)=%d, rev: %d\n", + i, j, Get_ts(sources[i].buffer[j]), + tn, index, Get_qs(sources[tn].buffer[index]), sources[i].buffer[j].rev); + } + + if(Get_te(sources[i].buffer[j]) != Get_qe(sources[tn].buffer[index])) + { + fprintf(stderr, "error7, Get_te(%d, %d)=%d, Get_qe(%d, %d)=%d, rev: %d\n", + i, j, Get_te(sources[i].buffer[j]), + tn, index, Get_qe(sources[tn].buffer[index]), + sources[i].buffer[j].rev); + } + + + + if(sources[i].buffer[j].ml != sources[tn].buffer[index].ml) + { + fprintf(stderr, "9 hahaha\n"); + } + + } + + } + } + + fprintf(stderr, "total_reads:%d, total_overlaps: %d\n", total_reads, total_overlaps); +} + + + +void debug_cut_ma_hit_t(ma_hit_t_alloc* sources, long long num_sources, ma_sub_t* coverage_cut) +{ + long long i, j, index; + uint32_t qn, tn; + long long total_overlaps = 0; + long long total_reads = 0; + + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + if(Get_qn(sources[i].buffer[j]) != i) + { + fprintf(stderr, "error 2\n"); + } + total_overlaps += sources[i].length; + + } + + if(sources[i].length != 0) + { + total_reads++; + } + + if((sources[i].length == 0 && coverage_cut[i].del == 1) + || + (sources[i].length != 0 && coverage_cut[i].del == 0)) + { + ; + } + else + { + fprintf(stderr, "i: %d, sources[i].length: %d, coverage_cut[i].del: %d\n", i, sources[i].length, coverage_cut[i].del); + } + + } + + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + qn = sources[i].buffer[j].qns>>32; + tn = sources[i].buffer[j].tn; + + long long k; + for (k = 0; k < sources[i].length; k++) + { + //here can be improved, since ma_hit_t_alloc has been sorted by tn + if(sources[i].buffer[k].tn == tn + && + ((uint32_t)(sources[i].buffer[k].qns>>32)) == qn) + { + if(k != j) + { + fprintf(stderr, "ERROR\n"); + } + } + } + } + } + + + + + + + + + + + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + /** + if(sources[i].buffer[j].bl != sources[i].buffer[j].ml) + { + fprintf(stderr, "error1\n"); + } + **/ + + qn = sources[i].buffer[j].qns>>32; + tn = sources[i].buffer[j].tn; + + index = get_specific_overlap(&(sources[tn]), tn, qn); + + + if(index == -1) + { + fprintf(stderr, "error3\n"); + } + else + { + if(sources[i].buffer[j].rev != sources[tn].buffer[index].rev) + { + fprintf(stderr, "hahaha\n"); + } + + if(Get_qn(sources[i].buffer[j]) != Get_tn(sources[tn].buffer[index])) + { + fprintf(stderr, "error4\n"); + } + + if(Get_tn(sources[i].buffer[j]) != Get_qn(sources[tn].buffer[index])) + { + fprintf(stderr, "error5\n"); + } + + if(Get_ts(sources[i].buffer[j]) != Get_qs(sources[tn].buffer[index])) + { + fprintf(stderr, "\nerror6, Get_ts(%d, %d)=%d, Get_qs(%d, %d)=%d, rev: %d\n", + i, j, Get_ts(sources[i].buffer[j]), + tn, index, Get_qs(sources[tn].buffer[index]), + sources[i].buffer[j].rev); + } + + if(Get_te(sources[i].buffer[j]) != Get_qe(sources[tn].buffer[index])) + { + fprintf(stderr, "\nerror7, Get_te(%d, %d)=%d, Get_qe(%d, %d)=%d, rev: %d\n", + i, j, Get_te(sources[i].buffer[j]), + tn, index, Get_qe(sources[tn].buffer[index]), + sources[i].buffer[j].rev); + } + + } + + } + } + + fprintf(stderr, "total_reads:%d, total_overlaps: %d\n", total_reads, total_overlaps); +} + +/********************************** + * Filter short potential unitigs * + **********************************/ +#define ASG_ET_MERGEABLE 0 +#define ASG_ET_TIP 1 +#define ASG_ET_MULTI_OUT 2 +#define ASG_ET_MULTI_NEI 3 + +static inline int asg_is_utg_end(const asg_t *g, uint32_t v, uint64_t *lw) +{ + + /** + .............................. + . w1--------------- . + . w2-------------- . + . w3-------------- .--->asg_arc_a(g, v^1) + . w4------------- . + . w5------------ . + .............................. + v--------------- + .............................. + . w1--------------- . + . w2-------------- . + . w3-------------- .--->asg_arc_a(g, v) + . w4------------- . + . w5------------ . + .............................. + !!!!!note here the graph has already been cleaned by transitive reduction, so idealy: + + ......................... + . w1--------------- .--->asg_arc_a(g, v^1) + ......................... + v--------------- + ......................... + . w5--------------- .--->asg_arc_a(g, v) + ......................... + + **/ + ///v^1 is the another direction of v + uint32_t w, nv, nw, nw0, nv0 = asg_arc_n(g, v^1); + int i, i0 = -1; + asg_arc_t *aw, *av = asg_arc_a(g, v^1); + + ///if this arc has not been deleted + for (i = nv = 0; i < nv0; ++i) + if (!av[i].del) i0 = i, ++nv; + + ///see the example below + if (nv == 0) return ASG_ET_TIP; // tip + + /** + since the graph has already been cleaned by transitive reduction, + w1 and w2 should not be overlapped with each other + that mean v has mutiple in-edges, and each of them is not overlapped with others + ......................... + . w2--------------- .--->asg_arc_a(g, v^1) + . w1--------------- . + ......................... + v--------------- + + **/ + if (nv > 1) return ASG_ET_MULTI_OUT; // multiple outgoing arcs + + + + + /** + * ///until here, nv == 1 + note the graph has already been cleaned by transitive reduction, + ......................... + . w1--------------- .--->asg_arc_a(g, v^1) + ......................... + v--------------- + **/ + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (based on query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (based on target) + p->ol: overlap length + **/ + ///until here, nv == 1 + if (lw) *lw = av[i0].ul<<32 | av[i0].v; + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qns direction of overlap length of this node (not overlap length) + (based on query) + p->v : |___________31___________|__________1___________| + tns reverse direction of overlap + (based on target) + p->ol: overlap length + **/ + w = av[i0].v ^ 1; + nw0 = asg_arc_n(g, w); + aw = asg_arc_a(g, w); + for (i = nw = 0; i < nw0; ++i) + if (!aw[i].del) ++nw; + + + /** + note nw is at least 1, since we have v + nw > 1 means + ......................... + . av[i0].v^1---------- .--->asg_arc_a(g, v^1) + ......................... + v--------------- + w--------------- + z--------------- + asg_arc_a(av[i0].v^1) is the (v, w, z), and v, w, z are not overlapped with each others + **/ + if (nw != 1) return ASG_ET_MULTI_NEI; + + /** + * nw == 1 means + note the graph has already been cleaned by transitive reduction, + ......................... + . w1--------------- .--->asg_arc_a(g, v^1) + ......................... + v--------------- + ......................... + . w5--------------- .--->asg_arc_a(g, v) + ......................... + + **/ + return ASG_ET_MERGEABLE; +} + + +int asg_extend(const asg_t *g, uint32_t v, int max_ext, asg64_v *a) +{ + int ret; + uint64_t lw; + a->n = 0; + kv_push(uint64_t, *a, v); + do { + /** + note that here the graph has been cleanned by transitive reduction + the following first line is to find the prefix of v^1: + (v^1)--->()---->()---->()----->.... + that is the suffix of v: + ...>(v) + **/ + ret = asg_is_utg_end(g, v^1, &lw); + /** + #define ASG_ET_MERGEABLE 0 + #define ASG_ET_TIP 1 + #define ASG_ET_MULTI_OUT 2 + #define ASG_ET_MULTI_NEI 3 + **/ + if (ret != 0) break; + kv_push(uint64_t, *a, lw); + /** + ret == 0 means: + v^1 and is the only prefix of (uint32_t)lw, + and (uint32_t)lw is the only prefix of v^1 + **/ + + v = (uint32_t)lw; + } while (--max_ext > 0); + return ret; +} + + +static inline int asg_is_single_edge(const asg_t *g, uint32_t v, uint32_t start_node) +{ + + /** + .............................. + . w1--------------- . + . w2-------------- . + . w3-------------- .--->asg_arc_a(g, v^1) + . w4------------- . + . w5------------ . + .............................. + v--------------- + .............................. + . w1--------------- . + . w2-------------- . + . w3-------------- .--->asg_arc_a(g, v) + . w4------------- . + . w5------------ . + .............................. + !!!!!note here the graph has already been cleaned by transitive reduction, so idealy: + + ......................... + . w1--------------- .--->asg_arc_a(g, v^1) + ......................... + v--------------- + ......................... + . w5--------------- .--->asg_arc_a(g, v) + ......................... + + **/ + ///v^1 is the another direction of v + uint32_t w, nv, nw, nw0, nv0 = asg_arc_n(g, v^1); + int i, i0 = -1; + asg_arc_t *av = asg_arc_a(g, v^1); + + int flag = 0; + ///if this arc has not been deleted + for (i = nv = 0; i < nv0; ++i) + { + ///if (!av[i].del) + { + i0 = i; + ++nv; + if(av[i].v>>1 == start_node) + { + flag = 1; + } + } + } + + if(flag == 0) + { + fprintf(stderr, "****ERROR\n"); + } + + return nv; +} + + +asg_t *ma_sg_gen(const ma_hit_t_alloc* sources, long long n_read, const ma_sub_t *coverage_cut, +int max_hang, int min_ovlp) +{ + double startTime = Get_T(); + size_t i, j; + asg_t *g; + ///just calloc + g = asg_init(); + + ///add seq to graph, seq just save the length of each read + for (i = 0; i < n_read; ++i) + { + ///if a read has been deleted, should we still add them? + asg_seq_set(g, i, coverage_cut[i].e - coverage_cut[i].s, coverage_cut[i].del); + } + + g->seq_vis = (uint8_t*)calloc(g->n_seq*2, sizeof(uint8_t)); + + for (i = 0; i < n_read; ++i) + { + for (j = 0; j < sources[i].length; j++) + { + int r; + asg_arc_t t, *p; + const ma_hit_t *h = &(sources[i].buffer[j]); + //high coverage region [sub[qn].e, sub[qn].s) in query + int ql = coverage_cut[Get_qn(*h)].e - coverage_cut[Get_qn(*h)].s; + //high coverage region [sub[qn].e, sub[qn].s) in target + int tl = coverage_cut[Get_tn(*h)].e - coverage_cut[Get_tn(*h)].s; + r = ma_hit2arc(h, ql, tl, max_hang, MAX_HANG_PRE, min_ovlp, &t); + /** + #define MA_HT_INT (-1) + #define MA_HT_QCONT (-2) + #define MA_HT_TCONT (-3) + #define MA_HT_SHORT_OVLP (-4) + the short overlaps and the overlaps with contain reads have already been removed + here we should have overhang + so r should always >= 0 + **/ + if (r >= 0) + { + ///push node? + p = asg_arc_pushp(g); + *p = t; + } + else + { + fprintf(stderr, "error\n"); + } + } + } + + asg_cleanup(g); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return g; +} + + +// pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well +//note!!!!!!!! here we don't exculde the deleted edges +static uint64_t asg_bub_finder_with_del(asg_t *g, uint32_t v0, int max_dist, buf_t *b, +uint32_t cut_in_node) +{ + uint32_t i, n_pending = 0; + uint64_t n_pop = 0; + ///if this node has been deleted + if (g->seq[v0>>1].del) return 0; // already deleted + ///asg_arc_n(n0) + if ((uint32_t)g->idx[v0] < 2) return 0; // no bubbles + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[v0].c = b->a[v0].d = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, v0); + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), d = b->a[v].d, c = b->a[v].c; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + ///why we have this assert? + /****************************may have bugs********************************/ + ///assert(nv > 0); + /****************************may have bugs********************************/ + + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l + binfo_t *t = &b->a[w]; + ///that means there is a circle, directly terminate the whole bubble poping + if (w == v0) + { + //fprintf(stderr, "n_pop error1\n"); + goto pop_reset; + } + + ///if this edge has been deleted + /****************************may have bugs********************************/ + ///if (av[i].del) continue; + /****************************may have bugs********************************/ + + ///push the edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + + ///find a too far path? directly terminate the whole bubble poping + if (d + l > max_dist) + { + //fprintf(stderr, "n_pop error2\n"); + break; // too far + } + + + + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p means the in-node of w is v + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l; + ///incoming edges of w + t->r = count_out_with_del(g, w^1); + if((w>>1) == cut_in_node) + { + t->r--; + } + ++n_pending; + } else { // visited before + ///c seems the max weight of node + if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; + if (c + 1 > t->c) t->c = c + 1; + ///update len(v0->w) + if (d + l < t->d) t->d = d + l; // update dist + } + /****************************may have bugs********************************/ + ///assert(t->r > 0); + /****************************may have bugs********************************/ + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = asg_arc_n(g, w); + if (x) kv_push(uint32_t, b->S, w); + else kv_push(uint32_t, b->T, w); // a tip + --n_pending; + } + } + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) + { + ///fprintf(stderr, "n_pop error3\n"); + goto pop_reset; + } + + } while (b->S.n > 1 || n_pending); + ///asg_bub_backtrack(g, v0, b); + ///n_pop = 1 | (uint64_t)b->T.n<<32; + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_t *t = &b->a[b->b.a[i]]; + t->s = t->c = t->d = 0; + } + ///fprintf(stderr, "n_pop: %d\n", n_pop); + return n_pop; +} + + +int asg_arc_del_triangular(asg_t *g, long long max_dist) +{ + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0; + buf_t b; + if (!g->is_symm) asg_symm(g); + memset(&b, 0, sizeof(buf_t)); + ///set information for each node + b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + int flag0, flag1, node; + for (v = 0; v < n_vtx; ++v) + { + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if (g->seq[v>>1].del) + { + continue; + } + + if(nv != 2) + { + continue; + } + + /**********************test first node************************/ + flag0 = asg_is_single_edge(g, av[0].v, v>>1); + flag1 = asg_is_single_edge(g, av[1].v, v>>1); + if(flag0 == flag1) + { + continue; + } + if(flag0 < 1 || flag0 > 2) + { + continue; + } + if(flag1 < 1 || flag1 > 2) + { + continue; + } + if(flag0 == 2) + { + node = 0; + } + else if(flag1 == 2) + { + node = 1; + } + /**********************test first node************************/ + + + + /**********************test second node************************/ + w = av[node].v^1; + asg_arc_t *aw = asg_arc_a(g, w); + uint32_t nw = asg_arc_n(g, w); + flag0 = asg_is_single_edge(g, aw[0].v, w>>1); + flag1 = asg_is_single_edge(g, aw[1].v, w>>1); + if(flag0 == flag1) + { + continue; + } + if(flag0 < 1 || flag0 > 2) + { + continue; + } + if(flag1 < 1 || flag1 > 2) + { + continue; + } + + + if(flag0 == 2 && (v>>1) != (aw[0].v>>1)) + { + fprintf(stderr, "error 0\n"); + } + + + if(flag1 == 2 && (v>>1) != (aw[1].v>>1)) + { + fprintf(stderr, "error 1\n"); + } + + + + /**********************test second node************************/ + + ///if not a bubble + if(asg_bub_finder_with_del(g, v, max_dist, &b, (w>>1)) == 0) + { + continue; + } + + + av[node].del = 1; + ///remove the reverse direction + asg_arc_del(g, av[node].v^1, av[node].ul>>32^1, 1); + n_reduced++; + + } + + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + + if (n_reduced) { + asg_cleanup(g); + asg_symm(g); + } + + fprintf(stderr, "[M::%s] removed %d short overlaps\n", __func__, n_reduced); +} + + + +// pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well +//note!!!!!!!! here we don't exculde the deleted edges +static uint64_t asg_bub_finder_with_del_advance(asg_t *g, uint32_t v0, int max_dist, buf_t *b) +{ + uint32_t i, n_pending = 0; + uint64_t n_pop = 0; + ///if this node has been deleted + if (g->seq[v0>>1].del) return 0; // already deleted + ///asg_arc_n(n0) + if ((uint32_t)g->idx[v0] < 2) return 0; // no bubbles + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[v0].c = b->a[v0].d = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, v0); + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), d = b->a[v].d, c = b->a[v].c; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + ///why we have this assert? + /****************************may have bugs********************************/ + ///assert(nv > 0); + /****************************may have bugs********************************/ + + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l + binfo_t *t = &b->a[w]; + ///that means there is a circle, directly terminate the whole bubble poping + if (w == v0) + { + //fprintf(stderr, "n_pop error1\n"); + goto pop_reset; + } + + ///if this edge has been deleted + /****************************may have bugs********************************/ + ///if (av[i].del) continue; + /****************************may have bugs********************************/ + + ///push the edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + + ///find a too far path? directly terminate the whole bubble poping + if (d + l > max_dist) + { + //fprintf(stderr, "n_pop error2\n"); + break; // too far + } + + + + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p means the in-node of w is v + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l; + ///incoming edges of w + t->r = count_out_with_del(g, w^1); + ++n_pending; + } else { // visited before + ///c seems the max weight of node + if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; + if (c + 1 > t->c) t->c = c + 1; + ///update len(v0->w) + if (d + l < t->d) t->d = d + l; // update dist + } + /****************************may have bugs********************************/ + ///assert(t->r > 0); + /****************************may have bugs********************************/ + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = asg_arc_n(g, w); + if (x) kv_push(uint32_t, b->S, w); + else kv_push(uint32_t, b->T, w); // a tip + --n_pending; + } + } + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) + { + ///fprintf(stderr, "n_pop error3\n"); + goto pop_reset; + } + + } while (b->S.n > 1 || n_pending); + ///asg_bub_backtrack(g, v0, b); + ///n_pop = 1 | (uint64_t)b->T.n<<32; + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_t *t = &b->a[b->b.a[i]]; + t->s = t->c = t->d = 0; + } + ///fprintf(stderr, "n_pop: %d\n", n_pop); + return n_pop; +} + + +// pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well +//note!!!!!!!! here we don't exculde the deleted edges +static uint64_t asg_bub_finder_without_del_advance(asg_t *g, uint32_t v0, int max_dist, +buf_t *b) +{ + uint32_t i, n_pending = 0; + uint64_t n_pop = 0; + ///if this node has been deleted + if (g->seq[v0>>1].del) return 0; // already deleted + ///asg_arc_n(n0) + if ((uint32_t)g->idx[v0] < 2) return 0; // no bubbles + if(count_out_without_del(g, v0) < 2) return 0; // no bubbles + + + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + ///for each node, b->a saves all related information + b->a[v0].c = b->a[v0].d = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, v0); + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), d = b->a[v].d, c = b->a[v].c; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + ///why we have this assert? + /****************************may have bugs********************************/ + ///assert(nv > 0); + /****************************may have bugs********************************/ + + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l + binfo_t *t = &b->a[w]; + ///that means there is a circle, directly terminate the whole bubble poping + if (w == v0) + { + //fprintf(stderr, "n_pop error1\n"); + goto pop_reset; + } + + ///if this edge has been deleted + /****************************may have bugs********************************/ + if (av[i].del) continue; + /****************************may have bugs********************************/ + + ///push the edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + + ///find a too far path? directly terminate the whole bubble poping + if (d + l > max_dist) + { + //fprintf(stderr, "n_pop error2\n"); + break; // too far + } + + + + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p means the in-node of w is v + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l; + ///incoming edges of w + t->r = count_out_without_del(g, w^1); + ++n_pending; + } else { // visited before + ///c seems the max weight of node + if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; + if (c + 1 > t->c) t->c = c + 1; + ///update len(v0->w) + if (d + l < t->d) t->d = d + l; // update dist + } + /****************************may have bugs********************************/ + ///assert(t->r > 0); + /****************************may have bugs********************************/ + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = asg_arc_n(g, w); + if (x) kv_push(uint32_t, b->S, w); + else kv_push(uint32_t, b->T, w); // a tip + --n_pending; + } + } + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) + { + ///fprintf(stderr, "n_pop error3\n"); + goto pop_reset; + } + + } while (b->S.n > 1 || n_pending); + ///asg_bub_backtrack(g, v0, b); + ///n_pop = 1 | (uint64_t)b->T.n<<32; + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_t *t = &b->a[b->b.a[i]]; + t->s = t->c = t->d = 0; + } + ///fprintf(stderr, "n_pop: %d\n", n_pop); + return n_pop; +} + + +// pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well +//note!!!!!!!! here we don't exculde the deleted edges +static uint64_t asg_bub_end_finder_with_del_advance(asg_t *g, uint32_t* v_Ns, uint32_t occ, +int max_dist, buf_t *b, uint32_t exculde_init, uint32_t exclude_node, uint32_t* sink) +{ + uint32_t i, j, n_pending = 0; + uint64_t n_pop = 0; + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + for (j = 0; j < occ; j++) + { + ///if this node has been deleted + if (g->seq[v_Ns[j]>>1].del) return 0; // already deleted + ///for each node, b->a saves all related information + b->a[v_Ns[j]].c = b->a[v_Ns[j]].d = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, (v_Ns[j]<<1)|exculde_init); + } + + + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), f = v & (uint32_t)1; + v = v >> 1; + uint32_t d = b->a[v].d, c = b->a[v].c; + + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l + binfo_t *t = &b->a[w]; + + for (j = 0; j < occ; j++) + { + if(w == v_Ns[j]) goto pop_reset; + } + + + + if(f && (exclude_node) == (w)) continue; + + ///if (av[i].del) continue; + + ///push the edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + + ///find a too far path? directly terminate the whole bubble poping + if (d + l > max_dist) + { + break; // too far + } + + + + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p means the in-node of w is v + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l; + ///incoming edges of w + t->r = count_out_with_del(g, w^1); + ///t->r = count_out_without_del(g, w^1); + ++n_pending; + } else { // visited before + ///c seems the max weight of node + if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; + if (c + 1 > t->c) t->c = c + 1; + ///update len(v0->w) + if (d + l < t->d) t->d = d + l; // update dist + } + /****************************may have bugs********************************/ + ///assert(t->r > 0); + /****************************may have bugs********************************/ + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = asg_arc_n(g, w); + //if (x) kv_push(uint32_t, b->S, w); + if (x) kv_push(uint32_t, b->S, w<<1); + else kv_push(uint32_t, b->T, w); // a tip + --n_pending; + } + } + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) + { + goto pop_reset; + } + + } while (b->S.n > 1 || n_pending); + + (*sink) = b->S.a[0]>>1; + ///asg_bub_backtrack(g, v0, b); + ///n_pop = 1 | (uint64_t)b->T.n<<32; + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_t *t = &b->a[b->b.a[i]]; + t->s = t->c = t->d = 0; + } + return n_pop; +} + +// pop bubbles from vertex v0; the graph MJUST BE symmetric: if u->v present, v'->u' must be present as well +//note!!!!!!!! here we don't exculde the deleted edges +static uint64_t asg_bub_end_finder_with_del_advance_debug(asg_t *g, uint32_t* v_Ns, uint32_t occ, +int max_dist, buf_t *b, uint32_t exculde_init, uint32_t exclude_node) +{ + uint32_t i, j, n_pending = 0; + uint64_t n_pop = 0; + ///S saves nodes with all incoming edges visited + b->S.n = b->T.n = b->b.n = b->e.n = 0; + for (j = 0; j < occ; j++) + { + ///if this node has been deleted + if (g->seq[v_Ns[j]>>1].del) return 0; // already deleted + ///for each node, b->a saves all related information + b->a[v_Ns[j]].c = b->a[v_Ns[j]].d = 0; + ///b->S is the nodes with all incoming edges visited + kv_push(uint32_t, b->S, (v_Ns[j]<<1)|exculde_init); + + fprintf(stderr, "init: %.*s\n", Get_NAME_LENGTH(R_INF, v_Ns[j]>>1), + Get_NAME(R_INF, v_Ns[j]>>1)); + } + + + + do { + ///v is a node that all incoming edges have been visited + ///d is the distance from v0 to v + uint32_t v = kv_pop(b->S), f = v & (uint32_t)1; + v = v >> 1; + uint32_t d = b->a[v].d, c = b->a[v].c; + + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + + fprintf(stderr, "v: %.*s, nv: %d, f: %d\n", + Get_NAME_LENGTH(R_INF, v>>1), Get_NAME(R_INF, v>>1), nv, f); + + ///all out-edges of v + for (i = 0; i < nv; ++i) { // loop through v's neighbors + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + uint32_t w = av[i].v, l = (uint32_t)av[i].ul; // v->w with length l + binfo_t *t = &b->a[w]; + + fprintf(stderr, "w: %.*s\n", Get_NAME_LENGTH(R_INF, w>>1), + Get_NAME(R_INF, w>>1)); + + if(f && (exclude_node) == (w)) + { + fprintf(stderr, "exclude_node: %.*s\n", Get_NAME_LENGTH(R_INF, exclude_node>>1), + Get_NAME(R_INF, exclude_node>>1)); + continue; + } + + ///if (av[i].del) continue; + + ///push the edge + kv_push(uint32_t, b->e, (g->idx[v]>>32) + i); + + ///find a too far path? directly terminate the whole bubble poping + if (d + l > max_dist) + { + break; // too far + } + + + + if (t->s == 0) { // this vertex has never been visited + kv_push(uint32_t, b->b, w); // save it for revert + ///t->p means the in-node of w is v + ///t->s = 1 means w has been visited + ///d is len(v0->v), l is len(v->w), so t->d is len(v0->w) + t->p = v, t->s = 1, t->d = d + l; + ///incoming edges of w + t->r = count_out_with_del(g, w^1); + ///t->r = count_out_without_del(g, w^1); + ++n_pending; + } else { // visited before + ///c seems the max weight of node + if (c + 1 > t->c || (c + 1 == t->c && d + l > t->d)) t->p = v; + if (c + 1 > t->c) t->c = c + 1; + ///update len(v0->w) + if (d + l < t->d) t->d = d + l; // update dist + } + /****************************may have bugs********************************/ + ///assert(t->r > 0); + /****************************may have bugs********************************/ + //if all incoming edges of w have visited + //push it to b->S + if (--(t->r) == 0) { + uint32_t x = asg_arc_n(g, w); + //if (x) kv_push(uint32_t, b->S, w); + if (x) kv_push(uint32_t, b->S, w<<1); + else kv_push(uint32_t, b->T, w); // a tip + --n_pending; + } + } + ///if i < nv, that means (d + l > max_dist) + if (i < nv || b->S.n == 0) + { + goto pop_reset; + } + + } while (b->S.n > 1 || n_pending); + ///asg_bub_backtrack(g, v0, b); + ///n_pop = 1 | (uint64_t)b->T.n<<32; + n_pop = 1; +pop_reset: + for (i = 0; i < b->b.n; ++i) { // clear the states of visited vertices + binfo_t *t = &b->a[b->b.a[i]]; + t->s = t->c = t->d = 0; + } + return n_pop; +} + + + +int if_node_exist(uint32_t* nodes, uint32_t length, uint32_t query) +{ + uint32_t i; + for (i = 0; i < length; ++i) + { + if((nodes[i]>>1) == query) + { + return 1; + } + } + + return 0; +} + + +int test_triangular(asg_t *g, uint32_t* nodes, uint32_t length, +uint32_t startNode, uint32_t endNode) +{ + + uint32_t i, v, w; + int flag0, flag1, node; + int n_reduced = 0; + for (i = 0; i < length; ++i) + { + v = nodes[i]; + + if((v>>1) == (startNode>>1) || (v>>1) == (endNode>>1)) + { + continue; + } + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 2) + { + continue; + } + /**********************test first node************************/ + flag0 = asg_is_single_edge(g, av[0].v, v>>1); + flag1 = asg_is_single_edge(g, av[1].v, v>>1); + if(flag0 == flag1) + { + continue; + } + if(flag0 < 1 || flag0 > 2) + { + continue; + } + if(flag1 < 1 || flag1 > 2) + { + continue; + } + if(flag0 == 2) + { + node = 0; + } + else if(flag1 == 2) + { + node = 1; + } + /**********************test first node************************/ + + + + /**********************test second node************************/ + w = av[node].v^1; + asg_arc_t *aw = asg_arc_a(g, w); + uint32_t nw = asg_arc_n(g, w); + flag0 = asg_is_single_edge(g, aw[0].v, w>>1); + flag1 = asg_is_single_edge(g, aw[1].v, w>>1); + if(flag0 == flag1) + { + continue; + } + if(flag0 < 1 || flag0 > 2) + { + continue; + } + if(flag1 < 1 || flag1 > 2) + { + continue; + } + + + if(flag0 == 2 && (v>>1) != (aw[0].v>>1)) + { + fprintf(stderr, "error 0\n"); + } + + + if(flag1 == 2 && (v>>1) != (aw[1].v>>1)) + { + fprintf(stderr, "error 1\n"); + } + /**********************test second node************************/ + + if(if_node_exist(nodes, length, (w>>1)) && ((w>>1) != (endNode>>1))) + { + av[node].del = 1; + ///remove the reverse direction + asg_arc_del(g, av[node].v^1, av[node].ul>>32^1, 1); + n_reduced++; + + ////fprintf(stderr, "v>>1: %u, w>>1: %u\n", v>>1, w>>1); + } + + } + + return n_reduced; +} + +long long single_edge_length(asg_t *g, uint32_t begNode, uint32_t endNode, long long edgeLen) +{ + + uint32_t v = begNode; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + long long rLen = 0; + + while (rLen < edgeLen && nv == 1) + { + rLen++; + if((av[0].v>>1) == endNode) + { + return rLen; + } + + if(asg_is_single_edge(g, av[0].v, v>>1) != 1) + { + return -1; + } + + v = av[0].v; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + } + + return -1; + +} + +uint32_t detect_single_path(asg_t *g, uint32_t begNode, uint32_t* endNode, long long* Len, buf_t* b) +{ + + uint32_t v = begNode; + uint32_t nv, rnv; + asg_arc_t *av; + (*Len) = 0; + + // if(begNode == 18001026) + // { + // fprintf(stderr, "(*Len): %d, v: %d, begNode: %d\n", (*Len), v, begNode); + // fflush(stderr); + // } + + // if(begNode == 18011441) + // { + // fprintf(stderr, "(*Len): %d, v: %d, begNode: %d\n", (*Len), v, begNode); + // fflush(stderr); + // } + + while (1) + { + (*Len)++; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + (*endNode) = v; + + if(b) kv_push(uint32_t, b->b, v>>1); + + + // if(begNode == 18001026) + // { + // fprintf(stderr, "!!!!!!!!!!!!(*Len): %d, v: %d, begNode: %d, nv: %d, nvr: %d\n", + // (*Len), v, begNode, nv, asg_arc_n(g, v^1)); + // fflush(stderr); + // } + + if(nv == 0) + { + return END_TIPS; + } + + if(nv == 2) + { + return TWO_OUTPUT; + } + + if(nv > 2) + { + return MUL_OUTPUT; + } + + ///up to here, nv=1 + ///rnv must >= 1 + rnv = asg_is_single_edge(g, av[0].v, v>>1); + v = av[0].v; + (*endNode) = v; + if(rnv == 2) + { + (*Len)++; + if(b) kv_push(uint32_t, b->b, v>>1); + return TWO_INPUT; + } + + if(rnv > 2) + { + (*Len)++; + if(b) kv_push(uint32_t, b->b, v>>1); + return MUL_INPUT; + } + + + if((v>>1) == (begNode>>1)) + { + return LOOP; + } + } + + return LONG_TIPS; +} + +int detect_bubble_end(asg_t *g, uint32_t begNode1, uint32_t begNode2, uint32_t* endNode, +long long* minLen, buf_t* b) +{ + uint32_t e1, e2; + long long l1, l2; + + if(detect_single_path(g, begNode1, &e1, &l1, b) == TWO_INPUT + && + detect_single_path(g, begNode2, &e2, &l2, b) == TWO_INPUT) + { + if(e1 == e2) + { + (*endNode) = e1; + (*minLen) = (l1 <= l2)? l1: l2; + return 1; + } + } + + return 0; +} + + +int detect_simple_bubble(asg_t *g, uint32_t begNode, uint32_t* endNode, long long* minLen, buf_t* b) +{ + uint32_t e1, e2; + long long l1, l2; + + if(asg_arc_n(g, begNode) != 2) + { + return 0; + } + + if(asg_is_single_edge(g, asg_arc_a(g, begNode)[0].v, begNode>>1)!=1 + || + asg_is_single_edge(g, asg_arc_a(g, begNode)[1].v, begNode>>1)!=1) + { + return 0; + } + + if(b) kv_push(uint32_t, b->b, begNode>>1); + + // if(begNode == 17939188) + // { + // fprintf(stderr, "#begNode: %d, nv: %d\n", begNode, asg_arc_n(g, begNode)); + // fprintf(stderr, "#asg_arc_a(g, begNode)[0].v: %d\n", asg_arc_a(g, begNode)[0].v); + // fprintf(stderr, "#asg_arc_a(g, begNode)[1].v: %d\n", asg_arc_a(g, begNode)[1].v); + // fflush(stderr); + // } + + + if(detect_single_path(g, asg_arc_a(g, begNode)[0].v, &e1, &l1, b) == TWO_INPUT + && + detect_single_path(g, asg_arc_a(g, begNode)[1].v, &e2, &l2, b) == TWO_INPUT) + { + if(e1 == e2) + { + (*endNode) = e1; + (*minLen) = (l1 <= l2)? l1: l2; + (*minLen)++; + return 1; + } + } + + return 0; +} + + +uint32_t detect_single_path_with_single_bubbles(asg_t *g, uint32_t begNode, uint32_t* endNode, +long long* Len, buf_t* b, uint32_t max_ext) +{ + + uint32_t v = begNode; + uint32_t nv, rnv; + asg_arc_t *av; + long long bLen; + long long pre_b_n; + (*Len) = 0; + + + // if(begNode == 18046652) + // { + // fprintf(stderr, "inner begNode: %d, vn: %d\n", begNode, asg_arc_n(g, begNode)); + // fflush(stderr); + // } + + // if(begNode == 18014006) + // { + // fprintf(stderr, "inner begNode: %d, vn: %d\n", begNode, asg_arc_n(g, begNode)); + // fflush(stderr); + // } + + + while (1) + { + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + (*endNode) = v; + (*Len)++; + + // if(begNode == 18046652) + // { + // fprintf(stderr, "(*Len): %d, v: %d, begNode: %d\n", (*Len), v, begNode); + // fflush(stderr); + // } + + // if(begNode == 18014006) + // { + // fprintf(stderr, "(*Len): %d, v: %d, begNode: %d\n", (*Len), v, begNode); + // fflush(stderr); + // } + + if((*Len) > max_ext) + { + return LONG_TIPS_UNDER_MAX_EXT; + } + + + if(b) kv_push(uint32_t, b->b, v>>1); + /** + if(b && b->b.n > 1000000) + { + fprintf(stderr, "begNode>>1: %u, v>>1: %u, b->b.n: %u\n", begNode>>1, v>>1, b->b.n); + } + **/ + + + if(nv == 0) + { + return END_TIPS; + } + + if(nv == 2) + { + + // if(begNode == 18014006) + // { + // fprintf(stderr, "bubble (*Len): %d, v: %d, begNode: %d\n", (*Len), v, begNode); + // fflush(stderr); + // } + + if(b) pre_b_n = b->b.n; + if(!detect_simple_bubble(g, v, &v, &bLen, b)) + { + if(b) b->b.n = pre_b_n; + return TWO_OUTPUT; + } + + + // if(begNode == 18014006) + // { + // fprintf(stderr, "bubble (*Len): %d, v: %d, begNode: %d\n", (*Len), v, begNode); + // fflush(stderr); + // } + + + + (*Len) = (*Len) + bLen - 2; + continue; + } + + if(nv > 2) + { + return MUL_OUTPUT; + } + + ///up to here, nv=1 + ///rnv must >= 1 + rnv = asg_is_single_edge(g, av[0].v, v>>1); + v = av[0].v; + (*endNode) = v; + if(rnv == 2) + { + if(b) kv_push(uint32_t, b->b, v>>1); + (*Len)++; + return TWO_INPUT; + } + + if(rnv > 2) + { + if(b) kv_push(uint32_t, b->b, v>>1); + (*Len)++; + return MUL_INPUT; + } + + if((v>>1) == (begNode>>1)) + { + return LOOP; + } + } + + return LONG_TIPS; +} + +int detect_bubble_end_with_bubbles(asg_t *g, uint32_t begNode1, uint32_t begNode2, +uint32_t* endNode, long long* minLen, buf_t* b) +{ + uint32_t e1, e2; + long long l1, l2; + + if(detect_single_path_with_single_bubbles(g, begNode1, &e1, &l1, b, (uint32_t)-1) == TWO_INPUT + && + detect_single_path_with_single_bubbles(g, begNode2, &e2, &l2, b, (uint32_t)-1) == TWO_INPUT) + { + if(e1 == e2) + { + (*endNode) = e1; + (*minLen) = (l1 <= l2)? l1: l2; + return 1; + } + } + + return 0; +} + + +int detect_mul_bubble_end_with_bubbles(asg_t *g, uint32_t* begs, uint32_t occ, +uint32_t* endNode, long long* minLen, buf_t* b) +{ + uint32_t e, flag, e_s; + long long l, i, l_s; + + if(occ < 1) return 0; + + flag = detect_single_path_with_single_bubbles(g, begs[0], &e, &l, b, (uint32_t)-1); + + if(flag == TWO_INPUT || flag == MUL_INPUT) + { + e_s = e; + l_s = l; + } + else + { + return 0; + } + + + + for (i = 1; i < occ; i++) + { + flag = detect_single_path_with_single_bubbles(g, begs[i], &e, &l, b, (uint32_t)-1); + if(flag == TWO_INPUT || flag == MUL_INPUT) + { + if(e != e_s) return 0; + if(l < l_s) l_s = l; + } + else + { + return 0; + } + } + + if(asg_arc_n(g, e_s^1) == occ) + { + (*endNode) = e_s; + (*minLen) = l_s; + return 1; + } + + return 0; +} + + +int detect_bubble_with_bubbles(asg_t *g, uint32_t begNode, uint32_t* endNode, long long* minLen, +buf_t* b, uint32_t max_ext) +{ + uint32_t e1, e2; + long long l1, l2; + + if(asg_arc_n(g, begNode) != 2) + { + return 0; + } + + if(asg_is_single_edge(g, asg_arc_a(g, begNode)[0].v, begNode>>1)!=1 + || + asg_is_single_edge(g, asg_arc_a(g, begNode)[1].v, begNode>>1)!=1) + { + return 0; + } + + if(b) kv_push(uint32_t, b->b, begNode>>1); + + // if(begNode == 18021291) + // { + // fprintf(stderr, "begNode: %d, vn: %d\n", begNode, asg_arc_n(g, begNode)); + // fprintf(stderr, "asg_arc_a(g, begNode)[0].v: %d\n", asg_arc_a(g, begNode)[0].v); + // fprintf(stderr, "asg_arc_a(g, begNode)[1].v: %d\n", asg_arc_a(g, begNode)[1].v); + // fflush(stderr); + // } + + + if(detect_single_path_with_single_bubbles(g, asg_arc_a(g, begNode)[0].v, &e1, &l1, b, max_ext) == TWO_INPUT + && + detect_single_path_with_single_bubbles(g, asg_arc_a(g, begNode)[1].v, &e2, &l2, b, max_ext) == TWO_INPUT) + { + if(e1 == e2) + { + (*endNode) = e1; + (*minLen) = (l1 <= l2)? l1: l2; + (*minLen)++; + return 1; + } + } + + return 0; +} + +int test_triangular_exact(asg_t *g, uint32_t* nodes, uint32_t length, +uint32_t startNode, uint32_t endNode, int max_dist, buf_t* bub) +{ + + uint32_t i, v, w; + ///int flag0, flag1, node; + int n_reduced = 0, todel; + long long NodeLen_first[3]; + long long NodeLen_second[3]; + + uint32_t Ns_first[3]; + uint32_t Ns_second[3]; + + + for (i = 0; i < length; ++i) + { + v = nodes[i]; + + if((v>>1) == (startNode>>1) || (v>>1) == (endNode>>1)) + { + continue; + } + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 2) + { + continue; + } + if(av[0].v == av[1].v) + { + continue; + } + /**********************test first node************************/ + NodeLen_first[0] = NodeLen_first[1] = NodeLen_first[2] = -1; + if(asg_is_single_edge(g, av[0].v, v>>1) <= 2 && asg_is_single_edge(g, av[1].v, v>>1) <= 2) + { + NodeLen_first[asg_is_single_edge(g, av[0].v, v>>1)] = 0; + NodeLen_first[asg_is_single_edge(g, av[1].v, v>>1)] = 1; + } + ///one node has one out-edge, another node has two out-edges + if(NodeLen_first[1] == -1 || NodeLen_first[2] == -1) + { + continue; + } + /**********************test first node************************/ + + ///if the potiential edge has already been removed + if(av[NodeLen_first[2]].del == 1) + { + continue; + } + + /**********************test second node************************/ + w = av[NodeLen_first[2]].v^1; + asg_arc_t *aw = asg_arc_a(g, w); + uint32_t nw = asg_arc_n(g, w); + if(nw != 2) + { + fprintf(stderr, "error\n"); + } + NodeLen_second[0] = NodeLen_second[1] = NodeLen_second[2] = -1; + if(asg_is_single_edge(g, aw[0].v, w>>1) <= 2 && asg_is_single_edge(g, aw[1].v, w>>1) <= 2) + { + NodeLen_second[asg_is_single_edge(g, aw[0].v, w>>1)] = 0; + NodeLen_second[asg_is_single_edge(g, aw[1].v, w>>1)] = 1; + } + ///one node has one out-edge, another node has two out-edges + if(NodeLen_second[1] == -1 || NodeLen_second[2] == -1) + { + continue; + } + + /**********************test second node************************/ + + if(if_node_exist(nodes, length, (w>>1)) && ((w>>1) != (endNode>>1))) + { + + //if(av[NodeLen_first[1]].el == 1 && av[NodeLen_first[2]].el == 0 && + //aw[NodeLen_second[1]].el == 1 && aw[NodeLen_second[2]].el == 0) + ///if(av[NodeLen_first[2]].el == 0 && aw[NodeLen_second[2]].el == 0) + // if( + // (av[NodeLen_first[1]].strong == 1 && av[NodeLen_first[2]].strong == 0 && + // aw[NodeLen_second[1]].strong == 1 && aw[NodeLen_second[2]].strong == 0) + // || + // (av[NodeLen_first[1]].el == 1 && av[NodeLen_first[2]].el == 0 && + // aw[NodeLen_second[1]].el == 1 && aw[NodeLen_second[2]].el == 0)) + uint32_t convex1, convex2, f1, f2; + long long l1, l2; + todel = 0; + f1 = detect_bubble_end_with_bubbles(g, av[0].v, av[1].v, &convex1, &l1, NULL); + f2 = detect_bubble_end_with_bubbles(g, aw[0].v, aw[1].v, &convex2, &l2, NULL); + if(f1 && f2) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1)) && + ((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l1 <= min_thres || l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f1) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1))) + { + if(l1 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f2) + { + if(((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + + + if(todel == 0) + { + if(!f1) + { + Ns_first[0] = av[0].v; Ns_first[1] = av[1].v; + f1 = asg_bub_end_finder_with_del_advance(g, Ns_first, 2, max_dist, + bub, 0, (u_int32_t)-1, &convex1); + l1 = min_thres + 10; + } + + if(!f2) + { + Ns_second[0] = aw[0].v; Ns_second[1] = aw[1].v; + f2 = asg_bub_end_finder_with_del_advance(g, Ns_second, 2, max_dist, + bub, 0, (u_int32_t)-1, &convex2); + l2 = min_thres + 10; + } + + + + + + if(f1 && f2) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1)) && + ((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l1 <= min_thres || l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f1) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1))) + { + if(l1 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f2) + { + if(((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + + } + + + if(todel) + { + av[NodeLen_first[2]].del = 1; + ///remove the reverse direction + asg_arc_del(g, av[NodeLen_first[2]].v^1, av[NodeLen_first[2]].ul>>32^1, 1); + n_reduced++; + ///fprintf(stderr, "***rm v>>1: %u, w>>1: %u\n", v>>1, w>>1); + } + + } + + } + + return n_reduced; +} + + +int find_single_link(asg_t *g, uint32_t link_beg, int linkLen, uint32_t* link_end) +{ + uint32_t v, w; + int i; + i = 0; + v = link_beg^1; + uint32_t nv, nw; + asg_arc_t *av; + int edgeLen = 0; + int flag = -1; + + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + if(nv != 1) + { + return 0; + } + v = av[0].v; + + while (edgeLen < linkLen) + { + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + if(nv != 1) + { + return 0; + } + + w = v^1; + nw = asg_arc_n(g, w); + if(nw == 2) + { + (*link_end) = w; + return 1; + } + else if(nw > 2) + { + return 0; + } + + v = av[0].v; + edgeLen++; + } + + + return 0; +} + + +int if_edge_exist(asg_arc_t* edges, uint32_t length, uint32_t query) +{ + uint32_t i; + for (i = 0; i < length; ++i) + { + if((edges[i].v>>1) == query) + { + return 1; + } + } + + return 0; +} + +int test_quadangular_with_addition_node(asg_t *g, uint32_t* nodes, uint32_t length, +uint32_t addition_node_length) +{ + + uint32_t i, v, w; + int flag, occ_v_0, occ_v_1, occ_w_0, occ_w_1; + int n_reduced = 0; + uint32_t v_out2_node, w_out2_node; + uint32_t cut_edge_v, cut_edge_w; + for (i = 0; i < length; ++i) + { + v = nodes[i]; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 1) + { + continue; + } + /**********************test first node************************/ + flag = asg_is_single_edge(g, av[0].v, v>>1); + if(flag != 2) + { + continue; + } + /**********************test first node************************/ + + if(!find_single_link(g, v, addition_node_length, &w)) + { + continue; + } + v = av[0].v^1; + ///up to now, v and w is the node what we want + asg_arc_t *aw = asg_arc_a(g, w); + uint32_t nw = asg_arc_n(g, w); + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + if(nv!=2 || nw != 2) + { + fprintf(stderr, "error\n"); + } + + if(!if_node_exist(nodes, length, (v>>1))) + { + continue; + } + if(!if_node_exist(nodes, length, (w>>1))) + { + continue; + } + /**********************for v************************/ + occ_v_0 = asg_is_single_edge(g, av[0].v, v>>1); + occ_v_1 = asg_is_single_edge(g, av[1].v, v>>1); + if(occ_v_0 == occ_v_1) + { + continue; + } + if(occ_v_0 < 1 || occ_v_0 > 2) + { + continue; + } + if(occ_v_1 < 1 || occ_v_1 > 2) + { + continue; + } + + if(occ_v_0 == 2) + { + v_out2_node = av[0].v^1; + cut_edge_v = 0; + } + else + { + v_out2_node = av[1].v^1; + cut_edge_v = 1; + } + if(!if_node_exist(nodes, length, (v_out2_node>>1))) + { + continue; + } + /**********************for v************************/ + + /**********************for w************************/ + occ_w_0 = asg_is_single_edge(g, aw[0].v, w>>1); + occ_w_1 = asg_is_single_edge(g, aw[1].v, w>>1); + if(occ_w_0 == occ_w_1) + { + continue; + } + if(occ_w_0 < 1 || occ_w_0 > 2) + { + continue; + } + if(occ_w_1 < 1 || occ_w_1 > 2) + { + continue; + } + + if(occ_w_0 == 2) + { + w_out2_node = aw[0].v^1; + cut_edge_w = 0; + } + else + { + w_out2_node = aw[1].v^1; + cut_edge_w = 1; + } + if(!if_node_exist(nodes, length, (w_out2_node>>1))) + { + continue; + } + /**********************for w************************/ + + + if(!if_edge_exist(asg_arc_a(g, w_out2_node), asg_arc_n(g, w_out2_node), (v_out2_node>>1))) + { + continue; + } + + if(!if_edge_exist(asg_arc_a(g, v_out2_node), asg_arc_n(g, v_out2_node), (w_out2_node>>1))) + { + continue; + } + + + av[cut_edge_v].del = 1; + ///remove the reverse direction + asg_arc_del(g, av[cut_edge_v].v^1, av[cut_edge_v].ul>>32^1, 1); + + aw[cut_edge_w].del = 1; + ///remove the reverse direction + asg_arc_del(g, aw[cut_edge_w].v^1, aw[cut_edge_w].ul>>32^1, 1); + + + n_reduced++; + } + + return n_reduced; +} + +int test_triangular_addition_exact(asg_t *g, uint32_t* nodes, uint32_t length, +uint32_t startNode, uint32_t endNode, int max_dist, buf_t* bub) +{ + + uint32_t i, j, v, w; + ///int flag0, flag1, node; + int n_reduced = 0, todel; + uint32_t Nodes1[2]; + uint32_t Nodes2[2]; + + uint32_t Ns_first[2]; + uint32_t Ns_second[2]; + + /** + if(startNode == 203) + { + uint32_t nv; + asg_arc_t *av; + + fprintf(stderr, "start: %d, end:%d, length: %d\n", startNode>>1, endNode>>1, length); + + v = startNode; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + fprintf(stderr, "******start: %d, nv: %d, dir: %d******\n", v>>1, nv, v&1); + for (j = 0; j < nv; j++) + { + fprintf(stderr, "j: %d, w: %d, dir: %d\n", j, av[j].v>>1, av[j].v&1); + } + fprintf(stderr, "*********************************************\n"); + + for (i = 0; i < length; ++i) + { + v = nodes[i]; + + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + fprintf(stderr, "******v: %d, nv: %d, dir: %d******\n", v>>1, nv, v&1); + for (j = 0; j < nv; j++) + { + fprintf(stderr, "j: %d, w: %d, dir: %d\n", j, av[j].v>>1, av[j].v&1); + } + fprintf(stderr, "*********************************************\n"); + } + + + v = endNode; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + fprintf(stderr, "******end: %d, nv: %d, dir: %d******\n", v>>1, nv, v&1); + for (j = 0; j < nv; j++) + { + fprintf(stderr, "j: %d, w: %d, dir: %d\n", j, av[j].v>>1, av[j].v&1); + } + fprintf(stderr, "*********************************************\n"); + } + **/ + + + for (i = 0; i < length; ++i) + { + v = nodes[i]; + /** + if(startNode == 203) + { + fprintf(stderr, "0 v: %d, i: %d\n", v, i); + } + **/ + + if((v>>1) == (startNode>>1) || (v>>1) == (endNode>>1)) + { + continue; + } + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 1) + { + continue; + } + if(asg_is_single_edge(g, av[0].v, v>>1) != 2) + { + continue; + } + + + w = v^1; + asg_arc_t *aw = asg_arc_a(g, w); + uint32_t nw = asg_arc_n(g, w); + if(nw != 1) + { + continue; + } + if(asg_is_single_edge(g, aw[0].v, w>>1) != 2) + { + continue; + } + + if((av[0].v>>1) == (aw[0].v>>1)) + { + continue; + } + + + + Nodes1[0] = av[0].v^1; + Nodes2[0] = aw[0].v^1; + + + for(j = 0; j < 2; j++) + { + if((asg_arc_a(g, Nodes1[0])[j].v>>1)!= (v>>1)) + { + Nodes1[1] = asg_arc_a(g, Nodes1[0])[j].v^1; + } + } + + for(j = 0; j < 2; j++) + { + if((asg_arc_a(g, Nodes2[0])[j].v>>1)!= (v>>1)) + { + Nodes2[1] = asg_arc_a(g, Nodes2[0])[j].v^1; + } + } + + if(asg_arc_n(g, Nodes1[1]) != 1 || asg_arc_n(g, Nodes2[1]) != 1) + { + continue; + } + + if((Nodes1[1]>>1) == (Nodes2[1]>>1)) + { + continue; + } + + + + if(asg_arc_a(g, Nodes1[1])[0].el == 0 || asg_arc_a(g, Nodes2[1])[0].el == 0) + { + continue; + } + + + + /** + if(startNode == 203) + { + fprintf(stderr, "1 v>>1: %d, i: %d\n", v>>1, i); + } + **/ + + + uint32_t convex1, convex2, f1, f2; + long long l1, l2; + todel = 0; + + if(Nodes1[0]^1 == startNode^1 || Nodes1[0]^1 == endNode) + { + continue; + } + if(Nodes2[1]^1 == startNode^1 || Nodes2[1]^1 == endNode) + { + continue; + } + + f1 = detect_bubble_end_with_bubbles(g, Nodes1[0]^1, Nodes2[1]^1, &convex1, &l1, NULL); + + + + + /** + if(startNode == 203) + { + fprintf(stderr, "2 v: %d, i: %d\n", v, i); + } + **/ + + if(Nodes2[0]^1 == startNode^1 || Nodes2[0]^1 == endNode) + { + continue; + } + if(Nodes1[1]^1 == startNode^1 || Nodes1[1]^1 == endNode) + { + continue; + } + + f2 = detect_bubble_end_with_bubbles(g, Nodes2[0]^1, Nodes1[1]^1, &convex2, &l2, NULL); + + + + + /** + if(startNode == 203) + { + fprintf(stderr, "3 v: %d, i: %d\n", v, i); + } + **/ + + if(f1 && f2) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1)) && + ((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l1 <= min_thres || l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f1) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1))) + { + if(l1 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f2) + { + if(((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + + /** + if(startNode == 203) + { + fprintf(stderr, "4 v: %d, i: %d\n", v, i); + } + **/ + + + if(todel == 0) + { + if(!f1) + { + Ns_first[0] = Nodes1[0]^1; Ns_first[1] = Nodes2[1]^1; + f1 = asg_bub_end_finder_with_del_advance(g, Ns_first, 2, max_dist, + bub, 0, (u_int32_t)-1, &convex1); + l1 = min_thres + 10; + } + + if(!f2) + { + Ns_second[0] = Nodes2[0]^1; Ns_second[1] = Nodes1[1]^1; + f2 = asg_bub_end_finder_with_del_advance(g, Ns_second, 2, max_dist, + bub, 0, (u_int32_t)-1, &convex2); + l2 = min_thres + 10; + } + + if(f1 && f2) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1)) && + ((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l1 <= min_thres || l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f1) + { + if(((convex1>>1) == (startNode>>1) || (convex1>>1) == (endNode>>1))) + { + if(l1 <= min_thres) + { + continue; + } + + todel = 1; + } + } + else if(f2) + { + if(((convex2>>1) == (startNode>>1) || (convex2>>1) == (endNode>>1))) + { + if(l2 <= min_thres) + { + continue; + } + + todel = 1; + } + } + + } + + + + + + + + + + + + + + + + + + + + + + + + + + if(todel) + { + if(av[0].el == 0 || aw[0].el == 0) + { + av[0].del = 1; + asg_arc_del(g, av[0].v^1, av[0].ul>>32^1, 1); + + aw[0].del = 1; + asg_arc_del(g, aw[0].v^1, aw[0].ul>>32^1, 1); + + n_reduced++; + } + } + } + + return n_reduced; +} + +int asg_arc_del_triangular_advance(asg_t *g, long long max_dist) +{ + double startTime = Get_T(); + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0, n_reduced_a = 0; + + + if (!g->is_symm) asg_symm(g); + + + buf_t b; + memset(&b, 0, sizeof(buf_t)); + b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + + + buf_t bub; + memset(&bub, 0, sizeof(buf_t)); + bub.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + + + + int flag0, flag1, node; + for (v = 0; v < n_vtx; ++v) + { + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if (g->seq[v>>1].del) + { + continue; + } + + if(nv < 2) + { + continue; + } + + // if(v == 17929187) + // { + // fprintf(stderr, "*********v: %d\n", v); + // fflush(stderr); + // } + + + + ///if this is a bubble + if(asg_bub_finder_with_del_advance(g, v, max_dist, &b) == 1) + { + n_reduced += test_triangular_exact(g, b.b.a, b.b.n, v, b.S.a[0], max_dist, &bub); + + // if(v == 17929187) + // { + // fprintf(stderr, "??????v: %d, b.S.a[0]: %d\n", v, b.S.a[0]); + // fflush(stderr); + // } + + n_reduced_a += test_triangular_addition_exact(g, b.b.a, b.b.n, v, b.S.a[0],max_dist, &bub); + } + + // if(v == 17929187) + // { + // fprintf(stderr, "##########v: %d\n", v); + // fflush(stderr); + // } + + } + + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + free(bub.a); free(bub.S.a); free(bub.T.a); free(bub.b.a); free(bub.e.a); + + if (n_reduced + n_reduced_a) { + asg_cleanup(g); + asg_symm(g); + } + + fprintf(stderr, "[M::%s] removed %d/%d triangular/triangular_a overlaps\n", + __func__, n_reduced, n_reduced_a); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return n_reduced + n_reduced_a; +} + + +int asg_arc_del_triangular_advance_debug(asg_t *g, long long max_dist) +{ + double startTime = Get_T(); + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0, n_reduced_a = 0; + + + if (!g->is_symm) asg_symm(g); + + + buf_t b; + memset(&b, 0, sizeof(buf_t)); + b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + + + buf_t bub; + memset(&bub, 0, sizeof(buf_t)); + bub.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + + fprintf(stderr, "n_vtx: %d\n", n_vtx); + + int flag0, flag1, node; + for (v = 0; v < n_vtx; ++v) + { + + fprintf(stderr, "0 v: %d\n", v); + fflush(stderr); + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if (g->seq[v>>1].del) + { + continue; + } + + if(nv < 2) + { + continue; + } + + // if(v == 17929187) + // { + // fprintf(stderr, "*********v: %d\n", v); + // fflush(stderr); + // } + + fprintf(stderr, "1 v: %d\n", v); + fflush(stderr); + + ///if this is a bubble + if(asg_bub_finder_with_del_advance(g, v, max_dist, &b) == 1) + { + fprintf(stderr, "2 v: %d\n", v); + fflush(stderr); + + n_reduced += test_triangular_exact(g, b.b.a, b.b.n, v, b.S.a[0], max_dist, &bub); + + fprintf(stderr, "3 v: %d\n", v); + fflush(stderr); + + if(v == 203) + { + fprintf(stderr, "??????v: %d, b.S.a[0]: %d, b.b.n: %d\n", v, b.S.a[0], b.b.n); + fflush(stderr); + } + + n_reduced_a += test_triangular_addition_exact(g, b.b.a, b.b.n, v, b.S.a[0],max_dist, &bub); + + fprintf(stderr, "5 v: %d\n", v); + fflush(stderr); + + } + + fprintf(stderr, "5 v: %d\n", v); + fflush(stderr); + + // if(v == 17929187) + // { + // fprintf(stderr, "##########v: %d\n", v); + // fflush(stderr); + // } + + } + + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + free(bub.a); free(bub.S.a); free(bub.T.a); free(bub.b.a); free(bub.e.a); + + if (n_reduced + n_reduced_a) { + asg_cleanup(g); + asg_symm(g); + } + + fprintf(stderr, "[M::%s] removed %d/%d triangular/triangular_a overlaps\n", + __func__, n_reduced, n_reduced_a); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return n_reduced + n_reduced_a; +} + + +int check_if_cross(asg_t *g, uint32_t v) +{ + uint32_t N_list[5]; + if (g->seq[v>>1].del) return 0; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 2) return 0; + if(asg_is_single_edge(g, av[0].v, v>>1) != 2 || asg_is_single_edge(g, av[1].v, v>>1) != 2) + { + return 0; + } + if(av[0].v == av[1].v) + { + return 0; + } + N_list[0] = v; + N_list[1] = av[0].v^1; + N_list[2] = av[1].v^1; + if(asg_arc_n(g, N_list[0]) != 2 || + asg_arc_n(g, N_list[1]) != 2 || + asg_arc_n(g, N_list[2]) != 2 ) + { + return 0; + } + + if(asg_arc_a(g, N_list[1])[0].v == asg_arc_a(g, N_list[1])[1].v) + { + return 0; + } + + if(asg_arc_a(g, N_list[2])[0].v == asg_arc_a(g, N_list[2])[1].v) + { + return 0; + } + + if(asg_arc_a(g, N_list[1])[0].v == (N_list[0]^1)) + { + N_list[3] = asg_arc_a(g, N_list[1])[1].v^1; + } + else if(asg_arc_a(g, N_list[1])[1].v == (N_list[0]^1)) + { + N_list[3] = asg_arc_a(g, N_list[1])[0].v^1; + } + + if(asg_arc_a(g, N_list[2])[0].v == (N_list[0]^1)) + { + N_list[4] = asg_arc_a(g, N_list[2])[1].v^1; + } + else if(asg_arc_a(g, N_list[2])[1].v == (N_list[0]^1)) + { + N_list[4] = asg_arc_a(g, N_list[2])[0].v^1; + } + + if(N_list[3] != N_list[4]) + { + return 0; + } + + if(asg_arc_n(g, N_list[0]) != 2 || + asg_arc_n(g, N_list[1]) != 2 || + asg_arc_n(g, N_list[2]) != 2 || + asg_arc_n(g, N_list[3]) != 2) + { + return 0; + } + + + uint32_t convex1, convex2, f1, f2; + long long l1, l2; + l1 = l2 = 0; + int todel = 0; + + f1 = detect_bubble_end_with_bubbles(g, N_list[0]^1, N_list[3]^1, &convex1, &l1, NULL); + f2 = detect_bubble_end_with_bubbles(g, N_list[1]^1, N_list[2]^1, &convex2, &l2, NULL); + + if(f1 && f2) + { + if(l1 > min_thres && l2 > min_thres) + { + todel = 1; + } + } + else if(f1) + { + if(l1 > min_thres) + { + todel = 1; + } + } + else if(f2) + { + if(l2 > min_thres) + { + todel = 1; + } + } + + + return todel; +} + +int asg_arc_identify_simple_bubbles(asg_t *g) +{ + double startTime = Get_T(); + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0; + buf_t b; + memset(&b, 0, sizeof(buf_t)); + memset(g->seq_vis, 0, g->n_seq*2*sizeof(uint8_t)); + long long l, i; + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v>>1].del) continue; + + b.b.n = 0; + + if(g->seq_vis[v] != 1) + { + ///if(detect_bubble_with_bubbles(g, v, &w, &l, &b, (uint32_t)-1)) + if(detect_bubble_with_bubbles(g, v, &w, &l, &b, SMALL_BUBBLE_SIZE)) + { + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i] != (v>>1) && b.b.a[i] != (w>>1)) + { + g->seq_vis[b.b.a[i]<<1] = 1; + g->seq_vis[(b.b.a[i]<<1) + 1] = 1; + } + } + g->seq_vis[v] = 1; + g->seq_vis[w^1] = 1; + + + // if(asg_arc_n(g, v) != 2 || asg_arc_n(g, w^1) != 2) + // { + // fprintf(stderr, "error\n"); + // } + // if(v>>1 != b.b.a[0] || w>>1 != b.b.a[b.b.n-1]) + // { + // fprintf(stderr, "sbsbsbs\n"); + // } + + } + } + + + if(check_if_cross(g, v)) + { + g->seq_vis[v] = 2; + } + } + free(b.b.a); + + long long nodes, bub_nodes, cross_nodes; + bub_nodes = nodes = cross_nodes = 0; + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v>>1].del) continue; + nodes++; + if(g->seq_vis[v] == 1) bub_nodes++; + if(g->seq_vis[v] == 2) cross_nodes++; + } + + + fprintf(stderr, "[M::%s] nodes:%d, bub_nodes: %d, cross_nodes: %d\n", __func__, + nodes, bub_nodes, cross_nodes); + + + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return n_reduced; +} + + +typedef struct { + int threadID; + int thread_num; + int check_cross; + asg_t *g; +} para_for_simple_bub; + +void* asg_arc_identify_simple_bubbles_pthread(void* arg) +{ + int thr_ID = ((para_for_simple_bub*)arg)->threadID; + int thr_num = ((para_for_simple_bub*)arg)->thread_num; + asg_t *g = ((para_for_simple_bub*)arg)->g; + int check_cross = ((para_for_simple_bub*)arg)->check_cross; + ///the reason is that each read has two direction (query->target, target->query) + + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0; + buf_t b; + memset(&b, 0, sizeof(buf_t)); + long long l, i; + ///for (v = 0; v < n_vtx; ++v) + for (v = thr_ID; v < n_vtx; v = v + thr_num) + { + if (g->seq[v>>1].del) continue; + + b.b.n = 0; + + ///if(v%10000 == 0) + //if(v >= 18020000) + // if(v == 18021291) + // { + // fprintf(stderr, "0 v: %d, thr_ID: %d\n", v, thr_ID); + // fflush(stderr); + // } + + + if(g->seq_vis[v] != 1) + { + ///if(detect_bubble_with_bubbles(g, v, &w, &l, &b, (uint32_t)-1)) + if(detect_bubble_with_bubbles(g, v, &w, &l, &b, SMALL_BUBBLE_SIZE)) + { + for (i = 0; i < b.b.n; i++) + { + if(b.b.a[i] != (v>>1) && b.b.a[i] != (w>>1)) + { + g->seq_vis[b.b.a[i]<<1] = 1; + g->seq_vis[(b.b.a[i]<<1) + 1] = 1; + } + } + g->seq_vis[v] = 1; + g->seq_vis[w^1] = 1; + } + } + + + ///if(v%10000 == 0) + //if(v >= 18020000) + // if(v == 18021291) + // { + // fprintf(stderr, "1 v: %d, thr_ID: %d\n", v, thr_ID); + // fflush(stderr); + // } + + + if(check_cross == 1 && check_if_cross(g, v)) + { + g->seq_vis[v] = 2; + } + + ///if(v%10000 == 0) + //if(v >= 18020000) + // if(v == 18021291) + // { + // fprintf(stderr, "2 v: %d, thr_ID: %d\n", v, thr_ID); + // fflush(stderr); + // } + } + free(b.b.a); + + free(arg); + + // fprintf(stderr, "thr_ID: %d end\n", thr_ID); + // fflush(stderr); +} + +int asg_arc_identify_simple_bubbles_multi(asg_t *g, int check_cross) +{ + double startTime = Get_T(); + memset(g->seq_vis, 0, g->n_seq*2*sizeof(uint8_t)); + + pthread_t *_r_threads; + + _r_threads = (pthread_t *)malloc(sizeof(pthread_t)*thread_num); + + int i = 0; + + for (i = 0; i < thread_num; i++) + { + para_for_simple_bub* arg = (para_for_simple_bub*)malloc(sizeof(*arg)); + arg->g = g; + arg->thread_num = thread_num; + arg->threadID = i; + arg->check_cross = check_cross; + + pthread_create(_r_threads + i, NULL, asg_arc_identify_simple_bubbles_pthread, (void*)arg); + } + + + for (i = 0; in_seq * 2; + long long nodes, bub_nodes, cross_nodes; + bub_nodes = nodes = cross_nodes = 0; + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v>>1].del) continue; + nodes++; + if(g->seq_vis[v] == 1) bub_nodes++; + if(g->seq_vis[v] == 2) cross_nodes++; + } + + + fprintf(stderr, "[M::%s] nodes:%d, bub_nodes: %d, cross_nodes: %d\n", __func__, + nodes, bub_nodes, cross_nodes); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return bub_nodes+cross_nodes; +} + + + +int check_small_bubble(asg_t *g, uint32_t begNode, uint32_t v, uint32_t w, +long long* vLen, long long* wLen, uint32_t* endNode) +{ + uint32_t nv = asg_arc_n(g, v); + uint32_t nw = asg_arc_n(g, w); + + asg_arc_t *av = asg_arc_a(g, v); + asg_arc_t *aw = asg_arc_a(g, w); + if(nv != 1 || nw != 1) + { + return 0; + } + + ///first node + ///nv must be 1 + if(asg_is_single_edge(g, av[0].v, v>>1) == 2) + { + uint32_t vv; + vv = av[0].v^1; + + if( + asg_is_single_edge(g, asg_arc_a(g, vv)[0].v, vv>>1) == 1 + && + asg_is_single_edge(g, asg_arc_a(g, vv)[1].v, vv>>1) == 1 + ) + { + ///walk along first path + long long pLen1; + pLen1 = single_edge_length(g, asg_arc_a(g, vv)[0].v, begNode>>1, 1000); + + ///walk along first path + long long pLen2; + pLen2 = single_edge_length(g, asg_arc_a(g, vv)[1].v, begNode>>1, 1000); + + + + if(pLen1 >= 0 && pLen2 >= 0) + { + if(((asg_arc_a(g, vv)[0].v) == (v^1)) && pLen1 == 1) + { + (*vLen) = pLen1; + (*wLen) = pLen2; + } + else if(((asg_arc_a(g, vv)[1].v) == (v^1)) && pLen2 == 1) + { + (*vLen) = pLen2; + (*wLen) = pLen1; + } + else + { + fprintf(stderr, "error\n"); + } + ///(*endNode) = vv>>1; + (*endNode) = vv; + return 1; + } + + /** + if(pLen1 != 1 && pLen2 != 1) + { + fprintf(stderr, "v: pLen1: %d, pLen2: %d\n", pLen1, pLen2); + pLen1 = single_edge_length(g, v^1, begNode>>1); + fprintf(stderr, "pLen_v: %d\n", pLen1); + fprintf(stderr, "vv>>1: %u, v>>1: %u, begNode>>1: %u\n", + vv>>1, v>>1, begNode>>1); + } + + if((asg_arc_a(g, vv)[0].v) != (v^1) && (asg_arc_a(g, vv)[1].v) != (v^1)) + { + fprintf(stderr, "vv: %u\n", vv); + } + **/ + + } + } + + + ///second node + ///nw must be 1 + if(asg_is_single_edge(g, aw[0].v, w>>1) == 2) + { + uint32_t ww; + ww = aw[0].v^1; + + if( + asg_is_single_edge(g, asg_arc_a(g, ww)[0].v, ww>>1) == 1 + && + asg_is_single_edge(g, asg_arc_a(g, ww)[1].v, ww>>1) == 1 + ) + { + ///walk along first path + long long pLen1; + pLen1 = single_edge_length(g, asg_arc_a(g, ww)[0].v, begNode>>1, 1000); + + ///walk along first path + long long pLen2; + pLen2 = single_edge_length(g, asg_arc_a(g, ww)[1].v, begNode>>1, 1000); + + if(pLen1 >= 0 && pLen2 >= 0) + { + if(((asg_arc_a(g, ww)[0].v) == (w^1)) && pLen1 == 1) + { + (*wLen) = pLen1; + (*vLen) = pLen2; + } + else if(((asg_arc_a(g, ww)[1].v) == (w^1)) && pLen2 == 1) + { + (*wLen) = pLen2; + (*vLen) = pLen1; + } + else + { + fprintf(stderr, "error\n"); + } + + //(*endNode) = ww>>1; + (*endNode) = ww; + + return 1; + } + + /** + if(pLen1 != 1 && pLen2 != 1) + { + fprintf(stderr, "w: pLen1: %d, pLen2: %d\n", pLen1, pLen2); + pLen1 = single_edge_length(g, w^1, begNode>>1); + fprintf(stderr, "pLen_w: %d\n", pLen1); + } + + if((asg_arc_a(g, ww)[0].v) != (w^1) && (asg_arc_a(g, ww)[1].v) != (w^1)) + { + fprintf(stderr, "ww: %u\n", ww); + } + **/ + } + } + + return 0; + +} + + +int test_single_node_bubble(asg_t *g, uint32_t* nodes, uint32_t length, +uint32_t startNode, uint32_t endNode) +{ + + uint32_t i, v, w; + uint32_t vEnd; + int flag0, flag1, node; + int n_reduced = 0; + long long Len[2], longLen; + long long longLen_thres = 4; + for (i = 0; i < length; ++i) + { + v = nodes[i]; + + if((v>>1) == (startNode>>1) || (v>>1) == (endNode>>1)) + { + continue; + } + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 2) + { + continue; + } + + + flag0 = asg_is_single_edge(g, av[0].v, v>>1); + flag1 = asg_is_single_edge(g, av[1].v, v>>1); + if(flag0 != 1 || flag1 != 1) + { + continue; + } + + if(check_small_bubble(g, v, av[0].v, av[1].v, &(Len[0]), &(Len[1]), &vEnd)) + { + + if(if_node_exist(nodes, length, vEnd>>1) && ((vEnd>>1) != (endNode>>1))) + { + + if(Len[0] == 1 && Len[1] != 1) + { + w = av[0].v; + longLen = Len[1]; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + // fprintf(stderr, "w>>1: %u, beg: %u, end: %u\n", + // w>>1, startNode>>1, endNode>>1); + asg_seq_del(g, w>>1); + n_reduced++; + }///up to here w is exactly overlapped in both directions + else if(longLen >= longLen_thres) + { + if(av[0].el == 1 && av[1].el == 1 + && + asg_arc_a(g, vEnd)[0].el == 1 && asg_arc_a(g, vEnd)[1].el == 1) + { + asg_seq_del(g, w>>1); + n_reduced++; + } + } + + /****************************may have bugs********************************/ + + } + else if(Len[0] != 1 && Len[1] == 1) + { + w = av[1].v; + longLen = Len[0]; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + // fprintf(stderr, "w>>1: %u, beg: %u, end: %u\n", + // w>>1, startNode>>1, endNode>>1); + asg_seq_del(g, w>>1); + n_reduced++; + }///up to here w is exactly overlapped in both directions + else if(longLen >= longLen_thres) + { + if(av[0].el == 1 && av[1].el == 1 + && + asg_arc_a(g, vEnd)[0].el == 1 && asg_arc_a(g, vEnd)[1].el == 1) + { + asg_seq_del(g, w>>1); + n_reduced++; + } + } + /****************************may have bugs********************************/ + } + else if(Len[0] == 1 && Len[1] == 1) + { + w = av[0].v; + flag0 = asg_arc_a(g, w)[0].el + asg_arc_a(g, w^1)[0].el; + w = av[1].v; + flag1 = asg_arc_a(g, w)[0].el + asg_arc_a(g, w^1)[0].el; + ///>=2 means this is an exact overlap + if(flag0 < 2 && flag1 >= 2) + { + w = av[0].v; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + // fprintf(stderr, "w>>1: %u, beg: %u, end: %u\n", + // w>>1, startNode>>1, endNode>>1); + asg_seq_del(g, w>>1); + n_reduced++; + } + /****************************may have bugs********************************/ + } + + if(flag0 >= 2 && flag1 < 2) + { + w = av[1].v; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + // fprintf(stderr, "w>>1: %u, beg: %u, end: %u\n", + // w>>1, startNode>>1, endNode>>1); + asg_seq_del(g, w>>1); + n_reduced++; + } + /****************************may have bugs********************************/ + } + } + else + { + fprintf(stderr, "error\n"); + } + + } + } + + } + + return n_reduced; +} + + +int test_single_node_bubble_directly(asg_t *g, uint32_t v, long long longLen_thres) +{ + + uint32_t w, vEnd; + int flag0, flag1, node; + int n_reduced = 0; + long long Len[2], longLen; + ///long long longLen_thres = 4; + + + + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 2) + { + return 0; + } + + + flag0 = asg_is_single_edge(g, av[0].v, v>>1); + flag1 = asg_is_single_edge(g, av[1].v, v>>1); + if(flag0 != 1 || flag1 != 1) + { + return 0; + } + + if(check_small_bubble(g, v, av[0].v, av[1].v, &(Len[0]), &(Len[1]), &vEnd)) + { + if(Len[0] == 1 && Len[1] != 1) + { + w = av[0].v; + longLen = Len[1]; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + asg_seq_del(g, w>>1); + n_reduced++; + }///up to here w is exactly overlapped in both directions + else if(longLen >= longLen_thres) + { + if(av[0].el == 1 && av[1].el == 1 + && + asg_arc_a(g, vEnd)[0].el == 1 && asg_arc_a(g, vEnd)[1].el == 1) + { + asg_seq_del(g, w>>1); + n_reduced++; + } + } + + /****************************may have bugs********************************/ + + } + else if(Len[0] != 1 && Len[1] == 1) + { + w = av[1].v; + longLen = Len[0]; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + asg_seq_del(g, w>>1); + n_reduced++; + }///up to here w is exactly overlapped in both directions + else if(longLen >= longLen_thres) + { + if(av[0].el == 1 && av[1].el == 1 + && + asg_arc_a(g, vEnd)[0].el == 1 && asg_arc_a(g, vEnd)[1].el == 1) + { + asg_seq_del(g, w>>1); + n_reduced++; + } + } + /****************************may have bugs********************************/ + } + else if(Len[0] == 1 && Len[1] == 1) + { + w = av[0].v; + flag0 = asg_arc_a(g, w)[0].el + asg_arc_a(g, w^1)[0].el; + w = av[1].v; + flag1 = asg_arc_a(g, w)[0].el + asg_arc_a(g, w^1)[0].el; + ///>=2 means this is an exact overlap + if(flag0 < 2 && flag1 >= 2) + { + w = av[0].v; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + asg_seq_del(g, w>>1); + n_reduced++; + } + /****************************may have bugs********************************/ + } + + if(flag0 >= 2 && flag1 < 2) + { + w = av[1].v; + + /****************************may have bugs********************************/ + if(asg_arc_a(g, w)[0].el == 0 || asg_arc_a(g, w^1)[0].el == 0) + { + asg_seq_del(g, w>>1); + n_reduced++; + } + /****************************may have bugs********************************/ + } + } + else + { + fprintf(stderr, "error\n"); + } + } + return n_reduced; +} + + +int asg_arc_del_single_node_bubble(asg_t *g, long long max_dist) +{ + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0; + buf_t b; + if (!g->is_symm) asg_symm(g); + memset(&b, 0, sizeof(buf_t)); + ///set information for each node + b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + int flag0, flag1, node; + for (v = 0; v < n_vtx; ++v) + { + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if (g->seq[v>>1].del) + { + continue; + } + + if(nv < 2) + { + continue; + } + + ///if this is a bubble + if(asg_bub_finder_with_del_advance(g, v, max_dist, &b) == 1) + { + n_reduced += test_single_node_bubble(g, b.b.a, b.b.n, v, b.S.a[0]); + } + + } + + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + + if (n_reduced) { + asg_cleanup(g); + asg_symm(g); + } + + fprintf(stderr, "[M::%s] removed %d short bubbles\n\n", __func__, n_reduced); + + return n_reduced; +} + +int asg_arc_del_single_node_directly(asg_t *g, long long longLen_thres) +{ + double startTime = Get_T(); + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0; + for (v = 0; v < n_vtx; ++v) + { + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if (g->seq[v>>1].del) + { + continue; + } + + if(nv != 2) + { + continue; + } + + n_reduced += test_single_node_bubble_directly(g, v, longLen_thres); + } + + + if (n_reduced) { + asg_cleanup(g); + asg_symm(g); + } + + fprintf(stderr, "[M::%s] removed %d small bubbles\n", __func__, n_reduced); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return n_reduced; +} + + +int asg_arc_del_self_circle_contig(asg_t *g) +{ + double startTime = Get_T(); + uint32_t v; + uint32_t n_vtx = g->n_seq * 2, n_reduced = 0; + uint32_t vEnd; + int flag0, flag1, node; + long long Len[3]; + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v>>1].del) + { + continue; + } + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 2) + { + continue; + } + if(av[0].v == av[1].v) + { + continue; + } + Len[0] = Len[1] = Len[2] = -1; + if(asg_is_single_edge(g, av[0].v, v>>1) <= 2 + && asg_is_single_edge(g, av[1].v, v>>1) <= 2) + { + Len[asg_is_single_edge(g, av[0].v, v>>1)] = 0; + Len[asg_is_single_edge(g, av[1].v, v>>1)] = 1; + } + + if(Len[1] == -1 || Len[2] == -1) + { + continue; + } + + + if(asg_arc_n(g, av[Len[2]].v) == 1 && + single_edge_length(g, av[Len[2]].v, v>>1, 100)!=-1) + { + av[Len[2]].del = 1; + ///remove the reverse direction + asg_arc_del(g, av[Len[2]].v^1, av[Len[2]].ul>>32^1, 1); + n_reduced++; + } + + } + + if (n_reduced) { + asg_cleanup(g); + asg_symm(g); + } + + fprintf(stderr, "[M::%s] removed %d self-circle contig\n", __func__, n_reduced); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return n_reduced; +} + + +int test_cross(asg_t *g, uint32_t* nodes, uint32_t length, +uint32_t startNode, uint32_t endNode) +{ + uint32_t a1, a2; + uint32_t N_list[5]; + uint32_t i, v; + int flag0, flag1; + int n_reduced = 0; + for (i = 0; i < length; ++i) + { + v = nodes[i]; + + if((v>>1) == (startNode>>1) || (v>>1) == (endNode>>1)) + { + continue; + } + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if(nv != 2) + { + continue; + } + if(av[0].v == av[1].v) + { + continue; + } + flag0 = asg_is_single_edge(g, av[0].v, v>>1); + flag1 = asg_is_single_edge(g, av[1].v, v>>1); + + if(flag0 != 2 || flag1 != 2) + { + continue; + } + + + N_list[0] = v; + N_list[1] = av[0].v^1; + N_list[2] = av[1].v^1; + + if(asg_arc_n(g, N_list[0]) != 2 || + asg_arc_n(g, N_list[1]) != 2 || + asg_arc_n(g, N_list[2]) != 2 ) + { + continue; + } + + if(asg_arc_a(g, N_list[1])[0].v == asg_arc_a(g, N_list[1])[1].v) + { + continue; + } + + if(asg_arc_a(g, N_list[2])[0].v == asg_arc_a(g, N_list[2])[1].v) + { + continue; + } + + + if(asg_arc_a(g, N_list[1])[0].v == (N_list[0]^1)) + { + N_list[3] = asg_arc_a(g, N_list[1])[1].v^1; + } + else if(asg_arc_a(g, N_list[1])[1].v == (N_list[0]^1)) + { + N_list[3] = asg_arc_a(g, N_list[1])[0].v^1; + } + else + { + fprintf(stderr, "ERROR\n"); + } + + if(asg_arc_a(g, N_list[2])[0].v == (N_list[0]^1)) + { + N_list[4] = asg_arc_a(g, N_list[2])[1].v^1; + } + else if(asg_arc_a(g, N_list[2])[1].v == (N_list[0]^1)) + { + N_list[4] = asg_arc_a(g, N_list[2])[0].v^1; + } + else + { + fprintf(stderr, "ERROR\n"); + } + + if(N_list[3] != N_list[4]) + { + continue; + } + + if(asg_arc_n(g, N_list[0]) != 2 || + asg_arc_n(g, N_list[1]) != 2 || + asg_arc_n(g, N_list[2]) != 2 || + asg_arc_n(g, N_list[3]) != 2) + { + continue; + } + /** + N_list[3] N_list[0] + + N_list[2] N_list[1] + **/ + if(asg_arc_a(g, N_list[0])[0].el == asg_arc_a(g, N_list[0])[1].el) + { + continue; + } + + if(asg_arc_a(g, N_list[0])[0].el == 1) + { + //a1 = asg_arc_a(g, N_list[0])[0].v >> 1; + a1 = 0; + } + else + { + ///a1 = asg_arc_a(g, N_list[0])[1].v >> 1; + a1 = 1; + } + + + + + + + if(asg_arc_a(g, N_list[3])[0].el == asg_arc_a(g, N_list[3])[1].el) + { + continue; + } + + if(asg_arc_a(g, N_list[3])[0].el == 1) + { + //a2 = asg_arc_a(g, N_list[3])[0].v >> 1; + a2 = 0; + } + else + { + //a2 = asg_arc_a(g, N_list[3])[1].v >> 1; + a2 = 1; + } + + if( + (asg_arc_a(g, N_list[0])[a1].v >> 1) + != + (asg_arc_a(g, N_list[3])[a2].v >> 1) + ) + { + if(((N_list[0]>>1) != (endNode>>1)) && + ((N_list[1]>>1) != (endNode>>1)) && + ((N_list[2]>>1) != (endNode>>1)) && + ((N_list[3]>>1) != (endNode>>1))) + { + asg_arc_a(g, N_list[0])[a1].del = 1; + asg_arc_del(g, asg_arc_a(g, N_list[0])[a1].v^1, + asg_arc_a(g, N_list[0])[a1].ul>>32^1, 1); + + + asg_arc_a(g, N_list[3])[a2].del = 1; + asg_arc_del(g, asg_arc_a(g, N_list[3])[a2].v^1, + asg_arc_a(g, N_list[3])[a2].ul>>32^1, 1); + /** + fprintf(stderr, "(N_list[0]>>1): %u, (N_list[1]>>1): %u, (N_list[2]>>1): %u, (N_list[3]>>1): %u\n", + (N_list[0]>>1), (N_list[1]>>1), (N_list[2]>>1), (N_list[3]>>1)); + fprintf(stderr, "a1: %u\n", asg_arc_a(g, N_list[0])[a1].v>>1); + fprintf(stderr, "a2: %u\n", asg_arc_a(g, N_list[3])[a2].v>>1); + **/ + + n_reduced++; + } + } + } + + return n_reduced; +} + +int asg_arc_del_cross_bubble(asg_t *g, long long max_dist) +{ + double startTime = Get_T(); + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, w, n_vtx = g->n_seq * 2, n_reduced = 0; + buf_t b; + if (!g->is_symm) asg_symm(g); + memset(&b, 0, sizeof(buf_t)); + ///set information for each node + b.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + int flag0, flag1, node; + for (v = 0; v < n_vtx; ++v) + { + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if (g->seq[v>>1].del) + { + continue; + } + + if(nv < 2) + { + continue; + } + + ///if this is a bubble + if(asg_bub_finder_with_del_advance(g, v, max_dist, &b) == 1) + { + n_reduced += test_cross(g, b.b.a, b.b.n, v, b.S.a[0]); + } + + } + + free(b.a); free(b.S.a); free(b.T.a); free(b.b.a); free(b.e.a); + + if (n_reduced) { + asg_cleanup(g); + asg_symm(g); + } + + fprintf(stderr, "[M::%s] removed %d cross\n", __func__, n_reduced); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return n_reduced; +} + + + +// transitive reduction; see Myers, 2005 +int asg_arc_del_trans(asg_t *g, int fuzz) +{ + double startTime = Get_T(); + + uint8_t *mark; + ///n_vtx = number of seq * 2 + ///the reason is that each read has two direction (query->target, target->query) + uint32_t v, n_vtx = g->n_seq * 2, n_reduced = 0; + ///at first, all nodes should be set to vacant + mark = (uint8_t*)calloc(n_vtx, 1); + + /**v is the id+direction of a node, + * the high 32-bit is the id, + * and the lowest 1-bit is the direction + * (0 means query-to-target, 1 means target-to-query)**/ + for (v = 0; v < n_vtx; ++v) { + ///nv is the number of overlaps with v(qn+direction) + uint32_t L, i, nv = asg_arc_n(g, v); + ///av is the array of v + asg_arc_t *av = asg_arc_a(g, v); + ///that means in this direction, read v is not overlapped with any other reads + if (nv == 0) continue; // no hits + + ///if the read itself has been removed + if (g->seq[v>>1].del) + { + for (i = 0; i < nv; ++i) av[i].del = 1, ++n_reduced; + continue; + } + + + + /** + ********************************query-to-target overlap**************************** + case 1: u = 0, rev = 0 in the view of target: direction is 1 + query: CCCCCCCCTAATTAAAAT target: TAATTAAAATGGGGGG (use ex-target as query) + |||||||||| <---> |||||||||| + target: TAATTAAAATGGGGGG query: CCCCCCCCTAATTAAAAT (use ex-query as target) + + case 2: u = 0, rev = 1 in the view of target: direction is 0 + query: CCCCCCCCTAATTAAAAT target: CCCCCCATTTTAATTA (use ex-target as query) + |||||||||| <---> |||||||||| + target: TAATTAAAATGGGGGG query: ATTTTAATTAGGGGGGGG (use ex-query as target) + ********************************query-to-target overlap**************************** + + ********************************target-to-query overlap**************************** + case 3: u = 1, rev = 0 in the view of target: direction is 0 + query: AAATAATATCCCCCCGCG target: GGGCCGGCAAATAATAT (use ex-target as query) + ||||||||| <---> ||||||||| + target: GGGCCGGCAAATAATAT query: AAATAATATCCCCCCGCG (use ex-query as target) + + case 4: u = 1, rev = 1 in the view of target: direction is 1 + query: AAATAATATCCCCCCGCG target: ATATTATTTGCCGGCCC (use ex-target as query) + ||||||||| <---> ||||||||| + target: GGGCCGGCAAATAATAT query: CGCGGGGGATATTATTT (use ex-query as target) + ********************************target-to-query overlap**************************** + + p->ul: |____________31__________|__________1___________|______________32_____________| + qns direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tns reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + + + //all outnode of v should be set to "not reduce" + for (i = 0; i < nv; ++i) mark[av[i].v] = 1; + + ///length of node (not overlap length) + ///av[nv-1] is longest out-dege + /** + * v--------------- + * w1--------------- + * w2-------------- + * w3-------------- + * w4-------------- + * w5------------- + * for v, the longest out-edge is v->w5 + **/ + L = asg_arc_len(av[nv-1]) + fuzz; + + + for (i = 0; i < nv; ++i) { + //w is an out-node of v + uint32_t w = av[i].v; + + uint32_t j, nw = asg_arc_n(g, w); + asg_arc_t *aw = asg_arc_a(g, w); + ///if w has already been reduced + if (mark[av[i].v] != 1) continue; + + for (j = 0; j < nw && asg_arc_len(aw[j]) + asg_arc_len(av[i]) <= L; ++j) + if (mark[aw[j].v]) mark[aw[j].v] = 2; + } + #if 0 + for (i = 0; i < nv; ++i) { + uint32_t w = av[i].v; + uint32_t j, nw = asg_arc_n(g, w); + asg_arc_t *aw = asg_arc_a(g, w); + for (j = 0; j < nw && (j == 0 || asg_arc_len(aw[j]) < fuzz); ++j) + if (mark[aw[j].v]) mark[aw[j].v] = 2; + } + #endif + //remove edges + for (i = 0; i < nv; ++i) { + if (mark[av[i].v] == 2) av[i].del = 1, ++n_reduced; + mark[av[i].v] = 0; + } + } + free(mark); + fprintf(stderr, "[M::%s] transitively reduced %d arcs\n", __func__, n_reduced); + if (n_reduced) { + asg_cleanup(g); + asg_symm(g); + } + + + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_reduced; +} + + + + + +///max_ext is 4 +int asg_cut_tip(asg_t *g, int max_ext) +{ + double startTime = Get_T(); + + asg64_v a = {0,0,0}; + uint32_t n_vtx = g->n_seq * 2, v, i, cnt = 0; + + for (v = 0; v < n_vtx; ++v) { + //if this seq has been deleted + if (g->seq[v>>1].del) continue; + ///check if the another direction of v has no overlaps + ///if the self direction of v has no overlaps, we don't have the overlaps of them + ///here is check if the reverse direction of v + /** + the following first line is to find (means v is a node has no prefix): + (v)--->()---->()---->()----->.... + another case is: + ......()---->()---->()----->()------>(v) + this case can be found by (v^1), so we don't need to process this case here + **/ + if (asg_is_utg_end(g, v, 0) != ASG_ET_TIP) continue; // not a tip + /** + the following second line is: + (v)--->()---->()---->()----->() + |--------max_ext-------| + **/ + ///that means here is a long tip, which is longer than max_ext + if (asg_extend(g, v, max_ext, &a) == ASG_ET_MERGEABLE) continue; // not a short unitig + + /** + * so combining the last two lines, they are designed to reomve(n(0), n(1), n(2)): + * ----->n(4) + * | + * n(0)--->n(1)---->n(2)---->n(3) + * | + * ----->n(5) + **/ + for (i = 0; i < a.n; ++i) + asg_seq_del(g, (uint32_t)a.a[i]>>1); + ++cnt; + } + free(a.a); + if (cnt > 0) asg_cleanup(g); + fprintf(stderr, "[M::%s] cut %d tips\n", __func__, cnt); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + + return cnt; +} + + +///max_ext is 4 +int debug_asg_cut_tip(asg_t *g, int max_ext) +{ + asg64_v a = {0,0,0}; + uint32_t n_vtx = g->n_seq * 2, v, i, cnt = 0; + + for (v = 0; v < n_vtx; ++v) { + //if this seq has been deleted + if (g->seq[v>>1].del) continue; + ///check if the another direction of v has no overlaps + ///if the self direction of v has no overlaps, we don't have the overlaps of them + ///here is check if the reverse direction of v + /** + the following first line is to find (means v is a node has no prefix): + (v)--->()---->()---->()----->.... + another case is: + ......()---->()---->()----->()------>(v) + this case can be found by (v^1), so we don't need to process this case here + **/ + if (asg_is_utg_end(g, v, 0) != ASG_ET_TIP) continue; // not a tip + /** + the following second line is: + (v)--->()---->()---->()----->() + |--------max_ext-------| + **/ + ///that means here is a long tip, which is longer than max_ext + if (asg_extend(g, v, max_ext, &a) == ASG_ET_MERGEABLE) continue; // not a short unitig + + /** + * so combining the last two lines, they are designed to reomve(n(0), n(1), n(2)): + * ----->n(4) + * | + * n(0)--->n(1)---->n(2)---->n(3) + * | + * ----->n(5) + **/ + for (i = 0; i < a.n; ++i) + { + asg_seq_del(g, (uint32_t)a.a[i]>>1); + fprintf(stderr, "removed node: %u\n", (uint32_t)a.a[i]>>1); + } + + ++cnt; + } + free(a.a); + if (cnt > 0) asg_cleanup(g); + fprintf(stderr, "[M::%s] cut %d tips\n", __func__, cnt); + return cnt; +} + + +// delete short arcs +///for best graph? +int asg_arc_del_short(asg_t *g, float drop_ratio) +{ + uint32_t v, n_vtx = g->n_seq * 2, n_short = 0; + for (v = 0; v < n_vtx; ++v) { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, thres, nv = asg_arc_n(g, v); + ///if there is just one overlap, do nothing + if (nv < 2) continue; + //av[0] has the most overlap length + ///remove short overlaps + thres = (uint32_t)(av[0].ol * drop_ratio + .499); + ///av has been sorted by overlap length + for (i = nv - 1; i >= 1 && av[i].ol < thres; --i); + for (i = i + 1; i < nv; ++i) + av[i].del = 1, ++n_short; + } + if (n_short) { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d short overlaps\n", __func__, n_short); + return n_short; +} + + +inline int check_weak_ma_hit(ma_hit_t_alloc* aim_paf, ma_hit_t_alloc* reverse_paf_list, +long long weakID, uint32_t w_qs, uint32_t w_qe) +{ + long long i = 0; + long long strongID, index; + for (i = 0; i < aim_paf->length; i++) + { + ///if this is a strong overlap + if ( + aim_paf->buffer[i].ml == 1 + && + Get_qs(aim_paf->buffer[i]) <= w_qs + && + Get_qe(aim_paf->buffer[i]) >= w_qe) + { + strongID = Get_tn(aim_paf->buffer[i]); + index = get_specific_overlap(&(reverse_paf_list[strongID]), strongID, weakID); + if(index != -1) + { + return 0; + } + } + + + } + + return 1; +} + + +inline int check_weak_ma_hit_debug(ma_hit_t_alloc* aim_paf, ma_hit_t_alloc* reverse_paf_list, +long long weakID) +{ + long long i = 0; + long long strongID, index; + for (i = 0; i < aim_paf->length; i++) + { + ///if this is a strong overlap + if (aim_paf->buffer[i].ml == 1) + { + strongID = Get_tn(aim_paf->buffer[i]); + index = get_specific_overlap(&(reverse_paf_list[strongID]), strongID, weakID); + if(index != -1) + { + return strongID; + } + } + + + } + + return 0; +} + +// delete short arcs +///for best graph? +int asg_arc_del_short_diploid(asg_t *g, float drop_ratio, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources) +{ + float second_drop_ratio = 0.3; + uint32_t v, n_vtx = g->n_seq * 2, n_short = 0; + uint32_t last_e, flag; + for (v = 0; v < n_vtx; ++v) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, thres, nv = asg_arc_n(g, v); + ///if there is just one overlap, do nothing + if (nv < 2) continue; + //av[0] has the most overlap length + ///remove short overlaps + thres = (uint32_t)(av[0].ol * drop_ratio + .499); + ///av has been sorted by overlap length + for (i = nv - 1; i >= 1 && av[i].ol < thres; --i); + last_e = i + 1; + + for (i = i + 1; i < nv; ++i) + av[i].del = 1, ++n_short; + + + if(nv >= 2 && av[1].del == 1) + { + thres = (uint32_t)(av[0].ol * second_drop_ratio + .499); + if(av[1].ol >= thres) + { + ///second longest + av[1].del = 0; + --n_short; + last_e++; + } + } + + /** + if(last_e > 1) + { + flag = 0; + ///at least one exact edge + for (i = 0; i < last_e; i++) + { + if(av[i].el) + { + flag = 1; + break; + } + } + + if(flag) + { + for (i = 0; i < last_e; i++) + { + //drop inexact overlaps + if(av[i].el == 0) + { + av[i].del = 1; + ++n_short; + } + } + } + } + **/ + + } + ///if (n_short) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d short overlaps\n", __func__, n_short); + return n_short; +} + + +// delete short arcs +///for best graph? +int asg_arc_del_short_diploid_unclean(asg_t *g, float drop_ratio, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources) +{ + double startTime = Get_T(); + + uint32_t v, n_vtx = g->n_seq * 2, n_short = 0; + uint32_t last_e, flag; + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v>>1].del) continue; + + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, thres, nv = asg_arc_n(g, v); + ///if there is just one overlap, do nothing + if (nv < 2) continue; + //av[0] has the most overlap length + ///remove short overlaps + thres = (uint32_t)(av[0].ol * drop_ratio + .499); + ///av has been sorted by overlap length + for (i = nv - 1; i >= 1 && av[i].ol < thres; --i); + last_e = i + 1; + + for (i = i + 1; i < nv; ++i) + av[i].del = 1, ++n_short; + + + if(nv >= 2 && av[1].del == 1) + { + ///second longest + av[1].del = 0; + --n_short; + last_e++; + } + + /** + if(last_e > 1) + { + flag = 0; + ///at least one exact edge + for (i = 0; i < last_e; i++) + { + if(av[i].el) + { + flag = 1; + break; + } + } + + if(flag) + { + for (i = 0; i < last_e; i++) + { + //drop inexact overlaps + if(av[i].el == 0) + { + av[i].del = 1; + ++n_short; + } + } + } + } + **/ + + } + ///if (n_short) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d short overlaps\n", __func__, n_short); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + + return n_short; +} + + +int asg_arc_del_short_diploid_unclean_exact(asg_t *g, float drop_ratio, ma_hit_t_alloc* sources) +{ + uint32_t v, n_vtx = g->n_seq * 2, n_short = 0; + uint32_t last_e, flag; + for (v = 0; v < n_vtx; ++v) + { + if (g->seq[v>>1].del) continue; + + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, nv = asg_arc_n(g, v); + ///if there is just one overlap, do nothing + if (nv < 2) continue; + ///keep the longest one + for (i = 1; i < nv; i++) + { + ///if it is an inexact overlap + if(av[i].el == 0 && + sources[v>>1].is_fully_corrected == 1&& + sources[(av[i].v>>1)].is_fully_corrected == 1) + { + av[i].del = 1; + ++n_short; + } + } + + } + + if (n_short) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d inexact overlaps\n", __func__, n_short); + return n_short; +} + +long long single_edge(asg_t *g, uint32_t begNode, long long edgeLen) +{ + + uint32_t v = begNode; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + long long rLen = 0; + + while (rLen < edgeLen && nv == 1) + { + rLen++; + + + if(asg_is_single_edge(g, av[0].v, v>>1) != 1) + { + return -1; + } + + if(rLen == edgeLen) + { + return rLen; + } + + v = av[0].v; + nv = asg_arc_n(g, v); + av = asg_arc_a(g, v); + } + + return -1; + +} + + +// delete short arcs +///for best graph? +int asg_arc_del_short_diploid_based_on_length_back(asg_t *g, float drop_ratio, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources) +{ + uint32_t v, n_vtx = g->n_seq * 2, n_short = 0; + uint32_t last_e, flag; + for (v = 0; v < n_vtx; ++v) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, thres, nv = asg_arc_n(g, v); + ///if there is just one overlap, do nothing + if (nv < 2) continue; + //av[0] has the most overlap length + ///remove short overlaps + thres = (uint32_t)(av[0].ol * drop_ratio + .499); + ///av has been sorted by overlap length + for (i = nv - 1; i >= 1 && av[i].ol < thres; --i); + last_e = i + 1; + + for (i = i + 1; i < nv; ++i) + av[i].del = 1, ++n_short; + + + if(nv >= 2 && av[1].del == 1) + { + if(single_edge(g, av[1].v, 4) != -1) + { + ///second longest + av[1].del = 0; + --n_short; + last_e++; + } + } + + + } + ///if (n_short) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d short overlaps\n", __func__, n_short); + return n_short; +} + + +///check if v has only one branch +static uint32_t asg_check_unambi1(asg_t *g, uint32_t v) +{ + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, nv = asg_arc_n(g, v); + uint32_t k = nv, kv; + for (i = 0, kv = 0; i < nv; ++i) + if (!av[i].del) ++kv, k = i; + if (kv != 1) return (uint32_t)-1; + return av[k].v; +} +///to see if it is a long tip +static int asg_topocut_aux(asg_t *g, uint32_t v, int max_ext) +{ + int32_t n_ext; + for (n_ext = 1; n_ext < max_ext && v != (uint32_t)-1; ++n_ext) { + if (asg_check_unambi1(g, v^1) == (uint32_t)-1) { + --n_ext; + break; + } + v = asg_check_unambi1(g, v); + } + return n_ext; +} + +// delete short arcs +///for best graph? +int asg_arc_del_short_diploid_by_length(asg_t *g, float drop_ratio, int max_ext) +{ + double startTime = Get_T(); + kvec_t(uint64_t) b; + memset(&b, 0, sizeof(b)); + + uint32_t v, n_vtx = g->n_seq * 2; + long long n_cut = 0; + + for (v = 0; v < n_vtx; ++v) + { + if(g->seq_vis[v] == 0) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t nv = asg_arc_n(g, v); + if (nv < 2) continue; + long long i; + for (i = 0; i < nv; ++i) + { + kv_push(uint64_t, b, (uint64_t)(av[i].ol << 32 | (av - g->arc + i))); + } + } + } + + fprintf(stderr, "[M::%s] %lld unsorted pending overlaps\n", __func__, b.n); + + radix_sort_arch64(b.a, b.a + b.n); + + fprintf(stderr, "[M::%s] %lld sorted pending overlaps\n", __func__, b.n); + + long long k; + for (k = 0; k < b.n; k++) + { + + asg_arc_t *a = &g->arc[(uint32_t)b.a[k]]; + ///v is self id, w is the id of another end + uint32_t i, iv, iw, v = (a->ul)>>32, w = a->v^1, to_del = 0; + uint32_t nv = asg_arc_n(g, v), nw = asg_arc_n(g, w), kv, kw; + uint32_t ov_max = 0, ow_max = 0; + asg_arc_t *av, *aw; + ///nv must be >= 2 + if (nv == 1 && nw == 1) continue; + av = asg_arc_a(g, v); + aw = asg_arc_a(g, w); + + + ///calculate the longest edge for v and w + for (i = 0, kv = 0; i < nv; ++i) { + if (av[i].del) continue; + if (ov_max < av[i].ol) ov_max = av[i].ol; + ++kv; + } + if (kv >= 2 && a->ol > ov_max * drop_ratio) continue; + + + for (i = 0, kw = 0; i < nw; ++i) { + if (aw[i].del) continue; + if (ow_max < aw[i].ol) ow_max = aw[i].ol; + ++kw; + } + if (kw >= 2 && a->ol > ow_max * drop_ratio) continue; + if (kv == 1 && kw == 1) continue; + + + ///to see which one is the current edge (from v and w) + for (iv = 0; iv < nv; ++iv) + if (av[iv].v == (w^1)) break; + for (iw = 0; iw < nw; ++iw) + if (aw[iw].v == (v^1)) break; + ///if one edge has been deleted, it should be deleted in both direction + if (av[iv].del && aw[iw].del) continue; + + ///kv and kw is the avialiable + if (kv > 1 && kw > 1) { + if (a->ol < ov_max * drop_ratio && a->ol < ow_max * drop_ratio) + to_del = 1; + } else if (kw == 1) { + if (asg_topocut_aux(g, w^1, max_ext) < max_ext) to_del = 1; + } else if (kv == 1) { + if (asg_topocut_aux(g, v^1, max_ext) < max_ext) to_del = 1; + } + if (to_del) + av[iv].del = aw[iw].del = 1, ++n_cut; + + } + + free(b.a); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d short overlaps\n", __func__, n_cut); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_cut; +} + + +inline int get_real_length(asg_t *g, uint32_t v, uint32_t* v_s) +{ + uint32_t i, kv = 0; + for (i = 0, kv = 0; i < asg_arc_n(g, v); i++) + { + if(!asg_arc_a(g, v)[i].del) + { + if(v_s) v_s[kv] = asg_arc_a(g, v)[i].v; + kv++; + } + } + + return kv; +} + +// delete short arcs +///for best graph? +int asg_arc_del_short_false_link_back(asg_t *g, float drop_ratio, int max_ext) +{ + double startTime = Get_T(); + kvec_t(uint64_t) b; + memset(&b, 0, sizeof(b)); + + uint32_t v, n_vtx = g->n_seq * 2; + long long n_cut = 0; + + for (v = 0; v < n_vtx; ++v) + { + if(g->seq_vis[v] == 0) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t nv = asg_arc_n(g, v); + if (nv != 2) continue; + if(asg_arc_n(g, v^1)!=1) continue; + + + long long i; + for (i = 0; i < nv; ++i) + { + kv_push(uint64_t, b, (uint64_t)(av[i].ol << 32 | (av - g->arc + i))); + } + } + } + + fprintf(stderr, "[M::%s] %lld unsorted pending overlaps\n", __func__, b.n); + + radix_sort_arch64(b.a, b.a + b.n); + + fprintf(stderr, "[M::%s] %lld sorted pending overlaps\n", __func__, b.n); + + uint32_t v_s[2]; + uint32_t w_s[4]; + + long long k; + for (k = 0; k < b.n; k++) + { + asg_arc_t *a = &g->arc[(uint32_t)b.a[k]]; + if(a->del) continue; + ///v is self id, w is the id of another end + uint32_t i, iv, v = (a->ul)>>32, to_del = 0; + uint32_t nv = asg_arc_n(g, v), kv; + asg_arc_t *av, *aw; + if (nv < 2) continue; + av = asg_arc_a(g, v); + + + kv = get_real_length(g, v, NULL); + if (kv != 2) continue; + + if(get_real_length(g, v^1, NULL)!=1) continue; + + + get_real_length(g, v^1, v_s); + if(get_real_length(g, v_s[0]^1, NULL) < 2) continue; + + + + + //check the length + for (i = 0, kv = 0; i < nv; ++i) { + if (av[i].del) continue; + v_s[kv] = av[i].ol; + ++kv; + } + + uint32_t s_max = 0; + for (i = 0; i < asg_arc_n(g, v^1); i++) + { + if(asg_arc_a(g, v^1)[i].del) continue; + s_max = asg_arc_a(g, v^1)[i].ol; + break; + } + + uint32_t ov_max, ov_min; + if(v_s[0] >= v_s[1]) + { + ov_max = v_s[0]; + ov_min = v_s[1]; + } + else + { + ov_max = v_s[1]; + ov_min = v_s[0]; + } + + if(ov_min < ov_max * drop_ratio) + { + continue; + } + + if(ov_max > s_max * 0.5) + { + continue; + } + //check the length + + + + + get_real_length(g, v, v_s); + v_s[0] = v_s[0]^1; + v_s[1] = v_s[1]^1; + + if(v_s[0] == v_s[1]) continue; + + if(get_real_length(g, v_s[0], NULL)!=2) continue; + if(get_real_length(g, v_s[1], NULL)!=2) continue; + + + get_real_length(g, v_s[0], w_s); + get_real_length(g, v_s[1], w_s + 2); + + for (i = 0; i < 2; i++) + { + if((w_s[i]>>1) != (v>>1)) + { + w_s[0] = w_s[i]; + } + } + + for (i = 2; i < 4; i++) + { + if((w_s[i]>>1) != (v>>1)) + { + w_s[1] = w_s[i]; + } + } + + if(w_s[0] == w_s[1]) + { + to_del = 1; + } + + uint32_t convex1, f1; + long long l1; + if(to_del == 0) + { + f1 = detect_bubble_end_with_bubbles(g, w_s[0], w_s[1], &convex1, &l1, NULL); + if(f1) + { + to_del = 1; + } + } + + if(to_del == 0) + { + v_s[0] = v_s[0]^1; + v_s[1] = v_s[1]^1; + + if(get_real_length(g, v_s[0], NULL)!=1) continue; + if(get_real_length(g, v_s[1], NULL)!=1) continue; + + if(v_s[0] == v_s[1]) + { + to_del = 1; + } + + f1 = detect_bubble_end_with_bubbles(g, v_s[0], v_s[1], &convex1, &l1, NULL); + if(f1) + { + to_del = 1; + } + } + + + if (to_del) + { + for (i = 0; i < nv; ++i) + { + if (av[i].del) continue; + ++n_cut; + av[i].del = 1; + asg_arc_del(g, av[i].v^1, av[i].ul>>32^1, 1); + } + + } + } + + free(b.a); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d false overlaps\n", __func__, n_cut); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_cut; +} + + +int asg_arc_del_short_false_link(asg_t *g, float drop_ratio, int max_dist) +{ + double startTime = Get_T(); + + kvec_t(uint64_t) b; + memset(&b, 0, sizeof(b)); + + + kvec_t(uint32_t) b_f; + memset(&b_f, 0, sizeof(b_f)); + + kvec_t(uint32_t) b_r; + memset(&b_r, 0, sizeof(b_r)); + + + uint32_t v, w, n_vtx = g->n_seq * 2, n_cut = 0; + uint32_t sink; + + buf_t bub; + if (!g->is_symm) asg_symm(g); + memset(&bub, 0, sizeof(buf_t)); + bub.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + + for (v = 0; v < n_vtx; ++v) + { + // if(v>>1 == 4581428) + // { + // fprintf(stderr, "%.*s, vis: %d\n", Get_NAME_LENGTH(R_INF, v>>1), Get_NAME(R_INF, v>>1), + // g->seq_vis[v]); + // } + + + if(g->seq_vis[v] == 0) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t nv = asg_arc_n(g, v); + + // if(v>>1 == 4581428) + // { + // fprintf(stderr, "nv: %d, rnv: %d\n", nv, asg_arc_n(g, v^1)); + // } + + if(nv == 1 && asg_arc_n(g, v^1) == 1) continue; + + + + uint64_t t_ol = 0; + long long i; + for (i = 0; i < nv; ++i) + { + t_ol += av[i].ol; + } + kv_push(uint64_t, b, (uint64_t)(t_ol << 32 | v)); + } + } + + fprintf(stderr, "[M::%s] %lld unsorted pending overlaps\n", __func__, b.n); + + radix_sort_arch64(b.a, b.a + b.n); + + fprintf(stderr, "[M::%s] %lld sorted pending overlaps\n", __func__, b.n); + + + + uint32_t min_edge; + + + long long k, t; + for (k = 0; k < b.n; k++) + { + ///v is the node + v = (uint32_t)b.a[k]; + if (g->seq[v>>1].del) continue; + uint32_t nv = asg_arc_n(g, v), nw, to_del_l, to_del_r; + if (nv < 2) continue; + uint32_t kv = get_real_length(g, v, NULL), kw; + if (kv < 2) continue; + uint32_t i; + asg_arc_t *av = asg_arc_a(g, v), *aw; + + b_f.n = 0; + b_r.n = 0; + to_del_l = 0; + for (i = 0; i < nv; i++) + { + if (av[i].del) continue; + + w = av[i].v^1; + nw = asg_arc_n(g, w); + if(nw < 2) break; + kw = get_real_length(g, w, NULL); + if(kw < 2) break; + + kv_push(uint32_t, b_f, av[i].v); + kv_push(uint32_t, b_r, w); + + aw = asg_arc_a(g, w); + min_edge = (u_int32_t)-1; + for (t = 0; t < nw; t++) + { + if(aw[t].del) continue; + if((aw[t].v>>1) == (v>>1)) continue; + if(aw[t].ol < min_edge) min_edge = aw[t].ol; + ///kv_push(uint32_t, b_r, aw[t].v); + } + + if(av[i].ol < min_edge * drop_ratio) to_del_l++; + } + + if(to_del_l != kv) continue; + + + + + + uint32_t convex1; + long long l1; + + + + + ////forward bubble + to_del_l = 0; + for (i = 0; i < b_f.n; i++) + { + if(b_f.a[i] == b_f.a[0]) + { + to_del_l = 1; + } + else + { + to_del_l = 0; + break; + } + } + //check the length + if(to_del_l == 0 && asg_bub_end_finder_with_del_advance(g, + b_f.a, b_f.n, max_dist, &bub, 0, (u_int32_t)-1, &sink)==1) + { + to_del_l = 1; + } + if(to_del_l == 0 && detect_mul_bubble_end_with_bubbles(g, b_f.a, b_f.n, &convex1, &l1, NULL)) + { + to_del_l = 1; + } + + ///if(v>>1 == 4581428) fprintf(stderr, "to_del_l: %d, b_f.n: %d\n", to_del_l, b_f.n); + + ///if(to_del_l == 0) continue; + + + + + + + + ////backward bubble + to_del_r = 0; + for (i = 0; i < b_r.n; i++) + { + if(b_r.a[i] == b_r.a[0]) + { + to_del_r = 1; + } + else + { + to_del_r = 0; + break; + } + } + if(to_del_r == 0 && asg_bub_end_finder_with_del_advance + (g, b_r.a, b_r.n, max_dist, &bub, 1, v^1, &sink)==1) + { + to_del_r = 1; + } + if(to_del_r == 0 && detect_mul_bubble_end_with_bubbles(g, b_r.a, b_r.n, &convex1, &l1, NULL)) + { + to_del_r = 1; + } + + + // if(v>>1 == 4581428) + // { + // fprintf(stderr, "to_del_l: %d, b_f.n: %d\n", to_del_l, b_f.n); + // asg_bub_end_finder_with_del_advance_debug(g, b_r.a, b_r.n, max_dist, &bub, 1, v^1); + // } + + + + + + if (to_del_l && to_del_r) + { + for (i = 0; i < nv; ++i) + { + if (av[i].del) continue; + ///fprintf(stderr, "%.*s\n", Get_NAME_LENGTH((R_INF), v>>1), Get_NAME((R_INF), v>>1)); + ++n_cut; + av[i].del = 1; + asg_arc_del(g, av[i].v^1, av[i].ul>>32^1, 1); + } + + } + } + + free(b.a); free(b_f.a); free(b_r.a); + free(bub.a); free(bub.S.a); free(bub.T.a); free(bub.b.a); free(bub.e.a); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d false overlaps\n", __func__, n_cut); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_cut; +} + + + +int asg_arc_del_tri_link(asg_t *g, int max_dist) +{ + double startTime = Get_T(); + + kvec_t(uint64_t) b; + memset(&b, 0, sizeof(b)); + + + kvec_t(uint32_t) b_f; + memset(&b_f, 0, sizeof(b_f)); + + kvec_t(uint32_t) b_r; + memset(&b_r, 0, sizeof(b_r)); + + + uint32_t v, w, n_vtx = g->n_seq * 2, n_cut = 0; + uint32_t sink; + + buf_t bub; + if (!g->is_symm) asg_symm(g); + memset(&bub, 0, sizeof(buf_t)); + bub.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + + uint32_t Ns_first[3]; + uint32_t Ns_second[3]; + long long NodeLen_first[3]; + long long NodeLen_second[3]; + for (v = 0; v < n_vtx; ++v) + { + if(g->seq_vis[v] == 0) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t nv = asg_arc_n(g, v); + + if(nv != 2) continue; + if(av[0].v == av[1].v) continue; + + /**********************test first node************************/ + NodeLen_first[0] = NodeLen_first[1] = NodeLen_first[2] = -1; + if(asg_is_single_edge(g, av[0].v, v>>1) <= 2 && asg_is_single_edge(g, av[1].v, v>>1) <= 2) + { + NodeLen_first[asg_is_single_edge(g, av[0].v, v>>1)] = 0; + NodeLen_first[asg_is_single_edge(g, av[1].v, v>>1)] = 1; + } + ///one node has one out-edge, another node has two out-edges + if(NodeLen_first[1] == -1 || NodeLen_first[2] == -1) + { + continue; + } + /**********************test first node************************/ + + + + /**********************test second node************************/ + w = av[NodeLen_first[2]].v^1; + asg_arc_t *aw = asg_arc_a(g, w); + uint32_t nw = asg_arc_n(g, w); + if(nw != 2) + { + fprintf(stderr, "error\n"); + } + NodeLen_second[0] = NodeLen_second[1] = NodeLen_second[2] = -1; + if(asg_is_single_edge(g, aw[0].v, w>>1) <= 2 && asg_is_single_edge(g, aw[1].v, w>>1) <= 2) + { + NodeLen_second[asg_is_single_edge(g, aw[0].v, w>>1)] = 0; + NodeLen_second[asg_is_single_edge(g, aw[1].v, w>>1)] = 1; + } + ///one node has one out-edge, another node has two out-edges + if(NodeLen_second[1] == -1 || NodeLen_second[2] == -1) + { + continue; + } + /**********************test second node************************/ + + uint64_t t_ol = av[NodeLen_first[2]].ol; + kv_push(uint64_t, b, (uint64_t)(t_ol << 32 | v)); + } + } + + fprintf(stderr, "[M::%s] %lld unsorted pending overlaps\n", __func__, b.n); + + radix_sort_arch64(b.a, b.a + b.n); + + fprintf(stderr, "[M::%s] %lld sorted pending overlaps\n", __func__, b.n); + + + + uint32_t min_edge; + + + long long k, t; + for (k = 0; k < b.n; k++) + { + ///v is the node + v = (uint32_t)b.a[k]; + if (g->seq[v>>1].del) continue; + uint32_t nv = asg_arc_n(g, v), nw, to_del; + uint32_t kv = get_real_length(g, v, NULL), kw; + + + ///at the begining, the nv of all nodes must be == 2; + ///here kv == 2, that means all edges are kept + ///so we can use normal method to delete edges + if (nv != 2) continue; + if (kv != 2) continue; + + + + uint32_t i; + asg_arc_t *av = asg_arc_a(g, v), *aw; + if(av[0].v == av[1].v) + { + continue; + } + + /**********************test first node************************/ + NodeLen_first[0] = NodeLen_first[1] = NodeLen_first[2] = -1; + if(asg_is_single_edge(g, av[0].v, v>>1) <= 2 && asg_is_single_edge(g, av[1].v, v>>1) <= 2) + { + NodeLen_first[asg_is_single_edge(g, av[0].v, v>>1)] = 0; + NodeLen_first[asg_is_single_edge(g, av[1].v, v>>1)] = 1; + } + ///one node has one out-edge, another node has two out-edges + if(NodeLen_first[1] == -1 || NodeLen_first[2] == -1) + { + continue; + } + /**********************test first node************************/ + + /**********************test second node************************/ + w = av[NodeLen_first[2]].v^1; + aw = asg_arc_a(g, w); + nw = asg_arc_n(g, w); + kw = get_real_length(g, w, NULL); + + ///at the begining, the nw of all nodes must be == 2; + ///here kw == 2, that means all edges are kept + ///so we can use normal method to delete edges + if(nw != 2) continue; + if(kw != 2) continue; + + + NodeLen_second[0] = NodeLen_second[1] = NodeLen_second[2] = -1; + if(asg_is_single_edge(g, aw[0].v, w>>1) <= 2 && asg_is_single_edge(g, aw[1].v, w>>1) <= 2) + { + NodeLen_second[asg_is_single_edge(g, aw[0].v, w>>1)] = 0; + NodeLen_second[asg_is_single_edge(g, aw[1].v, w>>1)] = 1; + } + ///one node has one out-edge, another node has two out-edges + if(NodeLen_second[1] == -1 || NodeLen_second[2] == -1) + { + continue; + } + + /**********************test second node************************/ + + + + + + + uint32_t convex1, convex2, f1, f2; + long long l1, l2; + to_del = 0; + f1 = detect_bubble_end_with_bubbles(g, av[0].v, av[1].v, &convex1, &l1, NULL); + f2 = detect_bubble_end_with_bubbles(g, aw[0].v, aw[1].v, &convex2, &l2, NULL); + if(f1 && f2) + { + if(l1 <= min_thres || l2 <= min_thres) + { + continue; + } + to_del = 1; + } + else if(f1) + { + if(l1 <= min_thres) + { + continue; + } + to_del = 1; + } + else if(f2) + { + if(l2 <= min_thres) + { + continue; + } + to_del = 1; + } + + + if(to_del == 0) + { + if(!f1) + { + Ns_first[0] = av[0].v; Ns_first[1] = av[1].v; + f1 = asg_bub_end_finder_with_del_advance(g, Ns_first, 2, max_dist, + &bub, 0, (u_int32_t)-1, &convex1); + l1 = min_thres + 10; + } + + if(!f2) + { + Ns_second[0] = aw[0].v; Ns_second[1] = aw[1].v; + f2 = asg_bub_end_finder_with_del_advance(g, Ns_second, 2, max_dist, + &bub, 0, (u_int32_t)-1, &convex2); + l2 = min_thres + 10; + } + + + + + + if(f1 && f2) + { + if(l1 <= min_thres || l2 <= min_thres) + { + continue; + } + to_del = 1; + } + else if(f1) + { + + if(l1 <= min_thres) + { + continue; + } + to_del = 1; + } + else if(f2) + { + + if(l2 <= min_thres) + { + continue; + } + + to_del = 1; + } + + } + + + if (to_del) + { + ++n_cut; + av[NodeLen_first[2]].del = 1; + asg_arc_del(g, av[NodeLen_first[2]].v^1, av[NodeLen_first[2]].ul>>32^1, 1); + } + } + + free(b.a); free(b_f.a); free(b_r.a); + free(bub.a); free(bub.S.a); free(bub.T.a); free(bub.b.a); free(bub.e.a); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d false overlaps\n", __func__, n_cut); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_cut; +} + +int asg_arc_del_complex_false_link(asg_t *g, float drop_ratio, int max_dist) +{ + double startTime = Get_T(); + + kvec_t(uint64_t) b; + memset(&b, 0, sizeof(b)); + + + kvec_t(uint32_t) b_f; + memset(&b_f, 0, sizeof(b_f)); + + kvec_t(uint32_t) b_r; + memset(&b_r, 0, sizeof(b_r)); + + + uint32_t v, w, n_vtx = g->n_seq * 2, n_cut = 0; + + + buf_t bub; + if (!g->is_symm) asg_symm(g); + memset(&bub, 0, sizeof(buf_t)); + bub.a = (binfo_t*)calloc(n_vtx, sizeof(binfo_t)); + + for (v = 0; v < n_vtx; ++v) + { + if(g->seq_vis[v] == 0) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t nv = asg_arc_n(g, v); + + if(nv == 1 && asg_arc_n(g, v^1) == 1) continue; + + uint64_t t_ol = 0; + long long i; + for (i = 0; i < nv; ++i) + { + t_ol += av[i].ol; + } + kv_push(uint64_t, b, (uint64_t)(t_ol << 32 | v)); + } + } + + fprintf(stderr, "[M::%s] %lld unsorted pending overlaps\n", __func__, b.n); + + radix_sort_arch64(b.a, b.a + b.n); + + fprintf(stderr, "[M::%s] %lld sorted pending overlaps\n", __func__, b.n); + + + + uint32_t min_edge; + + + long long k, t; + for (k = 0; k < b.n; k++) + { + ///v is the node + v = (uint32_t)b.a[k]; + if (g->seq[v>>1].del) continue; + uint32_t nv = asg_arc_n(g, v), nw, to_del; + if (nv < 2) continue; + uint32_t kv = get_real_length(g, v, NULL), kw; + if (kv < 2) continue; + uint32_t i; + asg_arc_t *av = asg_arc_a(g, v), *aw; + + b_f.n = 0; + b_r.n = 0; + to_del = 0; + for (i = 0; i < nv; i++) + { + if (av[i].del) continue; + + w = av[i].v^1; + nw = asg_arc_n(g, w); + if(nw < 2) break; + kw = get_real_length(g, w, NULL); + if(kw < 2) break; + + kv_push(uint32_t, b_f, av[i].v); + kv_push(uint32_t, b_r, w); + + aw = asg_arc_a(g, w); + min_edge = (u_int32_t)-1; + for (t = 0; t < nw; t++) + { + if(aw[t].del) continue; + if((aw[t].v>>1) == (v>>1)) continue; + if(aw[t].ol < min_edge) min_edge = aw[t].ol; + } + + if(av[i].ol < min_edge * drop_ratio) to_del++; + } + + if(to_del != kv) continue; + + for (i = 0; i < nv; ++i) + { + if (av[i].del) continue; + ++n_cut; + av[i].del = 1; + asg_arc_del(g, av[i].v^1, av[i].ul>>32^1, 1); + } + } + + + if(n_cut > 0) + { + for (v = 0; v < n_vtx; ++v) + { + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + if (g->seq[v>>1].del) + { + continue; + } + + if(nv < 2) + { + continue; + } + + + if(asg_bub_finder_without_del_advance(g, v, max_dist, &bub) == 1) + { + uint32_t i; + g->seq_vis[v] = 3; + g->seq_vis[v^1] = 3; + for (i = 0; i < bub.b.n; i++) + { + g->seq_vis[bub.b.a[i]] = 3; + g->seq_vis[bub.b.a[i]^1] = 3; + } + } + } + + + for (v = 0; v < n_vtx; ++v) + { + if(g->seq_vis[v] == 3) continue; + + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i; + for (i = 0; i < nv; ++i) + { + if (av[i].del && g->seq_vis[av[i].v] != 3) + { + av[i].del = 0; + asg_arc_del(g, av[i].v^1, av[i].ul>>32^1, 0); + } + } + } + } + + + + + free(b.a); free(b_f.a); free(b_r.a); + free(bub.a); free(bub.S.a); free(bub.T.a); free(bub.b.a); free(bub.e.a); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d false overlaps\n", __func__, n_cut); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_cut; +} + + + +int asg_arc_del_short_diploid_by_exact(asg_t *g, int max_ext, ma_hit_t_alloc* sources) +{ + double startTime = Get_T(); + kvec_t(uint64_t) b; + memset(&b, 0, sizeof(b)); + + uint32_t v, n_vtx = g->n_seq * 2; + long long n_cut = 0; + + for (v = 0; v < n_vtx; ++v) + { + if(g->seq_vis[v] == 0) + { + asg_arc_t *av = asg_arc_a(g, v); + uint32_t nv = asg_arc_n(g, v); + if (nv < 2) continue; + long long i; + for (i = 0; i < nv; ++i) + { + kv_push(uint64_t, b, (uint64_t)(av[i].ol << 32 | (av - g->arc + i))); + } + } + } + + fprintf(stderr, "[M::%s] %lld unsorted pending overlaps\n", __func__, b.n); + + radix_sort_arch64(b.a, b.a + b.n); + + fprintf(stderr, "[M::%s] %lld sorted pending overlaps\n", __func__, b.n); + + long long k; + for (k = 0; k < b.n; k++) + { + + asg_arc_t *a = &g->arc[(uint32_t)b.a[k]]; + ///v is self id, w is the id of another end + uint32_t i, iv, iw, v = (a->ul)>>32, w = a->v^1, to_del = 0; + uint32_t nv = asg_arc_n(g, v), nw = asg_arc_n(g, w), kv, kw; + uint32_t ov_max = 0, ow_max = 0, ov_max_i, ow_max_i; + asg_arc_t *av, *aw; + ///nv must be >= 2 + if (nv == 1 && nw == 1) continue; + av = asg_arc_a(g, v); + aw = asg_arc_a(g, w); + + + ///calculate the longest edge for v and w + for (i = 0, kv = 0; i < nv; ++i) { + if (av[i].del) continue; + if (ov_max < av[i].ol) + { + ov_max = av[i].ol; + ov_max_i = i; + } + ++kv; + } + if (kv >= 2 && a->ol == ov_max) continue; + + + for (i = 0, kw = 0; i < nw; ++i) { + if (aw[i].del) continue; + if (ow_max < aw[i].ol) + { + ow_max = aw[i].ol; + ow_max_i = i; + } + ++kw; + } + if (kw >= 2 && a->ol == ow_max) continue; + + if (kv == 1 && kw == 1) continue; + + + ///to see which one is the current edge (from v and w) + for (iv = 0; iv < nv; ++iv) + if (av[iv].v == (w^1)) break; + for (iw = 0; iw < nw; ++iw) + if (aw[iw].v == (v^1)) break; + ///if one edge has been deleted, it should be deleted in both direction + if (av[iv].del && aw[iw].del) continue; + + + + if(a->el == 0 && + sources[v>>1].is_fully_corrected == 1 && + sources[w>>1].is_fully_corrected == 1) + { + if (kv > 1 && kw > 1) { + to_del = 1; + } else if (kw == 1) { + if (asg_topocut_aux(g, w^1, max_ext) < max_ext) to_del = 1; + } else if (kv == 1) { + if (asg_topocut_aux(g, v^1, max_ext) < max_ext) to_del = 1; + } + } + + if(a->el == 0 && + sources[v>>1].is_fully_corrected == 1 && + sources[w>>1].is_fully_corrected == 0) + { + if(av[ov_max_i].el == 1 && sources[av[ov_max_i].v>>1].is_fully_corrected) + { + if (kv > 1 && kw > 1) { + to_del = 1; + } else if (kw == 1) { + if (asg_topocut_aux(g, w^1, max_ext) < max_ext) to_del = 1; + } else if (kv == 1) { + if (asg_topocut_aux(g, v^1, max_ext) < max_ext) to_del = 1; + } + } + } + + + + if (to_del) + av[iv].del = aw[iw].del = 1, ++n_cut; + + } + + free(b.a); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d inexact overlaps\n", __func__, n_cut); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_cut; +} + + +int asg_arc_del_false_node(asg_t *g, int max_ext) +{ + double startTime = Get_T(); + kvec_t(uint64_t) b; + memset(&b, 0, sizeof(b)); + + uint32_t v, n_vtx = g->n_seq * 2; + long long n_cut = 0; + + for (v = 0; v < n_vtx; ++v) + { + if(g->seq_vis[v] == 0) + { + if(asg_arc_n(g, v)!=1 || asg_arc_n(g, v^1)!=1) + { + continue; + } + + + if(asg_is_single_edge(g, asg_arc_a(g, v)[0].v, v>>1) < 2) + { + continue; + } + + if(asg_is_single_edge(g, asg_arc_a(g, v^1)[0].v, (v^1)>>1) < 2) + { + continue; + } + + if(asg_arc_a(g, v)[0].el == 1) + { + continue; + } + + asg_arc_t *av = asg_arc_a(g, v); + kv_push(uint64_t, b, (uint64_t)(av[0].ol << 32 | (av - g->arc))); + } + } + + radix_sort_arch64(b.a, b.a + b.n); + + long long k; + ///here all edges are inexact matches + for (k = 0; k < b.n; k++) + { + + asg_arc_t *a = &g->arc[(uint32_t)b.a[k]]; + ///v is self id, w is the id of another end + uint32_t i, iv, iw, v = (a->ul)>>32, w = a->v^1, to_del = 0; + uint32_t nv = asg_arc_n(g, v), nw = asg_arc_n(g, w), kv, kw; + asg_arc_t *av, *aw; + av = asg_arc_a(g, v); + aw = asg_arc_a(g, w); + + + /** + uint32_t en; + long long pathLen; + detect_single_path(g, w^1, &en, &pathLen, NULL); + ///<=2 means there is just one single read from w + if(pathLen <= 2) + { + continue; + } + **/ + + + + ///calculate the longest edge for v and w + for (i = 0, kv = 0; i < nv; ++i) { + if (av[i].del) continue; + ++kv; + } + + for (i = 0, kw = 0; i < nw; ++i) { + if (aw[i].del) continue; + ++kw; + } + + if (kv < 1 || kw < 2) continue; + + ///to see which one is the current edge (from v and w) + for (iv = 0; iv < nv; ++iv) + if (av[iv].v == (w^1)) break; + for (iw = 0; iw < nw; ++iw) + if (aw[iw].v == (v^1)) break; + ///if one edge has been deleted, it should be deleted in both direction + if (av[iv].del && aw[iw].del) continue; + + + + uint32_t el_edges = 0; + ///there should be at least two available edges in aw + for (i = 0; i < nw; i++) + { + if (aw[i].del) continue; + + if(i != iw && aw[i].el == 1) + { + el_edges++; + } + } + + if(el_edges > 0 && av[iv].el == 0) + { + asg_seq_del(g, v>>1); + ++n_cut; + } + } + + free(b.a); + if (n_cut) + { + asg_cleanup(g); + asg_symm(g); + } + fprintf(stderr, "[M::%s] removed %d single nodes\n", __func__, n_cut); + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); + return n_cut; +} + + + +#define arc_cnt(g, v) ((uint32_t)(g)->idx[(v)]) +#define arc_first(g, v) ((g)->arc[(g)->idx[(v)]>>32]) + +ma_ug_t *ma_ug_gen(asg_t *g) +{ + int32_t *mark; + uint32_t i, v, n_vtx = g->n_seq * 2; + ///is a queue + kdq_t(uint64_t) *q; + ma_ug_t *ug; + + ug = (ma_ug_t*)calloc(1, sizeof(ma_ug_t)); + ug->g = asg_init(); + mark = (int32_t*)calloc(n_vtx, 4); + + q = kdq_init(uint64_t); + for (v = 0; v < n_vtx; ++v) { + uint32_t w, x, l, start, end, len; + ma_utg_t *p; + if (g->seq[v>>1].del || arc_cnt(g, v) == 0 || mark[v]) continue; + mark[v] = 1; + q->count = 0, start = v, end = v^1, len = 0; + // forward + w = v; + while (1) { + + + /** + * w----->x + * w<-----x + * that means the only suffix of w is x, and the only prefix of x is w + **/ + if (arc_cnt(g, w) != 1) break; + x = arc_first(g, w).v; // w->x + if (arc_cnt(g, x^1) != 1) break; + + /** + * another direction of w would be marked as used (since w has been used) + **/ + mark[x] = mark[w^1] = 1; + ///l is the edge length, instead of overlap length + l = asg_arc_len(arc_first(g, w)); + kdq_push(uint64_t, q, (uint64_t)w<<32 | l); + end = x^1, len += l; + w = x; + if (x == v) break; + } + if (start != (end^1) || kdq_size(q) == 0) { // linear unitig + ///length of seq, instead of edge + l = g->seq[end>>1].len; + kdq_push(uint64_t, q, (uint64_t)(end^1)<<32 | l); + len += l; + } else { // circular unitig + start = end = UINT32_MAX; + goto add_unitig; // then it is not necessary to do the backward + } + // backward + x = v; + while (1) { // similar to forward but not the same + if (arc_cnt(g, x^1) != 1) break; + w = arc_first(g, x^1).v ^ 1; // w->x + if (arc_cnt(g, w) != 1) break; + mark[x] = mark[w^1] = 1; + l = asg_arc_len(arc_first(g, w)); + ///w is the seq id + direction, l is the length of edge + ///push element to the front of a queue + kdq_unshift(uint64_t, q, (uint64_t)w<<32 | l); + start = w, len += l; + x = w; + } +add_unitig: + if (start != UINT32_MAX) mark[start] = mark[end] = 1; + kv_pushp(ma_utg_t, ug->u, &p); + p->s = 0, p->start = start, p->end = end, p->len = len, p->n = kdq_size(q), p->circ = (start == UINT32_MAX); + p->m = p->n; + kv_roundup32(p->m); + p->a = (uint64_t*)malloc(8 * p->m); + //all elements are saved here + for (i = 0; i < kdq_size(q); ++i) + p->a[i] = kdq_at(q, i); + } + kdq_destroy(uint64_t, q); + + // add arcs between unitigs; reusing mark for a different purpose + //ug saves all unitigs + for (v = 0; v < n_vtx; ++v) mark[v] = -1; + + + for (i = 0; i < ug->u.n; ++i) { + if (ug->u.a[i].circ) continue; + mark[ug->u.a[i].start] = i<<1 | 0; + mark[ug->u.a[i].end] = i<<1 | 1; + } + + //scan all edges + for (i = 0; i < g->n_arc; ++i) { + asg_arc_t *p = &g->arc[i]; + if (p->del) continue; + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qns direction of overlap length of this node (not overlap length) + (based on query) + p->v : |___________31___________|__________1___________| + tns reverse direction of overlap + (based on target) + p->ol: overlap length + **/ + ///to connect two unitigs, we need to connect the end of unitig x to the start of unitig y + ///so we need to ^1 to get the reverse direction of (x's end)? + if (mark[p->ul>>32^1] >= 0 && mark[p->v] >= 0) { + asg_arc_t *q; + uint32_t u = mark[p->ul>>32^1]^1; + int l = ug->u.a[u>>1].len - p->ol; + if (l < 0) l = 1; + q = asg_arc_pushp(ug->g); + q->ol = p->ol, q->del = 0; + q->ul = (uint64_t)u<<32 | l; + q->v = mark[p->v]; + } + } + for (i = 0; i < ug->u.n; ++i) + asg_seq_set(ug->g, i, ug->u.a[i].len, 0); + asg_cleanup(ug->g); + free(mark); + return ug; +} + +static char comp_tab[] = { // complement base + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 'T', 'V', 'G', 'H', 'E', 'F', 'C', 'D', 'I', 'J', 'M', 'L', 'K', 'N', 'O', + 'P', 'Q', 'Y', 'S', 'A', 'A', 'B', 'W', 'X', 'R', 'Z', 91, 92, 93, 94, 95, + 64, 't', 'v', 'g', 'h', 'e', 'f', 'c', 'd', 'i', 'j', 'm', 'l', 'k', 'n', 'o', + 'p', 'q', 'y', 's', 'a', 'a', 'b', 'w', 'x', 'r', 'z', 123, 124, 125, 126, 127 +}; + +// generate unitig sequences +int ma_ug_seq(ma_ug_t *g, All_reads *RNF, const ma_sub_t *coverage_cut, +const long long n_read) +{ + UC_Read g_read; + init_UC_Read(&g_read); + utg_intv_t *tmp; + uint32_t i, j; + + + + tmp = (utg_intv_t*)calloc(n_read, sizeof(utg_intv_t)); + ///number of unitigs + for (i = 0; i < g->u.n; ++i) { + ma_utg_t *u = &g->u.a[i]; + uint32_t l = 0; + u->s = (char*)calloc(1, u->len + 1); + memset(u->s, 'N', u->len); + for (j = 0; j < u->n; ++j) { + utg_intv_t *t = &tmp[u->a[j]>>33]; + ///assert(t->len == 0); + t->utg = i, t->ori = u->a[j]>>32&1; + t->start = l, t->len = (uint32_t)u->a[j]; + l += t->len; + } + } + + + + int32_t id; + for (id = 0; id < n_read; id++) + { + utg_intv_t *t; + ma_utg_t *u; + if (id < 0 || tmp[id].len == 0) continue; + + t = &tmp[id]; + u = &g->u.a[t->utg]; + recover_UC_Read(&g_read, RNF, id); + + memmove(g_read.seq, g_read.seq + coverage_cut[id].s, coverage_cut[id].e - coverage_cut[id].s); + g_read.length = coverage_cut[id].e - coverage_cut[id].s; + + if (!t->ori) { // forward strand + for (i = 0; i < t->len; ++i) + u->s[t->start + i] = g_read.seq[i]; + } else { + for (i = 0; i < t->len; ++i) { + int c = (uint8_t)g_read.seq[g_read.length - 1 - i]; + u->s[t->start + i] = c >= 128? 'N' : comp_tab[c]; + } + } + } + + + free(tmp); + + destory_UC_Read(&g_read); + return 0; +} + +void ma_ug_print(const ma_ug_t *ug, All_reads *RNF, const ma_sub_t *coverage_cut, FILE *fp) +{ + uint32_t i, j, l; + char name[32]; + for (i = 0; i < ug->u.n; ++i) { // the Segment lines in GFA + ma_utg_t *p = &ug->u.a[i]; + sprintf(name, "utg%.6d%c", i + 1, "lc"[p->circ]); + fprintf(fp, "S\t%s\t%s\tLN:i:%d\n", name, p->s? p->s : "*", p->len); + + + + for (j = l = 0; j < p->n; l += (uint32_t)p->a[j++]) { + uint32_t x = p->a[j]>>33; + fprintf(fp, "a\t%s\t%d\t%.*s(%u):%d-%d\t%c\t%d\n", name, l, + Get_NAME_LENGTH((*RNF), x), Get_NAME((*RNF), x), x, + coverage_cut[x].s + 1, coverage_cut[x].e, "+-"[p->a[j]>>32&1], (uint32_t)p->a[j]); + + // if (sub) fprintf(fp, "a\t%s\t%d\t%s:%d-%d\t%c\t%d\n", name, l, d->seq[x].name, sub[x].s + 1, sub[x].e, "+-"[p->a[j]>>32&1], (uint32_t)p->a[j]); + // else fprintf(fp, "a\t%s\t%d\t%s\t%c\t%d\n", name, l, d->seq[x].name, "+-"[p->a[j]>>32&1], (uint32_t)p->a[j]); + + } + + + } + for (i = 0; i < ug->g->n_arc; ++i) { // the Link lines in GFA + uint32_t u = ug->g->arc[i].ul>>32, v = ug->g->arc[i].v; + fprintf(fp, "L\tutg%.6d%c\t%c\tutg%.6d%c\t%c\t%dM\tSD:i:%d\n", (u>>1)+1, "lc"[ug->u.a[u>>1].circ], "+-"[u&1], + (v>>1)+1, "lc"[ug->u.a[v>>1].circ], "+-"[v&1], ug->g->arc[i].ol, asg_arc_len(ug->g->arc[i])); + } + + + /** + for (i = 0; i < ug->u.n; ++i) { // summary of unitigs + uint32_t cnt[2]; + ma_utg_t *u = &ug->u.a[i]; + if (u->start == UINT32_MAX) { + fprintf(fp, "x\tutg%.6dc\t%d\t%d\n", i + 1, u->len, u->n); + } else + { + for (j = 0; j < 2; ++j) cnt[j] = asg_arc_n(ug->g, i<<1|j); + + fprintf(fp, "x\tutg%.6dl\t%d\t%d\t%d\t%d\t%.*s:%d-%d\t%c\t%.*s:%d-%d\t%c\n", + i + 1, u->len, u->n, cnt[1], cnt[0], + ///d->seq[u->start>>1].name, + Get_NAME_LENGTH((*RNF), u->start>>1), Get_NAME((*RNF), u->start>>1), + coverage_cut[u->start>>1].s + 1, coverage_cut[u->start>>1].e, + "+-"[u->start&1], + ///d->seq[u->end>>1].name, + Get_NAME_LENGTH((*RNF), u->end>>1), Get_NAME((*RNF), u->end>>1), + coverage_cut[u->end>>1].s + 1, coverage_cut[u->end>>1].e, + "+-"[u->end&1]); + + // if (sub) + // fprintf(fp, "x\tutg%.6dl\t%d\t%d\t%d\t%d\t%s:%d-%d\t%c\t%s:%d-%d\t%c\n", i + 1, u->len, u->n, cnt[1], cnt[0], + // d->seq[u->start>>1].name, sub[u->start>>1].s + 1, sub[u->start>>1].e, "+-"[u->start&1], + // d->seq[u->end>>1].name, sub[u->end>>1].s + 1, sub[u->end>>1].e, "+-"[u->end&1]); + // else + // fprintf(fp, "x\tutg%.6dl\t%d\t%d\t%d\t%d\t%s\t%c\t%s\t%c\n", i + 1, u->len, u->n, cnt[1], cnt[0], + // d->seq[u->start>>1].name, "+-"[u->start&1], d->seq[u->end>>1].name, "+-"[u->end&1]); + + + } + } + **/ + +} + +int asg_cut_internal(asg_t *g, int max_ext) +{ + asg64_v a = {0,0,0}; + uint32_t n_vtx = g->n_seq * 2, v, i, cnt = 0; + for (v = 0; v < n_vtx; ++v) { + if (g->seq[v>>1].del) continue; + if (asg_is_utg_end(g, v, 0) != ASG_ET_MULTI_NEI) continue; + if (asg_extend(g, v, max_ext, &a) != ASG_ET_MULTI_NEI) continue; + /** + * so combining the last two lines, they are designed to reomve(n(1), n(2))? + -----> <------- + | | + n(0)--->n(1)---->n(2)---->n(3) + | | + ------> <------- + **/ + for (i = 0; i < a.n; ++i) + asg_seq_del(g, (uint32_t)a.a[i]>>1); + ++cnt; + } + free(a.a); + if (cnt > 0) asg_cleanup(g); + fprintf(stderr, "[M::%s] cut %d internal sequences\n", __func__, cnt); + return cnt; +} + + + + + + +void clean_weak_ma_hit_t(ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long num_sources) +{ + double startTime = Get_T(); + long long bi_overlaps = 0; + long long si_overlaps = 0; + long long i, j, index; + uint32_t qn, tn; + ma_hit_t new_element; + long long qLen_0, qLen_1; + for (i = 0; i < num_sources; i++) + { + for (j = 0; j < sources[i].length; j++) + { + qn = Get_qn(sources[i].buffer[j]); + tn = Get_tn(sources[i].buffer[j]); + + //if this is a weak overlap + if(sources[i].buffer[j].ml == 0) + { + if(!check_weak_ma_hit(&(sources[qn]), reverse_sources, tn, + Get_qs(sources[i].buffer[j]), Get_qe(sources[i].buffer[j]))) + { + /** + if(memcmp("m64016_190918_162737/76808505/ccs", + Get_NAME(R_INF, i), Get_NAME_LENGTH(R_INF, i)) == 0) + { + fprintf(stderr, "#### %.*s, %.*s\n", + Get_NAME_LENGTH(R_INF, qn), Get_NAME(R_INF, qn), + Get_NAME_LENGTH(R_INF, tn), Get_NAME(R_INF, tn)); + } + **/ + + + sources[i].buffer[j].bl = 0; + index = get_specific_overlap(&(sources[tn]), tn, qn); + sources[tn].buffer[index].bl = 0; + } + } + } + } + + long long m = 0; + long long pre_overlaps, current_overlaps, exact_overlaps; + exact_overlaps = pre_overlaps = current_overlaps = 0; + for (i = 0; i < num_sources; i++) + { + m = 0; + + for (j = 0; j < sources[i].length; j++) + { + if(sources[i].buffer[j].bl != 0) + { + sources[i].buffer[m] = sources[i].buffer[j]; + if(sources[i].buffer[m].el) + { + exact_overlaps++; + } + m++; + } + } + + pre_overlaps += sources[i].length; + sources[i].length = m; + current_overlaps += sources[i].length; + } + + /** + fprintf(stdout, "pre_overlaps: %lld, current_overlaps: %lld, exact_overlaps: %lld\n", + pre_overlaps, current_overlaps, exact_overlaps); + **/ + + fprintf(stderr, "[M::%s] takes %0.2f s\n\n", __func__, Get_T()-startTime); +} + + +void build_string_graph(int min_dp, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, long long max_hang_length, +long long clean_round, float min_ovlp_drop_ratio, float max_ovlp_drop_ratio, +float final_ovlp_drop_ratio, char* output_file_name, long long bubble_dist) +{ + ma_sub_t* coverage_cut; + normalize_ma_hit_t(sources, n_read); + ///normalize_ma_hit_t_single_side(sources, n_read); + ///debug_normalize_ma_hit_t(sources, n_read); + clean_weak_ma_hit_t(sources, reverse_sources, n_read); + ///debug_normalize_ma_hit_t(sources, n_read); + + ma_hit_sub(min_dp, sources, n_read, readLen, mini_overlap_length, &coverage_cut); + ma_hit_cut(min_dp, sources, n_read, readLen, mini_overlap_length, &coverage_cut); + ///it seems we do not need ma_hit_flt + ma_hit_flt(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); + ///debug_cut_ma_hit_t(sources, n_read, coverage_cut); + ma_hit_contained(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); + ///debug_cut_ma_hit_t(sources, n_read, coverage_cut); + asg_t *sg = NULL; + sg = ma_sg_gen(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); + asg_arc_del_trans(sg, GAP_FUZZ); + asg_cut_tip(sg, MAX_SHORT_TIPS); + + asg_arc_del_triangular_advance(sg, bubble_dist); + + + if (asg_arc_del_short_diploid(sg, final_ovlp_drop_ratio, sources, reverse_sources) != 0) + { + ///asg_cut_tip(sg, MAX_SHORT_TIPS); + + } + + asg_arc_del_triangular_advance(sg, bubble_dist); + + while(asg_cut_tip(sg, MAX_SHORT_TIPS)!=0 && asg_arc_del_triangular_advance(sg, bubble_dist) != 0) + { + ; + } + + + + + + ma_ug_t *ug = NULL; + ug = ma_ug_gen(sg); + ma_ug_seq(ug, &R_INF, coverage_cut, n_read); + + + fprintf(stdout, "Writing GFA to disk ...... \n"); + char* gfa_name = (char*)malloc(strlen(output_file_name)+5); + sprintf(gfa_name, "%s.gfa", output_file_name); + FILE* output_file = fopen(gfa_name, "w"); + ma_ug_print(ug, &R_INF, coverage_cut, output_file); + + + asg_destroy(sg); + ma_ug_destroy(ug); + free(coverage_cut); + + free(gfa_name); + fclose(output_file); +} + + + +void debug_info_of_specfic_node(char* name, asg_t *g, char* command) +{ + fprintf(stderr, "\n\n\n"); + uint32_t v, n_vtx = g->n_seq * 2; + for (v = 0; v < n_vtx; ++v) + { + if(memcmp(name, Get_NAME(R_INF, (v>>1)), Get_NAME_LENGTH(R_INF, (v>>1))) == 0) + { + fprintf(stderr, "\nafter %s\n****************graph ref_read: %.*s, dir: %u****************\n", + command, Get_NAME_LENGTH(R_INF, (v>>1)), Get_NAME(R_INF, (v>>1)), v&1); + if(g->seq[v>>1].del) + { + fprintf(stderr, "read has already been deleted.\n"); + continue; + } + + asg_arc_t *av = asg_arc_a(g, v); + uint32_t i, nv = asg_arc_n(g, v); + for (i = 0; i < nv; ++i) + { + fprintf(stderr, "target: %.*s, el: %u, strong: %u, ol: %u, del: %u\n", + Get_NAME_LENGTH(R_INF, (av[i].v>>1)), + Get_NAME(R_INF, (av[i].v>>1)), + av[i].el, av[i].strong, av[i].ol, av[i].del); + } + } + } +} + +void debug_info_of_specfic_read(char* name, ma_hit_t_alloc* sources, +ma_hit_t_alloc* reverse_sources, int id, char* command) +{ + long long i, j, Len; + uint32_t qn, tn; + + if(id == -1) + { + i = 0; + Len = R_INF.total_reads; + } + else + { + i = id; + Len = id + 1; + } + + + for (; i < Len; i++) + { + if(memcmp(name, Get_NAME(R_INF, i), Get_NAME_LENGTH(R_INF, i)) == 0) + { + fprintf(stderr, "\n\n\nafter %s\n", command); + + fprintf(stderr, "****************ma_hit_t ref_read: %.*s****************\n", + Get_NAME_LENGTH(R_INF, i), Get_NAME(R_INF, i)); + + + fprintf(stderr, "sources Len: %d, is_fully_corrected: %d\n", + sources[i].length, sources[i].is_fully_corrected); + + for (j = 0; j < sources[i].length; j++) + { + qn = Get_qn(sources[i].buffer[j]); + tn = Get_tn(sources[i].buffer[j]); + fprintf(stderr, "target: %.*s, qs: %d, qe: %d, ts: %d, te: %d, ml: %d, rev: %d\n", + Get_NAME_LENGTH(R_INF, tn), Get_NAME(R_INF, tn), + Get_qs(sources[i].buffer[j]), + Get_qe(sources[i].buffer[j]), + Get_ts(sources[i].buffer[j]), + Get_te(sources[i].buffer[j]), + sources[i].buffer[j].ml, + sources[i].buffer[j].rev); + + + + /** + //if this is a weak overlap + if(sources[i].buffer[j].ml == 0) + { + if(!check_weak_ma_hit(&(sources[qn]), reverse_sources, tn, + Get_qs(sources[i].buffer[j]), Get_qe(sources[i].buffer[j]))) + { + int c_id = check_weak_ma_hit_debug(&(sources[qn]), reverse_sources, tn); + fprintf(stderr, "*************************conflict with %.*s\n", + Get_NAME_LENGTH(R_INF, c_id), Get_NAME(R_INF, c_id)); + } + } + **/ + } + + + + fprintf(stderr, "######reverse_query_read Len: %d\n", reverse_sources[i].length); + + + + for (j = 0; j < reverse_sources[i].length; j++) + { + qn = Get_qn(reverse_sources[i].buffer[j]); + tn = Get_tn(reverse_sources[i].buffer[j]); + fprintf(stderr, "target: %.*s, qs: %u, qe: %u, ts: %u, te: %u\n", + Get_NAME_LENGTH(R_INF, tn), Get_NAME(R_INF, tn), + Get_qs(reverse_sources[i].buffer[j]), + Get_qe(reverse_sources[i].buffer[j]), + Get_ts(reverse_sources[i].buffer[j]), + Get_te(reverse_sources[i].buffer[j])); + } + + } + } + + + + +} + +void ma_sg_print(const asg_t *g, const All_reads *RNF, const ma_sub_t *sub, FILE *fp) +{ + uint32_t i; + for (i = 0; i < g->n_arc; ++i) { + const asg_arc_t *p = &g->arc[i]; + if (sub) { + const ma_sub_t *sq = &sub[p->ul>>33], *st = &sub[p->v>>1]; + /** + fprintf(fp, "L\t%s:%d-%d\t%c\t%s:%d-%d\t%c\t%d:\tL1:i:%d\n", + d->seq[p->ul>>33].name, sq->s + 1, sq->e, "+-"[p->ul>>32&1], + d->seq[p->v>>1].name, st->s + 1, st->e, "+-"[p->v&1], p->ol, (uint32_t)p->ul); + **/ + /** + fprintf(stderr, "Get_NAME_LENGTH((*RNF), p->ul>>33): %u, p->ul>>33: %u\n", + Get_NAME_LENGTH((*RNF), (p->ul>>33)), + (p->ul>>33)); + + fprintf(stderr, "Get_NAME_LENGTH((*RNF), p->v>>1): %u, p->v>>1: %u\n", + Get_NAME_LENGTH((*RNF), (p->v>>1)), + (p->v>>1)); + **/ + + fprintf(fp, + "L\t%.*s:%d-%d\t%c\t%.*s:%d-%d\t%c\t%d:\tL1:i:%d\n", + Get_NAME_LENGTH((*RNF), p->ul>>33), + Get_NAME((*RNF), p->ul>>33), + sq->s + 1, sq->e, "+-"[p->ul>>32&1], + Get_NAME_LENGTH((*RNF), p->v>>1), + Get_NAME((*RNF), p->v>>1), + st->s + 1, st->e, "+-"[p->v&1], p->ol, (uint32_t)p->ul); + + + } + else + { + /** + fprintf(fp, "L\t%s\t%c\t%s\t%c\t%d:\tL1:i:%d\n", + d->seq[p->ul>>33].name, "+-"[p->ul>>32&1], + d->seq[p->v>>1].name, "+-"[p->v&1], p->ol, (uint32_t)p->ul); + **/ + fprintf(fp, "L\t%.*s\t%c\t%.*s\t%c\t%d:\tL1:i:%d\n", + Get_NAME_LENGTH((*RNF), p->ul>>33), + Get_NAME((*RNF), p->ul>>33), + "+-"[p->ul>>32&1], + Get_NAME_LENGTH((*RNF), p->v>>1), + Get_NAME((*RNF), p->v>>1), + "+-"[p->v&1], p->ol, (uint32_t)p->ul); + } + } +} + + +void ma_ug_print_simple(const ma_ug_t *ug, All_reads *RNF, const ma_sub_t *coverage_cut, FILE *fp) +{ + uint32_t i, j, l; + char name[32]; + for (i = 0; i < ug->u.n; ++i) { // the Segment lines in GFA + ma_utg_t *p = &ug->u.a[i]; + sprintf(name, "utg%.6d%c", i + 1, "lc"[p->circ]); + fprintf(fp, "S\t%s\t%s\tLN:i:%d\n", name, "*", p->len); + } + for (i = 0; i < ug->g->n_arc; ++i) { // the Link lines in GFA + uint32_t u = ug->g->arc[i].ul>>32, v = ug->g->arc[i].v; + fprintf(fp, "L\tutg%.6d%c\t%c\tutg%.6d%c\t%c\t%dM\tSD:i:%d\n", (u>>1)+1, "lc"[ug->u.a[u>>1].circ], "+-"[u&1], + (v>>1)+1, "lc"[ug->u.a[v>>1].circ], "+-"[v&1], ug->g->arc[i].ol, asg_arc_len(ug->g->arc[i])); + } +} + +void output_unitig_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, long long n_read) +{ + ma_ug_t *ug = NULL; + ug = ma_ug_gen(sg); + ma_ug_seq(ug, &R_INF, coverage_cut, n_read); + + fprintf(stdout, "Writing unitig GFA to disk ...... \n"); + char* gfa_name = (char*)malloc(strlen(output_file_name)+25); + sprintf(gfa_name, "%s.gfa", output_file_name); + FILE* output_file = fopen(gfa_name, "w"); + ma_ug_print(ug, &R_INF, coverage_cut, output_file); + fclose(output_file); + + + + + + sprintf(gfa_name, "%s.simple.gfa", output_file_name); + output_file = fopen(gfa_name, "w"); + ma_ug_print_simple(ug, &R_INF, coverage_cut, output_file); + fclose(output_file); + + free(gfa_name); + ma_ug_destroy(ug); +} + + +void output_read_graph(asg_t *sg, ma_sub_t* coverage_cut, char* output_file_name, long long n_read) +{ + fprintf(stdout, "Writing read GFA to disk ...... \n"); + char* gfa_name = (char*)malloc(strlen(output_file_name)+25); + sprintf(gfa_name, "%s.read.gfa", output_file_name); + FILE* output_file = fopen(gfa_name, "w"); + ma_sg_print(sg, &R_INF, coverage_cut, output_file); + free(gfa_name); + fclose(output_file); +} + +void read_ma(ma_hit_t* x, FILE* fp) +{ + fread(&(x->qns), sizeof(x->qns), 1, fp); + fread(&(x->qe), sizeof(x->qe), 1, fp); + fread(&(x->tn), sizeof(x->tn), 1, fp); + fread(&(x->ts), sizeof(x->ts), 1, fp); + fread(&(x->te), sizeof(x->te), 1, fp); + fread(&(x->el), sizeof(x->el), 1, fp); + + uint32_t t; + fread(&(t), sizeof(t), 1, fp); + x->ml = t; + + fread(&(t), sizeof(t), 1, fp); + x->rev = t; + + + fread(&(t), sizeof(t), 1, fp); + x->bl = t; + + fread(&(t), sizeof(t), 1, fp); + x->del = t; +} + +int load_ma_hit_ts(ma_hit_t_alloc** x, char* read_file_name) +{ + fprintf(stdout, "Loading ma_hit_ts to disk ...... \n"); + char* index_name = (char*)malloc(strlen(read_file_name)+15); + sprintf(index_name, "%s.bin", read_file_name); + FILE* fp = fopen(index_name, "r"); + if(!fp) + { + return 0; + } + + + long long n_read; + long long i, k; + fread(&n_read, sizeof(n_read), 1, fp); + (*x) = (ma_hit_t_alloc*)malloc(sizeof(ma_hit_t_alloc)*n_read); + + + for (i = 0; i < n_read; i++) + { + fread(&((*x)[i].is_fully_corrected), sizeof((*x)[i].is_fully_corrected), 1, fp); + fread(&((*x)[i].length), sizeof((*x)[i].length), 1, fp); + + (*x)[i].buffer = (ma_hit_t*)malloc(sizeof(ma_hit_t)*(*x)[i].length); + + for (k = 0; k < (*x)[i].length; k++) + { + read_ma(&((*x)[i].buffer[k]), fp); + } + } + + free(index_name); + fclose(fp); + fprintf(stdout, "ma_hit_ts has been read.\n"); +} + + +void write_ma(ma_hit_t* x, FILE* fp) +{ + fwrite(&(x->qns), sizeof(x->qns), 1, fp); + fwrite(&(x->qe), sizeof(x->qe), 1, fp); + fwrite(&(x->tn), sizeof(x->tn), 1, fp); + fwrite(&(x->ts), sizeof(x->ts), 1, fp); + fwrite(&(x->te), sizeof(x->te), 1, fp); + fwrite(&(x->el), sizeof(x->el), 1, fp); + + uint32_t t = x->ml; + fwrite(&(t), sizeof(t), 1, fp); + t = x->rev; + fwrite(&(t), sizeof(t), 1, fp); + + t = x->bl; + fwrite(&(t), sizeof(t), 1, fp); + t =x->del; + fwrite(&(t), sizeof(t), 1, fp); +} + + +void write_ma_hit_ts(ma_hit_t_alloc* x, long long n_read, char* read_file_name) +{ + fprintf(stdout, "Writing ma_hit_ts to disk ...... \n"); + char* index_name = (char*)malloc(strlen(read_file_name)+15); + sprintf(index_name, "%s.bin", read_file_name); + FILE* fp = fopen(index_name, "w"); + long long i, k; + fwrite(&n_read, sizeof(n_read), 1, fp); + + + for (i = 0; i < n_read; i++) + { + fwrite(&(x[i].is_fully_corrected), sizeof(x[i].is_fully_corrected), 1, fp); + fwrite(&(x[i].length), sizeof(x[i].length), 1, fp); + for (k = 0; k < x[i].length; k++) + { + write_ma(x[i].buffer + k, fp); + } + } + + + free(index_name); + fflush(fp); + fclose(fp); + fprintf(stdout, "ma_hit_ts has been written.\n"); +} + +void write_all_data_to_disk(ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, +All_reads *RNF, char* output_file_name) +{ + char* gfa_name = (char*)malloc(strlen(output_file_name)+25); + sprintf(gfa_name, "%s.gfa.aux", output_file_name); + write_All_reads(RNF, gfa_name); + + sprintf(gfa_name, "%s.gfa.aux.source", output_file_name); + write_ma_hit_ts(sources, RNF->total_reads, gfa_name); + + sprintf(gfa_name, "%s.gfa.aux.reverse", output_file_name); + write_ma_hit_ts(reverse_sources, RNF->total_reads, gfa_name); + + free(gfa_name); +} + +int load_all_data_from_disk(ma_hit_t_alloc **sources, ma_hit_t_alloc **reverse_sources, +char* output_file_name) +{ + char* gfa_name = (char*)malloc(strlen(output_file_name)+25); + sprintf(gfa_name, "%s.gfa.aux", output_file_name); + if(!load_All_reads(&R_INF, gfa_name)) + { + return 0; + } + + sprintf(gfa_name, "%s.gfa.aux.source", output_file_name); + if(!load_ma_hit_ts(sources, gfa_name)) + { + return 0; + } + + + sprintf(gfa_name, "%s.gfa.aux.reverse", output_file_name); + if(!load_ma_hit_ts(reverse_sources, gfa_name)) + { + return 0; + } + + return 1; +} + + + +void build_string_graph_without_clean( +int min_dp, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, +long long n_read, uint64_t* readLen, +long long mini_overlap_length, long long max_hang_length, long long clean_round, +float min_ovlp_drop_ratio, float max_ovlp_drop_ratio, float corase_ovlp_drop_ratio, +char* output_file_name, long long bubble_dist, int read_graph, int write) +{ + + if (write_index_to_disk && write) + { + write_all_data_to_disk(sources, reverse_sources, + &R_INF, output_file_name); + } + + + // debug_info_of_specfic_read("m64016_190918_162737/72220752/ccs", + // sources, reverse_sources, -1, "init"); + + ma_sub_t* coverage_cut; + normalize_ma_hit_t(sources, n_read); + ///normalize_ma_hit_t_single_side(sources, n_read); + + + // debug_info_of_specfic_read("m64016_190918_162737/72220752/ccs", + // sources, reverse_sources, -1, "normalize"); + + + + + + + + + ///debug_normalize_ma_hit_t(sources, n_read); + clean_weak_ma_hit_t(sources, reverse_sources, n_read); + ///debug_normalize_ma_hit_t(sources, n_read); + + + + + // debug_info_of_specfic_read("m64016_190918_162737/72220752/ccs", + // sources, reverse_sources, -1, "clean"); + + + + + + ma_hit_sub(min_dp, sources, n_read, readLen, mini_overlap_length, &coverage_cut); + ma_hit_cut(min_dp, sources, n_read, readLen, mini_overlap_length, &coverage_cut); + ///it seems we do not need ma_hit_flt + ma_hit_flt(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); + ///debug_cut_ma_hit_t(sources, n_read, coverage_cut); + ma_hit_contained(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); + ///debug_cut_ma_hit_t(sources, n_read, coverage_cut); + + + + + + // debug_info_of_specfic_read("m64016_190918_162737/72220752/ccs", + // sources, reverse_sources, -1, "contain"); + + + + + + + asg_t *sg = NULL; + sg = ma_sg_gen(sources, n_read, coverage_cut, max_hang_length, mini_overlap_length); + + + // debug_info_of_specfic_node("m64016_190918_162737/72220752/ccs", sg, "sg_gen"); + + + + + + + + asg_arc_del_trans(sg, GAP_FUZZ); + + // debug_info_of_specfic_node("m64016_190918_162737/72220752/ccs", sg, "del_trans"); + + ///goto out; + + asg_cut_tip(sg, MAX_SHORT_TIPS); + + + // debug_info_of_specfic_node("m64016_190918_162737/72220752/ccs", sg, "cut_tip"); + + + + + + asg_arc_del_short_diploid_unclean(sg, corase_ovlp_drop_ratio, sources, reverse_sources); + + + + // debug_info_of_specfic_node("m64016_190918_162737/72220752/ccs", sg, "cut_corase"); + + + + + // asg_arc_del_single_node_bubble(sg, bubble_dist); + // asg_cut_tip(sg, MAX_SHORT_TIPS); + asg_cut_tip(sg, MAX_SHORT_TIPS); + + ///clean_round = 0; + // fprintf(stderr, "\n\nWill perform %d round of clean...**********\n", + // clean_round); + + + if(clean_round > 0) + { + double cut_step; + if(clean_round == 1) + { + cut_step = max_ovlp_drop_ratio; + } + else + { + cut_step = (max_ovlp_drop_ratio - min_ovlp_drop_ratio) / (clean_round - 1); + } + double drop_ratio = min_ovlp_drop_ratio; + int i = 0; + for (i = 0; i < clean_round; i++, drop_ratio += cut_step) + { + if(drop_ratio > max_ovlp_drop_ratio) + { + drop_ratio = max_ovlp_drop_ratio; + } + + fprintf(stderr, "\n\n**********%d-th round drop: drop_ratio = %f**********\n", + i, drop_ratio); + + // debug_info_of_specfic_node("m64016_190918_162737/72220752/ccs", sg, "step"); + + + while(1) + { + int tri_flag = 0; + tri_flag += asg_arc_del_self_circle_contig(sg); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + ///asg_arc_del_single_node_bubble(sg, bubble_dist); + tri_flag += asg_arc_del_single_node_directly(sg, MAX_SHORT_TIPS); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + tri_flag += asg_arc_del_triangular_advance(sg, bubble_dist); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + tri_flag += asg_arc_del_cross_bubble(sg, bubble_dist); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + ///asg_arc_del_single_node_bubble(sg, bubble_dist); + tri_flag += asg_arc_del_single_node_directly(sg, MAX_SHORT_TIPS); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + + if(tri_flag == 0) + { + break; + } + } + + + + /****************************may have bugs********************************/ + //asg_arc_identify_simple_bubbles(sg); + asg_arc_identify_simple_bubbles_multi(sg, 1); + asg_arc_del_false_node(sg, MAX_SHORT_TIPS); + asg_cut_tip(sg, MAX_SHORT_TIPS); + /****************************may have bugs********************************/ + + /****************************may have bugs********************************/ + //asg_arc_identify_simple_bubbles(sg); + asg_arc_identify_simple_bubbles_multi(sg, 1); + ///asg_arc_del_short_diploid_unclean_exact(sg, drop_ratio, sources); + asg_arc_del_short_diploid_by_exact(sg, MAX_SHORT_TIPS, sources); + asg_cut_tip(sg, MAX_SHORT_TIPS); + /****************************may have bugs********************************/ + + + // fprintf(stderr, "bugs\n"); + // fflush(stderr); + + //asg_arc_identify_simple_bubbles(sg); + asg_arc_identify_simple_bubbles_multi(sg, 1); + + // fprintf(stderr, "asg_arc_identify_simple_bubbles_multi\n"); + // fflush(stderr); + + + asg_arc_del_short_diploid_by_length(sg, drop_ratio, MAX_SHORT_TIPS); + asg_cut_tip(sg, MAX_SHORT_TIPS); + + + // fprintf(stderr, "asg_arc_del_short_diploid_by_length\n"); + // fflush(stderr); + + + // debug_info_of_specfic_node("m64016_190918_162737/141297762/ccs", sg); + + + + + asg_arc_identify_simple_bubbles_multi(sg, 0); + asg_arc_del_short_false_link(sg, 0.6, bubble_dist); + + asg_arc_identify_simple_bubbles_multi(sg, 0); + asg_arc_del_complex_false_link(sg, 0.6, bubble_dist); + + asg_cut_tip(sg, MAX_SHORT_TIPS); + } + } + + + fprintf(stderr, "\n\n**********final clean**********\n"); + + ///debug_info_of_specfic_node("m64016_190918_162737/72220752/ccs", sg, "before final clean"); + + while(1) + { + int tri_flag = 0; + tri_flag += asg_arc_del_self_circle_contig(sg); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + ///asg_arc_del_single_node_bubble(sg, bubble_dist); + tri_flag += asg_arc_del_single_node_directly(sg, MAX_SHORT_TIPS); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + tri_flag += asg_arc_del_triangular_advance(sg, bubble_dist); + ///tri_flag += asg_arc_del_triangular_advance_debug(sg, bubble_dist); + + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + tri_flag += asg_arc_del_cross_bubble(sg, bubble_dist); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + ///asg_arc_del_single_node_bubble(sg, bubble_dist); + tri_flag += asg_arc_del_single_node_directly(sg, MAX_SHORT_TIPS); + // fprintf(stderr, "tri_flag: %d\n", tri_flag); + // fflush(stderr); + + if(tri_flag == 0) + { + break; + } + } + + + ///asg_arc_del_triangular_advance_debug(sg, bubble_dist); + + /** + fprintf(stderr, "\n\n**********final aggressive clean**********\n"); + while(1) + { + int tri_flag = 0; + asg_arc_identify_simple_bubbles_multi(sg, 0); + tri_flag = asg_arc_del_tri_link(sg, bubble_dist); + if(tri_flag == 0) + { + break; + } + } + **/ + + + + + + + /****************************may have bugs********************************/ + /** + long long c_tips = 1; + int i = 0; + while (c_tips && i < clean_round) + { + asg_arc_identify_simple_bubbles_multi(sg, 0); + c_tips = asg_arc_del_short_false_link(sg, 0.7, bubble_dist); + + asg_arc_identify_simple_bubbles_multi(sg, 0); + c_tips += asg_arc_del_complex_false_link(sg, 0.7, bubble_dist); + + if(c_tips) asg_cut_tip(sg, MAX_SHORT_TIPS); + i++; + } + **/ + + /****************************may have bugs********************************/ + + + // debug_info_of_specfic_node("m64016_190918_162737/141297762/ccs", sg); + + out: + + output_unitig_graph(sg, coverage_cut, output_file_name, n_read); + output_read_graph(sg, coverage_cut, output_file_name, n_read); + + + asg_destroy(sg); + free(coverage_cut); +} \ No newline at end of file diff --git a/Overlaps.h b/Overlaps.h new file mode 100644 index 0000000..f07872b --- /dev/null +++ b/Overlaps.h @@ -0,0 +1,322 @@ +#ifndef __OVERLAPS__ +#define __OVERLAPS__ +#include +#include "kvec.h" +#include "kdq.h" +///#include "Hash_Table.h" + +///#define MIN_OVERLAP_LEN 2000 +///#define MIN_OVERLAP_LEN 500 +#define MIN_OVERLAP_LEN 50 +#define MIN_OVERLAP_COVERAGE 1 +#define MAX_HANG_LEN 1000 +#define MAX_HANG_PRE 0.8 +#define GAP_FUZZ 1000 +#define MAX_SHORT_TIPS 3 +#define MAX_BUBBLE_DIST 10000000 +#define SMALL_BUBBLE_SIZE (uint32_t)-1 +//#define SMALL_BUBBLE_SIZE 1000 + + +#define Get_qn(RECORD) ((uint32_t)((RECORD).qns>>32)) +#define Get_qs(RECORD) ((uint32_t)((RECORD).qns)) +#define Get_qe(RECORD) ((RECORD).qe) +#define Get_tn(RECORD) ((RECORD).tn) +#define Get_ts(RECORD) ((RECORD).ts) +#define Get_te(RECORD) ((RECORD).te) + +#define LONG_TIPS 0 +#define TWO_INPUT 1 +#define TWO_OUTPUT 2 +#define MUL_INPUT 3 +#define MUL_OUTPUT 4 +#define END_TIPS 5 +#define LONG_TIPS_UNDER_MAX_EXT 6 +#define LOOP 7 + + +///query is the read itself +typedef struct { + uint64_t qns; + uint32_t qe, tn, ts, te; + uint32_t ml:31, rev:1; + uint32_t bl:31, del:1; + uint8_t el; +} ma_hit_t; + + +typedef struct { + ma_hit_t* buffer; + uint32_t size; + uint32_t length; + uint8_t is_fully_corrected; +} ma_hit_t_alloc; + + +void init_ma_hit_t_alloc(ma_hit_t_alloc* x); +void clear_ma_hit_t_alloc(ma_hit_t_alloc* x); +void resize_ma_hit_t_alloc(ma_hit_t_alloc* x, uint64_t size); +void destory_ma_hit_t_alloc(ma_hit_t_alloc* x); +void add_ma_hit_t_alloc(ma_hit_t_alloc* x, ma_hit_t* element); +void ma_hit_sort_tn(ma_hit_t *a, long long n); +void ma_hit_sort_qns(ma_hit_t *a, long long n); + +int load_all_data_from_disk(ma_hit_t_alloc **sources, ma_hit_t_alloc **reverse_sources, +char* output_file_name); + + +typedef struct { + ma_hit_t_alloc overlaps; +} Assembly_Graph; + +void init_Assembly_Graph(Assembly_Graph* x); +void destory_Assembly_Graph(Assembly_Graph* x); +void collect_ma_hit_t(ma_hit_t_alloc* dest, ma_hit_t_alloc* sources, long long num_sources); +void normalize_ma_hit_t(ma_hit_t_alloc* sources, long long num_sources); +void debug_normalize_ma_hit_t(ma_hit_t_alloc* sources, long long num_sources); + + +typedef struct { + uint32_t s:31, del:1, e; +} ma_sub_t; + +void ma_hit_sub(int min_dp, ma_hit_t_alloc* sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, ma_sub_t** coverage_cut); +void ma_hit_cut(int min_dp, ma_hit_t_alloc* sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, ma_sub_t** coverage_cut); +void ma_hit_flt(ma_hit_t_alloc* sources, long long n_read, const ma_sub_t *coverage_cut, +int max_hang, int min_ovlp); +long long get_specific_overlap(ma_hit_t_alloc* x, uint32_t qn, uint32_t tn); + +void debug_cut_ma_hit_t(ma_hit_t_alloc* sources, long long num_sources, ma_sub_t *coverage_cut); + +typedef struct { + uint64_t ul; + uint32_t v; + uint32_t ol:31, del:1; + uint8_t strong; + uint8_t el; +} asg_arc_t; + +typedef struct { + uint32_t len:31, del:1; +} asg_seq_t; + +typedef struct { + uint32_t m_arc, n_arc:31, is_srt:1; + asg_arc_t *arc; + uint32_t m_seq, n_seq:31, is_symm:1; + asg_seq_t *seq; + uint64_t *idx; + + uint8_t* seq_vis; +} asg_t; + +typedef struct { size_t n, m; uint64_t *a; } asg64_v; + + +#define MA_HT_INT (-1) +#define MA_HT_QCONT (-2) +#define MA_HT_TCONT (-3) +#define MA_HT_SHORT_OVLP (-4) + +///in default, max_hang = 1000, int_frac = 0.05, min_ovlp = 2000 +static inline int ma_hit2arc(const ma_hit_t *h, int ql, int tl, int max_hang, float int_frac, int min_ovlp, asg_arc_t *p) +{ + int32_t tl5, tl3, ext5, ext3, qs = (int32_t)h->qns; + uint32_t u, v, l; // u: query end; v: target end; l: length from u to v + + ///if query and target are in different strand + if (h->rev) tl5 = tl - h->te, tl3 = h->ts; // tl5: 5'-end overhang (on the query strand); tl3: similar + else tl5 = h->ts, tl3 = tl - h->te; + + ///ext5 and ext3 is the hang on left side and right side, respectively + ext5 = qs < tl5? qs : tl5; + ext3 = ql - h->qe < tl3? ql - h->qe : tl3; + + + /** + if (ext5 > max_hang || ext3 > max_hang || h->qe - qs < (h->qe - qs + ext5 + ext3) * int_frac) + return MA_HT_INT; + **/ + if (ext5 > max_hang || ext3 > max_hang + || h->qe - qs < (h->qe - qs + ext5 + ext3) * int_frac + || h->te - h->ts < (h->te - h->ts + ext5 + ext3) * int_frac) + { + return MA_HT_INT; + } + + /** + ********************************query-to-target overlap**************************** + case 1: u = 0, rev = 0 in the view of target: direction is 1 + query: CCCCCCCCTAATTAAAAT target: TAATTAAAATGGGGGG (use ex-target as query) + |||||||||| <---> |||||||||| + target: TAATTAAAATGGGGGG query: CCCCCCCCTAATTAAAAT (use ex-query as target) + + case 2: u = 0, rev = 1 in the view of target: direction is 0 + query: CCCCCCCCTAATTAAAAT target: CCCCCCATTTTAATTA (use ex-target as query) + |||||||||| <---> |||||||||| + target: TAATTAAAATGGGGGG query: ATTTTAATTAGGGGGGGG (use ex-query as target) + ********************************query-to-target overlap**************************** + + ********************************target-to-query overlap**************************** + case 3: u = 1, rev = 0 in the view of target: direction is 0 + query: AAATAATATCCCCCCGCG target: GGGCCGGCAAATAATAT (use ex-target as query) + ||||||||| <---> ||||||||| + target: GGGCCGGCAAATAATAT query: AAATAATATCCCCCCGCG (use ex-query as target) + + case 4: u = 1, rev = 1 in the view of target: direction is 1 + query: AAATAATATCCCCCCGCG target: ATATTATTTGCCGGCCC (use ex-target as query) + ||||||||| <---> ||||||||| + target: GGGCCGGCAAATAATAT query: CGCGGGGGATATTATTT (use ex-query as target) + ********************************target-to-query overlap**************************** + **/ + + if (qs <= tl5 && ql - h->qe <= tl3) return MA_HT_QCONT; // query contained in target + else if (qs >= tl5 && ql - h->qe >= tl3) return MA_HT_TCONT; // target contained in query + else if (qs > tl5) u = 0, v = !!h->rev, l = qs - tl5; ///u = 0 means query-to-target overlap, l is the length of node in string graph (not the overlap length) + else u = 1, v = !h->rev, l = (ql - h->qe) - tl3; ///u = 1 means target-to-query overlaps, l is the length of node in string graph (not the overlap length) + if (h->qe - qs + ext5 + ext3 < min_ovlp || h->te - h->ts + ext5 + ext3 < min_ovlp) return MA_HT_SHORT_OVLP; // short overlap + ///u = 0 / 1 means query-to-target / target-to-query overlaps, + ///l is the length of node in string graph (not the overlap length between two reads) + u |= h->qns>>32<<1, v |= h->tn<<1; + /** + p->ul: |____________31__________|__________1___________|______________32_____________| + qn direction of overlap length of this node (not overlap length) + (in the view of query) + p->v : |___________31___________|__________1___________| + tn reverse direction of overlap + (in the view of target) + p->ol: overlap length + **/ + p->ul = (uint64_t)u<<32 | l, p->v = v, p->ol = ql - l, p->del = 0; + ///l is the length of node in string graph (not the overlap length) + + p->strong = h->ml; + p->el = h->el; + return l; +} + + +void build_string_graph(int min_dp, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, long long max_hang_length, +long long clean_round, float min_ovlp_drop_ratio, float max_ovlp_drop_ratio, +float final_ovlp_drop_ratio, char* output_file_name, long long bubble_dist); + + +#define asg_arc_len(arc) ((uint32_t)(arc).ul) +#define asg_arc_n(g, v) ((uint32_t)(g)->idx[(v)]) +#define asg_arc_a(g, v) (&(g)->arc[(g)->idx[(v)]>>32]) + +// append an arc +static inline asg_arc_t *asg_arc_pushp(asg_t *g) +{ + if (g->n_arc == g->m_arc) { + g->m_arc = g->m_arc? g->m_arc<<1 : 16; + g->arc = (asg_arc_t*)realloc(g->arc, g->m_arc * sizeof(asg_arc_t)); + } + return &g->arc[g->n_arc++]; +} + +// set asg_arc_t::del for v->w +static inline void asg_arc_del(asg_t *g, uint32_t v, uint32_t w, int del) +{ + uint32_t i, nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + for (i = 0; i < nv; ++i) + if (av[i].v == w) av[i].del = !!del; +} + +// set asg_arc_t::del and asg_seq_t::del to 1 for sequence s and all its associated arcs +static inline void asg_seq_del(asg_t *g, uint32_t s) +{ + uint32_t k; + g->seq[s].del = 1; + for (k = 0; k < 2; ++k) { + uint32_t i, v = s<<1 | k; + uint32_t nv = asg_arc_n(g, v); + asg_arc_t *av = asg_arc_a(g, v); + for (i = 0; i < nv; ++i) { + av[i].del = 1; + asg_arc_del(g, av[i].v^1, v^1, 1); + } + } +} + + +typedef struct { + uint32_t len:31, circ:1; // len: length of the unitig; circ: circular if non-zero + uint32_t start, end; // start: starting vertex in the string graph; end: ending vertex + uint32_t m, n; // number of reads + uint64_t *a; // list of reads + char *s; // unitig sequence is not null +} ma_utg_t; + +typedef struct { size_t n, m; ma_utg_t *a; } ma_utg_v; + +typedef struct { + ma_utg_v u; + asg_t *g; +} ma_ug_t; + +typedef struct { + uint32_t utg:31, ori:1, start, len; +} utg_intv_t; + + +/****************** + * Bubble popping * + ******************/ + +typedef struct { + uint32_t p; // the optimal parent vertex + uint32_t d; // the shortest distance from the initial vertex + uint32_t c; // max count of reads + uint32_t r:31, s:1; // r: the number of remaining incoming arc; s: state + //s: state, s=0, this edge has not been visited, otherwise, s=1 +} binfo_t; + +typedef struct { + ///all information for each node + binfo_t *a; + kvec_t(uint32_t) S; // set of vertices without parents, nodes with all incoming edges visited + kvec_t(uint32_t) T; // set of tips + kvec_t(uint32_t) b; // visited vertices + kvec_t(uint32_t) e; // visited edges/arcs +} buf_t; + +// count the number of outgoing arcs, including reduced arcs +static inline int count_out_with_del(const asg_t *g, uint32_t v) +{ + uint32_t i, n, nv = asg_arc_n(g, v); + const asg_arc_t *av = asg_arc_a(g, v); + /** + for (i = n = 0; i < nv; ++i) + if (!av[i].del) ++n; + return n; + **/ + return nv; +} + + +// count the number of outgoing arcs, including reduced arcs +static inline int count_out_without_del(const asg_t *g, uint32_t v) +{ + uint32_t i, n, nv = asg_arc_n(g, v); + const asg_arc_t *av = asg_arc_a(g, v); + + for (i = n = 0; i < nv; ++i) + if (!av[i].del) ++n; + return n; +} + +void debug_info_of_specfic_read(char* name, ma_hit_t_alloc* sources, +ma_hit_t_alloc* reverse_sources, int id, char* fun); + +void build_string_graph_without_clean(int min_dp, ma_hit_t_alloc* sources, ma_hit_t_alloc* reverse_sources, long long n_read, uint64_t* readLen, +long long mini_overlap_length, long long max_hang_length, +long long clean_round, float min_ovlp_drop_ratio, float max_ovlp_drop_ratio, +float corase_ovlp_drop_ratio, char* output_file_name, long long bubble_dist, int read_graph, +int write); + +#endif \ No newline at end of file diff --git a/POA.cpp b/POA.cpp index 508a900..403482d 100644 --- a/POA.cpp +++ b/POA.cpp @@ -15,11 +15,13 @@ void init_Edge_alloc(Edge_alloc* list) { list->size = INIT_EDGE_SIZE; list->length = 0; + list->delete_length = 0; list->list = (Edge*)malloc(sizeof(Edge)*list->size); } else { list->length = 0; + list->delete_length = 0; } } @@ -27,6 +29,7 @@ void init_Edge_alloc(Edge_alloc* list) void clear_Edge_alloc(Edge_alloc* list) { list->length = 0; + list->delete_length = 0; } void destory_Edge_alloc(Edge_alloc* list) @@ -47,17 +50,138 @@ void append_Edge_alloc(Edge_alloc* list, uint64_t in_node, uint64_t out_node, u list->list[list->length].weight = weight; list->list[list->length].length = length; list->list[list->length].num_insertions = 0; + list->list[list->length].self_edge_ID = list->length; list->length++; } +int add_and_check_bi_direction_edge(Graph* graph, Node* in_node, Node* out_node, uint64_t weight, uint64_t flag) +{ + Edge* e_forward; + Edge* e_backward; + + //if there are no edge from in_node to out_node + if(!get_bi_Edge(graph, in_node, out_node, &e_forward, &e_backward)) + { + append_Edge_alloc(&(Output_Edges((*in_node))), (*in_node).ID, (*out_node).ID, weight, flag); + append_Edge_alloc(&(Input_Edges((*out_node))), (*in_node).ID, (*out_node).ID, weight, flag); + + Output_Edges((*in_node)).list[Output_Edges((*in_node)).length - 1].reverse_edge_ID + = Input_Edges((*out_node)).length - 1; + + Input_Edges((*out_node)).list[Input_Edges((*out_node)).length - 1].reverse_edge_ID + = Output_Edges((*in_node)).length - 1; + + return 1; + } + else//if there is an edge from in_node to out_node, do nothing + { + return 0; + } +} + + +void add_bi_direction_edge(Graph* graph, Node* in_node, Node* out_node, uint64_t weight, uint64_t flag) +{ + + append_Edge_alloc(&(Output_Edges((*in_node))), (*in_node).ID, (*out_node).ID, weight, flag); + append_Edge_alloc(&(Input_Edges((*out_node))), (*in_node).ID, (*out_node).ID, weight, flag); + + Output_Edges((*in_node)).list[Output_Edges((*in_node)).length - 1].reverse_edge_ID + = Input_Edges((*out_node)).length - 1; + + Input_Edges((*out_node)).list[Input_Edges((*out_node)).length - 1].reverse_edge_ID + = Output_Edges((*in_node)).length - 1; +} + + + +int remove_and_check_bi_direction_edge_from_nodes(Graph* graph, Node* in_node, Node* out_node) +{ + Edge* e_forward; + Edge* e_backward; + + //if there are no edge from in_node to out_node + //1. remove these two edges + //2. increase the edge_list.delete_length in both in_node and out_node + if(get_bi_Edge(graph, in_node, out_node, &e_forward, &e_backward)) + { + e_forward->in_node = (uint64_t)-1; + e_forward->out_node = (uint64_t)-1; + e_forward->weight = (uint64_t)-1; + e_forward->length = (uint64_t)-1; + e_forward->num_insertions = (uint64_t)-1; + e_forward->self_edge_ID = (uint64_t)-1; + e_forward->reverse_edge_ID = (uint64_t)-1; + + + e_backward->in_node = (uint64_t)-1; + e_backward->out_node = (uint64_t)-1; + e_backward->weight = (uint64_t)-1; + e_backward->length = (uint64_t)-1; + e_backward->num_insertions = (uint64_t)-1; + e_backward->self_edge_ID = (uint64_t)-1; + e_backward->reverse_edge_ID = (uint64_t)-1; + + Output_Edges(*in_node).delete_length++; + Input_Edges((*out_node)).delete_length++; + + return 1; + } + else//if there is an edge from in_node to out_node, do nothing + { + return 0; + } +} + + + +int remove_and_check_bi_direction_edge_from_edge(Graph* graph, Edge* e) +{ + Edge* e_forward; + Edge* e_backward; + + if(If_Edge_Exist(*e)) + { + get_bi_direction_edges(graph, e, &e_forward, &e_backward); + Output_Edges(G_Node(*graph, e_forward->in_node)).delete_length++; + Input_Edges(G_Node(*graph, e_forward->out_node)).delete_length++; + + e_forward->in_node = (uint64_t)-1; + e_forward->out_node = (uint64_t)-1; + e_forward->weight = (uint64_t)-1; + e_forward->length = (uint64_t)-1; + e_forward->num_insertions = (uint64_t)-1; + e_forward->self_edge_ID = (uint64_t)-1; + e_forward->reverse_edge_ID = (uint64_t)-1; + + + e_backward->in_node = (uint64_t)-1; + e_backward->out_node = (uint64_t)-1; + e_backward->weight = (uint64_t)-1; + e_backward->length = (uint64_t)-1; + e_backward->num_insertions = (uint64_t)-1; + e_backward->self_edge_ID = (uint64_t)-1; + e_backward->reverse_edge_ID = (uint64_t)-1; + + return 1; + } + else + { + return 0; + } + + +} + void init_Node_alloc(Node_alloc* list) { list->size = INIT_NODE_SIZE; list->length = 0; + list->delete_length = 0; list->list = (Node*)malloc(sizeof(Node)*list->size); list->sort.size = 0; list->sort.list = NULL; @@ -105,6 +229,7 @@ void clear_Node_alloc(Node_alloc* list) } list->length = 0; + list->delete_length = 0; } @@ -162,11 +287,14 @@ void init_Graph(Graph* g) g->s_start_nodeID = 0; g->seq = NULL; g->seqID = (uint64_t)-1; + + init_Queue(&(g->node_q)); } void destory_Graph(Graph* g) { destory_Node_alloc(&g->g_nodes); + destory_Queue(&(g->node_q)); } void clear_Graph(Graph* g) @@ -180,6 +308,8 @@ void clear_Graph(Graph* g) g->s_start_nodeID = 0; g->seq = NULL; g->seqID = (uint64_t)-1; + + clear_Queue(&(g->node_q)); } @@ -237,6 +367,282 @@ void addUnmatchedSeqToGraph(Graph* g, char* g_read_seq, long long g_read_length, +inline void add_insertionEdge_weight_print(Graph* g, long long alignNodeID, char* insert, long long insert_length) +{ + + long long nodeID; + long long edgeID; + Edge_alloc* edge = &(g->g_nodes.list[alignNodeID].insertion_edges); + + /******************************for homopolymer*************************/ + long long i = 0; + char hom; + if (insert_length > 0) + { + hom = insert[0]; + } + + for (i = 0; i < insert_length; i++) + { + if(insert[i] != hom) + { + break; + } + } + + fprintf(stderr, "###insert_length: %d\n", insert_length); + + ///if it is a homopolymer + if(i == insert_length) + { + ///single base + edgeID = getEdge(g, edge, 1, insert[0]); + if (edgeID != -1) + { + ///这条路均只有一个出度 + edge->list[edgeID].weight++; + } + else ///不存在这么一条边 + { + nodeID = add_Node_Graph(g, insert[0]); + append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + ///将新加入的节点通过insertion_edges接回backbone上 + ///应该连回到原节点，而不是原节点的下一个节点 + ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + } + + ///multiple bases + for (i = 1; i < insert_length; i++) + { + edgeID = get_insertion_Edges(g, edge, i + 1, insert); + if (edgeID != -1) + { + ///这条路均只有一个出度 + edge->list[edgeID].weight++; + } + else + { + create_insertion_Edges(g, alignNodeID, i + 1, insert); + } + } + + return; + } + /******************************for homopolymer*************************/ + + fprintf(stderr, "###not homopolymer: %d\n", insert_length); + + if (insert_length == 1) + { + edgeID = getEdge(g, edge, 1, insert[0]); + if (edgeID != -1) + { + ///这条路均只有一个出度 + edge->list[edgeID].weight++; + } + else ///不存在这么一条边 + { + nodeID = add_Node_Graph(g, insert[0]); + append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + ///将新加入的节点通过insertion_edges接回backbone上 + ///应该连回到原节点，而不是原节点的下一个节点 + ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + } + } + else if (insert_length == 2) + { + /*******************第0个字符********************* */ + edgeID = getEdge(g, edge, 1, insert[0]); + fprintf(stderr, "edgeID[0]: %d, length: %d\n", edgeID, edge->list[edgeID].length); + if (edgeID != -1) + { + ///这条路均只有一个出度 + edge->list[edgeID].weight++; + } + else ///不存在这么一条边 + { + nodeID = add_Node_Graph(g, insert[0]); + append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + ///将新加入的节点通过insertion_edges接回backbone上 + ///应该连回到原节点，而不是原节点的下一个节点 + ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + } + + fprintf(stderr, "edge->length: %d\n", edge->length); + + /*******************第0个字符********************* */ + + /*******************第1个字符********************* */ + if (insert[1] != insert[0]) + { + edgeID = getEdge(g, edge, 1, insert[1]); + fprintf(stderr, "edgeID[1]: %d, length: %d\n", edgeID, edge->list[edgeID].length); + if (edgeID != -1) + { + ///这条路均只有一个出度 + edge->list[edgeID].weight++; + } + else ///不存在这么一条边 + { + nodeID = add_Node_Graph(g, insert[1]); + append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + ///将新加入的节点通过insertion_edges接回backbone上 + ///应该连回到原节点，而不是原节点的下一个节点 + ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + } + + fprintf(stderr, "edge->length: %d\n", edge->length); + } + /*******************第1个字符********************* */ + + /**********************两个字符******************* */ + + edgeID = get_insertion_Edges(g, edge, 2, insert); + fprintf(stderr, "edgeID[len2]: %d, length: %d\n", edgeID, edge->list[edgeID].length); + if (edgeID != -1) + { + ///这条路均只有一个出度 + edge->list[edgeID].weight++; + } + else + { + create_insertion_Edges(g, alignNodeID, insert_length, insert); + } + + fprintf(stderr, "edge->length: %d\n", edge->length); + + for (i = 0; i < edge->length; i++) + { + fprintf(stderr, "edgeID[%d].length: %d\n", i, edge->list[i].length); + } + + + /**********************两个字符******************* */ + } + else if (insert_length > 2) + { + ////fprintf(stderr, "too long insertion\n"); + /*************************大于2个字符************************** */ + + edgeID = get_insertion_Edges(g, edge, insert_length, insert); + if (edgeID != -1) + { + ///这条路均只有一个出度 + edge->list[edgeID].weight++; + } + else + { + create_insertion_Edges(g, alignNodeID, insert_length, insert); + } + } + + + +} + + + +void addmatchedSeqToGraph_print(Graph* backbone, long long currentNodeID, char* x_string, long long x_length, + char* y_string, long long y_length, CIGAR* cigar, long long backbone_start, long long backbone_end) +{ + + int x_i, y_i, cigar_i; + x_i = 0; + y_i = 0; + cigar_i = 0; + int operation; + int operationLen; + int i; + int last_operation = -1; + + fprintf(stderr,"*******\n"); + ///note that node 0 is the start node + ///0 is match, 1 is mismatch, 2 is up, 3 is left + ///2是x缺字符（y多字符），而3是y缺字符（x多字符） + while (cigar_i < cigar->length) + { + operation = cigar->C_C[cigar_i]; + operationLen = cigar->C_L[cigar_i]; + + // fprintf(stderr, "operation: %d, operationLen: %d\n", + // operation, operationLen); + + ///这种情况代表匹配和mismatch + if (operation == 0 || operation == 1) + { + + for (i = 0; i < operationLen; i++) + { + //backbone->g_nodes.list[currentNodeID].weight++; + ///前面是插入，后面有可能是误配，也有可能是匹配 + add_mismatchEdge_weight(backbone, currentNodeID, y_string[y_i], last_operation); + x_i++; + y_i++; + currentNodeID++; + } + }///insertion + else if (operation == 2) + { + ///cigar的起始和结尾不可能是2，所以这里-1没问题 + ///if (operationLen <= CORRECT_INDEL_LENGTH) + { + add_insertionEdge_weight_print(backbone, currentNodeID, y_string + y_i, operationLen); + backbone->g_nodes.list[currentNodeID].num_insertions++; + } + + ///fprintf(stderr, "y_string: %.*s\n", operationLen, y_string+y_i); + y_i += operationLen; + } + else if (operation == 3) + { + ///3是y缺字符（x多字符），也就是backbone多字符 + ///这个相当于在backbone对应字符处变成了‘——’ + ///因此可以用mismatch类似的方法处理 + ///if (operationLen <= CORRECT_INDEL_LENGTH) + { + ///add_deletion_to_backbone(backbone, ¤tNodeID, operationLen); + ///在编辑距离中，前面是个插入，后面是个删除，这种情况是不存在的 + ///为了保险要不还给他加上吧 + ///先不加 + add_deletionEdge_weight(backbone, currentNodeID, operationLen); + } + + + currentNodeID += operationLen; + x_i += operationLen; + } + + last_operation = operation; + + cigar_i++; + } + + + + /** + ///cigar的起始和结尾不可能是2 + if (cigar->C_C[0] == 2 || cigar->C_C[cigar->length - 1] == 2) + { + fprintf(stderr, "error\n"); + } + + + if (x_i != x_length) + { + fprintf(stderr, "x_i: %d, x_length: %d\n", x_i, x_length); + } + + if (y_i != y_length) + { + fprintf(stderr, "y_i: %d, y_length: %d\n", y_i, y_length); + } + **/ + +} @@ -311,28 +717,6 @@ void addmatchedSeqToGraph(Graph* backbone, long long currentNodeID, char* x_stri cigar_i++; } - - - - /** - ///cigar的起始和结尾不可能是2 - if (cigar->C_C[0] == 2 || cigar->C_C[cigar->length - 1] == 2) - { - fprintf(stderr, "error\n"); - } - - - if (x_i != x_length) - { - fprintf(stderr, "x_i: %d, x_length: %d\n", x_i, x_length); - } - - if (y_i != y_length) - { - fprintf(stderr, "y_i: %d, y_length: %d\n", y_i, y_length); - } - **/ - } diff --git a/POA.h b/POA.h index 822fbcd..b54dc2e 100644 --- a/POA.h +++ b/POA.h @@ -23,6 +23,120 @@ 5. 有两个回溯矩阵，一个是graph的，一个是seq的 **/ + + +typedef struct +{ + long long beg; + ///end is the index of next input data, instead of the index of last data + long long end; + long long length; + long long size; + long long* buffer; +} Queue; + +inline void init_Queue(Queue* q) +{ + q->beg = 0; + q->end = 0; + q->length = 0; + q->size = 20; + q->buffer = (long long*)malloc(sizeof(long long)*q->size); +} + +inline void clear_Queue(Queue* q) +{ + q->beg = 0; + q->end = 0; + q->length = 0; +} + +inline void destory_Queue(Queue* q) +{ + free(q->buffer); +} + +inline int is_empty_Queue(Queue* q) +{ + ///end is the index of next input data, instead of the index of last data + if(q->beg == q->end) + { + return 1; + } + else + { + return 0; + } +} + +inline int is_full_Queue(Queue* q) +{ + ///end is the index of next input data, instead of the index of last data + if(q->end < q->size) + { + return 0; + } + else + { + return 1; + } +} + +inline void push_to_Queue(Queue* q, long long nodeID) +{ + if(is_full_Queue(q)) + { + long long move_length = q->beg; + ///end is the index of next input data, instead of the index of last data + long long current_length = q->end - q->beg; + ///recalloc directly + if(move_length == 0) + { + q->size = q->size * 2; + q->buffer = (long long*)realloc(q->buffer, q->size*sizeof(long long)); + } + else + { + ///won't overlap + if(current_length <= move_length) + { + memcpy(q->buffer, q->buffer+q->beg, sizeof(long long)*current_length); + } + else///may overlap + { + memmove(q->buffer, q->buffer+q->beg, sizeof(long long)*current_length); + } + + q->beg = 0; + q->end = current_length; + } + } + + + q->buffer[q->end] = nodeID; + q->end++; +} + +inline int pop_from_Queue(Queue* q, long long* nodeID) +{ + if(is_empty_Queue(q)) + { + (*nodeID) = -1; + return 0; + } + else + { + (*nodeID) = q->buffer[q->beg]; + q->beg++; + + return 1; + } +} + + + + + typedef struct { uint64_t in_node; @@ -32,6 +146,8 @@ typedef struct uint64_t num_insertions; ///这条路径上到backbone节点之前总共有多少节点 uint64_t length; + uint64_t self_edge_ID; + uint64_t reverse_edge_ID; } Edge; typedef struct @@ -39,8 +155,27 @@ typedef struct Edge* list; uint64_t size; uint64_t length; + uint64_t delete_length; } Edge_alloc; +#define Real_Length(X) ((X).length - (X).delete_length) +#define Input_Edges(Node) ((Node).insertion_edges) +#define Output_Edges(Node) ((Node).deletion_edges) +#define G_Node(G, Node) ((G).g_nodes.list[(Node)]) +#define If_Node_Exist(Node) ((Node).base != 'D') +#define If_Edge_Exist(E) ((E).out_node != (uint64_t)-1) +#define Visit(E) (E).length + +typedef struct +{ + long long index; +} RSet; + +inline void clear_RSet(RSet* set) +{ + set->index = 0; +} + typedef struct { uint64_t ID; @@ -73,6 +208,7 @@ typedef struct topo_Sorting_buffer sort; uint64_t size; uint64_t length; + uint64_t delete_length; } Node_alloc; typedef struct @@ -83,13 +219,222 @@ typedef struct Node_alloc g_nodes; - + Queue node_q; char* seq; uint64_t seqID; uint64_t s_start_nodeID; uint64_t s_end_nodeID; } Graph; +int add_and_check_bi_direction_edge(Graph* graph, Node* in_node, Node* out_node, uint64_t weight, uint64_t flag); +void add_bi_direction_edge(Graph* graph, Node* in_node, Node* out_node, uint64_t weight, uint64_t flag); +int remove_and_check_bi_direction_edge_from_nodes(Graph* graph, Node* in_node, Node* out_node); +int remove_and_check_bi_direction_edge_from_edge(Graph* graph, Edge* e); + + + +inline int Pop_Node(Graph* DAGCon, Node** node) +{ + long long nodeID = 0; + int return_flag = pop_from_Queue(&(DAGCon->node_q), &nodeID); + + (*node) = &(G_Node(*DAGCon, nodeID)); + return return_flag; +} + +inline int Push_Node(Graph* DAGCon, Node** node) +{ + push_to_Queue(&(DAGCon->node_q), (**node).ID); +} + +inline int getInputNodes(RSet* set, Graph* graph, Node* node, Node** get_Node) +{ + if(set->index >= Input_Edges(*node).length) + { + return 0; + } + + ///skip all deleted edges + while ( + set->index < Input_Edges(*node).length + && + !(If_Edge_Exist(Input_Edges(*node).list[set->index])) + ) + { + set->index++; + } + + + if( + set->index < Input_Edges(*node).length + && + If_Edge_Exist(Input_Edges(*node).list[set->index]) + ) + { + (*get_Node) = &(G_Node((*graph), Input_Edges(*node).list[set->index].in_node)); + set->index++; + return 1; + } + else + { + return 0; + } +} + + + +inline int getInputEdges(RSet* set, Graph* graph, Node* node, Edge** get_Edge) +{ + if(set->index >= Input_Edges(*node).length) + { + return 0; + } + + ///skip all deleted edges + while ( + set->index < Input_Edges(*node).length + && + !(If_Edge_Exist(Input_Edges(*node).list[set->index])) + ) + { + set->index++; + } + + + if( + set->index < Input_Edges(*node).length + && + If_Edge_Exist(Input_Edges(*node).list[set->index]) + ) + { + (*get_Edge) = &(Input_Edges(*node).list[set->index]); + set->index++; + return 1; + } + else + { + return 0; + } +} + + +inline int getOutputNodes(RSet* set, Graph* graph, Node* node, Node** get_Node) +{ + if(set->index >= Output_Edges(*node).length) + { + return 0; + } + + ///skip all deleted edges + while ( + set->index < Output_Edges(*node).length + && + !(If_Edge_Exist(Output_Edges(*node).list[set->index])) + ) + { + set->index++; + } + + if(set->index < Output_Edges(*node).length && + If_Edge_Exist(Output_Edges(*node).list[set->index])) + { + (*get_Node) = &(G_Node((*graph), Output_Edges(*node).list[set->index].out_node)); + set->index++; + return 1; + } + else + { + return 0; + } +} + + + +inline int getOutputEdges(RSet* set, Graph* graph, Node* node, Edge** get_Edge) +{ + if(set->index >= Output_Edges(*node).length) + { + return 0; + } + + ///skip all deleted edges + while ( + set->index < Output_Edges(*node).length + && + !(If_Edge_Exist(Output_Edges(*node).list[set->index])) + ) + { + set->index++; + } + + if(set->index < Output_Edges(*node).length && + If_Edge_Exist(Output_Edges(*node).list[set->index])) + { + (*get_Edge) = &(Output_Edges(*node).list[set->index]); + set->index++; + return 1; + } + else + { + return 0; + } +} + + +inline void get_bi_direction_edges(Graph* DAGCon, Edge* edge, Edge** e_forward, Edge** e_backward) +{ + long long in_node = edge->in_node; + long long out_node = edge->out_node; + + if( + edge->self_edge_ID < Output_Edges(G_Node(*DAGCon, in_node)).length + && + Output_Edges(G_Node(*DAGCon, in_node)).list[edge->self_edge_ID].in_node == in_node + && + Output_Edges(G_Node(*DAGCon, in_node)).list[edge->self_edge_ID].out_node == out_node + ) + { + (*e_forward) = &(Output_Edges(G_Node(*DAGCon, in_node)).list[edge->self_edge_ID]); + (*e_backward) = &(Input_Edges(G_Node(*DAGCon, out_node)).list[edge->reverse_edge_ID]); + } + else + { + (*e_forward) = &(Output_Edges(G_Node(*DAGCon, in_node)).list[edge->reverse_edge_ID]); + (*e_backward) = &(Input_Edges(G_Node(*DAGCon, out_node)).list[edge->self_edge_ID]); + } +} + + +inline long long get_bi_Edge(Graph* DAGCon, Node* inNode, Node* outNode, Edge** e_forward, Edge** e_backward) +{ + Edge* e; + RSet iter; + clear_RSet(&iter); + + if(If_Node_Exist(*inNode) && If_Node_Exist(*outNode)) + { + //find in-edge of outNode + while(getInputEdges(&iter, DAGCon, outNode, &e)) + { + if(e->in_node == inNode->ID) + { + get_bi_direction_edges(DAGCon, e, e_forward, e_backward); + return 1; + } + } + } + + return 0; +} + + +inline long long get_Edge_Weight(Graph* DAGCon, Node* inNode, Node* outNode) +{ + Edge* e_forward; + Edge* e_backward; + get_bi_Edge(DAGCon, inNode, outNode, &e_forward, &e_backward); + return e_forward->weight; +} void init_Edge_alloc(Edge_alloc* list); void clear_Edge_alloc(Edge_alloc* list); @@ -121,6 +466,48 @@ uint64_t inline add_Node_Graph(Graph* g, char base) return append_Node_alloc(&g->g_nodes, base); } +inline Node* add_Node_DAGCon(Graph* g, char base) +{ + return &(G_Node(*g, append_Node_alloc(&g->g_nodes, base))); +} + +///to delete a node +///1. set the corresponding base to be 'D' +///2. remove all related edges +///2. clear all related edges +///3. g_nodes.delete_length++, please do not substract g_nodes.length +uint64_t inline delete_Node_DAGCon(Graph* g, Node* node) +{ + g->g_nodes.delete_length++; + g->g_nodes.list[(*node).ID].base = 'D'; + g->g_nodes.list[(*node).ID].num_insertions = (uint64_t)-1; + g->g_nodes.list[(*node).ID].weight = (uint64_t)-1; + + + RSet iter; + Edge* e; + clear_RSet(&iter); + while (getOutputEdges(&iter, g, node, &e)) + { + remove_and_check_bi_direction_edge_from_edge(g, e); + } + + clear_RSet(&iter); + while (getInputEdges(&iter, g, node, &e)) + { + remove_and_check_bi_direction_edge_from_edge(g, e); + } + + + + clear_Edge_alloc(&(g->g_nodes.list[(*node).ID].insertion_edges)); + clear_Edge_alloc(&(g->g_nodes.list[(*node).ID].mismatch_edges)); + clear_Edge_alloc(&(g->g_nodes.list[(*node).ID].deletion_edges)); +} + + + + ///仅仅用于误配边 inline void add_mismatchEdge_weight(Graph* g, uint64_t in_node, char base, int last_operation) @@ -193,6 +580,7 @@ inline void add_single_deletionEdge_weight(Graph* g, long long alignNodeID, long inline void add_deletionEdge_weight(Graph* g, long long alignNodeID, long long deletion_length) { + /** if (deletion_length == 1) { add_single_deletionEdge_weight(g, alignNodeID, alignNodeID + 1, 0); @@ -208,6 +596,13 @@ inline void add_deletionEdge_weight(Graph* g, long long alignNodeID, long long d ///fprintf(stderr, "too long deletion!\n"); add_single_deletionEdge_weight(g, alignNodeID, alignNodeID + deletion_length, 0); } + **/ + long long i; + for (i = 0; i < deletion_length; i++) + { + add_single_deletionEdge_weight(g, alignNodeID + i, alignNodeID + i + 1, 0); + } + } @@ -266,8 +661,10 @@ inline int get_insertion_Edges(Graph* g, Edge_alloc* edge, uint64_t edge_length, return -1; } } - - return edgeID; + /****************************may have bugs********************************/ + ///return edgeID; + return return_edgeID; + /****************************may have bugs********************************/ } @@ -305,19 +702,183 @@ inline int create_insertion_Edges(Graph* g, long long alignNodeID, uint64_t edge } -inline void add_insertionEdge_weight(Graph* g, long long alignNodeID, char* insert, long long insert_length) + +inline void extract_path(Graph* backbone, int debug_node_in_backbone, int path_i, char* pre) +{ + int step = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[path_i].length; + int string_i, preNode, j; + if(step != 0) + { + string_i = 0; + preNode = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[path_i].out_node; + + for (j = 0; j < step; j++) + { + pre[string_i++] = G_Node(*backbone, preNode).base; + preNode = G_Node(*backbone, preNode).insertion_edges.list[0].out_node; + } + } + + pre[string_i] = '\0'; +} + + +inline void extract_path_debug(Graph* backbone, int debug_node_in_backbone, int path_i, char* pre) +{ + int step = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[path_i].length; + int string_i, preNode, preEdge, j; + if(step != 0) + { + string_i = 0; + preNode = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[path_i].out_node; + preEdge = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[path_i].length; + + for (j = 0; j < step; j++) + { + ///pre[string_i++] = G_Node(*backbone, preNode).base; + fprintf(stderr, "j: %d (%c%d), ", j, G_Node(*backbone, preNode).base, preEdge); + preEdge = G_Node(*backbone, preNode).insertion_edges.list[0].length; + preNode = G_Node(*backbone, preNode).insertion_edges.list[0].out_node; + } + } + + fprintf(stderr, "\n"); + + ///pre[string_i] = '\0'; +} + + +inline int getEdge_DEBUG(Graph* g, Edge_alloc* edge, uint64_t edge_length, char base) { long long i = 0; long long nodeID; + + for (i = 0; i < edge->length; i++) + { + ///fprintf(stderr, "************i:%d, edge->list[i].length: %d, edge_length: %d\n",i, edge->list[i].length, edge_length); + if (edge->list[i].length == edge_length) + { + nodeID = edge->list[i].out_node; + fprintf(stderr, "########i:%d, edge->list[i].length: %d, edge_length: %d, nodeID: %d, list[nodeID].base: %c, base: %c\n", + i, edge->list[i].length, edge_length, nodeID, g->g_nodes.list[nodeID].base, base); + + if(g->g_nodes.list[nodeID].base == base) + { + return i; + } + } + } + + return -1; +} + +inline int get_insertion_Edges_new(Graph* backbone, int debug_node_in_backbone, uint64_t edge_length, char* bases) +{ + int path_i, j, step, preNode; + + for (path_i = 0; path_i < G_Node(*backbone, debug_node_in_backbone).insertion_edges.length; path_i++) + { + step = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[path_i].length; + + + if(step != edge_length) + { + continue; + } + + + if(step != 0) + { + preNode = G_Node(*backbone, debug_node_in_backbone).insertion_edges.list[path_i].out_node; + + for (j = 0; j < step; j++) + { + ///pre[string_i++] = G_Node(*backbone, preNode).base; + ///fprintf(stderr, "path_i: %d, ID: %d\n", path_i, G_Node(*backbone, preNode).ID); + if(G_Node(*backbone, preNode).base != bases[j]) + { + break; + } + + preNode = G_Node(*backbone, preNode).insertion_edges.list[0].out_node; + } + + if(j == step) + { + return path_i; + } + } + } + + + return -1; +} + + +inline int get_insertion_Edges_debug(Graph* g, Edge_alloc* edge, uint64_t edge_length, char* bases) +{ + long long i = 0; + long long nodeID; + long long edgeID; + + if (edge_length < 1) + { + return -1; + } + + + ///fprintf(stderr, "edge_length: %d, edge: %.*s\n", edge_length, edge_length, bases); + + + edgeID = getEdge_DEBUG(g, edge, edge_length, bases[0]); + fprintf(stderr, "i: %d, edgeID: %d, edge_length - i: %d\n", i, edgeID, edge_length); + + + + + long long return_edgeID = edgeID; + + if(edgeID == -1) + { + return -1; + } + + + Edge_alloc* new_edge = edge; + + for (i = 1; i < edge_length; i++) + { + nodeID = new_edge->list[edgeID].out_node; + new_edge = &(g->g_nodes.list[nodeID].insertion_edges); + edgeID = getEdge_DEBUG(g, new_edge, edge_length - i, bases[i]); + fprintf(stderr, "i: %d, edgeID: %d, edge_length - i: %d\n", i, edgeID, edge_length - i); + if(edgeID == -1) + { + return -1; + } + } + /****************************may have bugs********************************/ + ///return edgeID; + return return_edgeID; + /****************************may have bugs********************************/ +} + +inline void add_insertionEdge_weight(Graph* g, long long alignNodeID, char* insert, long long insert_length) +{ + + long long nodeID; long long edgeID; Edge_alloc* edge = &(g->g_nodes.list[alignNodeID].insertion_edges); if (insert_length == 1) { edgeID = getEdge(g, edge, 1, insert[0]); + // if(edgeID != get_insertion_Edges_new(g, alignNodeID, insert_length, insert)) + // { + // fprintf(stderr, "error\n"); + // } if (edgeID != -1) { - ///这条路均只有一个出度 edge->list[edgeID].weight++; } else ///不存在这么一条边 @@ -330,68 +891,10 @@ inline void add_insertionEdge_weight(Graph* g, long long alignNodeID, char* inse append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); } } - else if (insert_length == 2) + else { - /*******************第0个字符********************* */ - edgeID = getEdge(g, edge, 1, insert[0]); - if (edgeID != -1) - { - ///这条路均只有一个出度 - edge->list[edgeID].weight++; - } - else ///不存在这么一条边 - { - nodeID = add_Node_Graph(g, insert[0]); - append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); - ///将新加入的节点通过insertion_edges接回backbone上 - ///应该连回到原节点，而不是原节点的下一个节点 - ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); - append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); - } - /*******************第0个字符********************* */ - - /*******************第1个字符********************* */ - if (insert[1] != insert[0]) - { - edgeID = getEdge(g, edge, 1, insert[1]); - if (edgeID != -1) - { - ///这条路均只有一个出度 - edge->list[edgeID].weight++; - } - else ///不存在这么一条边 - { - nodeID = add_Node_Graph(g, insert[1]); - append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); - ///将新加入的节点通过insertion_edges接回backbone上 - ///应该连回到原节点，而不是原节点的下一个节点 - ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); - append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); - } - } - /*******************第1个字符********************* */ - - /**********************两个字符******************* */ - - edgeID = get_insertion_Edges(g, edge, 2, insert); - if (edgeID != -1) - { - ///这条路均只有一个出度 - edge->list[edgeID].weight++; - } - else - { - create_insertion_Edges(g, alignNodeID, insert_length, insert); - } - - /**********************两个字符******************* */ - } - else if (insert_length > 2) - { - ////fprintf(stderr, "too long insertion\n"); - /*************************大于2个字符************************** */ - - edgeID = get_insertion_Edges(g, edge, insert_length, insert); + ///edgeID = get_insertion_Edges(g, edge, insert_length, insert); + edgeID = get_insertion_Edges_new(g, alignNodeID, insert_length, insert); if (edgeID != -1) { ///这条路均只有一个出度 @@ -402,10 +905,163 @@ inline void add_insertionEdge_weight(Graph* g, long long alignNodeID, char* inse create_insertion_Edges(g, alignNodeID, insert_length, insert); } } + + // /******************************for homopolymer*************************/ + // long long i = 0; + // char hom; + // if (insert_length > 0) + // { + // hom = insert[0]; + // } + + // for (i = 0; i < insert_length; i++) + // { + // if(insert[i] != hom) + // { + // break; + // } + // } + + // ///if it is a homopolymer + // if(i == insert_length) + // { + // ///single base + // edgeID = getEdge(g, edge, 1, insert[0]); + // if (edgeID != -1) + // { + // ///这条路均只有一个出度 + // edge->list[edgeID].weight++; + // } + // else ///不存在这么一条边 + // { + // nodeID = add_Node_Graph(g, insert[0]); + // append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + // ///将新加入的节点通过insertion_edges接回backbone上 + // ///应该连回到原节点，而不是原节点的下一个节点 + // ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + // append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + // } + + // ///multiple bases + // for (i = 1; i < insert_length; i++) + // { + // edgeID = get_insertion_Edges(g, edge, i + 1, insert); + // if (edgeID != -1) + // { + // ///这条路均只有一个出度 + // edge->list[edgeID].weight++; + // } + // else + // { + // create_insertion_Edges(g, alignNodeID, i + 1, insert); + // } + // } + + // return; + // } + // /******************************for homopolymer*************************/ + + + + // if (insert_length == 1) + // { + // edgeID = getEdge(g, edge, 1, insert[0]); + // if (edgeID != -1) + // { + // ///这条路均只有一个出度 + // edge->list[edgeID].weight++; + // } + // else ///不存在这么一条边 + // { + // nodeID = add_Node_Graph(g, insert[0]); + // append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + // ///将新加入的节点通过insertion_edges接回backbone上 + // ///应该连回到原节点，而不是原节点的下一个节点 + // ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + // append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + // } + // } + // else if (insert_length == 2) + // { + // /*******************第0个字符********************* */ + // edgeID = getEdge(g, edge, 1, insert[0]); + // if (edgeID != -1) + // { + // ///这条路均只有一个出度 + // edge->list[edgeID].weight++; + // } + // else ///不存在这么一条边 + // { + // nodeID = add_Node_Graph(g, insert[0]); + // append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + // ///将新加入的节点通过insertion_edges接回backbone上 + // ///应该连回到原节点，而不是原节点的下一个节点 + // ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + // append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + // } + // /*******************第0个字符********************* */ + + // /*******************第1个字符********************* */ + // if (insert[1] != insert[0]) + // { + // edgeID = getEdge(g, edge, 1, insert[1]); + // if (edgeID != -1) + // { + // ///这条路均只有一个出度 + // edge->list[edgeID].weight++; + // } + // else ///不存在这么一条边 + // { + // nodeID = add_Node_Graph(g, insert[1]); + // append_Edge_alloc(edge, alignNodeID, nodeID, 1, 1); + // ///将新加入的节点通过insertion_edges接回backbone上 + // ///应该连回到原节点，而不是原节点的下一个节点 + // ///append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID + 1, 1, 0); + // append_Edge_alloc(&(g->g_nodes.list[nodeID].insertion_edges), nodeID, alignNodeID, 1, 0); + // } + // } + // /*******************第1个字符********************* */ + + // /**********************两个字符******************* */ + + // edgeID = get_insertion_Edges(g, edge, 2, insert); + // if (edgeID != -1) + // { + // ///这条路均只有一个出度 + // edge->list[edgeID].weight++; + // } + // else + // { + // create_insertion_Edges(g, alignNodeID, insert_length, insert); + // } + + // /**********************两个字符******************* */ + // } + // else if (insert_length > 2) + // { + // ////fprintf(stderr, "too long insertion\n"); + // /*************************大于2个字符************************** */ + + // edgeID = get_insertion_Edges(g, edge, insert_length, insert); + // if (edgeID != -1) + // { + // ///这条路均只有一个出度 + // edge->list[edgeID].weight++; + // } + // else + // { + // create_insertion_Edges(g, alignNodeID, insert_length, insert); + // } + // } } +void addmatchedSeqToGraph_print(Graph* backbone, long long currentNodeID, char* x_string, long long x_length, + char* y_string, long long y_length, CIGAR* cigar, long long backbone_start, long long backbone_end); + + + #endif \ No newline at end of file diff --git a/Process_Read.cpp b/Process_Read.cpp index 743a52c..32101b6 100644 --- a/Process_Read.cpp +++ b/Process_Read.cpp @@ -74,9 +74,10 @@ void destory_All_reads(All_reads* r) void write_All_reads(All_reads* r, char* read_file_name) { fprintf(stdout, "Writing reads to disk ...... \n"); - char* index_name = (char*)malloc(strlen(read_file_name)+5); + char* index_name = (char*)malloc(strlen(read_file_name)+15); sprintf(index_name, "%s.bin", read_file_name); FILE* fp = fopen(index_name, "w"); + fwrite(&adapterLen, sizeof(adapterLen), 1, fp); fwrite(&r->index_size, sizeof(r->index_size), 1, fp); fwrite(&r->name_index_size, sizeof(r->name_index_size), 1, fp); fwrite(&r->total_reads, sizeof(r->total_reads), 1, fp); @@ -127,6 +128,7 @@ void write_All_reads(All_reads* r, char* read_file_name) free(index_name); + fflush(fp); fclose(fp); fprintf(stdout, "Reads has been written.\n"); } @@ -136,7 +138,7 @@ void write_All_reads(All_reads* r, char* read_file_name) int load_All_reads(All_reads* r, char* read_file_name) { fprintf(stdout, "Loading reads to disk ...... \n"); - char* index_name = (char*)malloc(strlen(read_file_name)+5); + char* index_name = (char*)malloc(strlen(read_file_name)+15); sprintf(index_name, "%s.bin", read_file_name); FILE* fp = fopen(index_name, "r"); if (!fp) @@ -144,7 +146,14 @@ int load_All_reads(All_reads* r, char* read_file_name) return 0; } - + int local_adapterLen; + fread(&local_adapterLen, sizeof(local_adapterLen), 1, fp); + if(local_adapterLen != adapterLen) + { + fprintf(stdout, "the adapterLen of index is: %d, but the adapterLen set by user is: %d\n", + local_adapterLen, adapterLen); + exit(1); + } fread(&r->index_size, sizeof(r->index_size), 1, fp); fread(&r->name_index_size, sizeof(r->name_index_size), 1, fp); fread(&r->total_reads, sizeof(r->total_reads), 1, fp); @@ -210,6 +219,8 @@ int load_All_reads(All_reads* r, char* read_file_name) r->cigars = (Compressed_Cigar_record*)malloc(sizeof(Compressed_Cigar_record)*r->total_reads); r->second_round_cigar = (Compressed_Cigar_record*)malloc(sizeof(Compressed_Cigar_record)*r->total_reads); + r->paf = (ma_hit_t_alloc*)malloc(sizeof(ma_hit_t_alloc)*r->total_reads); + r->reverse_paf = (ma_hit_t_alloc*)malloc(sizeof(ma_hit_t_alloc)*r->total_reads); for (i = 0; i < r->total_reads; i++) { r->second_round_cigar[i].size = r->cigars[i].size = 0; @@ -219,9 +230,14 @@ int load_All_reads(All_reads* r, char* read_file_name) r->second_round_cigar[i].lost_base_size = r->cigars[i].lost_base_size = 0; r->second_round_cigar[i].lost_base_length = r->cigars[i].lost_base_length = 0; r->second_round_cigar[i].lost_base = r->cigars[i].lost_base = NULL; + init_ma_hit_t_alloc(&(r->paf[i])); + init_ma_hit_t_alloc(&(r->reverse_paf[i])); } + + + free(index_name); fclose(fp); fprintf(stdout, "Reads has been loaded.\n"); @@ -279,6 +295,8 @@ void malloc_All_reads(All_reads* r) r->cigars = (Compressed_Cigar_record*)malloc(sizeof(Compressed_Cigar_record)*r->total_reads); r->second_round_cigar = (Compressed_Cigar_record*)malloc(sizeof(Compressed_Cigar_record)*r->total_reads); + r->paf = (ma_hit_t_alloc*)malloc(sizeof(ma_hit_t_alloc)*r->total_reads); + r->reverse_paf = (ma_hit_t_alloc*)malloc(sizeof(ma_hit_t_alloc)*r->total_reads); for (i = 0; i < r->total_reads; i++) { r->second_round_cigar[i].size = r->cigars[i].size = 0; @@ -288,9 +306,19 @@ void malloc_All_reads(All_reads* r) r->second_round_cigar[i].lost_base_size = r->cigars[i].lost_base_size = 0; r->second_round_cigar[i].lost_base_length = r->cigars[i].lost_base_length = 0; r->second_round_cigar[i].lost_base = r->cigars[i].lost_base = NULL; + init_ma_hit_t_alloc(&(r->paf[i])); + init_ma_hit_t_alloc(&(r->reverse_paf[i])); } + + + + + + + + r->name = (char*)malloc(sizeof(char)*r->total_name_length); r->N_site = (uint64_t**)calloc(r->total_reads, sizeof(uint64_t*)); @@ -658,21 +686,47 @@ void compress_base(uint8_t* dest, char* src, uint64_t src_l, uint64_t** N_site_l while (i + 4 <= src_l) { + + // fprintf(stderr, "0 i: %d, dest_i: %d, src_l: %d\n", + // i, dest_i, src_l); + // fflush(stderr); + tmp = 0; COMPRESS_BASE; tmp = tmp | (c<<6); + // fprintf(stderr, "*******1******1 i: %d, tmp: %d, c: %d\n", + // i, tmp, c); + // fflush(stderr); + COMPRESS_BASE; tmp = tmp | (c<<4); + // fprintf(stderr, "*******2******1 i: %d, tmp: %d, c: %d\n", + // i, tmp, c); + // fflush(stderr); + COMPRESS_BASE; tmp = tmp | (c<<2); + // fprintf(stderr, "*******3******1 i: %d, tmp: %d, c: %d\n", + // i, tmp, c); + // fflush(stderr); + COMPRESS_BASE; tmp = tmp | c; + // fprintf(stderr, "*******4******1 i: %d, tmp: %d, c: %d\n", + // i, tmp, c); + // fflush(stderr); + dest[dest_i] = tmp; + + // fprintf(stderr, "2 i: %d, dest_i: %d, src_l: %d\n", + // i, dest_i, src_l); + // fflush(stderr); + dest_i++; } @@ -716,7 +770,7 @@ inline void exchage_kstring_t(kstring_t* a, kstring_t* b) *b = tmp; } -int get_read(kseq_t *s) +int get_read(kseq_t *s, int adapterLen) { int l; @@ -727,6 +781,24 @@ int get_read(kseq_t *s) exchage_kstring_t(&seq->name, &s->name); exchage_kstring_t(&seq->qual, &s->qual); exchage_kstring_t(&seq->seq, &s->seq); + + if(adapterLen > 0) + { + if(s->seq.l <= adapterLen*2) + { + s->seq.l = 0; + } + else + { + long long i; + for (i = 0; i < (s->seq.l - adapterLen*2); i++) + { + s->seq.s[i] = s->seq.s[i + adapterLen]; + } + s->seq.l -= adapterLen*2; + } + + } return 1; } @@ -770,8 +842,8 @@ void init_R_buffer(int thread_num) void destory_R_buffer_block(R_buffer_block* curr_sub_block) { - - free(curr_sub_block->read); + kseq_destroy(curr_sub_block->read); + ///free(curr_sub_block->read); } @@ -790,7 +862,7 @@ void destory_R_buffer() inline void load_read_block(R_buffer_block* read_batch, int batch_read_size, - int* return_file_flag, int is_insert) + int* return_file_flag, int is_insert, int adapterLen) { int inner_i = 0; int file_flag = 1; @@ -801,18 +873,18 @@ inline void load_read_block(R_buffer_block* read_batch, int batch_read_size, while (inner_iread[inner_i]); + file_flag = get_read(&read_batch->read[inner_i], adapterLen); if (file_flag == 1) { read_batch->read[inner_i].ID = total_reads; total_reads++; - ///fprintf(stderr, "is_insert: %d\n", is_insert); if (is_insert) { - insert_read(&R_INF, &read_batch->read[inner_i].seq, &read_batch->read[inner_i].name); + insert_read(&R_INF, &read_batch->read[inner_i].seq, + &read_batch->read[inner_i].name); } inner_i++; @@ -896,7 +968,7 @@ void* input_reads_muti_threads(void* arg) - load_read_block(&tmp_buf, RDB.block_inner_size, &file_flag, is_insert); + load_read_block(&tmp_buf, RDB.block_inner_size, &file_flag, is_insert, adapterLen); if (file_flag == 0) { @@ -1022,7 +1094,7 @@ void Counting_block() load_read_block(&tmp_buf, RDB.block_inner_size, - &file_flag, 0); + &file_flag, 0, adapterLen); if (file_flag == 0) diff --git a/Process_Read.h b/Process_Read.h index edf32f1..fdc6b3a 100644 --- a/Process_Read.h +++ b/Process_Read.h @@ -6,6 +6,9 @@ #include #include #include "kseq.h" +#include "Overlaps.h" +#include "CommandLines.h" +///#include "Hash_Table.h" #define READ_INIT_NUMBER 1000 @@ -15,11 +18,11 @@ #define IS_FULL(buffer) ((buffer.num >= buffer.size)?1:0) #define IS_EMPTY(buffer) ((buffer.num == 0)?1:0) ///#define Get_READ_LENGTH(R_INF, ID) (R_INF.index[ID+1] - R_INF.index[ID]) -#define Get_READ_LENGTH(R_INF, ID) R_INF.read_length[ID] -#define Get_NAME_LENGTH(R_INF, ID) (R_INF.name_index[ID+1] - R_INF.name_index[ID]) +#define Get_READ_LENGTH(R_INF, ID) R_INF.read_length[(ID)] +#define Get_NAME_LENGTH(R_INF, ID) (R_INF.name_index[(ID)+1] - R_INF.name_index[(ID)]) ///#define Get_READ(R_INF, ID) R_INF.read + (R_INF.index[ID]>>2) + ID -#define Get_READ(R_INF, ID) R_INF.read_sperate[ID] -#define Get_NAME(R_INF, ID) R_INF.name + R_INF.name_index[ID] +#define Get_READ(R_INF, ID) R_INF.read_sperate[(ID)] +#define Get_NAME(R_INF, ID) R_INF.name + R_INF.name_index[(ID)] @@ -56,7 +59,56 @@ static char rc_Table[5] = {'T', 'G', 'C', 'A', 'N'}; void init_kseq(char* file); void destory_kseq(); -int get_read(kseq_t *s); +int get_read(kseq_t *s, int adapterLen); + + +typedef struct +{ + uint64_t x_id; + uint64_t x_pos_s; + uint64_t x_pos_e; + uint8_t x_pos_strand; + + uint64_t y_id; + uint64_t y_pos_s; + uint64_t y_pos_e; + uint8_t y_pos_strand; + + uint64_t matchLen; + uint64_t totalLen; + +} PAF; + +typedef struct +{ + PAF* list; + uint64_t size; + uint64_t length; +} PAF_alloc; + + +inline void init_PAF_alloc(PAF_alloc* list) +{ + list->size = 15; + list->length = 0; + list->list = (PAF*)malloc(sizeof(PAF)*list->size); +} + + +inline void append_PAF_alloc(PAF_alloc* list, PAF* e) +{ + if(list->length+1 > list->size) + { + list->size = list->size * 2; + list->list = (PAF*)realloc(list->list, sizeof(PAF)*list->size); + } + + list->list[list->length] = (*e); + list->length++; +} + + + typedef struct { @@ -100,6 +152,10 @@ typedef struct Compressed_Cigar_record* cigars; Compressed_Cigar_record* second_round_cigar; + ma_hit_t_alloc* paf; + ma_hit_t_alloc* reverse_paf; + ma_sub_t* coverage_cut; + } All_reads; extern All_reads R_INF; diff --git a/kdq.h b/kdq.h new file mode 100644 index 0000000..edd55b5 --- /dev/null +++ b/kdq.h @@ -0,0 +1,128 @@ +#ifndef __AC_KDQ_H +#define __AC_KDQ_H + +#include +#include + +#define __KDQ_TYPE(type) \ + typedef struct { \ + size_t front:58, bits:6, count, mask; \ + type *a; \ + } kdq_##type##_t; + +#define kdq_t(type) kdq_##type##_t +#define kdq_size(q) ((q)->count) +#define kdq_first(q) ((q)->a[(q)->front]) +#define kdq_last(q) ((q)->a[((q)->front + (q)->count - 1) & (q)->mask]) +#define kdq_at(q, i) ((q)->a[((q)->front + (i)) & (q)->mask]) + +#define __KDQ_IMPL(type, SCOPE) \ + SCOPE kdq_##type##_t *kdq_init_##type() \ + { \ + kdq_##type##_t *q; \ + q = (kdq_##type##_t*)calloc(1, sizeof(kdq_##type##_t)); \ + q->bits = 2, q->mask = (1ULL<bits) - 1; \ + q->a = (type*)malloc((1<bits) * sizeof(type)); \ + return q; \ + } \ + SCOPE void kdq_destroy_##type(kdq_##type##_t *q) \ + { \ + if (q == 0) return; \ + free(q->a); free(q); \ + } \ + SCOPE int kdq_resize_##type(kdq_##type##_t *q, int new_bits) \ + { \ + size_t new_size = 1ULL<bits; \ + if (new_size < q->count) { /* not big enough */ \ + int i; \ + for (i = 0; i < 64; ++i) \ + if (1ULL< q->count) break; \ + new_bits = i, new_size = 1ULL<bits) return q->bits; /* unchanged */ \ + if (new_bits > q->bits) q->a = (type*)realloc(q->a, (1ULL<front + q->count <= old_size) { /* unwrapped */ \ + if (q->front + q->count > new_size) /* only happens for shrinking */ \ + memmove(q->a, q->a + new_size, (q->front + q->count - new_size) * sizeof(type)); \ + } else { /* wrapped */ \ + memmove(q->a + (new_size - (old_size - q->front)), q->a + q->front, (old_size - q->front) * sizeof(type)); \ + q->front = new_size - (old_size - q->front); \ + } \ + q->bits = new_bits, q->mask = (1ULL<bits) - 1; \ + if (new_bits < q->bits) q->a = (type*)realloc(q->a, (1ULL<bits; \ + } \ + SCOPE type *kdq_pushp_##type(kdq_##type##_t *q) \ + { \ + if (q->count == 1ULL<bits) kdq_resize_##type(q, q->bits + 1); \ + return &q->a[((q->count++) + q->front) & (q)->mask]; \ + } \ + SCOPE void kdq_push_##type(kdq_##type##_t *q, type v) \ + { \ + if (q->count == 1ULL<bits) kdq_resize_##type(q, q->bits + 1); \ + q->a[((q->count++) + q->front) & (q)->mask] = v; \ + } \ + SCOPE type *kdq_unshiftp_##type(kdq_##type##_t *q) \ + { \ + if (q->count == 1ULL<bits) kdq_resize_##type(q, q->bits + 1); \ + ++q->count; \ + q->front = q->front? q->front - 1 : (1ULL<bits) - 1; \ + return &q->a[q->front]; \ + } \ + SCOPE void kdq_unshift_##type(kdq_##type##_t *q, type v) \ + { \ + type *p; \ + p = kdq_unshiftp_##type(q); \ + *p = v; \ + } \ + SCOPE type *kdq_pop_##type(kdq_##type##_t *q) \ + { \ + return q->count? &q->a[((--q->count) + q->front) & q->mask] : 0; \ + } \ + SCOPE type *kdq_shift_##type(kdq_##type##_t *q) \ + { \ + type *d = 0; \ + if (q->count == 0) return 0; \ + d = &q->a[q->front++]; \ + q->front &= q->mask; \ + --q->count; \ + return d; \ + } + +#define KDQ_INIT2(type, SCOPE) \ + __KDQ_TYPE(type) \ + __KDQ_IMPL(type, SCOPE) + +#ifndef klib_unused +#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3) +#define klib_unused __attribute__ ((__unused__)) +#else +#define klib_unused +#endif +#endif /* klib_unused */ + +#define KDQ_INIT(type) KDQ_INIT2(type, static inline klib_unused) + +#define KDQ_DECLARE(type) \ + __KDQ_TYPE(type) \ + kdq_##type##_t *kdq_init_##type(); \ + void kdq_destroy_##type(kdq_##type##_t *q); \ + int kdq_resize_##type(kdq_##type##_t *q, int new_bits); \ + type *kdq_pushp_##type(kdq_##type##_t *q); \ + void kdq_push_##type(kdq_##type##_t *q, type v); \ + type *kdq_unshiftp_##type(kdq_##type##_t *q); \ + void kdq_unshift_##type(kdq_##type##_t *q, type v); \ + type *kdq_pop_##type(kdq_##type##_t *q); \ + type *kdq_shift_##type(kdq_##type##_t *q); + +#define kdq_init(type) kdq_init_##type() +#define kdq_destroy(type, q) kdq_destroy_##type(q) +#define kdq_resize(type, q, new_bits) kdq_resize_##type(q, new_bits) +#define kdq_pushp(type, q) kdq_pushp_##type(q) +#define kdq_push(type, q, v) kdq_push_##type(q, v) +#define kdq_pop(type, q) kdq_pop_##type(q) +#define kdq_unshiftp(type, q) kdq_unshiftp_##type(q) +#define kdq_unshift(type, q, v) kdq_unshift_##type(q, v) +#define kdq_shift(type, q) kdq_shift_##type(q) + +#endif diff --git a/khash.h b/khash.h index 25b3883..9e010f2 100644 --- a/khash.h +++ b/khash.h @@ -209,8 +209,8 @@ static const double __ac_HASH_UPPER = 0.77; extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \ extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \ extern void kh_del_##name(kh_##name##_t *h, khint_t x);\ - extern void kh_write_##name(kh_##name##_t *h, FILE* fp)\ - extern void kh_load_##name(kh_##name##_t *h, FILE* fp) + extern void kh_write_##name(kh_##name##_t *h, FILE* fp);\ + extern void kh_load_##name(kh_##name##_t *h, FILE* fp); #define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ SCOPE kh_##name##_t *kh_init_##name(void) { \ diff --git a/kmer.cpp b/kmer.cpp index e9e9647..0d234cf 100644 --- a/kmer.cpp +++ b/kmer.cpp @@ -14,4 +14,182 @@ void init_Hash_code(Hash_code* code) { code->x[0] = 0; code->x[1] = 0; -} \ No newline at end of file +} + + + +void init_small_hash_table(small_hash_table* x) +{ + x->size = 0; + x->buffer = NULL; + x->length = 0; +} + +void clear_small_hash_table(small_hash_table* x) +{ + x->length = 0; +} + +void resize_small_hash_table(small_hash_table* x, uint64_t size) +{ + if(size > x->size) + { + x->size = size; + x->buffer = (k_v*)realloc(x->buffer, x->size*sizeof(k_v)); + } +} + +void destory_small_hash_table(small_hash_table* x) +{ + free(x->buffer); +} + + +void add_small_hash_table(small_hash_table* x, k_v* element) +{ + if(x->length + 1 > x->size) + { + x->size = (x->length + 1) * 2; + x->buffer = (k_v*)realloc(x->buffer, x->size*sizeof(k_v)); + } + + x->buffer[x->length] = (*element); + x->length++; +} + +//x > y, return 1; x < y, return -1, x == y, return 0 +int compare_k_mer(k_v* x, k_v* y) +{ + if(x->key.x[1] != y->key.x[1]) + { + return x->key.x[1] > y->key.x[1] ? 1: -1; + } + else + { + if(x->key.x[0] != y->key.x[0]) + { + return x->key.x[0] > y->key.x[0] ? 1: -1; + } + else + { + return 0; + } + } + +} + +int cmp_k_mer_kv(const void * a, const void * b) +{ + int flag = compare_k_mer((k_v*)a, (k_v*)b); + + if(flag == 0) + { + if ((*(k_v*)a).value != (*(k_v*)b).value) + { + return (*(k_v*)a).value > (*(k_v*)b).value ? 1: -1; + } + else + { + return 0; + } + + } + else + { + return flag; + } +} + +void sort_small_hash_table(small_hash_table* x) +{ + qsort(x->buffer, x->length, sizeof(k_v), cmp_k_mer_kv); +} + + +inline long long firstEqual(k_v* arr, long long arrLen, k_v* key) +{ + long long L = 0, R = arrLen - 1; //[L, R] + long long mid; + int flag; + while( L <= R) + { + mid = L + (R - L)/2; + + flag = compare_k_mer(&(arr[mid]), key); + + ///arr[mid] >= key + if(flag >= 0) + { + R = mid - 1; + } + else + { + L = mid + 1; + } + } + + + if(L < arrLen && (flag = compare_k_mer(&(arr[L]), key) == 0)) + { + return L; + } + + return -1; +} + +inline long long lastEqual(k_v* arr, long long arrLen, k_v* key) +{ + long long L = 0, R = arrLen - 1; //[L, R] + long long mid; + int flag; + while( L <= R) + { + mid = L + (R - L)/2; + flag = compare_k_mer(&(arr[mid]), key); + ///arr[mid] <= key + if(flag <= 0) + { + L = mid + 1; + } + else + { + R = mid - 1; + } + } + + if(R >= 0 && ((flag = compare_k_mer(&(arr[R]), key)) == 0)) + { + return R; + } + + return -1; +} + +int query_small_hash_table(small_hash_table* target, k_v* query, long long* l_end, long long* r_end) +{ + (*l_end) = -1; + (*r_end) = -1; + long long left_end; + long long right_end; + + left_end = firstEqual(target->buffer, target->length, query); + + if(left_end != -1) + { + right_end = lastEqual(target->buffer + left_end, target->length - left_end, query) + left_end; + + (*l_end) = left_end; + (*r_end) = right_end; + + if(right_end == -1) + { + fprintf(stderr, "error\n"); + } + + + return right_end - left_end + 1; + } + + return 0; +} + diff --git a/kmer.h b/kmer.h index c3cdab7..b49be00 100644 --- a/kmer.h +++ b/kmer.h @@ -21,6 +21,25 @@ typedef struct } Hash_code; +typedef struct { + Hash_code key; ///k-mer itself + uint64_t value; ///offset +} k_v; + +typedef struct { + k_v* buffer; + uint32_t size; + uint32_t length; +} small_hash_table; + +void init_small_hash_table(small_hash_table* x); +void clear_small_hash_table(small_hash_table* x); +void resize_small_hash_table(small_hash_table* x, uint64_t size); +void destory_small_hash_table(small_hash_table* x); +void add_small_hash_table(small_hash_table* x, k_v* element); +void sort_small_hash_table(small_hash_table* x); +int compare_k_mer(k_v* x, k_v* y); +int query_small_hash_table(small_hash_table* target, k_v* query, long long* l_end, long long* r_end); typedef struct diff --git a/ksort.h b/ksort.h new file mode 100644 index 0000000..0d3d82c --- /dev/null +++ b/ksort.h @@ -0,0 +1,187 @@ +/* The MIT License + + Copyright (c) 2008, 2011 Attractive Chaos + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +// This is a simplified version of ksort.h + +#ifndef AC_KSORT_H +#define AC_KSORT_H + +#include +#include + +typedef struct { + void *left, *right; + int depth; +} ks_isort_stack_t; + +#define member_size(type, member) sizeof(((type *)0)->member) + +#define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; } + +#define KSORT_INIT(name, type_t, __sort_lt) \ + static inline void __ks_insertsort_##name(type_t *s, type_t *t) \ + { \ + type_t *i, *j, swap_tmp; \ + for (i = s + 1; i < t; ++i) \ + for (j = i; j > s && __sort_lt(*j, *(j-1)); --j) { \ + swap_tmp = *j; *j = *(j-1); *(j-1) = swap_tmp; \ + } \ + } \ + void ks_combsort_##name(size_t n, type_t a[]) \ + { \ + const double shrink_factor = 1.2473309501039786540366528676643; \ + int do_swap; \ + size_t gap = n; \ + type_t tmp, *i, *j; \ + do { \ + if (gap > 2) { \ + gap = (size_t)(gap / shrink_factor); \ + if (gap == 9 || gap == 10) gap = 11; \ + } \ + do_swap = 0; \ + for (i = a; i < a + n - gap; ++i) { \ + j = i + gap; \ + if (__sort_lt(*j, *i)) { \ + tmp = *i; *i = *j; *j = tmp; \ + do_swap = 1; \ + } \ + } \ + } while (do_swap || gap > 2); \ + if (gap != 1) __ks_insertsort_##name(a, a + n); \ + } \ + void ks_introsort_##name(size_t n, type_t a[]) \ + { \ + int d; \ + ks_isort_stack_t *top, *stack; \ + type_t rp, swap_tmp; \ + type_t *s, *t, *i, *j, *k; \ + \ + if (n < 1) return; \ + else if (n == 2) { \ + if (__sort_lt(a[1], a[0])) { swap_tmp = a[0]; a[0] = a[1]; a[1] = swap_tmp; } \ + return; \ + } \ + for (d = 2; 1ul<>1) + 1; \ + if (__sort_lt(*k, *i)) { \ + if (__sort_lt(*k, *j)) k = j; \ + } else k = __sort_lt(*j, *i)? i : j; \ + rp = *k; \ + if (k != t) { swap_tmp = *k; *k = *t; *t = swap_tmp; } \ + for (;;) { \ + do ++i; while (__sort_lt(*i, rp)); \ + do --j; while (i <= j && __sort_lt(rp, *j)); \ + if (j <= i) break; \ + swap_tmp = *i; *i = *j; *j = swap_tmp; \ + } \ + swap_tmp = *i; *i = *t; *t = swap_tmp; \ + if (i-s > t-i) { \ + if (i-s > 16) { top->left = s; top->right = i-1; top->depth = d; ++top; } \ + s = t-i > 16? i+1 : t; \ + } else { \ + if (t-i > 16) { top->left = i+1; top->right = t; top->depth = d; ++top; } \ + t = i-s > 16? i-1 : s; \ + } \ + } else { \ + if (top == stack) { \ + free(stack); \ + __ks_insertsort_##name(a, a+n); \ + return; \ + } else { --top; s = (type_t*)top->left; t = (type_t*)top->right; d = top->depth; } \ + } \ + } \ + } + +#define ks_lt_generic(a, b) ((a) < (b)) +#define ks_lt_str(a, b) (strcmp((a), (b)) < 0) + +typedef const char *ksstr_t; + +#define KSORT_INIT_GENERIC(type_t) KSORT_INIT(type_t, type_t, ks_lt_generic) +#define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str) + +#define RS_MIN_SIZE 64 + +#define KRADIX_SORT_INIT(name, rstype_t, rskey, sizeof_key) \ + typedef struct { \ + rstype_t *b, *e; \ + } rsbucket_##name##_t; \ + void rs_insertsort_##name(rstype_t *beg, rstype_t *end) \ + { \ + rstype_t *i; \ + for (i = beg + 1; i < end; ++i) \ + if (rskey(*i) < rskey(*(i - 1))) { \ + rstype_t *j, tmp = *i; \ + for (j = i; j > beg && rskey(tmp) < rskey(*(j-1)); --j) \ + *j = *(j - 1); \ + *j = tmp; \ + } \ + } \ + void rs_sort_##name(rstype_t *beg, rstype_t *end, int n_bits, int s) \ + { \ + rstype_t *i; \ + int size = 1<b = k->e = beg; \ + for (i = beg; i != end; ++i) ++b[rskey(*i)>>s&m].e; \ + for (k = b + 1; k != be; ++k) \ + k->e += (k-1)->e - beg, k->b = (k-1)->e; \ + for (k = b; k != be;) { \ + if (k->b != k->e) { \ + rsbucket_##name##_t *l; \ + if ((l = b + (rskey(*k->b)>>s&m)) != k) { \ + rstype_t tmp = *k->b, swap; \ + do { \ + swap = tmp; tmp = *l->b; *l->b++ = swap; \ + l = b + (rskey(tmp)>>s&m); \ + } while (l != k); \ + *k->b++ = tmp; \ + } else ++k->b; \ + } else ++k; \ + } \ + for (b->b = beg, k = b + 1; k != be; ++k) k->b = (k-1)->e; \ + if (s) { \ + s = s > n_bits? s - n_bits : 0; \ + for (k = b; k != be; ++k) \ + if (k->e - k->b > RS_MIN_SIZE) rs_sort_##name(k->b, k->e, n_bits, s); \ + else if (k->e - k->b > 1) rs_insertsort_##name(k->b, k->e); \ + } \ + } \ + void radix_sort_##name(rstype_t *beg, rstype_t *end) \ + { \ + if (end - beg <= RS_MIN_SIZE) rs_insertsort_##name(beg, end); \ + else rs_sort_##name(beg, end, 8, sizeof_key * 8 - 8); \ + } + +#endif diff --git a/kvec.h b/kvec.h new file mode 100644 index 0000000..632fce4 --- /dev/null +++ b/kvec.h @@ -0,0 +1,110 @@ +/* The MIT License + + Copyright (c) 2008, by Attractive Chaos + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* + An example: + +#include "kvec.h" +int main() { + kvec_t(int) array; + kv_init(array); + kv_push(int, array, 10); // append + kv_a(int, array, 20) = 5; // dynamic + kv_A(array, 20) = 4; // static + kv_destroy(array); + return 0; +} +*/ + +/* + 2008-09-22 (0.1.0): + + * The initial version. + +*/ + +#ifndef AC_KVEC_H +#define AC_KVEC_H + +#include + +#define kv_roundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) + +#define kvec_t(type) struct { size_t n, m; type *a; } +#define kv_init(v) ((v).n = (v).m = 0, (v).a = 0) +#define kv_destroy(v) free((v).a) +#define kv_A(v, i) ((v).a[(i)]) +#define kv_pop(v) ((v).a[--(v).n]) +#define kv_size(v) ((v).n) +#define kv_max(v) ((v).m) + +#define kv_resize(type, v, s) do { \ + if ((v).m < (s)) { \ + (v).m = (s); \ + kv_roundup32((v).m); \ + (v).a = (type*)realloc((v).a, sizeof(type) * (v).m); \ + } \ + } while (0) + +#define kv_copy(type, v1, v0) do { \ + if ((v1).m < (v0).n) kv_resize(type, v1, (v0).n); \ + (v1).n = (v0).n; \ + memcpy((v1).a, (v0).a, sizeof(type) * (v0).n); \ + } while (0) \ + +#define kv_push(type, v, x) do { \ + if ((v).n == (v).m) { \ + (v).m = (v).m? (v).m<<1 : 2; \ + (v).a = (type*)realloc((v).a, sizeof(type) * (v).m); \ + } \ + (v).a[(v).n++] = (x); \ + } while (0) + +#define kv_pushp(type, v, p) do { \ + if ((v).n == (v).m) { \ + (v).m = (v).m? (v).m<<1 : 2; \ + (v).a = (type*)realloc((v).a, sizeof(type) * (v).m); \ + } \ + *(p) = &(v).a[(v).n++]; \ + } while (0) + +#define kv_a(type, v, i) ((v).m <= (size_t)(i)? \ + ((v).m = (v).n = (i) + 1, kv_roundup32((v).m), \ + (v).a = (type*)realloc((v).a, sizeof(type) * (v).m), 0) \ + : (v).n <= (size_t)(i)? (v).n = (i) \ + : 0), (v).a[(i)] + +#define kv_reverse(type, v, start) do { \ + if ((v).m > 0 && (v).n > (start)) { \ + size_t __i, __end = (v).n - (start); \ + type *__a = (v).a + (start); \ + for (__i = 0; __i < __end>>1; ++__i) { \ + type __t = __a[__end - 1 - __i]; \ + __a[__end - 1 - __i] = __a[__i]; __a[__i] = __t; \ + } \ + } \ + } while (0) + +#endif diff --git a/main.cpp b/main.cpp index 97333d5..735b92c 100644 --- a/main.cpp +++ b/main.cpp @@ -200,12 +200,6 @@ int main(int argc, char *argv[]) if (!CommandLine_process(argc, argv)) return 1; - - /** - debug_edit_distance(); - - return 1; - **/ fprintf(stdout, "Will perform %d round of error correction...\n", number_of_round); fprintf(stdout, "defined k_mer_min_freq by user: %d\n", k_mer_min_freq); @@ -213,10 +207,21 @@ int main(int argc, char *argv[]) fprintf(stdout, "k-mer length: %d\n",k_mer_length); + fprintf(stdout, "coverage: %d\n",coverage); + fprintf(stdout, "read_graph: %d\n", read_graph); + fprintf(stdout, "adapterLen: %d\n", adapterLen); + fflush(stdout); + + + + + if(required_read_name) + { + fprintf(stdout, "required_read_name: %s\n", required_read_name); + } + Correct_Reads(number_of_round); - - return 1; }