diff options
Diffstat (limited to 'seggetd/connection.cpp')
| -rw-r--r-- | seggetd/connection.cpp | 350 |
1 files changed, 350 insertions, 0 deletions
diff --git a/seggetd/connection.cpp b/seggetd/connection.cpp new file mode 100644 index 0000000..88f6069 --- /dev/null +++ b/seggetd/connection.cpp @@ -0,0 +1,350 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#include "connection.h" +#include "utils.h" +#include "settings.h" + +long script_waiting_connection_num=-1; +uint Tconnection::total_connections=0; +Tconnection connection_array[MAX_CONNECTS]; + +void init_connections(){ + for (ulong connection_num=0; connection_num<MAX_CONNECTS; connection_num++){ + connection_array[connection_num].connection_num=connection_num; + }; +} + +int Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num){ + try{ + segment=started_segment; + debug("Starting connection for distfile: "+segment->parent_distfile->name); + mirror_num=best_mirror_num; + network_num=network_number; + total_dld_bytes=0; + bytes_per_last_interval=0; + gettimeofday(&start_time,NULL); + debug("Connecting network"+toString(network_num)); + + segment->parent_distfile->set_status(DDOWNLOADING); + + if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ + connection_start_time_network_phase_for_pf_networks=segment->parent_distfile->network_distfile_brokers_array[network_num].phase; + } + + Tmirror *Pcurr_mirror; + switch (network_array[network_num].network_mode){ + case MODE_REMOTE:{ + url=segment->parent_distfile->url_list[mirror_num]; + Pcurr_mirror=find_mirror(strip_mirror_name(url)); + break; + } + case MODE_CORAL_CDN:{ + url=convert_to_coral_cdn_url(segment->parent_distfile->url_list[mirror_num]); + Pcurr_mirror=find_mirror(strip_mirror_name(url)); + break; + } + default:{ + Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num]; + url=Pcurr_mirror->url+segment->parent_distfile->name; + } + } + debug(" URL:"+url); + + if (run_user_python_script(connection_num)){ + if (segment->parent_distfile->active_connections_num<=0){ + segment->parent_distfile->set_status(DSCRIPTREJECTED); + } + return REJECTED_BY_USER_PYTHON_SCRIPT; + } + stats.active_connections_counter++; + ulong segment_start_result=segment->start(cm, connection_num, network_num, distfile_num, url); + if (segment_start_result){ + return segment_start_result; + }else{ + segment->parent_distfile->active_connections_num++; + active=true; + Pcurr_mirror->start(); + network_array[network_num].connect(); + } + debug("Started connection for distfile: "+segment->parent_distfile->name); + return 0; + }catch(...){ + error_log("Error in connection.cpp: start()"); + } + return ERROR_WHILE_PREPARING_CONNECTION; +} +/* +string explain_curl_error(int error_code){ + try{ + //curl_easy_strerror( + }catch(...){ + error_log("Error in connection.cpp: explain_curl_error()"); + } + return "Error in connection.cpp: explain_curl_error()"; +} +*/ +void Tconnection::stop(CURLcode connection_result){ + try{ + stats.active_connections_counter--; + Tmirror *Pcurr_mirror; + switch (network_array[network_num].network_mode){ + case MODE_REMOTE: + case MODE_CORAL_CDN:{ + Pcurr_mirror=find_mirror(strip_mirror_name(url)); + break; + } + default:{ + Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num]; + } + } + debug("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); + if (connection_result){ + string error_str=curl_easy_strerror(connection_result); + debug(" ERROR "+toString(connection_result)+": "+error_str); + error_log("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); + error_log(" ERROR "+toString(connection_result)+": "+error_str); + } + active=false; + network_array[network_num].disconnect(); +// network_array[network_num].benchmarked_mirror_list[mirror_num].stop(); + segment->segment_file.close(); + if (connection_result==0){ + if (! segment->segment_verification_is_ok()){ + connection_result=CURLE_READ_ERROR; + Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0); + debug("curl_lies - there is a problem downloading segment"); + error_log("curl_lies - there is a problem downloading segment"); + }else{ + Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),segment->segment_size); + } + }else{ + Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0); + } + segment->parent_distfile->active_connections_num--; + + msg_clean_connection(connection_num); + +/* + Tmirror *Pcurr_mirror; + if (network_array[network_num].network_mode==MODE_LOCAL){ + Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num]; + prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true; +// find_mirror(strip_mirror_name(segment->url)); + }else{ + Pcurr_mirror=find_mirror(strip_mirror_name(segment->url)); + } +*/ + debug("before gettimeofday"); + timeval now_time; + gettimeofday(&now_time,NULL); + + if (connection_result!=0){ + stats.fails_counter++; + switch (network_array[network_num].network_mode){ + case MODE_LOCAL:{ + // prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true; + debug("before setting mirror fail"); + segment->parent_distfile->network_distfile_brokers_array[network_num].local_mirror_failed(mirror_num); + debug("after setting mirror fail"); + // find_mirror(strip_mirror_name(segment->url)); + break; + } + case MODE_PROXY_FETCHER:{ + // prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true; + if (connection_start_time_network_phase_for_pf_networks==E_USE_AS_LOCAL_MIRRORS){ + segment->parent_distfile->network_distfile_brokers_array[network_num].local_mirror_failed(mirror_num); + }else{ // proxy-fetcher mirror failed, if everything correct it must be in phase E_PROXY_FETCHER_DOWNLOADED, + segment->parent_distfile->network_distfile_brokers_array[network_num].proxy_fetcher_mirror_failed(mirror_num); + } + // find_mirror(strip_mirror_name(segment->url)); + break; + } + default:{ // MODE REMOTE MIRRORS + } + } + // error -> start downloading again +// msg_status2(segment->connection_num, toString(connection_result)+"]- Failed download "+segment->file_name); + debug(toString(connection_result)+"]- Failed download "+segment->url); + if (segment->try_num>=settings.max_tries){ + segment->status=SFAILED; + segment->parent_distfile->set_status(DFAILED); + error_log("Segget failed to download distfile: "+segment->parent_distfile->name); + error_log("Segment:"+segment->file_name+" has reached max_tries limit - segment.status set to FAILED"); + } + else segment->status=SWAITING; + }else{ + // no error => count this one and start new + log("Succesfully downloaded "+segment->file_name+" on connection#"+toString(connection_num)); + debug(" Successful download "+segment->url); +// already done earlier in this function Pcurr_mirror=find_mirror(strip_mirror_name(segment->url)); + segment->status=SDOWNLOADED; + segment->parent_distfile->inc_dld_segments_count(segment); + }; + if ((segment->parent_distfile->get_status()!=DFAILED) + and (segment->parent_distfile->get_status()!=DDOWNLOADED) + and (segment->parent_distfile->get_status()!=DALL_LM_AND_PF_MIRRORS_FAILED)){ + if (segment->parent_distfile->active_connections_num>0){ + segment->parent_distfile->set_status(DDOWNLOADING); + }else{ + segment->parent_distfile->set_status(DWAITING); + } + } + }catch(...){ + error_log("Error in connection.cpp: stop()"); + } +} + +void Tconnection::inc_bytes_per_last_interval(ulong new_bytes_count){ + try{ + total_dld_bytes+=new_bytes_count; + bytes_per_last_interval+=new_bytes_count; + }catch(...){ + error_log("Error in connection.cpp: inc_bytes_per_last_interval()"); + } +} + +void Tconnection::show_connection_progress(ulong time_diff){ + try{ + stats.total_bytes_per_last_interval+=bytes_per_last_interval; + ulong speed=(bytes_per_last_interval*1000)/time_diff; + ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time); + string eta_string; + if (avg_speed==0){ + eta_string=" ETA: inf"; + }else{ + eta_string=" ETA: "+secsToString((segment->segment_size-segment->downloaded_bytes)/avg_speed); + } + string speed_str; + string avg_speed_str; + speed_str=" Speed: "+speedToString(speed); + avg_speed_str=" AVG speed: "+speedToString(avg_speed); + int percent=segment->downloaded_bytes*100/segment->segment_size; + string network_type_str; + switch (network_array[network_num].network_mode){ + case MODE_REMOTE: network_type_str="REM"; break; + case MODE_PROXY_FETCHER: network_type_str="P-F"; break; + case MODE_LOCAL: network_type_str="LOC"; break; + case MODE_CORAL_CDN: network_type_str="CDN"; break; + } + string progress_text=field("[Net",network_num,1) + +":"+network_type_str+"]" + +field(" Segment:",segment->segment_num, 5) + +field(" Try:",segment->try_num,4) + +field(" Bytes:",segment->downloaded_bytes,7) + +field(" / ",segment->segment_size,7) + +field(" = ",percent,3)+"%" + +speed_str + +avg_speed_str + +eta_string; + msg_segment_progress(segment->connection_num, progress_text); + bytes_per_last_interval=0; + }catch(...){ + error_log("Error in connection.cpp: show_connection_progress()"); + } +} + +string Tconnection::get_html_connection_progress(){ + try{ + ulong time_diff=time_left_since(stats.previous_time); +// stats.total_bytes_per_last_interval+=bytes_per_last_interval; + ulong speed=(bytes_per_last_interval*1000)/time_diff; + ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time); + string eta_string; + if (avg_speed==0){ + eta_string="inf"; + }else{ + eta_string=secsToString((segment->segment_size-segment->downloaded_bytes)/avg_speed); + } + string speed_str; + string avg_speed_str; + speed_str=speedToString(speed); + avg_speed_str=speedToString(avg_speed); + int segment_percent; + if (segment->segment_size>0){ + segment_percent=segment->downloaded_bytes*100/segment->segment_size; + }else{ + segment_percent=100; + } + int distfile_percent; + if (segment->parent_distfile->size>0){ + distfile_percent=segment->parent_distfile->dld_bytes*100/segment->parent_distfile->size; + }else{ + distfile_percent=100; + } + + int unfinished_segments_distfile_percent; + //TO-DO: it's necessary to check all connections and add dld bytes to be correct !!! + if (segment->parent_distfile->size>0){ + unfinished_segments_distfile_percent=segment->downloaded_bytes*100/segment->parent_distfile->size; + }else{ + unfinished_segments_distfile_percent=0; + } + string network_type_str; + switch (network_array[network_num].network_mode){ + case MODE_REMOTE: network_type_str="Remote"; break; + case MODE_PROXY_FETCHER: network_type_str="Proxy-Fetcher"; break; + case MODE_LOCAL: network_type_str="Local"; break; + case MODE_CORAL_CDN: network_type_str="CDN"; break; + } + string progress_text= + (string)"<td align=\"center\">\n<table width=\"100\" border=\"0\" cellpadding=\"0\" frame=\"void\">\n<tr>\n" + +((distfile_percent>0)?"<td bgcolor=\"#0000FF\">\n<img src=\"/img/blue_progress.jpg\" height=\"20\" width=\""+toString(distfile_percent)+"\"/>\n</td>\n" + :"") + +((unfinished_segments_distfile_percent>0)?"<td bgcolor=\"#00FF00\">\n<img src=\"/img/green_progress.jpg\" height=\"20\" width=\""+toString(unfinished_segments_distfile_percent)+"\"/>\n</td>\n" + :"") + +((100-distfile_percent-unfinished_segments_distfile_percent>0)?"<td bgcolor=\"#555555\">\n<img src=\"/img/bw_progress.jpg\" height=\"20\" width=\""+toString(100-distfile_percent-unfinished_segments_distfile_percent)+"\" />\n</td>\n" + :"") + +"</tr>\n</table>\n" +// +" "+toString(distfile_percent+unfinished_segments_distfile_percent)+"%" + +toString(distfile_percent)+"%" + +"</td>\n<td>\n" + +"<table>\n" + +"<tr>\n<td>"+segment->parent_distfile->name+"</td>\n</tr>\n" + +"<tr>\n<td>\n"+segment->url+"</td>\n</tr>\n" + +"</table>\n" + +"</td><td align=\"center\"><table width=\"100\" border=\"0\" cellpadding=\"0\" frame=\"void\"><tr>" + +((segment_percent>0)?"<td bgcolor=\"#00FF00\"><img src=\"/img/green_progress.jpg\" height=\"20\" width=\""+toString(segment_percent)+"\"/></td>" + :"") + +((100-segment_percent>0)?"<td bgcolor=\"#555555\"><img src=\"/img/bw_progress.jpg\" height=\"20\" width=\""+toString(100-segment_percent)+"\" /></td>" + :"") + +"</tr></table>" + +toString(segment_percent)+"%" + +"</td><td align=\"right\">"+toString(segment->segment_num) + +"</td><td align=\"right\">"+toString(segment->try_num) + +"</td><td align=\"right\">"+toString(network_num) + +"</td><td align=\"center\">"+network_type_str + +"</td><td align=\"right\">"+toString(segment->downloaded_bytes) + +"</td><td align=\"right\">"+toString(segment->segment_size) + +"</td><td align=\"right\">"+speed_str + +"</td><td align=\"right\">"+avg_speed_str + +"</td><td align=\"right\">"+eta_string + +"</td>"; + return progress_text; + }catch(...){ + error_log("Error in connection.cpp: show_connection_progress()"); + } + return ""; +}
\ No newline at end of file |