diff options
Diffstat (limited to 'request')
| -rw-r--r-- | request/Makefile | 29 | ||||
| -rw-r--r-- | request/config.cpp | 177 | ||||
| -rw-r--r-- | request/config.h | 54 | ||||
| -rw-r--r-- | request/distfile.cpp | 742 | ||||
| -rw-r--r-- | request/distfile.h | 155 | ||||
| -rw-r--r-- | request/phase.cpp | 37 | ||||
| -rw-r--r-- | request/phase.h | 43 | ||||
| -rw-r--r-- | request/pkg.h | 67 | ||||
| -rw-r--r-- | request/request.conf | 288 | ||||
| -rw-r--r-- | request/request.cpp | 247 | ||||
| -rw-r--r-- | request/request.h | 77 | ||||
| -rw-r--r-- | request/response.cpp | 65 | ||||
| -rw-r--r-- | request/response.h | 61 | ||||
| -rw-r--r-- | request/restrict.conf | 2 | ||||
| -rw-r--r-- | request/settings.cpp | 44 | ||||
| -rw-r--r-- | request/settings.h | 130 | ||||
| -rw-r--r-- | request/shorttypes.h | 32 | ||||
| -rw-r--r-- | request/str.cpp | 126 | ||||
| -rw-r--r-- | request/str.h | 49 | ||||
| -rw-r--r-- | request/utils.cpp | 44 | ||||
| -rw-r--r-- | request/utils.h | 39 |
21 files changed, 2508 insertions, 0 deletions
diff --git a/request/Makefile b/request/Makefile new file mode 100644 index 0000000..d7d45c7 --- /dev/null +++ b/request/Makefile @@ -0,0 +1,29 @@ +BINS = request +OBJS = $(addsuffix .o,$(BINS)) +PKGCONFIG_MODULES = json +MISSING_PLUGINS_LIBS = +LIBS_PKGCONFIG := $(foreach mod,$(PKGCONFIG_MODULES),$(shell pkg-config --libs $(mod))) +CXXFLAGS_PKGCONFIG := $(foreach mod,$(PKGCONFIG_MODULES),$(shell pkg-config --libs $(mod))) + +CXXFLAGS_WARNINGS = -pedantic -Wall -Wextra -Wformat -Weffc++ +#CXXFLAGS_WARNINGS = -pedantic -Wextra -Wformat -Weffc++ +CXXFLAGS_OPT = -O0 -g3 + +LIBS = $(LIBS_PKGCONFIG) +CXXFLAGS = $(CXXFLAGS_PKGCONFIG) $(CXXFLAGS_WARNINGS) $(CXXFLAGS_OPT) + +all: $(BINS) + +%.o: %.cxx + $(CXX) -c -o $@ $(CXXFLAGS) $^ + +request: request.o config.o phase.o response.o settings.o str.o utils.o + $(CXX) -o $@ $(LIBS) $(CIBS) $^ + +clean: + -rm -f $(BINS) + -rm -f $(OBJS) + -rm -f *~ + -rm -f *.o + +.PRECIOUS: %.o diff --git a/request/config.cpp b/request/config.cpp new file mode 100644 index 0000000..0cc5395 --- /dev/null +++ b/request/config.cpp @@ -0,0 +1,177 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "config.h" + +void Tconfig::load_settings_from_config_file(){ + try{ + ifstream file; + file.exceptions (ifstream::failbit | ifstream::badbit); + try{ + file.open((settings.conf_dir+"/"+config_file_name).c_str()); + } + catch(...){ + merror("Error opening settings file: "+settings.conf_dir+"/"+config_file_name+". Default settings will be used. Check if config file: "+config_file_name+" exists and segget has rights to access it."); + return; + } + try{ + //processing file + string line; + string name; + string value; + string inSection; + int posEqual; + while (not(file.eof())) { + getline(file,line); + if (! line.length()) continue; + if (line[0] == '#') continue; + if (line[0] == ';') continue; + line=noupper(line); + if (line[0] == '[') { + inSection=trim(line.substr(1,line.find(']')-1)); + continue; + } + posEqual=line.find('='); + name = trim(line.substr(0,posEqual)); + value = trim(line.substr(posEqual+1)); + content_[inSection+'/'+name]=noupper(value); + } + }catch(ifstream::failure e){ + if (!file.eof()){ + merror("Settings file: "+config_file_name+" was opened, but an error occured while reading it."); + return; + } + }catch(...){ + merror("Settings file: "+config_file_name+" was opened, but an error occured while reading settings from it."); + } + }catch(...){ + merror("Error in config.cpp: load_settings_from_config_file()"); + } +} + +int Tconfig::set(string const& section, string const& entry, string &dst) const { + try{ + map<string,string>::const_iterator ci = content_.find(section + '/' + entry); + if (ci == content_.end()){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+" has not been set."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+dst+". Default value forced."); + return 1; + } + else{ + dst=ci->second; + merror("Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+dst); + return 0; + } + }catch(...){ + merror("Error in config.cpp: set(string &dst, string const& section, string const& entry)"); + return 1; + } +} +int Tconfig::set(string const& section, string const& entry, ulong &dst, uint const& min_limit, uint const& max_limit) const { + try{ + uint return_value; + map<string,string>::const_iterator ci = content_.find(section + '/' + entry); + if (ci == content_.end()){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+" has not been set."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + else{ + return_value=atoi(ci->second.c_str()); + if (return_value==0) + if (toString(return_value)!=ci->second){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have an integer value in range from "+toString(min_limit) + +" to "+toString(max_limit) + +". Can't convert "+ci->second + +" to integer. "); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + if ((return_value>=min_limit) and (return_value<=max_limit)){ + //everything is ok + merror("Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(return_value)); + dst=return_value; + return 0; + }else{ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have an integer value in range from "+toString(min_limit) + +" to "+toString(max_limit)+"."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + } + }catch(...){ + merror("Error in config.cpp: set(ulong &dst, string const& section, string const& entry, uint const& min_limit, uint const& max_limit)"); + return 1; + } +} + +int Tconfig::set(string const& section, string const& entry, bool &dst) const { + try{ + uint return_value; + map<std::string,string>::const_iterator ci = content_.find(section + '/' + entry); + if (ci == content_.end()){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+" has not been set."); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + else{ + return_value=atoi(ci->second.c_str()); + if (return_value==0) + if (toString(return_value)!=ci->second){ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have a boolean value: 0 or 1" + +". Can't convert "+ci->second + +" to 0 or 1. "); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + if ((return_value==0) or (return_value==1)){ + //everything is ok + merror("Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(return_value)); + dst=return_value; + return 0; + }else{ + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry + +" must have a boolean value: 0 or 1"); + merror("! Settings in file:"+config_file_name+" ["+section+"]."+entry+"="+toString(dst)+". Default value forced."); + return 1; + } + } + }catch(...){ + merror("Error in config.cpp: set(bool &dst, string const& section, string const& entry)"); + return 1; + } +} + +void Tconfig::clear(){ + try{ + content_.clear(); + }catch(...){ + merror("Error in config.cpp: clear()"); + } +}
\ No newline at end of file diff --git a/request/config.h b/request/config.h new file mode 100644 index 0000000..9853d4c --- /dev/null +++ b/request/config.h @@ -0,0 +1,54 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __CONFIG_H__ +#define __CONFIG_H__ + +#include <string> +#include <map> +#include <fstream> +#include "str.h" +#include "settings.h" +#include "utils.h" +using namespace std; + +class Tconfig { + private: + map<string,string> content_; + string config_file_name; + public: + Tconfig(string const& file_name): + content_(), + config_file_name("") + {config_file_name=file_name;load_settings_from_config_file();}; + void load_settings_from_config_file(); + int set(string const& section, string const& entry, string &dst) const; + int set(string const& section, string const& entry, ulong &dst, uint const& min_limit, uint const& max_limit) const; + int set(string const& section, string const& entry, bool &dst) const; + void clear(); +}; + +#endif
\ No newline at end of file diff --git a/request/distfile.cpp b/request/distfile.cpp new file mode 100644 index 0000000..21c3ff5 --- /dev/null +++ b/request/distfile.cpp @@ -0,0 +1,742 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "distfile.h" + +//Make the necessary includes and set up the variables: +using namespace std; + +/* +#define R_LM_WAIT_FOR_LOCAL_MIRRORS 100 + +#define R_PF_ADDED_TO_PROXY_QUEUE 101 +#define R_PF_ALREADY_WAS_IN_QUEUE 102 +#define R_PF_DOWNLOADED 103 +#define R_PF_BE_MORE_PATIENT 104 +#define R_PF_ERROR_ADDING_TO_PROXY_QUEUE 105 +#define R_PF_FAILED 106 +#define R_PF_REJECTED 107 + +// 0 for succesfull return of provide_segment() +#define R_R_DOWNLOAD_STARTED 0 +#define R_R_WAITING 108 +#define R_R_DOWNLOADING 109 + +#define R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND 110 + +#define ALLOW_PROXY_FETCHER_NETWORKS 201 +#define DO_NOT_ALLOW_PROXY_FETCHER_NETWORKS 202 +#define ALLOW_REMOTE_NETWORKS 203 +#define DO_NOT_ALLOW_REMOTE_NETWORKS 204 +#define ALLOW_LOWER_PRIORITY_NETWORKS 205 +*/ + + +bool Tdistfile::allows_new_actions(){ +// if (downloaded) return false; +// else return true; +// int time_left=0; + if (status==DDOWNLOADED){ + debug("No new connection for distfile:"+name+". Distfile has DDOWNLOADED status"); + return false; + } + if (status==DFAILED){ + debug("No new connection for distfile:"+name+". Distfile has DFAILED status"); + return false; + } +// debug("Distfile "+Ppkg_array[pkg_num]->distfile_vector[distfile_num]->name+" allows new connections"); +// debug(" distfile_num:"+toString(distfile_num)); + if (active_connections_num<settings.max_connection_num_per_distfile){ + debug("Allow new connection for ============================================= Distfile "+name); + debug("max_connection limit has not been reached"); + return true; + }else{ + debug("No new connection for distfile: "+name+". It already has " + +toString(active_connections_num) + +" connections => choose another distfile."); + return false; + } +// if (((status==DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false; +//oterwise allow connections +// DNEW, +// D_NOT_PROXY_REQUESTED, +// DPROXY_REJECTED, +// DPROXY_FAILED, +// DPROXY_DOWNLOADED, +// DWAITING, +// DDOWNLOADING, +} + +void Tdistfile::init(){ + try{ + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + network_distfile_brokers_array[network_num].init(network_num); + } + }catch(...){ + error_log("Error: distfile.cpp: init()"); + } +} +void Tdistfile::load_url_list(json_object* json_array_distfile_urllist){ + try{ + url_count=json_object_array_length(json_array_distfile_urllist); + url_list= new string [url_count]; + for(uint urllist_array_item_num=0;urllist_array_item_num<url_count;urllist_array_item_num++){ + url_list[urllist_array_item_num]=json_object_get_string(json_object_array_get_idx(json_array_distfile_urllist,urllist_array_item_num)); + } + if (url_count){ + num=++stats.distfiles_count; + stats.inc_total_size(size); + } + }catch(...){ + error_log("Error: distfile.cpp: load_url_list()"); + } +} + +bool Tdistfile::check_if_dld(){ + try{ + ifstream filec((settings.distfiles_dir+"/"+name).c_str()); + ulong start = filec.tellg(); + filec.seekg (0, ios::end); + ulong end = filec.tellg(); + ulong d_size; + d_size = end - start; + //debug("seg:"+toString(seg_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size)); + filec.close(); + if (d_size==size){ + status=DDOWNLOADED; +// downloaded=true; + num=++stats.distfiles_count; + stats.inc_total_size(size); + stats.inc_dld_distfiles_count(); + stats.inc_dld_size(size); + debug("Distfile:"+name+" already downloaded"); + return true; + } + else{ + debug("Distfile:"+name+" not downloaded"); + return false; + } + }catch(...){ + error_log("Error: distfile.cpp: check_if_dld()"); + return false; + } +} + +void Tdistfile::load_distfile_from_json(json_object* json_obj_distfile){ + try{ + name=json_object_get_string(json_object_object_get(json_obj_distfile,"name")); + size=atoi(json_object_to_json_string(json_object_object_get(json_obj_distfile,"size"))); + RMD160=json_object_get_string(json_object_object_get(json_obj_distfile,"RMD160")); + SHA1=json_object_get_string(json_object_object_get(json_obj_distfile,"SHA1")); + SHA256=json_object_get_string(json_object_object_get(json_obj_distfile,"SHA256")); + if (not(check_if_dld())){ + json_data=json_object_to_json_string(json_obj_distfile); + split_into_segments(); + load_url_list(json_object_object_get(json_obj_distfile,"url_list")); + } + }catch(...){ + error_log("Error: distfile.cpp: load_distfile_from_json()"); + } +} + +void Tdistfile::split_into_segments(){ + try{ + int range_end; + if (size>settings.max_segment_size){ + segments_count=size/settings.max_segment_size; + // set segments to have equal size, only last segment may be somewhat large (up to segment_count bytes large) + segment_size=size/segments_count; + } + else{ + segment_size=size; + segments_count=1; + }; + dn_segments = new Tsegment [segments_count]; + //none downloaded yet + for (uint segment_num=0; segment_num<segments_count; segment_num++){ + if (segment_num == (segments_count-1)){ + range_end=size-1; + debug("last_segment ends"+toString(range_end)); + } + else{ + range_end=(segment_num+1)*segment_size-1; + debug("segment_range:"+toString(range_end)); + }; + dn_segments[segment_num].set_segment(this, segment_num, name, segment_size, range_end); + if (dn_segments[segment_num].status==SDOWNLOADED) + inc_dld_segments_count(&dn_segments[segment_num]); + } + }catch(...){ + error_log("Error: distfile.cpp: split_into_segments()"); + } +} + +Tdistfile::~Tdistfile(){ + try{ + delete [] url_list; + }catch(...){ + error_log("Error: distfile.cpp: ~Tdistfile()"); + } +} +bool Tdistfile::choose_best_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num){ + try{ + Tmirror *Pcurr_mirror; + Tmirror *Pbest_mirror=0; // the best isn't set let's find it + uint best_mirror_num=-1; // the best isn't set let's find it + + ulong best_mirror_self_rating=-1; + ulong curr_mirror_self_rating; + + for (url_num=0; url_num<url_count; url_num++){ + Pcurr_mirror=find_mirror(strip_mirror_name(url_list[url_num])); + debug("Evaluating url:"+url_list[url_num] + +" active_connections:"+toString(Pcurr_mirror->get_active_num()) + +" connection limit:"+toString(settings.max_connections_num_per_mirror)); + if (Pcurr_mirror->get_active_num()<settings.max_connections_num_per_mirror){ + debug("Url satisfies connection num per mirror limit"); + curr_mirror_self_rating=Pcurr_mirror->mirror_on_the_wall(); + if (curr_mirror_self_rating<best_mirror_self_rating){ + best_mirror_num=url_num; + best_mirror_self_rating=curr_mirror_self_rating; + Pbest_mirror=Pcurr_mirror; + } + if (best_mirror_self_rating==0) + // 0 can not be improved - it's one of the best + break; + } + } + if (Pbest_mirror){ + debug("Downloading from BEST_MIRROR:"+url_list[best_mirror_num]); +// Pbest_mirror->start(); +// active_connections_num++; + connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); + return R_R_DOWNLOAD_STARTED; + } + else{ + error_log("Can't choose mirror for segment:"+dn_segments[seg_num].file_name); + return 1; + } + }catch(...){ + error_log("Error: distfile.cpp: choose_best_mirror()"); + return 1; + } +} +bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num){ + try{ + long best_mirror_num=-1; // the best isn't set let's find it + bool all_mirrors_failed=true; + ulong best_mirror_self_rating=-1; + ulong curr_mirror_self_rating; + debug("Choosing mirror for network"+toString(network_num)); + for (ulong mirror_num=0; mirror_num<network_array[network_num].benchmarked_mirror_list.size(); mirror_num++){ + debug("Evaluating mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url); + if (network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]){ + debug("mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url+" has FAILED status - will be skipped"); + }else{ + all_mirrors_failed=false; + if (network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()<settings.max_connections_num_per_mirror){ + curr_mirror_self_rating=network_array[network_num].benchmarked_mirror_list[mirror_num].mirror_on_the_wall(); + if (curr_mirror_self_rating<best_mirror_self_rating){ + best_mirror_num=mirror_num; + best_mirror_self_rating=curr_mirror_self_rating; + } + if (best_mirror_self_rating==0) + // 0 can not be improved - it's one of the best + break; + }else{ + debug("Mirror already has:" + +toString(network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()) + +" connections which doesn't meet limit:"+toString(settings.max_connections_num_per_mirror)); + } + } + } + if (best_mirror_num!=-1){ + debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].benchmarked_mirror_list[best_mirror_num].url); +// active_connections_num++; + connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); + return R_R_DOWNLOAD_STARTED; + } + else{ + debug("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); + error_log("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); + if (all_mirrors_failed){ + debug("All local mirrors failed in network#"+toString(network_num)); + error_log("All local mirrors failed in network#"+toString(network_num)); + } + return 1; + } + }catch(...){ + error_log("Error: distfile.cpp: choose_best_local_mirror()"); + return 1; + } +} + +uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority){ + try{ + debug("network_priority="+toString(network_priority)); + //choose network +//---------------------------------------------------------------------------------------------------------- +// +// Several criterions can be used here to choose among networks with equal priority: +// min_active_connection, min_ratio_active_to_max_connection, best_speed_accocding_to_stats, etc +// add these options to segget.conf file +// +//---------------------------------------------------------------------------------------------------------- + int best_local_network_num=-1; + bool allow_proxy_fetcher_mirrors=true; + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + if (network_array[network_num].priority==network_priority){ + debug(" network_priority="+toString(network_array[network_num].priority)); + if (network_array[network_num].network_mode==MODE_LOCAL){ + debug(" network#:"+toString(network_num)+" has MODE_LOCAL"); + debug(" phase#"+phaseToString(network_distfile_brokers_array[network_num].phase)); + if (network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS){ + if (network_array[network_num].has_free_connections()){ + debug(" has free connections"); +// debug(" Allowed network#:"+toString(network_num)); + if ((best_local_network_num==-1) + or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_local_network_num=network_num; + debug(" Replace best LOCAL network to network#:"+toString(network_num)); + } + }else{ + debug(" does NOT have free connections"); + if (network_array[network_num].only_local_when_possible){ + allow_proxy_fetcher_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } + } + } + } + if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ + debug(" network#:"+toString(network_num)+" has MODE_PROXY_FETCHER"); + debug(" with phase:"+phaseToString(network_distfile_brokers_array[network_num].phase)); + if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS) + or (network_distfile_brokers_array[network_num].phase==E_PROXY_FETCHER_DOWNLOADED)) + { + debug("phase fits"); + if (network_array[network_num].has_free_connections()){ + debug(" has free connections"); +// debug(" Allowed network#:"+toString(network_num)); + if ((best_local_network_num==-1) + or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_local_network_num=network_num; + debug(" Replace best LOCAL network to network#:"+toString(network_num)); + } + }else{ + if (network_array[network_num].only_local_when_possible){ + allow_proxy_fetcher_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } + } + } + } + } + } + if (best_local_network_num!=-1){ + //best network has been found + //work with network + debug(" So best LOCAL network is network#:"+toString(best_local_network_num)); + return choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num); + }else{ + if (allow_proxy_fetcher_mirrors){ + return ALLOW_REQUESTS_TO_PROXY_FETCHERS; + }else{ + debug("NOT all local mirrors have failed - restricted to local mirrors only."); + return DO_NOT_ALLOW_REQUESTS_TO_PROXY_FETCHERS; + } + } + }catch(...){ + error_log("Error: distfile.cpp: provlocal_networkwork()"); + return 1; + } +} + +uint Tdistfile::request_proxy_fetcher_network(uint network_priority){ +// TO-DO: Add option to go for remote networks or low priority networks even if already DPROXY_QUEUED +// TO-DO: There can be several proxy-fetchers of the same priority level, define a rule to make a choice + try{ + debug("network_priority="+toString(network_priority)); + //choose network + int best_proxy_fetcher_network_num=-1; + bool allow_remote_mirrors=true; + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + if (network_array[network_num].priority==network_priority){ + if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ + //replace this one by does_not_reject_connections +// if (network_array[network_num].has_free_connections()){ + if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS) + or (network_distfile_brokers_array[network_num].phase==E_ALL_LOCAL_MIRRORS_FAILED)){ + debug("------//////////// Good phase !!!!!!!!!!!!!!!"); + decode_server_response(toString(network_distfile_brokers_array[network_num].proxy_fetcher_response)); + switch (network_distfile_brokers_array[network_num].proxy_fetcher_response){ + case R_PF_DOWNLOADED:break; + case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:break; + case R_PF_FAILED:break; + case R_PF_REJECTED:break; + default:{ + //if less then 30 secs left don't bother proxy-fetcher + if (2000>time_left_from(network_distfile_brokers_array[network_num].last_request_time)){ + debug("====================== Be more patient with proxy-fetcher - network#:"+toString(network_num) + +" because time left:"+toString(time_left_from(network_distfile_brokers_array[network_num].last_request_time))); + return R_PF_BE_MORE_PATIENT; + }else{ +// (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num) + best_proxy_fetcher_network_num=network_num; + debug("-------------------------------"); + debug(" Replace best_proxy_fetcher network_num to network#:"+toString(best_proxy_fetcher_network_num)); + }; + break; + } + } + } + if (best_proxy_fetcher_network_num!=-1) break; +/* + }else{ + if (network_array[network_num].only_local_when_possible){ + if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ + allow_remote_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } + } + } +*/ + } + } + } + if (best_proxy_fetcher_network_num != -1){ + // request from proxy fethcer + network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response=request(best_proxy_fetcher_network_num, json_data); + if ((network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response==R_PF_DOWNLOADED) + and (network_distfile_brokers_array[best_proxy_fetcher_network_num].phase==E_ALL_LOCAL_MIRRORS_FAILED)) + { + network_distfile_brokers_array[best_proxy_fetcher_network_num].phase=E_PROXY_FETCHER_DOWNLOADED; + } + debug("Trying to dowload distfile"+name+" via proxy_fetcher. status="+toString(status)); + if (network_distfile_brokers_array[best_proxy_fetcher_network_num].phase==E_PROXY_FETCHER_DOWNLOADED){ + // start download from the proxy_fetcher + debug(" So best proxy_fetcher_network is network#:"+toString(best_proxy_fetcher_network_num)); + return R_PF_DOWNLOADED; + //choose_best_local_mirror(cm, connection_num, best_proxy_fetcher_network_num, seg_num); + } + //return - don't switch to low priority networks + return network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response; + }else{ + if (allow_remote_mirrors){ + debug(" So best_proxy_fetcher_failed num="+toString(best_proxy_fetcher_network_num)); + return ALLOW_REMOTE_NETWORKS; + }else{ + debug("NOT all proxy-fetcher mirrors have failed - restricted to proxy-fetcher mirrors only."); + return DO_NOT_ALLOW_REMOTE_NETWORKS; + } + } + }catch(...){ + error_log("Error: distfile.cpp: provproxy_fetcher_networkwork()"); + return 1; + } +} + +uint Tdistfile::provide_remote_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority){ + try{ + debug("network_priority="+toString(network_priority)); + //choose network +//---------------------------------------------------------------------------------------------------------- +// +// Several criterions can be used here to choose among networks with equal priority: +// min_active_connection, min_ratio_active_to_max_connection, best_speed_accocding_to_stats, etc +// add these options to segget.conf file +// +//---------------------------------------------------------------------------------------------------------- + int best_remote_network_num=-1; + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + if (network_array[network_num].priority==network_priority){ + if (network_array[network_num].network_mode==MODE_REMOTE){ + if (network_array[network_num].has_free_connections()){ + if + ((best_remote_network_num==-1) + or + (network_array[best_remote_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_remote_network_num=network_num; + debug(" Replace best_remote_network_num to network#:"+toString(network_num)); + } + } + } + } + } + if (best_remote_network_num!=-1){ + //best network has been found + //work with network + debug(" So best network is network#:"+toString(best_remote_network_num)); + return choose_best_mirror(cm, connection_num, best_remote_network_num, seg_num); + } + // haven't found anything suitable + return ALLOW_LOWER_PRIORITY_NETWORKS; + }catch(...){ + error_log("Error: distfile.cpp: provide_remote_network()"); + return 1; + } +} + +int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ + try{ + int result=R_NOT_SET; + for (uint network_priority=10; network_priority>0; network_priority--){ + do { + debug("Switching to local networks with priority:"+toString(network_priority)); + result=provide_local_network(cm, connection_num, seg_num, network_priority); + if (result==ALLOW_REQUESTS_TO_PROXY_FETCHERS){ + debug("Requesting proxy-fetcher networks with priority:"+toString(network_priority)); + result=request_proxy_fetcher_network(network_priority); + switch (result){ + case ALLOW_REMOTE_NETWORKS:{ + debug("Switching to remote networks with priority:"+toString(network_priority)); + result=provide_remote_network(cm, connection_num, seg_num, network_priority); + // if not lower_priority_networks => result found => return it + if (result!=ALLOW_LOWER_PRIORITY_NETWORKS){ + return result; + }; + break; + } + // don't switch to remote and lower priority networks => return results + case DO_NOT_ALLOW_REMOTE_NETWORKS: return result; + case R_PF_DOWNLOADED: {debug("Yep, R_PF_DONWLOADED => LOOP AGAIN"); break;}; + // if already in queue, etc... + // don't switch to remote and lower priority networks => return results + default: return result; + } + }else{ + // don't switch to proxy-fetcher requests, remote, + // and lower priority networks => return results + return result; + } + // try to download from local mirrors again because one of the proxy-fetcher + // claims that distfile has been donwloaded + debug("----- JUST BEFORE THE LOOP ----"); + }while (result==R_PF_DOWNLOADED); + } + // haven't found anything suitable + return R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND; + }catch(...){ + error_log("Error: distfile.cpp: provide_segment()"); + return 1; + } +} +void Tdistfile::inc_dld_segments_count(Tsegment* current_segment){ + try{ + stats.inc_dld_size(current_segment->segment_size); + if (++dld_segments_count==segments_count) + combine_segments(); + }catch(...){ + error_log("Error: distfile.cpp: inc_dld_segments_count()"); + } +} +long is_symlink_restricted(string distfile_name){ + for(ulong pattern_num=0; pattern_num<settings.provide_mirror_files_restricted_patterns_vector.size(); pattern_num++){ + if (distfile_name.find(settings.provide_mirror_files_restricted_patterns_vector[pattern_num],0)!=distfile_name.npos){ + return pattern_num; + } + } + return -1; +} + +void Tdistfile::symlink_distfile_to_provide_mirror_dir(){ + try{ + long pattern_num=is_symlink_restricted(name); + if (pattern_num>=0){ + log("Symlink to distfile:"+name+" was restricted by pattern <" + +settings.provide_mirror_files_restricted_patterns_vector[pattern_num] + +"> from line "+toString(pattern_num+1)+" of restrict.conf file"); + return; + } + string new_mirror_name; + string old_distfile_name; + try{ + string old_distfile_path; + char current_path[FILENAME_MAX]; + if (!GetCurrentDir(current_path, sizeof(current_path))) + { + return; + } + if (settings.distfiles_dir.find("./",0)==0){ + old_distfile_path=current_path+settings.distfiles_dir.substr(1,settings.distfiles_dir.npos); + }else{ + old_distfile_path=settings.distfiles_dir; + } + new_mirror_name=settings.provide_mirror_dir+"/"+name; + old_distfile_name=old_distfile_path+"/"+name; + try{ + if (!symlink(old_distfile_name.c_str(), new_mirror_name.c_str())){ + log("Distfile:"+old_distfile_path+" was symlinked to the mirror dir:"); + }; + }catch(uint errno){ + switch (errno){ + case EACCES : error_log("Write access to the directory containing "+settings.provide_mirror_dir+" is denied, or one of the directories in the path prefix of "+settings.provide_mirror_dir+" did not allow search permission. (See also path_resolution(7)."); + break; + case EEXIST : error_log("There is already an existing file named "+new_mirror_name+"."); + break; + case EFAULT : error_log(old_distfile_path+" or "+settings.provide_mirror_dir+" points outside your accessible address space."); + break; + case EIO : error_log("A hardware error occurred while reading or writing data on the disk."); + break; + case ELOOP : error_log("Too many symbolic links were encountered in resolving "+settings.provide_mirror_dir+"."); + break; + case ENAMETOOLONG : error_log(old_distfile_path+" or "+settings.provide_mirror_dir+" was too long."); + break; + case ENOENT : error_log("A directory component in "+settings.provide_mirror_dir+" does not exist or is a dangling symbolic link, or "+old_distfile_path+" is the empty string."); + break; + case ENOMEM : error_log("Insufficient kernel memory was available."); + break; + case ENOSPC : error_log("The device containing the file has no room for the new directory entry."); + break; + case ENOTDIR : error_log("A component used as a directory in "+settings.provide_mirror_dir+" is not, in fact, a directory."); + break; + case EPERM : error_log("The file system containing "+settings.provide_mirror_dir+" does not support the creation of symbolic links."); + break; + case EROFS : error_log("The file "+new_mirror_name+" would exist on a read-only file system."); + break; + default: + error_log("Undocumented error while trying to symlink "+old_distfile_name+" to "+new_mirror_name); + } + }catch(...){ + error_log("Undocumented error (error description is not an integer) while trying to symlink "+old_distfile_name+" to "+new_mirror_name); + } + }catch(...){ + error_log("Error in distfile.cpp :: symlink_distfile_to_provide_mirror_dir() while trying to symlink "+old_distfile_name+" to "+new_mirror_name); + } + }catch(...){ + error_log("Error: distfile.cpp: symlink_distfile_to_provide_mirror_dir()"); + } +} +int Tdistfile::combine_segments(){ + try{ + debug("Combining distfile"+name); + ofstream distfile_file; + distfile_file.exceptions (ofstream::failbit | ofstream::badbit); + string distfile_path=settings.distfiles_dir+"/"+name; + try{ + distfile_file.open(distfile_path.c_str(),ofstream::binary|ios::trunc); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): opening distfile:"+distfile_path); + status=DFAILED; + return 1; + } + char * buffer; + ulong cur_seg_size; + try{ + for (uint seg_num=0; seg_num < segments_count; seg_num++){ + debug("Joining "+name+" segment "+toString(seg_num)+" "); + ifstream segment_file; + segment_file.exceptions (ofstream::failbit | ofstream::badbit); + try{ + segment_file.open((settings.segments_dir+"/"+dn_segments[seg_num].file_name).c_str(),ifstream::binary); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): opening segment:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); + status=DFAILED; + return 2; + } + try{ + // get size of file + ulong start=segment_file.tellg(); + segment_file.seekg(0,ifstream::end); + ulong end=segment_file.tellg(); + cur_seg_size=end-start; + segment_file.seekg(0); + debug(">>>>>SEG:"+dn_segments[seg_num].file_name+" Start: "+toString(start)+" End: "+toString(end)+" Size: "+toString(cur_seg_size)); + // allocate memory for file content + buffer = new char [cur_seg_size]; + // read content of infile + segment_file.read (buffer,cur_seg_size); + segment_file.close(); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): segment is open, but error occured while reading it:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); + status=DFAILED; + return 3; + } + try{ + // write to outfile + distfile_file.write (buffer,cur_seg_size); + }catch(...){ + error_log("Error: distfile.cpp: combine_segments(): distfile is open, but error occured while writing into it:"+settings.distfiles_dir+"/"+name); + status=DFAILED; + return 4; + } + // release dynamically-allocated memory + delete[] buffer; + if(remove((settings.segments_dir+"/"+dn_segments[seg_num].file_name).c_str()) != 0 ) + error_log("Tdistfile::combine_segments() - Error: can't delete:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); + else + debug(settings.segments_dir+"/"+dn_segments[seg_num].file_name+" deleted" ); + } + + distfile_file.close(); + stats.inc_dld_distfiles_count(); + log("Distfile "+name+" has been combined"); + }catch(...){ + error_log("Error in distfile.cpp: combine_segments() for distfile:"+settings.distfiles_dir+"/"+name); + status=DFAILED; + return 5; + } + try{ + if (rmd160_ok(settings.distfiles_dir+"/"+name,RMD160)) + log("RMD160 checksum for distfile:"+name+" is [OK]"); + else{ + log("Error: RMD160 checksum for distfile:"+name+" [FAILED]"); + error_log("Error: RMD160 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; + return 10; + } + + if (sha1_ok(settings.distfiles_dir+"/"+name,SHA1)) + log("SHA1 checksum for distfile:"+name+" is [OK]"); + else{ + log("Error: SHA1 checksum for distfile:"+name+" [FAILED]"); + error_log("Error: SHA1 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; + return 11; + } + + if (sha256_ok(settings.distfiles_dir+"/"+name,SHA256)) + log("SHA256 checksum for distfile:"+name+" is [OK]"); + else{ + log("Error: SHA256 checksum for distfile:"+name+" [FAILED]"); + error_log("Error: SHA256 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; + return 12; + } + status=DDOWNLOADED; + if (settings.provide_mirror_dir!="none"){ + symlink_distfile_to_provide_mirror_dir(); + } + }catch(...){ + error_log("Error: distfile.cpp: combine_segments() for segment:"+settings.distfiles_dir+"/"+name+" while checking checksums."); + return 30; + } + }catch(...){ + error_log("Error: distfile.cpp: combine_segments() for segment:"+settings.distfiles_dir+"/"+name+" during procedure."); + return 31; + } + return 0; +}
\ No newline at end of file diff --git a/request/distfile.h b/request/distfile.h new file mode 100644 index 0000000..8d42894 --- /dev/null +++ b/request/distfile.h @@ -0,0 +1,155 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __DISTFILE_H__ +#define __DISTFILE_H__ +#include <errno.h> +#include <stdlib.h> +#include <string.h> +#ifndef WIN32 +# include <unistd.h> +#endif +#include <curl/multi.h> + +#include <stdio.h> +#include <stdlib.h> +#include <fstream> +#include <iostream> +#include <json/json.h> +#ifdef WINDOWS + #include <direct.h> + #define GetCurrentDir _getcwd +#else + #include <unistd.h> + #define GetCurrentDir getcwd +#endif +#include "segment.h" +#include "mirror.h" +#include "checksum.h" +#include "network.h" +#include "networkbroker.h" +#include "segment.h" +#include "response.h" +class Tsegment; + +//#include <sys/types.h> +//#include <sys/socket.h> +//#include <stdio.h> +//#include <netinet/in.h> +//#include <arpa/inet.h> +//#include <sys/ioctl.h> +//#include <unistd.h> +//#include <stdlib.h> +//#include <string> +//#include <string.h> + + +//#include "client.cpp" + +using namespace std; + +typedef unsigned int uint; + +enum Tdistfile_status{ + DNEW, + D_NOT_PROXY_REQUESTED, + DPROXY_REJECTED, + DPROXY_QUEUED, + DPROXY_DOWNLOADING, + DPROXY_DOWNLOADED, + DPROXY_FAILED, + DWAITING, + DDOWNLOADING, + DDOWNLOADED, + DFAILED +}; + +long is_symlink_restricted(string distfile_name); + +class Tdistfile{ + private: + uint dld_segments_count; + bool choose_best_local_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num); + bool choose_best_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num); + public: + Tnetwork_distfile_broker network_distfile_brokers_array[MAX_NETWORKS]; + string json_data; +// bool downloaded; + Tdistfile_status status; + uint active_connections_num; + string *url_list; + uint url_num; + uint segment_num; + uint segments_count; + Tsegment *dn_segments; + string name; + uint num; + ulong size; + string RMD160; + string SHA1; + string SHA256; + uint url_count; + uint segment_size; + Tdistfile(): + dld_segments_count(0), + json_data(""), +// downloaded(0), + status(DNEW), + active_connections_num(0), + url_list(0), + url_num(0), + segment_num(0), + segments_count(0), + dn_segments(0), + name(""), + num(0), + size(0), + RMD160(""), + SHA1(""), + SHA256(""), + url_count(0), + segment_size(settings.max_segment_size) + {init();}; + Tdistfile(const Tdistfile &L); // copy constructor + Tdistfile & operator=(const Tdistfile &L); + ~Tdistfile(); + int request(ulong network_num, string msg); + void init(); + bool allows_new_actions(); + void load_distfile_from_json(json_object* json_obj_distfile); + void load_url_list(json_object* json_array_distfile_urllist); + void split_into_segments(); + uint provide_local_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority); + uint request_proxy_fetcher_network(uint network_priority); + uint provide_remote_network(CURLM* cm, uint connection_num, uint seg_num, uint network_priority); + int provide_segment(CURLM* cm, uint connection_num, uint seg_num); + void inc_dld_segments_count(Tsegment * current_segment); + void symlink_distfile_to_provide_mirror_dir(); + int combine_segments(); + bool check_if_dld(); +}; + +#endif
\ No newline at end of file diff --git a/request/phase.cpp b/request/phase.cpp new file mode 100644 index 0000000..04aef35 --- /dev/null +++ b/request/phase.cpp @@ -0,0 +1,37 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "phase.h" + +string phaseToString(int phase_num){ + switch(phase_num){ + case E_USE_AS_LOCAL_MIRRORS: return "E_USE_AS_LOCAL_MIRRORS"; + case E_ALL_LOCAL_MIRRORS_FAILED: return "E_ALL_LOCAL_MIRRORS_FAILED"; + case E_PROXY_FETCHER_DOWNLOADED: return "E_PROXY_FETCHER_DOWNLOADED"; + case E_ALL_PROXY_FETCHER_MIRRORS_FAILED: return "E_ALL_PROXY_FETCHER_MIRRORS_FAILED"; + default: return "Unknown phase"; + } +} diff --git a/request/phase.h b/request/phase.h new file mode 100644 index 0000000..221c33d --- /dev/null +++ b/request/phase.h @@ -0,0 +1,43 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __PHASE_H__ +#define __PHASE_H__ + +#include <string> + +using namespace std; + +enum Tnetwork_distfile_broker_phases{ + E_USE_AS_LOCAL_MIRRORS, + E_ALL_LOCAL_MIRRORS_FAILED, + E_PROXY_FETCHER_DOWNLOADED, + E_ALL_PROXY_FETCHER_MIRRORS_FAILED +}; + +string phaseToString(int phase_num); + +#endif
\ No newline at end of file diff --git a/request/pkg.h b/request/pkg.h new file mode 100644 index 0000000..a7a20b6 --- /dev/null +++ b/request/pkg.h @@ -0,0 +1,67 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef __PKG_H__ +#define __PKG_H__ + +#include <stdio.h> +#include <stdlib.h> +#include <fstream> +#include <iostream> +#include <json/json.h> +#include "distfile.h" + +using namespace std; +typedef unsigned int uint; +typedef Tdistfile* Pdistfile; + +#define Q_ADDED_TO_QUEUE 0 +#define Q_ALREADY_EXISTS 1 +#define Q_ERROR 100 + +class Tpkg{ + public: + string name; + string category; + uint distfile_count; + void load_distfile_list(json_object* json_array_distfile_list); + Tpkg(): + Pdistfile_list(), + name(""), + category(""), + distfile_count(0) + {}; + Tpkg(const Tpkg &L); // copy constructor + Tpkg & operator=(const Tpkg &L); + ~Tpkg(); + int push_back_distfile(json_object* json_obj_distfile); + int try_adding_distfile_to_proxy_fetchers_queue(json_object* json_obj_distfile); + void load_pkg_from_json(json_object* json_obj_pkg); +}; + +extern Tpkg **Ppkg_array; +extern Tpkg proxy_fetcher_pkg; + +#endif
\ No newline at end of file diff --git a/request/request.conf b/request/request.conf new file mode 100644 index 0000000..61dac25 --- /dev/null +++ b/request/request.conf @@ -0,0 +1,288 @@ +[folders] +# DISTFILES_DIR +# Define a dir to store distfiles +# Default: +# distfiles_dir=./distfiles +distfiles_dir=./distfiles + +# SEGMENTS_DIR +# Define a dir to store distfiles' segments +# Default: +# segments_dir=./tmp +segments_dir=./tmp + +[pkg_list] +# PKG_LIST_DIR +# Define a dir with pkg.list file +# Default: +# pkg_list_dir=./ +pkg_list_dir=./ + +# SYNOPSIS: del_pkg_list_when_dld_finished=0 | 1 +# - If del_pkg_list_when_dld_finished set to 1: +# Segget deletes pkg.list file, after all distfiles were successfuly fetched. +# Default: +# del_pkg_list_when_dld_finished=1 +del_pkg_list_when_dld_finished=1 + +[portage] +# SYNOPSIS: fetch_only=0 | 1 +# NOT IMPLEMENTED YET: fetch_only=0 + +[distfiles] +# MAX_CONNECTION_NUM_PER_DISTFILE +# Each distfile can have up to max_connection_num_per_distfile simultaneous +# connections. +# Minimum value: 1 +# Maximum value: 20 +# Default: +# max_connection_num_per_distfile=3 +max_connection_num_per_distfile=3 + +[segments] +# MAX_SEGMENT_SIZE +# Define maximum segment size in bytes. +# Minimum value: 10000 +# Maximum value: 10000000 +# Default: +# max_segment_size=500000 +max_segment_size=500000 + +# SYNOPSIS: resume_on=0 | 1 +# - If resume_on set to 1: +# Before starting downloading a segment segget checks whether this segment is +# already downloaded, checks segments size and if size mathces considers this +# segment to be downloaded and skips downloading process. +# - If resume_on set to 0: +# Segget always starts new fetch for a segment regardless of the fact whether it +# is downloaded or not. +# Default: +# resume_on=1 +resume_on=1 + +# MAX_TRIES +# If segment download was unsuccessful, new attempts are made. When attempts +# number reaches max_tries, segment gets FAILED status and error logged to error_log +# Minimum value: 1 +# Maximum value: -1 (for no limit) +# Default: +# max_tries=10 +max_tries=10 + +[connections] +# MAX_CONNECTIONS +# Define maximum number of connections +# Minimum value: 1 +# Maximum value: 20 +# Default: +# max_connections=10 +max_connections=10 + +# CURRENT_SPEED_TIME_INTERVAL_MSECS +# segget transfers may have bursty nature of their traffic. Therefore, while +# measuring current speed, segget actually calculates average speed during +# current_speed_time_interval_msecs time interval, defined in milliseconds. +# Minimum value: 100 +# Maximum value: 60000 +# Default: +# current_speed_time_interval_msecs=1000 +current_speed_time_interval_msecs=1000 + +# NOT IMPLEMENTED YET: max_total_speed=50000 + +[mirrors] +# MAX_CONNECTIONS_NUM_PER_MIRROR +# Define how many simultaneous downloads from one mirror segget is allowed to +# have. While choosing a mirror segget will skip mirrors with +# max_connections_num_per_mirror active downloads. +# Minimum value: 1 +# Maximum value: 10 +# Default: +# max_connections_num_per_mirror=1 +max_connections_num_per_mirror=1 + +# SYNOPSIS: collect_benchmark_stats_on=0 | 1 +# - If set to 1, stats on mirrors performance will be collected. +# default: +# collect_benchmark_stats_on=1 (Note: at the moment can NOT be changed) +collect_benchmark_stats_on=1 + +# SYNOPSIS: use_benchmark_stats=0 | 1 +# If use_benchmark_stats=1 statistics on mirrors is used to rate them +# and therefore improve performance. +# Each time connection from a particular mirror closes mirror->dld_time, +# and mirror->dld_size get increased (in case of unsuccessful connection +# only time gets increased), so avg speed for a mirror can be +# calculated: +# +# mirror->avg_speed=mirror->dld_size/mirror->dld_time. (1) +# +# When new segment is going to be started segget goes through the list of +# the mirrors distfile/segment has, and asks each mirror for self_rating: +# +# "ulong self_rating=mirror->mirror_on_the_wall();". +# +# This way segget chooses a mirror with the best self_rating. To calculate +# self_rating mirrors use the following formula: +# +# self_rating=dld_time/dld_size*honesty. (2) +# +# So mirrors actually say how bad they are. +# Even mirrors can have critical times, so to give mirrors another chance +# honesty was added to the formula (2). honesty can get values in +# interval (0,1]. Each time connection from a mirror opens or closes mirror +# sets its honesty=1. If mirror was asked for self_evaluation with +# mirror->mirror_on_the_wall(), but wasn't chosen its honesty decreases +# somewhat (see [mirror].benchmark_oblivion option), so next time it will +# lie little bit more about how bad it's. +# Default: +# use_benchmark_results=1 +# ***Note: at the moment use_benchmark_results can NOT be set to 0 +use_benchmark_stats=1 + +# BENCHMARK_OBLIVION +# benchmark_oblivion option allows to adjust how fast segget "forgets" +# benchmarking statistics on mirrors performance. +# Each time mirror->mirror_on_the_wall() called, mirror decreases its +# honesty (to have more chances next time) using the following formula: +# +# honesty=honesty*100/(100+settings.benchmark_oblivion) (3) +# +# Therefore, setting benchmark_oblivion=100 will make mirror look twice +# less bad next time mirror->mirror_on_the_wall() called. +# Minimum value: 0 (Beware: setting to 0, will can exclude unlucky mirrors) +# Maximum value: 1000 +# Default: +# benchmark_oblivion=5 +benchmark_oblivion=5 + +[provide_mirror_to_others] +# PROVIDE_MIRROR_DIR +# Define a dir for making symlinks to downloaded distfiles. This dir can be +# used to provide local mirror for other hosts (with help of Apache, vsftp, etc). +# If set to none, segget will not make symlinks. +# Default: +# provide_mirror_dir=none +provide_mirror_dir=./provide_mirror_dir + +# SYNOPSIS: PROVIDE_MIRROR_FILES_RESTRICT_LIST_ON= 0 | 1 +# If PROVIDE_MIRROR_DIR=none this option will be ignored. +# - If set to 1, segget will compare distfile name with the list of forbiden +# patterns from the restricted.conf file. If distfile name contains any of the +# patterns, no symlink will be provided to this distfile. +# Default: +# provide_mirror_files_restrict_list_on=0 +provide_mirror_files_restrict_list_on=0 + +[provide_proxy_fetcher_to_others] +# PROVIDE_PROXY_FETCHER_IP +# Define an ip address segget will use to provide access for tuiclients. +# The parameter should be a string holding your host dotted IP address. +# Default: +# provide_proxy_fetcher_ip=none +provide_proxy_fetcher_ip=127.0.0.1 + +# PROVIDE_PROXY_FETCHER_PORT +# Define a port segget will use to provide access for tuiclients. +# The parameter should be an integer. +# Minimum value: 1 +# Maximum value: 65535 +# Default: +# provide_proxy_fetcher_port=3131 +provide_proxy_fetcher_port=3130 + +[networks] +# NETWORK0_PRIORITY +# Define priority as a value in range from lowest 0 to highest 10. +# Segget tries to use networks with higher priority levels first, and in case of +# failure will switch to networks with lower priority levels. +# Segget will NOT use network if its priority level set to 0. Therefore at least +# one network must have priority level higher than 0. +# Networks with local mirrors usually would have higher priority than that of +# networks with remote mirrors. +# Segget can have up to 10 networks (from network0 to network9). +# Settings for each network should be defined in network#.conf file, where +# instead of # should be network number. For network0 it's network0.conf +# Default: +# network0_priority=10 +# network1_priority=0 +# network2_priority=0 +# network3_priority=0 +# network4_priority=0 +# network5_priority=0 +# network6_priority=0 +# network7_priority=0 +# network8_priority=0 +# network9_priority=0 +network0_priority=0 +network1_priority=0 +network2_priority=0 +network3_priority=0 +network4_priority=7 +network5_priority=0 +network6_priority=0 +network7_priority=0 +network8_priority=0 +network9_priority=0 + +[schedule] + +[user-data] +# USER_AGENT +# Set the User-Agent: header in the http request sent to the remote server. +# This can be used to fool servers or scripts. +# Default: +# user_agent=segget +user_agent=segget + +# Specify the user and password for authentication on a ftp servers. +# NOT IMPLEMENTED YET: ftp-user=anonymous +# NOT IMPLEMENTED YET: ftp-password=me@mail.ru + +[ui_server] +# tuiclient monitors segget's activity by establishing tcp connection +# with segget daemon (ui_server part of it). + +# UI_IP +# Define an ip address segget will use to provide access for tuiclients. +# The parameter should be a string holding your host dotted IP address. +# Default: +# ui_ip=127.0.0.1 +ui_ip=127.0.0.1 + +# UI_PORT +# Define a port segget will use to provide access for tuiclients. +# The parameter should be an integer. +# Minimum value: 1 +# Maximum value: 65535 +# Default: +# ui_port=9999 +ui_port=9999 + +[logs] +# LOGS_DIR +# Define a dir to store log files. +# Default: +# logs_dir=./logs +logs_dir=/home/mona/idfetcha/logs + +# GENERAL_LOG_FILE +# Define a file name to store general log. +# Set to none to disable loggin. +# Default: +# general_log_file=segget.log +general_log_file=segget.log + +# ERROR_LOG_FILE +# Define a file name to store error log. +# Set to none to disable loggin. +# Default: +# error_log_file=segget.log +error_log_file=error.log + +# DEBUG_LOG_FILE +# Define a file name to store debug log. +# Set to none to disable loggin. +# Default: +# debug_log_file=segget.log +debug_log_file=debug.log
\ No newline at end of file diff --git a/request/request.cpp b/request/request.cpp new file mode 100644 index 0000000..9847f75 --- /dev/null +++ b/request/request.cpp @@ -0,0 +1,247 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "request.h" + +int request(string msg){ +// gettimeofday(&network_distfile_brokers_array[network_num].last_request_time, NULL); + int len; + struct sockaddr_in address; + int result; + //Create a socket for the client: + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + //Name the socket, as agreed with the server: + address.sin_family = AF_INET; + address.sin_addr.s_addr = inet_addr(settings.request_ip.c_str()); + address.sin_port = htons(settings.request_port); + len = sizeof(address); + + //Connect your socket to the server’s socket: + result = connect(sockfd, (struct sockaddr *)&address, len); + if(result == -1) { + merror("Can't connect to segget daemon"); + return R_PF_ERROR_ADDING_TO_PROXY_QUEUE; + } + if (msg.length()>90000){return R_PF_ERROR_ADDING_TO_PROXY_QUEUE;}; + char send_buffer[100000]; + // char recv_buffer[256]; + strcpy(send_buffer,msg.c_str()); + //You can now read and write via sockfd: + if (write(sockfd, send_buffer, strlen(send_buffer))!=(int)msg.length()){ + merror("Error in request.cpp: request(): request msg size and sent data size are different."); + }; + + fd_set readfds, testfds; + FD_ZERO(&readfds); + FD_SET(sockfd, &readfds); + testfds = readfds; + + struct timeval response_timeout; + response_timeout.tv_sec=1; + response_timeout.tv_usec=0; + + result = select(FD_SETSIZE, &testfds, (fd_set *)0, + (fd_set *)0, &response_timeout); + + if(FD_ISSET(sockfd,&testfds)) { + int nread; + ioctl(sockfd, FIONREAD, &nread); + + char recv_buffer[1000]; + if(nread == 0) { + close(sockfd); + merror("Error in request.cpp: request(): no response from segget daemon"); + }else{ + if (nread!=read(sockfd, recv_buffer, nread)){ + merror("Error in request.cpp: request(): response msg size and received data size are different."); + }; + return decode_server_response(recv_buffer); + } + }else{ + merror("Error in request.cpp: request(): zero size response from segget daemon"); + } + close(sockfd); + return R_PF_ERROR_ADDING_TO_PROXY_QUEUE; +} + +int load_pkgs(){ + try{ + ifstream json_pkg_list_file; + json_pkg_list_file.exceptions (ofstream::failbit | ofstream::badbit); + try{ + json_pkg_list_file.open((settings.pkg_list_dir+"/pkg.list").c_str()); + }catch(...){ + merror("Error: Can't open "+settings.pkg_list_dir+"/pkg.list"+" file"); + return 1; + } + string buff((std::istreambuf_iterator<char>(json_pkg_list_file)), std::istreambuf_iterator<char>()); + try{ + json_pkg_list_file>>buff; + }catch(std::ifstream::failure e){ + if (!json_pkg_list_file.eof()){ + merror("Error: Can't read pkg.list file: "+(string)e.what()); + return 1; + } + }catch(...){ + perror("Unknown Error: Can't read pkg.list file"); + return 1; + } + try{ + json_pkg_list_file.close(); + }catch(...){ + perror("Error: Can't close pkg.list file"); + } + try{ + json_object *json_array_pkg_list; + json_array_pkg_list = json_tokener_parse(buff.c_str()); + if (is_error(json_array_pkg_list)) + merror("error parsing json:"+toString(json_tokener_errors[-(unsigned long)json_array_pkg_list])); + else { + uint pkg_count=json_object_array_length(json_array_pkg_list); + // create 0 pkg for distfiles to provide proxy-fetcher + for(uint array_item_num=0;array_item_num<pkg_count;array_item_num++){ + json_object* json_obj_pkg=json_object_array_get_idx(json_array_pkg_list,array_item_num); + json_object* json_array_distfile_list=json_object_object_get(json_obj_pkg,"distfile_list"); + ulong distfile_counter=json_object_array_length(json_array_distfile_list); + for(uint distfile_array_item_num=0;distfile_array_item_num<distfile_counter;distfile_array_item_num++){ + json_object* json_obj_distfile=json_object_array_get_idx(json_array_distfile_list,distfile_array_item_num); + request(json_object_to_json_string(json_obj_distfile)); + cout<< json_object_to_json_string(json_object_object_get(json_obj_distfile,"name"))<< endl; + } + } + } + return 0; + }catch(...){ + perror("Error: Can't parse json data from pkg.list file"); + return 1; + } + }catch(...){ + perror("Error in segget.cpp: load_pkgs()"); + return 1; + } +} +/* +void show_pkgs(){ + try{ + for (uint array_item_num=0;array_item_num<stats.pkg_count;array_item_num++){ + cout <<"PKG:"<<array_item_num<<") cat:"<< Ppkg_array[array_item_num]->category <<" name:"<< Ppkg_array[array_item_num]->name <<"\n"; + for(uint distfile_array_item_num=0;distfile_array_item_num<Ppkg_array[array_item_num]->distfile_count;distfile_array_item_num++){ + cout << " "<< distfile_array_item_num<<") distfile_name:"<< Ppkg_array[array_item_num]->Pdistfile_list[distfile_array_item_num]->name<<"\n"; + for(uint urllist_array_item_num=0;urllist_array_item_num<Ppkg_array[array_item_num]->Pdistfile_list[distfile_array_item_num]->url_count;urllist_array_item_num++){ + cout <<" "<<Ppkg_array[array_item_num]->Pdistfile_list[distfile_array_item_num]->url_list[urllist_array_item_num]<<"\n"; + } + } + } + }catch(...){ + error_log("Error in segget.cpp: show_pkgs()"); + } +} +*/ + +void segget_exit(int sig){ + try{ +// for(uint fd = 0; fd <= ui_server.max_fd_num; fd++) { +// close(fd); +// } + close(sockfd); + } + catch(...) + { + //error while clossing server_sockfd + } + cout<<"Exiting. (Cought signal:"+toString(sig)+")\n\n"; + exit(0); +} + +int routine(){ + try{ + signal(SIGABRT,segget_exit);//If program aborts go to assigned function "segget_exit". + signal(SIGTERM,segget_exit);//If program terminates go to assigned function "segget_exit". + signal(SIGINT,segget_exit);//If program terminates go to assigned function "segget_exit". +// prev_time=time((time_t *)NULL); + try{ + //load settings + settings.init(); + } + catch(...) + { + //error while loading settings + } + try{ + load_pkgs(); + }catch(...){ + //error while loading pkgs + } + try{ + //show_pkgs(); +// stats.show_totals(); + }catch(...){ + //error while showing stats + } + try{ +// download_pkgs(); + }catch(...){ + //error while downloading_pkgs + } + return 0; + }catch(...) + { + perror("Error in segget.cpp: routine()"); + } + return 1; +} +int parse_cli_arguments(int argc, char* argv[]){ + try{ + string option,name,value; + cout << "argc = " << argc << endl; + int posEqual; + for(int i = 0; i < argc; i++){ + cout << "argv[" << i << "] = " << argv[i] << endl; + option=argv[i]; + posEqual=option.find('='); + name = trim(option.substr(0,posEqual)); + value = trim(option.substr(posEqual+1)); + if (name=="--conf-dir") {settings.conf_dir=value; continue;}; + } + return 0; + }catch(...){ + perror("Error in segget.cpp: init_curses()"); + } + return 1; +} + +int main(int argc, char* argv[]) +{ + try{ + parse_cli_arguments(argc, argv); + routine(); + exit (0); + }catch(...){ + perror("Error in segget.cpp main()"); + } + exit(1); +}
\ No newline at end of file diff --git a/request/request.h b/request/request.h new file mode 100644 index 0000000..b627394 --- /dev/null +++ b/request/request.h @@ -0,0 +1,77 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __SEGGET_H__ +#define __SEGGET_H__ + +#include <stdio.h> +#include <string.h> +#include <signal.h> +#include <stdlib.h> +#include <stdio.h> +#include <fstream> +#include <iostream> +#include <json/json.h> +#include <pthread.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include <sys/socket.h> +//#include <stdio.h> +//#include <netinet/in.h> +#include <arpa/inet.h> +#include <sys/ioctl.h> +//#include <unistd.h> +//#include <stdlib.h> +//#include <string> +//#include <string.h> + + +#include "config.h" +#include "settings.h" +//#include "stats.h" +#include "str.h" +#include "utils.h" +#include "response.h" + +using namespace std; +int sockfd; + +int routine(); +void start_daemon_mode(); +int parse_cli_arguments(int argc, char* argv[]); +int init_curses(); +int load_pkgs(); +void show_pkgs(); +int choose_segment(uint connection_num); +int download_pkgs(); +int main(int argc, char* argv[]); +void *print_message_function( void *); + +#endif
\ No newline at end of file diff --git a/request/response.cpp b/request/response.cpp new file mode 100644 index 0000000..10956f4 --- /dev/null +++ b/request/response.cpp @@ -0,0 +1,65 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "response.h" +int decode_server_response(string server_response){ + int int_server_response=atoi(server_response.c_str()); + switch (int_server_response){ + case R_PF_NOT_REQUESTED_YET:{ +// debug("Server response:"+server_response+" - R_PF_NOT_REQUESTED_YET"); + return int_server_response; + }; + case R_PF_REJECTED:{ +// debug("Server response:"+server_response+" - R_PF_REJECTED"); + return int_server_response; + }; + case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:{ +// debug("Server response:"+server_response+" - R_PF_ERROR_ADDING_TO_PROXY_QUEUE"); + return int_server_response; + }; + case R_PF_ADDED_TO_PROXY_QUEUE:{ +// debug("Server response:"+server_response+" - R_PF_ADDED_TO_PROXY_QUEUE"); + return int_server_response; + }; + case R_PF_ALREADY_WAS_IN_QUEUE:{ +// debug("Server response:"+server_response+" - R_PF_ALREADY_WAS_IN_QUEUE"); + return int_server_response; + }; + case R_PF_DOWNLOADED:{ +// debug("Server response:"+server_response+" - R_PF_DOWNLOADED"); + return int_server_response; + }; + case R_PF_FAILED:{ +// debug("Server response:"+server_response+" - R_PF_FAILED"); + return int_server_response; + }; + default :{ +// debug("Server response:"+server_response+" - unknown => R_PF_FAILED"); + return R_PF_FAILED; + }; + } +} + diff --git a/request/response.h b/request/response.h new file mode 100644 index 0000000..d2db9af --- /dev/null +++ b/request/response.h @@ -0,0 +1,61 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __RESPONSE_H__ +#define __RESPONSE_H__ +#include <string> +#include <cstdlib> + +using namespace std; + +#define R_NOT_SET 300 +#define R_LM_WAIT_FOR_LOCAL_MIRRORS 50 + +#define R_PF_NOT_REQUESTED_YET 100 +#define R_PF_ADDED_TO_PROXY_QUEUE 101 +#define R_PF_ALREADY_WAS_IN_QUEUE 102 +#define R_PF_DOWNLOADED 103 +#define R_PF_BE_MORE_PATIENT 104 +#define R_PF_ERROR_ADDING_TO_PROXY_QUEUE 105 +#define R_PF_FAILED 106 +#define R_PF_REJECTED 107 + +// 0 for succesfull return of provide_segment() +#define R_R_DOWNLOAD_STARTED 0 +#define R_R_WAITING 108 +#define R_R_DOWNLOADING 109 + +#define R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND 110 + +#define ALLOW_REQUESTS_TO_PROXY_FETCHERS 201 +#define DO_NOT_ALLOW_REQUESTS_TO_PROXY_FETCHERS 202 +#define ALLOW_REMOTE_NETWORKS 203 +#define DO_NOT_ALLOW_REMOTE_NETWORKS 204 +#define ALLOW_LOWER_PRIORITY_NETWORKS 205 + +int decode_server_response(string server_response); + +#endif
\ No newline at end of file diff --git a/request/restrict.conf b/request/restrict.conf new file mode 100644 index 0000000..e554a5b --- /dev/null +++ b/request/restrict.conf @@ -0,0 +1,2 @@ +libX +bz2
\ No newline at end of file diff --git a/request/settings.cpp b/request/settings.cpp new file mode 100644 index 0000000..ba9fa7f --- /dev/null +++ b/request/settings.cpp @@ -0,0 +1,44 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "settings.h" + +Tsettings settings; + +void Tsettings::init(){ + try{ + Tconfig conf("segget.conf"); + conf.set("logs","general_log_file",general_log_file); + conf.set("logs","logs_dir",logs_dir); + conf.set("logs","error_log_file",error_log_file); + conf.set("logs","debug_log_file",debug_log_file); + conf.set("request_server","request_ip",request_ip); + conf.set("request_server","request_port",request_port,1,65535); + conf.clear(); + }catch(...){ + perror("Error in settings.cpp: init()"); + } +}
\ No newline at end of file diff --git a/request/settings.h b/request/settings.h new file mode 100644 index 0000000..097969a --- /dev/null +++ b/request/settings.h @@ -0,0 +1,130 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __SETTINGS_H__ +#define __SETTINGS_H__ + +#include <string> +#include "config.h" +//#include "tui.cpp" +#define MAX_CONNECTS 20 /* number of simultaneous transfers */ + +using namespace std; + +class Tsettings{ + public: + string conf_dir; + bool no_daemon_flag; + //folders + string distfiles_dir; + string segments_dir; + //pkg_list + string pkg_list_dir; + bool del_pkg_list_when_dld_finished; + //distfiles + ulong max_connection_num_per_distfile; + //segments + ulong max_segment_size; + bool resume_on; + ulong max_tries; + //connections + ulong max_connections; + ulong connection_timeout; + ulong ftp_response_timeout; + ulong time_out; + ulong low_connection_speed_limit; + ulong low_connection_speed_time; + ulong max_connection_speed; + ulong current_speed_time_interval_msecs; + string bind_interface; + //mirrors + ulong max_connections_num_per_mirror; + ulong benchmark_oblivion; + //provide_mirror_to_others + string provide_mirror_dir; + bool provide_mirror_files_restrict_list_on; + //provide_proxy_fether_to_others + string provide_proxy_fetcher_ip; + ulong provide_proxy_fetcher_port; + //ui_server + string request_ip; + ulong request_port; + //logs + string logs_dir; + string general_log_file; + string error_log_file; + string debug_log_file; + + Tsettings(): + conf_dir("/etc/seggetd"), + no_daemon_flag(false), + //folders + distfiles_dir("./distfiles"), + segments_dir("./tmp"), + //pkg_list + pkg_list_dir("./"), + del_pkg_list_when_dld_finished(1), + //distfiles + max_connection_num_per_distfile(3), + //segments + max_segment_size(500000), + resume_on(1), + max_tries(10), + //connections + max_connections(6), + connection_timeout(15), + ftp_response_timeout(180), + time_out(500), + low_connection_speed_limit(1000), + low_connection_speed_time(10), + max_connection_speed(0), + current_speed_time_interval_msecs(1000), + bind_interface("none"), + //mirrors + max_connections_num_per_mirror(1), + benchmark_oblivion(5), + //provide_mirror_to_others + provide_mirror_dir("none"), + provide_mirror_files_restrict_list_on(0), + //provide_proxy_fether_to_others + provide_proxy_fetcher_ip("127.0.0.1"), + provide_proxy_fetcher_port(9888), + //ui_server + request_ip("127.0.0.1"), + request_port(10000), + //logs + logs_dir("./logs"), + general_log_file("segget.log"), + error_log_file("error.log"), + debug_log_file("debug.log") + {}; + void set_resume(bool resume_setting){resume_on=resume_setting;}; + bool get_resume(){return resume_on;}; + void init(); +}; + +extern Tsettings settings; +#endif
\ No newline at end of file diff --git a/request/shorttypes.h b/request/shorttypes.h new file mode 100644 index 0000000..af0b982 --- /dev/null +++ b/request/shorttypes.h @@ -0,0 +1,32 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef _TYPES_H_ +#define _TYPES_H_ + +typedef unsigned long ulong; + +#endif
\ No newline at end of file diff --git a/request/str.cpp b/request/str.cpp new file mode 100644 index 0000000..d86d12c --- /dev/null +++ b/request/str.cpp @@ -0,0 +1,126 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "str.h" + +string toString(uint t){ + stringstream s; + s << t; + return s.str(); +} +string toString(int t){ + stringstream s; + s << t; + return s.str(); +} +string toString(ulong t){ + stringstream s; + s << t; + return s.str(); +} +string toString(long t){ + stringstream s; + s << t; + return s.str(); +} +string toString(bool t){ + stringstream s; + s << t; + return s.str(); +} +/* +template<typename T> string toString(T t){ + stringstream s; + s << t; + return s.str(); +} +*/ +/* +template<typename T> string field(string prefix,T t, int width){ + try{ + stringstream s1,s2; + s1 << t; + width=width+prefix.length(); + s2.width(width); + s2 << prefix+s1.str(); + return s2.str(); + }catch(...){ + error_log("Error in str.cpp: field()"); + return ""; + } +} +*/ +string field(string prefix,ulong t, int width){ + try{ + stringstream s1,s2; + s1 << t; + width=width+prefix.length(); + s2.width(width); + s2 << prefix+s1.str(); + return s2.str(); + }catch(...){ + perror("Error in str.cpp: field()"); + return ""; + } +} + +int lower_char(int c){ + try{ + return tolower((unsigned char)c); + }catch(...){ + perror("Error in str.cpp: lower_char()"); + return 0; + } +} + +string noupper(string s){ + try{ + transform(s.begin(), s.end(), s.begin(), lower_char); + return s; + }catch(...){ + perror("Error in str.cpp: noupper()"); + return ""; + } +} + +string trim(std::string const& source, char const* delims) { + try{ + string result(source); + string::size_type index = result.find_last_not_of(delims); + if(index != string::npos) + result.erase(++index); + + index = result.find_first_not_of(delims); + if(index != string::npos) + result.erase(0, index); + else + result.erase(); + return result; + }catch(...){ + perror("Error in str.cpp: trim()"); + return ""; + } +}
\ No newline at end of file diff --git a/request/str.h b/request/str.h new file mode 100644 index 0000000..f0c876a --- /dev/null +++ b/request/str.h @@ -0,0 +1,49 @@ + /* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __STR_H__ +#define __STR_H__ +#include <string> +#include <algorithm> +#include <sstream> +#include <stdio.h> +using namespace std; + +//template<typename T> string toString(T t); +//template<typename T> string field(string prefix,T t, int width); + +string trim(std::string const& source, char const* delims = " \t\r\n"); +int lower_char(int c); +string noupper(string s); + +string toString(uint t); +string toString(int t); +string toString(ulong t); +string toString(long t); +string toString(bool t); +string field(string prefix,ulong t, int width); + +#endif
\ No newline at end of file diff --git a/request/utils.cpp b/request/utils.cpp new file mode 100644 index 0000000..bb9b3bf --- /dev/null +++ b/request/utils.cpp @@ -0,0 +1,44 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "utils.h" +void merror(string error_msg){ + perror(error_msg.c_str()); +} + +ulong time_left_from(timeval from_time){ + try{ + timeval now_time; + gettimeofday(&now_time,NULL); + ulong timeleft=(now_time.tv_sec-from_time.tv_sec)*1000+(now_time.tv_usec-from_time.tv_usec)/1000; + if (timeleft<1) + timeleft=1; + return timeleft; + }catch(...){ + merror("Error in utils.cpp: time_left_from()"); + return 1; + } +}
\ No newline at end of file diff --git a/request/utils.h b/request/utils.h new file mode 100644 index 0000000..5e29a20 --- /dev/null +++ b/request/utils.h @@ -0,0 +1,39 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __UTILS_H__ +#define __UTILS_H__ + +#include "sys/time.h" +#include "shorttypes.h" +#include <string> +#include <stdio.h> +#include <cstdlib> +using namespace std; + +void merror(string error_msg); +ulong time_left_from(timeval from_time); +#endif
\ No newline at end of file |