From 995abc2a03def4c7d99c2d6748c3636cc86b815c Mon Sep 17 00:00:00 2001 From: Kostyantyn Ovechko Date: Sat, 10 Jul 2010 21:59:31 +0300 Subject: Move show_progress() to a thread --- segget/client.cpp | 0 segget/connection.cpp | 10 +- segget/connection.h | 4 +- segget/distfile.cpp | 213 +++++++++++++++++++++++++++++++------- segget/distfile.h | 39 ++++++- segget/network.cpp | 5 +- segget/network.h | 18 +++- segget/network0.conf | 72 +++++++++++-- segget/network0_mirrors.conf | 4 + segget/network1.conf | 56 ++++++++-- segget/network2.conf | 236 +++++++++++++++++++++++++++++++++++++++++++ segget/networkbroker.cpp | 2 +- segget/segget.conf | 22 +--- segget/segget.cpp | 209 ++++++++++++++++++++++++++------------ segget/segget.h | 1 + segget/segment.cpp | 26 +++-- segget/segment.h | 6 +- segget/stats.cpp | 1 + segget/tui.cpp | 19 ++-- 19 files changed, 779 insertions(+), 164 deletions(-) create mode 100644 segget/client.cpp create mode 100644 segget/network0_mirrors.conf create mode 100644 segget/network2.conf (limited to 'segget') diff --git a/segget/client.cpp b/segget/client.cpp new file mode 100644 index 0000000..e69de29 diff --git a/segget/connection.cpp b/segget/connection.cpp index dc1cc98..423c783 100644 --- a/segget/connection.cpp +++ b/segget/connection.cpp @@ -28,6 +28,7 @@ void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num){ try{ segment=started_segment; + debug("Started connection for distfile"+segment->parent_distfile->name); mirror_num=best_mirror_num; network_num=network_number; total_dld_bytes=0; @@ -43,6 +44,7 @@ void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegm void Tconnection::stop(uint connection_result){ try{ + debug("Finished connection for distfile"+segment->parent_distfile->name+" Status"+toString(connection_result)); msg_clean_connection(connection_num); active=false; network_array[network_num].disconnect(); @@ -52,7 +54,7 @@ void Tconnection::stop(uint connection_result){ prnt_distfile->active_connections_num--; Tmirror *Pcurr_mirror; - if (network_array[network_num].use_own_mirror_list_only_on){ + if (network_array[network_num].network_mode==MODE_LOCAL){ Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num]; prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true; // find_mirror(strip_mirror_name(segment->url)); @@ -69,16 +71,16 @@ void Tconnection::stop(uint connection_result){ debug(toString(connection_result)+"]- Failed download "+segment->url); Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),0); if (segment->try_num>=settings.max_tries){ - segment->status=FAILED; + segment->status=SFAILED; error_log("Segment:"+segment->file_name+" has reached max_tries limit - segment.status set to FAILED"); } - else segment->status=WAITING; + else segment->status=SWAITING; }else{ // no error => count this one and start new log("Succesfully downloaded "+segment->file_name+" on connection#"+toString(connection_num)); debug(" Successful download "+segment->url); Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),segment->segment_size); - segment->status=DOWNLOADED; + segment->status=SDOWNLOADED; prnt_distfile->inc_dld_segments_count(segment); }; }catch(...){ diff --git a/segget/connection.h b/segget/connection.h index fccf324..a5dd206 100644 --- a/segget/connection.h +++ b/segget/connection.h @@ -41,19 +41,19 @@ class Tconnection{ uint connection_num; uint network_num; uint mirror_num; - bool active; ulong total_dld_bytes; ulong bytes_per_last_interval; public: + bool active; timeval start_time; Tsegment *segment; Tconnection(): connection_num(total_connections), network_num(0), mirror_num(0), - active(0), total_dld_bytes(0), bytes_per_last_interval(0), + active(0), start_time(), segment(0){total_connections++;}; void start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num); diff --git a/segget/distfile.cpp b/segget/distfile.cpp index 023d0e7..9f0bb0e 100644 --- a/segget/distfile.cpp +++ b/segget/distfile.cpp @@ -26,6 +26,80 @@ #include "distfile.h" +//Make the necessary includes and set up the variables: +using namespace std; + +Tdistfile_status Tdistfile::request(string msg) +{ + int sockfd; + int len; + struct sockaddr_in address; + int result; + //Create a socket for the client: + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + //Name the socket, as agreed with the server: + address.sin_family = AF_INET; + address.sin_addr.s_addr = inet_addr("127.0.0.1"); + address.sin_port = htons(9797); + len = sizeof(address); + + //Connect your socket to the server’s socket: + result = connect(sockfd, (struct sockaddr *)&address, len); + if(result == -1) { + error_log("Can't connect to proxy-fetcher"); + return DPROXY_FAILED; + } + if (msg.length()>90000){return DPROXY_REJECTED;}; + char send_buffer[100000]; +// char recv_buffer[256]; + strcpy(send_buffer,msg.c_str()); + //You can now read and write via sockfd: + int i=write(sockfd, send_buffer, strlen(send_buffer)); + i++; +/* + fd_set readfds, testfds; + FD_ZERO(&readfds); + FD_SET(sockfd, &readfds); + testfds = readfds; + + result = select(FD_SETSIZE, &testfds, (fd_set *)0, + (fd_set *)0, (struct timeval *) 0); + + int nread; + ioctl(sockfd, FIONREAD, &nread); + + if(nread == 0) { + close(sockfd); + // FD_CLR(sockfd, &readfds); + /// printf("removing client on fd %d\n", sockfd); + }else{ + read(sockfd, recv_buffer, nread); + } +*/ + close(sockfd); + return DPROXY_QUEUED; +} + +bool Tdistfile::allows_new_actions(){ +// if (downloaded) return false; +// else return true; +// int time_left=0; + if (status==DDOWNLOADED) return false; +// if (((status==DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false; +//oterwise allow connections +// DNEW, +// D_NOT_PROXY_REQUESTED, +// DPROXY_REJECTED, +// DPROXY_FAILED, +// DPROXY_DOWNLOADED, +// DWAITING, +// DDOWNLOADING, +// DDOWNLOADED, +// DFAILED + return true; +} + void Tdistfile::init(){ try{ for (uint network_num=0; network_numstart(); + active_connections_num++; connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); return 0; } @@ -193,6 +270,7 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne if (best_mirror_num!=-1){ debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].benchmarked_mirror_list[best_mirror_num].url); network_array[network_num].benchmarked_mirror_list[best_mirror_num].start(); + active_connections_num++; connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); return 0; } @@ -208,7 +286,6 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ try{ - active_connections_num++; for (uint cur_network_priority=10; cur_network_priority>0; cur_network_priority--){ debug("cur_network_priority="+toString(cur_network_priority)); //choose network @@ -220,41 +297,62 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ // //---------------------------------------------------------------------------------------------------------- int best_local_network_num=-1; - int best_network_num=-1; + int best_proxy_fetcher_network_num=-1; + int best_remote_network_num=-1; bool allow_remote_mirrors=true; for (uint network_num=0; network_numnetwork_array[network_num].active_connections_num)){ - best_local_network_num=network_num; - debug(" Replace best LOCAL network to network#:"+toString(network_num)); - } - } - }else{ - if (network_array[network_num].only_local_when_possible){ + switch (network_array[network_num].network_mode){ + case MODE_LOCAL:{ + if (network_array[network_num].has_free_connections()){ if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ - allow_remote_mirrors=false; - debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); +// debug(" Allowed network#:"+toString(network_num)); + if ((best_local_network_num==-1) + or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_local_network_num=network_num; + debug(" Replace best LOCAL network to network#:"+toString(network_num)); + } + } + }else{ + if (network_array[network_num].only_local_when_possible){ + if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ + allow_remote_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } } } + break; + } + case MODE_PROXY_FETCHER:{ + //replace this one by does_not_reject_connections +// if (network_array[network_num].has_free_connections()){ + if + ((best_proxy_fetcher_network_num==-1) +// or +// (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num) + ){ + best_proxy_fetcher_network_num=network_num; + debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(network_num)); + debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(best_proxy_fetcher_network_num)); + } +// } + break; } - }else{ + case MODE_REMOTE:{ if (network_array[network_num].has_free_connections()){ if - ((best_network_num==-1) + ((best_remote_network_num==-1) or - (network_array[best_network_num].active_connections_num>network_array[network_num].active_connections_num)){ - best_network_num=network_num; - debug(" Replace best network to network to network#:"+toString(network_num)); + (network_array[best_remote_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_remote_network_num=network_num; + debug(" Replace best_remote_network_num to network#:"+toString(network_num)); } } + break; + } } //work with network } @@ -264,23 +362,57 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ //best network has been found //work with network debug(" So best LOCAL network is network#:"+toString(best_local_network_num)); - int res=choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num); - return res; + return choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num); }else{ - if (allow_remote_mirrors){ //since all local failed, go to remote - // remote_mirrors_go_second - if (best_network_num!=-1){ - //best network has been found - //work with network - debug(" So best network is network#:"+toString(best_network_num)); - return choose_best_mirror(cm, connection_num, best_network_num, seg_num); +//DDOWNLOADED) return false; +//DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false; + //oterwise allow connections +// DNEW, +// D_NOT_PROXY_REQUESTED, +// DPROXY_REJECTED, +// DPROXY_FAILED, +// DWAITING, +// DDOWNLOADING, +// DDOWNLOADED, +// DFAILED + if (allow_remote_mirrors){ //since all local failed, go to proxy_fetcher + debug("Remote mirrors are allowed"); + if (best_proxy_fetcher_network_num != -1){ + if (status == DPROXY_QUEUED){ + return 1; + }else{ + // request from proxy fethcer + status=request(json_data); + debug("Trying to dowload distfile"+name+" via proxy_fetcher. status="+toString(status)); + if (status==DPROXY_DOWNLOADED){ + // start download from the proxy_fetcher + debug(" So best proxy_fetcher_network is network#:"+toString(best_proxy_fetcher_network_num)); +// BEWARE -- CORRECT FOLLOWING LINES !!!!!!!!!!! + // start download from proxy_fether mirrors +// return choose_best_mirror(cm, connection_num, best_remote_network_num, seg_num); + } + //return - don't switch to low priority networks + return 0; + } + }else{ + // remote_mirrors_go_third + if (best_remote_network_num!=-1){ + //best network has been found + //work with network + debug(" So best_proxy_fetcher_failed num="+toString(best_proxy_fetcher_network_num)); + debug(" So best network is network#:"+toString(best_remote_network_num)); + return choose_best_mirror(cm, connection_num, best_remote_network_num, seg_num); + } } }else{ - debug("Restricted to local mirrors only when possible"); + debug("NOT all local mirrors have failed - restricted to local mirrors only."); + //return - don't switch to low priority networks + return 100; } } } - return 0; + // haven't found anything suitable + return 120; }catch(...){ error_log("Error: distfile.cpp: provide_segment()"); return 1; @@ -374,6 +506,7 @@ int Tdistfile::combine_segments(){ distfile_file.open(distfile_path.c_str(),ofstream::binary|ios::trunc); }catch(...){ error_log("Error: distfile.cpp: combine_segments(): opening distfile:"+distfile_path); + status=DFAILED; return 1; } char * buffer; @@ -387,6 +520,7 @@ int Tdistfile::combine_segments(){ segment_file.open((settings.segments_dir+"/"+dn_segments[seg_num].file_name).c_str(),ifstream::binary); }catch(...){ error_log("Error: distfile.cpp: combine_segments(): opening segment:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); + status=DFAILED; return 2; } try{ @@ -404,13 +538,15 @@ int Tdistfile::combine_segments(){ segment_file.close(); }catch(...){ error_log("Error: distfile.cpp: combine_segments(): segment is open, but error occured while reading it:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name); - return 3; + status=DFAILED; + return 3; } try{ // write to outfile distfile_file.write (buffer,cur_seg_size); }catch(...){ error_log("Error: distfile.cpp: combine_segments(): distfile is open, but error occured while writing into it:"+settings.distfiles_dir+"/"+name); + status=DFAILED; return 4; } // release dynamically-allocated memory @@ -426,6 +562,7 @@ int Tdistfile::combine_segments(){ log("Distfile "+name+" has been combined"); }catch(...){ error_log("Error in distfile.cpp: combine_segments() for distfile:"+settings.distfiles_dir+"/"+name); + status=DFAILED; return 5; } try{ @@ -434,6 +571,7 @@ int Tdistfile::combine_segments(){ else{ log("Error: RMD160 checksum for distfile:"+name+" [FAILED]"); error_log("Error: RMD160 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; return 10; } @@ -442,6 +580,7 @@ int Tdistfile::combine_segments(){ else{ log("Error: SHA1 checksum for distfile:"+name+" [FAILED]"); error_log("Error: SHA1 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; return 11; } @@ -450,8 +589,10 @@ int Tdistfile::combine_segments(){ else{ log("Error: SHA256 checksum for distfile:"+name+" [FAILED]"); error_log("Error: SHA256 checksum for distfile:"+name+" [FAILED]"); + status=DFAILED; return 12; } + status=DDOWNLOADED; if (settings.provide_mirror_dir!="none"){ symlink_distfile_to_provide_mirror_dir(); } diff --git a/segget/distfile.h b/segget/distfile.h index 2f9c3fa..1b2f110 100644 --- a/segget/distfile.h +++ b/segget/distfile.h @@ -52,10 +52,39 @@ #include "network.h" #include "networkbroker.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +//#include "client.cpp" + using namespace std; typedef unsigned int uint; +enum Tdistfile_status{ + DNEW, + D_NOT_PROXY_REQUESTED, + DPROXY_REJECTED, + DPROXY_QUEUED, + DPROXY_DOWNLOADING, + DPROXY_DOWNLOADED, + DPROXY_FAILED, + DWAITING, + DDOWNLOADING, + DDOWNLOADED, + DFAILED +}; + class Tdistfile{ private: uint dld_segments_count; @@ -63,7 +92,9 @@ class Tdistfile{ bool choose_best_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num); public: Tnetwork_distfile_broker network_distfile_brokers_array[MAX_NETWORKS]; - bool downloaded; + string json_data; +// bool downloaded; + Tdistfile_status status; uint active_connections_num; string *url_list; uint url_num; @@ -80,7 +111,9 @@ class Tdistfile{ uint segment_size; Tdistfile(): dld_segments_count(0), - downloaded(0), + json_data(""), +// downloaded(0), + status(DNEW), active_connections_num(0), url_list(0), url_num(0), @@ -99,7 +132,9 @@ class Tdistfile{ Tdistfile(const Tdistfile &L); // copy constructor Tdistfile & operator=(const Tdistfile &L); ~Tdistfile(); + Tdistfile_status request(string msg); void init(); + bool allows_new_actions(); void load_distfile_from_json(json_object* json_obj_distfile); void load_url_list(json_object* json_array_distfile_urllist); void split_into_segments(); diff --git a/segget/network.cpp b/segget/network.cpp index f0ea3b6..ad765f5 100644 --- a/segget/network.cpp +++ b/segget/network.cpp @@ -63,6 +63,7 @@ void Tnetwork::init(uint priority_value){ try{ priority=priority_value; Tconfig conf("network"+toString(network_num)+".conf"); + conf.set(network_mode, "mode", "network_mode",0,2); conf.set(bind_interface, "network_bind", "bind_interface"); conf.set(max_connections, "network_connections", "max_connections",1,MAX_CONNECTS); conf.set(connection_timeout, "network_connections", "connection_timeout",1,1000); @@ -80,9 +81,9 @@ void Tnetwork::init(uint priority_value){ conf.set(proxy_user, "network_proxy", "proxy_user"); conf.set(proxy_password, "network_proxy", "proxy_password"); - conf.set(use_own_mirror_list_only_on, "network_mirrors", "use_own_mirror_list_only_on"); +// conf.set(use_own_mirror_list_only_on, "network_mirrors", "use_own_mirror_list_only_on"); - if (use_own_mirror_list_only_on){ + if (network_mode==MODE_LOCAL){ conf.set(only_local_when_possible, "network_mirrors", "only_local_when_possible"); load_mirror_list(); log("Settings: Network"+toString(network_num)+" local mirror_list size:"+toString(mirror_list.size())); diff --git a/segget/network.h b/segget/network.h index c164bb7..a42f54d 100644 --- a/segget/network.h +++ b/segget/network.h @@ -38,6 +38,10 @@ using namespace std; #define MAX_NETWORKS 10 +#define MODE_REMOTE 0 +#define MODE_PROXY_FETCHER 1 +#define MODE_LOCAL 2 + class Tnetwork{ static uint network_count; private: @@ -48,6 +52,10 @@ class Tnetwork{ //network uint network_num; uint priority; + //mode + ulong network_mode; + //network_bind + string bind_interface; //connections ulong max_connections; ulong connection_timeout; @@ -57,7 +65,6 @@ class Tnetwork{ ulong low_connection_speed_time; ulong max_connection_speed; ulong current_speed_time_interval_msecs; - string bind_interface; //user-data string user_agent; //proxy @@ -67,7 +74,7 @@ class Tnetwork{ string proxy_user; string proxy_password; //mirrors - bool use_own_mirror_list_only_on; +// bool use_own_mirror_list_only_on; bool only_local_when_possible; Tnetwork(): benchmarked_mirror_list(), @@ -75,6 +82,10 @@ class Tnetwork{ //network network_num(network_count), priority(0), + //mode + network_mode(0), + //network_bind + bind_interface("none"), //connections max_connections(6), connection_timeout(15), @@ -84,7 +95,6 @@ class Tnetwork{ low_connection_speed_time(10), max_connection_speed(0), current_speed_time_interval_msecs(1000), - bind_interface("none"), //user-data user_agent("segget"), //proxy @@ -94,7 +104,7 @@ class Tnetwork{ proxy_user("none"), proxy_password("none"), //mirrors - use_own_mirror_list_only_on(0), +// use_own_mirror_list_only_on(0), only_local_when_possible(1) {network_count++;}; void init(uint priority_value); diff --git a/segget/network0.conf b/segget/network0.conf index 2fc7aea..d816e29 100644 --- a/segget/network0.conf +++ b/segget/network0.conf @@ -1,3 +1,38 @@ +[mode] +# SYNOPSIS: NETWORK_MODE= 0 | 1 | 2 +# 0 - remote mirrors (default) +# 1 - proxy fetcher +# 2 - local mirrors +# - If set to 2, segget will replace mirror list provided by portage +# system with the list from network0_mirrors.conf file +# - If set to 0, segget will use ONLY mirror list provided by portage +# system, and will NOT use the list from network0_mirrors.conf file +# In some cases it's necessary to make segget prefer local mirrors over +# the remote ones. For this purpose define settings for 2 networks: +# settings for network0 (to provide access to local mirrors), +# settings for network1 (to provide access to remote ones). +# 1) Set the following options in segget.conf file: +# [networks] +# network0_priority=10 +# network1_priority=9 +# As you can see network0 (with local mirrors) has higher priority than +# network1 (with mirrors provided by portage). +# 2) Set NETWORK_MODE=2 in network0.conf file. +# 3) Create network0_mirrors.conf file with the list of your local +# mirrors. +# For example, network0_mirrors.conf may look like this: +# http://192.168.210.12/ +# ftp://192.168.210.205/ +# http://192.168.210.56/ +# 4) Set NETWORK_MODE=0 in network1.conf file, +# so segget will use remote mirrors working via this network. +# NOTE: Actually network0 and network1 can be the same LAN with only +# one ip address set on the host. The only difference is that in case of +# network1 segget will have to use a gateway to access remote mirrors. +# Default: +# network_mode=0 +network_mode=2 + [network_bind] # BIND INTERFACE / IP # Pass a string as parameter. This sets the interface name to use as outgoing @@ -170,6 +205,22 @@ proxy_password=none # proxy_off=1 proxy_off=1 +[proxy-fetcher] +# Specify proxy-fetcher address and port +proxy-fetcher-ip=127.0.0.1 +proxy-fetcher-port=9797 +# Specify the user and password for authentication on a proxy-fetcher-server. +# proxy-fetcher-user=user +# proxy-fetcher-password=password + +# Set to forbid direct connections to Internet servers if it's possible to use +# proxy-fetcher. +# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-if-possible=1 +# Set to forbid direct connections to Internet servers. +# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-only=0 +# Set to forbid using proxy-fetcher. +# NOT IMPLEMENTED YET: no-proxy-fetcher + [network_mirrors] # SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 | 1 # - If set to 1, segget will replace mirror list provided by portage system with @@ -181,12 +232,21 @@ proxy_off=1 use_own_mirror_list_only_on=0 # SYNOPSIS: ONLY_LOCAL_WHEN_POSSIBLE=0 | 1 -# If NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 this option will be ignored. -# - If set to 1, segget will not use remote mirrors with equal or lower priority -# until all mirrors in network0_mirrors.conf file have failed. -# - If set to 0, segget will use remote mirrors with equal priority or mirrors -# with lower priority when this network has NO free connections (see option -# NETWORK_MAX_CONNECTIONS in [network_connections] section of this file). +# If NETWORK_MODE=0 or NETWORK_MODE=1 this option will be ignored. +# - If set to 1, segget will not use remote mirrors with equal or lower +# priority until all mirrors in network#_mirrors.conf file have failed. +# - If set to 0, segget will use remote mirrors with equal priority or +# mirrors with lower priority when this network has NO free connections +# (see option NETWORK_MAX_CONNECTIONS in [network_connections] section +# of this file). +# NOTE: Following example for NETWORK_MODE option, +# if in network0.conf has option ONLY_LOCAL_WHEN_POSSIBLE=1, segget +# will NOT start to use network1 for a particular distfile until all +# mirrors specified in network0_mirrors.conf file will have failed +# to provide this distfile. +# On the other hand if ONLY_LOCAL_WHEN_POSSIBLE=0 segget will start +# to use network1 as soon as NETWORK_MAX_CONNECTIONS limit, set +# in network0.conf file has been reached. # Default: # only_local_when_possible=1 only_local_when_possible=1 \ No newline at end of file diff --git a/segget/network0_mirrors.conf b/segget/network0_mirrors.conf new file mode 100644 index 0000000..4164689 --- /dev/null +++ b/segget/network0_mirrors.conf @@ -0,0 +1,4 @@ +ftp://192.168.56.11/ +ftp://192.168.56.12/ +ftp://192.168.56.13/ +ftp://192.168.56.14/ \ No newline at end of file diff --git a/segget/network1.conf b/segget/network1.conf index 7e0a72b..ea7e9bf 100644 --- a/segget/network1.conf +++ b/segget/network1.conf @@ -1,3 +1,38 @@ +[mode] +# SYNOPSIS: NETWORK_MODE= 0 | 1 | 2 +# 0 - remote mirrors (default) +# 1 - proxy fetcher +# 2 - local mirrors +# - If set to 2, segget will replace mirror list provided by portage +# system with the list from network0_mirrors.conf file +# - If set to 0, segget will use ONLY mirror list provided by portage +# system, and will NOT use the list from network0_mirrors.conf file +# In some cases it's necessary to make segget prefer local mirrors over +# the remote ones. For this purpose define settings for 2 networks: +# settings for network0 (to provide access to local mirrors), +# settings for network1 (to provide access to remote ones). +# 1) Set the following options in segget.conf file: +# [networks] +# network0_priority=10 +# network1_priority=9 +# As you can see network0 (with local mirrors) has higher priority than +# network1 (with mirrors provided by portage). +# 2) Set NETWORK_MODE=2 in network0.conf file. +# 3) Create network0_mirrors.conf file with the list of your local +# mirrors. +# For example, network0_mirrors.conf may look like this: +# http://192.168.210.12/ +# ftp://192.168.210.205/ +# http://192.168.210.56/ +# 4) Set NETWORK_MODE=0 in network1.conf file, +# so segget will use remote mirrors working via this network. +# NOTE: Actually network0 and network1 can be the same LAN with only +# one ip address set on the host. The only difference is that in case of +# network1 segget will have to use a gateway to access remote mirrors. +# Default: +# network_mode=0 +network_mode=1 + [network_bind] # BIND INTERFACE / IP # Pass a string as parameter. This sets the interface name to use as outgoing @@ -181,12 +216,21 @@ proxy_off=1 use_own_mirror_list_only_on=1 # SYNOPSIS: ONLY_LOCAL_WHEN_POSSIBLE=0 | 1 -# If NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 this option will be ignored. -# - If set to 1, segget will not use remote mirrors with equal or lower priority -# until all mirrors in network0_mirrors.conf file have failed. -# - If set to 0, segget will use remote mirrors with equal priority or mirrors -# with lower priority when this network has NO free connections (see option -# NETWORK_MAX_CONNECTIONS in [network_connections] section of this file). +# If NETWORK_MODE=0 or NETWORK_MODE=1 this option will be ignored. +# - If set to 1, segget will not use remote mirrors with equal or lower +# priority until all mirrors in network#_mirrors.conf file have failed. +# - If set to 0, segget will use remote mirrors with equal priority or +# mirrors with lower priority when this network has NO free connections +# (see option NETWORK_MAX_CONNECTIONS in [network_connections] section +# of this file). +# NOTE: Following example for NETWORK_MODE option, +# if in network0.conf has option ONLY_LOCAL_WHEN_POSSIBLE=1, segget +# will NOT start to use network1 for a particular distfile until all +# mirrors specified in network0_mirrors.conf file will have failed +# to provide this distfile. +# On the other hand if ONLY_LOCAL_WHEN_POSSIBLE=0 segget will start +# to use network1 as soon as NETWORK_MAX_CONNECTIONS limit, set +# in network0.conf file has been reached. # Default: # only_local_when_possible=1 only_local_when_possible=1 \ No newline at end of file diff --git a/segget/network2.conf b/segget/network2.conf new file mode 100644 index 0000000..bc2548d --- /dev/null +++ b/segget/network2.conf @@ -0,0 +1,236 @@ +[mode] +# SYNOPSIS: NETWORK_MODE= 0 | 1 | 2 +# 0 - remote mirrors (default) +# 1 - proxy fetcher +# 2 - local mirrors +# - If set to 2, segget will replace mirror list provided by portage +# system with the list from network0_mirrors.conf file +# - If set to 0, segget will use ONLY mirror list provided by portage +# system, and will NOT use the list from network0_mirrors.conf file +# In some cases it's necessary to make segget prefer local mirrors over +# the remote ones. For this purpose define settings for 2 networks: +# settings for network0 (to provide access to local mirrors), +# settings for network1 (to provide access to remote ones). +# 1) Set the following options in segget.conf file: +# [networks] +# network0_priority=10 +# network1_priority=9 +# As you can see network0 (with local mirrors) has higher priority than +# network1 (with mirrors provided by portage). +# 2) Set NETWORK_MODE=2 in network0.conf file. +# 3) Create network0_mirrors.conf file with the list of your local +# mirrors. +# For example, network0_mirrors.conf may look like this: +# http://192.168.210.12/ +# ftp://192.168.210.205/ +# http://192.168.210.56/ +# 4) Set NETWORK_MODE=0 in network1.conf file, +# so segget will use remote mirrors working via this network. +# NOTE: Actually network0 and network1 can be the same LAN with only +# one ip address set on the host. The only difference is that in case of +# network1 segget will have to use a gateway to access remote mirrors. +# Default: +# network_mode=0 +network_mode=0 + +[network_bind] +# BIND INTERFACE / IP +# Pass a string as parameter. This sets the interface name to use as outgoing +# network interface. The name can be an interface name, an IP address, or a host +# name. No binding is set by default. +# Default: +# bind_interface=none +bind_interface=none + +# BIND LOCALPORT +# Pass a long. This sets the local port number of the socket used for connection. +# This can be used in combination with BIND_INTERFACE and you are recommended to +# use BIND_LOCALPORTRANGE as well when this is set. +# Valid port numbers are 1 - 65535. + +# BIND_LOCALPORTRANGE +# Pass a long. This is the number of attempts segget should make to find a +# working local port number. It starts with the given BIND_LOCALPORT and adds +# one to the number for each retry. Setting this to 1 or below will make segget +# do only one try for the exact port number. Port numbers by nature are scarce +# resources that will be busy at times so setting this value to something too +# low might cause unnecessary connection setup failures. + +[network_connections] +# NETWORK_MAX_CONNECTIONS +# Define maximum number of connections +# Minimum value: 1 +# Maximum value: 20 +# Default: +# max_connections=10 +max_connections=3 + +# CONNECTION_TIMEOUT +# Set the number of seconds to wait while trying to connect. Use 0 to wait +# indefinitely. Pass a long. It should contain the maximum time in seconds that +# you allow the connection to the server to take. This only limits the connection +# phase, once it has connected, this option is of no more use. Set to zero to +# disable connection timeout (it will then only timeout on the system's internal +# timeouts). See also the TIMEOUT option. +# Minimum value: 1 +# Maximum value: 1000 +# Default: +# connection_timeout=15 +connection_timeout=15 + +# FTP_RESPONSE_TIMEOUT +# Set a timeout period (in seconds) on the amount of time that the server is +# allowed to take in order to generate a response message for a command before the +# session is considered hung. While awaiting for a response, this value overrides +# TIMEOUT. It is recommended that if used in conjunction with TIMEOUT, you set +# FTP_RESPONSE_TIMEOUT to a value smaller than TIMEOUT. +# Minimum value: 1 +# Maximum value: -1 (for no limit) +# Default: +# ftp_response_timeout=180 +ftp_response_timeout=180 + +# TIMEOUT +# maximum amount of time to download segment in seconds +# Set the maximum number of seconds for a connection to execute. +# Pass a long as parameter containing the maximum time in seconds that you allow +# the transfer operation to take. Normally, name lookups can take a considerable +# time and limiting operations to less than a few minutes risk aborting perfectly +# normal operations. +# Minimum value: 100 +# Maximum value: -1 (for no limit) +# Default: +# timeout=500 +timeout=500 + +# LOW_CONNECTION_SPEED_LIMIT +# Define the low speed limit for connection. Pass a long as parameter. It +# contains the transfer speed in bytes per second that the transfer should be +# below during LOW_CONNECTION_SPEED_TIME seconds to consider it too slow and abort. +# Minimum value: 1 +# Maximum value: -1 (-1 for no limit) +# Default: +# low_connection_speed_limit=1000 +low_connection_speed_limit=1000 + +# LOW_CONNECTION_SPEED_TIME +# Pass a long as parameter. It contains the time in seconds that the transfer +# should be below the LOW_CONNECTION_SPEED_LIMIT to consider it too slow and abort. +# Minimum value: 1 +# Maximum value: 600 +# Default: +# low_connection_speed_time=10 +low_connection_speed_time=10 + +# MAX_CONNECTION_SPEED +# If a download exceeds this speed (counted in bytes per second) on cumulative +# average during the transfer, the transfer will pause to keep the average rate +# less than or equal to the parameter value. Defaults to unlimited speed. +# Minimum value: 1 +# Maximum value: -1 (-1 for no limit) +# Default: +# max_connection_speed=0 +max_connection_speed=3000 + +[network_protocols] +# SYNOPSIS: http_on=0 | 1 +# Default: +# http_on=1 +# NOT IMPLEMENTED YET: http_on=1 + +# SYNOPSIS: ftp_on=0 | 1 +# Default: +# ftp_on=1 +# NOT IMPLEMENTED YET: ftp_on=1 + +[network_user_data] +# USER_AGENT +# Set the User-Agent: header in the http request sent to the remote server. +# This can be used to fool servers or scripts. +# Default: +# user_agent=segget +user_agent=segget + +# Specify the user and password for authentication on a ftp servers. +# NOT IMPLEMENTED YET: ftp-user=anonymous +# NOT IMPLEMENTED YET: ftp-password=me@mail.ru + +[network_proxy] +# PROXY_IP_OR_NAME +# Specify a proxy to use (address and port). +# Set HTTP proxy to use. The parameter should be a string holding the proxy host +# name or dotted IP address. To specify port number in this string, +# append :[port] to the end of the host name. The proxy string may be prefixed +# with [protocol]:// since any such prefix will be ignored. The proxy's port +# number may optionally be specified with the separate option. If not specified, +# by default port 1080 will be used for proxies. +# When you tell segget to use an HTTP proxy, segget will transparently convert +# operations to HTTP even if you specify an FTP URL etc. +# Segget respects the environment variables http_proxy, ftp_proxy, all_proxy etc, +# if any of those are set. The PROXY option does however override any possibly +# set environment variables. +# Default: +# proxy_ip_or_name=none +proxy_ip_or_name=none + +# PROXY_PORT +# Set the proxy port to connect to unless it is specified in the PROXY option. +# Minimum value: 1 +# Maximum value: 65535 +# Default: +# proxy_port=3128 +proxy_port=3128 + +# PROXY_USER +# Set user name to use for the transfer while connecting to Proxy. +# The PROXY_USER option should be used in same way as the PROXY_PASSWORD is used. +# In order to specify the password to be used in conjunction with the user name +# use the PROXY_PASSWORD option. +# Default: +# proxy_user=none +proxy_user=none + +# PROXY_PASSWORD +# Set password to use for the transfer while connecting to Proxy. +# The PROXY_PASSWORD option should be used in conjunction with the PROXY_USER +# option. +# Default: +# proxy_password=none +proxy_password=none + +# SYNOPSIS: proxy_off=0 | 1 +# Setting the proxy_off=1 will explicitly disable the use of a proxy, even if +# there is an environment variable set for it. +# Default: +# proxy_off=1 +proxy_off=1 + +[network_mirrors] +# SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 | 1 +# - If set to 1, segget will replace mirror list provided by portage system with +# the list from network1_mirrors.conf +# - If set to 0, segget will use ONLY mirror list provided by portage system, +# and will NOT use the list from network1_mirrors.conf file +# Default: +# use_own_mirror_list_only_on=0 +use_own_mirror_list_only_on=1 + +# SYNOPSIS: ONLY_LOCAL_WHEN_POSSIBLE=0 | 1 +# If NETWORK_MODE=0 or NETWORK_MODE=1 this option will be ignored. +# - If set to 1, segget will not use remote mirrors with equal or lower +# priority until all mirrors in network#_mirrors.conf file have failed. +# - If set to 0, segget will use remote mirrors with equal priority or +# mirrors with lower priority when this network has NO free connections +# (see option NETWORK_MAX_CONNECTIONS in [network_connections] section +# of this file). +# NOTE: Following example for NETWORK_MODE option, +# if in network0.conf has option ONLY_LOCAL_WHEN_POSSIBLE=1, segget +# will NOT start to use network1 for a particular distfile until all +# mirrors specified in network0_mirrors.conf file will have failed +# to provide this distfile. +# On the other hand if ONLY_LOCAL_WHEN_POSSIBLE=0 segget will start +# to use network1 as soon as NETWORK_MAX_CONNECTIONS limit, set +# in network0.conf file has been reached. +# Default: +# only_local_when_possible=1 +only_local_when_possible=1 \ No newline at end of file diff --git a/segget/networkbroker.cpp b/segget/networkbroker.cpp index 7ac083d..5ef8dbf 100644 --- a/segget/networkbroker.cpp +++ b/segget/networkbroker.cpp @@ -28,7 +28,7 @@ void Tnetwork_distfile_broker::init(ulong network_number){ try{ network_num=network_number; - if (network_array[network_num].use_own_mirror_list_only_on){ + if (network_array[network_num].network_mode==MODE_LOCAL){ // create flags for each mirror from this network mirror_fails_vector.insert(mirror_fails_vector.begin(),network_array[network_num].benchmarked_mirror_list.size(),false); } diff --git a/segget/segget.conf b/segget/segget.conf index 71c37f4..8b8217b 100644 --- a/segget/segget.conf +++ b/segget/segget.conf @@ -76,7 +76,7 @@ max_tries=10 # Maximum value: 20 # Default: # max_connections=10 -max_connections=7 +max_connections=6 # CURRENT_SPEED_TIME_INTERVAL_MSECS # segget transfers may have bursty nature of their traffic. Therefore, while @@ -195,14 +195,14 @@ network0_priority=10 # Define network specific settings in network1.conf # Default: # network1_priority=0 -network1_priority=10 +network1_priority=9 # NETWORK2_PRIORITY # Description: same as for NETWORK0_PRIORITY # Define network specific settings in network2.conf # Default: # network2_priority=0 -network2_priority=0 +network2_priority=8 # NETWORK3_PRIORITY # Description: same as for NETWORK0_PRIORITY @@ -267,22 +267,6 @@ user_agent=segget # NOT IMPLEMENTED YET: ftp-user=anonymous # NOT IMPLEMENTED YET: ftp-password=me@mail.ru -[proxy-fetcher] -# Specify proxy-fetcher address and port -# NOT IMPLEMENTED YET: proxy-fetcher-ip=none -# NOT IMPLEMENTED YET: proxy-fetcher-port=none -# Specify the user and password for authentication on a proxy-fetcher-server. -# proxy-fetcher-user=user -# proxy-fetcher-password=password - -# Set to forbid direct connections to Internet servers if it's possible to use -# proxy-fetcher. -# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-if-possible=1 -# Set to forbid direct connections to Internet servers. -# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-only=0 -# Set to forbid using proxy-fetcher. -# NOT IMPLEMENTED YET: no-proxy-fetcher - [logs] # LOGS_DIR # Define a dir to store log files. diff --git a/segget/segget.cpp b/segget/segget.cpp index edb6b0f..315ff35 100644 --- a/segget/segget.cpp +++ b/segget/segget.cpp @@ -27,6 +27,7 @@ #include "segget.h" int load_pkgs(){ + debug("uuuuuuuuuuuuuummmmmmmmmmmmmmmm"); try{ ifstream json_pkg_list_file; @@ -102,27 +103,37 @@ int choose_segment(uint connection_num){ while (pkg_numdistfile_count){ - if (not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->downloaded)){ + if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->allows_new_actions()){ + debug("Distfile "+Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->name+" allows new connections"); // debug(" distfile_num:"+toString(distfile_num)); - if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_numPdistfile_list[distfile_num]->active_connections_numPdistfile_list[distfile_num]->segments_count)); while (segment_numPdistfile_list[distfile_num]->segments_count){ + debug("segment_num:"+toString(segment_num)); // debug(" segment_num:"+toString(segment_num)); // segments_in_progress[connection_num]= // if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num); - if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==WAITING){ - Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num); - return 0; // success segment.max_tries has not been reached + debug("Let's get segment status"+statusToString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status)); + if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){ + if ( ! Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){ + return 0; // success + }; + }else{ + debug("status is not SWAITING - go for the next segment"); } - else - segment_num++; // segment already downloaded/downloading => go for the next one + // haven't managed to provide this segment, go for the next one + segment_num++; } - else + }else{ debug(" distfile "+Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->name+" has " +toString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_num) +" connections => choosing another distfile."); - segment_num=0; + } } distfile_num++; + segment_num=0; } distfile_num=0; pkg_num++; @@ -158,65 +169,90 @@ int download_pkgs(){ return EXIT_FAILURE; } - for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) { - if (choose_segment(connection_num)) - break; - }; - while (U) { - while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){}; - if (U) { - FD_ZERO(&R); - FD_ZERO(&W); - FD_ZERO(&E); - if (curl_multi_fdset(cm, &R, &W, &E, &M)) { - error_log("Error: curl_multi_fdset"); -// fprintf(stderr, "E: curl_multi_fdset\n"); - return EXIT_FAILURE; - } - if (curl_multi_timeout(cm, &L)) { - error_log("Error: curl_multi_timeout"); -// fprintf(stderr, "E: curl_multi_timeout\n"); - return EXIT_FAILURE; - } - if (L == -1) - L = 100; - if (M == -1) { - #ifdef WIN32 - Sleep(L); - #else - sleep(L / 1000); - #endif - } else { - T.tv_sec = L/1000; - T.tv_usec = (L%1000)*1000; - if (0 > select(M+1, &R, &W, &E, &T)) { -// fprintf(stderr, "E: select(%i,,,,%li): %i: %s\n", - error_log("Error: select ("+toString(M+1)+","+toString(L)+"):"+toString(errno)+": "+strerror(errno)); +// for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) { +// if ( ! connection_array[connection_num].active){ +// if (choose_segment(connection_num)) +// break; +// } +// }; + bool keep_running_flag=true; + while (keep_running_flag){ + U=1; + while (U) { + // Use free connections to download segments connections + for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) { + debug("connection_num:"+toString(connection_num)); + if ( ! connection_array[connection_num].active){ +// if ( + debug("connection is not active - choosing segment"); + choose_segment(connection_num); +// ) +// break; + } + else{ + debug("connection is active"); + } + }; + while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){}; + if (U) { + FD_ZERO(&R); + FD_ZERO(&W); + FD_ZERO(&E); + if (curl_multi_fdset(cm, &R, &W, &E, &M)) { + error_log("Error: curl_multi_fdset"); +// fprintf(stderr, "E: curl_multi_fdset\n"); + return EXIT_FAILURE; + } + if (curl_multi_timeout(cm, &L)) { + error_log("Error: curl_multi_timeout"); +// fprintf(stderr, "E: curl_multi_timeout\n"); return EXIT_FAILURE; } + if (L == -1) + L = 100; + if (M == -1) { + #ifdef WIN32 + Sleep(L); + #else + sleep(L / 1000); + #endif + } else { + T.tv_sec = L/1000; + T.tv_usec = (L%1000)*1000; + if (0 > select(M+1, &R, &W, &E, &T)) { +// fprintf(stderr, "E: select(%i,,,,%li): %i: %s\n", + error_log("Error: select ("+toString(M+1)+","+toString(L)+"):"+toString(errno)+": "+strerror(errno)); + return EXIT_FAILURE; + } + } } - } - while ((msg = curl_multi_info_read(cm, &Q))) { - if (msg->msg == CURLMSG_DONE) { - Tsegment *current_segment; - CURL *e = msg->easy_handle; - curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, ¤t_segment); - uint connection_result=msg->data.result; - string result_msg_text="RESULT:"+toString(connection_result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment"; - msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text); - curl_multi_remove_handle(cm, e); - - connection_array[current_segment->connection_num].stop(connection_result); - - if (not choose_segment(current_segment->connection_num)) { - U++; // just to prevent it from remaining at 0 if there are more URLs to get - }; - stats.show_totals(); - curl_easy_cleanup(e); - }else { - msg_error("ERROR: CURLMsg: "+msg->msg); + while ((msg = curl_multi_info_read(cm, &Q))) { + if (msg->msg == CURLMSG_DONE) { + Tsegment *current_segment; + CURL *e = msg->easy_handle; + curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, ¤t_segment); + uint connection_result=msg->data.result; + string result_msg_text="RESULT:"+toString(connection_result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment"; + msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text); + curl_multi_remove_handle(cm, e); + + connection_array[current_segment->connection_num].stop(connection_result); + + if (not choose_segment(current_segment->connection_num)) { + U++; // just to prevent it from remaining at 0 if there are more URLs to get + }; + stats.show_totals(); + curl_easy_cleanup(e); + }else { + error_log("ERROR: CURLMsg: "+msg->msg); + } } } + debug("bbbbbblllllllllllllaaaaaaaaaaaaa"); + // nothing to download - wait 5 secs and check upper_proxy_fetchers again + // later will be replaced by server_waiting_for_incoming connections and new tasks + // with timeout to check downloads from upper proxy fetcher + sleep(1); } curl_multi_cleanup(cm); curl_global_cleanup(); @@ -228,6 +264,8 @@ int download_pkgs(){ } } +void *print_message_function( void *ptr ); + int main() { try{ @@ -263,6 +301,31 @@ int main() catch(...){ //error while showing stats } + pthread_t thread1; +// , thread2; +// char *message1 = "Thread 1"; +// char *message2 = "Thread 2"; + int iret1; +// int iret2; + + /* Create independent threads each of which will execute function */ + + iret1 = pthread_create( &thread1, NULL, print_message_function, (void*) NULL); +// iret2 = pthread_create( &thread2, NULL, print_message_function, (void*) message2); + + /* Wait till threads are complete before main continues. Unless we */ + /* wait we run the risk of executing an exit which will terminate */ + /* the process and all threads before the threads have completed. */ + + debug("oooooooooooooo111111111111111111"); +// pthread_join( thread1, NULL); + debug("oooooooooooooo2222222222222222222222"); +// pthread_join( thread2, NULL); + +// printf("Thread 1 returns: %d\n",iret1); +// printf("Thread 2 returns: %d\n",iret2); + + try{ download_pkgs(); } @@ -282,4 +345,24 @@ int main() //error while ending curses } return 0; +} + +void *print_message_function(void *ptr){ +// char *message; + char * args; + args = (char *) ptr; + ulong i=0; + while (true){ + ulong time_diff_msecs=time_left_from(stats.previous_time); + if (time_diff_msecs >= settings.current_speed_time_interval_msecs){ + show_progress(time_diff_msecs); + }; + log("Thread:"+toString(i)); + //haven't downloaded anything - anyway, show totals + stats.show_totals(); + sleep(1); + i++; + } + return 0; +// printf("%s \n", message); } \ No newline at end of file diff --git a/segget/segget.h b/segget/segget.h index fc65f31..58a9e57 100644 --- a/segget/segget.h +++ b/segget/segget.h @@ -47,6 +47,7 @@ #include "str.cpp" #include "tui.cpp" #include "utils.cpp" +#include using namespace std; diff --git a/segget/segment.cpp b/segget/segment.cpp index 8e29b9c..6606bc8 100644 --- a/segget/segment.cpp +++ b/segget/segment.cpp @@ -26,6 +26,15 @@ #include "segment.h" +string statusToString(Tsegment_status the_status){ + switch (the_status){ + case SWAITING:return "SWAITING"; + case SDOWNLOADING:return "SDOWNLOADING"; + case SDOWNLOADED:return "SDOWNLOADED"; + case SFAILED:return "SFAILED"; + default :return "UNKNOWN STATUS"; + } +} void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end){ try{ parent_distfile=prnt_distfile; @@ -44,7 +53,7 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi debug("seg:"+toString(seg_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size)); file.close(); if (downloaded_size==segment_size){ - status=DOWNLOADED; + status=SDOWNLOADED; debug("seg:"+toString(seg_num)+" Downloaded"); } else{ @@ -58,8 +67,8 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi } void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num){ try{ - debug("NETWORK:"+toString(network_num)+(network_array[network_num].use_own_mirror_list_only_on ? " - LOCAL": " - REMOTE")); - if (network_array[network_num].use_own_mirror_list_only_on){ +// debug("NETWORK:"+toString(network_num)+(network_array[network_num].use_own_mirror_list_only_on ? " - LOCAL": " - REMOTE")); + if (network_array[network_num].network_mode==MODE_LOCAL){ url=network_array[network_num].benchmarked_mirror_list[mirror_num].url+parent_distfile->name; debug(" URL:"+url); }else{ @@ -67,7 +76,7 @@ void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, } msg_connecting(con_num,distfile_num, segment_num,"Downloading from "+url); segments_in_progress[con_num]=this; - status=DOWNLOADING; + status=SDOWNLOADING; downloaded_bytes=0; connection_num=con_num; // connection_array[con_num].start(network_num); @@ -177,11 +186,10 @@ size_t write_data(void *buffer, size_t size, size_t nmemb, void *cur_segment){ error_log("Can't write segment file:"+segment->file_name); } connection_array[segment->connection_num].inc_bytes_per_last_interval(bytes_written); - ulong time_diff_msecs=time_left_from(stats.previous_time); - if (time_diff_msecs >= settings.current_speed_time_interval_msecs){ - show_progress(time_diff_msecs); - stats.reset_previous_time(); - }; +// ulong time_diff_msecs=time_left_from(stats.previous_time); +// if (time_diff_msecs >= settings.current_speed_time_interval_msecs){ +// show_progress(time_diff_msecs); +// }; }catch(...){ error_log("Error in segment.cpp: write_data()"); return 0; diff --git a/segget/segment.h b/segget/segment.h index deb55b5..0c0ba83 100644 --- a/segget/segment.h +++ b/segget/segment.h @@ -48,7 +48,7 @@ extern Tsettings settings; unsigned long downloaded_bytes=0; size_t write_data(void *buffer, size_t size, size_t nmemb, void *cur_segment); -enum Tstatus{WAITING, DOWNLOADING, DOWNLOADED, FAILED}; +enum Tsegment_status{SWAITING, SDOWNLOADING, SDOWNLOADED, SFAILED}; class Tsegment{ private: @@ -56,7 +56,7 @@ class Tsegment{ char* urllist; public: string file_name; - Tstatus status; + Tsegment_status status; uint try_num; Tdistfile* parent_distfile; uint connection_num; @@ -70,7 +70,7 @@ class Tsegment{ easyhandle(0), urllist(0), file_name(""), - status(WAITING), + status(SWAITING), try_num(0), parent_distfile(0), connection_num(0), diff --git a/segget/stats.cpp b/segget/stats.cpp index 3683c5c..cfbe66c 100644 --- a/segget/stats.cpp +++ b/segget/stats.cpp @@ -63,6 +63,7 @@ void Tstats::show_totals(){ // +" Secs:"+toString(now_timee.tv_sec) // +" usecs:"+toString(now_timee.tv_usec) ); + reset_previous_time(); }catch(...){ error_log("Error in stats.cpp: show_totals()"); } diff --git a/segget/tui.cpp b/segget/tui.cpp index 3b880ae..341be4d 100644 --- a/segget/tui.cpp +++ b/segget/tui.cpp @@ -29,14 +29,19 @@ extern Tsettings settings; const uint CONNECTION_LINES=5; +bool msg_idle=true; void msg(uint y, uint x, string msg_text){ - try{ - move(y,x); - string ready_msg_text=msg_text+" "; - printw(ready_msg_text.c_str()); - refresh(); - }catch(...){ - error_log_no_msg("Error in tui.cpp: msg()"); + if (msg_idle){ + msg_idle=false; + try{ + move(y,x); + string ready_msg_text=msg_text+" "; + printw(ready_msg_text.c_str()); + refresh(); + }catch(...){ + error_log_no_msg("Error in tui.cpp: msg()"); + } + msg_idle=true; } } -- cgit v1.2.3-65-gdbad