summaryrefslogtreecommitdiff
path: root/segget
diff options
context:
space:
mode:
authorKostyantyn Ovechko <fastinetserver@gmail.com>2010-07-10 21:59:31 +0300
committerKostyantyn Ovechko <fastinetserver@gmail.com>2010-07-10 21:59:31 +0300
commit995abc2a03def4c7d99c2d6748c3636cc86b815c (patch)
treec1578512ade302c7a13108886073c27c5ff41611 /segget
parentFailprove .cpp files and log errors to error.log (diff)
downloadidfetch-995abc2a03def4c7d99c2d6748c3636cc86b815c.tar.gz
idfetch-995abc2a03def4c7d99c2d6748c3636cc86b815c.tar.bz2
idfetch-995abc2a03def4c7d99c2d6748c3636cc86b815c.zip
Move show_progress() to a thread
Diffstat (limited to 'segget')
-rw-r--r--segget/client.cpp0
-rw-r--r--segget/connection.cpp10
-rw-r--r--segget/connection.h4
-rw-r--r--segget/distfile.cpp213
-rw-r--r--segget/distfile.h39
-rw-r--r--segget/network.cpp5
-rw-r--r--segget/network.h18
-rw-r--r--segget/network0.conf72
-rw-r--r--segget/network0_mirrors.conf4
-rw-r--r--segget/network1.conf56
-rw-r--r--segget/network2.conf236
-rw-r--r--segget/networkbroker.cpp2
-rw-r--r--segget/segget.conf22
-rw-r--r--segget/segget.cpp209
-rw-r--r--segget/segget.h1
-rw-r--r--segget/segment.cpp26
-rw-r--r--segget/segment.h6
-rw-r--r--segget/stats.cpp1
-rw-r--r--segget/tui.cpp19
19 files changed, 779 insertions, 164 deletions
diff --git a/segget/client.cpp b/segget/client.cpp
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/segget/client.cpp
diff --git a/segget/connection.cpp b/segget/connection.cpp
index dc1cc98..423c783 100644
--- a/segget/connection.cpp
+++ b/segget/connection.cpp
@@ -28,6 +28,7 @@
void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num){
try{
segment=started_segment;
+ debug("Started connection for distfile"+segment->parent_distfile->name);
mirror_num=best_mirror_num;
network_num=network_number;
total_dld_bytes=0;
@@ -43,6 +44,7 @@ void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegm
void Tconnection::stop(uint connection_result){
try{
+ debug("Finished connection for distfile"+segment->parent_distfile->name+" Status"+toString(connection_result));
msg_clean_connection(connection_num);
active=false;
network_array[network_num].disconnect();
@@ -52,7 +54,7 @@ void Tconnection::stop(uint connection_result){
prnt_distfile->active_connections_num--;
Tmirror *Pcurr_mirror;
- if (network_array[network_num].use_own_mirror_list_only_on){
+ if (network_array[network_num].network_mode==MODE_LOCAL){
Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num];
prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true;
// find_mirror(strip_mirror_name(segment->url));
@@ -69,16 +71,16 @@ void Tconnection::stop(uint connection_result){
debug(toString(connection_result)+"]- Failed download "+segment->url);
Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),0);
if (segment->try_num>=settings.max_tries){
- segment->status=FAILED;
+ segment->status=SFAILED;
error_log("Segment:"+segment->file_name+" has reached max_tries limit - segment.status set to FAILED");
}
- else segment->status=WAITING;
+ else segment->status=SWAITING;
}else{
// no error => count this one and start new
log("Succesfully downloaded "+segment->file_name+" on connection#"+toString(connection_num));
debug(" Successful download "+segment->url);
Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),segment->segment_size);
- segment->status=DOWNLOADED;
+ segment->status=SDOWNLOADED;
prnt_distfile->inc_dld_segments_count(segment);
};
}catch(...){
diff --git a/segget/connection.h b/segget/connection.h
index fccf324..a5dd206 100644
--- a/segget/connection.h
+++ b/segget/connection.h
@@ -41,19 +41,19 @@ class Tconnection{
uint connection_num;
uint network_num;
uint mirror_num;
- bool active;
ulong total_dld_bytes;
ulong bytes_per_last_interval;
public:
+ bool active;
timeval start_time;
Tsegment *segment;
Tconnection():
connection_num(total_connections),
network_num(0),
mirror_num(0),
- active(0),
total_dld_bytes(0),
bytes_per_last_interval(0),
+ active(0),
start_time(),
segment(0){total_connections++;};
void start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num);
diff --git a/segget/distfile.cpp b/segget/distfile.cpp
index 023d0e7..9f0bb0e 100644
--- a/segget/distfile.cpp
+++ b/segget/distfile.cpp
@@ -26,6 +26,80 @@
#include "distfile.h"
+//Make the necessary includes and set up the variables:
+using namespace std;
+
+Tdistfile_status Tdistfile::request(string msg)
+{
+ int sockfd;
+ int len;
+ struct sockaddr_in address;
+ int result;
+ //Create a socket for the client:
+ sockfd = socket(AF_INET, SOCK_STREAM, 0);
+
+ //Name the socket, as agreed with the server:
+ address.sin_family = AF_INET;
+ address.sin_addr.s_addr = inet_addr("127.0.0.1");
+ address.sin_port = htons(9797);
+ len = sizeof(address);
+
+ //Connect your socket to the server’s socket:
+ result = connect(sockfd, (struct sockaddr *)&address, len);
+ if(result == -1) {
+ error_log("Can't connect to proxy-fetcher");
+ return DPROXY_FAILED;
+ }
+ if (msg.length()>90000){return DPROXY_REJECTED;};
+ char send_buffer[100000];
+// char recv_buffer[256];
+ strcpy(send_buffer,msg.c_str());
+ //You can now read and write via sockfd:
+ int i=write(sockfd, send_buffer, strlen(send_buffer));
+ i++;
+/*
+ fd_set readfds, testfds;
+ FD_ZERO(&readfds);
+ FD_SET(sockfd, &readfds);
+ testfds = readfds;
+
+ result = select(FD_SETSIZE, &testfds, (fd_set *)0,
+ (fd_set *)0, (struct timeval *) 0);
+
+ int nread;
+ ioctl(sockfd, FIONREAD, &nread);
+
+ if(nread == 0) {
+ close(sockfd);
+ // FD_CLR(sockfd, &readfds);
+ /// printf("removing client on fd %d\n", sockfd);
+ }else{
+ read(sockfd, recv_buffer, nread);
+ }
+*/
+ close(sockfd);
+ return DPROXY_QUEUED;
+}
+
+bool Tdistfile::allows_new_actions(){
+// if (downloaded) return false;
+// else return true;
+// int time_left=0;
+ if (status==DDOWNLOADED) return false;
+// if (((status==DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false;
+//oterwise allow connections
+// DNEW,
+// D_NOT_PROXY_REQUESTED,
+// DPROXY_REJECTED,
+// DPROXY_FAILED,
+// DPROXY_DOWNLOADED,
+// DWAITING,
+// DDOWNLOADING,
+// DDOWNLOADED,
+// DFAILED
+ return true;
+}
+
void Tdistfile::init(){
try{
for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){
@@ -62,8 +136,10 @@ bool Tdistfile::check_if_dld(){
//debug("seg:"+toString(seg_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size));
filec.close();
if (d_size==size){
- downloaded=true;
+ status=DDOWNLOADED;
+// downloaded=true;
num=++stats.distfiles_count;
+ stats.inc_total_size(size);
stats.inc_dld_distfiles_count();
stats.inc_dld_size(size);
debug("Distfile:"+name+" already downloaded");
@@ -86,8 +162,8 @@ void Tdistfile::load_distfile_from_json(json_object* json_obj_distfile){
RMD160=json_object_get_string(json_object_object_get(json_obj_distfile,"RMD160"));
SHA1=json_object_get_string(json_object_object_get(json_obj_distfile,"SHA1"));
SHA256=json_object_get_string(json_object_object_get(json_obj_distfile,"SHA256"));
-
if (not(check_if_dld())){
+ json_data=json_object_to_json_string(json_obj_distfile);
split_into_segments();
load_url_list(json_object_object_get(json_obj_distfile,"url_list"));
}
@@ -120,7 +196,7 @@ void Tdistfile::split_into_segments(){
debug("segment_range:"+toString(range_end));
};
dn_segments[segment_num].set_segment(this, segment_num, name, segment_size, range_end);
- if (dn_segments[segment_num].status==DOWNLOADED)
+ if (dn_segments[segment_num].status==SDOWNLOADED)
inc_dld_segments_count(&dn_segments[segment_num]);
}
}catch(...){
@@ -161,6 +237,7 @@ bool Tdistfile::choose_best_mirror(CURLM* cm, uint connection_num, uint network_
if (Pbest_mirror){
debug("Downloading from BEST_MIRROR:"+url_list[best_mirror_num]);
Pbest_mirror->start();
+ active_connections_num++;
connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num);
return 0;
}
@@ -193,6 +270,7 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne
if (best_mirror_num!=-1){
debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].benchmarked_mirror_list[best_mirror_num].url);
network_array[network_num].benchmarked_mirror_list[best_mirror_num].start();
+ active_connections_num++;
connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num);
return 0;
}
@@ -208,7 +286,6 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne
int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){
try{
- active_connections_num++;
for (uint cur_network_priority=10; cur_network_priority>0; cur_network_priority--){
debug("cur_network_priority="+toString(cur_network_priority));
//choose network
@@ -220,41 +297,62 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){
//
//----------------------------------------------------------------------------------------------------------
int best_local_network_num=-1;
- int best_network_num=-1;
+ int best_proxy_fetcher_network_num=-1;
+ int best_remote_network_num=-1;
bool allow_remote_mirrors=true;
for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){
//if network priority set then it's active
if (network_array[network_num].priority){
if (network_array[network_num].priority==cur_network_priority){
debug(" network_priority="+toString(network_array[network_num].priority));
- if (network_array[network_num].use_own_mirror_list_only_on){
- if (network_array[network_num].has_free_connections()){
- if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){
-// debug(" Allowed network#:"+toString(network_num));
- if ((best_local_network_num==-1)
- or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){
- best_local_network_num=network_num;
- debug(" Replace best LOCAL network to network#:"+toString(network_num));
- }
- }
- }else{
- if (network_array[network_num].only_local_when_possible){
+ switch (network_array[network_num].network_mode){
+ case MODE_LOCAL:{
+ if (network_array[network_num].has_free_connections()){
if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){
- allow_remote_mirrors=false;
- debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed");
+// debug(" Allowed network#:"+toString(network_num));
+ if ((best_local_network_num==-1)
+ or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){
+ best_local_network_num=network_num;
+ debug(" Replace best LOCAL network to network#:"+toString(network_num));
+ }
+ }
+ }else{
+ if (network_array[network_num].only_local_when_possible){
+ if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){
+ allow_remote_mirrors=false;
+ debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed");
+ }
}
}
+ break;
+ }
+ case MODE_PROXY_FETCHER:{
+ //replace this one by does_not_reject_connections
+// if (network_array[network_num].has_free_connections()){
+ if
+ ((best_proxy_fetcher_network_num==-1)
+// or
+// (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num)
+ ){
+ best_proxy_fetcher_network_num=network_num;
+ debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(network_num));
+ debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(best_proxy_fetcher_network_num));
+ }
+// }
+ break;
}
- }else{
+ case MODE_REMOTE:{
if (network_array[network_num].has_free_connections()){
if
- ((best_network_num==-1)
+ ((best_remote_network_num==-1)
or
- (network_array[best_network_num].active_connections_num>network_array[network_num].active_connections_num)){
- best_network_num=network_num;
- debug(" Replace best network to network to network#:"+toString(network_num));
+ (network_array[best_remote_network_num].active_connections_num>network_array[network_num].active_connections_num)){
+ best_remote_network_num=network_num;
+ debug(" Replace best_remote_network_num to network#:"+toString(network_num));
}
}
+ break;
+ }
}
//work with network
}
@@ -264,23 +362,57 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){
//best network has been found
//work with network
debug(" So best LOCAL network is network#:"+toString(best_local_network_num));
- int res=choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num);
- return res;
+ return choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num);
}else{
- if (allow_remote_mirrors){ //since all local failed, go to remote
- // remote_mirrors_go_second
- if (best_network_num!=-1){
- //best network has been found
- //work with network
- debug(" So best network is network#:"+toString(best_network_num));
- return choose_best_mirror(cm, connection_num, best_network_num, seg_num);
+//DDOWNLOADED) return false;
+//DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false;
+ //oterwise allow connections
+// DNEW,
+// D_NOT_PROXY_REQUESTED,
+// DPROXY_REJECTED,
+// DPROXY_FAILED,
+// DWAITING,
+// DDOWNLOADING,
+// DDOWNLOADED,
+// DFAILED
+ if (allow_remote_mirrors){ //since all local failed, go to proxy_fetcher
+ debug("Remote mirrors are allowed");
+ if (best_proxy_fetcher_network_num != -1){
+ if (status == DPROXY_QUEUED){
+ return 1;
+ }else{
+ // request from proxy fethcer
+ status=request(json_data);
+ debug("Trying to dowload distfile"+name+" via proxy_fetcher. status="+toString(status));
+ if (status==DPROXY_DOWNLOADED){
+ // start download from the proxy_fetcher
+ debug(" So best proxy_fetcher_network is network#:"+toString(best_proxy_fetcher_network_num));
+// BEWARE -- CORRECT FOLLOWING LINES !!!!!!!!!!!
+ // start download from proxy_fether mirrors
+// return choose_best_mirror(cm, connection_num, best_remote_network_num, seg_num);
+ }
+ //return - don't switch to low priority networks
+ return 0;
+ }
+ }else{
+ // remote_mirrors_go_third
+ if (best_remote_network_num!=-1){
+ //best network has been found
+ //work with network
+ debug(" So best_proxy_fetcher_failed num="+toString(best_proxy_fetcher_network_num));
+ debug(" So best network is network#:"+toString(best_remote_network_num));
+ return choose_best_mirror(cm, connection_num, best_remote_network_num, seg_num);
+ }
}
}else{
- debug("Restricted to local mirrors only when possible");
+ debug("NOT all local mirrors have failed - restricted to local mirrors only.");
+ //return - don't switch to low priority networks
+ return 100;
}
}
}
- return 0;
+ // haven't found anything suitable
+ return 120;
}catch(...){
error_log("Error: distfile.cpp: provide_segment()");
return 1;
@@ -374,6 +506,7 @@ int Tdistfile::combine_segments(){
distfile_file.open(distfile_path.c_str(),ofstream::binary|ios::trunc);
}catch(...){
error_log("Error: distfile.cpp: combine_segments(): opening distfile:"+distfile_path);
+ status=DFAILED;
return 1;
}
char * buffer;
@@ -387,6 +520,7 @@ int Tdistfile::combine_segments(){
segment_file.open((settings.segments_dir+"/"+dn_segments[seg_num].file_name).c_str(),ifstream::binary);
}catch(...){
error_log("Error: distfile.cpp: combine_segments(): opening segment:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name);
+ status=DFAILED;
return 2;
}
try{
@@ -404,13 +538,15 @@ int Tdistfile::combine_segments(){
segment_file.close();
}catch(...){
error_log("Error: distfile.cpp: combine_segments(): segment is open, but error occured while reading it:"+settings.segments_dir+"/"+dn_segments[seg_num].file_name);
- return 3;
+ status=DFAILED;
+ return 3;
}
try{
// write to outfile
distfile_file.write (buffer,cur_seg_size);
}catch(...){
error_log("Error: distfile.cpp: combine_segments(): distfile is open, but error occured while writing into it:"+settings.distfiles_dir+"/"+name);
+ status=DFAILED;
return 4;
}
// release dynamically-allocated memory
@@ -426,6 +562,7 @@ int Tdistfile::combine_segments(){
log("Distfile "+name+" has been combined");
}catch(...){
error_log("Error in distfile.cpp: combine_segments() for distfile:"+settings.distfiles_dir+"/"+name);
+ status=DFAILED;
return 5;
}
try{
@@ -434,6 +571,7 @@ int Tdistfile::combine_segments(){
else{
log("Error: RMD160 checksum for distfile:"+name+" [FAILED]");
error_log("Error: RMD160 checksum for distfile:"+name+" [FAILED]");
+ status=DFAILED;
return 10;
}
@@ -442,6 +580,7 @@ int Tdistfile::combine_segments(){
else{
log("Error: SHA1 checksum for distfile:"+name+" [FAILED]");
error_log("Error: SHA1 checksum for distfile:"+name+" [FAILED]");
+ status=DFAILED;
return 11;
}
@@ -450,8 +589,10 @@ int Tdistfile::combine_segments(){
else{
log("Error: SHA256 checksum for distfile:"+name+" [FAILED]");
error_log("Error: SHA256 checksum for distfile:"+name+" [FAILED]");
+ status=DFAILED;
return 12;
}
+ status=DDOWNLOADED;
if (settings.provide_mirror_dir!="none"){
symlink_distfile_to_provide_mirror_dir();
}
diff --git a/segget/distfile.h b/segget/distfile.h
index 2f9c3fa..1b2f110 100644
--- a/segget/distfile.h
+++ b/segget/distfile.h
@@ -52,10 +52,39 @@
#include "network.h"
#include "networkbroker.h"
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <stdio.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string>
+#include <string.h>
+
+
+//#include "client.cpp"
+
using namespace std;
typedef unsigned int uint;
+enum Tdistfile_status{
+ DNEW,
+ D_NOT_PROXY_REQUESTED,
+ DPROXY_REJECTED,
+ DPROXY_QUEUED,
+ DPROXY_DOWNLOADING,
+ DPROXY_DOWNLOADED,
+ DPROXY_FAILED,
+ DWAITING,
+ DDOWNLOADING,
+ DDOWNLOADED,
+ DFAILED
+};
+
class Tdistfile{
private:
uint dld_segments_count;
@@ -63,7 +92,9 @@ class Tdistfile{
bool choose_best_mirror(CURLM* cm, uint connection_num, uint network_num, uint seg_num);
public:
Tnetwork_distfile_broker network_distfile_brokers_array[MAX_NETWORKS];
- bool downloaded;
+ string json_data;
+// bool downloaded;
+ Tdistfile_status status;
uint active_connections_num;
string *url_list;
uint url_num;
@@ -80,7 +111,9 @@ class Tdistfile{
uint segment_size;
Tdistfile():
dld_segments_count(0),
- downloaded(0),
+ json_data(""),
+// downloaded(0),
+ status(DNEW),
active_connections_num(0),
url_list(0),
url_num(0),
@@ -99,7 +132,9 @@ class Tdistfile{
Tdistfile(const Tdistfile &L); // copy constructor
Tdistfile & operator=(const Tdistfile &L);
~Tdistfile();
+ Tdistfile_status request(string msg);
void init();
+ bool allows_new_actions();
void load_distfile_from_json(json_object* json_obj_distfile);
void load_url_list(json_object* json_array_distfile_urllist);
void split_into_segments();
diff --git a/segget/network.cpp b/segget/network.cpp
index f0ea3b6..ad765f5 100644
--- a/segget/network.cpp
+++ b/segget/network.cpp
@@ -63,6 +63,7 @@ void Tnetwork::init(uint priority_value){
try{
priority=priority_value;
Tconfig conf("network"+toString(network_num)+".conf");
+ conf.set(network_mode, "mode", "network_mode",0,2);
conf.set(bind_interface, "network_bind", "bind_interface");
conf.set(max_connections, "network_connections", "max_connections",1,MAX_CONNECTS);
conf.set(connection_timeout, "network_connections", "connection_timeout",1,1000);
@@ -80,9 +81,9 @@ void Tnetwork::init(uint priority_value){
conf.set(proxy_user, "network_proxy", "proxy_user");
conf.set(proxy_password, "network_proxy", "proxy_password");
- conf.set(use_own_mirror_list_only_on, "network_mirrors", "use_own_mirror_list_only_on");
+// conf.set(use_own_mirror_list_only_on, "network_mirrors", "use_own_mirror_list_only_on");
- if (use_own_mirror_list_only_on){
+ if (network_mode==MODE_LOCAL){
conf.set(only_local_when_possible, "network_mirrors", "only_local_when_possible");
load_mirror_list();
log("Settings: Network"+toString(network_num)+" local mirror_list size:"+toString(mirror_list.size()));
diff --git a/segget/network.h b/segget/network.h
index c164bb7..a42f54d 100644
--- a/segget/network.h
+++ b/segget/network.h
@@ -38,6 +38,10 @@ using namespace std;
#define MAX_NETWORKS 10
+#define MODE_REMOTE 0
+#define MODE_PROXY_FETCHER 1
+#define MODE_LOCAL 2
+
class Tnetwork{
static uint network_count;
private:
@@ -48,6 +52,10 @@ class Tnetwork{
//network
uint network_num;
uint priority;
+ //mode
+ ulong network_mode;
+ //network_bind
+ string bind_interface;
//connections
ulong max_connections;
ulong connection_timeout;
@@ -57,7 +65,6 @@ class Tnetwork{
ulong low_connection_speed_time;
ulong max_connection_speed;
ulong current_speed_time_interval_msecs;
- string bind_interface;
//user-data
string user_agent;
//proxy
@@ -67,7 +74,7 @@ class Tnetwork{
string proxy_user;
string proxy_password;
//mirrors
- bool use_own_mirror_list_only_on;
+// bool use_own_mirror_list_only_on;
bool only_local_when_possible;
Tnetwork():
benchmarked_mirror_list(),
@@ -75,6 +82,10 @@ class Tnetwork{
//network
network_num(network_count),
priority(0),
+ //mode
+ network_mode(0),
+ //network_bind
+ bind_interface("none"),
//connections
max_connections(6),
connection_timeout(15),
@@ -84,7 +95,6 @@ class Tnetwork{
low_connection_speed_time(10),
max_connection_speed(0),
current_speed_time_interval_msecs(1000),
- bind_interface("none"),
//user-data
user_agent("segget"),
//proxy
@@ -94,7 +104,7 @@ class Tnetwork{
proxy_user("none"),
proxy_password("none"),
//mirrors
- use_own_mirror_list_only_on(0),
+// use_own_mirror_list_only_on(0),
only_local_when_possible(1)
{network_count++;};
void init(uint priority_value);
diff --git a/segget/network0.conf b/segget/network0.conf
index 2fc7aea..d816e29 100644
--- a/segget/network0.conf
+++ b/segget/network0.conf
@@ -1,3 +1,38 @@
+[mode]
+# SYNOPSIS: NETWORK_MODE= 0 | 1 | 2
+# 0 - remote mirrors (default)
+# 1 - proxy fetcher
+# 2 - local mirrors
+# - If set to 2, segget will replace mirror list provided by portage
+# system with the list from network0_mirrors.conf file
+# - If set to 0, segget will use ONLY mirror list provided by portage
+# system, and will NOT use the list from network0_mirrors.conf file
+# In some cases it's necessary to make segget prefer local mirrors over
+# the remote ones. For this purpose define settings for 2 networks:
+# settings for network0 (to provide access to local mirrors),
+# settings for network1 (to provide access to remote ones).
+# 1) Set the following options in segget.conf file:
+# [networks]
+# network0_priority=10
+# network1_priority=9
+# As you can see network0 (with local mirrors) has higher priority than
+# network1 (with mirrors provided by portage).
+# 2) Set NETWORK_MODE=2 in network0.conf file.
+# 3) Create network0_mirrors.conf file with the list of your local
+# mirrors.
+# For example, network0_mirrors.conf may look like this:
+# http://192.168.210.12/
+# ftp://192.168.210.205/
+# http://192.168.210.56/
+# 4) Set NETWORK_MODE=0 in network1.conf file,
+# so segget will use remote mirrors working via this network.
+# NOTE: Actually network0 and network1 can be the same LAN with only
+# one ip address set on the host. The only difference is that in case of
+# network1 segget will have to use a gateway to access remote mirrors.
+# Default:
+# network_mode=0
+network_mode=2
+
[network_bind]
# BIND INTERFACE / IP
# Pass a string as parameter. This sets the interface name to use as outgoing
@@ -170,6 +205,22 @@ proxy_password=none
# proxy_off=1
proxy_off=1
+[proxy-fetcher]
+# Specify proxy-fetcher address and port
+proxy-fetcher-ip=127.0.0.1
+proxy-fetcher-port=9797
+# Specify the user and password for authentication on a proxy-fetcher-server.
+# proxy-fetcher-user=user
+# proxy-fetcher-password=password
+
+# Set to forbid direct connections to Internet servers if it's possible to use
+# proxy-fetcher.
+# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-if-possible=1
+# Set to forbid direct connections to Internet servers.
+# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-only=0
+# Set to forbid using proxy-fetcher.
+# NOT IMPLEMENTED YET: no-proxy-fetcher
+
[network_mirrors]
# SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 | 1
# - If set to 1, segget will replace mirror list provided by portage system with
@@ -181,12 +232,21 @@ proxy_off=1
use_own_mirror_list_only_on=0
# SYNOPSIS: ONLY_LOCAL_WHEN_POSSIBLE=0 | 1
-# If NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 this option will be ignored.
-# - If set to 1, segget will not use remote mirrors with equal or lower priority
-# until all mirrors in network0_mirrors.conf file have failed.
-# - If set to 0, segget will use remote mirrors with equal priority or mirrors
-# with lower priority when this network has NO free connections (see option
-# NETWORK_MAX_CONNECTIONS in [network_connections] section of this file).
+# If NETWORK_MODE=0 or NETWORK_MODE=1 this option will be ignored.
+# - If set to 1, segget will not use remote mirrors with equal or lower
+# priority until all mirrors in network#_mirrors.conf file have failed.
+# - If set to 0, segget will use remote mirrors with equal priority or
+# mirrors with lower priority when this network has NO free connections
+# (see option NETWORK_MAX_CONNECTIONS in [network_connections] section
+# of this file).
+# NOTE: Following example for NETWORK_MODE option,
+# if in network0.conf has option ONLY_LOCAL_WHEN_POSSIBLE=1, segget
+# will NOT start to use network1 for a particular distfile until all
+# mirrors specified in network0_mirrors.conf file will have failed
+# to provide this distfile.
+# On the other hand if ONLY_LOCAL_WHEN_POSSIBLE=0 segget will start
+# to use network1 as soon as NETWORK_MAX_CONNECTIONS limit, set
+# in network0.conf file has been reached.
# Default:
# only_local_when_possible=1
only_local_when_possible=1 \ No newline at end of file
diff --git a/segget/network0_mirrors.conf b/segget/network0_mirrors.conf
new file mode 100644
index 0000000..4164689
--- /dev/null
+++ b/segget/network0_mirrors.conf
@@ -0,0 +1,4 @@
+ftp://192.168.56.11/
+ftp://192.168.56.12/
+ftp://192.168.56.13/
+ftp://192.168.56.14/ \ No newline at end of file
diff --git a/segget/network1.conf b/segget/network1.conf
index 7e0a72b..ea7e9bf 100644
--- a/segget/network1.conf
+++ b/segget/network1.conf
@@ -1,3 +1,38 @@
+[mode]
+# SYNOPSIS: NETWORK_MODE= 0 | 1 | 2
+# 0 - remote mirrors (default)
+# 1 - proxy fetcher
+# 2 - local mirrors
+# - If set to 2, segget will replace mirror list provided by portage
+# system with the list from network0_mirrors.conf file
+# - If set to 0, segget will use ONLY mirror list provided by portage
+# system, and will NOT use the list from network0_mirrors.conf file
+# In some cases it's necessary to make segget prefer local mirrors over
+# the remote ones. For this purpose define settings for 2 networks:
+# settings for network0 (to provide access to local mirrors),
+# settings for network1 (to provide access to remote ones).
+# 1) Set the following options in segget.conf file:
+# [networks]
+# network0_priority=10
+# network1_priority=9
+# As you can see network0 (with local mirrors) has higher priority than
+# network1 (with mirrors provided by portage).
+# 2) Set NETWORK_MODE=2 in network0.conf file.
+# 3) Create network0_mirrors.conf file with the list of your local
+# mirrors.
+# For example, network0_mirrors.conf may look like this:
+# http://192.168.210.12/
+# ftp://192.168.210.205/
+# http://192.168.210.56/
+# 4) Set NETWORK_MODE=0 in network1.conf file,
+# so segget will use remote mirrors working via this network.
+# NOTE: Actually network0 and network1 can be the same LAN with only
+# one ip address set on the host. The only difference is that in case of
+# network1 segget will have to use a gateway to access remote mirrors.
+# Default:
+# network_mode=0
+network_mode=1
+
[network_bind]
# BIND INTERFACE / IP
# Pass a string as parameter. This sets the interface name to use as outgoing
@@ -181,12 +216,21 @@ proxy_off=1
use_own_mirror_list_only_on=1
# SYNOPSIS: ONLY_LOCAL_WHEN_POSSIBLE=0 | 1
-# If NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 this option will be ignored.
-# - If set to 1, segget will not use remote mirrors with equal or lower priority
-# until all mirrors in network0_mirrors.conf file have failed.
-# - If set to 0, segget will use remote mirrors with equal priority or mirrors
-# with lower priority when this network has NO free connections (see option
-# NETWORK_MAX_CONNECTIONS in [network_connections] section of this file).
+# If NETWORK_MODE=0 or NETWORK_MODE=1 this option will be ignored.
+# - If set to 1, segget will not use remote mirrors with equal or lower
+# priority until all mirrors in network#_mirrors.conf file have failed.
+# - If set to 0, segget will use remote mirrors with equal priority or
+# mirrors with lower priority when this network has NO free connections
+# (see option NETWORK_MAX_CONNECTIONS in [network_connections] section
+# of this file).
+# NOTE: Following example for NETWORK_MODE option,
+# if in network0.conf has option ONLY_LOCAL_WHEN_POSSIBLE=1, segget
+# will NOT start to use network1 for a particular distfile until all
+# mirrors specified in network0_mirrors.conf file will have failed
+# to provide this distfile.
+# On the other hand if ONLY_LOCAL_WHEN_POSSIBLE=0 segget will start
+# to use network1 as soon as NETWORK_MAX_CONNECTIONS limit, set
+# in network0.conf file has been reached.
# Default:
# only_local_when_possible=1
only_local_when_possible=1 \ No newline at end of file
diff --git a/segget/network2.conf b/segget/network2.conf
new file mode 100644
index 0000000..bc2548d
--- /dev/null
+++ b/segget/network2.conf
@@ -0,0 +1,236 @@
+[mode]
+# SYNOPSIS: NETWORK_MODE= 0 | 1 | 2
+# 0 - remote mirrors (default)
+# 1 - proxy fetcher
+# 2 - local mirrors
+# - If set to 2, segget will replace mirror list provided by portage
+# system with the list from network0_mirrors.conf file
+# - If set to 0, segget will use ONLY mirror list provided by portage
+# system, and will NOT use the list from network0_mirrors.conf file
+# In some cases it's necessary to make segget prefer local mirrors over
+# the remote ones. For this purpose define settings for 2 networks:
+# settings for network0 (to provide access to local mirrors),
+# settings for network1 (to provide access to remote ones).
+# 1) Set the following options in segget.conf file:
+# [networks]
+# network0_priority=10
+# network1_priority=9
+# As you can see network0 (with local mirrors) has higher priority than
+# network1 (with mirrors provided by portage).
+# 2) Set NETWORK_MODE=2 in network0.conf file.
+# 3) Create network0_mirrors.conf file with the list of your local
+# mirrors.
+# For example, network0_mirrors.conf may look like this:
+# http://192.168.210.12/
+# ftp://192.168.210.205/
+# http://192.168.210.56/
+# 4) Set NETWORK_MODE=0 in network1.conf file,
+# so segget will use remote mirrors working via this network.
+# NOTE: Actually network0 and network1 can be the same LAN with only
+# one ip address set on the host. The only difference is that in case of
+# network1 segget will have to use a gateway to access remote mirrors.
+# Default:
+# network_mode=0
+network_mode=0
+
+[network_bind]
+# BIND INTERFACE / IP
+# Pass a string as parameter. This sets the interface name to use as outgoing
+# network interface. The name can be an interface name, an IP address, or a host
+# name. No binding is set by default.
+# Default:
+# bind_interface=none
+bind_interface=none
+
+# BIND LOCALPORT
+# Pass a long. This sets the local port number of the socket used for connection.
+# This can be used in combination with BIND_INTERFACE and you are recommended to
+# use BIND_LOCALPORTRANGE as well when this is set.
+# Valid port numbers are 1 - 65535.
+
+# BIND_LOCALPORTRANGE
+# Pass a long. This is the number of attempts segget should make to find a
+# working local port number. It starts with the given BIND_LOCALPORT and adds
+# one to the number for each retry. Setting this to 1 or below will make segget
+# do only one try for the exact port number. Port numbers by nature are scarce
+# resources that will be busy at times so setting this value to something too
+# low might cause unnecessary connection setup failures.
+
+[network_connections]
+# NETWORK_MAX_CONNECTIONS
+# Define maximum number of connections
+# Minimum value: 1
+# Maximum value: 20
+# Default:
+# max_connections=10
+max_connections=3
+
+# CONNECTION_TIMEOUT
+# Set the number of seconds to wait while trying to connect. Use 0 to wait
+# indefinitely. Pass a long. It should contain the maximum time in seconds that
+# you allow the connection to the server to take. This only limits the connection
+# phase, once it has connected, this option is of no more use. Set to zero to
+# disable connection timeout (it will then only timeout on the system's internal
+# timeouts). See also the TIMEOUT option.
+# Minimum value: 1
+# Maximum value: 1000
+# Default:
+# connection_timeout=15
+connection_timeout=15
+
+# FTP_RESPONSE_TIMEOUT
+# Set a timeout period (in seconds) on the amount of time that the server is
+# allowed to take in order to generate a response message for a command before the
+# session is considered hung. While awaiting for a response, this value overrides
+# TIMEOUT. It is recommended that if used in conjunction with TIMEOUT, you set
+# FTP_RESPONSE_TIMEOUT to a value smaller than TIMEOUT.
+# Minimum value: 1
+# Maximum value: -1 (for no limit)
+# Default:
+# ftp_response_timeout=180
+ftp_response_timeout=180
+
+# TIMEOUT
+# maximum amount of time to download segment in seconds
+# Set the maximum number of seconds for a connection to execute.
+# Pass a long as parameter containing the maximum time in seconds that you allow
+# the transfer operation to take. Normally, name lookups can take a considerable
+# time and limiting operations to less than a few minutes risk aborting perfectly
+# normal operations.
+# Minimum value: 100
+# Maximum value: -1 (for no limit)
+# Default:
+# timeout=500
+timeout=500
+
+# LOW_CONNECTION_SPEED_LIMIT
+# Define the low speed limit for connection. Pass a long as parameter. It
+# contains the transfer speed in bytes per second that the transfer should be
+# below during LOW_CONNECTION_SPEED_TIME seconds to consider it too slow and abort.
+# Minimum value: 1
+# Maximum value: -1 (-1 for no limit)
+# Default:
+# low_connection_speed_limit=1000
+low_connection_speed_limit=1000
+
+# LOW_CONNECTION_SPEED_TIME
+# Pass a long as parameter. It contains the time in seconds that the transfer
+# should be below the LOW_CONNECTION_SPEED_LIMIT to consider it too slow and abort.
+# Minimum value: 1
+# Maximum value: 600
+# Default:
+# low_connection_speed_time=10
+low_connection_speed_time=10
+
+# MAX_CONNECTION_SPEED
+# If a download exceeds this speed (counted in bytes per second) on cumulative
+# average during the transfer, the transfer will pause to keep the average rate
+# less than or equal to the parameter value. Defaults to unlimited speed.
+# Minimum value: 1
+# Maximum value: -1 (-1 for no limit)
+# Default:
+# max_connection_speed=0
+max_connection_speed=3000
+
+[network_protocols]
+# SYNOPSIS: http_on=0 | 1
+# Default:
+# http_on=1
+# NOT IMPLEMENTED YET: http_on=1
+
+# SYNOPSIS: ftp_on=0 | 1
+# Default:
+# ftp_on=1
+# NOT IMPLEMENTED YET: ftp_on=1
+
+[network_user_data]
+# USER_AGENT
+# Set the User-Agent: header in the http request sent to the remote server.
+# This can be used to fool servers or scripts.
+# Default:
+# user_agent=segget
+user_agent=segget
+
+# Specify the user and password for authentication on a ftp servers.
+# NOT IMPLEMENTED YET: ftp-user=anonymous
+# NOT IMPLEMENTED YET: ftp-password=me@mail.ru
+
+[network_proxy]
+# PROXY_IP_OR_NAME
+# Specify a proxy to use (address and port).
+# Set HTTP proxy to use. The parameter should be a string holding the proxy host
+# name or dotted IP address. To specify port number in this string,
+# append :[port] to the end of the host name. The proxy string may be prefixed
+# with [protocol]:// since any such prefix will be ignored. The proxy's port
+# number may optionally be specified with the separate option. If not specified,
+# by default port 1080 will be used for proxies.
+# When you tell segget to use an HTTP proxy, segget will transparently convert
+# operations to HTTP even if you specify an FTP URL etc.
+# Segget respects the environment variables http_proxy, ftp_proxy, all_proxy etc,
+# if any of those are set. The PROXY option does however override any possibly
+# set environment variables.
+# Default:
+# proxy_ip_or_name=none
+proxy_ip_or_name=none
+
+# PROXY_PORT
+# Set the proxy port to connect to unless it is specified in the PROXY option.
+# Minimum value: 1
+# Maximum value: 65535
+# Default:
+# proxy_port=3128
+proxy_port=3128
+
+# PROXY_USER
+# Set user name to use for the transfer while connecting to Proxy.
+# The PROXY_USER option should be used in same way as the PROXY_PASSWORD is used.
+# In order to specify the password to be used in conjunction with the user name
+# use the PROXY_PASSWORD option.
+# Default:
+# proxy_user=none
+proxy_user=none
+
+# PROXY_PASSWORD
+# Set password to use for the transfer while connecting to Proxy.
+# The PROXY_PASSWORD option should be used in conjunction with the PROXY_USER
+# option.
+# Default:
+# proxy_password=none
+proxy_password=none
+
+# SYNOPSIS: proxy_off=0 | 1
+# Setting the proxy_off=1 will explicitly disable the use of a proxy, even if
+# there is an environment variable set for it.
+# Default:
+# proxy_off=1
+proxy_off=1
+
+[network_mirrors]
+# SYNOPSIS: NETWORK_USES_OWN_MIRROR_LIST_ONLY_ON=0 | 1
+# - If set to 1, segget will replace mirror list provided by portage system with
+# the list from network1_mirrors.conf
+# - If set to 0, segget will use ONLY mirror list provided by portage system,
+# and will NOT use the list from network1_mirrors.conf file
+# Default:
+# use_own_mirror_list_only_on=0
+use_own_mirror_list_only_on=1
+
+# SYNOPSIS: ONLY_LOCAL_WHEN_POSSIBLE=0 | 1
+# If NETWORK_MODE=0 or NETWORK_MODE=1 this option will be ignored.
+# - If set to 1, segget will not use remote mirrors with equal or lower
+# priority until all mirrors in network#_mirrors.conf file have failed.
+# - If set to 0, segget will use remote mirrors with equal priority or
+# mirrors with lower priority when this network has NO free connections
+# (see option NETWORK_MAX_CONNECTIONS in [network_connections] section
+# of this file).
+# NOTE: Following example for NETWORK_MODE option,
+# if in network0.conf has option ONLY_LOCAL_WHEN_POSSIBLE=1, segget
+# will NOT start to use network1 for a particular distfile until all
+# mirrors specified in network0_mirrors.conf file will have failed
+# to provide this distfile.
+# On the other hand if ONLY_LOCAL_WHEN_POSSIBLE=0 segget will start
+# to use network1 as soon as NETWORK_MAX_CONNECTIONS limit, set
+# in network0.conf file has been reached.
+# Default:
+# only_local_when_possible=1
+only_local_when_possible=1 \ No newline at end of file
diff --git a/segget/networkbroker.cpp b/segget/networkbroker.cpp
index 7ac083d..5ef8dbf 100644
--- a/segget/networkbroker.cpp
+++ b/segget/networkbroker.cpp
@@ -28,7 +28,7 @@
void Tnetwork_distfile_broker::init(ulong network_number){
try{
network_num=network_number;
- if (network_array[network_num].use_own_mirror_list_only_on){
+ if (network_array[network_num].network_mode==MODE_LOCAL){
// create flags for each mirror from this network
mirror_fails_vector.insert(mirror_fails_vector.begin(),network_array[network_num].benchmarked_mirror_list.size(),false);
}
diff --git a/segget/segget.conf b/segget/segget.conf
index 71c37f4..8b8217b 100644
--- a/segget/segget.conf
+++ b/segget/segget.conf
@@ -76,7 +76,7 @@ max_tries=10
# Maximum value: 20
# Default:
# max_connections=10
-max_connections=7
+max_connections=6
# CURRENT_SPEED_TIME_INTERVAL_MSECS
# segget transfers may have bursty nature of their traffic. Therefore, while
@@ -195,14 +195,14 @@ network0_priority=10
# Define network specific settings in network1.conf
# Default:
# network1_priority=0
-network1_priority=10
+network1_priority=9
# NETWORK2_PRIORITY
# Description: same as for NETWORK0_PRIORITY
# Define network specific settings in network2.conf
# Default:
# network2_priority=0
-network2_priority=0
+network2_priority=8
# NETWORK3_PRIORITY
# Description: same as for NETWORK0_PRIORITY
@@ -267,22 +267,6 @@ user_agent=segget
# NOT IMPLEMENTED YET: ftp-user=anonymous
# NOT IMPLEMENTED YET: ftp-password=me@mail.ru
-[proxy-fetcher]
-# Specify proxy-fetcher address and port
-# NOT IMPLEMENTED YET: proxy-fetcher-ip=none
-# NOT IMPLEMENTED YET: proxy-fetcher-port=none
-# Specify the user and password for authentication on a proxy-fetcher-server.
-# proxy-fetcher-user=user
-# proxy-fetcher-password=password
-
-# Set to forbid direct connections to Internet servers if it's possible to use
-# proxy-fetcher.
-# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-if-possible=1
-# Set to forbid direct connections to Internet servers.
-# NOT IMPLEMENTED YET: use-proxy-fetcher-demon-only=0
-# Set to forbid using proxy-fetcher.
-# NOT IMPLEMENTED YET: no-proxy-fetcher
-
[logs]
# LOGS_DIR
# Define a dir to store log files.
diff --git a/segget/segget.cpp b/segget/segget.cpp
index edb6b0f..315ff35 100644
--- a/segget/segget.cpp
+++ b/segget/segget.cpp
@@ -27,6 +27,7 @@
#include "segget.h"
int load_pkgs(){
+ debug("uuuuuuuuuuuuuummmmmmmmmmmmmmmm");
try{
ifstream json_pkg_list_file;
@@ -102,27 +103,37 @@ int choose_segment(uint connection_num){
while (pkg_num<stats.pkg_count){
// debug("pkg_num:"+toString(pkg_num));
while(distfile_num<Ppkg_array[pkg_num]->distfile_count){
- if (not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->downloaded)){
+ if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->allows_new_actions()){
+ debug("Distfile "+Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->name+" allows new connections");
// debug(" distfile_num:"+toString(distfile_num));
- if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile)
+ if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile){
+ debug("max_connection limit has not been reached");
+ debug("segment_num:"+toString(segment_num));
+ debug("segment_count:"+toString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->segments_count));
while (segment_num<Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->segments_count){
+ debug("segment_num:"+toString(segment_num));
// debug(" segment_num:"+toString(segment_num));
// segments_in_progress[connection_num]=
// if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num);
- if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==WAITING){
- Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num);
- return 0; // success segment.max_tries has not been reached
+ debug("Let's get segment status"+statusToString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status));
+ if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){
+ if ( ! Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){
+ return 0; // success
+ };
+ }else{
+ debug("status is not SWAITING - go for the next segment");
}
- else
- segment_num++; // segment already downloaded/downloading => go for the next one
+ // haven't managed to provide this segment, go for the next one
+ segment_num++;
}
- else
+ }else{
debug(" distfile "+Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->name+" has "
+toString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_num)
+" connections => choosing another distfile.");
- segment_num=0;
+ }
}
distfile_num++;
+ segment_num=0;
}
distfile_num=0;
pkg_num++;
@@ -158,65 +169,90 @@ int download_pkgs(){
return EXIT_FAILURE;
}
- for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) {
- if (choose_segment(connection_num))
- break;
- };
- while (U) {
- while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){};
- if (U) {
- FD_ZERO(&R);
- FD_ZERO(&W);
- FD_ZERO(&E);
- if (curl_multi_fdset(cm, &R, &W, &E, &M)) {
- error_log("Error: curl_multi_fdset");
-// fprintf(stderr, "E: curl_multi_fdset\n");
- return EXIT_FAILURE;
- }
- if (curl_multi_timeout(cm, &L)) {
- error_log("Error: curl_multi_timeout");
-// fprintf(stderr, "E: curl_multi_timeout\n");
- return EXIT_FAILURE;
- }
- if (L == -1)
- L = 100;
- if (M == -1) {
- #ifdef WIN32
- Sleep(L);
- #else
- sleep(L / 1000);
- #endif
- } else {
- T.tv_sec = L/1000;
- T.tv_usec = (L%1000)*1000;
- if (0 > select(M+1, &R, &W, &E, &T)) {
-// fprintf(stderr, "E: select(%i,,,,%li): %i: %s\n",
- error_log("Error: select ("+toString(M+1)+","+toString(L)+"):"+toString(errno)+": "+strerror(errno));
+// for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) {
+// if ( ! connection_array[connection_num].active){
+// if (choose_segment(connection_num))
+// break;
+// }
+// };
+ bool keep_running_flag=true;
+ while (keep_running_flag){
+ U=1;
+ while (U) {
+ // Use free connections to download segments connections
+ for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) {
+ debug("connection_num:"+toString(connection_num));
+ if ( ! connection_array[connection_num].active){
+// if (
+ debug("connection is not active - choosing segment");
+ choose_segment(connection_num);
+// )
+// break;
+ }
+ else{
+ debug("connection is active");
+ }
+ };
+ while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){};
+ if (U) {
+ FD_ZERO(&R);
+ FD_ZERO(&W);
+ FD_ZERO(&E);
+ if (curl_multi_fdset(cm, &R, &W, &E, &M)) {
+ error_log("Error: curl_multi_fdset");
+// fprintf(stderr, "E: curl_multi_fdset\n");
+ return EXIT_FAILURE;
+ }
+ if (curl_multi_timeout(cm, &L)) {
+ error_log("Error: curl_multi_timeout");
+// fprintf(stderr, "E: curl_multi_timeout\n");
return EXIT_FAILURE;
}
+ if (L == -1)
+ L = 100;
+ if (M == -1) {
+ #ifdef WIN32
+ Sleep(L);
+ #else
+ sleep(L / 1000);
+ #endif
+ } else {
+ T.tv_sec = L/1000;
+ T.tv_usec = (L%1000)*1000;
+ if (0 > select(M+1, &R, &W, &E, &T)) {
+// fprintf(stderr, "E: select(%i,,,,%li): %i: %s\n",
+ error_log("Error: select ("+toString(M+1)+","+toString(L)+"):"+toString(errno)+": "+strerror(errno));
+ return EXIT_FAILURE;
+ }
+ }
}
- }
- while ((msg = curl_multi_info_read(cm, &Q))) {
- if (msg->msg == CURLMSG_DONE) {
- Tsegment *current_segment;
- CURL *e = msg->easy_handle;
- curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &current_segment);
- uint connection_result=msg->data.result;
- string result_msg_text="RESULT:"+toString(connection_result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment";
- msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text);
- curl_multi_remove_handle(cm, e);
-
- connection_array[current_segment->connection_num].stop(connection_result);
-
- if (not choose_segment(current_segment->connection_num)) {
- U++; // just to prevent it from remaining at 0 if there are more URLs to get
- };
- stats.show_totals();
- curl_easy_cleanup(e);
- }else {
- msg_error("ERROR: CURLMsg: "+msg->msg);
+ while ((msg = curl_multi_info_read(cm, &Q))) {
+ if (msg->msg == CURLMSG_DONE) {
+ Tsegment *current_segment;
+ CURL *e = msg->easy_handle;
+ curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &current_segment);
+ uint connection_result=msg->data.result;
+ string result_msg_text="RESULT:"+toString(connection_result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment";
+ msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text);
+ curl_multi_remove_handle(cm, e);
+
+ connection_array[current_segment->connection_num].stop(connection_result);
+
+ if (not choose_segment(current_segment->connection_num)) {
+ U++; // just to prevent it from remaining at 0 if there are more URLs to get
+ };
+ stats.show_totals();
+ curl_easy_cleanup(e);
+ }else {
+ error_log("ERROR: CURLMsg: "+msg->msg);
+ }
}
}
+ debug("bbbbbblllllllllllllaaaaaaaaaaaaa");
+ // nothing to download - wait 5 secs and check upper_proxy_fetchers again
+ // later will be replaced by server_waiting_for_incoming connections and new tasks
+ // with timeout to check downloads from upper proxy fetcher
+ sleep(1);
}
curl_multi_cleanup(cm);
curl_global_cleanup();
@@ -228,6 +264,8 @@ int download_pkgs(){
}
}
+void *print_message_function( void *ptr );
+
int main()
{
try{
@@ -263,6 +301,31 @@ int main()
catch(...){
//error while showing stats
}
+ pthread_t thread1;
+// , thread2;
+// char *message1 = "Thread 1";
+// char *message2 = "Thread 2";
+ int iret1;
+// int iret2;
+
+ /* Create independent threads each of which will execute function */
+
+ iret1 = pthread_create( &thread1, NULL, print_message_function, (void*) NULL);
+// iret2 = pthread_create( &thread2, NULL, print_message_function, (void*) message2);
+
+ /* Wait till threads are complete before main continues. Unless we */
+ /* wait we run the risk of executing an exit which will terminate */
+ /* the process and all threads before the threads have completed. */
+
+ debug("oooooooooooooo111111111111111111");
+// pthread_join( thread1, NULL);
+ debug("oooooooooooooo2222222222222222222222");
+// pthread_join( thread2, NULL);
+
+// printf("Thread 1 returns: %d\n",iret1);
+// printf("Thread 2 returns: %d\n",iret2);
+
+
try{
download_pkgs();
}
@@ -282,4 +345,24 @@ int main()
//error while ending curses
}
return 0;
+}
+
+void *print_message_function(void *ptr){
+// char *message;
+ char * args;
+ args = (char *) ptr;
+ ulong i=0;
+ while (true){
+ ulong time_diff_msecs=time_left_from(stats.previous_time);
+ if (time_diff_msecs >= settings.current_speed_time_interval_msecs){
+ show_progress(time_diff_msecs);
+ };
+ log("Thread:"+toString(i));
+ //haven't downloaded anything - anyway, show totals
+ stats.show_totals();
+ sleep(1);
+ i++;
+ }
+ return 0;
+// printf("%s \n", message);
} \ No newline at end of file
diff --git a/segget/segget.h b/segget/segget.h
index fc65f31..58a9e57 100644
--- a/segget/segget.h
+++ b/segget/segget.h
@@ -47,6 +47,7 @@
#include "str.cpp"
#include "tui.cpp"
#include "utils.cpp"
+#include <pthread.h>
using namespace std;
diff --git a/segget/segment.cpp b/segget/segment.cpp
index 8e29b9c..6606bc8 100644
--- a/segget/segment.cpp
+++ b/segget/segment.cpp
@@ -26,6 +26,15 @@
#include "segment.h"
+string statusToString(Tsegment_status the_status){
+ switch (the_status){
+ case SWAITING:return "SWAITING";
+ case SDOWNLOADING:return "SDOWNLOADING";
+ case SDOWNLOADED:return "SDOWNLOADED";
+ case SFAILED:return "SFAILED";
+ default :return "UNKNOWN STATUS";
+ }
+}
void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end){
try{
parent_distfile=prnt_distfile;
@@ -44,7 +53,7 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi
debug("seg:"+toString(seg_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size));
file.close();
if (downloaded_size==segment_size){
- status=DOWNLOADED;
+ status=SDOWNLOADED;
debug("seg:"+toString(seg_num)+" Downloaded");
}
else{
@@ -58,8 +67,8 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi
}
void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num){
try{
- debug("NETWORK:"+toString(network_num)+(network_array[network_num].use_own_mirror_list_only_on ? " - LOCAL": " - REMOTE"));
- if (network_array[network_num].use_own_mirror_list_only_on){
+// debug("NETWORK:"+toString(network_num)+(network_array[network_num].use_own_mirror_list_only_on ? " - LOCAL": " - REMOTE"));
+ if (network_array[network_num].network_mode==MODE_LOCAL){
url=network_array[network_num].benchmarked_mirror_list[mirror_num].url+parent_distfile->name;
debug(" URL:"+url);
}else{
@@ -67,7 +76,7 @@ void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num,
}
msg_connecting(con_num,distfile_num, segment_num,"Downloading from "+url);
segments_in_progress[con_num]=this;
- status=DOWNLOADING;
+ status=SDOWNLOADING;
downloaded_bytes=0;
connection_num=con_num;
// connection_array[con_num].start(network_num);
@@ -177,11 +186,10 @@ size_t write_data(void *buffer, size_t size, size_t nmemb, void *cur_segment){
error_log("Can't write segment file:"+segment->file_name);
}
connection_array[segment->connection_num].inc_bytes_per_last_interval(bytes_written);
- ulong time_diff_msecs=time_left_from(stats.previous_time);
- if (time_diff_msecs >= settings.current_speed_time_interval_msecs){
- show_progress(time_diff_msecs);
- stats.reset_previous_time();
- };
+// ulong time_diff_msecs=time_left_from(stats.previous_time);
+// if (time_diff_msecs >= settings.current_speed_time_interval_msecs){
+// show_progress(time_diff_msecs);
+// };
}catch(...){
error_log("Error in segment.cpp: write_data()");
return 0;
diff --git a/segget/segment.h b/segget/segment.h
index deb55b5..0c0ba83 100644
--- a/segget/segment.h
+++ b/segget/segment.h
@@ -48,7 +48,7 @@ extern Tsettings settings;
unsigned long downloaded_bytes=0;
size_t write_data(void *buffer, size_t size, size_t nmemb, void *cur_segment);
-enum Tstatus{WAITING, DOWNLOADING, DOWNLOADED, FAILED};
+enum Tsegment_status{SWAITING, SDOWNLOADING, SDOWNLOADED, SFAILED};
class Tsegment{
private:
@@ -56,7 +56,7 @@ class Tsegment{
char* urllist;
public:
string file_name;
- Tstatus status;
+ Tsegment_status status;
uint try_num;
Tdistfile* parent_distfile;
uint connection_num;
@@ -70,7 +70,7 @@ class Tsegment{
easyhandle(0),
urllist(0),
file_name(""),
- status(WAITING),
+ status(SWAITING),
try_num(0),
parent_distfile(0),
connection_num(0),
diff --git a/segget/stats.cpp b/segget/stats.cpp
index 3683c5c..cfbe66c 100644
--- a/segget/stats.cpp
+++ b/segget/stats.cpp
@@ -63,6 +63,7 @@ void Tstats::show_totals(){
// +" Secs:"+toString(now_timee.tv_sec)
// +" usecs:"+toString(now_timee.tv_usec)
);
+ reset_previous_time();
}catch(...){
error_log("Error in stats.cpp: show_totals()");
}
diff --git a/segget/tui.cpp b/segget/tui.cpp
index 3b880ae..341be4d 100644
--- a/segget/tui.cpp
+++ b/segget/tui.cpp
@@ -29,14 +29,19 @@ extern Tsettings settings;
const uint CONNECTION_LINES=5;
+bool msg_idle=true;
void msg(uint y, uint x, string msg_text){
- try{
- move(y,x);
- string ready_msg_text=msg_text+" ";
- printw(ready_msg_text.c_str());
- refresh();
- }catch(...){
- error_log_no_msg("Error in tui.cpp: msg()");
+ if (msg_idle){
+ msg_idle=false;
+ try{
+ move(y,x);
+ string ready_msg_text=msg_text+" ";
+ printw(ready_msg_text.c_str());
+ refresh();
+ }catch(...){
+ error_log_no_msg("Error in tui.cpp: msg()");
+ }
+ msg_idle=true;
}
}