diff options
author | Jeremy Olexa <darkside@gentoo.org> | 2010-05-22 17:57:33 -0500 |
---|---|---|
committer | Jeremy Olexa <darkside@gentoo.org> | 2010-05-22 17:57:33 -0500 |
commit | 63d924470c8daa3333fc3f29d196fc5d240c6f72 (patch) | |
tree | b8296adf61cd583843df44964a214cfa614c4258 | |
download | gentoo-mirrorstats-63d924470c8daa3333fc3f29d196fc5d240c6f72.tar.gz gentoo-mirrorstats-63d924470c8daa3333fc3f29d196fc5d240c6f72.tar.bz2 gentoo-mirrorstats-63d924470c8daa3333fc3f29d196fc5d240c6f72.zip |
Initial commit from local disk on magpie.gentoo.org. Credit to Jeremy Olexa and Robin Johnson
-rwxr-xr-x | mirmon-portage.sh | 10 | ||||
-rwxr-xr-x | mirmon-source.sh | 10 | ||||
-rwxr-xr-x | portage_mirrors/get-mirror-list-portage.rb | 16 | ||||
-rw-r--r-- | portage_mirrors/mirmon.conf | 23 | ||||
-rwxr-xr-x | portage_mirrors/probe-mirmon-portage | 60 | ||||
-rwxr-xr-x | source_mirrors/get-mirror-list-source.rb | 16 | ||||
-rw-r--r-- | source_mirrors/mirmon.conf | 23 | ||||
-rwxr-xr-x | source_mirrors/probe-mirmon-source | 61 |
8 files changed, 219 insertions, 0 deletions
diff --git a/mirmon-portage.sh b/mirmon-portage.sh new file mode 100755 index 0000000..be1d8b6 --- /dev/null +++ b/mirmon-portage.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +cd /var/www/mirrorstats.gentoo.org/portage_mirrors +./get-mirror-list-portage.rb > ./var/g.mirrors +../common/mirmon -c ./mirmon.conf -q -get update +# Set up a nice link to our mirror page directly: +sed \ + -e 's#mirrors</H2>#<a href="http://www.gentoo.org/main/en/mirrors-rsync.xml">mirrors</a></H2>#' \ + >../htdocs/rsync/index.html <../htdocs/rsync/index-wip.html + diff --git a/mirmon-source.sh b/mirmon-source.sh new file mode 100755 index 0000000..20d02a1 --- /dev/null +++ b/mirmon-source.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +cd /var/www/mirrorstats.gentoo.org/source_mirrors +./get-mirror-list-source.rb > ./var/g.mirrors +../common/mirmon -c ./mirmon.conf -q -get update +# Set up a nice link to our mirror page directly: +sed \ + -e 's#mirrors</H2>#<a href="http://www.gentoo.org/main/en/mirrors2.xml">mirrors</a></H2>#' \ + >../htdocs/index.html <../htdocs/index-wip.html + diff --git a/portage_mirrors/get-mirror-list-portage.rb b/portage_mirrors/get-mirror-list-portage.rb new file mode 100755 index 0000000..7269ece --- /dev/null +++ b/portage_mirrors/get-mirror-list-portage.rb @@ -0,0 +1,16 @@ +#!/usr/bin/ruby + +%w[ rexml/document net/http ].each {|lib| require lib } + +m = Net::HTTP.get('www.gentoo.org', '/main/en/mirrors-rsync-data.xml?passthru=1') +x = REXML::Document.new(m) + +REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| + country = el.attributes['country'] + + el.each_element('mirror/uri/') do |mirror| + puts "#{country.downcase} #{mirror[0].to_s}" if mirror[0].to_s =~ /rsync\d+/ + end + +} + diff --git a/portage_mirrors/mirmon.conf b/portage_mirrors/mirmon.conf new file mode 100644 index 0000000..c22caf4 --- /dev/null +++ b/portage_mirrors/mirmon.conf @@ -0,0 +1,23 @@ +project_name Gentoo Linux +web_page /var/www/mirrorstats.gentoo.org/htdocs/rsync/index-wip.html +icons ../icons +countries /var/www/mirrorstats.gentoo.org/common/countries.list +mirror_list /var/www/mirrorstats.gentoo.org/portage_mirrors/var/g.mirrors +state /var/www/mirrorstats.gentoo.org/portage_mirrors/var/mirmon.state +probe /var/www/mirrorstats.gentoo.org/portage_mirrors/probe-mirmon-portage %TIMEOUT% %URL%gentoo-portage/metadata/timestamp +project_url http://www.gentoo.org/ +project_logo http://www.gentoo.org/images/gentoo-new.gif + + +# Fresh: 0 - 1h +# Oldish: 8 - 24h +# Old: 24h - inf +# Bad: No reachable, no state saved + +max_poll 60m +min_sync 30m +max_sync 30m + +put_histo top +timeout 30 + diff --git a/portage_mirrors/probe-mirmon-portage b/portage_mirrors/probe-mirmon-portage new file mode 100755 index 0000000..2709319 --- /dev/null +++ b/portage_mirrors/probe-mirmon-portage @@ -0,0 +1,60 @@ +#!/usr/bin/perl +# $Id: probe-mirmon,v 1.4 2009/08/19 23:15:46 karl Exp $ +# public domain. Originally written by Karl Berry, 2009. +# +# Probe rsync url's for mirmon; use wget for anything else. +# From description at http://people.cs.uu.nl/henkp/mirmon. +# +# Also requires a patch to mirmon itself to accept rsync urls +# (and I wanted https too): +# --- /usr/local/share/mirmon/ORIG/mirmon 2007-08-18 18:05:47.000000000 +0200 +# +++ /usr/local/share/mirmon/mirmon 2009-07-03 22:38:00.000000000 +0200 +# @@ -386,3 +386,3 @@ +# my ( $type, $site, $home ) ; +# - if ( $url =~ m!^(ftp|http)://([^/:]+)(:\d+)?/! ) +# + if ( $url =~ m!^(ftp|https?|rsync)://([^/:]+)(:\d+)?/! ) +# { $type = $1 ; $site = $2 ; $home = $& ; } + +main(@ARGV); + +use Date::Parse (); # dev-perl/TimeDate +use File::Tempdir; # dev-perl/File-Tempdir + +sub main { + my ( $timeout, $url ) = @_; + if ( $url =~ m,^rsync://, ) { + handle_rsync( $timeout, $url ); + } + else { + handle_wget( $timeout, $url ); + } +} + +sub handle_wget { + my ( $timeout, $url ) = @_; + exec {'/usr/bin/wget'} 'wget', qw( -q --passive-ftp -O - -T ), $timeout, '-t', 1, $url; +} + +sub handle_rsync { + my ( $timeout, $url ) = @_; + + my $tmpdir = File::Tempdir->new(); + my $dir = $tmpdir->name; + my $file = $url; + + $file =~ s/\W/_/g; # translate all non-letters to _ + + if ( my $fail = system '/usr/bin/rsync', qw( --no-motd --timeout ), $timeout, $url, "$dir/$file" ) { + warn "rsync failed, exit code $fail, $! $? $@\n"; + exit $fail; + } + + open my $fh, '<', "$dir/$file" or do { + warn "Opening Downloaded timestamp Failed"; + exit 900; # rediculous exit code. + }; + + print Date::Parse::str2time(<$fh>); + exit 0; + +} diff --git a/source_mirrors/get-mirror-list-source.rb b/source_mirrors/get-mirror-list-source.rb new file mode 100755 index 0000000..1aa22ea --- /dev/null +++ b/source_mirrors/get-mirror-list-source.rb @@ -0,0 +1,16 @@ +#!/usr/bin/ruby + +%w[ rexml/document net/http ].each {|lib| require lib } + +m = Net::HTTP.get('www.gentoo.org', '/main/en/mirrors3.xml?passthru=1') +x = REXML::Document.new(m) + +REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| + country = el.attributes['country'] + + el.each_element('mirror/uri/') do |mirror| + puts "#{country.downcase} #{mirror[0].to_s}" + end + +} + diff --git a/source_mirrors/mirmon.conf b/source_mirrors/mirmon.conf new file mode 100644 index 0000000..93bfd13 --- /dev/null +++ b/source_mirrors/mirmon.conf @@ -0,0 +1,23 @@ +project_name Gentoo Linux +web_page /var/www/mirrorstats.gentoo.org/htdocs/index-wip.html +icons ../icons +countries /var/www/mirrorstats.gentoo.org/common/countries.list +mirror_list /var/www/mirrorstats.gentoo.org/source_mirrors/var/g.mirrors +state /var/www/mirrorstats.gentoo.org/source_mirrors/var/mirmon.state +probe /var/www/mirrorstats.gentoo.org/source_mirrors/probe-mirmon-source %TIMEOUT% %URL%distfiles/timestamp.mirmon +project_url http://www.gentoo.org/ +project_logo http://www.gentoo.org/images/gentoo-new.gif + + +# Fresh: 0 - 8h +# Oldish: 8 - 24h +# Old: 24h - inf +# Bad: No reachable, no state saved + +max_poll 4h +min_sync 4h +max_sync 20h + +put_histo top +timeout 30 + diff --git a/source_mirrors/probe-mirmon-source b/source_mirrors/probe-mirmon-source new file mode 100755 index 0000000..6e8ae52 --- /dev/null +++ b/source_mirrors/probe-mirmon-source @@ -0,0 +1,61 @@ +#!/usr/bin/perl +# $Id: probe-mirmon,v 1.4 2009/08/19 23:15:46 karl Exp $ +# public domain. Originally written by Karl Berry, 2009. +# +# Probe rsync url's for mirmon; use wget for anything else. +# From description at http://people.cs.uu.nl/henkp/mirmon. +# +# Also requires a patch to mirmon itself to accept rsync urls +# (and I wanted https too): +# --- /usr/local/share/mirmon/ORIG/mirmon 2007-08-18 18:05:47.000000000 +0200 +# +++ /usr/local/share/mirmon/mirmon 2009-07-03 22:38:00.000000000 +0200 +# @@ -386,3 +386,3 @@ +# my ( $type, $site, $home ) ; +# - if ( $url =~ m!^(ftp|http)://([^/:]+)(:\d+)?/! ) +# + if ( $url =~ m!^(ftp|https?|rsync)://([^/:]+)(:\d+)?/! ) +# { $type = $1 ; $site = $2 ; $home = $& ; } + +main(@ARGV); + +use Date::Parse (); # dev-perl/TimeDate +use File::Tempdir; # dev-perl/File-Tempdir + +sub main { + my ( $timeout, $url ) = @_; + if ( $url =~ m,^rsync://, ) { + handle_rsync( $timeout, $url ); + } + else { + handle_wget( $timeout, $url ); + } +} + +sub handle_wget { + my ( $timeout, $url ) = @_; + exec {'/usr/bin/wget'} 'wget', qw( -q --passive-ftp -O - -T ), $timeout, '-t', 1, $url; +} + +sub handle_rsync { + my ( $timeout, $url ) = @_; + + my $tmpdir = File::Tempdir->new(); + my $dir = $tmpdir->name; + my $file = $url; + + $file =~ s/\W/_/g; # translate all non-letters to _ + + if ( my $fail = system '/usr/bin/rsync', qw( --no-motd --timeout ), $timeout, $url, "$dir/$file" ) { + warn "rsync failed, exit code $fail, $! $? $@\n"; + exit $fail; + } + + open my $fh, '<', "$dir/$file" or do { + warn "Opening Downloaded timestamp Failed"; + exit 900; # rediculous exit code. + }; + + print <$fh>; +# print Date::Parse::str2time(<$fh>); + exit 0; + +} |