diff options
author | Shyam Mani <fox2mike@gentoo.org> | 2008-12-21 04:11:59 +0800 |
---|---|---|
committer | Shyam Mani <fox2mike@gentoo.org> | 2008-12-21 04:11:59 +0800 |
commit | 457f231f50980fbd5029bd3f1addd74e51442c0c (patch) | |
tree | 5136ddc490c65ef15b3a3b362056b75798ec68c6 /gard-v2.py | |
download | gard-457f231f50980fbd5029bd3f1addd74e51442c0c.tar.gz gard-457f231f50980fbd5029bd3f1addd74e51442c0c.tar.bz2 gard-457f231f50980fbd5029bd3f1addd74e51442c0c.zip |
Initial version
Diffstat (limited to 'gard-v2.py')
-rwxr-xr-x | gard-v2.py | 279 |
1 files changed, 279 insertions, 0 deletions
diff --git a/gard-v2.py b/gard-v2.py new file mode 100755 index 0000000..3a623fc --- /dev/null +++ b/gard-v2.py @@ -0,0 +1,279 @@ +#!/usr/bin/python +# Header : $Header: /usr/local/cvsroot/shyam/scripts/gard-v2.py,v 1.2 2008/08/27 10:43:01 shyam Exp $ +# Purpose : Script to automatically check new Gentoo rsync and distfile mirrors +# Author : Shyam Mani <fox2mike@gentoo.org> +# Created : 05 Jul 2008 + +# Dedicated to robbat2 aka Robin, for sparking off the idea ;) + +import sys +import time +import os +import datetime +import subprocess + +rmaxlag=45 +dmaxlag=5.5 +relmaxlag=2 + +reportpath="/home/fox2mike/gentoo/mirrors/gard/reports" +bugurl="http://bugs.gentoo.org/buglist.cgi?query_format=advanced&product=Mirrors&status_whiteboard_type=allwordssubstr&status_whiteboard=due&bug_status=ASSIGNED&columnlist=bug_id,opendate,bug_status,resolution,short_desc,status_whiteboard&ctype=csv" +fname="/home/fox2mike/open-mirror-bugs" + +def mirrorcheck(target): + retcode1=subprocess.call(['/usr/bin/wget','-q','-T 30',target]) + if retcode1 > 0: + error1 = "*ERROR* : return value of wget during mirrorcheck was "+str(retcode1) + report.write(error1) + #report.close() + print error1 + return + else: + temp=open('timestamp.chk','r') + ots=temp.readline() + temp.close() + #Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python + t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S ")) + lag=(time.mktime(time.gmtime())-t1)/(60*60) + gmt=time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) + if lag > dmaxlag: + error1 = "*ERROR* : distfiles is over 5.5 hours behind, lag is : "+str(round(lag,2))+" hours"+'\n' + tstamp1 = "Timestamp on Mirror : "+str(ots) + curtime1 = "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl1 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(error1) + report.write(tstamp1) + report.write(curtime1) + report.write(bugurl1) + #report.close() + print error1,tstamp1,curtime1,bugurl1 + else: + info1 = addy+" is in sync, current delta : "+str(round(lag,2))+" hours"+'\n' + tstamp1 = "Timestamp on Mirror : "+str(ots) + curtime1 = "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl1 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(info1) + report.write(tstamp1) + report.write(curtime1) + report.write(bugurl1) + #report.close() + print info1,tstamp1,curtime1,bugurl1 + os.unlink('timestamp.chk') + return + +def relpermcheck(target): + retcode2=subprocess.call(['/usr/bin/wget','-q','-T 30',target]) + if retcode2 < 1: + error2 = "*ERROR* : return value of wget for /releases file perm check was "+str(retcode2)+" which means the file is world accessible!"+'\n' + report.write(error2) + report.close() + print error2 + os.unlink('THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt') + return + +def reltscheck(target): + retcode3=subprocess.call(['/usr/bin/wget','-q','-T 30',target]) + if retcode3 > 0: + error3 = "*ERROR* : return value of wget for release timestamp check was "+str(retcode3)+'\n' + report.write(error3) + #report.close() + print error3 + return + else: + temp=open('timestamp.x','r') + ots=temp.readline() + temp.close() + ts=ots.split(' ') + nots=ts[1]+" "+ts[2]+" "+ts[3]+" "+ts[4]+" "+ts[5]+" " + #Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python + t1=time.mktime(time.strptime(nots, "%a, %d %b %Y %H:%M:%S ")) + lag=(time.mktime(time.gmtime())-t1)/(60*60*24) + gmt=time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) + if lag > relmaxlag: + error3 = "*ERROR* : releases is over 2 days behind, lag is : "+str(round(lag,2))+" days"+'\n' + tstamp3 = "Timestamp on Mirror : "+nots+'\n' + curtime3 = "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl3 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(error3) + report.write(tstamp3) + report.write(curtime3) + report.write(bugurl3) + #report.close() + print error3,tstamp3,curtime3,bugurl3 + else: + info3 = addy+"'s releases dir is in sync, current delta : "+str(round(lag,2))+" days"+'\n' + tstamp3 = "Timestamp on Mirror : "+nots+'\n' + curtime3 = "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl3 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(info3) + report.write(tstamp3) + report.write(curtime3) + report.write(bugurl3) + #report.close() + print info3,tstamp3,curtime3,bugurl3 + os.unlink('timestamp.x') + return + + +retcode5=subprocess.call(['/usr/bin/wget','-q','-T 30', '-O', fname, bugurl]) +file=open(fname,'r') +print "\nInitializing G(entoo)A(utomated)R(sync)D(istfiles) checker script, version 2 (Katrina) - kicking mirror ass for Gentoo \n" + +# Format of CSV = 234083,"2008-08-06 13:01:32","ASSIGNED",,"[rsync,distfiles|all] New mirror Cambrium BV (The Netherlands)","due:2008/09/01;dist:http://mirror.cambrium.nl/pub/os/linux/gentoo/|ftp://mirror.cambrium.nl/pub/os/linux/gentoo/|rsync://mirror.cambrium.nl/gentoo/;portage:mirror.cambrium.nl" + +for line in file: + line=line.replace("\n","") + data=line.split(',') + bugnum=data[0] + swboard=data[5] + if bugnum == "bug_id": + continue + else: + details=swboard.split(';') + duedate=details[0] + duedate=duedate.replace("\"","") + #print duedate + urls=details[1].replace("dist:","") + port=details[2].split(':') + grsync=port[1].replace("\"","") + temp=urls.split('|') + http=temp[0] + ftp=temp[1] + drsync=temp[2] + #print http + #print ftp + #print drsync + #print grsync + if len(http) > 0: + boo=http.split('/') + addy=boo[2] + elif len(ftp) > 0: + boo=ftp.split('/') + addy=boo[2] + else: + addy=grsync + #hdomain='/'+addy[2] + if len(grsync) > 0: + fname=reportpath+"/"+grsync + if os.path.isdir(reportpath): + pass + else: + os.makedirs(reportpath) + report=open(fname,'a+') + info4=grsync+" wants to be a gentoo-portage mirror and is being checked as per Bug #"+bugnum+'\n' + report.write(info4) + print info4 + target=grsync+"::gentoo-portage/metadata/timestamp.chk" + retcode=subprocess.call(['rsync','-aqP','--no-motd','--contimeout=30',target,'.']) + if retcode > 0: + error4="*ERROR* : return value of rsync during gentoo-portage check was "+str(retcode) + report.write(error4) + report.close() + print error4 + else: + temp=open('timestamp.chk','r') + ots=temp.readline() + temp.close() + #Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python + t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S ")) + lag=(time.mktime(time.gmtime())-t1)/60 + if lag > rmaxlag: + error4 = "*ERROR* : gentoo-portage mirror is lagging by over 40 mins, lag is : "+str(round(lag,2))+" mins"+'\n' + tstamp4 = "Timestamp on Mirror : "+str(ots) + curtime4 = "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl4 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(error4) + report.write(tstamp4) + report.write(curtime4) + report.write(bugurl4) + report.close() + print error4,tstamp4,curtime4,bugurl4 + else: + info4 = grsync+" is in sync, current delta : "+str(round(lag,2))+" mins"+'\n' + tstamp4 = "Timestamp on Mirror : "+str(ots) + curtime4= "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl4 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(info4) + report.write(tstamp4) + report.write(curtime4) + report.write(bugurl4) + report.close() + print info4,tstamp4,curtime4,bugurl4 + os.unlink('timestamp.chk') + if len(http) > 0: + fname=reportpath+"/"+addy + if os.path.isdir(reportpath): + pass + else: + os.makedirs(reportpath) + report=open(fname,'a+') + info5 = addy+" wants to be distfiles mirror and is being checked as per Bug #"+bugnum+ " over http"+'\n' + report.write(info5) + print info5 + disttsdest=http+"/distfiles/timestamp.chk" + mirrorcheck(disttsdest) + reltsdest=http+"/releases/.test/timestamp.x" + reltscheck(reltsdest) + relpermdest=http+"/releases/.test/THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt" + relpermcheck(relpermdest) + if len(ftp) > 0: + fname=reportpath+"/"+addy + if os.path.isdir(reportpath): + pass + else: + os.makedirs(reportpath) + report=open(fname,'a+') + info6 = addy+" wants to be an distfiles mirror and is being checked as per Bug #"+bugnum+ " over ftp"+'\n' + report.write(info6) + print info6 + ftpdest=ftp+"/distfiles/timestamp.chk" + mirrorcheck(ftpdest) + reltsdest=ftp+"/releases/.test/timestamp.x" + reltscheck(reltsdest) + relpermdest=ftp+"/releases/.test/THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt" + relpermcheck(relpermdest) + if len(drsync) > 0: + fname=reportpath+"/"+addy + if os.path.isdir(reportpath): + pass + else: + os.makedirs(reportpath) + report=open(fname,'a+') + info7 = addy+" wants to be an distfiles mirror and is being checked as per Bug #"+bugnum+ " over rsync"+'\n' + report.write(info7) + print info7 + target=drsync+"/distfiles/timestamp.chk" + retcode4=subprocess.call(['rsync','-aqP','--no-motd','--contimeout=30',target,'.']) + if retcode4 > 0: + error7 = "*ERROR* : return value of rsync during distfiles check was "+str(retcode4) + report.write(error7) + print error7 + else: + temp=open('timestamp.chk','r') + ots=temp.readline() + temp.close() + t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S ")) + lag=(time.mktime(time.gmtime())-t1)/(60*60) + if lag > dmaxlag: + error7 = "*ERROR* : Mirror is lagging by over 5.5 hours, difference is : "+str(round(lag,2))+" hours"+'\n' + tstamp7 = "Timestamp on Mirror : "+str(ots) + curtime7= "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl7 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(error7) + report.write(tstamp7) + report.write(curtime7) + report.write(bugurl7) + report.close() + print error7,tstamp7,curtime7,bugurl7 + else: + info7 = addy+" is in sync, current lag : "+str(round(lag,2))+" hours"+'\n' + tstamp7 = "Timestamp on Mirror : "+str(ots) + curtime7 = "Current time in UTC : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n' + bugurl7 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n' + report.write(info7) + report.write(tstamp7) + report.write(curtime7) + report.write(bugurl7) + report.close() + print info7,tstamp7,curtime7,bugurl7 + os.unlink('timestamp.chk') |