path: root/gard-v2.py

#!/usr/bin/python
# Header   : $Header: /usr/local/cvsroot/shyam/scripts/gard-v2.py,v 1.2 2008/08/27 10:43:01 shyam Exp $
# Purpose  : Script to automatically check new Gentoo rsync and distfile mirrors
# Author   : Shyam Mani <fox2mike@gentoo.org>
# Created  : 05 Jul 2008

# Dedicated to robbat2 aka Robin, for sparking off the idea ;)

import sys
import time
import os
import datetime
import subprocess

rmaxlag=45
dmaxlag=5.5
relmaxlag=2

reportpath="/home/fox2mike/gentoo/mirrors/gard/reports"
bugurl="http://bugs.gentoo.org/buglist.cgi?query_format=advanced&product=Mirrors&status_whiteboard_type=allwordssubstr&status_whiteboard=due&bug_status=ASSIGNED&columnlist=bug_id,opendate,bug_status,resolution,short_desc,status_whiteboard&ctype=csv"
fname="/home/fox2mike/open-mirror-bugs"

def mirrorcheck(target):
	retcode1=subprocess.call(['/usr/bin/wget','-q','-T 30',target])
	if retcode1 > 0:
		error1 = "*ERROR* : return value of wget during mirrorcheck was "+str(retcode1)
		report.write(error1)
		#report.close()
		print error1
		return
	else:
		temp=open('timestamp.chk','r')
		ots=temp.readline()
		temp.close()
		#Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python
		t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S "))
		lag=(time.mktime(time.gmtime())-t1)/(60*60)
		gmt=time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
		if lag > dmaxlag:
			error1 = "*ERROR* : distfiles is over 5.5 hours behind, lag is : "+str(round(lag,2))+" hours"+'\n'
			tstamp1 = "Timestamp on Mirror   : "+str(ots)
			curtime1 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
			bugurl1 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
			report.write(error1)
			report.write(tstamp1)
			report.write(curtime1)
			report.write(bugurl1)
			#report.close()
			print error1,tstamp1,curtime1,bugurl1
		else:
			info1 = addy+" is in sync, current delta : "+str(round(lag,2))+" hours"+'\n'
			tstamp1 = "Timestamp on Mirror   : "+str(ots)
			curtime1 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
			bugurl1 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
			report.write(info1)
			report.write(tstamp1)
			report.write(curtime1)
			report.write(bugurl1)
			#report.close()
			print info1,tstamp1,curtime1,bugurl1
		os.unlink('timestamp.chk')
	return

def relpermcheck(target):
	retcode2=subprocess.call(['/usr/bin/wget','-q','-T 30',target])
	if retcode2 < 1:
		error2 = "*ERROR* : return value of wget for /releases file perm check was "+str(retcode2)+" which means the file is world accessible!"+'\n'
		report.write(error2)
		report.close()
		print error2
		os.unlink('THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt')
	return

def reltscheck(target):
	retcode3=subprocess.call(['/usr/bin/wget','-q','-T 30',target])
	if retcode3 > 0:
		error3 = "*ERROR* : return value of wget for release timestamp check was "+str(retcode3)+'\n'
		report.write(error3)
		#report.close()
		print error3
		return
	else:
		temp=open('timestamp.x','r')
		ots=temp.readline()
		temp.close()
		ts=ots.split(' ')
		nots=ts[1]+" "+ts[2]+" "+ts[3]+" "+ts[4]+" "+ts[5]+" "
		#Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python
		t1=time.mktime(time.strptime(nots, "%a, %d %b %Y %H:%M:%S "))
		lag=(time.mktime(time.gmtime())-t1)/(60*60*24)
		gmt=time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
		if lag > relmaxlag:
			error3 = "*ERROR* : releases is over 2 days behind, lag is : "+str(round(lag,2))+" days"+'\n'
			tstamp3 = "Timestamp on Mirror   : "+nots+'\n'
			curtime3 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
			bugurl3 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
			report.write(error3)
			report.write(tstamp3)
			report.write(curtime3)
			report.write(bugurl3)
			#report.close()
			print error3,tstamp3,curtime3,bugurl3
		else:
			info3 = addy+"'s releases dir is in sync, current delta : "+str(round(lag,2))+" days"+'\n'
			tstamp3 = "Timestamp on Mirror   : "+nots+'\n'
			curtime3 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
			bugurl3 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
			report.write(info3)
			report.write(tstamp3)
			report.write(curtime3)
			report.write(bugurl3)
			#report.close()
			print info3,tstamp3,curtime3,bugurl3
		os.unlink('timestamp.x')
	return


retcode5=subprocess.call(['/usr/bin/wget','-q','-T 30', '-O', fname, bugurl])
file=open(fname,'r')
print "\nInitializing G(entoo)A(utomated)R(sync)D(istfiles) checker script, version 2 (Katrina) - kicking mirror ass for Gentoo \n"

# Format of CSV = 234083,"2008-08-06 13:01:32","ASSIGNED",,"[rsync,distfiles|all] New mirror Cambrium BV (The Netherlands)","due:2008/09/01;dist:http://mirror.cambrium.nl/pub/os/linux/gentoo/|ftp://mirror.cambrium.nl/pub/os/linux/gentoo/|rsync://mirror.cambrium.nl/gentoo/;portage:mirror.cambrium.nl"

for line in file:
	line=line.replace("\n","")
	data=line.split(',')
	bugnum=data[0]
	swboard=data[5]
	if bugnum == "bug_id":
		continue
	else:
		details=swboard.split(';')
		duedate=details[0]
		duedate=duedate.replace("\"","")
		#print duedate
		urls=details[1].replace("dist:","")
		port=details[2].split(':')
		grsync=port[1].replace("\"","")
		temp=urls.split('|')
		http=temp[0]
		ftp=temp[1]
		drsync=temp[2]
		#print http
		#print ftp
		#print drsync
		#print grsync
		if len(http) > 0:
			boo=http.split('/')
			addy=boo[2]
		elif len(ftp) > 0:
			boo=ftp.split('/')
			addy=boo[2]
		else:
			addy=grsync
		#hdomain='/'+addy[2]
		if len(grsync) > 0:
			fname=reportpath+"/"+grsync
			if os.path.isdir(reportpath):
				pass
	        	else:
        	        	os.makedirs(reportpath)
			report=open(fname,'a+')
			info4=grsync+" wants to be a gentoo-portage mirror and is being checked as per Bug #"+bugnum+'\n'
			report.write(info4)
			print info4
			target=grsync+"::gentoo-portage/metadata/timestamp.chk"
			retcode=subprocess.call(['rsync','-aqP','--no-motd','--contimeout=30',target,'.'])
			if retcode > 0:
				error4="*ERROR* : return value of rsync during gentoo-portage check was "+str(retcode)
				report.write(error4)
				report.close()
				print error4
			else:	
				temp=open('timestamp.chk','r')
				ots=temp.readline()
				temp.close()
				#Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python
				t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S "))
				lag=(time.mktime(time.gmtime())-t1)/60
				if lag > rmaxlag:
					error4 = "*ERROR* : gentoo-portage mirror is lagging by over 40 mins, lag is : "+str(round(lag,2))+" mins"+'\n'
					tstamp4 = "Timestamp on Mirror   : "+str(ots)
					curtime4 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
					bugurl4 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
					report.write(error4)
					report.write(tstamp4)
					report.write(curtime4)
					report.write(bugurl4)
					report.close()
					print error4,tstamp4,curtime4,bugurl4
				else:
					info4 = grsync+" is in sync, current delta : "+str(round(lag,2))+" mins"+'\n'
					tstamp4 = "Timestamp on Mirror   : "+str(ots)
					curtime4= "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
					bugurl4 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
					report.write(info4)
					report.write(tstamp4)
					report.write(curtime4)
					report.write(bugurl4)
					report.close()
					print info4,tstamp4,curtime4,bugurl4
				os.unlink('timestamp.chk')
		if len(http) > 0:
			fname=reportpath+"/"+addy
			if os.path.isdir(reportpath):
      				pass
        		else:
       				os.makedirs(reportpath)
			report=open(fname,'a+')
			info5 = addy+" wants to be distfiles mirror and is being checked as per Bug #"+bugnum+ " over http"+'\n'
			report.write(info5)
			print info5
			disttsdest=http+"/distfiles/timestamp.chk"
			mirrorcheck(disttsdest)
			reltsdest=http+"/releases/.test/timestamp.x"
			reltscheck(reltsdest)
			relpermdest=http+"/releases/.test/THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt"
			relpermcheck(relpermdest)
		if len(ftp) > 0:
			fname=reportpath+"/"+addy
			if os.path.isdir(reportpath):
       				pass
       			else:
       				os.makedirs(reportpath)
			report=open(fname,'a+')
			info6 = addy+" wants to be an distfiles mirror and is being checked as per Bug #"+bugnum+ " over ftp"+'\n'
			report.write(info6)
			print info6
			ftpdest=ftp+"/distfiles/timestamp.chk"
			mirrorcheck(ftpdest)
			reltsdest=ftp+"/releases/.test/timestamp.x"
			reltscheck(reltsdest)
			relpermdest=ftp+"/releases/.test/THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt"
			relpermcheck(relpermdest)
		if len(drsync) > 0:
			fname=reportpath+"/"+addy
			if os.path.isdir(reportpath):
	       			pass
      			else:
               			os.makedirs(reportpath)
			report=open(fname,'a+')
			info7 = addy+" wants to be an distfiles mirror and is being checked as per Bug #"+bugnum+ " over rsync"+'\n'
			report.write(info7)
			print info7
			target=drsync+"/distfiles/timestamp.chk"
			retcode4=subprocess.call(['rsync','-aqP','--no-motd','--contimeout=30',target,'.'])
			if retcode4 > 0:
				error7 = "*ERROR* : return value of rsync during distfiles check was "+str(retcode4)
				report.write(error7)
				print error7
			else:
				temp=open('timestamp.chk','r')
				ots=temp.readline()
				temp.close()
				t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S "))
				lag=(time.mktime(time.gmtime())-t1)/(60*60)
				if lag > dmaxlag:
					error7 = "*ERROR* : Mirror is lagging by over 5.5 hours, difference is : "+str(round(lag,2))+" hours"+'\n'
					tstamp7 = "Timestamp on Mirror   : "+str(ots)
					curtime7= "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
					bugurl7 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
					report.write(error7)
					report.write(tstamp7)
					report.write(curtime7)
					report.write(bugurl7)
					report.close()
					print error7,tstamp7,curtime7,bugurl7
				else:
					info7 = addy+" is in sync, current lag : "+str(round(lag,2))+" hours"+'\n'
					tstamp7 = "Timestamp on Mirror   : "+str(ots)
					curtime7 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
					bugurl7 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
					report.write(info7)
					report.write(tstamp7)
					report.write(curtime7)
					report.write(bugurl7)
					report.close()
					print info7,tstamp7,curtime7,bugurl7
				os.unlink('timestamp.chk')