path: root/app-backup/duplicity/files/0.4.2-ftp-retry.patch

--- src/backends.py	2006-02-03 04:44:31.000000000 +0100
+++ src/backends.py.new	2006-10-19 18:41:24.000000000 +0200
@@ -18,8 +18,12 @@
 
 """Provides functions and classes for getting/sending files to destination"""
 
-import os, types, ftplib, tempfile
+import os, types, ftplib, tempfile, time, sys
 import log, path, dup_temp, file_naming
+import socket
+
+# TODO: move into globals?
+socket.setdefaulttimeout(10)
 
 class BackendException(Exception): pass
 class ParsingException(Exception): pass
@@ -110,8 +114,6 @@
 	and delete methods.
 
 	"""
-	def init(self, parsed_url): pass
-
 	def put(self, source_path, remote_filename = None):
 		"""Transfer source_path (Path object) to remote_filename (string)
 
@@ -126,7 +128,7 @@
 		"""Retrieve remote_filename and place in local_path"""
 		local_path.setdata()
 		pass
-	
+
 	def list(self):
 		"""Return list of filenames (strings) present in backend"""
 		pass
@@ -285,7 +287,7 @@
 		local_path.setdata()
 		if not local_path.exists():
 			raise BackendException("File %s not found" % local_path.name)
-		
+
 	def list(self):
 		"""List files available for scp
 
@@ -318,21 +320,72 @@
 
 class ftpBackend(Backend):
 	"""Connect to remote store using File Transfer Protocol"""
+	RETRY_SLEEP = 10 # time in seconds before reconnecting on errors (gets multiplied with the try counter)
+	RETRIES = 15 # number of retries
+
 	def __init__(self, parsed_url):
 		"""Create a new ftp backend object, log in to host"""
+		self.parsed_url = parsed_url
+		self.connect()
+
+	def connect(self):
+		"""Connect to self.parsed_url"""
 		self.ftp = ftplib.FTP()
-		if parsed_url.port is None: self.error_wrap('connect', parsed_url.host)
-		else: self.error_wrap('connect', parsed_url.host, parsed_url.port)
+		self.is_connected = False
+		if self.parsed_url.port is None:
+			self.error_wrap('connect', self.parsed_url.host)
+		else: self.error_wrap('connect', self.parsed_url.host,
+							  self.parsed_url.port)
+		self.is_connected = True
 
-		if parsed_url.user is not None:
-			self.error_wrap('login', parsed_url.user, self.get_password())
+		if self.parsed_url.user is not None:
+			self.error_wrap('login', self.parsed_url.user, self.get_password())
 		else: self.error_wrap('login')
-		self.ftp.cwd(parsed_url.path)
+		self.ftp.cwd(self.parsed_url.path)
 
 	def error_wrap(self, command, *args):
 		"""Run self.ftp.command(*args), but raise BackendException on error"""
-		try: return ftplib.FTP.__dict__[command](self.ftp, *args)
-		except ftplib.all_errors, e: raise BackendException(e)
+
+		# Log FTP command:
+		if command is 'login':
+			if log.verbosity > 8:
+				# Log full args at level 9:
+				log.Log("FTP: %s %s" % (command,args), 9)
+			else:
+				# replace password with stars:
+				log_args = list(args)
+				log_args[1] = '*' * len(log_args[1])
+				log.Log("FTP: %s %s" % (command,log_args), 5)
+		else:
+			log.Log("FTP: %s %s" % (command,args), 5)
+
+		# Execute:
+		tries = 0
+		while( True ):
+			tries = tries+1
+			try:
+				return ftplib.FTP.__dict__[command](self.ftp, *args)
+			except ftplib.all_errors, e:
+				if "450" in str(e) and command == 'nlst':
+					# 450 on list isn't an error, but indicates an empty dir
+					return []
+
+				if tries > self.RETRIES:
+					# Give up:
+					log.FatalError("Catched exception %s%s (%d exceptions in total), giving up.." % (sys.exc_info()[0],sys.exc_info()[1],tries,))
+					raise BackendException(e)
+
+				# Sleep and retry (after trying to reconnect, if possible):
+				sleep_time = self.RETRY_SLEEP * tries;
+				log.Warn("Catched exception %s%s (#%d), sleeping %ds before retry.." % (sys.exc_info()[0],sys.exc_info()[1],tries,sleep_time,))
+				time.sleep(sleep_time)
+				try:
+					if self.is_connected:
+						self.connect()
+					return ftplib.FTP.__dict__[command](self.ftp, *args)
+				except ftplib.all_errors, e:
+					continue
+			else: break
 
 	def get_password(self):
 		"""Get ftp password using environment if possible"""
@@ -364,7 +417,8 @@
 		# Some ftp servers raise error 450 if the directory is empty
 		try: return self.error_wrap('nlst')
 		except BackendException, e:
-			if "450" in str(e): return []
+			if "450" in str(e) or "500" in str(e) or "550" in str(e):
+				return []
 			raise
 
 	def delete(self, filename_list):
@@ -375,7 +429,10 @@
 
 	def close(self):
 		"""Shut down connection"""
-		self.error_wrap('quit')
+		try: self.error_wrap('quit')
+		except BackendException, e:
+			if "104" in str(e): return
+			raise
 
 
 class rsyncBackend(Backend):
@@ -405,7 +462,7 @@
 		local_path.setdata()
 		if not local_path.exists():
 			raise BackendException("File %s not found" % local_path.name)
-		
+
 	def list(self):
 		"""List files"""
 		def split (str):
@@ -447,11 +504,121 @@
 		for file in to_delete:
 			os.unlink (file)
 		os.rmdir (dir)
-		
+
+
+class BitBucketBackend(Backend):
+        """Backend for accessing Amazon S3 using the bitbucket.py module.
+
+        This backend supports access to Amazon S3 (http://aws.amazon.com/s3)
+        using a mix of environment variables and URL's. The access key and
+        secret key are taken from the environment variables S3KEY and S3SECRET
+        and the bucket name from the url. For example (in BASH):
+
+            $ export S3KEY='44CF9590006BF252F707'
+            $ export S3SECRET='OtxrzxIsfpFjA7SwPzILwy8Bw21TLhquhboDYROV'
+            $ duplicity /home/me s3+http://bucket_name
+
+        Note: / is disallowed in bucket names in case prefix support is implemented
+        in future.
+
+        TODO:
+            - support bucket prefixes with url's like s3+http://bucket_name/prefix
+            - bitbucket and amazon s3 are currently not very robust. We provide a
+              simplistic way of trying to re-connect and re-try an operation when
+              it fails. This is just a band-aid and should be removed if bitbucket
+              becomes more robust.
+            - Logging of actions.
+            - Better error messages for failures.
+        """
+
+        def __init__(self, parsed_url):
+                import bitbucket
+                self.module = bitbucket
+                self.bucket_name = parsed_url.suffix
+                if '/' in self.bucket_name:
+                    raise NotImplementedError("/ disallowed in bucket names and "
+                                              "bucket prefixes not supported.")
+                self.access_key = os.environ["S3KEY"]
+                self.secret_key = os.environ["S3SECRET"]
+                self._connect()
+
+        def _connect(self):
+                self.connection = self.module.connect(access_key=self.access_key,
+                                                      secret_key=self.secret_key)
+                self.bucket = self.connection.get_bucket(self.bucket_name)
+                # populate the bitbucket cache we do it here to be sure that
+                # even on re-connect we have a list of all keys on the server
+                self.bucket.fetch_all_keys()
+
+        def _logException(self, message=None):
+                # Simply dump the exception onto stderr since formatting it
+                # ourselves looks dangerous.
+                if message is not None:
+                    sys.stderr.write(message)
+                sys.excepthook(*sys.exc_info())
+
+        def put(self, source_path, remote_filename = None):
+		"""Transfer source_path (Path object) to remote_filename (string)
+
+		If remote_filename is None, get the filename from the last
+		path component of pathname.
+
+		"""
+		if not remote_filename:
+                    remote_filename = source_path.get_filename()
+                bits = self.module.Bits(filename=source_path.name)
+                try:
+                    self.bucket[remote_filename] = bits
+                except:
+                    self._logException("Error sending file %s, attempting to "
+                                       "re-connect.\n Got this Traceback:\n"
+                                       % remote_filename)
+                    self._connect()
+                    self.bucket[remote_filename] = bits
+
+	def get(self, remote_filename, local_path):
+		"""Retrieve remote_filename and place in local_path"""
+		local_path.setdata()
+                try:
+                    bits = self.bucket[remote_filename]
+                    bits.to_file(local_path.name)
+                except:
+                    self._logException("Error getting file %s, attempting to "
+                                       "re-connect.\n Got this Traceback:\n"
+                                       % remote_filename)
+                    self._connect()
+                    bits = self.bucket[remote_filename]
+                    bits.to_file(local_path.name)
+		local_path.setdata()
+
+	def list(self):
+		"""Return list of filenames (strings) present in backend"""
+                try:
+                    keys = self.bucket.keys()
+                except:
+                    self._logException("Error getting bucket keys, attempting to "
+                                       "re-connect.\n Got this Traceback:\n")
+                    self._connect()
+                    keys = self.bucket.keys()
+                return keys
+
+	def delete(self, filename_list):
+		"""Delete each filename in filename_list, in order if possible"""
+                for file in filename_list:
+                    try:
+                        del self.bucket[file]
+                    except:
+                        self._logException("Error deleting file %s, attempting to "
+                                           "re-connect.\n Got this Traceback:\n"
+                                           % file)
+                        self._connect()
+                        del self.bucket[file]
+
 # Dictionary relating protocol strings to backend_object classes.
 protocol_class_dict = {"scp": scpBackend,
 					   "ssh": scpBackend,
 					   "file": LocalBackend,
 					   "ftp": ftpBackend,
-					   "rsync": rsyncBackend}
+					   "rsync": rsyncBackend,
+					   "s3+http": BitBucketBackend}