diff options
author | 2010-07-31 00:46:07 -0300 | |
---|---|---|
committer | 2010-07-31 00:46:07 -0300 | |
commit | a250d9b7cc2e12cb0a3556b39c1305718263203f (patch) | |
tree | aea7d1070cf62e2e2aaf7919304861b192a8b906 | |
parent | added the new configuration option to the shipped configuration file (diff) | |
download | g-octave-a250d9b7cc2e12cb0a3556b39c1305718263203f.tar.gz g-octave-a250d9b7cc2e12cb0a3556b39c1305718263203f.tar.bz2 g-octave-a250d9b7cc2e12cb0a3556b39c1305718263203f.zip |
added contrib/manage_pkgdb.py
-rwxr-xr-x | contrib/manage_pkgdb.py | 219 |
1 files changed, 219 insertions, 0 deletions
diff --git a/contrib/manage_pkgdb.py b/contrib/manage_pkgdb.py new file mode 100755 index 0000000..4366df0 --- /dev/null +++ b/contrib/manage_pkgdb.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + manage_pkgdb.py + ~~~~~~~~~~~~~~~ + + a simple script to update a Git repository with a package database. + + :copyright: (c) 2010 by Rafael Goncalves Martins + :license: GPL-2, see LICENSE for more details. +""" + +from __future__ import print_function + +import datetime +import feedparser +import optparse +import os +import re +import shutil +import subprocess +import sys +import tarfile +import urllib + +from contextlib import closing + +class Git: + + def __init__(self, repo): + self._repo = repo + + def __call__(self, args): + args = args[:] + args.insert(0, 'git') + return subprocess.call(args, cwd=self._repo) + +re_tarball = re.compile(r'(([^/]+)-([0-9.]+)\.tar\.gz)$') + +class SfUpdates: + + # feed url from 'http://sourceforge.net/projects/octave/files/Octave%20Forge%20Packages/Individual%20Package%20Releases/' + feed_url = u'http://sourceforge.net/api/file/index/project-id/2888/mtime/desc/rss?path=%2FOctave%20Forge%20Packages%2FIndividual%20Package%20Releases' + + svnroot_url = u'https://octave.svn.sourceforge.net/svnroot/octave/trunk/octave-forge/' + categories = [u'main', u'extra', u'language', u'nonfree'] + + def __init__(self, local_dir): + self._local_dir = local_dir + self.feed = feedparser.parse(self.feed_url) + if self.feed.bozo == 1: + raise self.feed.bozo_exception + self.entries = self.feed.entries + + def remote_files(self): + # tarball name: { + # name, + # version, + # download url + # } + entries = {} + for entry in self.entries: + tarball = re_tarball.search(entry.summary) + if tarball is not None: + entries[tarball.group(1)] = { + 'name': tarball.group(2), + 'version': tarball.group(3), + 'url': entry.link, + } + return entries + + def local_files(self): + # tarball name: { + # name, + # version, + # category + # } + if not os.path.isdir(self._local_dir): + return {} + entries = {} + for category in os.listdir(self._local_dir): + for tarball in os.listdir(os.path.join(self._local_dir, category)): + my_tarball = re_tarball.search(unicode(tarball)) + if my_tarball is not None: + entries[my_tarball.group(1)] = { + 'name': my_tarball.group(2), + 'version': my_tarball.group(3), + 'category': unicode(category), + } + return entries + + def guess_category(self, pkgname): + for category in self.categories: + f = urllib.urlopen(self.svnroot_url + '/' + category + '/' + pkgname + '/DESCRIPTION') + if f.getcode() == 200: + return category + + def check_updates(self): + local_files = self.local_files() + remote_files = self.remote_files() + for remote in remote_files: + if remote not in local_files: + print('update found: %s; ' % remote, end='') + category = self.guess_category(remote_files[remote]['name']) + if category is None: + remote_name = remote_files[remote]['name'].lower() + for local in local_files: + local_name = local_files[local]['name'].lower() + if remote_name == local_name: + category = local_files[local]['category'] + break + remote_files[remote]['category'] = category + print('category: %s' % category) + if self.download(remote, remote_files[remote]) != os.EX_OK: + raise RuntimeError('Failed to download: %s' % remote) + + def download(self, tarball_name, entry): + return subprocess.call([ + 'wget', + '--output-document', os.path.join( + self._local_dir, + entry['category'], + tarball_name, + ), + entry['url'] + ]) + + def update_package_database(self, db_dir): + if not os.path.exists(db_dir): + os.makedirs(db_dir) + local_files = self.local_files() + for tarball_name in local_files: + entry = local_files[tarball_name] + description = os.path.join( + db_dir, + entry['category'], + entry['name'], + '%s-%s.DESCRIPTION' % (entry['name'], entry['version']) + ) + if not os.path.exists(description): + if not os.path.exists(os.path.dirname(description)): + os.makedirs(os.path.dirname(description)) + tarball = os.path.join( + self._local_dir, + entry['category'], + tarball_name + ) + with closing(tarfile.open(tarball, 'r:gz')) as src_tar: + to_extract = None + for f in src_tar: + if f.name.endswith('DESCRIPTION'): + to_extract = f + break + if f is None: + print('DESCRIPTION file not found: %s', tarball_name, file=sys.stderr) + continue + with closing(src_tar.extractfile(f)) as fp_tar: + with open(description, 'w') as fp: + shutil.copyfileobj(fp_tar, fp) + + +def main(argv): + + parser = optparse.OptionParser( + usage = '%prog [options] <sources directory> <package database directory>', + version = 'see g-octave --version', + description = 'a simple script to update a Git repository with a package database.' + ) + + parser.add_option( + '-c', '--commit', + action = 'store_true', + dest = 'commit', + default = False, + help = 'commit the changes to the Git repository' + ) + + parser.add_option( + '-p', '--push', + action = 'store_true', + dest = 'push', + default = False, + help = 'push the changes to the remote Git repository' + ) + + options, args = parser.parse_args(argv[1:]) + + if len(args) != 2: + print( + 'You need to provide 2 arguments:\n' + '- the directory where you store the source tarballs\n' + '- the directory where lives your package database (the git repository)', + file=sys.stderr + ) + return os.EX_USAGE + + print('* Fetching and parsing the Octave-Forge RSS feed ...') + sf = SfUpdates(args[0]) + print('* Looking for updates ...') + sf.check_updates() + print('* Trying to update the package database ...') + sf.update_package_database(args[1]) + if options.commit or options.push: + print('* Trying to commit the changes to the package database Git repository ...') + git = Git(args[1]) + date = datetime.datetime.utcnow().strftime('%H:%M:%S %Y/%m/%d') + if git(['add', '.']) == os.EX_OK and \ + git(['commit', '-m', 'Package database update - %s' % date]) == os.EX_OK: + if options.push: + print('Trying to push the changes to the remote Git repository ...') + return git(['push']) + return os.EX_OK + return os.EX_SOFTWARE + return os.EX_OK + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) |