diff options
author | 2013-09-04 20:02:36 +0200 | |
---|---|---|
committer | 2013-09-04 20:02:36 +0200 | |
commit | d2c815bf3f1557e83d550376da84ae6468c5eb8d (patch) | |
tree | ab2f0fbeec38a9c934321e8575e31164b55edd44 | |
parent | g_sorcery/logger: progress bar to stderr (diff) | |
download | g-sorcery-d2c815bf3f1557e83d550376da84ae6468c5eb8d.tar.gz g-sorcery-d2c815bf3f1557e83d550376da84ae6468c5eb8d.tar.bz2 g-sorcery-d2c815bf3f1557e83d550376da84ae6468c5eb8d.zip |
g_sorcery/package_db: check manifest parallely
-rw-r--r-- | g_sorcery/package_db.py | 42 |
1 files changed, 30 insertions, 12 deletions
diff --git a/g_sorcery/package_db.py b/g_sorcery/package_db.py index bd21857..e9402da 100644 --- a/g_sorcery/package_db.py +++ b/g_sorcery/package_db.py @@ -13,6 +13,7 @@ import glob import hashlib +import multiprocessing import os import shutil import sys @@ -193,6 +194,13 @@ class PackageDB(object): m_f = FileJSON(self.directory, 'manifest.json', []) m_f.write(manifest) + def _check_manifest_process(self, i, entries, errors): + for name, value in entries: + if hash_file(os.path.join(self.directory, name), hashlib.md5()) != \ + value: + errors.append(name) + + def check_manifest(self): """ Check database manifest. @@ -204,26 +212,36 @@ class PackageDB(object): self.logger.info("checking manifest") m_f = FileJSON(self.directory, 'manifest.json', []) manifest = m_f.read() - + + manager = multiprocessing.Manager() + result = True - errors = [] + errors = manager.list() names = [self.CATEGORIES_NAME] for name in names: if not name in manifest: raise DBStructureError('Bad manifest: no ' + name + ' entry') - progress_bar = ProgressBar(20, len(manifest)) - progress_bar.begin() - for name, value in manifest.items(): - progress_bar.increment() - if hash_file(os.path.join(self.directory, name), hashlib.md5()) != \ - value: - result = False - errors.append(name) + proc_num = multiprocessing.cpu_count() + 1 + + portion = len(manifest) // proc_num + procs = [] + entries = manifest.items() + + for i in range(proc_num - 1): + procs.append(multiprocessing.Process(target=self._check_manifest_process, args=(i, entries[i * portion:(i+1) * portion - 1], errors))) + procs.append(multiprocessing.Process(target=self._check_manifest_process, args=(proc_num - 1, entries[(proc_num - 1) * portion:], errors))) + + for proc in procs: + proc.start() + + for proc in procs: + proc.join() + + if errors: + result = False - progress_bar.end() - print("") return (result, errors) def clean(self): |