Added bugz package source module

package: allow merging files using direct binary data

stopped using a temporary file in the URL handler

added utility helpers to extract URLs from text and parse numeric ranges

5 files changed, 187 insertions, 8 deletions

diff --git a/pomu/package.py b/pomu/package.py
index e0714dd..2651774 100644
--- a/pomu/package.py
+++ b/pomu/package.py
@@ -81,11 +81,15 @@ class Package():
     def merge_into(self, dst):
         """Merges contents of the package into a specified directory (dst)"""
         for trg, src in self.filemap.items():
-            wd, _ = path.split(trg)
+            wd, filename = path.split(trg)
             dest = path.join(dst, wd)
             try:
                 makedirs(dest, exist_ok=True)
-                copy2(src, dest)
+                if isinstance(src, bytes):
+                    with open(path.join(dest, filename), 'wb') as f:
+                        f.write(src)
+                else:
+                    copy2(src, dest)
             except PermissionError:
                 return Result.Err('You do not have enough permissions')
         return Result.Ok().and_(self.apply_patches())
diff --git a/pomu/source/__init__.py b/pomu/source/__init__.py
index c54447c..07cc9d6 100644
--- a/pomu/source/__init__.py
+++ b/pomu/source/__init__.py
@@ -6,3 +6,4 @@ import pomu.source.portage
 import pomu.source.file
 # sealed until pbraw is released
 #import pomu.source.url
+#import pomu.source.bugz
diff --git a/pomu/source/bugz.py b/pomu/source/bugz.py
new file mode 100644
index 0000000..62916a3
--- /dev/null
+++ b/pomu/source/bugz.py
@@ -0,0 +1,127 @@
+"""
+A package source module to import ebuilds and patches from bugzilla
+"""
+
+import xmlrpc.client
+from os import path
+from urllib.parse import urlparse
+
+from pbraw import grab
+
+from pomu.package import Package
+from pomu.source import dispatcher
+from pomu.util.misc import extract_urls, parse_range
+from pomu.util.pkg import cpv_split, ver_str
+from pomu.util.query import query, QueryContext
+from pomu.util.result import Result
+
+class BzEbuild():
+    """A class to represent a local ebuild"""
+    __name__ = 'fs'
+
+    # slots?
+    def __init__(self, bug_id, category, name, version, filemap):
+        self.bug_id = bug_id
+        self.category = category
+        self.name = name
+        self.version = version
+        self.filemap = filemap
+
+    def fetch(self):
+        return Package(self.name, '/', self, self.category, self.version, self.filemap)
+
+    @staticmethod
+    def from_data_dir(pkgdir):
+        with open(path.join(pkgdir, 'BZ_BUG_ID'), 'r') as f:
+            return BugzillaSource.parse_bug(f.readline()).unwrap()
+
+    def write_meta(self, pkgdir):
+        with open(path.join(pkgdir, 'BZ_BUG_ID'), 'w') as f:
+            f.write(self.bug_id + '\n')
+
+    def __str__(self):
+        return '{}/{}-{} (from {})'.format(self.category, self.name, self.version, self.path)
+
+CLIENT_BASE = 'https://bugs.gentoo.org/xmlrpc.cgi'
+
+@dispatcher.source
+class BugzillaSource():
+    """The source module responsible for importing ebuilds and patches from bugzilla tickets"""
+    @dispatcher.handler(priority=1)
+    def parse_bug(uri):
+        if not uri.isdigit():
+            return Result.Err()
+        uri = int(uri)
+        proxy = xmlrpc.client.ServerProxy(CLIENT_BASE).Bug
+        payload = {'ids': [uri]}
+        try:
+            bug = proxy.get(payload)
+        except (xmlrpc.client.Fault, OverflowError) as err:
+            return Result.Err(str(err))
+        attachments = proxy.attachments(payload)['bugs'][str(uri)]
+        comments = proxy.comments(payload)['bugs'][str(uri)]['comments']
+        comment_links = []
+        for comment in comments:
+            comment_links.extend(extract_urls(text))
+        items = attachments + comment_links
+        if not items:
+            return Result.Err()
+        lines = ['Please select required items (ranges are accepted)']
+        for idx, item in enumerate(items):
+            if isinstance(item, str):
+                lines.append('{} - {}'.format(idx, item))
+            else:
+                lines.append('{} - Attachment: {}'.format(idx, item['file_name']))
+        lines.append('>>> ')
+        rng = query('items', '\n'.join(lines), 1)
+        idxs = parse_range(rng, len(items))
+        if not idxs:
+            return Result.Err()
+        filtered = [x for idx, x in enumerate(items) if idx in idxs]
+        files = []
+        for idx, item in enumerate(idxs):
+            if isinstance(item, str):
+                files.extend([(x[0], x[1].encode('utf-8')) for x in grab(item)])
+            else:
+                files.append((item['file_name'], item['data']))
+        if not files:
+            return Result.Err()
+        category = query('category', 'Please enter package category').expect()
+        name = query('name', 'Please enter package name')
+        ver = query('version', 'Please specify package version for {}'.format(name)).expect()
+        # TODO: ???
+        fmap = {}
+        for fn, data in files:
+            with QueryContext(path=None):
+                fpath = query('path', 'Please enter path for {} file'.format(fn), path.join(category, name, fn))
+                fmap[fpath] = data
+
+
+
+        return Result.Ok(BzEbuild(uri, category, name, ver, uri))
+
+    @dispatcher.handler(priority=2)
+    def parse_link(uri):
+        res = urlparse(uri)
+        if res.netloc != 'bugs.gentoo.org':
+            return Result.Err()
+        if res.path == '/show_bugs.cgi':
+            ps = [x.split('=') for x in res.params.split('&') if x.startswith('id=')][1]
+            return BugzillaSource.parse_bug(ps)
+        if res.path.lstrip('/').isdigit():
+            return BugzillaSource.parse_bug(res.path.lstrip('/'))
+        return Result.Err()
+
+
+    @dispatcher.handler()
+    def parse_full(uri):
+        if not uri.startswith('bug:'):
+            return Result.Err()
+        rem = uri[4:]
+        if rem.isdigit():
+            return BugzillaSource.parse_bug(rem)
+        return BugzillaSource.parse_link(rem)
+
+    @classmethod
+    def from_meta_dir(cls, metadir):
+        return BzEbuild.from_data_dir(cls, metadir)
diff --git a/pomu/source/url.py b/pomu/source/url.py
index fe346a4..0f43f01 100644
--- a/pomu/source/url.py
+++ b/pomu/source/url.py
@@ -28,11 +28,12 @@ class URLEbuild(PackageBase):
     def fetch(self):
         fd, tfile = tempfile.mkstemp()
         os.close(fd)
-        with open(tfile, 'w') as f:
-            if self.contents:
-                f.write(self.contents)
-            else:
-                fs = grab(self.url)
+        if self.contents:
+            if isinstance(self.contents, str):
+                self.content = self.content.encode('utf-8')
+        else:
+            fs = grab(self.url)
+            self.content = fs[0][1].encode('utf-8')
                 f.write(fs[0][1])
         return Package(self.name, '/', self, self.category, self.version,
                 filemap = {
@@ -40,7 +41,7 @@ class URLEbuild(PackageBase):
                         self.category,
                         self.name,
                         '{}/{}-{}.ebuild'.format(self.category, self.name, self.version)
-                    ) : tfile})
+                    ) : self.content})
     
     @staticmethod
     def from_data_dir(pkgdir):
diff --git a/pomu/util/misc.py b/pomu/util/misc.py
index a6ebb1b..7fa6fd1 100644
--- a/pomu/util/misc.py
+++ b/pomu/util/misc.py
@@ -1,4 +1,5 @@
 """Miscellaneous utility functions"""
+import re
 
 def list_add(dst, src):
     """
@@ -20,3 +21,48 @@ def pivot(string, idx, keep_pivot=False):
         return (string[:idx], string[idx:])
     else:
         return (string[:idx], string[idx+1:])
+
+def extract_urls(text):
+    """Extracts URLs from arbitrary text"""
+    schemas = ['http://', 'https://', 'ftp://', 'ftps://']
+    words = list(filter(lambda x: any(y in x for y in schemas), text.split()))
+    maxshift = lambda x: max(x.find(y) for y in schemas)
+    links = [x[maxshift(x):].rstrip('\'")>.,') for x in words]
+    return links
+
+def parse_range(text, max_num=None):
+    """Parses a numeric range (e.g. 1-2,5-16)"""
+    text = re.sub('\s*-\s*', '-', text)
+    subranges = [x.strip() for x in text.split(',')]
+    subs = []
+    maxint = -1
+    for sub in subranges:
+        l, _, r = sub.partition('-')
+        if not l and not _:
+            continue
+        if (l and not l.isdigit()) or (r and not r.isdigit()):
+            return Result.Err('Invalid subrange: {}'.format(sub))
+        if l:
+            l = int(l)
+            maxint = max(l, maxint)
+        if r:
+            r = int(r)
+            maxint = max(r, maxint)
+        if _:
+            subs.append((l if l else 1, r if r else -1))
+        else:
+            subs.append((l, None))
+    if max_num:
+        maxint = max_num
+    res = set()
+    add = lambda x: res.add(x) if x <= maxint else None
+    for l, r in subs:
+        if not r:
+            add(l)
+            continue
+        if r == -1:
+            r = maxint
+        for x in range(l, r + 1):
+            add(x)
+    return Result.Ok(res)
+