1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
#!/usr/bin/env python3
# pylint: disable=line-too-long,missing-docstring,invalid-name
# vim:noet sts=4 ts=4
# Copyright 2015 Doug Freed (dwfreed)
#
# - Acquire local copy of repo/gentoo (.git directory not required)
# - Ensure mtime of files matches last time they were committed to, including
# merge commits (for optimal behavior, this should be the last time they were
# actually modified, but the last time a commit affected them in any way is
# alright too)
# - Run egencache against the checkout (this step is NOT optional; manifest generation is torturously slow without md5-cache)
# - Run thicken-manifests.py
#
# Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory
# For GPG signing, add --sign parameter.
# It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment.
# You do NOT need to modify layout.conf for this script to work. Stick with GPG 2.0 for now, because 2.1 moves key operations into the agent, which makes them not parallel.
import os
import multiprocessing
import subprocess
import logging
import argparse
import errno
import math
import collections
import portage
import portage.exception
import portage.manifest
import portage.util
portage_settings = None
portage_portdbapi = None
logger = None
args = None
dir_contents = collections.defaultdict(list)
def gpg_sign(filename):
gpgcmd = portage_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
if gpgcmd in [None, '']:
raise portage.exception.MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset! Is make.globals missing?")
if "${PORTAGE_GPG_KEY}" in gpgcmd and "PORTAGE_GPG_KEY" not in portage_settings:
raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
if "${PORTAGE_GPG_DIR}" in gpgcmd:
if "PORTAGE_GPG_DIR" not in portage_settings:
portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser("~/.gnupg")
else:
portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser(portage_settings["PORTAGE_GPG_DIR"])
if not os.access(portage_settings["PORTAGE_GPG_DIR"], os.X_OK):
raise portage.exception.InvalidLocation("Unable to access directory: PORTAGE_GPG_DIR='%s'" % portage_settings["PORTAGE_GPG_DIR"])
gpgvars = {"FILE": filename}
for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
v = portage_settings.get(k)
if v is not None:
gpgvars[k] = v
gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
gpgcmd = portage.util.shlex_split(gpgcmd)
gpgcmd = [portage._unicode_encode(arg, encoding=portage._encodings['fs'], errors='strict') for arg in gpgcmd] # pylint: disable=protected-access
return_code = subprocess.call(gpgcmd)
if return_code == os.EX_OK:
os.rename(filename + ".asc", filename)
else:
raise portage.exception.PortageException("!!! gpg exited with '" + str(return_code) + "' status")
def worker_init():
global portage_settings, portage_portdbapi # pylint: disable=global-statement
portage_settings = portage.config(clone=portage.settings) # pylint: disable=no-member
portage_portdbapi = portage.portdbapi(portage_settings) # pylint: disable=no-member
def maybe_thicken_manifest(pkg_dir):
try:
manifest_mtime = math.floor(os.stat(os.path.join(pkg_dir, 'Manifest')).st_mtime)
except OSError as e:
if e.errno != errno.ENOENT:
logger.exception("%s OSError thrown trying to stat Manifest", pkg_dir)
manifest_mtime = 0
newest_mtime = manifest_mtime
manifest_entries = []
for root in pkg_dir, os.path.join(pkg_dir, 'files'):
for filename in dir_contents[root]:
if filename == 'Manifest' and root == pkg_dir:
continue
file_mtime = math.floor(os.stat(os.path.join(root, filename)).st_mtime)
if file_mtime > newest_mtime:
newest_mtime = file_mtime
if root != pkg_dir:
manifest_entries.append('AUX ' + filename)
elif filename.endswith('.ebuild'):
manifest_entries.append('EBUILD ' + filename)
else:
manifest_entries.append('MISC ' + filename)
if newest_mtime == manifest_mtime:
try:
with open(os.path.join(pkg_dir, 'Manifest'), 'r') as f:
for line in f:
if not line.startswith(('AUX', 'EBUILD', 'MISC')):
continue
manifest_entry = ' '.join(line.split(' ')[0:2])
if manifest_entry in manifest_entries:
manifest_entries.remove(manifest_entry)
else:
newest_mtime += 1
break
except:
pass
fetchlistdict = portage.FetchlistDict(pkg_dir, portage_settings, portage_portdbapi)
# We use an empty dir so that we do not introduce any changes to
# existing DIST entries, or inject DIST entries that were otherwise
# missing.
manifest = portage.manifest.Manifest(pkg_dir, fetchlist_dict=fetchlistdict, distdir='/var/empty')
try:
manifest.create(assumeDistHashesAlways=True)
except portage.exception.FileNotFound:
logger.exception("%s is missing DIST entries!", pkg_dir)
# If any exception was triggered on the Manifest, it is NOT safe to write.
return
# If it changed, write it out
if manifest.write():
if newest_mtime == manifest_mtime:
# portage says the manifest changed, but mtime is supposed to stay the same
# bump the mtime by 1 to be safe
newest_mtime += 1
if args.sign:
try:
gpg_sign(manifest.getFullname())
except:
logger.exception("%s Exception thrown during GPG signing", pkg_dir)
# Always reset mtime on manifest
os.utime(manifest.getFullname(), (newest_mtime, newest_mtime))
def main():
global logger, args, dir_contents # pylint: disable=global-statement
parser = argparse.ArgumentParser(description='Thicken ebuild manifests as needed')
parser.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int, help='Number of parallel jobs to run; default: number of CPUs')
parser.add_argument('-s', '--sign', action='store_true', help='Sign manifests with GPG')
parser.add_argument('location', nargs='?', default='.', help='The location to thicken manifests in; default: .')
args = parser.parse_args()
args.location = os.path.realpath(args.location)
logger = multiprocessing.log_to_stderr()
logger.setLevel(logging.WARN)
os.environ['PORTAGE_REPOSITORIES'] = '''
[DEFAULT]
main-repo = gentoo
[gentoo]
location = %s
''' % args.location
os.chdir(args.location)
pkg_dirs = []
for root, subdirectories, files in os.walk('.'):
if root == '.':
for dirname in ('eclass', 'licenses', 'metadata', 'profiles', 'scripts', '.git'):
if dirname in subdirectories:
subdirectories.remove(dirname)
if (not subdirectories or subdirectories == ['files']) and 'metadata.xml' in files:
if os.path.sep not in os.path.split(root)[0]:
# EDGE CASE: category with single package called "files"
continue
pkg_dirs.append(root[2:])
dir_contents[root[2:]] = files
if 'files' in subdirectories:
subdirectories.remove('files')
filesdir = os.path.join(root, 'files')
dir_contents[filesdir[2:]] = []
os.chdir(filesdir)
for root2, subdirectories2, files2 in os.walk('.'):
for filename in files2:
dir_contents[filesdir[2:]].append(os.path.join(root2[2:], filename))
os.chdir(args.location)
pkg_dirs.sort()
pool = multiprocessing.Pool(args.jobs, worker_init)
pool.map(maybe_thicken_manifest, pkg_dirs)
pool.close()
if __name__ == '__main__':
main()
|
GitWeb