1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
|
# config.py -- Portage Config
# Copyright 2007-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
import io
import stat
import portage
portage.proxy.lazyimport.lazyimport(globals(),
'portage.util:writemsg',
)
from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
from portage.localization import _
class LoaderError(Exception):
def __init__(self, resource, error_msg):
"""
@param resource: Resource that failed to load (file/sql/etc)
@type resource: String
@param error_msg: Error from underlying Loader system
@type error_msg: String
"""
self.resource = resource
self.error_msg = error_msg
def __str__(self):
return "Failed while loading resource: %s, error was: %s" % (
self.resource, self.error_msg)
def RecursiveFileLoader(filename):
"""
If filename is of type file, return a generate that yields filename
else if filename is of type directory, return a generator that fields
files in that directory.
Ignore files beginning with . or ending in ~.
Prune CVS directories.
@param filename: name of a file/directory to traverse
@rtype: list
@return: List of files to process
"""
try:
st = os.stat(filename)
except OSError:
return
if stat.S_ISDIR(st.st_mode):
for root, dirs, files in os.walk(filename):
for d in list(dirs):
if d[:1] == '.' or d == 'CVS':
dirs.remove(d)
for f in files:
try:
f = _unicode_decode(f,
encoding=_encodings['fs'], errors='strict')
except UnicodeDecodeError:
continue
if f[:1] == '.' or f[-1:] == '~':
continue
yield os.path.join(root, f)
else:
yield filename
class DataLoader(object):
def __init__(self, validator):
f = validator
if f is None:
# if they pass in no validator, just make a fake one
# that always returns true
def validate(key):
return True
f = validate
self._validate = f
def load(self):
"""
Function to do the actual work of a Loader
"""
raise NotImplementedError("Please override in a subclass")
class EnvLoader(DataLoader):
""" Class to access data in the environment """
def __init__(self, validator):
DataLoader.__init__(self, validator)
def load(self):
return os.environ
class TestTextLoader(DataLoader):
""" You give it some data, it 'loads' it for you, no filesystem access
"""
def __init__(self, validator):
DataLoader.__init__(self, validator)
self.data = {}
self.errors = {}
def setData(self, text):
"""Explicitly set the data field
Args:
text - a dict of data typical of Loaders
Returns:
None
"""
if isinstance(text, dict):
self.data = text
else:
raise ValueError("setData requires a dict argument")
def setErrors(self, errors):
self.errors = errors
def load(self):
return (self.data, self.errors)
class FileLoader(DataLoader):
""" Class to access data in files """
def __init__(self, filename, validator):
"""
Args:
filename : Name of file or directory to open
validator : class with validate() method to validate data.
"""
DataLoader.__init__(self, validator)
self.fname = filename
def load(self):
"""
Return the {source: {key: value}} pairs from a file
Return the {source: [list of errors] from a load
@param recursive: If set and self.fname is a directory;
load all files in self.fname
@type: Boolean
@rtype: tuple
@return:
Returns (data,errors), both may be empty dicts or populated.
"""
data = {}
errors = {}
# I tried to save a nasty lookup on lineparser by doing the lookup
# once, which may be expensive due to digging in child classes.
func = self.lineParser
for fn in RecursiveFileLoader(self.fname):
try:
with io.open(_unicode_encode(fn,
encoding=_encodings['fs'], errors='strict'), mode='r',
encoding=_encodings['content'], errors='replace') as f:
lines = f.readlines()
except EnvironmentError as e:
if e.errno == errno.EACCES:
writemsg(_("Permission denied: '%s'\n") % fn, noiselevel=-1)
del e
elif e.errno in (errno.ENOENT, errno.ESTALE):
del e
else:
raise
else:
for line_num, line in enumerate(lines):
func(line, line_num, data, errors)
return (data, errors)
def lineParser(self, line, line_num, data, errors):
""" This function parses 1 line at a time
Args:
line: a string representing 1 line of a file
line_num: an integer representing what line we are processing
data: a dict that contains the data we have extracted from the file
already
errors: a dict representing parse errors.
Returns:
Nothing (None). Writes to data and errors
"""
raise NotImplementedError("Please over-ride this in a child class")
class ItemFileLoader(FileLoader):
"""
Class to load data from a file full of items one per line
>>> item1
>>> item2
>>> item3
>>> item1
becomes { 'item1':None, 'item2':None, 'item3':None }
Note that due to the data store being a dict, duplicates
are removed.
"""
def __init__(self, filename, validator):
FileLoader.__init__(self, filename, validator)
def lineParser(self, line, line_num, data, errors):
line = line.strip()
if line.startswith('#'): # Skip commented lines
return
if not len(line): # skip empty lines
return
split = line.split()
if not len(split):
errors.setdefault(self.fname, []).append(
_("Malformed data at line: %s, data: %s")
% (line_num + 1, line))
return
key = split[0]
if not self._validate(key):
errors.setdefault(self.fname, []).append(
_("Validation failed at line: %s, data %s")
% (line_num + 1, key))
return
data[key] = None
class KeyListFileLoader(FileLoader):
"""
Class to load data from a file full of key [list] tuples
>>>>key foo1 foo2 foo3
becomes
{'key':['foo1','foo2','foo3']}
"""
def __init__(self, filename, validator=None, valuevalidator=None):
FileLoader.__init__(self, filename, validator)
f = valuevalidator
if f is None:
# if they pass in no validator, just make a fake one
# that always returns true
def validate(key):
return True
f = validate
self._valueValidate = f
def lineParser(self, line, line_num, data, errors):
line = line.strip()
if line.startswith('#'): # Skip commented lines
return
if not len(line): # skip empty lines
return
split = line.split()
if len(split) < 1:
errors.setdefault(self.fname, []).append(
_("Malformed data at line: %s, data: %s")
% (line_num + 1, line))
return
key = split[0]
value = split[1:]
if not self._validate(key):
errors.setdefault(self.fname, []).append(
_("Key validation failed at line: %s, data %s")
% (line_num + 1, key))
return
if not self._valueValidate(value):
errors.setdefault(self.fname, []).append(
_("Value validation failed at line: %s, data %s")
% (line_num + 1, value))
return
if key in data:
data[key].append(value)
else:
data[key] = value
class KeyValuePairFileLoader(FileLoader):
"""
Class to load data from a file full of key=value pairs
>>>>key=value
>>>>foo=bar
becomes:
{'key':'value',
'foo':'bar'}
"""
def __init__(self, filename, validator, valuevalidator=None):
FileLoader.__init__(self, filename, validator)
f = valuevalidator
if f is None:
# if they pass in no validator, just make a fake one
# that always returns true
def validate(key):
return True
f = validate
self._valueValidate = f
def lineParser(self, line, line_num, data, errors):
line = line.strip()
if line.startswith('#'): # skip commented lines
return
if not len(line): # skip empty lines
return
split = line.split('=', 1)
if len(split) < 2:
errors.setdefault(self.fname, []).append(
_("Malformed data at line: %s, data %s")
% (line_num + 1, line))
return
key = split[0].strip()
value = split[1].strip()
if not key:
errors.setdefault(self.fname, []).append(
_("Malformed key at line: %s, key %s")
% (line_num + 1, key))
return
if not self._validate(key):
errors.setdefault(self.fname, []).append(
_("Key validation failed at line: %s, data %s")
% (line_num + 1, key))
return
if not self._valueValidate(value):
errors.setdefault(self.fname, []).append(
_("Value validation failed at line: %s, data %s")
% (line_num + 1, value))
return
data[key] = value
|