source: people/peter.buschman/backup_monitoring/parsing/parsers/nsradmin/_nsradmin.py@ 976

Last change on this file since 976 was 976, checked in by peter, on Dec 6, 2011 at 10:19:33 AM

Raw checkin of current NetBackup / TSM parsing code.
File size: 13.6 KB
Line 
1
2import re
3import time
4import csv
5
6from ....builtins import *
7
8from ...exceptions import *
9from ...expressions import *
10from ...autotype import *
11from ...streams import *
12
13##
14## Wrap a dictionary and provide access to it's keys as attributes.
15##
16class DictAttr:
17
18 def __init__(self, dict):
19 self.__dict__['_dict'] = dict
20
21 def __getattr__(self, name):
22 if self._dict.has_key(name):
23 return self._dict[name]
24 else:
25 return None
26
27 def __setattr__(self, name, value):
28 self._dict[name] = value
29
30 def __getitem__(self, name):
31 if self._dict.has_key(name):
32 return self._dict[name]
33 else:
34 return None
35
36 def __setitem__(self, name, value):
37 self._dict[name] = value
38
39 def __str__(self):
40 return str(self._dict)
41
42#
43# Here are some observations on the output of nsradmin:
44#
45# 1. Status messages at the top need to be skipped (Current query set, Display options, etc.,)
46# 2. Attribute / value pairs are separated by a : in column 28.
47# 3. Attribute / value pairs are terminated by a ; and can span 1 or more lines.
48# 4. Attribute / value pairs will wrap long output lines with either a \ or ', ' (comma and space)
49# as the last character before the newline.
50# 5. An attribute can have one or more values. If multiple values are present, they are separated
51# by commas and may or may not be contained in doublequotes (").
52# 6. Multiple resource records are separated by a blank line.
53#
54
55##
56## ParseError
57##
58class ParseError(Exception):
59 """Exception to be raised when data cannot be successfully parsed."""
60
61
62##
63## nsradmin_strip_header()
64##
65## Strip any leading data from nsradmin output that does not represent
66## valid NSR resource data.
67##
68## EXAMPLE
69##
70## Current query set
71## Hidden display option turned on
72##
73## Display options:
74## Dynamic: Off;
75## Hidden: On;
76## Resource ID: Off;
77## Dynamic display option turned on
78##
79## Display options:
80## Dynamic: On;
81## Hidden: On;
82## Resource ID: Off;
83##
84def nsradmin_strip_header(lines):
85 i = 0
86 for line in lines:
87 # Read until we encounter a : at position 28 in the line
88 if len(line) >= 28 and line[28] == ':':
89 return lines[i:]
90 i += 1
91 raise ParseError
92
93
94##
95## Return true if data is valid nsradmin output
96##
97def nsradmin_validate(lines):
98
99 #
100 # Regular expressions that, if matched signify a fatal error in the data.
101 #
102 expressions = [
103 r'^Remote system error$',
104 r'^No resources found for query:',
105 r'^nsradmin: lost connection to server:',
106 r'^nsradmin: Failed to rebind to the server',
107 r'^nsradmin: RPC error:',
108 r'^nsradmin: There does not appear to be a NetWorker server running on',
109 ]
110
111 compiled_expressions = [ re.compile(regexp) for regexp in expressions ]
112
113 count = 0 # Count of key / value pairs
114
115 for line in lines:
116 for regexp in compiled_expressions:
117 if regexp.search(line):
118 return False
119 if len(line) > 28 and line[28] == ':':
120 count += 1
121 if count > 0:
122 return True
123 else:
124 return False
125
126
127##
128## nsradmin_unwrap()
129##
130## Return nsradmin resource data with all wrapped lines unwrapped, resulting in
131## one key / value pair per line.
132##
133def nsradmin_unwrap(lines):
134 i = 0
135 j = len(lines)
136 unwrapped_lines = []
137 while( i < j ):
138 line = lines[i]
139 line = line[:-1]
140 #
141 # Unwrap following lines if this line is terminated by a \
142 #
143 if re.compile(r'\\$').search(line):
144 line = re.compile(r'\\$').sub('', line)
145 while(1):
146 i += 1
147 line += lines[i][:-1].lstrip()
148 if re.compile(r';$').search(line):
149 break
150 else:
151 line = re.compile(r'\\$').sub('', line)
152 #
153 # Unwrap following lines if this line is terminated by ', '
154 #
155 elif re.compile(r', $').search(line):
156 while(1):
157 i += 1
158 try:
159 line += lines[i][:-1].lstrip()
160 except:
161 print line
162 if re.compile(r';$').search(line):
163 break
164 unwrapped_lines.append(line)
165 i += 1
166 return unwrapped_lines
167
168
169##
170## nsradmin_dict()
171##
172## Return the contents of an uwrapped nsradmin resource record as a dictionary
173## with spaces and slashes in the attribute names replaced by underscores for use
174## as dictionary keys.
175##
176def nsradmin_dict(lines):
177 dict = {}
178 count = 0 # Count of key / value pairs found
179 for line in lines:
180 if re.compile(r'^\s+[a-zA-Z/ ]+:.*;$').match(line):
181 count += 1
182 line = re.compile(r';$').sub('', line)
183 key = line[:28]
184 key = re.compile(r'^\s+').sub('', key)
185 key = key.lower()
186 key = key.replace(' ','_')
187 key = key.replace('/','_')
188 value = line[30:]
189 dict[key] = value
190 if count == 0:
191 raise ParseError
192 else:
193 return dict
194
195
196##
197## Object Oriented Wrapper for a Networker Resource
198##
199class NsrResource(DictAttr):
200
201 def __init__(self, lines):
202
203 dict = nsradmin_dict(nsradmin_unwrap(nsradmin_strip_header(lines)))
204 newdict = {}
205 keys = dict.keys()
206 keys.sort()
207 resource_type = None
208
209 #
210 # Keys that may be quoted or contain commas, but which should still be stored
211 # as single values.
212 #
213 single_keys = {
214 'nsr' : [ 'message', 'save_totals', 'totals_since' ],
215 'nsr client' : [ ],
216 'nsr device' : [ 'name', 'date_last_cleaned', 'message', 'volume_expiration' ],
217 'nsr directive' : [ ],
218 'nsr group' : [ 'completion', 'interval', 'last_start', 'start_time' ],
219 'nsr jukebox' : [ 'event_message' ],
220 'nsr label' : [ ],
221 'nsr license' : [ 'checksum' ],
222 'nsr notification' : [ ],
223 'nsr policy' : [ ],
224 'nsr pool' : [ ],
225 'nsr schedule' : [ ],
226 'nsr stage' : [ ],
227 }
228
229 #
230 # Keys that should always be treated as lists of one or more values.
231 #
232 list_keys = {
233 'nsr' : [ 'onc_transport', 'types_created', 'sequence_number', 'server_message_category',
234 'server_message_priority', 'server_message_source', 'message_list', 'server_message',
235 'privileges', 'administrator', 'member_of' ],
236 'nsr client' : [ 'onc_transport', 'aliases' ],
237 'nsr device' : [ 'onc_transport' ],
238 'nsr directive' : [ 'onc_transport' ],
239 'nsr group' : [ 'onc_transport' ],
240 'nsr jukebox' : [ 'onc_transport', 'devices', 'volumes', 'volume_ids', 'jukebox_features', 'physical_slots',
241 'stl_barcodes', 'messages', 'stl_device_reservation', 'load_enabled', 'administrator' ],
242 'nsr label' : [ 'onc_transport' ],
243 'nsr license' : [ 'onc_transport' ],
244 'nsr notification' : [ 'onc_transport', 'event', 'priority' ],
245 'nsr policy' : [ 'onc_transport' ],
246 'nsr pool' : [ 'onc_transport' ],
247 'nsr schedule' : [ 'onc_transport' ],
248 'nsr stage' : [ 'onc_transport' ],
249 }
250
251 #
252 # If the data contains a type field, set the resource_type variable so we can use
253 # it to handle certain keys as lists or dictionaries depending on the resource
254 # type.
255 #
256 if dict.has_key('type'):
257 resource_type = dict['type'].lower()
258
259 #
260 # Iterate through the rest of the key / value pairs, storing them appropriately
261 # in the object's data structure.
262 #
263 for key in keys:
264 value = dict[key]
265 #
266 # If the key is to be treated as a single value, do so.
267 #
268 if single_keys.has_key(resource_type) and key in single_keys[resource_type]:
269 value = re.compile(r'^"').sub('', value)
270 value = re.compile(r'"$').sub('', value)
271 value = value.lstrip()
272 value = value.rstrip()
273 value = value.replace('\\\\','\\') # Unescape backslashes
274 if re.compile(r'^\d+$').match(value):
275 value = int(value)
276 newdict[key] = value
277 #
278 # If the key is to be treated as a list, do so.
279 #
280 elif list_keys.has_key(resource_type) and key in list_keys[resource_type]:
281 list = []
282 for item in value.split(', '):
283 item = re.compile(r'^"').sub('', item)
284 item = re.compile(r'"$').sub('', item)
285 item = item.lstrip()
286 item = item.rstrip()
287 item = item.replace('\\\\','\\') # Unescape backslashes
288 if re.compile(r'^\d+$').match(item):
289 item = int(item)
290 list.append(item)
291 newdict[key] = list
292 #
293 # If the key consists of one or more attribute = value pairs, store it as a dictionary
294 # (wrapped using DictAttr).
295 #
296 elif re.compile('([a-zA-Z0-9_:" ]+=[a-zA-Z0-9_:" ]+, )+').search(value):
297 csvdict = {}
298 for pairs in value.split(', '):
299 try:
300 (sname, svalue) = pairs.split('=')
301 except:
302 print key, value
303 sname = sname.lstrip()
304 sname = sname.rstrip()
305 sname = sname.replace(' ','_')
306 sname = sname.replace('(','')
307 sname = sname.replace(')','')
308 sname = sname.lower()
309 svalue = svalue.lstrip()
310 svalue = svalue.rstrip()
311 sname = re.compile(r'^"').sub('', sname)
312 sname = re.compile(r'"$').sub('', sname)
313 svalue = re.compile(r'^"').sub('', svalue)
314 svalue = re.compile(r'"$').sub('', svalue)
315 if re.compile(r'^\d+$').match(svalue):
316 svalue = int(svalue)
317 csvdict[sname] = svalue
318 newdict[key] = DictAttr(csvdict)
319 #
320 # If the key is quoted, assume this is potentially a list of quoted strings and
321 # store the contents in a list, even if there is just one value.
322 #
323 elif re.compile('"[^"]*"').search(value):
324 list = []
325 matches = re.compile('"[^"]*"').findall(value)
326 for match in matches:
327 match = re.compile(r'^"').sub('', match)
328 match = re.compile(r'"$').sub('', match)
329 match = match.lstrip()
330 match = match.rstrip()
331 match = match.replace('\\\\','\\') # Unescape backslashes
332 match = match.replace('\\-','-') # Remove backslashes before hyphens
333 match = match.replace('-\\','-') # Remove backslashes after hyphens
334 if re.compile(r'^\d+$').match(match):
335 match = int(match)
336 list.append(match)
337 newdict[key] = list
338 #
339 # If no special treatment is required, just store the value.
340 #
341 else:
342 value = re.compile(r'^"').sub('', value)
343 value = re.compile(r'"$').sub('', value)
344 value = value.lstrip()
345 value = value.rstrip()
346 value = value.replace('\\\\','\\') # Unescape backslashes
347 if re.compile(r'^\d+$').match(value):
348 value = int(value)
349 newdict[key] = value
350 #
351 # Initialize the NsrResource object by calling the parent's init method.
352 #
353 DictAttr.__init__(self, newdict)
354
355
356##
357## Split an nsradmin text stream into a list of unique resource records
358##
359def nsradmin_split(lines):
360 i = 0
361 j = len(lines)
362 records = []
363 record = []
364 #
365 # Skip leading garbage
366 #
367 while i < j:
368 line = lines[i]
369 if len(line) >= 28 and line[28] == ':':
370 break
371 else:
372 i += 1
373 #
374 # Process resource records
375 #
376 previous_terminator = None
377 while i < j:
378 line = lines[i]
379 #
380 # If a new record starts here, add the previous to the return set before continuing
381 #
382 # (A blank line by itself is not enough to delimit a new record. The previous line must
383 # have been terminated by a semi-colon.)
384 #
385 if re.compile('^$').match(line) and previous_terminator == ';':
386 records.append(record)
387 record = []
388 else:
389 record.append(line)
390 i += 1
391 if len(line) > 1:
392 previous_terminator = line[-2]
393 #
394 # Add the final record to the list and return
395 #
396 records.append(record)
397 return records
398
399
Note: See TracBrowser for help on using the repository browser.