source: people/peter.buschman/backup_monitoring/parsing/parsers/bpdbjobs/_bpdbjobs.py@ 976

Last change on this file since 976 was 976, checked in by peter, on Dec 6, 2011 at 10:19:33 AM

Raw checkin of current NetBackup / TSM parsing code.
File size: 14.7 KB
Line 
1
2import csv
3import sys
4import datetime
5import re
6
7from ....builtins import *
8
9from ...exceptions import *
10from ...expressions import *
11from ...streams import *
12
13from ....dependencies.odict import OrderedDict
14
15csv.field_size_limit(2147483647) # increase csv field size limit; default is 131072 which can choke on some NBU jobs
16
17##
18## bpdbjobs has one record per-line
19##
20def stream(stream):
21
22 return NewLineStream(stream)
23
24
25##
26## NBU Job Types
27##
28job_types = {
29 '0' : 'backup',
30 '1' : 'archive',
31 '2' : 'restore',
32 '3' : 'verify',
33 '4' : 'duplication', # Changed to 'duplicate' in NBU 6.5
34 '5' : 'import',
35 '6' : 'dbbackup', # Changed to 'catalog backup' in NBU 6.5
36 '7' : 'vault',
37 '8' : 'label',
38 '9' : 'erase',
39 '10' : 'tpreq', # Changed to 'tape request' in NBU 6.5
40 '11' : 'tpclean', # Changed to 'clean' in NBU 6.5
41 '12' : 'tpformat', # Changed to 'format tape' in NBU 6.5
42 '13' : 'vmphyinv', # Changed to 'physical inventory' in NBU 6.5
43 '14' : 'dqts', # Changed to 'qualification' in NBU 6.5
44 '15' : 'dbrecover', # Not documented in NBU 6.5
45 '16' : 'mcontents', # Not documented in NBU 6.5
46 '17' : 'image_cleanup', # DSSU image cleanup
47}
48
49
50##
51## NBU Job States
52##
53job_states = {
54 '0' : 'queued',
55 '1' : 'active',
56 '2' : 'wait for retry',
57 '3' : 'done',
58 '4' : 'suspended',
59 '5' : 'incomplete',
60 '6' : '6',
61 '7' : '7',
62}
63
64
65##
66## NBU Schedule Types
67##
68schedule_types = {
69 '0' : 'FULL',
70 '1' : 'INCR',
71 '2' : 'UBAK',
72 '3' : 'UARC',
73 '4' : 'CINC',
74}
75
76
77##
78## NBU Job Subtypes
79##
80subtypes = {
81 '0' : 'immediate',
82 '1' : 'scheduled',
83 '2' : 'user-initiated',
84 '3' : 'quick erase',
85 '4' : 'long erase',
86}
87
88
89##
90## NBU Retention Units
91##
92retention_units = {
93 '0' : 'Unknown',
94 '1' : 'Days',
95}
96
97
98##
99## NBU Class / Policy Types
100##
101## http://seer.entsupport.symantec.com/docs/261264.htm
102##
103class_types = {
104 '0' : 'Standard',
105 '1' : 'Proxy', # NetBackup internal setting
106 '2' : 'Non-Standard', # NetBackup internal setting
107 '3' : 'Apollo-wbak', # NetBackup DataCenter only
108 '4' : 'Oracle',
109 '5' : 'Any policy type',
110 '6' : 'Informix-On-BAR',
111 '7' : 'Sybase',
112 '8' : 'MS-Sharepoint', # NetBackup Server / Enterprise Server only
113 '9' : 'MS-Windows', # Not active, use MS-Windows-NT (13) for Windows 2000/NT/XP/2003 clients.
114 '10' : 'NetWare',
115 '11' : 'DataTools-SQL-BackTrack',
116 '12' : 'Auspex-FastBackup',
117 '13' : 'MS-Windows-NT',
118 '14' : 'OS/2',
119 '15' : 'MS-SQL-Server',
120 '16' : 'MS-Exchange-Server',
121 '17' : 'SAP', # NetBackup Server / Enterprise Server only
122 '18' : 'DB2', # NetBackup Server / Enterprise Server only
123 '19' : 'NDMP',
124 '20' : 'FlashBackup', # NetBackup Server / Enterprise Server only
125 '21' : 'Split-Mirror', # NetBackup Server / Enterprise Server only
126 '22' : 'AFS', # NetBackup Server / Enterprise Server only
127 '23' : 'DFS', # Not an active policy type.
128 '24' : 'DataStore',
129 '25' : 'Lotus-Notes',
130 '26' : 'NCR-Teradata', # No longer supported, contact NCR to see if they can provide an extension.
131 '27' : 'OpenVMS',
132 '28' : 'MPE/iX',
133 '29' : 'FlashBackup-Windows', # NetBackup Server / Enterprise Server only
134 '30' : 'Vault',
135 '31' : 'BE-MS-SQL-Server',
136 '32' : 'BE-MS-Exchange-Server',
137 '33' : 'Macintosh', # Not active, use Standard (0) for Macintosh OSX clients.
138 '34' : 'Disk Staging', # NetBackup Server / Enterprise Server only
139 '35' : 'NBU-Catalog', # NetBackup 6.0 only
140}
141
142policy_types = class_types
143
144
145##
146## Operation Types
147##
148operation_types = {
149 '0' : 'mount',
150 '1' : 'position',
151 '2' : 'connect',
152 '3' : 'write',
153 '4' : 'vault initialize',
154 '5' : 'vault duplication',
155 '6' : 'vault duplication complete',
156 '7' : 'vault catalog backup',
157 '8' : 'vault eject',
158 '9' : '9',
159 '10' : 'report',
160 '11' : 'duplicate',
161 '12' : 'import',
162 '13' : 'verify',
163 '14' : 'restore',
164 '15' : 'catalog-backup',
165 '16' : 'vault',
166 '17' : 'label',
167 '18' : 'erase',
168}
169
170
171##
172## Parse a bpdbjobs record
173##
174## bpdbjobs -report -all_columns
175##
176def parse(record, format='bpdbjobs -report -all_columns', version=None, tz=None):
177
178 if format in ['bpdbjobs -report -all_columns', 'bpdbjobs -report -most_columns']:
179
180 record = record.replace('\r', '') # compensate for embedded carriage-returns
181 record = csv.reader([record], escapechar='\\').next()
182
183 i = 0
184 job = ExtendedDict()
185
186 #
187 # Leading Columns
188 #
189 job['jobid'] = int(record[0]) if re_integer.match(record[0]) else None
190 job['jobtype'] = job_types[record[1]] if record[1] in job_types else None
191
192 if job.jobtype == 'backup':
193 job.backup = True
194 else:
195 job.backup = False
196
197 if job.jobtype == 'duplication':
198 job.duplication = True
199 else:
200 job.duplication = False
201
202 job['state'] = job_states[record[2]] if record[2] in job_states else None
203
204 if job.state == 'active':
205 job.active = True
206 else:
207 job.active = False
208
209 if job.state == 'queued':
210 job.queued = True
211 else:
212 job.queued = False
213
214 if job.state == 'done':
215 job.done = True
216 else:
217 job.done = False
218
219 job['status'] = int(record[3]) if re_integer.match(record[3]) else None
220 #job['class'] = record[4] if record[4] else None
221 job['policy'] = record[4] if record[4] else None
222 job['schedule'] = record[5] if record[5] else None
223 job['client'] = record[6] if record[6] else None
224 job['server'] = record[7] if record[7] else None
225 job['started'] = datetime.datetime.fromtimestamp(int(record[8]), tz) if re_integer.match(record[8]) else None
226 job['elapsed'] = int(record[9]) if re_integer.match(record[9]) else None
227 job['ended'] = datetime.datetime.fromtimestamp(int(record[10]), tz) if re_integer.match(record[10]) else None
228 job['stunit'] = record[11] if record[11] else None
229 job['try'] = int(record[12]) if re_integer.match(record[12]) else None
230 job['operation'] = operation_types[record[13]] if record[13] in operation_types else None
231 job['kbytes'] = long(record[14]) if re_integer.match(record[14]) else None
232 job['files'] = long(record[15]) if re_integer.match(record[15]) else None
233 job['pathlastwritten'] = record[16] if record[16] else None
234 job['percent'] = int(record[17]) if re_integer.match(record[17]) else None
235 job['jobpid'] = int(record[18]) if re_integer.match(record[18]) else None
236 job['owner'] = record[19] if record[19] else None
237 job['subtype'] = subtypes[record[20]] if record[20] in subtypes else None
238 #job['classtype'] = class_types[record[21]] if record[21] in class_types else None
239 job['policytype'] = policy_types[record[21]] if record[21] in policy_types else None
240 job['schedule_type'] = schedule_types[record[22]] if record[22] in schedule_types else None
241 job['priority'] = record[23] if record[23] else None
242 job['group'] = record[24] if record[24] else None
243 job['masterserver'] = record[25] if record[25] else None
244 job['retentionunits'] = retention_units[record[26]] if record[26] in retention_units else None
245 job['retentionperiod'] = record[27] if record[27] else None
246 job['compression'] = record[28] if record[28] else None
247 job['kbyteslastwritten'] = long(record[29]) if re_integer.match(record[29]) else None
248 job['fileslastwritten'] = long(record[30]) if re_integer.match(record[30]) else None
249
250 #
251 # Files
252 #
253 filelistcount = int(record[31]) if re_integer.match(record[31]) else None
254 filelist = []
255
256 i = 32
257
258 if filelistcount is not None and filelistcount > 0:
259 for i in range(i, i + filelistcount):
260 file = record[i]
261 if file:
262 filelist.append(file)
263 i += 1
264
265 job['filelistcount'] = filelistcount
266 job['filelist'] = filelist
267
268 #
269 # Tries
270 #
271 trycount = int(record[i]) if record[i] else None
272 tries = []
273
274 if trycount:
275
276 for t in range(trycount):
277
278 Try = ExtendedDict()
279 #Try = OrderedDict()
280
281 #Try['trypid'] = record[i+1] if record[i+1] else None
282 Try['pid'] = record[i+1] if record[i+1] else None
283 #Try['trystunit'] = record[i+2] if record[i+2] else None
284 Try['stunit'] = record[i+2] if record[i+2] else None
285 #Try['tryserver'] = record[i+3] if record[i+3] else None
286 Try['server'] = record[i+3] if record[i+3] else None
287 #Try['trystarted'] = datetime.datetime.fromtimestamp(int(record[i+4]), tz) if re_integer.match(record[i+4]) else None
288 Try['started'] = datetime.datetime.fromtimestamp(int(record[i+4]), tz) if re_integer.match(record[i+4]) else None
289 Try['elapsed'] = int(record[i+5]) if re_integer.match(record[i+5]) else None
290 #Try['tryended'] = datetime.datetime.fromtimestamp(int(record[i+6]), tz) if re_integer.match(record[i+6]) else None
291 Try['ended'] = datetime.datetime.fromtimestamp(int(record[i+6]), tz) if re_integer.match(record[i+6]) else None
292 #Try['trystatus'] = int(record[i+7]) if re_integer.match(record[i+7]) else None
293 Try['status'] = int(record[i+7]) if re_integer.match(record[i+7]) else None
294 #Try['trystatusdescription'] = record[i+8] if record[i+8] else None
295 Try['statusdescription'] = record[i+8] if record[i+8] else None
296
297 trystatuscount = int(record[i+9]) if re_integer.match(record[i+9]) else None
298 trystatuslines = []
299
300 i += 10
301
302 #
303 # Process status lines if there are any
304 #
305 if trystatuscount is not None and trystatuscount > 0:
306
307 for i in range(i, i + trystatuscount):
308 trystatuslines.append(record[i])
309
310 #Try['trystatuscount'] = trystatuscount
311 Try['statuscount'] = trystatuscount
312 #Try['trystatuslines'] = trystatuslines
313 Try['statuslines'] = trystatuslines
314 #Try['trybyteswritten'] = long(record[i+1]) if re_integer.match(record[i+1]) else None
315 #Try['byteswritten'] = long(record[i+1]) if re_integer.match(record[i+1]) else None
316 #Try['trykbyteswritten'] = long(record[i+1]) if re_integer.match(record[i+1]) else None
317 #Try['kbyteswritten'] = long(record[i+1]) if re_integer.match(record[i+1]) else None
318 #Try['trykbytes'] = long(record[i+1]) if re_integer.match(record[i+1]) else None
319 Try['kbytes'] = long(record[i+1]) if re_integer.match(record[i+1]) else None
320 #Try['tryfileswritten'] = long(record[i+2]) if re_integer.match(record[i+2]) else None
321 Try['fileswritten'] = long(record[i+2]) if re_integer.match(record[i+2]) else None
322
323 tries.append(Try)
324
325 #
326 # Only increment by 1 if there were no status lines
327 #
328 if trystatuscount is not None and trystatuscount > 0:
329 i += 2
330 else:
331 i += 1
332
333 job['trycount'] = trycount
334 job['tries'] = tries
335
336 #
337 # Trailing Columns
338 #
339 remaining = len(record[i+1:])
340 job['parentjob'] = record[i+1] if record[i+1] else None
341 job['kbpersec'] = record[i+2] if record[i+2] else None
342 job['copy'] = record[i+3] if record[i+3] else None
343 job['robot'] = record[i+4] if record[i+4] else None
344 job['vault'] = record[i+5] if record[i+5] else None
345 job['profile'] = record[i+6] if record[i+6] else None
346 job['session'] = record[i+7] if record[i+7] else None
347 job['ejecttapes'] = record[i+8] if record[i+8] else None
348 job['srcstunit'] = record[i+9] if record[i+9] else None
349 job['srcserver'] = record[i+10] if record[i+10] else None
350 job['srcmedia'] = record[i+11] if record[i+11] else None
351 job['dstmedia'] = record[i+12] if record[i+12] else None
352 job['stream'] = record[i+13] if record[i+13] else None
353 job['suspendable'] = record[i+14] if record[i+14] else None
354 job['resumable'] = record[i+15] if record[i+15] else None
355 job['restartable'] = record[i+16] if record[i+16] else None
356 job['datamovement'] = record[i+17] if record[i+17] else None
357 job['frozenimage'] = record[i+18] if record[i+18] else None
358 job['backupid'] = record[i+19] if record[i+19] else None
359 job['killable'] = record[i+20] if record[i+20] else None
360 job['controllinghost'] = record[i+21] if record[i+21] else None
361
362 #
363 # DEBUG
364 #j = 0
365 #for key in job.keys():
366 # j += 1
367 # print '%2d: %20s: %s' % (j, key, job[key])
368
369 return job
370
371 else:
372
373 raise ParseError, 'Unknown format %s' % (format)
374
Note: See TracBrowser for help on using the repository browser.