Changeset 1205 for obs


Ignore:
Timestamp:
Nov 29, 2015, 11:21:15 PM (9 years ago)
Author:
joergs
Message:

refactoring Jenkins functions to JenkinsTrigger class

File:
1 edited

Legend:

Unmodified
Added
Removed

  • obs/repo-status/repo-status.py

    r1204 r1205  
    11#!/usr/bin/env python
    22
    3 import argparse
    4 from csv import DictReader
    5 from glob import glob
    6 import logging
    7 import os
    8 from pprint import pprint,pformat
    9 import jenkinsapi.custom_exceptions
    10 from   jenkinsapi.jenkins import Jenkins
    11 import shlex
    12 import subprocess
    13 import sys
    14 import xml.etree.ElementTree as etree
    15 
    16 
    17 osc="osc"
    18 #osc_paramter="--apiurl=https://obs.dass-it"
    19 #prj="bareos:playground"
    20 #pkg="bareos"
    21 osc_paramter=""
    22 prj=""
    23 pkg=""
     3import  argparse
     4from    csv import DictReader
     5from    glob import glob
     6import  logging
     7import  os
     8from    pprint import pprint,pformat
     9import  jenkinsapi.custom_exceptions
     10from    jenkinsapi.jenkins import Jenkins
     11import  shlex
     12import  subprocess
     13import  sys
     14import  xml.etree.ElementTree as etree
     15
     16
    2417destdir=""
    25 wait=False
    2618
    2719class STATE:
    … …  
    3123    failed="FAILED"
    3224    succeeded="SUCCEEDED"
    33 
    34 
    35 jenkins_status = {
    36     'status': STATE.disabled,
    37     'jobname': "",
    38 }
    39 
    40 def format_command( cmd ):
    41     logger=logging.getLogger(__name__)
    42     #logger.debug( "cmd1:" + str(cmd) )
    43     cmd2=" ".join( cmd )
    44     logger.debug( "cmd2:" + str(cmd2) )
    45     cmd3=shlex.split( cmd2 )
    46     #logger.debug( "cmd3:" + str(cmd3) )
    47     return cmd3
    48 
    49 def show( string, array ):
    50     if array:
    51         print string + ":"
    52         for i in array:
    53             print " ", i
    5425
    5526def write_status(obs, jenkins_status):
    … …  
    135106    return pkg
    136107
     108class Osc:
     109    def __init__(self, osc_paramter):
     110        self.osc="osc"
     111        self.paramter=osc_paramter
     112
     113    def get_command(self, args):
     114        return self.__format_command([ self.osc, self.paramter ] + args)
     115
     116    def __format_command(self, cmd):
     117        logger=logging.getLogger(__name__)
     118        #logger.debug( "cmd1:" + str(cmd) )
     119        cmd2=" ".join( cmd )
     120        logger.debug( "cmd2:" + str(cmd2) )
     121        cmd3=shlex.split( cmd2 )
     122        #logger.debug( "cmd3:" + str(cmd3) )
     123        return cmd3
     124
     125
    137126class ObsStatus:
    138     def __init__(self, prj, pkg):
     127    def __init__(self, osc, prj, pkg):
    139128        self.logger = logging.getLogger()
     129        self.osc = osc
    140130        self.prj = prj
    141131        self.pkg = pkg
    … …  
    167157        # Debian_5.0|i586|published|False|succeeded|
    168158        # Debian_5.0|x86_64|published|False|succeeded|
    169         cmd = format_command( [ osc, osc_paramter, "results", "--csv", "--format='%(state)s|%(repository)s|%(arch)s|%(dirty)s|%(code)s|%(details)s'", "--verbose", self.prj, self.pkg ] )
     159        cmd = self.osc.get_command( ["results", "--csv", "--format='%(state)s|%(repository)s|%(arch)s|%(dirty)s|%(code)s|%(details)s'", "--verbose", self.prj, self.pkg] )
    170160        # "--last-build": NO, because if --last-build, disabled in 'code' is replaced by succeeded/failed
    171161        results=subprocess.Popen( cmd, stdout=subprocess.PIPE)
    … …  
    185175    def __get_obs_last_build(self, repository, arch):
    186176        # {'rev': '99', 'version': '1.2.1170-1.3', 'srcmd5': '611f626d431d06dc81a32e0e021da0d7', 'time': '2014-03-12 16:43:55'}
    187         cmd = format_command( [ osc, osc_paramter, "buildhist", "--csv", self.prj, self.pkg, repository, arch ] )
     177        cmd = self.osc.get_command( [ "buildhist", "--csv", self.prj, self.pkg, repository, arch ] )
    188178        buildhist=subprocess.Popen( cmd, stdout=subprocess.PIPE )
    189179        rc=buildhist.wait()
    … …  
    207197
    208198    def __update_packages(self):
    209         cmd = format_command( [ osc, osc_paramter, "results", "--xml", self.prj ] )
     199        cmd = self.osc.get_command( [ "results", "--xml", self.prj ] )
    210200        # "--last-build": NO, because if --last-build, disabled in 'code' is replaced by succeeded/failed
    211201        results=subprocess.Popen(cmd, stdout=subprocess.PIPE)
    … …  
    333323
    334324
    335 
    336 def add_jenkins_build_parameter( xmlConfig, build_params_add ):
    337     logger=logging.getLogger(__name__)
    338 
    339     # check, if build_params_add are already parameter of Jenkins job
    340     for child in xmlConfig.findall("./properties/hudson.model.ParametersDefinitionProperty/parameterDefinitions/hudson.model.StringParameterDefinition"):
    341         name=child.find('name').text
    342         if name in build_params_add.keys():
    343             # remove item from configuration parameter
    344             logger.debug( "build parameter " + name + " already in jenkins build configuration" )
    345             build_params_add.pop( name )
    346 
    347     #etree.dump( xmlConfig )
    348 
    349     # add remaining build parameter
    350     parameterDefinitions = xmlConfig.find("./properties/hudson.model.ParametersDefinitionProperty/parameterDefinitions")
    351 
    352     if len(parameterDefinitions) == 0:
    353         logger.error( "no jenkins build paramter defined. This should not happen. Skipping adding build parameter" )
     325class JenkinsTrigger:
     326    def __init__(self, url, jobname, obs):
     327        self.logger = logging.getLogger()
     328        self.url = url
     329        self.jobname = jobname
     330        self.obs = obs
     331        self.status = {
     332            'status': STATE.unknown,
     333            'jobname': jobname,
     334        }
     335        self.jenkins = Jenkins(self.url)
     336        # J.keys() # Jenkins objects appear to be dict-like, mapping keys (job-names) to
     337        #['foo', 'test_jenkinsapi']
     338        self.job = self.jenkins.get_job(jobname)
     339       
     340        self.distreleases = get_repo_list(self.obs.get_status('finished'), jenkins=True)
     341        self.version = self.obs.get_pkg_info()
     342       
     343        self.project_packages = get_packages_string(self.obs.get_successful_packages())
     344
     345
     346    def __add_jenkins_build_parameter(self, xmlConfig, build_params_add):
     347        # check, if build_params_add are already parameter of Jenkins job
     348        for child in xmlConfig.findall("./properties/hudson.model.ParametersDefinitionProperty/parameterDefinitions/hudson.model.StringParameterDefinition"):
     349            name=child.find('name').text
     350            if name in build_params_add.keys():
     351                # remove item from configuration parameter
     352                self.logger.debug( "build parameter " + name + " already in jenkins build configuration" )
     353                build_params_add.pop( name )
     354
     355        #etree.dump( xmlConfig )
     356
     357        # add remaining build parameter
     358        parameterDefinitions = xmlConfig.find("./properties/hudson.model.ParametersDefinitionProperty/parameterDefinitions")
     359
     360        if len(parameterDefinitions) == 0:
     361            self.logger.error( "no jenkins build paramter defined. This should not happen. Skipping adding build parameter" )
     362            return
     363
     364        #etree.dump( parameterDefinitions )
     365
     366        for item in build_params_add:
     367            self.logger.debug( "add build parameter " + str(item) )
     368            # example:
     369            #<hudson.model.StringParameterDefinition>
     370                #<name>BUILD_VERSION</name>
     371                #<description>automatically set by obs-status</description>
     372                #<defaultValue />
     373            #</hudson.model.StringParameterDefinition>
     374            new = etree.Element('hudson.model.StringParameterDefinition')
     375
     376            # add name
     377            name = etree.Element( 'name' )
     378            name.text = item
     379            new.append(name)
     380
     381            # add description
     382            description = etree.Element( 'description' )
     383            description.text = "BUILD parameter, required by and added from repo-status"
     384            new.append(description)
     385
     386            # add defaultValue
     387            defaultValue = etree.Element( 'defaultValue' )
     388            # no default value
     389            new.append(defaultValue)
     390
     391            parameterDefinitions.append(new)
    354392        return
    355393
    356     #etree.dump( parameterDefinitions )
    357 
    358     for item in build_params_add:
    359 
    360         logger.debug( "add build parameter " + str(item) )
    361 
    362         # example:
    363         #<hudson.model.StringParameterDefinition>
    364             #<name>BUILD_VERSION</name>
    365             #<description>automatically set by obs-status</description>
    366             #<defaultValue />
    367         #</hudson.model.StringParameterDefinition>
    368 
    369         new = etree.Element('hudson.model.StringParameterDefinition')
    370 
    371         # add name
    372         name = etree.Element( 'name' )
    373         name.text = item
    374         new.append(name)
    375 
    376         # add description
    377         description = etree.Element( 'description' )
    378         description.text = "BUILD parameter, required by and added from repo-status"
    379         new.append(description)
    380 
    381         # add defaultValue
    382         defaultValue = etree.Element( 'defaultValue' )
    383         # no default value
    384         new.append(defaultValue)
    385 
    386         parameterDefinitions.append(new)
    387     return
    388 
    389 
    390 def get_jenkins_build_parameter( job, parameter ):
    391     logger=logging.getLogger(__name__)
    392     try:
    393         build=job.get_last_build()
    394         parameters = build.get_actions()['parameters']
    395         for i in parameters:
    396             if i['name'] == parameter:
    397                 return i['value']
    398     except jenkinsapi.custom_exceptions.NoBuildData:
    399         pass
    400     logger.warn( "jenkins build parameter " + parameter + " not defined" )
    401     return
    402 
    403 def set_jenkins_matrix_distrelease( xmlConfig, distreleases ):
    404     logger=logging.getLogger(__name__)
    405     for child in xmlConfig.findall("./axes/hudson.matrix.TextAxis"):
    406         name=child.find('name').text
    407         if name == "DISTRELEASE":
    408             #etree.dump( child )
    409             values=child.find('values')
    410             for value in child.findall('values/string'):
    411                 logger.debug( "distrelease old: " + value.text )
    412                 values.remove( value )
    413             for i in distreleases:
    414                 new = etree.Element('string')
    415                 new.text=i
    416                 values.append( new )
    417             #etree.dump( child )
    418             #print "new:"
    419             for value in child.findall('values/string'):
    420                 logger.debug( "distrelease new: " + str(value.text) )
    421     return
    422 
    423 
    424 
    425 def check_jenkins_status(url, jobname, status, distreleases, version, project_packages):
    426     logger=logging.getLogger(__name__)
    427     logger.debug( "check_jenkins_status" )
    428     logger.debug( str(distreleases) )
    429 
    430     jenkins = Jenkins( url )
    431 
    432     # J.keys() # Jenkins objects appear to be dict-like, mapping keys (job-names) to
    433     #['foo', 'test_jenkinsapi']
    434 
    435     job=jenkins.get_job( jobname )
    436     try:
    437         build=job.get_last_build()
    438         status['BUILD_NUMBER']=build.get_number()
    439         status['BUILD_URL']=build.get_result_url()
    440     except jenkinsapi.custom_exceptions.NoBuildData:
    441         #except NoBuildData:
    442         pass
    443 
    444     if job.is_queued_or_running():
    445         # TODO: if job is queue, but not running, BUILD_NUMBER is wrong
    446         logger.debug( "jenkins job " + jobname + " is running" )
    447         status['status']=STATE.pending
    448         return status['status']
    449 
    450     # jenkins job is not running
    451     logger.debug( "no jenkins job is running for " + jobname )
    452     jenkins_job_build_version=get_jenkins_build_parameter( job, 'BUILD_VERSION' )
    453 
    454     logger.debug( "OBS version: " + str(version['version']) + ", " + "jenkins version: " + str(jenkins_job_build_version) )
    455 
    456     if jenkins_job_build_version == version['version']:
    457         # TODO: check if number DISTRELEASES have changed
    458         logger.debug( "skipped jenkins, as it has already tested the current OBS version" )
    459         if build.is_good():
    460             # success
    461             status['status']=STATE.succeeded
    462         else:
    463             status['status']=STATE.failed
    464         return status['status']
    465 
    466     # jenkins job is not running and last_build has not been for current version
    467 
    468     # get config data in xml format
    469     configString=job.get_config()
    470 
    471     # https://docs.python.org/2.7/library/xml.etree.elementtree.html
    472     xmlConfig=etree.fromstring( configString )
    473     #etree.dump( xmlConfig )
    474 
    475     pkg = get_packages_string(project_packages)
    476     build_params={ 'BUILD_VERSION': version['version'], 'BUILD_REV': version['rev'], 'BUILD_SRCMD5': version['srcmd5'], 'PACKAGES': pkg }
    477     # build paramter that must be present as Jenkins parameter configuration
    478     add_jenkins_build_parameter( xmlConfig, build_params.copy() )
    479     set_jenkins_matrix_distrelease( xmlConfig, distreleases )
    480 
    481     #etree.dump(xmlConfig)
    482 
    483     xmlString=etree.tostring( xmlConfig )
    484     job.update_config( xmlString )
    485 
    486     #for i in b.get_matrix_runs():
    487 
    488     logger.info( "starting jenkins for obs build " + version['version'] )
    489     invocation=job.invoke( build_params=build_params )
    490 
    491     if wait:
     394
     395    def __get_jenkins_build_parameter(self, parameter):
    492396        try:
    493             invocation.block()
    494         except jenkinsapi.custom_exceptions.TimeOut as e:
    495             logger.exception( "timeout while waiting for jenkins job" )
    496 
    497     # recusive
    498     return check_jenkins_status( url, jobname, status, distreleases, version, project_packages )
    499 
    500     #b.is_good()
    501 
    502     #pprint( dir(invocation) )
    503 
    504     #pprint( dir(invocation.job) )
    505 
    506     #build=invocation.get_build()
    507 
    508     #if invocation.is_queued_or_running():
    509         #status['status']=STATE.pending
    510     #else:
    511         #logger.debug( "duration: " + build.get_duration() )
    512         ##print build.get_actions()
    513         ## recusive
    514 
    515     #return status['status']
    516 
     397            build=self.job.get_last_build()
     398            parameters = build.get_actions()['parameters']
     399            for i in parameters:
     400                if i['name'] == parameter:
     401                    return i['value']
     402        except jenkinsapi.custom_exceptions.NoBuildData:
     403            pass
     404        logger.warn( "jenkins build parameter " + parameter + " not defined" )
     405        return
     406
     407
     408    def __set_jenkins_matrix_distrelease(self, xmlConfig):
     409        for child in xmlConfig.findall("./axes/hudson.matrix.TextAxis"):
     410            name=child.find('name').text
     411            if name == "DISTRELEASE":
     412                #etree.dump( child )
     413                values=child.find('values')
     414                for value in child.findall('values/string'):
     415                    logger.debug( "distrelease old: " + value.text )
     416                    values.remove( value )
     417                for i in self.distreleases:
     418                    new = etree.Element('string')
     419                    new.text=i
     420                    values.append( new )
     421                #etree.dump( child )
     422                #print "new:"
     423                for value in child.findall('values/string'):
     424                    self.logger.debug( "distrelease new: " + str(value.text) )
     425        return
     426
     427
     428    def checkAndTrigger(self, wait = False):
     429        self.logger.debug("check_jenkins_status")
     430
     431        try:
     432            build=self.job.get_last_build()
     433            self.status['BUILD_NUMBER']=build.get_number()
     434            self.status['BUILD_URL']=build.get_result_url()
     435        except jenkinsapi.custom_exceptions.NoBuildData:
     436            pass
     437
     438        if self.job.is_queued_or_running():
     439            # TODO: if job is queue, but not running, BUILD_NUMBER is wrong
     440            self.logger.debug( "jenkins job " + self.jobname + " is running" )
     441            self.status['status']=STATE.pending
     442            return self.status['status']
     443
     444        # jenkins job is not running
     445        self.logger.debug( "no jenkins job is running for " + self.jobname )
     446        jenkins_job_build_version=self.__get_jenkins_build_parameter('BUILD_VERSION')
     447
     448        self.logger.debug( "OBS version: " + str(self.version['version']) + ", " + "jenkins version: " + str(jenkins_job_build_version) )
     449
     450        if jenkins_job_build_version == self.obs.get_pkg_version():
     451            # TODO: check if number DISTRELEASES have changed
     452            logger.debug( "skipped jenkins, as it has already tested the current OBS version" )
     453            if build.is_good():
     454                # success
     455                self.status['status']=STATE.succeeded
     456            else:
     457                self.status['status']=STATE.failed
     458            return self.status['status']
     459
     460        # jenkins job is not running and last_build has not been for current version
     461
     462        # get config data in xml format
     463        configString=self.job.get_config()
     464
     465        # https://docs.python.org/2.7/library/xml.etree.elementtree.html
     466        xmlConfig=etree.fromstring( configString )
     467
     468        build_params = {
     469            'BUILD_VERSION': self.version['version'],
     470            'BUILD_REV': self.version['rev'],
     471            'BUILD_SRCMD5': self.version['srcmd5'],
     472            'PACKAGES': self.project_packages,
     473        }
     474        # build paramter that must be present as Jenkins parameter configuration
     475        self.__add_jenkins_build_parameter( xmlConfig, build_params.copy() )
     476        self.__set_jenkins_matrix_distrelease(xmlConfig)
     477
     478        xmlString=etree.tostring( xmlConfig )
     479        self.job.update_config( xmlString )
     480
     481        self.logger.info( "starting jenkins for obs build " + self.version['version'] )
     482        invocation=self.job.invoke( build_params=build_params )
     483
     484        if wait:
     485            try:
     486                invocation.block()
     487            except jenkinsapi.custom_exceptions.TimeOut as e:
     488                self.logger.exception( "timeout while waiting for jenkins job" )
     489
     490        # recusive
     491        return self.checkAndTrigger()
     492
     493
     494    def get_status(self):
     495        return self.status
    517496
    518497
    … …  
    548527        destdir=args.destdir
    549528
    550     if args.jenkinsjob:
    551         jenkins_status['jobname'] = args.jenkinsjob
    552 
    553     if args.jenkinsurl and args.jenkinsjob:
    554         # enable jenkins
    555         jenkins_status['status']=STATE.unknown
    556 
     529    jenkins_status = {
     530        'status': STATE.disabled
     531    }
     532
     533    osc = Osc(osc_paramter)
    557534    # check obs
    558     obs = ObsStatus(prj, pkg)
     535    obs = ObsStatus(osc, prj, pkg)
    559536    obs.update()
    560537    if obs.get_state() == STATE.succeeded and obs.get_pkg_version():
    561538        if args.jenkinsurl and args.jenkinsjob:
    562539            # run and check jenkins
    563             check_jenkins_status(args.jenkinsurl, args.jenkinsjob, jenkins_status, get_repo_list( obs.get_status('finished'), jenkins=True ), obs.get_pkg_info(), obs.get_successful_packages())
     540            jenkins = JenkinsTrigger(args.jenkinsurl, args.jenkinsjob, obs)
     541            jenkins.checkAndTrigger(wait)
     542            jenkins_status = jenkins.get_status()
    564543    else:
    565544        logger.info( "skipped jenkins tests, because prior steps" )
Note: See TracChangeset for help on using the changeset viewer.