#!/usr/bin/env python from __future__ import print_function import fnmatch import os import sys import argparse import subprocess from collections import OrderedDict from six import iteritems import globals import toolconfig import multi_facility_process_helper_client import json import getpass from globals import UserParamsException def doWeNeedToSwitchTheMasterLink(facility, name, type, release, versionsAndReleases): '''If we are going to a release and a facility is already in that release, we should not switch the master link''' if not versionsAndReleases[facility][0]: if globals.verbose: print("Release has not been pushed to facility {0}; so no need to consider upgrading master link".format( facility)) return False if type in ['SIOC', 'HIOC']: masterRelease = versionsAndReleases[facility][0]['master'] return masterRelease != release else: masterRelease = versionsAndReleases[facility][0] return masterRelease != release def determineIOCLinksThatNeedToChangeBecauseTheMasterLinkIsChanging(facility, name, type, newMasterRelease, versionsAndReleases): '''Loop thru the IOCs for this facility and determine all the IOC's that need to change because the master link is changing We deem that the IOC link is changing even if previously it was pointing to the master release. This is because the master link itself is now changing ''' ret = [] if not versionsAndReleases[facility][0]: if globals.verbose: print("Release has not been pushed to facility {0}; so we do not have the master link changing.".format( facility)) return ret iocs = versionsAndReleases[facility][0]['iocs'] for ioc in iocs: if ioc['release'] != newMasterRelease: ret.append((facility, ioc['ioc'])) elif ioc['isMaster'] == False: ret.append((facility, ioc['ioc'])) return ret def generateFacilityIOCFromVersionsAndReleases(versionsAndReleases): '''Generate a list of facility, IOC tuples from versionsAndReleases''' facilityIOCList = [] for facility, vr in iteritems(versionsAndReleases): if 'iocs' not in vr[0]: continue for ioc in vr[0]['iocs']: facilityIOCList.append((facility, ioc['ioc'])) if globals.verbose: print("Facility IOC list {0}".format(facilityIOCList)) return facilityIOCList def getFaciltyIOC(facilityarg, searchioc, facilityIOCList): '''Iterate thru facilityIOCList and give us a facility/IOC tuple that tells us which facility an IOC is located In case we have more than one facility with IOCs that named the same, we expect the facility to be specified to help disambiguate ''' # Do not count total matches; instead, count facilities numberOfIOCsWithTheSameName = len(set(facilityIOCTuple[0] for facilityIOCTuple in facilityIOCList if fnmatch.fnmatch(facilityIOCTuple[1], searchioc))) if numberOfIOCsWithTheSameName <= 0: raise UserParamsException("Cannot find IOC with name {0} in {1}. If you expected to see an IOC here, you can use cram validate to see if the softlinks are set up correctly".format(searchioc, facilityarg)) elif numberOfIOCsWithTheSameName > 1: if (not facilityarg or facilityarg == 'ALL'): raise UserParamsException("Found more than one IOC with the same name " + searchioc + ". In this case we need an unambiguous facility argument to disambiguate the IOC.") else: return [facilityIOCTuple for facilityIOCTuple in facilityIOCList if fnmatch.fnmatch(facilityIOCTuple[1], searchioc) and facilityIOCTuple[0] == facilityarg] elif numberOfIOCsWithTheSameName == 1: if facilityarg: iocFacCombos = [facilityIOCTuple for facilityIOCTuple in facilityIOCList if fnmatch.fnmatch(facilityIOCTuple[1], searchioc) and facilityIOCTuple[0] == facilityarg] if len(iocFacCombos) < 1: raise UserParamsException("Cannot find IOC with name {0} in {1}. If you expected to see an IOC here, you can use cram validate to see if the softlinks are set up correctly".format(searchioc, facilityarg)) return iocFacCombos else: return [facilityIOCTuple for facilityIOCTuple in facilityIOCList if fnmatch.fnmatch(facilityIOCTuple[1], searchioc)] else: raise UserParamsException("We should not be here. Please contact the developer {0} in {1} using {2} gives {3}".format(searchioc, facilityarg, facilityIOCList, numberOfIOCsWithTheSameName)) def determineFacilityIOCCombos(facility, ioc, name, type, release, versionsAndReleases): '''Determine the facilities and IOCs that this call applies to based on the type of application and the arguments being passed in. This returns a list of tuples - First element in tuple is the facility and second element is the IOC instance identifier. For the IOC master release, for HLA, and for PyDM, this will be MASTER This method determines the various use cases we are currently supporting''' iocFacilityCombos = [] facilityConfig = toolconfig.getFacilityConfig() facilityList = toolconfig.getListOfFacilities() if type in ['SIOC', 'HIOC']: # For IOC's, we support these options # ALL IOCs in one or more or ALL facilities. # A specified list of one or more IOCs - in this case the facility argument should be ignored if not ioc: raise UserParamsException("For IOC's, we need to specify the IOC's that need to be upgraded by using the -i/--ioc argument. To upgrade all IOCS, use -i ALL. When upgrading ALL IOC's, also specify one or more or ALL facilities. Specify more than once IOC using a command separated list of IOC names. For example, -i sioc-ms-001,sioc-ms-002.") if ioc == 'ALL': if not facility: raise UserParamsException("For IOC's, we need to specify the IOC's that need to be upgraded by using the -i/--ioc argument. To upgrade all IOCS, use -i ALL. When upgrading ALL IOC's, also specify one or more or ALL facilities. Specify more than once IOC using a command separated list of IOC names. For example, -i sioc-ms-001,sioc-ms-002.") if facility == 'ALL': for specificfacility in facilityConfig: if doWeNeedToSwitchTheMasterLink(specificfacility['name'], name, type, release, versionsAndReleases): iocFacilityCombos.append((specificfacility['name'], 'MASTER')) iocFacilityCombos.extend(determineIOCLinksThatNeedToChangeBecauseTheMasterLinkIsChanging(specificfacility['name'], name, type, release, versionsAndReleases)) else: facilities = facility.split(',') for facility in facilities: if facility not in facilityList: raise UserParamsException("Facility " + facility + " is not present in the global facility config") if doWeNeedToSwitchTheMasterLink(facility, name, type, release, versionsAndReleases): iocFacilityCombos.append((facility, 'MASTER')) iocFacilityCombos.extend(determineIOCLinksThatNeedToChangeBecauseTheMasterLinkIsChanging(facility, name, type, release, versionsAndReleases)) else: if ('*' in ioc or '?' in ioc) and not facility: raise UserParamsException("When specifying IOCs using a wildcard, the facility must also be specified.") facilityIOCList = generateFacilityIOCFromVersionsAndReleases(versionsAndReleases) ioclist = ioc.split(",") for specificioc in ioclist: # Determine which facility the IOC is in facilityIOC = getFaciltyIOC(facility, specificioc, facilityIOCList) if not facilityIOC: raise UserParamsException("Could not locate IOC " + specificioc + " for software package " + name) iocFacilityCombos.extend(facilityIOC) # Remove duplicates iocFacilityCombos = list(OrderedDict.fromkeys(iocFacilityCombos)) if globals.verbose: print("IOCs that need upgrading {0}".format(iocFacilityCombos)) else: # For non IOC's, we support these options # One or more specified facilities # All facilities if 'facility': if facility == 'ALL': for specificfacility in facilityConfig: if doWeNeedToSwitchTheMasterLink(specificfacility['name'], name, type, release, versionsAndReleases): iocFacilityCombos.append((specificfacility['name'], 'MASTER')) else: facilities = facility.split(",") for specificfacility in facilities: if specificfacility not in facilityList: raise UserParamsException("Facility " + specificfacility + " is not present in the global facility config") if doWeNeedToSwitchTheMasterLink(specificfacility, name, type, release, versionsAndReleases): iocFacilityCombos.append((specificfacility, 'MASTER')) else: raise UserParamsException("For applications of type - " + type + ", you need to specify at least one facility. You can use ALL for all facilities or you can specify more than one facility using a comma separated list. Use -h to get a list of facilities") return iocFacilityCombos def makeSureReleaseFolderExistsOnFacilities(name, type, release, iocFacilityCombos, versionsAndReleases): """Make sure that the release folder exists on all the facilities being affected before we actually switch the link In addition, we also get the current versions for IOCs/HLA apps and return it """ facilities = [] for iocFacilityCombo in iocFacilityCombos: facility = iocFacilityCombo[0] if facility not in facilities: facilities.append(facility) for facility in facilities: vr = versionsAndReleases[facility] releases = vr[1] if release not in map(lambda x : x['name'], releases): raise UserParamsException("Release " + release + " was not found in facility " + facility + ". Please push this release again to all facilities.") def determineExistingVersionsAndAvailableReleases(name, type): '''For the given software package as determined by the name and type, in one call, determine the current versions and the available releases across all facilities Return is dictionary of facility -> a list of two elements - first is the version info as appropriate to the type of software package and the second is a list of available releases. ''' ret = {} for facility in toolconfig.getListOfFacilities(): commandsAndArgs = [] arg = {} arg['name'] = name arg['releaseFolder'] = toolconfig.determineReleaseFolder(facility, name, type) arg['linkFolder'] = toolconfig.determineLinkFolder(facility, name, type) if type in ['SIOC', 'HIOC']: determineVersionCommand = 'determineIOCVersions' elif type == 'PyDM': determineVersionCommand = 'determinePyDMDisplayVersion' else: determineVersionCommand = 'determineHLAVersion' commandsAndArgs.append((determineVersionCommand, arg)) releaseFolder = os.path.join(toolconfig.determineReleaseFolder(facility, name, type), name) commandsAndArgs.append(('ls', [releaseFolder])) results = multi_facility_process_helper_client.executeCommandsOnFacility(facility, commandsAndArgs) ret[facility] = results return ret def upgradeRelease(args): """ Switch the links in iocCommon/pydmDisplay/HLACommon/etc to point to the specified release""" packageInfo = toolconfig.determinePackageInfoFromArgs(args) facilityConfig = toolconfig.getFacilityConfig() if 'name' not in packageInfo: raise UserParamsException("Cannot identify package. Please specify the name of the package by either using the --name argument or by running this command in the folder that contains the release for this package.") if 'type' not in packageInfo: raise UserParamsException("Cannot identify type of package. One way to fix this is to run this command in a folder that contains the release for this package") name = packageInfo['name'] type = packageInfo['type'] release = args.release if globals.verbose: print("Switching to release {0} for package {1} of type {2}".format(release, name, type)) versionsAndReleases = determineExistingVersionsAndAvailableReleases(name, type) if globals.verbose: print("Versions and releases {0}".format(json.dumps(versionsAndReleases))) # Determine facilities, IOC's etc. iocFacilityCombos = determineFacilityIOCCombos(args.facility, args.ioc, name, type, release, versionsAndReleases) # We now have in iocFacilityCombos a list of tuples # First element in tuple is the facility # Second element is the IOC instance identifier. For the IOC master release, for HLA and for PyDM, this will be MASTER if globals.verbose: print("IOCFacilityCombos is {0}".format(iocFacilityCombos)) if args.list: print("As part of this operation, these IOCs/HLA/PyDM/Master links will be upgraded") for iocFac in iocFacilityCombos: print("Master IOC Link on " + iocFac[0] if iocFac[1] == 'MASTER' else "IOC " + iocFac[1]) return # Make sure the release folder exists on all the facilities makeSureReleaseFolderExistsOnFacilities(name, type, release, iocFacilityCombos, versionsAndReleases) for iocFacilityCombo in iocFacilityCombos: if globals.verbose: print("Switching to release {0} for package {1} in facility {2} for IOC {3}".format(args.release, name, iocFacilityCombo[0], iocFacilityCombo[1])) facility = iocFacilityCombo[0] ioc = iocFacilityCombo[1] commandsAndArgs = [] arg = {} arg['name'] = name arg['type'] = type arg['releaseFolder'] = toolconfig.determineReleaseFolder(facility, name, type) arg['linkFolder'] = toolconfig.determineLinkFolder(facility, name, type) arg['isMaster'] = ioc == 'MASTER' arg['facility'] = facility arg['ioc'] = ioc arg['release'] = release arg['user'] = getpass.getuser() commandsAndArgs.append(('upgrade', arg)) results = multi_facility_process_helper_client.executeCommandsOnFacility(facility, commandsAndArgs) if not results[0]['status']: raise Exception("Upgrading " + name + " of type " + type + " isMaster" + ioc == 'MASTER' + "ioc/facilty" + ioc + facility + "to" + release + " failed") if 'preDeployResults' in results[0] and 'stdout' in results[0]['preDeployResults']: print(results[0]['preDeployResults']['stdout']) if globals.verbose: print(results[0]['preDeployResults']['stderr']) if 'postDeployResults' in results[0] and 'stdout' in results[0]['postDeployResults']: print(results[0]['postDeployResults']['stdout']) if globals.verbose: print(results[0]['postDeployResults']['stderr']) print("Upgraded {0} package {1} in facilty {2} of type {3} to {4}".format(("ioc " + ioc + " in ") if type in ['SIOC', 'HIOC'] else "", name, facility, type, release)) def upgradeReleaseOnFacility(appName, appType, facility, ioc, release): commandsAndArgs = [] arg = {} arg['name'] = appName arg['type'] = appType arg['releaseFolder'] = toolconfig.determineReleaseFolder(facility, appName, appType) arg['linkFolder'] = toolconfig.determineLinkFolder(facility, appName, appType) arg['isMaster'] = ioc == 'MASTER' arg['facility'] = facility arg['ioc'] = ioc arg['release'] = release arg['user'] = getpass.getuser() commandsAndArgs.append(('upgrade', arg)) results = multi_facility_process_helper_client.executeCommandsOnFacility(facility, commandsAndArgs) if not results[0]['status']: raise Exception("Upgrading " + appName + " of type " + type + " isMaster" + ioc == 'MASTER' + "ioc/facilty" + ioc + facility + "to" + release + " failed") if 'preDeployResults' in results[0] and 'stdout' in results[0]['preDeployResults']: print(results[0]['preDeployResults']['stdout']) if globals.verbose: print(results[0]['preDeployResults']['stderr']) if 'postDeployResults' in results[0] and 'stdout' in results[0]['postDeployResults']: print(results[0]['postDeployResults']['stdout']) if globals.verbose: print(results[0]['postDeployResults']['stderr']) print( "Upgraded", ("ioc " + ioc + " in ") if type in ['SIOC', 'HIOC'] else "", "package", appName, "in facilty", facility, "of type", appType, "to", release) def execute(args): parser = argparse.ArgumentParser(prog="cram upgrade", description="Upgrades the specified IOC/HLA app/PyDM display to the specified release. Currently, one can specify one or ALL facilities and one or ALL IOCs.") parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose logging') parser.add_argument('-n', '--name', help='Specify the application name; we automatically pick this up from the package info if we are running this command in the folder containing the release') parser.add_argument('-t', '--type', help='Specifies the type of application from the deployment perspective. Possible values are ' + ", ".join(toolconfig.appTypes.keys())) parser.add_argument('-i', '--ioc', help='Deploys only to the specified ioc(s). If this is ALL, we deploy to all IOCs in the facilities as identified by the facility argument.') parser.add_argument('-f', '--facility', help='Deploys only to the specified facility(ies). If this is ALL, we deploy to all facilities. Possible values are - ALL, ' + ", ".join(toolconfig.getListOfFacilities()) + ". If you deploying to specific IOC's, you can skip this argument; cram will figure out which facility the IOC belongs to") parser.add_argument('-l', '--list', help='Only list the IOCs/master links that will affected. Do not actually make the change') parser.add_argument("release", help="Please specify a release that was previously submitted. You can use cram ls -p to determine a list of available releases.") args = parser.parse_args(args) globals.verbose = args.verbose try: upgradeRelease(args) except UserParamsException as upe: if not globals.verbose and not globals.isunittests: print(upe) sys.exit(-1) else: raise if __name__ == "__main__": execute(sys.argv[1:])