#!/usr/bin/env python2 #============================================================== # # Abs: Module for executing processes across ssh connections # # Name: multi_facility_process_helper.py # # Facility: SLAC Controls (Development) # # Auth: 08-Jul-2013, M. Shankar (mshankar) #-------------------------------------------------------------- # Mods: (Latest to oldest) # 20-Aug-2021, A. Le (anle) # Add support for PyDM displays # Use "lexists" to catch broken softlinks # 19-Aug-2021, A. Le (anle) # Prevent listCurrentReleases from erroring out # 18-Aug-2021, A. Le (anle) # Handle "current" link error # 30-Aug-2019, A. Le (anle) # Add ability to force rewrite templates # 19-Aug-2019, A. Le (anle) # Suppress CVS stdout # 08-Aug-2019, A. Le (anle) # Update screeniocs on Prod instead of Dev # Move getScreenIOCSEntry and updateScreenIOCSEntry from multi_facility_process_helper.py to iocIsBorn.py # 07-Aug-2019, A. Le (anle) # Modify local copy of screeniocs instead of remote # #============================================================== """This provides functionality to make executing processes across ssh connections easier. We expect to drive this from python on the dev server by issuing commands thru a PIPE. Typical usage would be to subprocess a ssh executing this script (which is in the PATH) using a PIPE for stdin. Then, commands sent to the stdin PIPE on the dev side will be executed in the script running on the prod side. Commands execute well defined methods defined in this script. For example, a command to walk a folder and determine current versions of IOCS for a given package """ import os import sys import argparse import subprocess import json import string import pwd import globals import toolconfig import shutil import random import datetime import envPathsParser import cpuEnvPathsParser import cdCommandsParser import multiplePVGet import logging logger = logging.getLogger(__name__) def cd(arg): """Change the current working directory on the prod side to arg. arg is a string""" if globals.verbose: logger.debug("Changing current working folder to %s", arg) os.chdir(arg) def getCurDir(arg): """Prints the current working directory""" return json.dumps(os.path.abspath(os.path.curdir)) def getEnvVars(arg): "Returns the environment on the other side" if not arg: return os.environ.copy() elif type(arg) is str or type(arg) is unicode: if arg in os.environ: return os.environ[arg] elif type(arg) is list: ret = {} for nm in arg: if nm in os.environ: ret[nm] = (os.environ[nm]) return ret return False def mkdirs(arg): """Makes the specified folder if it does not exist The name of the folder is specified in the 'name' argument. A 'mode' argument can be used to specify the creation mode for the folder The mode argument is optional and looks like so 0744 The final call looks like so os.makedirs("testMode", 0744) """ if globals.verbose: logger.debug("Making the specified folder %s", arg) justcreated = False dirname = os.path.expandvars(arg['name']) if not os.path.exists(dirname): os.umask(0000) if'mode' in arg: os.makedirs(dirname, arg['mode']) else: os.makedirs(dirname) justcreated = True details = {'name' : arg, 'isFolder' : os.path.isdir(dirname), 'isLink' : os.path.islink(dirname), 'date' : os.path.getmtime(dirname), 'size' : os.path.getsize(dirname), 'justcreated' : justcreated } return details def link(arg): ''' Create a softlink between the specified files/folder. We expect two parameters target - the absolute path of the link target; this is the path that the "link" points to. link - the absolute path of the link; we expect the directory containing the link to already exist. ''' targetPath = os.path.expandvars(arg['target']) linkPath = os.path.expandvars(arg['link']) relpath = os.path.relpath(targetPath, os.path.dirname(linkPath)) os.chdir(os.path.dirname(linkPath)) if os.path.exists(linkPath): if globals.verbose: logger.debug("Removing existing link %s", linkPath) os.unlink(linkPath) if globals.verbose: logger.debug("Changing relative link %s in folder %s", relpath, os.getcwd()) os.symlink(relpath, os.path.basename(linkPath)) def ls(arg): '''Return a list of dicts of the contents of a folder As input, we expect a list of foldernames (strings) We return a list of dicts. Each dict contains name - The name of the file/subfolder parent - The name of the parent folder isFolder - Is this a folder? isLink - Is this a link? pointingTo - If this is a link, this is what the link is pointing to date - The date associated with the file/subfolder size - The size of the file/folder''' folders = arg result = [] for folder in folders: if not os.path.exists(folder): continue for path in sorted(os.listdir(folder)): full = os.path.join(folder, path) # This seems to be a good test of valid/invalid links. if os.path.exists(full): details = {'name' : path, 'parent' : folder, 'isFolder' : os.path.isdir(full), 'isLink' : os.path.islink(full), 'date' : os.path.getmtime(full), 'size' : os.path.getsize(full)} if os.path.islink(full): details['pointingTo'] = os.readlink(full) result.append(details) return result def fileinfo(arg): '''Return information about a list of files. Differs from ls in that it returns info about the files/folders themselves - not their contents. Think ls -ld As input, we expect a list of pathnames (strings) - these will be mostly full paths We return a dict of dicts - key is name. Value is a dict containing name - The path name of the file/folder isFolder - Is this a folder? isLink - Is this a link? pointingTo - If this is a link, this is what the link is pointing to date - The date associated with the file/subfolder size - The size of the file/folder''' paths = arg result = {} for path in paths: path = os.path.expandvars(path) if not os.path.exists(path): continue details = {'name' : path, 'isFolder' : os.path.isdir(path), 'isLink' : os.path.islink(path), 'date' : os.path.getmtime(path), 'size' : os.path.getsize(path)} if os.path.islink(path): details['pointingTo'] = os.readlink(path) result[path] = details return result def loadJSON(arg): '''Loads the contents of a file as JSON and returns it. As input, we expect a path to the file containing the data. ''' jsonpath = arg with open(jsonpath,'r') as f: ret = json.load(f) return ret def determineIOCVersions(arg): """Determine the versions that IOC's belonging to a particular software package are pointing to. We determine where the master release for the package is pointing to. We determine where the iocSpecificRelease is pointing to; if it is pointing to the master release, we use that value instead. As inputs, we expect a dict with name - package name releaseFolder - The release folder; basically iocTop. IOC's whose iocSpecificRelease points to a folder in releaseFolder/name are considered to be part of this package. releaseFolder/current points to the master release. linkFolder - This is where the IOC links are; basically, iocCommon We output a dict with two elements master - This is the master release iocs - A list of dicts. Each dict has a ioc, release and boolean isMaster element. """ name = arg['name'] releaseFolder = arg['releaseFolder'] linkFolder = arg['linkFolder'] if globals.verbose: logger.debug("Determing IOC versions for %s using the release folder %s and link folder %s", name, releaseFolder, linkFolder) # Determine the master release thisPackageReleasePath = os.path.join(releaseFolder, name) if globals.verbose: logger.debug("Package release path %s", thisPackageReleasePath) currentPath = os.path.join(os.path.realpath(thisPackageReleasePath), "current") if not os.path.islink(currentPath): if os.path.exists(currentPath): raise Exception(currentPath + " is not a link. Cannot determine master release") elif os.path.exists(thisPackageReleasePath): # We only care that "current" exists if the package release path also exists raise Exception('"current" link not found. Cannot determine master release') return {} masterReleasePath = os.path.realpath(currentPath) masterRelease = os.path.basename(masterReleasePath) if globals.verbose: logger.debug("Master release is %s", masterRelease) iocReleases = [] linkFolderDirs = filter(lambda x : os.path.isdir(x), map(lambda x : os.path.join(linkFolder, x), os.listdir(linkFolder))) for linkFolderDir in linkFolderDirs: if os.path.islink(os.path.join(linkFolderDir, 'iocSpecificRelease')): # We are in an IOC folder linkisPointingTo = os.readlink(os.path.join(linkFolderDir, 'iocSpecificRelease')) # Normalized path is the path without the ..'s. iocNormalizedPath = os.path.normpath(os.path.join(linkFolderDir, linkisPointingTo)) # If the last part of the normalized path is current, we are pointing to the master release if globals.verbose: logger.debug("%s %s -> %s", linkFolderDir, linkisPointingTo, iocNormalizedPath) if thisPackageReleasePath == os.path.dirname(iocNormalizedPath): iocRelease = os.path.basename(iocNormalizedPath) iocIsPointingToMasterRelease = False if iocRelease == 'current': iocIsPointingToMasterRelease = True iocRelease = masterRelease iocReleases.append({"ioc" : os.path.basename(linkFolderDir), "release" : iocRelease, "isMaster" : iocIsPointingToMasterRelease}) result = {} result['master'] = masterRelease result['iocs'] = sorted(iocReleases, key=lambda x: x['ioc']) return result def determineIOCRunningVersions(arg): """Determine the currently running version on the IOC. As inputs, we expect a dict with name - package name releaseFolder - The release folder; basically iocTop. IOC's whose iocSpecificRelease points to a folder in releaseFolder/name are considered to be part of this package. releaseFolder/current points to the master release. linkFolder - This is where the IOC links are; basically, iocCommon timeout - Time in seconds to wait to get the PV information. We output a dict of IOC running versions. Each dict has ioc as the key and a release as the value. The release is computed by converting the IOC name into the APP_DIR PVs For example, IOC sioc-sys0-ml00 is converted to two PV's - SIOC:SYS0:ML00:APP_DIR1 and SIOC:SYS0:ML00:APP_DIR2. We combine the values of these - /afs/slac.stanford.edu/g/lcls/vol8/epic and s/iocTop/MatlabSupport/MatlabSupport-R3 and get the last folder name to get the running version - MatlabSupport-R3 """ name = arg['name'] releaseFolder = arg['releaseFolder'] linkFolder = arg['linkFolder'] timeout = int(arg['timeout']) if globals.verbose: logger.debug("Determing running IOC versions for %s using the release folder %s and link folder %s using timeout", name, releaseFolder, linkFolder, timeout) # Determine the master release thisPackageReleasePath = os.path.join(releaseFolder, name) if globals.verbose: logger.debug("Package release path %s", thisPackageReleasePath) currentPath = os.path.join(os.path.realpath(thisPackageReleasePath), "current") if not os.path.islink(currentPath): if os.path.exists(currentPath): raise Exception(currentPath + " is not a link. Cannot determine master release") else: return {} masterReleasePath = os.path.realpath(currentPath) masterRelease = os.path.basename(masterReleasePath) if globals.verbose: logger.debug("Master release is %s", masterRelease) iocRunningReleases = {} iocRunningReleasesPVNames = [] ioc2PVNames = {} linkFolderDirs = filter(lambda x : os.path.isdir(x), map(lambda x : os.path.join(linkFolder, x), os.listdir(linkFolder))) for linkFolderDir in linkFolderDirs: if os.path.islink(os.path.join(linkFolderDir, 'iocSpecificRelease')): # We are in an IOC folder linkisPointingTo = os.readlink(os.path.join(linkFolderDir, 'iocSpecificRelease')) # Normalized path is the path without the ..'s. iocNormalizedPath = os.path.normpath(os.path.join(linkFolderDir, linkisPointingTo)) # If the last part of the normalized path is current, we are pointing to the master release if globals.verbose: logger.debug("%s %s -> %s", linkFolderDir, linkisPointingTo, iocNormalizedPath) if thisPackageReleasePath == os.path.dirname(iocNormalizedPath): iocName = os.path.basename(linkFolderDir) appDir1PVName = iocName.replace('-', ':').upper() + ':APP_DIR1' appDir2PVName = iocName.replace('-', ':').upper() + ':APP_DIR2' if globals.verbose: logger.debug( "Determining IOC running versions using %s %s for IOC %s", appDir1PVName, appDir2PVName, iocName) iocRunningReleasesPVNames.append(appDir1PVName) iocRunningReleasesPVNames.append(appDir2PVName) ioc2PVNames[iocName] = (appDir1PVName, appDir2PVName) pv2vals = multiplePVGet.getMultiplePVs(iocRunningReleasesPVNames, timeout) if globals.verbose: logger.debug(pv2vals) result = {} for (iocName, runPVNames) in ioc2PVNames.iteritems(): if runPVNames[0] in pv2vals and runPVNames[1] in pv2vals: completeVersion = pv2vals[runPVNames[0]] + pv2vals[runPVNames[1]] if globals.verbose: logger.debug( "Complete version for IOC %s is %s", iocName, completeVersion) runningVersion = os.path.basename(completeVersion) result[iocName] = runningVersion return result def determineHLAVersion(arg): '''Determine the HLA version for the specified software package. As inputs, we expect a dict with name - package name releaseFolder - The release folder; basically PHYSICS_TOP/releases or path to tools/scripts/releases. linkFolder - This is where the HLA links are; basically, PHYSICS_TOP or tools/scripts Return is a simple string''' name = arg['name'] releaseFolder = arg['releaseFolder'] linkFolder = arg['linkFolder'] if globals.verbose: logger.debug( "Determing HLA version for %s using the release folder %s and link folder %s", name, releaseFolder, linkFolder) hlaMasterLinkPath = os.path.join(linkFolder, name) if not os.path.islink(hlaMasterLinkPath): if os.path.exists(hlaMasterLinkPath): raise Exception(hlaMasterLinkPath + " is not a link. Cannot determine release for " + name) else: return None masterReleasePath = os.path.realpath(hlaMasterLinkPath) masterRelease = os.path.basename(masterReleasePath) if globals.verbose: logger.debug( "Master release is %s", masterRelease) return masterRelease def determinePyDMDisplayVersion(arg): '''Determine the version for the specified PyDM display package. As inputs, we expect a dict with name - package name releaseFolder - The release folder; basically $PYDM_DISPLAYS_PATH/release or tools/pydm/display/release. linkFolder - This is where the PyDM subsystem links are; basically, PYDM_DISPLAYS_PATH or tools/pydm/display Return is a simple string''' name = arg['name'] # Remove pydm prefix from package name to get subsystem if name.startswith('pydm-'): linkName = name.replace('pydm-', '', 1) else: linkName = name releaseFolder = arg['releaseFolder'] linkFolder = arg['linkFolder'] if globals.verbose: logger.debug( "Determining PyDM display version for %s using the release folder %s and link folder %s", name, releaseFolder, linkFolder) pydmDisplayMasterLinkPath = os.path.join(linkFolder, linkName) if not os.path.islink(pydmDisplayMasterLinkPath): if os.path.exists(pydmDisplayMasterLinkPath): raise Exception(pydmDisplayMasterLinkPath + " is not a link. Cannot determine release for " + name) else: return None masterReleasePath = os.path.realpath(pydmDisplayMasterLinkPath) masterRelease = os.path.basename(masterReleasePath) if globals.verbose: logger.debug( "Master release is %s", masterRelease) return masterRelease def generateReplaceDictForEnvPaths(releaseFolder, name, release): ''' Generate a replacement dictionary for envPaths based on iocTop, enviromnent variables and other parameters''' replaceDict = {} # TOP => The path to the release that we are upgrading to - // replaceDict['TOP'] = os.path.join(releaseFolder, name, release) # EPICS_SITE_TOP => Value of the environment variable EPICS_TOP if 'EPICS_TOP' in os.environ: replaceDict['EPICS_SITE_TOP'] = os.environ['EPICS_TOP'] # BASE_SITE_TOP => Value of the environment variable EPICS_BASE_TOP if 'EPICS_BASE_TOP' in os.environ: replaceDict['BASE_SITE_TOP'] = os.environ['EPICS_BASE_TOP'] # IOC_SITE_TOP => iocTop for this facility (as defined in the facilities.cfg) replaceDict['IOC_SITE_TOP'] = releaseFolder if 'PACKAGE_TOP' in os.environ: replaceDict['PACKAGE_SITE_TOP'] = os.environ['PACKAGE_TOP'] if 'TOOLS' in os.environ: replaceDict['TOOLS_SITE_TOP'] = os.environ['TOOLS'] replaceDict['ALARM_CONFIGS_TOP'] = os.path.join(os.environ['TOOLS'], 'AlarmConfigsTop') return replaceDict def fixEnvPaths(name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release): '''This runs standard preDeploy scripts liking fixing envPaths, copying over display files etc Currently, here's the list of actions in sequence 1) If this is an IOC app, we fix the envPaths for this IOC. This action does not run if this is an HLA or a PyDM display package. If this is an IOC master link, we fix the envPaths for all IOCs in this release. ''' if type in ['SIOC', 'HIOC']: for bootFolderName in ['iocBoot', 'cpuBoot', 'build', 'children']: iocBootFolder = os.path.join(releaseFolder, name, release, bootFolderName) if globals.verbose: logger.debug( "Changing envPaths for %s in %s", "ALL IOCs" if ioc == 'MASTER' else "IOC " + ioc, iocBootFolder) for root, dirs, files in os.walk(iocBootFolder): for file in files: if ioc != "MASTER" and os.path.split(root)[1] != ioc: logger.debug("Skipping changing envpaths for some other ioc %s", os.path.split(root)[1]) continue if file == 'envPaths' or file == 'cpuEnv.sh': if globals.verbose: logger.debug( "Changing env paths in %s", os.path.join(root, file)) replaceDict = generateReplaceDictForEnvPaths(releaseFolder, name, release) if file == 'envPaths': envPathsParser.parseAndReplaceEnvPathsFile(os.path.join(root, file), replaceDict) elif file == 'cpuEnv.sh': cpuEnvPathsParser.parseAndReplaceEnvPathsFile(os.path.join(root, file), replaceDict, 'PACKAGE_SITE_TOP') elif file == 'cdCommands': if globals.verbose: logger.debug( "Changing cdCommands in %s", os.path.join(root, file)) replaceDict = generateReplaceDictForEnvPaths(releaseFolder, name, release) cdCommandsParser.parseAndReplaceCDCommandsFile(os.path.join(root, file), replaceDict) def fixEnvPathsForAllIOCs(arg): ''' After a push (especially after a push --freshen), we'd want to at least fix the envPaths for a all IOCs in this release As inputs, we can expect a dict with name - package name type - package type facility - This facility releaseFolder - The folder where the releases are stored for this package linkFolder - The folder containing the softlinks release - The release whose envPaths we want to fix. ''' name = arg['name'] type = arg['type'] facility = arg['facility'] releaseFolder = arg['releaseFolder'] linkFolder = arg['linkFolder'] release = arg['release'] if globals.verbose: logger.debug( "Fixing envPaths for all IOCs for %s of type %s with release and link folders %s %s for release %s", name, type, releaseFolder, linkFolder, release) fixEnvPaths(name, type, releaseFolder, linkFolder, True, facility, 'MASTER', release) def runPrePostDeployScript(scriptpath, name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release): '''Runs the named script in the releaseFolder for this release it is exists. For example, preDeploy is releaseFolder///bin/preDeploy In this case, the scriptpath is bin/preDeploy Several environment variables are passed in. Environment variables being with CRAM_ ''' scriptPath = os.path.join(releaseFolder, name, release, scriptpath) if os.path.exists(scriptPath): if globals.verbose: logger.debug("Executing script {0} for {1} of type {2} in {3}" \ " with links {4} {5} facility {6} ioc {7} and" \ " release {8}".format(scriptpath, name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release)) myenvs = dict(os.environ) myenvs['CRAM_SCRIPT'] = scriptpath myenvs['CRAM_PACKAGENAME'] = name myenvs['CRAM_PACKAGETYPE'] = type myenvs['CRAM_RELEASEFOLDER'] = os.path.join(releaseFolder, name, release) myenvs['CRAM_RELEASE'] = release myenvs['CRAM_LINKFOLDER'] = linkFolder myenvs['CRAM_ISMASTER'] = 'true' if isMaster else 'false' myenvs['CRAM_FACILITY'] = facility myenvs['CRAM_IOC'] = ioc startingDir = os.path.realpath(os.curdir) try: os.chdir(os.path.join(releaseFolder, name, release)) ret = subprocess.Popen(scriptpath, env=myenvs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, err = ret.communicate() if globals.verbose: logger.debug("Return value from script is %s", ret) return {'retcode' : ret.returncode, 'stdout' : output, 'stderr': err} except Exception as e: logger.exception("Exception when executing %s in folder %s", scriptpath, os.curdir) raise e else: os.chdir(startingDir) else: if globals.verbose: logger.debug("Not present script {0} for {1} of type {2} in {3}" \ " with links {4} {5} facility {6} ioc {7} and " \ "release {8}".format(scriptpath, name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release)) return {'retcode' : 0, "reason": "Script %s not present" % (scriptpath)} def run(arg): '''Run the specified script in the release folder for the specified IOC/HLA app/PyDM display As inputs, we can expect a dict with script - relative path to the script name - package name type - package type releaseFolder - The folder where the releases are stored for this package linkFolder - The folder containing the softlinks isMaster - Are we switching the master link? In the case of HLA applications and PyDM displays, this is always true. facility - The facility that this IOC belongs to. Being passed here so that we can pass it to the preDeploy and postDeploy ioc - The name of the IOC for which we are upgrading. In case of the master link, HLA applications, or PyDM displays, this will be MASTER release - The release we want to go to. ''' script = arg['script'] name = arg['name'] type = arg['type'] releaseFolder = arg['releaseFolder'] linkFolder = arg['linkFolder'] isMaster = arg['isMaster'] facility = arg['facility'] ioc = arg['ioc'] release = arg['release'] if globals.verbose: logger.debug( "Running %s for %s of type %s with release and link folders %s %s isMaster %s ioc/facilty %s %s within %s", script, name, type, releaseFolder, linkFolder, isMaster, ioc, facility, release) runRes = runPrePostDeployScript(script, name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) return runRes def updateHistory(name, type, releaseFolder, ioc, release, user): '''Adds an entry to the history file - releaseFolder/multi_facility_hist releaseFolder/multi_facility_hist is a dict persisted with shelve. Keys are ioc (MASTER for the master link/HLA/PyDM). Values are a ordered list of release numbers that this ioc went thru; recent releases first; older releases last This is not really transactional; we should look into whether we can use locks on NFS. ''' histFolder = os.path.join(releaseFolder, name) histFilePath = os.path.join(releaseFolder, name, 'multi_facility_hist') newFilePath = os.path.join(histFolder, "tmp_" + str(int(random.random()*1000000))) while os.path.exists(newFilePath): logger.debug("Generating a new name for the temporary file\n") newFilePath = os.path.join(histFolder, "tmp_" + str(int(random.random()*1000000))) histdict = {} if os.path.exists(histFilePath): with open(histFilePath, 'r') as f: histdict = json.load(f) releasehist = [] if ioc in histdict: releasehist = histdict[ioc] releaseinfo = {} releaseinfo['release'] = release releaseinfo['user'] = user # To parse this, use datetime.datetime.strptime(datetime.datetime.now().replace(microsecond=0).isoformat(), '%Y-%m-%dT%H:%M:%S') releaseinfo['datetime'] = datetime.datetime.now().replace(microsecond=0).isoformat() releasehist.insert(0, releaseinfo) histdict[ioc] = releasehist with open(newFilePath, 'w') as fn: json.dump(histdict, fn) os.rename(newFilePath, histFilePath) def upgrade(arg): '''All the steps necessary for upgrading a release As inputs, we can expect a dict with name - package name type - package type releaseFolder - The folder where the releases are stored for this package linkFolder - The folder containing the softlinks isMaster - Are we switching the master link? In the case of HLA applications and PyDM displays, this is always true. facility - The facility that this IOC belongs to. Being passed here so that we can pass it to the preDeploy and postDeploy ioc - The name of the IOC for which we are upgrading. In case of the master link or HLA applications and PyDM displays, this will be MASTER release - The release we want to go to. user - The user who initiated this command What we do. 1) Call the predeploy if it exists 2) Switch the link - we use a relative link. 3) Custom processing that's built into cram - like copying over EDM displays etc, updating the path for EPICS base. 4) Call the postdeploy if it exists We return True if this succeeded. ''' name = arg['name'] type = arg['type'] releaseFolder = arg['releaseFolder'] linkFolder = arg['linkFolder'] isMaster = arg['isMaster'] facility = arg['facility'] ioc = arg['ioc'] release = arg['release'] user = arg['user'] if globals.verbose: logger.debug("Upgrading %s of type %s with release and link folders %s %s isMaster %s ioc/facilty %s %s to %s", name, type, releaseFolder, linkFolder, isMaster, ioc, facility, release) if type in ['SIOC', 'HIOC']: if isMaster: newReleasePath = os.path.join(releaseFolder, name, release) masterLinkPath = os.path.join(releaseFolder, name, 'current') relpath = os.path.relpath(newReleasePath, os.path.dirname(masterLinkPath)) if globals.verbose: logger.debug( "Changing master - relative path is %s", relpath) os.chdir(os.path.dirname(masterLinkPath)) preRunRes = runPrePostDeployScript('bin/preDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) fixEnvPaths(name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) if os.path.lexists('current'): os.remove('current') os.symlink(relpath, 'current') postRunRes = runPrePostDeployScript('bin/postDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) else: # We check to see is the new version is the same as the master release version. # If so, we point to the master release. masterLinkPath = os.path.join(releaseFolder, name, 'current') masterRelease = os.path.basename(os.path.realpath(masterLinkPath)) if release == masterRelease: iocLinkPath = os.path.join(linkFolder, ioc, 'iocSpecificRelease') relpath = os.path.relpath(masterLinkPath, os.path.dirname(iocLinkPath)) if globals.verbose: logger.debug( "Repointing to master - relative path is %s", relpath) os.chdir(os.path.dirname(iocLinkPath)) preRunRes = runPrePostDeployScript('bin/preDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) fixEnvPaths(name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) if os.path.lexists('iocSpecificRelease'): os.remove('iocSpecificRelease') os.symlink(relpath, 'iocSpecificRelease') postRunRes = runPrePostDeployScript('bin/postDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) else: newReleasePath = os.path.join(releaseFolder, name, release) iocLinkPath = os.path.join(linkFolder, ioc, 'iocSpecificRelease') relpath = os.path.relpath(newReleasePath, os.path.dirname(iocLinkPath)) if globals.verbose: logger.debug( "Pointing IOC to specific release, relative path is %s", relpath) os.chdir(os.path.dirname(iocLinkPath)) preRunRes = runPrePostDeployScript('bin/preDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) fixEnvPaths(name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) if os.path.lexists('iocSpecificRelease'): os.remove('iocSpecificRelease') os.symlink(relpath, 'iocSpecificRelease') postRunRes = runPrePostDeployScript('bin/postDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) else: # For PyDM displays, the link name is the subsystem, without "pydm-" if type == 'PyDM' and name.startswith('pydm-'): linkName = name.replace('pydm-', '', 1) else: linkName = name newReleasePath = os.path.join(releaseFolder, name, release) masterLinkPath = os.path.join(linkFolder, linkName) relpath = os.path.relpath(newReleasePath, os.path.dirname(masterLinkPath)) if globals.verbose: logger.debug( "Relative path is %s", relpath) os.chdir(os.path.dirname(masterLinkPath)) preRunRes = runPrePostDeployScript('bin/preDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) fixEnvPaths(name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) if os.path.lexists(linkName): os.remove(linkName) os.symlink(relpath, linkName) postRunRes = runPrePostDeployScript('bin/postDeploy', name, type, releaseFolder, linkFolder, isMaster, facility, ioc, release) try: updateHistory(name, type, releaseFolder, ioc, release, user) except: logger.exception("Unexpected exception adding release to history db") logger.debug( "Files in current folder %s", os.listdir(os.getcwd())) pass return {'status' : True, 'preDeployResults' : preRunRes, 'postDeployResults' : postRunRes} def hist(arg): '''Returns the entire history of a software package As inputs, we can expect a dict with name - package name type - package type releaseFolder - The folder where the releases are stored for this package We return the history - this is a dict with IOC names as keys and a linked list of releases as values - see updateHistory above. ''' name = arg['name'] type = arg['type'] releaseFolder = arg['releaseFolder'] histFolder = os.path.join(releaseFolder, name) histFilePath = os.path.join(releaseFolder, name, 'multi_facility_hist') histdict = {} if os.path.exists(histFilePath): with open(histFilePath, 'r') as f: histdict = json.load(f) return histdict def removeFolder(arg): '''Removes the folder specified by the argument. As input, we expect the name of the folder to delete.''' folderName = arg if os.path.exists(folderName): logger.debug("Removing folder %s", folderName) shutil.rmtree(folderName) return True def templateReplace(arg): ''' Template replaces a source file into a dest file. In addition to the environment, we expect a dict of additional arguments that are to be expanded. As inputs we expect a dict with source - Source of the template content - Optionally, you can send the content itself as the value of content dest - The template is expanded into this file. templatevars - An optional dict that contains additional name/value pairs ''' if 'source' in arg: srcFile = os.path.expandvars(arg['source']) if not os.path.exists(srcFile): logger.debug("Template source {} does not exist".format(srcFile)) return {'status' : 'Template source does not exist'} with open(srcFile, 'r') as f: lines = f.readlines() elif 'content' in arg: lines = [ x + "\n" for x in arg['content'].split("\n")] else: logger.debug("One of 'content or 'source' must be specified'") return {'status' : "One of 'content or 'source' must be specified'"} destFile = os.path.expandvars(arg['dest']) templateVars = os.environ.copy() if 'templatevars' in arg: templateVars.update(arg['templateVars']) if os.path.exists(destFile) and not ('force' in arg and arg['force']): logger.debug("Template destination {} already exists".format(destFile)) return {'status' : 'Template destination already exists'} with open(destFile, 'w') as f: for line in lines: op0 = string.Template(line).substitute(templateVars) op1 = op0.replace("^^", "$") f.write(op1) return {'status' : True} def updateScreenIOCs(arg): '''Updates screeniocs on prod''' # Go to $IOC_SCREEN iocScreenDir = os.path.expandvars("$IOC_SCREEN") os.chdir(iocScreenDir) # Update screeniocs subprocess.call(['cvs', '-Q', 'update', 'screeniocs']) return {'status' : True} def exit(arg): """Exit from python on the prod side""" sys.exit() commandMappings = { "cd" : cd, "pwd" : getCurDir, "env" : getEnvVars, "mkdirs" : mkdirs, "link": link, "ls" : ls, "fileinfo": fileinfo, "loadJSON": loadJSON, "determineIOCVersions" : determineIOCVersions, "determineIOCRunningVersions" : determineIOCRunningVersions, "determineHLAVersion" : determineHLAVersion, "determinePyDMDisplayVersion" : determinePyDMDisplayVersion, "run" : run, "upgrade" : upgrade, "hist": hist, "removeFolder" : removeFolder, "fixEnvPathsForAllIOCs" : fixEnvPathsForAllIOCs, "templateReplace": templateReplace, "updateScreenIOCs" : updateScreenIOCs, "exit": exit } if __name__ == "__main__": '''Read the stdin and process commands. Command syntax is simple. Commands have one and only one argument. Commands come as a list of tuples of size 2. First item in tuple is the command name that is one of the keys in commandMappings above. Second item is the argument to this command. Note each arg is different - for example, it can be a dict with multiple keys. We batch execute all the commands and send the results as a JSON list. Bear in mind the command and the JSON'ed args cannot have newlines. Note, if you pass in a -v argument when calling this across the wire, your JSON parsing on the client side will not work. Use the -v only for debugging and do not pass it thru ''' root = logging.getLogger() root.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stderr) ch.setLevel(logging.DEBUG) root.addHandler(ch) parser = argparse.ArgumentParser(prog="cram") parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose logging') args = parser.parse_args() globals.verbose = args.verbose commandAndArgs = json.load(sys.stdin) logger.debug("Received %s from the client", commandAndArgs) results = [] for commandAndArg in commandAndArgs: command = commandAndArg[0] arg = commandAndArg[1] if globals.verbose: logger.debug("Processing %s with arguments %s", command, arg) commandresult = commandMappings[command](arg) if globals.verbose: logger.debug("Command Result %s", commandresult) results.append(commandresult) json.dump(results, sys.stdout)