Job handling in Ganga Jakub T. Moscicki ARDA/LHCb GANGA-DIRAC Meeting, June, 2005

2 Ganga Object Model

3 Gaudi Application Object class Gaudi(GangaObject): _schema = Schema(Version(1,0),{ 'optsfile': FileItem(), 'version': SimpleItem(None), 'platform': SimpleItem(None), 'package': SimpleItem(None), 'appname': SimpleItem(None), 'cmt_release_area': SimpleItem(None), 'cmt_user_path': SimpleItem(None), 'masterpackage': SimpleItem(None), 'extraopts': SimpleItem(None)}) _category='applications' _name='Gaudi' def _auto__init__(self):... def configure(self):... extra_cfg=GaudiExtras() extra_cfg.flatopts=FileParser.writeString(gaudiopts,"expand") return (modified, extra_cfg) def list_choices(self,property):...

4 Job Submit

5 class GaudiLFSRunTimeHandler: def prepare(self,app,extra): (algpack,alg,algver)=app.masterpackage.split('/',3) script="""#!/usr/bin/env bash export CMTPATH=###CMTUSERPATH### export ###THEAPP###_release_area=###CMTRELEASEAREA### if [ -f ${LHCBHOME}/scripts/ProjectEnv.sh ]; then. ${LHCBHOME}/scripts/ProjectEnv.sh ###THEAPP### ###VERSION### else echo "Could not find the ProjectEnv.sh script. Your job will probably fail" fi mkdir -p cmttemp/v1/cmt cat >cmttemp/v1/cmt/requirements <<EOF use ###ALG### ###ALGVER### ###ALGPACK### EOF cmt setup -sh -quiet -pack=cmttemp -version=v1 -path=$PWD >cmttemp/v1/cmt/setup.sh. cmttemp/v1/cmt/setup.sh $###THEAPP###_release_area/###APPUPPER###/###APPUPPER###_###VERSION###/###PACKAGE###/###THEAPP###/###VE RSION###/###PLATFORM###/###THEAPP###.exe myopts.opts """ script=script.replace('###CMTUSERPATH###',app.cmt_user_path) script=script.replace('###THEAPP###',app.appname) script=script.replace('###CMTRELEASEAREA###',app.cmt_release_area) script=script.replace('###VERSION###',app.version) script=script.replace('###ALG###',alg) script=script.replace('###ALGVER###',algver) script=script.replace('###ALGPACK###',algpack) script=script.replace('###APPUPPER###',app.appname.toupper()) script=script.replace('###PACKAGE###',app.package) script=script.replace('###PLATFORM###',app.platform) return {'jobscript': ('myscript',script), 'inputbox':[('myopts.opts',extra.flatopts)]}

6 LSF Submit (1) def submit(self,jobid, jobconfig): inw = FileWorkspace.InputWorkspace() outw = FileWorkspace.OutputWorkspace() logger.info('LSF: submitting job %d',jobid) inw.create(jobid) outw.create(jobid) scriptpath = self.preparejob(jobid,jobconfig,inw,outw) # FIXME: garbbing stdout is done by shell magic and probably should be implemented in python directly rc,soutfile = shell_cmd('cd %s; bsub %s' % (inw.getPath(),scriptpath)) if rc == 0: sout = file(soutfile).read() import re m = re.compile(r"^Job \d*)> is submitted to (\S*) queue \S*)>.", re.M).search(sout) if m is None: logger.warning('could not match the output and extract the LSF job identifier!') logger.warning('command output \n %s ',sout) else: self.id = m.group('id') queue = m.group('queue') if self.queue != queue: self.queue = queue logger.warning('you requested queue "%s" but the job was submitted to queue "%s"',self.queue,queue) logger.warning('command output \n %s ',sout) logger.info('job %d submission OK',jobid) return rc == 0

7 LSF Submit (2) def preparejob(self,jobid,jobconfig,inw,outw): appscriptpath = inw.writefile(jobconfig['jobscript'],executable=1) # put files into job workdir (also to protect the originals while the job is running) sharedinputbox = map(lambda f: inw.writefile(f), jobconfig['inputbox']) sharedoutputbox=outw.getPath() print sharedoutputbox text = """#!/usr/bin/env python import shutil sharedinputbox = ###SHAREDINPUTBOX### sharedoutputbox= ###SHAREDOUTPUTBOX### for fn in sharedinputbox: shutil.copy(fn,'.') s = os.system('###APPSCRIPTNAME###') print 'DEBUG: Job finshed with exit code: ',s if s == 0: for fn in os.listdir('.'): if not os.path.isdir(fn): shutil.copy(fn,sharedoutputbox) # FIXME: needs recursive copy sys.exit(s) """ text = text.replace('###SHAREDINPUTBOX###',repr(sharedinputbox)) text = text.replace('###APPSCRIPTNAME###',appscriptpath) text = text.replace('###SHAREDOUTPUTBOX###',repr(sharedoutputbox)) return inw.writefile(('__jobscript__',text),executable=1)

8 Job Submit Sequence

9 Files/Job Repository File Workspace  store input/output sandbox of jobs and user files Local Workspace ~/__Ganga4__/workspace/input/* ~/__Ganga4__/workspace/output/* Remote Workspace not implemented: Lightweight SE using ARDA/MD Server Job Repository  store job information, NO FILES just references Remote Repository ARDA/MD Server with PostgreSQL Local Repository ~/__Ganga4__/repository/ganga_user

10 The END

11 LSF backend object class LSF(GangaObject): _schema = Schema(Version(1,0), {'queue' : SimpleItem(defvalue='8nm'), 'id' : SimpleItem(defvalue=None,protected=1,copyable=0), 'status' : SimpleItem(defvalue=None,protected=1,copyable=0) }) _category = 'backends' _name = 'LSF' def __init__(self): super(LSF,self).__init__()

12 LSF Monitoring def updateMonitoringInformation(jobs): rc,soutfile = shell_cmd('bjobs -a',allowed_exit=[0,255]) sout = file(soutfile).read() if rc == 0: import re m1 = re.compile(r"JOBID\s+USER\s+STAT\s+QUEUE").search(sout) if not m1: logger.warning('problem with understanding the bjobs output:\n%s',sout) else: items = re.compile(r"^(?P \d+)(\s*)(\S*)(\s*)(\S*)", re.M).findall(sout) ids = map(lambda x: x[0], items) for j in jobs: try: idx = ids.index(j.backend.id) new_status = items[idx][4] if j.backend.status != new_status: logger.info('%d: LSF job status changed to %s',j.id,new_status) j.backend.status = new_status if j.backend.status == 'DONE' or j.backend.status == 'ERROR': j.status = "completed" except ValueError: pass updateMonitoringInformation = staticmethod(updateMonitoringInformation)

13 Hello CLI Hello World:  # execute hello script locally  from Ganga.CLI import *  Job(exe='hello').submit() Hello DaVinci:  # execute DaVinci on the GRID # analysis will start at a worker node somewhere far far away ;-)  j = Job(name='serious analysis',backend="Glite")  j.application = DaVinciApplication(version='v12r3')  j.application.optionsfile = "DV-demo.opts"  j.application.outputfiles = ["DVNtuples.hbook"]  j.submit()

14 Jobs  # registry of persistent jobs  jobs() Statistics: 2 jobs registry ID status name # 1 new serious analysis # 2 submitted hello  # looping and selecting jobs  j = jobs()[1]  for j in jobs(): print j  for j in jobs()[2:9]: j.name = 'important!'  important = jobs()['important!']

15 Plugin Components Applications & Backends  # list plugin components  backends() ['TestSubmitter', 'Local', 'Glite']  applications() ['DaVinciApplication', 'TestApplication', 'Executable']  # creating objects  app = DaVinciApplication(optionsfile='some.opts')  bk = Local()  j.application, j.backend = app, bk  # creating objects by a string name  j.application = 'DaVinciApplication'  j.application.optionsfile = 'some.opts'  j.backend = 'Local'

16 Templates and Copying Copy jobs  # reuse existing jobs configuration to create new jobs  j = other_job.copy()  j = Job(template = other_job) Job templates  # job templates are just like any other jobs # except that their sole purpose it to store job configuration  t = JobTemplate(backend=LSF(queue='8nm'))  j = Job(template = t)  # templates are stored in a separate container  templates() Statistics: 1 jobs templates ID status name # 1 TEMPLATE None

17 Design Principles CLI Design Principles  Be predictable and follow python way of thinking  Increase complexity of interface with complexity of task: Simple tasks – simple! Complicated tasks – also simple ;) !  Try to prevent users from slient mistakes: job.id = 5 # FAILS: id is a read-only property finished_job.name = 'newname' # FAILS: job is finished so can't modify  Hide implementation: job._impl.attrs['id'] = 5  Be convinient and guide users j.application.exe j.exe # ALIASES of properties TAB completion shows properties and hides internals  Be flexible: good for writing complex macros/scripts...