From 8170e49bdf12c25d25e310ab47f57dede618b557 Mon Sep 17 00:00:00 2001 From: Stefano Belforte Date: Fri, 18 Oct 2024 23:26:29 +0200 Subject: [PATCH] do not download user sandbox in TW (#8745) --- src/python/CRABInterface/RESTCache.py | 15 ++++++----- .../TaskWorker/Actions/DagmanCreator.py | 27 ++++++++++++------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/python/CRABInterface/RESTCache.py b/src/python/CRABInterface/RESTCache.py index 478f3314c5..1ccc6213f7 100644 --- a/src/python/CRABInterface/RESTCache.py +++ b/src/python/CRABInterface/RESTCache.py @@ -123,14 +123,17 @@ def get(self, subresource, objecttype, taskname, username, tarballname, clientme if not objecttype: raise MissingParameter("objecttype is missing") if objecttype == 'sandbox': + # sandboxes can be reused across tasks, so are user, but not task, specific + # object path in the S3 container is: username/sandboxes/tarballname + # username indicates the user who owns the sandbox which may be different from + # the user who is making the REST call (authenticatedUserName). + # We rely on client to provide correct username. + # But when uploading from CRABClient, username is not known to the caller code + # for historical reasons so we settle for the authenticatedUserName. + # Having a special condition here is ugly but easier than fixing CRABClient if not tarballname: raise MissingParameter("tarballname is missing") - if subresource == 'upload' or (subresource == 'download' and clientmethod == 'head_object'): - ownerName = authenticatedUserName - else: - ownerName = username - if not ownerName: - raise MissingParameter("username is missing") + ownerName = username if username else authenticatedUserName # sandbox goes in bucket/username/sandboxes/ objectPath = ownerName + '/sandboxes/' + tarballname else: diff --git a/src/python/TaskWorker/Actions/DagmanCreator.py b/src/python/TaskWorker/Actions/DagmanCreator.py index dedb6fd52a..3a347d514e 100644 --- a/src/python/TaskWorker/Actions/DagmanCreator.py +++ b/src/python/TaskWorker/Actions/DagmanCreator.py @@ -17,7 +17,7 @@ from ast import literal_eval from ServerUtilities import MAX_DISK_SPACE, MAX_IDLE_JOBS, MAX_POST_JOBS, TASKLIFETIME -from ServerUtilities import getLock, downloadFromS3 +from ServerUtilities import getLock, downloadFromS3, checkS3Object, uploadToS3 import TaskWorker.DataObjects.Result from TaskWorker.Actions.TaskAction import TaskAction @@ -544,8 +544,7 @@ def makeJobSubmit(self, task): info['additional_environment_options'] += ' CRAB_TASKMANAGER_TARBALL=local' else: raise TaskWorkerException("Cannot find TaskManagerRun.tar.gz inside the cwd: %s" % os.getcwd()) - if os.path.exists("sandbox.tar.gz"): - info['additional_input_file'] += ", sandbox.tar.gz" + info['additional_input_file'] += ", sandbox.tar.gz" # it will be present on SPOOL_DIR after dab_bootstrap info['additional_input_file'] += ", run_and_lumis.tar.gz" info['additional_input_file'] += ", input_files.tar.gz" info['additional_input_file'] += ", submit_env.sh" @@ -728,6 +727,11 @@ def prepareLocal(self, dagSpecs, info, kw, inputFiles, subdags): tf.add(ifname) finally: tf.close() + # also upload InputFiles.tar.gz to s3 + task = kw['task']['tm_taskname'] + uploadToS3(crabserver=self.crabserver, filepath='InputFiles.tar.gz', + objecttype='runtimefiles', taskname=task, + logger=self.logger) def createSubdag(self, splitterResult, **kwargs): @@ -1124,6 +1128,7 @@ def extractMonitorFiles(self, inputFiles, **kw): Modifies inputFiles list by appending the debug folder if the extraction succeeds. """ + tarFileName = 'sandbox.tar.gz' if not os.path.isfile('debug_files.tar.gz') else 'debug_files.tar.gz' try: debugTar = tarfile.open(tarFileName) @@ -1195,20 +1200,24 @@ def executeInternal(self, *args, **kw): # Bootstrap the ISB if we are running in the TW if self.crabserver: username = kw['task']['tm_username'] + taskname = kw['task']['tm_taskname'] sandboxName = kw['task']['tm_user_sandbox'] dbgFilesName = kw['task']['tm_debug_files'] + self.logger.debug(f"Checking if sandbox file is available: {sandboxName}") try: - downloadFromS3(crabserver=self.crabserver, objecttype='sandbox', username=username, - tarballname=sandboxName, filepath=sandboxTarBall, logger=self.logger) + checkS3Object(crabserver=self.crabserver, objecttype='sandbox', taskname=taskname, + username=username, tarballname=sandboxName, logger=self.logger) kw['task']['tm_user_sandbox'] = sandboxTarBall except Exception as ex: - raise TaskWorkerException("The CRAB server backend could not download the input sandbox with your code " + \ + raise TaskWorkerException("The CRAB server backend could not find the input sandbox with your code " + \ "from S3.\nThis could be a temporary glitch; please try to submit a new task later " + \ - "(resubmit will not work) and contact the experts if the error persists.\nError reason: %s" % str(ex)) from ex + "(resubmit will not work) and contact the experts if the error persists." + \ + f"\nError reason: {ex}") from ex + # still need this download, until we change AdjustSites.py to do it there try: downloadFromS3(crabserver=self.crabserver, objecttype='sandbox', username=username, tarballname=dbgFilesName, filepath=debugTarBall, logger=self.logger) - except Exception as ex: # pylint: disable=broad-except + except Exception as ex: # pylint: disable=broad-except self.logger.exception(ex) # Bootstrap the runtime if it is available. @@ -1227,8 +1236,6 @@ def executeInternal(self, *args, **kw): self.extractMonitorFiles(inputFiles, **kw) - if kw['task'].get('tm_user_sandbox') == 'sandbox.tar.gz': - inputFiles.append('sandbox.tar.gz') if os.path.exists("CMSRunAnalysis.tar.gz"): inputFiles.append("CMSRunAnalysis.tar.gz") if os.path.exists("TaskManagerRun.tar.gz"):