From c91d11cd1c712639c9639ba6878cdd18d008a2f6 Mon Sep 17 00:00:00 2001 From: tlatorre Date: Mon, 18 May 2020 11:18:27 -0500 Subject: always resubmit jobs in the RETRY state --- utils/submit-grid-jobs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'utils/submit-grid-jobs') diff --git a/utils/submit-grid-jobs b/utils/submit-grid-jobs index 1796e99..29f7d9c 100755 --- a/utils/submit-grid-jobs +++ b/utils/submit-grid-jobs @@ -400,7 +400,11 @@ def main(conn, dqxx_dir, max_retries, max_jobs): else: # Don't know what to do here for Removed or Submission_err log.warn("Job %i is in the state %i. Don't know what to do." % (id, job_status)) - elif state == 'RETRY' and nretry < max_retries: + elif state == 'RETRY': + if njobs >= max_jobs: + log.verbose("Skipping job %i because there are already %i jobs in the queue" % (id,njobs)) + continue + log.notice("Resubmitting job %i from RETRY state" % id) if submit_job(submit_file): log.warn("Failed to resubmit job %i. Setting it to FAILED state." % id) -- cgit