add a script to submit jobs to the grid

7 files changed, 156 insertions, 2 deletions

diff --git a/Makefile b/Makefile
index 2b589f0..27445de 100644
--- a/Makefile
+++ b/Makefile
@@ -5,4 +5,5 @@ default: all
 
 .DEFAULT:
 	cd src && "$(MAKE)" $@
+	cd utils && "$(MAKE)" $@
 
diff --git a/src/Makefile b/src/Makefile
index bd77dfb..a0eef06 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -3,6 +3,10 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh')
 CFLAGS=-fdiagnostics-color -O2 -Wall -g -DSWAP_BYTES
 LDLIBS=-fdiagnostics-color -lm -lgsl -lgslcblas -lnlopt_cxx -lstdc++
 
+PREFIX?=$(HOME)/local
+INSTALL_BIN=$(PREFIX)/bin
+INSTALL=install
+
 all: test test-vector test-likelihood fit test-charge test-path calculate-csda-range test-time-pdf test-zebra test-find-peaks zdab-cat
 
 Makefile.dep:
@@ -36,6 +40,10 @@ calculate-csda-range: calculate-csda-range.o
 
 zdab-cat: zdab-cat.o zebra.o pmt.o vector.o misc.o zdab_utils.o pack2b.o db.o dqxx.o dict.o siphash.o release.o dc.o sort.o util.o
 
+install:
+	@mkdir -p $(INSTALL_BIN)
+	$(INSTALL) fit $(INSTALL_BIN)
+
 clean:
 	rm -f *.o calculate_limits test test-vector test-likelihood fit test-charge test-path calculate-csda-range test-time-pdf test-zebra test-find-peaks Makefile.dep zdab-cat
 
diff --git a/src/fit.c b/src/fit.c
index 22028df..0307437 100644
--- a/src/fit.c
+++ b/src/fit.c
@@ -6173,7 +6173,8 @@ skip_mc:
             nhit = get_nhit(&ev);
 
             if (fout) {
-                fprintf(fout, "      - gtid: %i\n", ev.gtid);
+                fprintf(fout, "      - run: %i\n", ev.run);
+                fprintf(fout, "        gtid: %i\n", ev.gtid);
                 fprintf(fout, "        nhit: %zu\n", nhit);
             }
 
diff --git a/src/zdab-cat.c b/src/zdab-cat.c
index a109b46..d891c78 100644
--- a/src/zdab-cat.c
+++ b/src/zdab-cat.c
@@ -92,7 +92,9 @@ int main(int argc, char **argv)
             fprintf(stderr, "failed to open '%s': %s\n", output, strerror(errno));
             return 1;
         }
+    }
 
+    if (fout) {
         fprintf(fout, "git_sha1: %s\n", GitSHA1());
         fprintf(fout, "git_dirty: %s\n", GitDirty());
         fprintf(fout, "data:\n");
@@ -293,7 +295,8 @@ skip_mc:
             nhit = get_nhit(&ev);
 
             if (fout) {
-                fprintf(fout, "      - gtid: %i\n", ev.gtid);
+                fprintf(fout, "      - run: %i\n", ev.run);
+                fprintf(fout, "        gtid: %i\n", ev.gtid);
                 fprintf(fout, "        nhit: %zu\n", nhit);
             }
 
diff --git a/utils/Makefile b/utils/Makefile
new file mode 100644
index 0000000..348c4f2
--- /dev/null
+++ b/utils/Makefile
@@ -0,0 +1,16 @@
+# install python scripts with `make install`
+
+PREFIX?=$(HOME)/local
+INSTALL_BIN=$(PREFIX)/bin
+INSTALL=install
+
+all:
+
+install:
+	@mkdir -p $(INSTALL_BIN)
+	$(INSTALL) plot $(INSTALL_BIN)
+	$(INSTALL) plot-fit-results $(INSTALL_BIN)
+	$(INSTALL) plot-likelihood $(INSTALL_BIN)
+	$(INSTALL) submit-grid-jobs $(INSTALL_BIN)
+
+.PHONY: install
diff --git a/utils/submit-grid-jobs b/utils/submit-grid-jobs
new file mode 100755
index 0000000..1ada09f
--- /dev/null
+++ b/utils/submit-grid-jobs
@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# Copyright (c) 2019, Anthony Latorre <tlatorre at uchicago>
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import print_function, division
+import yaml
+import string
+from os.path import split, splitext, join
+import uuid
+from subprocess import check_call
+
+# all files required to run the fitter (except the DQXX files)
+INPUT_FILES = ["muE_water_liquid.txt","pmt_response_qoca_d2o_20060216.dat","rsp_rayleigh.dat","e_water_liquid.txt","pmt_pcath_response.dat","pmt.txt","muE_deuterium_oxide_liquid.txt","pmt_response.dat","proton_water_liquid.txt"]
+
+# generate a UUID to append to all the filenames so that if we run the same job
+# twice we don't overwrite the first job
+ID = uuid.uuid1()
+
+class MyTemplate(string.Template):
+    delimiter = '@'
+
+def submit_job(filename, run, gtid, dir, dqxx_dir, min_nhit, max_particles):
+    print("submitting job for %s gtid %i" % (filename, gtid))
+    head, tail = split(filename)
+    root, ext = splitext(tail)
+
+    # all output files are prefixed with FILENAME_GTID_UUID
+    prefix = "%s_%08i_%s" % (root,gtid,ID.hex)
+
+    # fit output filename
+    output = "%s.txt" % prefix
+    # condor submit filename
+    condor_submit = "%s.submit" % prefix
+
+    # set up the arguments for the template
+    executable = join(dir,"fit")
+    args = [tail,"-o",output,"--gtid",gtid,"--min-nhit",min_nhit,"--max-particles",max_particles]
+    transfer_input_files = ",".join([filename,join(dqxx_dir,"DQXX_%010i.dat" % run)] + [join(dir,filename) for filename in INPUT_FILES])
+    transfer_output_files = ",".join([output])
+
+    condor_error = "%s.error" % prefix
+    condor_output = "%s.output" % prefix
+    condor_log = "%s.log" % prefix
+
+    with open("submit_template.submit") as f:
+        s = f.read()
+
+    template = MyTemplate(s)
+
+    submit_string = template.safe_substitute(
+            executable=executable,
+            args=" ".join(map(str,args)),
+            transfer_input_files=transfer_input_files,
+            transfer_output_files=transfer_output_files,
+            error=condor_error,
+            output=condor_output,
+            log=condor_log)
+    
+    # write out the formatted template
+    with open(condor_submit, "w") as f:
+        f.write(submit_string)
+
+    # submit the job
+    check_call(["condor_submit",condor_submit])
+
+if __name__ == '__main__':
+    import argparse
+    from subprocess import check_output
+
+    parser = argparse.ArgumentParser("submit grid jobs")
+    parser.add_argument("filenames", nargs='+', help="input files")
+    parser.add_argument("--dir", type=str, help="fitter directory", required=True)
+    parser.add_argument("--dqxx-dir", type=str, help="dqxx directory", required=True)
+    parser.add_argument("--min-nhit", type=int, help="minimum nhit to fit an event", default=100)
+    parser.add_argument("--max-particles", type=int, help="maximum number of particles to fit for", default=3)
+    args = parser.parse_args()
+
+    for filename in args.filenames:
+        output = check_output([join(args.dir,"zdab-cat"),filename])
+        data = yaml.load(output)
+
+        for i, event in enumerate(data['data']):
+            for ev in event['ev']:
+                run = ev['run']
+                gtid = ev['gtid']
+                nhit = ev['nhit']
+
+                if nhit >= 100:
+                    submit_job(filename, run, gtid, args.dir, args.dqxx_dir, args.min_nhit, args.max_particles)
diff --git a/utils/submit_template.submit b/utils/submit_template.submit
new file mode 100644
index 0000000..4ce7ce7
--- /dev/null
+++ b/utils/submit_template.submit
@@ -0,0 +1,24 @@
+# We need the job to run our executable script, with the
+#  input.txt filename as an argument, and to transfer the
+#  relevant input and output files:
+executable = @executable
+arguments = @args
+transfer_input_files = @transfer_input_files
+
+transfer_output_files = @transfer_output_files
+
+error = @error
+output = @output
+log = @log
+
+# The below are good base requirements for first testing jobs on OSG, 
+#  if you don't have a good idea of memory and disk usage.
+requirements = (OSGVO_OS_STRING == "RHEL 7") && (OpSys == "LINUX")
+request_cpus = 1
+request_memory = 1 GB
+request_disk = 1 GB
+
+# Queue one job with the above specifications.
+queue 1
+
++ProjectName = "SNOplus"