add a script to move grid files from successful fits to a new directory

2 files changed, 87 insertions, 1 deletions

diff --git a/utils/delete-zdabs b/utils/delete-zdabs
index cf59986..a5b1ee0 100755
--- a/utils/delete-zdabs
+++ b/utils/delete-zdabs
@@ -13,7 +13,7 @@ if __name__ == '__main__':
     from os.path import join, split, exists
     import numpy as np
 
-    parser = argparse.ArgumentParser("concatenate fit results from grid jobs into a single file")
+    parser = argparse.ArgumentParser("delete zdab files which have all been successfully fit")
     parser.add_argument("--db", type=str, help="database file", default=None)
     parser.add_argument('--loglevel',
                         help="logging level (debug, verbose, notice, warning)",
diff --git a/utils/mv-grid-files b/utils/mv-grid-files
new file mode 100755
index 0000000..4e1c231
--- /dev/null
+++ b/utils/mv-grid-files
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+"""
+Script to move grid output files which have been successfully fit. For example:
+
+    $ cd fit_results
+    $ mv-grid-files --new-dir fit_results_to_move
+
+will take all the files associated with a successful grid job (submit file,
+hdf5 output, condor output, condor error, and condor log) and move them with
+the same directory structure to fit_results_to_move.
+
+The idea here is that we first move these files locally on the grid login node
+and then we can set up a cron job:
+
+    0 0 * * * rsync -avzP --remove-source-files username@osgconnect.net:fit_results_to_move/ ~/fit_results/
+
+to copy all the data back and delete it from the grid login node. This ensures
+we leave the files there for jobs which didn't successfully fit since we
+probably want to resubmit them at some point.
+"""
+from sddm.logger import Logger
+
+log = Logger()
+
+def mv(src,dst):
+    log.notice("mv %s %s" % (src,dst))
+    if os.path.exists(src):
+        os.renames(src,dst)
+    else:
+        log.debug("skipping %s because it doesn't exist" % src)
+
+if __name__ == '__main__':
+    import argparse
+    import sqlite3
+    import os
+    from os.path import join, split, exists
+    import numpy as np
+    import glob
+    from sddm import splitext
+
+    parser = argparse.ArgumentParser("delete grid output files which have been successfully fit")
+    parser.add_argument("--db", type=str, help="database file", default=None)
+    parser.add_argument('--loglevel',
+                        help="logging level (debug, verbose, notice, warning)",
+                        default='notice')
+    parser.add_argument('--logfile', default=None,
+                        help="filename for log file")
+    parser.add_argument('--new-dir', default=None,
+                        help="directory to move successful fits to")
+    args = parser.parse_args()
+
+    log.set_verbosity(args.loglevel)
+
+    if args.logfile:
+        log.set_logfile(args.logfile)
+
+    home = os.path.expanduser("~")
+
+    if args.db is None:
+        args.db = join(home,'state.db')
+
+    if args.new_dir is None:
+        args.new_dir = join(home,"fit_results_to_move")
+
+    conn = sqlite3.connect(args.db)
+
+    c = conn.cursor()
+
+    results = c.execute('SELECT filename, uuid, gtid, particle_id, state FROM state WHERE state = "SUCCESS" ORDER BY timestamp ASC')
+
+    for filename, uuid, gtid, particle_id, state in results.fetchall():
+        head, tail = split(filename)
+        root, ext = splitext(tail)
+
+        # all output files are prefixed with FILENAME_GTID_UUID
+        prefix = "%s_%08i_%i_%s" % (root,gtid,particle_id,uuid)
+
+        new_dir = "%s_%s" % (root,uuid)
+
+        if state == 'SUCCESS':
+            # If it successfully fit, then we move all the associated files to
+            # a new directory. From there, they can be copied back
+            for filename in glob.glob("%s/%s.*" % (new_dir,prefix)):
+                mv(filename,join(args.new_dir,filename))
+
+    conn.close()