fix a few bugs in cat-grid-jobs since I now add 2 fit results per particle combo

1 files changed, 27 insertions, 10 deletions

diff --git a/utils/cat-grid-jobs b/utils/cat-grid-jobs
index 2de0126..62dcd42 100755
--- a/utils/cat-grid-jobs
+++ b/utils/cat-grid-jobs
@@ -222,12 +222,24 @@ def cat_grid_jobs(conn, output_dir):
         output = join(output_dir,"%s_%s_fit_results.hdf5" % (root,uuid))
 
         if os.path.exists(output):
+            total_fits = 0
+            for fit_result_filename in fit_results:
+                fit_result_head, fit_result_tail = split(fit_result_filename)
+
+                if not os.path.exists(fit_result_filename):
+                    log.warn("File '%s' does not exist!" % filename)
+                    continue
+
+                with h5py.File(fit_result_filename,'r') as f:
+                    if 'git_sha1' not in f.attrs:
+                        log.warn("No git sha1 found for '%s'. Skipping..." % fit_result_filename)
+                        continue
+                    total_fits += f['fits'].shape[0]
+
             with h5py.File(output,'r') as fout:
                 if 'fits' in fout:
-                    total_fits = fout['fits'].shape[0]
-
-                    if total_fits >= len(fit_results):
-                        log.debug("skipping %s because there are already %i fit results" % (tail,len(fit_results)))
+                    if fout['fits'].shape[0] >= total_fits:
+                        log.debug("skipping %s because there are already %i fit results" % (tail,total_fits))
                         continue
 
         # First we get the full event list along with the data cleaning word, FTP
@@ -246,15 +258,20 @@ def cat_grid_jobs(conn, output_dir):
 
         with h5py.File(output,"a") as fout:
             total_events = fout['ev'].shape[0]
-            for filename in fit_results:
-                head, tail = split(filename)
-                with h5py.File(filename) as f:
+            for fit_result_filename in fit_results:
+                fit_result_head, fit_result_tail = split(fit_result_filename)
+
+                if not os.path.exists(fit_result_filename):
+                    log.warn("File '%s' does not exist!" % fit_result_filename)
+                    continue
+
+                with h5py.File(fit_result_filename) as f:
                     if 'git_sha1' not in f.attrs:
-                        log.warn("No git sha1 found for %s. Skipping..." % tail)
+                        log.warn("No git sha1 found for %s. Skipping..." % fit_result_tail)
                         continue
                     # Check to see if the git sha1 match
                     if fout.attrs['git_sha1'] != f.attrs['git_sha1']:
-                        log.debug("git_sha1 is %s for current version but %s for %s" % (fout.attrs['git_sha1'],f.attrs['git_sha1'],tail))
+                        log.debug("git_sha1 is %s for current version but %s for %s" % (fout.attrs['git_sha1'],f.attrs['git_sha1'],fit_result_tail))
                     # get fits which match up with the events
                     valid_fits = f['fits'][np.isin(f['fits'][:][['run','gtid']],fout['ev'][:][['run','gtid']])]
                     # Add the fit results
@@ -263,7 +280,7 @@ def cat_grid_jobs(conn, output_dir):
                     events_with_fit += len(np.unique(valid_fits[['run','gtid']]))
                     total_fits += len(np.unique(f['fits']['run','gtid']))
 
-        log.notice("%s_%s: added %i/%i fit results to a total of %i events" % (filename, uuid, events_with_fit, total_fits, total_events))
+        log.notice("%s (%s): added %i/%i fit results to a total of %i events" % (tail, uuid, events_with_fit, total_fits, total_events))
 
 if __name__ == '__main__':
     import argparse