switch from YAML output to HDF5 to speed things up

1 files changed, 193 insertions, 0 deletions

diff --git a/src/hdf5_utils.c b/src/hdf5_utils.c
new file mode 100644
index 0000000..05d1002
--- /dev/null
+++ b/src/hdf5_utils.c
@@ -0,0 +1,193 @@
+#include "hdf5_utils.h"
+#include "hdf5.h"
+#include <string.h> /* for strlen() */
+
+char *GitSHA1(void);
+char *GitDirty(void);
+
+/* Write out data to an HDF5 file.
+ *
+ * This function writes out three datasets to an HDF5 file:
+ *
+ * 1. ev: the event dataset
+ *
+ *    This dataset contains event level information similar to the EV bank in
+ *    SNOMAN. It also contains a data cleaning word and the FTP, FTK, and RSP
+ *    fit values for the position and energy.
+ *
+ * 2. mcgn: the primary particle tracks
+ *
+ *     This dataset contains vertex and track information for all the primary
+ *     particle tracks used to simulate the event.
+ *
+ * 3. fits: the fit information
+ *
+ *     This dataset contains the reconstructed fit information for each event.
+ *     Note that this dataset will in general contain multiple rows for a
+ *     single event since we fit for different particle hypotheses.
+ *
+ * Currently the ev and mcgn datasets are fixed size, and the fits dataset is
+ * chunked and extensible to allow the cat-grid-jobs script to work.
+ *
+ */
+int save_output(const char *output, HDF5Event *hdf5_events, size_t nevents, HDF5MCGN *hdf5_mcgn, size_t nmcgn, HDF5Fit *hdf5_fits, size_t nfits)
+{
+    hid_t      hdf5_events_tid;     /* File datatype identifier */
+    hid_t      file, dataset, space; /* Handles */
+    herr_t     status;
+    hsize_t    dim[1];   /* Dataspace dimensions */
+    hid_t      hdf5_mcgn_tid;     /* File datatype identifier */
+    hid_t      dataset_mcgn, space_mcgn; /* Handles */
+    hsize_t    dim_mcgn[1];   /* Dataspace dimensions */
+    hid_t      hdf5_fits_tid;     /* File datatype identifier */
+    hid_t      dataset_fits, space_fits; /* Handles */
+    hsize_t    dim_fits[1];   /* Dataspace dimensions */
+    hsize_t    chunk_dims[1];   /* Dataspace dimensions */
+    hid_t      dcpl_id; /* Handles */
+    hsize_t    maxdims[1] = {H5S_UNLIMITED};
+
+    /*
+     * Create the data space.
+     */
+    dim[0] = nevents;
+    space = H5Screate_simple(1, dim, NULL);
+    dim_mcgn[0] = nmcgn;
+    space_mcgn = H5Screate_simple(1, dim_mcgn, NULL);
+    dim_fits[0] = nfits;
+    space_fits = H5Screate_simple(1, dim_fits, maxdims);
+    chunk_dims[0] = 10;
+    dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
+    H5Pset_chunk(dcpl_id, 1, chunk_dims);
+
+    /*
+     * Create the file.
+     */
+    file = H5Fcreate(output, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+
+    hid_t   attr1, attr2; /* Attribute identifiers */
+    hid_t   aid1, aid2;    /* Attribute dataspace identifiers */
+    hid_t   atype1, atype2;    /* Attribute type */
+
+    char *git_sha1 = GitSHA1();
+    char *git_dirty = GitDirty();
+
+    /*
+     * Create string attribute.
+     */
+    aid1  = H5Screate(H5S_SCALAR);
+    atype1 = H5Tcopy(H5T_C_S1);
+             H5Tset_size(atype1, strlen(git_sha1));
+             H5Tset_strpad(atype1,H5T_STR_NULLTERM);
+    attr1 = H5Acreate2(file, "git_sha1", atype1, aid1, H5P_DEFAULT, H5P_DEFAULT);
+
+    /*
+     * Create string attribute.
+     */
+    aid2  = H5Screate(H5S_SCALAR);
+    atype2 = H5Tcopy(H5T_C_S1);
+             H5Tset_size(atype2, strlen(git_dirty));
+             H5Tset_strpad(atype2,H5T_STR_NULLTERM);
+    attr2 = H5Acreate2(file, "git_dirty", atype2, aid2, H5P_DEFAULT, H5P_DEFAULT);
+
+    herr_t  ret;                /* Return value */
+
+    /*
+     * Write string attribute.
+     */
+    ret = H5Awrite(attr1, atype1, git_sha1);
+    ret = H5Awrite(attr2, atype2, git_dirty);
+
+    /*
+     * Create the memory data type.
+     */
+    hdf5_events_tid = H5Tcreate(H5T_COMPOUND, sizeof(HDF5Event));
+    H5Tinsert(hdf5_events_tid, "run",        HOFFSET(HDF5Event, run),        H5T_NATIVE_INT);
+    H5Tinsert(hdf5_events_tid, "evn",        HOFFSET(HDF5Event, evn),        H5T_NATIVE_INT);
+    H5Tinsert(hdf5_events_tid, "gtr",        HOFFSET(HDF5Event, gtr),        H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_events_tid, "nhit",       HOFFSET(HDF5Event, nhit),       H5T_NATIVE_INT);
+    H5Tinsert(hdf5_events_tid, "gtid",       HOFFSET(HDF5Event, gtid),       H5T_NATIVE_UINT32);
+    H5Tinsert(hdf5_events_tid, "trg_type",   HOFFSET(HDF5Event, trg_type),   H5T_NATIVE_UINT32);
+    H5Tinsert(hdf5_events_tid, "dc",         HOFFSET(HDF5Event, dc),         H5T_NATIVE_UINT32);
+    H5Tinsert(hdf5_events_tid, "ftp_x",      HOFFSET(HDF5Event, ftp_x),      H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_events_tid, "ftp_y",      HOFFSET(HDF5Event, ftp_y),      H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_events_tid, "ftp_z",      HOFFSET(HDF5Event, ftp_z),      H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_events_tid, "ftk_energy", HOFFSET(HDF5Event, ftk_energy), H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_events_tid, "rsp_energy", HOFFSET(HDF5Event, rsp_energy), H5T_NATIVE_DOUBLE);
+
+    hdf5_mcgn_tid = H5Tcreate(H5T_COMPOUND, sizeof(HDF5MCGN));
+    H5Tinsert(hdf5_mcgn_tid, "run",    HOFFSET(HDF5MCGN, run),    H5T_NATIVE_INT);
+    H5Tinsert(hdf5_mcgn_tid, "evn",    HOFFSET(HDF5MCGN, evn),    H5T_NATIVE_INT);
+    H5Tinsert(hdf5_mcgn_tid, "id",     HOFFSET(HDF5MCGN, id),     H5T_NATIVE_INT);
+    H5Tinsert(hdf5_mcgn_tid, "energy", HOFFSET(HDF5MCGN, energy), H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_mcgn_tid, "x",      HOFFSET(HDF5MCGN, x),      H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_mcgn_tid, "y",      HOFFSET(HDF5MCGN, y),      H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_mcgn_tid, "z",      HOFFSET(HDF5MCGN, z),      H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_mcgn_tid, "dirx",   HOFFSET(HDF5MCGN, dirx),   H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_mcgn_tid, "diry",   HOFFSET(HDF5MCGN, diry),   H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_mcgn_tid, "dirz",   HOFFSET(HDF5MCGN, dirz),   H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_mcgn_tid, "time",   HOFFSET(HDF5MCGN, time),   H5T_NATIVE_DOUBLE);
+
+    hdf5_fits_tid = H5Tcreate(H5T_COMPOUND, sizeof(HDF5Fit));
+    H5Tinsert(hdf5_fits_tid, "run",     HOFFSET(HDF5Fit, run),     H5T_NATIVE_INT);
+    H5Tinsert(hdf5_fits_tid, "gtid",    HOFFSET(HDF5Fit, gtid),    H5T_NATIVE_UINT32);
+    H5Tinsert(hdf5_fits_tid, "n",       HOFFSET(HDF5Fit, n),       H5T_NATIVE_INT);
+    H5Tinsert(hdf5_fits_tid, "x",       HOFFSET(HDF5Fit, x),       H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "y",       HOFFSET(HDF5Fit, y),       H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "z",       HOFFSET(HDF5Fit, z),       H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "t0",      HOFFSET(HDF5Fit, t0),      H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "id1",     HOFFSET(HDF5Fit, id1),     H5T_NATIVE_INT);
+    H5Tinsert(hdf5_fits_tid, "energy1", HOFFSET(HDF5Fit, energy1), H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "theta1",  HOFFSET(HDF5Fit, theta1),  H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "phi1",    HOFFSET(HDF5Fit, phi1),    H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "id2",     HOFFSET(HDF5Fit, id2),     H5T_NATIVE_INT);
+    H5Tinsert(hdf5_fits_tid, "energy2", HOFFSET(HDF5Fit, energy2), H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "theta2",  HOFFSET(HDF5Fit, theta2),  H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "phi2",    HOFFSET(HDF5Fit, phi2),    H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "id3",     HOFFSET(HDF5Fit, id3),     H5T_NATIVE_INT);
+    H5Tinsert(hdf5_fits_tid, "energy3", HOFFSET(HDF5Fit, energy3), H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "theta3",  HOFFSET(HDF5Fit, theta3),  H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "phi3",    HOFFSET(HDF5Fit, phi3),    H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "fmin",    HOFFSET(HDF5Fit, fmin),    H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "time",    HOFFSET(HDF5Fit, time),    H5T_NATIVE_DOUBLE);
+    H5Tinsert(hdf5_fits_tid, "psi",     HOFFSET(HDF5Fit, psi),     H5T_NATIVE_DOUBLE);
+
+    /*
+     * Create the dataset.
+     */
+    dataset = H5Dcreate2(file, "ev", hdf5_events_tid, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    dataset_mcgn = H5Dcreate2(file, "mcgn", hdf5_mcgn_tid, space_mcgn, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    dataset_fits = H5Dcreate2(file, "fits", hdf5_fits_tid, space_fits, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
+
+    /*
+     * Wtite data to the dataset;
+     */
+    if (nevents)
+        status = H5Dwrite(dataset, hdf5_events_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, hdf5_events);
+    if (nmcgn)
+        status = H5Dwrite(dataset_mcgn, hdf5_mcgn_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, hdf5_mcgn);
+    if (nfits)
+        status = H5Dwrite(dataset_fits, hdf5_fits_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, hdf5_fits);
+
+    /*
+     * Release resources
+     */
+    H5Sclose(aid1);
+    H5Sclose(aid2);
+    H5Tclose(atype1);
+    H5Tclose(atype2);
+    H5Aclose(attr1);
+    H5Aclose(attr2);
+    H5Tclose(hdf5_events_tid);
+    H5Tclose(hdf5_mcgn_tid);
+    H5Tclose(hdf5_fits_tid);
+    H5Sclose(space);
+    H5Sclose(space_mcgn);
+    H5Sclose(space_fits);
+    H5Pclose(dcpl_id);
+    H5Dclose(dataset);
+    H5Dclose(dataset_mcgn);
+    H5Dclose(dataset_fits);
+    H5Fclose(file);
+
+    return 0;
+}