update submit-grid-jobs and cat-grid-jobs

This commit updates submit-grid-jobs so that it keeps a database of jobs. This
allows the script to make sure that we only have a certain number of jobs in
the job queue at a single time and automatically resubmitting failed jobs. The
idea is that it can now be run once to add jobs to the database:

    $ submit-grid-jobs ~/zdabs/SNOCR_0000010000_000_p4_reduced.xzdab.gz

and then be run periodically via crontab:

    PATH=/usr/bin:$HOME/local/bin
    SDDM_DATA=$HOME/sddm/src
    DQXX_DIR=$HOME/dqxx

    0 * * * * submit-grid-jobs --auto --logfile ~/submit.log

Similarly I updated cat-grid-jobs so that it uses the same database and can
also be run via a cron job:

    PATH=/usr/bin:$HOME/local/bin
    SDDM_DATA=$HOME/sddm/src
    DQXX_DIR=$HOME/dqxx

    0 * * * * cat-grid-jobs --logfile cat.log --output-dir $HOME/fit_results

I also updated fit so that it keeps track of the total time elapsed including
the initial fits instead of just counting the final fits.

1 files changed, 17 insertions, 12 deletions

diff --git a/src/fit.c b/src/fit.c
index 34bb583..9a94ec6 100644
--- a/src/fit.c
+++ b/src/fit.c
@@ -5432,6 +5432,7 @@ int fit_event2(event *ev, double *xopt, double *fmin, int *id, size_t n, double
     size_t *result;
     size_t nvertices;
     int status;
+    int rv;
 
     /* Create the minimizer object. */
     opt = nlopt_create(NLOPT_LN_BOBYQA, 4+3*n);
@@ -5516,6 +5517,8 @@ int fit_event2(event *ev, double *xopt, double *fmin, int *id, size_t n, double
     for (i = 0; i < n; i++)
         fpars.id[i] = id[i];
 
+    time_elapsed = 0.0;
+
     for (i = 0; i < nvertices; i++) {
         /* Copy the starting parameters to the `x` array. */
         memcpy(x,x0,sizeof(x));
@@ -5581,6 +5584,8 @@ int fit_event2(event *ev, double *xopt, double *fmin, int *id, size_t n, double
 
         long long elapsed = (tv_stop.tv_sec - tv_start.tv_sec)*1000 + (tv_stop.tv_usec - tv_start.tv_usec)/1000;
 
+        time_elapsed += elapsed/1000.0;
+
         printf("%4zu/%4zu %7.2f %7.2f %7.2f %6.2f ",
                i+1,
                nvertices,
@@ -5661,13 +5666,13 @@ close:
         exit(0);
     }
 
-    gettimeofday(&tv_start, NULL);
-    nlopt_optimize(opt,x,&fval);
+    gettimeofday(&tv_start, NULL); 
+    rv = nlopt_optimize(opt,x,&fval);
     gettimeofday(&tv_stop, NULL);
 
-    time_elapsed = tv_stop.tv_sec - tv_start.tv_sec + (tv_stop.tv_usec - tv_start.tv_usec)/1e6;
+    time_elapsed += tv_stop.tv_sec - tv_start.tv_sec + (tv_stop.tv_usec - tv_start.tv_usec)/1e6;
 
-    if (time_elapsed > maxtime) goto end;
+    if (time_elapsed > maxtime || rv == NLOPT_MAXTIME_REACHED) goto end;
 
     if (stop) goto stop;
 
@@ -5678,20 +5683,20 @@ close:
         nlopt_set_maxtime(opt, maxtime-time_elapsed);
 
         gettimeofday(&tv_start, NULL);
-        nlopt_optimize(opt,x,&fval);
+        rv = nlopt_optimize(opt,x,&fval);
         gettimeofday(&tv_stop, NULL);
 
         time_elapsed += tv_stop.tv_sec - tv_start.tv_sec + (tv_stop.tv_usec - tv_start.tv_usec)/1e6;
 
         if (stop) goto stop;
-    } while (fval < *fmin && fabs(fval-*fmin) > 1e-2 && time_elapsed < maxtime);
+    } while (fval < *fmin && fabs(fval-*fmin) > 1e-2 && time_elapsed < maxtime && rv != NLOPT_MAXTIME_REACHED);
 
     if (fval < *fmin) {
         *fmin = fval;
         memcpy(xopt,x,sizeof(x));
     }
 
-    if (time_elapsed >= maxtime) goto end;
+    if (time_elapsed > maxtime || rv == NLOPT_MAXTIME_REACHED) goto end;
 
     /* Now, minimize with SBPLX.
     *
@@ -5720,12 +5725,12 @@ close:
     memcpy(x,xopt,sizeof(x));
 
     gettimeofday(&tv_start, NULL);
-    nlopt_optimize(opt,x,&fval);
+    rv = nlopt_optimize(opt,x,&fval);
     gettimeofday(&tv_stop, NULL);
 
-    time_elapsed = tv_stop.tv_sec - tv_start.tv_sec + (tv_stop.tv_usec - tv_start.tv_usec)/1e6;
+    time_elapsed += tv_stop.tv_sec - tv_start.tv_sec + (tv_stop.tv_usec - tv_start.tv_usec)/1e6;
 
-    if (time_elapsed > maxtime) goto end;
+    if (time_elapsed > maxtime || rv == NLOPT_MAXTIME_REACHED) goto end;
 
     if (stop) goto stop;
 
@@ -5736,13 +5741,13 @@ close:
         nlopt_set_maxtime(opt, maxtime-time_elapsed);
 
         gettimeofday(&tv_start, NULL);
-        nlopt_optimize(opt,x,&fval);
+        rv = nlopt_optimize(opt,x,&fval);
         gettimeofday(&tv_stop, NULL);
 
         time_elapsed += tv_stop.tv_sec - tv_start.tv_sec + (tv_stop.tv_usec - tv_start.tv_usec)/1e6;
 
         if (stop) goto stop;
-    } while (fval < *fmin && fabs(fval-*fmin) > 1e-2 && time_elapsed < maxtime);
+    } while (fval < *fmin && fabs(fval-*fmin) > 1e-2 && time_elapsed < maxtime && rv != NLOPT_MAXTIME_REACHED);
 
     if (fval < *fmin) {
         *fmin = fval;