v492 dev25 - improved pre-processing scripts.

docs/changes/162.bugfix.md

@@ -0,0 +1 @@
+Fix retrieving of laser run numbers for list of laser runs in db scripts.

docs/changes/162.feature.md

@@ -0,0 +1 @@
+Add new script `scripts/preprocessing/check_preprocessing_completeness.sh` to check for completeness of archive productions at several analysis stages.

scripts/ANALYSIS.anasum_allcuts.sh

@@ -47,6 +47,9 @@ if [[ ${RUNTYPE} == "PRECUTS" ]]; then
     TMPDIR="$VERITAS_IRFPRODUCTION_DIR/$EDMAJORVERSION/${VERITAS_ANALYSIS_TYPE:0:2}/BDTtraining/BackgroundRates/${EPOCH}"
 fi
 
+# TMP
+IGNORETYPE="IGNOREIRF"
+
 PREDIR="$VERITAS_PREPROCESSED_DATA_DIR/${VERITAS_ANALYSIS_TYPE:0:2}/mscw/"
 echo "$PREDIR"
 echo "TMP $TMPDIR"

scripts/IRF.generalproduction.sh

@@ -11,22 +11,22 @@ IRF production for all epochs
 required parameters:
 
     <sim type>              simulation type
-                            Main types: GRISU, CARE_24_20, CARE_RedHV_Feb2024, CARE_UV_2212
-                            CARE_RedHV_Feb2024, CARE_202404, CARE_24_20)
-                            V6 basic types: CARE_202404, CARE_RedHV_Feb2024
+                            Main types: GRISU  CARE_24_20  CARE_RedHV_Feb2024  CARE_UV_2212
+                            CARE_RedHV_Feb2024  CARE_202404  CARE_24_20)
+                            V6 basic types: CARE_202404  CARE_RedHV_Feb2024
 
     <IRF type>              type of instrument response function to produce.
-                            EVNDISP,
-                            MAKETABLES, COMBINETABLES,
-                            TRAINMVANGRES,
-                            TRAINXGBANGRES, ANAXGBANGRES,
-                            TRAINXGBGH, ANAXGBGH,
-                            ANALYSETABLES, ANALYSETABLESXGBTRAIN
-                            PRESELECTEFFECTIVEAREAS, COMBINEPRESELECTEFFECTIVEAREAS,
-                            TRAINTMVA, OPTIMIZETMVA,
-                            ANATABLESEFFAREAS,
-                            EFFECTIVEAREAS, COMBINEEFFECTIVEAREAS,
-                            (EVNDISPCOMPRESS, MVAEVNDISP)
+                            EVNDISP
+                            MAKETABLES  COMBINETABLES
+                            TRAINMVANGRES
+                            TRAINXGBANGRES  ANAXGBANGRES
+                            TRAINXGBGH  ANAXGBGH
+                            ANALYSETABLES  ANALYSETABLESXGBTRAIN
+                            PRESELECTEFFECTIVEAREAS  COMBINEPRESELECTEFFECTIVEAREAS
+                            TRAINTMVA  OPTIMIZETMVA
+                            ANATABLESEFFAREAS
+                            EFFECTIVEAREAS  COMBINEEFFECTIVEAREAS
+                            (EVNDISPCOMPRESS  MVAEVNDISP)
 
 --------------------------------------------------------------------------------
 "

scripts/IRF.trainXGBforAngularReconstruction.sh

@@ -75,7 +75,7 @@ MSCWLIST="$ODIR/xgbFiles.list"
 rm -f "${MSCWLIST}"
 touch "${MSCWLIST}"
 
-INDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/MSCW_RECID${RECID}_DISP"
+INDIR="$VERITAS_IRFPRODUCTION_DIR/${EDVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/MSCW_RECID${RECID}_DISP"
 
 STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json"
 TRAIN_ANGLES=$(jq -r ".zenith[] | select(.id==\"$ZA\") | .train | join(\" \")" "$STEREO_PAR")

scripts/RUNLIST.whichRunsAreOnDisk.sh

@@ -151,7 +151,9 @@ while read -r RUNID RUNDATE ; do
 # You have to do it this way, because using a pipe | calls the command in a
 # subshell, and that prevents variables from being saved within the 'while' loop
 # http://stackoverflow.com/questions/14585045/is-it-possible-to-avoid-pipes-when-reading-from-mysql-in-bash
-done < <("${MYSQL[@]}" -e "USE VERITAS ; SELECT run_id, data_start_time FROM tblRun_Info WHERE $SUB")
+# Use the database start time for runs whose data start time is NULL (for
+# example, aborted runs).
+done < <("${MYSQL[@]}" -e "USE VERITAS ; SELECT run_id, COALESCE(data_start_time, db_start_time) FROM tblRun_Info WHERE $SUB")
 
 if $DELETEFLAG; then
     if (( ${#FILES_TO_DELETE[@]} == 0 )); then

scripts/db_scripts/db_pack_new_directories.sh

@@ -3,7 +3,7 @@
 # with query_run_list.sh
 #
 
-DBTEXTDIR="$VERITAS_DATA_DIR/shared/DBTEXT/"
+DBTEXTDIR="$VERITAS_DATA_DIR/shared/DBTEXT"
 
 get_run_directory()
 {
@@ -15,17 +15,35 @@ get_run_directory()
     fi
     echo "${DBTEXTDIR}/${SRUN}"
 }
-LDIR=$(find "${DBTEXTDIR}" -type d -name "[0-9][0-9][0-9]*")
-
 PDIR=$(pwd)
 
-for L in ${LDIR}
+while IFS= read -r -d '' L
 do
     RUN=$(basename "$L")
     TDIR=$(get_run_directory "$RUN")
+
+    # Only operate on the run directory directly below its expected parent.
+    if [[ "$(dirname "$L")" != "$TDIR" ]]; then
+        echo "Skipping unexpected directory: $L" >&2
+        continue
+    fi
+
     echo "$RUN" "$TDIR"/"$RUN"
     cd "$TDIR" || exit
-    tar -czf "${RUN}".tar.gz "${RUN}"
-done
+
+    ARCHIVE="${RUN}.tar.gz"
+    TEMP_ARCHIVE="${ARCHIVE}.$$"
+
+    # Do not remove the source directory unless the temporary archive was
+    # created successfully and can be read back by tar.
+    if tar -czf "$TEMP_ARCHIVE" -- "$RUN" \
+        && tar -tzf "$TEMP_ARCHIVE" >/dev/null \
+        && mv -f -- "$TEMP_ARCHIVE" "$ARCHIVE"; then
+        rm -rf -- "$RUN"
+    else
+        echo "Failed to create or validate $TDIR/$ARCHIVE; keeping $TDIR/$RUN" >&2
+        rm -f -- "$TEMP_ARCHIVE"
+    fi
+done < <(find "${DBTEXTDIR}" -type d -name "[0-9][0-9][0-9]*" -print0)
 
 cd "${PDIR}" || exit

scripts/db_scripts/db_run.sh

@@ -51,9 +51,9 @@ get_file_status()
     elif [[ -e ${TFIL} ]]; then
         echo "2"
     else
-        TARF="$(getDBTextFileDirectory "${RRUN}").tar.gz"
+        TARF="$(getDBTextFileDirectory "${TRUN}").tar.gz"
         if [[ -e ${TARF} ]]; then
-            FFIL="${RRUN}/$(basename "${TFIL}")"
+            FFIL="${TRUN}/$(basename "${TFIL}")"
             CFIL=$(tar -tzf "${TARF}" "${FFIL}" 2>/dev/null)
             if [[ "${CFIL}" == "${FFIL}" ]]; then
                 echo "1"
@@ -119,13 +119,11 @@ get_end_time()
 get_laser_run()
 {
     OFIL="$(getDBTextFileDirectory "${RUN}")/${RUN}.laserrun"
-    LASERRUN=""
     while IFS="|" read -ra a; do
         if [[ ${a[0]} != "run_id" ]]; then
-            LASERRUN="${LASERRUN} ${a[0]}"
+            printf '%s\n' "${a[0]}"
         fi
     done < "${OFIL}"
-    echo "${LASERRUN}"
 }
 
 get_excluded_telescopes()

scripts/helper_scripts/ANALYSIS.anasum_sub.sh

@@ -76,7 +76,6 @@ fi
 rm -f $OUTPUTLOGFILE
 touch $OUTPUTLOGFILE
 
-
 inspect_executables()
 {
     if [ -n "$EVNDISP_APPTAINER" ]; then

scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh

@@ -123,7 +123,11 @@ inspect_executables()
 
 get_disp_dir()
 {
-    if [ "$HVSETTINGS" == "obsLowHV" ]; then
+    # V5 has no separate redHV XML training.  Some V5 run metadata contains
+    # obsLowHV, but those runs must still use the standard V5 directory.
+    if [ "$EPOCH" == "V5" ]; then
+        DISPDIR="DispBDTs//${ANATYPE}/${EPOCH}_ATM${ATMO}/"
+    elif [ "$HVSETTINGS" == "obsLowHV" ]; then
         DISPDIR="DispBDTs/${ANATYPE}/${EPOCH}_ATM${ATMO}_redHV/"
     elif [ "$HVSETTINGS" == "obsFilter" ]; then
         DISPDIR="DispBDTs/${ANATYPE}/${EPOCH}_ATM${ATMO}_UV/"

scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh

@@ -10,7 +10,14 @@
 RUNLIST=RRUNLIST
 ODIR=OODIR
 CUT=CCUT
-V2DL3VERSION="0.8.0"
+# Set to "" to use the unversioned environment v2dl3Eventdisplay.
+# V2DL3VERSION="v0.8.1"
+V2DL3VERSION=""
+if [[ -n "${V2DL3VERSION}" ]]; then
+    CONDA_ENV="v2dl3Eventdisplay-${V2DL3VERSION}"
+else
+    CONDA_ENV="v2dl3Eventdisplay"
+fi
 
 # temporary (scratch) directory
 if [[ -n $TMPDIR ]]; then
@@ -41,23 +48,23 @@ check_conda_installation()
         exit
     fi
     env_info=$(conda info --envs)
-    env_name="v2dl3Eventdisplay-${V2DL3VERSION}"
-    if [[ "$env_info" == *"$env_name"* ]]; then
-        echo "Found conda environment '$env_name'"
+    if awk '$1 !~ /^#/ {print $1}' <<< "$env_info" | grep -Fxq "$CONDA_ENV"; then
+        echo "Found conda environment '$CONDA_ENV'"
     else
-        echo "Error: the conda environment '$env_name' does not exist."
+        echo "Error: the conda environment '$CONDA_ENV' does not exist."
         echo "exiting..."
         exit
     fi
 }
 
 check_conda_installation
-# shellcheck source=/dev/null
 
-source activate base
-conda activate v2dl3Eventdisplay-${V2DL3VERSION}
-# Install only if not already present (avoid slow per-job reinstall)
-pip show v2dl3-eventdisplay &>/dev/null 2>&1 || pip install -e "${V2DL3SYS%/}-v${V2DL3VERSION}"
+CONDA_BASE=$(conda info --base) || exit 1
+# shellcheck source=/dev/null
+source "${CONDA_BASE}/etc/profile.d/conda.sh" || exit 1
+conda activate "${CONDA_ENV}" || exit 1
+command -v v2dl3-eventdisplay >/dev/null 2>&1 || exit 1
+command -v v2dl3-eventdisplay-query-runparameters >/dev/null 2>&1 || exit 1
 
 V2DL3OPT=(
     --fuzzy_boundary zenith 0.05
@@ -112,9 +119,19 @@ do
         continue
     fi
     echo "   ANASUM file: ${ANASUMFILE}"
-    result=$(v2dl3-eventdisplay-query-runparameters "${ANASUMFILE}" "${RUN}")
-    EPOCH=$(echo "$result" |  awk '{print $2}')
-    EFFAREA=$(echo "$result" | awk '{print $5}')
+    QUERY_ERROR_LOG="${TEMPDIR}/${RUN}.v2dl3-query.stderr.log"
+    : > "${QUERY_ERROR_LOG}"
+    result=$(v2dl3-eventdisplay-query-runparameters "${ANASUMFILE}" "${RUN}" 2>"${QUERY_ERROR_LOG}")
+    EPOCH=$(printf '%s\n' "$result" | awk -F': ' '/^Epoch:/ {print $2; exit}')
+    EFFAREA=$(printf '%s\n' "$result" | awk -F': ' '/^Effective Area:/ {print $2; exit}')
+    if [[ -z "$EPOCH" || -z "$EFFAREA" ]]; then
+        echo "Error: could not extract epoch/effective area for run ${RUN}."
+        cat "${QUERY_ERROR_LOG}"
+        echo "Query output: ${result}"
+        echo "Skipping run ${RUN}"
+        continue
+    fi
+    EVNDISPVERSION=$(echo "${EFFAREA}" | grep -oE 'v[0-9]+' | head -n 1)
     echo "   Effective area file: $EFFAREA Epoch: $EPOCH"
     DBFITSFILE=$(getNumberedDirectory "$RUN" "$VERITAS_DATA_DIR"/shared/DBFITS)/$RUN.db.fits.gz
     INTERPOLATOR=$(getInterpolator "$EFFAREA")
@@ -127,6 +144,19 @@ do
 
     for m in "point-like" "full-enclosure"
     do
+        if [[ "$m" == "full-enclosure" && \
+              "${EVNDISPVERSION,,}" == *v490* && \
+              ( "${EFFAREA,,}" == *redhv* || \
+                "${EFFAREA,,}" == *uv* ) ]]; then
+            echo "   Skipping full-enclosure conversion for EVNDISPVERSION=${EVNDISPVERSION} and RedHV/UV effective-area file"
+            rm -f \
+                "${ODIR}/full-enclosure/${RUN}.fits.gz" \
+                "${ODIR}/full-enclosure/${RUN}.log" \
+                "${ODIR}/full-enclosure-all-events/${RUN}.fits.gz" \
+                "${ODIR}/full-enclosure-all-events/${RUN}.log"
+            continue
+        fi
+
         echo "   Converting (${m}, ${V2DL3OPT[*]})"
 
         for p in "" "-all-events"
@@ -140,20 +170,30 @@ do
             echo "EVENTFILTER ${V2DL3SELECT[*]}"
 
             mkdir -p ${ODIR}/${m}${p}
-            rm -f ${ODIR}/${m}${p}/"${RUN}".log
+            RUN_LOG="${ODIR}/${m}${p}/${RUN}.log"
+            EVENTDISPLAY_STDERR_LOG="${TEMPDIR}/${RUN}.${m}${p}.stderr.log"
+            rm -f ${ODIR}/${m}${p}/"${RUN}".log "$EVENTDISPLAY_STDERR_LOG"
+
+            if [[ -s "${QUERY_ERROR_LOG}" ]]; then
+                cat "${QUERY_ERROR_LOG}" >> ${ODIR}/${m}${p}/"${RUN}".log
+            fi
 
             v2dl3-eventdisplay \
                 --${m} \
                 "${V2DL3OPT[@]}" "${V2DL3SELECT[@]}" \
                 --file_pair "${ANASUMFILE}" "$VERITAS_EVNDISP_AUX_DIR"/EffectiveAreas/"${EFFAREA}" \
-                --logfile ${ODIR}/${m}${p}/"${RUN}".log \
+                --logfile "$RUN_LOG" \
                 --instrument_epoch "${EPOCH}" \
                 --interpolator_name "${INTERPOLATOR}" \
                 --db_fits_file "${DBFITSFILE}" \
-                ${ODIR}/${m}${p}/"${RUN}".fits.gz
+                ${ODIR}/${m}${p}/"${RUN}".fits.gz \
+                2> "$EVENTDISPLAY_STDERR_LOG"
+
+            cat "$EVENTDISPLAY_STDERR_LOG" >> "$RUN_LOG"
+            rm -f "$EVENTDISPLAY_STDERR_LOG"
 
-            python --version >> ${ODIR}/${m}${p}/"${RUN}".log
-            conda list -n v2dl3Eventdisplay-${V2DL3VERSION} >> ${ODIR}/${m}${p}/"${RUN}".log
+            python --version >> ${ODIR}/${m}${p}/"${RUN}".log 2>&1
+            conda list -n "${CONDA_ENV}" >> ${ODIR}/${m}${p}/"${RUN}".log 2>&1
             PDIR=$(pwd)
             cd "${PDIR}" || exit
         done

scripts/helper_scripts/IRF.effective_area_parallel_sub.sh

@@ -112,7 +112,7 @@ PARAMFILE="
 * ENERGYAXISBINS 60
 * ENERGYAXISBINHISTOS 30
 * EBIASBINHISTOS 75
-* ANGULARRESOLUTIONBINHISTOS 40
+* ANGULARRESOLUTIONBINHISTOS 200
 * RESPONSEMATRICESEBINS 200
 * AZIMUTHBINS 1
 * FILLMONTECARLOHISTOS 0

scripts/preprocessing/README.md

@@ -51,6 +51,26 @@ Move Eventdisplay data products from all stages into an runs_with_issues directo
 ./check_dl3_number_of_files_per_cut.sh <directory>
 ```
 
+### Check completeness of a large preprocessing production
+
+Compare every numeric `<run>.root` input below the selected reference directory
+(by default `<production-directory>/evndisp`) with the standard evndisp, mscw,
+anasum, and DL3 products. The checker inventories each tree once and writes compact
+reports, so it can be used for productions with more than 50,000 files:
+
+```bash
+./check_preprocessing_completeness.sh <production-directory> [report-directory] [reference-subdirectory] [run-list-file]
+```
+
+The command exits `0` when all products are present, `1` for missing or duplicate
+products, and `2` for usage or filesystem errors. If no report directory is given,
+one is created below the production directory. The reference subdirectory defaults
+to `evndisp`. Runs listed in the optional run-list file (one numeric run per line)
+are excluded from the check. See the report's `summary.tsv` and `missing-*.txt`
+files for machine-readable results. The filtered reference run list is saved as
+`reference-runs.txt` in the report directory. Directory symlinks are followed,
+so an `evndisp` link to another filesystem can be used as the reference.
+
 ### Check if runs read from a run list are processed with evndis/mscw
 
 ```bash

scripts/preprocessing/check_evndisp_mscw_processing.sh

@@ -18,7 +18,7 @@ EDVERSION=$(cat "$VERITAS_EVNDISP_AUX_DIR"/IRFMINORVERSION)
 
 file_on_disk()
 {
-    ARCHIVEDIR="$VERITAS_DATA_DIR/processed_data_${EDVERSION}/${VERITAS_ANALYSIS_TYPE:0:2}/${DTYPE}/"
+    ARCHIVEDIR="$VERITAS_DATA_DIR/shared/processed_data_${EDVERSION}/${VERITAS_ANALYSIS_TYPE:0:2}/${DTYPE}/"
     TRUN="$1"
     if [[ ${TRUN} -lt 100000 ]]; then
         EDIR="${ARCHIVEDIR}/${TRUN:0:1}/"