diff --git a/docs/changes/162.bugfix.md b/docs/changes/162.bugfix.md new file mode 100644 index 0000000..a9acde4 --- /dev/null +++ b/docs/changes/162.bugfix.md @@ -0,0 +1 @@ +Fix retrieving of laser run numbers for list of laser runs in db scripts. diff --git a/docs/changes/162.feature.md b/docs/changes/162.feature.md new file mode 100644 index 0000000..afbdd31 --- /dev/null +++ b/docs/changes/162.feature.md @@ -0,0 +1 @@ +Add new script `scripts/preprocessing/check_preprocessing_completeness.sh` to check for completeness of archive productions at several analysis stages. diff --git a/scripts/ANALYSIS.anasum_allcuts.sh b/scripts/ANALYSIS.anasum_allcuts.sh index 8c3ad9e..648329a 100755 --- a/scripts/ANALYSIS.anasum_allcuts.sh +++ b/scripts/ANALYSIS.anasum_allcuts.sh @@ -47,6 +47,9 @@ if [[ ${RUNTYPE} == "PRECUTS" ]]; then TMPDIR="$VERITAS_IRFPRODUCTION_DIR/$EDMAJORVERSION/${VERITAS_ANALYSIS_TYPE:0:2}/BDTtraining/BackgroundRates/${EPOCH}" fi +# TMP +IGNORETYPE="IGNOREIRF" + PREDIR="$VERITAS_PREPROCESSED_DATA_DIR/${VERITAS_ANALYSIS_TYPE:0:2}/mscw/" echo "$PREDIR" echo "TMP $TMPDIR" diff --git a/scripts/IRF.generalproduction.sh b/scripts/IRF.generalproduction.sh index ae5b4aa..f0f39c1 100755 --- a/scripts/IRF.generalproduction.sh +++ b/scripts/IRF.generalproduction.sh @@ -11,22 +11,22 @@ IRF production for all epochs required parameters: simulation type - Main types: GRISU, CARE_24_20, CARE_RedHV_Feb2024, CARE_UV_2212 - CARE_RedHV_Feb2024, CARE_202404, CARE_24_20) - V6 basic types: CARE_202404, CARE_RedHV_Feb2024 + Main types: GRISU CARE_24_20 CARE_RedHV_Feb2024 CARE_UV_2212 + CARE_RedHV_Feb2024 CARE_202404 CARE_24_20) + V6 basic types: CARE_202404 CARE_RedHV_Feb2024 type of instrument response function to produce. - EVNDISP, - MAKETABLES, COMBINETABLES, - TRAINMVANGRES, - TRAINXGBANGRES, ANAXGBANGRES, - TRAINXGBGH, ANAXGBGH, - ANALYSETABLES, ANALYSETABLESXGBTRAIN - PRESELECTEFFECTIVEAREAS, COMBINEPRESELECTEFFECTIVEAREAS, - TRAINTMVA, OPTIMIZETMVA, - ANATABLESEFFAREAS, - EFFECTIVEAREAS, COMBINEEFFECTIVEAREAS, - (EVNDISPCOMPRESS, MVAEVNDISP) + EVNDISP + MAKETABLES COMBINETABLES + TRAINMVANGRES + TRAINXGBANGRES ANAXGBANGRES + TRAINXGBGH ANAXGBGH + ANALYSETABLES ANALYSETABLESXGBTRAIN + PRESELECTEFFECTIVEAREAS COMBINEPRESELECTEFFECTIVEAREAS + TRAINTMVA OPTIMIZETMVA + ANATABLESEFFAREAS + EFFECTIVEAREAS COMBINEEFFECTIVEAREAS + (EVNDISPCOMPRESS MVAEVNDISP) -------------------------------------------------------------------------------- " diff --git a/scripts/IRF.trainXGBforAngularReconstruction.sh b/scripts/IRF.trainXGBforAngularReconstruction.sh index e38a55a..079b901 100755 --- a/scripts/IRF.trainXGBforAngularReconstruction.sh +++ b/scripts/IRF.trainXGBforAngularReconstruction.sh @@ -75,7 +75,7 @@ MSCWLIST="$ODIR/xgbFiles.list" rm -f "${MSCWLIST}" touch "${MSCWLIST}" -INDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/MSCW_RECID${RECID}_DISP" +INDIR="$VERITAS_IRFPRODUCTION_DIR/${EDVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/MSCW_RECID${RECID}_DISP" STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json" TRAIN_ANGLES=$(jq -r ".zenith[] | select(.id==\"$ZA\") | .train | join(\" \")" "$STEREO_PAR") diff --git a/scripts/RUNLIST.whichRunsAreOnDisk.sh b/scripts/RUNLIST.whichRunsAreOnDisk.sh index 8f92446..4ad8ecb 100755 --- a/scripts/RUNLIST.whichRunsAreOnDisk.sh +++ b/scripts/RUNLIST.whichRunsAreOnDisk.sh @@ -151,7 +151,9 @@ while read -r RUNID RUNDATE ; do # You have to do it this way, because using a pipe | calls the command in a # subshell, and that prevents variables from being saved within the 'while' loop # http://stackoverflow.com/questions/14585045/is-it-possible-to-avoid-pipes-when-reading-from-mysql-in-bash -done < <("${MYSQL[@]}" -e "USE VERITAS ; SELECT run_id, data_start_time FROM tblRun_Info WHERE $SUB") +# Use the database start time for runs whose data start time is NULL (for +# example, aborted runs). +done < <("${MYSQL[@]}" -e "USE VERITAS ; SELECT run_id, COALESCE(data_start_time, db_start_time) FROM tblRun_Info WHERE $SUB") if $DELETEFLAG; then if (( ${#FILES_TO_DELETE[@]} == 0 )); then diff --git a/scripts/db_scripts/db_pack_new_directories.sh b/scripts/db_scripts/db_pack_new_directories.sh index 3643b6c..412fbf3 100755 --- a/scripts/db_scripts/db_pack_new_directories.sh +++ b/scripts/db_scripts/db_pack_new_directories.sh @@ -3,7 +3,7 @@ # with query_run_list.sh # -DBTEXTDIR="$VERITAS_DATA_DIR/shared/DBTEXT/" +DBTEXTDIR="$VERITAS_DATA_DIR/shared/DBTEXT" get_run_directory() { @@ -15,17 +15,35 @@ get_run_directory() fi echo "${DBTEXTDIR}/${SRUN}" } -LDIR=$(find "${DBTEXTDIR}" -type d -name "[0-9][0-9][0-9]*") - PDIR=$(pwd) -for L in ${LDIR} +while IFS= read -r -d '' L do RUN=$(basename "$L") TDIR=$(get_run_directory "$RUN") + + # Only operate on the run directory directly below its expected parent. + if [[ "$(dirname "$L")" != "$TDIR" ]]; then + echo "Skipping unexpected directory: $L" >&2 + continue + fi + echo "$RUN" "$TDIR"/"$RUN" cd "$TDIR" || exit - tar -czf "${RUN}".tar.gz "${RUN}" -done + + ARCHIVE="${RUN}.tar.gz" + TEMP_ARCHIVE="${ARCHIVE}.$$" + + # Do not remove the source directory unless the temporary archive was + # created successfully and can be read back by tar. + if tar -czf "$TEMP_ARCHIVE" -- "$RUN" \ + && tar -tzf "$TEMP_ARCHIVE" >/dev/null \ + && mv -f -- "$TEMP_ARCHIVE" "$ARCHIVE"; then + rm -rf -- "$RUN" + else + echo "Failed to create or validate $TDIR/$ARCHIVE; keeping $TDIR/$RUN" >&2 + rm -f -- "$TEMP_ARCHIVE" + fi +done < <(find "${DBTEXTDIR}" -type d -name "[0-9][0-9][0-9]*" -print0) cd "${PDIR}" || exit diff --git a/scripts/db_scripts/db_run.sh b/scripts/db_scripts/db_run.sh index ab15b02..31a0fbf 100755 --- a/scripts/db_scripts/db_run.sh +++ b/scripts/db_scripts/db_run.sh @@ -51,9 +51,9 @@ get_file_status() elif [[ -e ${TFIL} ]]; then echo "2" else - TARF="$(getDBTextFileDirectory "${RRUN}").tar.gz" + TARF="$(getDBTextFileDirectory "${TRUN}").tar.gz" if [[ -e ${TARF} ]]; then - FFIL="${RRUN}/$(basename "${TFIL}")" + FFIL="${TRUN}/$(basename "${TFIL}")" CFIL=$(tar -tzf "${TARF}" "${FFIL}" 2>/dev/null) if [[ "${CFIL}" == "${FFIL}" ]]; then echo "1" @@ -119,13 +119,11 @@ get_end_time() get_laser_run() { OFIL="$(getDBTextFileDirectory "${RUN}")/${RUN}.laserrun" - LASERRUN="" while IFS="|" read -ra a; do if [[ ${a[0]} != "run_id" ]]; then - LASERRUN="${LASERRUN} ${a[0]}" + printf '%s\n' "${a[0]}" fi done < "${OFIL}" - echo "${LASERRUN}" } get_excluded_telescopes() diff --git a/scripts/helper_scripts/ANALYSIS.anasum_sub.sh b/scripts/helper_scripts/ANALYSIS.anasum_sub.sh index f3ce12f..5f74580 100755 --- a/scripts/helper_scripts/ANALYSIS.anasum_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.anasum_sub.sh @@ -76,7 +76,6 @@ fi rm -f $OUTPUTLOGFILE touch $OUTPUTLOGFILE - inspect_executables() { if [ -n "$EVNDISP_APPTAINER" ]; then diff --git a/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh b/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh index 811d986..4e6c18d 100755 --- a/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh @@ -123,7 +123,11 @@ inspect_executables() get_disp_dir() { - if [ "$HVSETTINGS" == "obsLowHV" ]; then + # V5 has no separate redHV XML training. Some V5 run metadata contains + # obsLowHV, but those runs must still use the standard V5 directory. + if [ "$EPOCH" == "V5" ]; then + DISPDIR="DispBDTs//${ANATYPE}/${EPOCH}_ATM${ATMO}/" + elif [ "$HVSETTINGS" == "obsLowHV" ]; then DISPDIR="DispBDTs/${ANATYPE}/${EPOCH}_ATM${ATMO}_redHV/" elif [ "$HVSETTINGS" == "obsFilter" ]; then DISPDIR="DispBDTs/${ANATYPE}/${EPOCH}_ATM${ATMO}_UV/" diff --git a/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh b/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh index 4a44aa1..ef0a4cf 100755 --- a/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh @@ -10,7 +10,14 @@ RUNLIST=RRUNLIST ODIR=OODIR CUT=CCUT -V2DL3VERSION="0.8.0" +# Set to "" to use the unversioned environment v2dl3Eventdisplay. +# V2DL3VERSION="v0.8.1" +V2DL3VERSION="" +if [[ -n "${V2DL3VERSION}" ]]; then + CONDA_ENV="v2dl3Eventdisplay-${V2DL3VERSION}" +else + CONDA_ENV="v2dl3Eventdisplay" +fi # temporary (scratch) directory if [[ -n $TMPDIR ]]; then @@ -41,23 +48,23 @@ check_conda_installation() exit fi env_info=$(conda info --envs) - env_name="v2dl3Eventdisplay-${V2DL3VERSION}" - if [[ "$env_info" == *"$env_name"* ]]; then - echo "Found conda environment '$env_name'" + if awk '$1 !~ /^#/ {print $1}' <<< "$env_info" | grep -Fxq "$CONDA_ENV"; then + echo "Found conda environment '$CONDA_ENV'" else - echo "Error: the conda environment '$env_name' does not exist." + echo "Error: the conda environment '$CONDA_ENV' does not exist." echo "exiting..." exit fi } check_conda_installation -# shellcheck source=/dev/null -source activate base -conda activate v2dl3Eventdisplay-${V2DL3VERSION} -# Install only if not already present (avoid slow per-job reinstall) -pip show v2dl3-eventdisplay &>/dev/null 2>&1 || pip install -e "${V2DL3SYS%/}-v${V2DL3VERSION}" +CONDA_BASE=$(conda info --base) || exit 1 +# shellcheck source=/dev/null +source "${CONDA_BASE}/etc/profile.d/conda.sh" || exit 1 +conda activate "${CONDA_ENV}" || exit 1 +command -v v2dl3-eventdisplay >/dev/null 2>&1 || exit 1 +command -v v2dl3-eventdisplay-query-runparameters >/dev/null 2>&1 || exit 1 V2DL3OPT=( --fuzzy_boundary zenith 0.05 @@ -112,9 +119,19 @@ do continue fi echo " ANASUM file: ${ANASUMFILE}" - result=$(v2dl3-eventdisplay-query-runparameters "${ANASUMFILE}" "${RUN}") - EPOCH=$(echo "$result" | awk '{print $2}') - EFFAREA=$(echo "$result" | awk '{print $5}') + QUERY_ERROR_LOG="${TEMPDIR}/${RUN}.v2dl3-query.stderr.log" + : > "${QUERY_ERROR_LOG}" + result=$(v2dl3-eventdisplay-query-runparameters "${ANASUMFILE}" "${RUN}" 2>"${QUERY_ERROR_LOG}") + EPOCH=$(printf '%s\n' "$result" | awk -F': ' '/^Epoch:/ {print $2; exit}') + EFFAREA=$(printf '%s\n' "$result" | awk -F': ' '/^Effective Area:/ {print $2; exit}') + if [[ -z "$EPOCH" || -z "$EFFAREA" ]]; then + echo "Error: could not extract epoch/effective area for run ${RUN}." + cat "${QUERY_ERROR_LOG}" + echo "Query output: ${result}" + echo "Skipping run ${RUN}" + continue + fi + EVNDISPVERSION=$(echo "${EFFAREA}" | grep -oE 'v[0-9]+' | head -n 1) echo " Effective area file: $EFFAREA Epoch: $EPOCH" DBFITSFILE=$(getNumberedDirectory "$RUN" "$VERITAS_DATA_DIR"/shared/DBFITS)/$RUN.db.fits.gz INTERPOLATOR=$(getInterpolator "$EFFAREA") @@ -127,6 +144,19 @@ do for m in "point-like" "full-enclosure" do + if [[ "$m" == "full-enclosure" && \ + "${EVNDISPVERSION,,}" == *v490* && \ + ( "${EFFAREA,,}" == *redhv* || \ + "${EFFAREA,,}" == *uv* ) ]]; then + echo " Skipping full-enclosure conversion for EVNDISPVERSION=${EVNDISPVERSION} and RedHV/UV effective-area file" + rm -f \ + "${ODIR}/full-enclosure/${RUN}.fits.gz" \ + "${ODIR}/full-enclosure/${RUN}.log" \ + "${ODIR}/full-enclosure-all-events/${RUN}.fits.gz" \ + "${ODIR}/full-enclosure-all-events/${RUN}.log" + continue + fi + echo " Converting (${m}, ${V2DL3OPT[*]})" for p in "" "-all-events" @@ -140,20 +170,30 @@ do echo "EVENTFILTER ${V2DL3SELECT[*]}" mkdir -p ${ODIR}/${m}${p} - rm -f ${ODIR}/${m}${p}/"${RUN}".log + RUN_LOG="${ODIR}/${m}${p}/${RUN}.log" + EVENTDISPLAY_STDERR_LOG="${TEMPDIR}/${RUN}.${m}${p}.stderr.log" + rm -f ${ODIR}/${m}${p}/"${RUN}".log "$EVENTDISPLAY_STDERR_LOG" + + if [[ -s "${QUERY_ERROR_LOG}" ]]; then + cat "${QUERY_ERROR_LOG}" >> ${ODIR}/${m}${p}/"${RUN}".log + fi v2dl3-eventdisplay \ --${m} \ "${V2DL3OPT[@]}" "${V2DL3SELECT[@]}" \ --file_pair "${ANASUMFILE}" "$VERITAS_EVNDISP_AUX_DIR"/EffectiveAreas/"${EFFAREA}" \ - --logfile ${ODIR}/${m}${p}/"${RUN}".log \ + --logfile "$RUN_LOG" \ --instrument_epoch "${EPOCH}" \ --interpolator_name "${INTERPOLATOR}" \ --db_fits_file "${DBFITSFILE}" \ - ${ODIR}/${m}${p}/"${RUN}".fits.gz + ${ODIR}/${m}${p}/"${RUN}".fits.gz \ + 2> "$EVENTDISPLAY_STDERR_LOG" + + cat "$EVENTDISPLAY_STDERR_LOG" >> "$RUN_LOG" + rm -f "$EVENTDISPLAY_STDERR_LOG" - python --version >> ${ODIR}/${m}${p}/"${RUN}".log - conda list -n v2dl3Eventdisplay-${V2DL3VERSION} >> ${ODIR}/${m}${p}/"${RUN}".log + python --version >> ${ODIR}/${m}${p}/"${RUN}".log 2>&1 + conda list -n "${CONDA_ENV}" >> ${ODIR}/${m}${p}/"${RUN}".log 2>&1 PDIR=$(pwd) cd "${PDIR}" || exit done diff --git a/scripts/helper_scripts/IRF.effective_area_parallel_sub.sh b/scripts/helper_scripts/IRF.effective_area_parallel_sub.sh index a6d0aa8..41758d0 100755 --- a/scripts/helper_scripts/IRF.effective_area_parallel_sub.sh +++ b/scripts/helper_scripts/IRF.effective_area_parallel_sub.sh @@ -112,7 +112,7 @@ PARAMFILE=" * ENERGYAXISBINS 60 * ENERGYAXISBINHISTOS 30 * EBIASBINHISTOS 75 -* ANGULARRESOLUTIONBINHISTOS 40 +* ANGULARRESOLUTIONBINHISTOS 200 * RESPONSEMATRICESEBINS 200 * AZIMUTHBINS 1 * FILLMONTECARLOHISTOS 0 diff --git a/scripts/preprocessing/README.md b/scripts/preprocessing/README.md index b7ac0a4..127f369 100644 --- a/scripts/preprocessing/README.md +++ b/scripts/preprocessing/README.md @@ -51,6 +51,26 @@ Move Eventdisplay data products from all stages into an runs_with_issues directo ./check_dl3_number_of_files_per_cut.sh ``` +### Check completeness of a large preprocessing production + +Compare every numeric `.root` input below the selected reference directory +(by default `/evndisp`) with the standard evndisp, mscw, +anasum, and DL3 products. The checker inventories each tree once and writes compact +reports, so it can be used for productions with more than 50,000 files: + +```bash +./check_preprocessing_completeness.sh [report-directory] [reference-subdirectory] [run-list-file] +``` + +The command exits `0` when all products are present, `1` for missing or duplicate +products, and `2` for usage or filesystem errors. If no report directory is given, +one is created below the production directory. The reference subdirectory defaults +to `evndisp`. Runs listed in the optional run-list file (one numeric run per line) +are excluded from the check. See the report's `summary.tsv` and `missing-*.txt` +files for machine-readable results. The filtered reference run list is saved as +`reference-runs.txt` in the report directory. Directory symlinks are followed, +so an `evndisp` link to another filesystem can be used as the reference. + ### Check if runs read from a run list are processed with evndis/mscw ```bash diff --git a/scripts/preprocessing/check_evndisp_mscw_processing.sh b/scripts/preprocessing/check_evndisp_mscw_processing.sh index d892f66..3b5f76e 100755 --- a/scripts/preprocessing/check_evndisp_mscw_processing.sh +++ b/scripts/preprocessing/check_evndisp_mscw_processing.sh @@ -18,7 +18,7 @@ EDVERSION=$(cat "$VERITAS_EVNDISP_AUX_DIR"/IRFMINORVERSION) file_on_disk() { - ARCHIVEDIR="$VERITAS_DATA_DIR/processed_data_${EDVERSION}/${VERITAS_ANALYSIS_TYPE:0:2}/${DTYPE}/" + ARCHIVEDIR="$VERITAS_DATA_DIR/shared/processed_data_${EDVERSION}/${VERITAS_ANALYSIS_TYPE:0:2}/${DTYPE}/" TRUN="$1" if [[ ${TRUN} -lt 100000 ]]; then EDIR="${ARCHIVEDIR}/${TRUN:0:1}/" diff --git a/scripts/preprocessing/check_preprocessing_completeness.sh b/scripts/preprocessing/check_preprocessing_completeness.sh new file mode 100755 index 0000000..b316776 --- /dev/null +++ b/scripts/preprocessing/check_preprocessing_completeness.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Check completeness of a large preprocessing production. + +set -u +set -o pipefail + +usage() +{ + cat <<'EOF' +Usage: check_preprocessing_completeness.sh [report-directory] [reference-subdirectory] [run-list-file] + +Check every baseline .root file against the standard evndisp, mscw, anasum, +and DL3 products. Reports are written to the optional report directory. The +reference subdirectory defaults to 'evndisp'. Runs listed in the optional +run-list file are excluded from the completeness check. +EOF +} + +if [[ $# -lt 1 || $# -gt 4 || ${1:-} == "-h" || ${1:-} == "--help" ]]; then + usage + exit 2 +fi + +if [[ ! -d "$1" || ! -r "$1" || ! -x "$1" ]]; then + echo "Error: production directory '$1' is not a readable directory" >&2 + exit 2 +fi + +ROOT=$(cd "$1" && pwd -P) || exit 2 +REFERENCE_SUBDIR=${3:-evndisp} +EXCLUDE_RUN_LIST=${4:-} +case "/$REFERENCE_SUBDIR/" in + //|*/../*|*/./*|//*) + echo "Error: reference subdirectory must be a non-empty relative path without '.' or '..' components" >&2 + exit 2 + ;; +esac +REFERENCE_DIR="$ROOT/$REFERENCE_SUBDIR" +if [[ ! -d "$REFERENCE_DIR" || ! -r "$REFERENCE_DIR" || ! -x "$REFERENCE_DIR" ]]; then + echo "Error: reference directory '$REFERENCE_DIR' is not a readable directory" >&2 + exit 2 +fi +REFERENCE_DIR=$(cd "$REFERENCE_DIR" && pwd -P) || { + echo "Error: cannot resolve reference directory '$REFERENCE_DIR'" >&2 + exit 2 +} + +if [[ -n "$EXCLUDE_RUN_LIST" && ( ! -f "$EXCLUDE_RUN_LIST" || ! -r "$EXCLUDE_RUN_LIST" ) ]]; then + echo "Error: run-list file '$EXCLUDE_RUN_LIST' is not a readable regular file" >&2 + exit 2 +fi + +if [[ $# -ge 2 ]]; then + REPORT=$2 +else + REPORT="$ROOT/preprocessing-completeness-$(date +%Y%m%dT%H%M%S)" +fi + +if ! mkdir -p "$REPORT"; then + echo "Error: cannot create report directory '$REPORT'" >&2 + exit 2 +fi +if [[ ! -d "$REPORT" || ! -w "$REPORT" ]]; then + echo "Error: report directory '$REPORT' is not writable" >&2 + exit 2 +fi + +# Edit these lists when a new standard cut is introduced. NN productions use +# only the supersoftNN2tel cut; all other productions use the standard cuts. +if [[ "${ROOT##*/}" == "NN" ]]; then + CUTS=(supersoftNN2tel) +else + CUTS=(hard2tel hard3tel moderate2tel soft2tel) +fi + +TARGET_NAMES=(evndisp mscw) +TARGET_REGEXES=('^([0-9]+)\.root$' '^([0-9]+)\.mscw\.root$') +TARGET_GLOBS=('*.root' '*.mscw.root') + +for cut in "${CUTS[@]}"; do + TARGET_NAMES+=("anasum_${cut}") + TARGET_REGEXES+=('^([0-9]+)\.anasum\.root$') + TARGET_GLOBS+=('*.anasum.root') +done + +for mode in pointlike fullenclosure; do + for all_events in '' '-all-events'; do + for cut in "${CUTS[@]}"; do + TARGET_NAMES+=("dl3_${mode}${all_events}_${cut}") + TARGET_REGEXES+=('^([0-9]+)\.fits\.gz$') + TARGET_GLOBS+=('*.fits.gz') + done + done +done + +TMP_WORK=$(mktemp -d "${TMPDIR:-/tmp}/check-preprocessing-completeness.XXXXXX") || { + echo "Error: cannot create temporary working directory" >&2 + exit 2 +} +# shellcheck disable=SC2329,SC2317 # cleanup is invoked indirectly by trap +cleanup() +{ + rm -rf "$TMP_WORK" +} +trap cleanup EXIT HUP INT TERM + +export LC_ALL=C + +EXPECTED_RAW="$TMP_WORK/expected.raw.tsv" +EXPECTED_RUNS="$TMP_WORK/expected-runs.txt" +EXPECTED_PATHS="$TMP_WORK/expected-paths.tsv" +BASELINE_PATHS0="$TMP_WORK/baseline-paths.null" +BASELINE_DUPLICATES="$REPORT/baseline-duplicates.tsv" +BASELINE_DUPLICATES_RUNS="$TMP_WORK/baseline-duplicate-runs.txt" +EXCLUDED_RUNS_FILE="$TMP_WORK/excluded-runs.txt" +: > "$EXPECTED_RAW" +: > "$EXPECTED_PATHS" +baseline_file_count=0 + +declare -A EXCLUDED_RUNS=() +if [[ -n "$EXCLUDE_RUN_LIST" ]]; then + if ! awk ' + /^[[:space:]]*($|#)/ { next } + { + run = $0 + sub(/^[[:space:]]+/, "", run) + sub(/[[:space:]]+$/, "", run) + if (run !~ /^[0-9]+$/) { + invalid = 1 + next + } + print run + } + END { exit invalid } + ' "$EXCLUDE_RUN_LIST" | sort -nu > "$EXCLUDED_RUNS_FILE"; then + echo "Error: run-list file '$EXCLUDE_RUN_LIST' must contain one numeric run per line" >&2 + exit 2 + fi + while IFS= read -r run; do + [[ -n "$run" ]] && EXCLUDED_RUNS["$run"]=1 + done < "$EXCLUDED_RUNS_FILE" +fi + +run_is_excluded() +{ + [[ -n ${EXCLUDED_RUNS[$1]+yes} ]] +} + +filesystem_error=0 +if ! find -H "$REFERENCE_DIR" -type f -name '*.root' -print0 > "$BASELINE_PATHS0"; then + echo "Error: reference traversal failed for '$REFERENCE_DIR'" >&2 + filesystem_error=1 +fi + +while IFS= read -r -d '' path; do + name=${path##*/} + if [[ "$name" =~ ^([0-9]+)\.root$ ]]; then + baseline_file_count=$((baseline_file_count + 1)) + run=${BASH_REMATCH[1]} + run_is_excluded "$run" && continue + printf '%s\t%s\n' "$run" "$path" >> "$EXPECTED_PATHS" + printf '%s\n' "$run" >> "$EXPECTED_RAW" + fi +done < "$BASELINE_PATHS0" + +sort -u "$EXPECTED_RAW" > "$EXPECTED_RUNS" +awk -F '\t' '{ count[$1]++ } END { for (run in count) if (count[run] > 1) print run }' \ + "$EXPECTED_PATHS" | sort -u > "$BASELINE_DUPLICATES_RUNS" +: > "$BASELINE_DUPLICATES" +if [[ -s "$BASELINE_DUPLICATES_RUNS" ]]; then + awk -F '\t' 'NR == FNR { duplicate[$1] = 1; next } duplicate[$1] { print }' \ + "$BASELINE_DUPLICATES_RUNS" "$EXPECTED_PATHS" | sort -t $'\t' -k1,1 -k2,2 > "$BASELINE_DUPLICATES" +fi + +expected_count=$(wc -l < "$EXPECTED_RUNS" | tr -d '[:space:]') +baseline_duplicate_count=$(wc -l < "$BASELINE_DUPLICATES_RUNS" | tr -d '[:space:]') +REFERENCE_RUNS="$REPORT/reference-runs.txt" +if ! cp "$EXPECTED_RUNS" "$REFERENCE_RUNS"; then + echo "Error: cannot write reference run list '$REFERENCE_RUNS'" >&2 + exit 2 +fi + +if [[ "$expected_count" -eq 0 && "$baseline_file_count" -eq 0 ]]; then + echo "Error: no baseline files matching .root found below '$REFERENCE_DIR'" >&2 + exit 2 +fi + +SUMMARY="$REPORT/summary.tsv" +cat > "$SUMMARY" <<'EOF' +target status expected present missing unexpected duplicate_runs +EOF +printf 'Baseline runs: %s\n' "$expected_count" +if [[ "${#EXCLUDED_RUNS[@]}" -gt 0 ]]; then + printf 'Excluded runs: %s\n' "${#EXCLUDED_RUNS[@]}" +fi +if [[ "$baseline_duplicate_count" -gt 0 ]]; then + printf 'Baseline duplicate runs: %s (see %s)\n' "$baseline_duplicate_count" "$BASELINE_DUPLICATES" +fi + +incomplete=0 +if [[ "$baseline_duplicate_count" -gt 0 ]]; then + incomplete=1 +fi + +for index in "${!TARGET_NAMES[@]}"; do + target=${TARGET_NAMES[$index]} + target_dir="$ROOT/$target" + target_runs0="$TMP_WORK/${target}.paths.null" + target_paths="$TMP_WORK/${target}.paths.tsv" + target_runs="$TMP_WORK/${target}.runs.txt" + duplicate_runs="$TMP_WORK/${target}.duplicate-runs.txt" + missing_report="$REPORT/missing-${target}.txt" + unexpected_report="$REPORT/unexpected-${target}.txt" + duplicate_report="$REPORT/duplicates-${target}.tsv" + : > "$target_paths" + : > "$target_runs" + : > "$duplicate_runs" + : > "$missing_report" + : > "$unexpected_report" + : > "$duplicate_report" + + if [[ ! -d "$target_dir" || ! -r "$target_dir" || ! -x "$target_dir" ]]; then + cp "$EXPECTED_RUNS" "$missing_report" + if [[ "$expected_count" -eq 0 ]]; then + printf '%s\tok\t0\t0\t0\t0\t0\n' "$target" >> "$SUMMARY" + printf '%-42s ok (no expected runs)\n' "$target" + continue + fi + printf '%s\tmissing-directory\t%s\t0\t%s\t0\t0\n' \ + "$target" "$expected_count" "$expected_count" >> "$SUMMARY" + printf '%-42s missing directory\n' "$target" + incomplete=1 + continue + fi + + if ! find -H "$target_dir" -type f -name "${TARGET_GLOBS[$index]}" -print0 > "$target_runs0"; then + echo "Error: traversal failed for '$target_dir'" >&2 + filesystem_error=1 + fi + + while IFS= read -r -d '' path; do + name=${path##*/} + if [[ "$name" =~ ${TARGET_REGEXES[$index]} ]]; then + run=${BASH_REMATCH[1]} + run_is_excluded "$run" && continue + printf '%s\t%s\n' "$run" "$path" >> "$target_paths" + printf '%s\n' "$run" >> "$target_runs" + fi + done < "$target_runs0" + + sort -u "$target_runs" -o "$target_runs" + awk -F '\t' '{ count[$1]++ } END { for (run in count) if (count[run] > 1) print run }' \ + "$target_paths" | sort -u > "$duplicate_runs" + if [[ -s "$duplicate_runs" ]]; then + awk -F '\t' 'NR == FNR { duplicate[$1] = 1; next } duplicate[$1] { print }' \ + "$duplicate_runs" "$target_paths" | sort -t $'\t' -k1,1 -k2,2 > "$duplicate_report" + fi + + comm -23 "$EXPECTED_RUNS" "$target_runs" > "$missing_report" + comm -13 "$EXPECTED_RUNS" "$target_runs" > "$unexpected_report" + missing_count=$(wc -l < "$missing_report" | tr -d '[:space:]') + unexpected_count=$(wc -l < "$unexpected_report" | tr -d '[:space:]') + duplicate_count=$(wc -l < "$duplicate_runs" | tr -d '[:space:]') + present_count=$((expected_count - missing_count)) + status=ok + if [[ "$missing_count" -gt 0 || "$duplicate_count" -gt 0 ]]; then + status=incomplete + incomplete=1 + fi + if [[ "$unexpected_count" -gt 0 ]]; then + status=${status/ok/warning} + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$target" "$status" "$expected_count" "$present_count" "$missing_count" \ + "$unexpected_count" "$duplicate_count" >> "$SUMMARY" + printf '%-42s %s (present %s, missing %s, unexpected %s, duplicates %s)\n' \ + "$target" "$status" "$present_count" "$missing_count" "$unexpected_count" "$duplicate_count" +done + +cat > "$REPORT/README" <.root basenames below the evndisp reference +directory (/evndisp). +EOF + +printf 'Reports: %s\n' "$REPORT" +if [[ "$filesystem_error" -ne 0 ]]; then + exit 2 +fi +if [[ "$incomplete" -ne 0 ]]; then + exit 1 +fi +exit 0 diff --git a/scripts/preprocessing/prepare_runlist_after_dqm.sh b/scripts/preprocessing/prepare_runlist_after_dqm.sh index 1437031..9f36ecb 100755 --- a/scripts/preprocessing/prepare_runlist_after_dqm.sh +++ b/scripts/preprocessing/prepare_runlist_after_dqm.sh @@ -165,9 +165,21 @@ do # Check DQM # DQM string DQMSTRING=$(tar -axf "${DBTEXTFILE}" "${DQMFILE}" -O) + # A zero-sized archive member is possible when no DQM record exists. + # Treat it like a missing DQM file instead of trying to parse it. + if [[ -z "${DQMSTRING//[[:space:]]/}" ]]; then + echo " RUN $R empty DQM file ${DQMFILE} found (NODQMFILE CUT APPLIED)" + echo "${R}" >> runlist_NODQM.dat + continue + fi echo "$DQMSTRING" # data category - RCAT=$(echo "${DQMSTRING}" | cut -d '|' -f 2 "${RDQM}" | grep -v data_category) + RCAT=$(echo "${DQMSTRING}" | cut -d '|' -f 2 | grep -v data_category) + if [[ -z "${RCAT//[[:space:]]/}" ]]; then + echo " RUN $R malformed DQM file ${DQMFILE} (no data category; NODQMFILE CUT APPLIED)" + echo "${R}" >> runlist_NODQM.dat + continue + fi # (especially early runs do not have a science category) if [[ ${RCAT} != "science" ]] \ && [[ ${RCAT} != "reducedhv" ]] \ @@ -177,7 +189,7 @@ do continue fi # DQM status - RSTATUS=$(echo "${DQMSTRING}" | cut -d '|' -f 3 "${RDQM}" | grep -v status) + RSTATUS=$(echo "${DQMSTRING}" | cut -d '|' -f 3 | grep -v status) if [[ ${RSTATUS} == "do_not_use" ]] || [[ ${RSTATUS} == "NULL" ]]; then # early V4 runs without DQM if [[ ${RSTATUS} == "do_not_use" ]] || [[ $R -gt 46642 ]]; then @@ -191,7 +203,7 @@ do INFOSTRING=$(tar -axf "${DBTEXTFILE}" "${INFOFILE}" -O) echo "$INFOSTRING" # usable duration - RUSABLE=$(echo "${DQMSTRING}" | cut -d '|' -f 6 "${RDQM}" | grep -v usable_duration) + RUSABLE=$(echo "${DQMSTRING}" | cut -d '|' -f 6 | grep -v usable_duration) if [[ $RUSABLE != "NULL" ]]; then RTUSABLE=$(echo "$RUSABLE" | awk 'NR==1 {split($1, arr, "[:]"); print arr[2]}') if [[ $((10#$RTUSABLE)) -lt 2 ]]; then @@ -204,8 +216,8 @@ do continue fi # data duration frum run info - RDATAT1=$(echo "${INFOSTRING}" | cut -d '|' -f 7 "${RDQM}" | grep -v data_start_time) - RDATAT2=$(echo "${INFOSTRING}" | cut -d '|' -f 8 "${RDQM}" | grep -v data_end_time) + RDATAT1=$(echo "${INFOSTRING}" | cut -d '|' -f 7 | grep -v data_start_time) + RDATAT2=$(echo "${INFOSTRING}" | cut -d '|' -f 8 | grep -v data_end_time) echo " RUN $R $RDATAT1 $RDATAT2" RDATAT1=$(date -u -d "$RDATAT1" +%s) RDATAT2=$(date -u -d "$RDATAT2" +%s) @@ -216,7 +228,7 @@ do continue fi # time mask - RCUTMASK=$(echo "${DQMSTRING}" | cut -d '|' -f 7 "${RDQM}" | grep -v time_cut_mask) + RCUTMASK=$(echo "${DQMSTRING}" | cut -d '|' -f 7 | grep -v time_cut_mask) if [[ $RCUTMASK != "NULL" ]]; then IFS=',' for TCUT in $RCUTMASK diff --git a/scripts/RUNLIST.preprocessing.sh b/scripts/preprocessing/prepare_runlist_from_db.sh similarity index 96% rename from scripts/RUNLIST.preprocessing.sh rename to scripts/preprocessing/prepare_runlist_from_db.sh index 70fce6c..040afd9 100755 --- a/scripts/RUNLIST.preprocessing.sh +++ b/scripts/preprocessing/prepare_runlist_from_db.sh @@ -7,7 +7,7 @@ echo " EVNDISP runlist script: generate a simple run list (one run per line) with loose quality cuts used for preprocessing. -RUNLIST.preprocessing.sh [start date] [end date] +./prepare_runlist_from_db [start date] [end date] [start date] select all runs on or after this date (default: 2011-01-01, format = YYYY-MM-DD) @@ -24,7 +24,7 @@ exit fi # Run init script -bash "$( cd "$( dirname "$0" )" && pwd )/helper_scripts/UTILITY.script_init.sh" || exit 1 +bash "$( cd "$( dirname "$0" )" && pwd )/../helper_scripts/UTILITY.script_init.sh" || exit 1 # Parse command line arguments [[ "$1" ]] && START_DATE=$1" 00:00:00" || START_DATE="2011-01-01 00:00:00" diff --git a/scripts/preprocessing/prepare_yearly_runlists_from_db.sh b/scripts/preprocessing/prepare_yearly_runlists_from_db.sh new file mode 100755 index 0000000..208e1fc --- /dev/null +++ b/scripts/preprocessing/prepare_yearly_runlists_from_db.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Generates a simple run list (one run per line) with quality cuts + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +for y in $(seq 2012 2025); do + "$script_dir/prepare_runlist_from_db.sh" "${y}-08-01" "$((y+1))-09-01" /afs/ifh.de/group/cta/scratch/maierg/EVNDISP/EVNDISP-400/GITHUB_Eventdisplay/EventDisplay_Preprocessing/processing/runlists_good_observation_runs/runs_not_processed.dat >| "runs_${y}_$((y+1)).txt" +done diff --git a/scripts/preprocessing/prepro_check_and_clean_files.sh b/scripts/preprocessing/prepro_check_and_clean_files.sh index 5232e9a..9a183b5 100755 --- a/scripts/preprocessing/prepro_check_and_clean_files.sh +++ b/scripts/preprocessing/prepro_check_and_clean_files.sh @@ -31,6 +31,24 @@ move_list() done } +# for xgb products: require the eventdisplay-ml completion message +if [[ $FTYPE == "xgb" ]]; then + xgb_bad_logs="" + shopt -s nullglob + for F in "$FTYPE"/*.log; do + if ! grep -qF "INFO:eventdisplay_ml.models:Total processed events written" "$F"; then + xgb_bad_logs+="$F "$'\n' + fi + done + shopt -u nullglob + + if [[ -n $xgb_bad_logs ]]; then + file_count=$(echo "$xgb_bad_logs" | wc -w) + echo "FOUND $file_count xgb log files without the eventdisplay-ml completion message" + move_list error "$xgb_bad_logs" + fi +fi + # for anasum products: require VERITAS_ANALYSIS_TYPE in the last log line if [[ $FTYPE == anasum* ]]; then anasum_bad_logs="" diff --git a/scripts/preprocessing/prepro_rsync_desy_dcache.sh b/scripts/preprocessing/prepro_rsync_desy_dcache.sh index e2d70c7..1f8aa30 100755 --- a/scripts/preprocessing/prepro_rsync_desy_dcache.sh +++ b/scripts/preprocessing/prepro_rsync_desy_dcache.sh @@ -36,6 +36,14 @@ process_sync() { # ---- Jobs ---- +# DBFITS +echo "Syncing DBFITS" +process_sync "$IDIR/DBFITS/" "$BDIR/DBFITS/" + +# DBTEXT +echo "Syncing DBTEXT" +process_sync "$IDIR/DBTEXT/" "$BDIR/DBTEXT/" + # v490.7 echo "Syncing evndisp v490.7 AP" process_sync "$IDIR/processed_data_v490.7/AP/evndisp/" "$BDIR/v490.7/AP/evndisp/" @@ -51,11 +59,3 @@ echo "Syncing DL3 v491.0" process_sync "$IDIR/processed_data_v491.0/AP/" "$BDIR/v491.0/DL3/" "dl3*.tar.gz" echo "Syncing mscw v491.0" process_sync "$IDIR/processed_data_v491.0/AP/mscw/" "$BDIR/v491.0/AP/mscw/" - -# DBFITS -echo "Syncing DBFITS" -process_sync "$IDIR/DBFITS" "$BDIR/DBFITS" - -# DBTEXT -echo "Syncing DBTEXT" -process_sync "$IDIR/DBTEXT" "$BDIR/DBTEXT" diff --git a/scripts/set_environment.sh b/scripts/set_environment.sh index 60b2e67..9e74f1c 100755 --- a/scripts/set_environment.sh +++ b/scripts/set_environment.sh @@ -62,7 +62,7 @@ export V2DL3SYS=${USERAFSDIR}/EVNDISP/EVNDISP-400/GITHUB_Eventdisplay/PreProcess export EVNDISP_ML_SYS=${USERAFSDIR}/EVNDISP/EVNDISP-400/GITHUB_Eventdisplay/Eventdisplay-ML # EVENTDISPLAY using apptainers if [[ $PROCESS == "apptainer"* ]]; then - export EVNDISP_APPTAINER="$VERITAS_DATA_DIR/shared/APPTAINERS/eventdisplay_v4_${EVNDISPVERSION}-rc6.sif" + export EVNDISP_APPTAINER="$VERITAS_DATA_DIR/shared/APPTAINERS/eventdisplay_v4_${EVNDISPVERSION}-rc7.sif" export EVNDISP_ENV="--env VERITAS_DATA_DIR=${VERITAS_DATA_DIR},VERITAS_EVNDISP_AUX_DIR=${VERITAS_EVNDISP_AUX_DIR},VERITAS_USER_DATA_DIR=${VERITAS_USER_DATA_DIR},VERITAS_USER_LOG_DIR=${VERITAS_USER_LOG_DIR}" export EVNDISPSYS="apptainer exec --no-mount bind-paths --cleanenv ${EVNDISP_APPTAINER} /opt/EventDisplay_v4/" # Alma Linux 9 (al9) processing