Bash script to evaluate all epoch checkpoints for a run

bash_scripts/run_evaluation_array.sh

@@ -10,7 +10,7 @@
 #SBATCH -e slurm_array.%A-%a.%N.err
 #SBATCH --mail-type=ALL
 #SBATCH [email protected]
-#SBATCH --array=0-2%3
+#SBATCH --array=0-5%3
 
 
 # NOTE on SBATCH command for array jobs
@@ -21,7 +21,7 @@
 # Source bashrc
 # ----------------------
 # Otherwise `which python` points to the miniconda module's Python
-source ~/.bashrc
+# source ~/.bashrc
 
 
 # memory
@@ -40,19 +40,23 @@ set -o pipefail  # make the pipe fail if any part of it fails
 # Define variables
 # ----------------------
 
+# mlflow
+MLFLOW_FOLDER=/ceph/zoo/users/sminano/ml-runs-all/ml-runs-scratch
+
 # List of models to evaluate
-MLFLOW_CKPTS_FOLDER=/ceph/zoo/users/sminano/ml-runs-all/ml-runs/317777717624044570/fe9a6c2f491a4496aade5034c75316cc/checkpoints
+MLFLOW_CKPTS_FOLDER=/ceph/zoo/users/sminano/ml-runs-all/ml-runs/317777717624044570/7a6d5551ca974d578a293928d6385d5a/checkpoints
 LIST_CKPT_FILES=("$MLFLOW_CKPTS_FOLDER"/*.ckpt)
 
 # selected model
 CKPT_PATH=${LIST_CKPT_FILES[${SLURM_ARRAY_TASK_ID}]}
 
-# destination mlflow folder
-# EXPERIMENT_NAME="Sept2023" ----> get from training job
-MLFLOW_FOLDER=/ceph/zoo/users/sminano/ml-runs-all/ml-runs
+# select whether to evaluate on the validation set or on the
+# test set
+EVALUATION_SPLIT=validation
 
 # version of the codebase
-GIT_BRANCH=main
+# GIT_BRANCH=main-------------------------------------------
+GIT_BRANCH=smg/eval-bash-script-cluster
 
 # --------------------
 # Check inputs
@@ -71,7 +75,7 @@ module load miniconda
 
 # Define a environment for each job in the
 # temporary directory of the compute node
-ENV_NAME=crabs-dev-$SPLIT_SEED-$SLURM_ARRAY_JOB_ID
+ENV_NAME=crabs-dev-$SLURM_ARRAY_JOB_ID-$SLURM_ARRAY_TASK_ID
 ENV_PREFIX=$TMPDIR/$ENV_NAME
 
 # create environment
@@ -81,7 +85,7 @@ conda create \
     python=3.10
 
 # activate environment
-conda activate $ENV_PREFIX
+source activate $ENV_PREFIX
 
 # install crabs package in virtual env
 python -m pip install git+https://github.com/SainsburyWellcomeCentre/crabs-exploration.git@$GIT_BRANCH
@@ -105,12 +109,21 @@ echo $(nvidia-smi --query-gpu=name,memory.total,memory.free,memory.used --format
 echo "-----"
 
 
-# -------------------
-# Run training script
-# -------------------
-echo "Evaluating trained model at $CKPT_PATH: "
+# -------------------------
+# Run evaluation script
+# -------------------------
+echo "Evaluating trained model at $CKPT_PATH on $EVALUATION_SPLIT set: "
+
+# conditionally append flag to command
+if [ "$EVALUATION_SPLIT" = "validation" ]; then
+    USE_TEST_SET_FLAG=""
+elif [ "$EVALUATION_SPLIT" = "test" ]; then
+    USE_TEST_SET_FLAG="--use_test_set"
+fi
+
 evaluate-detector  \
  --trained_model_path $CKPT_PATH \
  --accelerator gpu \
  --mlflow_folder $MLFLOW_FOLDER \
+ $USE_TEST_SET_FLAG
 echo "-----"