diff --git a/.github/workflows/calibration.yaml b/.github/workflows/calibration.yaml index 3a6547aef7..9ac68940af 100644 --- a/.github/workflows/calibration.yaml +++ b/.github/workflows/calibration.yaml @@ -2,14 +2,16 @@ name: Calibration on: push: - branches: "master" + branches: "mg/nightly-calibration" workflow_dispatch: inputs: commit: - description: 'Commit' + description: '40-character commit hash' required: true default: "" type: string + schedule: + - cron: 31 2 * * * env: REPO_PATH: /mnt/tlo/TLOmodel @@ -44,8 +46,17 @@ jobs: else SHA=${{ github.sha }} fi + SHA=e7cb080ca2b100c82b3d9c7063a6c91f73e63a25 ENV="/mnt/tlo/env-${SHA}" + if [[ -d "${ENV}" ]]; then + echo "Virtual environment directory ${ENV} already exists, leaving..." + # exit 1 + fi WORKTREE_PATH="/mnt/tlo/${SHA}" + if [[ -d "${WORKTREE_PATH}" ]]; then + echo "Worktree directory ${WORKTREE_PATH} already exists, leaving..." + # exit 1 + fi echo "SHA=${SHA}" echo "SHA=${SHA}" >> "${GITHUB_ENV}" echo "ENV=${ENV}" @@ -88,6 +99,10 @@ jobs: - name: Generate list of tasks id: tasks run: | + if [[ -d "${{ output_dir }}" ]]; then + echo "Output directory ${{ needs.setup.outputs.output_dir }} already exists, setting RUNS_NUMBER to 0" + RUNS_NUMBER=0 + fi RUNS="[" for run in $(seq 0 $((${RUNS_NUMBER} - 1))); do RUNS="${RUNS}\"${run}\"," @@ -100,7 +115,7 @@ jobs: tasks: needs: setup name: Run task ${{ matrix.index }} - runs-on: [tlo-dev-vm-1, tasks] # Use only runners dedicated to running the tasks. + runs-on: [tlo-dev-vm-1] # Use only runners dedicated to running the tasks. timeout-minutes: 5760 # = 4 * 24 * 60 minutes = 4 days strategy: fail-fast: false @@ -109,38 +124,38 @@ jobs: steps: - name: Run the task run: | - source "${{ needs.setup.outputs.environment_path }}/bin/activate" - draw=0 - task_output_dir="${{ needs.setup.outputs.output_dir }}/${RUN_NAME}/${draw}/${{ matrix.index }}" - mkdir -p "${task_output_dir}" + echo "index = ${{ matrix.index }}" + # source "${{ needs.setup.outputs.environment_path }}/bin/activate" + # draw=0 + # task_output_dir="${{ needs.setup.outputs.output_dir }}/${RUN_NAME}/${draw}/${{ matrix.index }}" + # mkdir -p "${task_output_dir}" - tlo scenario-run --output-dir "${task_output_dir}" --draw "${draw}" ${{ matrix.index }} "${{ needs.setup.outputs.worktree_path }}/src/scripts/calibration_analyses/scenarios/long_run_all_diseases.py" - working-directory: "${{ needs.setup.outputs.worktree_path }}" + # tlo scenario-run --output-dir "${task_output_dir}" --draw "${draw}" ${{ matrix.index }} "${{ needs.setup.outputs.worktree_path }}/src/scripts/calibration_analyses/scenarios/long_run_all_diseases.py" + # working-directory: "${{ needs.setup.outputs.worktree_path }}" - # Do the postprocessing - postprocess: - name: Post processing - needs: [setup, tasks] - runs-on: [tlo-dev-vm-1, postprocess] # Use only the runners dedicated to postprocessing - strategy: - fail-fast: false - steps: - - name: Run post-processing - run: | - source "${{ needs.setup.outputs.environment_path }}/bin/activate" - task_output_dir="${{ needs.setup.outputs.output_dir }}/${PROCESS_NAME}" - mkdir -p "${task_output_dir}" + # # Do the postprocessing + # postprocess: + # name: Post processing + # needs: [setup, tasks] + # runs-on: [tlo-dev-vm-1, postprocess] # Use only the runners dedicated to postprocessing + # strategy: + # fail-fast: false + # steps: + # - name: Run post-processing + # run: | + # source "${{ needs.setup.outputs.environment_path }}/bin/activate" + # task_output_dir="${{ needs.setup.outputs.output_dir }}/${PROCESS_NAME}" + # mkdir -p "${task_output_dir}" - python3 "${{ needs.setup.outputs.worktree_path }}/src/scripts/calibration_analyses/analysis_scripts/process.py" "${task_output_dir}" "${RUN_NAME}" "${{ needs.setup.outputs.worktree_path }}/resources" - working-directory: "${{ needs.setup.outputs.worktree_path }}" + # python3 "${{ needs.setup.outputs.worktree_path }}/src/scripts/calibration_analyses/analysis_scripts/process.py" "${task_output_dir}" "${RUN_NAME}" "${{ needs.setup.outputs.worktree_path }}/resources" + # working-directory: "${{ needs.setup.outputs.worktree_path }}" # Cleanup stage, to remove temporary directories and such cleanup: name: Cleanup job - # It depends on all the previous jobs, but it always runs, regardless of - # their success (or maybe check that only `setup` was - # successful?) - if: ${{ always() }} + # It depends on all the previous jobs, but it runs only if `setup` was + # successful + if: ${{ needs.setup.result == "success" }} timeout-minutes: 10 needs: [setup, tasks, postprocess] runs-on: [tlo-dev-vm-1] @@ -149,8 +164,8 @@ jobs: steps: - name: Cleanup worktree run: | - git -C "${REPO_PATH}" worktree remove -f "${{ needs.setup.outputs.worktree_path }}" || true + echo git -C "${REPO_PATH}" worktree remove -f "${{ needs.setup.outputs.worktree_path }}" || true - name: Cleanup virtual environment run: | - rm -rvf "${{ needs.setup.outputs.environment_path }}" + echo "${{ needs.setup.outputs.environment_path }}"