Decoupler pseudo bulk (#303) #61
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Galaxy Tool Deployment | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| env: | |
| GALAXY_FORK: galaxyproject | |
| GALAXY_BRANCH: release_21.01 | |
| MAX_CHUNKS: 4 | |
| jobs: | |
| # This is a setup modified from that at | |
| # https://github.com/galaxyproject/tools-iuc/blob/master/.github/workflows/pr.yaml | |
| # The setup job does two things: | |
| # 1. cache the pip cache and .planemo | |
| # 2. determine the list of changed repositories | |
| # it produces one artifact which contains | |
| # - a file with the latest SHA from the chosen branch of the Galaxy repo | |
| # - a file containing the list of changed repositories | |
| # which are needed in subsequent steps. | |
| setup: | |
| name: Setup cache and Planemo | |
| runs-on: self-hosted | |
| outputs: | |
| galaxy_head_sha: ${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} | |
| strategy: | |
| matrix: | |
| python-version: [3.8] | |
| steps: | |
| - name: Print github context properties | |
| run: | | |
| echo 'event: ${{ github.event_name }}' | |
| echo 'sha: ${{ github.sha }}' | |
| echo 'ref: ${{ github.ref }}' | |
| echo 'head_ref: ${{ github.head_ref }}' | |
| echo 'base_ref: ${{ github.base_ref }}' | |
| echo 'event.before: ${{ github.event.before }}' | |
| echo 'event.after: ${{ github.event.after }}' | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Determine latest commit in the Galaxy repo | |
| id: get-galaxy-sha | |
| run: echo "::set-output name=galaxy_head_sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" | |
| - name: Cache .cache/pip | |
| uses: actions/cache@v2 | |
| id: cache-pip | |
| with: | |
| path: ~/.cache/pip | |
| key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} | |
| - name: Cache .planemo | |
| uses: actions/cache@v2 | |
| id: cache-planemo | |
| with: | |
| path: ~/.planemo | |
| key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} | |
| # Install the `wheel` package so that when installing other packages which | |
| # are not available as wheels, pip will build a wheel for them, which can be cached. | |
| - name: Install wheel | |
| run: pip install wheel | |
| - name: Install Planemo | |
| run: pip install planemo | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Fake a Planemo run to update cache | |
| uses: galaxyproject/planemo-ci-action@v1 | |
| id: discover | |
| with: | |
| create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} | |
| galaxy-fork: ${{ env.GALAXY_FORK }} | |
| galaxy-branch: ${{ env.GALAXY_BRANCH }} | |
| max-chunks: ${{ env.MAX_CHUNKS }} | |
| python-version: ${{ matrix.python-version }} | |
| find_changed: | |
| name: Determine changed repositories | |
| needs: [setup] | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.8] | |
| outputs: | |
| commit_range: ${{ steps.get-commit-range.outputs.commit_range }} | |
| nchunks: ${{ steps.get-chunks.outputs.nchunks }} | |
| chunk_list: ${{ steps.get-chunks.outputs.chunk_list }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| # The range of commits to check for changes is: | |
| # - for events on the develop branch we compare against the sha before the event | |
| # (note that this does not work for feature branch events since we want all | |
| # commits on the feature branch and not just the commits of the last event) | |
| - name: Set commit range (push to the develop branch, e.g. merge) | |
| run: echo "COMMIT_RANGE=${{ github.event.before }}.." >> $GITHUB_ENV | |
| - id: get-commit-range | |
| run: echo "::set-output name=commit_range::$COMMIT_RANGE" | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache .cache/pip | |
| uses: actions/cache@v2 | |
| id: cache-pip | |
| with: | |
| path: ~/.cache/pip | |
| key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} | |
| - name: Install Planemo | |
| run: pip install planemo | |
| - name: Planemo ci_find_repos | |
| run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --output changed_repositories.list | |
| - name: Show repo list | |
| run: cat changed_repositories.list | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: Workflow artifacts | |
| path: changed_repositories.list | |
| - name: Planemo ci_find_tools for the changed repos | |
| run: | | |
| touch changed_tools.list | |
| if [ -s changed_repositories.list ]; then | |
| planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) | |
| fi | |
| - name: Show tool list | |
| run: cat changed_tools.list | |
| - name: Compute chunks | |
| id: get-chunks | |
| run: | | |
| nchunks=$(wc -l < changed_tools.list) | |
| if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then | |
| nchunks=$MAX_CHUNKS | |
| elif [ "$nchunks" -eq 0 ]; then | |
| nchunks=1 | |
| fi | |
| echo "::set-output name=nchunks::$nchunks" | |
| echo "::set-output name=chunk_list::[$(seq -s ", " 0 $(($nchunks - 1)))]" | |
| - name: Show chunks | |
| run: | | |
| echo 'Using ${{ steps.get-chunks.outputs.nchunks }} chunks (${{ steps.get-chunks.outputs.chunk_list }})' | |
| # deploy the tools to the toolshed | |
| deploy: | |
| name: Deploy | |
| needs: [setup, find_changed] | |
| strategy: | |
| matrix: | |
| python-version: [3.8] | |
| runs-on: self-hosted | |
| if: github.ref == 'refs/heads/develop' && github.repository_owner == 'ebi-gene-expression-group' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: Workflow artifacts | |
| path: ../workflow_artifacts/ | |
| - name: Cache .cache/pip | |
| uses: actions/cache@v2 | |
| id: cache-pip | |
| with: | |
| path: ~/.cache/pip | |
| key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} | |
| - name: Install Planemo | |
| run: pip install planemo | |
| - name: Deploy on toolshed | |
| env: | |
| SHED_KEY: ${{ secrets.TS_API_KEY }} | |
| run: | | |
| while read -r DIR; do | |
| if [ -d "$DIR/test-data" ]; then | |
| max_test_file_size=$(du $DIR/test-data/$(ls -S $DIR/test-data | head -n 1) | awk '{print $1}') | |
| if [ $max_test_file_size -gt 10000 ]; then | |
| echo "Deleting test files, found one > 10Mb" 1>&2 | |
| rm -rf $DIR/test-data | |
| fi | |
| fi | |
| planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; | |
| done < ../workflow_artifacts/changed_repositories.list |