-
Notifications
You must be signed in to change notification settings - Fork 14
185 lines (180 loc) · 6.84 KB
/
deploy.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
name: Galaxy Tool Deployment
on:
push:
branches:
- develop
env:
GALAXY_FORK: galaxyproject
GALAXY_BRANCH: release_23.1
MAX_CHUNKS: 4
jobs:
# This is a setup modified from that at
# https://github.com/galaxyproject/tools-iuc/blob/master/.github/workflows/pr.yaml
# The setup job does two things:
# 1. cache the pip cache and .planemo
# 2. determine the list of changed repositories
# it produces one artifact which contains
# - a file with the latest SHA from the chosen branch of the Galaxy repo
# - a file containing the list of changed repositories
# which are needed in subsequent steps.
setup:
name: Setup cache and Planemo
runs-on: ubuntu-latest
outputs:
galaxy_head_sha: ${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }}
strategy:
matrix:
python-version: ['3.7']
steps:
- name: Print github context properties
run: |
echo 'event: ${{ github.event_name }}'
echo 'sha: ${{ github.sha }}'
echo 'ref: ${{ github.ref }}'
echo 'head_ref: ${{ github.head_ref }}'
echo 'base_ref: ${{ github.base_ref }}'
echo 'event.before: ${{ github.event.before }}'
echo 'event.after: ${{ github.event.after }}'
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Determine latest commit in the Galaxy repo
id: get-galaxy-sha
run: echo "::set-output name=galaxy_head_sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)"
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }}
- name: Cache .planemo
uses: actions/cache@v3
id: cache-planemo
with:
path: ~/.planemo
key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }}
# Install the `wheel` package so that when installing other packages which
# are not available as wheels, pip will build a wheel for them, which can be cached.
- name: Install wheel
run: pip install wheel
- name: Install Planemo
run: pip install planemo
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Fake a Planemo run to update cache
uses: galaxyproject/planemo-ci-action@v1
id: discover
with:
create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }}
galaxy-fork: ${{ env.GALAXY_FORK }}
galaxy-branch: ${{ env.GALAXY_BRANCH }}
max-chunks: ${{ env.MAX_CHUNKS }}
python-version: ${{ matrix.python-version }}
find_changed:
name: Determine changed repositories
needs: [setup]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ['3.7']
outputs:
commit_range: ${{ steps.get-commit-range.outputs.commit_range }}
nchunks: ${{ steps.get-chunks.outputs.nchunks }}
chunk_list: ${{ steps.get-chunks.outputs.chunk_list }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
# The range of commits to check for changes is:
# - for events on the develop branch we compare against the sha before the event
# (note that this does not work for feature branch events since we want all
# commits on the feature branch and not just the commits of the last event)
- name: Set commit range (push to the develop branch, e.g. merge)
run: echo "COMMIT_RANGE=${{ github.event.before }}.." >> $GITHUB_ENV
- id: get-commit-range
run: echo "::set-output name=commit_range::$COMMIT_RANGE"
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }}
- name: Install Planemo
run: pip install planemo
- name: Planemo ci_find_repos
run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --output changed_repositories.list
- name: Show repo list
run: cat changed_repositories.list
- uses: actions/upload-artifact@v3
with:
name: Workflow artifacts
path: changed_repositories.list
- name: Planemo ci_find_tools for the changed repos
run: |
touch changed_tools.list
if [ -s changed_repositories.list ]; then
planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list)
fi
- name: Show tool list
run: cat changed_tools.list
- name: Compute chunks
id: get-chunks
run: |
nchunks=$(wc -l < changed_tools.list)
if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then
nchunks=$MAX_CHUNKS
elif [ "$nchunks" -eq 0 ]; then
nchunks=1
fi
echo "::set-output name=nchunks::$nchunks"
echo "::set-output name=chunk_list::[$(seq -s ", " 0 $(($nchunks - 1)))]"
- name: Show chunks
run: |
echo 'Using ${{ steps.get-chunks.outputs.nchunks }} chunks (${{ steps.get-chunks.outputs.chunk_list }})'
# deploy the tools to the toolshed
deploy:
name: Deploy
needs: [setup, find_changed]
strategy:
matrix:
python-version: ['3.7']
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/develop' && github.repository_owner == 'ebi-gene-expression-group'
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/download-artifact@v2
with:
name: Workflow artifacts
path: ../workflow_artifacts/
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }}
- name: Install Planemo
run: pip install planemo
- name: Deploy on toolshed
env:
SHED_KEY: ${{ secrets.TS_API_KEY }}
run: |
while read -r DIR; do
if [ -d "$DIR/test-data" ]; then
max_test_file_size=$(du $DIR/test-data/$(ls -S $DIR/test-data | head -n 1) | awk '{print $1}')
if [ $max_test_file_size -gt 10000 ]; then
echo "Deleting test files, found one > 10Mb" 1>&2
rm -rf $DIR/test-data
fi
fi
planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1;
done < ../workflow_artifacts/changed_repositories.list