Group CI scripts in subdirectories (apache#9653)

Reviewed the scripts and removed some of the old unused ones.

.github/workflows/ci.yml

@@ -62,17 +62,17 @@ jobs:
           path: ~/.cache/pre-commit
           key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Static checks"
         if: success()
         env:
           PYTHON_VERSION: 3.6
         run: |
           python -m pip install pre-commit \
               --constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
-          ./scripts/ci/ci_run_static_checks.sh pylint-tests mypy yamllint flake8
+          ./scripts/ci/static_checks/ci_run_static_checks.sh pylint-tests mypy yamllint flake8
 
   static-checks-2:
     timeout-minutes: 60
@@ -95,14 +95,14 @@ jobs:
           path: ~/.cache/pre-commit
           key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Static checks"
         run: |
           python -m pip install pre-commit \
               --constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
-          ./scripts/ci/ci_run_static_checks.sh
+          ./scripts/ci/static_checks/ci_run_static_checks.sh
 
   docs:
     timeout-minutes: 60
@@ -113,9 +113,9 @@ jobs:
     steps:
       - uses: actions/checkout@master
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Build docs"
-        run: ./scripts/ci/ci_docs.sh
+        run: ./scripts/ci/docs/ci_docs.sh
 
   build-prod-image:
     timeout-minutes: 60
@@ -130,7 +130,7 @@ jobs:
     steps:
       - uses: actions/checkout@master
       - name: "Build PROD image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
 
   prepare-backport-packages:
     timeout-minutes: 60
@@ -148,12 +148,12 @@ jobs:
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Prepare & test backport packages"
         run: |
-          ./scripts/ci/ci_prepare_and_test_backport_packages.sh
+          ./scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
 
   trigger-tests:
     timeout-minutes: 10
@@ -166,7 +166,7 @@ jobs:
       - name: "Get count of changed python files"
         run: |
           set +e
-          ./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} \
+          ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \
               '^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests'
           echo "::set-output name=count::$?"
         id: trigger-tests
@@ -209,15 +209,15 @@ jobs:
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - uses: engineerd/[email protected]
         name: Setup Kind Cluster
         with:
           version: "${{ matrix.kind-version }}"
           name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}}
           config: "scripts/ci/kubernetes/kind-cluster-conf.yaml"
       - name: "Deploy app to cluster"
-        run: ./scripts/ci/ci_deploy_app_to_kubernetes.sh
+        run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
       - name: Cache virtualenv for kubernetes testing
         uses: actions/cache@v2
         env:
@@ -227,7 +227,7 @@ jobs:
           key: "${{ env.cache-name }}-${{ github.job }}-\
 ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}"
       - name: "Tests"
-        run: ./scripts/ci/ci_run_kubernetes_tests.sh
+        run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
       - uses: actions/upload-artifact@v2
         name: Upload KinD logs
         # Always run this, even if one of th previous steps failed.
@@ -262,11 +262,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   tests-mysql:
     timeout-minutes: 80
@@ -294,11 +294,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   tests-sqlite:
     timeout-minutes: 80
@@ -324,11 +324,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   tests-quarantined:
     timeout-minutes: 80
@@ -357,11 +357,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   helm-tests:
     timeout-minutes: 5
@@ -391,11 +391,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
       - uses: actions/checkout@master
       - uses: actions/setup-python@v1
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Generate requirements"
-        run: ./scripts/ci/ci_generate_requirements.sh
+        run: ./scripts/ci/requirements/ci_generate_requirements.sh
 
   push-prod-images-to-github-cache:
     timeout-minutes: 80
@@ -419,11 +419,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     steps:
       - uses: actions/checkout@master
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build PROD images ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
       - name: "Push PROD images ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_push_production_images.sh
+        run: ./scripts/ci/images/ci_push_production_images.sh
 
   push-ci-images-to-github-cache:
     timeout-minutes: 40
@@ -451,11 +451,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     steps:
       - uses: actions/checkout@master
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Push CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_push_ci_image.sh
+        run: ./scripts/ci/images/ci_push_ci_image.sh
 
   tag-repo-nightly:
     timeout-minutes: 60
@@ -473,7 +473,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     steps:
       - uses: actions/checkout@master
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Tag commit"
         run: |
           BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///')

.pre-commit-config.yaml

@@ -205,43 +205,43 @@ repos:
       - id: lint-dockerfile
         name: Lint dockerfile
         language: system
-        entry: "./scripts/ci/pre_commit_lint_dockerfile.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh"
         files: ^Dockerfile.*$
         pass_filenames: true
       - id: setup-order
         name: Checks for an order of dependencies in setup.py
         language: python
         files: ^setup.py$
         pass_filenames: false
-        entry: ./scripts/ci/pre_commit_check_order_setup.py
+        entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
       - id: update-breeze-file
         name: Update output of breeze command in BREEZE.rst
-        entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh"
         language: system
         files: ^BREEZE.rst$|^breeze$|^breeze-complete$
         pass_filenames: false
       - id: update-local-yml-file
         name: Update mounts in the local yml file
-        entry: "./scripts/ci/pre_commit_local_yml_mounts.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh"
         language: system
         files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml"
         pass_filenames: false
       - id: update-setup-cfg-file
         name: Update setup.cfg file with all licenses
-        entry: "./scripts/ci/pre_commit_setup_cfg_file.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh"
         language: system
         files: ^setup.cfg$
         pass_filenames: false
       - id: build-providers-dependencies
         name: Build cross-dependencies for providers packages
-        entry: "./scripts/ci/pre_commit_build_providers_dependencies.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh"
         language: system
         files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$
         pass_filenames: false
       - id: update-extras
         name: Update extras in documentation
-        entry: "./scripts/ci/pre_commit_update_extras.sh"
-        language: system
+        entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
+        language: python
         files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$
         pass_filenames: false
       - id: pydevd
@@ -324,62 +324,62 @@ repos:
         pass_filenames: true
       - id: check-integrations
         name: Check if integration list is aligned
-        entry: ./scripts/ci/pre_commit_check_integrations.sh
+        entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
         language: system
         pass_filenames: false
         files: ^common/_common_values.sh$|^breeze-complete$
       - id: build
         name: Check if image build is needed
-        entry: ./scripts/ci/pre_commit_ci_build.sh 3.6 false
+        entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false
         language: system
         always_run: true
         pass_filenames: false
       - id: check-apache-license
         name: Check if licenses are OK for Apache
-        entry: "./scripts/ci/pre_commit_check_license.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh"
         language: system
         files: ^.*LICENSE.*$|^.*LICENCE.*$
         pass_filenames: false
       - id: airflow-config-yaml
         name: Checks for consistency between config.yml and default_config.cfg
         language: python
+        entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
         files: "config.yml$|default_airflow.cfg$|default.cfg$"
         pass_filenames: false
-        require_serial: false
-        entry: ./scripts/ci/pre_commit_yaml_to_cfg.py
+        require_serial: true
         additional_dependencies: ['pyyaml']
       - id: mypy
         name: Run mypy
         language: system
-        entry: "./scripts/ci/pre_commit_mypy.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
         files: \.py$
         exclude: ^dev|^backport_packages
         require_serial: true
       - id: pylint
         name: Run pylint for main sources
         language: system
-        entry: "./scripts/ci/pre_commit_pylint_main.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_pylint_main.sh"
         files: \.py$
         exclude: ^tests/.*\.py$|^scripts/.*\.py$|^dev|^backport_packages|^kubernetes_tests
         pass_filenames: true
         require_serial: true  # Pylint tests should be run in one chunk to detect all cycles
       - id: pylint-tests
         name: Run pylint for tests
         language: system
-        entry: "./scripts/ci/pre_commit_pylint_tests.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_pylint_tests.sh"
         files: ^tests/.*\.py$
         pass_filenames: true
         require_serial: true
       - id: flake8
         name: Run flake8
         language: system
-        entry: "./scripts/ci/pre_commit_flake8.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh"
         files: \.py$
         pass_filenames: true
       - id: bat-tests
         name: Run BATS bash tests for changed bash files
         language: system
-        entry: "./scripts/ci/pre_commit_bat_tests.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh"
         files: ^breeze$|^breeze-complete$|\.sh$|\.bash$
         pass_filenames: false
       - id: stylelint

.rat-excludes

@@ -91,3 +91,4 @@ input_notebook.ipynb
 
 # .git might be a file in case of worktree
 .git
+tmp

BREEZE.rst

@@ -922,7 +922,7 @@ by the root user, you can fix the ownership of those files by running this scrip
 
 .. code-block::
 
-  ./scripts/ci/ci_fix_ownership.sh
+  ./scripts/ci/tools/ci_fix_ownership.sh
 
 Mounting Local Sources to Breeze
 --------------------------------