From faec41ec9a05a037b88fd0213b1936cde2b5c454 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 16 Jul 2020 18:05:35 +0200 Subject: [PATCH] Group CI scripts in subdirectories (#9653) Reviewed the scripts and removed some of the old unused ones. --- .github/workflows/ci.yml | 76 +++++++++---------- .pre-commit-config.yaml | 36 ++++----- .rat-excludes | 1 + BREEZE.rst | 2 +- STATIC_CODE_CHECKS.rst | 30 ++++---- TESTING.rst | 8 +- backport_packages/build_source_package.sh | 4 +- breeze | 4 +- docs/start_doc_server.sh | 4 +- hooks/build | 4 +- hooks/push | 5 -- .../ci_prepare_and_test_backport_packages.sh | 16 ++-- .../ci_prepare_backport_packages.sh | 4 +- .../ci_prepare_backport_readme.sh | 4 +- ...st_backport_packages_import_all_classes.sh | 4 +- ...st_backport_packages_install_separately.sh | 4 +- scripts/ci/ci_load_image_to_kind.sh | 33 -------- .../ci/ci_perform_kind_cluster_operation.sh | 32 -------- scripts/ci/{ => docs}/ci_docs.sh | 4 +- scripts/ci/{ => images}/ci_build_dockerhub.sh | 4 +- .../{ => images}/ci_prepare_ci_image_on_ci.sh | 4 +- .../ci_prepare_prod_image_on_ci.sh | 4 +- scripts/ci/{ => images}/ci_push_ci_image.sh | 4 +- .../{ => images}/ci_push_production_images.sh | 4 +- .../in_container/_in_container_script_init.sh | 4 +- .../ci/in_container/_in_container_utils.sh | 22 +++--- .../deploy_airflow_to_kubernetes.sh | 23 ------ scripts/ci/in_container/entrypoint_ci.sh | 16 ++-- scripts/ci/{ => in_container}/run_cli_tool.sh | 0 scripts/ci/in_container/run_pylint_main.sh | 3 +- scripts/ci/in_container/run_pylint_tests.sh | 4 +- scripts/ci/in_container/run_system_tests.sh | 4 +- .../ci_deploy_app_to_kubernetes.sh | 4 +- .../ci_run_kubernetes_tests.sh | 4 +- scripts/ci/{ => libraries}/_all_libs.sh | 31 ++++---- scripts/ci/libraries/_build_images.sh | 2 +- scripts/ci/libraries/_initialization.sh | 6 +- scripts/ci/libraries/_kind.sh | 40 ++++++---- scripts/ci/{ => libraries}/_script_init.sh | 13 ++-- scripts/ci/minikdc.properties | 27 ------- .../{ => pre_commit}/pre_commit_bat_tests.sh | 4 +- .../pre_commit_breeze_cmd_line.sh | 10 ++- ...pre_commit_build_providers_dependencies.sh | 9 +-- .../pre_commit_check_integrations.sh | 9 +-- .../pre_commit_check_license.sh | 2 +- .../pre_commit_check_order_setup.py | 2 +- .../{ => pre_commit}/pre_commit_ci_build.sh | 4 +- .../ci/{ => pre_commit}/pre_commit_flake8.sh | 4 +- .../pre_commit_generate_requirements.sh | 4 +- .../ci/pre_commit/pre_commit_insert_extras.py | 4 +- .../pre_commit_lint_dockerfile.sh | 4 +- .../pre_commit_local_yml_mounts.sh | 8 +- .../ci/{ => pre_commit}/pre_commit_mypy.sh | 4 +- .../pre_commit_pylint_main.sh | 4 +- .../pre_commit_pylint_tests.sh | 4 +- .../pre_commit_setup_cfg_file.sh | 8 +- .../pre_commit_yaml_to_cfg.py | 11 ++- scripts/ci/pre_commit_update_extras.sh | 31 -------- .../ci_generate_requirements.sh | 4 +- .../ci/{ => static_checks}/ci_bat_tests.sh | 3 + .../{ => static_checks}/ci_check_license.sh | 4 +- scripts/ci/{ => static_checks}/ci_flake8.sh | 4 +- .../{ => static_checks}/ci_lint_dockerfile.sh | 4 +- scripts/ci/{ => static_checks}/ci_mypy.sh | 4 +- .../ci/{ => static_checks}/ci_pylint_main.sh | 4 +- .../ci/{ => static_checks}/ci_pylint_tests.sh | 4 +- .../ci_refresh_pylint_todo.sh | 4 +- .../ci_run_static_checks.sh | 4 +- .../{ => testing}/ci_run_airflow_testing.sh | 33 ++++---- .../ci/{ => tools}/ci_count_changed_files.sh | 4 +- scripts/ci/{ => tools}/ci_fix_ownership.sh | 10 +-- scripts/ci/{ => tools}/ci_free_space_on_ci.sh | 4 +- tests/bats/bats_utils.bash | 4 +- 73 files changed, 289 insertions(+), 429 deletions(-) rename scripts/ci/{ => backport_packages}/ci_prepare_and_test_backport_packages.sh (68%) rename scripts/ci/{ => backport_packages}/ci_prepare_backport_packages.sh (88%) rename scripts/ci/{ => backport_packages}/ci_prepare_backport_readme.sh (88%) rename scripts/ci/{ => backport_packages}/ci_test_backport_packages_import_all_classes.sh (94%) rename scripts/ci/{ => backport_packages}/ci_test_backport_packages_install_separately.sh (93%) delete mode 100755 scripts/ci/ci_load_image_to_kind.sh delete mode 100755 scripts/ci/ci_perform_kind_cluster_operation.sh rename scripts/ci/{ => docs}/ci_docs.sh (92%) rename scripts/ci/{ => images}/ci_build_dockerhub.sh (95%) rename scripts/ci/{ => images}/ci_prepare_ci_image_on_ci.sh (87%) rename scripts/ci/{ => images}/ci_prepare_prod_image_on_ci.sh (87%) rename scripts/ci/{ => images}/ci_push_ci_image.sh (88%) rename scripts/ci/{ => images}/ci_push_production_images.sh (88%) delete mode 100755 scripts/ci/in_container/deploy_airflow_to_kubernetes.sh rename scripts/ci/{ => in_container}/run_cli_tool.sh (100%) rename scripts/ci/{ => kubernetes}/ci_deploy_app_to_kubernetes.sh (92%) rename scripts/ci/{ => kubernetes}/ci_run_kubernetes_tests.sh (96%) rename scripts/ci/{ => libraries}/_all_libs.sh (67%) rename scripts/ci/{ => libraries}/_script_init.sh (73%) delete mode 100644 scripts/ci/minikdc.properties rename scripts/ci/{ => pre_commit}/pre_commit_bat_tests.sh (87%) rename scripts/ci/{ => pre_commit}/pre_commit_breeze_cmd_line.sh (89%) rename scripts/ci/{ => pre_commit}/pre_commit_build_providers_dependencies.sh (85%) rename scripts/ci/{ => pre_commit}/pre_commit_check_integrations.sh (86%) rename scripts/ci/{ => pre_commit}/pre_commit_check_license.sh (92%) rename scripts/ci/{ => pre_commit}/pre_commit_check_order_setup.py (99%) rename scripts/ci/{ => pre_commit}/pre_commit_ci_build.sh (88%) rename scripts/ci/{ => pre_commit}/pre_commit_flake8.sh (87%) rename scripts/ci/{ => pre_commit}/pre_commit_generate_requirements.sh (85%) rename tests/insert_extras.py => scripts/ci/pre_commit/pre_commit_insert_extras.py (94%) mode change 100644 => 100755 rename scripts/ci/{ => pre_commit}/pre_commit_lint_dockerfile.sh (84%) rename scripts/ci/{ => pre_commit}/pre_commit_local_yml_mounts.sh (86%) rename scripts/ci/{ => pre_commit}/pre_commit_mypy.sh (87%) rename scripts/ci/{ => pre_commit}/pre_commit_pylint_main.sh (86%) rename scripts/ci/{ => pre_commit}/pre_commit_pylint_tests.sh (86%) rename scripts/ci/{ => pre_commit}/pre_commit_setup_cfg_file.sh (87%) rename scripts/ci/{ => pre_commit}/pre_commit_yaml_to_cfg.py (95%) delete mode 100755 scripts/ci/pre_commit_update_extras.sh rename scripts/ci/{ => requirements}/ci_generate_requirements.sh (88%) rename scripts/ci/{ => static_checks}/ci_bat_tests.sh (90%) rename scripts/ci/{ => static_checks}/ci_check_license.sh (94%) rename scripts/ci/{ => static_checks}/ci_flake8.sh (94%) rename scripts/ci/{ => static_checks}/ci_lint_dockerfile.sh (92%) rename scripts/ci/{ => static_checks}/ci_mypy.sh (93%) rename scripts/ci/{ => static_checks}/ci_pylint_main.sh (95%) rename scripts/ci/{ => static_checks}/ci_pylint_tests.sh (95%) rename scripts/ci/{ => static_checks}/ci_refresh_pylint_todo.sh (92%) rename scripts/ci/{ => static_checks}/ci_run_static_checks.sh (91%) rename scripts/ci/{ => testing}/ci_run_airflow_testing.sh (82%) rename scripts/ci/{ => tools}/ci_count_changed_files.sh (92%) rename scripts/ci/{ => tools}/ci_fix_ownership.sh (82%) rename scripts/ci/{ => tools}/ci_free_space_on_ci.sh (87%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6818c810f1a89..0df0c66bab88b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,9 +62,9 @@ jobs: path: ~/.cache/pre-commit key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }} - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Static checks" if: success() env: @@ -72,7 +72,7 @@ jobs: run: | python -m pip install pre-commit \ --constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt - ./scripts/ci/ci_run_static_checks.sh pylint-tests mypy yamllint flake8 + ./scripts/ci/static_checks/ci_run_static_checks.sh pylint-tests mypy yamllint flake8 static-checks-2: timeout-minutes: 60 @@ -95,14 +95,14 @@ jobs: path: ~/.cache/pre-commit key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }} - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Static checks" run: | python -m pip install pre-commit \ --constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt - ./scripts/ci/ci_run_static_checks.sh + ./scripts/ci/static_checks/ci_run_static_checks.sh docs: timeout-minutes: 60 @@ -113,9 +113,9 @@ jobs: steps: - uses: actions/checkout@master - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Build docs" - run: ./scripts/ci/ci_docs.sh + run: ./scripts/ci/docs/ci_docs.sh build-prod-image: timeout-minutes: 60 @@ -130,7 +130,7 @@ jobs: steps: - uses: actions/checkout@master - name: "Build PROD image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh prepare-backport-packages: timeout-minutes: 60 @@ -148,12 +148,12 @@ jobs: with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Prepare & test backport packages" run: | - ./scripts/ci/ci_prepare_and_test_backport_packages.sh + ./scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh trigger-tests: timeout-minutes: 10 @@ -166,7 +166,7 @@ jobs: - name: "Get count of changed python files" run: | set +e - ./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} \ + ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \ '^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests' echo "::set-output name=count::$?" id: trigger-tests @@ -209,7 +209,7 @@ jobs: with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - uses: engineerd/setup-kind@v0.4.0 name: Setup Kind Cluster with: @@ -217,7 +217,7 @@ jobs: name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}} config: "scripts/ci/kubernetes/kind-cluster-conf.yaml" - name: "Deploy app to cluster" - run: ./scripts/ci/ci_deploy_app_to_kubernetes.sh + run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh - name: Cache virtualenv for kubernetes testing uses: actions/cache@v2 env: @@ -227,7 +227,7 @@ jobs: key: "${{ env.cache-name }}-${{ github.job }}-\ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}" - name: "Tests" - run: ./scripts/ci/ci_run_kubernetes_tests.sh + run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh - uses: actions/upload-artifact@v2 name: Upload KinD logs # Always run this, even if one of th previous steps failed. @@ -262,11 +262,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh tests-mysql: timeout-minutes: 80 @@ -294,11 +294,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh tests-sqlite: timeout-minutes: 80 @@ -324,11 +324,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh tests-quarantined: timeout-minutes: 80 @@ -357,11 +357,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh helm-tests: timeout-minutes: 5 @@ -391,11 +391,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - uses: actions/checkout@master - uses: actions/setup-python@v1 - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Generate requirements" - run: ./scripts/ci/ci_generate_requirements.sh + run: ./scripts/ci/requirements/ci_generate_requirements.sh push-prod-images-to-github-cache: timeout-minutes: 80 @@ -419,11 +419,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') steps: - uses: actions/checkout@master - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build PROD images ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - name: "Push PROD images ${{ matrix.python-version }}" - run: ./scripts/ci/ci_push_production_images.sh + run: ./scripts/ci/images/ci_push_production_images.sh push-ci-images-to-github-cache: timeout-minutes: 40 @@ -451,11 +451,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') steps: - uses: actions/checkout@master - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Push CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_push_ci_image.sh + run: ./scripts/ci/images/ci_push_ci_image.sh tag-repo-nightly: timeout-minutes: 60 @@ -473,7 +473,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') steps: - uses: actions/checkout@master - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Tag commit" run: | BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///') diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 771aa13d001f8..011c72824a4db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -205,7 +205,7 @@ repos: - id: lint-dockerfile name: Lint dockerfile language: system - entry: "./scripts/ci/pre_commit_lint_dockerfile.sh" + entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh" files: ^Dockerfile.*$ pass_filenames: true - id: setup-order @@ -213,35 +213,35 @@ repos: language: python files: ^setup.py$ pass_filenames: false - entry: ./scripts/ci/pre_commit_check_order_setup.py + entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py - id: update-breeze-file name: Update output of breeze command in BREEZE.rst - entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh" + entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh" language: system files: ^BREEZE.rst$|^breeze$|^breeze-complete$ pass_filenames: false - id: update-local-yml-file name: Update mounts in the local yml file - entry: "./scripts/ci/pre_commit_local_yml_mounts.sh" + entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh" language: system files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml" pass_filenames: false - id: update-setup-cfg-file name: Update setup.cfg file with all licenses - entry: "./scripts/ci/pre_commit_setup_cfg_file.sh" + entry: "./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh" language: system files: ^setup.cfg$ pass_filenames: false - id: build-providers-dependencies name: Build cross-dependencies for providers packages - entry: "./scripts/ci/pre_commit_build_providers_dependencies.sh" + entry: "./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh" language: system files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$ pass_filenames: false - id: update-extras name: Update extras in documentation - entry: "./scripts/ci/pre_commit_update_extras.sh" - language: system + entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py + language: python files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$ pass_filenames: false - id: pydevd @@ -324,41 +324,41 @@ repos: pass_filenames: true - id: check-integrations name: Check if integration list is aligned - entry: ./scripts/ci/pre_commit_check_integrations.sh + entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh language: system pass_filenames: false files: ^common/_common_values.sh$|^breeze-complete$ - id: build name: Check if image build is needed - entry: ./scripts/ci/pre_commit_ci_build.sh 3.6 false + entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false language: system always_run: true pass_filenames: false - id: check-apache-license name: Check if licenses are OK for Apache - entry: "./scripts/ci/pre_commit_check_license.sh" + entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh" language: system files: ^.*LICENSE.*$|^.*LICENCE.*$ pass_filenames: false - id: airflow-config-yaml name: Checks for consistency between config.yml and default_config.cfg language: python + entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py files: "config.yml$|default_airflow.cfg$|default.cfg$" pass_filenames: false - require_serial: false - entry: ./scripts/ci/pre_commit_yaml_to_cfg.py + require_serial: true additional_dependencies: ['pyyaml'] - id: mypy name: Run mypy language: system - entry: "./scripts/ci/pre_commit_mypy.sh" + entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" files: \.py$ exclude: ^dev|^backport_packages require_serial: true - id: pylint name: Run pylint for main sources language: system - entry: "./scripts/ci/pre_commit_pylint_main.sh" + entry: "./scripts/ci/pre_commit/pre_commit_pylint_main.sh" files: \.py$ exclude: ^tests/.*\.py$|^scripts/.*\.py$|^dev|^backport_packages|^kubernetes_tests pass_filenames: true @@ -366,20 +366,20 @@ repos: - id: pylint-tests name: Run pylint for tests language: system - entry: "./scripts/ci/pre_commit_pylint_tests.sh" + entry: "./scripts/ci/pre_commit/pre_commit_pylint_tests.sh" files: ^tests/.*\.py$ pass_filenames: true require_serial: true - id: flake8 name: Run flake8 language: system - entry: "./scripts/ci/pre_commit_flake8.sh" + entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh" files: \.py$ pass_filenames: true - id: bat-tests name: Run BATS bash tests for changed bash files language: system - entry: "./scripts/ci/pre_commit_bat_tests.sh" + entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh" files: ^breeze$|^breeze-complete$|\.sh$|\.bash$ pass_filenames: false - id: stylelint diff --git a/.rat-excludes b/.rat-excludes index 4dc43a9633637..dd8a89f403dd6 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -91,3 +91,4 @@ input_notebook.ipynb # .git might be a file in case of worktree .git +tmp diff --git a/BREEZE.rst b/BREEZE.rst index 25223dc1f954f..cf7fd051c3989 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -922,7 +922,7 @@ by the root user, you can fix the ownership of those files by running this scrip .. code-block:: - ./scripts/ci/ci_fix_ownership.sh + ./scripts/ci/tools/ci_fix_ownership.sh Mounting Local Sources to Breeze -------------------------------- diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 9157c7d20115f..42522c37da3c6 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -228,15 +228,15 @@ contributor to Airflow, you can choose one of the sub-tasks as your first issue To fix a pylint issue, do the following: 1. Remove module/modules from the - `scripts/ci/pylint_todo.txt `__. + `scripts/ci/static_checks/pylint_todo.txt `__. -2. Run `scripts/ci/ci_pylint_main.sh `__ and - `scripts/ci/ci_pylint_tests.sh `__. +2. Run `scripts/ci/static_checks/ci_pylint_main.sh `__ and + `scripts/ci/ci_pylint_tests.sh `__. 3. Fix all the issues reported by pylint. -4. Re-run `scripts/ci/ci_pylint_main.sh `__ and - `scripts/ci/ci_pylint_tests.sh `__. +4. Re-run `scripts/ci/static_checks/ci_pylint_main.sh `__ and + `scripts/ci/ci_pylint_tests.sh `__. 5. If you see "success", submit a PR following `Pull Request guidelines <#pull-request-guidelines>`__. @@ -368,15 +368,15 @@ Running Static Code Checks via Scripts from the Host .................................................... You can trigger the static checks from the host environment, without entering the Docker container. To do -this, run the following scripts (the same is done in the CI builds): +this, run the following scripts: -* ``_ - checks the licenses. -* ``_ - checks that documentation can be built without warnings. -* ``_ - runs Flake8 source code style enforcement tool. -* ``_ - runs lint checker for the dockerfiles. -* ``_ - runs a check for mypy type annotation consistency. -* ``_ - runs pylint static code checker for main files. -* ``_ - runs pylint static code checker for tests. +* ``_ - checks that documentation can be built without warnings. +* ``_ - checks the licenses. +* ``_ - runs Flake8 source code style enforcement tool. +* ``_ - runs lint checker for the dockerfiles. +* ``_ - runs a check for mypy type annotation consistency. +* ``_ - runs pylint static code checker for main files. +* ``_ - runs pylint static code checker for tests. The scripts may ask you to rebuild the images, if needed. @@ -421,8 +421,8 @@ On the host: .. code-block:: - ./scripts/ci/ci_pylint.sh ./airflow/example_dags/ + ./scripts/ci/static_checks/ci_pylint.sh ./airflow/example_dags/ .. code-block:: - ./scripts/ci/ci_pylint.sh ./airflow/example_dags/test_utils.py + ./scripts/ci/static_checks/ci_pylint.sh ./airflow/example_dags/test_utils.py diff --git a/TESTING.rst b/TESTING.rst index fc6f5093907bd..6ff4994128bda 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -378,10 +378,10 @@ to run the tests manually one by one. Running kubernetes tests - ./scripts/ci/ci_run_kubernetes_tests.sh - runs all kubernetes tests - ./scripts/ci/ci_run_kubernetes_tests.sh TEST [TEST ...] - runs selected kubernetes tests (from kubernetes_tests folder) - ./scripts/ci/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in - ./scripts/ci/ci_run_kubernetes_tests.sh [--help] - Prints this help message + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh - runs all kubernetes tests + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh TEST [TEST ...] - runs selected kubernetes tests (from kubernetes_tests folder) + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [--help] - Prints this help message You can also run the same tests command with Breeze, using ``kind-cluster test`` command (to run all diff --git a/backport_packages/build_source_package.sh b/backport_packages/build_source_package.sh index 769d9a1b3b6ae..2a68d3bc76919 100755 --- a/backport_packages/build_source_package.sh +++ b/backport_packages/build_source_package.sh @@ -23,8 +23,8 @@ # or it needs to be available in your keychain set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "${MY_DIR}"/.. +BACKPORT_PACKAGES_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "${BACKPORT_PACKAGES_DIR}"/.. function check_version() { : "${VERSION:?"Please export VERSION variable with the version of source package to prepare"}" diff --git a/breeze b/breeze index 2b34b177db013..4c0516759e849 100755 --- a/breeze +++ b/breeze @@ -62,8 +62,8 @@ function setup_default_breeze_variables() { # load all the common functions here - those are the functions that are shared between Breeze # and CI scripts (CI scripts do not use Breeze as execution environment) - # shellcheck source=scripts/ci/_all_libs.sh - . "${SCRIPTS_CI_DIR}/_all_libs.sh" + # shellcheck source=scripts/ci/libraries/_all_libs.sh + . "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh" # We have different versions of images depending on the python version used. We keep up with the # Latest patch-level changes in Python (this is done automatically during CI builds) so we have diff --git a/docs/start_doc_server.sh b/docs/start_doc_server.sh index 26248ece7a929..51cc7a77def2f 100755 --- a/docs/start_doc_server.sh +++ b/docs/start_doc_server.sh @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -(cd "${MY_DIR}"/_build/html || exit; +DOCS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +(cd "${DOCS_DIR}"/_build/html || exit; # The below command works on both Python 2 and Python 3 python -m http.server 8000 && python -m SimpleHTTPServer 8000 ) diff --git a/hooks/build b/hooks/build index a98a9237bbee1..d3d3efb9f1c1f 100755 --- a/hooks/build +++ b/hooks/build @@ -20,9 +20,9 @@ # on Travis CI to potentially rebuild (and refresh layers that # are not cached) Docker images that are used to run CI jobs -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +_HOOK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Dockerhub builds are run inside Docker container export SKIP_IN_CONTAINER_CHECK="true" -exec "${MY_DIR}/../scripts/ci/ci_build_dockerhub.sh" +exec "${_HOOK_DIR}/../scripts/ci/images/ci_build_dockerhub.sh" diff --git a/hooks/push b/hooks/push index 91cf0969ea6f3..84805b0d64e70 100755 --- a/hooks/push +++ b/hooks/push @@ -20,11 +20,6 @@ # and it is difficult to pass list of the built images from the build to push phase set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -echo "My dir: ${MY_DIR}" - - echo echo "Skip pushing the image. All images were built and pushed in the build hook already!" echo diff --git a/scripts/ci/ci_prepare_and_test_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh similarity index 68% rename from scripts/ci/ci_prepare_and_test_backport_packages.sh rename to scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh index eeb50bc2ef590..7982606de5f09 100755 --- a/scripts/ci/ci_prepare_and_test_backport_packages.sh +++ b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh @@ -17,19 +17,19 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" -"${MY_DIR}/ci_prepare_backport_readme.sh" -"${MY_DIR}/ci_prepare_backport_packages.sh" -"${MY_DIR}/ci_test_backport_packages_install_separately.sh" -"${MY_DIR}/ci_test_backport_packages_import_all_classes.sh" +"${SCRIPTS_CI_DIR}/backport_packages/ci_prepare_backport_readme.sh" +"${SCRIPTS_CI_DIR}/backport_packages/ci_prepare_backport_packages.sh" +"${SCRIPTS_CI_DIR}/backport_packages/ci_test_backport_packages_install_separately.sh" +"${SCRIPTS_CI_DIR}/backport_packages/ci_test_backport_packages_import_all_classes.sh" -cd "${MY_DIR}/../../backport_packages" || exit 1 +cd "${AIRFLOW_SOURCES}/backport_packages" || exit 1 DUMP_FILE="/tmp/airflow_provider_packages_$(date +"%Y%m%d-%H%M%S").tar.gz" -cd "${MY_DIR}/../../dist" || exit 1 +cd "${AIRFLOW_SOURCES}/dist" || exit 1 tar -cvzf "${DUMP_FILE}" . echo "Packages are in dist and also tar-gzipped in ${DUMP_FILE}" diff --git a/scripts/ci/ci_prepare_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh similarity index 88% rename from scripts/ci/ci_prepare_backport_packages.sh rename to scripts/ci/backport_packages/ci_prepare_backport_packages.sh index 7a701fe1558ec..e1c8bd000cf4c 100755 --- a/scripts/ci/ci_prepare_backport_packages.sh +++ b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" get_environment_for_builds_on_ci diff --git a/scripts/ci/ci_prepare_backport_readme.sh b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh similarity index 88% rename from scripts/ci/ci_prepare_backport_readme.sh rename to scripts/ci/backport_packages/ci_prepare_backport_readme.sh index 4b73b9eb1e8a0..b869128c90553 100755 --- a/scripts/ci/ci_prepare_backport_readme.sh +++ b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" get_environment_for_builds_on_ci diff --git a/scripts/ci/ci_test_backport_packages_import_all_classes.sh b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh similarity index 94% rename from scripts/ci/ci_test_backport_packages_import_all_classes.sh rename to scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh index c4151542aa01f..0ba120f453f2b 100755 --- a/scripts/ci/ci_test_backport_packages_import_all_classes.sh +++ b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_test_package_import_all_classes() { docker run "${EXTRA_DOCKER_FLAGS[@]}" \ diff --git a/scripts/ci/ci_test_backport_packages_install_separately.sh b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh similarity index 93% rename from scripts/ci/ci_test_backport_packages_install_separately.sh rename to scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh index 21eb55cbec972..3ed23c72a5a84 100755 --- a/scripts/ci/ci_test_backport_packages_install_separately.sh +++ b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_test_package_installation_separately() { docker run "${EXTRA_DOCKER_FLAGS[@]}" \ diff --git a/scripts/ci/ci_load_image_to_kind.sh b/scripts/ci/ci_load_image_to_kind.sh deleted file mode 100755 index dda1e387f72e2..0000000000000 --- a/scripts/ci/ci_load_image_to_kind.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -cd "${AIRFLOW_SOURCES}" || exit 1 - -export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"} -export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"} - -prepare_prod_build -echo -echo "Loading the ${AIRFLOW_PROD_IMAGE} to cluster ${KIND_CLUSTER_NAME} from docker" -echo -"${AIRFLOW_SOURCES}/.build/bin/kind" load docker-image --name "${KIND_CLUSTER_NAME}" "${AIRFLOW_PROD_IMAGE}" -echo -echo "Loaded the ${AIRFLOW_PROD_IMAGE} to cluster ${KIND_CLUSTER_NAME}" -echo diff --git a/scripts/ci/ci_perform_kind_cluster_operation.sh b/scripts/ci/ci_perform_kind_cluster_operation.sh deleted file mode 100755 index 4d3ddd27d9725..0000000000000 --- a/scripts/ci/ci_perform_kind_cluster_operation.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -# adding trap to exiting trap -HANDLERS="$( trap -p EXIT | cut -f2 -d \' )" -# shellcheck disable=SC2064 -trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT - -get_environment_for_builds_on_ci -make_sure_kubernetes_tools_are_installed -initialize_kind_variables -perform_kind_cluster_operation "${@}" - -check_cluster_ready_for_airflow diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/docs/ci_docs.sh similarity index 92% rename from scripts/ci/ci_docs.sh rename to scripts/ci/docs/ci_docs.sh index 761e9a0cf344d..847cb49f1a1b8 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/docs/ci_docs.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_docs() { docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \ diff --git a/scripts/ci/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh similarity index 95% rename from scripts/ci/ci_build_dockerhub.sh rename to scripts/ci/images/ci_build_dockerhub.sh index e8a4e7c795d11..c488a8a3da89f 100755 --- a/scripts/ci/ci_build_dockerhub.sh +++ b/scripts/ci/images/ci_build_dockerhub.sh @@ -47,8 +47,8 @@ echo "DOCKER_TAG=${DOCKER_TAG}" echo "Detected PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}" echo -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" if [[ ${DOCKER_TAG} == *python*-ci ]]; then echo diff --git a/scripts/ci/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh similarity index 87% rename from scripts/ci/ci_prepare_ci_image_on_ci.sh rename to scripts/ci/images/ci_prepare_ci_image_on_ci.sh index 8ced22035004e..5fd7913a8ad9f 100755 --- a/scripts/ci/ci_prepare_ci_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" build_ci_image_on_ci diff --git a/scripts/ci/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh similarity index 87% rename from scripts/ci/ci_prepare_prod_image_on_ci.sh rename to scripts/ci/images/ci_prepare_prod_image_on_ci.sh index 066a43b63318c..ab4e7c0f640da 100755 --- a/scripts/ci/ci_prepare_prod_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" build_prod_image_on_ci diff --git a/scripts/ci/ci_push_ci_image.sh b/scripts/ci/images/ci_push_ci_image.sh similarity index 88% rename from scripts/ci/ci_push_ci_image.sh rename to scripts/ci/images/ci_push_ci_image.sh index 09e2a7aef50fd..664d81a63872e 100755 --- a/scripts/ci/ci_push_ci_image.sh +++ b/scripts/ci/images/ci_push_ci_image.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" prepare_ci_build diff --git a/scripts/ci/ci_push_production_images.sh b/scripts/ci/images/ci_push_production_images.sh similarity index 88% rename from scripts/ci/ci_push_production_images.sh rename to scripts/ci/images/ci_push_production_images.sh index d6c6e7d68f439..db439d8a3c57e 100755 --- a/scripts/ci/ci_push_production_images.sh +++ b/scripts/ci/images/ci_push_production_images.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" prepare_prod_build diff --git a/scripts/ci/in_container/_in_container_script_init.sh b/scripts/ci/in_container/_in_container_script_init.sh index 35400162f4432..50a558e80107e 100755 --- a/scripts/ci/in_container/_in_container_script_init.sh +++ b/scripts/ci/in_container/_in_container_script_init.sh @@ -19,10 +19,10 @@ set -euo pipefail # This should only be sourced from in_container directory! -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +IN_CONTAINER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=scripts/ci/in_container/_in_container_utils.sh -. "${MY_DIR}/_in_container_utils.sh" +. "${IN_CONTAINER_DIR}/_in_container_utils.sh" in_container_basic_sanity_check diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 4b2aa4392d516..e4d3f9d54771d 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -151,15 +151,15 @@ function in_container_refresh_pylint_todo() { -name "*.py" \ -not -name 'webserver_config.py' | \ grep ".*.py$" | \ - xargs pylint | tee "${MY_DIR}/../pylint_todo_main.txt" + xargs pylint | tee "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_main.txt" - grep -v "\*\*" < "${MY_DIR}/../pylint_todo_main.txt" | \ + grep -v "\*\*" < "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_main.txt" | \ grep -v "^$" | grep -v "\-\-\-" | grep -v "^Your code has been" | \ - awk 'BEGIN{FS=":"}{print "./"$1}' | sort | uniq > "${MY_DIR}/../pylint_todo_new.txt" + awk 'BEGIN{FS=":"}{print "./"$1}' | sort | uniq > "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_new.txt" if [[ ${VERBOSE} == "true" ]]; then echo - echo "So far found $(wc -l <"${MY_DIR}/../pylint_todo_new.txt") files" + echo "So far found $(wc -l <"${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_new.txt") files" echo echo @@ -167,18 +167,18 @@ function in_container_refresh_pylint_todo() { echo fi find "./tests" -name "*.py" -print0 | \ - xargs -0 pylint --disable="${DISABLE_CHECKS_FOR_TESTS}" | tee "${MY_DIR}/../pylint_todo_tests.txt" + xargs -0 pylint --disable="${DISABLE_CHECKS_FOR_TESTS}" | tee "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_tests.txt" - grep -v "\*\*" < "${MY_DIR}/../pylint_todo_tests.txt" | \ + grep -v "\*\*" < "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_tests.txt" | \ grep -v "^$" | grep -v "\-\-\-" | grep -v "^Your code has been" | \ - awk 'BEGIN{FS=":"}{print "./"$1}' | sort | uniq >> "${MY_DIR}/../pylint_todo_new.txt" + awk 'BEGIN{FS=":"}{print "./"$1}' | sort | uniq >> "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_new.txt" - rm -fv "${MY_DIR}/../pylint_todo_main.txt" "${MY_DIR}/../pylint_todo_tests.txt" - mv -v "${MY_DIR}/../pylint_todo_new.txt" "${MY_DIR}/../pylint_todo.txt" + rm -fv "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_main.txt" "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_tests.txt" + mv -v "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo_new.txt" "${AIRFLOW_SOURCES}/scripts/ci/pylint_todo.txt" if [[ ${VERBOSE} == "true" ]]; then echo - echo "Found $(wc -l <"${MY_DIR}/../pylint_todo.txt") files" + echo "Found $(wc -l <"${AIRFLOW_SOURCES}/scripts/ci/pylint_todo.txt") files" echo fi } @@ -212,7 +212,7 @@ function setup_kerberos() { PASS="airflow" KRB5_KTNAME=/etc/airflow.keytab - sudo cp "${MY_DIR}/krb5/krb5.conf" /etc/krb5.conf + sudo cp "${AIRFLOW_SOURCES}/scripts/ci/in_container/krb5/krb5.conf" /etc/krb5.conf echo -e "${PASS}\n${PASS}" | \ sudo kadmin -p "${ADMIN}/admin" -w "${PASS}" -q "addprinc -randkey airflow/${FQDN}" 2>&1 \ diff --git a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh b/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh deleted file mode 100755 index a124fba263cc4..0000000000000 --- a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# Script to run flake8 on all code. Can be started from any working directory -# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" - -"${MY_DIR}/kubernetes/docker/rebuild_airflow_image.sh" -"${MY_DIR}/kubernetes/app/deploy_app.sh" diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh index 6ff9134180ec1..d006eaa6652b4 100755 --- a/scripts/ci/in_container/entrypoint_ci.sh +++ b/scripts/ci/in_container/entrypoint_ci.sh @@ -22,7 +22,7 @@ fi # shellcheck source=scripts/ci/in_container/_in_container_script_init.sh . /opt/airflow/scripts/ci/in_container/_in_container_script_init.sh -AIRFLOW_SOURCES=$(cd "${MY_DIR}/../../.." || exit 1; pwd) +AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../../.." || exit 1; pwd) PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.6} BACKEND=${BACKEND:=sqlite} @@ -47,8 +47,8 @@ INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION:=""}" if [[ ${CI} == "false" ]]; then # Create links for useful CLI tools - # shellcheck source=scripts/ci/run_cli_tool.sh - source <(bash scripts/ci/run_cli_tool.sh) + # shellcheck source=scripts/ci/in_container/run_cli_tool.sh + source <(bash scripts/ci/in_container/run_cli_tool.sh) fi if [[ ${AIRFLOW_VERSION} == *1.10* || ${INSTALL_AIRFLOW_VERSION} == *1.10* ]]; then @@ -98,10 +98,10 @@ export PATH=${PATH}:${AIRFLOW_SOURCES} unset AIRFLOW__CORE__UNIT_TEST_MODE mkdir -pv "${AIRFLOW_HOME}/logs/" -cp -f "${MY_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg" +cp -f "${IN_CONTAINER_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg" set +e -"${MY_DIR}/check_environment.sh" +"${IN_CONTAINER_DIR}/check_environment.sh" ENVIRONMENT_EXIT_CODE=$? set -e if [[ ${ENVIRONMENT_EXIT_CODE} != 0 ]]; then @@ -150,7 +150,7 @@ done ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null # shellcheck source=scripts/ci/in_container/configure_environment.sh -. "${MY_DIR}/configure_environment.sh" +. "${IN_CONTAINER_DIR}/configure_environment.sh" cd "${AIRFLOW_SOURCES}" @@ -211,7 +211,7 @@ fi ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${TESTS_TO_RUN[@]}") if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then - "${MY_DIR}/run_system_tests.sh" "${ARGS[@]}" + "${IN_CONTAINER_DIR}/run_system_tests.sh" "${ARGS[@]}" else - "${MY_DIR}/run_ci_tests.sh" "${ARGS[@]}" + "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${ARGS[@]}" fi diff --git a/scripts/ci/run_cli_tool.sh b/scripts/ci/in_container/run_cli_tool.sh similarity index 100% rename from scripts/ci/run_cli_tool.sh rename to scripts/ci/in_container/run_cli_tool.sh diff --git a/scripts/ci/in_container/run_pylint_main.sh b/scripts/ci/in_container/run_pylint_main.sh index ded109fb4414f..3f2510099287b 100755 --- a/scripts/ci/in_container/run_pylint_main.sh +++ b/scripts/ci/in_container/run_pylint_main.sh @@ -24,7 +24,7 @@ set +e if [[ ${#@} == "0" ]]; then echo - echo "Running pylint for all sources except 'tests' folder" + echo "Running pylint for all sources except 'tests' and 'kubernetes_tests' folder" echo # Using path -prune is much better in the local environment on OSX because we have host @@ -39,6 +39,7 @@ if [[ ${#@} == "0" ]]; then -path "./docs/_build" -prune -o \ -path "./build" -prune -o \ -path "./tests" -prune -o \ + -path "./kubernetes_tests" -prune -o \ -name "*.py" \ -not -name 'webserver_config.py' | \ grep ".*.py$" | \ diff --git a/scripts/ci/in_container/run_pylint_tests.sh b/scripts/ci/in_container/run_pylint_tests.sh index c9fc35ee4d9d0..881b03f756880 100755 --- a/scripts/ci/in_container/run_pylint_tests.sh +++ b/scripts/ci/in_container/run_pylint_tests.sh @@ -24,9 +24,9 @@ set +e if [[ ${#@} == "0" ]]; then echo - echo "Running pylint for 'tests' folder" + echo "Running pylint for 'tests' and 'kubernetes_tests' folder" echo - find "./tests" -name "*.py" | \ + find "./tests" "./kubernetes_tests" -name "*.py" | \ grep -vFf scripts/ci/pylint_todo.txt | \ # running pylint using built-in parallel functionality might speed it up xargs pylint -j 0 --disable="${DISABLE_CHECKS_FOR_TESTS}" --output-format=colorized diff --git a/scripts/ci/in_container/run_system_tests.sh b/scripts/ci/in_container/run_system_tests.sh index 11dcf06436bf0..8cb3c3e440b36 100755 --- a/scripts/ci/in_container/run_system_tests.sh +++ b/scripts/ci/in_container/run_system_tests.sh @@ -20,10 +20,10 @@ # Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails) set -euo pipefail -MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) +IN_CONTAINER_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) # shellcheck source=scripts/ci/in_container/_in_container_utils.sh -. "${MY_DIR}/_in_container_utils.sh" +. "${IN_CONTAINER_DIR}/_in_container_utils.sh" in_container_basic_sanity_check diff --git a/scripts/ci/ci_deploy_app_to_kubernetes.sh b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh similarity index 92% rename from scripts/ci/ci_deploy_app_to_kubernetes.sh rename to scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh index 307bf00def4a5..d4bb4b1e5b856 100755 --- a/scripts/ci/ci_deploy_app_to_kubernetes.sh +++ b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" set -euo pipefail diff --git a/scripts/ci/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh similarity index 96% rename from scripts/ci/ci_run_kubernetes_tests.sh rename to scripts/ci/kubernetes/ci_run_kubernetes_tests.sh index 4d49e9e60c6d2..3d8194a4bca50 100755 --- a/scripts/ci/ci_run_kubernetes_tests.sh +++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" # adding trap to exiting trap HANDLERS="$( trap -p EXIT | cut -f2 -d \' )" diff --git a/scripts/ci/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh similarity index 67% rename from scripts/ci/_all_libs.sh rename to scripts/ci/libraries/_all_libs.sh index 2bc16c38b4668..dca8dbd5cca2b 100755 --- a/scripts/ci/_all_libs.sh +++ b/scripts/ci/libraries/_all_libs.sh @@ -16,36 +16,35 @@ # specific language governing permissions and limitations # under the License. -SCRIPTS_CI_DIR=$(dirname "${BASH_SOURCE[0]}") +LIBRARIES_DIR=$(dirname "${BASH_SOURCE[0]}") -# must be first to initialize arrays TODO: For sure? # shellcheck source=scripts/ci/libraries/_initialization.sh -. "${SCRIPTS_CI_DIR}"/libraries/_initialization.sh +. "${LIBRARIES_DIR}"/_initialization.sh # shellcheck source=scripts/ci/libraries/_sanity_checks.sh -. "${SCRIPTS_CI_DIR}"/libraries/_sanity_checks.sh +. "${LIBRARIES_DIR}"/_sanity_checks.sh # shellcheck source=scripts/ci/libraries/_build_images.sh -. "${SCRIPTS_CI_DIR}"/libraries/_build_images.sh +. "${LIBRARIES_DIR}"/_build_images.sh # shellcheck source=scripts/ci/libraries/_kind.sh -. "${SCRIPTS_CI_DIR}"/libraries/_kind.sh +. "${LIBRARIES_DIR}"/_kind.sh # shellcheck source=scripts/ci/libraries/_local_mounts.sh -. "${SCRIPTS_CI_DIR}"/libraries/_local_mounts.sh +. "${LIBRARIES_DIR}"/_local_mounts.sh # shellcheck source=scripts/ci/libraries/_md5sum.sh -. "${SCRIPTS_CI_DIR}"/libraries/_md5sum.sh +. "${LIBRARIES_DIR}"/_md5sum.sh # shellcheck source=scripts/ci/libraries/_parameters.sh -. "${SCRIPTS_CI_DIR}"/libraries/_parameters.sh +. "${LIBRARIES_DIR}"/_parameters.sh # shellcheck source=scripts/ci/libraries/_permissions.sh -. "${SCRIPTS_CI_DIR}"/libraries/_permissions.sh +. "${LIBRARIES_DIR}"/_permissions.sh # shellcheck source=scripts/ci/libraries/_push_pull_remove_images.sh -. "${SCRIPTS_CI_DIR}"/libraries/_push_pull_remove_images.sh +. "${LIBRARIES_DIR}"/_push_pull_remove_images.sh # shellcheck source=scripts/ci/libraries/_pylint.sh -. "${SCRIPTS_CI_DIR}"/libraries/_pylint.sh +. "${LIBRARIES_DIR}"/_pylint.sh # shellcheck source=scripts/ci/libraries/_runs.sh -. "${SCRIPTS_CI_DIR}"/libraries/_runs.sh +. "${LIBRARIES_DIR}"/_runs.sh # shellcheck source=scripts/ci/libraries/_spinner.sh -. "${SCRIPTS_CI_DIR}"/libraries/_spinner.sh +. "${LIBRARIES_DIR}"/_spinner.sh # shellcheck source=scripts/ci/libraries/_start_end.sh -. "${SCRIPTS_CI_DIR}"/libraries/_start_end.sh +. "${LIBRARIES_DIR}"/_start_end.sh # shellcheck source=scripts/ci/libraries/_verbosity.sh -. "${SCRIPTS_CI_DIR}"/libraries/_verbosity.sh +. "${LIBRARIES_DIR}"/_verbosity.sh diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index ed894a127e1b7..343d9e2e885a0 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -415,7 +415,7 @@ function rebuild_ci_image_if_needed() { if [[ ${SYSTEM} != "Darwin" ]]; then ROOT_FILES_COUNT=$(find "airflow" "tests" -user root | wc -l | xargs) if [[ ${ROOT_FILES_COUNT} != "0" ]]; then - ./scripts/ci/ci_fix_ownership.sh + ./scripts/ci/tools/ci_fix_ownership.sh fi fi print_info diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index d159792ed729f..6ee89303f9911 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -37,10 +37,6 @@ function initialize_common_environment { # shellcheck disable=SC2034 FILES_TO_CLEANUP_ON_EXIT=() - # Sets to where airflow sources are located - AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${MY_DIR}/../../" && pwd)} - export AIRFLOW_SOURCES - # Sets to the build cache directory - status of build and convenience scripts are stored there BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build" export BUILD_CACHE_DIR @@ -176,7 +172,7 @@ function initialize_common_environment { fi # Read airflow version from the version.py - AIRFLOW_VERSION=$(grep version "airflow/version.py" | awk '{print $3}' | sed "s/['+]//g") + AIRFLOW_VERSION=$(grep version "${AIRFLOW_SOURCES}/airflow/version.py" | awk '{print $3}' | sed "s/['+]//g") export AIRFLOW_VERSION # default version of python used to tag the "master" and "latest" images in DockerHub diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index f24b3aa2a564c..2cf43c35fb670 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -125,6 +125,16 @@ function delete_cluster() { } function perform_kind_cluster_operation() { + ALLOWED_KIND_OPERATIONS="[ start restart stop deploy test shell ]" + + set +u + if [[ -z "${1}" ]]; then + echo >&2 + echo >&2 "Operation must be provided as first parameter. One of: ${ALLOWED_KIND_OPERATIONS}" + echo >&2 + exit 1 + fi + set -u OPERATION="${1}" ALL_CLUSTERS=$(kind get clusters || true) @@ -181,17 +191,16 @@ function perform_kind_cluster_operation() { echo echo "Testing with KinD" echo - "${AIRFLOW_SOURCES}/scripts/ci/ci_run_kubernetes_tests.sh" + "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh" elif [[ ${OPERATION} == "shell" ]]; then echo echo "Entering an interactive shell for kubernetes testing" echo - "${AIRFLOW_SOURCES}/scripts/ci/ci_run_kubernetes_tests.sh" "-i" + "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh" "-i" else - echo - echo "Wrong cluster operation: ${OPERATION}. Should be one of:" - echo "${FORMATTED_KIND_OPERATIONS}" - echo + echo >&2 + echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of: ${ALLOWED_KIND_OPERATIONS}" + echo >&2 exit 1 fi else @@ -208,15 +217,14 @@ function perform_kind_cluster_operation() { create_cluster elif [[ ${OPERATION} == "stop" || ${OEPRATON} == "deploy" || \ ${OPERATION} == "test" || ${OPERATION} == "shell" ]]; then - echo - echo "Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation" - echo + echo >&2 + echo >&2 "Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation" + echo >&2 exit 1 else - echo - echo "Wrong cluster operation: ${OPERATION}. Should be one of:" - echo "${FORMATTED_KIND_OPERATIONS}" - echo + echo >&2 + echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of ${ALLOWED_KIND_OPERATIONS}" + echo >&2 exit 1 fi fi @@ -262,9 +270,9 @@ function forward_port_to_kind_webserver() { set +e while ! curl http://localhost:8080/health -s | grep -q healthy; do if [[ ${num_tries} == 6 ]]; then - echo - echo "ERROR! Could not setup a forward port to Airflow's webserver after ${num_tries}! Exiting." - echo + echo >&2 + echo >&2 "ERROR! Could not setup a forward port to Airflow's webserver after ${num_tries}! Exiting." + echo >&2 exit 1 fi echo diff --git a/scripts/ci/_script_init.sh b/scripts/ci/libraries/_script_init.sh similarity index 73% rename from scripts/ci/_script_init.sh rename to scripts/ci/libraries/_script_init.sh index 2acbf105c62dc..804037eac0083 100755 --- a/scripts/ci/_script_init.sh +++ b/scripts/ci/libraries/_script_init.sh @@ -18,17 +18,18 @@ set -euo pipefail -# This should only be sourced from CI directory! +_CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") -SCRIPTS_CI_DIR="$( dirname "${BASH_SOURCE[0]}" )" +SCRIPTS_CI_DIR="$(cd "${_CURRENT_DIR}"/.. && pwd)" export SCRIPTS_CI_DIR -# shellcheck source=scripts/ci/_all_libs.sh -. "${SCRIPTS_CI_DIR}"/_all_libs.sh +# Sets to where airflow sources are located +AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${SCRIPTS_CI_DIR}/../../" && pwd)} +export AIRFLOW_SOURCES -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export MY_DIR +# shellcheck source=scripts/ci/libraries/_all_libs.sh +. "${SCRIPTS_CI_DIR}"/libraries/_all_libs.sh initialize_common_environment diff --git a/scripts/ci/minikdc.properties b/scripts/ci/minikdc.properties deleted file mode 100644 index c70ff8448bf6d..0000000000000 --- a/scripts/ci/minikdc.properties +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -org.name=TEST -org.domain=LOCAL -kdc.bind.address=localhost -kdc.port=8888 -instance=DefaultKrbServer -max.ticket.lifetime=86400000 -max.renewable.lifetime=604800000 -transport=TCP -debug=true diff --git a/scripts/ci/pre_commit_bat_tests.sh b/scripts/ci/pre_commit/pre_commit_bat_tests.sh similarity index 87% rename from scripts/ci/pre_commit_bat_tests.sh rename to scripts/ci/pre_commit/pre_commit_bat_tests.sh index 4ae72c336bf36..b820c7e60fd9d 100755 --- a/scripts/ci/pre_commit_bat_tests.sh +++ b/scripts/ci/pre_commit/pre_commit_bat_tests.sh @@ -24,5 +24,5 @@ else PARAMS=("${@}") fi -# shellcheck source=scripts/ci/ci_bat_tests.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_bat_tests.sh" "${PARAMS[@]}" +# shellcheck source=scripts/ci/static_checks/ci_bat_tests.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_bat_tests.sh" "${PARAMS[@]}" diff --git a/scripts/ci/pre_commit_breeze_cmd_line.sh b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh similarity index 89% rename from scripts/ci/pre_commit_breeze_cmd_line.sh rename to scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh index eeac2e6ecc7e2..4d62cc1ed068e 100755 --- a/scripts/ci/pre_commit_breeze_cmd_line.sh +++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh @@ -18,12 +18,14 @@ set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd); +cd "${AIRFLOW_SOURCES}" || exit 1 + + TMP_FILE=$(mktemp) TMP_OUTPUT=$(mktemp) -cd "${MY_DIR}/../../" || exit; - echo " .. code-block:: text " >"${TMP_FILE}" @@ -46,7 +48,7 @@ if (( MAX_LEN > MAX_SCREEN_WIDTH + 2 )); then exit 1 fi -BREEZE_RST_FILE="${MY_DIR}/../../BREEZE.rst" +BREEZE_RST_FILE="${AIRFLOW_SOURCES}/BREEZE.rst" LEAD='^ \.\. START BREEZE HELP MARKER$' TAIL='^ \.\. END BREEZE HELP MARKER$' diff --git a/scripts/ci/pre_commit_build_providers_dependencies.sh b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh similarity index 85% rename from scripts/ci/pre_commit_build_providers_dependencies.sh rename to scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh index 6eee5f45519f7..a212266a82852 100755 --- a/scripts/ci/pre_commit_build_providers_dependencies.sh +++ b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh @@ -18,12 +18,9 @@ set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -cd "${MY_DIR}/../../" || exit; - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd); +cd "${AIRFLOW_SOURCES}" || exit 1 PYTHONPATH="$(pwd)" export PYTHONPATH diff --git a/scripts/ci/pre_commit_check_integrations.sh b/scripts/ci/pre_commit/pre_commit_check_integrations.sh similarity index 86% rename from scripts/ci/pre_commit_check_integrations.sh rename to scripts/ci/pre_commit/pre_commit_check_integrations.sh index 69ace38a2a800..c21a3294833fc 100755 --- a/scripts/ci/pre_commit_check_integrations.sh +++ b/scripts/ci/pre_commit/pre_commit_check_integrations.sh @@ -17,12 +17,9 @@ # under the License. set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -cd "${MY_DIR}/../../" || exit; - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd); +cd "${AIRFLOW_SOURCES}" || exit 1 . breeze-complete diff --git a/scripts/ci/pre_commit_check_license.sh b/scripts/ci/pre_commit/pre_commit_check_license.sh similarity index 92% rename from scripts/ci/pre_commit_check_license.sh rename to scripts/ci/pre_commit/pre_commit_check_license.sh index 4ab6964c57bbd..109df313ef4a5 100755 --- a/scripts/ci/pre_commit_check_license.sh +++ b/scripts/ci/pre_commit/pre_commit_check_license.sh @@ -21,5 +21,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" # Hide lines between ****/**** (detailed list of files) -"$( dirname "${BASH_SOURCE[0]}" )/ci_check_license.sh" 2>&1 | \ +"$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_check_license.sh" 2>&1 | \ (sed "/Files with Apache License headers will be marked AL.*$/,/^\**$/d" || true) diff --git a/scripts/ci/pre_commit_check_order_setup.py b/scripts/ci/pre_commit/pre_commit_check_order_setup.py similarity index 99% rename from scripts/ci/pre_commit_check_order_setup.py rename to scripts/ci/pre_commit/pre_commit_check_order_setup.py index ffbad393d1e9f..57f56624edfbc 100755 --- a/scripts/ci/pre_commit_check_order_setup.py +++ b/scripts/ci/pre_commit/pre_commit_check_order_setup.py @@ -43,7 +43,7 @@ def _check_list_sorted(the_list: List[str], message: str) -> None: def setup() -> str: - setup_py_file_path = abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, 'setup.py')) + setup_py_file_path = abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir, 'setup.py')) with open(setup_py_file_path) as setup_file: setup_context = setup_file.read() return setup_context diff --git a/scripts/ci/pre_commit_ci_build.sh b/scripts/ci/pre_commit/pre_commit_ci_build.sh similarity index 88% rename from scripts/ci/pre_commit_ci_build.sh rename to scripts/ci/pre_commit/pre_commit_ci_build.sh index 6f25c39a4fc56..861785bcc5157 100755 --- a/scripts/ci/pre_commit_ci_build.sh +++ b/scripts/ci/pre_commit/pre_commit_ci_build.sh @@ -18,8 +18,8 @@ export PYTHON_MAJOR_MINOR_VERSION="${1}" export REMEMBER_LAST_ANSWER="${2}" -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" forget_last_answer diff --git a/scripts/ci/pre_commit_flake8.sh b/scripts/ci/pre_commit/pre_commit_flake8.sh similarity index 87% rename from scripts/ci/pre_commit_flake8.sh rename to scripts/ci/pre_commit/pre_commit_flake8.sh index dd2a3da0b5283..95f1b987bceeb 100755 --- a/scripts/ci/pre_commit_flake8.sh +++ b/scripts/ci/pre_commit/pre_commit_flake8.sh @@ -18,5 +18,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_flake8.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_flake8.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_flake8.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_flake8.sh" "${@}" diff --git a/scripts/ci/pre_commit_generate_requirements.sh b/scripts/ci/pre_commit/pre_commit_generate_requirements.sh similarity index 85% rename from scripts/ci/pre_commit_generate_requirements.sh rename to scripts/ci/pre_commit/pre_commit_generate_requirements.sh index f9e952e8471b1..d0c2deb34cf02 100755 --- a/scripts/ci/pre_commit_generate_requirements.sh +++ b/scripts/ci/pre_commit/pre_commit_generate_requirements.sh @@ -20,5 +20,5 @@ export REMEMBER_LAST_ANSWER="true" export PYTHON_MAJOR_MINOR_VERSION="${1}" -# shellcheck source=scripts/ci/ci_generate_requirements.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_generate_requirements.sh" +# shellcheck source=scripts/ci/requirements/ci_generate_requirements.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../generate_requirements/ci_generate_requirements.sh" diff --git a/tests/insert_extras.py b/scripts/ci/pre_commit/pre_commit_insert_extras.py old mode 100644 new mode 100755 similarity index 94% rename from tests/insert_extras.py rename to scripts/ci/pre_commit/pre_commit_insert_extras.py index 8ed6a64662210..b95fe992eae93 --- a/tests/insert_extras.py +++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -20,8 +21,9 @@ from textwrap import wrap from typing import List -AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir) +AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir) +sys.path.insert(0, AIRFLOW_SOURCES_DIR) # flake8: noqa: F401 # pylint: disable=wrong-import-position from setup import EXTRAS_REQUIREMENTS # isort:skip diff --git a/scripts/ci/pre_commit_lint_dockerfile.sh b/scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh similarity index 84% rename from scripts/ci/pre_commit_lint_dockerfile.sh rename to scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh index bce941b120b39..857c27e9f3544 100755 --- a/scripts/ci/pre_commit_lint_dockerfile.sh +++ b/scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh @@ -17,5 +17,5 @@ # under the License. export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_lint_dockerfile.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_lint_dockerfile.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_lint_dockerfile.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_lint_dockerfile.sh" "${@}" diff --git a/scripts/ci/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh similarity index 86% rename from scripts/ci/pre_commit_local_yml_mounts.sh rename to scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh index b638a08b65362..43ece7e949a7f 100755 --- a/scripts/ci/pre_commit_local_yml_mounts.sh +++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh @@ -18,17 +18,15 @@ set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" TMP_OUTPUT=$(mktemp) # Remove temp file if it's hanging around trap 'rm -rf -- "${TMP_OUTPUT}" 2>/dev/null' EXIT -LOCAL_YML_FILE="${MY_DIR}/docker-compose/local.yml" +LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml" LEAD=' # START automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh' TAIL=' # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh' diff --git a/scripts/ci/pre_commit_mypy.sh b/scripts/ci/pre_commit/pre_commit_mypy.sh similarity index 87% rename from scripts/ci/pre_commit_mypy.sh rename to scripts/ci/pre_commit/pre_commit_mypy.sh index 6f0025f3485c6..f5ab2541a9b9f 100755 --- a/scripts/ci/pre_commit_mypy.sh +++ b/scripts/ci/pre_commit/pre_commit_mypy.sh @@ -18,5 +18,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_mypy.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_mypy.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_mypy.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_mypy.sh" "${@}" diff --git a/scripts/ci/pre_commit_pylint_main.sh b/scripts/ci/pre_commit/pre_commit_pylint_main.sh similarity index 86% rename from scripts/ci/pre_commit_pylint_main.sh rename to scripts/ci/pre_commit/pre_commit_pylint_main.sh index 9a1281f29082f..39c9d785eda87 100755 --- a/scripts/ci/pre_commit_pylint_main.sh +++ b/scripts/ci/pre_commit/pre_commit_pylint_main.sh @@ -18,5 +18,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_pylint_main.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_pylint_main.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_pylint_main.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_pylint_main.sh" "${@}" diff --git a/scripts/ci/pre_commit_pylint_tests.sh b/scripts/ci/pre_commit/pre_commit_pylint_tests.sh similarity index 86% rename from scripts/ci/pre_commit_pylint_tests.sh rename to scripts/ci/pre_commit/pre_commit_pylint_tests.sh index 14a0aa1396709..3fc196500867f 100755 --- a/scripts/ci/pre_commit_pylint_tests.sh +++ b/scripts/ci/pre_commit/pre_commit_pylint_tests.sh @@ -18,5 +18,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_pylint_tests.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_pylint_tests.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_pylint_tests.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_pylint_tests.sh" "${@}" diff --git a/scripts/ci/pre_commit_setup_cfg_file.sh b/scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh similarity index 87% rename from scripts/ci/pre_commit_setup_cfg_file.sh rename to scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh index f7a92acfc332c..90a0aa5689ed7 100755 --- a/scripts/ci/pre_commit_setup_cfg_file.sh +++ b/scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh @@ -18,16 +18,16 @@ set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd); +cd "${AIRFLOW_SOURCES}" || exit 1 TMP_FILE=$(mktemp) TMP_OUTPUT=$(mktemp) -cd "${MY_DIR}/../../" || exit; - find "licenses" -type f -exec echo " " {} \; | LC_ALL=C sort >>"${TMP_FILE}" -SETUP_CFG_FILE="${MY_DIR}/../../setup.cfg" +SETUP_CFG_FILE="${AIRFLOW_SOURCES}/setup.cfg" LEAD='^# Start of licenses generated automatically$' TAIL='^# End of licences generated automatically$' diff --git a/scripts/ci/pre_commit_yaml_to_cfg.py b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py similarity index 95% rename from scripts/ci/pre_commit_yaml_to_cfg.py rename to scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py index 7c326d790f7a0..9b31cef30f1c6 100755 --- a/scripts/ci/pre_commit_yaml_to_cfg.py +++ b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -141,7 +141,9 @@ def _write_option(configfile, idx, option): if __name__ == '__main__': airflow_config_dir = os.path.join( - os.path.dirname(__file__), "../../airflow/config_templates") + os.path.dirname(__file__), + os.pardir, os.pardir, os.pardir, + "airflow", "config_templates") airflow_default_config_path = os.path.join(airflow_config_dir, "default_airflow.cfg") airflow_config_yaml_file_path = os.path.join(airflow_config_dir, "config.yml") @@ -150,7 +152,10 @@ def _write_option(configfile, idx, option): default_cfg_file_path=airflow_default_config_path ) - providers_dir = os.path.join(os.path.dirname(__file__), "../../airflow/providers") + providers_dir = os.path.join( + os.path.dirname(__file__), + os.pardir, os.pardir, os.pardir, + "airflow", "providers") for root, dir_names, file_names in os.walk(providers_dir): for file_name in file_names: if root.endswith("config_templates") and file_name == 'config.yml' and \ diff --git a/scripts/ci/pre_commit_update_extras.sh b/scripts/ci/pre_commit_update_extras.sh deleted file mode 100755 index be43a91f4ab07..0000000000000 --- a/scripts/ci/pre_commit_update_extras.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -cd "${MY_DIR}/../../" || exit; - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -PYTHONPATH="$(pwd)" -export PYTHONPATH - -python3 tests/insert_extras.py diff --git a/scripts/ci/ci_generate_requirements.sh b/scripts/ci/requirements/ci_generate_requirements.sh similarity index 88% rename from scripts/ci/ci_generate_requirements.sh rename to scripts/ci/requirements/ci_generate_requirements.sh index f55799ff84b2b..5cc4a0e09ab47 100755 --- a/scripts/ci/ci_generate_requirements.sh +++ b/scripts/ci/requirements/ci_generate_requirements.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" get_environment_for_builds_on_ci diff --git a/scripts/ci/ci_bat_tests.sh b/scripts/ci/static_checks/ci_bat_tests.sh similarity index 90% rename from scripts/ci/ci_bat_tests.sh rename to scripts/ci/static_checks/ci_bat_tests.sh index 25011ce2c6c4c..e50c55941ed00 100755 --- a/scripts/ci/ci_bat_tests.sh +++ b/scripts/ci/static_checks/ci_bat_tests.sh @@ -16,6 +16,9 @@ # specific language governing permissions and limitations # under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + function run_bats_tests() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then diff --git a/scripts/ci/ci_check_license.sh b/scripts/ci/static_checks/ci_check_license.sh similarity index 94% rename from scripts/ci/ci_check_license.sh rename to scripts/ci/static_checks/ci_check_license.sh index da5aebd94f0a5..3d887c4219f48 100755 --- a/scripts/ci/ci_check_license.sh +++ b/scripts/ci/static_checks/ci_check_license.sh @@ -18,8 +18,8 @@ export MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true" export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_check_license() { echo diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/static_checks/ci_flake8.sh similarity index 94% rename from scripts/ci/ci_flake8.sh rename to scripts/ci/static_checks/ci_flake8.sh index 33504c066c56f..4ebd060328899 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/static_checks/ci_flake8.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_flake8() { FILES=("$@") diff --git a/scripts/ci/ci_lint_dockerfile.sh b/scripts/ci/static_checks/ci_lint_dockerfile.sh similarity index 92% rename from scripts/ci/ci_lint_dockerfile.sh rename to scripts/ci/static_checks/ci_lint_dockerfile.sh index 29a5f685e1c61..2e48043f5c78e 100755 --- a/scripts/ci/ci_lint_dockerfile.sh +++ b/scripts/ci/static_checks/ci_lint_dockerfile.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_docker_lint() { FILES=("$@") diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/static_checks/ci_mypy.sh similarity index 93% rename from scripts/ci/ci_mypy.sh rename to scripts/ci/static_checks/ci_mypy.sh index 5647f4cbe9afe..8d6aeba19cd32 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/static_checks/ci_mypy.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_mypy() { FILES=("$@") diff --git a/scripts/ci/ci_pylint_main.sh b/scripts/ci/static_checks/ci_pylint_main.sh similarity index 95% rename from scripts/ci/ci_pylint_main.sh rename to scripts/ci/static_checks/ci_pylint_main.sh index 4c971ff9efb76..09083b6e9e92a 100755 --- a/scripts/ci/ci_pylint_main.sh +++ b/scripts/ci/static_checks/ci_pylint_main.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_pylint_main() { FILES=("$@") diff --git a/scripts/ci/ci_pylint_tests.sh b/scripts/ci/static_checks/ci_pylint_tests.sh similarity index 95% rename from scripts/ci/ci_pylint_tests.sh rename to scripts/ci/static_checks/ci_pylint_tests.sh index ed073fef88d08..191af0459030b 100755 --- a/scripts/ci/ci_pylint_tests.sh +++ b/scripts/ci/static_checks/ci_pylint_tests.sh @@ -18,8 +18,8 @@ export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_pylint_tests() { FILES=("$@") diff --git a/scripts/ci/ci_refresh_pylint_todo.sh b/scripts/ci/static_checks/ci_refresh_pylint_todo.sh similarity index 92% rename from scripts/ci/ci_refresh_pylint_todo.sh rename to scripts/ci/static_checks/ci_refresh_pylint_todo.sh index f0ca1052f50f8..202f6b345fd9c 100755 --- a/scripts/ci/ci_refresh_pylint_todo.sh +++ b/scripts/ci/static_checks/ci_refresh_pylint_todo.sh @@ -17,8 +17,8 @@ # under the License. export FORCE_ANSWER_TO_QUESTIONS=quit -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function refresh_pylint_todo() { docker run "${EXTRA_DOCKER_FLAGS[@]}" \ diff --git a/scripts/ci/ci_run_static_checks.sh b/scripts/ci/static_checks/ci_run_static_checks.sh similarity index 91% rename from scripts/ci/ci_run_static_checks.sh rename to scripts/ci/static_checks/ci_run_static_checks.sh index 69562e25e4f3a..4bfe98dc3be69 100755 --- a/scripts/ci/ci_run_static_checks.sh +++ b/scripts/ci/static_checks/ci_run_static_checks.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then echo diff --git a/scripts/ci/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh similarity index 82% rename from scripts/ci/ci_run_airflow_testing.sh rename to scripts/ci/testing/ci_run_airflow_testing.sh index f125e7b3e9b78..884b164f55907 100755 --- a/scripts/ci/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -17,28 +17,29 @@ # under the License. export VERBOSE=${VERBOSE:="false"} +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then + echo + echo "Skipping running tests !!!!!" + echo + exit +fi + + function run_airflow_testing_in_docker() { set +u # shellcheck disable=SC2016 docker-compose --log-level INFO \ - -f "${MY_DIR}/docker-compose/base.yml" \ - -f "${MY_DIR}/docker-compose/backend-${BACKEND}.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml" \ "${INTEGRATIONS[@]}" \ "${DOCKER_COMPOSE_LOCAL[@]}" \ run airflow "${@}" set -u } -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then - echo - echo "Skipping running tests !!!!!" - echo - exit -fi - get_environment_for_builds_on_ci prepare_ci_build @@ -64,17 +65,17 @@ export FORWARD_CREDENTIALS=${FORWARD_CREDENTIALS:="false"} export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then - DOCKER_COMPOSE_LOCAL=("-f" "${MY_DIR}/docker-compose/local.yml") + DOCKER_COMPOSE_LOCAL=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml") else DOCKER_COMPOSE_LOCAL=() fi if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${MY_DIR}/docker-compose/forward-credentials.yml") + DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml") fi if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${MY_DIR}/docker-compose/remove-sources.yml") + DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml") fi echo @@ -97,7 +98,7 @@ fi for _INT in ${ENABLED_INTEGRATIONS} do INTEGRATIONS+=("-f") - INTEGRATIONS+=("${MY_DIR}/docker-compose/integration-${_INT}.yml") + INTEGRATIONS+=("${SCRIPTS_CI_DIR}/docker-compose/integration-${_INT}.yml") done RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""} diff --git a/scripts/ci/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh similarity index 92% rename from scripts/ci/ci_count_changed_files.sh rename to scripts/ci/tools/ci_count_changed_files.sh index d1ccd4bc3cb9a..e3d2c9eb0c76a 100755 --- a/scripts/ci/ci_count_changed_files.sh +++ b/scripts/ci/tools/ci_count_changed_files.sh @@ -22,8 +22,8 @@ # $1: Revision to compare # $2: Pattern to match -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" get_environment_for_builds_on_ci diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh similarity index 82% rename from scripts/ci/ci_fix_ownership.sh rename to scripts/ci/tools/ci_fix_ownership.sh index 7e85152a7c43f..8cde42dc55c52 100755 --- a/scripts/ci/ci_fix_ownership.sh +++ b/scripts/ci/tools/ci_fix_ownership.sh @@ -21,8 +21,8 @@ # export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" export AIRFLOW_CI_IMAGE=\ ${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci @@ -39,7 +39,7 @@ export HOST_OS export BACKEND="sqlite" docker-compose \ - -f "${MY_DIR}/docker-compose/base.yml" \ - -f "${MY_DIR}/docker-compose/local.yml" \ - -f "${MY_DIR}/docker-compose/forward-credentials.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml" \ run airflow /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh diff --git a/scripts/ci/ci_free_space_on_ci.sh b/scripts/ci/tools/ci_free_space_on_ci.sh similarity index 87% rename from scripts/ci/ci_free_space_on_ci.sh rename to scripts/ci/tools/ci_free_space_on_ci.sh index 5d8f8510a3b08..a50add7dbff85 100755 --- a/scripts/ci/ci_free_space_on_ci.sh +++ b/scripts/ci/tools/ci_free_space_on_ci.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" sudo swapoff -a sudo rm -f /swapfile diff --git a/tests/bats/bats_utils.bash b/tests/bats/bats_utils.bash index 7a29da9432271..c06b182763422 100644 --- a/tests/bats/bats_utils.bash +++ b/tests/bats/bats_utils.bash @@ -18,5 +18,5 @@ AIRFLOW_SOURCES=$(pwd) export AIRFLOW_SOURCES export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci -# shellcheck source=scripts/ci/_all_libs.sh -source "${SCRIPTS_CI_DIR}/_all_libs.sh" +# shellcheck source=scripts/ci/libraries/_all_libs.sh +source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"