airbyte-sync-parallel.yaml

id: airbyte-sync-parallel
namespace: company.team

tasks:
  - id: data_ingestion
    type: io.kestra.plugin.core.flow.Parallel
    tasks:
      - id: salesforce
        type: io.kestra.plugin.airbyte.connections.Sync
        connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12ab

      - id: google_analytics
        type: io.kestra.plugin.airbyte.connections.Sync
        connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12cd

      - id: facebook_ads
        type: io.kestra.plugin.airbyte.connections.Sync
        connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12ef

pluginDefaults:
  - type: io.kestra.plugin.airbyte.connections.Sync
    values:
      url: http://host.docker.internal:8000/
      username: "{{ secret('AIRBYTE_USERNAME') }}"
      password: "{{ secret('AIRBYTE_PASSWORD') }}"

extend:
  title: Trigger multiple Airbyte syncs in parallel
  description: >
    This flow syncs data from multiple sources in parallel using Airbyte.

    The Airbyte server credentials, referenced in the `pluginDefaults`, are
    stored as secrets. The `pluginDefaults` flow property helps remove
    boilerplate code, allowing to define common values such as `url`, `username`
    and `password` in one place.
      
    This flow executes three Airbyte connections in parallel. To control how
    many tasks run in parallel, use the `concurrent` property of the
    `io.kestra.core.tasks.flows.Parallel` task.
  tags:
    - Ingest
    - Parallel
  ee: false
  demo: false
  meta_description: This flow syncs data from multiple sources in parallel using Airbyte.