fivetran-dbt-cloud.yaml

id: fivetran-dbt-cloud
namespace: company.team

tasks:
  - id: fivetran_syncs
    type: io.kestra.plugin.core.flow.Parallel
    tasks:
      - id: salesforce
        type: io.kestra.plugin.fivetran.connectors.Sync
        connectorId: enterYourFivetranConnectorId

      - id: google_analytics
        type: io.kestra.plugin.fivetran.connectors.Sync
        connectorId: enterYourFivetranConnectorId

      - id: facebook
        type: io.kestra.plugin.fivetran.connectors.Sync
        connectorId: enterYourFivetranConnectorId

  - id: dbt_cloud_job
    type: io.kestra.plugin.dbt.cloud.TriggerRun
    jobId: "396284"
    accountId: "{{ secret('DBT_CLOUD_ACCOUNT_ID') }}"
    token: "{{ secret('DBT_CLOUD_API_TOKEN') }}"
    wait: true

pluginDefaults:
  - type: io.kestra.plugin.fivetran.connectors.Sync
    values:
      apiKey: "{{ secret('FIVETRAN_API_KEY') }}"
      apiSecret: "{{ secret('FIVETRAN_API_SECRET') }}"

extend:
  title: Trigger multiple Fivetran syncs in parallel, then run a dbt Cloud job
  description: >-
    This flow will sync data from multiple sources using Fivetran and then
    trigger a dbt Cloud job.

    Given that you may have tens or hundreds of Fivetran connectors that need to
    be synced before running dbt, `pluginDefaults` property is used to avoid
    boilerplate configuration.

    The `wait` flag on the dbt Cloud job task is set to `true` allowing to
    capture logs and job status and ensuring that dynamically generated tasks
    (based on dbt models and tests) will be displayed in the Gantt view. This
    will also ensure that Kestra will wait for the dbt Cloud job to finish
    before continuing any downstream tasks, and it will fail the execution if
    dbt Cloud job fails.
  tags:
    - Ingest
    - dbt
    - Parallel
  ee: false
  demo: false
  meta_description: >
    This flow will sync data from multiple sources using Fivetran and then
    trigger a dbt Cloud job.