dlt-zendesk-to-weaviate.yaml

id: dlt-zendesk-to-weaviate
namespace: company.team

tasks:
  - id: dlt_pipeline
    type: io.kestra.plugin.scripts.python.Script
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    containerImage: python:3.11
    beforeCommands:
      - pip install dlt[weaviate]
      - dlt --non-interactive init zendesk weaviate
    warningOnStdErr: false
    env:
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__PASSWORD: "{{ secret('ZENDESK_PASSWORD') }}"
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__SUBDOMAIN: "{{ secret('ZENDESK_SUBDOMAIN') }}"
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__EMAIL: "{{ secret('ZENDESK_EMAIL') }}"
      DESTINATION__WEAVIATE__CREDENTIALS__URL: "{{ secret('WEAVIATE_URL') }}"
      DESTINATION__WEAVIATE__CREDENTIALS__API_KEY: "{{ secret('WEAVIATE_API_KEY') }}"
    script: |
      import dlt
      from zendesk import zendesk_support

      pipeline = dlt.pipeline(
          pipeline_name="zendesk_pipeline",
          destination="weaviate",
          dataset_name="zendesk",
      )

      zendesk_source = zendesk_support(load_all=False)
      tickets = zendesk_source.tickets

      load_info = pipeline.run(
          weaviate_adapter(
              tickets,
              vectorize=["subject", "description"],
          )
      )

extend:
  title: Ingest Zendesk data into Weaviate using dlt
  description: >
    This flow demonstrates how to extract data from Zendesk into Weaviate using
    dlt. The entire workflow logic is contained in a single Python script that
    uses the dlt Python library to ingest the data into Weaviate. 

    The credentials to access the Zendesk API and Weaviate are stored using
    Secret.
  tags:
    - Ingest
  ee: false
  demo: false
  meta_description: "This flow extract data from Zendesk into Weaviate using dlt.
    The entire workflow logic is contained in a single Python script that uses
    the dlt Python library to ingest the data into Weaviate. "