From 79d246bd08f08800ebf73b1c6e10524714597df6 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle Date: Wed, 4 Dec 2024 16:15:54 +0100 Subject: [PATCH] refactor: migrate to property v2 migration of abstract class --- .../plugin/git/AbstractCloningTask.java | 5 +- .../io/kestra/plugin/git/AbstractGitTask.java | 35 +--- .../kestra/plugin/git/AbstractPushTask.java | 48 ++--- .../kestra/plugin/git/AbstractSyncTask.java | 26 +-- src/main/java/io/kestra/plugin/git/Clone.java | 13 +- src/main/java/io/kestra/plugin/git/Push.java | 26 +-- .../java/io/kestra/plugin/git/PushFlows.java | 29 +-- .../kestra/plugin/git/PushNamespaceFiles.java | 27 +-- src/main/java/io/kestra/plugin/git/Sync.java | 13 +- .../java/io/kestra/plugin/git/SyncFlows.java | 27 ++- .../kestra/plugin/git/SyncNamespaceFiles.java | 15 +- .../plugin/git/services/GitService.java | 13 +- .../java/io/kestra/plugin/git/CloneTest.java | 9 +- .../io/kestra/plugin/git/PushFlowsTest.java | 193 ++++++++---------- .../plugin/git/PushNamespaceFilesTest.java | 165 +++++++-------- .../java/io/kestra/plugin/git/PushTest.java | 159 ++++++++------- .../io/kestra/plugin/git/SyncFlowsTest.java | 93 ++++----- .../plugin/git/SyncNamespaceFilesTest.java | 57 ++---- .../java/io/kestra/plugin/git/SyncTest.java | 43 ++-- 19 files changed, 463 insertions(+), 533 deletions(-) diff --git a/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java b/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java index e8b11db..66c347c 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractCloningTask.java @@ -1,6 +1,6 @@ package io.kestra.plugin.git; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.NoArgsConstructor; @@ -13,6 +13,5 @@ public abstract class AbstractCloningTask extends AbstractGitTask { @Schema( title = "Whether to clone submodules." ) - @PluginProperty - protected Boolean cloneSubmodules; + protected Property cloneSubmodules; } diff --git a/src/main/java/io/kestra/plugin/git/AbstractGitTask.java b/src/main/java/io/kestra/plugin/git/AbstractGitTask.java index 4f7916a..948d47b 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractGitTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractGitTask.java @@ -1,6 +1,6 @@ package io.kestra.plugin.git; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.Task; import io.kestra.core.runners.RunContext; import io.kestra.plugin.git.services.SshTransportConfigCallback; @@ -18,68 +18,55 @@ @NoArgsConstructor @Getter public abstract class AbstractGitTask extends Task { - private static final Pattern PEBBLE_TEMPLATE_PATTERN = Pattern.compile("^\\s*\\{\\{"); @Schema( title = "The URI to clone from." ) - @PluginProperty(dynamic = true) - protected String url; + protected Property url; @Schema( title = "The username or organization." ) - @PluginProperty(dynamic = true) - protected String username; + protected Property username; @Schema( title = "The password or Personal Access Token (PAT). When you authenticate the task with a PAT, any flows or files pushed to Git from Kestra will be pushed from the user associated with that PAT. This way, you don't need to configure the commit author (the `authorName` and `authorEmail` properties)." ) - @PluginProperty(dynamic = true) - protected String password; + protected Property password; @Schema( title = "PEM-format private key content that is paired with a public key registered on Git.", description = "To generate an ECDSA PEM format key from OpenSSH, use the following command: `ssh-keygen -t ecdsa -b 256 -m PEM`. " + "You can then set this property with your private key content and put your public key on Git." ) - @PluginProperty(dynamic = true) - protected String privateKey; + protected Property privateKey; @Schema( title = "The passphrase for the `privateKey`." ) - @PluginProperty(dynamic = true) - protected String passphrase; + protected Property passphrase; @Schema( title = "The initial Git branch." ) - @PluginProperty(dynamic = true) - public abstract String getBranch(); + public abstract Property getBranch(); public > T authentified(T command, RunContext runContext) throws Exception { if (this.username != null && this.password != null) { command.setCredentialsProvider(new UsernamePasswordCredentialsProvider( - runContext.render(this.username), - runContext.render(this.password) + runContext.render(this.username).as(String.class).orElseThrow(), + runContext.render(this.password).as(String.class).orElseThrow() )); } if (this.privateKey != null) { command.setTransportConfigCallback(new SshTransportConfigCallback( - runContext.render(this.privateKey).getBytes(StandardCharsets.UTF_8), - runContext.render(this.passphrase) + runContext.render(this.privateKey).as(String.class).orElseThrow().getBytes(StandardCharsets.UTF_8), + runContext.render(this.passphrase).as(String.class).orElse(null) )); } return command; } - - protected void detectPasswordLeaks() { - if (this.password != null && !PEBBLE_TEMPLATE_PATTERN.matcher(this.password).find()) { - throw new IllegalArgumentException("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables)."); - } - } } diff --git a/src/main/java/io/kestra/plugin/git/AbstractPushTask.java b/src/main/java/io/kestra/plugin/git/AbstractPushTask.java index 170d5e6..fc8e5d7 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractPushTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractPushTask.java @@ -1,7 +1,7 @@ package io.kestra.plugin.git; import io.kestra.core.exceptions.IllegalVariableEvaluationException; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; @@ -46,41 +46,39 @@ @NoArgsConstructor @Getter public abstract class AbstractPushTask extends AbstractCloningTask implements RunnableTask { - @PluginProperty(dynamic = true) - protected String commitMessage; + protected Property commitMessage; @Schema( title = "If `true`, the task will only output modifications without pushing any file to Git yet. If `false` (default), all listed files will be pushed to Git immediately." ) - @PluginProperty @Builder.Default - private boolean dryRun = false; + private Property dryRun = Property.of(false); @Schema( title = "The commit author email.", description = "If null, no author will be set on this commit." ) - @PluginProperty(dynamic = true) - private String authorEmail; + private Property authorEmail; @Schema( title = "The commit author name.", description = "If null, the username will be used instead.", defaultValue = "`username`" ) - @PluginProperty(dynamic = true) - private String authorName; + private Property authorName; - public abstract String getCommitMessage(); + public String renderCommitMessage(RunContext runContext) throws IllegalVariableEvaluationException { + return runContext.render(this.getCommitMessage()).as(String.class).orElseThrow(); + } - public abstract String getGitDirectory(); + public abstract Property getGitDirectory(); public abstract Object globs(); - public abstract String fetchedNamespace(); + public abstract Property fetchedNamespace(); private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException { - Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()))); + Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElseThrow())); flowDirectory.toFile().mkdirs(); return flowDirectory; } @@ -130,14 +128,14 @@ protected void writeResourceFile(Path path, InputStream inputStream) throws IOEx Files.copy(inputStream, path, REPLACE_EXISTING); } - private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException { + private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException, IllegalVariableEvaluationException { File diffFile = runContext.workingDir().createTempFile(".ion").toFile(); try (DiffFormatter diffFormatter = new DiffFormatter(null); BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) { diffFormatter.setRepository(git.getRepository()); DiffCommand diff = git.diff(); - if (this.dryRun) { + if (runContext.render(this.dryRun).as(Boolean.class).orElse(false)) { diff = diff.setCached(true); } else { diff = diff.setOldTree(treeIterator(git, "HEAD~1")) @@ -204,21 +202,21 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw String commitId = null; ObjectId commit; try { - String httpUrl = gitService.getHttpUrl(runContext.render(this.url)); - if (this.isDryRun()) { + String httpUrl = gitService.getHttpUrl(runContext.render(this.url).as(String.class).orElse(null)); + if (runContext.render(dryRun).as(Boolean.class).orElse(false)) { logger.info( "Dry run — no changes will be pushed to {} for now until you set the `dryRun` parameter to false", httpUrl ); } else { - String renderedBranch = runContext.render(this.getBranch()); + String renderedBranch = runContext.render(this.getBranch()).as(String.class).orElse(null); logger.info( "Pushing to {} on branch {}", httpUrl, renderedBranch ); - String message = runContext.render(this.getCommitMessage()); + String message = this.renderCommitMessage(runContext); ObjectId head = git.getRepository().resolve(Constants.HEAD); commit = git.commit() .setAllowEmpty(false) @@ -247,8 +245,9 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw } private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluationException { - String name = Optional.ofNullable(this.authorName).orElse(runContext.render(this.username)); - String authorEmail = this.authorEmail; + String name = runContext.render(this.authorName).as(String.class) + .orElse(runContext.render(this.username).as(String.class).orElse(null)); + String authorEmail = runContext.render(this.authorEmail).as(String.class).orElse(null); if (authorEmail == null || name == null) { return null; } @@ -274,9 +273,10 @@ public O run(RunContext runContext) throws Exception { GitService gitService = new GitService(this); gitService.namespaceAccessGuard(runContext, this.fetchedNamespace()); - this.detectPasswordLeaks(); - Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules); + Git git = gitService.cloneBranch(runContext, + runContext.render(this.getBranch()).as(String.class).orElse(null), + runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false)); Path localGitDirectory = this.createGitDirectory(runContext); @@ -291,7 +291,7 @@ public O run(RunContext runContext) throws Exception { this.writeResourceFiles(contentByPath); AddCommand add = git.add(); - add.addFilepattern(runContext.render(this.getGitDirectory())); + add.addFilepattern(runContext.render(this.getGitDirectory()).as(String.class).orElse(null)); add.call(); Output pushOutput = this.push(git, runContext, gitService); diff --git a/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java b/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java index d280e27..51e2fb5 100644 --- a/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java +++ b/src/main/java/io/kestra/plugin/git/AbstractSyncTask.java @@ -2,6 +2,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; @@ -43,18 +44,17 @@ public abstract class AbstractSyncTask ext @Schema( title = "If `true`, the task will only output modifications without performing any modification to Kestra. If `false` (default), all listed modifications will be applied." ) - @PluginProperty @Builder.Default - private boolean dryRun = false; + private Property dryRun = Property.of(false); public abstract boolean isDelete(); - public abstract String getGitDirectory(); + public abstract Property getGitDirectory(); - public abstract String fetchedNamespace(); + public abstract Property fetchedNamespace(); private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException { - Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()))); + Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElseThrow())); syncDirectory.toFile().mkdirs(); return syncDirectory; } @@ -81,7 +81,7 @@ protected Map> gitResourcesContentByUri(Path baseDire protected boolean traverseDirectories() { return true; } - + protected boolean mustKeep(RunContext runContext, T instanceResource) { return false; } @@ -100,7 +100,7 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map< try (BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) { List syncResults = new ArrayList<>(); - String renderedGitDirectory = runContext.render(this.getGitDirectory()); + String renderedGitDirectory = runContext.render(this.getGitDirectory()).as(String.class).orElse(null); if (deletedResources != null) { deletedResources.stream() .map(throwFunction(deletedResource -> wrapper( @@ -146,17 +146,19 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map< } public O run(RunContext runContext) throws Exception { - this.detectPasswordLeaks(); GitService gitService = new GitService(this); gitService.namespaceAccessGuard(runContext, this.fetchedNamespace()); - Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules); + Git git = gitService.cloneBranch(runContext, + runContext.render(this.getBranch()).as(String.class).orElse(null), + runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false) + ); Path localGitDirectory = this.createGitDirectory(runContext); Map> gitContentByUri = this.gitResourcesContentByUri(localGitDirectory); - String renderedNamespace = runContext.render(this.fetchedNamespace()); + String renderedNamespace = runContext.render(this.fetchedNamespace()).as(String.class).orElse(null); Map beforeUpdateResourcesByUri = this.fetchResources(runContext, renderedNamespace) .stream() @@ -170,7 +172,7 @@ public O run(RunContext runContext) throws Exception { .map(throwFunction(e -> { InputStream inputStream = e.getValue().get(); T resource; - if (this.dryRun) { + if (runContext.render(dryRun).as(Boolean.class).orElseThrow()) { resource = this.simulateResourceWrite(runContext, renderedNamespace, e.getKey(), inputStream); } else { resource = this.writeResource(runContext, renderedNamespace, e.getKey(), inputStream); @@ -198,7 +200,7 @@ public O run(RunContext runContext) throws Exception { return; } - if (!this.dryRun) { + if (!runContext.render(dryRun).as(Boolean.class).orElseThrow()) { this.deleteResource(runContext, renderedNamespace, e.getValue()); } diff --git a/src/main/java/io/kestra/plugin/git/Clone.java b/src/main/java/io/kestra/plugin/git/Clone.java index 5b2233e..6435d04 100644 --- a/src/main/java/io/kestra/plugin/git/Clone.java +++ b/src/main/java/io/kestra/plugin/git/Clone.java @@ -3,6 +3,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -78,7 +79,7 @@ code = """ id: git_python namespace: company.team - + tasks: - id: file_system type: io.kestra.plugin.core.flow.WorkingDirectory @@ -105,7 +106,7 @@ public class Clone extends AbstractCloningTask implements RunnableTask branch; @Schema( title = "Creates a shallow clone with a history truncated to the specified number of commits." @@ -118,7 +119,7 @@ public class Clone extends AbstractCloningTask implements RunnableTask getUrl() { return super.getUrl(); } diff --git a/src/main/java/io/kestra/plugin/git/Push.java b/src/main/java/io/kestra/plugin/git/Push.java index b32c8c5..5638c70 100644 --- a/src/main/java/io/kestra/plugin/git/Push.java +++ b/src/main/java/io/kestra/plugin/git/Push.java @@ -6,6 +6,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.InputFilesInterface; import io.kestra.core.models.tasks.NamespaceFiles; import io.kestra.core.models.tasks.NamespaceFilesInterface; @@ -58,7 +59,7 @@ code = """ id: push_to_git namespace: company.team - + tasks: - id: commit_and_push type: io.kestra.plugin.git.Push @@ -71,7 +72,7 @@ username: git_username password: "{{ secret('GITHUB_ACCESS_TOKEN') }}" commitMessage: "add flows and scripts {{ now() }}" - + triggers: - id: schedule_push type: io.kestra.plugin.core.trigger.Schedule @@ -86,12 +87,12 @@ code = """ id: push_new_file_to_git namespace: company.team - + inputs: - id: commit_message type: STRING defaults: add a new file to Git - + tasks: - id: wdir type: io.kestra.plugin.core.flow.WorkingDirectory @@ -133,14 +134,13 @@ public class Push extends AbstractCloningTask implements RunnableTask branch; @Schema( title = "Commit message." ) - @PluginProperty(dynamic = true) @NotNull - private String commitMessage; + private Property commitMessage; private NamespaceFiles namespaceFiles; @@ -173,7 +173,7 @@ private boolean branchExists(RunContext runContext, String branch) throws Except } } - return authentified(Git.lsRemoteRepository().setRemote(runContext.render(url)), runContext) + return authentified(Git.lsRemoteRepository().setRemote(runContext.render(url).as(String.class).orElseThrow()), runContext) .callAsMap() .containsKey(R_HEADS + branch); } @@ -187,7 +187,7 @@ public Output run(RunContext runContext) throws Exception { basePath = runContext.workingDir().resolve(Path.of(runContext.render(this.directory))); } - String branch = runContext.render(this.branch); + String branch = runContext.render(this.branch).as(String.class).orElseThrow(); if (this.url != null) { boolean branchExists = branchExists(runContext, branch); @@ -204,7 +204,7 @@ public Output run(RunContext runContext) throws Exception { if (branchExists) { cloneHead.toBuilder() - .branch(branch) + .branch(Property.of(branch)) .build() .run(runContext); } else { @@ -295,7 +295,7 @@ public Output run(RunContext runContext) throws Exception { try { commitId = git.commit() .setAllowEmpty(false) - .setMessage(runContext.render(this.commitMessage)) + .setMessage(runContext.render(this.commitMessage).as(String.class).orElse(null)) .setAuthor(author(runContext)) .call() .getId(); @@ -322,7 +322,9 @@ private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluati return new PersonIdent(runContext.render(this.author.name), runContext.render(this.author.email)); } if (this.author.email != null && this.username != null) { - return new PersonIdent(runContext.render(this.username), runContext.render(this.author.email)); + return new PersonIdent(runContext.render(this.username).as(String.class).orElseThrow(), + runContext.render(this.author.email) + ); } return null; diff --git a/src/main/java/io/kestra/plugin/git/PushFlows.java b/src/main/java/io/kestra/plugin/git/PushFlows.java index 597bd22..113d655 100644 --- a/src/main/java/io/kestra/plugin/git/PushFlows.java +++ b/src/main/java/io/kestra/plugin/git/PushFlows.java @@ -6,6 +6,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.runners.DefaultRunContext; import io.kestra.core.runners.RunContext; @@ -46,7 +47,7 @@ Using this task, you can push one or more flows from a given namespace (and opti code = """ id: push_to_git namespace: system - + tasks: - id: commit_and_push type: io.kestra.plugin.git.PushFlows @@ -61,7 +62,7 @@ Using this task, you can push one or more flows from a given namespace (and opti branch: kestra # optional, uses "kestra" by default commitMessage: "add flows {{ now() }}" # optional string dryRun: true # if true, you'll see what files will be added, modified or deleted based on the state in Git without overwriting the files yet - + triggers: - id: schedule_push type: io.kestra.plugin.core.trigger.Schedule @@ -74,12 +75,12 @@ Using this task, you can push one or more flows from a given namespace (and opti code = """ id: myflow namespace: prod - + inputs: - id: push type: BOOLEAN defaults: false - + tasks: - id: if type: io.kestra.plugin.core.flow.If @@ -104,9 +105,8 @@ public class PushFlows extends AbstractPushTask { title = "The branch to which files should be committed and pushed.", description = "If the branch doesn't exist yet, it will be created." ) - @PluginProperty(dynamic = true) @Builder.Default - private String branch = "kestra"; + private Property branch = Property.of("kestra"); @Schema( title = "Directory to which flows should be pushed.", @@ -116,16 +116,15 @@ public class PushFlows extends AbstractPushTask { If the `includeChildNamespaces` property is set to true, this task will also push all flows from child namespaces into their corresponding nested directories, e.g., flows from the child namespace called prod.marketing will be added to the marketing folder within the _flows folder. Note that the targetNamespace (here prod) is specified in the flow code; therefore, kestra will not create the prod directory within _flows. You can use the PushFlows task to push flows from the sourceNamespace, and use SyncFlows to then sync PR-approved flows to the targetNamespace, including all child namespaces.""" ) - @PluginProperty(dynamic = true) @Builder.Default - private String gitDirectory = "_flows"; + private Property gitDirectory = Property.of("_flows"); @Schema( title = "The source namespace from which flows should be synced to the `gitDirectory`." ) @PluginProperty(dynamic = true) @Builder.Default - private String sourceNamespace = "{{ flow.namespace }}"; + private Property sourceNamespace = new Property<>("{{ flow.namespace }}"); @Schema( title = "The target namespace, intended as the production namespace.", @@ -151,7 +150,7 @@ By default, all flows from the specified sourceNamespace will be pushed (and opt title = "Whether you want to push flows from child namespaces as well.", description = """ By default, it’s `false`, so the task will push only flows from the explicitly declared namespace without pushing flows from child namespaces. If set to `true`, flows from child namespaces will be pushed to child directories in Git. See the example below for a practical explanation: - + | Source namespace in the flow code | Git directory path | Synced to target namespace | | --------------------------------- | ------------------------------ | ----------------------------- | | namespace: dev | _flows/flow1.yml | namespace: prod | @@ -170,8 +169,9 @@ By default, all flows from the specified sourceNamespace will be pushed (and opt defaultValue = "Add flows from sourceNamespace" ) @Override - public String getCommitMessage() { - return Optional.ofNullable(this.commitMessage).orElse("Add flows from " + this.sourceNamespace + " namespace"); + public String renderCommitMessage(RunContext runContext) throws IllegalVariableEvaluationException { + return runContext.render(this.commitMessage).as(String.class) + .orElse("Add flows from " + runContext.render(this.sourceNamespace).as(String.class).orElse(runContext.render("{{flow.namespace}}")) + " namespace"); } @Override @@ -180,7 +180,7 @@ public Object globs() { } @Override - public String fetchedNamespace() { + public Property fetchedNamespace() { return this.sourceNamespace; } @@ -190,7 +190,8 @@ protected Map> instanceResourcesContentByPath(RunCon Map flowProps = Optional.ofNullable((Map) runContext.getVariables().get("flow")).orElse(Collections.emptyMap()); String tenantId = flowProps.get("tenantId"); List flowsToPush; - String renderedSourceNamespace = runContext.render(this.sourceNamespace); + String renderedSourceNamespace = runContext.render(this.sourceNamespace).as(String.class) + .orElse(runContext.render("{{flow.namespace}}")); if (Boolean.TRUE.equals(this.includeChildNamespaces)) { flowsToPush = flowRepository.findWithSource(null, tenantId, null, renderedSourceNamespace, null); } else { diff --git a/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java b/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java index 45ffa2e..b78e181 100644 --- a/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java +++ b/src/main/java/io/kestra/plugin/git/PushNamespaceFiles.java @@ -1,8 +1,10 @@ package io.kestra.plugin.git; +import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.storages.Namespace; import io.kestra.core.utils.PathMatcherPredicate; @@ -41,7 +43,7 @@ code = """ id: push_to_git namespace: system - + tasks: - id: commit_and_push type: io.kestra.plugin.git.PushNamespaceFiles @@ -54,7 +56,7 @@ branch: dev # optional, uses "kestra" by default commitMessage: "add namespace files" # optional string dryRun: true # if true, you'll see what files will be added, modified or deleted based on the state in Git without overwriting the files yet - + triggers: - id: schedule_push_to_git type: io.kestra.plugin.core.trigger.Schedule @@ -70,20 +72,19 @@ public class PushNamespaceFiles extends AbstractPushTask branch = Property.of("kestra"); @Schema( title = "The namespace from which files should be pushed to the `gitDirectory`." ) - @PluginProperty(dynamic = true) @Builder.Default - private String namespace = "{{ flow.namespace }}"; + private Property namespace = new Property<>("{{ flow.namespace }}"); @Schema( title = "Directory to which Namespace Files should be pushed.", description = """ If not set, files will be pushed to a Git directory named _files. See the table below for an example mapping of Namespace Files to Git paths: - + | Namespace File Path | Git directory path | | --------------------- | ---------------------------- | | scripts/app.py | _files/scripts/app.py | @@ -92,9 +93,8 @@ public class PushNamespaceFiles extends AbstractPushTask gitDirectory = Property.of("_files"); @Schema( title = "Which Namespace Files should be included in the commit.", @@ -114,8 +114,10 @@ Given that this is a glob pattern string (or a list of glob patterns), you can i defaultValue = "Add files from `namespace` namespace" ) @Override - public String getCommitMessage() { - return Optional.ofNullable(this.commitMessage).orElse("Add files from " + this.namespace + " namespace"); + public String renderCommitMessage(RunContext runContext) throws IllegalVariableEvaluationException { + return runContext.render(this.commitMessage).as(String.class) + .orElse("Add files from " + runContext.render(this.namespace).as(String.class) + .orElse(runContext.render("{{flow.namespace}}")) + " namespace"); } @Override @@ -124,14 +126,15 @@ public Object globs() { } @Override - public String fetchedNamespace() { + public Property fetchedNamespace() { return this.namespace; } @Override protected Map> instanceResourcesContentByPath(RunContext runContext, Path baseDirectory, List globs) throws Exception { - Namespace storage = runContext.storage().namespace(runContext.render(this.namespace)); + Namespace storage = runContext.storage().namespace(runContext.render(this.namespace).as(String.class) + .orElse(runContext.render("{{flow.namespace}}"))); Predicate matcher = (globs != null) ? PathMatcherPredicate.matches(globs) : (path -> true); return storage diff --git a/src/main/java/io/kestra/plugin/git/Sync.java b/src/main/java/io/kestra/plugin/git/Sync.java index b1986e2..56e27cb 100644 --- a/src/main/java/io/kestra/plugin/git/Sync.java +++ b/src/main/java/io/kestra/plugin/git/Sync.java @@ -4,6 +4,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.models.tasks.VoidOutput; import io.kestra.core.repositories.FlowRepositoryInterface; @@ -59,7 +60,7 @@ code = """ id: sync_from_git namespace: company.team - + tasks: - id: git type: io.kestra.plugin.git.Sync @@ -70,7 +71,7 @@ gitDirectory: your_git_dir # optional, otherwise all files namespaceFilesDirectory: your_namespace_files_location # optional, otherwise the namespace root directory dryRun: true # if true, print the output of what files will be added/modified or deleted without overwriting the files yet - + triggers: - id: every_minute type: io.kestra.plugin.core.trigger.Schedule @@ -96,13 +97,13 @@ public class Sync extends AbstractCloningTask implements RunnableTask branch; @Schema( title = "If true, the task will only display modifications without syncing any files yet. If false (default), all namespace files and flows will be overwritten based on the state in Git." ) @PluginProperty - private Boolean dryRun; + private Property dryRun; @Override public VoidOutput run(RunContext runContext) throws Exception { @@ -110,7 +111,7 @@ public VoidOutput run(RunContext runContext) throws Exception { Map flowProps = (Map) runContext.getVariables().get("flow"); String namespace = flowProps.get("namespace"); String tenantId = flowProps.get("tenantId"); - boolean dryRun = this.dryRun != null && this.dryRun; + boolean dryRun = runContext.render(this.dryRun).as(Boolean.class).orElse(false); Clone clone = Clone.builder() .depth(1) @@ -284,7 +285,7 @@ private static void logUpdate(Logger logger, String path) { @Override @NotNull - public String getUrl() { + public Property getUrl() { return super.getUrl(); } } diff --git a/src/main/java/io/kestra/plugin/git/SyncFlows.java b/src/main/java/io/kestra/plugin/git/SyncFlows.java index 224bdfa..588e23a 100644 --- a/src/main/java/io/kestra/plugin/git/SyncFlows.java +++ b/src/main/java/io/kestra/plugin/git/SyncFlows.java @@ -4,6 +4,7 @@ import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.flows.Flow; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.DefaultRunContext; import io.kestra.core.runners.RunContext; import io.kestra.core.services.FlowService; @@ -68,9 +69,8 @@ public class SyncFlows extends AbstractSyncTask { @Schema( title = "The branch from which flows will be synced to Kestra." ) - @PluginProperty(dynamic = true) @Builder.Default - private String branch = "main"; + private Property branch = Property.of("main"); @Schema( title = "The target namespace to which flows from the `gitDirectory` should be synced.", @@ -78,11 +78,11 @@ public class SyncFlows extends AbstractSyncTask { If the top-level namespace specified in the flow source code is different than the `targetNamespace`, it will be overwritten by this target namespace. This facilitates moving between environments and projects. If `includeChildNamespaces` property is set to true, the top-level namespace in the source code will also be overwritten by the `targetNamespace` in children namespaces. For example, if the `targetNamespace` is set to `prod` and `includeChildNamespaces` property is set to `true`, then: - - `namespace: dev` in flow source code will be overwritten by `namespace: prod`, + - `namespace: dev` in flow source code will be overwritten by `namespace: prod`, - `namespace: dev.marketing.crm` will be overwritten by `namespace: prod.marketing.crm`. See the table below for a practical explanation: - + | Source namespace in the flow code | Git directory path | Synced to target namespace | | --------------------------------- | ------------------------------ | ----------------------------- | | namespace: dev | _flows/flow1.yml | namespace: prod | @@ -93,9 +93,8 @@ public class SyncFlows extends AbstractSyncTask { | namespace: dev.marketing.crm | _flows/marketing/crm/flow6.yml | namespace: prod.marketing.crm | """ ) - @PluginProperty(dynamic = true) @NotNull - private String targetNamespace; + private Property targetNamespace; @Schema( title = "Directory from which flows should be synced.", @@ -104,13 +103,13 @@ public class SyncFlows extends AbstractSyncTask { If `includeChildNamespaces` property is set to `true`, this task will push all flows from nested subdirectories into their corresponding child namespaces, e.g. if `targetNamespace` is set to `prod`, then: - - flows from the `_flows` directory will be synced to the `prod` namespace, - - flows from the `_flows/marketing` subdirectory in Git will be synced to the `prod.marketing` namespace, + - flows from the `_flows` directory will be synced to the `prod` namespace, + - flows from the `_flows/marketing` subdirectory in Git will be synced to the `prod.marketing` namespace, - flows from the `_flows/marketing/crm` subdirectory will be synced to the `prod.marketing.crm` namespace.""" ) @PluginProperty(dynamic = true) @Builder.Default - private String gitDirectory = "_flows"; + private Property gitDirectory = Property.of("_flows"); @Schema( title = "Whether you want to sync flows from child namespaces as well.", @@ -140,7 +139,7 @@ private FlowService flowService(RunContext runContext) { @Override - public String fetchedNamespace() { + public Property fetchedNamespace() { return this.targetNamespace; } @@ -155,7 +154,7 @@ protected Flow simulateResourceWrite(RunContext runContext, String renderedNames return null; } - return flowService(runContext).importFlow(runContext.tenantId(), SyncFlows.replaceNamespace(renderedNamespace, uri, inputStream), true); + return flowService(runContext).importFlow(runContext.flowInfo().tenantId(), SyncFlows.replaceNamespace(renderedNamespace, uri, inputStream), true); } @Override @@ -179,7 +178,7 @@ protected Flow writeResource(RunContext runContext, String renderedNamespace, UR String flowSource = SyncFlows.replaceNamespace(renderedNamespace, uri, inputStream); - return flowService(runContext).importFlow(runContext.tenantId(), flowSource); + return flowService(runContext).importFlow(runContext.flowInfo().tenantId(), flowSource); } private static String replaceNamespace(String renderedNamespace, URI uri, InputStream inputStream) throws IOException { @@ -225,10 +224,10 @@ protected SyncResult wrapper(RunContext runContext, String renderedGitDirectory, @Override protected List fetchResources(RunContext runContext, String renderedNamespace) { if (this.includeChildNamespaces) { - return flowService(runContext).findByNamespacePrefix(runContext.tenantId(), renderedNamespace); + return flowService(runContext).findByNamespacePrefix(runContext.flowInfo().tenantId(), renderedNamespace); } - return flowService(runContext).findByNamespace(runContext.tenantId(), renderedNamespace); + return flowService(runContext).findByNamespace(runContext.flowInfo().tenantId(), renderedNamespace); } @Override diff --git a/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java b/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java index b5d3682..2be862b 100644 --- a/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java +++ b/src/main/java/io/kestra/plugin/git/SyncNamespaceFiles.java @@ -3,6 +3,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.storages.Namespace; import io.kestra.core.storages.NamespaceFile; @@ -38,7 +39,7 @@ code = """ id: sync_from_git namespace: system - + tasks: - id: git type: io.kestra.plugin.git.SyncNamespaceFiles @@ -50,7 +51,7 @@ username: git_username password: "{{ secret('GITHUB_ACCESS_TOKEN') }}" dryRun: true # if true, the task will only log which flows from Git will be added/modified or deleted in kestra without making any changes in kestra backend yet - + triggers: - id: every_minute type: io.kestra.plugin.core.trigger.Schedule @@ -63,16 +64,14 @@ public class SyncNamespaceFiles extends AbstractSyncTask branch = Property.of("kestra"); @Schema( title = "The namespace from which files should be synced from the `gitDirectory` to Kestra." ) - @PluginProperty(dynamic = true) @Builder.Default - private String namespace = "{{ flow.namespace }}"; + private Property namespace = new Property<>("{{ flow.namespace }}"); @Schema( title = "Directory from which Namespace Files should be synced.", @@ -80,7 +79,7 @@ public class SyncNamespaceFiles extends AbstractSyncTask gitDirectory = Property.of("_files"); @Schema( title = "Whether you want to delete Namespace Files present in kestra but not present in Git.", @@ -91,7 +90,7 @@ public class SyncNamespaceFiles extends AbstractSyncTask fetchedNamespace() { return this.namespace; } diff --git a/src/main/java/io/kestra/plugin/git/services/GitService.java b/src/main/java/io/kestra/plugin/git/services/GitService.java index 37fe206..a0fa804 100644 --- a/src/main/java/io/kestra/plugin/git/services/GitService.java +++ b/src/main/java/io/kestra/plugin/git/services/GitService.java @@ -1,6 +1,7 @@ package io.kestra.plugin.git.services; import io.kestra.core.exceptions.IllegalVariableEvaluationException; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.DefaultRunContext; import io.kestra.core.runners.RunContext; import io.kestra.core.services.FlowService; @@ -28,13 +29,13 @@ public Git cloneBranch(RunContext runContext, String branch, Boolean withSubmodu .password(gitTask.getPassword()) .privateKey(gitTask.getPrivateKey()) .passphrase(gitTask.getPassphrase()) - .cloneSubmodules(withSubmodules) + .cloneSubmodules(Property.of(withSubmodules)) .build(); boolean branchExists = this.branchExists(runContext, branch); if (branchExists) { cloneHead.toBuilder() - .branch(branch) + .branch(Property.of(branch)) .build() .run(runContext); } else { @@ -56,7 +57,7 @@ public Git cloneBranch(RunContext runContext, String branch, Boolean withSubmodu } public boolean branchExists(RunContext runContext, String branch) throws Exception { - return gitTask.authentified(Git.lsRemoteRepository().setRemote(runContext.render(gitTask.getUrl())), runContext) + return gitTask.authentified(Git.lsRemoteRepository().setRemote(runContext.render(gitTask.getUrl()).as(String.class).orElseThrow()), runContext) .callAsMap() .containsKey(R_HEADS + branch); } @@ -81,12 +82,12 @@ public String getHttpUrl(String gitUrl) { return httpUrl; } - public void namespaceAccessGuard(RunContext runContext, String namespaceToAccess) throws IllegalVariableEvaluationException { + public void namespaceAccessGuard(RunContext runContext, Property namespaceToAccess) throws IllegalVariableEvaluationException { FlowService flowService = ((DefaultRunContext)runContext).getApplicationContext().getBean(FlowService.class); RunContext.FlowInfo flowInfo = runContext.flowInfo(); flowService.checkAllowedNamespace( - runContext.tenantId(), - runContext.render(namespaceToAccess), + runContext.flowInfo().tenantId(), + runContext.render(namespaceToAccess).as(String.class).orElse(runContext.render("{{flow.namespace}}")), flowInfo.tenantId(), flowInfo.namespace() ); diff --git a/src/test/java/io/kestra/plugin/git/CloneTest.java b/src/test/java/io/kestra/plugin/git/CloneTest.java index cd8e572..7f9b9af 100644 --- a/src/test/java/io/kestra/plugin/git/CloneTest.java +++ b/src/test/java/io/kestra/plugin/git/CloneTest.java @@ -1,5 +1,6 @@ package io.kestra.plugin.git; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.micronaut.context.annotation.Value; @@ -25,7 +26,7 @@ void publicRepository() throws Exception { RunContext runContext = runContextFactory.of(); Clone task = Clone.builder() - .url("https://github.com/kestra-io/plugin-template") + .url(Property.of("https://github.com/kestra-io/plugin-template")) .build(); Clone.Output runOutput = task.run(runContext); @@ -42,9 +43,9 @@ void privateRepository() throws Exception { RunContext runContext = runContextFactory.of(); Clone task = Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) .build(); Clone.Output runOutput = task.run(runContext); diff --git a/src/test/java/io/kestra/plugin/git/PushFlowsTest.java b/src/test/java/io/kestra/plugin/git/PushFlowsTest.java index b26aa2c..19f5977 100644 --- a/src/test/java/io/kestra/plugin/git/PushFlowsTest.java +++ b/src/test/java/io/kestra/plugin/git/PushFlowsTest.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import io.kestra.core.models.flows.Flow; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; @@ -50,24 +51,6 @@ public class PushFlowsTest extends AbstractGitTest { @Inject private FlowRepositoryInterface flowRepositoryInterface; - @Test - void hardcodedPassword() { - PushFlows pushFlows = PushFlows.builder() - .id("pushFlows") - .type(PushFlows.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> pushFlows.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_SingleRegex() throws Exception { String tenantId = "my-tenant"; @@ -85,18 +68,18 @@ void defaultCase_SingleRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) .targetNamespace("{{targetNamespace}}") .flows("second*") .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -107,10 +90,10 @@ void defaultCase_SingleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -159,19 +142,19 @@ void defaultCase_SingleRegexDryRun() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) .targetNamespace("{{targetNamespace}}") .flows("second*") .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") - .dryRun(true) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .dryRun(Property.of(true)) .build(); PushFlows.Output pushOutput = pushFlows.run(runContext); @@ -208,17 +191,17 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) .targetNamespace("{{targetNamespace}}") .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -227,10 +210,10 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -309,17 +292,17 @@ void defaultCase_NoRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) .targetNamespace("{{targetNamespace}}") .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -328,10 +311,10 @@ void defaultCase_NoRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -387,18 +370,18 @@ void defaultCase_MultipleRegex() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .sourceNamespace("{{sourceNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) .targetNamespace("{{targetNamespace}}") .flows(List.of("first*", "second*")) .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -407,10 +390,10 @@ void defaultCase_MultipleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -465,15 +448,15 @@ void defaultCase_NoRegexNoChildNsNoAuthorName() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .sourceNamespace("{{sourceNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) .targetNamespace("{{targetNamespace}}") - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -482,10 +465,10 @@ void defaultCase_NoRegexNoChildNsNoAuthorName() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -529,15 +512,15 @@ void defaultCase_NoRegexNoAuthor() throws Exception { PushFlows pushFlows = PushFlows.builder() .id("pushFlows") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .sourceNamespace("{{sourceNamespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .sourceNamespace(new Property<>("{{sourceNamespace}}")) .targetNamespace("{{targetNamespace}}") .includeChildNamespaces(true) - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -546,10 +529,10 @@ void defaultCase_NoRegexNoAuthor() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -625,9 +608,9 @@ private FlowWithSource createFlow(String tenantId, String namespace) { private FlowWithSource createFlow(String tenantId, String flowId, String namespace) { String flowSource = """ id:\s""" + flowId + """ - + namespace:\s""" + namespace + """ - + tasks: - id: my-task type: io.kestra.core.tasks.log.Log diff --git a/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java b/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java index 5785bab..8dfa9e0 100644 --- a/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java +++ b/src/test/java/io/kestra/plugin/git/PushNamespaceFilesTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.git; import com.fasterxml.jackson.core.type.TypeReference; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.serializers.JacksonMapper; @@ -45,24 +46,6 @@ public class PushNamespaceFilesTest extends AbstractGitTest { @Inject private StorageInterface storage; - @Test - void hardcodedPassword() { - PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() - .id("pushNamespaceFiles") - .type(PushNamespaceFiles.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> pushNamespaceFiles.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_SingleRegex() throws Exception { String tenantId = "my-tenant"; @@ -81,16 +64,16 @@ void defaultCase_SingleRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) .files("nested/*") - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -101,10 +84,10 @@ void defaultCase_SingleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -154,17 +137,17 @@ void defaultCase_SingleRegexDryRun() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) .files("second*") - .gitDirectory("{{gitDirectory}}") - .dryRun(true) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .dryRun(Property.of(true)) .build(); PushNamespaceFiles.Output pushOutput = pushNamespaceFiles.run(runContext); @@ -205,15 +188,15 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushFlows.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -222,10 +205,10 @@ void defaultCase_SingleRegex_DeleteScopedToRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -306,15 +289,15 @@ void defaultCase_NoRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -323,10 +306,10 @@ void defaultCase_NoRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -385,16 +368,16 @@ void defaultCase_MultipleRegex() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .authorEmail("{{email}}") - .authorName("{{name}}") - .namespace("{{namespace}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .authorEmail(new Property<>("{{email}}")) + .authorName(new Property<>("{{name}}")) + .namespace(new Property<>("{{namespace}}")) .files(List.of("first*", "second*")) - .gitDirectory("{{gitDirectory}}") + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -403,10 +386,10 @@ void defaultCase_MultipleRegex() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -458,13 +441,13 @@ void defaultCase_NoRegexNoAuthor() throws Exception { PushNamespaceFiles pushNamespaceFiles = PushNamespaceFiles.builder() .id("pushNamespaceFiles") .type(PushNamespaceFiles.class.getName()) - .branch("{{branch}}") - .url("{{url}}") - .commitMessage("Push from CI - {{description}}") - .username("{{pat}}") - .password("{{pat}}") - .namespace("{{namespace}}") - .gitDirectory("{{gitDirectory}}") + .branch(new Property<>("{{branch}}")) + .url(new Property<>("{{url}}")) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .namespace(new Property<>("{{namespace}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) .build(); try { @@ -473,10 +456,10 @@ void defaultCase_NoRegexNoAuthor() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branch) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branch)) .build(); RunContext cloneRunContext = runContextFactory.of(); diff --git a/src/test/java/io/kestra/plugin/git/PushTest.java b/src/test/java/io/kestra/plugin/git/PushTest.java index a38b78b..710ccc8 100644 --- a/src/test/java/io/kestra/plugin/git/PushTest.java +++ b/src/test/java/io/kestra/plugin/git/PushTest.java @@ -2,6 +2,7 @@ import io.kestra.core.models.flows.Flow; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.NamespaceFiles; import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.runners.RunContext; @@ -61,10 +62,10 @@ void cloneThenPush_OnlyNeedsCredentialsForPush() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -85,13 +86,13 @@ void cloneThenPush_OnlyNeedsCredentialsForPush() throws Exception { .flows(Push.FlowFiles.builder() .enabled(false) .build()) - .commitMessage("Push from CI - Clone then push") + .commitMessage(Property.of("Push from CI - Clone then push")) .inputFiles(Map.of( INPUT_FILE_NAME, expectedInputFileContent )) - .username(pat) - .password(pat) - .branch(BRANCH) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build(); Push.Output pushOutput = push.run(cloneRunContext); @@ -122,10 +123,10 @@ void cloneThenPush_PushBranchContentToAnother() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build(); RunContext cloneRunContext = runContextFactory.of(); @@ -139,10 +140,10 @@ void cloneThenPush_PushBranchContentToAnother() throws Exception { .flows(Push.FlowFiles.builder() .enabled(false) .build()) - .commitMessage("Push from CI - Clone then push") - .username(pat) - .password(pat) - .branch(otherBranch) + .commitMessage(Property.of("Push from CI - Clone then push")) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(Property.of(otherBranch)) .build(); Push.Output pushOutput = push.run(cloneRunContext); @@ -186,13 +187,13 @@ void oneTaskPush_ExistingBranch() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") + .url(Property.of(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) .flows(Push.FlowFiles.builder() .enabled(false) .build()) - .username(pat) - .password(pat) + .username(Property.of(pat)) + .password(Property.of(pat)) .inputFiles(Map.of( INPUT_FILE_NAME, expectedInputFileContent, shouldNotBeCommitted, "should not be committed" @@ -205,7 +206,7 @@ void oneTaskPush_ExistingBranch() throws Exception { INPUT_FILE_NAME, namespaceFileName )) - .branch(BRANCH) + .branch(new Property<>(BRANCH)) .build(); push.run(runContext); @@ -213,10 +214,10 @@ void oneTaskPush_ExistingBranch() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -236,10 +237,10 @@ void oneTaskPush_NonExistingBranch() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); Assertions.assertThrows(TransportException.class, () -> clone.run(runContextFactory.of())); @@ -248,17 +249,17 @@ void oneTaskPush_NonExistingBranch() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) + .url(Property.of(repositoryUrl)) .inputFiles(Map.of( toDeleteFileName, "some content" )) .flows(Push.FlowFiles.builder() .enabled(false) .build()) - .commitMessage("Branch creation") - .username(pat) - .password(pat) - .branch(branchName) + .commitMessage(Property.of("Branch creation")) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); push.run(runContextFactory.of()); @@ -286,27 +287,27 @@ void oneTaskPush_NoChangeShouldNotCommit() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); String toDeleteFileName = "to_delete.txt"; Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) + .url(Property.of(repositoryUrl)) .inputFiles(Map.of( toDeleteFileName, "some content" )) .flows(Push.FlowFiles.builder() .enabled(false) .build()) - .commitMessage("Branch creation") - .username(pat) - .password(pat) - .branch(branchName) + .commitMessage(Property.of("Branch creation")) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); RunContext runContext = runContextFactory.of(); Push.Output firstPush = push.run(runContext); @@ -344,13 +345,13 @@ void oneTaskPush_WithSpecifiedDirectory() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - One-task push with specified directory") + .url(Property.of(repositoryUrl)) + .commitMessage(Property.of("Push from CI - One-task push with specified directory")) .flows(Push.FlowFiles.builder() .enabled(false) .build()) - .username(pat) - .password(pat) + .username(Property.of(pat)) + .password(Property.of(pat)) .directory(directory) .inputFiles(Map.of( "not_included_file.txt", "not included", @@ -358,7 +359,7 @@ void oneTaskPush_WithSpecifiedDirectory() throws Exception { directory + "/" + INPUT_FILE_NAME, expectedInputFileContent, directory + "/" + nestedFilePath, expectedNestedInputFileContent )) - .branch(BRANCH) + .branch(new Property<>(BRANCH)) .build(); push.run(runContext); @@ -366,10 +367,10 @@ void oneTaskPush_WithSpecifiedDirectory() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -400,11 +401,11 @@ void oneTaskPush_WithFlows() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") - .username(pat) - .password(pat) - .branch(branchName) + .url(Property.of(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); try { @@ -413,10 +414,10 @@ void oneTaskPush_WithFlows() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -454,18 +455,18 @@ void oneTaskPush_WithFlowsNoChildNs() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") + .url(Property.of(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) .flows(Push.FlowFiles.builder() .childNamespaces(false) .build()) - .username(pat) - .password(pat) + .username(Property.of(pat)) + .password(Property.of(pat)) .author(Push.Author.builder() .name(gitUserName) .email(gitUserEmail) .build()) - .branch(branchName) + .branch(new Property<>(branchName)) .build(); try { @@ -474,10 +475,10 @@ void oneTaskPush_WithFlowsNoChildNs() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); @@ -513,14 +514,14 @@ void oneTaskPush_WithFlowsAndDirectory() throws Exception { Push push = Push.builder() .id("push") .type(Push.class.getName()) - .url(repositoryUrl) - .commitMessage("Push from CI - {{description}}") - .username(pat) - .password(pat) + .url(Property.of(repositoryUrl)) + .commitMessage(new Property<>("Push from CI - {{description}}")) + .username(Property.of(pat)) + .password(Property.of(pat)) .author(Push.Author.builder() .email(gitUserEmail) .build()) - .branch(branchName) + .branch(new Property<>(branchName)) .flows(Push.FlowFiles.builder() .gitDirectory("my-flows") .build()) @@ -532,10 +533,10 @@ void oneTaskPush_WithFlowsAndDirectory() throws Exception { Clone clone = Clone.builder() .id("clone") .type(Clone.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(branchName) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(branchName)) .build(); Clone.Output cloneOutput = clone.run(runContextFactory.of()); diff --git a/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java b/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java index a8e9a16..cd0dcef 100644 --- a/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java +++ b/src/test/java/io/kestra/plugin/git/SyncFlowsTest.java @@ -4,6 +4,7 @@ import io.kestra.core.models.executions.LogEntry; import io.kestra.core.models.flows.Flow; import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.models.property.Property; import io.kestra.core.queues.QueueInterface; import io.kestra.core.repositories.FlowRepositoryInterface; import io.kestra.core.runners.RunContext; @@ -63,24 +64,6 @@ void init() { }); } - @Test - void hardcodedPassword() { - SyncFlows syncFlows = SyncFlows.builder() - .id("syncFlows") - .type(PushNamespaceFiles.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> syncFlows.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_WithDelete() throws Exception { RunContext runContext = runContext(); @@ -145,12 +128,12 @@ void defaultCase_WithDelete() throws Exception { flows.forEach(f -> previousRevisionByUid.put(f.uidWithoutRevision(), f.getRevision())); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) .delete(true) .includeChildNamespaces(true) .build(); @@ -161,10 +144,10 @@ void defaultCase_WithDelete() throws Exception { RunContext cloneRunContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build() .run(cloneRunContext); assertFlows(cloneRunContext.workingDir().path().resolve(Path.of(GIT_DIRECTORY)).toFile(), true, selfFlowSource); @@ -239,12 +222,12 @@ void defaultCase_WithoutDelete() throws Exception { flows.forEach(f -> previousRevisionByUid.put(f.uidWithoutRevision(), f.getRevision())); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) .includeChildNamespaces(true) .build(); SyncFlows.Output syncOutput = task.run(runContext); @@ -254,10 +237,10 @@ void defaultCase_WithoutDelete() throws Exception { RunContext cloneRunContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build() .run(cloneRunContext); assertFlows(cloneRunContext.workingDir().path().resolve(Path.of(GIT_DIRECTORY)).toFile(), true, selfFlowSource, nonVersionedFlowSource); @@ -338,12 +321,12 @@ void defaultCase_WithDeleteNoChildNs() throws Exception { flows.forEach(f -> previousRevisionByUid.put(f.uidWithoutRevision(), f.getRevision())); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) .delete(true) .includeChildNamespaces(false) .build(); @@ -354,10 +337,10 @@ void defaultCase_WithDeleteNoChildNs() throws Exception { RunContext cloneRunContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build() .run(cloneRunContext); assertFlows(cloneRunContext.workingDir().path().resolve(Path.of(GIT_DIRECTORY)).toFile(), false, selfFlowSource, unversionedFlowSourceInChildNamespace); @@ -435,15 +418,15 @@ void dryRun_WithDelete() throws Exception { .toArray(String[]::new); SyncFlows task = SyncFlows.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .targetNamespace("{{namespace}}") + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .targetNamespace(new Property<>("{{namespace}}")) .delete(true) .includeChildNamespaces(true) - .dryRun(true) + .dryRun(Property.of(true)) .build(); SyncFlows.Output syncOutput = task.run(runContext); diff --git a/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java b/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java index 965cffe..e33f50b 100644 --- a/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java +++ b/src/test/java/io/kestra/plugin/git/SyncNamespaceFilesTest.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import io.kestra.core.junit.annotations.KestraTest; import io.kestra.core.models.executions.LogEntry; +import io.kestra.core.models.property.Property; import io.kestra.core.queues.QueueInterface; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; @@ -48,24 +49,6 @@ void init() throws IOException { storage.deleteByPrefix(TENANT_ID, NAMESPACE, URI.create(StorageContext.namespaceFilePrefix(NAMESPACE))); } - @Test - void hardcodedPassword() { - SyncNamespaceFiles syncNamespaceFiles = SyncNamespaceFiles.builder() - .id("syncNamespaceFiles") - .type(PushNamespaceFiles.class.getName()) - .url(repositoryUrl) - .password("my-password") - .build(); - - IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> syncNamespaceFiles.run(runContextFactory.of(Map.of( - "flow", Map.of( - "tenantId", "tenantId", - "namespace", "system" - )))) - ); - assertThat(illegalArgumentException.getMessage(), is("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).")); - } - @Test void defaultCase_WithDelete() throws Exception { RunContext runContext = runContext(); @@ -108,12 +91,12 @@ void defaultCase_WithDelete() throws Exception { ); SyncNamespaceFiles task = SyncNamespaceFiles.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .namespace("{{namespace}}") + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .namespace(new Property<>("{{namespace}}")) .delete(true) .build(); SyncNamespaceFiles.Output syncOutput = task.run(runContext); @@ -174,12 +157,12 @@ void defaultCase_WithoutDelete() throws Exception { ); SyncNamespaceFiles task = SyncNamespaceFiles.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .namespace("{{namespace}}") + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .namespace(new Property<>("{{namespace}}")) .build(); SyncNamespaceFiles.Output syncOutput = task.run(runContext); @@ -239,13 +222,13 @@ void defaultCase_DryRunWithDeleteFlag_ShouldStillNotifyWhatWouldBeDeleted() thro ); SyncNamespaceFiles task = SyncNamespaceFiles.builder() - .url("{{url}}") - .username("{{pat}}") - .password("{{pat}}") - .branch("{{branch}}") - .gitDirectory("{{gitDirectory}}") - .namespace("{{namespace}}") - .dryRun(true) + .url(new Property<>("{{url}}")) + .username(new Property<>("{{pat}}")) + .password(new Property<>("{{pat}}")) + .branch(new Property<>("{{branch}}")) + .gitDirectory(new Property<>("{{gitDirectory}}")) + .namespace(new Property<>("{{namespace}}")) + .dryRun(Property.of(true)) .delete(true) .build(); SyncNamespaceFiles.Output syncOutput = task.run(runContext); diff --git a/src/test/java/io/kestra/plugin/git/SyncTest.java b/src/test/java/io/kestra/plugin/git/SyncTest.java index 975cf98..0bb40d7 100644 --- a/src/test/java/io/kestra/plugin/git/SyncTest.java +++ b/src/test/java/io/kestra/plugin/git/SyncTest.java @@ -2,6 +2,7 @@ import io.kestra.core.models.executions.LogEntry; import io.kestra.core.models.flows.Flow; +import io.kestra.core.models.property.Property; import io.kestra.core.queues.QueueFactoryInterface; import io.kestra.core.queues.QueueInterface; import io.kestra.core.repositories.FlowRepositoryInterface; @@ -172,10 +173,10 @@ void reconcileNsFilesAndFlows() throws Exception { String clonedGitDirectory = "to_clone"; String destinationDirectory = "sync_directory"; Sync task = Sync.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .gitDirectory(clonedGitDirectory) .namespaceFilesDirectory(destinationDirectory) .build(); @@ -193,10 +194,10 @@ void reconcileNsFilesAndFlows() throws Exception { RunContext runContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build() .run(runContext); assertFlows(TENANT_ID, runContext.workingDir().path().resolve(Path.of(clonedGitDirectory, "_flows")).toFile(), selfFlowSource); @@ -277,10 +278,10 @@ void reconcile_MinimumSetup() throws Exception { // region WHEN Sync task = Sync.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build(); task.run(runContextFactory.of(Map.of("flow", Map.of( @@ -297,10 +298,10 @@ void reconcile_MinimumSetup() throws Exception { RunContext runContext = runContextFactory.of(); Clone.builder() - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .build() .run(runContext); assertFlows(TENANT_ID, runContext.workingDir().path().resolve("_flows").toFile(), selfFlowSource); @@ -374,12 +375,12 @@ void reconcile_DryRun_ShouldDoNothing() throws Exception { Sync task = Sync.builder() .id("reconcile") .type(Sync.class.getName()) - .url(repositoryUrl) - .username(pat) - .password(pat) - .branch(BRANCH) + .url(Property.of(repositoryUrl)) + .username(Property.of(pat)) + .password(Property.of(pat)) + .branch(new Property<>(BRANCH)) .gitDirectory("to_clone") - .dryRun(true) + .dryRun(Property.of(true)) .build(); RunContext runContext = TestsUtils.mockRunContext(runContextFactory, task, Collections.emptyMap()); task.run(runContext);