Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: migrate to dynamic properties #108

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

refactor: migrate to dynamic properties #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions src/main/java/io/kestra/plugin/git/AbstractCloningTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.kestra.plugin.git;

import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.NoArgsConstructor;
Expand All @@ -13,6 +13,5 @@ public abstract class AbstractCloningTask extends AbstractGitTask {
@Schema(
title = "Whether to clone submodules."
)
@PluginProperty
protected Boolean cloneSubmodules;
protected Property<Boolean> cloneSubmodules;
}
35 changes: 11 additions & 24 deletions src/main/java/io/kestra/plugin/git/AbstractGitTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.kestra.plugin.git;

import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.runners.RunContext;
import io.kestra.plugin.git.services.SshTransportConfigCallback;
Expand All @@ -18,68 +18,55 @@
@NoArgsConstructor
@Getter
public abstract class AbstractGitTask extends Task {
private static final Pattern PEBBLE_TEMPLATE_PATTERN = Pattern.compile("^\\s*\\{\\{");

@Schema(
title = "The URI to clone from."
)
@PluginProperty(dynamic = true)
protected String url;
protected Property<String> url;

@Schema(
title = "The username or organization."
)
@PluginProperty(dynamic = true)
protected String username;
protected Property<String> username;

@Schema(
title = "The password or Personal Access Token (PAT). When you authenticate the task with a PAT, any flows or files pushed to Git from Kestra will be pushed from the user associated with that PAT. This way, you don't need to configure the commit author (the `authorName` and `authorEmail` properties)."
)
@PluginProperty(dynamic = true)
protected String password;
protected Property<String> password;

@Schema(
title = "PEM-format private key content that is paired with a public key registered on Git.",
description = "To generate an ECDSA PEM format key from OpenSSH, use the following command: `ssh-keygen -t ecdsa -b 256 -m PEM`. " +
"You can then set this property with your private key content and put your public key on Git."
)
@PluginProperty(dynamic = true)
protected String privateKey;
protected Property<String> privateKey;

@Schema(
title = "The passphrase for the `privateKey`."
)
@PluginProperty(dynamic = true)
protected String passphrase;
protected Property<String> passphrase;


@Schema(
title = "The initial Git branch."
)
@PluginProperty(dynamic = true)
public abstract String getBranch();
public abstract Property<String> getBranch();

public <T extends TransportCommand<T, ?>> T authentified(T command, RunContext runContext) throws Exception {
if (this.username != null && this.password != null) {
command.setCredentialsProvider(new UsernamePasswordCredentialsProvider(
runContext.render(this.username),
runContext.render(this.password)
runContext.render(this.username).as(String.class).orElseThrow(),
runContext.render(this.password).as(String.class).orElseThrow()
));
}

if (this.privateKey != null) {
command.setTransportConfigCallback(new SshTransportConfigCallback(
runContext.render(this.privateKey).getBytes(StandardCharsets.UTF_8),
runContext.render(this.passphrase)
runContext.render(this.privateKey).as(String.class).orElseThrow().getBytes(StandardCharsets.UTF_8),
runContext.render(this.passphrase).as(String.class).orElse(null)
));
}

return command;
}

protected void detectPasswordLeaks() {
if (this.password != null && !PEBBLE_TEMPLATE_PATTERN.matcher(this.password).find()) {
throw new IllegalArgumentException("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).");
}
}
}
48 changes: 24 additions & 24 deletions src/main/java/io/kestra/plugin/git/AbstractPushTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.kestra.plugin.git;

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
Expand Down Expand Up @@ -46,41 +46,39 @@
@NoArgsConstructor
@Getter
public abstract class AbstractPushTask<O extends AbstractPushTask.Output> extends AbstractCloningTask implements RunnableTask<O> {
@PluginProperty(dynamic = true)
protected String commitMessage;
protected Property<String> commitMessage;

@Schema(
title = "If `true`, the task will only output modifications without pushing any file to Git yet. If `false` (default), all listed files will be pushed to Git immediately."
)
@PluginProperty
@Builder.Default
private boolean dryRun = false;
private Property<Boolean> dryRun = Property.of(false);

@Schema(
title = "The commit author email.",
description = "If null, no author will be set on this commit."
)
@PluginProperty(dynamic = true)
private String authorEmail;
private Property<String> authorEmail;

@Schema(
title = "The commit author name.",
description = "If null, the username will be used instead.",
defaultValue = "`username`"
)
@PluginProperty(dynamic = true)
private String authorName;
private Property<String> authorName;

public abstract String getCommitMessage();
public String renderCommitMessage(RunContext runContext) throws IllegalVariableEvaluationException {
return runContext.render(this.getCommitMessage()).as(String.class).orElseThrow();
}

public abstract String getGitDirectory();
public abstract Property<String> getGitDirectory();

public abstract Object globs();

public abstract String fetchedNamespace();
public abstract Property<String> fetchedNamespace();

private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException {
Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory())));
Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElseThrow()));
flowDirectory.toFile().mkdirs();
return flowDirectory;
}
Expand Down Expand Up @@ -130,14 +128,14 @@ protected void writeResourceFile(Path path, InputStream inputStream) throws IOEx
Files.copy(inputStream, path, REPLACE_EXISTING);
}

private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException {
private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException, IllegalVariableEvaluationException {
File diffFile = runContext.workingDir().createTempFile(".ion").toFile();
try (DiffFormatter diffFormatter = new DiffFormatter(null);
BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) {
diffFormatter.setRepository(git.getRepository());

DiffCommand diff = git.diff();
if (this.dryRun) {
if (runContext.render(this.dryRun).as(Boolean.class).orElse(false)) {
diff = diff.setCached(true);
} else {
diff = diff.setOldTree(treeIterator(git, "HEAD~1"))
Expand Down Expand Up @@ -204,21 +202,21 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw
String commitId = null;
ObjectId commit;
try {
String httpUrl = gitService.getHttpUrl(runContext.render(this.url));
if (this.isDryRun()) {
String httpUrl = gitService.getHttpUrl(runContext.render(this.url).as(String.class).orElse(null));
if (runContext.render(dryRun).as(Boolean.class).orElse(false)) {
logger.info(
"Dry run — no changes will be pushed to {} for now until you set the `dryRun` parameter to false",
httpUrl
);
} else {
String renderedBranch = runContext.render(this.getBranch());
String renderedBranch = runContext.render(this.getBranch()).as(String.class).orElse(null);
logger.info(
"Pushing to {} on branch {}",
httpUrl,
renderedBranch
);

String message = runContext.render(this.getCommitMessage());
String message = this.renderCommitMessage(runContext);
ObjectId head = git.getRepository().resolve(Constants.HEAD);
commit = git.commit()
.setAllowEmpty(false)
Expand Down Expand Up @@ -247,8 +245,9 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw
}

private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluationException {
String name = Optional.ofNullable(this.authorName).orElse(runContext.render(this.username));
String authorEmail = this.authorEmail;
String name = runContext.render(this.authorName).as(String.class)
.orElse(runContext.render(this.username).as(String.class).orElse(null));
String authorEmail = runContext.render(this.authorEmail).as(String.class).orElse(null);
if (authorEmail == null || name == null) {
return null;
}
Expand All @@ -274,9 +273,10 @@ public O run(RunContext runContext) throws Exception {
GitService gitService = new GitService(this);

gitService.namespaceAccessGuard(runContext, this.fetchedNamespace());
this.detectPasswordLeaks();

Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules);
Git git = gitService.cloneBranch(runContext,
runContext.render(this.getBranch()).as(String.class).orElse(null),
runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false));

Path localGitDirectory = this.createGitDirectory(runContext);

Expand All @@ -291,7 +291,7 @@ public O run(RunContext runContext) throws Exception {
this.writeResourceFiles(contentByPath);

AddCommand add = git.add();
add.addFilepattern(runContext.render(this.getGitDirectory()));
add.addFilepattern(runContext.render(this.getGitDirectory()).as(String.class).orElse(null));
add.call();

Output pushOutput = this.push(git, runContext, gitService);
Expand Down
26 changes: 14 additions & 12 deletions src/main/java/io/kestra/plugin/git/AbstractSyncTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
Expand Down Expand Up @@ -43,18 +44,17 @@ public abstract class AbstractSyncTask<T, O extends AbstractSyncTask.Output> ext
@Schema(
title = "If `true`, the task will only output modifications without performing any modification to Kestra. If `false` (default), all listed modifications will be applied."
)
@PluginProperty
@Builder.Default
private boolean dryRun = false;
private Property<Boolean> dryRun = Property.of(false);

public abstract boolean isDelete();

public abstract String getGitDirectory();
public abstract Property<String> getGitDirectory();

public abstract String fetchedNamespace();
public abstract Property<String> fetchedNamespace();

private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException {
Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory())));
Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElseThrow()));
syncDirectory.toFile().mkdirs();
return syncDirectory;
}
Expand All @@ -81,7 +81,7 @@ protected Map<URI, Supplier<InputStream>> gitResourcesContentByUri(Path baseDire
protected boolean traverseDirectories() {
return true;
}

protected boolean mustKeep(RunContext runContext, T instanceResource) {
return false;
}
Expand All @@ -100,7 +100,7 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map<
try (BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) {
List<SyncResult> syncResults = new ArrayList<>();

String renderedGitDirectory = runContext.render(this.getGitDirectory());
String renderedGitDirectory = runContext.render(this.getGitDirectory()).as(String.class).orElse(null);
if (deletedResources != null) {
deletedResources.stream()
.map(throwFunction(deletedResource -> wrapper(
Expand Down Expand Up @@ -146,17 +146,19 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map<
}

public O run(RunContext runContext) throws Exception {
this.detectPasswordLeaks();
GitService gitService = new GitService(this);

gitService.namespaceAccessGuard(runContext, this.fetchedNamespace());

Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules);
Git git = gitService.cloneBranch(runContext,
runContext.render(this.getBranch()).as(String.class).orElse(null),
runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false)
);

Path localGitDirectory = this.createGitDirectory(runContext);
Map<URI, Supplier<InputStream>> gitContentByUri = this.gitResourcesContentByUri(localGitDirectory);

String renderedNamespace = runContext.render(this.fetchedNamespace());
String renderedNamespace = runContext.render(this.fetchedNamespace()).as(String.class).orElse(null);

Map<URI, T> beforeUpdateResourcesByUri = this.fetchResources(runContext, renderedNamespace)
.stream()
Expand All @@ -170,7 +172,7 @@ public O run(RunContext runContext) throws Exception {
.map(throwFunction(e -> {
InputStream inputStream = e.getValue().get();
T resource;
if (this.dryRun) {
if (runContext.render(dryRun).as(Boolean.class).orElseThrow()) {
resource = this.simulateResourceWrite(runContext, renderedNamespace, e.getKey(), inputStream);
} else {
resource = this.writeResource(runContext, renderedNamespace, e.getKey(), inputStream);
Expand Down Expand Up @@ -198,7 +200,7 @@ public O run(RunContext runContext) throws Exception {
return;
}

if (!this.dryRun) {
if (!runContext.render(dryRun).as(Boolean.class).orElseThrow()) {
this.deleteResource(runContext, renderedNamespace, e.getValue());
}

Expand Down
13 changes: 7 additions & 6 deletions src/main/java/io/kestra/plugin/git/Clone.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.swagger.v3.oas.annotations.media.Schema;
Expand Down Expand Up @@ -78,7 +79,7 @@
code = """
id: git_python
namespace: company.team

tasks:
- id: file_system
type: io.kestra.plugin.core.flow.WorkingDirectory
Expand All @@ -105,7 +106,7 @@ public class Clone extends AbstractCloningTask implements RunnableTask<Clone.Out
@PluginProperty(dynamic = true)
private String directory;

private String branch;
private Property<String> branch;

@Schema(
title = "Creates a shallow clone with a history truncated to the specified number of commits."
Expand All @@ -118,7 +119,7 @@ public class Clone extends AbstractCloningTask implements RunnableTask<Clone.Out
@Override
public Clone.Output run(RunContext runContext) throws Exception {
Logger logger = runContext.logger();
String url = runContext.render(this.url);
String url = runContext.render(this.url).as(String.class).orElse(null);

Path path = runContext.workingDir().path();
if (this.directory != null) {
Expand All @@ -131,15 +132,15 @@ public Clone.Output run(RunContext runContext) throws Exception {
.setDirectory(path.toFile());

if (this.branch != null) {
cloneCommand.setBranch(runContext.render(this.branch));
cloneCommand.setBranch(runContext.render(this.branch).as(String.class).orElseThrow());
}

if (this.depth != null) {
cloneCommand.setDepth(this.depth);
}

if (this.cloneSubmodules != null) {
cloneCommand.setCloneSubmodules(this.cloneSubmodules);
cloneCommand.setCloneSubmodules(runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false));
}

cloneCommand = authentified(cloneCommand, runContext);
Expand All @@ -155,7 +156,7 @@ public Clone.Output run(RunContext runContext) throws Exception {

@Override
@NotNull
public String getUrl() {
public Property<String> getUrl() {
return super.getUrl();
}

Expand Down
Loading
Loading