Skip to content

Commit

Permalink
refactor: migrate to property v2
Browse files Browse the repository at this point in the history
migration of abstract class
  • Loading branch information
mgabelle committed Dec 4, 2024
1 parent 77d6ad2 commit 77dadaf
Show file tree
Hide file tree
Showing 19 changed files with 463 additions and 533 deletions.
5 changes: 2 additions & 3 deletions src/main/java/io/kestra/plugin/git/AbstractCloningTask.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.kestra.plugin.git;

import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.NoArgsConstructor;
Expand All @@ -13,6 +13,5 @@ public abstract class AbstractCloningTask extends AbstractGitTask {
@Schema(
title = "Whether to clone submodules."
)
@PluginProperty
protected Boolean cloneSubmodules;
protected Property<Boolean> cloneSubmodules;
}
35 changes: 11 additions & 24 deletions src/main/java/io/kestra/plugin/git/AbstractGitTask.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.kestra.plugin.git;

import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.runners.RunContext;
import io.kestra.plugin.git.services.SshTransportConfigCallback;
Expand All @@ -18,68 +18,55 @@
@NoArgsConstructor
@Getter
public abstract class AbstractGitTask extends Task {
private static final Pattern PEBBLE_TEMPLATE_PATTERN = Pattern.compile("^\\s*\\{\\{");

@Schema(
title = "The URI to clone from."
)
@PluginProperty(dynamic = true)
protected String url;
protected Property<String> url;

@Schema(
title = "The username or organization."
)
@PluginProperty(dynamic = true)
protected String username;
protected Property<String> username;

@Schema(
title = "The password or Personal Access Token (PAT). When you authenticate the task with a PAT, any flows or files pushed to Git from Kestra will be pushed from the user associated with that PAT. This way, you don't need to configure the commit author (the `authorName` and `authorEmail` properties)."
)
@PluginProperty(dynamic = true)
protected String password;
protected Property<String> password;

@Schema(
title = "PEM-format private key content that is paired with a public key registered on Git.",
description = "To generate an ECDSA PEM format key from OpenSSH, use the following command: `ssh-keygen -t ecdsa -b 256 -m PEM`. " +
"You can then set this property with your private key content and put your public key on Git."
)
@PluginProperty(dynamic = true)
protected String privateKey;
protected Property<String> privateKey;

@Schema(
title = "The passphrase for the `privateKey`."
)
@PluginProperty(dynamic = true)
protected String passphrase;
protected Property<String> passphrase;


@Schema(
title = "The initial Git branch."
)
@PluginProperty(dynamic = true)
public abstract String getBranch();
public abstract Property<String> getBranch();

public <T extends TransportCommand<T, ?>> T authentified(T command, RunContext runContext) throws Exception {
if (this.username != null && this.password != null) {
command.setCredentialsProvider(new UsernamePasswordCredentialsProvider(
runContext.render(this.username),
runContext.render(this.password)
runContext.render(this.username).as(String.class).orElseThrow(),
runContext.render(this.password).as(String.class).orElseThrow()
));
}

if (this.privateKey != null) {
command.setTransportConfigCallback(new SshTransportConfigCallback(
runContext.render(this.privateKey).getBytes(StandardCharsets.UTF_8),
runContext.render(this.passphrase)
runContext.render(this.privateKey).as(String.class).orElseThrow().getBytes(StandardCharsets.UTF_8),
runContext.render(this.passphrase).as(String.class).orElse(null)
));
}

return command;
}

protected void detectPasswordLeaks() {
if (this.password != null && !PEBBLE_TEMPLATE_PATTERN.matcher(this.password).find()) {
throw new IllegalArgumentException("It looks like you have hard-coded Git credentials. Make sure to pass the credential securely using a Pebble expression (e.g. using secrets or environment variables).");
}
}
}
48 changes: 24 additions & 24 deletions src/main/java/io/kestra/plugin/git/AbstractPushTask.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.kestra.plugin.git;

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
Expand Down Expand Up @@ -46,41 +46,39 @@
@NoArgsConstructor
@Getter
public abstract class AbstractPushTask<O extends AbstractPushTask.Output> extends AbstractCloningTask implements RunnableTask<O> {
@PluginProperty(dynamic = true)
protected String commitMessage;
protected Property<String> commitMessage;

@Schema(
title = "If `true`, the task will only output modifications without pushing any file to Git yet. If `false` (default), all listed files will be pushed to Git immediately."
)
@PluginProperty
@Builder.Default
private boolean dryRun = false;
private Property<Boolean> dryRun = Property.of(false);

@Schema(
title = "The commit author email.",
description = "If null, no author will be set on this commit."
)
@PluginProperty(dynamic = true)
private String authorEmail;
private Property<String> authorEmail;

@Schema(
title = "The commit author name.",
description = "If null, the username will be used instead.",
defaultValue = "`username`"
)
@PluginProperty(dynamic = true)
private String authorName;
private Property<String> authorName;

public abstract String getCommitMessage();
public String renderCommitMessage(RunContext runContext) throws IllegalVariableEvaluationException {
return runContext.render(this.getCommitMessage()).as(String.class).orElseThrow();
}

public abstract String getGitDirectory();
public abstract Property<String> getGitDirectory();

public abstract Object globs();

public abstract String fetchedNamespace();
public abstract Property<String> fetchedNamespace();

private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException {
Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory())));
Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElseThrow()));
flowDirectory.toFile().mkdirs();
return flowDirectory;
}
Expand Down Expand Up @@ -130,14 +128,14 @@ protected void writeResourceFile(Path path, InputStream inputStream) throws IOEx
Files.copy(inputStream, path, REPLACE_EXISTING);
}

private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException {
private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException, IllegalVariableEvaluationException {
File diffFile = runContext.workingDir().createTempFile(".ion").toFile();
try (DiffFormatter diffFormatter = new DiffFormatter(null);
BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) {
diffFormatter.setRepository(git.getRepository());

DiffCommand diff = git.diff();
if (this.dryRun) {
if (runContext.render(this.dryRun).as(Boolean.class).orElse(false)) {
diff = diff.setCached(true);
} else {
diff = diff.setOldTree(treeIterator(git, "HEAD~1"))
Expand Down Expand Up @@ -204,21 +202,21 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw
String commitId = null;
ObjectId commit;
try {
String httpUrl = gitService.getHttpUrl(runContext.render(this.url));
if (this.isDryRun()) {
String httpUrl = gitService.getHttpUrl(runContext.render(this.url).as(String.class).orElse(null));
if (runContext.render(dryRun).as(Boolean.class).orElse(false)) {
logger.info(
"Dry run — no changes will be pushed to {} for now until you set the `dryRun` parameter to false",
httpUrl
);
} else {
String renderedBranch = runContext.render(this.getBranch());
String renderedBranch = runContext.render(this.getBranch()).as(String.class).orElse(null);
logger.info(
"Pushing to {} on branch {}",
httpUrl,
renderedBranch
);

String message = runContext.render(this.getCommitMessage());
String message = this.renderCommitMessage(runContext);
ObjectId head = git.getRepository().resolve(Constants.HEAD);
commit = git.commit()
.setAllowEmpty(false)
Expand Down Expand Up @@ -247,8 +245,9 @@ private Output push(Git git, RunContext runContext, GitService gitService) throw
}

private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluationException {
String name = Optional.ofNullable(this.authorName).orElse(runContext.render(this.username));
String authorEmail = this.authorEmail;
String name = runContext.render(this.authorName).as(String.class)
.orElse(runContext.render(this.username).as(String.class).orElse(null));
String authorEmail = runContext.render(this.authorEmail).as(String.class).orElse(null);
if (authorEmail == null || name == null) {
return null;
}
Expand All @@ -274,9 +273,10 @@ public O run(RunContext runContext) throws Exception {
GitService gitService = new GitService(this);

gitService.namespaceAccessGuard(runContext, this.fetchedNamespace());
this.detectPasswordLeaks();

Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules);
Git git = gitService.cloneBranch(runContext,
runContext.render(this.getBranch()).as(String.class).orElse(null),
runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false));

Path localGitDirectory = this.createGitDirectory(runContext);

Expand All @@ -291,7 +291,7 @@ public O run(RunContext runContext) throws Exception {
this.writeResourceFiles(contentByPath);

AddCommand add = git.add();
add.addFilepattern(runContext.render(this.getGitDirectory()));
add.addFilepattern(runContext.render(this.getGitDirectory()).as(String.class).orElse(null));
add.call();

Output pushOutput = this.push(git, runContext, gitService);
Expand Down
26 changes: 14 additions & 12 deletions src/main/java/io/kestra/plugin/git/AbstractSyncTask.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
Expand Down Expand Up @@ -43,18 +44,17 @@ public abstract class AbstractSyncTask<T, O extends AbstractSyncTask.Output> ext
@Schema(
title = "If `true`, the task will only output modifications without performing any modification to Kestra. If `false` (default), all listed modifications will be applied."
)
@PluginProperty
@Builder.Default
private boolean dryRun = false;
private Property<Boolean> dryRun = Property.of(false);

public abstract boolean isDelete();

public abstract String getGitDirectory();
public abstract Property<String> getGitDirectory();

public abstract String fetchedNamespace();
public abstract Property<String> fetchedNamespace();

private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException {
Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory())));
Path syncDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory()).as(String.class).orElseThrow()));
syncDirectory.toFile().mkdirs();
return syncDirectory;
}
Expand All @@ -81,7 +81,7 @@ protected Map<URI, Supplier<InputStream>> gitResourcesContentByUri(Path baseDire
protected boolean traverseDirectories() {
return true;
}

protected boolean mustKeep(RunContext runContext, T instanceResource) {
return false;
}
Expand All @@ -100,7 +100,7 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map<
try (BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) {
List<SyncResult> syncResults = new ArrayList<>();

String renderedGitDirectory = runContext.render(this.getGitDirectory());
String renderedGitDirectory = runContext.render(this.getGitDirectory()).as(String.class).orElse(null);
if (deletedResources != null) {
deletedResources.stream()
.map(throwFunction(deletedResource -> wrapper(
Expand Down Expand Up @@ -146,17 +146,19 @@ private URI createDiffFile(RunContext runContext, String renderedNamespace, Map<
}

public O run(RunContext runContext) throws Exception {
this.detectPasswordLeaks();
GitService gitService = new GitService(this);

gitService.namespaceAccessGuard(runContext, this.fetchedNamespace());

Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules);
Git git = gitService.cloneBranch(runContext,
runContext.render(this.getBranch()).as(String.class).orElse(null),
runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false)
);

Path localGitDirectory = this.createGitDirectory(runContext);
Map<URI, Supplier<InputStream>> gitContentByUri = this.gitResourcesContentByUri(localGitDirectory);

String renderedNamespace = runContext.render(this.fetchedNamespace());
String renderedNamespace = runContext.render(this.fetchedNamespace()).as(String.class).orElse(null);

Map<URI, T> beforeUpdateResourcesByUri = this.fetchResources(runContext, renderedNamespace)
.stream()
Expand All @@ -170,7 +172,7 @@ public O run(RunContext runContext) throws Exception {
.map(throwFunction(e -> {
InputStream inputStream = e.getValue().get();
T resource;
if (this.dryRun) {
if (runContext.render(dryRun).as(Boolean.class).orElseThrow()) {
resource = this.simulateResourceWrite(runContext, renderedNamespace, e.getKey(), inputStream);
} else {
resource = this.writeResource(runContext, renderedNamespace, e.getKey(), inputStream);
Expand Down Expand Up @@ -198,7 +200,7 @@ public O run(RunContext runContext) throws Exception {
return;
}

if (!this.dryRun) {
if (!runContext.render(dryRun).as(Boolean.class).orElseThrow()) {
this.deleteResource(runContext, renderedNamespace, e.getValue());
}

Expand Down
13 changes: 7 additions & 6 deletions src/main/java/io/kestra/plugin/git/Clone.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.swagger.v3.oas.annotations.media.Schema;
Expand Down Expand Up @@ -78,7 +79,7 @@
code = """
id: git_python
namespace: company.team
tasks:
- id: file_system
type: io.kestra.plugin.core.flow.WorkingDirectory
Expand All @@ -105,7 +106,7 @@ public class Clone extends AbstractCloningTask implements RunnableTask<Clone.Out
@PluginProperty(dynamic = true)
private String directory;

private String branch;
private Property<String> branch;

@Schema(
title = "Creates a shallow clone with a history truncated to the specified number of commits."
Expand All @@ -118,7 +119,7 @@ public class Clone extends AbstractCloningTask implements RunnableTask<Clone.Out
@Override
public Clone.Output run(RunContext runContext) throws Exception {
Logger logger = runContext.logger();
String url = runContext.render(this.url);
String url = runContext.render(this.url).as(String.class).orElse(null);

Path path = runContext.workingDir().path();
if (this.directory != null) {
Expand All @@ -131,15 +132,15 @@ public Clone.Output run(RunContext runContext) throws Exception {
.setDirectory(path.toFile());

if (this.branch != null) {
cloneCommand.setBranch(runContext.render(this.branch));
cloneCommand.setBranch(runContext.render(this.branch).as(String.class).orElseThrow());
}

if (this.depth != null) {
cloneCommand.setDepth(this.depth);
}

if (this.cloneSubmodules != null) {
cloneCommand.setCloneSubmodules(this.cloneSubmodules);
cloneCommand.setCloneSubmodules(runContext.render(this.cloneSubmodules).as(Boolean.class).orElse(false));
}

cloneCommand = authentified(cloneCommand, runContext);
Expand All @@ -155,7 +156,7 @@ public Clone.Output run(RunContext runContext) throws Exception {

@Override
@NotNull
public String getUrl() {
public Property<String> getUrl() {
return super.getUrl();
}

Expand Down
Loading

0 comments on commit 77dadaf

Please sign in to comment.