Upgrade to rules_js and ts-proto

.aspect/rules/external_repository_action_cache/npm_translate_lock_LTE4Nzc1MDcwNjU=

@@ -0,0 +1,4 @@
+# Input hashes for repository rule npm_translate_lock(name = "npm", pnpm_lock = "//:pnpm-lock.yaml").
+# This file should be checked into version control along with the pnpm-lock.yaml file.
+pnpm-lock.yaml=374681099
+package.json=-1895465316

.gitignore

@@ -26,3 +26,5 @@ lerna-debug.log
 go.sum
 
 .df-credentials.json
+
+.aspect/

BUILD

@@ -1,71 +1,31 @@
+load("@aspect_rules_js//js:defs.bzl", "js_binary")
+load("@bazel_gazelle//:def.bzl", "gazelle")
+load("@npm//:defs.bzl", "npm_link_all_packages")
+load("@aspect_rules_ts//ts:defs.bzl", "ts_config")
+
 package(default_visibility = ["//visibility:public"])
 
-load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
+npm_link_all_packages(name = "node_modules")
 
 exports_files([
-    "tsconfig.json",
     "package.json",
     "readme.md",
     "version.bzl",
 ])
 
-PROTOBUF_DEPS = [
-    "@npm//protobufjs",
-    "@npm//protobufjs-cli",
-    # these deps are needed even though they are not automatic transitive deps of
-    # protobufjs since if they are not in the runfiles then protobufjs attempts to
-    # run `npm install` at runtime to get thhem which fails as it tries to access
-    # the npm cache outside of the sandbox
-    "@npm//semver",
-    "@npm//chalk",
-    "@npm//glob",
-    "@npm//jsdoc",
-    "@npm//minimist",
-    "@npm//tmp",
-    "@npm//uglify-js",
-    "@npm//uglify-es",
-    "@npm//espree",
-    "@npm//escodegen",
-    "@npm//estraverse",
-]
-
-nodejs_binary(
-    name = "pbjs",
-    data = PROTOBUF_DEPS,
-    entry_point = "@npm//:node_modules/protobufjs-cli/bin/pbjs",
-    install_source_map_support = False,
-)
-
-nodejs_binary(
-    name = "pbts",
-    data = PROTOBUF_DEPS,
-    entry_point = "@npm//:node_modules/protobufjs-cli/bin/pbts",
-    install_source_map_support = False,
+ts_config(
+    name = "tsconfig",
+    src = "tsconfig.json",
 )
 
-nodejs_binary(
-    name = "tslint",
-    data = [
-        "@npm//tslint",
-    ],
-    entry_point = "@npm//:node_modules/tslint/bin/tslint",
-    install_source_map_support = False,
-    templated_args = ["--node_options=--preserve-symlinks"],
+js_binary(
+    name = "run_tslint",
+    entry_point = "//:node_modules/tslint/bin/tslint",
+    node_options = ["--preserve-symlinks"],
+    # install_source_map_support = False,
 )
 
-load("@bazel_gazelle//:def.bzl", "gazelle")
-
 # gazelle:prefix github.com/dataform-co/dataform
 # gazelle:proto package
 # gazelle:proto_group go_package
 gazelle(name = "gazelle")
-
-load("//tools:ts_library.bzl", "ts_library")
-
-# TODO: This is only here in order to workaround a bug in the way bazel resolves
-# workspace imports when in nested repositories, and can be removed once that is fixed.
-ts_library(
-    name = "modules-fix",
-    srcs = [],
-    module_name = "df",
-)

WORKSPACE

@@ -1,6 +1,7 @@
 workspace(name = "df")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 http_archive(
     name = "bazel_skylib",
@@ -33,39 +34,130 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
 
+########
+
+# http_archive(
+#     name = "rules_proto_grpc",
+#     sha256 = "7954abbb6898830cd10ac9714fbcacf092299fda00ed2baf781172f545120419",
+#     strip_prefix = "rules_proto_grpc-3.1.1",
+#     urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/3.1.1.tar.gz"],
+# )
+
+# TODO: We need `base_env` added (used in //tools/ts_proto_library), so have to pull from a recent commit sha.
+# When avaialble, update to rules_proto_grpc 4.4.0 when available, and undo custom version.
+git_repository(
+    name = "rules_proto_grpc",
+    commit = "e9b7119d712c8e1f4159a10893b2752b38d1b664",
+    remote = "https://github.com/rules-proto-grpc/rules_proto_grpc.git",
+)
+
+load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")
+
+rules_proto_grpc_toolchains()
+
+rules_proto_grpc_repos()
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
+rules_proto_dependencies()
+
+rules_proto_toolchains()
+
 http_archive(
     name = "build_bazel_rules_nodejs",
-    sha256 = "d14076339deb08e5460c221fae5c5e9605d2ef4848eee1f0c81c9ffdc1ab31c1",
-    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/1.6.1/rules_nodejs-1.6.1.tar.gz"],
+    sha256 = "dcc55f810142b6cf46a44d0180a5a7fb923c04a5061e2e8d8eb05ccccc60864b",
+    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.8.0/rules_nodejs-5.8.0.tar.gz"],
 )
 
-load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install")
+# # or for core module
+# http_archive(
+#     name = "rules_nodejs",
+#     sha256 = "08337d4fffc78f7fe648a93be12ea2fc4e8eb9795a4e6aa48595b66b34555626",
+#     urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.8.0/rules_nodejs-core-5.8.0.tar.gz"],
+# )
 
-node_repositories(
-    node_repositories = {
-        "16.16.0-darwin_amd64": ("node-v16.16.0-darwin-x64.tar.gz", "node-v16.16.0-darwin-x64", "982edd0fad364ad6e2d72161671544ab9399bd0ca8c726bde3cd07775c4c709a"),
-        "16.16.0-linux_amd64": ("node-v16.16.0-linux-x64.tar.xz", "node-v16.16.0-linux-x64", "edcb6e9bb049ae365611aa209fc03c4bfc7e0295dbcc5b2f1e710ac70384a8ec"),
-        "16.16.0-windows_amd64": ("node-v16.16.0-win-x64.zip", "node-v16.16.0-win-x64", "c657acc98af55018c8fd6113c7e08d67c8083af75ba0306f9561b0117abc39d4"),
-    },
-    node_version = "16.16.0",
-    package_json = ["//:package.json"],
-    yarn_version = "1.13.0",
+http_archive(
+    name = "aspect_rules_js",
+    sha256 = "ad666b12324bab8bc151772bb2eff9aadace7bfd4d624157c2ac3931860d1c95",
+    strip_prefix = "rules_js-1.11.1",
+    url = "https://github.com/aspect-build/rules_js/archive/refs/tags/v1.11.1.tar.gz",
 )
 
-yarn_install(
+load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies")
+
+rules_js_dependencies()
+
+http_archive(
+    name = "aspect_rules_ts",
+    sha256 = "e81f37c4fe014fc83229e619360d51bfd6cb8ac405a7e8018b4a362efa79d000",
+    strip_prefix = "rules_ts-1.0.4",
+    url = "https://github.com/aspect-build/rules_ts/archive/refs/tags/v1.0.4.tar.gz",
+)
+
+load("@aspect_rules_ts//ts:repositories.bzl", "rules_ts_dependencies")
+
+rules_ts_dependencies(
+    ts_version_from = "//:package.json",
+)
+
+load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains")
+
+nodejs_register_toolchains(
+    name = "nodejs",
+    node_version = DEFAULT_NODE_VERSION,
+)
+
+load("@aspect_rules_js//npm:npm_import.bzl", "npm_translate_lock")
+
+npm_translate_lock(
     name = "npm",
-    package_json = "//:package.json",
-    symlink_node_modules = False,
-    yarn_lock = "//:yarn.lock",
+    data = [
+        "//:package.json",
+    ],
+    pnpm_lock = "//:pnpm-lock.yaml",
+    update_pnpm_lock = True,
+    verify_node_modules_ignored = "//:.bazelignore",
 )
 
-load("@npm//:install_bazel_dependencies.bzl", "install_bazel_dependencies")
+load("@npm//:repositories.bzl", "npm_repositories")
+
+npm_repositories()
+
+#########
+
+# http_archive(
+#     name = "rules_nodejs",
+#     sha256 = "08337d4fffc78f7fe648a93be12ea2fc4e8eb9795a4e6aa48595b66b34555626",
+#     urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.8.0/rules_nodejs-core-5.8.0.tar.gz"],
+# )
+
+# load("@rules_nodejs//:index.bzl", "node_repositories", "yarn_install")
+
+# node_repositories(
+#     node_repositories = {
+#         "16.16.0-darwin_amd64": ("node-v16.16.0-darwin-x64.tar.gz", "node-v16.16.0-darwin-x64", "982edd0fad364ad6e2d72161671544ab9399bd0ca8c726bde3cd07775c4c709a"),
+#         "16.16.0-linux_amd64": ("node-v16.16.0-linux-x64.tar.xz", "node-v16.16.0-linux-x64", "edcb6e9bb049ae365611aa209fc03c4bfc7e0295dbcc5b2f1e710ac70384a8ec"),
+#         "16.16.0-windows_amd64": ("node-v16.16.0-win-x64.zip", "node-v16.16.0-win-x64", "c657acc98af55018c8fd6113c7e08d67c8083af75ba0306f9561b0117abc39d4"),
+#     },
+#     node_version = "16.16.0",
+#     package_json = ["//:package.json"],
+#     yarn_version = "1.13.0",
+# )
+
+# yarn_install(
+#     name = "npm",
+#     package_json = "//:package.json",
+#     symlink_node_modules = False,
+#     yarn_lock = "//:yarn.lock",
+# )
+
+# load("@npm//:install_bazel_dependencies.bzl", "install_bazel_dependencies")
 
-install_bazel_dependencies()
+# install_bazel_dependencies()
 
-load("@npm_bazel_typescript//:index.bzl", "ts_setup_workspace")
+# load("@npm_bazel_typescript//:index.bzl", "ts_setup_workspace")
 
-ts_setup_workspace()
+# ts_setup_workspace()
 
 # Go/Gazelle requirements/dependencies.
 http_archive(

api/BUILD

@@ -19,7 +19,7 @@ ts_library(
         "//protos:ts",
         "//sandbox/vm:compile_loader",
         "//sqlx",
-        "//sandbox/vm:vm",
+        "//sandbox/vm",
         "@npm//@google-cloud/bigquery",
         "@npm//@types/glob",
         "@npm//@types/js-beautify",
@@ -45,7 +45,7 @@ ts_library(
         "@npm//presto-client",
         "@npm//pretty-ms",
         "@npm//promise-pool-executor",
-        "@npm//protobufjs",
+        # "@npm//protobufjs",
         "@npm//semver",
         "@npm//snowflake-sdk",
         "@npm//sql-formatter",

api/commands/build.ts

@@ -5,11 +5,12 @@ import { StringifiedMap, StringifiedSet } from "df/common/strings/stringifier";
 import { adapters } from "df/core";
 import { targetStringifier } from "df/core/targets";
 import * as utils from "df/core/utils";
-import { dataform } from "df/protos/ts";
+import * as core from "df/protos/core";
+import * as execution from "df/protos/execution";
 
 export async function build(
-  compiledGraph: dataform.ICompiledGraph,
-  runConfig: dataform.IRunConfig,
+  compiledGraph: core.CompiledGraph,
+  runConfig: execution.RunConfig,
   dbadapter: dbadapters.IDbAdapter
 ) {
   runConfig = {
@@ -19,7 +20,7 @@ export async function build(
 
   const prunedGraph = prune(compiledGraph, runConfig);
 
-  const allInvolvedTargets = new StringifiedSet<dataform.ITarget>(
+  const allInvolvedTargets = new StringifiedSet<core.Target>(
     targetStringifier,
     prunedGraph.tables.map(table => table.target)
   );
@@ -44,9 +45,9 @@ export class Builder {
   private readonly adapter: adapters.IAdapter;
 
   constructor(
-    private readonly prunedGraph: dataform.ICompiledGraph,
-    private readonly runConfig: dataform.IRunConfig,
-    private readonly warehouseState: dataform.IWarehouseState
+    private readonly prunedGraph: core.CompiledGraph,
+    private readonly runConfig: execution.RunConfig,
+    private readonly warehouseState: dataform.WarehouseState
   ) {
     this.adapter = adapters.create(
       prunedGraph.projectConfig,
@@ -60,14 +61,14 @@ export class Builder {
       throw new Error(`Project has unresolved compilation or validation errors.`);
     }
 
-    const tableMetadataByTarget = new StringifiedMap<dataform.ITarget, dataform.ITableMetadata>(
+    const tableMetadataByTarget = new StringifiedMap<core.Target, execution.TableMetadata>(
       targetStringifier
     );
     this.warehouseState.tables.forEach(tableState => {
       tableMetadataByTarget.set(tableState.target, tableState);
     });
 
-    const actions: dataform.IExecutionAction[] = [].concat(
+    const actions: dataform.ExecutionAction[] = [].concat(
       this.prunedGraph.tables.map(t =>
         this.buildTable(t, tableMetadataByTarget.get(t.target), this.runConfig)
       ),
@@ -84,9 +85,9 @@ export class Builder {
   }
 
   private buildTable(
-    table: dataform.ITable,
-    tableMetadata: dataform.ITableMetadata,
-    runConfig: dataform.IRunConfig
+    table: core.Target,
+    tableMetadata: execution.TableMetadata,
+    runConfig: execution.RunConfig
   ) {
     if (table.protected && this.runConfig.fullRefresh) {
       throw new Error("Protected datasets cannot be fully refreshed.");
@@ -99,35 +100,33 @@ export class Builder {
       tasks: table.disabled
         ? []
         : this.adapter.publishTasks(table, runConfig, tableMetadata).build(),
-      hermeticity: table.hermeticity || dataform.ActionHermeticity.HERMETIC
+      hermeticity: table.hermeticity || core.ActionHermeticity.HERMETIC
     };
   }
 
-  private buildOperation(operation: dataform.IOperation) {
+  private buildOperation(operation: core.Operation) {
     return {
       ...this.toPartialExecutionAction(operation),
       type: "operation",
       tasks: operation.disabled
         ? []
         : operation.queries.map(statement => ({ type: "statement", statement })),
-      hermeticity: operation.hermeticity || dataform.ActionHermeticity.NON_HERMETIC
+      hermeticity: operation.hermeticity || core.ActionHermeticity.NON_HERMETIC
     };
   }
 
-  private buildAssertion(assertion: dataform.IAssertion) {
+  private buildAssertion(assertion: core.Assertion) {
     return {
       ...this.toPartialExecutionAction(assertion),
       type: "assertion",
       tasks: assertion.disabled
         ? []
         : this.adapter.assertTasks(assertion, this.prunedGraph.projectConfig).build(),
-      hermeticity: assertion.hermeticity || dataform.ActionHermeticity.HERMETIC
+      hermeticity: assertion.hermeticity || core.ActionHermeticity.HERMETIC
     };
   }
 
-  private toPartialExecutionAction(
-    action: dataform.ITable | dataform.IOperation | dataform.IAssertion
-  ) {
+  private toPartialExecutionAction(action: core.Target | core.Operation | core.Assertion) {
     return dataform.ExecutionAction.create({
       target: action.target,
       fileName: action.fileName,