build: perform package.json substitutions in bazel build

kormide · dgp1130 · commit 4b5c52b0d822 · 2022-01-10T10:25:49.000-08:00
diff --git a/.bazelrc b/.bazelrc
@@ -74,6 +74,8 @@ build:release --stamp
 build:snapshot --workspace_status_command="yarn -s ng-dev release build-env-stamp --mode=snapshot"
 build:snapshot --stamp
 
+build:local --//:enable_package_json_tar_deps
+
 ###############################
 # Output                      #
 ###############################
diff --git a/.prettierignore b/.prettierignore
@@ -12,4 +12,5 @@
 .yarn/
 dist/
 third_party/
-/tests/legacy-cli/e2e/assets/9.0-project/
+/tests/legacy-cli/e2e/assets/9.0-project/
+/tools/test/*.json
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -2,6 +2,8 @@
 #
 # Use of this source code is governed by an MIT-style license that can be
 # found in the LICENSE file at https://angular.io/license
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
@@ -11,10 +13,24 @@ exports_files([
     "tsconfig.json",  # @external
     "tsconfig-test.json",  # @external
     "tsconfig-build.json",  # @external
+    "package.json",
 ])
 
 # Detect if the build is running under --stamp
 config_setting(
     name = "stamp",
     values = {"stamp": "true"},
 )
+
+# If set will replace dependency versions with tarballs for packages in this repo
+bool_flag(
+    name = "enable_package_json_tar_deps",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "package_json_use_tar_deps",
+    flag_values = {
+        ":enable_package_json_tar_deps": "true",
+    },
+)
diff --git a/WORKSPACE b/WORKSPACE
@@ -5,6 +5,15 @@ workspace(
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
+    ],
+)
+
 http_archive(
     name = "build_bazel_rules_nodejs",
     sha256 = "cfc289523cf1594598215901154a6c2515e8bf3671fd708264a6f6aefe02bf39",
@@ -17,6 +26,10 @@ http_archive(
     urls = ["https://github.com/bazelbuild/rules_pkg/releases/download/0.5.1/rules_pkg-0.5.1.tar.gz"],
 )
 
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+
+bazel_skylib_workspace()
+
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 
 rules_pkg_dependencies()
@@ -50,3 +63,16 @@ yarn_install(
     package_json = "//:package.json",
     yarn_lock = "//:yarn.lock",
 )
+
+http_archive(
+    name = "aspect_bazel_lib",
+    sha256 = "534c9c61b72c257c95302d544984fd8ee63953c233292c5b6952ca5b33cd225e",
+    strip_prefix = "bazel-lib-0.4.2",
+    url = "https://github.com/aspect-build/bazel-lib/archive/v0.4.2.tar.gz",
+)
+
+load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "register_jq_toolchains")
+
+aspect_bazel_lib_dependencies()
+
+register_jq_toolchains(version = "1.6")
diff --git a/packages/angular/cli/BUILD.bazel b/packages/angular/cli/BUILD.bazel
@@ -316,7 +316,14 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+        "//packages/angular_devkit/build_angular:package.json",
+        "//packages/angular_devkit/build_webpack:package.json",
+        "//packages/angular_devkit/core:package.json",
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/schematics/angular:package.json",
+    ],
     deps = [
         ":README.md",
         ":angular-cli",
diff --git a/packages/angular/pwa/BUILD.bazel b/packages/angular/pwa/BUILD.bazel
@@ -76,7 +76,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/schematics/angular:package.json",
+    ],
     deps = [
         ":README.md",
         ":license",
diff --git a/packages/angular_devkit/architect/BUILD.bazel b/packages/angular_devkit/architect/BUILD.bazel
@@ -99,7 +99,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":architect",
diff --git a/packages/angular_devkit/architect_cli/BUILD.bazel b/packages/angular_devkit/architect_cli/BUILD.bazel
@@ -37,7 +37,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":architect_cli",
diff --git a/packages/angular_devkit/benchmark/BUILD.bazel b/packages/angular_devkit/benchmark/BUILD.bazel
@@ -78,7 +78,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         "src/test/exit-code-one.js",
         "src/test/fibonacci.js",
diff --git a/packages/angular_devkit/build_angular/BUILD.bazel b/packages/angular_devkit/build_angular/BUILD.bazel
@@ -219,7 +219,11 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+        "//packages/angular_devkit/build_webpack:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":build_angular",
diff --git a/packages/angular_devkit/build_optimizer/BUILD.bazel b/packages/angular_devkit/build_optimizer/BUILD.bazel
@@ -68,7 +68,6 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
     deps = [
         ":README.md",
         ":build_optimizer",
diff --git a/packages/angular_devkit/build_webpack/BUILD.bazel b/packages/angular_devkit/build_webpack/BUILD.bazel
@@ -109,7 +109,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/architect:package.json",
+    ],
     deps = [
         ":README.md",
         ":build_webpack",
diff --git a/packages/angular_devkit/core/BUILD.bazel b/packages/angular_devkit/core/BUILD.bazel
@@ -85,7 +85,6 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
     deps = [
         ":README.md",
         ":core",
diff --git a/packages/angular_devkit/schematics/BUILD.bazel b/packages/angular_devkit/schematics/BUILD.bazel
@@ -73,7 +73,9 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":collection-schema.json",
diff --git a/packages/angular_devkit/schematics_cli/BUILD.bazel b/packages/angular_devkit/schematics_cli/BUILD.bazel
@@ -98,7 +98,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         ":README.md",
         ":license",
diff --git a/packages/ngtools/webpack/BUILD.bazel b/packages/ngtools/webpack/BUILD.bazel
@@ -76,7 +76,6 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
     deps = [
         ":README.md",
         ":license",
diff --git a/packages/schematics/angular/BUILD.bazel b/packages/schematics/angular/BUILD.bazel
@@ -140,7 +140,10 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
-    srcs = [":package.json"],
+    pkg_deps = [
+        "//packages/angular_devkit/schematics:package.json",
+        "//packages/angular_devkit/core:package.json",
+    ],
     deps = [
         "library/library-long.md",
         ":README.md",
diff --git a/scripts/build-bazel.ts b/scripts/build-bazel.ts
@@ -113,13 +113,6 @@ async function _build(logger: logging.Logger, mode: BuildMode): Promise<string[]
   const queryTargetsCmd = `${bazelCmd} query --output=label "attr(name, npm_package_archive, //packages/...)"`;
   const targets = (await _exec(queryTargetsCmd, true, queryLogger)).split(/\r?\n/);
 
-  let configArg = '';
-  if (mode === 'snapshot') {
-    configArg = '--config=snapshot';
-  } else if (mode === 'release') {
-    configArg = '--config=release';
-  }
-
   const buildLogger = logger.createChild('build');
 
   // If we are in release mode, run `bazel clean` to ensure the execroot and action cache
@@ -132,7 +125,7 @@ async function _build(logger: logging.Logger, mode: BuildMode): Promise<string[]
     await _exec(`${bazelCmd} clean`, false, buildLogger);
   }
 
-  await _exec(`${bazelCmd} build ${configArg} ${targets.join(' ')}`, false, buildLogger);
+  await _exec(`${bazelCmd} build --config=${mode} ${targets.join(' ')}`, false, buildLogger);
 
   return targets;
 }
diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel
@@ -7,6 +7,10 @@ load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
 
 package(default_visibility = ["//visibility:public"])
 
+exports_files([
+    "package_json_release_filter.jq",
+])
+
 nodejs_binary(
     name = "ng_cli_schema",
     data = [
diff --git a/tools/defaults.bzl b/tools/defaults.bzl
@@ -1,9 +1,13 @@
 """Re-export of some bazel rules with repository-wide defaults."""
 
 load("@npm//@bazel/typescript:index.bzl", _ts_library = "ts_library")
-load("@build_bazel_rules_nodejs//:index.bzl", _pkg_npm = "pkg_npm")
+load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin", _pkg_npm = "pkg_npm")
 load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@npm//@angular/dev-infra-private/bazel:extract_js_module_output.bzl", "extract_js_module_output")
+load("@aspect_bazel_lib//lib:utils.bzl", "to_label")
+load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
+load("//tools:link_package_json_to_tarballs.bzl", "link_package_json_to_tarballs")
 
 _DEFAULT_TSCONFIG = "//:tsconfig-build.json"
 _DEFAULT_TSCONFIG_TEST = "//:tsconfig-test.json"
@@ -44,8 +48,20 @@ def ts_library(
         **kwargs
     )
 
-def pkg_npm(name, use_prodmode_output = False, **kwargs):
-    """Default values for pkg_npm"""
+def pkg_npm(name, pkg_deps = [], use_prodmode_output = False, **kwargs):
+    """Override of pkg_npm to produce package outputs and version substitutions conventional to the angular-cli project.
+
+    Produces a package and a tar of that package. Expects a package.json file
+    in the same folder to exist.
+
+    Args:
+        name: Name of the pkg_npm rule. '_archive.tar.gz' is appended to create the tarball.
+        pkg_deps: package.json files of dependent packages. These are used for local path substitutions when --config=local is set.
+        use_prodmode_output: False to ship ES5 devmode output, True to ship ESM output. Defaults to False.
+        **kwargs: Additional arguments passed to the real pkg_npm.
+    """
+    pkg_json = ":package.json"
+
     visibility = kwargs.pop("visibility", None)
 
     common_substitutions = dict(kwargs.pop("substitutions", {}))
@@ -79,6 +95,57 @@ def pkg_npm(name, use_prodmode_output = False, **kwargs):
         deps = deps,
     )
 
+    # Merge package.json with root package.json and perform various substitutions to
+    # prepare it for release. For jq docs, see https://stedolan.github.io/jq/manual/.
+    jq(
+        name = "basic_substitutions",
+        # Note: this jq filter relies on the order of the inputs
+        # buildifier: do not sort
+        srcs = ["//:package.json", pkg_json],
+        filter_file = "//tools:package_json_release_filter.jq",
+        args = ["--slurp"],
+        out = "substituted/package.json",
+    )
+
+    # Copy package.json files to bazel-out so we can use their bazel-out paths to determine
+    # the corresponding package npm package tar.gz path for substitutions.
+    copy_to_bin(
+        name = "package_json_copy",
+        srcs = [pkg_json],
+    )
+    pkg_deps_copies = []
+    for pkg_dep in pkg_deps:
+        pkg_label = to_label(pkg_dep)
+        if pkg_label.name != "package.json":
+            fail("ERROR: only package.json files allowed in pkg_deps of pkg_npm macro")
+        pkg_deps_copies.append("@%s//%s:package_json_copy" % (pkg_label.workspace_name, pkg_label.package))
+
+    # Substitute dependencies on other packages in this repo with tarballs.
+    link_package_json_to_tarballs(
+        name = "tar_substitutions",
+        src = "substituted/package.json",
+        pkg_deps = [":package_json_copy"] + pkg_deps_copies,
+        out = "substituted_with_tars/package.json",
+    )
+
+    # Move the generated package.json along with other deps into a directory for pkg_npm
+    # to package up because pkg_npm requires that all inputs be in the same directory.
+    copy_to_directory(
+        name = "package",
+        srcs = select({
+            # Do tar substitution if config_setting 'package_json_use_tar_deps' is true (local builds)
+            "//:package_json_use_tar_deps": [":%s_js_module_output" % name, "substituted_with_tars/package.json"],
+            "//conditions:default": [":%s_js_module_output" % name, "substituted/package.json"],
+        }),
+        replace_prefixes = {
+            "substituted_with_tars/": "",
+            "substituted/": "",
+        },
+        exclude_prefixes = [
+            "packages",  # Exclude compiled outputs of dependent packages
+        ],
+    )
+
     _pkg_npm(
         name = name,
         # We never set a `package_name` for NPM packages, neither do we enable validation.
@@ -101,7 +168,7 @@ def pkg_npm(name, use_prodmode_output = False, **kwargs):
             "//conditions:default": substitutions,
         }),
         visibility = visibility,
-        deps = [":%s_js_module_output" % name],
+        nested_packages = ["package"],
         tgz = None,
         **kwargs
     )
diff --git a/tools/link_package_json_to_tarballs.bzl b/tools/link_package_json_to_tarballs.bzl
diff --git a/tools/package_json_release_filter.jq b/tools/package_json_release_filter.jq
diff --git a/tools/test/BUILD.bazel b/tools/test/BUILD.bazel
diff --git a/tools/test/expected_package.json b/tools/test/expected_package.json
diff --git a/tools/test/project_package.json b/tools/test/project_package.json
diff --git a/tools/test/root_package.json b/tools/test/root_package.json
