Build with JDK 21 (#1556)

* Build with JDK 21

Add -Djava.security.manager=allow jvm flag to:
- tools executed via ctx.actions.run if host jdk is >= 17
- generated test executables if target jdk is >= 17

* allow security manager for coverage worker

* Add ci build with jdk21

* lint

* test all with jdk 21

* Add libxml

* Uncomment JunitRuntimePlatform test

Make sure compilation toolchain matches runtime target version

This test was failing with --java_language_version=21

Because it got compiled with newer jdk than was run.

Author: simuons

.bazelci/presubmit.yml

@@ -90,3 +90,15 @@ tasks:
     platform: ubuntu2004
     run_targets:
     -  "//tools:lint_check"
+  test_rules_scala_jdk21:
+    name: "./test_rules_scala with jdk21"
+    platform: ubuntu2004
+    shell_commands:
+      - sudo apt update && sudo apt install -y libxml2-utils
+      - mv tools/bazel.rc.buildkite tools/bazel.rc
+      - echo "import %workspace%/tools/bazel.rc" > .bazelrc
+      - echo "build --java_language_version=21" >> .bazelrc
+      - echo "build --java_runtime_version=21" >> .bazelrc
+      - echo "build --tool_java_language_version=21" >> .bazelrc
+      - echo "build --tool_java_runtime_version=21" >> .bazelrc
+      - "./test_rules_scala.sh"

WORKSPACE

@@ -203,3 +203,9 @@ repositories(
     ],
     maven_servers = MAVEN_SERVER_URLS,
 )
+
+load("//test/toolchains:jdk.bzl", "remote_jdk21_repositories", "remote_jdk21_toolchains")
+
+remote_jdk21_repositories()
+
+remote_jdk21_toolchains()

scala/private/common_attributes.bzl

@@ -84,6 +84,9 @@ implicit_deps = {
     "_java_runtime": attr.label(
         default = Label("@bazel_tools//tools/jdk:current_java_runtime"),
     ),
+    "_java_host_runtime": attr.label(
+        default = Label("@bazel_tools//tools/jdk:current_host_java_runtime"),
+    ),
     "_scalac": attr.label(
         executable = True,
         cfg = "exec",

scala/private/phases/phase_coverage.bzl

@@ -8,6 +8,10 @@ load(
     "@io_bazel_rules_scala//scala/private:coverage_replacements_provider.bzl",
     _coverage_replacements_provider = "coverage_replacements_provider",
 )
+load(
+    "@io_bazel_rules_scala//scala/private:rule_impls.bzl",
+    _allow_security_manager = "allow_security_manager",
+)
 
 def phase_coverage_library(ctx, p):
     args = struct(
@@ -60,7 +64,7 @@ def _phase_coverage(ctx, p, srcjars):
             outputs = [output_jar],
             executable = ctx.attr._code_coverage_instrumentation_worker.files_to_run,
             execution_requirements = {"supports-workers": "1"},
-            arguments = [args],
+            arguments = ["--jvm_flag=%s" % f for f in _allow_security_manager(ctx)] + [args],
         )
 
         replacements = {input_jar: output_jar}

scala/private/phases/phase_scalafmt.bzl

@@ -7,6 +7,10 @@ load(
     "@io_bazel_rules_scala//scala/private:paths.bzl",
     _scala_extension = "scala_extension",
 )
+load(
+    "//scala/private:rule_impls.bzl",
+    _allow_security_manager = "allow_security_manager",
+)
 
 def phase_scalafmt(ctx, p):
     if ctx.attr.format:
@@ -25,14 +29,15 @@ def _build_format(ctx):
     files = []
     srcs = []
     manifest_content = []
+    jvm_flags = ["-Dfile.encoding=UTF-8"] + _allow_security_manager(ctx)
     for src in ctx.files.srcs:
         # only format scala source files, not generated files
         if src.path.endswith(_scala_extension) and src.is_source:
             srcs.append(src)
             file = ctx.actions.declare_file("{}.fmt.output".format(src.short_path))
             files.append(file)
             ctx.actions.run(
-                arguments = ["--jvm_flag=-Dfile.encoding=UTF-8", _format_args(ctx, src, file)],
+                arguments = ["--jvm_flag=%s" % f for f in jvm_flags] + [_format_args(ctx, src, file)],
                 executable = ctx.executable._fmt,
                 outputs = [file],
                 inputs = [ctx.file.config, src],

scala/private/phases/phase_write_executable.bzl

@@ -5,6 +5,7 @@
 #
 load(
     "@io_bazel_rules_scala//scala/private:rule_impls.bzl",
+    "allow_security_manager",
     "expand_location",
     "first_non_empty",
     "is_windows",
@@ -22,14 +23,12 @@ def phase_write_executable_scalatest(ctx, p):
         ctx.toolchains["@io_bazel_rules_scala//scala:toolchain_type"].scala_test_jvm_flags,
     )
 
-    expanded_jvm_flags = [
-        "-DRULES_SCALA_MAIN_WS_NAME=%s" % ctx.workspace_name,
-        "-DRULES_SCALA_ARGS_FILE=%s" % p.runfiles.args_file.short_path.replace("../", "external/"),
-    ] + expand_location(ctx, final_jvm_flags)
-
     args = struct(
         rjars = p.coverage_runfiles.rjars,
-        jvm_flags = _allow_security_manager(ctx, expanded_jvm_flags),
+        jvm_flags = [
+            "-DRULES_SCALA_MAIN_WS_NAME=%s" % ctx.workspace_name,
+            "-DRULES_SCALA_ARGS_FILE=%s" % p.runfiles.args_file.short_path.replace("../", "external/"),
+        ] + expand_location(ctx, final_jvm_flags) + _allow_security_manager_for_specified_java_runtime(ctx),
         use_jacoco = ctx.configuration.coverage_enabled,
     )
     return _phase_write_executable_default(ctx, p, args)
@@ -44,7 +43,7 @@ def phase_write_executable_repl(ctx, p):
 def phase_write_executable_junit_test(ctx, p):
     args = struct(
         rjars = p.coverage_runfiles.rjars,
-        jvm_flags = _allow_security_manager(ctx, p.jvm_flags + ctx.attr.jvm_flags),
+        jvm_flags = p.jvm_flags + ctx.attr.jvm_flags + _allow_security_manager_for_specified_java_runtime(ctx),
         main_class = "com.google.testing.junit.runner.BazelTestRunner",
         use_jacoco = ctx.configuration.coverage_enabled,
     )
@@ -187,7 +186,12 @@ def _jar_path_based_on_java_bin(ctx):
     jar_path = java_bin_var.rpartition("/")[0] + "/jar"
     return jar_path
 
-def _allow_security_manager(ctx, jvm_flags):
-    java_toolchain = ctx.toolchains["@bazel_tools//tools/jdk:toolchain_type"]
-    java_runtime = specified_java_runtime(ctx, java_toolchain.java.java_runtime if java_toolchain else None)
-    return jvm_flags + (["-Djava.security.manager=allow"] if java_runtime and java_runtime.version >= 17 else [])
+# Allow security manager for generated test executables if they will be run with jdk >= 17
+def _allow_security_manager_for_specified_java_runtime(ctx):
+    return allow_security_manager(
+        ctx,
+        specified_java_runtime(
+            ctx,
+            default_runtime = ctx.attr._java_runtime[java_common.JavaRuntimeInfo],
+        ),
+    )

scala/private/rule_impls.bzl

@@ -121,7 +121,10 @@ def compile_scala(
     )
 
     # scalac_jvm_flags passed in on the target override scalac_jvm_flags passed in on the toolchain
-    final_scalac_jvm_flags = first_non_empty(scalac_jvm_flags, toolchain.scalac_jvm_flags)
+    # TODO: scalac worker is run with @bazel_tools//tools/jdk:runtime_toolchain_type
+    # which is different from rules_java where compilation runtime is used from
+    # @bazel_tools//tools/jdk:toolchain_type
+    final_scalac_jvm_flags = first_non_empty(scalac_jvm_flags, toolchain.scalac_jvm_flags) + allow_security_manager(ctx)
 
     ctx.actions.run(
         inputs = ins,
@@ -219,3 +222,10 @@ def java_bin_windows(ctx):
 
 def is_windows(ctx):
     return ctx.configuration.host_path_separator == ";"
+
+# Return a jvm flag allowing security manager for jdk runtime >= 17
+# If no runtime is supplied then runtime is taken from ctx.attr._java_host_runtime
+# This must be a runtime used in generated java_binary script (usually workers using SecurityManager)
+def allow_security_manager(ctx, runtime = None):
+    java_runtime = runtime if runtime else ctx.attr._java_host_runtime[java_common.JavaRuntimeInfo]
+    return ["-Djava.security.manager=allow"] if java_runtime.version >= 17 else []

scala/scalafmt/phase_scalafmt_ext.bzl

@@ -23,6 +23,9 @@ ext_scalafmt = {
             default = "//scala/scalafmt",
             executable = True,
         ),
+        "_java_host_runtime": attr.label(
+            default = Label("@bazel_tools//tools/jdk:current_host_java_runtime"),
+        ),
         "_runner": attr.label(
             allow_single_file = True,
             default = "//scala/scalafmt:runner",

scala_proto/private/scala_proto_aspect.bzl

@@ -5,6 +5,7 @@ load(
     "//scala/private:rule_impls.bzl",
     "compile_scala",
     "specified_java_compile_toolchain",
+    _allow_security_manager = "allow_security_manager",
 )
 load("//scala/private/toolchain_deps:toolchain_deps.bzl", "find_deps_info_on")
 load(
@@ -73,7 +74,7 @@ def _generate_sources(ctx, toolchain, proto):
 
     ctx.actions.run(
         executable = toolchain.worker,
-        arguments = [toolchain.worker_flags, args],
+        arguments = ["--jvm_flag=%s" % f for f in _allow_security_manager(ctx)] + [toolchain.worker_flags, args],
         inputs = depset(transitive = [descriptors, toolchain.generators_jars]),
         outputs = outputs.values(),
         tools = [toolchain.protoc],
@@ -203,6 +204,9 @@ def make_scala_proto_aspect(*extras):
         "_java_toolchain": attr.label(
             default = Label("@bazel_tools//tools/jdk:current_java_toolchain"),
         ),
+        "_java_host_runtime": attr.label(
+            default = Label("@bazel_tools//tools/jdk:current_host_java_runtime"),
+        ),
     }
     return aspect(
         implementation = _scala_proto_aspect_impl,

test/BUILD

@@ -781,6 +781,8 @@ scala_library(
     srcs = [
         "src/main/scala/scalarules/test/junit/runtime_platform/JunitRuntimePlatformTest.java",
     ],
+    # make sure java compilation toolchain matches runtime toolchain ie --target
+    java_compile_toolchain = "@bazel_tools//tools/jdk:toolchain_java11",
     deps = ["@io_bazel_rules_scala_junit_junit"],
 )