Centralize SDK generation logic (#1515)

Presently, the majority of Java code generation is driven by the
`pulumi-java-gen` binary, since Java usage began before we had time to
implement the `Generate*` family of language host gRPC methods.
Recently, these gRPC methods were implemented, to support (among other
things) conformance testing. Unfortunately, while both routes end in
`pkg/codegen/java`'s `Generate*` functions, each had accumulated its own
special "setup logic" ahead of the call into `pkg/codegen`. This commit
attempts to sort this out, pushing all that logic into `pkg/codegen` so
that both routes behave identically. As a result of this, we should be
able to deprecate `pulumi-java-gen` more safely when the time comes,
remove direct build-time dependencies on `pulumi-java` from
`pulumi/pulumi` and fix some issues that have arisen as a result of the
historic differences, such as #1404 (which looks like it may have
already been fixed, but this should cement it), and the Java side of
#1508.

Alongside new unit tests and existing conformance tests, this changeset
has been manually tested using `pulumi-azure-native` and changes akin to
those in pulumi/pulumi-azure-native#3776 (using a locally modified
`pulumi package gen-sdk` that can be mainstreamed when these changes
have been merged and released).

Closes #1404
Part of #1508

Author: lunaris

pkg/cmd/pulumi-java-gen/generate.go

@@ -16,12 +16,16 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/blang/semver"
+	"github.com/hashicorp/hcl/v2"
 
 	javagen "github.com/pulumi/pulumi-java/pkg/codegen/java"
 	pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 )
 
 type generateJavaOptions struct {
@@ -53,10 +57,11 @@ func generateJava(cfg generateJavaOptions) error {
 		return fmt.Errorf("failed to read schema from %s: %w", cfg.Schema, err)
 	}
 
-	pkgSpec, err := dedupTypes(rawPkgSpec)
+	pkgSpec, diags, err := javagen.DeduplicateTypes(rawPkgSpec)
 	if err != nil {
 		return fmt.Errorf("failed to dedup types in schema from %s: %w", cfg.Schema, err)
 	}
+	printDiagnostics(diags)
 
 	pkg, err := pschema.ImportSpec(*pkgSpec, nil)
 	if err != nil {
@@ -80,7 +85,13 @@ func generateJava(cfg generateJavaOptions) error {
 	if err != nil {
 		return err
 	}
-	files, err := javagen.GeneratePackage("pulumi-java-gen", pkg, extraFiles, cfg.Local)
+	files, err := javagen.GeneratePackage(
+		"pulumi-java-gen",
+		pkg,
+		extraFiles,
+		nil, /*localDependencies*/
+		cfg.Local,
+	)
 	if err != nil {
 		return err
 	}
@@ -99,3 +110,15 @@ func generateJava(cfg generateJavaOptions) error {
 
 	return nil
 }
+
+// printDiagnostics prints the given diagnostics to stdout and stderr.
+func printDiagnostics(diagnostics hcl.Diagnostics) {
+	sink := diag.DefaultSink(os.Stdout, os.Stderr, diag.FormatOptions{Color: cmdutil.GetGlobalColorization()})
+	for _, diagnostic := range diagnostics {
+		if diagnostic.Severity == hcl.DiagError {
+			sink.Errorf(diag.Message("", "%s"), diagnostic)
+		} else {
+			sink.Warningf(diag.Message("", "%s"), diagnostic)
+		}
+	}
+}

pkg/cmd/pulumi-language-java/main.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	pbempty "github.com/golang/protobuf/ptypes/empty"
+	"github.com/hashicorp/hcl/v2"
 	"github.com/pkg/errors"
 	hclsyntax "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
 	"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
@@ -32,7 +33,6 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
 	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
-	"golang.org/x/exp/maps"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -703,57 +703,40 @@ func (host *javaLanguageHost) GeneratePackage(
 		return nil, err
 	}
 
-	pkg, diags, err := schema.BindSpec(spec, loader)
+	diags := hcl.Diagnostics{}
+
+	// Historically, Java has "deduplicated" PackageSpecs to reduce sets of multiple types whose names differ only in
+	// case down to just one type that is then shared (assuming that, apart from name, the types are otherwise
+	// identical). We thus perform that deduplication here before we bind the schema and resolve any references.
+	dedupedSpec, dedupeDiags, err := codegen.DeduplicateTypes(&spec)
 	if err != nil {
 		return nil, err
 	}
-	rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
-	if diags.HasErrors() {
+	diags = diags.Extend(dedupeDiags)
+	if dedupeDiags.HasErrors() {
 		return &pulumirpc.GeneratePackageResponse{
-			Diagnostics: rpcDiagnostics,
+			Diagnostics: plugin.HclDiagnosticsToRPCDiagnostics(diags),
 		}, nil
 	}
 
-	if pkg.Description == "" {
-		pkg.Description = " "
-	}
-	if pkg.Repository == "" {
-		pkg.Repository = "https://example.com"
-	}
-
-	// Presently, we only support generating Java SDKs which use Gradle as a build system. Specify that here, as well as
-	// the set of dependencies that all generated SDKs rely on.
-	pkgInfo := codegen.PackageInfo{
-		BuildFiles: "gradle",
-		Dependencies: map[string]string{
-			"com.google.code.gson:gson":       "2.8.9",
-			"com.google.code.findbugs:jsr305": "3.0.2",
-		},
+	pkg, bindDiags, err := schema.BindSpec(*dedupedSpec, loader)
+	if err != nil {
+		return nil, err
 	}
-
-	repositories := map[string]bool{}
-
-	for name, dep := range req.LocalDependencies {
-		parts := strings.Split(dep, ":")
-		if len(parts) < 3 {
-			return nil, fmt.Errorf(
-				"invalid dependency for %s %s; must be of the form groupId:artifactId:version[:repositoryPath]",
-				name, dep,
-			)
-		}
-
-		k := parts[0] + ":" + parts[1]
-		pkgInfo.Dependencies[k] = parts[2]
-
-		if len(parts) == 4 {
-			repositories[parts[3]] = true
-		}
+	diags = diags.Extend(bindDiags)
+	if bindDiags.HasErrors() {
+		return &pulumirpc.GeneratePackageResponse{
+			Diagnostics: plugin.HclDiagnosticsToRPCDiagnostics(diags),
+		}, nil
 	}
 
-	pkgInfo.Repositories = maps.Keys(repositories)
-	pkg.Language["java"] = pkgInfo
-
-	files, err := codegen.GeneratePackage("pulumi-language-java", pkg, req.ExtraFiles, req.Local)
+	files, err := codegen.GeneratePackage(
+		"pulumi-language-java",
+		pkg,
+		req.ExtraFiles,
+		req.LocalDependencies,
+		req.Local,
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -772,7 +755,7 @@ func (host *javaLanguageHost) GeneratePackage(
 	}
 
 	return &pulumirpc.GeneratePackageResponse{
-		Diagnostics: rpcDiagnostics,
+		Diagnostics: plugin.HclDiagnosticsToRPCDiagnostics(diags),
 	}, nil
 }
 

pkg/cmd/pulumi-java-gen/dedup.go renamed to pkg/codegen/java/dedup.go

@@ -1,6 +1,6 @@
-// Copyright 2022, Pulumi Corporation.  All rights reserved.
+// Copyright 2024, Pulumi Corporation.  All rights reserved.
 
-package main
+package java
 
 import (
 	"bytes"
@@ -9,15 +9,16 @@ import (
 	"reflect"
 	"strings"
 
-	pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 )
 
-// Detects cases when identical types have similar names modulo case
-// such as `azure-native:network:IpAllocationMethod` vs
-// `azure-native:network:IPAllocationMethod`, deterministically picks
-// one of these names, and rewrites the schema as if there was only
-// one such type.
-func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
+// DeduplicateTypes detects multiple types in a PackageSpec whose names are the same modulo case, such as
+// `azure-native:network:IpAllocationMethod` and `azure-native:network:IPAllocationMethod`, deterministically picks one
+// of these names, and rewrites the schema as if there was only one such type.
+func DeduplicateTypes(spec *schema.PackageSpec) (*schema.PackageSpec, hcl.Diagnostics, error) {
+	diags := hcl.Diagnostics{}
+
 	normalizedTokens := map[string]string{}
 	for typeToken := range spec.Types {
 		key := strings.ToUpper(typeToken)
@@ -41,12 +42,12 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 
 	var buf bytes.Buffer
 	if err := json.NewEncoder(&buf).Encode(spec); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	var rawSchema interface{}
 	if err := json.NewDecoder(bytes.NewReader(buf.Bytes())).Decode(&rawSchema); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	types := map[string]interface{}{}
@@ -64,13 +65,19 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 			transformJSONTree(stripDescription, types[newToken]),
 		)
 		if eq {
-			fmt.Printf("WARN renaming %s to %s in the schema\n",
-				oldToken, newToken)
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary:  fmt.Sprintf("Renaming '%s' to '%s' in the schema", oldToken, newToken),
+			})
 			delete(types, oldToken)
 		} else {
-			fmt.Printf("WARN not renaming %s to %s in the schema "+
-				"because they differ structurally\n",
-				oldToken, newToken)
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary: fmt.Sprintf(
+					"Not renaming '%s' to '%s' in the schema because they differ structurally",
+					oldToken, newToken,
+				),
+			})
 		}
 	}
 
@@ -89,7 +96,10 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 			return node
 		}
 		if r, isRenamed := renamedRefs[s]; isRenamed {
-			fmt.Printf("Rewritten %s to %s\n", s, r)
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary:  fmt.Sprintf("Rewrote reference '%s' to '%s'", s, r),
+			})
 			return r
 		}
 		return node
@@ -100,15 +110,15 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 	buf.Reset()
 
 	if err := json.NewEncoder(&buf).Encode(&rawSchema); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	var fixedSpec pschema.PackageSpec
+	var fixedSpec schema.PackageSpec
 	if err := json.NewDecoder(bytes.NewReader(buf.Bytes())).Decode(&fixedSpec); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	return &fixedSpec, nil
+	return &fixedSpec, diags, nil
 }
 
 func transformJSONTree(t func(interface{}) interface{}, tree interface{}) interface{} {