add keep-git-dir and submodules controls for Git querystring URLs

client/llb/source.go

@@ -341,6 +341,11 @@ func Git(url, fragment string, opts ...GitOption) State {
 		addCap(&gi.Constraints, pb.CapSourceGitChecksum)
 	}
 
+	if gi.SkipSubmodules {
+		attrs[pb.AttrGitSkipSubmodules] = "true"
+		addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules)
+	}
+
 	addCap(&gi.Constraints, pb.CapSourceGit)
 
 	source := NewSource("git://"+id, attrs, gi.Constraints)
@@ -367,6 +372,7 @@ type GitInfo struct {
 	Checksum         string
 	Ref              string
 	SubDir           string
+	SkipSubmodules   bool
 }
 
 func GitRef(v string) GitOption {
@@ -381,6 +387,12 @@ func GitSubDir(v string) GitOption {
 	})
 }
 
+func GitSkipSubmodules() GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		gi.SkipSubmodules = true
+	})
+}
+
 func KeepGitDir() GitOption {
 	return gitOptionFunc(func(gi *GitInfo) {
 		gi.KeepGitDir = true

frontend/dockerfile/dfgitutil/git_ref.go

@@ -3,6 +3,7 @@ package dfgitutil
 
 import (
 	"net/url"
+	"strconv"
 	"strings"
 
 	cerrdefs "github.com/containerd/errdefs"
@@ -49,6 +50,12 @@ type GitRef struct {
 	// Discouraged, although not deprecated.
 	// Instead, consider using an encrypted TCP connection such as "[email protected]/foo/bar.git" or "https://github.com/foo/bar.git".
 	UnencryptedTCP bool
+
+	// KeepGitDir is true for URL that controls whether to keep the .git directory.
+	KeepGitDir *bool
+
+	// Submodules is true for URL that controls whether to fetch git submodules.
+	Submodules *bool
 }
 
 // ParseGitRef parses a git ref.
@@ -121,11 +128,14 @@ func (gf *GitRef) loadQuery(query url.Values) error {
 	var tag, branch string
 	for k, v := range query {
 		switch len(v) {
-		case 0:
-			return errors.Errorf("query %q has no value", k)
-		case 1:
-			if v[0] == "" {
-				return errors.Errorf("query %q has no value", k)
+		case 0, 1:
+			if len(v) == 0 || v[0] == "" {
+				switch k {
+				case "submodules", "keep-git-dir":
+					v = nil
+				default:
+					return errors.Errorf("query %q has no value", k)
+				}
 			}
 			// NOP
 		default:
@@ -148,6 +158,30 @@ func (gf *GitRef) loadQuery(query url.Values) error {
 			gf.SubDir = v[0]
 		case "checksum", "commit":
 			gf.Checksum = v[0]
+		case "keep-git-dir":
+			var vv bool
+			if len(v) == 0 {
+				vv = true
+			} else {
+				var err error
+				vv, err = strconv.ParseBool(v[0])
+				if err != nil {
+					return errors.Errorf("invalid keep-git-dir value: %q", v[0])
+				}
+			}
+			gf.KeepGitDir = &vv
+		case "submodules":
+			var vv bool
+			if len(v) == 0 {
+				vv = true
+			} else {
+				var err error
+				vv, err = strconv.ParseBool(v[0])
+				if err != nil {
+					return errors.Errorf("invalid submodules value: %q", v[0])
+				}
+			}
+			gf.Submodules = &vv
 		default:
 			return errors.Errorf("unexpected query %q", k)
 		}

frontend/dockerfile/dockerfile2llb/convert.go

@@ -1520,7 +1520,15 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				llb.WithCustomName(pgName),
 				llb.GitRef(gitRef.Ref),
 			}
-			if cfg.keepGitDir {
+			if cfg.keepGitDir != nil && gitRef.KeepGitDir != nil {
+				if *cfg.keepGitDir != *gitRef.KeepGitDir {
+					return errors.New("inconsistent keep-git-dir configuration")
+				}
+			}
+			if gitRef.KeepGitDir != nil {
+				cfg.keepGitDir = gitRef.KeepGitDir
+			}
+			if cfg.keepGitDir != nil && *cfg.keepGitDir {
 				gitOptions = append(gitOptions, llb.KeepGitDir())
 			}
 			if cfg.checksum != "" && gitRef.Checksum != "" {
@@ -1537,6 +1545,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 			if gitRef.SubDir != "" {
 				gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir))
 			}
+			if gitRef.Submodules != nil && !*gitRef.Submodules {
+				gitOptions = append(gitOptions, llb.GitSkipSubmodules())
+			}
 
 			st := llb.Git(gitRef.Remote, "", gitOptions...)
 			opts := append([]llb.CopyOption{&llb.CopyInfo{
@@ -1711,7 +1722,7 @@ type copyConfig struct {
 	chown           string
 	chmod           string
 	link            bool
-	keepGitDir      bool
+	keepGitDir      *bool
 	checksum        string
 	parents         bool
 	location        []parser.Range

frontend/dockerfile/dockerfile_addgit_test.go

@@ -356,8 +356,25 @@ func testGitQueryString(t *testing.T, sb integration.Sandbox) {
 	integration.SkipOnPlatform(t, "windows")
 	f := getFrontend(t, sb)
 
-	gitDir, err := os.MkdirTemp("", "buildkit")
+	subModDir := t.TempDir()
+	defer os.RemoveAll(subModDir)
+
+	err := runShell(subModDir, []string{
+		"git init",
+		"git config --local user.email test",
+		"git config --local user.name test",
+		"echo 123 >file",
+		"git add file",
+		"git commit -m initial",
+		"git update-server-info",
+	}...)
 	require.NoError(t, err)
+
+	subModServer := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(subModDir))))
+	defer subModServer.Close()
+	submodServerURL := subModServer.URL
+
+	gitDir := t.TempDir()
 	defer os.RemoveAll(gitDir)
 	err = runShell(gitDir, []string{
 		"git init",
@@ -368,13 +385,20 @@ func testGitQueryString(t *testing.T, sb integration.Sandbox) {
 	require.NoError(t, err)
 
 	err = os.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte(`
+FROM scratch AS withgit
+COPY .git/HEAD out
+
+FROM scratch as withsubmod
+COPY submod/file out
+
 FROM scratch
 COPY foo out
 `), 0600)
 	require.NoError(t, err)
 
 	err = runShell(gitDir, []string{
-		"git add Dockerfile foo",
+		"git submodule add " + submodServerURL + "/.git submod",
+		"git add Dockerfile foo submod",
 		"git commit -m initial",
 		"git tag v0.0.1",
 		"git branch base",
@@ -426,6 +450,7 @@ COPY foo out
 	type tcase struct {
 		name      string
 		url       string
+		target    string
 		expectOut string
 		expectErr string
 	}
@@ -513,15 +538,73 @@ COPY foo out
 			url:       serverURL + "/.git?subdir=sub&ref=feature",
 			expectOut: "subfeature\n",
 		},
+		{
+			name:      "withgit",
+			url:       serverURL + "/.git?keep-git-dir=true",
+			expectOut: commitHashLatest + "\n",
+			target:    "withgit",
+		},
+		{
+			name:      "withgitandtag",
+			url:       serverURL + "/.git?tag=v0.0.2&keep-git-dir=true",
+			expectOut: commitHashV2 + "\n",
+			target:    "withgit",
+		},
+		{
+			name:      "withgit-default",
+			url:       serverURL + "/.git",
+			expectErr: ".git/HEAD\": not found",
+			target:    "withgit",
+		},
+		{
+			name:      "withgit-valueless",
+			url:       serverURL + "/.git?keep-git-dir&submodules",
+			expectOut: commitHashLatest + "\n",
+			target:    "withgit",
+		},
+		{
+			name:      "withgit-forbidden",
+			url:       serverURL + "/.git?keep-git-dir=false",
+			expectErr: ".git/HEAD\": not found",
+			target:    "withgit",
+		},
+		{
+			name:      "withsubmod",
+			url:       serverURL + "/.git",
+			expectOut: "123\n",
+			target:    "withsubmod",
+		},
+		{
+			name:      "withsubmodset",
+			url:       serverURL + "/.git?submodules=true",
+			expectOut: "123\n",
+			target:    "withsubmod",
+		},
+		{
+			name:      "withsubmodempty",
+			url:       serverURL + "/.git?submodules",
+			expectOut: "123\n",
+			target:    "withsubmod",
+		},
+		{
+			name:      "withoutsubmod",
+			url:       serverURL + "/.git?submodules=false",
+			expectErr: "submod/file\": not found",
+			target:    "withsubmod",
+		},
 	}
 
 	for _, tc := range tcases {
 		t.Run("context_"+tc.name, func(t *testing.T) {
 			dest := t.TempDir()
+			attrs := map[string]string{
+				"context": tc.url,
+			}
+			if tc.target != "" {
+				attrs["target"] = tc.target
+			}
 			_, err = f.Solve(sb.Context(), c, client.SolveOpt{
-				FrontendAttrs: map[string]string{
-					"context": tc.url,
-				},
+				FrontendAttrs: attrs,
 				Exports: []client.ExportEntry{
 					{
 						Type:      client.ExporterLocal,
@@ -551,15 +634,28 @@ COPY foo out
 
 	for _, tc := range tcases {
 		dockerfile2 := fmt.Sprintf(`
-FROM scratch
+FROM scratch AS main
 ADD %s /repo/
+
+FROM scratch as withsubmod
+COPY --from=main /repo/submod/file /repo/foo
+
+FROM scratch AS withgit
+COPY --from=main /repo/.git/HEAD /repo/foo
+
+FROM main
 		`, tc.url)
 		inDir := integration.Tmpdir(t,
 			fstest.CreateFile("Dockerfile", []byte(dockerfile2), 0600),
 		)
 		t.Run("add_"+tc.name, func(t *testing.T) {
 			dest := t.TempDir()
+			attrs := map[string]string{}
+			if tc.target != "" {
+				attrs["target"] = tc.target
+			}
 			_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+				FrontendAttrs: attrs,
 				Exports: []client.ExportEntry{
 					{
 						Type:      client.ExporterLocal,

frontend/dockerfile/instructions/commands.go

@@ -244,7 +244,7 @@ type AddCommand struct {
 	Chmod           string
 	Link            bool
 	ExcludePatterns []string
-	KeepGitDir      bool // whether to keep .git dir, only meaningful for git sources
+	KeepGitDir      *bool // whether to keep .git dir, only meaningful for git sources
 	Checksum        string
 	Unpack          *bool
 }

frontend/dockerfile/instructions/parse.go

@@ -354,13 +354,19 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 		unpack = &b
 	}
 
+	var keepGit *bool
+	if _, ok := req.flags.used["keep-git-dir"]; ok {
+		b := flKeepGitDir.Value == "true"
+		keepGit = &b
+	}
+
 	return &AddCommand{
 		withNameAndCode: newWithNameAndCode(req),
 		SourcesAndDest:  *sourcesAndDest,
 		Chown:           flChown.Value,
 		Chmod:           flChmod.Value,
 		Link:            flLink.Value == "true",
-		KeepGitDir:      flKeepGitDir.Value == "true",
+		KeepGitDir:      keepGit,
 		Checksum:        flChecksum.Value,
 		ExcludePatterns: stringValuesFromFlagIfPossible(flExcludes),
 		Unpack:          unpack,

frontend/dockerui/context.go

@@ -69,9 +69,9 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
 		bctx.dockerfileLocalName = v
 	}
 
-	keepGit := false
+	var keepGit *bool
 	if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil {
-		keepGit = v
+		keepGit = &v
 	}
 	if st, ok, err := DetectGitContext(opts[localNameContext], keepGit); ok {
 		if err != nil {
@@ -143,7 +143,7 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
 	return bctx, nil
 }
 
-func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) {
+func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) {
 	g, isGit, err := dfgitutil.ParseGitRef(ref)
 	if err != nil {
 		return nil, isGit, err
@@ -152,7 +152,10 @@ func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) {
 		llb.GitRef(g.Ref),
 		WithInternalName("load git source " + ref),
 	}
-	if keepGit {
+	if g.KeepGitDir != nil && *g.KeepGitDir {
+		gitOpts = append(gitOpts, llb.KeepGitDir())
+	}
+	if keepGit != nil && *keepGit {
 		gitOpts = append(gitOpts, llb.KeepGitDir())
 	}
 	if g.SubDir != "" {
@@ -161,6 +164,9 @@ func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) {
 	if g.Checksum != "" {
 		gitOpts = append(gitOpts, llb.GitChecksum(g.Checksum))
 	}
+	if g.Submodules != nil && !*g.Submodules {
+		gitOpts = append(gitOpts, llb.GitSkipSubmodules())
+	}
 
 	st := llb.Git(g.Remote, "", gitOpts...)
 	return &st, true, nil

frontend/dockerui/namedcontext.go

@@ -138,7 +138,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
 		}
 		return &st, &img, nil
 	case "git":
-		st, ok, err := DetectGitContext(nc.input, true)
+		st, ok, err := DetectGitContext(nc.input, nil)
 		if !ok {
 			return nil, nil, errors.Errorf("invalid git context %s", nc.input)
 		}
@@ -147,7 +147,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
 		}
 		return st, nil, nil
 	case "http", "https":
-		st, ok, err := DetectGitContext(nc.input, true)
+		st, ok, err := DetectGitContext(nc.input, nil)
 		if !ok {
 			httpst := llb.HTTP(nc.input, llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input))
 			st = &httpst