Scan GitLab Groups (#4320)

* gitlab groups init

* added list group projects api

* list group projects updated

* added duplicate repo scan check

* comments addressed

* added error when repo and group id flags are provided at the same time

* added test case for gitlab group projects

Author: shahzadhaider1

main.go

@@ -136,6 +136,7 @@ var (
 	gitlabScanEndpoint     = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
 	gitlabScanRepos        = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
 	gitlabScanToken        = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+	gitlabScanGroupIds     = gitlabScan.Flag("group-id", "GitLab group ID. If provided, it will scan the group and its subgroups. You can repeat this flag.").Strings()
 	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 	gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
@@ -788,10 +789,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			return scanMetrics, fmt.Errorf("could not create filter: %v", err)
 		}
 
+		if len(*gitlabScanRepos) > 0 && len(*gitlabScanGroupIds) > 0 {
+			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both repositories and groups at the same time")
+		}
+
 		cfg := sources.GitlabConfig{
 			Endpoint:     *gitlabScanEndpoint,
 			Token:        *gitlabScanToken,
 			Repos:        *gitlabScanRepos,
+			GroupIds:     *gitlabScanGroupIds,
 			IncludeRepos: *gitlabScanIncludeRepos,
 			ExcludeRepos: *gitlabScanExcludeRepos,
 			Filter:       filter,

pkg/engine/gitlab.go

@@ -46,6 +46,10 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) (source
 		connection.Repositories = c.Repos
 	}
 
+	if len(c.GroupIds) > 0 {
+		connection.GroupIds = c.GroupIds
+	}
+
 	if len(c.IncludeRepos) > 0 {
 		connection.IncludeRepos = c.IncludeRepos
 	}

pkg/pb/sourcespb/sources.pb.go

@@ -44,6 +44,7 @@ type Source struct {
 	token        string
 	url          string
 	repos        []string
+	groupIds     []string
 	ignoreRepos  []string
 	includeRepos []string
 
@@ -158,6 +159,7 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 	}
 
 	s.repos = conn.GetRepositories()
+	s.groupIds = conn.GetGroupIds()
 	s.ignoreRepos = conn.GetIgnoreRepos()
 	s.includeRepos = conn.GetIncludeRepos()
 	s.enumerateSharedProjects = !conn.ExcludeProjectsSharedIntoGroups
@@ -266,14 +268,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar
 				return ctx.Err()
 			},
 		}
-		if feature.UseSimplifiedGitlabEnumeration.Load() {
-			if err := s.getAllProjectReposV2(ctx, apiClient, ignoreRepo, reporter); err != nil {
-				return err
-			}
-		} else {
-			if err := s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter); err != nil {
-				return err
-			}
+
+		if err := s.listProjects(ctx, apiClient, ignoreRepo, reporter); err != nil {
+			return err
 		}
 
 	} else {
@@ -287,6 +284,21 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar
 	return s.scanRepos(ctx, chunksChan)
 }
 
+func (s *Source) listProjects(ctx context.Context,
+	apiClient *gitlab.Client,
+	ignoreProject func(string) bool,
+	visitor sources.UnitReporter) error {
+	if len(s.groupIds) > 0 {
+		return s.getAllProjectReposInGroups(ctx, apiClient, ignoreProject, visitor)
+	}
+
+	if feature.UseSimplifiedGitlabEnumeration.Load() {
+		return s.getAllProjectReposV2(ctx, apiClient, ignoreProject, visitor)
+	}
+
+	return s.getAllProjectRepos(ctx, apiClient, ignoreProject, visitor)
+}
+
 func (s *Source) scanTargets(ctx context.Context, client *gitlab.Client, targets []sources.ChunkingTarget, chunksChan chan *sources.Chunk) error {
 	ctx = context.WithValues(ctx, "scan_type", "targeted")
 	for _, tgt := range targets {
@@ -401,16 +413,9 @@ func (s *Source) Validate(ctx context.Context) []error {
 		},
 	}
 
-	if feature.UseSimplifiedGitlabEnumeration.Load() {
-		if err := s.getAllProjectReposV2(ctx, apiClient, ignoreProject, visitor); err != nil {
-			errs = append(errs, err)
-			return errs
-		}
-	} else {
-		if err := s.getAllProjectRepos(ctx, apiClient, ignoreProject, visitor); err != nil {
-			errs = append(errs, err)
-			return errs
-		}
+	if err := s.listProjects(ctx, apiClient, ignoreProject, visitor); err != nil {
+		errs = append(errs, err)
+		return errs
 	}
 
 	if len(repos) == 0 {
@@ -478,7 +483,6 @@ func (s *Source) getAllProjectRepos(
 	reporter sources.UnitReporter,
 ) error {
 	gitlabReposEnumerated.WithLabelValues(s.name).Set(0)
-
 	// Projects without repo will get user projects, groups projects, and subgroup projects.
 	user, _, err := apiClient.Users.CurrentUser()
 	if err != nil {
@@ -728,6 +732,118 @@ func (s *Source) getAllProjectReposV2(
 	return nil
 }
 
+// getAllProjectReposInGroups fetches all projects in a GitLab group and its subgroups.
+// It uses the group projects API with include_subgroups=true parameter.
+func (s *Source) getAllProjectReposInGroups(
+	ctx context.Context,
+	apiClient *gitlab.Client,
+	ignoreRepo func(string) bool,
+	reporter sources.UnitReporter,
+) error {
+	gitlabReposEnumerated.WithLabelValues(s.name).Set(0)
+	gitlabGroupsEnumerated.WithLabelValues(s.name).Set(float64(len(s.groupIds)))
+
+	processedProjects := make(map[string]bool)
+
+	var projectsWithNamespace []string
+	const (
+		orderBy         = "id"
+		paginationLimit = 100
+	)
+
+	listOpts := gitlab.ListOptions{PerPage: paginationLimit}
+	projectOpts := &gitlab.ListGroupProjectsOptions{
+		ListOptions:      listOpts,
+		OrderBy:          gitlab.Ptr(orderBy),
+		IncludeSubGroups: gitlab.Ptr(true),
+		WithShared:       gitlab.Ptr(true),
+	}
+
+	// For non gitlab.com instances, you might want to adjust access levels
+	if s.url != gitlabBaseURL {
+		projectOpts.MinAccessLevel = gitlab.Ptr(gitlab.GuestPermissions)
+	}
+
+	ctx.Logger().Info("starting group projects enumeration",
+		"group_ids", s.groupIds,
+		"include_subgroups", true,
+		"list_options", listOpts)
+
+	for _, groupID := range s.groupIds {
+		groupCtx := context.WithValues(ctx, "group_id", groupID)
+
+		projectOpts.Page = 0
+		groupCtx.Logger().V(2).Info("processing group", "group_id", groupID)
+
+		for {
+			projects, res, err := apiClient.Groups.ListGroupProjects(groupID, projectOpts)
+			if err != nil {
+				err = fmt.Errorf("received error on listing projects for group %s: %w", groupID, err)
+				if err := reporter.UnitErr(ctx, err); err != nil {
+					return err
+				}
+				break
+			}
+
+			groupCtx.Logger().V(3).Info("listed group projects", "count", len(projects))
+
+			for _, proj := range projects {
+				projCtx := context.WithValues(ctx,
+					"project_id", proj.ID,
+					"project_name", proj.NameWithNamespace,
+					"group_id", groupID)
+
+				if processedProjects[proj.HTTPURLToRepo] {
+					projCtx.Logger().V(3).Info("skipping project", "reason", "already processed")
+					continue
+				}
+				processedProjects[proj.HTTPURLToRepo] = true
+
+				// skip projects configured to be ignored.
+				if ignoreRepo(proj.PathWithNamespace) {
+					projCtx.Logger().V(3).Info("skipping project", "reason", "ignored in config")
+					continue
+				}
+
+				// report an error if we could not convert the project into a URL.
+				if _, err := url.Parse(proj.HTTPURLToRepo); err != nil {
+					projCtx.Logger().V(3).Info("skipping project",
+						"reason", "URL parse failure",
+						"url", proj.HTTPURLToRepo,
+						"parse_error", err)
+
+					err = fmt.Errorf("could not parse url %q given by project: %w", proj.HTTPURLToRepo, err)
+					if err := reporter.UnitErr(ctx, err); err != nil {
+						return err
+					}
+					continue
+				}
+
+				// report the unit.
+				projCtx.Logger().V(3).Info("accepting project")
+
+				unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo}
+				gitlabReposEnumerated.WithLabelValues(s.name).Inc()
+				projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)
+
+				if err := reporter.UnitOk(ctx, unit); err != nil {
+					return err
+				}
+			}
+
+			// handle pagination.
+			projectOpts.Page = res.NextPage
+			if res.NextPage == 0 {
+				break
+			}
+		}
+	}
+
+	ctx.Logger().Info("Enumerated GitLab group projects", "count", len(projectsWithNamespace))
+
+	return nil
+}
+
 func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	// If there is resume information available, limit this scan to only the repos that still need scanning.
 	reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
@@ -937,11 +1053,11 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
 		_ = reporter.UnitErr(ctx, fmt.Errorf("could not compile include/exclude repo glob: %w", err))
 	})
 
-	if feature.UseSimplifiedGitlabEnumeration.Load() {
-		return s.getAllProjectReposV2(ctx, apiClient, ignoreRepo, reporter)
-	} else {
-		return s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter)
+	if err := s.listProjects(ctx, apiClient, ignoreRepo, reporter); err != nil {
+		return err
 	}
+
+	return nil
 }
 
 // ChunkUnit downloads and reports chunks for the given GitLab repository unit.

pkg/sources/gitlab/gitlab.go

@@ -136,6 +136,41 @@ func TestSource_Scan(t *testing.T) {
 			},
 			wantReposScanned: 1,
 		},
+		{
+			name: "token auth, group projects enumeration with include_subgroups",
+			init: init{
+				name: "test source group enumeration",
+				connection: &sourcespb.GitLab{
+					Credential: &sourcespb.GitLab_Token{
+						Token: token,
+					},
+					GroupIds: []string{"15013490"},
+				},
+			},
+			wantChunk: &sources.Chunk{
+				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
+				SourceName: "test source group enumeration",
+			},
+			wantReposScanned: 5,
+		},
+		{
+			name: "token auth, group projects enumeration with include_subgroups and exclude repositories",
+			init: init{
+				name: "test source group enumeration with exclude repos",
+				connection: &sourcespb.GitLab{
+					Credential: &sourcespb.GitLab_Token{
+						Token: token,
+					},
+					GroupIds:    []string{"15013490"},
+					IgnoreRepos: []string{"tes1188/test-user-count"},
+				},
+			},
+			wantChunk: &sources.Chunk{
+				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
+				SourceName: "test source group enumeration with exclude repos",
+			},
+			wantReposScanned: 4,
+		},
 	}
 
 	for _, tt := range tests {

pkg/sources/gitlab/gitlab_integration_test.go

@@ -342,6 +342,8 @@ type GitlabConfig struct {
 	Token string
 	// Repos is the list of repositories to scan.
 	Repos []string
+	// GroupIds is the list of groups to scan.
+	GroupIds []string
 	// Filter is the filter to use to scan the source.
 	Filter *common.Filter
 	// SkipBinaries allows skipping binary files from the scan.

pkg/sources/sources.go

@@ -233,6 +233,7 @@ message GitLab {
   repeated string include_repos = 9;
   bool exclude_projects_shared_into_groups = 10;
   bool remove_auth_in_url = 11;
+  repeated string group_ids = 12;
 }
 
 message GitHub {