Add option to remove stale profiles when running app-store-connect fetch-signing-files (#469)

Author: priitlatt

CHANGELOG.md

@@ -1,3 +1,14 @@
+Version 0.60.0
+-------------
+
+This release contains changes from [PR #469](https://github.com/codemagic-ci-cd/cli-tools/pull/469).
+
+**Features**
+- Add option `--delete-stale-profiles` to action `app-store-connect fetch-signing-files` to delete encountered stale provisioning profiles. Those profiles are not shown in Apple Developer Portal and cannot be queried using normal App Store Connect API profiles read and listing endpoints.
+
+**Docs**
+- Update docs for `app-store-connect fetch-signing-files`.
+
 Version 0.59.1
 -------------
 

docs/app-store-connect/fetch-signing-files.md

@@ -24,6 +24,7 @@ app-store-connect fetch-signing-files [-h] [--log-stream STREAM] [--no-color] [-
     [--type PROFILE_TYPE]
     [--strict-match-identifier]
     [--create]
+    [--delete-stale-profiles]
     BUNDLE_ID_IDENTIFIER
 ```
 ### Required arguments for action `fetch-signing-files`
@@ -62,6 +63,10 @@ Only match Bundle IDs that have exactly the same identifier specified by `BUNDLE
 
 
 Whether to create the resource if it does not exist yet
+##### `--delete-stale-profiles`
+
+
+Whether to delete invalid provisioning profiles that are discovered for specified bundle identifiers. Those profiles have active status with expiration date in the past, rendering them invalid. Such profiles are not shown in the Apple Developer Portal and they cannot be discovered using conventional read or list API queries.
 ### Optional arguments for command `app-store-connect`
 
 ##### `--log-api-calls`

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemagic-cli-tools"
-version = "0.59.1"
+version = "0.60.0"
 description = "CLI tools used in Codemagic builds"
 readme = "README.md"
 authors = [

src/codemagic/__version__.py

@@ -1,5 +1,5 @@
 __title__ = "codemagic-cli-tools"
 __description__ = "CLI tools used in Codemagic builds"
-__version__ = "0.59.1.dev"
+__version__ = "0.60.0.dev"
 __url__ = "https://github.com/codemagic-ci-cd/cli-tools"
 __licence__ = "GNU General Public License v3.0"

src/codemagic/tools/app_store_connect/actions/fetch_signing_files_action.py

@@ -6,6 +6,7 @@
 from typing import List
 from typing import Optional
 from typing import Sequence
+from typing import Set
 from typing import Tuple
 from typing import Union
 
@@ -18,6 +19,7 @@
 from codemagic.apple.resources import Profile
 from codemagic.apple.resources import ProfileState
 from codemagic.apple.resources import ProfileType
+from codemagic.apple.resources import ResourceId
 from codemagic.apple.resources import SigningCertificate
 from codemagic.cli import Colors
 from codemagic.models import PrivateKey
@@ -31,6 +33,15 @@
 from ..errors import AppStoreConnectError
 
 
+class _StaleProfileError(Exception):
+    """
+    App Store Connect API response for listing bundle identifier profiles contains resources that are
+    already "deleted". These profiles aren't available in the Developer Portal UI nor via read requests,
+    but can be deleted via API. They significantly slow down actions as the deleted profiles accumulate
+    and eventually many unnecessary 404 requests need to be performed.
+    """
+
+
 class FetchSigningFilesAction(AbstractBaseAction, metaclass=ABCMeta):
     @cli.action(
         "fetch-signing-files",
@@ -42,6 +53,7 @@ class FetchSigningFilesAction(AbstractBaseAction, metaclass=ABCMeta):
         ProfileArgument.PROFILE_TYPE,
         BundleIdArgument.IDENTIFIER_STRICT_MATCH,
         CommonArgument.CREATE_RESOURCE,
+        ProfileArgument.DELETE_STALE_PROFILES,
     )
     def fetch_signing_files(
         self,
@@ -53,6 +65,7 @@ def fetch_signing_files(
         profile_type: ProfileType = ProfileType.IOS_APP_DEVELOPMENT,
         bundle_id_identifier_strict_match: bool = False,
         create_resource: bool = False,
+        delete_stale_profiles: bool = False,
     ) -> Tuple[List[Profile], List[SigningCertificate]]:
         """
         Fetch provisioning profiles and code signing certificates
@@ -90,6 +103,7 @@ def fetch_signing_files(
             profile_type,
             create_resource,
             platform,
+            delete_stale_profiles,
         )
         self.echo("")
 
@@ -148,40 +162,106 @@ def _get_or_create_certificates(
             certificates.append(certificate)
         return certificates
 
-    def _get_or_create_profiles(
-        self,
-        bundle_ids: Sequence[BundleId],
-        certificates: Sequence[SigningCertificate],
-        profile_type: ProfileType,
-        create_resource: bool,
-        platform: Optional[BundleIdPlatform] = None,
-    ):
-        def has_certificate(profile) -> bool:
-            try:
-                profile_certificates = self.api_client.profiles.list_certificate_ids(profile)
-                return bool(certificate_ids.issubset({c.id for c in profile_certificates}))
-            except AppStoreConnectApiError as err:
-                error = f"Listing {SigningCertificate.s} for {Profile} {profile.id} failed unexpectedly"
-                self.logger.warning(Colors.YELLOW(f"{error}: {err.error_response}"))
-                return False
+    def _has_certificate(self, profile: Profile, available_certificate_ids: Set[ResourceId]) -> bool:
+        try:
+            profile_certificates = self.api_client.profiles.list_certificate_ids(profile)
+        except AppStoreConnectApiError as err:
+            if f"There is no resource of type 'profiles' with id '{profile.id}'" in str(err.error_response):
+                raise _StaleProfileError() from err
+            error = f"Listing {SigningCertificate.s} for {Profile} {profile.id} failed unexpectedly"
+            self.logger.warning(Colors.YELLOW(f"{error}: {err.error_response}"))
+            return False
+
+        # Do not use set.issubset as empty set is subset of another empty set.
+        return bool({c.id for c in profile_certificates} & available_certificate_ids)
+
+    def _has_profile(self, bundle_id: BundleId, available_profile_ids: Set[ResourceId]) -> bool:
+        try:
+            bundle_id_profiles = self.api_client.bundle_ids.list_profile_ids(bundle_id)
+        except AppStoreConnectApiError as err:
+            error = f"Listing {Profile.s} for {BundleId} {bundle_id.id} failed unexpectedly"
+            self.logger.warning(Colors.YELLOW(f"{error}: {err.error_response}"))
+            return False
+
+        # Do not use set.issubset as empty set is subset of another empty set.
+        return bool({p.id for p in bundle_id_profiles} & available_profile_ids)
+
+    def _handle_stale_profiles(self, stale_profiles: Sequence[Profile], delete_stale_profiles: bool) -> None:
+        """
+        Listing profiles for bundle identifiers can return "zombie" profiles, which cannot be used.
+        Notify about such findings and if user has granted a permission, then attempt to delete
+        such profiles so that they wouldn't be picked up on the following action invocations.
+        """
+        if not stale_profiles:
+            return
+
+        stale_profile_ids = ", ".join(p.id for p in stale_profiles)
+
+        if not delete_stale_profiles:
+            self.logger.warning(Colors.RED(f"\nFound {len(stale_profiles)} stale profiles: {stale_profile_ids}."))
+            message = (
+                "These profiles are expired with invalid status and cannot be used or seen in "
+                "Apple Developer Portal. Requesting information about such provisioning profiles "
+                "causes unnecessarily slow-downs of the action. Get rid of them and speed up future "
+                "action invocations by using optional flag "
+                f"{Colors.BRIGHT_WHITE(ProfileArgument.DELETE_STALE_PROFILES.flag)}."
+            )
+            self.logger.warning(message)
+            return
 
-        def missing_profile(bundle_id) -> bool:
+        self.logger.info(f"\nFound {len(stale_profiles)} stale profiles, deleting them.")
+        for stale_profile in stale_profiles:
             try:
-                bundle_ids_profiles = self.api_client.bundle_ids.list_profile_ids(bundle_id)
-                return not (profile_ids & {p.id for p in bundle_ids_profiles})
+                self.api_client.profiles.delete(stale_profile)
             except AppStoreConnectApiError as err:
-                error = f"Listing {Profile.s} for {BundleId} {bundle_id.id} failed unexpectedly"
-                self.logger.warning(Colors.YELLOW(f"{error}: {err.error_response}"))
-                return True
+                error_message = f"- Failed to delete stale {Profile} {stale_profile.id}: {err.error_response}"
+                self.logger.warning(Colors.RED(error_message))
+            else:
+                self.logger.info(Colors.GREEN(f"- Deleted stale {Profile} {stale_profile.id}"))
 
-        certificate_ids = {c.id for c in certificates}
-        profiles = self.list_bundle_id_profiles(
+    def _find_usable_profiles(
+        self,
+        bundle_ids: Sequence[BundleId],
+        certificates: Sequence[SigningCertificate],
+        profile_type: ProfileType,
+        delete_stale_profiles: bool,
+    ) -> List[Profile]:
+        all_profiles = self.list_bundle_id_profiles(
             [bundle_id.id for bundle_id in bundle_ids],
             profile_type=profile_type,
             profile_state=ProfileState.ACTIVE,
             should_print=False,
         )
-        profiles = list(filter(has_certificate, profiles))
+
+        usable_profiles, stale_profiles = [], []
+        certificate_ids = {c.id for c in certificates}
+        for profile in all_profiles:
+            try:
+                if self._has_certificate(profile, certificate_ids):
+                    usable_profiles.append(profile)
+            except _StaleProfileError:
+                stale_profiles.append(profile)
+
+        self._handle_stale_profiles(stale_profiles, delete_stale_profiles)
+        self.logger.info("")
+
+        return usable_profiles
+
+    def _get_or_create_profiles(
+        self,
+        bundle_ids: Sequence[BundleId],
+        certificates: Sequence[SigningCertificate],
+        profile_type: ProfileType,
+        create_resource: bool,
+        platform: Optional[BundleIdPlatform] = None,
+        delete_stale_profiles: bool = False,
+    ):
+        profiles = self._find_usable_profiles(
+            bundle_ids,
+            certificates,
+            profile_type,
+            delete_stale_profiles,
+        )
 
         certificate_names = ", ".join(c.get_display_info() for c in certificates)
         message = f"that contain {SigningCertificate.plural(len(certificates))} {certificate_names}"
@@ -190,8 +270,9 @@ def missing_profile(bundle_id) -> bool:
             self.logger.info(f"- {profile.get_display_info()}")
 
         profile_ids = {p.id for p in profiles}
-        bundle_ids_without_profiles = list(filter(missing_profile, bundle_ids))
+        bundle_ids_without_profiles = [bid for bid in bundle_ids if not self._has_profile(bid, profile_ids)]
         if bundle_ids_without_profiles and not create_resource:
+            self.logger.info("")
             missing = ", ".join(f'"{bid.attributes.identifier}" [{bid.id}]' for bid in bundle_ids_without_profiles)
             raise AppStoreConnectError(f"Did not find {profile_type} {Profile.s} for {BundleId.s}: {missing}")
 

src/codemagic/tools/app_store_connect/arguments.py

@@ -1551,6 +1551,21 @@ class ProfileArgument(cli.Argument):
             "metavar": "device-id",
         },
     )
+    DELETE_STALE_PROFILES = cli.ArgumentProperties(
+        key="delete_stale_profiles",
+        flags=("--delete-stale-profiles",),
+        type=bool,
+        description=(
+            "Whether to delete invalid provisioning profiles that are discovered for specified bundle identifiers. "
+            "Those profiles have active status with expiration date in the past, rendering them invalid. Such "
+            "profiles are not shown in the Apple Developer Portal and they cannot be discovered using conventional "
+            "read or list API queries."
+        ),
+        argparse_kwargs={
+            "required": False,
+            "action": "store_true",
+        },
+    )
 
 
 class CommonArgument(cli.Argument):