[Bug] argocd_project destroy fails because applications from argocd_application_set are not fully removed

[docbyte86]

Terraform Version, ArgoCD Provider Version and ArgoCD Version

OpenTofu version: latest
ArgoCD provider version: 7.10.0
ArgoCD version: Helm 8.2.5

Affected Resource(s)

Terraform Configuration Files

terraform {
  required_providers {
    stackit = {
      source  = "stackitcloud/stackit"
      version = "0.59.0"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "3.0.2"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "2.38.0"
    }
    argocd = {
      source = "argoproj-labs/argocd"
      version = "7.10.0"
    }
  }
}

provider "stackit" {
  default_region        = "eu01"
  service_account_token = "***"
}

provider "helm" {
  kubernetes = {
    host                   = yamldecode(stackit_ske_kubeconfig.ske.kube_config).clusters[0].cluster.server
    client_certificate     = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).users[0].user["client-certificate-data"])
    client_key             = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).users[0].user["client-key-data"])
    cluster_ca_certificate = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).clusters[0].cluster["certificate-authority-data"])
  }
}

provider "kubernetes" {
  host                   = yamldecode(stackit_ske_kubeconfig.ske.kube_config).clusters[0].cluster.server
  client_certificate     = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).users[0].user["client-certificate-data"])
  client_key             = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).users[0].user["client-key-data"])
  cluster_ca_certificate = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).clusters[0].cluster["certificate-authority-data"])
}

provider "argocd" {
  username                    = "admin"
  password                    = "3Algf7U5h2kMjP9O"
  plain_text                  = true
  port_forward_with_namespace = "system-argo"
  kubernetes {
    host                   = yamldecode(stackit_ske_kubeconfig.ske.kube_config).clusters[0].cluster.server
    client_certificate     = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).users[0].user["client-certificate-data"])
    client_key             = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).users[0].user["client-key-data"])
    cluster_ca_certificate = base64decode(yamldecode(stackit_ske_kubeconfig.ske.kube_config).clusters[0].cluster["certificate-authority-data"])
  }
}


resource "stackit_ske_kubeconfig" "ske" {
  project_id   = "*****"
  cluster_name = "*****"
  refresh      = true
}

resource "kubernetes_namespace" "argo_cd" {
  metadata {

    labels = {*}

    name = "system-argo"
  }
}

resource "helm_release" "argo_cd" {
  name       = "argo"
  repository = "https://argoproj.github.io/argo-helm"
  chart      = "argo-cd"
  version    = "8.2.5"
  namespace  = kubernetes_namespace.argo_cd.metadata[0].name
  values     = [file("${path.module}/argocd.yml")]
}

resource "argocd_repository" "repositories" {
  depends_on = [
    data.kubernetes_secret.argocd_admin
  ]
  for_each = local.argocd_repositories
  repo     = each.value.repo
  name     = each.value.name
  type     = each.value.type
  username = each.value.username
  password = each.value.password
}

resource "argocd_project" "rundeck" {
  depends_on = [ argocd_repository.repositories ]
  metadata {
    name      = "rundeck"
    namespace = helm_release.argo_cd.namespace
  }
  spec {
    description       = "rundeck argocd project"
    source_namespaces = ["*"]
    source_repos      = ["*"]
    destination {
      server    = "*"
      name      = "*"
      namespace = "*"
    }
    cluster_resource_whitelist {
      group = "*"
      kind  = "*"
    }
    namespace_resource_whitelist {
      group = "*"
      kind  = "*"
    }
  }
}

resource "argocd_application_set" "rundeck" {
  metadata {
    name      = "rundeck"
    namespace = helm_release.argo_cd.namespace
  }
  spec {
    generator {
      git {
        repo_url = "****"
        revision = "logging"
        directory {
          path = "rundeck/argocd/*"
        }
        directory {
          path = "rundeck/argocd/_global"
          exclude = true
        }
      }
    }
    template {
      metadata {
        name      = "{{path.basename}}"
        namespace = "system-argo"
      }
      spec {
        project = argocd_project.rundeck.metadata[0].name
        destination {
          namespace = "{{path.basename}}"
          server    = "https://kubernetes.default.svc/"
        }
        source {
          repo_url       = "****"
          target_revision = "argo"
          path           = "rundeck"
          helm {
            value_files = [
              "$rundeck-values/rundeck/argocd/_global/values.yml",
              "$rundeck-values/rundeck/argocd/{{path.basename}}/values.yml"
            ]
          }
        }
        source {
          repo_url = "****"
          target_revision = "logging"
          ref = "rundeck-values"
        }
        sync_policy {
          automated {
            prune = true
            self_heal = true
          }
          sync_options = [
            "CreateNamespace=true"
          ]
        }
      }
    }
  }
}

data "kubernetes_secret" "argocd_admin" {
  metadata {
    name      = "argocd-initial-admin-secret"
    namespace = helm_release.argo_cd.namespace
  }
}

data "kubernetes_ingress_v1" "argo_argocd_server" {
  metadata {
    name = "argo-argocd-server"
    namespace = helm_release.argo_cd.namespace
  }
}

output "ingress_host" {
  value = data.kubernetes_ingress_v1.argo_argocd_server.spec[0].rule[0].host
}

locals {
  argocd_repositories = {
    application-rundeck = {
      repo     = "*****"
      name     = "application-rundeck"
      type     = "git"
      username = "****"
      password = "***"
    }
    helm-rundeck = {
      repo     = "*****"
      name     = "helm-rundeck"
      type     = "git"
      username = "****"
      password = "***"
    }
  }
}

Steps to Reproduce

Create a Terraform configuration that defines an argocd_project and an argocd_application_set resource, where the ApplicationSet deploys applications within the project.
Run terraform apply to create the resources.
Run terraform destroy to remove the resources.

Expected Behavior

The argocd_application_set resource should first delete the applications it created and only be considered completely destroyed once all applications have been removed. Subsequently, the argocd_project should be successfully deleted, as it no longer has any dependent applications.

Actual Behavior

When running terraform destroy, the argocd_application_set resource is deleted before the applications it created are fully torn down. This causes the subsequent attempt to delete the argocd_project to fail with the error message rpc error: code = InvalidArgument desc = project is referenced by 5 applications.

Important Factoids

The resources are managed on a standard Kubernetes cluster on STACKIT Cloud using the ArgoCD provider. There are no unusual configurations.

Community Note

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

[docbyte86]

Same issue if project got renamed.

When attempting to rename a project that has an existing ApplicationSet dependency, the OpenTofu deployment fails. The operation tries to delete the old project name but is blocked by the ApplicationSet still referencing it, leading to the following error:

Error: failed to delete project testproject
rpc error: code = InvalidArgument desc = project is referenced by x applications

Steps to Reproduce:

1. Deploy a project using OpenTofu.
2. Create an ApplicationSet that references this project.
3. Attempt to rename the project in the OpenTofu configuration.
4. Run tofu apply.

Expected Behavior:

OpenTofu should recognize the dependency and destroy the ApplicationSet before attempting to delete the old project. It should then be able to successfully delete the old project and create the new one with the updated name, along with the corresponding ApplicationSet.

Actual Behavior:

The tofu apply command fails with the project is referenced by 2 applications error, and the project rename operation is aborted.

Impact:

This bug prevents users from renaming projects, requiring manual intervention to destroy dependent resources before a rename can be completed. This adds unnecessary complexity and risk to the deployment process.

[the-technat]

Does the issue still appear when setting a depends_on block on the argocd_application_set resource?

Theoretically terraform should be smart to realize that project = argocd_project.rundeck.metadata[0].name implicitly gives this dependency, but maybe that could help nail down where the bug is.