Fix config connector healthcheck race condition with observedGeneration, add generation tests

Signed-off-by: danko <[email protected]>

Author: adam-elementor

resource_customizations/cloudfunctions.cnrm.cloud.google.com/CloudFunctionsFunction/health.lua

@@ -4,34 +4,38 @@ local hs = {
 }
 if obj.status ~= nil then
   if obj.status.conditions ~= nil then
-    for i, condition in ipairs(obj.status.conditions) do
+    
+    -- Don't change health if the resource status is stale, skip if observedGeneration is not set
+    if obj.status.observedGeneration == nil or obj.status.observedGeneration == obj.metadata.generation then
+      for i, condition in ipairs(obj.status.conditions) do
 
-      -- Up To Date
-      if condition.reason == "UpToDate" and condition.status == "True" then
-        hs.status = "Healthy"
-        hs.message = condition.message
-        return hs
-      end
+        -- Up To Date
+        if condition.reason == "UpToDate" and condition.status == "True" then
+          hs.status = "Healthy"
+          hs.message = condition.message
+          return hs
+        end
 
-      -- Update Failed
-      if condition.reason == "UpdateFailed" then
-        hs.status = "Degraded"
-        hs.message = condition.message
-        return hs
-      end
+        -- Update Failed
+        if condition.reason == "UpdateFailed" then
+          hs.status = "Degraded"
+          hs.message = condition.message
+          return hs
+        end
 
-      -- Dependency Not Found
-      if condition.reason == "DependencyNotFound" then
-        hs.status = "Degraded"
-        hs.message = condition.message
-        return hs
-      end
+        -- Dependency Not Found
+        if condition.reason == "DependencyNotFound" then
+          hs.status = "Degraded"
+          hs.message = condition.message
+          return hs
+        end
 
-      -- Dependency Not Ready
-      if condition.reason == "DependencyNotReady" then
-        hs.status = "Suspended"
-        hs.message = condition.message
-        return hs
+        -- Dependency Not Ready
+        if condition.reason == "DependencyNotReady" then
+          hs.status = "Suspended"
+          hs.message = condition.message
+          return hs
+        end
       end
     end
   end

resource_customizations/cloudfunctions.cnrm.cloud.google.com/CloudFunctionsFunction/health_test.yaml

@@ -19,3 +19,7 @@ tests:
     status: Progressing
     message: "Update in progress"
   inputPath: testdata/update_in_progress.yaml
+- healthStatus:
+    status: Progressing
+    message: 'Update in progress'
+  inputPath: testdata/generation.yaml

resource_customizations/cloudfunctions.cnrm.cloud.google.com/CloudFunctionsFunction/testdata/dependency_not_found.yaml

@@ -1,6 +1,9 @@
 apiVersion: cloudfunctions.cnrm.cloud.google.com/v1beta1
 kind: CloudFunctionsFunction
+metadata:
+  generation: 1
 status:
+  observedGeneration: 1
   conditions:
     - lastTransitionTime: '2022-07-01T12:56:21Z'
       message: Dependency not found

resource_customizations/cloudfunctions.cnrm.cloud.google.com/CloudFunctionsFunction/testdata/dependency_not_ready.yaml

@@ -1,6 +1,9 @@
 apiVersion: cloudfunctions.cnrm.cloud.google.com/v1beta1
 kind: CloudFunctionsFunction
+metadata:
+  generation: 1
 status:
+  observedGeneration: 1
   conditions:
     - lastTransitionTime: '2022-07-01T12:56:21Z'
       message: Dependency not ready

resource_customizations/cloudfunctions.cnrm.cloud.google.com/CloudFunctionsFunction/testdata/generation.yaml

@@ -0,0 +1,12 @@
+apiVersion: cloudfunctions.cnrm.cloud.google.com/v1beta1
+kind: CloudFunctionsFunction
+metadata:
+  generation: 2
+status:
+  observedGeneration: 1
+  conditions:
+    - lastTransitionTime: '2022-05-09T08:49:18Z'
+      message: The resource is up to date
+      reason: UpToDate
+      status: 'True'
+      type: Ready

resource_customizations/cloudfunctions.cnrm.cloud.google.com/CloudFunctionsFunction/testdata/up_to_date.yaml

@@ -1,6 +1,9 @@
 apiVersion: cloudfunctions.cnrm.cloud.google.com/v1beta1
 kind: CloudFunctionsFunction
+metadata:
+  generation: 1
 status:
+  observedGeneration: 1
   conditions:
     - lastTransitionTime: '2022-05-09T08:49:18Z'
       message: The resource is up to date

resource_customizations/cloudfunctions.cnrm.cloud.google.com/CloudFunctionsFunction/testdata/update_failed.yaml

@@ -1,6 +1,9 @@
 apiVersion: cloudfunctions.cnrm.cloud.google.com/v1beta1
 kind: CloudFunctionsFunction
+metadata:
+  generation: 1
 status:
+  observedGeneration: 1
   conditions:
     - lastTransitionTime: '2022-07-01T12:56:21Z'
       message: Update failed

resource_customizations/cloudscheduler.cnrm.cloud.google.com/CloudSchedulerJob/health.lua

@@ -4,34 +4,38 @@ local hs = {
 }
 if obj.status ~= nil then
   if obj.status.conditions ~= nil then
-    for i, condition in ipairs(obj.status.conditions) do
+    
+    -- Don't change health if the resource status is stale, skip if observedGeneration is not set
+    if obj.status.observedGeneration == nil or obj.status.observedGeneration == obj.metadata.generation then
+      for i, condition in ipairs(obj.status.conditions) do
 
-      -- Up To Date
-      if condition.reason == "UpToDate" and condition.status == "True" then
-        hs.status = "Healthy"
-        hs.message = condition.message
-        return hs
-      end
+        -- Up To Date
+        if condition.reason == "UpToDate" and condition.status == "True" then
+          hs.status = "Healthy"
+          hs.message = condition.message
+          return hs
+        end
 
-      -- Update Failed
-      if condition.reason == "UpdateFailed" then
-        hs.status = "Degraded"
-        hs.message = condition.message
-        return hs
-      end
+        -- Update Failed
+        if condition.reason == "UpdateFailed" then
+          hs.status = "Degraded"
+          hs.message = condition.message
+          return hs
+        end
 
-      -- Dependency Not Found
-      if condition.reason == "DependencyNotFound" then
-        hs.status = "Degraded"
-        hs.message = condition.message
-        return hs
-      end
+        -- Dependency Not Found
+        if condition.reason == "DependencyNotFound" then
+          hs.status = "Degraded"
+          hs.message = condition.message
+          return hs
+        end
 
-      -- Dependency Not Ready
-      if condition.reason == "DependencyNotReady" then
-        hs.status = "Suspended"
-        hs.message = condition.message
-        return hs
+        -- Dependency Not Ready
+        if condition.reason == "DependencyNotReady" then
+          hs.status = "Suspended"
+          hs.message = condition.message
+          return hs
+        end
       end
     end
   end

resource_customizations/cloudscheduler.cnrm.cloud.google.com/CloudSchedulerJob/health_test.yaml

@@ -19,3 +19,7 @@ tests:
     status: Progressing
     message: "Update in progress"
   inputPath: testdata/update_in_progress.yaml
+- healthStatus:
+    status: Progressing
+    message: 'Update in progress'
+  inputPath: testdata/generation.yaml

resource_customizations/cloudscheduler.cnrm.cloud.google.com/CloudSchedulerJob/testdata/dependency_not_found.yaml

@@ -1,6 +1,9 @@
 apiVersion: cloudscheduler.cnrm.cloud.google.com/v1beta1
 kind: CloudSchedulerJob
+metadata:
+  generation: 1
 status:
+  observedGeneration: 1
   conditions:
     - lastTransitionTime: '2022-07-01T12:56:21Z'
       message: Dependency not found