Skip to content

Commit 3e965c1

Browse files
vipul-ybvipul-yb
committed
[PLAT-18466]: Delete PITR schedule at the start of the Upgrade and recreate after rollback/finalize
Summary: Added changes to delete the existing PITR schedule at the start of any(PG-15/non-PG-15) upgrade and re-create it after upgrade completion(rollback/finalise). As PITR schedules are created as part of xCluster DR operations, I added the same changes for DR's PITR schedule. As a result of this change, customers cannot perform any xcluster DR operation when either cluster is in the upgrade process, except for failover when the target universe is not in the upgrade phase. Test Plan: Added unit tests Tested manually: - Create a universe with pitr enabled, perform an upgrade and verify that pitr schedule is deleted from the universe. Perform finalise/rollback and verify a new pitr schedule is created after the task with the same parameters. - Created a xcluster DR setup and performed an upgrade on the target universe and verified that the target universe pitr is deleted in the monitoring phase and re-created after the upgrade completion. Also, verified the same for the source universe. Reviewers: anijhawan, vkumar, hzare, hvontela Reviewed By: hzare Subscribers: yugaware Differential Revision: https://phorge.dev.yugabyte.com/D46581
1 parent fbe9dfa commit 3e965c1

23 files changed

+590
-143
lines changed

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@
3636
import com.yugabyte.yw.commissioner.tasks.subtasks.CheckUnderReplicatedTablets;
3737
import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteCapacityReservation;
3838
import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteClusterFromUniverse;
39+
import com.yugabyte.yw.commissioner.tasks.subtasks.DisablePitrConfig;
3940
import com.yugabyte.yw.commissioner.tasks.subtasks.DoCapacityReservation;
41+
import com.yugabyte.yw.commissioner.tasks.subtasks.EnablePitrConfig;
4042
import com.yugabyte.yw.commissioner.tasks.subtasks.InstanceActions;
4143
import com.yugabyte.yw.commissioner.tasks.subtasks.InstanceExistCheck;
4244
import com.yugabyte.yw.commissioner.tasks.subtasks.ManageCatalogUpgradeSuperUser.Action;
@@ -3892,15 +3894,39 @@ protected void createFinalizeUpgradeTasks(
38923894
log.info("Skipping upgrade finalization for universe : " + universe.getUniverseUUID());
38933895
}
38943896

3895-
// Update PITR configs to set intermittentMinRecoverTimeInMillis to current time
3896-
// as PITR configs are only valid from the completion of software upgrade finalization
3897-
createUpdatePitrConfigIntermittentMinRecoverTimeTask();
3897+
// Re-enable PITR configs after successful upgrade finalization
3898+
// This also updates intermittentMinRecoverTimeInMillis for all PITR configs
3899+
createEnablePitrConfigTask();
38983900

38993901
createUpdateUniverseSoftwareUpgradeStateTask(
39003902
UniverseDefinitionTaskParams.SoftwareUpgradeState.Ready,
39013903
false /* isSoftwareRollbackAllowed */);
39023904
}
39033905

3906+
protected void createEnablePitrConfigTask() {
3907+
SubTaskGroup subTaskGroup =
3908+
createSubTaskGroup("EnablePitrConfig", SubTaskGroupType.ConfigureUniverse);
3909+
EnablePitrConfig.Params params = new EnablePitrConfig.Params();
3910+
params.setUniverseUUID(taskParams().getUniverseUUID());
3911+
3912+
EnablePitrConfig task = createTask(EnablePitrConfig.class);
3913+
task.initialize(params);
3914+
subTaskGroup.addSubTask(task);
3915+
getRunnableTask().addSubTaskGroup(subTaskGroup);
3916+
}
3917+
3918+
public void createDisablePitrConfigTask() {
3919+
SubTaskGroup subTaskGroup =
3920+
createSubTaskGroup("DisablePitrConfig", SubTaskGroupType.ConfigureUniverse);
3921+
DisablePitrConfig.Params params = new DisablePitrConfig.Params();
3922+
params.setUniverseUUID(taskParams().getUniverseUUID());
3923+
params.ignoreErrors = true; // Ignore errors to not block upgrade if PITR configs don't exist
3924+
DisablePitrConfig task = createTask(DisablePitrConfig.class);
3925+
task.initialize(params);
3926+
subTaskGroup.addSubTask(task);
3927+
getRunnableTask().addSubTaskGroup(subTaskGroup);
3928+
}
3929+
39043930
protected void createSetYBMajorVersionUpgradeCompatibility(
39053931
Universe universe, ServerType serverType, List<NodeDetails> nodes, String flagValue) {
39063932
if (nodes.isEmpty()) {

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/DisablePitrConfig.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright (c) YugaByte, Inc.
2+
package com.yugabyte.yw.commissioner.tasks.subtasks;
3+
4+
import com.yugabyte.yw.commissioner.BaseTaskDependencies;
5+
import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase;
6+
import com.yugabyte.yw.forms.UniverseTaskParams;
7+
import com.yugabyte.yw.models.PitrConfig;
8+
import com.yugabyte.yw.models.Universe;
9+
import java.util.List;
10+
import javax.inject.Inject;
11+
import lombok.extern.slf4j.Slf4j;
12+
import org.yb.client.DeleteSnapshotScheduleResponse;
13+
import org.yb.client.ListSnapshotSchedulesResponse;
14+
import org.yb.client.SnapshotScheduleInfo;
15+
import org.yb.client.YBClient;
16+
17+
@Slf4j
18+
public class DisablePitrConfig extends UniverseTaskBase {
19+
20+
@Inject
21+
protected DisablePitrConfig(BaseTaskDependencies baseTaskDependencies) {
22+
super(baseTaskDependencies);
23+
}
24+
25+
public static class Params extends UniverseTaskParams {
26+
// The universe UUID must be stored in universeUUID field.
27+
28+
public boolean ignoreErrors;
29+
}
30+
31+
@Override
32+
protected Params taskParams() {
33+
return (Params) taskParams;
34+
}
35+
36+
@Override
37+
public String getName() {
38+
return String.format("%s(Universe=%s)", super.getName(), taskParams().getUniverseUUID());
39+
}
40+
41+
@Override
42+
public void run() {
43+
Universe universe = Universe.getOrBadRequest(taskParams().getUniverseUUID());
44+
List<PitrConfig> pitrConfigs = PitrConfig.getByUniverseUUID(taskParams().getUniverseUUID());
45+
46+
if (pitrConfigs.isEmpty()) {
47+
log.info("No PITR configs found for universe {}", taskParams().getUniverseUUID());
48+
return;
49+
}
50+
51+
log.info(
52+
"Found {} PITR configs to disable for universe {}",
53+
pitrConfigs.size(),
54+
taskParams().getUniverseUUID());
55+
56+
try (YBClient client = ybService.getUniverseClient(universe)) {
57+
ListSnapshotSchedulesResponse scheduleListResp = client.listSnapshotSchedules(null);
58+
59+
for (PitrConfig pitrConfig : pitrConfigs) {
60+
try {
61+
// Find and delete the snapshot schedule for this PITR config
62+
for (SnapshotScheduleInfo scheduleInfo : scheduleListResp.getSnapshotScheduleInfoList()) {
63+
if (scheduleInfo.getSnapshotScheduleUUID().equals(pitrConfig.getUuid())) {
64+
DeleteSnapshotScheduleResponse resp =
65+
client.deleteSnapshotSchedule(pitrConfig.getUuid());
66+
if (resp.hasError()) {
67+
String errorMsg =
68+
"Failed to delete snapshot schedule for PITR config "
69+
+ pitrConfig.getUuid()
70+
+ " due to error: "
71+
+ resp.errorMessage();
72+
log.error(errorMsg);
73+
if (!taskParams().ignoreErrors) {
74+
throw new RuntimeException(errorMsg);
75+
}
76+
} else {
77+
log.debug(
78+
"Successfully deleted snapshot schedule for PITR config {}",
79+
pitrConfig.getUuid());
80+
}
81+
break;
82+
}
83+
}
84+
85+
// Mark the PITR config as disabled
86+
pitrConfig.updateDisabledStatus(true);
87+
log.info(
88+
"Successfully disabled PITR config {} for universe {}",
89+
pitrConfig.getUuid(),
90+
taskParams().getUniverseUUID());
91+
} catch (Exception e) {
92+
log.error("Error disabling PITR config {}: {}", pitrConfig.getUuid(), e.getMessage());
93+
if (!taskParams().ignoreErrors) {
94+
throw new RuntimeException("Failed to disable PITR config " + pitrConfig.getUuid(), e);
95+
}
96+
}
97+
}
98+
} catch (Exception e) {
99+
log.error("{} hit error : {}", getName(), e.getMessage());
100+
if (taskParams().ignoreErrors) {
101+
log.debug(
102+
"Ignoring error disabling PITR configs for universe: {} as ignoreErrors is set to true."
103+
+ " Error: {}",
104+
taskParams().getUniverseUUID(),
105+
e.getMessage());
106+
return;
107+
}
108+
throw new RuntimeException(e);
109+
}
110+
}
111+
}

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/EnablePitrConfig.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Copyright (c) YugaByte, Inc.
2+
package com.yugabyte.yw.commissioner.tasks.subtasks;
3+
4+
import static com.yugabyte.yw.common.backuprestore.BackupUtil.TABLE_TYPE_TO_YQL_DATABASE_MAP;
5+
6+
import com.yugabyte.yw.commissioner.BaseTaskDependencies;
7+
import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase;
8+
import com.yugabyte.yw.forms.CreatePitrConfigParams;
9+
import com.yugabyte.yw.forms.UniverseTaskParams;
10+
import com.yugabyte.yw.models.PitrConfig;
11+
import com.yugabyte.yw.models.Universe;
12+
import io.ebean.annotation.Transactional;
13+
import java.util.List;
14+
import java.util.Objects;
15+
import java.util.UUID;
16+
import javax.inject.Inject;
17+
import lombok.extern.slf4j.Slf4j;
18+
import org.yb.client.CreateSnapshotScheduleResponse;
19+
import org.yb.client.YBClient;
20+
21+
@Slf4j
22+
public class EnablePitrConfig extends UniverseTaskBase {
23+
24+
@Inject
25+
protected EnablePitrConfig(BaseTaskDependencies baseTaskDependencies) {
26+
super(baseTaskDependencies);
27+
}
28+
29+
public static class Params extends UniverseTaskParams {
30+
// The universe UUID must be stored in universeUUID field.
31+
}
32+
33+
@Override
34+
protected Params taskParams() {
35+
return (Params) taskParams;
36+
}
37+
38+
@Override
39+
public String getName() {
40+
return String.format("%s(Universe=%s)", super.getName(), taskParams().getUniverseUUID());
41+
}
42+
43+
@Override
44+
public void run() {
45+
Universe universe = Universe.getOrBadRequest(taskParams().getUniverseUUID());
46+
List<PitrConfig> disabledPitrConfigs =
47+
PitrConfig.getDisabledByUniverseUUID(taskParams().getUniverseUUID());
48+
49+
if (disabledPitrConfigs.isEmpty()) {
50+
log.info("No disabled PITR configs found for universe {}", taskParams().getUniverseUUID());
51+
return;
52+
}
53+
54+
log.info(
55+
"Found {} disabled PITR configs to enable for universe {}",
56+
disabledPitrConfigs.size(),
57+
taskParams().getUniverseUUID());
58+
59+
try (YBClient client = ybService.getUniverseClient(universe)) {
60+
for (PitrConfig pitrConfig : disabledPitrConfigs) {
61+
try {
62+
// Find the keyspace id of the keyspace name specified in the pitr config.
63+
String keyspaceId =
64+
getKeyspaceNameKeyspaceIdMap(client, pitrConfig.getTableType())
65+
.get(pitrConfig.getDbName());
66+
if (Objects.isNull(keyspaceId)) {
67+
String errorMsg =
68+
String.format(
69+
"A keyspace with name %s and table type %s could not be found for PITR config"
70+
+ " %s",
71+
pitrConfig.getDbName(), pitrConfig.getTableType(), pitrConfig.getUuid());
72+
log.error(errorMsg);
73+
throw new IllegalArgumentException(errorMsg);
74+
}
75+
log.debug(
76+
"Found keyspace id {} for keyspace name {} for PITR config {}",
77+
keyspaceId,
78+
pitrConfig.getDbName(),
79+
pitrConfig.getUuid());
80+
81+
// Create the PITR config on DB with existing parameters.
82+
CreateSnapshotScheduleResponse resp =
83+
client.createSnapshotSchedule(
84+
TABLE_TYPE_TO_YQL_DATABASE_MAP.get(pitrConfig.getTableType()),
85+
pitrConfig.getDbName(),
86+
keyspaceId,
87+
pitrConfig.getRetentionPeriod(),
88+
pitrConfig.getScheduleInterval());
89+
if (resp.hasError()) {
90+
String errorMsg =
91+
"Failed to create snapshot schedule for PITR config "
92+
+ pitrConfig.getUuid()
93+
+ " due to error: "
94+
+ resp.errorMessage();
95+
log.error(errorMsg);
96+
throw new RuntimeException(errorMsg);
97+
}
98+
99+
reCreatePitrConfig(pitrConfig, resp.getSnapshotScheduleUUID());
100+
} catch (Exception e) {
101+
log.error("Error enabling PITR config {}: {}", pitrConfig.getUuid(), e.getMessage());
102+
throw new RuntimeException("Failed to enable PITR config " + pitrConfig.getUuid(), e);
103+
}
104+
}
105+
} catch (Exception e) {
106+
log.error("{} hit error : {}", getName(), e.getMessage());
107+
throw new RuntimeException(e);
108+
}
109+
}
110+
111+
@Transactional
112+
private void reCreatePitrConfig(PitrConfig pitrConfig, UUID scheduleUUID) {
113+
CreatePitrConfigParams params = pitrConfig.toCreatePitrConfigParams();
114+
pitrConfig.delete();
115+
116+
PitrConfig newPitrConfig = PitrConfig.create(scheduleUUID, params);
117+
118+
log.info(
119+
"Successfully enabled new PITR config {} for universe {}",
120+
newPitrConfig.getUuid(),
121+
taskParams().getUniverseUUID());
122+
}
123+
}

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdatePitrConfigIntermittentMinRecoverTime.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
88
import com.yugabyte.yw.models.PitrConfig;
99
import com.yugabyte.yw.models.Universe;
10-
import java.util.Date;
1110
import java.util.List;
1211
import javax.inject.Inject;
1312
import lombok.extern.slf4j.Slf4j;
@@ -16,7 +15,11 @@
1615
* Subtask to update intermittentMinRecoverTimeInMillis for all PITR configs associated with a
1716
* universe. This is used during software upgrade and rollback operations to ensure PITR configs are
1817
* only valid from the completion of these operations.
18+
*
19+
* @deprecated This functionality has been merged into EnablePitrConfig task. Use EnablePitrConfig
20+
* instead.
1921
*/
22+
@Deprecated
2023
@Slf4j
2124
public class UpdatePitrConfigIntermittentMinRecoverTime extends UniverseTaskBase {
2225

@@ -52,9 +55,7 @@ public void run() {
5255
currentTimeInMillis);
5356

5457
for (PitrConfig pitrConfig : pitrConfigs) {
55-
pitrConfig.setIntermittentMinRecoverTimeInMillis(currentTimeInMillis);
56-
pitrConfig.setUpdateTime(new Date());
57-
pitrConfig.update();
58+
pitrConfig.updateIntermittentMinRecoverTimeInMillis(currentTimeInMillis);
5859
log.debug(
5960
"Updated PITR config {} with intermittentMinRecoverTimeInMillis={}",
6061
pitrConfig.getUuid(),

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackKubernetesUpgrade.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,12 +162,12 @@ public void run() {
162162
if (requireAdditionalSuperUserForCatalogUpgrade) {
163163
createManageCatalogUpgradeSuperUserTask(Action.DELETE_USER);
164164
}
165-
166-
// Update PITR configs to set intermittentMinRecoverTimeInMillis to current time
167-
// as PITR configs are only valid from the completion of software rollback
168-
createUpdatePitrConfigIntermittentMinRecoverTimeTask();
169165
}
170166

167+
// Re-enable PITR configs after successful rollback
168+
// This also updates intermittentMinRecoverTimeInMillis for all PITR configs
169+
createEnablePitrConfigTask();
170+
171171
// Update Software version
172172
createUpdateSoftwareVersionTask(targetVersion, false /*isSoftwareUpdateViaVm*/)
173173
.setSubTaskGroupType(getTaskSubGroupType());

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackUpgrade.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -175,12 +175,12 @@ public void run() {
175175
if (requireAdditionalSuperUserForCatalogUpgrade) {
176176
createManageCatalogUpgradeSuperUserTask(Action.DELETE_USER);
177177
}
178-
179-
// Update PITR configs to set intermittentMinRecoverTimeInMillis to current time
180-
// as PITR configs are only valid from the completion of software rollback
181-
createUpdatePitrConfigIntermittentMinRecoverTimeTask();
182178
}
183179

180+
// Re-enable PITR configs after successful rollback
181+
// This also updates intermittentMinRecoverTimeInMillis for all PITR configs
182+
createEnablePitrConfigTask();
183+
184184
// Check software version on each node.
185185
createCheckSoftwareVersionTask(allNodes, oldVersion);
186186

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeYB.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ public void run() {
9494

9595
createStoreAutoFlagConfigVersionTask(taskParams().getUniverseUUID(), newVersion);
9696

97+
// Disable PITR configs at the start of software upgrade
98+
createDisablePitrConfigTask();
99+
97100
String password = null;
98101
boolean catalogUpgradeCompleted = false;
99102

‎managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareUpgradeYB.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ public void run() {
9292
UniverseDefinitionTaskParams.SoftwareUpgradeState.Upgrading,
9393
true /* isSoftwareRollbackAllowed */);
9494

95+
// Disable PITR configs at the start of software upgrade
96+
createDisablePitrConfigTask();
97+
9598
if (!universe
9699
.getUniverseDetails()
97100
.xClusterInfo

0 commit comments

Comments
 (0)