Skip to content

feat: Add per-workspace backup status tracking with annotations #1550

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tolusha wants to merge 1 commit into
base: main
Choose a base branch
from
+769 −14
Open

feat: Add per-workspace backup status tracking with annotations #1550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 57 additions & 14 deletions controllers/backupcronjob/backupcronjob_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package controllers
import (
"context"
"reflect"
"time"

"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
Expand All @@ -40,6 +41,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
Expand Down Expand Up @@ -93,7 +95,8 @@ func (r *BackupCronJobReconciler) SetupWithManager(mgr ctrl.Manager) error {

return ctrl.NewControllerManagedBy(mgr).
Named("BackupCronJob").
Watches(&controllerv1alpha1.DevWorkspaceOperatorConfig{},
Watches(
&controllerv1alpha1.DevWorkspaceOperatorConfig{},
handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request {
operatorNamespace, err := infrastructure.GetNamespace()
// Ignore events from other namespaces
Expand All @@ -111,17 +114,22 @@ func (r *BackupCronJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
},
}
}),
builder.WithPredicates(configPredicate),
).
Watches(
&batchv1.Job{},
r.getBackupJobEventHandler(),
builder.WithPredicates(r.getBackupJobPredicate()),
).
WithEventFilter(configPredicate).
Complete(r)
}

// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=serviceaccounts;,verbs=get;list;create;update;patch;delete
// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete
// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch
// +kubebuilder:rbac:groups=controller.devfile.io,resources=devworkspaceoperatorconfigs,verbs=get;list;update;patch;watch
// +kubebuilder:rbac:groups=workspace.devfile.io,resources=devworkspaces,verbs=get;list
// +kubebuilder:rbac:groups=workspace.devfile.io,resources=devworkspaces,verbs=get;list;update;patch
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=builds,verbs=get
// +kubebuilder:rbac:groups="",resources=builds/details,verbs=update
Expand Down Expand Up @@ -215,7 +223,7 @@ func (r *BackupCronJobReconciler) stopCron(log logr.Logger) {
}

// executeBackupSync executes the backup job for all DevWorkspaces in the cluster that
// have been stopped in the last N minutes.
// have been stopped since their last backup.
func (r *BackupCronJobReconciler) executeBackupSync(ctx context.Context, dwOperatorConfig *controllerv1alpha1.DevWorkspaceOperatorConfig, log logr.Logger) error {
log.Info("Executing backup sync for all DevWorkspaces")

Expand Down Expand Up @@ -264,31 +272,65 @@ func (r *BackupCronJobReconciler) executeBackupSync(ctx context.Context, dwOpera
return nil
}

// wasStoppedSinceLastBackup checks if the DevWorkspace was stopped since the last backup time.
func (r *BackupCronJobReconciler) wasStoppedSinceLastBackup(workspace *dw.DevWorkspace, lastBackupTime *metav1.Time, log logr.Logger) bool {
// wasStoppedSinceLastBackup checks if the DevWorkspace was stopped since its last backup.
// It reads the last backup time from the DevWorkspace annotation, or falls back to the
// provided globalLastBackupTime if the annotation doesn't exist.
func (r *BackupCronJobReconciler) wasStoppedSinceLastBackup(
workspace *dw.DevWorkspace,
globalLastBackupTime *metav1.Time,
log logr.Logger,
) bool {
if workspace.Status.Phase != dw.DevWorkspaceStatusStopped {
return false
}
log.Info("DevWorkspace is currently stopped, checking if it was stopped since last backup", "namespace", workspace.Namespace, "name", workspace.Name)
// Check if the workspace was stopped in the last N minutes

// Get the last backup time and success status from the workspace annotations
var lastBackupTime *metav1.Time
var lastBackupSuccessful bool
if workspace.Annotations != nil {
if lastBackupTimeStr, ok := workspace.Annotations[constants.DevWorkspaceLastBackupTimeAnnotation]; ok {
parsedTime, err := time.Parse(time.RFC3339Nano, lastBackupTimeStr)
if err != nil {
log.Error(err, "Failed to parse last backup time annotation, treating as no previous backup", "value", lastBackupTimeStr)
} else {
lastBackupTime = &metav1.Time{Time: parsedTime}
}
}

lastBackupSuccessful = workspace.Annotations[constants.DevWorkspaceLastBackupSuccessfulAnnotation] == "true"
}

// Fall back to globalLastBackupTime if annotation doesn't exist
if lastBackupTime == nil && globalLastBackupTime != nil {
lastBackupTime = globalLastBackupTime
}

if lastBackupTime == nil {
return true
}

if !lastBackupSuccessful {
return true
}

// Check if the workspace was stopped since the last successful backup
if workspace.Status.Conditions != nil {
lastTimeStopped := metav1.Time{}
for _, condition := range workspace.Status.Conditions {
if condition.Type == conditions.Started && condition.Status == corev1.ConditionFalse {
lastTimeStopped = condition.LastTransitionTime
}
}

if !lastTimeStopped.IsZero() {
if lastBackupTime == nil {
// No previous backup, so consider it stopped since last backup
return true
}
if lastTimeStopped.Time.After(lastBackupTime.Time) {
log.Info("DevWorkspace was stopped since last backup", "namespace", workspace.Namespace, "name", workspace.Name)
log.Info("DevWorkspace was stopped since last successful backup", "namespace", workspace.Namespace, "name", workspace.Name)
return true
}
}
}

return false
}

Expand Down Expand Up @@ -336,6 +378,7 @@ func (r *BackupCronJobReconciler) createBackupJob(
Namespace: workspace.Namespace,
Labels: map[string]string{
constants.DevWorkspaceIDLabel: dwID,
constants.DevWorkspaceNameLabel: workspace.Name,
constants.DevWorkspaceBackupJobLabel: "true",
},
},
Expand Down Expand Up @@ -532,7 +575,7 @@ func (r *BackupCronJobReconciler) copySecret(ctx context.Context, workspace *dw.
}
err = r.Create(ctx, namespaceSecret)
if err == nil {
log.Info("Sucesfully created secret", "name", namespaceSecret.Name, "namespace", workspace.Namespace)
log.Info("Successfully created secret", "name", namespaceSecret.Name, "namespace", workspace.Namespace)
}
return namespaceSecret, err
}
Loading
Loading