diff --git a/config/crds/migration.kcp.io_logicalclustermigrations.yaml b/config/crds/migration.kcp.io_logicalclustermigrations.yaml index c8cb4536e92..fcbc3ae7fcb 100644 --- a/config/crds/migration.kcp.io_logicalclustermigrations.yaml +++ b/config/crds/migration.kcp.io_logicalclustermigrations.yaml @@ -132,6 +132,7 @@ spec: - Migrating - OriginCleanup - DestinationFinalize + - Aborting - Completed - Failed type: string diff --git a/config/root-phase0/apiexport-migration.kcp.io.yaml b/config/root-phase0/apiexport-migration.kcp.io.yaml index ed510d59a93..62ebcd547f7 100644 --- a/config/root-phase0/apiexport-migration.kcp.io.yaml +++ b/config/root-phase0/apiexport-migration.kcp.io.yaml @@ -11,7 +11,7 @@ spec: crd: {} - group: migration.kcp.io name: logicalclustermigrations - schema: v260604-70219e163.logicalclustermigrations.migration.kcp.io + schema: v260624-65ebd3edd.logicalclustermigrations.migration.kcp.io storage: crd: {} status: {} diff --git a/config/root-phase0/apiresourceschema-logicalclustermigrations.migration.kcp.io.yaml b/config/root-phase0/apiresourceschema-logicalclustermigrations.migration.kcp.io.yaml index f1811e467aa..f87eae4ba68 100644 --- a/config/root-phase0/apiresourceschema-logicalclustermigrations.migration.kcp.io.yaml +++ b/config/root-phase0/apiresourceschema-logicalclustermigrations.migration.kcp.io.yaml @@ -1,7 +1,7 @@ apiVersion: apis.kcp.io/v1alpha1 kind: APIResourceSchema metadata: - name: v260604-70219e163.logicalclustermigrations.migration.kcp.io + name: v260624-65ebd3edd.logicalclustermigrations.migration.kcp.io spec: group: migration.kcp.io names: @@ -127,6 +127,7 @@ spec: - Migrating - OriginCleanup - DestinationFinalize + - Aborting - Completed - Failed type: string diff --git a/pkg/reconciler/core/logicalclusterdeletion/logicalcluster_deletion_controller.go b/pkg/reconciler/core/logicalclusterdeletion/logicalcluster_deletion_controller.go index 614f9858b4b..b3474d9ac7f 100644 --- a/pkg/reconciler/core/logicalclusterdeletion/logicalcluster_deletion_controller.go +++ b/pkg/reconciler/core/logicalclusterdeletion/logicalcluster_deletion_controller.go @@ -252,6 +252,14 @@ func (c *Controller) process(ctx context.Context, key string) error { return nil } + // If the LC is currently being migrated, skip reconciliation. The + // migration controller handles etcd cleanup directly for migrating + // clusters via its Aborting phase. + if logicalCluster.Annotations["internal.kcp.io/migrating"] != "" { + logger.V(4).Info("logical cluster is migrating, skipping deletion reconciliation") + return nil + } + logicalClusterCopy := logicalCluster.DeepCopy() logger.V(2).Info("deleting logical cluster") diff --git a/pkg/reconciler/migration/logicalclustermigration/controller.go b/pkg/reconciler/migration/logicalclustermigration/controller.go index 60b2b130662..06b81fd37cc 100644 --- a/pkg/reconciler/migration/logicalclustermigration/controller.go +++ b/pkg/reconciler/migration/logicalclustermigration/controller.go @@ -24,6 +24,7 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -84,6 +85,7 @@ func NewController( etcdStoragePrefix: etcdStoragePrefix, logicalClusterLister: localLogicalClusterInformer.Lister(), shardLister: cachedShardInformer.Lister(), + migrationIndexer: cachedLogicalClusterMigrationInformer.Informer().GetIndexer(), migratingLogicalClusters: migratingLogicalClusters, cancelLogicalClusterConnections: cancelLogicalClusterConnections, ddsif: ddsif, @@ -102,6 +104,14 @@ func NewController( DeleteFunc: func(obj interface{}) { c.enqueue(obj) }, }) + // Watch local LogicalClusters for deletion during migration. + // When a workspace is deleted, the LC gets a DeletionTimestamp; + // we need to enqueue the corresponding migration to trigger abort. + _, _ = localLogicalClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, obj interface{}) { c.enqueueFromLogicalCluster(obj) }, + DeleteFunc: func(obj interface{}) { c.enqueueFromLogicalCluster(obj) }, + }) + return c, nil } @@ -119,6 +129,7 @@ type Controller struct { logicalClusterLister corev1alpha1listers.LogicalClusterClusterLister shardLister corev1alpha1listers.ShardClusterLister + migrationIndexer cache.Indexer migratingLogicalClusters *MigratingLogicalClusters cancelLogicalClusterConnections func(logicalcluster.Path, error) @@ -138,6 +149,41 @@ func (c *Controller) enqueue(obj interface{}) { c.queue.Add(key) } +// enqueueFromLogicalCluster handles events from the local LogicalCluster +// informer. If the LC is currently migrating, it enqueues the corresponding +// LogicalClusterMigration so the reconciler can detect deletion and abort. +func (c *Controller) enqueueFromLogicalCluster(obj interface{}) { + metaObj, err := meta.Accessor(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + + lcName := logicalcluster.From(metaObj) + if !c.migratingLogicalClusters.IsMigrating(lcName) { + return + } + + // Find the migration that references this LC by scanning the cached informer store. + for _, item := range c.migrationIndexer.List() { + migration, ok := item.(*migrationv1alpha1.LogicalClusterMigration) + if !ok { + continue + } + if migration.Spec.LogicalCluster == string(lcName) { + key, err := kcpcache.MetaClusterNamespaceKeyFunc(migration) + if err != nil { + utilruntime.HandleError(err) + continue + } + logger := logging.WithQueueKey(logging.WithReconciler(klog.Background(), ControllerName), key) + logger.V(4).Info("queueing LogicalClusterMigration from LogicalCluster event", "logicalCluster", lcName) + c.queue.Add(key) + return + } + } +} + func (c *Controller) Start(ctx context.Context, numThreads int) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() diff --git a/pkg/reconciler/migration/logicalclustermigration/reconciler.go b/pkg/reconciler/migration/logicalclustermigration/reconciler.go index 1f2b6c2670a..9ae78f5ad00 100644 --- a/pkg/reconciler/migration/logicalclustermigration/reconciler.go +++ b/pkg/reconciler/migration/logicalclustermigration/reconciler.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -42,6 +43,32 @@ func (c *Controller) reconcile(ctx context.Context, migration *migrationv1alpha1 return false, nil } + // Detect LC deletion during migration. If the origin shard sees the LC + // has a DeletionTimestamp and we're not already in a terminal or aborting + // phase, transition to Aborting to clean up data on both shards. + if isOrigin && !isTerminalOrAborting(migration.Status.Phase) { + lcName := logicalcluster.Name(migration.Spec.LogicalCluster) + lc, err := c.logicalClusterLister.Cluster(lcName).Get(corev1alpha1.LogicalClusterName) + if err == nil && !lc.DeletionTimestamp.IsZero() { + logger.V(2).Info("LogicalCluster is being deleted, transitioning to Aborting", "logicalCluster", lcName) + migration.Status.Phase = migrationv1alpha1.LogicalClusterMigrationPhaseAborting + return true, nil + } + // If the LC is not found in the lister, it may have already been + // fully deleted. Transition to Aborting to ensure cleanup. + if apierrors.IsNotFound(err) { + logger.V(2).Info("LogicalCluster not found in lister, transitioning to Aborting", "logicalCluster", lcName) + migration.Status.Phase = migrationv1alpha1.LogicalClusterMigrationPhaseAborting + return true, nil + } + } + + // Requeue on the origin shard during phases where the destination is + // active. This ensures the deletion detection above re-runs even if + // the LC informer event is missed. + requeueForDeletionDetection := isOrigin && (migration.Status.Phase == migrationv1alpha1.LogicalClusterMigrationPhaseMigrating || + migration.Status.Phase == migrationv1alpha1.LogicalClusterMigrationPhaseDestinationFinalize) + // Route through the process. The overview of the process id // described in the doc.go, details are in the methods. switch migration.Status.Phase { @@ -65,7 +92,7 @@ func (c *Controller) reconcile(ctx context.Context, migration *migrationv1alpha1 if isDestination { return c.reconcileMigrating(ctx, migration) } - return false, nil + return requeueForDeletionDetection, nil case migrationv1alpha1.LogicalClusterMigrationPhaseOriginCleanup: if isOrigin { @@ -77,7 +104,10 @@ func (c *Controller) reconcile(ctx context.Context, migration *migrationv1alpha1 if isDestination { return c.reconcileDestinationFinalize(ctx, migration) } - return false, nil + return requeueForDeletionDetection, nil + + case migrationv1alpha1.LogicalClusterMigrationPhaseAborting: + return c.reconcileAborting(ctx, migration) default: logger.V(2).Info("unknown phase", "phase", migration.Status.Phase) @@ -275,3 +305,58 @@ func (c *Controller) reconcileDestinationFinalize(ctx context.Context, migration logger.V(2).Info("migration completed", "logicalCluster", lcName) return false, nil } + +// reconcileAborting handles the Aborting phase, which is entered when the +// LogicalCluster is being deleted during an active migration. Both origin and +// destination shards clean up their local etcd data independently. +func (c *Controller) reconcileAborting(ctx context.Context, migration *migrationv1alpha1.LogicalClusterMigration) (bool, error) { + logger := klog.FromContext(ctx) + lcName := logicalcluster.Name(migration.Spec.LogicalCluster) + + logger.V(2).Info("aborting migration due to LogicalCluster deletion", "logicalCluster", lcName) + + // Each shard cleans up its own local etcd data. + // deleteOriginData() is idempotent — if no data exists locally, it's a no-op. + if c.isOrigin(migration) || c.isDestination(migration) { + if err := c.deleteOriginData(ctx, lcName); err != nil { + return true, err + } + c.migratingLogicalClusters.Remove(lcName) + } + + // Track per-shard completion via conditions. + if c.isOrigin(migration) { + conditions.MarkTrue(migration, migrationv1alpha1.LCMigrationOriginCleaned) + } + if c.isDestination(migration) { + conditions.MarkTrue(migration, migrationv1alpha1.LCMigrationDestinationCleaned) + } + + // Both shards done? Transition to terminal Failed state. + // If data was never copied to the destination (DataCopied is not True), + // there's nothing to clean on the destination — only origin cleanup is needed. + originDone := conditions.IsTrue(migration, migrationv1alpha1.LCMigrationOriginCleaned) + destinationDone := conditions.IsTrue(migration, migrationv1alpha1.LCMigrationDestinationCleaned) + destinationNeverHadData := !conditions.IsTrue(migration, migrationv1alpha1.LCMigrationDataCopied) + + if originDone && (destinationDone || destinationNeverHadData) { + migration.Status.Phase = migrationv1alpha1.LogicalClusterMigrationPhaseFailed + conditions.MarkTrue(migration, migrationv1alpha1.LCMigrationAborted) + logger.V(2).Info("abort complete, transitioning to Failed", "logicalCluster", lcName) + } + + return false, nil +} + +// isTerminalOrAborting returns true if the migration is in a terminal state +// or already aborting. +func isTerminalOrAborting(phase migrationv1alpha1.LogicalClusterMigrationPhaseType) bool { + switch phase { + case migrationv1alpha1.LogicalClusterMigrationPhaseCompleted, + migrationv1alpha1.LogicalClusterMigrationPhaseFailed, + migrationv1alpha1.LogicalClusterMigrationPhaseAborting: + return true + default: + return false + } +} diff --git a/pkg/server/filters/migratinglogicalcluster.go b/pkg/server/filters/migratinglogicalcluster.go index 842f75db801..2ddb8d6b7d2 100644 --- a/pkg/server/filters/migratinglogicalcluster.go +++ b/pkg/server/filters/migratinglogicalcluster.go @@ -49,10 +49,13 @@ func WithBlockMigratingLogicalClusters(handler http.Handler, isMigrating func(lo } userInfo, ok := request.UserFrom(req.Context()) - if ok && slices.Contains(userInfo.GetGroups(), bootstrappolicy.SystemExternalLogicalClusterAdmin) { - // allow system:kcp:external-logical-cluster-admin to pass, - // required for the destination shard to get - // logicalclusterdump from the migrating workspace + if ok && (slices.Contains(userInfo.GetGroups(), bootstrappolicy.SystemExternalLogicalClusterAdmin) || + slices.Contains(userInfo.GetGroups(), bootstrappolicy.SystemMastersGroup)) { + // allow system:kcp:external-logical-cluster-admin and + // system:masters to pass, required for the destination shard + // to get logicalclusterdump from the migrating workspace and + // for internal controllers (e.g. workspace deletion) to manage + // the LogicalCluster during migration. handler.ServeHTTP(w, req) return } diff --git a/staging/src/github.com/kcp-dev/sdk/apis/migration/v1alpha1/types_logicalclustermigration.go b/staging/src/github.com/kcp-dev/sdk/apis/migration/v1alpha1/types_logicalclustermigration.go index 1a0331170dd..24527b76940 100644 --- a/staging/src/github.com/kcp-dev/sdk/apis/migration/v1alpha1/types_logicalclustermigration.go +++ b/staging/src/github.com/kcp-dev/sdk/apis/migration/v1alpha1/types_logicalclustermigration.go @@ -63,7 +63,7 @@ type LogicalClusterMigrationSpec struct { // LogicalClusterMigrationPhaseType is the type of the current phase of the migration. // -// +kubebuilder:validation:Enum=Preparing;Migrating;OriginCleanup;DestinationFinalize;Completed;Failed +// +kubebuilder:validation:Enum=Preparing;Migrating;OriginCleanup;DestinationFinalize;Aborting;Completed;Failed type LogicalClusterMigrationPhaseType string const ( @@ -71,6 +71,7 @@ const ( LogicalClusterMigrationPhaseMigrating LogicalClusterMigrationPhaseType = "Migrating" LogicalClusterMigrationPhaseOriginCleanup LogicalClusterMigrationPhaseType = "OriginCleanup" LogicalClusterMigrationPhaseDestinationFinalize LogicalClusterMigrationPhaseType = "DestinationFinalize" + LogicalClusterMigrationPhaseAborting LogicalClusterMigrationPhaseType = "Aborting" LogicalClusterMigrationPhaseCompleted LogicalClusterMigrationPhaseType = "Completed" LogicalClusterMigrationPhaseFailed LogicalClusterMigrationPhaseType = "Failed" ) @@ -114,9 +115,17 @@ const ( // belonging to the migration logical cluster. LCMigrationOriginCleaned conditionsv1alpha1.ConditionType = "OriginCleaned" + // LCMigrationDestinationCleaned indicates the destination shard has cleaned up + // all copied data during an abort. + LCMigrationDestinationCleaned conditionsv1alpha1.ConditionType = "DestinationCleaned" + // LCMigrationCompleted indicates the migration has fully completed and the // logical cluster is available on the destination shard. LCMigrationCompleted conditionsv1alpha1.ConditionType = "Completed" + + // LCMigrationAborted indicates the migration was aborted (e.g. due to + // workspace deletion) and both shards have cleaned up their data. + LCMigrationAborted conditionsv1alpha1.ConditionType = "Aborted" ) func (in *LogicalClusterMigration) SetConditions(c conditionsv1alpha1.Conditions) { diff --git a/test/e2e/logicalclustermigration/migration_abort_test.go b/test/e2e/logicalclustermigration/migration_abort_test.go new file mode 100644 index 00000000000..9108fad48e7 --- /dev/null +++ b/test/e2e/logicalclustermigration/migration_abort_test.go @@ -0,0 +1,242 @@ +/* +Copyright 2026 The kcp Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logicalclustermigration + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" + "github.com/kcp-dev/logicalcluster/v3" + apisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" + "github.com/kcp-dev/sdk/apis/core" + migrationv1alpha1 "github.com/kcp-dev/sdk/apis/migration/v1alpha1" + conditionsv1alpha1 "github.com/kcp-dev/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" + kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" + kcptesting "github.com/kcp-dev/sdk/testing" + + "github.com/kcp-dev/kcp/test/e2e/framework" +) + +// TestMigrationAbortOnWorkspaceDeletion verifies that deleting a workspace +// while its logical cluster is being migrated correctly aborts the migration +// and cleans up all data on both shards. It uses a non-existent destination +// shard to stall the migration in the Preparing phase (origin completes +// preparation but the destination never picks up). +func TestMigrationAbortOnWorkspaceDeletion(t *testing.T) { + t.Parallel() + framework.Suite(t, "control-plane") + + server := kcptesting.SharedKcpServer(t) + + cfg := server.BaseConfig(t) + kcpClusterClient, err := kcpclientset.NewForConfig(cfg) + require.NoError(t, err) + kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cfg) + require.NoError(t, err) + + originShard := server.ShardNames()[0] + // Use a non-existent shard as destination to stall the migration. + // The origin will complete Preparing but the destination will never + // pick up, leaving the migration stuck in the Migrating phase. + nonExistentShard := "non-existent-shard-for-abort-test" + + t.Logf("Using origin shard %q and non-existent destination %q", originShard, nonExistentShard) + + // Create an org workspace and a child workspace pinned to the origin shard. + // Since this blocks until the lc is ready, we can proceed directly. + orgPath, _ := kcptesting.NewWorkspaceFixture(t, server, core.RootCluster.Path()) + wsPath, ws := kcptesting.NewWorkspaceFixture(t, server, orgPath, kcptesting.WithShard(originShard)) + lcName := logicalcluster.Name(ws.Spec.Cluster) + wsName := ws.Name + + t.Logf("Workspace %s (logical cluster %s) created on shard %s", wsPath, lcName, originShard) + + // Create test data to verify cleanup. + numConfigMaps := 3 + for i := range numConfigMaps { + _, err := kubeClusterClient.Cluster(wsPath).CoreV1().ConfigMaps("default").Create( + t.Context(), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("abort-test-cm-%d", i)}, + Data: map[string]string{"index": fmt.Sprintf("%d", i)}, + }, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + } + + // Bind the migration API in the org workspace. + bindMigrationAPI(t, kcpClusterClient, orgPath) + + // Create the migration targeting the non-existent shard. + t.Logf("Creating LogicalClusterMigration targeting non-existent shard %q", nonExistentShard) + var lcm *migrationv1alpha1.LogicalClusterMigration + require.EventuallyWithT(t, func(c *assert.CollectT) { + var createErr error + lcm, createErr = kcpClusterClient.Cluster(orgPath).MigrationV1alpha1().LogicalClusterMigrations().Create( + t.Context(), + &migrationv1alpha1.LogicalClusterMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "abort-test-migration", + }, + Spec: migrationv1alpha1.LogicalClusterMigrationSpec{ + LogicalCluster: lcName.String(), + DestinationShard: nonExistentShard, + }, + }, + metav1.CreateOptions{}, + ) + require.NoError(c, createErr) + }, wait.ForeverTestTimeout, 100*time.Millisecond, "failed to create LogicalClusterMigration") + + // Wait for the migration to reach the Migrating phase. + // The origin shard will complete Preparing (annotate LC, purge informers, + // block access), then transition to Migrating. But since the destination + // shard doesn't exist, it stays stuck in Migrating. + t.Logf("Waiting for migration to reach Migrating phase (stalled)") + require.EventuallyWithT(t, func(c *assert.CollectT) { + migration, err := kcpClusterClient.Cluster(orgPath).MigrationV1alpha1().LogicalClusterMigrations().Get(t.Context(), lcm.Name, metav1.GetOptions{}) + require.NoError(c, err) + t.Logf("migration phase: %q, conditions: %v", migration.Status.Phase, conditionsSummary(migration.Status.Conditions)) + require.Equal(c, migrationv1alpha1.LogicalClusterMigrationPhaseMigrating, migration.Status.Phase) + }, wait.ForeverTestTimeout, 500*time.Millisecond, "migration did not reach Migrating phase") + + // Verify the migration has the origin shard set. + migration, err := kcpClusterClient.Cluster(orgPath).MigrationV1alpha1().LogicalClusterMigrations().Get(t.Context(), lcm.Name, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, originShard, migration.Status.OriginShard, "origin shard should be set in status") + + // Verify the OriginReady condition is true. + require.True(t, hasCondition(migration.Status.Conditions, "OriginReady", corev1.ConditionTrue), + "OriginReady condition should be True") + + // Now delete the workspace. This is the core action under test. + t.Logf("Deleting workspace %s to trigger migration abort", wsName) + require.EventuallyWithT(t, func(c *assert.CollectT) { + err := kcpClusterClient.Cluster(orgPath).TenancyV1alpha1().Workspaces().Delete(t.Context(), wsName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + require.NoError(c, err) + } + }, wait.ForeverTestTimeout, 100*time.Millisecond, "failed to delete workspace") + + // Verify the migration transitions to Aborting phase. + t.Logf("Waiting for migration to transition to Aborting phase") + require.EventuallyWithT(t, func(c *assert.CollectT) { + migration, err := kcpClusterClient.Cluster(orgPath).MigrationV1alpha1().LogicalClusterMigrations().Get(t.Context(), lcm.Name, metav1.GetOptions{}) + require.NoError(c, err) + t.Logf("migration phase: %q, conditions: %v", migration.Status.Phase, conditionsSummary(migration.Status.Conditions)) + phase := migration.Status.Phase + require.True(c, + phase == migrationv1alpha1.LogicalClusterMigrationPhaseAborting || + phase == migrationv1alpha1.LogicalClusterMigrationPhaseFailed, + "expected Aborting or Failed phase, got %q", phase) + }, wait.ForeverTestTimeout, 500*time.Millisecond, "migration did not transition to Aborting") + + // Verify the migration reaches the terminal Failed phase with Aborted condition. + t.Logf("Waiting for migration to reach Failed phase with Aborted condition") + require.EventuallyWithT(t, func(c *assert.CollectT) { + migration, err := kcpClusterClient.Cluster(orgPath).MigrationV1alpha1().LogicalClusterMigrations().Get(t.Context(), lcm.Name, metav1.GetOptions{}) + require.NoError(c, err) + t.Logf("migration phase: %q, conditions: %v", migration.Status.Phase, conditionsSummary(migration.Status.Conditions)) + require.Equal(c, migrationv1alpha1.LogicalClusterMigrationPhaseFailed, migration.Status.Phase) + require.True(c, hasCondition(migration.Status.Conditions, "Aborted", corev1.ConditionTrue), + "Aborted condition should be True") + require.True(c, hasCondition(migration.Status.Conditions, "OriginCleaned", corev1.ConditionTrue), + "OriginCleaned condition should be True") + }, wait.ForeverTestTimeout, 500*time.Millisecond, "migration did not reach Failed phase with expected conditions") + + // Verify the workspace is fully deleted. + t.Logf("Verifying workspace is fully deleted") + require.EventuallyWithT(t, func(c *assert.CollectT) { + _, err := kcpClusterClient.Cluster(orgPath).TenancyV1alpha1().Workspaces().Get(t.Context(), wsName, metav1.GetOptions{}) + require.True(c, apierrors.IsNotFound(err), "workspace should be gone, got: %v", err) + }, wait.ForeverTestTimeout, 500*time.Millisecond, "workspace was not fully deleted") + + // Verify we cannot access the logical cluster anymore (all data gone). + t.Logf("Verifying logical cluster data is inaccessible") + require.EventuallyWithT(t, func(c *assert.CollectT) { + _, err := kubeClusterClient.Cluster(wsPath).CoreV1().ConfigMaps("default").List(t.Context(), metav1.ListOptions{}) + // The LC is gone - we expect either NotFound or Forbidden (depending on + // whether the front-proxy still resolves the path). + require.True(c, apierrors.IsNotFound(err) || apierrors.IsForbidden(err), + "expected NotFound or Forbidden accessing deleted LC, got: %v", err) + }, wait.ForeverTestTimeout, 500*time.Millisecond, "logical cluster data still accessible after abort") +} + +// bindMigrationAPI creates and waits for the migration.kcp.io APIBinding to +// become bound in the given workspace. +func bindMigrationAPI(t *testing.T, kcpClusterClient kcpclientset.ClusterInterface, wsPath logicalcluster.Path) { + t.Helper() + t.Logf("Creating APIBinding for migration.kcp.io in %s", wsPath) + _, err := kcpClusterClient.Cluster(wsPath).ApisV1alpha2().APIBindings().Create( + t.Context(), + &apisv1alpha2.APIBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "migration"}, + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: core.RootCluster.Path().String(), + Name: "migration.kcp.io", + }, + }, + }, + }, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + + require.EventuallyWithT(t, func(c *assert.CollectT) { + binding, err := kcpClusterClient.Cluster(wsPath).ApisV1alpha2().APIBindings().Get(t.Context(), "migration", metav1.GetOptions{}) + require.NoError(c, err) + require.Equal(c, apisv1alpha2.APIBindingPhaseBound, binding.Status.Phase) + }, wait.ForeverTestTimeout, 100*time.Millisecond, "migration APIBinding never reached Bound phase") +} + +// hasCondition checks if a condition with the given type and status exists. +func hasCondition(conditions conditionsv1alpha1.Conditions, condType conditionsv1alpha1.ConditionType, status corev1.ConditionStatus) bool { + for _, c := range conditions { + if c.Type == condType && c.Status == status { + return true + } + } + return false +} + +// conditionsSummary returns a brief string summary of conditions for logging. +func conditionsSummary(conditions conditionsv1alpha1.Conditions) string { + if len(conditions) == 0 { + return "" + } + result := "" + for _, c := range conditions { + if result != "" { + result += ", " + } + result += fmt.Sprintf("%s=%s", c.Type, c.Status) + } + return result +}