package operations import ( "context" "fmt" "testing" "time" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/model" opMock "github.com/alcionai/corso/src/internal/operations/inject/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" "github.com/alcionai/corso/src/pkg/store" ) func getKopiaHandles( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument ) (*kopia.Wrapper, *kopia.ModelStore) { st := storeTD.NewPrefixedS3Storage(t) k := kopia.NewConn(st) err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) kw, err := kopia.NewWrapper(k) // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe // to close here. k.Close(ctx) require.NoError(t, err, "getting kopia wrapper: %s", clues.ToCore(err)) ms, err := kopia.NewModelStore(k) require.NoError(t, err, "getting model store: %s", clues.ToCore(err)) return kw, ms } type MaintenanceOpIntegrationSuite struct { tester.Suite } func TestMaintenanceOpIntegrationSuite(t *testing.T) { suite.Run(t, &MaintenanceOpIntegrationSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}), }) } func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() kw, ms := getKopiaHandles(t, ctx) defer kw.Close(ctx) defer ms.Close(ctx) mo, err := NewMaintenanceOperation( ctx, control.DefaultOptions(), kw, store.NewWrapper(ms), repository.Maintenance{ Type: repository.MetadataMaintenance, }, evmock.NewBus()) require.NoError(t, err, clues.ToCore(err)) err = mo.Run(ctx) assert.NoError(t, err, clues.ToCore(err)) } type MaintenanceOpNightlySuite struct { tester.Suite } func TestMaintenanceOpNightlySuite(t *testing.T) { suite.Run(t, &MaintenanceOpNightlySuite{ Suite: tester.NewNightlySuite( t, [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}), }) } func (suite *MaintenanceOpNightlySuite) TestRepoMaintenance_GarbageCollection() { var ( t = suite.T() acct = tconfig.NewM365Account(suite.T()) tenantID = acct.Config[account.AzureTenantIDKey] opts = control.DefaultOptions() osel = selectors.NewOneDriveBackup([]string{userID}) // Default policy used by SDK clients failurePolicy = control.FailAfterRecovery T1 = time.Now().Truncate(0) ) ctx, flush := tester.NewContext(t) defer flush() osel.Include(selTD.OneDriveBackupFolderScope(osel)) pathElements := []string{odConsts.DrivesPathDir, "drive-id", odConsts.RootPathDir, folderID} tmp, err := path.Build(tenantID, userID, path.OneDriveService, path.FilesCategory, false, pathElements...) require.NoError(suite.T(), err, clues.ToCore(err)) locPath := path.Builder{}.Append(tmp.Folders()...) kw, ms := getKopiaHandles(t, ctx) storer := store.NewWrapper(ms) var bupIDs []model.StableID // Make two failed backups so the garbage collection code will try to delete // something. for i := 0; i < 2; i++ { suite.Run(fmt.Sprintf("Setup%d", i), func() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() cs := []data.BackupCollection{ makeBackupCollection( tmp, locPath, []dataMock.Item{ makeMockItem("file1", nil, T1, false, nil), makeMockItem("file2", nil, T1, false, assert.AnError), }), } prefixPath, err := path.BuildMetadata( tenantID, userID, path.OneDriveService, path.FilesCategory, false) require.NoError(t, err, clues.ToCore(err)) mc, err := graph.MakeMetadataCollection( prefixPath, makeMetadataCollectionEntries("url/1", driveID, folderID, tmp), func(*support.ControllerOperationStatus) {}) require.NoError(t, err, clues.ToCore(err)) cs = append(cs, mc) bp := opMock.NewMockBackupProducer(cs, data.CollectionStats{}, false) opts.FailureHandling = failurePolicy bo, err := NewBackupOperation( ctx, opts, kw, storer, &bp, acct, osel.Selector, selectors.Selector{DiscreteOwner: userID}, evmock.NewBus(), count.New()) require.NoError(t, err, clues.ToCore(err)) err = bo.Run(ctx) assert.Error(t, err, clues.ToCore(err)) require.NotEmpty(t, bo.Results.BackupID) bupIDs = append(bupIDs, bo.Results.BackupID) }) } // Double check we have two backup models. This is not an exhaustive check but // will give us some comfort that things are working as expected. bups, err := storer.GetBackups(ctx) require.NoError( t, err, "checking backup model existence: %s", clues.ToCore(err)) var gotBupIDs []model.StableID for _, bup := range bups { gotBupIDs = append(gotBupIDs, bup.ID) } require.ElementsMatch(t, bupIDs, gotBupIDs) // Run maintenance with garbage collection. suite.Run("RunMaintenance", func() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() mo, err := NewMaintenanceOperation( ctx, control.DefaultOptions(), kw, store.NewWrapper(ms), repository.Maintenance{ Type: repository.CompleteMaintenance, // Set buffer to 0 so things will actually be garbage collected. CleanupBuffer: ptr.To(time.Duration(0)), }, evmock.NewBus()) require.NoError(t, err, clues.ToCore(err)) err = mo.Run(ctx) assert.NoError(t, err, clues.ToCore(err)) // Check for backup models again. Only the second one should still be present. bups, err = storer.GetBackups(ctx) require.NoError( t, err, "checking backup model existence after maintenance: %s", clues.ToCore(err)) gotBupIDs = nil for _, bup := range bups { gotBupIDs = append(gotBupIDs, bup.ID) } assert.ElementsMatch(t, bupIDs[1:], gotBupIDs) }) }