Reduce select query for url cache delta query

This commit is contained in:
Abhishek Pandey 2023-08-18 20:18:17 +05:30
parent 06d4f764e7
commit 302be45a9a
3 changed files with 38 additions and 32 deletions

View File

@ -473,7 +473,7 @@ func (c *Collections) addURLCacheToDriveCollections(
driveID,
prevDelta,
urlCacheRefreshInterval,
c.handler.NewItemPager(driveID, "", api.DriveItemSelectDefault()),
c.handler.NewItemPager(driveID, "", api.DriveItemSelectURLCache()),
errs)
if err != nil {
return err

View File

@ -369,41 +369,41 @@ func (op *BackupOperation) do(
lastBackupVersion = mans.MinBackupVersion()
}
// Run 3 times and exit
cs := []data.BackupCollection{}
canUsePreviousBackup := false
// // Run 3 times and exit
// cs := []data.BackupCollection{}
// canUsePreviousBackup := false
var maxCount int = 2
// var maxCount int = 2
for i := 0; i < maxCount; i++ {
logger.Ctx(ctx).Info("delta query iteration")
// for i := 0; i < maxCount; i++ {
logger.Ctx(ctx).Info("delta query iteration")
cs, _, canUsePreviousBackup, err := produceBackupDataCollections(
ctx,
op.bp,
op.ResourceOwner,
op.Selectors,
mdColls,
lastBackupVersion,
op.Options,
op.Errors)
if err != nil {
return nil, clues.Wrap(err, "producing backup data collections")
}
ctx = clues.Add(
ctx,
"can_use_previous_backup", canUsePreviousBackup,
"collection_count", len(cs))
// sleep for 5 mins
//time.Sleep(5 * time.Minute)
if i == maxCount-1 {
return nil, clues.New("unable to produce backup collections").WithClues(ctx)
}
cs, ssmb, canUsePreviousBackup, err := produceBackupDataCollections(
ctx,
op.bp,
op.ResourceOwner,
op.Selectors,
mdColls,
lastBackupVersion,
op.Options,
op.Errors)
if err != nil {
return nil, clues.Wrap(err, "producing backup data collections")
}
ctx = clues.Add(
ctx,
"can_use_previous_backup", canUsePreviousBackup,
"collection_count", len(cs))
// sleep for 5 mins
//time.Sleep(5 * time.Minute)
// if i == maxCount-1 {
// return nil, clues.New("unable to produce backup collections").WithClues(ctx)
// }
// }
writeStats, deets, toMerge, err := consumeBackupCollections(
ctx,
op.kopia,
@ -411,7 +411,7 @@ func (op *BackupOperation) do(
reasons,
mans,
cs,
nil,
ssmb,
backupID,
op.incremental && canUseMetadata && canUsePreviousBackup,
op.Errors)

View File

@ -94,6 +94,12 @@ func idAnd(ss ...string) []string {
// exported
// ---------------------------------------------------------------------------
func DriveItemSelectURLCache() []string {
return idAnd(
"deleted",
)
}
func DriveItemSelectDefault() []string {
return idAnd(
"content.downloadUrl",