package data import ( "context" "io" "sync" "time" "github.com/alcionai/clues" "github.com/spatialcurrent/go-lazy/pkg/lazy" "github.com/alcionai/corso/src/internal/common/errs" "github.com/alcionai/corso/src/internal/common/readers" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" ) var ( _ Item = &unindexedPrefetchedItem{} _ ItemModTime = &unindexedPrefetchedItem{} _ Item = &prefetchedItem{} _ ItemInfo = &prefetchedItem{} _ ItemModTime = &prefetchedItem{} _ Item = &unindexedLazyItem{} _ ItemModTime = &unindexedLazyItem{} _ Item = &lazyItem{} _ ItemInfo = &lazyItem{} _ ItemModTime = &lazyItem{} ) func NewDeletedItem(itemID string) Item { return &unindexedPrefetchedItem{ id: itemID, deleted: true, // TODO(ashmrtn): This really doesn't need to be set since deleted items are // never passed to the actual storage engine. Setting it for now so tests // don't break. modTime: time.Now().UTC(), } } func NewUnindexedPrefetchedItem( reader io.ReadCloser, itemID string, modTime time.Time, ) (*unindexedPrefetchedItem, error) { r, err := readers.NewVersionedBackupReader( readers.SerializationFormat{Version: readers.DefaultSerializationVersion}, reader) if err != nil { return nil, clues.Stack(err) } return &unindexedPrefetchedItem{ id: itemID, reader: r, modTime: modTime, }, nil } // unindexedPrefetchedItem represents a single item retrieved from the remote // service. // // This item doesn't implement ItemInfo so it's safe to use for items like // metadata that shouldn't appear in backup details. type unindexedPrefetchedItem struct { id string reader io.ReadCloser // modTime is the modified time of the item. It should match the modTime in // info if info is present. Here as a separate field so that deleted items // don't error out by trying to source it from info. modTime time.Time // deleted flags if this item has been removed in the remote service and // should be removed in storage as well. deleted bool } func (i unindexedPrefetchedItem) ID() string { return i.id } func (i *unindexedPrefetchedItem) ToReader() io.ReadCloser { return i.reader } func (i unindexedPrefetchedItem) Deleted() bool { return i.deleted } func (i unindexedPrefetchedItem) ModTime() time.Time { return i.modTime } func NewPrefetchedItem( reader io.ReadCloser, itemID string, info details.ItemInfo, ) (*prefetchedItem, error) { inner, err := NewUnindexedPrefetchedItem(reader, itemID, info.Modified()) if err != nil { return nil, clues.Stack(err) } return &prefetchedItem{ unindexedPrefetchedItem: inner, info: info, }, nil } // prefetchedItem represents a single item retrieved from the remote service. // // This item implements ItemInfo so it should be used for things that need to // appear in backup details. type prefetchedItem struct { *unindexedPrefetchedItem info details.ItemInfo } func (i prefetchedItem) Info() (details.ItemInfo, error) { return i.info, nil } type ItemDataGetter interface { GetData( context.Context, *fault.Bus, ) (io.ReadCloser, *details.ItemInfo, bool, error) } func NewUnindexedLazyItem( ctx context.Context, itemGetter ItemDataGetter, itemID string, modTime time.Time, errs *fault.Bus, ) *unindexedLazyItem { return &unindexedLazyItem{ ctx: ctx, id: itemID, itemGetter: itemGetter, modTime: modTime, errs: errs, } } // unindexedLazyItem represents a single item retrieved from the remote service. // It lazily fetches the item's data when the first call to ToReader().Read() is // made. // // This item doesn't implement ItemInfo so it's safe to use for items like // metadata that shouldn't appear in backup details. type unindexedLazyItem struct { ctx context.Context mu sync.Mutex id string errs *fault.Bus itemGetter ItemDataGetter modTime time.Time // info holds the details information for this item. Store a pointer in this // struct so we can tell if it's been set already or not. // // This also helps with garbage collection because now the golang garbage // collector can collect the lazyItem struct once the storage engine is done // with it. The ItemInfo struct needs to stick around until the end of the // backup though as backup details is written last. info *details.ItemInfo delInFlight bool } func (i *unindexedLazyItem) ID() string { return i.id } func (i *unindexedLazyItem) ToReader() io.ReadCloser { return lazy.NewLazyReadCloser(func() (io.ReadCloser, error) { // Don't allow getting Item info while trying to initialize said info. // GetData could be a long running call, but in theory nothing should happen // with the item until a reader is returned anyway. i.mu.Lock() defer i.mu.Unlock() reader, info, delInFlight, err := i.itemGetter.GetData(i.ctx, i.errs) if err != nil { return nil, clues.Stack(err) } format := readers.SerializationFormat{ Version: readers.DefaultSerializationVersion, } // If an item was deleted then return an empty file so we don't fail the // backup and return a sentinel error when asked for ItemInfo so we don't // display the item in the backup. // // The item will be deleted from storage on the next backup when either the // delta token shows it's removed or we do a full backup (token expired // etc.) and the item isn't enumerated in that set. if delInFlight { logger.Ctx(i.ctx).Info("item not found") i.delInFlight = true format.DelInFlight = true r, err := readers.NewVersionedBackupReader(format) return r, clues.Stack(err).OrNil() } i.info = info r, err := readers.NewVersionedBackupReader(format, reader) return r, clues.Stack(err).OrNil() }) } func (i *unindexedLazyItem) Deleted() bool { return false } func (i *unindexedLazyItem) ModTime() time.Time { return i.modTime } func NewLazyItem( ctx context.Context, itemGetter ItemDataGetter, itemID string, modTime time.Time, errs *fault.Bus, ) *lazyItem { return &lazyItem{ unindexedLazyItem: NewUnindexedLazyItem( ctx, itemGetter, itemID, modTime, errs), } } // lazyItem represents a single item retrieved from the remote service. It // lazily fetches the item's data when the first call to ToReader().Read() is // made. // // This item implements ItemInfo so it should be used for things that need to // appear in backup details. type lazyItem struct { *unindexedLazyItem } func (i *lazyItem) Info() (details.ItemInfo, error) { i.mu.Lock() defer i.mu.Unlock() if i.delInFlight { return details.ItemInfo{}, clues.Stack(errs.NotFound).WithClues(i.ctx) } else if i.info == nil { return details.ItemInfo{}, clues.New("requesting ItemInfo before data retrieval"). WithClues(i.ctx) } return *i.info, nil }