From 8c939c0f0d50501fdfa369ba75706fbc9568b5bd Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Thu, 10 Aug 2023 11:38:34 +0530 Subject: [PATCH] Split services and collections for OneDrive & SharePoint (#4002) Only code movement, no code changes. Moved services to `/internal/m365/services/{onedrive,sharepoint,exchange}` Moved collections to `/internal/m365/collection/{drive,site}` --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/factory/impl/common.go | 4 +- src/cmd/factory/impl/exchange.go | 2 +- src/internal/kopia/data_collection_test.go | 2 +- src/internal/kopia/merge_collection_test.go | 2 +- src/internal/kopia/upload_test.go | 2 +- src/internal/kopia/wrapper_test.go | 4 +- src/internal/m365/backup.go | 6 +- src/internal/m365/backup_test.go | 4 +- .../drive}/collection.go | 6 +- .../drive}/collection_test.go | 10 +- .../drive}/collections.go | 6 +- .../drive}/collections_test.go | 8 +- .../drive}/folder_cache.go | 2 +- .../drive}/handlers.go | 2 +- .../drive/helper_test.go} | 2 +- .../{onedrive => collection/drive}/item.go | 4 +- .../drive}/item_collector.go | 2 +- .../drive}/item_collector_test.go | 2 +- .../drive}/item_handler.go | 8 +- .../drive}/item_handler_test.go | 2 +- .../drive}/item_test.go | 2 +- .../drive}/library_handler.go | 27 +- .../drive}/library_handler_test.go | 2 +- .../drive}/metadata/consts.go | 0 .../drive}/metadata/metadata.go | 0 .../drive}/metadata/permissions.go | 0 .../drive}/metadata/permissions_test.go | 0 .../drive}/metadata/testdata/permissions.go | 2 +- .../drive}/permission.go | 6 +- .../drive}/permission_test.go | 6 +- .../{onedrive => collection/drive}/restore.go | 213 +----------- .../drive}/restore_caches.go | 4 +- .../drive}/restore_test.go | 301 +---------------- .../drive}/url_cache.go | 2 +- .../drive}/url_cache_test.go | 2 +- .../{sharepoint => collection/site}/backup.go | 235 ++++--------- .../m365/collection/site/backup_test.go | 73 ++++ .../site}/collection.go | 10 +- .../site}/collection_test.go | 10 +- .../site}/datacategory_string.go | 2 +- .../site}/helper_test.go | 4 +- .../{sharepoint => collection/site}/lists.go | 22 +- .../site}/lists_test.go | 14 +- .../{sharepoint => collection/site}/pages.go | 2 +- .../site}/pages_test.go | 2 +- .../site}/restore.go | 14 +- src/internal/m365/controller_test.go | 2 +- src/internal/m365/export.go | 2 +- src/internal/m365/graph/metadata/metadata.go | 2 +- .../m365/graph/metadata/metadata_test.go | 2 +- src/internal/m365/helper_test.go | 8 +- src/internal/m365/onedrive_test.go | 6 +- src/internal/m365/restore.go | 9 +- .../m365/{ => service}/exchange/attachment.go | 0 .../m365/{ => service}/exchange/attendees.go | 0 .../m365/{ => service}/exchange/backup.go | 0 .../{ => service}/exchange/backup_test.go | 0 .../{ => service}/exchange/cache_container.go | 0 .../m365/{ => service}/exchange/collection.go | 0 .../{ => service}/exchange/collection_test.go | 0 .../m365/{ => service}/exchange/consts.go | 0 .../{ => service}/exchange/contacts_backup.go | 0 .../exchange/contacts_container_cache.go | 0 .../exchange/contacts_restore.go | 0 .../exchange/contacts_restore_test.go | 2 +- .../exchange/container_resolver.go | 0 .../exchange/container_resolver_test.go | 0 .../{ => service}/exchange/events_backup.go | 0 .../exchange/events_container_cache.go | 0 .../exchange/events_instance_restore.go | 0 .../{ => service}/exchange/events_restore.go | 0 .../exchange/events_restore_test.go | 2 +- .../m365/{ => service}/exchange/handlers.go | 0 .../{ => service}/exchange/helper_test.go | 0 .../{ => service}/exchange/mail_backup.go | 0 .../exchange/mail_container_cache.go | 0 .../exchange/mail_container_cache_test.go | 0 .../{ => service}/exchange/mail_restore.go | 0 .../exchange/mail_restore_test.go | 2 +- .../exchange/mock/collections.go | 0 .../{ => service}/exchange/mock/contact.go | 0 .../m365/{ => service}/exchange/mock/event.go | 0 .../m365/{ => service}/exchange/mock/mail.go | 0 .../{ => service}/exchange/mock/mock_test.go | 0 .../m365/{ => service}/exchange/restore.go | 0 .../{ => service}/exchange/restore_test.go | 2 +- .../exchange/testdata/handlers.go | 2 +- .../m365/{ => service}/exchange/transform.go | 0 .../{ => service}/exchange/transform_test.go | 2 +- .../m365/{ => service}/groups/backup.go | 0 .../m365/{ => service}/groups/restore.go | 0 .../m365/{ => service}/onedrive/backup.go | 5 +- .../{ => service}/onedrive/backup_test.go | 0 .../{ => service}/onedrive/consts/consts.go | 0 .../m365/{ => service}/onedrive/export.go | 5 +- .../{ => service}/onedrive/export_test.go | 4 +- .../{ => service}/onedrive/mock/handlers.go | 2 +- .../m365/{ => service}/onedrive/mock/item.go | 0 src/internal/m365/service/onedrive/restore.go | 221 ++++++++++++ .../m365/service/onedrive/restore_test.go | 317 ++++++++++++++++++ .../m365/{ => service}/onedrive/stub/stub.go | 4 +- .../{ => service}/onedrive/testdata/item.go | 0 .../sharepoint/api/beta_service.go | 0 .../sharepoint/api/beta_service_test.go | 0 .../{ => service}/sharepoint/api/pages.go | 0 .../sharepoint/api/pages_test.go | 8 +- .../sharepoint/api/serialization.go | 0 .../sharepoint/api/serialization_test.go | 2 +- .../m365/service/sharepoint/backup.go | 133 ++++++++ .../{ => service}/sharepoint/backup_test.go | 74 +--- .../{ => service}/sharepoint/mock/list.go | 0 .../sharepoint/mock/mock_test.go | 2 +- .../{ => service}/sharepoint/mock/page.go | 0 .../m365/service/sharepoint/restore.go | 122 +++++++ src/internal/m365/stub/stub.go | 4 +- src/internal/operations/backup_test.go | 4 +- src/internal/operations/export_test.go | 2 +- src/internal/operations/restore.go | 2 +- src/internal/operations/restore_test.go | 2 +- src/internal/operations/test/exchange_test.go | 6 +- src/internal/operations/test/helper_test.go | 4 +- src/internal/operations/test/onedrive_test.go | 18 +- .../operations/test/sharepoint_test.go | 7 +- src/pkg/backup/details/details.go | 2 +- src/pkg/backup/details/details_test.go | 4 +- src/pkg/path/drive_test.go | 2 +- src/pkg/repository/repository.go | 2 +- src/pkg/selectors/onedrive_test.go | 2 +- src/pkg/selectors/sharepoint_test.go | 2 +- src/pkg/services/m365/api/client_test.go | 2 +- src/pkg/services/m365/api/contacts_test.go | 2 +- src/pkg/services/m365/api/drive_pager.go | 2 +- src/pkg/services/m365/api/events_test.go | 2 +- src/pkg/services/m365/api/mail_test.go | 2 +- 134 files changed, 1132 insertions(+), 918 deletions(-) rename src/internal/m365/{onedrive => collection/drive}/collection.go (99%) rename src/internal/m365/{onedrive => collection/drive}/collection_test.go (98%) rename src/internal/m365/{onedrive => collection/drive}/collections.go (99%) rename src/internal/m365/{onedrive => collection/drive}/collections_test.go (99%) rename src/internal/m365/{onedrive => collection/drive}/folder_cache.go (97%) rename src/internal/m365/{onedrive => collection/drive}/handlers.go (99%) rename src/internal/m365/{onedrive/service_test.go => collection/drive/helper_test.go} (98%) rename src/internal/m365/{onedrive => collection/drive}/item.go (98%) rename src/internal/m365/{onedrive => collection/drive}/item_collector.go (99%) rename src/internal/m365/{onedrive => collection/drive}/item_collector_test.go (99%) rename src/internal/m365/{onedrive => collection/drive}/item_handler.go (97%) rename src/internal/m365/{onedrive => collection/drive}/item_handler_test.go (98%) rename src/internal/m365/{onedrive => collection/drive}/item_test.go (99%) rename src/internal/m365/{sharepoint => collection/drive}/library_handler.go (93%) rename src/internal/m365/{sharepoint => collection/drive}/library_handler_test.go (98%) rename src/internal/m365/{onedrive => collection/drive}/metadata/consts.go (100%) rename src/internal/m365/{onedrive => collection/drive}/metadata/metadata.go (100%) rename src/internal/m365/{onedrive => collection/drive}/metadata/permissions.go (100%) rename src/internal/m365/{onedrive => collection/drive}/metadata/permissions_test.go (100%) rename src/internal/m365/{onedrive => collection/drive}/metadata/testdata/permissions.go (94%) rename src/internal/m365/{onedrive => collection/drive}/permission.go (98%) rename src/internal/m365/{onedrive => collection/drive}/permission_test.go (95%) rename src/internal/m365/{onedrive => collection/drive}/restore.go (82%) rename src/internal/m365/{onedrive => collection/drive}/restore_caches.go (97%) rename src/internal/m365/{onedrive => collection/drive}/restore_test.go (70%) rename src/internal/m365/{onedrive => collection/drive}/url_cache.go (99%) rename src/internal/m365/{onedrive => collection/drive}/url_cache_test.go (99%) rename src/internal/m365/{sharepoint => collection/site}/backup.go (53%) create mode 100644 src/internal/m365/collection/site/backup_test.go rename src/internal/m365/{sharepoint => collection/site}/collection.go (97%) rename src/internal/m365/{sharepoint => collection/site}/collection_test.go (95%) rename src/internal/m365/{sharepoint => collection/site}/datacategory_string.go (97%) rename src/internal/m365/{sharepoint => collection/site}/helper_test.go (97%) rename src/internal/m365/{sharepoint => collection/site}/lists.go (96%) rename src/internal/m365/{sharepoint => collection/site}/lists_test.go (89%) rename src/internal/m365/{sharepoint => collection/site}/pages.go (98%) rename src/internal/m365/{sharepoint => collection/site}/pages_test.go (98%) rename src/internal/m365/{sharepoint => collection/site}/restore.go (95%) rename src/internal/m365/{ => service}/exchange/attachment.go (100%) rename src/internal/m365/{ => service}/exchange/attendees.go (100%) rename src/internal/m365/{ => service}/exchange/backup.go (100%) rename src/internal/m365/{ => service}/exchange/backup_test.go (100%) rename src/internal/m365/{ => service}/exchange/cache_container.go (100%) rename src/internal/m365/{ => service}/exchange/collection.go (100%) rename src/internal/m365/{ => service}/exchange/collection_test.go (100%) rename src/internal/m365/{ => service}/exchange/consts.go (100%) rename src/internal/m365/{ => service}/exchange/contacts_backup.go (100%) rename src/internal/m365/{ => service}/exchange/contacts_container_cache.go (100%) rename src/internal/m365/{ => service}/exchange/contacts_restore.go (100%) rename src/internal/m365/{ => service}/exchange/contacts_restore_test.go (98%) rename src/internal/m365/{ => service}/exchange/container_resolver.go (100%) rename src/internal/m365/{ => service}/exchange/container_resolver_test.go (100%) rename src/internal/m365/{ => service}/exchange/events_backup.go (100%) rename src/internal/m365/{ => service}/exchange/events_container_cache.go (100%) rename src/internal/m365/{ => service}/exchange/events_instance_restore.go (100%) rename src/internal/m365/{ => service}/exchange/events_restore.go (100%) rename src/internal/m365/{ => service}/exchange/events_restore_test.go (99%) rename src/internal/m365/{ => service}/exchange/handlers.go (100%) rename src/internal/m365/{ => service}/exchange/helper_test.go (100%) rename src/internal/m365/{ => service}/exchange/mail_backup.go (100%) rename src/internal/m365/{ => service}/exchange/mail_container_cache.go (100%) rename src/internal/m365/{ => service}/exchange/mail_container_cache_test.go (100%) rename src/internal/m365/{ => service}/exchange/mail_restore.go (100%) rename src/internal/m365/{ => service}/exchange/mail_restore_test.go (99%) rename src/internal/m365/{ => service}/exchange/mock/collections.go (100%) rename src/internal/m365/{ => service}/exchange/mock/contact.go (100%) rename src/internal/m365/{ => service}/exchange/mock/event.go (100%) rename src/internal/m365/{ => service}/exchange/mock/mail.go (100%) rename src/internal/m365/{ => service}/exchange/mock/mock_test.go (100%) rename src/internal/m365/{ => service}/exchange/restore.go (100%) rename src/internal/m365/{ => service}/exchange/restore_test.go (99%) rename src/internal/m365/{ => service}/exchange/testdata/handlers.go (92%) rename src/internal/m365/{ => service}/exchange/transform.go (100%) rename src/internal/m365/{ => service}/exchange/transform_test.go (98%) rename src/internal/m365/{ => service}/groups/backup.go (100%) rename src/internal/m365/{ => service}/groups/restore.go (100%) rename src/internal/m365/{ => service}/onedrive/backup.go (95%) rename src/internal/m365/{ => service}/onedrive/backup_test.go (100%) rename src/internal/m365/{ => service}/onedrive/consts/consts.go (100%) rename src/internal/m365/{ => service}/onedrive/export.go (95%) rename src/internal/m365/{ => service}/onedrive/export_test.go (98%) rename src/internal/m365/{ => service}/onedrive/mock/handlers.go (99%) rename src/internal/m365/{ => service}/onedrive/mock/item.go (100%) create mode 100644 src/internal/m365/service/onedrive/restore.go create mode 100644 src/internal/m365/service/onedrive/restore_test.go rename src/internal/m365/{ => service}/onedrive/stub/stub.go (98%) rename src/internal/m365/{ => service}/onedrive/testdata/item.go (100%) rename src/internal/m365/{ => service}/sharepoint/api/beta_service.go (100%) rename src/internal/m365/{ => service}/sharepoint/api/beta_service_test.go (100%) rename src/internal/m365/{ => service}/sharepoint/api/pages.go (100%) rename src/internal/m365/{ => service}/sharepoint/api/pages_test.go (92%) rename src/internal/m365/{ => service}/sharepoint/api/serialization.go (100%) rename src/internal/m365/{ => service}/sharepoint/api/serialization_test.go (97%) create mode 100644 src/internal/m365/service/sharepoint/backup.go rename src/internal/m365/{ => service}/sharepoint/backup_test.go (69%) rename src/internal/m365/{ => service}/sharepoint/mock/list.go (100%) rename src/internal/m365/{ => service}/sharepoint/mock/mock_test.go (95%) rename src/internal/m365/{ => service}/sharepoint/mock/page.go (100%) create mode 100644 src/internal/m365/service/sharepoint/restore.go diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index 5904e09d4..f6532828b 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -17,9 +17,9 @@ import ( "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" - odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" + odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" diff --git a/src/cmd/factory/impl/exchange.go b/src/cmd/factory/impl/exchange.go index b7ad4840d..d4513fe4a 100644 --- a/src/cmd/factory/impl/exchange.go +++ b/src/cmd/factory/impl/exchange.go @@ -5,8 +5,8 @@ import ( . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/resource" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" diff --git a/src/internal/kopia/data_collection_test.go b/src/internal/kopia/data_collection_test.go index 318af2682..a4da94ee4 100644 --- a/src/internal/kopia/data_collection_test.go +++ b/src/internal/kopia/data_collection_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/data" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" diff --git a/src/internal/kopia/merge_collection_test.go b/src/internal/kopia/merge_collection_test.go index 4ffd8d394..9aaf751a1 100644 --- a/src/internal/kopia/merge_collection_test.go +++ b/src/internal/kopia/merge_collection_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index 1edc4c9bf..95c39c46d 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -21,7 +21,7 @@ import ( pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" "github.com/alcionai/corso/src/internal/data" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/identity" diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 646c88d68..582c3ff78 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -25,8 +25,8 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data/mock" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/identity" diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index 6b7af571f..9e7194511 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -8,10 +8,10 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" - "github.com/alcionai/corso/src/internal/m365/exchange" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" - "github.com/alcionai/corso/src/internal/m365/sharepoint" + "github.com/alcionai/corso/src/internal/m365/service/exchange" + "github.com/alcionai/corso/src/internal/m365/service/onedrive" + "github.com/alcionai/corso/src/internal/m365/service/sharepoint" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index 6560977dd..3e35b0030 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -11,9 +11,9 @@ import ( "github.com/stretchr/testify/suite" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" - "github.com/alcionai/corso/src/internal/m365/exchange" "github.com/alcionai/corso/src/internal/m365/resource" - "github.com/alcionai/corso/src/internal/m365/sharepoint" + "github.com/alcionai/corso/src/internal/m365/service/exchange" + "github.com/alcionai/corso/src/internal/m365/service/sharepoint" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" diff --git a/src/internal/m365/onedrive/collection.go b/src/internal/m365/collection/drive/collection.go similarity index 99% rename from src/internal/m365/onedrive/collection.go rename to src/internal/m365/collection/drive/collection.go index 377152e48..a7ea841d1 100644 --- a/src/internal/m365/onedrive/collection.go +++ b/src/internal/m365/collection/drive/collection.go @@ -1,5 +1,5 @@ -// Package onedrive provides support for retrieving M365 OneDrive objects -package onedrive +// Package drive provides support for retrieving M365 Drive objects +package drive import ( "context" @@ -15,8 +15,8 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/backup/details" diff --git a/src/internal/m365/onedrive/collection_test.go b/src/internal/m365/collection/drive/collection_test.go similarity index 98% rename from src/internal/m365/onedrive/collection_test.go rename to src/internal/m365/collection/drive/collection_test.go index 3c30cac22..3636b7663 100644 --- a/src/internal/m365/onedrive/collection_test.go +++ b/src/internal/m365/collection/drive/collection_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "bytes" @@ -20,11 +20,11 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + metaTD "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata/testdata" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" - metaTD "github.com/alcionai/corso/src/internal/m365/onedrive/metadata/testdata" - "github.com/alcionai/corso/src/internal/m365/onedrive/mock" - odTD "github.com/alcionai/corso/src/internal/m365/onedrive/testdata" + "github.com/alcionai/corso/src/internal/m365/service/onedrive/mock" + odTD "github.com/alcionai/corso/src/internal/m365/service/onedrive/testdata" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" diff --git a/src/internal/m365/onedrive/collections.go b/src/internal/m365/collection/drive/collections.go similarity index 99% rename from src/internal/m365/onedrive/collections.go rename to src/internal/m365/collection/drive/collections.go index a73d46fec..ac976b015 100644 --- a/src/internal/m365/onedrive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" @@ -14,9 +14,9 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/control" diff --git a/src/internal/m365/onedrive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go similarity index 99% rename from src/internal/m365/onedrive/collections_test.go rename to src/internal/m365/collection/drive/collections_test.go index b1ca92963..f118c0f03 100644 --- a/src/internal/m365/onedrive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" @@ -17,10 +17,10 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" - "github.com/alcionai/corso/src/internal/m365/onedrive/mock" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" + "github.com/alcionai/corso/src/internal/m365/service/onedrive/mock" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" diff --git a/src/internal/m365/onedrive/folder_cache.go b/src/internal/m365/collection/drive/folder_cache.go similarity index 97% rename from src/internal/m365/onedrive/folder_cache.go rename to src/internal/m365/collection/drive/folder_cache.go index 696d42819..1fa4643db 100644 --- a/src/internal/m365/onedrive/folder_cache.go +++ b/src/internal/m365/collection/drive/folder_cache.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "github.com/microsoftgraph/msgraph-sdk-go/models" diff --git a/src/internal/m365/onedrive/handlers.go b/src/internal/m365/collection/drive/handlers.go similarity index 99% rename from src/internal/m365/onedrive/handlers.go rename to src/internal/m365/collection/drive/handlers.go index cb33b373d..239bcbef5 100644 --- a/src/internal/m365/onedrive/handlers.go +++ b/src/internal/m365/collection/drive/handlers.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" diff --git a/src/internal/m365/onedrive/service_test.go b/src/internal/m365/collection/drive/helper_test.go similarity index 98% rename from src/internal/m365/onedrive/service_test.go rename to src/internal/m365/collection/drive/helper_test.go index a2766b8ee..0c9ec8f8c 100644 --- a/src/internal/m365/onedrive/service_test.go +++ b/src/internal/m365/collection/drive/helper_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "testing" diff --git a/src/internal/m365/onedrive/item.go b/src/internal/m365/collection/drive/item.go similarity index 98% rename from src/internal/m365/onedrive/item.go rename to src/internal/m365/collection/drive/item.go index 3bf35000e..19da4a30e 100644 --- a/src/internal/m365/onedrive/item.go +++ b/src/internal/m365/collection/drive/item.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "bytes" @@ -13,8 +13,8 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/readers" "github.com/alcionai/corso/src/internal/common/str" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/pkg/services/m365/api" ) diff --git a/src/internal/m365/onedrive/item_collector.go b/src/internal/m365/collection/drive/item_collector.go similarity index 99% rename from src/internal/m365/onedrive/item_collector.go rename to src/internal/m365/collection/drive/item_collector.go index f34f02998..d737c4abd 100644 --- a/src/internal/m365/onedrive/item_collector.go +++ b/src/internal/m365/collection/drive/item_collector.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/collection/drive/item_collector_test.go similarity index 99% rename from src/internal/m365/onedrive/item_collector_test.go rename to src/internal/m365/collection/drive/item_collector_test.go index b1fd4102d..f8aca7eb6 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/collection/drive/item_collector_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" diff --git a/src/internal/m365/onedrive/item_handler.go b/src/internal/m365/collection/drive/item_handler.go similarity index 97% rename from src/internal/m365/onedrive/item_handler.go rename to src/internal/m365/collection/drive/item_handler.go index 64701da8f..929649aae 100644 --- a/src/internal/m365/onedrive/item_handler.go +++ b/src/internal/m365/collection/drive/item_handler.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" @@ -10,7 +10,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/ptr" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" @@ -29,6 +29,10 @@ type itemBackupHandler struct { scope selectors.OneDriveScope } +func NewItemBackupHandler(ac api.Drives, scope selectors.OneDriveScope) *itemBackupHandler { + return &itemBackupHandler{ac, scope} +} + func (h itemBackupHandler) Get( ctx context.Context, url string, diff --git a/src/internal/m365/onedrive/item_handler_test.go b/src/internal/m365/collection/drive/item_handler_test.go similarity index 98% rename from src/internal/m365/onedrive/item_handler_test.go rename to src/internal/m365/collection/drive/item_handler_test.go index dbc2c0b61..76767acce 100644 --- a/src/internal/m365/onedrive/item_handler_test.go +++ b/src/internal/m365/collection/drive/item_handler_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "testing" diff --git a/src/internal/m365/onedrive/item_test.go b/src/internal/m365/collection/drive/item_test.go similarity index 99% rename from src/internal/m365/onedrive/item_test.go rename to src/internal/m365/collection/drive/item_test.go index f8f11992f..dfec42e2d 100644 --- a/src/internal/m365/onedrive/item_test.go +++ b/src/internal/m365/collection/drive/item_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "bytes" diff --git a/src/internal/m365/sharepoint/library_handler.go b/src/internal/m365/collection/drive/library_handler.go similarity index 93% rename from src/internal/m365/sharepoint/library_handler.go rename to src/internal/m365/collection/drive/library_handler.go index 3f16c6eae..4649e458c 100644 --- a/src/internal/m365/sharepoint/library_handler.go +++ b/src/internal/m365/collection/drive/library_handler.go @@ -1,4 +1,4 @@ -package sharepoint +package drive import ( "context" @@ -9,8 +9,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/m365/onedrive" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" @@ -18,13 +17,17 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api" ) -var _ onedrive.BackupHandler = &libraryBackupHandler{} +var _ BackupHandler = &libraryBackupHandler{} type libraryBackupHandler struct { ac api.Drives scope selectors.SharePointScope } +func NewLibraryBackupHandler(ac api.Drives, scope selectors.SharePointScope) libraryBackupHandler { + return libraryBackupHandler{ac, scope} +} + func (h libraryBackupHandler) Get( ctx context.Context, url string, @@ -78,7 +81,7 @@ func (h libraryBackupHandler) AugmentItemInfo( size int64, parentPath *path.Builder, ) details.ItemInfo { - return augmentItemInfo(dii, item, size, parentPath) + return augmentLibraryItemInfo(dii, item, size, parentPath) } // constructWebURL is a helper function for recreating the webURL @@ -154,12 +157,16 @@ func (h libraryBackupHandler) IncludesDir(dir string) bool { // Restore // --------------------------------------------------------------------------- -var _ onedrive.RestoreHandler = &libraryRestoreHandler{} +var _ RestoreHandler = &libraryRestoreHandler{} type libraryRestoreHandler struct { ac api.Client } +func NewLibraryRestoreHandler(ac api.Client) libraryRestoreHandler { + return libraryRestoreHandler{ac} +} + func (h libraryRestoreHandler) PostDrive( ctx context.Context, siteID, driveName string, @@ -167,10 +174,6 @@ func (h libraryRestoreHandler) PostDrive( return h.ac.Lists().PostDrive(ctx, siteID, driveName) } -func NewRestoreHandler(ac api.Client) *libraryRestoreHandler { - return &libraryRestoreHandler{ac} -} - func (h libraryRestoreHandler) NewDrivePager( resourceOwner string, fields []string, @@ -184,7 +187,7 @@ func (h libraryRestoreHandler) AugmentItemInfo( size int64, parentPath *path.Builder, ) details.ItemInfo { - return augmentItemInfo(dii, item, size, parentPath) + return augmentLibraryItemInfo(dii, item, size, parentPath) } func (h libraryRestoreHandler) DeleteItem( @@ -263,7 +266,7 @@ func (h libraryRestoreHandler) GetRootFolder( // Common // --------------------------------------------------------------------------- -func augmentItemInfo( +func augmentLibraryItemInfo( dii details.ItemInfo, item models.DriveItemable, size int64, diff --git a/src/internal/m365/sharepoint/library_handler_test.go b/src/internal/m365/collection/drive/library_handler_test.go similarity index 98% rename from src/internal/m365/sharepoint/library_handler_test.go rename to src/internal/m365/collection/drive/library_handler_test.go index 254af56aa..1646868e0 100644 --- a/src/internal/m365/sharepoint/library_handler_test.go +++ b/src/internal/m365/collection/drive/library_handler_test.go @@ -1,4 +1,4 @@ -package sharepoint +package drive import ( "testing" diff --git a/src/internal/m365/onedrive/metadata/consts.go b/src/internal/m365/collection/drive/metadata/consts.go similarity index 100% rename from src/internal/m365/onedrive/metadata/consts.go rename to src/internal/m365/collection/drive/metadata/consts.go diff --git a/src/internal/m365/onedrive/metadata/metadata.go b/src/internal/m365/collection/drive/metadata/metadata.go similarity index 100% rename from src/internal/m365/onedrive/metadata/metadata.go rename to src/internal/m365/collection/drive/metadata/metadata.go diff --git a/src/internal/m365/onedrive/metadata/permissions.go b/src/internal/m365/collection/drive/metadata/permissions.go similarity index 100% rename from src/internal/m365/onedrive/metadata/permissions.go rename to src/internal/m365/collection/drive/metadata/permissions.go diff --git a/src/internal/m365/onedrive/metadata/permissions_test.go b/src/internal/m365/collection/drive/metadata/permissions_test.go similarity index 100% rename from src/internal/m365/onedrive/metadata/permissions_test.go rename to src/internal/m365/collection/drive/metadata/permissions_test.go diff --git a/src/internal/m365/onedrive/metadata/testdata/permissions.go b/src/internal/m365/collection/drive/metadata/testdata/permissions.go similarity index 94% rename from src/internal/m365/onedrive/metadata/testdata/permissions.go rename to src/internal/m365/collection/drive/metadata/testdata/permissions.go index a3ccc5cb3..846ee19ed 100644 --- a/src/internal/m365/onedrive/metadata/testdata/permissions.go +++ b/src/internal/m365/collection/drive/metadata/testdata/permissions.go @@ -6,7 +6,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" ) func AssertMetadataEqual(t *testing.T, expect, got metadata.Metadata) { diff --git a/src/internal/m365/onedrive/permission.go b/src/internal/m365/collection/drive/permission.go similarity index 98% rename from src/internal/m365/onedrive/permission.go rename to src/internal/m365/collection/drive/permission.go index 900d8c989..4125231c6 100644 --- a/src/internal/m365/onedrive/permission.go +++ b/src/internal/m365/collection/drive/permission.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" @@ -11,7 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -76,7 +76,7 @@ func getCollectionMetadata( metaName = metadata.DirMetaFileSuffix } - meta, err := fetchAndReadMetadata(ctx, dc, metaName) + meta, err := FetchAndReadMetadata(ctx, dc, metaName) if err != nil { return metadata.Metadata{}, clues.Wrap(err, "collection metadata") } diff --git a/src/internal/m365/onedrive/permission_test.go b/src/internal/m365/collection/drive/permission_test.go similarity index 95% rename from src/internal/m365/onedrive/permission_test.go rename to src/internal/m365/collection/drive/permission_test.go index 7782fccd9..c241f8a98 100644 --- a/src/internal/m365/onedrive/permission_test.go +++ b/src/internal/m365/collection/drive/permission_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "strings" @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" ) diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/collection/drive/restore.go similarity index 82% rename from src/internal/m365/onedrive/restore.go rename to src/internal/m365/collection/drive/restore.go index 900f37e60..ad7cad33f 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/collection/drive/restore.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" @@ -6,7 +6,6 @@ import ( "fmt" "io" "runtime/trace" - "sort" "strings" "sync" "sync/atomic" @@ -15,12 +14,11 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" - "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/operations/inject" @@ -39,81 +37,6 @@ const ( maxUploadRetries = 3 ) -// ConsumeRestoreCollections will restore the specified data collections into OneDrive -func ConsumeRestoreCollections( - ctx context.Context, - rh RestoreHandler, - rcc inject.RestoreConsumerConfig, - backupDriveIDNames idname.Cacher, - dcs []data.RestoreCollection, - deets *details.Builder, - errs *fault.Bus, - ctr *count.Bus, -) (*support.ControllerOperationStatus, error) { - var ( - restoreMetrics support.CollectionMetrics - el = errs.Local() - caches = NewRestoreCaches(backupDriveIDNames) - fallbackDriveName = rcc.RestoreConfig.Location - ) - - ctx = clues.Add(ctx, "backup_version", rcc.BackupVersion) - - err := caches.Populate(ctx, rh, rcc.ProtectedResource.ID()) - if err != nil { - return nil, clues.Wrap(err, "initializing restore caches") - } - - // Reorder collections so that the parents directories are created - // before the child directories; a requirement for permissions. - data.SortRestoreCollections(dcs) - - // Iterate through the data collections and restore the contents of each - for _, dc := range dcs { - if el.Failure() != nil { - break - } - - var ( - err error - metrics support.CollectionMetrics - ictx = clues.Add( - ctx, - "category", dc.FullPath().Category(), - "full_path", dc.FullPath()) - ) - - metrics, err = RestoreCollection( - ictx, - rh, - rcc, - dc, - caches, - deets, - fallbackDriveName, - errs, - ctr.Local()) - if err != nil { - el.AddRecoverable(ctx, err) - } - - restoreMetrics = support.CombineMetrics(restoreMetrics, metrics) - - if errors.Is(err, context.Canceled) { - break - } - } - - status := support.CreateStatus( - ctx, - support.Restore, - len(dcs), - restoreMetrics, - rcc.RestoreConfig.Location) - - return status, el.Failure() -} - // RestoreCollection handles restoration of an individual collection. // returns: // - the collection's item and byte count metrics @@ -518,7 +441,7 @@ func restoreV1File( // Fetch item permissions from the collection and restore them. metaName := trimmedName + metadata.MetaFileSuffix - meta, err := fetchAndReadMetadata(ctx, fibn, metaName) + meta, err := FetchAndReadMetadata(ctx, fibn, metaName) if err != nil { return details.ItemInfo{}, clues.Wrap(err, "restoring file") } @@ -556,7 +479,7 @@ func restoreV6File( // Get metadata file so we can determine the file name. metaName := trimmedName + metadata.MetaFileSuffix - meta, err := fetchAndReadMetadata(ctx, fibn, metaName) + meta, err := FetchAndReadMetadata(ctx, fibn, metaName) if err != nil { return details.ItemInfo{}, clues.Wrap(err, "restoring file") } @@ -932,7 +855,7 @@ func restoreFile( return ptr.Val(newItem.GetId()), dii, nil } -func fetchAndReadMetadata( +func FetchAndReadMetadata( ctx context.Context, fibn data.FetchItemByNamer, metaName string, @@ -974,132 +897,6 @@ func getMetadata(metar io.ReadCloser) (metadata.Metadata, error) { return meta, nil } -// Augment restore path to add extra files(meta) needed for restore as -// well as do any other ordering operations on the paths -// -// Only accepts StoragePath/RestorePath pairs where the RestorePath is -// at least as long as the StoragePath. If the RestorePath is longer than the -// StoragePath then the first few (closest to the root) directories will use -// default permissions during restore. -func AugmentRestorePaths( - backupVersion int, - paths []path.RestorePaths, -) ([]path.RestorePaths, error) { - // Keyed by each value's StoragePath.String() which corresponds to the RepoRef - // of the directory. - colPaths := map[string]path.RestorePaths{} - - for _, p := range paths { - first := true - - for { - sp, err := p.StoragePath.Dir() - if err != nil { - return nil, err - } - - drivePath, err := path.ToDrivePath(sp) - if err != nil { - return nil, err - } - - if len(drivePath.Folders) == 0 { - break - } - - if len(p.RestorePath.Elements()) < len(sp.Elements()) { - return nil, clues.New("restorePath shorter than storagePath"). - With("restore_path", p.RestorePath, "storage_path", sp) - } - - rp := p.RestorePath - - // Make sure the RestorePath always points to the level of the current - // collection. We need to track if it's the first iteration because the - // RestorePath starts out at the collection level to begin with. - if !first { - rp, err = p.RestorePath.Dir() - if err != nil { - return nil, err - } - } - - paths := path.RestorePaths{ - StoragePath: sp, - RestorePath: rp, - } - - colPaths[sp.String()] = paths - p = paths - first = false - } - } - - // Adds dirmeta files as we need to make sure collections for all - // directories involved are created and not just the final one. No - // need to add `.meta` files (metadata for files) as they will - // anyways be looked up automatically. - // TODO: Stop populating .dirmeta for newer versions once we can - // get files from parent directory via `Fetch` in a collection. - // As of now look up metadata for parent directories from a - // collection. - for _, p := range colPaths { - el := p.StoragePath.Elements() - - if backupVersion >= version.OneDrive6NameInMeta { - mPath, err := p.StoragePath.AppendItem(".dirmeta") - if err != nil { - return nil, err - } - - paths = append( - paths, - path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) - } else if backupVersion >= version.OneDrive4DirIncludesPermissions { - mPath, err := p.StoragePath.AppendItem(el.Last() + ".dirmeta") - if err != nil { - return nil, err - } - - paths = append( - paths, - path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) - } else if backupVersion >= version.OneDrive1DataAndMetaFiles { - pp, err := p.StoragePath.Dir() - if err != nil { - return nil, err - } - - mPath, err := pp.AppendItem(el.Last() + ".dirmeta") - if err != nil { - return nil, err - } - - prp, err := p.RestorePath.Dir() - if err != nil { - return nil, err - } - - paths = append( - paths, - path.RestorePaths{StoragePath: mPath, RestorePath: prp}) - } - } - - // This sort is done primarily to order `.meta` files after `.data` - // files. This is only a necessity for OneDrive as we are storing - // metadata for files/folders in separate meta files and we the - // data to be restored before we can restore the metadata. - // - // This sorting assumes stuff in the same StoragePath directory end up in the - // same RestorePath collection. - sort.Slice(paths, func(i, j int) bool { - return paths[i].StoragePath.String() < paths[j].StoragePath.String() - }) - - return paths, nil -} - type PostDriveAndGetRootFolderer interface { PostDriver GetRootFolderer diff --git a/src/internal/m365/onedrive/restore_caches.go b/src/internal/m365/collection/drive/restore_caches.go similarity index 97% rename from src/internal/m365/onedrive/restore_caches.go rename to src/internal/m365/collection/drive/restore_caches.go index 096e0bff8..e2b4953dd 100644 --- a/src/internal/m365/onedrive/restore_caches.go +++ b/src/internal/m365/collection/drive/restore_caches.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" @@ -10,8 +10,8 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/pkg/services/m365/api" ) diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/collection/drive/restore_test.go similarity index 70% rename from src/internal/m365/onedrive/restore_test.go rename to src/internal/m365/collection/drive/restore_test.go index b948a8cab..2b64ce7a2 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/collection/drive/restore_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" @@ -14,8 +14,8 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/mock" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" + "github.com/alcionai/corso/src/internal/m365/service/onedrive/mock" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -34,301 +34,6 @@ func TestRestoreUnitSuite(t *testing.T) { suite.Run(t, &RestoreUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *RestoreUnitSuite) TestAugmentRestorePaths() { - // Adding a simple test here so that we can be sure that this - // function gets updated whenever we add a new version. - require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version") - - table := []struct { - name string - version int - input []string - output []string - }{ - { - name: "no change v0", - version: 0, - input: []string{ - "file.txt.data", - "file.txt", // v0 does not have `.data` - }, - output: []string{ - "file.txt", // ordering artifact of sorting - "file.txt.data", - }, - }, - { - name: "one folder v0", - version: 0, - input: []string{ - "folder/file.txt.data", - "folder/file.txt", - }, - output: []string{ - "folder/file.txt", - "folder/file.txt.data", - }, - }, - { - name: "no change v1", - version: version.OneDrive1DataAndMetaFiles, - input: []string{ - "file.txt.data", - }, - output: []string{ - "file.txt.data", - }, - }, - { - name: "one folder v1", - version: version.OneDrive1DataAndMetaFiles, - input: []string{ - "folder/file.txt.data", - }, - output: []string{ - "folder.dirmeta", - "folder/file.txt.data", - }, - }, - { - name: "nested folders v1", - version: version.OneDrive1DataAndMetaFiles, - input: []string{ - "folder/file.txt.data", - "folder/folder2/file.txt.data", - }, - output: []string{ - "folder.dirmeta", - "folder/file.txt.data", - "folder/folder2.dirmeta", - "folder/folder2/file.txt.data", - }, - }, - { - name: "no change v4", - version: version.OneDrive4DirIncludesPermissions, - input: []string{ - "file.txt.data", - }, - output: []string{ - "file.txt.data", - }, - }, - { - name: "one folder v4", - version: version.OneDrive4DirIncludesPermissions, - input: []string{ - "folder/file.txt.data", - }, - output: []string{ - "folder/file.txt.data", - "folder/folder.dirmeta", - }, - }, - { - name: "nested folders v4", - version: version.OneDrive4DirIncludesPermissions, - input: []string{ - "folder/file.txt.data", - "folder/folder2/file.txt.data", - }, - output: []string{ - "folder/file.txt.data", - "folder/folder.dirmeta", - "folder/folder2/file.txt.data", - "folder/folder2/folder2.dirmeta", - }, - }, - { - name: "no change v6", - version: version.OneDrive6NameInMeta, - input: []string{ - "file.txt.data", - }, - output: []string{ - "file.txt.data", - }, - }, - { - name: "one folder v6", - version: version.OneDrive6NameInMeta, - input: []string{ - "folder/file.txt.data", - }, - output: []string{ - "folder/.dirmeta", - "folder/file.txt.data", - }, - }, - { - name: "nested folders v6", - version: version.OneDrive6NameInMeta, - input: []string{ - "folder/file.txt.data", - "folder/folder2/file.txt.data", - }, - output: []string{ - "folder/.dirmeta", - "folder/file.txt.data", - "folder/folder2/.dirmeta", - "folder/folder2/file.txt.data", - }, - }, - } - - for _, test := range table { - suite.Run(test.name, func() { - t := suite.T() - - _, flush := tester.NewContext(t) - defer flush() - - base := "id/onedrive/user/files/drives/driveID/root:/" - - inPaths := []path.RestorePaths{} - for _, ps := range test.input { - p, err := path.FromDataLayerPath(base+ps, true) - require.NoError(t, err, "creating path", clues.ToCore(err)) - - pd, err := p.Dir() - require.NoError(t, err, "creating collection path", clues.ToCore(err)) - - inPaths = append( - inPaths, - path.RestorePaths{StoragePath: p, RestorePath: pd}) - } - - outPaths := []path.RestorePaths{} - for _, ps := range test.output { - p, err := path.FromDataLayerPath(base+ps, true) - require.NoError(t, err, "creating path", clues.ToCore(err)) - - pd, err := p.Dir() - require.NoError(t, err, "creating collection path", clues.ToCore(err)) - - outPaths = append( - outPaths, - path.RestorePaths{StoragePath: p, RestorePath: pd}) - } - - actual, err := AugmentRestorePaths(test.version, inPaths) - require.NoError(t, err, "augmenting paths", clues.ToCore(err)) - - // Ordering of paths matter here as we need dirmeta files - // to show up before file in dir - assert.Equal(t, outPaths, actual, "augmented paths") - }) - } -} - -// TestAugmentRestorePaths_DifferentRestorePath tests that RestorePath -// substitution works properly. Since it's only possible for future backup -// versions to need restore path substitution (i.e. due to storing folders by -// ID instead of name) this is only tested against the most recent backup -// version at the moment. -func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() { - // Adding a simple test here so that we can be sure that this - // function gets updated whenever we add a new version. - require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version") - - type pathPair struct { - storage string - restore string - } - - table := []struct { - name string - version int - input []pathPair - output []pathPair - errCheck assert.ErrorAssertionFunc - }{ - { - name: "nested folders", - version: version.Backup, - input: []pathPair{ - {storage: "folder-id/file.txt.data", restore: "folder"}, - {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, - }, - output: []pathPair{ - {storage: "folder-id/.dirmeta", restore: "folder"}, - {storage: "folder-id/file.txt.data", restore: "folder"}, - {storage: "folder-id/folder2-id/.dirmeta", restore: "folder/folder2"}, - {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, - }, - errCheck: assert.NoError, - }, - { - name: "restore path longer one folder", - version: version.Backup, - input: []pathPair{ - {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, - }, - output: []pathPair{ - {storage: "folder-id/.dirmeta", restore: "corso_restore/folder"}, - {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, - }, - errCheck: assert.NoError, - }, - { - name: "restore path shorter one folder", - version: version.Backup, - input: []pathPair{ - {storage: "folder-id/file.txt.data", restore: ""}, - }, - errCheck: assert.Error, - }, - } - - for _, test := range table { - suite.Run(test.name, func() { - t := suite.T() - - _, flush := tester.NewContext(t) - defer flush() - - base := "id/onedrive/user/files/drives/driveID/root:/" - - inPaths := []path.RestorePaths{} - for _, ps := range test.input { - p, err := path.FromDataLayerPath(base+ps.storage, true) - require.NoError(t, err, "creating path", clues.ToCore(err)) - - r, err := path.FromDataLayerPath(base+ps.restore, false) - require.NoError(t, err, "creating path", clues.ToCore(err)) - - inPaths = append( - inPaths, - path.RestorePaths{StoragePath: p, RestorePath: r}) - } - - outPaths := []path.RestorePaths{} - for _, ps := range test.output { - p, err := path.FromDataLayerPath(base+ps.storage, true) - require.NoError(t, err, "creating path", clues.ToCore(err)) - - r, err := path.FromDataLayerPath(base+ps.restore, false) - require.NoError(t, err, "creating path", clues.ToCore(err)) - - outPaths = append( - outPaths, - path.RestorePaths{StoragePath: p, RestorePath: r}) - } - - actual, err := AugmentRestorePaths(test.version, inPaths) - test.errCheck(t, err, "augmenting paths", clues.ToCore(err)) - - if err != nil { - return - } - - // Ordering of paths matter here as we need dirmeta files - // to show up before file in dir - assert.Equal(t, outPaths, actual, "augmented paths") - }) - } -} - func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { const mndiID = "mndi-id" diff --git a/src/internal/m365/onedrive/url_cache.go b/src/internal/m365/collection/drive/url_cache.go similarity index 99% rename from src/internal/m365/onedrive/url_cache.go rename to src/internal/m365/collection/drive/url_cache.go index ebd67d8b1..6c06866c6 100644 --- a/src/internal/m365/onedrive/url_cache.go +++ b/src/internal/m365/collection/drive/url_cache.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" diff --git a/src/internal/m365/onedrive/url_cache_test.go b/src/internal/m365/collection/drive/url_cache_test.go similarity index 99% rename from src/internal/m365/onedrive/url_cache_test.go rename to src/internal/m365/collection/drive/url_cache_test.go index bf4f25350..f2fd257b8 100644 --- a/src/internal/m365/onedrive/url_cache_test.go +++ b/src/internal/m365/collection/drive/url_cache_test.go @@ -1,4 +1,4 @@ -package onedrive +package drive import ( "context" diff --git a/src/internal/m365/sharepoint/backup.go b/src/internal/m365/collection/site/backup.go similarity index 53% rename from src/internal/m365/sharepoint/backup.go rename to src/internal/m365/collection/site/backup.go index 4b924ff44..14f1333be 100644 --- a/src/internal/m365/sharepoint/backup.go +++ b/src/internal/m365/collection/site/backup.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "context" @@ -7,11 +7,10 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" - betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" + betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" "github.com/alcionai/corso/src/internal/m365/support" - "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" @@ -21,173 +20,9 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api" ) -func ProduceBackupCollections( - ctx context.Context, - bpc inject.BackupProducerConfig, - ac api.Client, - creds account.M365Config, - su support.StatusUpdater, - errs *fault.Bus, -) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { - b, err := bpc.Selector.ToSharePointBackup() - if err != nil { - return nil, nil, false, clues.Wrap(err, "sharePointDataCollection: parsing selector") - } - - var ( - el = errs.Local() - collections = []data.BackupCollection{} - categories = map[path.CategoryType]struct{}{} - ssmb = prefixmatcher.NewStringSetBuilder() - canUsePreviousBackup bool - ) - - ctx = clues.Add( - ctx, - "site_id", clues.Hide(bpc.ProtectedResource.ID()), - "site_url", clues.Hide(bpc.ProtectedResource.Name())) - - for _, scope := range b.Scopes() { - if el.Failure() != nil { - break - } - - progressBar := observe.MessageWithCompletion( - ctx, - observe.Bulletf("%s", scope.Category().PathType())) - defer close(progressBar) - - var spcs []data.BackupCollection - - switch scope.Category().PathType() { - case path.ListsCategory: - spcs, err = collectLists( - ctx, - bpc, - ac, - creds.AzureTenantID, - su, - errs) - if err != nil { - el.AddRecoverable(ctx, err) - continue - } - - // Lists don't make use of previous metadata - // TODO: Revisit when we add support of lists - canUsePreviousBackup = true - - case path.LibrariesCategory: - spcs, canUsePreviousBackup, err = collectLibraries( - ctx, - bpc, - ac.Drives(), - creds.AzureTenantID, - ssmb, - scope, - su, - errs) - if err != nil { - el.AddRecoverable(ctx, err) - continue - } - - case path.PagesCategory: - spcs, err = collectPages( - ctx, - bpc, - creds, - ac, - su, - errs) - if err != nil { - el.AddRecoverable(ctx, err) - continue - } - - // Lists don't make use of previous metadata - // TODO: Revisit when we add support of pages - canUsePreviousBackup = true - } - - collections = append(collections, spcs...) - - categories[scope.Category().PathType()] = struct{}{} - } - - if len(collections) > 0 { - baseCols, err := graph.BaseCollections( - ctx, - collections, - creds.AzureTenantID, - bpc.ProtectedResource.ID(), - path.SharePointService, - categories, - su, - errs) - if err != nil { - return nil, nil, false, err - } - - collections = append(collections, baseCols...) - } - - return collections, ssmb.ToReader(), canUsePreviousBackup, el.Failure() -} - -func collectLists( - ctx context.Context, - bpc inject.BackupProducerConfig, - ac api.Client, - tenantID string, - su support.StatusUpdater, - errs *fault.Bus, -) ([]data.BackupCollection, error) { - logger.Ctx(ctx).Debug("Creating SharePoint List Collections") - - var ( - el = errs.Local() - spcs = make([]data.BackupCollection, 0) - ) - - lists, err := preFetchLists(ctx, ac.Stable, bpc.ProtectedResource.ID()) - if err != nil { - return nil, err - } - - for _, tuple := range lists { - if el.Failure() != nil { - break - } - - dir, err := path.Build( - tenantID, - bpc.ProtectedResource.ID(), - path.SharePointService, - path.ListsCategory, - false, - tuple.name) - if err != nil { - el.AddRecoverable(ctx, clues.Wrap(err, "creating list collection path").WithClues(ctx)) - } - - collection := NewCollection( - dir, - ac, - List, - su, - bpc.Options) - collection.AddJob(tuple.id) - - spcs = append(spcs, collection) - } - - return spcs, el.Failure() -} - -// collectLibraries constructs a onedrive Collections struct and Get()s +// CollectLibraries constructs a onedrive Collections struct and Get()s // all the drives associated with the site. -func collectLibraries( +func CollectLibraries( ctx context.Context, bpc inject.BackupProducerConfig, ad api.Drives, @@ -201,8 +36,8 @@ func collectLibraries( var ( collections = []data.BackupCollection{} - colls = onedrive.NewCollections( - &libraryBackupHandler{ad, scope}, + colls = drive.NewCollections( + drive.NewLibraryBackupHandler(ad, scope), tenantID, bpc.ProtectedResource.ID(), su, @@ -217,9 +52,9 @@ func collectLibraries( return append(collections, odcs...), canUsePreviousBackup, nil } -// collectPages constructs a sharepoint Collections struct and Get()s the associated +// CollectPages constructs a sharepoint Collections struct and Get()s the associated // M365 IDs for the associated Pages. -func collectPages( +func CollectPages( ctx context.Context, bpc inject.BackupProducerConfig, creds account.M365Config, @@ -273,7 +108,57 @@ func collectPages( Pages, su, bpc.Options) - collection.betaService = betaService + collection.SetBetaService(betaService) + collection.AddJob(tuple.ID) + + spcs = append(spcs, collection) + } + + return spcs, el.Failure() +} + +func CollectLists( + ctx context.Context, + bpc inject.BackupProducerConfig, + ac api.Client, + tenantID string, + su support.StatusUpdater, + errs *fault.Bus, +) ([]data.BackupCollection, error) { + logger.Ctx(ctx).Debug("Creating SharePoint List Collections") + + var ( + el = errs.Local() + spcs = make([]data.BackupCollection, 0) + ) + + lists, err := PreFetchLists(ctx, ac.Stable, bpc.ProtectedResource.ID()) + if err != nil { + return nil, err + } + + for _, tuple := range lists { + if el.Failure() != nil { + break + } + + dir, err := path.Build( + tenantID, + bpc.ProtectedResource.ID(), + path.SharePointService, + path.ListsCategory, + false, + tuple.Name) + if err != nil { + el.AddRecoverable(ctx, clues.Wrap(err, "creating list collection path").WithClues(ctx)) + } + + collection := NewCollection( + dir, + ac, + List, + su, + bpc.Options) collection.AddJob(tuple.ID) spcs = append(spcs, collection) diff --git a/src/internal/m365/collection/site/backup_test.go b/src/internal/m365/collection/site/backup_test.go new file mode 100644 index 000000000..de0d91c50 --- /dev/null +++ b/src/internal/m365/collection/site/backup_test.go @@ -0,0 +1,73 @@ +package site + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/idname/mock" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type SharePointPagesSuite struct { + tester.Suite +} + +func TestSharePointPagesSuite(t *testing.T) { + suite.Run(t, &SharePointPagesSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *SharePointPagesSuite) SetupSuite() { + ctx, flush := tester.NewContext(suite.T()) + defer flush() + + graph.InitializeConcurrencyLimiter(ctx, false, 4) +} + +func (suite *SharePointPagesSuite) TestCollectPages() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + siteID = tconfig.M365SiteID(t) + a = tconfig.NewM365Account(t) + ) + + creds, err := a.M365Config() + require.NoError(t, err, clues.ToCore(err)) + + ac, err := api.NewClient(creds, control.DefaultOptions()) + require.NoError(t, err, clues.ToCore(err)) + + bpc := inject.BackupProducerConfig{ + LastBackupVersion: version.NoBackup, + Options: control.DefaultOptions(), + ProtectedResource: mock.NewProvider(siteID, siteID), + } + + col, err := CollectPages( + ctx, + bpc, + creds, + ac, + (&MockGraphService{}).UpdateStatus, + fault.New(true)) + assert.NoError(t, err, clues.ToCore(err)) + assert.NotEmpty(t, col) +} diff --git a/src/internal/m365/sharepoint/collection.go b/src/internal/m365/collection/site/collection.go similarity index 97% rename from src/internal/m365/sharepoint/collection.go rename to src/internal/m365/collection/site/collection.go index 23a7cb447..6d115ca3b 100644 --- a/src/internal/m365/sharepoint/collection.go +++ b/src/internal/m365/collection/site/collection.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "bytes" @@ -13,7 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/graph" - betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" + betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/backup/details" @@ -81,6 +81,10 @@ func NewCollection( return c } +func (sc *Collection) SetBetaService(betaService *betaAPI.BetaService) { + sc.betaService = betaService +} + // AddJob appends additional objectID to job field func (sc *Collection) AddJob(objID string) { sc.jobs = append(sc.jobs, objID) @@ -254,7 +258,7 @@ func (sc *Collection) retrieveLists( sc.data <- &Item{ id: ptr.Val(lst.GetId()), data: io.NopCloser(bytes.NewReader(byteArray)), - info: listToSPInfo(lst, size), + info: ListToSPInfo(lst, size), modTime: t, } diff --git a/src/internal/m365/sharepoint/collection_test.go b/src/internal/m365/collection/site/collection_test.go similarity index 95% rename from src/internal/m365/sharepoint/collection_test.go rename to src/internal/m365/collection/site/collection_test.go index 0462a5c8e..9c7d1ab88 100644 --- a/src/internal/m365/sharepoint/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "bytes" @@ -14,8 +14,8 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" - betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" - spMock "github.com/alcionai/corso/src/internal/m365/sharepoint/mock" + betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" + spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" @@ -118,7 +118,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { data := &Item{ id: name, data: io.NopCloser(bytes.NewReader(byteArray)), - info: listToSPInfo(listing, int64(len(byteArray))), + info: ListToSPInfo(listing, int64(len(byteArray))), } return data @@ -207,7 +207,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { listData := &Item{ id: testName, data: io.NopCloser(bytes.NewReader(byteArray)), - info: listToSPInfo(listing, int64(len(byteArray))), + info: ListToSPInfo(listing, int64(len(byteArray))), } destName := testdata.DefaultRestoreConfig("").Location diff --git a/src/internal/m365/sharepoint/datacategory_string.go b/src/internal/m365/collection/site/datacategory_string.go similarity index 97% rename from src/internal/m365/sharepoint/datacategory_string.go rename to src/internal/m365/collection/site/datacategory_string.go index b3281ff7f..eac0006cc 100644 --- a/src/internal/m365/sharepoint/datacategory_string.go +++ b/src/internal/m365/collection/site/datacategory_string.go @@ -1,6 +1,6 @@ // Code generated by "stringer -type=DataCategory"; DO NOT EDIT. -package sharepoint +package site import "strconv" diff --git a/src/internal/m365/sharepoint/helper_test.go b/src/internal/m365/collection/site/helper_test.go similarity index 97% rename from src/internal/m365/sharepoint/helper_test.go rename to src/internal/m365/collection/site/helper_test.go index 006a5648c..ca953a9b4 100644 --- a/src/internal/m365/sharepoint/helper_test.go +++ b/src/internal/m365/collection/site/helper_test.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "testing" @@ -43,7 +43,7 @@ func (ms *MockGraphService) UpdateStatus(*support.ControllerOperationStatus) { } // --------------------------------------------------------------------------- -// Helper Functions +// Helper functions // --------------------------------------------------------------------------- func createTestService(t *testing.T, credentials account.M365Config) *graph.Service { diff --git a/src/internal/m365/sharepoint/lists.go b/src/internal/m365/collection/site/lists.go similarity index 96% rename from src/internal/m365/sharepoint/lists.go rename to src/internal/m365/collection/site/lists.go index 0555516af..e717f8d67 100644 --- a/src/internal/m365/sharepoint/lists.go +++ b/src/internal/m365/collection/site/lists.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "context" @@ -14,9 +14,9 @@ import ( "github.com/alcionai/corso/src/pkg/fault" ) -// listToSPInfo translates models.Listable metadata into searchable content +// ListToSPInfo translates models.Listable metadata into searchable content // List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 -func listToSPInfo(lst models.Listable, size int64) *details.SharePointInfo { +func ListToSPInfo(lst models.Listable, size int64) *details.SharePointInfo { var ( name = ptr.Val(lst.GetDisplayName()) webURL = ptr.Val(lst.GetWebUrl()) @@ -34,9 +34,9 @@ func listToSPInfo(lst models.Listable, size int64) *details.SharePointInfo { } } -type listTuple struct { - name string - id string +type ListTuple struct { + ID string + Name string } func preFetchListOptions() *sites.ItemListsRequestBuilderGetRequestConfiguration { @@ -51,15 +51,15 @@ func preFetchListOptions() *sites.ItemListsRequestBuilderGetRequestConfiguration return options } -func preFetchLists( +func PreFetchLists( ctx context.Context, gs graph.Servicer, siteID string, -) ([]listTuple, error) { +) ([]ListTuple, error) { var ( builder = gs.Client().Sites().BySiteId(siteID).Lists() options = preFetchListOptions() - listTuples = make([]listTuple, 0) + listTuples = make([]ListTuple, 0) ) for { @@ -72,11 +72,11 @@ func preFetchLists( var ( id = ptr.Val(entry.GetId()) name = ptr.Val(entry.GetDisplayName()) - temp = listTuple{id: id, name: name} + temp = ListTuple{ID: id, Name: name} ) if len(name) == 0 { - temp.name = id + temp.Name = id } listTuples = append(listTuples, temp) diff --git a/src/internal/m365/sharepoint/lists_test.go b/src/internal/m365/collection/site/lists_test.go similarity index 89% rename from src/internal/m365/sharepoint/lists_test.go rename to src/internal/m365/collection/site/lists_test.go index 61265003e..6942f0e83 100644 --- a/src/internal/m365/sharepoint/lists_test.go +++ b/src/internal/m365/collection/site/lists_test.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "testing" @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" @@ -28,6 +29,11 @@ func (suite *ListsUnitSuite) SetupSuite() { require.NoError(t, err, clues.ToCore(err)) suite.creds = m365 + + ctx, flush := tester.NewContext(suite.T()) + defer flush() + + graph.InitializeConcurrencyLimiter(ctx, false, 4) } func TestListsUnitSuite(t *testing.T) { @@ -57,10 +63,10 @@ func (suite *ListsUnitSuite) TestLoadList() { defer flush() service := createTestService(t, suite.creds) - tuples, err := preFetchLists(ctx, service, "root") + tuples, err := PreFetchLists(ctx, service, "root") require.NoError(t, err, clues.ToCore(err)) - job := []string{tuples[0].id} + job := []string{tuples[0].ID} lists, err := loadSiteLists(ctx, service, "root", job, fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) assert.Greater(t, len(lists), 0) @@ -98,7 +104,7 @@ func (suite *ListsUnitSuite) TestSharePointInfo() { t := suite.T() list, expected := test.listAndDeets() - info := listToSPInfo(list, 10) + info := ListToSPInfo(list, 10) assert.Equal(t, expected.ItemType, info.ItemType) assert.Equal(t, expected.ItemName, info.ItemName) assert.Equal(t, expected.WebURL, info.WebURL) diff --git a/src/internal/m365/sharepoint/pages.go b/src/internal/m365/collection/site/pages.go similarity index 98% rename from src/internal/m365/sharepoint/pages.go rename to src/internal/m365/collection/site/pages.go index c5e0bb633..23e4e0e9f 100644 --- a/src/internal/m365/sharepoint/pages.go +++ b/src/internal/m365/collection/site/pages.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "time" diff --git a/src/internal/m365/sharepoint/pages_test.go b/src/internal/m365/collection/site/pages_test.go similarity index 98% rename from src/internal/m365/sharepoint/pages_test.go rename to src/internal/m365/collection/site/pages_test.go index d89b0d921..a1c044aaf 100644 --- a/src/internal/m365/sharepoint/pages_test.go +++ b/src/internal/m365/collection/site/pages_test.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "testing" diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/collection/site/restore.go similarity index 95% rename from src/internal/m365/sharepoint/restore.go rename to src/internal/m365/collection/site/restore.go index bb894f5ea..875ac5115 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -1,4 +1,4 @@ -package sharepoint +package site import ( "context" @@ -15,9 +15,9 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" + "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" - betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" + betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" @@ -41,9 +41,9 @@ func ConsumeRestoreCollections( ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - lrh = libraryRestoreHandler{ac} + lrh = drive.NewLibraryRestoreHandler(ac) restoreMetrics support.CollectionMetrics - caches = onedrive.NewRestoreCaches(backupDriveIDNames) + caches = drive.NewRestoreCaches(backupDriveIDNames) el = errs.Local() ) @@ -75,7 +75,7 @@ func ConsumeRestoreCollections( switch dc.FullPath().Category() { case path.LibrariesCategory: - metrics, err = onedrive.RestoreCollection( + metrics, err = drive.RestoreCollection( ictx, lrh, rcc, @@ -200,7 +200,7 @@ func restoreListItem( } } - dii.SharePoint = listToSPInfo(restoredList, int64(len(byteArray))) + dii.SharePoint = ListToSPInfo(restoredList, int64(len(byteArray))) return dii, nil } diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 7076455a9..ec2c8c72c 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -17,10 +17,10 @@ import ( inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index 4da037e26..0003353fb 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -8,7 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" + "github.com/alcionai/corso/src/internal/m365/service/onedrive" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" diff --git a/src/internal/m365/graph/metadata/metadata.go b/src/internal/m365/graph/metadata/metadata.go index 9b61a3fc0..d213cd481 100644 --- a/src/internal/m365/graph/metadata/metadata.go +++ b/src/internal/m365/graph/metadata/metadata.go @@ -1,7 +1,7 @@ package metadata import ( - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/pkg/path" ) diff --git a/src/internal/m365/graph/metadata/metadata_test.go b/src/internal/m365/graph/metadata/metadata_test.go index f7c1b81fe..15b190a19 100644 --- a/src/internal/m365/graph/metadata/metadata_test.go +++ b/src/internal/m365/graph/metadata/metadata_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + odmetadata "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph/metadata" - odmetadata "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" ) diff --git a/src/internal/m365/helper_test.go b/src/internal/m365/helper_test.go index 78e9cb365..f4c80a479 100644 --- a/src/internal/m365/helper_test.go +++ b/src/internal/m365/helper_test.go @@ -17,10 +17,10 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/m365/onedrive" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" - odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" + "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/resource" + odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/control" @@ -737,7 +737,7 @@ func compareDriveItem( ) if !isMeta { - oitem := item.(*onedrive.Item) + oitem := item.(*drive.Item) info := oitem.Info() if info.OneDrive != nil { diff --git a/src/internal/m365/onedrive_test.go b/src/internal/m365/onedrive_test.go index ba81a477a..53b45be52 100644 --- a/src/internal/m365/onedrive_test.go +++ b/src/internal/m365/onedrive_test.go @@ -14,11 +14,11 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" - "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" + "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index de9e0bb13..3455e650f 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -7,10 +7,11 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" - "github.com/alcionai/corso/src/internal/m365/exchange" + "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" - "github.com/alcionai/corso/src/internal/m365/sharepoint" + "github.com/alcionai/corso/src/internal/m365/service/exchange" + "github.com/alcionai/corso/src/internal/m365/service/onedrive" + "github.com/alcionai/corso/src/internal/m365/service/sharepoint" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" @@ -71,7 +72,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( case path.OneDriveService: status, err = onedrive.ConsumeRestoreCollections( ctx, - onedrive.NewRestoreHandler(ctrl.AC), + drive.NewRestoreHandler(ctrl.AC), rcc, ctrl.backupDriveIDNames, dcs, diff --git a/src/internal/m365/exchange/attachment.go b/src/internal/m365/service/exchange/attachment.go similarity index 100% rename from src/internal/m365/exchange/attachment.go rename to src/internal/m365/service/exchange/attachment.go diff --git a/src/internal/m365/exchange/attendees.go b/src/internal/m365/service/exchange/attendees.go similarity index 100% rename from src/internal/m365/exchange/attendees.go rename to src/internal/m365/service/exchange/attendees.go diff --git a/src/internal/m365/exchange/backup.go b/src/internal/m365/service/exchange/backup.go similarity index 100% rename from src/internal/m365/exchange/backup.go rename to src/internal/m365/service/exchange/backup.go diff --git a/src/internal/m365/exchange/backup_test.go b/src/internal/m365/service/exchange/backup_test.go similarity index 100% rename from src/internal/m365/exchange/backup_test.go rename to src/internal/m365/service/exchange/backup_test.go diff --git a/src/internal/m365/exchange/cache_container.go b/src/internal/m365/service/exchange/cache_container.go similarity index 100% rename from src/internal/m365/exchange/cache_container.go rename to src/internal/m365/service/exchange/cache_container.go diff --git a/src/internal/m365/exchange/collection.go b/src/internal/m365/service/exchange/collection.go similarity index 100% rename from src/internal/m365/exchange/collection.go rename to src/internal/m365/service/exchange/collection.go diff --git a/src/internal/m365/exchange/collection_test.go b/src/internal/m365/service/exchange/collection_test.go similarity index 100% rename from src/internal/m365/exchange/collection_test.go rename to src/internal/m365/service/exchange/collection_test.go diff --git a/src/internal/m365/exchange/consts.go b/src/internal/m365/service/exchange/consts.go similarity index 100% rename from src/internal/m365/exchange/consts.go rename to src/internal/m365/service/exchange/consts.go diff --git a/src/internal/m365/exchange/contacts_backup.go b/src/internal/m365/service/exchange/contacts_backup.go similarity index 100% rename from src/internal/m365/exchange/contacts_backup.go rename to src/internal/m365/service/exchange/contacts_backup.go diff --git a/src/internal/m365/exchange/contacts_container_cache.go b/src/internal/m365/service/exchange/contacts_container_cache.go similarity index 100% rename from src/internal/m365/exchange/contacts_container_cache.go rename to src/internal/m365/service/exchange/contacts_container_cache.go diff --git a/src/internal/m365/exchange/contacts_restore.go b/src/internal/m365/service/exchange/contacts_restore.go similarity index 100% rename from src/internal/m365/exchange/contacts_restore.go rename to src/internal/m365/service/exchange/contacts_restore.go diff --git a/src/internal/m365/exchange/contacts_restore_test.go b/src/internal/m365/service/exchange/contacts_restore_test.go similarity index 98% rename from src/internal/m365/exchange/contacts_restore_test.go rename to src/internal/m365/service/exchange/contacts_restore_test.go index d55c1d261..f2030ea16 100644 --- a/src/internal/m365/exchange/contacts_restore_test.go +++ b/src/internal/m365/service/exchange/contacts_restore_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/control" diff --git a/src/internal/m365/exchange/container_resolver.go b/src/internal/m365/service/exchange/container_resolver.go similarity index 100% rename from src/internal/m365/exchange/container_resolver.go rename to src/internal/m365/service/exchange/container_resolver.go diff --git a/src/internal/m365/exchange/container_resolver_test.go b/src/internal/m365/service/exchange/container_resolver_test.go similarity index 100% rename from src/internal/m365/exchange/container_resolver_test.go rename to src/internal/m365/service/exchange/container_resolver_test.go diff --git a/src/internal/m365/exchange/events_backup.go b/src/internal/m365/service/exchange/events_backup.go similarity index 100% rename from src/internal/m365/exchange/events_backup.go rename to src/internal/m365/service/exchange/events_backup.go diff --git a/src/internal/m365/exchange/events_container_cache.go b/src/internal/m365/service/exchange/events_container_cache.go similarity index 100% rename from src/internal/m365/exchange/events_container_cache.go rename to src/internal/m365/service/exchange/events_container_cache.go diff --git a/src/internal/m365/exchange/events_instance_restore.go b/src/internal/m365/service/exchange/events_instance_restore.go similarity index 100% rename from src/internal/m365/exchange/events_instance_restore.go rename to src/internal/m365/service/exchange/events_instance_restore.go diff --git a/src/internal/m365/exchange/events_restore.go b/src/internal/m365/service/exchange/events_restore.go similarity index 100% rename from src/internal/m365/exchange/events_restore.go rename to src/internal/m365/service/exchange/events_restore.go diff --git a/src/internal/m365/exchange/events_restore_test.go b/src/internal/m365/service/exchange/events_restore_test.go similarity index 99% rename from src/internal/m365/exchange/events_restore_test.go rename to src/internal/m365/service/exchange/events_restore_test.go index b8db6f052..ed0fbc60c 100644 --- a/src/internal/m365/exchange/events_restore_test.go +++ b/src/internal/m365/service/exchange/events_restore_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/control" diff --git a/src/internal/m365/exchange/handlers.go b/src/internal/m365/service/exchange/handlers.go similarity index 100% rename from src/internal/m365/exchange/handlers.go rename to src/internal/m365/service/exchange/handlers.go diff --git a/src/internal/m365/exchange/helper_test.go b/src/internal/m365/service/exchange/helper_test.go similarity index 100% rename from src/internal/m365/exchange/helper_test.go rename to src/internal/m365/service/exchange/helper_test.go diff --git a/src/internal/m365/exchange/mail_backup.go b/src/internal/m365/service/exchange/mail_backup.go similarity index 100% rename from src/internal/m365/exchange/mail_backup.go rename to src/internal/m365/service/exchange/mail_backup.go diff --git a/src/internal/m365/exchange/mail_container_cache.go b/src/internal/m365/service/exchange/mail_container_cache.go similarity index 100% rename from src/internal/m365/exchange/mail_container_cache.go rename to src/internal/m365/service/exchange/mail_container_cache.go diff --git a/src/internal/m365/exchange/mail_container_cache_test.go b/src/internal/m365/service/exchange/mail_container_cache_test.go similarity index 100% rename from src/internal/m365/exchange/mail_container_cache_test.go rename to src/internal/m365/service/exchange/mail_container_cache_test.go diff --git a/src/internal/m365/exchange/mail_restore.go b/src/internal/m365/service/exchange/mail_restore.go similarity index 100% rename from src/internal/m365/exchange/mail_restore.go rename to src/internal/m365/service/exchange/mail_restore.go diff --git a/src/internal/m365/exchange/mail_restore_test.go b/src/internal/m365/service/exchange/mail_restore_test.go similarity index 99% rename from src/internal/m365/exchange/mail_restore_test.go rename to src/internal/m365/service/exchange/mail_restore_test.go index 5b85321b6..58fdcd7cb 100644 --- a/src/internal/m365/exchange/mail_restore_test.go +++ b/src/internal/m365/service/exchange/mail_restore_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/control" diff --git a/src/internal/m365/exchange/mock/collections.go b/src/internal/m365/service/exchange/mock/collections.go similarity index 100% rename from src/internal/m365/exchange/mock/collections.go rename to src/internal/m365/service/exchange/mock/collections.go diff --git a/src/internal/m365/exchange/mock/contact.go b/src/internal/m365/service/exchange/mock/contact.go similarity index 100% rename from src/internal/m365/exchange/mock/contact.go rename to src/internal/m365/service/exchange/mock/contact.go diff --git a/src/internal/m365/exchange/mock/event.go b/src/internal/m365/service/exchange/mock/event.go similarity index 100% rename from src/internal/m365/exchange/mock/event.go rename to src/internal/m365/service/exchange/mock/event.go diff --git a/src/internal/m365/exchange/mock/mail.go b/src/internal/m365/service/exchange/mock/mail.go similarity index 100% rename from src/internal/m365/exchange/mock/mail.go rename to src/internal/m365/service/exchange/mock/mail.go diff --git a/src/internal/m365/exchange/mock/mock_test.go b/src/internal/m365/service/exchange/mock/mock_test.go similarity index 100% rename from src/internal/m365/exchange/mock/mock_test.go rename to src/internal/m365/service/exchange/mock/mock_test.go diff --git a/src/internal/m365/exchange/restore.go b/src/internal/m365/service/exchange/restore.go similarity index 100% rename from src/internal/m365/exchange/restore.go rename to src/internal/m365/service/exchange/restore.go diff --git a/src/internal/m365/exchange/restore_test.go b/src/internal/m365/service/exchange/restore_test.go similarity index 99% rename from src/internal/m365/exchange/restore_test.go rename to src/internal/m365/service/exchange/restore_test.go index a30d56dd0..88983114e 100644 --- a/src/internal/m365/exchange/restore_test.go +++ b/src/internal/m365/service/exchange/restore_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" diff --git a/src/internal/m365/exchange/testdata/handlers.go b/src/internal/m365/service/exchange/testdata/handlers.go similarity index 92% rename from src/internal/m365/exchange/testdata/handlers.go rename to src/internal/m365/service/exchange/testdata/handlers.go index 559c23b2c..2a62e609f 100644 --- a/src/internal/m365/exchange/testdata/handlers.go +++ b/src/internal/m365/service/exchange/testdata/handlers.go @@ -7,8 +7,8 @@ import ( "github.com/alcionai/clues" "github.com/stretchr/testify/require" - "github.com/alcionai/corso/src/internal/m365/exchange" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/service/exchange" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" diff --git a/src/internal/m365/exchange/transform.go b/src/internal/m365/service/exchange/transform.go similarity index 100% rename from src/internal/m365/exchange/transform.go rename to src/internal/m365/service/exchange/transform.go diff --git a/src/internal/m365/exchange/transform_test.go b/src/internal/m365/service/exchange/transform_test.go similarity index 98% rename from src/internal/m365/exchange/transform_test.go rename to src/internal/m365/service/exchange/transform_test.go index 1bd8070dc..020406803 100644 --- a/src/internal/m365/exchange/transform_test.go +++ b/src/internal/m365/service/exchange/transform_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/services/m365/api" ) diff --git a/src/internal/m365/groups/backup.go b/src/internal/m365/service/groups/backup.go similarity index 100% rename from src/internal/m365/groups/backup.go rename to src/internal/m365/service/groups/backup.go diff --git a/src/internal/m365/groups/restore.go b/src/internal/m365/service/groups/restore.go similarity index 100% rename from src/internal/m365/groups/restore.go rename to src/internal/m365/service/groups/restore.go diff --git a/src/internal/m365/onedrive/backup.go b/src/internal/m365/service/onedrive/backup.go similarity index 95% rename from src/internal/m365/onedrive/backup.go rename to src/internal/m365/service/onedrive/backup.go index ddf410958..169aba08c 100644 --- a/src/internal/m365/onedrive/backup.go +++ b/src/internal/m365/service/onedrive/backup.go @@ -7,6 +7,7 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" @@ -47,8 +48,8 @@ func ProduceBackupCollections( logger.Ctx(ctx).Debug("creating OneDrive collections") - nc := NewCollections( - &itemBackupHandler{ac.Drives(), scope}, + nc := drive.NewCollections( + drive.NewItemBackupHandler(ac.Drives(), scope), tenant, bpc.ProtectedResource.ID(), su, diff --git a/src/internal/m365/onedrive/backup_test.go b/src/internal/m365/service/onedrive/backup_test.go similarity index 100% rename from src/internal/m365/onedrive/backup_test.go rename to src/internal/m365/service/onedrive/backup_test.go diff --git a/src/internal/m365/onedrive/consts/consts.go b/src/internal/m365/service/onedrive/consts/consts.go similarity index 100% rename from src/internal/m365/onedrive/consts/consts.go rename to src/internal/m365/service/onedrive/consts/consts.go diff --git a/src/internal/m365/onedrive/export.go b/src/internal/m365/service/onedrive/export.go similarity index 95% rename from src/internal/m365/onedrive/export.go rename to src/internal/m365/service/onedrive/export.go index 9868a9b71..8c0af44a2 100644 --- a/src/internal/m365/onedrive/export.go +++ b/src/internal/m365/service/onedrive/export.go @@ -7,7 +7,8 @@ import ( "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" @@ -120,7 +121,7 @@ func getItemName( trimmedName := strings.TrimSuffix(id, metadata.DataFileSuffix) metaName := trimmedName + metadata.MetaFileSuffix - meta, err := fetchAndReadMetadata(ctx, fin, metaName) + meta, err := drive.FetchAndReadMetadata(ctx, fin, metaName) if err != nil { return "", clues.Wrap(err, "getting metadata").WithClues(ctx) } diff --git a/src/internal/m365/onedrive/export_test.go b/src/internal/m365/service/onedrive/export_test.go similarity index 98% rename from src/internal/m365/onedrive/export_test.go rename to src/internal/m365/service/onedrive/export_test.go index ce707885f..3468a7661 100644 --- a/src/internal/m365/onedrive/export_test.go +++ b/src/internal/m365/service/onedrive/export_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/data" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" diff --git a/src/internal/m365/onedrive/mock/handlers.go b/src/internal/m365/service/onedrive/mock/handlers.go similarity index 99% rename from src/internal/m365/onedrive/mock/handlers.go rename to src/internal/m365/service/onedrive/mock/handlers.go index 75dd3c3f1..20beb6bca 100644 --- a/src/internal/m365/onedrive/mock/handlers.go +++ b/src/internal/m365/service/onedrive/mock/handlers.go @@ -8,7 +8,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" diff --git a/src/internal/m365/onedrive/mock/item.go b/src/internal/m365/service/onedrive/mock/item.go similarity index 100% rename from src/internal/m365/onedrive/mock/item.go rename to src/internal/m365/service/onedrive/mock/item.go diff --git a/src/internal/m365/service/onedrive/restore.go b/src/internal/m365/service/onedrive/restore.go new file mode 100644 index 000000000..a1dc65182 --- /dev/null +++ b/src/internal/m365/service/onedrive/restore.go @@ -0,0 +1,221 @@ +package onedrive + +import ( + "context" + "sort" + + "github.com/alcionai/clues" + "github.com/pkg/errors" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +// ConsumeRestoreCollections will restore the specified data collections into OneDrive +func ConsumeRestoreCollections( + ctx context.Context, + rh drive.RestoreHandler, + rcc inject.RestoreConsumerConfig, + backupDriveIDNames idname.Cacher, + dcs []data.RestoreCollection, + deets *details.Builder, + errs *fault.Bus, + ctr *count.Bus, +) (*support.ControllerOperationStatus, error) { + var ( + restoreMetrics support.CollectionMetrics + el = errs.Local() + caches = drive.NewRestoreCaches(backupDriveIDNames) + fallbackDriveName = rcc.RestoreConfig.Location + ) + + ctx = clues.Add(ctx, "backup_version", rcc.BackupVersion) + + err := caches.Populate(ctx, rh, rcc.ProtectedResource.ID()) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } + + // Reorder collections so that the parents directories are created + // before the child directories; a requirement for permissions. + data.SortRestoreCollections(dcs) + + // Iterate through the data collections and restore the contents of each + for _, dc := range dcs { + if el.Failure() != nil { + break + } + + var ( + err error + metrics support.CollectionMetrics + ictx = clues.Add( + ctx, + "category", dc.FullPath().Category(), + "full_path", dc.FullPath()) + ) + + metrics, err = drive.RestoreCollection( + ictx, + rh, + rcc, + dc, + caches, + deets, + fallbackDriveName, + errs, + ctr.Local()) + if err != nil { + el.AddRecoverable(ctx, err) + } + + restoreMetrics = support.CombineMetrics(restoreMetrics, metrics) + + if errors.Is(err, context.Canceled) { + break + } + } + + status := support.CreateStatus( + ctx, + support.Restore, + len(dcs), + restoreMetrics, + rcc.RestoreConfig.Location) + + return status, el.Failure() +} + +// Augment restore path to add extra files(meta) needed for restore as +// well as do any other ordering operations on the paths +// +// Only accepts StoragePath/RestorePath pairs where the RestorePath is +// at least as long as the StoragePath. If the RestorePath is longer than the +// StoragePath then the first few (closest to the root) directories will use +// default permissions during restore. +func AugmentRestorePaths( + backupVersion int, + paths []path.RestorePaths, +) ([]path.RestorePaths, error) { + // Keyed by each value's StoragePath.String() which corresponds to the RepoRef + // of the directory. + colPaths := map[string]path.RestorePaths{} + + for _, p := range paths { + first := true + + for { + sp, err := p.StoragePath.Dir() + if err != nil { + return nil, err + } + + drivePath, err := path.ToDrivePath(sp) + if err != nil { + return nil, err + } + + if len(drivePath.Folders) == 0 { + break + } + + if len(p.RestorePath.Elements()) < len(sp.Elements()) { + return nil, clues.New("restorePath shorter than storagePath"). + With("restore_path", p.RestorePath, "storage_path", sp) + } + + rp := p.RestorePath + + // Make sure the RestorePath always points to the level of the current + // collection. We need to track if it's the first iteration because the + // RestorePath starts out at the collection level to begin with. + if !first { + rp, err = p.RestorePath.Dir() + if err != nil { + return nil, err + } + } + + paths := path.RestorePaths{ + StoragePath: sp, + RestorePath: rp, + } + + colPaths[sp.String()] = paths + p = paths + first = false + } + } + + // Adds dirmeta files as we need to make sure collections for all + // directories involved are created and not just the final one. No + // need to add `.meta` files (metadata for files) as they will + // anyways be looked up automatically. + // TODO: Stop populating .dirmeta for newer versions once we can + // get files from parent directory via `Fetch` in a collection. + // As of now look up metadata for parent directories from a + // collection. + for _, p := range colPaths { + el := p.StoragePath.Elements() + + if backupVersion >= version.OneDrive6NameInMeta { + mPath, err := p.StoragePath.AppendItem(".dirmeta") + if err != nil { + return nil, err + } + + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) + } else if backupVersion >= version.OneDrive4DirIncludesPermissions { + mPath, err := p.StoragePath.AppendItem(el.Last() + ".dirmeta") + if err != nil { + return nil, err + } + + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) + } else if backupVersion >= version.OneDrive1DataAndMetaFiles { + pp, err := p.StoragePath.Dir() + if err != nil { + return nil, err + } + + mPath, err := pp.AppendItem(el.Last() + ".dirmeta") + if err != nil { + return nil, err + } + + prp, err := p.RestorePath.Dir() + if err != nil { + return nil, err + } + + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: prp}) + } + } + + // This sort is done primarily to order `.meta` files after `.data` + // files. This is only a necessity for OneDrive as we are storing + // metadata for files/folders in separate meta files and we the + // data to be restored before we can restore the metadata. + // + // This sorting assumes stuff in the same StoragePath directory end up in the + // same RestorePath collection. + sort.Slice(paths, func(i, j int) bool { + return paths[i].StoragePath.String() < paths[j].StoragePath.String() + }) + + return paths, nil +} diff --git a/src/internal/m365/service/onedrive/restore_test.go b/src/internal/m365/service/onedrive/restore_test.go new file mode 100644 index 000000000..0af13eccb --- /dev/null +++ b/src/internal/m365/service/onedrive/restore_test.go @@ -0,0 +1,317 @@ +package onedrive + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/path" +) + +type RestoreUnitSuite struct { + tester.Suite +} + +func TestRestoreUnitSuite(t *testing.T) { + suite.Run(t, &RestoreUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *RestoreUnitSuite) TestAugmentRestorePaths() { + // Adding a simple test here so that we can be sure that this + // function gets updated whenever we add a new version. + require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version") + + table := []struct { + name string + version int + input []string + output []string + }{ + { + name: "no change v0", + version: 0, + input: []string{ + "file.txt.data", + "file.txt", // v0 does not have `.data` + }, + output: []string{ + "file.txt", // ordering artifact of sorting + "file.txt.data", + }, + }, + { + name: "one folder v0", + version: 0, + input: []string{ + "folder/file.txt.data", + "folder/file.txt", + }, + output: []string{ + "folder/file.txt", + "folder/file.txt.data", + }, + }, + { + name: "no change v1", + version: version.OneDrive1DataAndMetaFiles, + input: []string{ + "file.txt.data", + }, + output: []string{ + "file.txt.data", + }, + }, + { + name: "one folder v1", + version: version.OneDrive1DataAndMetaFiles, + input: []string{ + "folder/file.txt.data", + }, + output: []string{ + "folder.dirmeta", + "folder/file.txt.data", + }, + }, + { + name: "nested folders v1", + version: version.OneDrive1DataAndMetaFiles, + input: []string{ + "folder/file.txt.data", + "folder/folder2/file.txt.data", + }, + output: []string{ + "folder.dirmeta", + "folder/file.txt.data", + "folder/folder2.dirmeta", + "folder/folder2/file.txt.data", + }, + }, + { + name: "no change v4", + version: version.OneDrive4DirIncludesPermissions, + input: []string{ + "file.txt.data", + }, + output: []string{ + "file.txt.data", + }, + }, + { + name: "one folder v4", + version: version.OneDrive4DirIncludesPermissions, + input: []string{ + "folder/file.txt.data", + }, + output: []string{ + "folder/file.txt.data", + "folder/folder.dirmeta", + }, + }, + { + name: "nested folders v4", + version: version.OneDrive4DirIncludesPermissions, + input: []string{ + "folder/file.txt.data", + "folder/folder2/file.txt.data", + }, + output: []string{ + "folder/file.txt.data", + "folder/folder.dirmeta", + "folder/folder2/file.txt.data", + "folder/folder2/folder2.dirmeta", + }, + }, + { + name: "no change v6", + version: version.OneDrive6NameInMeta, + input: []string{ + "file.txt.data", + }, + output: []string{ + "file.txt.data", + }, + }, + { + name: "one folder v6", + version: version.OneDrive6NameInMeta, + input: []string{ + "folder/file.txt.data", + }, + output: []string{ + "folder/.dirmeta", + "folder/file.txt.data", + }, + }, + { + name: "nested folders v6", + version: version.OneDrive6NameInMeta, + input: []string{ + "folder/file.txt.data", + "folder/folder2/file.txt.data", + }, + output: []string{ + "folder/.dirmeta", + "folder/file.txt.data", + "folder/folder2/.dirmeta", + "folder/folder2/file.txt.data", + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + _, flush := tester.NewContext(t) + defer flush() + + base := "id/onedrive/user/files/drives/driveID/root:/" + + inPaths := []path.RestorePaths{} + for _, ps := range test.input { + p, err := path.FromDataLayerPath(base+ps, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + pd, err := p.Dir() + require.NoError(t, err, "creating collection path", clues.ToCore(err)) + + inPaths = append( + inPaths, + path.RestorePaths{StoragePath: p, RestorePath: pd}) + } + + outPaths := []path.RestorePaths{} + for _, ps := range test.output { + p, err := path.FromDataLayerPath(base+ps, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + pd, err := p.Dir() + require.NoError(t, err, "creating collection path", clues.ToCore(err)) + + outPaths = append( + outPaths, + path.RestorePaths{StoragePath: p, RestorePath: pd}) + } + + actual, err := AugmentRestorePaths(test.version, inPaths) + require.NoError(t, err, "augmenting paths", clues.ToCore(err)) + + // Ordering of paths matter here as we need dirmeta files + // to show up before file in dir + assert.Equal(t, outPaths, actual, "augmented paths") + }) + } +} + +// TestAugmentRestorePaths_DifferentRestorePath tests that RestorePath +// substitution works properly. Since it's only possible for future backup +// versions to need restore path substitution (i.e. due to storing folders by +// ID instead of name) this is only tested against the most recent backup +// version at the moment. +func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() { + // Adding a simple test here so that we can be sure that this + // function gets updated whenever we add a new version. + require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version") + + type pathPair struct { + storage string + restore string + } + + table := []struct { + name string + version int + input []pathPair + output []pathPair + errCheck assert.ErrorAssertionFunc + }{ + { + name: "nested folders", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: "folder"}, + {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, + }, + output: []pathPair{ + {storage: "folder-id/.dirmeta", restore: "folder"}, + {storage: "folder-id/file.txt.data", restore: "folder"}, + {storage: "folder-id/folder2-id/.dirmeta", restore: "folder/folder2"}, + {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, + }, + errCheck: assert.NoError, + }, + { + name: "restore path longer one folder", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, + }, + output: []pathPair{ + {storage: "folder-id/.dirmeta", restore: "corso_restore/folder"}, + {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, + }, + errCheck: assert.NoError, + }, + { + name: "restore path shorter one folder", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: ""}, + }, + errCheck: assert.Error, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + _, flush := tester.NewContext(t) + defer flush() + + base := "id/onedrive/user/files/drives/driveID/root:/" + + inPaths := []path.RestorePaths{} + for _, ps := range test.input { + p, err := path.FromDataLayerPath(base+ps.storage, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + r, err := path.FromDataLayerPath(base+ps.restore, false) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + inPaths = append( + inPaths, + path.RestorePaths{StoragePath: p, RestorePath: r}) + } + + outPaths := []path.RestorePaths{} + for _, ps := range test.output { + p, err := path.FromDataLayerPath(base+ps.storage, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + r, err := path.FromDataLayerPath(base+ps.restore, false) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + outPaths = append( + outPaths, + path.RestorePaths{StoragePath: p, RestorePath: r}) + } + + actual, err := AugmentRestorePaths(test.version, inPaths) + test.errCheck(t, err, "augmenting paths", clues.ToCore(err)) + + if err != nil { + return + } + + // Ordering of paths matter here as we need dirmeta files + // to show up before file in dir + assert.Equal(t, outPaths, actual, "augmented paths") + }) + } +} diff --git a/src/internal/m365/onedrive/stub/stub.go b/src/internal/m365/service/onedrive/stub/stub.go similarity index 98% rename from src/internal/m365/onedrive/stub/stub.go rename to src/internal/m365/service/onedrive/stub/stub.go index da313a98c..933e98762 100644 --- a/src/internal/m365/onedrive/stub/stub.go +++ b/src/internal/m365/service/onedrive/stub/stub.go @@ -8,8 +8,8 @@ import ( "github.com/alcionai/clues" "github.com/google/uuid" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" m365Stub "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" diff --git a/src/internal/m365/onedrive/testdata/item.go b/src/internal/m365/service/onedrive/testdata/item.go similarity index 100% rename from src/internal/m365/onedrive/testdata/item.go rename to src/internal/m365/service/onedrive/testdata/item.go diff --git a/src/internal/m365/sharepoint/api/beta_service.go b/src/internal/m365/service/sharepoint/api/beta_service.go similarity index 100% rename from src/internal/m365/sharepoint/api/beta_service.go rename to src/internal/m365/service/sharepoint/api/beta_service.go diff --git a/src/internal/m365/sharepoint/api/beta_service_test.go b/src/internal/m365/service/sharepoint/api/beta_service_test.go similarity index 100% rename from src/internal/m365/sharepoint/api/beta_service_test.go rename to src/internal/m365/service/sharepoint/api/beta_service_test.go diff --git a/src/internal/m365/sharepoint/api/pages.go b/src/internal/m365/service/sharepoint/api/pages.go similarity index 100% rename from src/internal/m365/sharepoint/api/pages.go rename to src/internal/m365/service/sharepoint/api/pages.go diff --git a/src/internal/m365/sharepoint/api/pages_test.go b/src/internal/m365/service/sharepoint/api/pages_test.go similarity index 92% rename from src/internal/m365/sharepoint/api/pages_test.go rename to src/internal/m365/service/sharepoint/api/pages_test.go index f3052c7a4..ae02d87c1 100644 --- a/src/internal/m365/sharepoint/api/pages_test.go +++ b/src/internal/m365/service/sharepoint/api/pages_test.go @@ -10,10 +10,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/m365/collection/site" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/sharepoint" - "github.com/alcionai/corso/src/internal/m365/sharepoint/api" - spMock "github.com/alcionai/corso/src/internal/m365/sharepoint/mock" + "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" + spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" @@ -108,7 +108,7 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() { //nolint:lll byteArray := spMock.Page("Byte Test") - pageData := sharepoint.NewItem( + pageData := site.NewItem( testName, io.NopCloser(bytes.NewReader(byteArray)), ) diff --git a/src/internal/m365/sharepoint/api/serialization.go b/src/internal/m365/service/sharepoint/api/serialization.go similarity index 100% rename from src/internal/m365/sharepoint/api/serialization.go rename to src/internal/m365/service/sharepoint/api/serialization.go diff --git a/src/internal/m365/sharepoint/api/serialization_test.go b/src/internal/m365/service/sharepoint/api/serialization_test.go similarity index 97% rename from src/internal/m365/sharepoint/api/serialization_test.go rename to src/internal/m365/service/sharepoint/api/serialization_test.go index 099691d16..8673cd95e 100644 --- a/src/internal/m365/sharepoint/api/serialization_test.go +++ b/src/internal/m365/service/sharepoint/api/serialization_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/suite" bmodels "github.com/alcionai/corso/src/internal/m365/graph/betasdk/models" - spMock "github.com/alcionai/corso/src/internal/m365/sharepoint/mock" + spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" "github.com/alcionai/corso/src/internal/tester" ) diff --git a/src/internal/m365/service/sharepoint/backup.go b/src/internal/m365/service/sharepoint/backup.go new file mode 100644 index 000000000..479d4ac24 --- /dev/null +++ b/src/internal/m365/service/sharepoint/backup.go @@ -0,0 +1,133 @@ +package sharepoint + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/prefixmatcher" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/site" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +func ProduceBackupCollections( + ctx context.Context, + bpc inject.BackupProducerConfig, + ac api.Client, + creds account.M365Config, + su support.StatusUpdater, + errs *fault.Bus, +) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { + b, err := bpc.Selector.ToSharePointBackup() + if err != nil { + return nil, nil, false, clues.Wrap(err, "sharePointDataCollection: parsing selector") + } + + var ( + el = errs.Local() + collections = []data.BackupCollection{} + categories = map[path.CategoryType]struct{}{} + ssmb = prefixmatcher.NewStringSetBuilder() + canUsePreviousBackup bool + ) + + ctx = clues.Add( + ctx, + "site_id", clues.Hide(bpc.ProtectedResource.ID()), + "site_url", clues.Hide(bpc.ProtectedResource.Name())) + + for _, scope := range b.Scopes() { + if el.Failure() != nil { + break + } + + progressBar := observe.MessageWithCompletion( + ctx, + observe.Bulletf("%s", scope.Category().PathType())) + defer close(progressBar) + + var spcs []data.BackupCollection + + switch scope.Category().PathType() { + case path.ListsCategory: + spcs, err = site.CollectLists( + ctx, + bpc, + ac, + creds.AzureTenantID, + su, + errs) + if err != nil { + el.AddRecoverable(ctx, err) + continue + } + + // Lists don't make use of previous metadata + // TODO: Revisit when we add support of lists + canUsePreviousBackup = true + + case path.LibrariesCategory: + spcs, canUsePreviousBackup, err = site.CollectLibraries( + ctx, + bpc, + ac.Drives(), + creds.AzureTenantID, + ssmb, + scope, + su, + errs) + if err != nil { + el.AddRecoverable(ctx, err) + continue + } + + case path.PagesCategory: + spcs, err = site.CollectPages( + ctx, + bpc, + creds, + ac, + su, + errs) + if err != nil { + el.AddRecoverable(ctx, err) + continue + } + + // Lists don't make use of previous metadata + // TODO: Revisit when we add support of pages + canUsePreviousBackup = true + } + + collections = append(collections, spcs...) + + categories[scope.Category().PathType()] = struct{}{} + } + + if len(collections) > 0 { + baseCols, err := graph.BaseCollections( + ctx, + collections, + creds.AzureTenantID, + bpc.ProtectedResource.ID(), + path.SharePointService, + categories, + su, + errs) + if err != nil { + return nil, nil, false, err + } + + collections = append(collections, baseCols...) + } + + return collections, ssmb.ToReader(), canUsePreviousBackup, el.Failure() +} diff --git a/src/internal/m365/sharepoint/backup_test.go b/src/internal/m365/service/sharepoint/backup_test.go similarity index 69% rename from src/internal/m365/sharepoint/backup_test.go rename to src/internal/m365/service/sharepoint/backup_test.go index dfd717711..2a7c6aad8 100644 --- a/src/internal/m365/sharepoint/backup_test.go +++ b/src/internal/m365/service/sharepoint/backup_test.go @@ -9,14 +9,9 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/idname/mock" - "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/m365/collection/drive" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/internal/tester/tconfig" - "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -50,12 +45,12 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { const ( tenantID = "tenant" - site = "site" + siteID = "site" driveID = "driveID1" ) pb := path.Builder{}.Append(testBaseDrivePath.Elements()...) - ep, err := libraryBackupHandler{}.CanonicalPath(pb, tenantID, site) + ep, err := drive.NewLibraryBackupHandler(api.Drives{}, nil).CanonicalPath(pb, tenantID, siteID) require.NoError(suite.T(), err, clues.ToCore(err)) tests := []struct { @@ -99,15 +94,15 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { itemColls = map[string]map[string]string{ driveID: {}, } - collMap = map[string]map[string]*onedrive.Collection{ + collMap = map[string]map[string]*drive.Collection{ driveID: {}, } ) - c := onedrive.NewCollections( - &libraryBackupHandler{api.Drives{}, test.scope}, + c := drive.NewCollections( + drive.NewLibraryBackupHandler(api.Drives{}, test.scope), tenantID, - site, + siteID, nil, control.DefaultOptions()) @@ -169,56 +164,3 @@ func driveRootItem(id string) models.DriveItemable { return item } - -type SharePointPagesSuite struct { - tester.Suite -} - -func TestSharePointPagesSuite(t *testing.T) { - suite.Run(t, &SharePointPagesSuite{ - Suite: tester.NewIntegrationSuite( - t, - [][]string{tconfig.M365AcctCredEnvs}), - }) -} - -func (suite *SharePointPagesSuite) SetupSuite() { - ctx, flush := tester.NewContext(suite.T()) - defer flush() - - graph.InitializeConcurrencyLimiter(ctx, false, 4) -} - -func (suite *SharePointPagesSuite) TestCollectPages() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - siteID = tconfig.M365SiteID(t) - a = tconfig.NewM365Account(t) - ) - - creds, err := a.M365Config() - require.NoError(t, err, clues.ToCore(err)) - - ac, err := api.NewClient(creds, control.DefaultOptions()) - require.NoError(t, err, clues.ToCore(err)) - - bpc := inject.BackupProducerConfig{ - LastBackupVersion: version.NoBackup, - Options: control.DefaultOptions(), - ProtectedResource: mock.NewProvider(siteID, siteID), - } - - col, err := collectPages( - ctx, - bpc, - creds, - ac, - (&MockGraphService{}).UpdateStatus, - fault.New(true)) - assert.NoError(t, err, clues.ToCore(err)) - assert.NotEmpty(t, col) -} diff --git a/src/internal/m365/sharepoint/mock/list.go b/src/internal/m365/service/sharepoint/mock/list.go similarity index 100% rename from src/internal/m365/sharepoint/mock/list.go rename to src/internal/m365/service/sharepoint/mock/list.go diff --git a/src/internal/m365/sharepoint/mock/mock_test.go b/src/internal/m365/service/sharepoint/mock/mock_test.go similarity index 95% rename from src/internal/m365/sharepoint/mock/mock_test.go rename to src/internal/m365/service/sharepoint/mock/mock_test.go index 52070c2cb..61590fb9e 100644 --- a/src/internal/m365/sharepoint/mock/mock_test.go +++ b/src/internal/m365/service/sharepoint/mock/mock_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/m365/sharepoint/api" + "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" "github.com/alcionai/corso/src/internal/tester" ) diff --git a/src/internal/m365/sharepoint/mock/page.go b/src/internal/m365/service/sharepoint/mock/page.go similarity index 100% rename from src/internal/m365/sharepoint/mock/page.go rename to src/internal/m365/service/sharepoint/mock/page.go diff --git a/src/internal/m365/service/sharepoint/restore.go b/src/internal/m365/service/sharepoint/restore.go new file mode 100644 index 000000000..35e1c67cd --- /dev/null +++ b/src/internal/m365/service/sharepoint/restore.go @@ -0,0 +1,122 @@ +package sharepoint + +import ( + "context" + "errors" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/m365/collection/site" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +// ConsumeRestoreCollections will restore the specified data collections into OneDrive +func ConsumeRestoreCollections( + ctx context.Context, + rcc inject.RestoreConsumerConfig, + ac api.Client, + backupDriveIDNames idname.Cacher, + dcs []data.RestoreCollection, + deets *details.Builder, + errs *fault.Bus, + ctr *count.Bus, +) (*support.ControllerOperationStatus, error) { + var ( + lrh = drive.NewLibraryRestoreHandler(ac) + restoreMetrics support.CollectionMetrics + caches = drive.NewRestoreCaches(backupDriveIDNames) + el = errs.Local() + ) + + err := caches.Populate(ctx, lrh, rcc.ProtectedResource.ID()) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } + + // Reorder collections so that the parents directories are created + // before the child directories; a requirement for permissions. + data.SortRestoreCollections(dcs) + + // Iterate through the data collections and restore the contents of each + for _, dc := range dcs { + if el.Failure() != nil { + break + } + + var ( + err error + category = dc.FullPath().Category() + metrics support.CollectionMetrics + ictx = clues.Add(ctx, + "category", category, + "restore_location", clues.Hide(rcc.RestoreConfig.Location), + "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), + "full_path", dc.FullPath()) + ) + + switch dc.FullPath().Category() { + case path.LibrariesCategory: + metrics, err = drive.RestoreCollection( + ictx, + lrh, + rcc, + dc, + caches, + deets, + control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem), + errs, + ctr) + + case path.ListsCategory: + metrics, err = site.RestoreListCollection( + ictx, + ac.Stable, + dc, + rcc.RestoreConfig.Location, + deets, + errs) + + case path.PagesCategory: + metrics, err = site.RestorePageCollection( + ictx, + ac.Stable, + dc, + rcc.RestoreConfig.Location, + deets, + errs) + + default: + return nil, clues.Wrap(clues.New(category.String()), "category not supported").With("category", category) + } + + restoreMetrics = support.CombineMetrics(restoreMetrics, metrics) + + if err != nil { + el.AddRecoverable(ctx, err) + } + + if errors.Is(err, context.Canceled) { + break + } + } + + status := support.CreateStatus( + ctx, + support.Restore, + len(dcs), + restoreMetrics, + rcc.RestoreConfig.Location) + + return status, el.Failure() +} diff --git a/src/internal/m365/stub/stub.go b/src/internal/m365/stub/stub.go index da3340f60..cb986cc4d 100644 --- a/src/internal/m365/stub/stub.go +++ b/src/internal/m365/stub/stub.go @@ -7,10 +7,10 @@ import ( "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/data" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/mock" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/resource" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" ) diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index a3dc19a18..fd39ddc82 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -20,8 +20,8 @@ import ( "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - odMock "github.com/alcionai/corso/src/internal/m365/onedrive/mock" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" + odMock "github.com/alcionai/corso/src/internal/m365/service/onedrive/mock" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations/inject" diff --git a/src/internal/operations/export_test.go b/src/internal/operations/export_test.go index 10dec2ab1..2fbc843a7 100644 --- a/src/internal/operations/export_test.go +++ b/src/internal/operations/export_test.go @@ -18,8 +18,8 @@ import ( "github.com/alcionai/corso/src/internal/data" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 1e6cc62a5..68c8c1b5f 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -16,7 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/kopia" - "github.com/alcionai/corso/src/internal/m365/onedrive" + "github.com/alcionai/corso/src/internal/m365/service/onedrive" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/operations/inject" diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index c97812a63..856505bfd 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -17,10 +17,10 @@ import ( evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/m365" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/tester" diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index 8dd917719..7fc1ff58e 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -18,11 +18,11 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" - "github.com/alcionai/corso/src/internal/m365/exchange" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" - exchTD "github.com/alcionai/corso/src/internal/m365/exchange/testdata" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/resource" + "github.com/alcionai/corso/src/internal/m365/service/exchange" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" + exchTD "github.com/alcionai/corso/src/internal/m365/service/exchange/testdata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 7cbe25f5e..5ed5e5f2e 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -19,10 +19,10 @@ import ( evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/m365" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/resource" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations" "github.com/alcionai/corso/src/internal/operations/inject" diff --git a/src/internal/operations/test/onedrive_test.go b/src/internal/operations/test/onedrive_test.go index 60578f318..c4faedec6 100644 --- a/src/internal/operations/test/onedrive_test.go +++ b/src/internal/operations/test/onedrive_test.go @@ -20,9 +20,9 @@ import ( "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/m365" + "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/streamstore" @@ -132,8 +132,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { return id } - grh := func(ac api.Client) onedrive.RestoreHandler { - return onedrive.NewRestoreHandler(ac) + grh := func(ac api.Client) drive.RestoreHandler { + return drive.NewRestoreHandler(ac) } runDriveIncrementalTest( @@ -157,7 +157,7 @@ func runDriveIncrementalTest( category path.CategoryType, includeContainers func([]string) selectors.Selector, getTestDriveID func(*testing.T, context.Context) string, - getRestoreHandler func(api.Client) onedrive.RestoreHandler, + getRestoreHandler func(api.Client) drive.RestoreHandler, skipPermissionsTests bool, ) { t := suite.T() @@ -388,7 +388,7 @@ func runDriveIncrementalTest( { name: "add permission to new file", updateFiles: func(t *testing.T, ctx context.Context) { - err = onedrive.UpdatePermissions( + err = drive.UpdatePermissions( ctx, rh, driveID, @@ -406,7 +406,7 @@ func runDriveIncrementalTest( { name: "remove permission from new file", updateFiles: func(t *testing.T, ctx context.Context) { - err = onedrive.UpdatePermissions( + err = drive.UpdatePermissions( ctx, rh, driveID, @@ -425,7 +425,7 @@ func runDriveIncrementalTest( name: "add permission to container", updateFiles: func(t *testing.T, ctx context.Context) { targetContainer := containerInfos[container1].id - err = onedrive.UpdatePermissions( + err = drive.UpdatePermissions( ctx, rh, driveID, @@ -444,7 +444,7 @@ func runDriveIncrementalTest( name: "remove permission from container", updateFiles: func(t *testing.T, ctx context.Context) { targetContainer := containerInfos[container1].id - err = onedrive.UpdatePermissions( + err = drive.UpdatePermissions( ctx, rh, driveID, diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index 635c8c1ac..dea0c23bf 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -13,10 +13,9 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/resource" - "github.com/alcionai/corso/src/internal/m365/sharepoint" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" @@ -74,8 +73,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { return id } - grh := func(ac api.Client) onedrive.RestoreHandler { - return sharepoint.NewRestoreHandler(ac) + grh := func(ac api.Client) drive.RestoreHandler { + return drive.NewLibraryRestoreHandler(ac) } runDriveIncrementalTest( diff --git a/src/pkg/backup/details/details.go b/src/pkg/backup/details/details.go index d90e056f8..ec2fdfcd5 100644 --- a/src/pkg/backup/details/details.go +++ b/src/pkg/backup/details/details.go @@ -7,7 +7,7 @@ import ( "github.com/alcionai/clues" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/pkg/path" ) diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index 4646b484a..b804c04cf 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/dttm" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/path" diff --git a/src/pkg/path/drive_test.go b/src/pkg/path/drive_test.go index 131c17b9c..e457a4423 100644 --- a/src/pkg/path/drive_test.go +++ b/src/pkg/path/drive_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" ) diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 3f8813406..8d84d0cf3 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -14,7 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/m365" - "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/observe" diff --git a/src/pkg/selectors/onedrive_test.go b/src/pkg/selectors/onedrive_test.go index be8553bcd..71a7132d3 100644 --- a/src/pkg/selectors/onedrive_test.go +++ b/src/pkg/selectors/onedrive_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/dttm" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 1609783f0..d2b75469f 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -12,7 +12,7 @@ import ( "golang.org/x/exp/slices" "github.com/alcionai/corso/src/internal/common/dttm" - odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" diff --git a/src/pkg/services/m365/api/client_test.go b/src/pkg/services/m365/api/client_test.go index 6385ce41c..20a3007cc 100644 --- a/src/pkg/services/m365/api/client_test.go +++ b/src/pkg/services/m365/api/client_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" diff --git a/src/pkg/services/m365/api/contacts_test.go b/src/pkg/services/m365/api/contacts_test.go index 865adf9a1..afc344cc5 100644 --- a/src/pkg/services/m365/api/contacts_test.go +++ b/src/pkg/services/m365/api/contacts_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/backup/details" diff --git a/src/pkg/services/m365/api/drive_pager.go b/src/pkg/services/m365/api/drive_pager.go index 3ba6e4b46..7a8c100a3 100644 --- a/src/pkg/services/m365/api/drive_pager.go +++ b/src/pkg/services/m365/api/drive_pager.go @@ -13,7 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" - onedrive "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + onedrive "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/pkg/logger" ) diff --git a/src/pkg/services/m365/api/events_test.go b/src/pkg/services/m365/api/events_test.go index 383376cce..cf7d9873f 100644 --- a/src/pkg/services/m365/api/events_test.go +++ b/src/pkg/services/m365/api/events_test.go @@ -13,8 +13,8 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/backup/details" diff --git a/src/pkg/services/m365/api/mail_test.go b/src/pkg/services/m365/api/mail_test.go index 812e86a0c..5c0f1ccd8 100644 --- a/src/pkg/services/m365/api/mail_test.go +++ b/src/pkg/services/m365/api/mail_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/backup/details"