From dba6c2bf2c9187e2057e5f861cabc0a086ca34ef Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Thu, 20 Jul 2023 13:19:03 +0530 Subject: [PATCH 01/62] Update CHANGELOG for 0.11.1 (#3863) --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7fdca0d..44a5075af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +## [v0.11.1] (beta) - 2023-07-20 + ### Fixed - Allow repo connect to succeed when a `corso.toml` file was not provided but configuration is specified using environment variables and flags. @@ -328,7 +330,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Miscellaneous - Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35)) -[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.0...HEAD +[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.1...HEAD +[v0.11.1]: https://github.com/alcionai/corso/compare/v0.11.0...v0.11.1 [v0.11.0]: https://github.com/alcionai/corso/compare/v0.10.0...v0.11.0 [v0.10.0]: https://github.com/alcionai/corso/compare/v0.9.0...v0.10.0 [v0.9.0]: https://github.com/alcionai/corso/compare/v0.8.1...v0.9.0 From a8b375ae608ce96200fbdba0bf6924aef26c1163 Mon Sep 17 00:00:00 2001 From: neha-Gupta1 Date: Thu, 20 Jul 2023 15:09:49 +0530 Subject: [PATCH 02/62] error handling --- src/go.sum | 7 --- src/pkg/services/m365/api/groups.go | 65 ++++++++++++++---------- src/pkg/services/m365/api/groups_test.go | 2 +- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/src/go.sum b/src/go.sum index 34860c7c1..030dabe8a 100644 --- a/src/go.sum +++ b/src/go.sum @@ -71,7 +71,6 @@ github.com/aws/aws-sdk-go v1.44.302/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8 github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -124,7 +123,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -227,7 +225,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -235,7 +232,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= @@ -309,7 +305,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= @@ -446,7 +441,6 @@ go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLk go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -798,7 +792,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/pkg/services/m365/api/groups.go b/src/pkg/services/m365/api/groups.go index 19c381323..9889e2bde 100644 --- a/src/pkg/services/m365/api/groups.go +++ b/src/pkg/services/m365/api/groups.go @@ -8,11 +8,15 @@ import ( "github.com/alcionai/corso/src/internal/common/tform" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" ) -const teamService = "Team" +const ( + teamsAdditionalDataLabel = "Team" + ResourceProvisioningOptions = "resourceProvisioningOptions" +) // --------------------------------------------------------------------------- // controller @@ -22,20 +26,18 @@ func (c Client) Groups() Groups { return Groups{c} } -// On creation of each Microsoft Teams a corrsponding group gets created from them. -// Most of the information like events, drive and mail info will be fetched directly -// from groups. So we pull in group and process only the once which are associated with -// a team for further proccessing of teams. +// On creation of each Teams team a corrsponding group gets created. +// The group acts as the protected resource, and all teams data like events, +// drive and mail messages are owned by that group. // Teams is an interface-compliant provider of the client. type Groups struct { Client } -// GetAll retrieves all groups. -func (c Groups) GetAll( +// GetAllTeams retrieves all groups. +func (c Groups) GetAllTeams( ctx context.Context, - filterTeams bool, errs *fault.Bus, ) ([]models.Groupable, error) { service, err := c.Service() @@ -43,13 +45,26 @@ func (c Groups) GetAll( return nil, err } - return getGroups(ctx, filterTeams, errs, service) + return getGroups(ctx, true, errs, service) +} + +// GetAllGroups retrieves all groups. +func (c Groups) GetAll( + ctx context.Context, + errs *fault.Bus, +) ([]models.Groupable, error) { + service, err := c.Service() + if err != nil { + return nil, err + } + + return getGroups(ctx, false, errs, service) } // GetAll retrieves all groups. func getGroups( ctx context.Context, - filterTeams bool, + getOnlyTeams bool, errs *fault.Bus, service graph.Servicer, ) ([]models.Groupable, error) { @@ -81,8 +96,8 @@ func getGroups( if err != nil { el.AddRecoverable(ctx, graph.Wrap(ctx, err, "validating groups")) } else { - isTeam := IsTeam(item) - if !filterTeams || isTeam { + isTeam := IsTeam(ctx, item) + if !getOnlyTeams || isTeam { groups = append(groups, item) } } @@ -97,15 +112,18 @@ func getGroups( return groups, el.Failure() } -func IsTeam(g models.Groupable) bool { - if g.GetAdditionalData()["resourceProvisioningOptions"] != nil { - val, _ := tform.AnyValueToT[[]any]("resourceProvisioningOptions", g.GetAdditionalData()) +func IsTeam(ctx context.Context, g models.Groupable) bool { + log := logger.Ctx(ctx) + + if g.GetAdditionalData()[ResourceProvisioningOptions] != nil { + val, _ := tform.AnyValueToT[[]any](ResourceProvisioningOptions, g.GetAdditionalData()) for _, v := range val { s, err := str.AnyToString(v) if err != nil { + log.Debug("could not be converted to string value: ", ResourceProvisioningOptions) return false } - if s == teamService { + if s == teamsAdditionalDataLabel { return true } } @@ -125,21 +143,12 @@ func (c Groups) GetByID( resp, err := service.Client().Groups().ByGroupId(identifier).Get(ctx, nil) if err != nil { - return nil, graph.Wrap(ctx, err, "getting group by ID") - } - - if err != nil { - err := graph.Wrap(ctx, err, "getting teams by id") - - // TODO: check if its applicable here - if graph.IsErrItemNotFound(err) { - err = clues.Stack(graph.ErrResourceOwnerNotFound, err) - } + err := graph.Wrap(ctx, err, "getting group by id") return nil, err } - return resp, err + return resp, graph.Stack(ctx, err).OrNil() } // --------------------------------------------------------------------------- @@ -154,7 +163,7 @@ func ValidateGroup(item models.Groupable) error { } if item.GetDisplayName() == nil { - return clues.New("missing principalName") + return clues.New("missing display name") } return nil diff --git a/src/pkg/services/m365/api/groups_test.go b/src/pkg/services/m365/api/groups_test.go index 96e131273..0bb3e3686 100644 --- a/src/pkg/services/m365/api/groups_test.go +++ b/src/pkg/services/m365/api/groups_test.go @@ -45,7 +45,7 @@ func (suite *TeamsIntgSuite) TestGetAll() { teams, err := suite.its.ac. Groups(). - GetAll(ctx, true, fault.New(true)) + GetAllTeams(ctx, fault.New(true)) require.NoError(t, err) require.NotZero(t, len(teams), "must have at least one team") From e130f85da334c138ba39decae767e5197758ce55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Jul 2023 18:45:19 +0000 Subject: [PATCH 03/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.302=20to=201.44.304=20in=20/src=20(#?= =?UTF-8?q?3862)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.302 to 1.44.304.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.304 (2023-07-20)

Service Client Updates

  • service/savingsplans: Updates service documentation

Release v1.44.303 (2023-07-19)

Service Client Updates

  • service/cloudformation: Updates service API and documentation
    • SDK and documentation updates for GetTemplateSummary API (unrecognized resources)
  • service/ec2: Updates service documentation
    • Amazon EC2 documentation updates.
  • service/grafana: Updates service API, documentation, and paginators
  • service/medical-imaging: Adds new service
  • service/ram: Updates service API and documentation
  • service/ssm-sap: Updates service API and documentation
  • service/wafv2: Updates service API and documentation
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.302&new-version=1.44.304)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 99581228f..6ebbb2adb 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.302 + github.com/aws/aws-sdk-go v1.44.304 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 030dabe8a..a66eb73f4 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.302 h1:ST3ko6GrJKn3Xi+nAvxjG3uk/V1pW8KC52WLeIxqqNk= -github.com/aws/aws-sdk-go v1.44.302/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.304 h1:crcJBVeewWcVAXDQChzJWZYGFq9i1TYLycAtQ6Xpi4c= +github.com/aws/aws-sdk-go v1.44.304/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 9359679f9967b7f0eaae245d89c96f2a3727f79a Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 20 Jul 2023 15:05:30 -0600 Subject: [PATCH 04/62] add api funcs for creating documentLibs (#3793) Adds api handlers for creating document libraries in sharepoint. This is the first step in allowing us to restore drives that were deleted between backup and restore. --- #### Does this PR need a docs update or release note? - [x] :clock1: Yes, but in a later PR #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- CHANGELOG.md | 4 + src/internal/common/idname/idname.go | 33 +- src/internal/data/mock/collection.go | 48 ++ src/internal/m365/controller.go | 20 + src/internal/m365/controller_test.go | 90 +++- src/internal/m365/mock/connector.go | 2 + src/internal/m365/onedrive/handlers.go | 15 +- .../m365/onedrive/item_collector_test.go | 4 +- src/internal/m365/onedrive/item_handler.go | 14 + src/internal/m365/onedrive/mock/handlers.go | 16 + src/internal/m365/onedrive/restore.go | 214 ++++++++- src/internal/m365/onedrive/restore_test.go | 436 +++++++++++++++++- src/internal/m365/restore.go | 6 + .../m365/sharepoint/library_handler.go | 36 +- src/internal/m365/sharepoint/restore.go | 19 +- src/internal/operations/inject/inject.go | 11 + src/internal/operations/restore.go | 14 +- src/pkg/control/restore.go | 9 +- src/pkg/services/m365/api/lists.go | 64 +++ src/pkg/services/m365/api/lists_test.go | 57 +++ website/docs/support/known-issues.md | 2 - 21 files changed, 1051 insertions(+), 63 deletions(-) create mode 100644 src/pkg/services/m365/api/lists.go create mode 100644 src/pkg/services/m365/api/lists_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a5075af..296b01a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +### Fixed +- SharePoint document libraries deleted after the last backup can now be restored. + ## [v0.11.1] (beta) - 2023-07-20 ### Fixed @@ -23,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Return a ServiceNotEnabled error when a tenant has no active SharePoint license. - Added retries for http/2 stream connection failures when downloading large item content. +- SharePoint document libraries that were deleted after the last backup can now be restored. ### Known issues - If a link share is created for an item with inheritance disabled diff --git a/src/internal/common/idname/idname.go b/src/internal/common/idname/idname.go index d56fab025..0367d954b 100644 --- a/src/internal/common/idname/idname.go +++ b/src/internal/common/idname/idname.go @@ -40,6 +40,11 @@ type Cacher interface { ProviderForName(id string) Provider } +type CacheBuilder interface { + Add(id, name string) + Cacher +} + var _ Cacher = &cache{} type cache struct { @@ -47,17 +52,29 @@ type cache struct { nameToID map[string]string } -func NewCache(idToName map[string]string) cache { - nti := make(map[string]string, len(idToName)) - - for id, name := range idToName { - nti[name] = id +func NewCache(idToName map[string]string) *cache { + c := cache{ + idToName: map[string]string{}, + nameToID: map[string]string{}, } - return cache{ - idToName: idToName, - nameToID: nti, + if len(idToName) > 0 { + nti := make(map[string]string, len(idToName)) + + for id, name := range idToName { + nti[name] = id + } + + c.idToName = idToName + c.nameToID = nti } + + return &c +} + +func (c *cache) Add(id, name string) { + c.idToName[strings.ToLower(id)] = name + c.nameToID[strings.ToLower(name)] = id } // IDOf returns the id associated with the given name. diff --git a/src/internal/data/mock/collection.go b/src/internal/data/mock/collection.go index 63f2b2dd8..55f291a7f 100644 --- a/src/internal/data/mock/collection.go +++ b/src/internal/data/mock/collection.go @@ -1,12 +1,24 @@ package mock import ( + "context" "io" "time" + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" ) +// --------------------------------------------------------------------------- +// stream +// --------------------------------------------------------------------------- + +var _ data.Stream = &Stream{} + type Stream struct { ID string Reader io.ReadCloser @@ -52,3 +64,39 @@ type errReader struct { func (er errReader) Read([]byte) (int, error) { return 0, er.readErr } + +// --------------------------------------------------------------------------- +// collection +// --------------------------------------------------------------------------- + +var ( + _ data.Collection = &Collection{} + _ data.BackupCollection = &Collection{} + _ data.RestoreCollection = &Collection{} +) + +type Collection struct{} + +func (c Collection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Stream { + return nil +} + +func (c Collection) FullPath() path.Path { + return nil +} + +func (c Collection) PreviousPath() path.Path { + return nil +} + +func (c Collection) State() data.CollectionState { + return data.NewState +} + +func (c Collection) DoNotMergeItems() bool { + return true +} + +func (c Collection) FetchItemByName(ctx context.Context, name string) (data.Stream, error) { + return &Stream{}, clues.New("not implemented") +} diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index b0c8792e5..9b037350b 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -47,6 +48,11 @@ type Controller struct { // mutex used to synchronize updates to `status` mu sync.Mutex status support.ControllerOperationStatus // contains the status of the last run status + + // backupDriveIDNames is populated on restore. It maps the backup's + // drive names to their id. Primarily for use when creating or looking + // up a new drive. + backupDriveIDNames idname.CacheBuilder } func NewController( @@ -142,6 +148,20 @@ func (ctrl *Controller) incrementAwaitingMessages() { ctrl.wg.Add(1) } +func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { + if ctrl.backupDriveIDNames == nil { + ctrl.backupDriveIDNames = idname.NewCache(map[string]string{}) + } + + if dii.SharePoint != nil { + ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) + } + + if dii.OneDrive != nil { + ctrl.backupDriveIDNames.Add(dii.OneDrive.DriveID, dii.OneDrive.DriveName) + } +} + // --------------------------------------------------------------------------- // Resource Lookup Handling // --------------------------------------------------------------------------- diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 6d04b7e9e..ef729493b 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -12,8 +12,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" + dataMock "github.com/alcionai/corso/src/internal/data/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" @@ -22,6 +24,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" @@ -260,6 +263,82 @@ func (suite *ControllerUnitSuite) TestController_Wait() { assert.Equal(t, int64(4), result.Bytes) } +func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { + var ( + odid = "od-id" + odname = "od-name" + spid = "sp-id" + spname = "sp-name" + // intentionally declared outside the test loop + ctrl = &Controller{ + wg: &sync.WaitGroup{}, + region: &trace.Region{}, + backupDriveIDNames: idname.NewCache(nil), + } + ) + + table := []struct { + name string + service path.ServiceType + cat path.CategoryType + dii details.ItemInfo + expectID string + expectName string + }{ + { + name: "exchange", + dii: details.ItemInfo{ + Exchange: &details.ExchangeInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "folder", + dii: details.ItemInfo{ + Folder: &details.FolderInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "onedrive", + dii: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + DriveID: odid, + DriveName: odname, + }, + }, + expectID: odid, + expectName: odname, + }, + { + name: "sharepoint", + dii: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + DriveID: spid, + DriveName: spname, + }, + }, + expectID: spid, + expectName: spname, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctrl.CacheItemInfo(test.dii) + + name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID) + assert.Equal(t, test.expectName, name) + + id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName) + assert.Equal(t, test.expectID, id) + }) + } +} + // --------------------------------------------------------------------------- // Integration tests // --------------------------------------------------------------------------- @@ -315,7 +394,7 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { RestorePermissions: true, ToggleFeatures: control.Toggles{}, }, - nil, + []data.RestoreCollection{&dataMock.Collection{}}, fault.New(true), count.New()) assert.Error(t, err, clues.ToCore(err)) @@ -397,13 +476,8 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() { test.col, fault.New(true), count.New()) - require.NoError(t, err, clues.ToCore(err)) - assert.NotNil(t, deets) - - stats := suite.ctrl.Wait() - assert.Zero(t, stats.Objects) - assert.Zero(t, stats.Folders) - assert.Zero(t, stats.Successes) + require.Error(t, err, clues.ToCore(err)) + assert.Nil(t, deets) }) } } diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 05cb8e159..977306883 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -69,3 +69,5 @@ func (ctrl Controller) ConsumeRestoreCollections( ) (*details.Details, error) { return ctrl.Deets, ctrl.Err } + +func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {} diff --git a/src/internal/m365/onedrive/handlers.go b/src/internal/m365/onedrive/handlers.go index dfea5ee17..cb33b373d 100644 --- a/src/internal/m365/onedrive/handlers.go +++ b/src/internal/m365/onedrive/handlers.go @@ -35,6 +35,7 @@ type BackupHandler interface { api.Getter GetItemPermissioner GetItemer + NewDrivePagerer // PathPrefix constructs the service and category specific path prefix for // the given values. @@ -49,7 +50,6 @@ type BackupHandler interface { // ServiceCat returns the service and category used by this implementation. ServiceCat() (path.ServiceType, path.CategoryType) - NewDrivePager(resourceOwner string, fields []string) api.DrivePager NewItemPager(driveID, link string, fields []string) api.DriveItemDeltaEnumerator // FormatDisplayPath creates a human-readable string to represent the // provided path. @@ -61,6 +61,10 @@ type BackupHandler interface { IncludesDir(dir string) bool } +type NewDrivePagerer interface { + NewDrivePager(resourceOwner string, fields []string) api.DrivePager +} + type GetItemPermissioner interface { GetItemPermission( ctx context.Context, @@ -86,7 +90,9 @@ type RestoreHandler interface { GetItemsByCollisionKeyser GetRootFolderer ItemInfoAugmenter + NewDrivePagerer NewItemContentUploader + PostDriver PostItemInContainerer DeleteItemPermissioner UpdateItemPermissioner @@ -145,6 +151,13 @@ type UpdateItemLinkSharer interface { ) (models.Permissionable, error) } +type PostDriver interface { + PostDrive( + ctx context.Context, + protectedResourceID, driveName string, + ) (models.Driveable, error) +} + type PostItemInContainerer interface { PostItemInContainer( ctx context.Context, diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/onedrive/item_collector_test.go index 6e4a79be6..ec2ab26af 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -361,8 +361,8 @@ func (suite *OneDriveIntgSuite) TestCreateGetDeleteFolder() { Folders: folderElements, } - caches := NewRestoreCaches() - caches.DriveIDToRootFolderID[driveID] = ptr.Val(rootFolder.GetId()) + caches := NewRestoreCaches(nil) + caches.DriveIDToDriveInfo[driveID] = driveInfo{rootFolderID: ptr.Val(rootFolder.GetId())} rh := NewRestoreHandler(suite.ac) diff --git a/src/internal/m365/onedrive/item_handler.go b/src/internal/m365/onedrive/item_handler.go index 0b1420cf0..64701da8f 100644 --- a/src/internal/m365/onedrive/item_handler.go +++ b/src/internal/m365/onedrive/item_handler.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" + "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -133,6 +134,19 @@ func NewRestoreHandler(ac api.Client) *itemRestoreHandler { return &itemRestoreHandler{ac.Drives()} } +func (h itemRestoreHandler) PostDrive( + context.Context, + string, string, +) (models.Driveable, error) { + return nil, clues.New("creating drives in oneDrive is not supported") +} + +func (h itemRestoreHandler) NewDrivePager( + resourceOwner string, fields []string, +) api.DrivePager { + return h.ac.NewUserDrivePager(resourceOwner, fields) +} + // AugmentItemInfo will populate a details.OneDriveInfo struct // with properties from the drive item. ItemSize is specified // separately for restore processes because the local itemable diff --git a/src/internal/m365/onedrive/mock/handlers.go b/src/internal/m365/onedrive/mock/handlers.go index 92b4573e6..75dd3c3f1 100644 --- a/src/internal/m365/onedrive/mock/handlers.go +++ b/src/internal/m365/onedrive/mock/handlers.go @@ -249,9 +249,25 @@ type RestoreHandler struct { PostItemResp models.DriveItemable PostItemErr error + DrivePagerV api.DrivePager + + PostDriveResp models.Driveable + PostDriveErr error + UploadSessionErr error } +func (h RestoreHandler) PostDrive( + ctx context.Context, + protectedResourceID, driveName string, +) (models.Driveable, error) { + return h.PostDriveResp, h.PostDriveErr +} + +func (h RestoreHandler) NewDrivePager(string, []string) api.DrivePager { + return h.DrivePagerV +} + func (h *RestoreHandler) AugmentItemInfo( details.ItemInfo, models.DriveItemable, diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index 84b8f1cd0..f951419be 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -15,6 +15,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -37,9 +38,17 @@ const ( maxUploadRetries = 3 ) +type driveInfo struct { + id string + name string + rootFolderID string +} + type restoreCaches struct { + BackupDriveIDName idname.Cacher collisionKeyToItemID map[string]api.DriveItemIDType - DriveIDToRootFolderID map[string]string + DriveIDToDriveInfo map[string]driveInfo + DriveNameToDriveInfo map[string]driveInfo Folders *folderCache OldLinkShareIDToNewID map[string]string OldPermIDToNewID map[string]string @@ -48,10 +57,74 @@ type restoreCaches struct { pool sync.Pool } -func NewRestoreCaches() *restoreCaches { +func (rc *restoreCaches) AddDrive( + ctx context.Context, + md models.Driveable, + grf GetRootFolderer, +) error { + di := driveInfo{ + id: ptr.Val(md.GetId()), + name: ptr.Val(md.GetName()), + } + + ctx = clues.Add(ctx, "drive_info", di) + + root, err := grf.GetRootFolder(ctx, di.id) + if err != nil { + return clues.Wrap(err, "getting drive root id") + } + + di.rootFolderID = ptr.Val(root.GetId()) + + rc.DriveIDToDriveInfo[di.id] = di + rc.DriveNameToDriveInfo[di.name] = di + + return nil +} + +// Populate looks up drive items available to the protectedResource +// and adds their info to the caches. +func (rc *restoreCaches) Populate( + ctx context.Context, + gdparf GetDrivePagerAndRootFolderer, + protectedResourceID string, +) error { + drives, err := api.GetAllDrives( + ctx, + gdparf.NewDrivePager(protectedResourceID, nil), + true, + maxDrivesRetries) + if err != nil { + return clues.Wrap(err, "getting drives") + } + + for _, md := range drives { + if err := rc.AddDrive(ctx, md, gdparf); err != nil { + return clues.Wrap(err, "caching drive") + } + } + + return nil +} + +type GetDrivePagerAndRootFolderer interface { + GetRootFolderer + NewDrivePagerer +} + +func NewRestoreCaches( + backupDriveIDNames idname.Cacher, +) *restoreCaches { + // avoid nil panics + if backupDriveIDNames == nil { + backupDriveIDNames = idname.NewCache(nil) + } + return &restoreCaches{ + BackupDriveIDName: backupDriveIDNames, collisionKeyToItemID: map[string]api.DriveItemIDType{}, - DriveIDToRootFolderID: map[string]string{}, + DriveIDToDriveInfo: map[string]driveInfo{}, + DriveNameToDriveInfo: map[string]driveInfo{}, Folders: NewFolderCache(), OldLinkShareIDToNewID: map[string]string{}, OldPermIDToNewID: map[string]string{}, @@ -73,19 +146,27 @@ func ConsumeRestoreCollections( backupVersion int, restoreCfg control.RestoreConfig, opts control.Options, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - restoreMetrics support.CollectionMetrics - caches = NewRestoreCaches() - el = errs.Local() + restoreMetrics support.CollectionMetrics + el = errs.Local() + caches = NewRestoreCaches(backupDriveIDNames) + protectedResourceID = dcs[0].FullPath().ResourceOwner() + fallbackDriveName = restoreCfg.Location ) ctx = clues.Add(ctx, "backup_version", backupVersion) + err := caches.Populate(ctx, rh, protectedResourceID) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } + // Reorder collections so that the parents directories are created // before the child directories; a requirement for permissions. data.SortRestoreCollections(dcs) @@ -102,7 +183,7 @@ func ConsumeRestoreCollections( ictx = clues.Add( ctx, "category", dc.FullPath().Category(), - "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), + "resource_owner", clues.Hide(protectedResourceID), "full_path", dc.FullPath()) ) @@ -115,6 +196,7 @@ func ConsumeRestoreCollections( caches, deets, opts.RestorePermissions, + fallbackDriveName, errs, ctr.Local()) if err != nil { @@ -152,18 +234,20 @@ func RestoreCollection( caches *restoreCaches, deets *details.Builder, restorePerms bool, // TODD: move into restoreConfig + fallbackDriveName string, errs *fault.Bus, ctr *count.Bus, ) (support.CollectionMetrics, error) { var ( - metrics = support.CollectionMetrics{} - directory = dc.FullPath() - el = errs.Local() - metricsObjects int64 - metricsBytes int64 - metricsSuccess int64 - wg sync.WaitGroup - complete bool + metrics = support.CollectionMetrics{} + directory = dc.FullPath() + protectedResourceID = directory.ResourceOwner() + el = errs.Local() + metricsObjects int64 + metricsBytes int64 + metricsSuccess int64 + wg sync.WaitGroup + complete bool ) ctx, end := diagnostics.Span(ctx, "gc:drive:restoreCollection", diagnostics.Label("path", directory)) @@ -174,15 +258,23 @@ func RestoreCollection( return metrics, clues.Wrap(err, "creating drive path").WithClues(ctx) } - if _, ok := caches.DriveIDToRootFolderID[drivePath.DriveID]; !ok { - root, err := rh.GetRootFolder(ctx, drivePath.DriveID) - if err != nil { - return metrics, clues.Wrap(err, "getting drive root id") - } - - caches.DriveIDToRootFolderID[drivePath.DriveID] = ptr.Val(root.GetId()) + di, err := ensureDriveExists( + ctx, + rh, + caches, + drivePath, + protectedResourceID, + fallbackDriveName) + if err != nil { + return metrics, clues.Wrap(err, "ensuring drive exists") } + // clobber the drivePath details with the details retrieved + // in the ensure func, as they might have changed to reflect + // a different drive as a restore location. + drivePath.DriveID = di.id + drivePath.Root = di.rootFolderID + // Assemble folder hierarchy we're going to restore into (we recreate the folder hierarchy // from the backup under this the restore folder instead of root) // i.e. Restore into `/` @@ -704,7 +796,7 @@ func createRestoreFolders( driveID = drivePath.DriveID folders = restoreDir.Elements() location = path.Builder{}.Append(driveID) - parentFolderID = caches.DriveIDToRootFolderID[drivePath.DriveID] + parentFolderID = caches.DriveIDToDriveInfo[drivePath.DriveID].rootFolderID ) ctx = clues.Add( @@ -1113,3 +1205,79 @@ func AugmentRestorePaths( return paths, nil } + +type PostDriveAndGetRootFolderer interface { + PostDriver + GetRootFolderer +} + +// ensureDriveExists looks up the drive by its id. If no drive is found with +// that ID, a new drive is generated with the same name. If the name collides +// with an existing drive, a number is appended to the drive name. Eg: foo -> +// foo 1. This will repeat as many times as is needed. +// Returns the root folder of the drive +func ensureDriveExists( + ctx context.Context, + pdagrf PostDriveAndGetRootFolderer, + caches *restoreCaches, + drivePath *path.DrivePath, + protectedResourceID, fallbackDriveName string, +) (driveInfo, error) { + driveID := drivePath.DriveID + + // the drive might already be cached by ID. it's okay + // if the name has changed. the ID is a better reference + // anyway. + if di, ok := caches.DriveIDToDriveInfo[driveID]; ok { + return di, nil + } + + var ( + newDriveName = fallbackDriveName + newDrive models.Driveable + err error + ) + + // if the drive wasn't found by ID, maybe we can find a + // drive with the same name but different ID. + // start by looking up the old drive's name + oldName, ok := caches.BackupDriveIDName.NameOf(driveID) + if ok { + // check for drives that currently have the same name + if di, ok := caches.DriveNameToDriveInfo[oldName]; ok { + return di, nil + } + + // if no current drives have the same name, we'll make + // a new drive with that name. + newDriveName = oldName + } + + nextDriveName := newDriveName + + // For sharepoint, document libraries can collide by name with + // item types beyond just drive. Lists, for example, cannot share + // names with document libraries (they're the same type, actually). + // In those cases we need to rename the drive until we can create + // one without a collision. + for i := 1; ; i++ { + ictx := clues.Add(ctx, "new_drive_name", clues.Hide(nextDriveName)) + + newDrive, err = pdagrf.PostDrive(ictx, protectedResourceID, nextDriveName) + if err != nil && !errors.Is(err, graph.ErrItemAlreadyExistsConflict) { + return driveInfo{}, clues.Wrap(err, "creating new drive") + } + + if err == nil { + break + } + + nextDriveName = fmt.Sprintf("%s %d", newDriveName, i) + } + + if err := caches.AddDrive(ctx, newDrive, pdagrf); err != nil { + return driveInfo{}, clues.Wrap(err, "adding drive to cache").OrNil() + } + + return caches.DriveIDToDriveInfo[ptr.Val(newDrive.GetId())], nil +} diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index 4128661f5..dbb7317c9 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" @@ -21,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" + apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type RestoreUnitSuite struct { @@ -491,7 +493,7 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { mndi.SetId(ptr.To(mndiID)) var ( - caches = NewRestoreCaches() + caches = NewRestoreCaches(nil) rh = &mock.RestoreHandler{ PostItemResp: models.NewDriveItem(), DeleteItemErr: test.deleteErr, @@ -617,3 +619,435 @@ func (suite *RestoreUnitSuite) TestCreateFolder() { }) } } + +type mockGRF struct { + err error + rootFolder models.DriveItemable +} + +func (m *mockGRF) GetRootFolder( + context.Context, + string, +) (models.DriveItemable, error) { + return m.rootFolder, m.err +} + +func (suite *RestoreUnitSuite) TestRestoreCaches_AddDrive() { + rfID := "this-is-id" + driveID := "another-id" + name := "name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + table := []struct { + name string + mock *mockGRF + expectErr require.ErrorAssertionFunc + expectID string + checkValues bool + }{ + { + name: "good", + mock: &mockGRF{rootFolder: rf}, + expectErr: require.NoError, + expectID: rfID, + checkValues: true, + }, + { + name: "err", + mock: &mockGRF{err: assert.AnError}, + expectErr: require.Error, + expectID: "", + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := NewRestoreCaches(nil) + err := rc.AddDrive(ctx, md, test.mock) + test.expectErr(t, err, clues.ToCore(err)) + + if test.checkValues { + idResult := rc.DriveIDToDriveInfo[driveID] + assert.Equal(t, driveID, idResult.id, "drive id") + assert.Equal(t, name, idResult.name, "drive name") + assert.Equal(t, test.expectID, idResult.rootFolderID, "root folder id") + + nameResult := rc.DriveNameToDriveInfo[name] + assert.Equal(t, driveID, nameResult.id, "drive id") + assert.Equal(t, name, nameResult.name, "drive name") + assert.Equal(t, test.expectID, nameResult.rootFolderID, "root folder id") + } + }) + } +} + +type mockGDPARF struct { + err error + rootFolder models.DriveItemable + pager *apiMock.DrivePager +} + +func (m *mockGDPARF) GetRootFolder( + context.Context, + string, +) (models.DriveItemable, error) { + return m.rootFolder, m.err +} + +func (m *mockGDPARF) NewDrivePager( + string, + []string, +) api.DrivePager { + return m.pager +} + +func (suite *RestoreUnitSuite) TestRestoreCaches_Populate() { + rfID := "this-is-id" + driveID := "another-id" + name := "name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + table := []struct { + name string + mock *apiMock.DrivePager + expectErr require.ErrorAssertionFunc + expectLen int + checkValues bool + }{ + { + name: "no results", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Drives: []models.Driveable{}}, + }, + }, + expectErr: require.NoError, + expectLen: 0, + }, + { + name: "one result", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Drives: []models.Driveable{md}}, + }, + }, + expectErr: require.NoError, + expectLen: 1, + checkValues: true, + }, + { + name: "error", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Err: assert.AnError}, + }, + }, + expectErr: require.Error, + expectLen: 0, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + gdparf := &mockGDPARF{ + rootFolder: rf, + pager: test.mock, + } + + rc := NewRestoreCaches(nil) + err := rc.Populate(ctx, gdparf, "shmoo") + test.expectErr(t, err, clues.ToCore(err)) + + assert.Len(t, rc.DriveIDToDriveInfo, test.expectLen) + assert.Len(t, rc.DriveNameToDriveInfo, test.expectLen) + + if test.checkValues { + idResult := rc.DriveIDToDriveInfo[driveID] + assert.Equal(t, driveID, idResult.id, "drive id") + assert.Equal(t, name, idResult.name, "drive name") + assert.Equal(t, rfID, idResult.rootFolderID, "root folder id") + + nameResult := rc.DriveNameToDriveInfo[name] + assert.Equal(t, driveID, nameResult.id, "drive id") + assert.Equal(t, name, nameResult.name, "drive name") + assert.Equal(t, rfID, nameResult.rootFolderID, "root folder id") + } + }) + } +} + +type mockPDAGRF struct { + i int + postResp []models.Driveable + postErr []error + + grf mockGRF +} + +func (m *mockPDAGRF) PostDrive( + ctx context.Context, + protectedResourceID, driveName string, +) (models.Driveable, error) { + defer func() { m.i++ }() + + md := m.postResp[m.i] + if md != nil { + md.SetName(&driveName) + } + + return md, m.postErr[m.i] +} + +func (m *mockPDAGRF) GetRootFolder( + ctx context.Context, + driveID string, +) (models.DriveItemable, error) { + return m.grf.rootFolder, m.grf.err +} + +func (suite *RestoreUnitSuite) TestEnsureDriveExists() { + rfID := "this-is-id" + driveID := "another-id" + oldID := "old-id" + name := "name" + otherName := "other name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + grf := mockGRF{rootFolder: rf} + + makeMD := func() models.Driveable { + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + return md + } + + dp := &path.DrivePath{ + DriveID: driveID, + Root: "root:", + Folders: path.Elements{}, + } + + oldDP := &path.DrivePath{ + DriveID: oldID, + Root: "root:", + Folders: path.Elements{}, + } + + populatedCache := func(id string) *restoreCaches { + rc := NewRestoreCaches(nil) + di := driveInfo{ + id: id, + name: name, + } + rc.DriveIDToDriveInfo[id] = di + rc.DriveNameToDriveInfo[name] = di + + return rc + } + + oldDriveIDNames := idname.NewCache(nil) + oldDriveIDNames.Add(oldID, name) + + idSwitchedCache := func() *restoreCaches { + rc := NewRestoreCaches(oldDriveIDNames) + di := driveInfo{ + id: "diff", + name: name, + } + rc.DriveIDToDriveInfo["diff"] = di + rc.DriveNameToDriveInfo[name] = di + + return rc + } + + table := []struct { + name string + dp *path.DrivePath + mock *mockPDAGRF + rc *restoreCaches + expectErr require.ErrorAssertionFunc + fallbackName string + expectName string + expectID string + skipValueChecks bool + }{ + { + name: "drive already in cache", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: populatedCache(driveID), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + { + name: "drive with same name but different id exists", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: idSwitchedCache(), + expectErr: require.NoError, + fallbackName: otherName, + expectName: name, + expectID: "diff", + }, + { + name: "drive created with old name", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: NewRestoreCaches(oldDriveIDNames), + expectErr: require.NoError, + fallbackName: otherName, + expectName: name, + expectID: driveID, + }, + { + name: "drive created with fallback name", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.NoError, + fallbackName: otherName, + expectName: otherName, + expectID: driveID, + }, + { + name: "error creating drive", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil}, + postErr: []error{assert.AnError}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.Error, + fallbackName: name, + expectName: "", + skipValueChecks: true, + expectID: driveID, + }, + { + name: "drive name already exists", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: populatedCache("beaux"), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + { + name: "list with name already exists", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.NoError, + fallbackName: name, + expectName: name + " 1", + expectID: driveID, + }, + { + name: "list with old name already exists", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: NewRestoreCaches(oldDriveIDNames), + expectErr: require.NoError, + fallbackName: name, + expectName: name + " 1", + expectID: driveID, + }, + { + name: "drive and list with name already exist", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: populatedCache(driveID), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := test.rc + + di, err := ensureDriveExists( + ctx, + test.mock, + rc, + test.dp, + "prID", + test.fallbackName) + test.expectErr(t, err, clues.ToCore(err)) + + if !test.skipValueChecks { + assert.Equal(t, test.expectName, di.name, "ensured drive has expected name") + assert.Equal(t, test.expectID, di.id, "ensured drive has expected id") + + nameResult := rc.DriveNameToDriveInfo[test.expectName] + assert.Equal(t, test.expectName, nameResult.name, "found drive entry with expected name") + } + }) + } +} diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index 3c5e3e646..31e36e2bb 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -38,6 +38,10 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) ctx = clues.Add(ctx, "restore_config", restoreCfg) // TODO(rkeepers): needs PII control + if len(dcs) == 0 { + return nil, clues.New("no data collections to restore") + } + var ( status *support.ControllerOperationStatus deets = &details.Builder{} @@ -54,6 +58,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( backupVersion, restoreCfg, opts, + ctrl.backupDriveIDNames, dcs, deets, errs, @@ -65,6 +70,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctrl.AC, restoreCfg, opts, + ctrl.backupDriveIDNames, dcs, deets, errs, diff --git a/src/internal/m365/sharepoint/library_handler.go b/src/internal/m365/sharepoint/library_handler.go index 07c997fcb..3f16c6eae 100644 --- a/src/internal/m365/sharepoint/library_handler.go +++ b/src/internal/m365/sharepoint/library_handler.go @@ -157,11 +157,25 @@ func (h libraryBackupHandler) IncludesDir(dir string) bool { var _ onedrive.RestoreHandler = &libraryRestoreHandler{} type libraryRestoreHandler struct { - ac api.Drives + ac api.Client +} + +func (h libraryRestoreHandler) PostDrive( + ctx context.Context, + siteID, driveName string, +) (models.Driveable, error) { + return h.ac.Lists().PostDrive(ctx, siteID, driveName) } func NewRestoreHandler(ac api.Client) *libraryRestoreHandler { - return &libraryRestoreHandler{ac.Drives()} + return &libraryRestoreHandler{ac} +} + +func (h libraryRestoreHandler) NewDrivePager( + resourceOwner string, + fields []string, +) api.DrivePager { + return h.ac.Drives().NewSiteDrivePager(resourceOwner, fields) } func (h libraryRestoreHandler) AugmentItemInfo( @@ -177,21 +191,21 @@ func (h libraryRestoreHandler) DeleteItem( ctx context.Context, driveID, itemID string, ) error { - return h.ac.DeleteItem(ctx, driveID, itemID) + return h.ac.Drives().DeleteItem(ctx, driveID, itemID) } func (h libraryRestoreHandler) DeleteItemPermission( ctx context.Context, driveID, itemID, permissionID string, ) error { - return h.ac.DeleteItemPermission(ctx, driveID, itemID, permissionID) + return h.ac.Drives().DeleteItemPermission(ctx, driveID, itemID, permissionID) } func (h libraryRestoreHandler) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, ) (map[string]api.DriveItemIDType, error) { - m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) + m, err := h.ac.Drives().GetItemsInContainerByCollisionKey(ctx, driveID, containerID) if err != nil { return nil, err } @@ -203,7 +217,7 @@ func (h libraryRestoreHandler) NewItemContentUpload( ctx context.Context, driveID, itemID string, ) (models.UploadSessionable, error) { - return h.ac.NewItemContentUpload(ctx, driveID, itemID) + return h.ac.Drives().NewItemContentUpload(ctx, driveID, itemID) } func (h libraryRestoreHandler) PostItemPermissionUpdate( @@ -211,7 +225,7 @@ func (h libraryRestoreHandler) PostItemPermissionUpdate( driveID, itemID string, body *drives.ItemItemsItemInvitePostRequestBody, ) (drives.ItemItemsItemInviteResponseable, error) { - return h.ac.PostItemPermissionUpdate(ctx, driveID, itemID, body) + return h.ac.Drives().PostItemPermissionUpdate(ctx, driveID, itemID, body) } func (h libraryRestoreHandler) PostItemLinkShareUpdate( @@ -219,7 +233,7 @@ func (h libraryRestoreHandler) PostItemLinkShareUpdate( driveID, itemID string, body *drives.ItemItemsItemCreateLinkPostRequestBody, ) (models.Permissionable, error) { - return h.ac.PostItemLinkShareUpdate(ctx, driveID, itemID, body) + return h.ac.Drives().PostItemLinkShareUpdate(ctx, driveID, itemID, body) } func (h libraryRestoreHandler) PostItemInContainer( @@ -228,21 +242,21 @@ func (h libraryRestoreHandler) PostItemInContainer( newItem models.DriveItemable, onCollision control.CollisionPolicy, ) (models.DriveItemable, error) { - return h.ac.PostItemInContainer(ctx, driveID, parentFolderID, newItem, onCollision) + return h.ac.Drives().PostItemInContainer(ctx, driveID, parentFolderID, newItem, onCollision) } func (h libraryRestoreHandler) GetFolderByName( ctx context.Context, driveID, parentFolderID, folderName string, ) (models.DriveItemable, error) { - return h.ac.GetFolderByName(ctx, driveID, parentFolderID, folderName) + return h.ac.Drives().GetFolderByName(ctx, driveID, parentFolderID, folderName) } func (h libraryRestoreHandler) GetRootFolder( ctx context.Context, driveID string, ) (models.DriveItemable, error) { - return h.ac.GetRootFolder(ctx, driveID) + return h.ac.Drives().GetRootFolder(ctx, driveID) } // --------------------------------------------------------------------------- diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index 417d6d87c..c38b82e08 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -10,6 +10,8 @@ import ( "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -33,17 +35,25 @@ func ConsumeRestoreCollections( ac api.Client, restoreCfg control.RestoreConfig, opts control.Options, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - restoreMetrics support.CollectionMetrics - caches = onedrive.NewRestoreCaches() - el = errs.Local() + lrh = libraryRestoreHandler{ac} + protectedResourceID = dcs[0].FullPath().ResourceOwner() + restoreMetrics support.CollectionMetrics + caches = onedrive.NewRestoreCaches(backupDriveIDNames) + el = errs.Local() ) + err := caches.Populate(ctx, lrh, protectedResourceID) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } + // Reorder collections so that the parents directories are created // before the child directories; a requirement for permissions. data.SortRestoreCollections(dcs) @@ -69,13 +79,14 @@ func ConsumeRestoreCollections( case path.LibrariesCategory: metrics, err = onedrive.RestoreCollection( ictx, - libraryRestoreHandler{ac.Drives()}, + lrh, restoreCfg, backupVersion, dc, caches, deets, opts.RestorePermissions, + control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem), errs, ctr) diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index ae2c8d534..912b46743 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -46,6 +46,17 @@ type ( ) (*details.Details, error) Wait() *data.CollectionStats + + CacheItemInfoer + } + + CacheItemInfoer interface { + // CacheItemInfo is used by the consumer to cache metadata that is + // sourced from per-item info, but may be valuable to the restore at + // large. + // Ex: pairing drive ids with drive names as they appeared at the time + // of backup. + CacheItemInfo(v details.ItemInfo) } RepoMaintenancer interface { diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index e77b1104b..0f853a853 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -219,7 +219,13 @@ func (op *RestoreOperation) do( observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) - paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.Errors) + paths, err := formatDetailsForRestoration( + ctx, + bup.Version, + op.Selectors, + deets, + op.rc, + op.Errors) if err != nil { return nil, clues.Wrap(err, "formatting paths from details") } @@ -359,6 +365,7 @@ func formatDetailsForRestoration( backupVersion int, sel selectors.Selector, deets *details.Details, + cii inject.CacheItemInfoer, errs *fault.Bus, ) ([]path.RestorePaths, error) { fds, err := sel.Reduce(ctx, deets, errs) @@ -366,6 +373,11 @@ func formatDetailsForRestoration( return nil, err } + // allow restore controllers to iterate over item metadata + for _, ent := range fds.Entries { + cii.CacheItemInfo(ent.ItemInfo) + } + paths, err := pathtransformer.GetPaths(ctx, backupVersion, fds.Items(), errs) if err != nil { return nil, clues.Wrap(err, "getting restore paths") diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 5fc5f7be8..2b4129d9f 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -52,8 +52,9 @@ type RestoreConfig struct { // Defaults to "Corso_Restore_" Location string - // Drive specifies the drive into which the data will be restored. - // If empty, data is restored to the same drive that was backed up. + // Drive specifies the name of the drive into which the data will be + // restored. If empty, data is restored to the same drive that was backed + // up. // Defaults to empty. Drive string } @@ -65,6 +66,10 @@ func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { } } +func DefaultRestoreContainerName(timeFormat dttm.TimeFormat) string { + return defaultRestoreLocation + dttm.FormatNow(timeFormat) +} + // EnsureRestoreConfigDefaults sets all non-supported values in the config // struct to the default value. func EnsureRestoreConfigDefaults( diff --git a/src/pkg/services/m365/api/lists.go b/src/pkg/services/m365/api/lists.go new file mode 100644 index 000000000..fb6abaa48 --- /dev/null +++ b/src/pkg/services/m365/api/lists.go @@ -0,0 +1,64 @@ +package api + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" +) + +// --------------------------------------------------------------------------- +// controller +// --------------------------------------------------------------------------- + +func (c Client) Lists() Lists { + return Lists{c} +} + +// Lists is an interface-compliant provider of the client. +type Lists struct { + Client +} + +// PostDrive creates a new list of type drive. Specifically used to create +// documentLibraries for SharePoint Sites. +func (c Lists) PostDrive( + ctx context.Context, + siteID, driveName string, +) (models.Driveable, error) { + list := models.NewList() + list.SetDisplayName(&driveName) + list.SetDescription(ptr.To("corso auto-generated restore destination")) + + li := models.NewListInfo() + li.SetTemplate(ptr.To("documentLibrary")) + list.SetList(li) + + // creating a list of type documentLibrary will result in the creation + // of a new drive owned by the given site. + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists() + + newList, err := builder.Post(ctx, list, nil) + if graph.IsErrItemAlreadyExistsConflict(err) { + return nil, clues.Stack(graph.ErrItemAlreadyExistsConflict, err).WithClues(ctx) + } + + if err != nil { + return nil, graph.Wrap(ctx, err, "creating documentLibrary list") + } + + // drive information is not returned by the list creation. + drive, err := builder. + ByListId(ptr.Val(newList.GetId())). + Drive(). + Get(ctx, nil) + + return drive, graph.Wrap(ctx, err, "fetching created documentLibrary").OrNil() +} diff --git a/src/pkg/services/m365/api/lists_test.go b/src/pkg/services/m365/api/lists_test.go new file mode 100644 index 000000000..5864427f2 --- /dev/null +++ b/src/pkg/services/m365/api/lists_test.go @@ -0,0 +1,57 @@ +package api_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/control/testdata" +) + +type ListsAPIIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func (suite *ListsAPIIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func TestListsAPIIntgSuite(t *testing.T) { + suite.Run(t, &ListsAPIIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *ListsAPIIntgSuite) TestLists_PostDrive() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + driveName = testdata.DefaultRestoreConfig("list_api_post_drive").Location + siteID = suite.its.siteID + ) + + // first post, should have no errors + list, err := acl.PostDrive(ctx, siteID, driveName) + require.NoError(t, err, clues.ToCore(err)) + // the site name cannot be set when posting, only its DisplayName. + // so we double check here that we're still getting the name we expect. + assert.Equal(t, driveName, ptr.Val(list.GetName())) + + // second post, same name, should error on name conflict] + _, err = acl.PostDrive(ctx, siteID, driveName) + require.ErrorIs(t, err, graph.ErrItemAlreadyExistsConflict, clues.ToCore(err)) +} diff --git a/website/docs/support/known-issues.md b/website/docs/support/known-issues.md index 754bddfb6..e6bc12809 100644 --- a/website/docs/support/known-issues.md +++ b/website/docs/support/known-issues.md @@ -16,8 +16,6 @@ Below is a list of known Corso issues and limitations: from M365 while a backup creation is running. The next backup creation will correct any missing data. -* SharePoint document library data can't be restored after the library has been deleted. - * Sharing information of items in OneDrive/SharePoint using sharing links aren't backed up and restored. * Permissions/Access given to a site group can't be restored. From d555501093ad7b2a8da889816d3a29e39cbc815d Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 20 Jul 2023 19:13:09 -0600 Subject: [PATCH 05/62] add integration tests for missing drives (#3847) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Issue(s) * #3652 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .github/actions/purge-m365-data/action.yml | 15 +- .github/workflows/ci_test_cleanup.yml | 1 + src/cmd/purge/scripts/onedrivePurge.ps1 | 95 ++++++- src/internal/m365/graph/errors.go | 8 +- src/internal/operations/restore_test.go | 197 -------------- .../operations/test/sharepoint_test.go | 244 +++++++++++++++++- 6 files changed, 352 insertions(+), 208 deletions(-) diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index d7681e55b..725fc711f 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -19,7 +19,9 @@ inputs: site: description: Sharepoint site where data is to be purged. libraries: - description: List of library names within site where data is to be purged. + description: List of library names within the site where data is to be purged. + library-prefix: + description: List of library names within the site where the library will get deleted entirely. folder-prefix: description: Name of the folder to be purged. If falsy, will purge the set of static, well known folders instead. older-than: @@ -76,7 +78,10 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 \ + -User ${{ inputs.user }} \ + -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") \ + -PurgeBeforeTimestamp ${{ inputs.older-than }} ################################################################################################################ # Sharepoint @@ -90,4 +95,8 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -Site ${{ inputs.site }} \ + -LibraryNameList "${{ inputs.libraries }}".split(",") \ + -FolderPrefixPurgeList ${{ inputs.folder-prefix }} \ + -LibraryPrefixDeleteList ${{ inputs.library-prefix }} \ + -PurgeBeforeTimestamp ${{ inputs.older-than }} diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 65e678e4b..9f51d3e14 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -62,6 +62,7 @@ jobs: site: ${{ vars[matrix.site] }} folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} + library-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} azure-client-secret: ${{ secrets.CLIENT_SECRET }} diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index 7372245aa..b4717d35e 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -19,14 +19,17 @@ Param ( [datetime]$PurgeBeforeTimestamp, [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")] - [String[]]$FolderPrefixPurgeList + [String[]]$FolderPrefixPurgeList, + + [Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")] + [String[]]$LibraryPrefixDeleteList ) Set-StrictMode -Version 2.0 # Attempt to set network timeout to 10min [System.Net.ServicePointManager]::MaxServicePointIdleTime = 600000 -function Get-TimestampFromName { +function Get-TimestampFromFolderName { param ( [Parameter(Mandatory = $True, HelpMessage = "Folder ")] [Microsoft.SharePoint.Client.Folder]$folder @@ -54,6 +57,36 @@ function Get-TimestampFromName { return $timestamp } + +function Get-TimestampFromListName { + param ( + [Parameter(Mandatory = $True, HelpMessage = "List ")] + [Microsoft.SharePoint.Client.List]$list + ) + + $name = $list.Title + + #fallback on list create time + [datetime]$timestamp = $list.LastItemUserModifiedDate + + try { + # Assumes that the timestamp is at the end and starts with yyyy-mm-ddT and is ISO8601 + if ($name -imatch "(\d{4}}-\d{2}-\d{2}T.*)") { + $timestamp = [System.Convert]::ToDatetime($Matches.0) + } + + # Assumes that the timestamp is at the end and starts with dd-MMM-yyyy_HH-MM-SS + if ($name -imatch "(\d{2}-[a-zA-Z]{3}-\d{4}_\d{2}-\d{2}-\d{2})") { + $timestamp = [datetime]::ParseExact($Matches.0, "dd-MMM-yyyy_HH-mm-ss", [CultureInfo]::InvariantCulture, "AssumeUniversal") + } + } + catch {} + + Write-Verbose "List: $name, create timestamp: $timestamp" + + return $timestamp +} + function Purge-Library { [CmdletBinding(SupportsShouldProcess)] Param ( @@ -77,7 +110,7 @@ function Purge-Library { foreach ($f in $folders) { $folderName = $f.Name - $createTime = Get-TimestampFromName -Folder $f + $createTime = Get-TimestampFromFolderName -Folder $f if ($PurgeBeforeTimestamp -gt $createTime) { foreach ($p in $FolderPrefixPurgeList) { @@ -97,7 +130,7 @@ function Purge-Library { if ($f.ServerRelativeUrl -imatch "$SiteSuffix/{0,1}(.+?)/{0,1}$folderName$") { $siteRelativeParentPath = $Matches.1 } - + if ($PSCmdlet.ShouldProcess("Name: " + $f.Name + " Parent: " + $siteRelativeParentPath, "Remove folder")) { Write-Host "Deleting folder: "$f.Name" with parent: $siteRelativeParentPath" try { @@ -110,6 +143,54 @@ function Purge-Library { } } +function Delete-LibraryByPrefix { + [CmdletBinding(SupportsShouldProcess)] + Param ( + [Parameter(Mandatory = $True, HelpMessage = "Document library root")] + [String]$LibraryNamePrefix, + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")] + [datetime]$PurgeBeforeTimestamp, + + [Parameter(Mandatory = $True, HelpMessage = "Site suffix")] + [String[]]$SiteSuffix + ) + + Write-Host "`nDeleting library: $LibraryNamePrefix" + + $listsToDelete = @() + $lists = Get-PnPList + + foreach ($l in $lists) { + $listName = $l.Title + $createTime = Get-TimestampFromListName -List $l + + if ($PurgeBeforeTimestamp -gt $createTime) { + foreach ($p in $FolderPrefixPurgeList) { + if ($listName -like "$p*") { + $listsToDelete += $l + } + } + } + } + + Write-Host "Found"$listsToDelete.count"lists to delete" + + foreach ($l in $listsToDelete) { + $listName = $l.Title + + if ($PSCmdlet.ShouldProcess("Name: " + $l.Title + "Remove folder")) { + Write-Host "Deleting list: "$l.Title + try { + Remove-PnPList -Identity $l.Id -Force + } + catch [ System.Management.Automation.ItemNotFoundException ] { + Write-Host "List: "$f.Name" is already deleted. Skipping..." + } + } + } +} + ######## MAIN ######### # Setup SharePointPnP @@ -176,4 +257,8 @@ $FolderPrefixPurgeList = $FolderPrefixPurgeList | ForEach-Object { @($_.Split(', foreach ($library in $LibraryNameList) { Purge-Library -LibraryName $library -PurgeBeforeTimestamp $PurgeBeforeTimestamp -FolderPrefixPurgeList $FolderPrefixPurgeList -SiteSuffix $siteSuffix -} \ No newline at end of file +} + +foreach ($libraryPfx in $LibraryPrefixDeleteList) { + Delete-LibraryByPrefix -LibraryNamePrefix $libraryPfx -PurgeBeforeTimestamp $PurgeBeforeTimestamp -SiteSuffix $siteSuffix +} diff --git a/src/internal/m365/graph/errors.go b/src/internal/m365/graph/errors.go index f0df6b4ec..dad2674a4 100644 --- a/src/internal/m365/graph/errors.go +++ b/src/internal/m365/graph/errors.go @@ -271,7 +271,9 @@ func Wrap(ctx context.Context, e error, msg string) *clues.Err { e = clues.Stack(e, clues.New(mainMsg)) } - return setLabels(clues.Wrap(e, msg).WithClues(ctx).With(data...), innerMsg) + ce := clues.Wrap(e, msg).WithClues(ctx).With(data...).WithTrace(1) + + return setLabels(ce, innerMsg) } // Stack is a helper function that extracts ODataError metadata from @@ -292,7 +294,9 @@ func Stack(ctx context.Context, e error) *clues.Err { e = clues.Stack(e, clues.New(mainMsg)) } - return setLabels(clues.Stack(e).WithClues(ctx).With(data...), innerMsg) + ce := clues.Stack(e).WithClues(ctx).With(data...).WithTrace(1) + + return setLabels(ce, innerMsg) } // stackReq is a helper function that extracts ODataError metadata from diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index f02ee1731..5a314aaf4 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -10,8 +10,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" - inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -21,7 +19,6 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" - "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/tester" @@ -32,7 +29,6 @@ import ( "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/services/m365/api" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" "github.com/alcionai/corso/src/pkg/store" ) @@ -143,13 +139,6 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { // integration // --------------------------------------------------------------------------- -type bupResults struct { - selectorResourceOwners []string - backupID model.StableID - items int - ctrl *m365.Controller -} - type RestoreOpIntegrationSuite struct { tester.Suite @@ -267,192 +256,6 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { } } -func setupExchangeBackup( - t *testing.T, - kw *kopia.Wrapper, - sw *store.Wrapper, - acct account.Account, - owner string, -) bupResults { - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - users = []string{owner} - esel = selectors.NewExchangeBackup(users) - ) - - esel.DiscreteOwner = owner - esel.Include( - esel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()), - esel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), - esel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) - - ctrl, sel := ControllerWithSelector(t, ctx, acct, resource.Users, esel.Selector, nil, nil) - - bo, err := NewBackupOperation( - ctx, - control.Defaults(), - kw, - sw, - ctrl, - acct, - sel, - inMock.NewProvider(owner, owner), - evmock.NewBus()) - require.NoError(t, err, clues.ToCore(err)) - - err = bo.Run(ctx) - require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, bo.Results.BackupID) - - return bupResults{ - selectorResourceOwners: users, - backupID: bo.Results.BackupID, - // Discount metadata collection files (1 delta and one prev path for each category). - // These meta files are used to aid restore, but are not themselves - // restored (ie: counted as writes). - items: bo.Results.ItemsWritten - 6, - ctrl: ctrl, - } -} - -func setupSharePointBackup( - t *testing.T, - kw *kopia.Wrapper, - sw *store.Wrapper, - acct account.Account, - owner string, -) bupResults { - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - sites = []string{owner} - ssel = selectors.NewSharePointBackup(sites) - ) - - // assume a folder name "test" exists in the drive. - // this is brittle, and requires us to backfill anytime - // the site under test changes, but also prevents explosive - // growth from re-backup/restore of restored files. - ssel.Include(ssel.LibraryFolders([]string{"test"}, selectors.PrefixMatch())) - ssel.DiscreteOwner = owner - - ctrl, sel := ControllerWithSelector(t, ctx, acct, resource.Sites, ssel.Selector, nil, nil) - - bo, err := NewBackupOperation( - ctx, - control.Defaults(), - kw, - sw, - ctrl, - acct, - sel, - inMock.NewProvider(owner, owner), - evmock.NewBus()) - require.NoError(t, err, clues.ToCore(err)) - - err = bo.Run(ctx) - require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, bo.Results.BackupID) - - return bupResults{ - selectorResourceOwners: sites, - backupID: bo.Results.BackupID, - // Discount metadata files (2: 1 delta, 1 prev path) - // assume only one folder, and therefore 1 dirmeta per drive - // (2 drives: documents and more documents) - // assume only one file in each folder, and therefore 1 meta per drive - // (2 drives: documents and more documents) - // Meta files are used to aid restore, but are not themselves - // restored (ie: counted as writes). - items: bo.Results.ItemsWritten - 6, - ctrl: ctrl, - } -} - -func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { - tables := []struct { - name string - owner string - restoreCfg control.RestoreConfig - getSelector func(t *testing.T, owners []string) selectors.Selector - setup func(t *testing.T, kw *kopia.Wrapper, sw *store.Wrapper, acct account.Account, owner string) bupResults - }{ - { - name: "Exchange_Restore", - owner: tconfig.M365UserID(suite.T()), - restoreCfg: testdata.DefaultRestoreConfig(""), - getSelector: func(t *testing.T, owners []string) selectors.Selector { - rsel := selectors.NewExchangeRestore(owners) - rsel.Include(rsel.AllData()) - - return rsel.Selector - }, - setup: setupExchangeBackup, - }, - { - name: "SharePoint_Restore", - owner: tconfig.M365SiteID(suite.T()), - restoreCfg: control.DefaultRestoreConfig(dttm.SafeForTesting), - getSelector: func(t *testing.T, owners []string) selectors.Selector { - rsel := selectors.NewSharePointRestore(owners) - rsel.Include(rsel.Library(tconfig.LibraryDocuments), rsel.Library(tconfig.LibraryMoreDocuments)) - - return rsel.Selector - }, - setup: setupSharePointBackup, - }, - } - - for _, test := range tables { - suite.Run(test.name, func() { - var ( - t = suite.T() - mb = evmock.NewBus() - bup = test.setup(t, suite.kw, suite.sw, suite.acct, test.owner) - ) - - ctx, flush := tester.NewContext(t) - defer flush() - - require.NotZero(t, bup.items) - require.NotEmpty(t, bup.backupID) - - ro, err := NewRestoreOperation( - ctx, - control.Options{FailureHandling: control.FailFast}, - suite.kw, - suite.sw, - bup.ctrl, - tconfig.NewM365Account(t), - bup.backupID, - test.getSelector(t, bup.selectorResourceOwners), - test.restoreCfg, - mb, - count.New()) - require.NoError(t, err, clues.ToCore(err)) - - ds, err := ro.Run(ctx) - - require.NoError(t, err, "restoreOp.Run() %+v", clues.ToCore(err)) - require.NotEmpty(t, ro.Results, "restoreOp results") - require.NotNil(t, ds, "restored details") - assert.Equal(t, ro.Status, Completed, "restoreOp status") - assert.Equal(t, ro.Results.ItemsWritten, len(ds.Items()), "item write count matches len details") - assert.Less(t, 0, ro.Results.ItemsRead, "restore items read") - assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read") - assert.Equal(t, 1, ro.Results.ResourceOwners, "resource Owners") - assert.NoError(t, ro.Errors.Failure(), "non-recoverable error", clues.ToCore(ro.Errors.Failure())) - assert.Empty(t, ro.Errors.Recovered(), "recoverable errors") - assert.Equal(t, bup.items, ro.Results.ItemsWritten, "backup and restore wrote the same num of items") - assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") - assert.Equal(t, 1, mb.TimesCalled[events.RestoreEnd], "restore-end events") - }) - } -} - func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { t := suite.T() diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index 1b5a52dc2..ad2e5d79a 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -5,6 +5,9 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/google/uuid" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -19,6 +22,8 @@ import ( "github.com/alcionai/corso/src/internal/version" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" + ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" @@ -196,7 +201,7 @@ func (suite *SharePointRestoreIntgSuite) SetupSuite() { } func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { - sel := selectors.NewSharePointBackup([]string{suite.its.userID}) + sel := selectors.NewSharePointBackup([]string{suite.its.siteID}) sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Filter(sel.Library("documents")) sel.DiscreteOwner = suite.its.siteID @@ -209,3 +214,240 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedO suite.its.siteDriveID, suite.its.siteDriveRootFolderID) } + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives() { + t := suite.T() + + // despite the client having a method for drive.Patch and drive.Delete, both only return + // the error code and message `invalidRequest`. + t.Skip("graph api doesn't allow patch or delete on drives, so we cannot run any conditions") + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := ctrlTD.DefaultRestoreConfig("restore_deleted_drives") + rc.OnCollision = control.Copy + + // create a new drive + md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.siteID, rc.Location) + require.NoError(t, err, clues.ToCore(err)) + + driveID := ptr.Val(md.GetId()) + + // get the root folder + mdi, err := suite.its.ac.Drives().GetRootFolder(ctx, driveID) + require.NoError(t, err, clues.ToCore(err)) + + rootFolderID := ptr.Val(mdi.GetId()) + + // add an item to it + itemName := uuid.NewString() + + item := models.NewDriveItem() + item.SetName(ptr.To(itemName + ".txt")) + + file := models.NewFile() + item.SetFile(file) + + _, err = suite.its.ac.Drives().PostItemInContainer( + ctx, + driveID, + rootFolderID, + item, + control.Copy) + require.NoError(t, err, clues.ToCore(err)) + + // run a backup + var ( + mb = evmock.NewBus() + opts = control.Defaults() + graphClient = suite.its.ac.Stable.Client() + ) + + bsel := selectors.NewSharePointBackup([]string{suite.its.siteID}) + bsel.Include(selTD.SharePointBackupFolderScope(bsel)) + bsel.Filter(bsel.Library(rc.Location)) + bsel.DiscreteOwner = suite.its.siteID + + bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + // test cases: + + // first test, we take the current drive and rename it. + // the restore should find the drive by id and restore items + // into it like normal. Due to collision handling, this should + // create a copy of the current item. + suite.Run("renamed drive", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + patchBody := models.NewDrive() + patchBody.SetName(ptr.To("some other name")) + + md, err = graphClient. + Drives(). + ByDriveId(driveID). + Patch(ctx, patchBody, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 2) + + for _, item := range items { + assert.Contains(t, ptr.Val(item.GetName()), itemName) + } + }) + + // second test, we delete the drive altogether. the restore should find + // no existing drives, but it should have the old drive's name and attempt + // to recreate that drive by name. + suite.Run("deleted drive", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + err = graphClient. + Drives(). + ByDriveId(driveID). + Delete(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + pgr := suite.its.ac. + Drives(). + NewSiteDrivePager(suite.its.siteID, []string{"id", "name"}) + + drives, err := api.GetAllDrives(ctx, pgr, false, -1) + require.NoError(t, err, clues.ToCore(err)) + + var created models.Driveable + + for _, drive := range drives { + if ptr.Val(drive.GetName()) == ptr.Val(created.GetName()) && + ptr.Val(drive.GetId()) != driveID { + created = drive + break + } + } + + require.NotNil(t, created, "found the restored drive by name") + md = created + driveID = ptr.Val(md.GetId()) + + mdi, err := suite.its.ac.Drives().GetRootFolder(ctx, driveID) + require.NoError(t, err, clues.ToCore(err)) + + rootFolderID = ptr.Val(mdi.GetId()) + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 1) + + assert.Equal(t, ptr.Val(items[0].GetName()), itemName+".txt") + }) + + // final test, run a follow-up restore. This should match the + // drive we created in the prior test by name, but not by ID. + suite.Run("different drive - same name", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 2) + + for _, item := range items { + assert.Contains(t, ptr.Val(item.GetName()), itemName) + } + }) +} From 5ac0c135bd3dfa11a621374c91287da7fd9c6b2f Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 20 Jul 2023 20:22:03 -0600 Subject: [PATCH 06/62] single-line cleanup scripts in gh (#3871) - [x] :bug: Bugfix #### Test Plan - [x] :green_heart: E2E --- .github/actions/purge-m365-data/action.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index 725fc711f..ec26b987d 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -78,10 +78,7 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 \ - -User ${{ inputs.user }} \ - -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") \ - -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} ################################################################################################################ # Sharepoint @@ -95,8 +92,4 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} \ - -LibraryNameList "${{ inputs.libraries }}".split(",") \ - -FolderPrefixPurgeList ${{ inputs.folder-prefix }} \ - -LibraryPrefixDeleteList ${{ inputs.library-prefix }} \ - -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} From 1f7941d19bdd4a1463f086bab869975e55143f64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 06:04:51 +0000 Subject: [PATCH 07/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20sass=20from?= =?UTF-8?q?=201.63.6=20to=201.64.0=20in=20/website=20(#3874)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sass](https://github.com/sass/dart-sass) from 1.63.6 to 1.64.0.
Release notes

Sourced from sass's releases.

Dart Sass 1.64.0

To install Sass 1.64.0, download one of the packages below and add it to your PATH, or see the Sass website for full installation instructions.

Changes

  • Comments that appear before or between @use and @forward rules are now emitted in source order as much as possible, instead of always being emitted after the CSS of all module dependencies.

  • Fix a bug where an interpolation in a custom property name crashed if the file was loaded by a @use nested in an @import.

JavaScript API

  • Add a new SassCalculation type that represents the calculation objects added in Dart Sass 1.40.0.

  • Add Value.assertCalculation(), which returns the value if it's a SassCalculation and throws an error otherwise.

  • Produce a better error message when an environment that supports some Node.js APIs loads the browser entrypoint but attempts to access the filesystem.

Embedded Sass

  • Fix a bug where nested relative @imports failed to load when using the deprecated functions render or renderSync and those relative imports were loaded multiple times across different files.

See the full changelog for changes in earlier releases.

Changelog

Sourced from sass's changelog.

1.64.0

  • Comments that appear before or between @use and @forward rules are now emitted in source order as much as possible, instead of always being emitted after the CSS of all module dependencies.

  • Fix a bug where an interpolation in a custom property name crashed if the file was loaded by a @use nested in an @import.

JavaScript API

  • Add a new SassCalculation type that represents the calculation objects added in Dart Sass 1.40.0.

  • Add Value.assertCalculation(), which returns the value if it's a SassCalculation and throws an error otherwise.

  • Produce a better error message when an environment that supports some Node.js APIs loads the browser entrypoint but attempts to access the filesystem.

Embedded Sass

  • Fix a bug where nested relative @imports failed to load when using the deprecated functions render or renderSync and those relative imports were loaded multiple times across different files.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sass&package-manager=npm_and_yarn&previous-version=1.63.6&new-version=1.64.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 14 +++++++------- website/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 21df1ab55..a4561faa5 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -24,7 +24,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.63.6", + "sass": "^1.64.0", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" @@ -12571,9 +12571,9 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.63.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", - "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.0.tgz", + "integrity": "sha512-m7YtAGmQta9uANIUJwXesAJMSncqH+3INc8kdVXs6eV6GUC8Qu2IYKQSN8PRLgiQfpca697G94klm2leYMxSHw==", "dependencies": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -23802,9 +23802,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sass": { - "version": "1.63.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", - "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.0.tgz", + "integrity": "sha512-m7YtAGmQta9uANIUJwXesAJMSncqH+3INc8kdVXs6eV6GUC8Qu2IYKQSN8PRLgiQfpca697G94klm2leYMxSHw==", "requires": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", diff --git a/website/package.json b/website/package.json index 2c7272ce9..55573ce46 100644 --- a/website/package.json +++ b/website/package.json @@ -30,7 +30,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.63.6", + "sass": "^1.64.0", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" From 876f0d8bd90abe4a28b9d5aab8e29ae41019d436 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 06:10:03 +0000 Subject: [PATCH 08/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20docusaurus-pl?= =?UTF-8?q?ugin-sass=20from=200.2.4=20to=200.2.5=20in=20/website=20(#3873)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docusaurus-plugin-sass](https://github.com/rlamana/docusaurus-plugin-sass) from 0.2.4 to 0.2.5.
Release notes

Sourced from docusaurus-plugin-sass's releases.

v0.2.5

What's Changed

New Contributors

Full Changelog: https://github.com/rlamana/docusaurus-plugin-sass/compare/v0.2.4...v0.2.5-2

Commits
  • 70e42fc Merge pull request #21 from roydukkey/issue/20
  • 7a5720c Merge pull request #33 from noomorph/patch-1
  • d9f4fad build: support docusarus@3.0.0-alpha.0
  • 2fe9818 Merge branch 'rlamana:master' into issue/20
  • bb70d7d Add missing newline
  • c482d9a Add TypeScript support for Sass/SCSS modules
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docusaurus-plugin-sass&package-manager=npm_and_yarn&previous-version=0.2.4&new-version=0.2.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 16 ++++++++-------- website/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index a4561faa5..bbf48e4b8 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -16,7 +16,7 @@ "animate.css": "^4.1.1", "clsx": "^2.0.0", "docusaurus-plugin-image-zoom": "^1.0.1", - "docusaurus-plugin-sass": "^0.2.4", + "docusaurus-plugin-sass": "^0.2.5", "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", @@ -6541,14 +6541,14 @@ } }, "node_modules/docusaurus-plugin-sass": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.4.tgz", - "integrity": "sha512-r9bLXW6X2z64bzQUQZB1SxmNlGvSO9swTFALgiMjr/1O4FRDti6BseU4Sw2mlZkYvVQTq8cJMJIP6w7z/5We8Q==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz", + "integrity": "sha512-Z+D0fLFUKcFpM+bqSUmqKIU+vO+YF1xoEQh5hoFreg2eMf722+siwXDD+sqtwU8E4MvVpuvsQfaHwODNlxJAEg==", "dependencies": { "sass-loader": "^10.1.1" }, "peerDependencies": { - "@docusaurus/core": "^2.0.0-beta", + "@docusaurus/core": "^2.0.0-beta || ^3.0.0-alpha", "sass": "^1.30.0" } }, @@ -19867,9 +19867,9 @@ } }, "docusaurus-plugin-sass": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.4.tgz", - "integrity": "sha512-r9bLXW6X2z64bzQUQZB1SxmNlGvSO9swTFALgiMjr/1O4FRDti6BseU4Sw2mlZkYvVQTq8cJMJIP6w7z/5We8Q==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz", + "integrity": "sha512-Z+D0fLFUKcFpM+bqSUmqKIU+vO+YF1xoEQh5hoFreg2eMf722+siwXDD+sqtwU8E4MvVpuvsQfaHwODNlxJAEg==", "requires": { "sass-loader": "^10.1.1" } diff --git a/website/package.json b/website/package.json index 55573ce46..3e1388b77 100644 --- a/website/package.json +++ b/website/package.json @@ -22,7 +22,7 @@ "animate.css": "^4.1.1", "clsx": "^2.0.0", "docusaurus-plugin-image-zoom": "^1.0.1", - "docusaurus-plugin-sass": "^0.2.4", + "docusaurus-plugin-sass": "^0.2.5", "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", From d9525310a1f8f5150986973e4769df19ca00c2bb Mon Sep 17 00:00:00 2001 From: neha-Gupta1 Date: Fri, 21 Jul 2023 16:06:06 +0530 Subject: [PATCH 09/62] add test cases --- src/internal/tester/tconfig/config.go | 16 +++ .../tester/tconfig/protected_resources.go | 26 ++++ src/pkg/services/m365/api/groups.go | 37 +++-- src/pkg/services/m365/api/groups_test.go | 130 +++++++++++++++++- src/pkg/services/m365/api/helper_test.go | 9 ++ 5 files changed, 190 insertions(+), 28 deletions(-) diff --git a/src/internal/tester/tconfig/config.go b/src/internal/tester/tconfig/config.go index c6bcd6e4b..1d9a89310 100644 --- a/src/internal/tester/tconfig/config.go +++ b/src/internal/tester/tconfig/config.go @@ -25,6 +25,8 @@ const ( TestCfgAzureTenantID = "azure_tenantid" TestCfgSiteID = "m365siteid" TestCfgSiteURL = "m365siteurl" + TestCfgTeamID = "m365teamid" + TestCfgGroupID = "m365groupid" TestCfgUserID = "m365userid" TestCfgSecondaryUserID = "secondarym365userid" TestCfgTertiaryUserID = "tertiarym365userid" @@ -38,6 +40,8 @@ const ( const ( EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL" + EnvCorsoM365TestTeamID = "CORSO_M365_TEST_TEAM_ID" + EnvCorsoM365TestGroupID = "CORSO_M365_TEST_Group_ID" EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID" @@ -148,6 +152,18 @@ func ReadTestConfig() (map[string]string, error) { os.Getenv(EnvCorsoM365TestSiteID), vpr.GetString(TestCfgSiteID), "10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") + fallbackTo( + testEnv, + TestCfgTeamID, + os.Getenv(EnvCorsoM365TestTeamID), + vpr.GetString(TestCfgTeamID), + "d288d6bc-4595-4ff5-87a1-6e7fd750aa42") + fallbackTo( + testEnv, + TestCfgGroupID, + os.Getenv(EnvCorsoM365TestGroupID), + vpr.GetString(TestCfgGroupID), + "3d1129b1-52e1-4f49-a47a-a515b14c8a8e") fallbackTo( testEnv, TestCfgSiteURL, diff --git a/src/internal/tester/tconfig/protected_resources.go b/src/internal/tester/tconfig/protected_resources.go index b9e31ce06..13c1c9fb9 100644 --- a/src/internal/tester/tconfig/protected_resources.go +++ b/src/internal/tester/tconfig/protected_resources.go @@ -209,3 +209,29 @@ func UnlicensedM365UserID(t *testing.T) string { return strings.ToLower(cfg[TestCfgSecondaryUserID]) } + +// Teams + +// M365TeamsID returns a teamID string representing the m365TeamsID described +// by either the env var CORSO_M365_TEST_TEAM_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func M365TeamsID(t *testing.T) string { + cfg, err := ReadTestConfig() + require.NoError(t, err, "retrieving m365 team id from test configuration: %+v", clues.ToCore(err)) + + return strings.ToLower(cfg[TestCfgTeamID]) +} + +// Groups + +// M365GroupID returns a groupID string representing the m365GroupID described +// by either the env var CORSO_M365_TEST_Group_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func M365GroupID(t *testing.T) string { + cfg, err := ReadTestConfig() + require.NoError(t, err, "retrieving m365 group id from test configuration: %+v", clues.ToCore(err)) + + return strings.ToLower(cfg[TestCfgTeamID]) +} diff --git a/src/pkg/services/m365/api/groups.go b/src/pkg/services/m365/api/groups.go index 9889e2bde..8eef3fba3 100644 --- a/src/pkg/services/m365/api/groups.go +++ b/src/pkg/services/m365/api/groups.go @@ -4,13 +4,14 @@ import ( "context" "github.com/alcionai/clues" + msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/tform" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" - msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - "github.com/microsoftgraph/msgraph-sdk-go/models" ) const ( @@ -22,8 +23,8 @@ const ( // controller // --------------------------------------------------------------------------- -func (c Client) Groups() Groups { - return Groups{c} +func (c Client) Teams() Teams { + return Teams{c} } // On creation of each Teams team a corrsponding group gets created. @@ -31,12 +32,12 @@ func (c Client) Groups() Groups { // drive and mail messages are owned by that group. // Teams is an interface-compliant provider of the client. -type Groups struct { +type Teams struct { Client } // GetAllTeams retrieves all groups. -func (c Groups) GetAllTeams( +func (c Teams) GetAll( ctx context.Context, errs *fault.Bus, ) ([]models.Groupable, error) { @@ -48,19 +49,6 @@ func (c Groups) GetAllTeams( return getGroups(ctx, true, errs, service) } -// GetAllGroups retrieves all groups. -func (c Groups) GetAll( - ctx context.Context, - errs *fault.Bus, -) ([]models.Groupable, error) { - service, err := c.Service() - if err != nil { - return nil, err - } - - return getGroups(ctx, false, errs, service) -} - // GetAll retrieves all groups. func getGroups( ctx context.Context, @@ -68,7 +56,6 @@ func getGroups( errs *fault.Bus, service graph.Servicer, ) ([]models.Groupable, error) { - resp, err := service.Client().Groups().Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting all groups") @@ -123,16 +110,18 @@ func IsTeam(ctx context.Context, g models.Groupable) bool { log.Debug("could not be converted to string value: ", ResourceProvisioningOptions) return false } + if s == teamsAdditionalDataLabel { return true } } } + return false } // GetID retrieves team by groupID/teamID. -func (c Groups) GetByID( +func (c Teams) GetByID( ctx context.Context, identifier string, ) (models.Groupable, error) { @@ -148,6 +137,12 @@ func (c Groups) GetByID( return nil, err } + if !IsTeam(ctx, resp) { + err := clues.New("given teamID is not related to any team") + + return nil, err + } + return resp, graph.Stack(ctx, err).OrNil() } diff --git a/src/pkg/services/m365/api/groups_test.go b/src/pkg/services/m365/api/groups_test.go index 0bb3e3686..dcb039dc5 100644 --- a/src/pkg/services/m365/api/groups_test.go +++ b/src/pkg/services/m365/api/groups_test.go @@ -3,13 +3,18 @@ package api_test import ( "testing" + "github.com/alcionai/clues" + "github.com/google/uuid" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/fault" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type TeamsUnitSuite struct { @@ -20,6 +25,61 @@ func TestTeamsUnitSuite(t *testing.T) { suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) } +func (suite *TeamsUnitSuite) TestValidateGroup() { + team := models.NewTeam() + team.SetDisplayName(ptr.To("testgroup")) + team.SetId(ptr.To("testID")) + + tests := []struct { + name string + args models.Groupable + errCheck assert.ErrorAssertionFunc + errIsSkippable bool + }{ + { + name: "Valid group ", + args: func() *models.Group { + s := models.NewGroup() + s.SetId(ptr.To("id")) + s.SetDisplayName(ptr.To("testTeam")) + return s + }(), + errCheck: assert.NoError, + }, + { + name: "No name", + args: func() *models.Group { + s := models.NewGroup() + s.SetId(ptr.To("id")) + return s + }(), + errCheck: assert.Error, + }, + { + name: "No ID", + args: func() *models.Group { + s := models.NewGroup() + s.SetDisplayName(ptr.To("testTeam")) + return s + }(), + errCheck: assert.Error, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + err := api.ValidateGroup(test.args) + test.errCheck(t, err, clues.ToCore(err)) + + if test.errIsSkippable { + assert.ErrorIs(t, err, api.ErrKnownSkippableCase) + } + }) + } +} + type TeamsIntgSuite struct { tester.Suite its intgTesterSetup @@ -37,19 +97,75 @@ func (suite *TeamsIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } -func (suite *TeamsIntgSuite) TestGetAll() { +func (suite *TeamsIntgSuite) TestGetAllTeams() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() teams, err := suite.its.ac. - Groups(). - GetAllTeams(ctx, fault.New(true)) + Teams(). + GetAll(ctx, fault.New(true)) require.NoError(t, err) require.NotZero(t, len(teams), "must have at least one team") for _, team := range teams { - assert.NotEmpty(t, ptr.Val(team.GetDisplayName()), "must not return onedrive teams") + assert.True(t, api.IsTeam(ctx, team), "must not return non teams groups") + } +} + +func (suite *TeamsIntgSuite) TestTeams_GetByID() { + var ( + t = suite.T() + teamID = tconfig.M365TeamsID(t) + ) + + teamsAPI := suite.its.ac.Teams() + + table := []struct { + name string + id string + expectErr func(*testing.T, error) + }{ + { + name: "3 part id", + id: teamID, + expectErr: func(t *testing.T, err error) { + assert.NoError(t, err, clues.ToCore(err)) + }, + }, + { + name: "malformed id", + id: uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + { + name: "random id", + id: uuid.NewString() + "," + uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + + { + name: "malformed url", + id: "barunihlda", + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + _, err := teamsAPI.GetByID(ctx, test.id) + test.expectErr(t, err) + }) } } diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index 05e16b00e..984411f4a 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -82,6 +82,7 @@ type intgTesterSetup struct { siteID string siteDriveID string siteDriveRootFolderID string + teamID string } func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { @@ -130,5 +131,13 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) + // teams + its.teamID = tconfig.M365TeamsID(t) + + team, err := its.ac.Teams().GetByID(ctx, its.teamID) + require.NoError(t, err, clues.ToCore(err)) + + its.teamID = ptr.Val(team.GetId()) + return its } From de742bbd2f40154871466409c12eab6c7f040e1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:13:15 +0000 Subject: [PATCH 10/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.304=20to=201.44.305=20in=20/src=20(#?= =?UTF-8?q?3867)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.304 to 1.44.305.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.305 (2023-07-20)

Service Client Updates

  • service/connectcases: Updates service API and documentation
  • service/models.lex.v2: Updates service API and documentation
  • service/route53resolver: Updates service API, documentation, and paginators
  • service/s3: Updates service examples
    • Improve performance of S3 clients by simplifying and optimizing endpoint resolution.
  • service/sagemaker: Updates service API, documentation, and paginators
    • Cross account support for SageMaker Feature Store
  • service/sagemaker-featurestore-runtime: Updates service API and documentation
  • service/securitylake: Updates service API and documentation
  • service/transcribe: Updates service API and documentation
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.304&new-version=1.44.305)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 6ebbb2adb..5cb7b980e 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.304 + github.com/aws/aws-sdk-go v1.44.305 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index a66eb73f4..fe18cc952 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.304 h1:crcJBVeewWcVAXDQChzJWZYGFq9i1TYLycAtQ6Xpi4c= -github.com/aws/aws-sdk-go v1.44.304/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.305 h1:fU/5lY3WyBjGU9fkmQYd8o4fZu+2RaOv/i+sPaJVvFg= +github.com/aws/aws-sdk-go v1.44.305/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 5c80d7645525cede346fcdc87c5b62c832876394 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 21 Jul 2023 10:56:26 -0600 Subject: [PATCH 11/62] give library prefix delete list a default val (#3872) #### Type of change - [x] :bug: Bugfix --- .github/actions/purge-m365-data/action.yml | 9 +++------ .github/workflows/ci_test_cleanup.yml | 14 ++++++-------- src/cmd/purge/scripts/onedrivePurge.ps1 | 2 +- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index ec26b987d..dd67f2326 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -53,8 +53,7 @@ runs: AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} - run: | - ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + run: ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} - name: Reset retention for all mailboxes to 0 if: ${{ inputs.user == '' }} @@ -63,8 +62,7 @@ runs: env: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: | - ./exchangeRetention.ps1 + run: ./exchangeRetention.ps1 ################################################################################################################ # OneDrive @@ -91,5 +89,4 @@ runs: env: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + run: ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 9f51d3e14..11ba94097 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -12,7 +12,7 @@ jobs: continue-on-error: true strategy: matrix: - user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID, '' ] + user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, '' ] steps: - uses: actions/checkout@v3 @@ -22,8 +22,7 @@ jobs: # At 20 minutes ago, we should be safe from conflicts. # The additional 10 minutes is just to be good citizens. - name: Set purge boundary - run: | - echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV + run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - name: Purge CI-Produced Folders for Users uses: ./.github/actions/purge-m365-data @@ -43,7 +42,7 @@ jobs: continue-on-error: true strategy: matrix: - site: [ CORSO_M365_TEST_SITE_URL, EXT_SDK_TEST_SITE_URL ] + site: [ CORSO_M365_TEST_SITE_URL ] steps: - uses: actions/checkout@v3 @@ -53,15 +52,14 @@ jobs: # At 20 minutes ago, we should be safe from conflicts. # The additional 10 minutes is just to be good citizens. - name: Set purge boundary - run: | - echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV + run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - name: Purge CI-Produced Folders for Sites uses: ./.github/actions/purge-m365-data with: site: ${{ vars[matrix.site] }} - folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} - libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} + folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} + libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} library-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index b4717d35e..e8f258b95 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -22,7 +22,7 @@ Param ( [String[]]$FolderPrefixPurgeList, [Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")] - [String[]]$LibraryPrefixDeleteList + [String[]]$LibraryPrefixDeleteList = @() ) Set-StrictMode -Version 2.0 From 47e5707ae4188f3fb3c82deedbacdf8d3aa1eaef Mon Sep 17 00:00:00 2001 From: ryanfkeepers Date: Fri, 21 Jul 2023 12:51:28 -0600 Subject: [PATCH 12/62] update all graph packages --- src/go.mod | 13 ++++++------- src/go.sum | 26 ++++++++++++-------------- src/internal/m365/graph/errors.go | 2 +- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/go.mod b/src/go.mod index 5cb7b980e..b589ca542 100644 --- a/src/go.mod +++ b/src/go.mod @@ -5,7 +5,7 @@ go 1.20 replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 require ( - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go v1.44.305 @@ -14,12 +14,12 @@ require ( github.com/google/uuid v1.3.0 github.com/h2non/gock v1.2.0 github.com/kopia/kopia v0.12.2-0.20230327171220-747baeebdab1 - github.com/microsoft/kiota-abstractions-go v1.0.0 + github.com/microsoft/kiota-abstractions-go v1.1.0 github.com/microsoft/kiota-authentication-azure-go v1.0.0 github.com/microsoft/kiota-http-go v1.0.0 github.com/microsoft/kiota-serialization-form-go v1.0.0 - github.com/microsoft/kiota-serialization-json-go v1.0.2 - github.com/microsoftgraph/msgraph-sdk-go v1.4.0 + github.com/microsoft/kiota-serialization-json-go v1.0.4 + github.com/microsoftgraph/msgraph-sdk-go v1.12.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 github.com/puzpuzpuz/xsync/v2 v2.4.1 @@ -42,7 +42,6 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect @@ -62,9 +61,9 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/src/go.sum b/src/go.sum index fe18cc952..718084162 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,14 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= @@ -102,7 +102,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= @@ -272,20 +271,20 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microsoft/kiota-abstractions-go v1.0.0 h1:teQS3yOmcTyps+O48AD17LI8TR1B3wCEwGFcwC6K75c= -github.com/microsoft/kiota-abstractions-go v1.0.0/go.mod h1:2yaRQnx2KU7UaenYSApiTT4pf7fFkPV0B71Rm2uYynQ= +github.com/microsoft/kiota-abstractions-go v1.1.0 h1:X1aKlsYCRs/0RSChr/fbq4j/+kxRzbSY5GeWhtHQNYI= +github.com/microsoft/kiota-abstractions-go v1.1.0/go.mod h1:RkxyZ5x87Njik7iVeQY9M2wtrrL1MJZcXiI/BxD/82g= github.com/microsoft/kiota-authentication-azure-go v1.0.0 h1:29FNZZ/4nnCOwFcGWlB/sxPvWz487HA2bXH8jR5k2Rk= github.com/microsoft/kiota-authentication-azure-go v1.0.0/go.mod h1:rnx3PRlkGdXDcA/0lZQTbBwyYGmc+3POt7HpE/e4jGw= github.com/microsoft/kiota-http-go v1.0.0 h1:F1hd6gMlLeEgH2CkRB7z13ow7LxMKMWEmms/t0VfS+k= github.com/microsoft/kiota-http-go v1.0.0/go.mod h1:eujxJliqodotsYepIc6ihhK+vXMMt5Q8YiSNL7+7M7U= github.com/microsoft/kiota-serialization-form-go v1.0.0 h1:UNdrkMnLFqUCccQZerKjblsyVgifS11b3WCx+eFEsAI= github.com/microsoft/kiota-serialization-form-go v1.0.0/go.mod h1:h4mQOO6KVTNciMF6azi1J9QB19ujSw3ULKcSNyXXOMA= -github.com/microsoft/kiota-serialization-json-go v1.0.2 h1:RXan8v7yWBD88XxVZ2W38BBcqu2UqWtgS54nCbOS5ow= -github.com/microsoft/kiota-serialization-json-go v1.0.2/go.mod h1:AUItT9exyxmjZQE8IeFD9ygP77q9GKVb+AQE2V5Ikho= +github.com/microsoft/kiota-serialization-json-go v1.0.4 h1:5TaISWwd2Me8clrK7SqNATo0tv9seOq59y4I5953egQ= +github.com/microsoft/kiota-serialization-json-go v1.0.4/go.mod h1:rM4+FsAY+9AEpBsBzkFFis+b/LZLlNKKewuLwK9Q6Mg= github.com/microsoft/kiota-serialization-text-go v1.0.0 h1:XOaRhAXy+g8ZVpcq7x7a0jlETWnWrEum0RhmbYrTFnA= github.com/microsoft/kiota-serialization-text-go v1.0.0/go.mod h1:sM1/C6ecnQ7IquQOGUrUldaO5wj+9+v7G2W3sQ3fy6M= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0 h1:ibNwMDEZ6HikA9BVXu+TljCzCiE+yFsD6wLpJbTc1tc= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0/go.mod h1:JIDL1xENx92B60NjO2ACyqGeKvtYkdl9rirgajIgryw= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0 h1:/jZJ1KCtVlvxStKq31VsEPOQQ5Iy26R1pgvc+RYt7XI= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0/go.mod h1:ccLv84FJFtwdSzYWM/HlTes5FLzkzzBsYh9kg93/WS8= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 h1:7NWTfyXvOjoizW7PmxNp3+8wCKPgpODs/D1cUZ3fkAY= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0/go.mod h1:tQb4q3YMIj2dWhhXhQSJ4ELpol931ANKzHSYK5kX1qE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -303,7 +302,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= diff --git a/src/internal/m365/graph/errors.go b/src/internal/m365/graph/errors.go index dad2674a4..f35b91385 100644 --- a/src/internal/m365/graph/errors.go +++ b/src/internal/m365/graph/errors.go @@ -365,7 +365,7 @@ func errData(err odataerrors.ODataErrorable) (string, []any, string) { msgConcat += ptr.Val(d.GetMessage()) } - inner := mainErr.GetInnererror() + inner := mainErr.GetInnerError() if inner != nil { data = appendIf(data, "odataerror_inner_cli_req_id", inner.GetClientRequestId()) data = appendIf(data, "odataerror_inner_req_id", inner.GetRequestId()) From 9bcf1601874501bd533f8506866786e049be621d Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 21 Jul 2023 13:08:00 -0600 Subject: [PATCH 13/62] atttempt ternary for value (#3877) #### Type of change - [x] :bug: Bugfix - [x] :robot: Supportability/Tests --- .github/actions/purge-m365-data/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index dd67f2326..cf013a054 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -89,4 +89,4 @@ runs: env: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + run: ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix && inputs.library-prefix || '[]' }} -PurgeBeforeTimestamp ${{ inputs.older-than }} From 916bb0b27c140cf276e588440f0034b377fc1f39 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:05:38 -0700 Subject: [PATCH 14/62] Create an interface and implementation for Reason struct (#3868) Intermediate step to a few different goals including * moving interface definitions to better locations while avoid cycles * adding a flag to disable kopia-assisted incrementals Create an interface and implementation for the existing Reason struct. The goal is to set stuff up so that eventually the kopia package can ask the struct for the subtree path to work with when merging the hierarchy instead of having the backup operation pass that information in Code changes are mostly just turning stuff into a struct and fixing up compile errors. Some functions have been excluded from the struct (i.e. `Key`) and made into functions in the kopia package itself --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #2360 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/kopia/backup_bases.go | 20 +- src/internal/kopia/backup_bases_test.go | 35 +-- src/internal/kopia/base_finder.go | 115 +++++++--- src/internal/kopia/base_finder_test.go | 225 ++++++-------------- src/internal/kopia/inject/inject.go | 2 +- src/internal/kopia/wrapper_test.go | 68 +++--- src/internal/operations/backup.go | 46 ++-- src/internal/operations/backup_test.go | 94 ++++---- src/internal/operations/manifests.go | 12 +- src/internal/operations/manifests_test.go | 208 ++++++------------ src/internal/operations/test/helper_test.go | 10 +- 11 files changed, 343 insertions(+), 492 deletions(-) diff --git a/src/internal/kopia/backup_bases.go b/src/internal/kopia/backup_bases.go index 0505fc829..c0b8ecfaa 100644 --- a/src/internal/kopia/backup_bases.go +++ b/src/internal/kopia/backup_bases.go @@ -24,7 +24,7 @@ type BackupBases interface { MergeBackupBases( ctx context.Context, other BackupBases, - reasonToKey func(Reason) string, + reasonToKey func(Reasoner) string, ) BackupBases } @@ -109,10 +109,10 @@ func (bb *backupBases) ClearAssistBases() { // some migration that disrupts lookup), and that the BackupBases used to call // this function contains the current version. // -// reasonToKey should be a function that, given a Reason, will produce some -// string that represents Reason in the context of the merge operation. For -// example, to merge BackupBases across a ResourceOwner migration, the Reason's -// service and category can be used as the key. +// reasonToKey should be a function that, given a Reasoner, will produce some +// string that represents Reasoner in the context of the merge operation. For +// example, to merge BackupBases across a ProtectedResource migration, the +// Reasoner's service and category can be used as the key. // // Selection priority, for each reason key generated by reasonsToKey, follows // these rules: @@ -125,7 +125,7 @@ func (bb *backupBases) ClearAssistBases() { func (bb *backupBases) MergeBackupBases( ctx context.Context, other BackupBases, - reasonToKey func(reason Reason) string, + reasonToKey func(reason Reasoner) string, ) BackupBases { if other == nil || (len(other.MergeBases()) == 0 && len(other.AssistBases()) == 0) { return bb @@ -159,7 +159,7 @@ func (bb *backupBases) MergeBackupBases( // Calculate the set of mergeBases to pull from other into this one. for _, m := range other.MergeBases() { - useReasons := []Reason{} + useReasons := []Reasoner{} for _, r := range m.Reasons { k := reasonToKey(r) @@ -210,7 +210,7 @@ func (bb *backupBases) MergeBackupBases( // Add assistBases from other to this one as needed. for _, m := range other.AssistBases() { - useReasons := []Reason{} + useReasons := []Reasoner{} // Assume that all complete manifests in assist overlap with MergeBases. if len(m.IncompleteReason) == 0 { @@ -267,8 +267,8 @@ func findNonUniqueManifests( } for _, reason := range man.Reasons { - reasonKey := reason.ResourceOwner + reason.Service.String() + reason.Category.String() - reasons[reasonKey] = append(reasons[reasonKey], man) + mapKey := reasonKey(reason) + reasons[mapKey] = append(reasons[mapKey], man) } } diff --git a/src/internal/kopia/backup_bases_test.go b/src/internal/kopia/backup_bases_test.go index f902d4e37..04afb5408 100644 --- a/src/internal/kopia/backup_bases_test.go +++ b/src/internal/kopia/backup_bases_test.go @@ -16,7 +16,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -func makeManifest(id, incmpl, bID string, reasons ...Reason) ManifestEntry { +func makeManifest(id, incmpl, bID string, reasons ...Reasoner) ManifestEntry { bIDKey, _ := makeTagKV(TagBackupID) return ManifestEntry{ @@ -223,14 +223,10 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { ir = "checkpoint" } - reasons := make([]Reason, 0, len(i.cat)) + reasons := make([]Reasoner, 0, len(i.cat)) for _, c := range i.cat { - reasons = append(reasons, Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: c, - }) + reasons = append(reasons, NewReason("", ro, path.ExchangeService, c)) } m := makeManifest(baseID, ir, "b"+baseID, reasons...) @@ -457,8 +453,8 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { got := bb.MergeBackupBases( ctx, other, - func(reason Reason) string { - return reason.Service.String() + reason.Category.String() + func(r Reasoner) string { + return r.Service().String() + r.Category().String() }) AssertBackupBasesEqual(t, expect, got) }) @@ -469,13 +465,8 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { ro := "resource_owner" makeMan := func(pct path.CategoryType, id, incmpl, bID string) ManifestEntry { - reason := Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: pct, - } - - return makeManifest(id, incmpl, bID, reason) + r := NewReason("", ro, path.ExchangeService, pct) + return makeManifest(id, incmpl, bID, r) } // Make a function so tests can modify things without messing with each other. @@ -606,11 +597,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, - Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }) + NewReason("", ro, path.ExchangeService, path.ContactsCategory)) res.assistBases = res.mergeBases return res @@ -619,11 +606,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, - Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }) + NewReason("", ro, path.ExchangeService, path.ContactsCategory)) res.assistBases = res.mergeBases return res diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index 9ac651512..f8119587b 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -29,39 +29,98 @@ const ( userTagPrefix = "tag:" ) -type Reason struct { - ResourceOwner string - Service path.ServiceType - Category path.CategoryType +// TODO(ashmrtn): Move this into some inject package. Here to avoid import +// cycles. +type Reasoner interface { + Tenant() string + ProtectedResource() string + Service() path.ServiceType + Category() path.CategoryType + // SubtreePath returns the path prefix for data in existing backups that have + // parameters (tenant, protected resourced, etc) that match this Reasoner. + SubtreePath() (path.Path, error) + // TODO(ashmrtn): Remove this when kopia generates tags from Reasons. + TagKeys() []string } -func (r Reason) TagKeys() []string { - return []string{ - r.ResourceOwner, - serviceCatString(r.Service, r.Category), +func NewReason( + tenant, resource string, + service path.ServiceType, + category path.CategoryType, +) Reasoner { + return reason{ + tenant: tenant, + resource: resource, + service: service, + category: category, } } -// Key is the concatenation of the ResourceOwner, Service, and Category. -func (r Reason) Key() string { - return r.ResourceOwner + r.Service.String() + r.Category.String() +type reason struct { + // tenant appears here so that when this is moved to an inject package nothing + // needs changed. However, kopia itself is blind to the fields in the reason + // struct and relies on helper functions to get the information it needs. + tenant string + resource string + service path.ServiceType + category path.CategoryType +} + +func (r reason) Tenant() string { + return r.tenant +} + +func (r reason) ProtectedResource() string { + return r.resource +} + +func (r reason) Service() path.ServiceType { + return r.service +} + +func (r reason) Category() path.CategoryType { + return r.category +} + +func (r reason) SubtreePath() (path.Path, error) { + p, err := path.ServicePrefix( + r.Tenant(), + r.ProtectedResource(), + r.Service(), + r.Category()) + + return p, clues.Wrap(err, "building path").OrNil() +} + +// TODO(ashmrtn): Remove this when kopia generates tags based off Reasons. Here +// at the moment so things compile. +func (r reason) TagKeys() []string { + return []string{ + r.ProtectedResource(), + serviceCatString(r.Service(), r.Category()), + } +} + +// reasonKey returns the concatenation of the ProtectedResource, Service, and Category. +func reasonKey(r Reasoner) string { + return r.ProtectedResource() + r.Service().String() + r.Category().String() } type BackupEntry struct { *backup.Backup - Reasons []Reason + Reasons []Reasoner } type ManifestEntry struct { *snapshot.Manifest - // Reason contains the ResourceOwners and Service/Categories that caused this + // Reasons contains the ResourceOwners and Service/Categories that caused this // snapshot to be selected as a base. We can't reuse OwnersCats here because // it's possible some ResourceOwners will have a subset of the Categories as // the reason for selecting a snapshot. For example: // 1. backup user1 email,contacts -> B1 // 2. backup user1 contacts -> B2 (uses B1 as base) // 3. backup user1 email,contacts,events (uses B1 for email, B2 for contacts) - Reasons []Reason + Reasons []Reasoner } func (me ManifestEntry) GetTag(key string) (string, bool) { @@ -157,7 +216,7 @@ func (b *baseFinder) getBackupModel( // most recent complete backup as the base. func (b *baseFinder) findBasesInSet( ctx context.Context, - reason Reason, + reason Reasoner, metas []*manifest.EntryMetadata, ) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { // Sort manifests by time so we can go through them sequentially. The code in @@ -190,7 +249,7 @@ func (b *baseFinder) findBasesInSet( kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }) logger.Ctx(ictx).Info("found incomplete backup") @@ -211,7 +270,7 @@ func (b *baseFinder) findBasesInSet( kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }) logger.Ctx(ictx).Info("found incomplete backup") @@ -235,7 +294,7 @@ func (b *baseFinder) findBasesInSet( kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }) logger.Ctx(ictx).Infow( @@ -253,13 +312,13 @@ func (b *baseFinder) findBasesInSet( me := ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, } kopiaAssistSnaps = append(kopiaAssistSnaps, me) return &BackupEntry{ Backup: bup, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }, &me, kopiaAssistSnaps, nil } @@ -270,12 +329,12 @@ func (b *baseFinder) findBasesInSet( func (b *baseFinder) getBase( ctx context.Context, - reason Reason, + r Reasoner, tags map[string]string, ) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { allTags := map[string]string{} - for _, k := range reason.TagKeys() { + for _, k := range r.TagKeys() { allTags[k] = "" } @@ -292,12 +351,12 @@ func (b *baseFinder) getBase( return nil, nil, nil, nil } - return b.findBasesInSet(ctx, reason, metas) + return b.findBasesInSet(ctx, r, metas) } func (b *baseFinder) FindBases( ctx context.Context, - reasons []Reason, + reasons []Reasoner, tags map[string]string, ) BackupBases { var ( @@ -310,14 +369,14 @@ func (b *baseFinder) FindBases( kopiaAssistSnaps = map[manifest.ID]ManifestEntry{} ) - for _, reason := range reasons { + for _, searchReason := range reasons { ictx := clues.Add( ctx, - "search_service", reason.Service.String(), - "search_category", reason.Category.String()) + "search_service", searchReason.Service().String(), + "search_category", searchReason.Category().String()) logger.Ctx(ictx).Info("searching for previous manifests") - baseBackup, baseSnap, assistSnaps, err := b.getBase(ictx, reason, tags) + baseBackup, baseSnap, assistSnaps, err := b.getBase(ictx, searchReason, tags) if err != nil { logger.Ctx(ctx).Info( "getting base, falling back to full backup for reason", diff --git a/src/internal/kopia/base_finder_test.go b/src/internal/kopia/base_finder_test.go index f76b3c81a..cb3239ca1 100644 --- a/src/internal/kopia/base_finder_test.go +++ b/src/internal/kopia/base_finder_test.go @@ -39,61 +39,24 @@ var ( testUser2 = "user2" testUser3 = "user3" - testAllUsersAllCats = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, + testAllUsersAllCats = []Reasoner{ + // User1 email and events. + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + // User2 email and events. + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + // User3 email and events. + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EventsCategory), } - testAllUsersMail = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + testAllUsersMail = []Reasoner{ + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), } - testUser1Mail = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + testUser1Mail = []Reasoner{ + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), } ) @@ -322,12 +285,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() { sm: mockEmptySnapshotManager{}, bg: mockEmptyModelGetter{}, } - reasons := []Reason{ - { - ResourceOwner: "a-user", - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons := []Reasoner{ + NewReason("", "a-user", path.ExchangeService, path.EmailCategory), } bb := bf.FindBases(ctx, reasons, nil) @@ -345,12 +304,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { sm: &mockSnapshotManager{findErr: assert.AnError}, bg: mockEmptyModelGetter{}, } - reasons := []Reason{ - { - ResourceOwner: "a-user", - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons := []Reasoner{ + NewReason("", "a-user", path.ExchangeService, path.EmailCategory), } bb := bf.FindBases(ctx, reasons, nil) @@ -361,14 +316,14 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { func (suite *BaseFinderUnitSuite) TestGetBases() { table := []struct { name string - input []Reason + input []Reasoner manifestData []manifestInfo // Use this to denote the Reasons a base backup or base manifest is // selected. The int maps to the index of the backup or manifest in data. - expectedBaseReasons map[int][]Reason + expectedBaseReasons map[int][]Reasoner // Use this to denote the Reasons a kopia assised incrementals manifest is // selected. The int maps to the index of the manifest in data. - expectedAssistManifestReasons map[int][]Reason + expectedAssistManifestReasons map[int][]Reasoner backupData []backupInfo }{ { @@ -394,10 +349,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 1: testUser1Mail, }, backupData: []backupInfo{ @@ -428,10 +383,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, 1: testUser1Mail, }, @@ -463,10 +418,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, 1: testUser1Mail, }, @@ -492,10 +447,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser3, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, }, backupData: []backupInfo{ @@ -519,10 +474,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser3, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testAllUsersAllCats, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testAllUsersAllCats, }, backupData: []backupInfo{ @@ -557,76 +512,28 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser3, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: { - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), }, 1: { - Reason{ - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", testUser3, path.ExchangeService, path.EventsCategory), }, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: { - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), }, 1: { - Reason{ - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", testUser3, path.ExchangeService, path.EventsCategory), }, }, backupData: []backupInfo{ @@ -657,10 +564,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, 1: testUser1Mail, }, @@ -693,10 +600,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 1: testUser1Mail, }, backupData: []backupInfo{ @@ -728,8 +635,8 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{}, - expectedAssistManifestReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{}, + expectedAssistManifestReasons: map[int][]Reasoner{ 1: testUser1Mail, }, backupData: []backupInfo{ @@ -752,10 +659,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, }, backupData: []backupInfo{ @@ -787,10 +694,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, }, backupData: []backupInfo{ @@ -857,17 +764,17 @@ func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() { table := []struct { name string - input []Reason + input []Reasoner tags map[string]string // Use this to denote which manifests in data should be expected. Allows // defining data in a table while not repeating things between data and // expected. - expectedIdxs map[int][]Reason + expectedIdxs map[int][]Reasoner }{ { name: "no tags specified", tags: nil, - expectedIdxs: map[int][]Reason{ + expectedIdxs: map[int][]Reasoner{ 0: testUser1Mail, }, }, @@ -877,14 +784,14 @@ func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() { "fnords": "", "smarf": "", }, - expectedIdxs: map[int][]Reason{ + expectedIdxs: map[int][]Reasoner{ 0: testUser1Mail, }, }, { name: "subset of custom tags", tags: map[string]string{"fnords": ""}, - expectedIdxs: map[int][]Reason{ + expectedIdxs: map[int][]Reasoner{ 0: testUser1Mail, }, }, @@ -925,7 +832,7 @@ func checkManifestEntriesMatch( t *testing.T, retSnaps []ManifestEntry, allExpected []manifestInfo, - expectedIdxsAndReasons map[int][]Reason, + expectedIdxsAndReasons map[int][]Reasoner, ) { // Check the proper snapshot manifests were returned. expected := make([]*snapshot.Manifest, 0, len(expectedIdxsAndReasons)) @@ -941,7 +848,7 @@ func checkManifestEntriesMatch( assert.ElementsMatch(t, expected, got) // Check the reasons for selecting each manifest are correct. - expectedReasons := make(map[manifest.ID][]Reason, len(expectedIdxsAndReasons)) + expectedReasons := make(map[manifest.ID][]Reasoner, len(expectedIdxsAndReasons)) for idx, reasons := range expectedIdxsAndReasons { expectedReasons[allExpected[idx].man.ID] = reasons } @@ -967,7 +874,7 @@ func checkBackupEntriesMatch( t *testing.T, retBups []BackupEntry, allExpected []backupInfo, - expectedIdxsAndReasons map[int][]Reason, + expectedIdxsAndReasons map[int][]Reasoner, ) { // Check the proper snapshot manifests were returned. expected := make([]*backup.Backup, 0, len(expectedIdxsAndReasons)) @@ -983,7 +890,7 @@ func checkBackupEntriesMatch( assert.ElementsMatch(t, expected, got) // Check the reasons for selecting each manifest are correct. - expectedReasons := make(map[model.StableID][]Reason, len(expectedIdxsAndReasons)) + expectedReasons := make(map[model.StableID][]Reasoner, len(expectedIdxsAndReasons)) for idx, reasons := range expectedIdxsAndReasons { expectedReasons[allExpected[idx].b.ID] = reasons } diff --git a/src/internal/kopia/inject/inject.go b/src/internal/kopia/inject/inject.go index 6921c353d..ed0a8f5e8 100644 --- a/src/internal/kopia/inject/inject.go +++ b/src/internal/kopia/inject/inject.go @@ -37,7 +37,7 @@ type ( BaseFinder interface { FindBases( ctx context.Context, - reasons []kopia.Reason, + reasons []kopia.Reasoner, tags map[string]string, ) kopia.BackupBases } diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 12857904f..5af044f94 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -703,17 +703,19 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { "brunhilda": "", } - reasons := []Reason{ - { - ResourceOwner: suite.storePath1.ResourceOwner(), - Service: suite.storePath1.Service(), - Category: suite.storePath1.Category(), - }, - { - ResourceOwner: suite.storePath2.ResourceOwner(), - Service: suite.storePath2.Service(), - Category: suite.storePath2.Category(), - }, + reasons := []Reasoner{ + NewReason( + testTenant, + suite.storePath1.ResourceOwner(), + suite.storePath1.Service(), + suite.storePath1.Category(), + ), + NewReason( + testTenant, + suite.storePath2.ResourceOwner(), + suite.storePath2.Service(), + suite.storePath2.Category(), + ), } for _, r := range reasons { @@ -837,12 +839,12 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { "brunhilda": "", } - reasons := []Reason{ - { - ResourceOwner: storePath.ResourceOwner(), - Service: storePath.Service(), - Category: storePath.Category(), - }, + reasons := []Reasoner{ + NewReason( + testTenant, + storePath.ResourceOwner(), + storePath.Service(), + storePath.Category()), } for _, r := range reasons { @@ -1017,13 +1019,9 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { w := &Wrapper{k} tags := map[string]string{} - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) - for _, k := range reason.TagKeys() { + for _, k := range r.TagKeys() { tags[k] = "" } @@ -1113,13 +1111,9 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { loc1 := path.Builder{}.Append(suite.storePath1.Folders()...) loc2 := path.Builder{}.Append(suite.storePath2.Folders()...) tags := map[string]string{} - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) - for _, k := range reason.TagKeys() { + for _, k := range r.TagKeys() { tags[k] = "" } @@ -1392,13 +1386,9 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { } tags := map[string]string{} - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) - for _, k := range reason.TagKeys() { + for _, k := range r.TagKeys() { tags[k] = "" } @@ -1437,11 +1427,7 @@ func (c *i64counter) Count(i int64) { } func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) subtreePathTmp, err := path.Build( testTenant, @@ -1459,7 +1445,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { tags := map[string]string{} - for _, k := range reason.TagKeys() { + for _, k := range r.TagKeys() { tags[k] = "" } diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 00eb82884..0d6509d1d 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -280,8 +280,8 @@ func (op *BackupOperation) do( backupID model.StableID, ) (*details.Builder, error) { var ( - reasons = selectorToReasons(op.Selectors, false) - fallbackReasons = makeFallbackReasons(op.Selectors) + reasons = selectorToReasons(op.account.ID(), op.Selectors, false) + fallbackReasons = makeFallbackReasons(op.account.ID(), op.Selectors) lastBackupVersion = version.NoBackup ) @@ -370,10 +370,10 @@ func (op *BackupOperation) do( return deets, nil } -func makeFallbackReasons(sel selectors.Selector) []kopia.Reason { +func makeFallbackReasons(tenant string, sel selectors.Selector) []kopia.Reasoner { if sel.PathService() != path.SharePointService && sel.DiscreteOwner != sel.DiscreteOwnerName { - return selectorToReasons(sel, true) + return selectorToReasons(tenant, sel, true) } return nil @@ -420,9 +420,13 @@ func produceBackupDataCollections( // Consumer funcs // --------------------------------------------------------------------------- -func selectorToReasons(sel selectors.Selector, useOwnerNameForID bool) []kopia.Reason { +func selectorToReasons( + tenant string, + sel selectors.Selector, + useOwnerNameForID bool, +) []kopia.Reasoner { service := sel.PathService() - reasons := []kopia.Reason{} + reasons := []kopia.Reasoner{} pcs, err := sel.PathCategories() if err != nil { @@ -438,28 +442,24 @@ func selectorToReasons(sel selectors.Selector, useOwnerNameForID bool) []kopia.R for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} { for _, cat := range sl { - reasons = append(reasons, kopia.Reason{ - ResourceOwner: owner, - Service: service, - Category: cat, - }) + reasons = append(reasons, kopia.NewReason(tenant, owner, service, cat)) } } return reasons } -func builderFromReason(ctx context.Context, tenant string, r kopia.Reason) (*path.Builder, error) { - ctx = clues.Add(ctx, "category", r.Category.String()) +func builderFromReason(ctx context.Context, tenant string, r kopia.Reasoner) (*path.Builder, error) { + ctx = clues.Add(ctx, "category", r.Category().String()) // This is hacky, but we want the path package to format the path the right // way (e.x. proper order for service, category, etc), but we don't care about // the folders after the prefix. p, err := path.Build( tenant, - r.ResourceOwner, - r.Service, - r.Category, + r.ProtectedResource(), + r.Service(), + r.Category(), false, "tmp") if err != nil { @@ -474,7 +474,7 @@ func consumeBackupCollections( ctx context.Context, bc kinject.BackupConsumer, tenantID string, - reasons []kopia.Reason, + reasons []kopia.Reasoner, bbs kopia.BackupBases, cs []data.BackupCollection, pmr prefixmatcher.StringSetReader, @@ -530,8 +530,8 @@ func consumeBackupCollections( } paths = append(paths, pb) - services[reason.Service.String()] = struct{}{} - categories[reason.Category.String()] = struct{}{} + services[reason.Service().String()] = struct{}{} + categories[reason.Category().String()] = struct{}{} } ids[m.ID] = struct{}{} @@ -609,11 +609,11 @@ func consumeBackupCollections( return kopiaStats, deets, itemsSourcedFromBase, err } -func matchesReason(reasons []kopia.Reason, p path.Path) bool { +func matchesReason(reasons []kopia.Reasoner, p path.Path) bool { for _, reason := range reasons { - if p.ResourceOwner() == reason.ResourceOwner && - p.Service() == reason.Service && - p.Category() == reason.Category { + if p.ResourceOwner() == reason.ProtectedResource() && + p.Service() == reason.Service() && + p.Category() == reason.Category() { return true } } diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index ffa164c81..49f8d4aa5 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -395,25 +395,23 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections tenant, path.ExchangeService.String(), resourceOwner, - path.EmailCategory.String(), - ) + path.EmailCategory.String()) contactsBuilder = path.Builder{}.Append( tenant, path.ExchangeService.String(), resourceOwner, - path.ContactsCategory.String(), - ) + path.ContactsCategory.String()) - emailReason = kopia.Reason{ - ResourceOwner: resourceOwner, - Service: path.ExchangeService, - Category: path.EmailCategory, - } - contactsReason = kopia.Reason{ - ResourceOwner: resourceOwner, - Service: path.ExchangeService, - Category: path.ContactsCategory, - } + emailReason = kopia.NewReason( + "", + resourceOwner, + path.ExchangeService, + path.EmailCategory) + contactsReason = kopia.NewReason( + "", + resourceOwner, + path.ExchangeService, + path.ContactsCategory) manifest1 = &snapshot.Manifest{ ID: "id1", @@ -434,7 +432,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections input: kopia.NewMockBackupBases().WithMergeBases( kopia.ManifestEntry{ Manifest: manifest1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ emailReason, }, }).ClearMockAssistBases(), @@ -452,7 +450,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections input: kopia.NewMockBackupBases().WithMergeBases( kopia.ManifestEntry{ Manifest: manifest1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ emailReason, contactsReason, }, @@ -472,14 +470,14 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections input: kopia.NewMockBackupBases().WithMergeBases( kopia.ManifestEntry{ Manifest: manifest1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ emailReason, contactsReason, }, }, kopia.ManifestEntry{ Manifest: manifest2, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ emailReason, contactsReason, }, @@ -506,13 +504,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections input: kopia.NewMockBackupBases().WithMergeBases( kopia.ManifestEntry{ Manifest: manifest1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ emailReason, }, }).WithAssistBases( kopia.ManifestEntry{ Manifest: manifest2, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ contactsReason, }, }), @@ -629,16 +627,16 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems DetailsID: "did2", } - pathReason1 = kopia.Reason{ - ResourceOwner: itemPath1.ResourceOwner(), - Service: itemPath1.Service(), - Category: itemPath1.Category(), - } - pathReason3 = kopia.Reason{ - ResourceOwner: itemPath3.ResourceOwner(), - Service: itemPath3.Service(), - Category: itemPath3.Category(), - } + pathReason1 = kopia.NewReason( + "", + itemPath1.ResourceOwner(), + itemPath1.Service(), + itemPath1.Category()) + pathReason3 = kopia.NewReason( + "", + itemPath3.ResourceOwner(), + itemPath3.Service(), + itemPath3.Category()) ) itemParents1, err := path.GetDriveFolderPath(itemPath1) @@ -684,7 +682,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems }, DetailsID: "foo", }, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -703,7 +701,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -730,13 +728,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -763,7 +761,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -822,7 +820,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -849,7 +847,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -879,7 +877,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -909,7 +907,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -940,7 +938,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -971,13 +969,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, { Backup: &backup2, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason3, }, }, @@ -1064,11 +1062,11 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde locPath1 = path.Builder{}.Append(itemPath1.Folders()...) - pathReason1 = kopia.Reason{ - ResourceOwner: itemPath1.ResourceOwner(), - Service: itemPath1.Service(), - Category: itemPath1.Category(), - } + pathReason1 = kopia.NewReason( + "", + itemPath1.ResourceOwner(), + itemPath1.Service(), + itemPath1.Category()) backup1 = kopia.BackupEntry{ Backup: &backup.Backup{ @@ -1077,7 +1075,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde }, DetailsID: "did1", }, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, } diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index 5e1c79e4f..1c5d1716c 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -23,7 +23,7 @@ func produceManifestsAndMetadata( ctx context.Context, bf inject.BaseFinder, rp inject.RestoreProducer, - reasons, fallbackReasons []kopia.Reason, + reasons, fallbackReasons []kopia.Reasoner, tenantID string, getMetadata bool, ) (kopia.BackupBases, []data.RestoreCollection, bool, error) { @@ -47,8 +47,8 @@ func produceManifestsAndMetadata( bb = bb.MergeBackupBases( ctx, fbb, - func(r kopia.Reason) string { - return r.Service.String() + r.Category.String() + func(r kopia.Reasoner) string { + return r.Service().String() + r.Category().String() }) if !getMetadata { @@ -115,9 +115,9 @@ func collectMetadata( Append(fn). ToServiceCategoryMetadataPath( tenantID, - reason.ResourceOwner, - reason.Service, - reason.Category, + reason.ProtectedResource(), + reason.Service(), + reason.Category(), true) if err != nil { return nil, clues. diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index e4ae9b6d3..5fdf22424 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -47,7 +47,7 @@ type mockBackupFinder struct { func (bf *mockBackupFinder) FindBases( _ context.Context, - reasons []kopia.Reason, + reasons []kopia.Reasoner, _ map[string]string, ) kopia.BackupBases { if len(reasons) == 0 { @@ -58,7 +58,7 @@ func (bf *mockBackupFinder) FindBases( return kopia.NewMockBackupBases() } - b := bf.data[reasons[0].ResourceOwner] + b := bf.data[reasons[0].ProtectedResource()] if b == nil { return kopia.NewMockBackupBases() } @@ -102,7 +102,7 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { table := []struct { name string manID string - reasons []kopia.Reason + reasons []kopia.Reasoner fileNames []string expectPaths func(*testing.T, []string) []path.Path expectErr error @@ -110,12 +110,8 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "single reason, single file", manID: "single single", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -133,12 +129,8 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "single reason, multiple files", manID: "single multi", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -156,17 +148,9 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "multiple reasons, single file", manID: "multi single", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -187,17 +171,9 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "multiple reasons, multiple file", manID: "multi multi", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -243,17 +219,13 @@ func buildReasons( ro string, service path.ServiceType, cats ...path.CategoryType, -) []kopia.Reason { - var reasons []kopia.Reason +) []kopia.Reasoner { + var reasons []kopia.Reasoner for _, cat := range cats { reasons = append( reasons, - kopia.Reason{ - ResourceOwner: ro, - Service: service, - Category: cat, - }) + kopia.NewReason("", ro, service, cat)) } return reasons @@ -280,7 +252,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name string bf *mockBackupFinder rp mockRestoreProducer - reasons []kopia.Reason + reasons []kopia.Reasoner getMeta bool assertErr assert.ErrorAssertionFunc assertB assert.BoolAssertionFunc @@ -291,7 +263,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { { name: "don't get metadata, no mans", rp: mockRestoreProducer{}, - reasons: []kopia.Reason{}, + reasons: []kopia.Reasoner{}, getMeta: false, assertErr: assert.NoError, assertB: assert.False, @@ -308,12 +280,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, rp: mockRestoreProducer{}, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: false, assertErr: assert.NoError, @@ -333,12 +301,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, rp: mockRestoreProducer{}, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -365,17 +329,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id1"}}}, }, }, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -421,12 +377,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}}, }, }, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -454,12 +406,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}}, }, }, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -480,12 +428,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, rp: mockRestoreProducer{err: assert.AnError}, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.Error, @@ -588,24 +532,24 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb } } - emailReason := kopia.Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + emailReason := kopia.NewReason( + "", + ro, + path.ExchangeService, + path.EmailCategory) - fbEmailReason := kopia.Reason{ - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + fbEmailReason := kopia.NewReason( + "", + fbro, + path.ExchangeService, + path.EmailCategory) table := []struct { name string bf *mockBackupFinder rp mockRestoreProducer - reasons []kopia.Reason - fallbackReasons []kopia.Reason + reasons []kopia.Reasoner + fallbackReasons []kopia.Reasoner getMeta bool assertErr assert.ErrorAssertionFunc assertB assert.BoolAssertionFunc @@ -624,7 +568,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb }, }, rp: mockRestoreProducer{}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: false, assertErr: assert.NoError, assertB: assert.False, @@ -649,7 +593,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - fallbackReasons: []kopia.Reason{fbEmailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -680,8 +624,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -708,8 +652,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id2"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -744,8 +688,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id2"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -776,8 +720,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -808,8 +752,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id2"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -838,21 +782,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{ + reasons: []kopia.Reasoner{ emailReason, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), }, - fallbackReasons: []kopia.Reason{ + fallbackReasons: []kopia.Reasoner{ fbEmailReason, - { - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -882,13 +818,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{ - { - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{ + kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -921,21 +853,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{ + reasons: []kopia.Reasoner{ emailReason, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), }, - fallbackReasons: []kopia.Reason{ + fallbackReasons: []kopia.Reasoner{ fbEmailReason, - { - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 93a609365..31dbb9544 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -242,13 +242,7 @@ func checkBackupIsInManifests( for _, category := range categories { t.Run(category.String(), func(t *testing.T) { var ( - reasons = []kopia.Reason{ - { - ResourceOwner: resourceOwner, - Service: sel.PathService(), - Category: category, - }, - } + r = kopia.NewReason("", resourceOwner, sel.PathService(), category) tags = map[string]string{kopia.TagBackupCategory: ""} found bool ) @@ -256,7 +250,7 @@ func checkBackupIsInManifests( bf, err := kw.NewBaseFinder(sw) require.NoError(t, err, clues.ToCore(err)) - mans := bf.FindBases(ctx, reasons, tags) + mans := bf.FindBases(ctx, []kopia.Reasoner{r}, tags) for _, man := range mans.MergeBases() { bID, ok := man.GetTag(kopia.TagBackupID) if !assert.Truef(t, ok, "snapshot manifest %s missing backup ID tag", man.ID) { From 4ddbb1cc30d945e2cfb9d7509e6a00bdebb300c4 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 21 Jul 2023 15:41:14 -0600 Subject: [PATCH 15/62] remove the getM365 cmd (#3881) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup --- src/cmd/getM365/exchange/get_item.go | 157 -------------------- src/cmd/getM365/main.go | 36 ----- src/cmd/getM365/onedrive/get_item.go | 207 --------------------------- 3 files changed, 400 deletions(-) delete mode 100644 src/cmd/getM365/exchange/get_item.go delete mode 100644 src/cmd/getM365/main.go delete mode 100644 src/cmd/getM365/onedrive/get_item.go diff --git a/src/cmd/getM365/exchange/get_item.go b/src/cmd/getM365/exchange/get_item.go deleted file mode 100644 index cc6e8cd6a..000000000 --- a/src/cmd/getM365/exchange/get_item.go +++ /dev/null @@ -1,157 +0,0 @@ -// get_item.go is a source file designed to retrieve an m365 object from an -// existing M365 account. Data displayed is representative of the current -// serialization abstraction versioning used by Microsoft Graph and stored by Corso. - -package exchange - -import ( - "context" - "fmt" - "os" - - "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" - kw "github.com/microsoft/kiota-serialization-json-go" - "github.com/spf13/cobra" - - "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/common/str" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/credentials" - "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/services/m365/api" -) - -// Required inputs from user for command execution -var ( - user, tenant, m365ID, category string -) - -func AddCommands(parent *cobra.Command) { - exCmd := &cobra.Command{ - Use: "exchange", - Short: "Get an M365ID item JSON", - RunE: handleExchangeCmd, - } - - fs := exCmd.PersistentFlags() - fs.StringVar(&m365ID, "id", "", "m365 identifier for object") - fs.StringVar(&category, "category", "", "type of M365 data (contacts, email, events)") - fs.StringVar(&user, "user", "", "m365 user id of M365 user") - fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant") - - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user")) - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id")) - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("category")) - - parent.AddCommand(exCmd) -} - -func handleExchangeCmd(cmd *cobra.Command, args []string) error { - if utils.HasNoFlagsAndShownHelp(cmd) { - return nil - } - - tid := str.First(tenant, os.Getenv(account.AzureTenantID)) - - ctx := clues.Add( - cmd.Context(), - "item_id", m365ID, - "resource_owner", user, - "tenant", tid) - - creds := account.M365Config{ - M365: credentials.GetM365(), - AzureTenantID: tid, - } - - err := runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true)) - if err != nil { - cmd.SilenceUsage = true - cmd.SilenceErrors = true - - return clues.Wrap(err, "getting item") - } - - return nil -} - -func runDisplayM365JSON( - ctx context.Context, - creds account.M365Config, - user, itemID string, - errs *fault.Bus, -) error { - var ( - bs []byte - err error - cat = path.ToCategoryType(category) - sw = kw.NewJsonSerializationWriter() - ) - - ac, err := api.NewClient(creds) - if err != nil { - return err - } - - switch cat { - case path.EmailCategory: - bs, err = getItem(ctx, ac.Mail(), user, itemID, true, errs) - case path.EventsCategory: - bs, err = getItem(ctx, ac.Events(), user, itemID, true, errs) - case path.ContactsCategory: - bs, err = getItem(ctx, ac.Contacts(), user, itemID, true, errs) - default: - return fmt.Errorf("unable to process category: %s", cat) - } - - if err != nil { - return err - } - - err = sw.WriteStringValue("", ptr.To(string(bs))) - if err != nil { - return clues.Wrap(err, "Error writing string value: "+itemID) - } - - array, err := sw.GetSerializedContent() - if err != nil { - return clues.Wrap(err, "Error serializing item: "+itemID) - } - - fmt.Println(string(array)) - - return nil -} - -type itemer interface { - GetItem( - ctx context.Context, - user, itemID string, - immutableID bool, - errs *fault.Bus, - ) (serialization.Parsable, *details.ExchangeInfo, error) - Serialize( - ctx context.Context, - item serialization.Parsable, - user, itemID string, - ) ([]byte, error) -} - -func getItem( - ctx context.Context, - itm itemer, - user, itemID string, - immutableIDs bool, - errs *fault.Bus, -) ([]byte, error) { - sp, _, err := itm.GetItem(ctx, user, itemID, immutableIDs, errs) - if err != nil { - return nil, clues.Wrap(err, "getting item") - } - - return itm.Serialize(ctx, sp, user, itemID) -} diff --git a/src/cmd/getM365/main.go b/src/cmd/getM365/main.go deleted file mode 100644 index c7acd3175..000000000 --- a/src/cmd/getM365/main.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "context" - "os" - - "github.com/spf13/cobra" - - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cmd/getM365/exchange" - "github.com/alcionai/corso/src/cmd/getM365/onedrive" - "github.com/alcionai/corso/src/pkg/logger" -) - -var rootCmd = &cobra.Command{ - Use: "getM365", -} - -func main() { - ls := logger.Settings{ - Level: logger.LLDebug, - Format: logger.LFText, - } - ctx, _ := logger.CtxOrSeed(context.Background(), ls) - - ctx = SetRootCmd(ctx, rootCmd) - defer logger.Flush(ctx) - - exchange.AddCommands(rootCmd) - onedrive.AddCommands(rootCmd) - - if err := rootCmd.Execute(); err != nil { - Err(ctx, err) - os.Exit(1) - } -} diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go deleted file mode 100644 index 05b5395ce..000000000 --- a/src/cmd/getM365/onedrive/get_item.go +++ /dev/null @@ -1,207 +0,0 @@ -// get_item.go is a source file designed to retrieve an m365 object from an -// existing M365 account. Data displayed is representative of the current -// serialization abstraction versioning used by Microsoft Graph and stored by Corso. - -package onedrive - -import ( - "context" - "encoding/json" - "io" - "net/http" - "os" - - "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" - kjson "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/spf13/cobra" - - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/common/str" - "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/credentials" - "github.com/alcionai/corso/src/pkg/services/m365/api" -) - -const downloadURLKey = "@microsoft.graph.downloadUrl" - -// Required inputs from user for command execution -var ( - user, tenant, m365ID string -) - -func AddCommands(parent *cobra.Command) { - exCmd := &cobra.Command{ - Use: "onedrive", - Short: "Get an M365ID item", - RunE: handleOneDriveCmd, - } - - fs := exCmd.PersistentFlags() - fs.StringVar(&m365ID, "id", "", "m365 identifier for object") - fs.StringVar(&user, "user", "", "m365 user id of M365 user") - fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant") - - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user")) - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id")) - - parent.AddCommand(exCmd) -} - -func handleOneDriveCmd(cmd *cobra.Command, args []string) error { - if utils.HasNoFlagsAndShownHelp(cmd) { - return nil - } - - tid := str.First(tenant, os.Getenv(account.AzureTenantID)) - - ctx := clues.Add( - cmd.Context(), - "item_id", m365ID, - "resource_owner", user, - "tenant", tid) - - // get account info - creds := account.M365Config{ - M365: credentials.GetM365(), - AzureTenantID: tid, - } - - gr := graph.NewNoTimeoutHTTPWrapper() - - ac, err := api.NewClient(creds) - if err != nil { - return Only(ctx, clues.Wrap(err, "getting api client")) - } - - err = runDisplayM365JSON(ctx, ac, gr, creds, user, m365ID) - if err != nil { - cmd.SilenceUsage = true - cmd.SilenceErrors = true - - return Only(ctx, clues.Wrap(err, "getting item")) - } - - return nil -} - -type itemData struct { - Size int `json:"size"` -} - -type itemPrintable struct { - Info json.RawMessage `json:"info"` - Permissions json.RawMessage `json:"permissions"` - Data itemData `json:"data"` -} - -func (i itemPrintable) MinimumPrintable() any { - return i -} - -func runDisplayM365JSON( - ctx context.Context, - ac api.Client, - gr graph.Requester, - creds account.M365Config, - userID, itemID string, -) error { - drive, err := ac.Users().GetDefaultDrive(ctx, userID) - if err != nil { - return err - } - - driveID := ptr.Val(drive.GetId()) - - it := itemPrintable{} - - item, err := ac.Drives().GetItem(ctx, driveID, itemID) - if err != nil { - return err - } - - if item != nil { - content, err := getDriveItemContent(ctx, gr, item) - if err != nil { - return err - } - - // We could get size from item.GetSize(), but the - // getDriveItemContent call is to ensure that we are able to - // download the file. - it.Data.Size = len(content) - } - - sInfo, err := serializeObject(item) - if err != nil { - return err - } - - err = json.Unmarshal([]byte(sInfo), &it.Info) - if err != nil { - return err - } - - perms, err := ac.Drives().GetItemPermission(ctx, driveID, itemID) - if err != nil { - return err - } - - sPerms, err := serializeObject(perms) - if err != nil { - return err - } - - err = json.Unmarshal([]byte(sPerms), &it.Permissions) - if err != nil { - return err - } - - PrettyJSON(ctx, it) - - return nil -} - -func serializeObject(data serialization.Parsable) (string, error) { - sw := kjson.NewJsonSerializationWriter() - - err := sw.WriteObjectValue("", data) - if err != nil { - return "", clues.Wrap(err, "writing serializing info") - } - - content, err := sw.GetSerializedContent() - if err != nil { - return "", clues.Wrap(err, "getting serializing info") - } - - return string(content), err -} - -func getDriveItemContent( - ctx context.Context, - gr graph.Requester, - item models.DriveItemable, -) ([]byte, error) { - url, ok := item.GetAdditionalData()[downloadURLKey].(*string) - if !ok { - return nil, clues.New("retrieving download url") - } - - resp, err := gr.Request(ctx, http.MethodGet, *url, nil, nil) - if err != nil { - return nil, clues.New("requesting item content").With("error", err) - } - defer resp.Body.Close() - - content, err := io.ReadAll(resp.Body) - if err != nil { - return nil, clues.New("reading item content").With("error", err) - } - - return content, nil -} From 7677299ace2d4ca61b173453ade793c88b7b4a3e Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 21 Jul 2023 16:19:13 -0600 Subject: [PATCH 16/62] use replace collisions for nightly tests (#3882) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../actions/backup-restore-test/action.yml | 5 +++ src/internal/m365/controller_test.go | 3 +- src/internal/m365/onedrive_test.go | 31 ++++++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/actions/backup-restore-test/action.yml b/.github/actions/backup-restore-test/action.yml index 4e31ad836..2d161af63 100644 --- a/.github/actions/backup-restore-test/action.yml +++ b/.github/actions/backup-restore-test/action.yml @@ -24,6 +24,10 @@ inputs: log-dir: description: Folder to store test log files required: true + on-collision: + description: Value for the --collisions flag + requried: false + default: "replace" outputs: backup-id: @@ -57,6 +61,7 @@ runs: ./corso restore '${{ inputs.service }}' \ --no-stats \ --hide-progress \ + --collisions ${{ inputs.on-collision }} \ ${{ inputs.restore-args }} \ --backup '${{ steps.backup.outputs.result }}' \ 2>&1 | diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index ef729493b..487603b39 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -692,6 +692,7 @@ func runRestoreBackupTestVersions( tenant string, resourceOwners []string, opts control.Options, + crc control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -702,7 +703,7 @@ func runRestoreBackupTestVersions( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: crc, } totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( diff --git a/src/internal/m365/onedrive_test.go b/src/internal/m365/onedrive_test.go index 3fbd5f531..eade30c9d 100644 --- a/src/internal/m365/onedrive_test.go +++ b/src/internal/m365/onedrive_test.go @@ -22,6 +22,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -516,6 +517,9 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( collectionsLatest: expected, } + rc := testdata.DefaultRestoreConfig("od_restore_and_backup_multi") + rc.OnCollision = control.Replace + runRestoreBackupTestVersions( t, testData, @@ -524,7 +528,8 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( control.Options{ RestorePermissions: true, ToggleFeatures: control.Toggles{}, - }) + }, + rc) }) } } @@ -763,6 +768,9 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } + rc := testdata.DefaultRestoreConfig("perms_restore_and_backup") + rc.OnCollision = control.Replace + runRestoreBackupTestVersions( t, testData, @@ -771,7 +779,8 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { control.Options{ RestorePermissions: true, ToggleFeatures: control.Toggles{}, - }) + }, + rc) }) } } @@ -851,6 +860,9 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } + rc := testdata.DefaultRestoreConfig("perms_backup_no_restore") + rc.OnCollision = control.Replace + runRestoreBackupTestVersions( t, testData, @@ -859,7 +871,8 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { control.Options{ RestorePermissions: false, ToggleFeatures: control.Toggles{}, - }) + }, + rc) }) } } @@ -1054,6 +1067,9 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio collectionsLatest: expected, } + rc := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup") + rc.OnCollision = control.Replace + runRestoreBackupTestVersions( t, testData, @@ -1062,7 +1078,8 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio control.Options{ RestorePermissions: true, ToggleFeatures: control.Toggles{}, - }) + }, + rc) }) } } @@ -1247,6 +1264,9 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion collectionsLatest: expected, } + rc := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup") + rc.OnCollision = control.Replace + runRestoreBackupTestVersions( t, testData, @@ -1255,7 +1275,8 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion control.Options{ RestorePermissions: true, ToggleFeatures: control.Toggles{}, - }) + }, + rc) }) } } From 62d4c68c047204287dd9f0fe4c1fda1d112cc774 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 21 Jul 2023 15:49:42 -0700 Subject: [PATCH 17/62] Have kopia package generate tags during backup based on Reasons (#3869) Move tag generation from the backup op to the kopia package. This makes it match the pattern that finding base backups uses where a set of Reasons and a separate set of additional tags are provided --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #2360 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/kopia/base_finder.go | 8 +--- src/internal/kopia/inject/inject.go | 1 + src/internal/kopia/wrapper.go | 17 ++++++- src/internal/kopia/wrapper_test.go | 60 +++++++++++-------------- src/internal/operations/backup.go | 7 +-- src/internal/operations/backup_test.go | 5 ++- src/internal/streamstore/streamstore.go | 1 + 7 files changed, 50 insertions(+), 49 deletions(-) diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index f8119587b..00561c833 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -39,8 +39,6 @@ type Reasoner interface { // SubtreePath returns the path prefix for data in existing backups that have // parameters (tenant, protected resourced, etc) that match this Reasoner. SubtreePath() (path.Path, error) - // TODO(ashmrtn): Remove this when kopia generates tags from Reasons. - TagKeys() []string } func NewReason( @@ -92,9 +90,7 @@ func (r reason) SubtreePath() (path.Path, error) { return p, clues.Wrap(err, "building path").OrNil() } -// TODO(ashmrtn): Remove this when kopia generates tags based off Reasons. Here -// at the moment so things compile. -func (r reason) TagKeys() []string { +func tagKeys(r Reasoner) []string { return []string{ r.ProtectedResource(), serviceCatString(r.Service(), r.Category()), @@ -334,7 +330,7 @@ func (b *baseFinder) getBase( ) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { allTags := map[string]string{} - for _, k := range r.TagKeys() { + for _, k := range tagKeys(r) { allTags[k] = "" } diff --git a/src/internal/kopia/inject/inject.go b/src/internal/kopia/inject/inject.go index ed0a8f5e8..22ae0d429 100644 --- a/src/internal/kopia/inject/inject.go +++ b/src/internal/kopia/inject/inject.go @@ -15,6 +15,7 @@ type ( BackupConsumer interface { ConsumeBackupCollections( ctx context.Context, + backupReasons []kopia.Reasoner, bases []kopia.IncrementalBase, cs []data.BackupCollection, pmr prefixmatcher.StringSetReader, diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index f65827f76..59235cce2 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -17,6 +17,7 @@ import ( "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/kopia/kopia/snapshot/snapshotmaintenance" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/ptr" @@ -145,10 +146,11 @@ type IncrementalBase struct { // complete backup of all data. func (w Wrapper) ConsumeBackupCollections( ctx context.Context, + backupReasons []Reasoner, previousSnapshots []IncrementalBase, collections []data.BackupCollection, globalExcludeSet prefixmatcher.StringSetReader, - tags map[string]string, + additionalTags map[string]string, buildTreeWithBase bool, errs *fault.Bus, ) (*BackupStats, *details.Builder, DetailsMergeInfoer, error) { @@ -190,6 +192,19 @@ func (w Wrapper) ConsumeBackupCollections( return nil, nil, nil, clues.Wrap(err, "building kopia directories") } + // Add some extra tags so we can look things up by reason. + tags := maps.Clone(additionalTags) + if tags == nil { + // Some platforms seem to return nil if the input is nil. + tags = map[string]string{} + } + + for _, r := range backupReasons { + for _, k := range tagKeys(r) { + tags[k] = "" + } + } + s, err := w.makeSnapshotWithRoot( ctx, previousSnapshots, diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 5af044f94..733cdaadd 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -718,15 +718,17 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { ), } + expectedTags := map[string]string{} + + maps.Copy(expectedTags, tags) + for _, r := range reasons { - for _, k := range r.TagKeys() { - tags[k] = "" + for _, k := range tagKeys(r) { + expectedTags[k] = "" } } - expectedTags := map[string]string{} - - maps.Copy(expectedTags, normalizeTagKVs(tags)) + expectedTags = normalizeTagKVs(expectedTags) table := []struct { name string @@ -757,6 +759,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { stats, deets, _, err := suite.w.ConsumeBackupCollections( suite.ctx, + reasons, prevSnaps, collections, nil, @@ -847,15 +850,17 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { storePath.Category()), } + expectedTags := map[string]string{} + + maps.Copy(expectedTags, tags) + for _, r := range reasons { - for _, k := range r.TagKeys() { - tags[k] = "" + for _, k := range tagKeys(r) { + expectedTags[k] = "" } } - expectedTags := map[string]string{} - - maps.Copy(expectedTags, normalizeTagKVs(tags)) + expectedTags = normalizeTagKVs(expectedTags) table := []struct { name string @@ -942,6 +947,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { stats, deets, prevShortRefs, err := suite.w.ConsumeBackupCollections( suite.ctx, + reasons, prevSnaps, collections, nil, @@ -1018,13 +1024,8 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { w := &Wrapper{k} - tags := map[string]string{} r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) - for _, k := range r.TagKeys() { - tags[k] = "" - } - dc1 := exchMock.NewCollection(suite.storePath1, suite.locPath1, 1) dc2 := exchMock.NewCollection(suite.storePath2, suite.locPath2, 1) @@ -1036,10 +1037,11 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { stats, _, _, err := w.ConsumeBackupCollections( ctx, + []Reasoner{r}, nil, []data.BackupCollection{dc1, dc2}, nil, - tags, + nil, true, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -1110,13 +1112,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { loc1 := path.Builder{}.Append(suite.storePath1.Folders()...) loc2 := path.Builder{}.Append(suite.storePath2.Folders()...) - tags := map[string]string{} r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) - for _, k := range r.TagKeys() { - tags[k] = "" - } - collections := []data.BackupCollection{ &mockBackupCollection{ path: suite.storePath1, @@ -1158,10 +1155,11 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { stats, deets, _, err := suite.w.ConsumeBackupCollections( suite.ctx, + []Reasoner{r}, nil, collections, nil, - tags, + nil, true, fault.New(true)) require.Error(t, err, clues.ToCore(err)) @@ -1233,6 +1231,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollectionsHandlesNoCollections() s, d, _, err := suite.w.ConsumeBackupCollections( ctx, nil, + nil, test.collections, nil, nil, @@ -1385,19 +1384,15 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { collections = append(collections, collection) } - tags := map[string]string{} r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) - for _, k := range r.TagKeys() { - tags[k] = "" - } - stats, deets, _, err := suite.w.ConsumeBackupCollections( suite.ctx, + []Reasoner{r}, nil, collections, nil, - tags, + nil, false, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -1443,12 +1438,6 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { man, err := suite.w.c.LoadSnapshot(suite.ctx, suite.snapshotID) require.NoError(suite.T(), err, "getting base snapshot: %v", clues.ToCore(err)) - tags := map[string]string{} - - for _, k := range r.TagKeys() { - tags[k] = "" - } - table := []struct { name string excludeItem bool @@ -1537,6 +1526,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { stats, _, _, err := suite.w.ConsumeBackupCollections( suite.ctx, + []Reasoner{r}, []IncrementalBase{ { Manifest: man, @@ -1547,7 +1537,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { }, test.cols(), excluded, - tags, + nil, true, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 0d6509d1d..98ceab012 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -495,12 +495,6 @@ func consumeBackupCollections( kopia.TagBackupCategory: "", } - for _, reason := range reasons { - for _, k := range reason.TagKeys() { - tags[k] = "" - } - } - // AssistBases should be the upper bound for how many snapshots we pass in. bases := make([]kopia.IncrementalBase, 0, len(bbs.AssistBases())) // Track IDs we've seen already so we don't accidentally duplicate some @@ -578,6 +572,7 @@ func consumeBackupCollections( kopiaStats, deets, itemsSourcedFromBase, err := bc.ConsumeBackupCollections( ctx, + reasons, bases, cs, pmr, diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 49f8d4aa5..cd4a83737 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -107,6 +107,7 @@ func checkPaths(t *testing.T, expected, got []path.Path) { type mockBackupConsumer struct { checkFunc func( + backupReasons []kopia.Reasoner, bases []kopia.IncrementalBase, cs []data.BackupCollection, tags map[string]string, @@ -115,6 +116,7 @@ type mockBackupConsumer struct { func (mbu mockBackupConsumer) ConsumeBackupCollections( ctx context.Context, + backupReasons []kopia.Reasoner, bases []kopia.IncrementalBase, cs []data.BackupCollection, excluded prefixmatcher.StringSetReader, @@ -123,7 +125,7 @@ func (mbu mockBackupConsumer) ConsumeBackupCollections( errs *fault.Bus, ) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error) { if mbu.checkFunc != nil { - mbu.checkFunc(bases, cs, tags, buildTreeWithBase) + mbu.checkFunc(backupReasons, bases, cs, tags, buildTreeWithBase) } return &kopia.BackupStats{}, &details.Builder{}, nil, nil @@ -537,6 +539,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections mbu := &mockBackupConsumer{ checkFunc: func( + backupReasons []kopia.Reasoner, bases []kopia.IncrementalBase, cs []data.BackupCollection, tags map[string]string, diff --git a/src/internal/streamstore/streamstore.go b/src/internal/streamstore/streamstore.go index 9deb0176d..6f5918c81 100644 --- a/src/internal/streamstore/streamstore.go +++ b/src/internal/streamstore/streamstore.go @@ -234,6 +234,7 @@ func write( backupStats, _, _, err := bup.ConsumeBackupCollections( ctx, nil, + nil, dbcs, prefixmatcher.NopReader[map[string]struct{}](), nil, From 0d6b08204db692a5e62b1cb0e2bea40bf6f11d2a Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 21 Jul 2023 18:00:09 -0600 Subject: [PATCH 18/62] allow users to limit page size (#3875) allows cli users to limit the page size of delta queries by calling a new hidden flag: --delta-page-size. This also adds the control.Options struct to the api client, so that configurations such as this can be easily handed into, and used by, the client. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cli/backup/exchange.go | 3 +- src/cli/backup/exchange_test.go | 37 ++++++++++--------- src/cli/backup/onedrive.go | 2 +- src/cli/flags/options.go | 14 +++++++ src/cli/utils/options.go | 6 +++ src/cli/utils/options_test.go | 3 ++ src/cli/utils/testdata/flags.go | 2 + src/cli/utils/users.go | 8 ++-- src/internal/events/events.go | 4 +- src/internal/m365/backup_test.go | 2 +- src/internal/m365/controller.go | 2 +- src/internal/m365/exchange/backup_test.go | 2 +- .../m365/exchange/container_resolver_test.go | 3 +- src/internal/m365/exchange/helper_test.go | 3 +- .../exchange/mail_container_cache_test.go | 3 +- src/internal/m365/exchange/restore_test.go | 2 +- .../m365/onedrive/item_collector_test.go | 2 +- src/internal/m365/onedrive/service_test.go | 3 +- src/internal/m365/onedrive/url_cache_test.go | 2 +- src/internal/m365/sharepoint/backup_test.go | 2 +- .../m365/sharepoint/collection_test.go | 2 +- src/internal/operations/backup_test.go | 2 +- src/internal/operations/test/exchange_test.go | 2 +- src/internal/operations/test/helper_test.go | 2 +- src/pkg/control/options.go | 10 +++-- src/pkg/services/m365/api/client.go | 11 +++++- src/pkg/services/m365/api/contacts_pager.go | 2 +- src/pkg/services/m365/api/events_pager.go | 2 +- src/pkg/services/m365/api/helper_test.go | 3 +- src/pkg/services/m365/api/mail_pager.go | 2 +- src/pkg/services/m365/m365.go | 3 +- 31 files changed, 96 insertions(+), 50 deletions(-) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 99bb0ff78..0f11bd6bd 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -94,6 +94,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { flags.AddDisableDeltaFlag(c) flags.AddEnableImmutableIDFlag(c) flags.AddDisableConcurrencyLimiterFlag(c) + flags.AddDeltaPageSizeFlag(c) case listCommand: c, fs = utils.AddCommand(cmd, exchangeListCmd()) @@ -175,7 +176,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { sel := exchangeBackupCreateSelectors(flags.UserFV, flags.CategoryDataFV) - ins, err := utils.UsersMap(ctx, *acct, fault.New(true)) + ins, err := utils.UsersMap(ctx, *acct, utils.Control(), fault.New(true)) if err != nil { return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users")) } diff --git a/src/cli/backup/exchange_test.go b/src/cli/backup/exchange_test.go index 6bd078797..d260ca290 100644 --- a/src/cli/backup/exchange_test.go +++ b/src/cli/backup/exchange_test.go @@ -37,11 +37,11 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { expectRunE func(*cobra.Command, []string) error }{ { - "create exchange", - createCommand, - expectUse + " " + exchangeServiceCommandCreateUseSuffix, - exchangeCreateCmd().Short, - []string{ + name: "create exchange", + use: createCommand, + expectUse: expectUse + " " + exchangeServiceCommandCreateUseSuffix, + expectShort: exchangeCreateCmd().Short, + flags: []string{ flags.UserFN, flags.CategoryDataFN, flags.DisableIncrementalsFN, @@ -50,28 +50,29 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { flags.FetchParallelismFN, flags.SkipReduceFN, flags.NoStatsFN, + flags.DeltaPageSizeFN, }, - createExchangeCmd, + expectRunE: createExchangeCmd, }, { - "list exchange", - listCommand, - expectUse, - exchangeListCmd().Short, - []string{ + name: "list exchange", + use: listCommand, + expectUse: expectUse, + expectShort: exchangeListCmd().Short, + flags: []string{ flags.BackupFN, flags.FailedItemsFN, flags.SkippedItemsFN, flags.RecoveredErrorsFN, }, - listExchangeCmd, + expectRunE: listExchangeCmd, }, { - "details exchange", - detailsCommand, - expectUse + " " + exchangeServiceCommandDetailsUseSuffix, - exchangeDetailsCmd().Short, - []string{ + name: "details exchange", + use: detailsCommand, + expectUse: expectUse + " " + exchangeServiceCommandDetailsUseSuffix, + expectShort: exchangeDetailsCmd().Short, + flags: []string{ flags.BackupFN, flags.ContactFN, flags.ContactFolderFN, @@ -90,7 +91,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { flags.EventStartsBeforeFN, flags.EventSubjectFN, }, - detailsExchangeCmd, + expectRunE: detailsExchangeCmd, }, { "delete exchange", diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index 62ce242d4..b9d94fc41 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -157,7 +157,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error { sel := oneDriveBackupCreateSelectors(flags.UserFV) - ins, err := utils.UsersMap(ctx, *acct, fault.New(true)) + ins, err := utils.UsersMap(ctx, *acct, utils.Control(), fault.New(true)) if err != nil { return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users")) } diff --git a/src/cli/flags/options.go b/src/cli/flags/options.go index 046d3c8d7..81a893f93 100644 --- a/src/cli/flags/options.go +++ b/src/cli/flags/options.go @@ -5,6 +5,7 @@ import ( ) const ( + DeltaPageSizeFN = "delta-page-size" DisableConcurrencyLimiterFN = "disable-concurrency-limiter" DisableDeltaFN = "disable-delta" DisableIncrementalsFN = "disable-incrementals" @@ -21,6 +22,7 @@ const ( ) var ( + DeltaPageSizeFV int DisableConcurrencyLimiterFV bool DisableDeltaFV bool DisableIncrementalsFV bool @@ -72,6 +74,18 @@ func AddSkipReduceFlag(cmd *cobra.Command) { cobra.CheckErr(fs.MarkHidden(SkipReduceFN)) } +// AddDeltaPageSizeFlag adds a hidden flag that allows callers to reduce delta +// query page sizes below 500. +func AddDeltaPageSizeFlag(cmd *cobra.Command) { + fs := cmd.Flags() + fs.IntVar( + &DeltaPageSizeFV, + DeltaPageSizeFN, + 500, + "Control quantity of items returned in paged queries. Valid range is [1-500]. Default: 500") + cobra.CheckErr(fs.MarkHidden(DeltaPageSizeFN)) +} + // AddFetchParallelismFlag adds a hidden flag that allows callers to reduce call // paralellism (ie, the corso worker pool size) from 4 to as low as 1. func AddFetchParallelismFlag(cmd *cobra.Command) { diff --git a/src/cli/utils/options.go b/src/cli/utils/options.go index 0cc44c839..7f9176a90 100644 --- a/src/cli/utils/options.go +++ b/src/cli/utils/options.go @@ -14,6 +14,12 @@ func Control() control.Options { opt.FailureHandling = control.FailFast } + dps := int32(flags.DeltaPageSizeFV) + if dps > 500 || dps < 1 { + dps = 500 + } + + opt.DeltaPageSize = dps opt.DisableMetrics = flags.NoStatsFV opt.RestorePermissions = flags.RestorePermissionsFV opt.SkipReduce = flags.SkipReduceFV diff --git a/src/cli/utils/options_test.go b/src/cli/utils/options_test.go index 746558aa1..1a8f7ddcd 100644 --- a/src/cli/utils/options_test.go +++ b/src/cli/utils/options_test.go @@ -35,6 +35,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { assert.True(t, flags.SkipReduceFV, flags.SkipReduceFN) assert.Equal(t, 2, flags.FetchParallelismFV, flags.FetchParallelismFN) assert.True(t, flags.DisableConcurrencyLimiterFV, flags.DisableConcurrencyLimiterFN) + assert.Equal(t, 499, flags.DeltaPageSizeFV, flags.DeltaPageSizeFN) }, } @@ -48,6 +49,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { flags.AddSkipReduceFlag(cmd) flags.AddFetchParallelismFlag(cmd) flags.AddDisableConcurrencyLimiterFlag(cmd) + flags.AddDeltaPageSizeFlag(cmd) // Test arg parsing for few args cmd.SetArgs([]string{ @@ -60,6 +62,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { "--" + flags.SkipReduceFN, "--" + flags.FetchParallelismFN, "2", "--" + flags.DisableConcurrencyLimiterFN, + "--" + flags.DeltaPageSizeFN, "499", }) err := cmd.Execute() diff --git a/src/cli/utils/testdata/flags.go b/src/cli/utils/testdata/flags.go index f97529b57..d29198072 100644 --- a/src/cli/utils/testdata/flags.go +++ b/src/cli/utils/testdata/flags.go @@ -48,6 +48,8 @@ var ( Destination = "destination" RestorePermissions = true + DeltaPageSize = "deltaPageSize" + AzureClientID = "testAzureClientId" AzureTenantID = "testAzureTenantId" AzureClientSecret = "testAzureClientSecret" diff --git a/src/cli/utils/users.go b/src/cli/utils/users.go index 610f0e2c6..affa520fd 100644 --- a/src/cli/utils/users.go +++ b/src/cli/utils/users.go @@ -7,6 +7,7 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -15,9 +16,10 @@ import ( func UsersMap( ctx context.Context, acct account.Account, + co control.Options, errs *fault.Bus, ) (idname.Cacher, error) { - au, err := makeUserAPI(acct) + au, err := makeUserAPI(acct, co) if err != nil { return nil, clues.Wrap(err, "constructing a graph client") } @@ -25,13 +27,13 @@ func UsersMap( return au.GetAllIDsAndNames(ctx, errs) } -func makeUserAPI(acct account.Account) (api.Users, error) { +func makeUserAPI(acct account.Account, co control.Options) (api.Users, error) { creds, err := acct.M365Config() if err != nil { return api.Users{}, clues.Wrap(err, "getting m365 account creds") } - cli, err := api.NewClient(creds) + cli, err := api.NewClient(creds, co) if err != nil { return api.Users{}, clues.Wrap(err, "constructing api client") } diff --git a/src/internal/events/events.go b/src/internal/events/events.go index baa2c2117..1252052f7 100644 --- a/src/internal/events/events.go +++ b/src/internal/events/events.go @@ -82,8 +82,8 @@ var ( RudderStackDataPlaneURL string ) -func NewBus(ctx context.Context, s storage.Storage, tenID string, opts control.Options) (Bus, error) { - if opts.DisableMetrics { +func NewBus(ctx context.Context, s storage.Storage, tenID string, co control.Options) (Bus, error) { + if co.DisableMetrics { return Bus{}, nil } diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index bb59741f8..b80bd4ddc 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -57,7 +57,7 @@ func (suite *DataCollectionIntgSuite) SetupSuite() { suite.tenantID = creds.AzureTenantID - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 9b037350b..4dd2c19e8 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -69,7 +69,7 @@ func NewController( return nil, clues.Wrap(err, "retrieving m365 account configuration").WithClues(ctx) } - ac, err := api.NewClient(creds) + ac, err := api.NewClient(creds, co) if err != nil { return nil, clues.Wrap(err, "creating api client").WithClues(ctx) } diff --git a/src/internal/m365/exchange/backup_test.go b/src/internal/m365/exchange/backup_test.go index 8ac8c14dd..34735eda8 100644 --- a/src/internal/m365/exchange/backup_test.go +++ b/src/internal/m365/exchange/backup_test.go @@ -414,7 +414,7 @@ func (suite *BackupIntgSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) suite.tenantID = creds.AzureTenantID diff --git a/src/internal/m365/exchange/container_resolver_test.go b/src/internal/m365/exchange/container_resolver_test.go index 8b5fa7c95..54cd23c67 100644 --- a/src/internal/m365/exchange/container_resolver_test.go +++ b/src/internal/m365/exchange/container_resolver_test.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -698,7 +699,7 @@ func (suite *ContainerResolverSuite) SetupSuite() { } func (suite *ContainerResolverSuite) TestPopulate() { - ac, err := api.NewClient(suite.credentials) + ac, err := api.NewClient(suite.credentials, control.Defaults()) require.NoError(suite.T(), err, clues.ToCore(err)) eventFunc := func(t *testing.T) graph.ContainerResolver { diff --git a/src/internal/m365/exchange/helper_test.go b/src/internal/m365/exchange/helper_test.go index 7e604c466..f8cadd227 100644 --- a/src/internal/m365/exchange/helper_test.go +++ b/src/internal/m365/exchange/helper_test.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -30,7 +31,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.creds = creds - its.ac, err = api.NewClient(creds) + its.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) its.userID = tconfig.GetM365UserID(ctx) diff --git a/src/internal/m365/exchange/mail_container_cache_test.go b/src/internal/m365/exchange/mail_container_cache_test.go index b95a9a170..64f453092 100644 --- a/src/internal/m365/exchange/mail_container_cache_test.go +++ b/src/internal/m365/exchange/mail_container_cache_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -83,7 +84,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() { ctx, flush := tester.NewContext(t) defer flush() - ac, err := api.NewClient(suite.credentials) + ac, err := api.NewClient(suite.credentials, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) acm := ac.Mail() diff --git a/src/internal/m365/exchange/restore_test.go b/src/internal/m365/exchange/restore_test.go index 4d91329e9..42e61a915 100644 --- a/src/internal/m365/exchange/restore_test.go +++ b/src/internal/m365/exchange/restore_test.go @@ -44,7 +44,7 @@ func (suite *RestoreIntgSuite) SetupSuite() { require.NoError(t, err, clues.ToCore(err)) suite.credentials = m365 - suite.ac, err = api.NewClient(m365) + suite.ac, err = api.NewClient(m365, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/onedrive/item_collector_test.go index ec2ab26af..fc2cccd62 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -313,7 +313,7 @@ func (suite *OneDriveIntgSuite) SetupSuite() { suite.creds = creds - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/onedrive/service_test.go b/src/internal/m365/onedrive/service_test.go index 4569acffc..a39a65a76 100644 --- a/src/internal/m365/onedrive/service_test.go +++ b/src/internal/m365/onedrive/service_test.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -20,7 +21,7 @@ type oneDriveService struct { } func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) { - ac, err := api.NewClient(credentials) + ac, err := api.NewClient(credentials, control.Defaults()) if err != nil { return nil, err } diff --git a/src/internal/m365/onedrive/url_cache_test.go b/src/internal/m365/onedrive/url_cache_test.go index 8adcf36cc..7946da840 100644 --- a/src/internal/m365/onedrive/url_cache_test.go +++ b/src/internal/m365/onedrive/url_cache_test.go @@ -53,7 +53,7 @@ func (suite *URLCacheIntegrationSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user) diff --git a/src/internal/m365/sharepoint/backup_test.go b/src/internal/m365/sharepoint/backup_test.go index 6e878f0b9..973a55670 100644 --- a/src/internal/m365/sharepoint/backup_test.go +++ b/src/internal/m365/sharepoint/backup_test.go @@ -201,7 +201,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds) + ac, err := api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) col, err := collectPages( diff --git a/src/internal/m365/sharepoint/collection_test.go b/src/internal/m365/sharepoint/collection_test.go index babe6114e..42f9ad9a1 100644 --- a/src/internal/m365/sharepoint/collection_test.go +++ b/src/internal/m365/sharepoint/collection_test.go @@ -43,7 +43,7 @@ func (suite *SharePointCollectionSuite) SetupSuite() { suite.creds = m365 - ac, err := api.NewClient(m365) + ac, err := api.NewClient(m365, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) suite.ac = ac diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index cd4a83737..f15f88f02 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -1232,7 +1232,7 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index 647c7a397..e33cdd0ae 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -278,7 +278,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds) + ac, err := api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 31dbb9544..f1da62cbe 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -585,7 +585,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds) + its.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index 23375f229..fbb3d08a9 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -7,14 +7,17 @@ import ( // Options holds the optional configurations for a process type Options struct { + // DeltaPageSize controls the quantity of items fetched in each page + // during multi-page queries, such as graph api delta endpoints. + DeltaPageSize int32 `json:"deltaPageSize"` DisableMetrics bool `json:"disableMetrics"` FailureHandling FailurePolicy `json:"failureHandling"` + ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` + Parallelism Parallelism `json:"parallelism"` + Repo repository.Options `json:"repo"` RestorePermissions bool `json:"restorePermissions"` SkipReduce bool `json:"skipReduce"` ToggleFeatures Toggles `json:"toggleFeatures"` - Parallelism Parallelism `json:"parallelism"` - Repo repository.Options `json:"repo"` - ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` } type Parallelism struct { @@ -39,6 +42,7 @@ const ( func Defaults() Options { return Options{ FailureHandling: FailAfterRecovery, + DeltaPageSize: 500, ToggleFeatures: Toggles{}, Parallelism: Parallelism{ CollectionBuffer: 4, diff --git a/src/pkg/services/m365/api/client.go b/src/pkg/services/m365/api/client.go index 957da03db..c74bf215b 100644 --- a/src/pkg/services/m365/api/client.go +++ b/src/pkg/services/m365/api/client.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" ) @@ -36,11 +37,13 @@ type Client struct { // arbitrary urls instead of constructing queries using the // graph api client. Requester graph.Requester + + options control.Options } // NewClient produces a new exchange api client. Must be used in // place of creating an ad-hoc client struct. -func NewClient(creds account.M365Config) (Client, error) { +func NewClient(creds account.M365Config, co control.Options) (Client, error) { s, err := NewService(creds) if err != nil { return Client{}, err @@ -53,7 +56,11 @@ func NewClient(creds account.M365Config) (Client, error) { rqr := graph.NewNoTimeoutHTTPWrapper() - return Client{creds, s, li, rqr}, nil + if co.DeltaPageSize < 1 || co.DeltaPageSize > maxDeltaPageSize { + co.DeltaPageSize = maxDeltaPageSize + } + + return Client{creds, s, li, rqr, co}, nil } // initConcurrencyLimit ensures that the graph concurrency limiter is diff --git a/src/pkg/services/m365/api/contacts_pager.go b/src/pkg/services/m365/api/contacts_pager.go index f997bd2e7..9a86f1e00 100644 --- a/src/pkg/services/m365/api/contacts_pager.go +++ b/src/pkg/services/m365/api/contacts_pager.go @@ -277,7 +277,7 @@ func (c Contacts) NewContactDeltaIDsPager( Select: idAnd(parentFolderID), // do NOT set Top. It limits the total items received. }, - Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)), + Headers: newPreferHeaders(preferPageSize(c.options.DeltaPageSize), preferImmutableIDs(immutableIDs)), } var builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder diff --git a/src/pkg/services/m365/api/events_pager.go b/src/pkg/services/m365/api/events_pager.go index d70e1d281..2874d37e5 100644 --- a/src/pkg/services/m365/api/events_pager.go +++ b/src/pkg/services/m365/api/events_pager.go @@ -244,7 +244,7 @@ func (c Events) NewEventDeltaIDsPager( immutableIDs bool, ) (itemIDPager, error) { options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{ - Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)), + Headers: newPreferHeaders(preferPageSize(c.options.DeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemCalendarsItemEventsDeltaRequestBuilderGetQueryParameters{ // do NOT set Top. It limits the total items received. }, diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index 05e16b00e..8a98a5d56 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) @@ -96,7 +97,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds) + its.ac, err = api.NewClient(creds, control.Defaults()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index 5472239f8..0648a906c 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -310,7 +310,7 @@ func (c Mail) NewMailDeltaIDsPager( Select: idAnd("isRead"), // do NOT set Top. It limits the total items received. }, - Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)), + Headers: newPreferHeaders(preferPageSize(c.options.DeltaPageSize), preferImmutableIDs(immutableIDs)), } var builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index 91141696f..9dd803cf5 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -328,7 +329,7 @@ func makeAC( return api.Client{}, clues.Wrap(err, "getting m365 account creds") } - cli, err := api.NewClient(creds) + cli, err := api.NewClient(creds, control.Defaults()) if err != nil { return api.Client{}, clues.Wrap(err, "constructing api client") } From 6880064c64b2d0ccca8379f55a740908271fa939 Mon Sep 17 00:00:00 2001 From: Niraj Tolia Date: Fri, 21 Jul 2023 18:04:18 -0700 Subject: [PATCH 19/62] First draft of a community-contributed blog post on multi-tenant backup (#3885) - Minor edits made for clarity and to pass the linter - Added image --- #### Does this PR need a docs update or release note? - [x] :no_entry: No --- ...23-07-24-multi-tenant-backup-with-corso.md | 175 ++++++++++++++++++ website/blog/images/data-center.jpg | Bin 0 -> 231909 bytes 2 files changed, 175 insertions(+) create mode 100644 website/blog/2023-07-24-multi-tenant-backup-with-corso.md create mode 100644 website/blog/images/data-center.jpg diff --git a/website/blog/2023-07-24-multi-tenant-backup-with-corso.md b/website/blog/2023-07-24-multi-tenant-backup-with-corso.md new file mode 100644 index 000000000..e828bad2c --- /dev/null +++ b/website/blog/2023-07-24-multi-tenant-backup-with-corso.md @@ -0,0 +1,175 @@ +--- +slug: multi-tenant-backup-with-corso +title: "Using Corso to Build a Self-Hosted Multi-Tenant Office 365 Backup Solution" +description: "" +authors: + - name: meuchels + title: Corso Community Member, IT Lead + url: https://github.com/meuchels + image_url: https://avatars.githubusercontent.com/u/77171293?v=4 +tags: [corso, microsoft 365, backups, msp, multi-tenant] +date: 2023-07-24 +image: ./images/data-center.jpg +--- + +![A woman engineer holding a laptop in front of a data center](./images/data-center.jpg) + +This community-contributed blog post shows how MSPs in the community are using Corso to build out a multi-tenant backup +solution for their Microsoft 365 customers. If you have questions, come find the author (or us) on +[Discord](https://www.alcion.ai/discord). + + + +First of all, I offer a fully managed backup solution. My clients have no access to the backup software or the data. I +require them to request recovery in a ticket. For my use case I have a self-hosted instance of MinIO that I won't be +going over but there is [another blog post on it](./2023-2-4-where-to-store-corso.md#local-s3-testing). I will show the +layout and an example of how to backup emails using the exchange option in Corso. + +## Organizing the file structure on your storage + +I wanted my S3 bucket to be laid out in the following fashion utilizing 1 bucket with prefixes for the tenants. For now, +all I did is create a bucket with access to a user for corso. While it's possible to use a single bucket and use prefix +paths per tenant within it, I didn't do that in my setup. The will be generated later with the backup initialization. + +```bash +BUCKET + tenant1-exchange + tenant1-onedrive + tenant1-sharepoint + tenant2-exchange + tenant2-onedrive + tenant2-sharepoint +``` + +If I don’t backup a particular service for a client, it will be clear by looking at whether the bucket exists or not. + +I have a short name for each tenant to differentiate them. + +## The backup compute server layout + +I utilize Ubuntu Server for this task. In my setup, everything is done as the root user. I have put the corso +executable in `/opt/corso/` and will be building everything under there. Here is the folder layout before I go into +usage. + +```bash +# For logs +/opt/corso/logs +# For config files +/opt/corso/toml +# Root of the scripts folder +/opt/corso/scripts +# For building out the environment loaders +/opt/corso/scripts/environments +# For building out the backup scripts +/opt/corso/scripts/back-available +# For adding a link to the backups that will be run +/opt/corso/scripts/back-active +``` + +## The environment files + +For [configuration](../../docs/setup/configuration/), create an environment file +`/opt/corso/scripts/environments/blank-exchange` with the following content for a template. You can copy this template +to `-exchange` in the same folder to setup your client exchange backup environment. + +```bash +##################################### +#EDIT THIS SECTION TO MEET YOUR NEEDS +##################################### + +# this is a shortname for your tenant to setup storage +export tenantshortname="" + +# this is your tenant info from the app setup on O365 +export AZURE_TENANT_ID="" +export AZURE_CLIENT_ID="" +export AZURE_CLIENT_SECRET="" + +# this is your credentials for your s3 storage +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="EvwruJe!0E|yM~|M1Uuj_~rddS@8r zcmC#OXXX9Q^Y2XS=;>tl&VSw+*U8Gv9RPqu{YUO$WnufyAKn?&O-n=SokaiuL{yvq zV)OrE4=bN{I{^SGkc+RIosF#rC8GrsC8waE0HwT@x1*JZ2ir$83kNedOG+t_lZ%~5I5@n$ zz1i)oEZF~X=)cqdtHQrE|JU$e^|Aj`-@koFDQ#tK=IQ7``HxdAK#m|!H%fOGGYcz9 zw*O}k{~s6pFSGv34pt2-Yb!S^=l7~~-lNRU+2*~uoh|J=>_E0Wjwi09YIp0E|)Idk)mU+f5ls2k_6Gr%Q48ueg8b_x%5P z{J$NbDDpfU$;gg9(C(f=PoZf&s&H!VJUA!ED2v!92pk!{WeF!m_~%!^*>I!&<<)!Un;{ zz-GZ#z_!Bvgq?%kfxU!%gF}NOg=2vef>VIggR_D2hWiGW23H2x0yhY^0Cxa)4-XHI z56=M44=)d|4{s0e4<7@c3ttES9exIW5B?4T0f7jC1p$blhG2%^f$$9>6QKs74`CMJ z0O1cJ3L*s}7osep0irYFSHv{LD#TvIS;RxcXCzD{IwT<^H6%+UU!-`X5~NO~DWn6W zXJjm724ohIu zBUDe+IMi~~@2D%NH)v>R^l0K}2525=acGrj18AFQf6#Hz+0hlyEzyI~v(a18XVEV( zkTB>lBrr@c{4mlmnlPp?&M=WM=`p1+%`k&7voSj`moV?JaIiSB)UceeVz8>PMzM~t z5wPj8Ww0%=L$Qmo2e9{WU~p(~q;M>7LU2lO25}B>;c*#o6>#lwqj76-A-I=#*myj6 zI(R;K*?2v8+xRf}^!N(+j`;ETjrfcBPXuHH5(HKRkpy6ZS%L>b5<+o8E5azkI>H6Q zCn8EB86pRwM4~pL4PsbgW@2?>Z{j@SLEHHwg;Wr#XKDs&ZR)Sob<`U)$TR{p)-)+J{WO=f6tpU|0kqY$t8_?o0(7=?>2$+% z5A+Q5y7b}nZS+SBgbWG{ehk$N>x^iOVvMegMU3-I@Js?s4otaB)6CG!yv(-D+00Wc z&@6l`b}YFpGpul|f~?N0MXXC~sB98!UToEDJM8%E%Isg++t@ESXgLfx5;=xB-ahbt zaQsm6VT}`;Q<3v4XD8=17c-XyS2ou?HyXDrcQAK5_Z1Hdj}=cI&oVC#uL^HCZ$IxV zA3vWPADHi$pO*g!Pv)m)P&f?(WJ)|!_>^Q=@Z;1-A`4Y zUd+_Yip=iJ70h$YFD;}j(k)Ic#Vu1TkF9`KNmhr}K#6JAo2`g+Wk3=0W|zG{K?42cP9WSA0SK;_zki>xZvNUvEQnLpnprLO+M@g~^3g ze?$M~_H8L#I6OZBHo`6f63HEz8Tl4v88sIDAv!JkCB`CVES57iBld5cbsQv~FFr2; zKEXL*F;OhBA_*(WFKI7XCAl?)G9@bIHq|tBG>s=MKOH&UGkrTlIio$3HZvjfCCfHz zAzL!LE{8NHGUp-JGIuslJP({tmLHw}RA5)ITqsxAT0~!zRt#V4Rs5?&r)0QPptQ1# zxGcKtwcNRUyF#PlXC;4SWff^vd^L2nXZ3N7QO$I%bZtAB4P01ZQE|wZ=dNdx-L>xu70 z?hWnz+vnGJ_ub|D*$lFV~`?TP6=Zwfq@2uGDk2&eNp?QV*@ded|nMLizX%Eq?<)sOj~u^eA_)c(mNBon!9Uz7JH}r?)xtX zp})|6r5#cnRvvL3bsx(fPoEf`9GtqGKA(L%$2!lsV7zGlE&hArQt$HM%I)g!b<7RX zP5CX)ZU5cJyUly2`{#$KKSX~j9t9qUpLCxNpS@oYUNT;pUOV5E-q!y*{r&rQ4G;=| zhJk^Bg@J{GgMEK=!NVgWA|N0lq9Gx_Z)iBE=&0`>94uTcjQ5?K5D$-#oQ8~)jE0$x zj+vK>S4PGN>i_Qv{`Lc~5Mgv+|H42~1E8^>V6dS6{saiV_h(>X{uu=QUkMfl4just z8WDhm3gZW)|4)^i^m;dbi3KJus>c9tPZ9>7j-J~BRR|~R9a}tOKJXuN%hO2-f26Q zaU=BYDGv78MNM}GC%*pHY+1Pkx?0@V;~y=^JEQg)-2546z)%s8CJ%iO|vPl*k9+weL9wuyc$xudtk3XhM%@#TaYrPZXXg`#iETHvieEyBQ zCFgctYUF2Xx*}p2INGupcVs>6m5f%{GTHbEc7u-Y%j$IX6Yck>1k)aeEO3y_gDJ!( z3-KrLo||wRTAYJ$CkZAI%ljrvv^tKzC~A~bT|Crq1YOd&KdK?1h|=6P#MZPcY`Lad z0yP~W7VP}%KHky8!^2{YoOB&;@@`UhtS;h|c!g^z%6gmu$$#b{#T}#I%5=6ai_L#7 zrj7^oB9or?7?-XVZcVH>L4Nra1h3!La_fg?b^~nE!Gkp{^1M{(Gj(Z*YE*d z0dsQb?$bh|DY4~uf?Sk@8L)l&YG8oW-2wB9<5~3&-uO+9t=^MU)ve0`%{RCpP91Sf za&E8dD&FtOyN^XNM@6qa<2Wws$c~gX+6gW%p#cwho*s?(5PBrAx+Bb*zdul zn!)9qHW@e9d>`$CKkTf_5c^CIIA>2dyAC2$i4mUc*2!iI_is^1lksg9^b=HPSzjDTU z1;>d=B~E*?b*T>}+FpC>-Rv)S_bqgH%->^`PI0!K^@gB1qsclLYK8=i3@|2geLY^v zslD}b3fA_YSmZHf?C1^D(>_^V&RNO&F2zbG5y~Nn)+L)J%UUVaI||brzp?;8+hH(3 zn*F1R`t7X*XvC>$z7b9)2tD7t`EzwBuP5e)Uw1k&_jqQ-WOe`41Z^sSJdD=wk&vF$ zM`hVXtz8YPss&{=sqgdCj?mdwp~#aaS5+>*{zED;%F0ZK{c=pCXePHTUQ01MZ90MdTlyi?;DSUC8rHv#V<3M_(3`Gk=e< zVJ03DC4uAR1Uv4v62)0C@=)$C@^3zmwGOh(&n?34A0nBVc}^(>Vk3yC@m?nU8mVdM z>3%?P>A*j2WTS52m6AqH@Iz;mXxLzM6kT1+G+3qt^OPu^&voc@UGYyVYO~B&S+7!m z*m4o4iRXlMLfF8ldnS5nZn3?2D5I1(nmMsfbYo{{Keyw9J17XBw{{n2^ z7)@M>?hTDV(TzJ8`4qOIKO&`}==jyD}R3OUDqI3|=*HQCSVm?{#PujJT9jFyK zqAL+&I135^O4h|C)nO$-B=Do{jwslXw^`>@m4?ioQ5^e60n%a-t3^DRGdj_`~bF%ghV83gZQD42H&k9iVup7a|Pwn`3^2@++quJpTArHQ;?#~l2*Ni`LbU$TcTREt52l8`NTmPFB!2`C~>2DC+Ev+`SAOeAFt`*dBby-Ml2R?p#AAhTg)4GZarkp z$?wdYk1mtkKS5C8a4d|XCRvDR)xiDSB*~DIedZ$@k>P+e{n6Ka0Hh8QpQ`gxJRG<& zVDOT2jDTk|-^?c3N-jjd_{4IXGw^u6#ZwNs=NRu95mwh*)t6jUn&-MAIuGe1*uc|K zWMI)zBJwF-@#+O9;Sm|S<_EaN9?suXv^RD7T%5SL{CRLx>YuRmW4D`<#=B#U+r`RV z^ey*j{laOMa*Hs$a!7t9SOdhNC6iaY}Dl#+JF2U9kXy6%3pFD(Drx@yI-J3xc>I7O-&7&U-HEf!YBcaQtP2uw{uHkN zGL%kmtUDp9`dKDjST#O3q9tKP>Fh56NmDDnI0q$CUUz&0gC({X2ljVNFJZ`SbsCFE zY*n6xa0J2=zT;1VrD?l^AE}*{`0{(~ z{sK&gJI7cQl7GOoRz<`2DoNosvSqf?Nh}R*u{3sZ4pwYlZ4^2_aau~W_I^H(Z}ayV zRT(9@zep#Kq%$i=+@==t-~ftkeLtnB3x*S&n^xrhrM7CK#u#Lb#^pR^8gHw6zddQH zU$2Bt8%Y4?{lIX>omQQL@0v85wR(h0LKnv(u(5 zF8r0hjq%f>`Whyy!KiIXLF5{>z)7Y&R#%P;9J|cmJas)&cFAM+7L#kTtguwH4J$4m z44m#KO2fCHK>5eG2XB>ZqGmT@pMzj8ZCa~seL0!T@xK7Epv0HLHS<2*zX0|QmAGd? zy4t-AJ0j1Pn(|sYTVst1jb77sg!1*j06}4N`bod%Rw5%SUV&=+TSc6~py$06Ao+HN zs>Tz(%aF|>b|c-{$=WO{2xo1%6aIH2*nDW21=x!h_o@q-^iR(5)n`p|sCZb|jo*Ex zA>539+N@2AeY^zb`H;QVwivKaLWrKtEgF3uD%FRm-D}+qPIOpi?R<3FvTxi50vHU@ zhK_wMvSfgjwD_)Iw#RFQ(wgLct6990P48-Juf@UD*wn)sN1&%KQ6``FT+VNT4yw2O z%&*HchH~>CjMmtfHaU(krmK;ms!FdnHHUhjtX%3ha@ZZn$7$ya2+Z=)&FBQpTvc(C zV`SuCCoUu=!j|vyLcL@3O&8Qs1CutNnnNUX(c2sL-zkI?c- zj|L8ICgDs;j9y-+@0xPpXcYbVCUq9{9&b^;^ky90G;ShX1gC25Nzp(YJIGwN4-`;0 zMg7$c61pN`|6nmIQGvlyPCVttK z1PbBPOUAR{)#$9~w>8h6!RUG3Io(NS*Uf`7@e=o~k@yBNvIoMh4aj;on6YDQ`j~~C zwKt&EX8u-?FVNR`S?+h-F0swpUjTnop1p>+q=B=pL#hVEup(16v6M%M6(?e!kx80` zx)kAXm1a~qUJ-(d6XG3yQF48R7Gh7hls0+Gb6FMd99B@U(YU;!eypBOwArR(s zZ4IMUB}hqyj$v-(!CH&|&Y^ii=VJDoXruGTN$G*>KAEj{wQBsn(pn8Cez(C zRFJ0t!#$9oAVLm*7hLTA+x)6dHzF=y?~U%kX!Wi0vF&y#&syrSk1xK*vL)e}xTC&! z6yKOMNiBZp6>8#@*9z;_d!I{4Zx<{|#9*rUhoPrUPpJ6Uu*p@a=r@t+Un>e;i`$yo zXYh%+I5kQ0!aH)f#)EHc7FaD@lC@%4iEaYPjqCIdlZ`DEZ6yO~8H71gg?zX8CuNya zMn6_bQwpP1_31)Dk^#!kwU^W1o|xPTuq3SA5Asbd3w#2FAu@1e%qK z-Ad#gHzCpGXNsY9p)BUNy7WumT9I^ZtCfIo%P&qF%rh;iI`=z$-IFLs1t^`z;5T+n z>=fM`R>AVZN*s9W<^i+u6(#EpvWJ44{*-v!St9Z|kjc68`AeUgm%iG99BTmdIa8&V z>+phV)AWxQ2v1P94`_}_RG@BFORd2w`%8euEizk19`P}2rbW878-5B;QIJ^JsXZOJ zfr!P&lPeq(LF;>E;KlA&G z|8J{^$6@4i=Z~W&cE0mmHcEv9FTPJ7eRdzf&b2_CZ@J&#!TF#BZ#WDil>Xv1>)Wq8xwUttTPLS-#5v|cv-r21xpgvkf4wXeS` z36^FTRaaeA*{eo~f7og@nEo;2AEQANmpHoT&X=jQaM zu2y8ua(eoGIp_>(pZV)aiRiGY1LUCd^Y5np3w+1I&YD$uRg+~s%b~SOXmO!Ddi>L< zEyN|=sIZG-c)wo<0NGq#-txH|=023)>E=Ottf;d2x0(}Tbe%q`pgoQ@Alxz07KFd| z@(R@Y>&-iP=9ZvgU*7`fXoJZt*4@;{dJue6V=e4h!&2+3S>EejdbKv22*a*^# zAAJObpO>3YOfRugn!Tuh>Y3#VrS(ATA(-CN*r6N}u%58mB8w83{tZF5FBg<$zEmGV zCnYQ7Z@%8qCnDCqesMnXoWm=JuEC^M;eHbU(w7zYU8u#J)mixXA&g#V7S%+8CQV;F z7T8yBb?a6Jfj=RI&#pRozJSvWIViacE;LhufZx<`+UC>iQH3Muj8m38rJ;Kk=g(X112i zJGpm*Wuk0>BJC)r`aWxd%cT}(`L9R;?$0(?qVN&9&FlSIiKJw!B0TLnV+nDE#0Eba z^E_Hg4rPSf)~t3%7L5MTZOIJRWKIc>^L|Dy!kczcq=}ldGC)hIcBV#^}ehT@1ir}S0KPQEfB(9Bjws|$xm2f$(vp~9Ye!(P)T{hPHT)6f14 zI)gQQzT$uMZ<`K5Gdbu5#yKLR#xPFW_;2z2FbClFgAdJ?2i2`1d92I8fPrW`k9gWLZ z?mzii?#}XtRuGjmc8B z=V%4l$yS^nCi)ufGHa!EPkYzDkMbc$_vB+Bw>dz^j?N7m0=#m3@l^YP`wv(`wjak}dY;3_9N@Kwi5Ynze@es}O?FTzy zi|I+{fmT$GB-LFJS)1mYPsIU_YCjqsf+zMc3Oj@q-M_tMxl-k$wO2)?E_j<#4CclR z7m_xLCyYgQj-g-o_89ymz9?cWcxlV+P2v`ktuo$t*>iW9*Yv90xq%<}&=qZeaWu`c z!7!&48?Ly*8fRH+B?bA!d1%x-Q1-Sh`6*0dJtt&&lk?;?jikSmnIjDN z{IjR&mw%f?JRVYj(R}Se&Dc-}e}gZo#k!3pQ~52zV0=qHkyY)@%*)KIo{Bnd>6GGx zQ$pMiAm$^|-Qm7YIolIcO29Js_3$vi=B*Gk5nfspb@Zc2k%={ma|_T3lUtf!X^J_u zlOPCdG1Q;Rl;0k6Ys@b&XH%34oHE*NakWtPd*t7sUK8WaOxuuQM6ghrEK{Q5!Hy$L z0aFeQv#5zqD|3LoehZ@~&*mDBp(h@nyVvrzv`^w|mNJ%gv=({sb_Rb?2j&&6G3#n% zqvR9zOlydbU+&AKR!Pq*G!nePn%o_FHNMj+wBfCCu75YUAi#A6 zD@^1dX!VzpUabxXu0F9JdENdYdjK$CwaC|Y$PUjntfR)Xz*H#cTfkKq-MIW*Wg(Xyah{lbq{28M>ntKnX8;VYzK4-H) zsl6$<*U&$L4mqbE&Aq(4e3~LD6wWs`VfJCXH>3SX;~`rrfqgOBMsMzZ@cLdB4>@)V zq`GhKFA%=B22PgB&QF3gskX$P`r>=0#zkvis-9A>ZzetaWbxC0fp|B8r87BevqUPp zcFd{xsf0PLjKgmp^cqHif?sR(gphf4{EQtePXuwh?=GjLCUngFIOP=hDO2M@vjY8`VT%%53r;WYtjO}$-EMf0+M2;Hk(K9Pq)~iCQ ztr`VeIzy^-f8s(6j#-LmqC5+wBLxY2%m{;`NdD5-HQ zcPx6PT=aGW#hLPHI9DhwsWs&0A{evzqEG|Ot)H@tbWrp$vH5Pz9LbnZ1E*sj6^!ZXXN?`fp!DnfG?=Yj zcq!_t>LM6;1sx-=)*bbFUp21Ql?BnAwnm(O3+s5|&+-HLM@S_L?WIMcnoTMRI=Q|i zK5bopHR$pf{@?*7&{I*d)#h4}aAPpLitl>lUjStWD~W<0y_Sw+u9H)q^CLEZY~ww0 z@Po9es^MCps;1Z^ef-jViO0S+1M1-`@PW+;MqvjLJ)jJ(bEV)c_2qdlhW1nbyWA#8 zt!&HRU`soH=5qPltrmaq$RbwTA{stb8ysC~a~GRI-fUr)lG9xyIxfT{bA#;>aZM$4 zNSL!rhaALP^h;!Fq9?6C@`zlFT)oqM62s6m#hl%S3RwvbOcajXt?(?!?R&BcpX z<>9RyOtMQSPn<8dMMF^EWQv72a&k$gSP$E+f&39%!Xk1T%b=2ArC#z^scTa=u;`b7 zqSLYGptzTxuQ7jW>%OlT8xI53j`FOYV0s?A3cWW2uB-i8yx7exba0Zu$TkJGUVV%d zd=6I}#!nd)V9~-~rgUUlC_e&|w#l?UE|x2~9V$JclLWctkxXL=02kh`B%Qa5UQ zM@s#|g0CC;gGUTLc%W|~y8qNnykjYo%0f?DiEa7YRLs_aZ9)Iz?Tge2+f~*I*jxs< z+goz1uaZW`Paj=#IQPMjY(7w2(#m zNb;q%T7C%Vpu@Z&5Qx4(-62eubZv@T6lK@*(rHu%d-wHu#Zkv zaMgN4^6>RHu3PJTIe1X^$@OU|!BqDq&!2AgqAQB*nKIfZhYHvle$!JikJqPm(3Udz ztZXEv@-nLZ1FTp)$umW;0p;OKKID=VsTiL-x2mMGw4!yrmo&J(O%Nlcf~G_mW(Cb| z?%c@2`^*}3bRZJ9Zv2go5y9$+-0hg{%{*Q2zK?2NIn~fRmm5o9qq?0PEZc!<@KJgo z%3Y8J%2%S_2U2m-sn}rC71!dF>8zrj4bV3t60G9)>AG-LA8uM*EAm&LeEr(1E^X|h zXvlOM*V_SQi0l~bJfw7r%j0>)#(~2qB`q8oEa<@tQ(&{iwR0oL6U@^wSLGYl_&Aff zZ`_yv%7Kt5Y&Zd&jXGc%POZ62LNDCwh15@7lL(*gXYh`p!y);B&R&6Ytj%)=4@>q> zs9WQg1Pe-Bsb<48Svwd3DjV3;s()(p`^hY}GiF$<-9P$ij0(uz4n_`tTakU*@(hrB znmWtV^t3xWy@@5jWApH-Yj-tP2Wj*P__r+*v2s~-eRsj*^}G2ChILPq$B`9;Vw$Xm=vz{#@t%bw)jl+KY<}DSW z(^!9!S&>twvrLsDW=e)*j5KNDzW~VMWs9@iDW z2uA7kl3Q=*Pf;=~J)?@Rh^v~HdFPNTx^KnYVftwwv5BVqT8cj{{mg!eICY+W_+7bI zHFDNZv@48seO|6T#rQcj6~e$P!28_OGQtqy9*NBf=WQ;uDgsIBsd)G*?dZG3@3Kc4 zEIERQrY;cX$4+z*5%oUvSk(9H0}(@^=xBU4{1*>7g=<}hnZcSNzj*8MX$`+B{r&rS zTIOM}xzRW^cxGJV{&f^W22(J3B}eE9!UTSD++`>qYPw|*iHP?qDfKStpWZRDa<7E>LK1 zMMdlA0#YJ@NP6==_E{?MpbyrCjVMzkr%;U;@rGi%liIaXo#Z(m6SVilty|mSCO+XD zs$UOBx~o6DhjL$GuX(*V_MoXAHljx|pY0^C-}G5yFb1}-I~${9u&5I^XF(E&U8)VV zIEev9oCcBvL_SKVxA!0pT$lNi4g08}Ahi;{oM1L!L(uH4I^gryF+yp%t2j$_BM4S4 zSo841NJuonqT-;#NvBvrwIg!zQ*W3ZswB`L4^<+-{<@?LeR%5HIlo(0BU_5F9y$--HUvso60O#R@O)j z*U;rhR*{Tk&0W<^jU}hXV&?_^?+%k5bnzS(=Ud`Rfdu>k!tRp(-F=2lZro(lUd7%d zP*Orir4I6SdCtxm^E!4%8GkGcNJfT5)_4PJlj@nCI2q%zd~tYOZjD-McF?P%-}}J} zTYi3c@1}nk72{NV$h*Wsmh+H#<9k6+(unQIJYn9>{oYPcAXsz_ zMSq^1+7S?m2i+6Mu+@#Q8AQX5-asj7nWt_#MJ;IJeFSmTFIROqMqXYF$n>5e&T-Mu zGg_08I+W`ebX=7QDYTOxCt1`dazH%E^E>W*%qhn?>9p)G&mmloCGuAUx5%E*Yh$Kd z(`wo>o^9pq*?!2S!HX?46>KJv7{$gDPVJRTV!PAcy0TJU?)t;Ta*aIvghiH%oY~gT zTi3JyvhKgQV;Pb*D~kWqMsR0L=mvNG0mgf2s*n2CIr_Wa)TY3FIfj=`x!|X3Z_OL> za^COjTWcag25-Gxc578-rxAXp8GbfX_wN(N-FjA6tb-FI;dFO4YUO zs_K!#(2g>;k8e{cNl@e`f6EQ(61jE*8PkHKlB^j1B5I^Xt$##as;t|Z$>Sbr>bI2G zS;UF5g5@AH^|;g)fNzpSfloVbrtD3`tE{U-ov*X&3N6CojR5Otm0j%Vn0s`xMM|qn zY(AZWKljb0UDos#8B@S=C$z}ujk8xn>B)SN;v*jymdskjp zS?Pk6pxIjSmb0DtJ%`M~yjRLP@vv3I$zMv~IMdY4*_?W3B`IDf{rpQ-vL!#;FB!YqyNBbn`r{q_J&rOhgX8q1_1^^96UGBQU6mN+blMo1rA{&9^^ZRc)MQ#lqq;VI zeWC$yS^~`yce9lCe1}eoo67EMZp~c`^rT~n|4oxqd6v)Uvk127Emp?iQ3tiej)tlZd7zIO0)_Uh-R38+H4}I&=>}&Z;^(>IK}pxcKWSvicN*EI-|~jh#8qdn0>pQQz^_8gETKOrq3G z4reQaBBUNx)a4)7W^y)Wh=4fT_EgB0GP5Gb7HC^}pekxd6Nx@wug!Z7s`>>mGDt6M zv8ztMYO#dFZHA5AkBQ0truOW^*otJcmeOs;d1MdkJug`Sa-6=UQm6O_T0I#8Y1UCam(r=m+p2#S7L<y5p_loZInme zaF|^X4|r;=Y5Xz$8X9b(WD^_oEm{IJJDK$K7HHj6bBZrqYRVqoU*rhdQ8ue3j6U-YxgX;z*i z<-PYbg!U|A&I&JH(gh7ay`gq0zO`8xz4nn|b3~Tc>GsDiL++w7FB*p8wXVFyZvZmiEk&44V+GQ^#nhaEz-}0B(GC57ZR@ zroro9fHZ#?AN+5U!1~Y~C8PQEf|s`@W`B~EZr`63PXVLMGiWA)i3z$^C(mBMo&nbL z3ago|kmA{HS?nmQ_qE=o#xIjv^G(nv;Z<%DdGB#vy5JmB#@AuMTVCtzfmohl!oL7^ zo=OILd``PIDYdkU-drP6{rKhH&$&_HBJdKv3@ATJv{PNwg=H}KwnXWSv2d}&HD$@KR}MMU5U`lUoo?OD4{?|zt8*gW&s|BW9o0LpL`?eWE;pB( z)s{`PW33m!VvkVKZm^k0KACsDB%rHw8fI@;-p5!|NyBQiFiY}QYF?s%C zZ{B-}JIl9??myZiycW_XNs`1fk+mPkorjz2!cW!gejm^l^0kewq?hudpVkS=h^!z7fsR+Hjw>8wi0B#ON?32ljaZge` zZnOBTcr&cc(k2JZ=tGZtGz))@gkcdyT0q7%EinXjzav4*bm1;%P8d?ysoRh>Z%IjB+T+BRVs*nMZ*6#opCNmfO zJ$W}3LEvm=j!--k!_k8%kh@E2SWT^cwl2+-)i*~pshd6aO_XetWeTn|_%iII)M*t2 z+JNDlJ)+_^Ob&gucO;d8=&>?W4_a(Ue%C}6V!;4hK`mDABmZTiAZC0ZHt`tb{844 zhTFq~_{C1c{A6!72KY%^zXv?6d(=HcEr5*2m$4!)xSLf-by?8Be(C4sM2jo|AI3-* zOa5#O#$WNOrKXhj;@w%BJrfqZV)$|-_qcjt{{gqQgo^uGjs6;M@|LkuG+6_!o&H^H%1^(TvtKH+WMtB0lOlq9w5 z=W=FQgZ(nj5~=O~{?5`jPaCL7>wrX3ONBlsyK_Q^b%!bMr+6?d~jE4Um*5-yHQb}g&)MS;tZ4wMAQ z2W$^-;3m&cJ2us`SDJytgrKj#jfL2zG41H+`WP2CY6J9SAMY?vI*khUat`^mg=|3H z4mmr*oA<&E`A!?Lw^LrT@W;Y_A69JFBGWPJioZ)*fq{Xe^QHEAib`UtG_^TbPaadn}D@PRoxT!=SL$FN)qs311> z7(e`i=2^$hgEX*tjOw4mMQn6S-$)JsCCV znpQ}>q`qQ&W>HMk)QdNT?NbUdm?oL&Ny?p0#(U5b8duet!1FR#fCll)b}@sTjly4m zrH7RQ`%EO(&&f~N`mhbN;p_>=eof%rgk4sJH0$RKvhq-(1W4*(T#Wn4h5D)Di6tm= zYa?$=oT#$!g|=PcyJ|FV2Ma4shj z=j%IjPnc}k!c)VAXV013R3fbMaMFnc%QTm9ggVHz@07 z@|Hh`K`2rcs5Y^?D#$2bW6@i1)hLkESgje#9ScSNRUDka)&7t}*kqtd2F6KZ1i|Fe~oR4i_E9u!t zS00S9U~pJwIdpyaP?uk%(-+C{)0rqD8q0Dn)f%Np2n3A_3u{bFpz_S(d+CgWd4_)){MJim3p1}O0L#oQ0KmLdquuwhnC&!Dqe}OFTm$0O6ZQX z>W!cITXlsA>F-VWmU1}~zr`j~P^WiF|9HPc=kcg8vZ0{Lyl@9ofW%~cb7x!RlV925 zN4jF1%9cnL7Ec@cqct+v^t7^Ig@{*VADLZ8tihIZ1+gXi^lhi14yizQpe2H^&5^=Ya#=gwgLcfvL3 z?$%RhXs>WF;~^)REAU$t>7S)u_MjNhq|}Zoz8$$Zt|)3&EHJegy##4NMHb7*d()9KRCtSBFOG_czJVIS$&SGTT;tu%ayso2SX*2NKJ{o3#zZX5IGv$sKfS)lhcV_)w3OI`b&Vf0TdSiu})fugF z1|1y!beS=xLOwAXa)ik}&jmyqP<@Am(Lq`Fe7^Qa1ks zmOyF0iy}{8Q&x45oZI%r>}XqyONK&Xt?miMBkP2^Mf9JOKpdx9*ez84yF$fq2Tfy} zRWLq3N~lC9p0q=sSw4w`;8; zV99`wA|vtl?DxledT>er`|2F(wyo`RuD!{It8(#cwwS!&LwfDwi~y8#%z`C01OA6J zTHE3_{QR7rFFwE89Ek)F@)Z$dQY7O^(=Fq}jFv|Ac|nmguH8j*4ptK2XzMt=M3K(z zJCxT~k*kG*aJNxRk<`ap20+HHqBRLy+_dDn zwqeTxr0?6Nw6W(!w<}S6YI9<~AmZ;}AzZv!n;syDom8mdDYe!XKqAJ^8IKGv>?fn| zjIrPAGP|{Bv9Axz)qzn+9Hf-H8p!p9Qzi`ZwWcgwQ2)PiVLLoTPz$KjHbW&Xt4o}cDprzwB za=L~{$0jX=#RRuVIHZCyq+>osrNV^tfVS0LX;#qs&{*R_mt7~aZXs8W%gZJy3RDVny;Jb&`hxc?7Oo$ zwZ-R6jyc%cF&eNVBnSutTn_k)C(3oXm59{v5ILTuJp|ukD1(gX&8isex;1V|d9Q0S685(ehdeSC(V!{rXFsIY7mQD94ku{Z>Ac5t&w?vC)4h1x03 z;I854d9eO>@46ctkz{~dIHeF#IaLzZm4F#EUC2QK5bi9YS7#m7e6QJ6(`=0s^*zSv zY@a62Oj{z|@CX_pJ(lNDjK#FrA&Ho14x(~nIUuBrM2bBVkehAaF&0c#zquO@DsV|u zPR74ciWH$hM;tkf+Pvw?i&>GUTsgO8XegBHM`=E=b@= z6P?k@hi16WuT7R2MsW&_y10@A(XM=L)dI3aLTFg9jnQ0gN7~_v?JR|9IBlHK)m+`t zBka%YkyvrV>02;ietA6?#_rn>;8C^DRBa0)QSFRULu)t_-wo3Ld zKdmNapLdDLM+l~~2^I@RxlLdabx(zam|DWj`RV7JrWr?M4P zawU7%QcT}#l!%*euEXt@K~7Rr_Kkg)Tg@QynIbK<#%ov0=<>kChQiWHay<>E_Gk`* zqsRh<$ytMR2x|#T}^!i85Tu=pC8;$kGxf|Vul(gopDG{6-<3}G| zw5gG@+UZz*G*cCl5N|XHB^V+TjJ*RZhD<2!Ly65R(GjxNsP9#OLUQLua!p zR#nLnq=|69cTtygyuhM4xu-Evl};gH)$Ukwu!|C1Eu4g`f~nFz+2Y>Msq| z#4ZZo(=vvf6w*Cs>>aWlQtM~bq9T8ClzJFr-}!*I0RFgzRBhMF6sC$ zb08BQ-IMkTo99j2%P4}VyXQY`z&|takakT z0FJA?0yz{fN9Z6AEGRFg{Ig`dpk!{517!3H9@hPrx7oV~X65*ujdu}9G^f?9YBTX?p^6VYp1nybP>kBSP%)M zPC#l=Ruam$k>K5z#@96JEE2LK9H9HB7nIfRWeM$xc$E^_dU4o%iP!YjgIYDIHMUm* zB8gvObTUlGA~3VM>BRBkz)oQ15t9g3%zLv;(jdnQdsaCgbQG{0b|=J0*x1JjDk@nY zQUP(Zp%E1JP0Vk?f+|XyB7=7{%mp{<-!81%t^zQ63VzIf)TEM+5ZckNVG5o){{Uuk zeH=029b>hqGn95gK`v0`TGGyCrJl-SO!uHCJfYAjJqu@Um?jOI*Q zQeo;?zLd+6hCikFJm!VGR+V}Lm(xCL;gZsPZOUD;L1Ao0X4)yWd50XtA`!W@)xHy^cN0U#a_+93Kx`Isb$v2~s-4(ejohG+@y_>^ z7gb{;C9To~pkp#dLkgE`aBs4$E1dcbM>B#HJ-0zdSs^ZnQm7D|q*h%XMNe&3B3M1! z^Y6d&*Q=^JbvCT2I=|63n3mkVsg`4G_3|j^1^WB;ZaH&0h1Ji660z9DJqX!>FE%HpwwbT_cCU43?E905u((bJop z9@`(YI5)RxZX|00MK#iqr_o3zJSpihv4plD@uI`Hdnc6pw$nw%{j>~pcDPYDw^h&= znqaAD^uR0X_HOMTJCM|kL}XFblVK3`w!q8BgsUp_iUzlIqlX)`JRu=-=m@w0Q#o%% zBaT@Yb2pZ|+kU5&hFB_BmeH;_WjLlz-M}69^NkL>?xB0mGW!Q@PRMOf5Q=5j47@e2 zL@Y-3j?E?wT(G0^%xRPr z6ZXP6hrGsBYo5_8aFr)!zcu`wHGreCs+30vGzi-3piDbvU;NS4K*%jxmq{h}T`41?1L#KJ-^{Tbz%D%~}q{Dv& zAfuFDB$Jv@N{PugA)wf%d&)Zu^miov9QnU4aEcw~r-f;HH+^pKyRg>t3sbwo>5MYIBbI5t0o zausqRIMa3D{jNjLD+sZ9Orx{=Ynld7XJ(wL}J{ISnr&BxlpQ#jsO6lp!U zYHOIpxi`TXV#gWms@kGUfeg=&L5)qk$+OTd4RfOl{CSU)wxH{67-t%mj3{t zW_Li6zLoNkH&Yb$*i5fx?^5#H=k1Vu6*@aX%Lv*#68)xWY!qj3V~5&|wve8LI$IZv zv8|-y0kJ@2?C8c|PXUS4QV4^tM*ZoH79lyBH2nlPU0Cpe1=x1ge=X16^JQ%~hs}Yw z2BHEtWpuld6oA~>8-zv0i!vrEs7*5Yk@RPU?4ZL}+2+czu-Wi&$bDaS#iPis?@BuM zDD6yWrB$ZMqpEVg*9qCVsaXpPViXRQmIko82IfW3T?x>qP-N#WN=?DiB_ccGNhH^- zh}$`YCOU3vsir&HK)NLUR|DzXYxHLV;!v|WbKYVMo!GG(xK{z3Z!)TpBNGl@TdEZNAjKsz5@L`dWw{gEMG4J{ z>E+`0OLgJWTyb&dwYj^r+c_s{s;?|^M2ymk^IpbSW=EMZw!B!xYLUgHvKVCku`b#W zp*#akG{*pNj#-$2tBr5bmnBX$21g!pD-hRJjLpeRb#u!pX31jiIvsTzHrp3aXe*Y; z;eA!)B@Q{U3!J_oeie@ZhZx^A>$O*StoqcS!H}}US?oa8Y5ECY{4k(QbxG0 zh~*r$gEt=1#NtDFelGYyx#&G5bG9r0(wXc|lyJlaQGfZk=3Ss)H2B z?mn$2fH}TE8N5Em3zLHj!Y2Hy)fIH;obu$|y!aEc*d1YqS zzEMe2+o$eD5ZfKr$+caT0%GE*lw+k6nJ~$Y*yPix!6|@En%>Ik2Ub}zl-&gR6hk<7 zYp%(;686dR>!QfFk#ufJI}i!nf6(b`GeC3CT5d-3+m=Edt%P-yxi*+(hL-(IHu)*w zjOn#eL9n6kL3G@5a~eXLwiunE36xGj!5|2S4zgX^HWmw2&t#PDOmYWjF7V4BMIGY_ z5eyZwyk3;o1FJeROKrIrF*-N$9`9m7EaRxMD)JVBEg*ZKlV}}j&U$r`%oHvl?7}r= ze~(B){{SnhCT82{v-!!$M_XX+GH&XNNN3?PezNiWV>_L7TOrZe?m}x&+HRH#qqAGY zv(nmbsKiuGFJ$vsJ7$BU@jGp_(hIY$n!_qy*%;1?1mYrvshQ!0s3ejvHV;X3(^|Q? z*G(f4ss%BGwRW69tmgN5x4SXpbzD@FXRT@<8@$sJL$Kj0(kbNgAQA0RNb%fHYIkZV zkGCs13AWs*EvqClXjIe>RTsvgk|WmK!_`pcw*vCf8cH5datGK|KIHaKeot66@hJOK z85**A2y2Dt!V(y~sJ@!QF!pN>x@*!7AFff54U<&`Z@s3A$|1fxlTXQ~XW7@Bo2@Z8 z)n=d*7QpRg-Y|;Iu6Ul?Qd1XYvZ)dmV2zMOAal0nqb`HZnCBAkh)P!LAYMELb}Kj0 zB2jXf%L(X&P%YzC5Ux)Qu(zueZ{+pxf-QQW>(u~j^AeikI&nZmf})cC85ogQa$W#UUf$B(9N7+cy8NICj`FP zFGP6)lFJ?O)@Q)nDBBau8Z%oq0b#^8Q}ED^sQC^#nIevC>B-*oCr3voUPm4xnJqsO z=K)(%Nf{ku5ab=IX$X!cvMOP+=pK?Hx;jRmQCvNuL~B1Tb=N*lhX$8J9i#$nw~>ru zIzO3i$a3Z5`DoID!5Dba)P&WV41S5i=vKV0F0rLX`6`442^tN)&S{$iBootNkOtSq z#GZ!sQ_VLmPcj>7Bnu)sdBz>vqq-c`HqkV7eFJ98--+n$W0NEPZ6RjsOnF0 z1PY3=T<@YIq^>hMVNCOLo7_2~E=5H21`lqwMH`eOAu&`Gy#}I41SL3z^&m*Dux3@} z#(30(WslzWtJ~`Z4(T*Xq#sHv4SiFXL5T|)4Bf+12&1eP)(wz1q3)50#ukPw{} zOwcP2Bl6_QMHbrUb6fgRo$KpAAV8M#w^{!HDw4GJ!5k*^cAg~%$}5bmlxQvpM`Y^d zq89$GDxM6uk3fAHTT2=u>#sNa3BvKd^g91Pa8Z=n} zHkiUE#zRYW>~%*ib_IN^<_k2hGWs4zY;?%R9#0 z5g&_>l=M0`B^blxvLB;XP-gN7T$(YRo!S8bdJ70E0>Nap!Uu*@Xd{bWD}c@JR{;pilRnkK$RXkQ%rbcqlPbBUwt=pI7^^< zRD4!(BD{wg&$WzmtV;7XKv9v{2rq1Ak%<_NjtN}cQTi~XK8y1`hgUw!hgV?=tFqBm z6{fWZV)mx9+{Lj&bGUihqH;hM9dT751Z#}2Ylj_CTM0!r=_#}_fr1l;Oye1f^dn9& zrZnagSk;1}n$2o#*y~Cd_SDv``7bLNTV~PY}|GPH)Sr(yuE+?;1YRz23s9pEW9M-Z(cIE(Y)f`NgP4d6yq=_OOahChRiQmc=5Lw6<&eX6or0dqr!Xvf3vMm_uEF zx+XvoOSstD8F9ATMS59F5uU=Tfwisimj;CrE124x6}eSd@FpDDk>NgNq?17x-hRBGe4Z2+C^J6bSIc?@DG69{pRAsI0W|=YTCy+{+21gN<$;NIXTw}B~ z9xcCaLR8L%LR;`;pr*J*xuKut_~kQIMqZIgqq3;7R}%+_p3k09P+7S^6UnzmIj%8e zmXjQRXLq%w2Nt%go^{H!L^QrZH8~3^w6z!w`cPYNy)e%g5OMea_fY=Df;h>R9Ty-`qSC@VBD#! zNZdVXtxQ2IR?1XlG_NAdic}(O65>WGn+xd9&V>UMnqyB!d8vY|XKo4Y(caM7_Qt5p z7FR(yw{DQgG{z$)g&?#BQZYn_NZ{z2v^lmsGw$!=e$>s~xAWco&akA@td9(q)Y53| zf-w$8n&;1FvWJ(q1fe^m+Gw0|xsu95NL#B4%Axap=}^sxbVAE$ksd!auAURES)DNb z235VRb>=je91p)H_`u4A=#K91CaPV@sjl3bC%==XA~Z+_^fcIJzhlK2I$VgyN0J_- zj@N62!BmgKZWsi(fXHDEhYvE85jWL?9fQsx55Q6ezJ!j|BtHH~PiMkP>T&J~M?wil)3{q0De z&T-4Sn~aIJzEu@TY)q%3FR|ytu|mb!?NC*j`@Hsw(uKX=)q3hFcH2#UxA8|Ek;HlH zBC_=s-c>QGY?Rz@RZlBXRjIOcNHOasx(?mE1D&&BHIhomYD6ctOmIItBZ$x6C}Jsb zkx~yjYTl?6hLxIt_gG}65{*f~M*%Bvnb=M#z%wHgeet|wG(u(2#cBbQ=ZfmMi@%rIEqN6|Jrx(+B^Q;H!^mez zl<;FYI%uDAvPg+Se2WRB85{Y%WKB_14{Qmh$gZYC=e5yN{;Zs2C$+$!Oe;Zg)HjTb zk}oMu%aW<>MRrA%N?|g1Gc`~KAvo3`6p|E4MRJ8Thw>_W>RVKF@~d8hdEqycaBXl- z)Kdgb9N-J?JvFda;ayQjV|^4(#`@(sZGEEklNYk8%Rn8C8zZldO~CEVhRM?R6~~96 z-(dJgit1sEXOkAH>B&I-PigJ6Lb_IVPkNgr&u`MccH(X51xsu#RV_`n-UPA8#v>6V z7uGwH*Kwmi3HwOy*IgACA~`+-+sAQq1DGL`ST8ipqDZ)9_g*nHGj9Z&tDxW`1V&31 zD9|7y6nB45^6yRLv`yTR)*N`z$3$2gD#AcmNXOe$^yDQia^xgkJnnEYo!l-}L-b= zi>DXn%sZb1TLUA}v9v@W=p7 zi`}lO4mE6aY&|(v^Tb{mM2KfSim+u&``hy8-tu;Rt~<6^V;q)jMkutgpR#U*8m^%A zB%mBdp8EH}%|1Nqe-y!nE~?#au<=oHh=>bWJk&(>WTnX+TB{O*L0EFOPh7z^OTx8t zchx0gAKuew+?r@99Et6eqmxt!Og*N%U@D>;BOT+&gn5d3$&LCnNc)tN;9NGwX%NdH zuuM}y$ipOMn6~)=YJZGM#@>UxDw{JOqYsR`8;eUEX5(6{E0q@rMKb8Co=jmqU&`sa zE6K~@a`CE4hOi7k3XJ5{t;ZzjgH+kI%!h*N%y2PW6%@tn=<3&^Xu69=<-}SnE}+#~#e$GHbd@2%7AkjqJC%w~#`6M9w8~rd6&EQF$w08N>W_ zTTpA9-N9dU4R;2)IUv*@M+Yii43;S6>Ez^()OAhHO$R2%bc}-j%z0FPU6?i!kTXw%6T#? zWo@|oSIb)(W;6HGaO0}}AHXK)c^K~5a7Yp=GQ)tLNoLV3?l&B8i7qS|fR-VW7RP9Z zM+<2jfr`GU(b?Wu80cukw@3(6SSOW$F>#BLeU5JX z470d|wAEKhBt`wGX2FyLsD}7d2=Uw%T2pbyEM5<7LJBA(iG|usYtC#^RTk7^)YcgA z=S*b5P;Jvz?W(D@eeJW)@=5RAsR(L{{5m@Xt7?|)em#v<8<25rICR%UND_2GRq6B{ z3j`+S9OKuuxpHkS$TErV$VR1V{vb`s?$|^eUE1s`o0+CkS*}kynAat$BqwX^7@evb z5zFswhWC@oE$wvQ$-xX*>&}Dw&fs+?!BU)>UEZQ?jd{DFomaN^vi{v{Dp3%owjo<2 zv{n!4FC1L4(Dsi!H5_8LdW#}W4uqr;R+{vLvOR5!h~jNmU`v}$&?Xju>qU@Y*xnh` zR5EL1s+i?Nk+-rOXD*hyg;m#Zqr9;-R12z#iY{1vHw%(MVOh- z=!_%N;oy~)Uk;J{FTiy_l)DcOgdydjKau-(e{grE4N(Uk%gL?YMBX-%< zdkSri?4rI^8{U}RT-OC#N&!pQeYsII1s%>><8hEySpZH-P+4;<*mkPSuNAgeWg}Y7 z+Sfm5l$eTZ?boxg>i1&_>WZc@hORf)MD5<==C#DEKZ1O1sy*9?x^HWGYYiekH^vn< zF~3CWv)J{-J(odV;))0&p@uj6xV$Wpgt`m)u3TIYHxx44r)~|AHj}rVtyZ3UmSv^wjA7RoIAEF*&{rnV&mxytSs*=?(vnojChGx$Od>gX z6-G@r*T|dbm%F#ekyR&A)@!%IeooVUJ;?c}D+Ifvt{P0?E!DniaS($a8BC(nY~fO_ z)84gW08?-jV;-g8#l-ZM|OJ>y2O%lE$J`KLlX;s`Q* z>emR(6eb?`H3-`a|Bdsc(<_(jpU&cqUuk@Qt>f^BSi9@mj_|yiohz^Lr)x-O20k z6{vgOb9P(aPDyWhkH)_8qDK43ho$c$)W^Ke(LZ>NRX=-|UA^Wz+xyAbPu}E3A~IotBQP^UQgV?LG-9E#1|)*P(G^2ebMQ4lW0In>;Uz@E@fK8r(nbH;00;pA z00BP&>dJLGZq=#QPlex9XjB`Ndb&C%jM2#Csp?d$l}c@tX*-&QJyYsoVJa@4U!n&q zm@aOKfjoAX{{ZIZIUb=19-zD{HLkUqvue|fP2waG!CPBJz*nbTl`-d24p>R>+^iPo z*=-GY(|GVy>N2G^xu!#ei>>xn+N_mnA9!WWbX6GQ#|R;3D!Ux*oIDf@g~3~MrDi?` z?W){cKFWUq{ED|{OJqNdm=EN(Ki-_rij_vIC$k7%QR)RiNeY{Av7>;x-&dou)ILh9 zlgdBt5$Xv|rzT1bL8vtVp#ZIx>Nas2yWv&qR<8U{Jc1-OVJA|cEyt4D8FWA8wNvOF zNwPODu(jjS)Kx6C*{eFvFy@~mTGOabvasorb()~@L=oyJ=!iIx86z+gYd9fMhrv2| zrWH|umCzHSb71BcYZqD+To|ylDQd;wjnURVrWO-*Ef()2-s+~YOUfD!N)uY2uT~H} zL2y8;1+N6qsZ_GhSHIbnTA5`--okTfA|x-_&7$D%vpYj2hlhp#0JIhm<_gUjwLE%~y~{#aDQ*-KpaC)+9p=mAuAy=M2bOtjdo1Gh^!)jr8*J<5u!dN)D7sG4O{ zekKEX0O}L0)XteFSza{F7Llmx&`(D|qqBH%36+NMvN{yhVLpp@WQ{is$`qcbVG`aK zXa45Y)*{e=MmMS{{>{pkg*PBj!kDpmEN0;=3c`#<(1AeQE*EMQmG9a#Iwy3sEiCR6 z7(-PTxs?IXwEkgOElsr9n*RW=JD$e5g{MNT_ESQ;*}&winb^quB@uAACug&p;OmZi zH|4GRg=ug8k!SoCx~=~Jm`W!FS}*wz(5pbhv~3J`J0g0~J{TRe5wfpyk>^q03J_(f2Y5@kI78P2pDlEkF z0b1?#tvL?OJ{)@1+ComkyDU>AYsk03{ywz9XRcea(M+MP5 zXje&5x`e_W3bVD%#L_u+p*qh@_$OJNyENtzsJfegMQ%fSj;m_8ENwms*rSSTgsSXr z35@O`W~ZoqTwVb}n&^TQg6P6^E(ksB$gDvZxb9sUR3cSxWPFu3!A7%74o5`_ns6A3 z?5s-?o3~#Hrr?3RDsOWh*SAsQ6XBAy7FA;{3>26jmqf=&ZXM-p3vWkTqjrwJZdm$C zJs=skUP{62h~}zy-Ok}~OwVUj@HaO-pImTz3bZhtm1_JpmOK7o3cV_w1tMBTEKpAV zO4Mf2LH_{aP-#uPqK)heVo?H}NAb$))i6?fxCLCxiB3nWg#8hzz!!27x=P>S5zT)3 zM|tx%iC7fK7SGaY4O%c=R?;^3|D?+*}>QC0U&tgdIW^N^PZ9$5GNC zrk1cSnTw{TnNl3=vMIGeI>)MY3TJY4nC>XC<4}g4pe#iP=7|o)2I>uKm_Z7)5X{BU zq@Y6G>M*X23QMAD=v%=W3S82KMxfxl)P(1q$oQMFDNGXRPo!GelrW*}%EX&hpafkf zt-GoX;R}MW+C?Y2o7_bjxm3a=`6n{9p1Es*X&L!$TQb-!7G{~uYjc_lNITOk=FeWE zH%?rYq(!2j{8tNtvu9(p4-2TD-K|6LwEk!eRy%Wl3lGq%XKP|?)9^LT`WQ#GVcE{$ z9M7U<vq|#g3BKfRSI%FMzoTuyA$w@6Fn6uPgI#M0R|A%;SJOT z>BM)Y>7p$g!os0Kg$fgh2sPCYf@L^Cs5J(l)sPaaTU}GLNW*?V=$T9fZXJ)~Lss>n zLbbH?7Eki-#8ibM;N!)1ejlRf+wmbpb5$D(ld)3=)*5U(ldRkdbvKROcWw?kC9*B%&1h`*9jXKo zBI|oQ0o)3=N7`z0>IGxYopJb3R$(Hz{{Vzlm`g1`<^F{}k^aU5;jLVM@>X+;WH>C(nr_bSh!h8hV@4l14|83h{E zovlKMl^0=9p+bwE-~web^CdQ#4L~5(3bkrfSuQH2N|n!SLw7g&rnLaJgMAtK(DzUW zsR%VvvzIFLyA%fob+MI=NV<6DskyBOBr082t{oMR4O*C8R5n43TzGli9;DM@)FWpb ziZt9c9cWOZ=v1<(6T7BG&1~eWP%;o@L89r-jtZSd5vs`rjhs=b;86~Dx{F9b%Msj* zoQsXnpsDLo@2BTU+>qX=oPw$%qm>VTlCPsamu~f7-Y=nD^lG-FecjU%IApC<#X7Rw zJsZX*Va>321+X@(#h#)05AwRW#YyAAZ96;Z)|2mP)C*?!-aiTo!|kJs92JCmO0DBX z>L+JZ@U=Am0L<81KRX4!pTmLw08EuME-0gLTOpHaXV&Ks0X!A5w@X>>;!}}PcpKCx zP@zJF3KkU#6evYK(3wjSlA6RCgH#|DrrO$P3p-Ezhc!|7PCBobf2|dr-6I2`2B6in zo9Ny11@Ln>!i3sHrcl!*vOi{PREm|(2K+n9v+SLm%I|kaiw|T2oGc+=SPIXpC1Y5E z$qC|&9YPHN$T7`RJ%J}R_PYHaMSx+)kIo|b9;C5{ax^#}sx;m=BBQAaE)|>R4~9C5seh%+U%i-nMb{{z&Ba4-%|jd8f@-^0u?b5K;S$q;Pd3`NG+w_uFMJj*U<1 zsCQe*3y)EuY)U7jbN2xsTeh`a7B~JZlhIJ2LWJSDLWK$xC{Uq7g%`chsnI0KL5MYB zL8#$qCB^_%t(7YFY4%(3Z#D%rt|GCZ;dUd%3eQB7sZ`XJjZakd2x-(KMZ1bLo;N7FCA~%4)1&~MTeF%K z8gwdAYf60@o1H5M6dv}b+6MFpI(6dcy9$D)>p!&!aJtLATK&~b!@_Xakrx1jDQd)| z^>G)1i^=BH~iH2ZJyuY+4?zIrk>rQA~EIl3XsE>0!PscjHpn6 zRNI=_jKgIfH*Lzfv79qbL?}?90lh+n9Y|XBN^_GbXC*bLF<@9#qR~!(&ojB9SE}ds z>lgz@?p5ute`T2kXW8y{Z^<%~TG#tt+ECFFWS7~BW77(xn zfT%v9nyBi6CxYMVB6SGUX5xzr)MK~$h)|(nS%po|_Ma3r^F(G=p|7qN3xcuQooA_wyuu=* zH4Yq=R&*7?rtD01YFN+Cjo0;1g27Gb2pz+JRz z^p=N|lvYP6BKkbqH|+Rgpmr|_fy0L^U8k!iL@0omT;qOj~s4Ul3zw-0&B z-TOmgp&^H)cZQ1Hw(VA+aXtsaG*7IbSd?B>dHX$|EDzk^SEL^5=>m|U zU<(h5?W|r45Kt3@b(^XhEM5vl|mVNiWe2J%LkP(mlVh{*K;AvU?*nOE_B zRw2ncjW=#ivb%9kVH$`{sZpu6hMjUk4vH^@7s7>wGhd>AecaWb-d3+>-gi?oOmZPs z@AJEVP<13DTA25ct`BvtK1yoGvyN(NcqWi~fdyizP4r(vtw7fIskr;D;mEC(v>#@x z_)e_H(`AL4flaD^K3sem1e+P@8eQ1!HC^)W?n1%H0*6?(l%B6ndHtxi?CcDpn^G z0aB$2twD%294!)6ds%i($SWteYTL~8enC?nkSjBLAz@g1C2WD{s(+Z}3KrHMyVU*_ zXKgcTq95Ef*^88`PTASEI(TW2=+UgvTeiKf)J`Wuei5Rm^!kXrlqoo-&`+c2=??{h zr-5Cet4hQWJ}VJXPvQD5jk}yy8k|8=;j2o<)sn&#C-qLKCg_g0;wa{sgbm$I==zd$ zP~HgB@l|NOu05W!YCXHtZ3B4vCe&lMq5)2`F!(1~o;+7+z6rLMZwZwc?G4e@g$j;s z3cGzP`KoSnPKv9l8hN*U)DLHoRmNkkFmH4x$vupRis;?(E`glY7O+3M5LH{*L@YUi zyD_cd1l3DWwnL`PcH6T9XKTI_`m7L93wpNq*m$u8Y1OBT0{;Nr(mZgjqRH8e9|k|k z>8`~>#QrvR{{YjHZ&jgenm&^k3%0}k8(Zd_jo@aTnd;A9Hy8??C{1E(5NfSmw3|Bk zD%OY8IimM)TRgbhn7?Ql=sc5spysTjvJh&+ky|KbU;NW_X11{X+ONtLPkWl9Q_(d( zx^+hsDLAHC(=5uJl&*qZ90HJoj}<&XD-Av$MSyO9)nR{uU7@Q=#1L|s)Jnk43kL@; zN#IdDMd1{8T~9>upI9tqV;49qDij{E(G4zksmx=wXil=XafItNLArT$n_8oHrrJbi z6KXZu6k6A6B|gsG$kK4!n_8oH%4Ma8T(5-+E~e_S*=ngc%#70qX+5(|;_5n{O1Q{% z;Q&_afHJywd?9qNDguNM3J|y^Esa_jF0|6=D80Nw+IDcq({T5j`GiH*oY_O=O;vMc zFO_@LPlHvW&T3ih1L5gah1j-bm+-!$f0}Pqi)Jz?PZM_3t%|$6GEPSGlX>-kluuLw zuUc1HvOtUcB}HL-RCN0 z#i9_PNlddUc3%Zf$f?=F=)hei7S(w!3Mu?QMS;H&SYNrouSh-Bj422c#ZMg6GvWFy zBjPF_;8eJ3!m&2ub3*f_)Kk4nbX7MjMPgl*3j)Ga8{2dGh7O`rtklk#Ct02}vI ze<``O-aJ=M2;i9X;KFoHvv4XZQ#oXrY9qW8YEyoO=uoH*!+h1CdP(9F2W%#HLM>F= z+X}GsW5D&{VZJVlp#Y%)Hv|(EmZ27B!r6B2o|HSj+qS)yX<{3l`lqrKDx>#mB?Rxg z(&kgUl)Tr!SbRm}>eaIT>u7JmTpy@}%@K1r^(pGCjl*ZAJQZi)r1x+NtBEB!8=54i z89h)W!gOdZg%>r2{F5oBu?DJ-r%AHnsX5ge9Day3Z6GoC0x06w$weIh)}IIodRp zlLv~GD-teL>1QQ1iL70gi6vSsh0}+!-YNCub~`Qj&+=61J^jgRPOdtYvswGIPzz}{ z_ir@{6_4*UzGb#Y?r!s36^Gdi>|cAe^B3(8yVUs#wD*rCVYh)|q8iCKMr8^X`%b1C z^a_T_pw98d`2^N6f*JtoGuu`2T@$#ds1_$Kkz)%GK&(X=I)JcK;IMu&u)l~a(tIn) zKSUIpi6s;@Lr+o{6+eWjVcxI>gsjx4YUtJnUdv5-4(Tbn9NiU<_TIRTnu}fH+?AL1 zvt?G+p{9QjRV=$6-pDf3WQpXGr0nXTy=ug1+x!qs+m@B@l@PpS*nVZ815kvAyuOs%G|h)x_p1*ehQzbCzg6E@M$fV+G`N-$JY`z8n?;$erZATb<8m)W z%5JAkF^Q~IDmI&CT5W$HlLB+x=XQjh)W`Ew?4{7^neZEe4HB#a(0Qm^M1Jads8Fb7 zS)Vf6qHM0Z!R&?hPrq9K0EYW-u&MGD(F&HTOeZH6kE*fk`z@ZELm8md=6dY*^?!wC zrQBCY<&1A?np_l9_?Jicg27NJs250AN*74gh;g~YE~AVpmVz9OrRtrgq%XA|3+-ou zpvy&0gVCQ#{46_((Nw1TbsonEb6i?Sh`e4AFr|lrz}1D{67BE{5Z(%g6@hk_lI*9q zvQvBHAo2pIYpY0t$CjT13d8$XwP_FyY-5AMv;Nvr%cY=4i%hB;e`DH2bj;7)eqXAz zY)y-5&!PaG>@cmLvJD=>&=28wR*i|YDEFwkIE=zHo;3;WDlT!1C&u?vVb!UWKz2uR zbtkuXQqQU6tv<#704Y{}>a#Xq%qwVfdMGj8ADX3D$7#G(Y$cC_?=*f&6ev)zqQI~a zfx1-PDpa6)fddO%$C}yh^s3X9cczuH_JL*!XGyx(WP7a`hrHNTXuXBt?_f7l(f` zr*CU553A6ENP#<(HjSd2>bZmh{LV@HQrhinyYx@0_jjC6GMfNmDbFr5dss?qGfZM@ zZrX*+vbn=hx?05dvvJMa)s}-;!2vPB&vSkiLBG7$@>U;adEIaOtCT2I{iAuUpeD@Z zRfE|Jx2r$ybx2wsj4^je!3pI)SRx`W>iJuDeUMm3H z<%FhFsA)O!A+C*c`dy8@RyOfOw5rtU>WEa1NyuupT@ru%VV@OR)jv5_$6~eGPa4&K z?mbYrBKFJ@y9Hni3%(`W;Ia4>=+6a!ZY4wbto{|*MY$)8HivV$+tW{ICldj5j#O>7 zie1W%Z^~g=ziD=rGo4P5aB{JJ*jWq;)^g7g+D*4?OLK6SQh$SjKeO8st~Lr$zq0KW zr}iDc+**8|Kccn%%k63!(8M^iHD1p4%UlRN4n;H^BKGQzsIjMVG`JL4;kjA;r;@Hc z1y7Y(H1k$(bgiP;!Td_k(ifJiYuSZ{aN}`5Op|L>ABi%xCmmq$Di#%mVOUXMD+_`L zsu0ht(m_^O=>ZDZBDqaeUk+*Dsk{*15q8jM=^4qvMoNySdtJ+0lM~ST{ zYL>>J`EaXl<6D@g@P_OBj#Ob3AB!pxC z5EP(xT=cFFEG(E3{UW+b^?TXeJE;F=LvG9cZ{!#gDb}1%9A(E~*etwALBW;xb?{%< zC#dMp)%$kLBjF!OWV*<`oUg-MEp|-S{{GAF0wPa#5_rIxC7@3@V|0Ke zj-X#`!y%G4x*ww+RK7Gy@BPTk>v4W{klj{)vgcXz+PE_apdnUOY5Se}WbkZ99x z#;w&u(5C+4twQh}ovGO!$qmo3?i27H4jK1i)bS^9=+Xous9rW8Bd^&IJP*3L7~>Qc zzbI^0vUf)J8k~H;#ISLp8gE9O5XxB;#pm}Di_n)WjiFh(wt5$k8mm1v84lj~z36nG z&2l}i z&Zt|6;;%wdixR)}h;to<=$SjmR3lBh0A%yjo#a8@Mzj;!p~7^VlsnHW zoT3)h`>flRWEn3|=)#nG-0)N*mti#%A$KrrHpOfynLf#qNa|i**!g3S0j4N+<<$;8 zGWw)_xu};S#FWrBe`=bOVpDi4tMPjF4}`*TVR&w-Pn6o<{?f4d+>#OUW_2B6(V>5y zeNpD4ZvJJdd>F^SD#UEaxNz4zKU`RN^}Oo-8kBir$IR=#dn5-3MW`}pjFayyUVegE zd9@$(Hb||Rj-dn9&>1lBv-ISz`YQ?f)R^SVrYaq!h25N~dv)bsy~0?D4IQQb_Ra*A zs0MJ1qbqIC`XXr0Ql|fB`|&9JvW9>(uR(5UgoM9@X`#JGNRo>@jfpEff#yweT%}}v zCk3g_X67MO4}V>(OF9Bi>=VCAO(yj8gMf{BV;5mwPv-VY+hWHebuDfr>+-(eDq%XH z8N-@(S)1ppn#pya3)%HHo86skofqFPBM(Gx{R*9Nkn8qypwIoyee zaxbmhJ;-08wdtOEEag4u>fa*26g|tBV9fE&N*524_-M^8D+Q=$XJPFjr+CsRS7a?M z@$DE!Gie@^pH;B{UXTZr4e5;8sz^Qp1u#{~162l|eO}8negQ?rUuwTA2-ym`fxvWx z8515RTW{qy39t8B$U}+QKz>zlFkpyHCl`_M|YA&FRrxg3rohIiikrFmF~ z#`gPC-+3(2whTs;`F&`)S1oD8(3NTEVmcO6GKMpIxBLyGN=O z5C4&*R?@H4rOutH^o&Kkb&csgn|TmuX^H&$MB@+;`?bn>t(})bO%)V?lw(kqH}#;o zyITsvy!Nm>kiLk3az6=N7>^0dL@i_&C!-?wa#06|O$dsetcPfJlHZczA)C3c?{z=x zfBC;4!z6x~8s7nadT6W?6{v+-DLE9P`$w{8nk6U8NAqn6Y$b6Fo|{vRz5TVeV39$( z`v!qoy%1Q#-Ct{e-GTz>k{j0bRm>wWlR$Y+nYc>nJ<*xcbo?{g0bb*OB#7+NqOr}X zs~FpMg3Ii~o|N1%Cx8V(%jO4z>b^bC$6)y4C5vmh&Omae(UTMu$;sYJb&(L^_;<-U4IW}EdjmLBf))W|{5u+&PZWDq$P^PLfX z3mHCQ&Y7IPiMH_s)BI)rVoPKEO~`)!25Uh-Op#6hL38vUk)qkkk1&I^uXDmGEF3Dr}I8wM8ujI)$l)2FSvb`uLn%L-k(33|#O z5=*dR5C=XKWRMv?C>!r;S=!L$-v~h#aK{f}dL5e;&d-2ea#q^xaiKL5A_*eCPClh3 zdmV+PVNukVf@w5`9=SirH6~G}_qwNOUS84);zx>ws9PS*93~F21_Q@r^dv1T9uL~c zR+$v)m;Lx(y2C#)?V$*wO5bytDf=eL`i~^HYr+ua@7*3-z5~*;Wr}dJ0T+b)7p|r` z{itZx9w1HbV8@D6vvDxzd6<1N+Hws~Wi4`Q| z)<&(e2PRc*7AWC~xg#R}yc{pQhufal{4SH?o$Lcrca-~JfR_yB&ULN``V(I`7&BMW zVfun=5%uD?Ev-th4dLb@*tV|Go|mfP#AO{tGj(3cUkg|Aag@e7pkWWb=z5u^fC_N&p<^ok$4N&8XX_eN`3EuuiOdMh^q&oKZn z=%YhrDoVGqWf#pU=N4O${v(+!|3~7EIwt7h{gadTwC#UAh-6NU60Lp^sMWku`j3RF ze+4R4-Mv!V-2dV%9ntOG5)3#0S_#g2vR)zES`@&?-sa!C3>LxIONGVPHDqftp}78| zv7G2u>0uaAZ$nNL8(<=%}(`y4s^QFm`+#IN2G9o z+hi)38<&vx!g|oZCK?ajbQC^)bb-(7H8Fi9Bv3oli+BZBkS*j)tuZkY#f@G}YhFI% z_@N$IXc?d|xty+7#8h|ivO}dvqhxBOUKRP>*yt6dS`Ls9CvnZtR023D&2O!J8Ko{3NuMul9K5^ZwOkD1m89WV~exD~t zL98!CbZw10KQMH*%kl^Ko5j>!z8v1k9drj@h8>-{u0rcB`z+S-d zcqI0i#<`F>^GylSXj`d|qzVx8VrL}U$j_$f8DG!bSuXA$$pY6Wpc+u3U}6Z&7@>{A zETmf9qM@0P8Or0oCQ>#nt``uJGf?p&n-4PAKjmVAqo<_VmG5%wF}75ID{+_P{xX*j zu@w}gEBX38foo!s>9TCdttRuVISU7qO#5J}((5QU9!Y|n+gw^KXip&-(QTMNaDEWv z!fR0R6meh|0Ttts1P|pXFSpwFdtWiOuNrqy(ZWyci8XyxUxg1~{?NQ8LP;u<+^~Kk zm%dI+Qeam1SJ@mQ%9abfBNe(YI*ZFs>Kmn$kLLu50HV4Cn_~7|JsCyl|C-y!*>@nQ+ z9;jRiC(Rm`RHxIf*cyS<6Fz#C<@+-KTxt4{<4^nNABifB+{{#DEKk8NJNtF?of<^R zjJ4z>yJjg-YJ`hyu($5f)kwK`D`9ruWfjBTZ@%Jdp6mz~YkEuQz*VU>Z#Ah0oW)d#rS6{#>Hq(;;t%_Kx`Zl-0-D!` z@Wn{34WhVlagMnp7#PW75Bn zEz>P79s2;;xG%-LljM_<*6c?1-InlPto@Pb54!d@)RNkSt9|bixo?Z1<%^VaTU>5W z>I{x3XWfSzMI$F?!iEc4OCipzjdQ|Y(j+!vl!7NH$`d=X{S*s($J2RE&D?@Z%xDZ2 zUNc2AZzeV>gGx(Z*$1YT<$Hb*+Vk{@qJ4}fSRaHx7cZnPWWRkWE~A1M-rDn%(+6bs zjXTeTO1$BBuV-m!NP3TU-t|LgrdEkmjcI5PY*29v@q{)QZg^R4=VDEw3VHgw4LKt^ zQqgq|Lbmp{@P{yDoWBbg7CEls;W3NMGdpndf_1o3$s| zYR>)tjE3%(s)0;b@290d~d@N{}Xb{y6d7T%}iHc&eI#F-Z^wbW3Ea?0>#mLhf>W!V1S7V?p&MCW}Sg>+f%wdKBDfcGrKwIbb-m?@(p&?e2J+M5S9n zkC^J#n~WN7BsNE^EY&peO8#gp5ytU$`i7Vhc3zm_lIP7p67txSp!e%&;-U_$)6(|K zSCHa^CRfQO#UrB5fx~4#R|endZT$264D@FMbge{kr!!G*grBa7YoRNC(c*gWP&D)k z!0x>JISH*U9~O0gSPu}r0OfxNc9XH>kyzQ3wQ}#%N3DQzuEg5F?2}gX<*z4)YWIQY z|8(o~wd;=jh)ZG01|w`GoiGumE_-)Amz;h=f=!U)13J%b(wal12Tkw~of~82ke=aF zfm&dTtfV@?q^xIzXDt0Ax0>KzaoB;I=&91_JeX`D&AX)Khglxt&vFNHensjR+70r> zLgaXvWL}leg(4b&oe9lthhIiqzfS%jO$RU$erxJC%5&oWQ{vUwkIP%#%yu9q+!qmz z%LadqbvXdBv)e2+UhqYJUaRhLwNFM_n5kU_<6OXOQwkGEQK0#;CrtikSN^UcnrDiO z_a#4$EAcNlAnI$Xi^H}$Z=H34Tkda4m_~k)eG|Oe?G_eN@pcHNIPm)gvh3wC1j#Cy zGpT0UBR1%mB)q37n+ao?t(b9L6_B1}InparL&#<7*RDfira9g=q_>NmN@Fc88l{d1 zKqjX<${ux^9%DF8s^ohgWy_V=j&+B{u!hu4`~fz>ucIIyAIIdw>UT}mVi6#3)_9go zD4SPlT6IFHnU?#TF^Zea+Z?%6x!+h?6TKSBqK=h`~XjSieDhM&~7O~Js@7?$3jn< z9)xDMW^V-m8YvDSOruMYANQe&opY_JO+lFav($3D!FiZ{1GQtovMh{Njy4M~!e#A0 z8{(lP`#<-m6e@)_DE_8;z^9QiuL_lDd>iIdOPpveOsKOHz8Lj z@}IeQufR>k?2+9-N8^@dP~4%urK{xZ0eE1Qx1l2i8ytfsTjnO z@cBVQN7W4)&;M3z^CR0Jnak~6eoA@!Kf0FA69f9rV(N~|uN)L3@+x6(;!Rx+n7%!7 z&w#)HOMJeZ0uK30?QSjA8T2(9?_SB#?YTfG`{$kx1u;i#7w{b1fklWeM4e=s#wb8|h8qe%$Hm!E|)+C4jQm4DMz zhAoAr$##d!U$S&(^7 z_>hA$T=|>4rL!FDHmv%&Qkwo`Ye!ygr>0CJ9kpCar{KDd? zHsOWyv*q)dp&;n)vf3@5+h~js8-0J3T@&ph$8lcG&Cjz3uWy7(-2rdfjNHtht+a)Z zPl<hWUYHd*${X?BWbXwB<*_I%g>nb7`Z= zS<%n=Uq{yQ$QrY zcQYU7up`(lsJiy}r4B^rww>OwuDIoL(a?RS(E}WN{7k3GW>?o{-xv|_z}pC$W1s0G z)NoRXL1!)=V!?ENGL<$#`CFAT8?8=MDM({H+=`@yQ2bcL^S(%*Na>s5O-RYi$bw=F z&bH>YI9KONyY4qP7|VoXgv;){5N%(!9OmvX4A7-6b582$Yu$*oQIQk(>%KWrYR46+ z=2g%5h3aUXmr7()Tf@s`ENnQQa5>@A;aNfsiLg00%aca2PE2(J!3>=Y_Lsco zE6+@U)@aDovY+^7h0TgqIdA=k;PKTdxw|F1<$)pz+jD6ae|l1ZE<{Iv1LL=tqD zj}Sa=`wx^B9uh%bgkKK!pSu~uD)~sl+KE$&y-e%_6|&)*Kjqka9-}cF9OB(OGYZj1RQdJPa2n!ju&OcScr8VXJ5VOcFSS;Y=%4Z{iIfShM_Y^C zo-SvajJyr9hx)2hYb8e@xhd0Ku?LQJ4a$!wO;F%^v0U98*NgHK%Q66{bX4bnTuUXp zyz)M?m*-dtKZ9-}y*_l zi+8FobI5SN{L}y^uG^3?=%^I}Ficja+kWvyKyq}-eJ_5C*^xdI91segMw##AYPX=) z9J&;S`}8y_CZ7JrN^G`U5P5~w3Mtpa1u+Snz$kcsd7J*R&V+IBVIa(_p=HT&X(s>C zgVW_6bdN)7y%^^B?#+QD+;#s~58z$)7hC5Zr;}gZ;D|c3=gqJq?xRz@rPB@?ZSdW4 zx90T-li#X*7=RU2>k?GlZ&@`(ejzl zTfMS}*uDEq<7Ievnv!qBD_QO;KDihC4it`?J{$3V3H-4sB=C*O}1_df~yi;JiH2Vl6_AIv> z1No5xTUD9Prd{07GVCL!IhO~aUsJ6J`8^Dtu$Xz$)2Sh2x$>;wn63tS94&klUKn@9 zz!5`Bm)KN!Y+<|4i9paUvsZDfMa+y|=l`NY^MM^{{>!f4bv{*+2#AFz;Pf(`6mqSR z4Z>qy!nz)0M6H2=MFH`xtbs-wlAZjH=5@h&BL}(8HzA>26IAjs)^lQs_8up zhjN#-ORAQxJ++f(bfxO60-!P(l(!u?@YY|8Io*YczjxaOzNM{uhKc|0b@{uHYEp8* zlYaTAKPsuYIeP1P+lTB-8~WZLOgjwOg7S#MxfowTWm&6Q$Ve6G3OfM&o~|?w-+=@a zW}f!v5V33AGX@~-2UV~@63(&$4|M#@=463Gp5?1z}xVz@$1ds9vR(c;~O9xlf!G>GF(r|58f^ zNP15@D0Hn?&6tAaik0gc_Vj&gh?3o|h!}k8JDJqCN}b|6O-s;y=N%e^^`++CSw(B` z2*gpeG|f4++3bD;wb>+Fnwd}eeX@u=2UmgHXSdY&|H~HVCzI^|V zjC2&vDc}vYDEo7nLg&ISDzEeX3+J3O86-% zaD)(WooI9Z{MUiVIn{Pfn^l|DmvLf`j0gx&{J#%RSgG^Y@<(Z(ovx?zO(KoFL+Y-v?1LC73+9p9?Q8~|~^ zcHj?Fioy_HqxPY0xsK&bd4|_L4gcX!{+xb-ysBGBt?*79Mg#O-&79c3`4hO=L|y{I zq~!+&Te7`m(dmn(t2A|%Le0X|Y5ccPozqp{5&5EV!_5lDi-0+-^XfK|Q`k%fiB zQm=2|a`I0vxNA}e_ zd>j)C!~y^N85zJsFWzlD!aM`+jm4C_K88IF8~%L6&sPTaJ&j>%>-A?MuDF%^M*=SK ztwrr~Fw9I6L^i8(34g$y*?#8D>O_r%D#bq`ZbTl-LTdhH*W-UA>lK{+io}H>zLgt6 z+3FrJ#6g5{XW@lG* zqupw5;Ub}8>S69KE^ByVLcVWxaV_^n#z?U^Y97W@1Q61*tjf_%INATzNkedm;3@tt zi1;Xf+tTMX#|1E|30z`0P31LG+4XQ7!;LN?=!bq5y0o(?n_~tzD>AH^hBf4s+Zz7gT#HH z|2y^xXL<#n9`!|TX!(h1J6;M#m>!l(fBA<4%G=KFU$u!dV-TPgU zFuupd4IT+!<$n+YNU-03^n>TF*@&#R&2}z|%^df5dHM-x*-k3+%ub~NQnO0t1-KTI zsV2{zxhxk}qf_f1B@y>BcNzmu$gA;>JiMBgj@m3F-k1BU=M}D`fAyD7!2jpV-0FGt z7%kSeTbe)5;m%uxmEGN=b(9qQN|f;{#%|hR1n+aklL|ap zV;_JpqIdS;4`c`IrLPsLoEINqga$Wvc)f};@mR@tQ8ub2IF8EeAF+9Ln(9rH47sBQ zAI~gky^!$*pr1VEj$n4?gh!~Icn+MkEj+3@0_fZ3ef*x3pzj5HKD{G(A+{TR0Wgg~ zVZ^uJbKZhjJCIUf6(%?II+$>yz4nR%RzDS~a4$~G9CKe;ZF+>0seIKcbV6qn0!SC; zwGf6V>I`L;chHwoaQ?$^PvD$dE{x~HNi;JT zyAIu~TD%AxuD`rPydq7h( zmER%tTu;@0E(#$+0@l0il;#*iV*mbbKVZBu63uWkmsx3h*Bw;c9aI=p-u?eR(r{DO z?>yXHn7u&ti4eGH**kt`Ap=M9Nt(*NCo2!|9NmiJ)~|%*_DDVbD$OYrKUN_o_~btD z10gI?)bl)g0K_|r;su%x1MuZ=zRxt~~Vb(-?wo?o~MrR4}@ZY3<(Gr`Y6mKZNJ%1oDx z@MLp4wyh@Yt9n@HyW6I2D8C67{)C}lSq-h5-<2s;m8T`xlCm;Gz%L$7CCw$otO)4} zzf8hHxVI{%Pb9{|oRu|E3PMrQ;OkH85dQnS{b@01);qevq zER^)tCaR(F)fjJ?#(be6{;%7eT;EC^Qw$YW8Uo*QO7mr0#(GXKLvn>47QC#r(fi?F z{t8;q!;whx&O=^fp~-u!?@}2B1nu7A_C?3Wr-{B#f2nS#n@4TWQa5#c7f1S<9u)@; zi2%#V+0T{!eg5?7O8by%4$(j?y%p}=C+*l>+IMXIp7Kj}P-2K$%tM)b8f03ia*L$U zL6$EG+TYleTTUG(xf)a!)h{0^VC1snMSD27Q^~$NKzWh{c*3JIJbSljLyJ^Pfk&&( zZrl_4q`iLci$YR;0BFd?xG3XWCRJ2Dd;#~eWy#i5k#O+I!WK8;8sJ^|1VfB12bP=k zEnCo;o8zSGRHX}5rOf3T^JU-iKkexChLwUtlS6%90inirGkuDf_w^7OjWE*1LGG-b zgoc&VXTk5NhOM$c9=v8xl?GRkJbH2PC=+Gx$#?4CTd9gg)^bj&+YS}H-V->n6<6g} zXmwTg{;2z`ghzqrbMCj(VTO-xOPzb5*MSd6zoTaB2pDPUw&&>SN3#|3<7E3IF$hyV zf5x*`<|7@FcAXKn=VG+?>51Xf)h$gfslkWV(V667OehBnPGNNWV@_(U6#dhxi3$4W z0Fn3Tr;eRYy9!USrmUmGB2Kn~-6r4XOjeLG4{RhJ3+Y0mP^KaD;F6x2S#h@1Ncfy! zZ+msS(~qLzjh2$Hkl%?Bvg_^i--{`K-IDPBoE_$aujeOed@i~e3^pF#-E!LvKY4s9~yS!no*BeD1Dfbw^rc%bk>gV;{VQaryz0PJD>-G85ouVenN)%_gu=C}W)O0@ zc`v|hEkgIDP_iW?c-6{oct$UwQP3-`dBtXhX zS=7>8M{304-Z_dn6!rALyb9R(Y5NS4@>w7F$FXtYuML6r@EzE_67-8=2hivr!mDR= zH5?s9n`zn|CGzn=`RFWd9-mmah2)%LX%BnC=g|^DP1O??TXl`jHue>O?e}Ei2$!k} zm^=*st4XK$(F7rfoAIYkk$frh4L%!^)T#+1^P{-P(`P9I+>r(v$dA zS7J#zMAS`|@m)B*uwKc?M9rdYoj+TPx;$cu+D>A0NZil<3$y0aqiFsMMkP`3IuX1& za9o>Aqe|@wKF$YLqwpxqgI9I>`>5cdoaAy*_>a?>el6u6Jw|m*`$=DHG zxx0JDab9RgaakUW*tNfU5cFU+@7DFQj=O3+s|5cJG?%z%27p2_x5ba@bD~{^f8RcLf-28yowE}Szrb3@> z5@996^#Tqh%n_gt2t)Sx4~S|bR+iiK&AKmY&jX>z?ghCeBDQ@DRmcYwDJ0D7k4HGp z;hG1`?p2C##^oGqU-rLoHCn1UF6gAPxs87&mm}U7jpjXm#Bn#3J)HBDp}{f8?L=chmGFJ z_2aNg(w@6cCH6*y$n4j1dPx2t9f0g^G%JDAAW}v>bT{o0^N{xj$a>0K9ZrEYo{e6G4;_ebW5d*&)oy7_NL{l)d(st)Icl!=HKdAf<0N7Cb z{PVz5B~jg#s`xS?naKCAvCcPtCDrlaqD|Lx+nob{UbqN3p6Y$;vdtw5YvmtP4b%~Z zAo}X~<)$Xr55Qw}H2o@*n;-Wa&xqPNFp$-2cDG&**FYJd{76ki`XRK!i6Y~d zmt1y@1$7I6BBI-%2AaH)AR@2wuHu)qn8wn3pO_x4+`hZ|`nV6mSM=X@=slY|P?ogM z1U^Tp&aUOsfp{A73SC;;3X2>*7nRy0etE+?GgIccE12*8BYC57-aI7sG~u<(ea8Hg z2hK8pmntZqWQD>bI5~y5->YbI@|?`CclFAa;=evK8FbKsJ$yF4yk;U*9JvwSg%pw+#slh8zGK934Js#0_AEA1KwhnA z(WejQ;8%htaA|{Sa;JueKf7fU`T~gU^pi2N>1!1iziWWgE|_-B?akj&2D}UE^NH$H zLQmN`J`C+l{5|jW{}9Q4?&J zTRVa!qpgi+h=001<)y!-KlfX?0^)OwUXKb)JIsI<95Vvg7s{mu-oVjib2?`h@#Uc; z00*?AgS|!!`{?I-HD@_oaakczX5{XBckhS!A0epLRL=1W*F?WaQ@LcM08BHKlGdhm&9lU*w_Ak|>OPD4_B2)hseJfA%hnG)-U<2Dn1C>e1y*~?8}4ZP6;6juiJ!%~ ze=ASoa-(OW2>iT&h~|Oq)S8HD3Q~%|4Bn2{YF*;fzG!$0T9i_UBiC&jY ztY#IO%~LVeAL~zNr)`_$VJ6s^_3q~YnD6a+o-{*Z1VJbKe zx6kK|AM|(i1-1mR9mD>>#VI5$y4IU`3&VPe^;3xCn_52>M=x=v7&Dx?dW2e~-eT?w zDDRddk4jw>oiaD4khliS?HB983pymnaBpR*=G*eAAy&WYIAx(CbNZ2fr$ZP_(rI`C zloe3z_w8YU&S*ERdrx2|+wJLV<77?SSHofY-ocK^ch`HoRN~FjhM3W|%hXC1;3p|e zlb6NG0ocv3n+o>fIXA&BX>L}8$ zsF`1!TfOWIzj1qnga%YVT>H18WBgbsY;mob(=g%YZOu;9F|4CY^%@jZkYc6nkdK~k z3ng2+Y%qEF=SE9%Gkck;+zUK~b?taXoS2bDH0bjp+Z)=dMH9!sx=9;pewsCvej+LA zYM{&g&20Nd`bWt>Hw5PP9-fT71o6{)7TclB?2?N`hD)=?cO)rZ5%sTv!WPt^{`r4u zc7F$3w1Y2z-fY2?TR>V~$JtBmxT;jzsf(IjkvifS0!$327d8J#IOi>Hl5fyAsQld} zJQ3<`eHK>l)E*QQ4|?DbcLsk_&NwqRX^dcaj9O*y^Z*Sj|s3f-5 zhpi;L^(xt86P|YRrIrp)_GouH?;Xez+gm=sj1qSGEO_wmC?6CJg2DeqS&aEw!{HkQ zZ{gkRNt6Rf_hssyp$xj>5r&5?S-Zpw5MFk{=e9XWfi9&&a3+--8#|5hAQ5 z6_~Ho^u~qMM(a(gY7~BLu^+5AIV&dgzw*^MesMeOfvNMfQ(@_^j<+-dZ-efDT_SAz zy`fLYKQ7>l_n4(uQ>dD*XUc`L7xiE>V@mDki;e9B*ZgxcNM`Y#wvbEEV#oLa8J2vE zplV2vI5z_jvAU4(0p9rk9rcgbZLhd848cg_u96<6{VgK=A+%>W}MKc5z1}u@kYYB0^FH7A}9l6}vXA-jv7#dsKynM>7`*55>_2uthwE|iJ+80ubf^Enn82Gke z2EU}P+qQ?hV2;!-q*?^Lyq;-fR(Q+b^q~JtEqbO>^3E!P=p^ zq5rZ11P^ux$whkCnp|s-+Z{vph$>1gD9*)L=j&MySdZm5v!Lu$E~}9&G+z2M)S5UZyb|u&J)vz%kS9;EB{(67ptDtOGFO_F^aFbpP13& zycJ36xJaVqiF7rP&eSkrCw{z8UH>77$w0%ID zkpw}-y)1zLE7Guz=zFaTuy~H)be~rY(C@PHZoHaq;-EgE$Jd3Kt}we#F={@!OiXTI zly!OF!guj3KQf!=O@$HPE6BRBkcROJ(nHXPHu||4qkWO}+`7g68BR+j8+_zuds6j} zsi&KtknK&f=*ryZcL)ftlzuxR2f)=^?GTVvO)Y<4!@{+Z%J*4@O8e;F$&Byl%xw{g zvAasD0{nX+_4B6p<#kAjdR2JDef5_P8Hw0uyA@wIpF?NRkwlHe{-o-^by(exluvb0 zpLatQw9JDe#|*Ami^FR7vUTnyI(JaHOxNS94oC>Kr@9q!xxAoeruDbs^E$xyy&cJB zO(rLz=hb*~ywWChOfeCL;9S#oj$Q@r$97IuTrw>zLfX~NQRaWMh=)lmyELke2O004 zX?BBF%sKRKZPteL!M`pa(N3s7=`$++M?!mSJg_Of)n;^Lyx2#y+<^y9ml1C(ztT#E zUs+mm(d&6$xGg9icM72S>U82mB;0*=Cd_?`IPPu zVF!M9Yl_1(p2m?}zftOwNbp*|t9X*s&S#X}bT6vOnGfn=nS*otuJh3O-fl|kvL37= zUvm^*dy=pTat%g3K}_Y;wYk46ai4Ac$+9v|r(GzLigB1>O3#a)NOP-_sDL)U_5Dn- zgPPk*)5S@rPqD_Y2r1NK9SR(cWI~raLDv~sdp^2cTH*Ga5osfuX-qqrdZg8MjW zLAcIK^A(h*-wK|&pxFyrHTL~f1Oc*xlyCp^{i~x;(1fmz zFUrlQ2tr3`?zx6#kNwzp0AF;p;CkzU! z9HYOdTw`2=q(B3+WI~Ggrn#L~45jZy%^o_}u)w2&B$yjnA77R;QN<@+slI#N6luFB zUk~Ldh7RdVo`=u7zP545HiWc0fE(H}x|2_Q63MQM9R&BQb{BqeNk-g}d1UEXf#Y}6 zbuiZK>9QBZ#xK_R(W&6yba%4V*^py@Lf`mvK`(x}Fs4FxP$iGwCMQZf==5^;Xz3OYwb76p9i;>#(MlsgX^(iGK&yfolt6dbux^p-*?QI9(!;@?DB8>rwsCT_Hn!0&e?nxf5B64CRw#yg!(E>!`xf!3z|-Kd_3cIj7u4@EzQMObHdXW znj4lWGHzloU8WpLE?f|A<015s+3F-s@$U$c_qbPPe1>Zf4Q+*w5^k|d$CRp#80+Dm zwjU*KW!7^U`b$p*1`8{pkhb_V#UE;f=C}_Re0%do-i4oX;q^1FHuh^3GrgT+(<<|6 z13^)DP8I6bt4e0-r+j4xEF~wd8H(Kp;$L>(mM)kPji;Nc2sb`jV;aR{oHB0-Kaa4% zr@@qWyOhE!z3$8ulMKAQX1a4^Y_j&=DXtk^c`E$~>7$bRyo#BbY;6>cZ&&^EL4+~r%8O0 zC#Ihl@V5=h{h=-nX8KM`!nl6h?*w@uGwRT;8f|5q_ADyKOP5@q@CBiwk7FkPRI+AT zyNWx;4ndgfZyJwz$K z{RHB4!fpzm?zywKJ^!LHH*)mb5rmLhR{2u<9&il&B`3u()bY^WLjc6l7Jv6r?c3;Y zEu49!qmMVUJOirpSL2!=N_G}VgDL(ZZ%R+*U)GA1W!A$>{kZqL{X`dOfNxoR4D^nw z-RWv{=B=Azekm^NSFoADsnJxi^aW>vm2qs@crIk?1MwvrC6bDRu~(|>qi`msTo6*_ zE8Saixd#BCOA>&i&befgpYBRy-UaX3BcI&#WkPA``gYyC^Iwq^)}2xA4x6U(g{4x^ zzWMWV&Toc9n|Gii3}*EweToJJ_#jJfxTdRQ|b9`c7M!_VQGW-&UBo22ZG1*1%_vq9AP-ZQ>IxQ6wso zoU7S!i;C1r#J$t0oYL=L5xwB!Z185DfdG37abjc81GnS#elhg^5$N}T#^3xS?$d!% z3BrII${~lC#ee3#yM|1BoxjI{B|DGY=wv2yReQS+=`2sGm2x_iscDFNvMDBU5Ybc`4qAt@~)jWh!U z>1H5ZBJlam@B425>^Po3_B_vh-Pe7^d7USs%nE(~0us~7_t#3LA1@1>!i6#M&B%@y zpZiG~m))3GnQ`GK{WOX3hdS%-H;xgrH~K`;wD|Ds3qR;kv8_LnE+*%D?Y2EyL2mRF znibD^g`jEw()$sYAL-`ZW;S)C`LuC0(w+6F&EDAMb;6q0b>YmBso4>7lA0{KJ{6?f zY_k$V0cSBNoXpehdi~R`b#(3J!|ZuCni|Zg(S-45YK5U5>Em98rw2?E(?^^x#p{Lv_BE4hYaPs2?~Pk^#-#&9-L4XQBKUSD zpJ_f@O>#8!xa|MvXlB+iD%?<>?6sHOqaBu7ON=vp&g8$zY?=IoWa&)L*1f;ucE+M{`t!>^j{69@K7)y9|SH7Ar@*tQsx=u%Hv+2~`+DsTHixW8leQGY|e@cHJe zX>&lwNGyy)zxp?bIjo|=`M!JvnA+(Ok`u@7u`h zzsg!G&eT(EpRyIJnCk8AtqhX=t^q+}^QZg8Jv&+T!VUFovA-Q!-hl6wIjz1RP;`G0 zDm={dyWj z@%;ZSN+?*cf&(<~2P1W|Ex z`%Js9SbM8x>MHjc;!3F$xPZqVMi9IfZVDof5q(EtS-(DwxLwvXeBV6jl4l5P&8;`F zh}_UQ(UYyZ+)66v$|jd3Fo?cmvWMA2oSaNUHNQ)1l;PT1DC1b?zETo}%6o1BLq)I2 zUi+i=k&I&LK(+h;u9=3tPLYY(IG`p(hb+kUaM5jsgRA+jHCW!Qu z{47^X!_mTFgrVE?j7DC%Go)djhzh2pt{$Y!7tvBJSuF_3)2uV|PP7S%>M?@wE$ z$5W(d@gLsosiE@hof}9;zsk_hQsW3h4Pg=1DD=X0hSR~_F*^DPAK|`C7JH-eJIhae z>mSSM2>H&a&Wbr*7?NVoDG^>#Q5(EmnG1RoZh()kW7XT}$T+yrx65l59DfvRtK;UJ zapZ%9d#wf}tn&Du$5Vm^&-wJbz_vGDR28k7#K$`1xK&_M3|UVqs3E_6@# zgjoRD#?B1V%>5VWhEh)8s=#QTFUZBYQyTL{pc+AsN!}?T$Aog=C2S)U_sViIPop2BJ@T;N53wMqEFe4mu|sgV#3*dP z7%wR1jrh*v|M0$h?O>>P;)@bf50r$zbE*{&LRwHifMz3Mq+xSne)D*X9dA7J!{VC; zDdrdE!&QtK7md?*vZq9-p9XTj@FT_%lkMcTGK+=xdRvPWu|Lz8w!<@VZm97d9ek4F zo0w5AdX6IP7{RZmyO|P>9xj_G1Nj5acE5B8u?~4zu=4)qem0m4e~qTrJ~Ke-@8f@? zt@1V3W8zLJa(i@F`J4;j!;ULndL!UGwM;Q+AHuq2uk7&kk4I=@mxv}Df}r##oVF%1 zf=h=?!Oh|$zm!KA$TkPwsPy!=cQS27v)o5Zqpm4A|MHI)5dME`&J5<+7e^Ko0})%P zhm00NKpRs=HHw^S`kMTw^}SEuJw*2e<`J77^h;+I4kQT|$T6OS4%R&%dyeAt}Fc54!N4rp#U zNCS+_mDxHop@&aqX%$zau_dcixnVXhTq{bD$HY$MQ}C2y1AkbQi26G_ z=L2o!5kp+kLYWdqFyHVWo^S6_0ZcB0aX16?-KpnWj3U@akoR08N%$wNw>`rM2-K^V zRO3M=!&Y0V9%GvAhdg`FXo-a_TOR7NA=qa-^! zkoQ-hO3|vHMl?kVl@8`G!j7WzSRu8t6Z2tF@jm@en~S_Fbfbw@Q$4UFT@-e{ zvL{xRr?GOpbH21v*X2_A4k0F*AL14nbE#~oc|cnQ7$NCGJeFXShiC9}y@=Cav;0Ki zvP3&WL?Q}nGYnWqwzl`uYJ>V+;#8Z=Bw+>g#Xtx9D-G%rAtNszirFRhjBGi31J2U- ze%XPwG)uxaLvssmadnrAC$jjd^Y6XhKRI8iy2SSu4NHYu;M(hyn@Q{p=JSc&hg1qM zt~xV8>e_#stZYZaDk*60>QF2HVykQ$`l`{OzS%L$xqRuuDFfB1UO=QKL}pyV)!p(! z#%oYrW~WxlQl#t$-!?-q*R~_M6=A=B)G0uS%Gzpx7b?ozTKn~n*6-y5{?`Xn0HBjE zj+fdUS5m7|Z-s75I;{*`zn9*N%EL{B>O!QJyswDG6vu0}k2wjii@HzTPDOlou7PI% z;dP94Xznf=6)vxIVgTAoA_!ml{ib)$xrA$I{3M;!SR9XzJ>=|Ht{rvZg8+0S`!ql! zBZ4RSd0@le+%!pl#i!c!r;uq=3!p3u{BBdpdH7}VS8W_gq<#*+0h#jC6Ek<{O35;@ zjbW*6seM?j62PnZ8b0dqY=`S&7-xX^znRIUm$L%+pOTMd6CW zB&h}G9*js%m}I%EK;JN01HZpDqFRCm$}YiWWTizRQrY=KgLvZU^9FL_NW-Ml`DNhW z=BvCv+@{pO1(HX+r}HYtcB1l2s{~aF|HEr$s(;9oE7Npl(#NK$di9|hWm>&ZQ=xPA zHcc_%?8P>{*9fDbqJ2U`egN^(&GR%k0(is%be2gjewMu5Ho%*{Cp`aX$9f4TP`3jO z&QtB783e99%O3TFOc?+b>-{@QsGUuq*=7BT_z{{tJ}0eB^)FegW@j$!_o&Ys6i1yy zN0x$n#G2z5rHYZ}(OQtL0Q z!0hDE3(in9%t8m}=Iu=Qzd5PLk{ZZ|7m*1e_ynO|UgQ~}4|GzxuLYq)7MZuQD1hrD zQrNYvk1eX%CY`m1s}g3uw5l+)>uj?hvZN}b&?opf`2JS0gx6(!DN#2jk{djF6b%df zG9_bg!G2$36PLxb5Y5}>sr(3lTL44U3!M8o*GW$S6XtzeE2Z{S?|Xbq)E!KP+%`~V zfKFdaKf!q5lY+OmtmuwD&(LlSBX#TgS4T-M{m*-TE5-NUA>VVAV5M%f|8SAZ2BhY5 zdBDqnU*bZ@SVM;`e{~7i;HQqr!+`493sPEM+KZd-NFj0YR-ra#3%9OGCFi!4vzH=) ztVDab29iAbdsUQE{U)q!^bdGJR?v^*B*O%U;&SIV2pdsdD?Bz@t4R`}P(q9@n4prx z$f}0w#kYM)Hy_Dv)wG4}pqVcGsv{TR=aaUr>RsN|T5X>oo0GD%sA(^76l|*QfGv(I zoJ{U=(h1^3s??67zA<{tZ=(QLsap9QOcrtcnKFz@I8$nbG-h)D`nSb7j%07oRM@cU zRm0%##^w+e)d??SI*h<4!W7MOJ>NK_-H_$V#M(1`htfq?-ro6OCh$3$bETL~E>o$x$Yew=a!OU-O5F!E^o}BKcu% zeqHQ+F+Dp;`wcCX7SjxKAhv>~bCyL4F}GTu;e#Zdh0|0dqUP^c=T7G|C6lKHt0IeT z{N)O9U%os7$}WigL(kP@gAyCsCHXXPQ)w6Z2@lzZ?tBo$(A!Ts5khukbd~UrHIDWzRMM z8i$seoh{N&k=#Q_YOBBrV`U2=jx9y~{FCxPA%SbeJw2=VS+k0-X@AASR3E}_@+yly zGt$}!@?Y3D$t8EMb}&zoPBiVIdvw)y;orI(eL3i~R;L~K!<8n+LZ-D2uPttdj&7pD zvyX57!z(m+xc5F=4#~9!j*s@Y8kFG{;D^RQQ%3Cl5ICGclcHG?+O2%?QX3{@gh4I_ zj~paD!|2^P5B%&p8G08!b`I+p)jm^i#oV8?g*&Y+Phsd3-M_$1EDxw|p>Ob9wWd}Q zkzGpF+=y2zWhq8Q(a+ug!{bvWp>RQ5ryqN1Xv5ujG0|#7Y}XVC=q=frEiB!BhJB++ zk>S_0n$(|@?_tlai{00}zFq33oT+f|(CxGLl`9JpLkR)ufVglPe*$F_ryxsKO|H=a z7B>YG)zZm&y+te<4rC>=mTnI$lNgnCj-|$ngydZ?{uXX$mva=3s)yOSB1P zDq0eDsoeD$Bi_WYXaIM>}F1s#FD_I|z@ zc_^PWsy* zCR}Bx>;F338Nz+X80W1;5XV;ZK6K4U0!4@t=pGoC#7yq@sXi&W_e%uHrK(Yn!lHsK)+nb2u?4CaV43nuy=?GmA$FtTjiV+7UsWF&vV--Fv zooSHlA0FNr<3qqp=LY{`56HCC*Nv_2oD)RwBZyNAa3Qzkg&W^2DDDfiZ+>K(v-G>R zsR(Vnl!DjZHkNR`{9c5QnV>iabob*T$&RU27cMIZya(QeQazmXqZRD7NSmp!+@CRi+X%w#`tgeVZ(cJ(Z6V>1&d zC0%fmS3Juw{B!m`Y=-qfJ}$^|ib(U@$Q)9fk=m?~fE|<0PLa$s0*pMMdJXlPIl`I)X({^%~cChqSS{KcuLeD=?W* zi?=8LDmVJHXB2sErfBM+}nG z#}>G;Z@0x@P}C%uV{J&!4TO_CM2PJBcSd5h(DL$-oPg9noO3{^g(y$YK50_BZNRxyF zeU9k&lm5JOe%5Xzj3c<3bvjbtP|W#_p#}9wXXZwMUP9N$<2I&VHz;N*kcNkew2k)l zrchGS%>EAxoS_~=>Fm{ z$HGTO47EKA%ASeLDr0K?&Ib%OMkM^4GQ|_!Co!*uZJi7w6ir#dZM4 zFDzT7^4sFAD#bDBV6>ZW?U6;~R5qHZ*~?_t+lMk(y#YIQ*p%6t)}6I>{~65n9FU>a z@UgA=;PTuAMeAO`+p8BCV}{~>MahOkJveH0hV9-k>#yWNX@`X)At=(&o49ZDdBe$F zw_?ohZJrs~QAO3N6q(L|#)t5-M-)7IdKp%cA?!jO+kp{75IF#xfqtEg;@%W+Qr_3H zwRyg_Q3q1#{sPBz{02@b^;0!#Xw?VMu)F0Hoi~N3vaWh1CKpT)u~hor5sK=S)vf^0 zPB2cbHE;W9ecY?o2v{U7J zjA4>=Unh?)DO1(2Wys$UHk<2Y=ATR$)gv4(dq(4nt5u=6<;qQCIgF+`%js)cKu;t8 z4l&bLsefV8CN)FWNl%Gf3b%?OGa+QorN$nw=8vSDQUHzGm;kSYbZ*99WlXvHPxG(z zy<1lbQ}Hl6OcVq;uil*$R|r?7<{tidT0!={q8^&%N9=I<*%!@}5D3aAEfUl$mhPQzq-CmFS%`HFb?qX_@(dk&q$P@g!^dDaG+8w(1 zmEr&!3ikTGa`XFL1rsgVq(`7|jQbddEqp+~;TsP>5}uAt->#Qf3o&d@>29z7U`JU@ zoR^WCJ|jC%3>iMj(uxPl-hzXT4(j)^y>fr+E&PW^$88*hR@Q4O^weBsMS2b&w(1F) zzJj^pGW-2~l*#P_B|xbH6Xz&tMTbsgEXhz~5*EQJIw$VC@sbDfeCW@I5Udfe^eZ4` z@pUL75}d3Hz1P=LKs1%==@7u+!;S{BPdk2ljCTYR0_F}a`Z|(ppqe*$17@}=bcTIK zdfoKwS>4Mo7d?dM%dnNjmva66XAtl4hXCKoKVfnmEZ>)&1~&S!3>W(#lAr1%u50-0 zbwWzheobu^9vMPC8oxzOPOi0Td02_aAX_3TRiS9pqtzBgOf9=q#KGzc;@XwwMcL^? z!e;-6L>Dd9TroUHff1I$P)FGNxW7~?f?_&#bvx*{BE=O)6;c3#JpKG*!md|k@|F4T zCs>o@a^gfm%*0b=(;n&qBdaCV1ExtE1f>pMNwHs)Vj~X6Udbft`IA$ts#0$H z*|mshdy|>izQ|llf-bLT$&>E9qejX&EaQ1I-T4!9&p#`1$lr-$6Y{VTlf2Gm<$n4B zv*-b|pSF13VR141-04DnOZ#cqJR|qdzT04{_(SPGOESFJU%T@Uf(s8G#=KRYVC(AS zv*1KC8(6H1{04rN{5tV$%}UXL2Ju%N2mP2}ZKS@d<^u>S-Y0DF#+Rrmda<7RF>#&_pH6My_rbXADk=Cy9BL7iRP;4oe2T2-_d)RRp@`IbZ-=4K+dT{#W;B5IY^BBR72Zwa9 z|5%+`q-ywbg#zEK+`G)c-DR!LG}^Ghh#a-vV5I(Z>o12yNLq)=N=-7cBb^zV07bso zH6m{H{NUiv*q)ctZzB$|$QDpLO$-E6m^;%_O&a)FV-mfU_d^V1U)Q{2*qV0KQNoS! z&FwIp;4Io}JjRevFl!msFtm_ZZinQq`4y$X7Af_O-mSDlDc&$MX^mRjBs{()@Txq! zEq$%MQhIANPvcfwHgB!C9Sp{e{om{8UAV5&t7{_xHA9g-hZ~?G^LS6>Pd2PPD1b;n zu6WvXp2M+hs^eQQ98>3gzi!yv&5}+429TbMdF8|#B95NK|hf+kVNg_=ryeAWA%Q zE0@DB*x2pX$~9-`7VT1%^$1Qy*>bE+ktGz92W{)6{F0Z$r1ZggF~H~Lnp6u?JbSC9 zIrKxcVVEMkL-!niL%37=UYSkrr^(9>dj0Wm@P0B=3`(_Hknb07$2vu1=EqMGh0-oY zb2Mn5APz-=w)P-V+(_AHmj*cP_QgArjpfo~e|XR%^UYyCYVM7@$g&Hor=YSSrml#>IgXokw`HD|U zX4pI38_xX??=caWa3((AcFfqQwQsQE2g*f?eYTk)?G{m+yd-?85AR<3)0yftLSqBX zRva%|_iG(X=l0d;VqtPB!~%1>+s@iX^bc6dg$OEvz!y`{HA7sz_dZt=Jy4%$rv`^* z#&v*bR%%+M+o3@X4FBOp5)+j8$P~{yaYL-Ma=$;uXT_C2!jr%%)E{|WHYaTKBDmOqa);$3rve#bQ1L}O9hg1S6UVx#yP+bJ z*0cgV)Lj1Nq;Uy%R$J2QAhT?r?x{uy16uKC&4Kn4u5jB8CONnA9mmuPsp5C+Dk+t` z%Go|6EI6e?wK4_11I^I1N$E8?$l14DeImk9Fh3)mo(jE+W2y<8l#^6^QYq(`XIdKN zxXL1@2!$L9SAd+YrS48n{LYO#AukyWUU-_PdzA-N;GNxRf=t|$vK9ELVAm9JK|CvN zPG|b3YP%LMQ+!Dq>pwY`n~A0u=GJn8OoK)@XI$bfDL$ykSEu^4e7|Qo+fq%@p+5koErb!UCfHpy)wQlV)If8 zrJw8Xuk_^Jj>2<%oL8hbx{BG-G`|5v>G(D}*i=8{!FkCw59Kl_MqXB#jn5$GUw_dn z?=|n_ta20Mhm~I^i+SOOO89NoQ$I+XWw0BV2DbX5J8c=q-04?j&>4D^1u&xP$iZr^ zhjirSPF~KK*HO)1wB8Bf6)Zv~)EuCiwy4N0k|H?QZb33mdgon0-#gvOx@Mr;loeC< z*!UNNw|gt4;Cz${Z@f2v& zVq7T-DgBbv5bZEgkVW|$hQ-v$NLdFp`=~%Dk_Thtfw zaL?bw>CEjrgv*OGW-a%p(4)IL9b>8bOfB}DP<);AzHh|A2I*wOk@YoJzhyEge&P4VJK^5^2CJ2iNF|Fcbbj4+`=)i??IZHLOa%qtimMSo-snxs1u zJMidMrR;?7(LkBYyHC=*rQg21N;>mWu>{viLb!ssHmHtg$Nlr`$BfIG=*Sja>jz6F zA$!Z{iUmN{nTyvSfvUVk)TU48A@7F8_=SmVWb5zC$%(sXTe$eo6C1MLVpxl5UT&oDth=Ip2tQ7~A|YhR#Rz>$P@?o0`wv{?*Z`hJo$oEj$+KS54hJ`a7V^(5nd` zFP>+T_Ud6Y8LaYtpc=UG4HxUG7MJsBAv~}A+j{t<^15&M2NS^8;$xXq|NKHe_7uJ> z;ZU<*=BhUC3aZH?AcrZ$_?|khZ|J5FhH4bQ_XO_!K4I%Wyc-@}zBE3p{#>O3T#SWl z5*AwKM{9^pYIYLgE?wlBBo|a+de)wM>8ghX_ZC2nNtvuuUeNm?qAG${sa!>P%XBQq zEmsKYO;H`Chr>Z%m4qv)UQ<{4;9c{6UG*@huF|yU%<6=#S+1?4%yjRU9A}J{Rhs)#yd(?Ibm& z-q?p#JtAm3+674irTae6!JQYWoPCrS{W&7Pkp&)0=eOWim@WVH_R6x`yH?yj9$45Tap90Vot*zWqI2hu1*b+2;d`gM%du|M zu0=2^+UwwcMLm-KJIOmvx#tfEzKaO#afW$`7G$M zAT0fFup8_^m02xT2%AST?75g7DdIC;cEo(Wi393RwiID^IPq*>@?vI7 z{Pwi9dO4UM9C+We^=Gi`pO0r#J2b$9q*AN5kvLm!f_{v4Z{`MJxrrUhW>ob zIg;-V{4G3aFXq1-3`l&C{pZ`=W%L?*hL&80HgN7gJc%B`7)9N4>y#BcS+@oB7g}so zYK{%+K>=ewvu!O7P-|jvQ>5%T8)FN77=DeUU*{WK$CN3gha#p`Sa0w1Nlo}_$ECs_ z%EFA_Jm%AAp*dXLr)8om^LrX$m(&kq~$w zHTr6?y^wVxvX#1!L&cRW3pYuXiyPW+6$4Ei`dA@AM$?WHLwg3n{l~aL(t^5tr|Ny5 zhs6O_ih3~yuFQ1b7N*_`UZ#1z3J^M%wtAGferDLTw8~uPB5r&cU9WtjjKHW#?Y&+^ z2lfDddc}VOB&K#)@4cPXd{w%PO1KfdL(Oc1<{EJj0;zKY^fn7$TF+Lg{!s~)5-M7} zrmuqeUUdlYT39t_RQ2WRxe9$7Z?FA2mX+UD%qxWaJa*5`#Bt)jfNblAE*9LAn1T5j zsIsy)M`$2fOJ8~6(9s2Nh`JCe$ihiM_&*w)l`t@T!~}q#Z+OSLfQe6%P@JN>m`@|#(`qe=HF^wsp0x$2^djs6~piawg>@w)wO!#m3j^PaMouU zsB5{*XdVR_ritzJD?uaH&-GX77x(ik&nXZu5&l1;L&X41fmS#w`Hs}M8j9Hka&k#7hw}>S?BCH zS+wBfse!kxl<Eh!eB6*P51ylXoE{WCv=*w4az;iW<+Hua z5n+XYkVB_f^_phdWIFGhP<)*ep$H#s(T^J!YY^I3m?;S(*22avW@g)P+r6!tj5$t# z9UgeVAEEm{v(^s4`Wx#C%abIOXE`HLzTiFlGx z3oTrSmbbS}yQ&iTv}Bgnpn_8qX^-^5{V`GX*XZf^SgC4c9Hmc}Mk>RHX%1Hk8#6OV zai0cB=+-l!R7b$M=3HR-ES-;fW;4=C@|{QWO5 zzQk2~d@8=1}*xge`*Hu+4~7k7hYqS<&i1_)MIf(9vDh& zTfe^9{vRG!9SdHaaU6x_5~WzfISq6~|JZ8+K&SP47g;Zq6X(r~Udch%+S#)YvCWJt zg^wP_zIX4K&gR!Oj-!k5GLRzo)?`6$&V`Np1PZK9uca*qhgPvSV>*HwuwNY&}%+ybz|)NkSt zZ`ZN4yHEJupM8kHLu(;vK9v00FuWMn>TYw^X+(rrd_9E?IKQTiA769sd20MJdLh?H z9arKWueF3WG+$2nz&-8qo`fwY*_Dd*|0k!fZ?M zSAL?5by|P&+grI5@AFbp)%3%=l*!#@-<5@AUMxi(g^%zD!4mp+ z9;<82+Up*mP!VoloTo8bY@>OI5c_A_J%wPZ9||8IBdC^JUM4D`$#KC(tcq-Zudl_L z5nln*2gtj(V6jsX$pr3zmh$WD2i#XliPxw>;?8RDNUtp_PMcG(2gLvCjnTZ^=} zc&z-s#MUX9~$xwpQwp`epm+2+r*#!-C94!5xN{?iI?lZ+;yx+hK*<{H7|3pmW5w$S^tE zX5kT-ZLAhlPnau=CWc$KL;Ue82>0@fhvzig9xAvp5Tjn<8Q*m3~?o_qKC#(E#-^ju$@4?@b}g_ zMW{{?Q<-k1R}7&sYJZ&CHGQ1qS!TQNk#RKK2AlyJYzywy|AUAXI)MQAi8dHu3md;F zoGCJY2P`+xIOWC(&j$$peSF><)Rcs5#xO38F!?>DW2uz<6=owS9BG_p-nTjyhur^l z)+v$=?w4{pP-HWpu1rY8{7rQ{YD} zZaIiEW?V#$i0D_g%AyiQhu^es0Dj@BU&p3z8KaiYMrA`Q;E|wli6uDa9O_O0(qZwMCEiWzYljVgx@3cJdwm zEN3asH&o*F;hP(Qzc#n4OV8vr4f7M;SM4~>017%z&Z)+x87-nE5Pj@Y5LaWd5-)8+ z)VGDq7ge)4n=Y6WsFxLqk+t8DQ&_DN1?)3FNQNMn43r_|EB61N0hj(GXLbmmBlt4( za|R=xH#>QU-Wqu<)M^#L@c+eYAsI1Lv=shAOGW@0L}cGZF;;hiLS}yq4SA*dqa5t= zO1|>TyAMCwicihU&7YTc2ZHV$vg%LyZf0BuW{%)k#*t%0P`Q(Q_}T4XTz7^qBCeqT zhoP6(IQ>cHG+Vm7t*Ax^8Mn%`id0RfENp8h+kl^SXHydt#Qg@i@~Pn@t^ZAa;^4G@ zV2ivoQz*WL*@NpxSWjrSP%MC3=4GcRGfAaU={QNJh9#H)E`+;dew?>6lr3LBg9Hl3 zi51`#$BCKxBX*7Ycj`qTE|TzqX{~3}<=U7$rtl_6F)gl9SheI@KMu570b0#`n+T|s zb#iZB(4TTQ8xr_qCJdbcu{HTTn1-R$c2qUH829qf_UIw`*UE8pn(&LyTXAxNe&$hm zFEed^Z)1eY2WjITk%@z=mh)u`D|_K|fP#0c7#!oT-+_6Om2NXigsr*Q9LY4<;XO@q z9?wDUYJkjXx9I;7xSQYlOe||rRe{;oKOLu_Irwt*UkpPJrdmgcEl83H(9(H|CDY>q z=wOv7zhn#f^_K&@dKNT%dpAc8O()i zq>0I>%ZWK_WknB|DG~eCl1S9{^e1np+1NI*B8Vyn)P(zfpu-&xKScNT(o0j`Qw}uf zx;NzKzX%G6nZAD|6kkv}W66MMF#9W$t;gg*X97C{NqmRf6f{!>BYi@$t`$Lzav@law{H z%&1*Vl#2b(EKG7`u`P*SF^<#5W%GKWTdwK%pPGpHoVaFCaOu;JOuloZ_ZJb9&<1Sb0( z?BpW>iYk4~lGagFeq*cV6h_e*sLm(zl%0G14{kKTSbCX3R!2c~{2YwiB3rTiB!{x1 z7Ev+eX2e-$G)$8Jf`}|boVDo!c%LJ1n>%|r&n*nVLZUyFymRh>sw<<*JzOkIZuiaj)Y=0|zcp2N80CQ7& z!Ll!NeD!vMtu^NZi>M_n5$-e*$};K&Q~iqrKm2n{#v}zMS>|4AA^$TMa6TC|&c((T zOxS}nCr=``wu-Ue_{}P>NCaK39|fkiR|D)Ed>zXpyxSLBoUpoBri`CIysdR-P?W|`%`h3_PeB)R>es5j))l&tRW@VuN zog2EU?XW_O!2}*M44>gl#Y3Y!@m;(mJa?R|0I^jsRIaw}uZ>9*t1#@)_XQ2cc@vlXfv7`|2}1Cg8e1o{WBZyb z*kgT0(Oz-i-Aj*CQ6;epccZvODHwkK;rL&|j~0Mwb!z_4Sa2rt95I}c7I3T%9K`Qs z5P>-J(-(2poU5AcyH;c!REAKoQf?hr_1Y#p&zM`>s$~ny*eJzy$?KrL&(b^;b&Rl9 zK_&_M_FfP;WaSh-_F_XV;@mLzsb?gy@5ghJ3%89rY*Twq_P~O-iCN@;Y|cCrV8%Bn zu>k#U!egQn=G-xsYFjju)UeRqmUxO$)L(pA$0@M07OR4@gMXF*lQQQbhq4A6(l`9` z_X@NW|871@M1w%KiqXlrIDSI4XaPfV3T+>6%?VxGMrPbohpL6?Nm5jvy>C9KB4EAsge!qxV zeqO%!5ax~j7Yt7@p!%-jB~DpK#9$ey|>}bPN4E z;52jj_SS=v;9H0)jZuLqV_^S`U%(=rI}2FAkbn=BPGTHuL}0}4L}{F4#ziNhDgwn_ z;!E?kl&jinY);^|alA}3XXUvavH?;yah2+KOEY{JuNtBd@*xluL857()+I}@<-7w( z2fC|x0(RSB;wCG3>0GdMU4)1)a%)>uvT_`JXhcS19Qf-qOH(S2qNT0kzaBhcT08;* zd_qDZLVVl@@81WXj$1<2$cEM?6|90(BikQ{B5nf!5Oa1lax@@Q`|iW;$4(qL zS7ZBbOYlrNDw1~A=uGRC@nTAK*cpy{+L=?=LUiZVgP-&C=X;-5t_SPnv`x9p*^?=l z$5(c$KWg;r{{89f7M5zsgXZ5!m=+^x)N=xCeb=3xV`$V+V2*h!1<=B2oIJ|T3b;Qx zJdwZ}P54ybTb-VE`t0o!cAD_eauyNU+IMPO}2U()OcEVCwPb&1y<>j z4&=mJdBk-&JejvqJlbZD>=UfPRte81A3Y{&q59|`E%@@o4}7{Qn}N68L$ne5HdN!; z2ND@FQzkNRC^YBhK`3!=- zDtR|bxU|LdaOFX($>QCoY2kv$GP_(hwTX#I%W$RI=}#Q2dqm!GAALm!?v#bp=3RE^ zpy_;4!##Qd6Rt*jA-1ML^+MKD%mkPXfaX(t4*9pDu=lMKu;GTbAOwad|tdoe?1+}A{M!QGA1vgo?;+~Yj7OYkZ z|Nc$mWO;(_Vw*U~_W7eDttZb3sT0S0TDe~r36F_=vceO9V0M+&*^?`Mv>#DSnAV9= z%2$?Okov`XM#ON1kr}HT*zl*6Eq3;3lAPc6e^-wNRguj8n1cJ#5zzt_d0y}E)N&mc zZklFiR7(m7U*aS&C7IZ3amL{q9~l87|KYd$FkGujmzTnle4`^uPvc(dmbQNSHUW&( zVUmoyXk?y0O5;0iL>)qhHkQH4q4`q@Ie91#iPDGSR5Gs6I&Z;=T<+G|*JPB2)}A}v z(7F_ruwUxDC}EOy+vCDdM4)7yN9Pkts5m5GvO zt#44&h$S~H|HAw7YU*@GP#>otN&FYDdW40JgUn;~Bv86HZ9iq=+w)Zl-f>}ajxmB? zx00MGg?%?<{I;)~tSJ%sd$m=KZxUpFExg+0aTO6*;FWvZ6oKk5WT3M+5NJ&^m6=*z z;ioN{L5Pq)CI*2D`!=yEFg8{KL8uZ0q6voV-nksk|5 zwUa^`3sjIvW2>DMvnP}9mAcdIn`(DR2LHp;5lKJKAbniBbjBaoM>J$Oc1I`h{Vn-g z@Xk7ysWNjYtOtB735^v^sTYTqL8-0w8OI<8zvr1uO{(#55VuB5d!O2FsQgWPSD zW>l-K5~?&F3obA)9hL{hx%Omh)O1146Qu5!g4L#XkV_*B`uLnEHvWm_fiR~g0Ud=y zevHeFZ4!qB`I&fBt#j2Qg(u%G%2H&qER4eANS9I@_Iv04*a7j>5OWng?v-?4U<4abHxVTboVNhr;wb{!xor3IS z(X@<|{4#RmnPnck+26Z`QD$E#jOn^DGGtEKx^{J{Rr&^p##+pZe|Y~oKK}Ylhbz>D zF*+uznFk`I_3TW2GdQBUXpz?QFEP-l)hKxOE{8DP`RT%s&Ot&eu24oUFbmp*r=2z;ufI2#I0u&C5BGz8?jImcLdks6P>U53fWxrm%1a z+rGB}X>n)CBlcQF*lIZ%U&nycRA2m5vt21Ykqj(ZSc>>g9$O17G8kmqpidCC$z^@` zvtdvl9Lzvv&agjhPQ%0J!g3%6X*s(=rn@v#Y z@?Vi9y6iW)g;|kS9FDu^r(ZEF4H^%yuxCd}%m`WC(@Awh@Kq{BbXQ4{7%^YUF|ip{Y#oB)Sx( z8VEDwQ0=clkTKF~DQBr>#44X6UZGnA5^^+X`4OWHAY|-A=Y}{il3rNOqz{PzzXT9s zJxA6dLeXc)R00LOhoQB){VMtj8vSH9ivct8t~NUr5lzY zyd{v=NS8)q_ky%%{{V_4*iT`iYz^YW%3lL?Kfpkxze)6TYcP$!12-}C7-fN2uHA)_AWTjwdf@aqG*txZjgkb zCfp<>bo^+)LgOQ0O9v4?i%3Z;q>FGtHzJlGWWyR&^M*>BaFMwUO=@0NBw3W&yFm=V znTP)XgKaZDz10i-V=CR%kZrZljEkiPuqlR4a3dDALJ!k4%q(r*;=!m{M$p|$fetK- z2rwXlo(Vp}I*=uh2?dU+?hgE-bXY{q{sTleWGwKu5{`ls=_jC^Nxwu^og~QBWROg5 zh-(tYoSX-1qX}*#r0&+^9t6#*LhKn*{R+tE20~Pdhbl-HmKTOsxbe)()fAufrceH0 z4mzim3Gy><;n4yyp`jweiotj2WNiNc)O?CC`iLWDd-O~4;+5&AQ-K2#oSuT$TU6*e zYmnIGwd(zZIH79)0Hn6xE|M(?kJ)II(Z-$8A&h1yek|L<6tJ7Rm+_$uz+7|w2)oCJ zZOHYft;D|sSY&w%ut})jp&c0egaUtr8i+QD9IC>a&vEsSxd~B1R-)X5kMUaSn=q^A z4B0)!n!8wnLQop8#HO(%P7q9%O)g4{c4CfHd>b!xT<|c-B-E3W0LF2F9JG)@14I$g z0}wQ10zxU1&zVH;ckoyn*eA^sK1q^q`wOI`8DiKwI#SEN za?t(OMNFGT|$C{`^Y1&xC#QQDZT_CoVRRpkEw2JRi*18HN>w4MuDXE`aFiXNnP z7)OCCp$$-*sN10Tj$x-M{s}QB4Yn+o(8H>cHZY02v!WYQNzW?`Yx2DYn2)(S&o+gz zeTZZfsSMl+51|^EsV$ieDZc2HR4*AEP~q^zyNiZrp85(&Or%*b(0q`HAc6*n{0l-k zjxC9hiC=m=h4Xp$noTp$&a)FtV08$xGNoSd#ZGqiUh5 zG0aDSg7O%2EA(SzB{4T8g*8vw86Ox%Ft%6kA`PErlW`vSpV5l{0HSZ8^xf!CKC3d( z#V<{Rm|i*%VTCAx78VyoF}sP?GB_Yv!t>Gh91V1fWQo3b~APMC=ctLlL!onY3PD4 zsRVRnG@eAGM2QaeC+pGW)<){EM6H=*8TtsR=H0oNVLNp(YCJy3gsqUYvNfbVmT(T(YeKgePSi&XIs1Yn z*1L_v!}x9Z7~TQkPt<0N*gKIf-pr<7D2<(lXf+633Dmju1ay65u*E*&#M4DPPonsA z@P11n<$6$Mrz1K0teVBjMw)QcK6);=itNX)E>)IDeTt-aWREQzGWv14Hlt=yTkI%m zQ<6Ub(2A)Pi0z4;quDYD)li&e4w|PS(#O_7y1H6p?Wv^Kp7or*dM zM2bBG5KE4ghAC$0*F;9n4(LIYndn;$FyvYG`Z2b?;WHHA{qW#pn)tTjVnERXLRuug zh0+Y2Rp4%hI57425;itANXFw4yJD}QEbw30+erP{HSu@S@OC)F1^b_6Ds9bqNbND2P7fh>>A-?EH5^N^KSkP)9 z)1xUXqT2iwFplJEPTYHo8)UwWX{Qf9oeB&fh7q>YxZu!bQ#C-0+U(s=YFr6ua28O4|i zBkqX77LQYWpA@dN3<*S|Od1h)ZVPGr;MU*W?#(<@l@%u==k{iScoQ`nVcd7JA3iE3 z$E`mm!{9=_gB~LAC&h`;^c|2Rp>$~10)fKMmsCgKhRUkZoBqc)VAcb&P zK_M=PniA+o(!`G9HAE8$UMOycF)67`VMDcuYTZe?nG@@}8HFQBwl3TIz|&eUSD>n- zmw~q93E1T3goIlWZCpL|8CzSo(Wlgu2}=}>UF!W2A~b}M(hAXZ_{F46rF&h(cQBLf z4U(=eqiM2qu%Vmi(3hx$p4u`NL`dcjLPZbTh=rbpKW*l$%vWXp2v~smf07udoxHqOosg?CbIx7de&W>*8ipQ*AOC8YbIzi(#;0o@Hc-oimpWFtdVA&CMaQPWQpFt=3P5vAi2P2(OpkSL*WJbX9q|zHN1oV=#pXm&1z}RjqNru<&&`fV< zJ@q=S`p&rKv1rjH^K9IfH zlXs8M!&Y4?n5n0clgC03kdb0q`iM)SDRS8z5)wC7K28i=8xh{8YPq^fT|@9+i7YzL9D28VIgF%sgx!yT#k(X~xm5k&V-)`Y zYaaaybSX1KzT2?{&9VrAR*6SP)9LVRqc)6#C|m?Gv zdT*_tqZOsB6ie3v^H}hSH|mBhwd#M+NVVsEhC8n4rRpUUVqn%fHfh1BciP=&IO68wqf^~xxT0Vz4$ z3ynZbHZ9+wF-xUn7W31ndt}?uElbmRa!+kjUFLe14}`2mZs7?#4wf$X9-{afR3YqY z#)S6|B#9um$eLQploLFOCj6hDu=p8dOVlk$WY|eE=${OWVTPOJ)g~=;ErD(uB5N`^ zvKK^LB2ch~b%R+N^jF;rX&`3^C)F;A@XjDaOoqP_8rNZMbLcHw4@$c@A;+y1%e#N3 z!tiKXI5t9ZOVn9tEvtgAX(J>&iYuO7@DO8|d#EWYKg4h+UeHJ;e{9GmpOhY`c`q<#)mfZ=a8y1*raKSEpBtEE1s6(io#~o_5GPZIzLatQc zlsk}Qg(qDkg)pnjf5SxJpl(g@8TH$os!M8OO}+Y3FBGu@ut+oS0B9uVE1bFojkLkHseeYYnW z?s4@F=gbd#;zSxA3fbimrKjYFtBUAWqgnibl6W;}MhAA8p|3oXNvF8nw{k=Z10={} z=^*fqHWM2GkuPkSF|!$xC%ent^g<*L(EIECeY-)$C67B)N>;9{SK z;`c`}c1c}kS88Z%bV!Wme^P-addPXHUc_2xRJ$2VQ}skeDgOWw8iug#7#;}9-cDAe z#Q6yH$z#CbeyIG2MBQ4z%%`Ivt=IHRWHJXc(JZ}$nu<&zD3)hoEV2wjUZsU`ac$gS zYr{(HC-##u604Z3a=HQ%k~x7rR3s#cQP^UKfoHN92eIl?ljbl(5r2A&ue48X`Ta*D zMp5ABf_+4UxyjvkLzZ9Q<*v*&{{Wj5vj>Kcor)NizrC*vdI;SWGWyVN;f=!+HyR42 z$SEO{>-Cz+pyPNLQBe(skWoDf($pCDZAKHBvqs*TEt_3KU>V=`1DCn9Zn88*gS&w4S>Uc)ST)7)^lzF0TiU z7SeM&cSAY)&w05TW^H zU6Lk}=1HHh>>|F&t?DZ6s8RRe(~CQC=1*K@F3fW9Vc<dodp* z{{Z|zMwd+*NR>!rlzK?55-Gq_R|1ME;3jXoW;J`{Hl52vG3~~SacoWITNJ^z%foHKG`c^-nEF)1@FYIVE9>d^Qi{cIGzdyF*TcdTS z;BB)100VB$%;i!#Hg%{61557glu+hjsF02f5BkK5-&v^Q0fq}H-VxJN@^2yAX4{8{hZ;8Pt=Ltc4Qex zzreNO+}JU++VlN3y-4$Uf`AL-`hlx&EJ!^>uVL;Jd&qW(>EC>zCo)af;T|#WMqF># zP9-$cg4}PqpK+*7K!rHli!yEOf1uK?8x~TUYL>A)iCLj$qmnmC0$?j}O1NSo6iw+i z(8p%gqe&93lP^)gOWMdPZB5eT_#^5k1VlH6EFj_}zrg^THk#FC4s9(&+$V-UXnB`qr45gbX`{y4yY$iPu-VrWz4-(9zpC^M4sUGC~KnZ5Rvqg*%$|uReAXh19ptQ zehRbd%l7{OlRm$8Ume^D2IF#yM6J+mU10wJ7|VJ_p&kUhoQPb-yiZv2hpXv~EgM*g z(?o1TQpZYl`XtLsBdA>XW0cdlgc)+#lQ9fG=E4o7a`=xvN$9uoYWg+#DkerAN*grC zR4L?*kG4o85m91yC8~~ZQE7(CJdoqoi0Aeo*Wr>|JPyj1lDe+MKQa2kEVl;Y8ti6D z6=9+%-4=&@y&2BDui)aj_H*(j{(@xV**^lu*fXEi9=IH7ptH%mPNKJcgM<-oKUfSg zmb6vL`xMe+$kR|aD-iW771gZ4as44tiUegDz-ug@tPoMzr$HI)}8o zBPi2AZzf9ylwugm#2))zOqCqq?B$^oX76VyNqk{qHTvxbn z_bc&z7uwy7o4d3r!p@gCmd3ToL?xRnXNB$xaXwx=jdh()&w zJsDn1aVd@1<7W?bksZLAps!Nd72Cx_+!~{HHwr^bVP7I28k>l?EOce=EG~&`OEny4 z?ZfDQq5KF}DFav0t%b6Y#dPkY(1?8zX3P0RF0JfA{4qx;@%$84hBlb@Oc9?0j zZB9osRN9Jyp}q@M9KjNcxh9HQjYZ5#bSREuo+wi=5|aqfNU6w?A{pGSipIk6G1`H- z7u~N?=ysz$$V%l3Zi=MBta4^e-vcwejQWD{LU~tpp*ZG5wor$H$3nD$OLdLR&h_vXoDeH&FC?nH4Pkh_j?i)*wSOcI3(D z^-s~j7dueQ5pU$FlLDN{A*%J@P6;B!@gA&OyY7!TDJtInw;f!NIKVs_+ z%1TqUk74i-xPj8-bCy$VTCuJk32C}%!cjqqICHr)ys}KNw@rYRXcLWqaYJPkJV;-( z`Ip?)5(~nUdPll`7(|}%Uqonv=qNE^{z7yfO7eqH`5G?l`2;qW>-83dYEy@xiPm~k z>@=CTse1k^>TXC+SWVV;LyFo%Ae5F->L+Zj6^wi3mmBQ;l@Haa^I=V0#Z~xPA~(!K zgVZIV3ajI{aph7>3hpG5+mBKvLet!095=Fc6W=d69iN!>=!fso*i%_2VOK47GnKT6 z)umhp#2F?{rV~p+Uh7urc4Ls!W+N-Og_I{j?M2|7t>y*(Ms~)f$8)$m9hp4~kLWeY z>`v!~R9IUMZy+V^i;Km;&jqXCt2BM6})|H_6W)7#Z zC$h)%T1zJ6!O6%>Yo+!OzQIegs)QhTRp-RA#pwm7=W|GBvnk87d3J7u8)wyT`1}u(d7cIq7Mhx3$)oiqLKBor%}wf0jBu)py-rG*Hl#aH5^XDU zxf~K#&IZ>lzX*c_2bMy`OiZ1v1?aTs_ali~m7^*@;gmge#TvxOgga6#<~<>akNE*2 zNa~v=ro^R^Nr1tm%TL@Q1hzo*eson+*fUMh=wlgr9a^5aN&G0;c_!_a$$2S34a-D&$IQlL z;ciI+me!KT{{WarWLxZLlD1~EjRsC=MQEp;!h4B25}^wST1LuGpn#{X7_X5M+*X1% zE#8GWFp_RZL@`4PR?>t_p010HM;WuCjl>o9kx2<>v2L=HP*h7+Rlz7j-9(n70nP-7 zSd2hUG?<^+c$oy*4Ddo|PAqAhqmP`EUxdgCS@Dqr4=rJCT+@E4tmrQZM>})5yi+=Dq zgphJFl7x(-)5Npau_ALslT2<=DsR+NNmI6By#;Q-gqEcNHR+9`6Kc0HMXWMLNUuAu z&`3I*(V;_fmwFk)*#^McpiI#NrA*L~u_LXYA{$FBxaL_p)ZFsEsh#2y9~1p8q$$O< za7kmlH~m5_69v`LNvmR%O3vT8o=5XZDPX4*z6NsM5aBq6o@P#bf3Sos^jTjb>LZ1; zA%O$bME*Y{9k;-`Mu7cI51-MIBc%pEBwh0!@r$vW_JJY%Ay<8*w}4xqB3fxNL0Cih ze+8d!A|FC!=$>0hU5_E;F_Fpt07OQS#EC^xX#H5;a^6xSY>DBY0xZ;Y@N4iB47?pr zpx1mKsN6L6YW)-uF}JD{yadFNcShWj7c)=Yli_SI`vgHthcGcVpGIIB7m?$pL??X{ z>at{kS*X&=xNnB;UyL zorR!4<%Z6)k6x{UN0*iy2#*gE*ZFkGN5#mi8*ho+fnjqk7l`472Pf_?Lzx+VC;9wL+|+ttWwYs8R4=(Jgiyuumh%$54m3V`l`pgeh9SaUz4X z^>4W=5^%=VxY((Pja&@lcGmpZr6uL_em5a{RHX#DI7Pgoj*>#y)TiY_esDH}CQVFE z(1oCL_8 z%R#Q$2Sss!7p)K_fG;Eff{?6rkYNv#SbMIP6nZj9!5D93jCAuVHQNTYdV(q5g&BJKGqcVZ#V<0Ph~Ac|bY zZrPOAu|>qd(&)a-mz!QurZjLrP>{Cv82f3I=b7SiMK_8k7Mlo%u=f?3T3nnFS`%Ce z0fXFbGc_$}N==qLB%#`?X66;$AAv}ORjq~Ir4L~m8g2}twH`#04f{-g2f;{?mXznn z?QU<1Y5W|{M9moFthc0!mffCki&u)bm{>$Mv`QO7t%k9=1ouPrJE1B`BT_2dfJ3qE z2FBi!B25Y9$)VgLY`cpt$ajL36Izomq7XwOPiS|eMfFj4$ZtqsOdDdZ{26LVXzmtQX$Kw6p$hjU-7*jii z1A7e(@+bcQu98^|m^;0R`$X{c8!@=%(?U$rNL#bb;8o!;ku~`m+G(DNWe40ALA-ub z(HawtVp^;ovN|N&Qgnt$w;uCiohNa_a^kn-Xk_avVvQyF$mR?sHX`? zLRkuE{v!26=})A%skfaA4m{Yd%nzA65YQ;FL5fC{{R*%z#T)l zN+9D8QZ#ux5%K)dWj#=}fg0C2cr&S@dxCdLVjPJ|jN_Kv3;31^UkN#8R09KEg5}+2 z{GU;*j(8Dr;6$PcQG!b~nq8#B)eNpGzu@%=ebG?0b-4?88#SPe{F5h)G^omdoDC$b zrZpcVsq&3}vNa+}Hnj!+0Jf7!6tib>*2$!H(~3Qd$@MQJq|%c~%Tr?2sEubd8YWcD z6WR1csYC|1Vyl0U<>tOkfJkOsJ&UM|v>v^f`43?A*7!f89Pb2pz+F?_fZhlD-BJ`sJSM9siIZSzGR!kWVs z3}dOc1u~o&=$E}k8HjOTBeiXCG2s@Q%ArS?BzO~a5>ht~;fZSNf+x=X!^t>M=SbLY zoTwLbE?J>_N$57}UbR*R8I32FlLxnAgjs!{x)oAjRF+$D`y0Jc=3rKdxNzo0QXQ{h z%VSH%;cn#4*R?$IQJU)Sp&z;7kMl+<78fc}@P%r1VsA440EJJ5Ey*jniCH6Gy^rPG zG7HZ`QQ&3YJ(=i_k)jp+5BBan_QS`2D8n>>OxH&|MTv5(oqaz?+hXX{?1|r{Svk5k+i*R7qiEQd*#`iu?(w zIEtUx*%l<+QU=k>X@#$QE_O=ieO@@PS;9Cy@O@|hQogm5*E(++Z-g%_Y@4Q#1Ul!nq^o$!**BqYAU zyt0Ko5rVb~j4@T%pV}bs8`dGC;i)@kk@e+e8Adfald!VmAeh3DGUM+;B=bECZ*lJi zZr>y*(nni_CY=zi-1156$7sF|#@P)XlOJ7)6Of{HlLk-7j?}AgnXwFO4?(OWZSZlh z@My{Q3JmbpgAz|d$+o^y5cXPJ6>T-fqI=UNAvBrJM@g?zBGow7#U^1~ae4I{P0iZS z_{fsvt*K=?CRsRGbQ2Hv4wSzwp@NE%CFqH)lS`cHI1Y`ZTs*9wNZ$nsCbBUSC^U$r zdy%4R`l7RAMy4SiP2=3tud@^M*+ZR?Wrh7%)7f-nhc5Hlm+gk97hXcMBk%qng)g8m z>gc2XvNiV_cs-z=oO$~*_`aWKx;b8grXQfPu`ewOlW_ws2tA2g^iZeN12WGm+{z{XuPhAtYn}08nJSgj&kA`^a309l|;nK`7hc*CM8n4U(cg6zE9` zTJB7wmTqt_TTqncNw@nZs}YpZyRor|BMLlZMQPlY zlenZLXQbe~c2qF3!21hnI!cs!DjFv;hCYc;FAQJZJop?Y;BquOgV<~A7VS~JF!S2< zxGhg(+n4Oe^bv~e%o1=%vFrUJ#JCum#2(MQnFWK}i|V1LjVHh2`hB+Ou#ioFFzWNH z>)1*TWzLUb?|oPBTFiG}!8;A@{1lmXhwz;x^Zx(^Euk$XqohceveKKSmw~v0Rc@|g zF|3Vd))-IGeakDJz<} zRxJr-GEruzR-}_1aFnM6D^R)XLn(ZT0wDL{R?5+15pd}CKl3{YMFSxdBytY|1#)oot zTs=(5>L^2%hbW#~(UPo&=CS}00d*OC!PQ*ti>KM^K0 z11Bl$H59aS{*EBXqqx1gbJ3wt8bpM=KctKH2W%hMFf*G-J#S7-{7+;*v!OaN{*r1I z82*WuV1E-&!1^=lIO_)>NVN-t5I~`knO%B9j9hmczkpQ{ccRz9qm-IXeY!(_gFp>6 zLs?h33HuS+g*RBCrP9WnBAjxL-{x~(c3T;VPQ?(+5=s*zqF6HMkf2eF-7lFXul6FQ zX(8o+m84S5FgAdegi+dh5}{IF2*l_bp^YICxg}M6hSFT%xn6}#r%9%@PoXy_3$DEx zvMHw@0c8nHvY2`rEt4jj*{dBFfrZAgqn-JPCT+bpVZ z>_X=EBpk6s&TZ_Y$c(y}l0~S2+CIY?2YAZRR9>uvb!<1%D9HT_oEn7LVl)*yM(y6B zMyNMBSnM>ur8J=k=zSc}E#lPrL?rklW&ynuR)?qd%pzCNI|sRUxKf_PfGAc<+=9WR zYBcuxW_RchF2mZPT6f@Z_|LA~^+6W>=}|9>fCRHCVO+?5^x%NNj)U`P79G| z1?*nJPN+d9eIg{7vjZI=BAv~Ts}V$ShrCvHH`%;dWw)8N3pgC-8V$vgwmp4EF4rD3 zl{%ik-&|n)seXi4MA3x1k{vb_-e>kX6kan|xTKce9X>3E;V`LE#VDGHgGnbj9t70N z@?o}YnoQKuti-hw3#fkr%Vc)Q&m>D@9yk_Q(iA3%5pPILmi+~!w+58olfjDI+{sVm zMEQlu6$O{~Q@cT?{7MrRMI$jTL8&sn1en`=jwvj)5S)@uG#zR~SjhLAR6)2tHWDK6 z?9DQUhb1pmW(X-ZxfRGY=-`vVBj8^mOsixS32;s8I$L^9AV%!noiiQ@MdA#tJB`}a zDr}jXgqJ!PtuZF+3~htG1iT(7nJuO}45U-h45UhH8WE{>QR?ihq|{6ta}EdHC&sB& zBs3-+5K2>7i^hi`pI%to^?DkFG^I-5NeKlKEW#YJ0~DZ@;$F(cvHry_0%KGqm$8vY zN#4etluKli@5X|DDgL5u1Kx}=4SS92zKL%vVotmXQgyu*mn3p2Hb0<*Gfi%HAJo4j zk3A_pGZ$gG(2;)fB}1|k$li#L!X)WLl1SMj{jG#We#GRQjii+KF78bGjhlR)lO}!o zEUZdU$j4IOaat^m(iWj`u{*s>sz|M^cr+dsN3}u=!D2Nr*uBVMw4mCh;1jtQERwN(xt%bTTX(_ zqkN0HV%r!yg6iG|h1i7u08JK{V%o+bnHA$A!?$l?5Uo`lXDFaF@mLed&5M#wf--uc zWE!t@MP&UD(yyw@0T&+XJTGsN+Du*@LxOFW$Vp08*N#}ho3`@Zj?kGN(j!r{9?~rA zQs^fFTVqU#h3;B`2ViV{Q2S4{JgYa(ft}u6-HOhu;9)_=OiE&f(31*GcKB3{W0M?kN*HBa~9)FdyHf$sxTNLz^oU5Zi#q!?J+ter3vRFKU*4m z0rG4;TYU7GPq(0qT0E3w$>3-w9tUdLnf{A%AlW^Mk7UK$<^25mX2ho+;%K7Y#}Y|6 z)%wcI88(X-1Ys!TO_G?ZF>r@6His$lJUK;~J%tLQndGbn)Xg!FuYKuil4{_SED*+r zWKn#vN=BM!azyG+@d>=7-^(9ZUbuc!GrAk0aZluL4`5`^1d=Y6U*JbVGJ6KMYL$9D z#OVrQp(uvPF6vLr)5JPL_)Sp|@4a3?js+l&z?Z9L$ildS82)QWi`ybQaw>O?P{ z_+nqX`!bx;7|PLD>uuALYT%vpK*v01Z9TdqF|Wa3vD_r|3m6F%hvb1vKAS@)xe2zE z(vgx%#4g5B8C8~|Do(^9!FNPx0F|S{BtuA*FiHv7*`RR{OlLF{l!QLC6vF`SMVJQ) zLD1o~uQT6~HN6{KsHX%eTolgU}xz@qir`Z2&H=(h=PbETnqW5;+Xl2cJQ zArBLea=%{D0n@~|+HPJ8-2{|Ko`+4NqqN=zp3mTFAGAxrBS`mjcm5u3xSvJ%?%atP zPmsf4lrcSrRT0gLlto3N)rpICS%+F|F!3m)8nXU~BelTLigNEN2Wbd#36cAd>W_3H zijESY)A;Qs)GHEQ(e)+t*yK(N~xOPwTPt(2rw z%|Ki5CFIGHof16CD0?CFOLf>qEe#^gSx?lT_!td~p**}jx->o^#I>F8^Lx*@-J8Eql;M4EaLDJC@AN=t2G z*QNT6DOx(cGDfg8@yUjNNRLvj;b;+F#lGD-JRzfSJsAY-`z;pkm$8m-fic)jXr(os zW>EGMwx!&JQJ_Jp0$cfY1(V6haua+JT}Gk?OTgCzRIZ?tPZ|!yG77APB|ZC-)3~Xg z;E1Wtu|LRC3w%E*GN3TpmaAbn)}Ajp=p z3J;~0w7rxgz!ny|w!OzcV?HP&DtwpZT{jk94ME5GA0wcoXt6;wEeCEunt~%8$gVLw ziu_X*x356}dPIvNe?%-Ay9si&GgsxD4?rn&lyMFz#T7$9Ds44e-h->0BIlK-8Vnxk z(GJtbqyvlL0V97=8g4|mKE@-i@iL^#G^F~G_96feB+mg(5*xUcYDU_GEIEZX z@f!;xop*LoPirBr%+{D!u0_M`%)>aE7Vm*-h7xf%Z@~%eN{Kq1Yt+U@&Akki7zC7Y z4G<9zVoEw+IoO41c@XHd+Mv0Yu`Q|&c6tqF*z3?2%!iI-(Hsk+sxb1zjN})+=uT0z zC72{7Ujb8}K@Pc+a5tg@swDdnNx?MkcFFyUMEM%%z$mIeaQV(p}LG!_zf6^=c5fW)h{}d#fWW8 zgkJAO!BYzo2ey!0NE7`23va9W`pa=6*WyPlPl4GZ^!H`)s! zI(>{H4fjMh=(EY(cXaw`Pj3gvB#^X3$Tp5FR+G*N==UMq8^-TNL$KSVkG0_ zJD8MV(K;1NDKq5O7n1onYItEqzebLuCOUDz-WBm*xPYL#Q)J1nAe|x-%9)AkxGvI3 zM=flCS01X98yJm!Pp4 z`A$Ti;S0aue?w!PJ*1aliT;Bt z;mS4lLTEfJcL>DFZrHHNj_$os%`~dF7R14g6b1g>L6Qe1m7u?w4%LY3mYDH14%>&)#j|Cm5 zl?LFS{{Ygzp$I}^6A>bP36s%+FX0(V8w33hS|;-nC)c+~%4yJD-QyC_N$!>I^&aed zFtRayx@3BO(76n&*yOjfrr`s$B5yVuRz`XWek<5i}1&N$;V>{VG`Q*=|U)7 z?cjpBX*xvh_Yvvz?l-+JKf$g8!P)znBSdG&niJ%7vu1E`I*fmVC~II&$9gD)40DoF z!Fn|*{{X>Zn}imWT=^)zW1BWvPzZ<2=-5`a($6SCCTYT%oH^Wi0hMcy5M-)hDps!O zFZac(wC%!(QT?l9L;mB|Z2(X5dq zlgt&yR=t@d>gv688oZdXs~Vjl3Ir=5swHOSi%ohUzP?UK-_nP&Pj%2DeX>UGgqn`- z(jMWDO18>h`b%=bCS%42XqOhUOZz{FODz;|J*?NM`+5MaM+a(Hn3_eTS8iN+C$U` zMg*MEE+k7qPEwmoe#GV061m)jF|Y}6cO6ujZ8F^G$pp|>7)|J>qnNGb?Mp0&j=x$|o7kqqIovDpEIdV)hUW+g*W`@v^$npsz(Cdm?t3vM!sEpI#$h3_&5*QocmVQSs{4=>&(WZQm&V-MpG3gt1*&NK5xYLw5C$N*@-VT43koEha6`un$ zech5oZu|cL2M;}vVRjJ4WGdeS(AFnIQJO{}9T{{-Ou<6{$`ZGhDxKG$N?y*F#K}v@ zvwMbx32m3*Asv>9FzzU-CD>LCUrrZr+@4sw9}<0 zeff`pzrp)txIJg!o$ugmn*9u)a;tNyl5MhI^v7+2!zWjX%$Omxk{5sK9f=NrsF$I} zNn1-9ZYmmtIhnH4RZqa$C!$S45p_~eJDQUZB$7QL3J+8yQFB0oxenven=$vndjpg| za&TT$3{T*At6d%pV`p_r^+&#cBa_i3U2L*SiwsK=E5wcNOnXF4nS_{TA!*$*{{Vw( z6iy#thKTglcpVw!yk0~!F$iLGaA5~jN+gjQyQ97kn>K4ZkJDc3YqKEM?V=-$Q8{JH z!pW5=%9!2uCht0D><=AAh4Ky)V}6@HNV576@dt++hB^uGOQRDYk~9tQNU8jfU3Y?Q zQQFPccK(R}0HQ^3kGnA;(H*B`hWz@&T}G9XN>cj?V)S5I-Qar3?PlxyDET7M`-WFg zUgF9cZHJ$W*K#bVsnyeeju@LAuayGy8i&Aon$Q%UczcInuu4E zaj2Ln&`kcZBGr78NkEmizwomu)HL$~7a)JJy+}loPei3nYR46%QoSi{YU6C$ATDPg-HCmjc1(AeMw!HB6&ouNKBJrS0^-*e*$Xe;{6YE zaW8BXYern`J0-hPfLHB3Vn8Skj(_WbW={ z33U+Y#Al%Zr9fK0r|i(LBMjX*AEO&6D>fP2HZaFhx^VOSD4n7}qaE&8O zJ=n?&6N6BRD4|w>#l#Ci4%j(NL4{PyH(uQ@1fT~-@F*Y#p?Z*DS7_@MQ_6tDKU~s zMxMg~>mLNrpG1?=Ui}VL7SH=W7XJV+B9xM7+^Fxl!{QcsRFu@vO_ob341qGf7Xgm! zM9cOcRyLn997%-;bS|thk`a=|W_KIZrbz&L@}@P~^=vV;A7W~t*hzUu8VxB!Nu?z` zozSLSZy|$K$!=F>FC{Hhv94ufO0=7==rSh@VX5y>%c97#4Pq8=)SGtbeb`CwIw)IR?BpVj5NEuaY%yxE{BxAMi+RF)rgIy|_uM#SM99-N_cI3QA4(v0-_u!m@Ln|c{)CXJaYHw&C(x1LcqD9#s}htq!7r+GJ#HZHRSLPf=sX2T%~WU9 zq%=*hLm2MTa)+parw1KK#d@YXEKpIjx1BGMv@x^FYjn>}{TU>>5VXOO-WrDPJeqFu zF)HMQhpfZ|AEq9rbB>l|SIC@q7n4gycO*!AnXKOn+?HLDFWj3sol-XtjYj&&(!fMY ziBh|NdH6yv_TvJf@SP2*fG|*(8$Qb|a!#TT96! z5~3z8)?*2aMPz5_%Wg@6o{6;N)qN%>wji}RV$tN5m#zJp5q3%7hg)8P&^ir9H+@OV zz{UM`G?h01gxVd_c3@LYgn{ap832oy&;(li?8vx1xvP;S-U~ zOneJ%drS8v)Yd1vq3k%DB@^VRX)Rv^CeAEuv%G83xFpnTo8Kf?(PP>YMK_Xw?QR$| za)tEXy7&?vYTJc`;u?vE{Rm2~8ZVe8h^0Fmq&FnyiART#+1f5PHm**}PFYsF?fN3@ z7iPsVI7y6crb>wpA-;9|58Q+NQe2{U5W7bURQ?bD0ESW1vFw~+ijpQI*T`8GZDeq( zwPjR>2$S6tX{l{AWM-6bOIn=6w`TfxG7>~JNT9%1M|24hC!pF$v6W(6MNH(hLnNO1 z4oEciAJ!khVU@`FLv3#(u|An>PMTQE@NX{Yhv^uHKA94jmvM=`h$jC4hrrry{lqm# z!z3@-GDdklf~T+{tt$kDM`9bY{l~gPk-oxe*hpK8E28PBoHfzsq%NzsO+>`#H%MYa z9M$Pxp|4X>0O+MJwm$3&27=h3HmpiCYp?o1!o{Vb{ujjzUE2H5qP+v^*wxS|l-cN9 zQti<+I%!0?Z$ZT8&@nEPQxl`?IE)$xcPvp*)!bWmW5m`JhaQ9un26AG zBxPcTiMZs?B^o^$G$neFWH*fv>KkUnJy@I$&z#D=3kkbwi%R( zY1pEDt`BlE4mwFTOQ2^Vx&~P^FM$x6mOao-dk{;{Xrts&#KEDF({1w~9RP^nILQb_ zEeWBLU+ZEP;}4?Kqsa5oehZ>~a4^M%g0fO%Ylng>G)%Uu%zp-yi!__;cQ2oV2Y_ys`RQl;af;fF$(n9Qi!L(#5ZWCFmVE@RI~F0y=y_|{ zogu%7V=ZTp(O;N%3!-VoXjs#g9TJWa8}iQL9kf~zYu8s^1ea0}xe_bc$gG-=`Yw*U z*ZhfB&IB&IG$LMTwP_<|DUs$qe>c*naRNDX`!<9hBjnEMC6QFF$yCrO7PQkz3|H{{Z$VjMoB-V)wx*e!*4?Oupo15?PyT;DAz>bDap++D)kkB=jYY zLe;Q5%&l7}F|WhHNh8!Xn^;+6L_E@fhLLjmtpzrk zWBnTs-4CHWsAP2b3Fcq1e?+%=HbIBT!2JW3?>v!eF`kH7konlh95x`ZaRP?7m=!Rs zTKyJ7B|b<+qlF%VZ49JZ*{O#{KA4R^@#HX6!(@7&tNw+tM&)ts?+8ZKAuZ70 z5w!}rM2#mo)P?T`IV$P348EbPUh7r<4JjdSr>hv&CRP`gKAoj=RGH^ea|shKVjO}L zr@2Sxr0=8SH!BEkB}KCRj;Bnia2eh1^)iUzs9N|PBWBSll!$_ug{fz#V&$%{0<0yi zI9X$OaVXIz1R|JX3t=C;H^T0tjiXH^&3PUk+8M3hBDc6lS9Fj zjTUkvZog$wLJb)M2~GwnW09kkxSXLe`q284{{RihLnY*f+=r;9O*rKi5cY4*1a*P9sOq5Yt4(%j4$ddiWxFk|tA7r1PFtCj3h)b(tFPu*%7msiB zbLaLyy}!{v4fnYnRmNyrp28(5BiznDe=(23Y@V8QWbDMR2x~sH{lE!233mMj zBh+p*!xeM)(3{}D2k2&kbK}vq48F0kHAtrSHQ5rSvxv>dVZ}6-?2Abnals9qf^`(+l8q}{w zV>Skbnu1S>9f_*({{TmpDAI*a$2^N^)F})90LfVlSA=&=L9c^2&IYZ9k?xOhSem{= zlOsc>qFW{y&7`es-Nz-gF{CtaB}JVEq?`>QK|-=eSAnEIxO2XwN3_pWC$^dlmU%4& zEulEipqY%a-)^!})KsEPKwwu*H5QW))=b$GwOXF#gcyv^ztG^>nbhfE4@FZ zDGNZIm)~LizrqiPx9JgrehnL%tX6pwiCm3+6Zn&fl)!H1P1%Q5M0;pd`s^lPk@;^X z$c&$1I<$SOG)baMmZ@s^C%`j|UU8u6fcJ9)PD);sXN<_(lW#|)@E}V@y<^fNMp5!! zUqo+(f@@tT#2R=KL`ZBxN$SGfv%wFpF++gO3=RV6G(iEO1IKVg`AA`nJBvvPa*@JZ z3R1h|yY>o`q!E& z_e7jH)l)J3kYN|bAECk2q>RM798<9}-}G$EUn_Cj)s^>RO?G3JI%~Jc&t4Gg8F!;B ztA(IOfQXd6#D%forF4{KH0J_J8eXsb6Kyn>K|T$0!cTU5=_j(Hs7h?8N|_wn_axBE zri1AAC;9966a3Z(G@97H8VS!2y^%J)ui=VX&OdfHUw?LCuO>O}IF3H}A0kn_klEZP z2~<8!9mx}~{f<4%6t?I_I#PBIfx;H5fJ!o{rLq`LLWQs)^m#YFHn8HG0SffrufEEUmO=NRu z!9!+KhtoB{$m%f#9m*2@`WUGoOrm}x_7M1gMoo7MqM3BhxZHXC%m#!BNTt|eB8}4) zoFOu~PpLsA{{Sbx!&L1MPXhD~NNVl4*(ZVYcC;o)ZMfTVMZFT((1&AMp`t=?!~G0~ z$gQ%wu8CWUDSM3N<~>L?PdW7|;M}G)JPAb2lhb5QnsuhMOUBz1C|*pViDObuZR}C< zMxpC+P70lr3Y?ZF1Lb+Lks(F&q%5-SFIHPjN>X;l?PIkSmgH9ZZb-Yppvw!QXk*DY z*1G6yZP!Su=-TwHMDX5sf9ZK=-}+K-@5stg=ju6%f`l~O%8hbtl#)>46VLWA z8?nTaNhT))rbAx>6S3%jM2-{mY$l(L1Saaql&J3Y5jet!l?=L*1e%%=ay}XdX(*SX zXdzkUg@ic#p`#IaaB5xUR)LXUgKmBrc?MEWsaT>1}AG926R0}Cw&@SW;GEt(P|TE)j|IN zZF$^UM>75hOpCq2AB$4V;oUDBEPY5#C&i4Kk57R}f!SD}7%uW!ktlR5yIX%pB=J^$ zh+6IZREFKP{j-`ja z=pvT2`{Zq5dw-v?zUTZO)=4=fvL;J-6&$!6u(%V}E~{%vgtvj|jeRNUw@pc9JrR2Y0BWL0eb-8&IU0SY-+F4U;BuT$xQq1Q2gHUhJ7rsWT$R;n%>^5vd8v zR@#VG+GBE)k}@VxHRBe!4jZy9`eo#cWC&$_ZbnBmsz`Ad-Ry;>6d|NAa6t3Zz~Sgd z!i1L?MTbeDeKEO*!WR2St_ko&=>GtK{{Td7sWDi#3GPF3!aEHbVj=D!43vzCt-@`R z7k-wp_K{0jcOT6bCW%5NyNc;84BU{^-8X|2uoOdefezJ+| zEWd-D=p~cG8&K2^bz@l?3T9Nxa3r`K$2$ul4Ay;0kAS9>-^jr2Z~YD>g3(*p)lPF? zV*$;ipQ|k%w~u0jod(!YLTONxQb&*BE2NY|EJ9KGK-iVD2l#awg|A;dBFCP&)JYYp zY2-w>L{iW5qD7`QFft~hQh{q}l8;2VA)?ben$5ROwl*}A!6`&XVGXMdpF`hC={9PG zPEtm#JZ@t)i-z&2+-736baq;kDsEE6OY#|3mvh;nOf57`%N|CeQ3P1F{EwPJ>kAmM z4rM!a{36~^(0VsyiOO0eJu>DEP1KS}QWqq{F3GtOO-q(jq$-Fd)1y3;$U~o32^o|Z z;$qXdq>#gb)Ep9a4Ym>#;~%LINILs&AhN?X(Q{`LJH)y-c-XAE5^N?(A{c6SUnC4+ zhlxaqwObBrt09evwAG2?AY*YhHUgRnjjAqI*TMSAT(*K!JAZz_km*r=8GDN0Tjg)j zj>tp>h9*fGSdq}AP8w&i@It8*?h52S<4O?Fq{?h+lReQYYqPtL>U2t8GTO@`xWu!xdt&EW$-N?DLDn?v*@+MAL%t9*DsHR2c$7m>s z&f>Xk!Yn{ItGJnT+tki^CBNd=1?)EV8+#lBalvo3ibUt(5*(h{n1!zI^hhT|_!b;> zE7&T3VuZ!vdNC%^J}9U8>3iee50H={3P~44$82xhvyxPYv@j&z5Z6GG6`p}AtK98+ zjn@vu!!3sSE8n0>bcXaoW}~*u?^0JSJ^ui^jqF+f00z&2;gmI$F;iwosDZUigCq_I zk(!ERjt(X$H86-2641R!yypX*gp!rngqFIGSy_4;l6V`QEsv87X(o~+o~(6Zw%Qyd zgnfzV`aO%J^<>C}(Im=mRjcqs%Q$_##@j(?l4>mic#;h|oJi?rX5u3x?XEmOqQyQm zU6TYa;Tj`9v0-%NFT+8}9tK*N{{T#&n|&b{sNL_;6By$x`DiHA%ftE#sk=C!>8?)b1g;(%ikeiCGUAjFWNFe8=n#pl==-&VCWAco34)^cX&e>bfy3mgER* zif!maV*D(T4+g2*_)+^E?k*O@(5qoS>npPFp<9eghO&l)&17vY43%FY$kS085P0gG zG4>SJM(0|QH4RE}$|tfG!pTo;tDpbR)d)nu%~+cD2z38cJ#pHUR7?$X_)dtgnD-;L#l}p zsarqJ&{|O^Jy<#v@;h^&)Jv7m%wy1l`7I?<$&B7~B=8 z=P8%x`WX`qWo$?-#iB2fdg3D)rlF%f76(G!b{Zuebk+twjh@4h!e?R0j1EbpP}+V4 zP&*N+q}s(jw{jvp?WR*ZH6lWRv|08gB8({svOPNpb<8Jj47}>aDNaX? z3ob<24Z9+avDZQy=%CIb^MMa9@ePW4PhI*l0f{z})(cc|>Ln((lzfTToxhcWEu-f7 zITGc(`}xZ-zWe_GrcVr5bZk!Sy-0cwqf-%df(rVZ2t{eCPBTgZkI*03P0DjDA?b8O ztZpcw(s{S56`1CDD-qE!vxL@r65vd4AL$dJZ?7aZq-{jQS{YL7OsNKn|xFLnvF7*fyzaE9z|WRasy4!Sb~91c1tRlZ40QNaXReuKSpV`7`juS7g4ctHtf*p+DEE;Qlt zM{0u*+!;hr($>gPk#HBWjXb1n98s@I5xjd4mSMuNCllqCMCCq+OJg1sYp}%yOJzdT zsldnAr{h0@GL#d*n(1E^QYN@2SYh%Jk~Dsw;El)F{S?iOl@?JS5cDNY1WWozI0?=7 zEy_7HFzUJ^mGV;xb^wJ1E`{p1CSm0mL?I?uQF0=`%3j{u3G*sgA0W=*I&? zXTUdfQylciCEK`-pl1dJ7aw$w0rHI&D9E3Ry~yiV={ZARgp=3ge+Aaqd%a9u6&Hf> z(sz53`gR@%BR&jG45Z5?twf@VQ3=bD&h5wC3{0hd{Tpn_u1u#f)Eq%Joj5X_-qL%q z*H+tesYd005ocyuTiuPYJK~3yr6+GuJ;_n>H#}*wk}y~|L)vzSR_0q<4o#%e7%!fo z8wtL~fnMxH-&((5Xm2bZW=gfB_L*r;RYND>OHJV{!OY-VMg-Y1gK#$NSi?`$h$5X?XiT~#Qs7g=Mj{|?7jc8} z(3QGJeDEbKMh-!-Qf`w!;ra<|wiKzT@-w$Ytava-M`J|-RdK0}BwjjBCDHtQuy`ZYRY zSCs~Xb-5;@V$6$CD#@%RG&*Imd;2kgrdlydbTJMF;ey4}-abarpAbtb;70lsbQ0rH zDJGeUNjOd&pj6&u)eSpg$wrO9Qqo|eNef$Yeg-y_)4_NZ92sg0B?&4+<&ZvzDm{m&S-j|q7o1_ zB-)kafZ}@@@Y8`D8)`Q<>W1KEgh<&d8|e2<5LsXFP0j_8jiruAaa)qyK1sX@yMIGw zn(!+rsZ@OJI2OQm7vZfR18zm7`jabzaNP+>Hi;oIQh$O=Phpo`HVa7_>B@s#qEVtN zFr-1BOvHr62IXCreTDnj@n{u?<4(k;tva*IDu&`5=GMLKm9!X159ybG6iE~@B}6y* zbVD~5dK)3)4xgfP!bceIUBE0@1I}j$XB|$JIs$(TNXD?|a z$z?V(lW_$#X-MBpnGu|iBy{97PpDm@$--_ScQi5C38O`eLdo+XK_fH9-*cvYeQx^O*en8y|O-JXmU*B zBXAiqRd6Ky9%dx2fpNA5mr5kKqR`M{NSSla46Qa)gv5L+e0-4!FL++!iZ}FLuE%nO z{;Y4V2CYdoIY*Teg;FC?n)FO>s(*s$@CKZ4rD|$o_51fCZm-YZ(8;4Up&xBTNqQDP zfy}}1&*w(fj{t@|t4S~KLdqtWM&NABEXCMj#V{egC{npE3dgNYMNJyiBYBG(5^dmw zK2P{WPK!iDE)dDO)vR&3bFo@jpiRij(KjqNFIL25mROSA2X(CgzqN=!%u-TkzN9L{PhBX*Wz_(322x(*2D;O0ty^-BobVyA$J-dL&r@@ zD1dLqDWCnKG|$^6UAQFeEHPaCKK&9W7dLL+Iw#S72~K|n^km;l=VIghB$j)U>F7)+ zAL!}9q8+g?sKyu!243AnGHlMfAKD@2Qp$AG!(2Y8*P|LvnBf)DR5%dE(WxQE@-(kQ z(98|+)*m9(O~O|cF_6{%#If=jJ~9@RTD?nVyv&YUiz3HquRhAe^}|K~03lS;7dC$f zm-h^1U68~{FJcf79BeiY3=uOUl0HaHU}=p;gwW^kT>dQAqi!fphcjnXHN%>xaa66o z{gZ!x_J3ymzWu@9Z@+Lj^pO09%?wZ3Lkv*TIzg!^z|k?d6EmYmy&@|mUBgD@FZxm1 zpx#iodv@qz6(pJHB$8!&*uu&QN*A6+QIRad z$2Nb#RJ725vpNd<V3A6J2I4hQI9lT49M8X8p&@w4jnSjtaC^8A`JQ2S_p~6prBkx$(+=@{75EH?A2GcS(A$&l1&9(uYhFlLrMMF~0eB`?eVSoM#zZgnaC%4boflt%+b!;Ii@KeBiv%Z4VfNKQ*}(<`QIYdtMr!Bo_28aZFl@z!pV zXzM1E9UYSLD@2T;_axY=)Q=rk@Ykm)Ch|l&N!$uc8(W`s46J)1gY0Nffc3 z;&u1yB-t1#y#)OnHYbS*$dLA93R}WUc0-h$x~F6LV;;{)~z=qh7=q-(=~`@->jQ+kNbmV7mf?L=6!lR-CQd3mSy5jSGTiW{Fl!ifXXB zOd05x$+4b<);K76p%0LA6ly$%_!zP&Z41ERbR^0Pj-tx&g~AZ2Q81uJgyJy{Jx4SE8Uwva@c} z@<|<0&ITuzJ+iI44;t%rTkI(vl2ks7fBEcpi7Vd%5|UN2brNmK;X>`PsnKN%JCZe} zb{<91XgZ^i(@pD#{1e}xz7-9zNh3lE4U?Xxi=_2%j$-Jdi{XdQY73C*K|+Y9&C*Xp|b2A?cw^O5JN=~^*y>K zC?OCNmO#s9A*tIm{RK3csP#nhWKJ03Jg&sqS;3(u5wAv(tD_D{Q6$4=AjoW%RGAyW zp(Bh|sA%sfvdZDX0-|{uCrKD2hK89SQX9DQgfH?^BvngfI1wcMv1op6ze6?wF!@)Y z%A2o)pMaAZG>E)2=cF=@>!paZ~nYE)Ro-PWX?{1oekksBb5j;G9e_09!H7LlBm%* ze#78?5r%SJucL1QP)QJv0w;0ca8E;`LKzvNf?W(Z10kEyBaK;7I%*c9B#@n)4m+gc z5jB`^WIgyIj_Si9!Kd6ah>))$Tn{34Z|qT#@I`?`Tr*weJ@hg;M8HB@CI$tt{RK3R z;vzyzo>^XmY=zwl45Hd-dKN<$38@^=9P!4Sti0t2N$kjIaMRiwaj=@=YHMSVkxCvq zJ(?8>5>2y9A`xV>35`fOEDu8q1HA|wYr`;+$!&>+42Jl7{{RDd(tDBTKHt;A8J|)! zR#Eu|3D~YtGlicz65EFdXrxTU5+G?kOhRYV+6J?Pu2)94amSRK9hCItL%gNO$+G^# zW60epb-a{mCxEOu5n6^XqLua-+=+c5QH*88PTznbUHo_EzHlDM__l3(4 zIHp}n`lki6!KrV!pEPo5K6NbJvI}!$P4#A@V4D8`$5}mbvORktq$gUyns4>@_D$NH z@;cx`%}gws@K3{{bVN{LBPSLmiax0zQQz=3*xM7n=2?`t)%+1c(!E7gQfW69C{IT* z;Fi!+9J5j5ZWS6evD7EY=Y%tYtNq&RcCfYB%HS9l?`6Ubk(a@(S>!Uq} zcD_h!6^+CcKM2t>zKC1&T?&E_R6JawVo7Aw8g#Li&n|nm@E(uVzv$-NP|xUeQAF=Erz5y+Q+3~NyPl^Se7 z>JlN^d0QY6*g-~!Q%gx7*6wL_8fW0UB7HXKmgtff29t9+MUv=ptZYk`U5USBE5(Wl z-vWP{rCI$)EJHuL{{Vna?X;Hc)30%D>e;OLU$MMGryWxvjC0cu{1nXzE$9&nQa;gv z3e`kS9mk&|X;}^D&|C`dhmo^zuaQv%$6?{v$Y>8NR>u5^ky9LWH6ks$fX(N8iK^^` zwSK#cR?FEUCF3HybKrBJszeFM@Kp|D$r6Pbe?jEA5+F*fmBz(#L})a%Au*=hD;|;4|PV7 za6i%@NR*9H=7=qB*94PG_BLp08WHWrbu}cCNn448*K{pY%7@&xz=B$m)3;9r%00)@ z4T{abM)>^3qp+KasMyCW=g^f))YoP?+|jd=Qyg+pp#zEOy@|DG;C{q5ie3XcE>*iC zI;Q5OJaLx9AyB+)=cr1RZG2#ebm4Vz-!1&Z0* zK;8^X65Hjxl)3IU&vKc^3nZcvA9XuM`W&-;7h+p`wjl5DK8|Ciy)UO|++WZ!p{1i( zvAtGJ2DSI#b5MwjcFDhAzkHMwp}?UokhK(~%OtKX(if6BP1STbX}S%hp?aASB5)i- zDd;AMNNcQ!tJ!Wv=ezz0`CE;nk>%LZs)GHrBJ3ri8o=U-Weg=)CCrI8mMOOIgUe_W z5(R?|oG8OX4f{IPUU z%d=9m57@#M@Aes!6UDf5mQhkU)N5T@4AZAq_$Cwj1~Ighw@38}k!;(-2uCIbdt{ky zq<0C4``oVC3EN);b3KF-5`j4l@3uS{t{HBgsP-DfZHh@po~fAhEELp)&{DlV(G9nW z*JQRn4FPg}QB9~xB}r_ANkSk#Mlvf>Y9T7%O-T-%9_*o$dk;JiS~IdFad0*U*piOM z6M|2$V;bDOMi%HYwIzg!Z zXndW_J=5XpMn;(TGCLY>jx_SNXR$kzXmQh$N1^0XCBWJ?Em40WH|HfyP6@L)L{T`> z^Da0Smv_w*9r zj&&GPlag$P6lu1DZKm%8;!EB77OdE&vi*b_`b`h1>DZzN1)=I)mW__{3vP`{Wzu-E z9pKSI8rZQUx4@Kr`~5!u00pz#V9?1EHZ+J>`RL`f z*jI#H^Y9W;3;vDn6W#14p(2E_u<|!mJ;ddS1oVMWwQP?594-1Yp1L8(z}sxF_V^!i z$0CJ29LU3hAx>FJk-&{dst;02zapae`;GjQyr#k*M@cj$U?k-swwBHjoAX;RGbbR=sc3LCkJuq4A6NIWbSB`ExN)t_YlVo zRvnH9p-D2dd-TE=M6HQi662;iT_NK`$t@|WY~O_Z5f#jIYAFztAJT8Jes6EsX=CYg zWTy{Eep<$s*_LwsSk}jJA-QfG{*k14c`dhQvwH^alqIqe;QkTG_WuAz!kZzUIA{DA z)F{|ru`^SJyZs0!2wh)Ao`pLISLie}91TTd*=~B&u^h77!pj^~?{t}M#2S6^_t5_U zf6+*j4#R-wGt06SqXWu0dPbsD47qaspYrK&q{m2D`RhUGwKl8KZT^rb_uPdkd|Y=C zRTLzI@3C<#x**Ge#34Bndx?F>ZMYYFDQ@E5pkqz7Khn`d9S&LyoM)pjSi?P(4y_v|unz25OK+&Pk^$?#yK*mxf zy^9Nq9j7ujIm{GnK}cBC(4TGI{{Y}{m`qD3{{Tf8T77|AjPIljcB5AUW0*V1?{3^b zLn=*e!~2mDwDPN^G2oJF3G6PDWw<;ZMo_g)#OWQl%QYZq&Mj-8OgTH^o^TRE-Pg)F zSVV@;qErnEYF#baq+ajR7}({uCCeOYJv4iYTZEkOYqP`l#_dKH+6;+1+V(|hW85)u z=VjdTchL`+$JGrQ^m;fJhp5>qY{k+tM?^fK*O-%=cH^Bp3NPHY`H?vL_wEM!;B4G* zLfId|JbwYmjj&8fw(z@a*={LxA`!2{%Pex2hN#prWj@19EObNBCq$cr(FuT3 zN}f6N7!@Rxl2}WGHF82mgq!_Y5zT?3JG*T4bdmUr7G;pJ(FjRmNKX?1nqJ_cv?iPk zog_&d4ha;uOk}xa97;$`xcwPpqdOw9ctso!9lA*3I#U{KSZXKaZZaYU-leC|ZYEdmfY_e$xS`zSh8LhQF%L5Z8D{h5NFcRZS zC&_@#jUuFjQ-TsiWbzdeC*@481=P~5H>9DnOPa!6Ijdgv6XtE{+Hp(_-XBF~| zvM}uzghea#h;!Fb@6r4deHt5-Y95P;q6jsXn+UB!z?EZ}kuxDKNiYk}gz|lcmgI>r zlHHGeZv6^UYULiYvmjouO@^Q8~XCCA^ zv}zLeCb1|-fzdGpqO z+Qx=9iC6Q{Svl#kD{;^XUtnxlnxbK-W!M`-S1J zFa8jiNhcyC-+q?1;!-3Uk}@YIPEzHo9l>3T(T@`lHDY?Wfi#I^I^R(}*ey=vKlE{o zj_^$gP+mp5B8QqS87g`ZXazwnl3w4s76CHeW(z28hfLTD=Wd92yCzNdA+PL34t$W`Co3B^31- z<-7NPr-tO5-%2Z;AF3V_n(|WDJ_~Z-+PKV-v)_^l^6|?@lr?!L1dZhqiY7;a3K1#E zlwxD0BxXz54G6;9LjaF2*vNVbic&^PdkxDOmWKZT!jg`46rCp7&OMm&$cc2Yv_h(N zd%nYpg@?QDN=6o5i01b{NKL(UZ~9Ii=8LfR4YCZ|Z3d1QWNnSYVlW$TEXdK-^}#(7 zWe_sjIoUBb;!9J)1+uX*$kfQ>R^zj`{D(K)ritK#yja}aWXt~mgprt8%M!*zQi4Jj zzui822_T`j3`o;)fkX{jiLm<)&2j1^sfos_3*WG$*9^Q33A~t14V$6ShQewIe2nNv z1f`)a{e?~jC~c4!<|J3Z)n(au%^|C8>WkDDmiu~jKizwBT zLX!##lojr#%&0qR%vm8Z|%BXu3Mx?XB;AH2ZvPh-4O3h=y zZ4jw&B-@0IN?yofMi$7n;VEXZJU?T0qYGqWF}{L(rtkEIwr!0{-VaSff`x~>?mVqM z5kBZ+!1V^%-gbW339`b`2`we={{RS)jR_2DTjbxc>j?|p_cqGYlS3UR4Uf1a-$jvh zPvW{@5*Rp6q>UEj9falkLNO94m#1-RXiGh|Hfk6#?%VJD5=Y3H@=>Ex6ZX-i$nOm$ zxZ{!|&L#GA#3OAbZJU{Ue?*9(1{M6srOgotEk>g@=@!z3Y!h(;Diw_*Au48;NgIzP z4ql_#QaR8}ZmE~44hj^GOvGv;iO89On{IGHG^bHQ4{}iHF=))xBzX%D-97~DgPEtf zBlZ)I-4C6Uo{f*IcfiA=BH4-9(8*Dr%?*0mx<=6&Q$&zjaw$DfXk-i%C@K-tH6`|w zOcORJmP*^fB%5ie{{Tq@+Z>K&2Jr3#t5PH^N>4=)GBA?laYn;rgyZjlr6nXSw656v zD2|EHvgq8$3nQc?IzumwM_8k!GmbSegoH4Jq}AJHgHY^?%q`d+@7?+t!&c7wv-UHB z+d-~J$o~LD6<3{E+ZJ2GJKpcmlmu;^xAuQUsIwzy*k94xOKfq;`ymi4j>90dnjc8V z;Xj6g&`Z@PgqD`grI9^3MEt+?iO5L9S>JZmgf?QtvCD72?DUYB>L+nIHG485^lW2i zQsZJETWy7jdx}lKsYOXGwxKw_o064TB$7r=v@D>i6LVn8t;dF&p8_XXveHCZ4%s=_ zYdnq{ci?Z6Obu8T<8BGOv?)kPjtunAu+6lFO%iI{33(Ba(t%b&5^h2^LyZv&GB_aH z1Bp%>6;6b=s|$Du^h3ZJ{VC8}a&y#{PTn4aQT;W9+tPhHH9f^1Dl=6C5y;bd_7PXnoKk6{jE zlk!AXTdjniKDm+P-o;Weay82y)=MRiwbRgQjmYRuK_ZE^QwaI!n=XP}G0wd;4-AcJ z6xj&KNQ{^I@>(`XrJ?iKpjfQx#AGEb?fXyY?Hsec?EQ@3mYO1oqT5zF!N9fdP;bcl z5W!(YT6 zl{%I~V#xA(2_+njI&?UK2Px>49F^E;Mir+I&gQl^=t`KAkZ8EYp0)glz68a9sN|nK2+6j9l&BF2 zDnS#-;ciE9K{jYCS=o-72Hz%I@wWHM(mG}@QY7+bODM7|Fhp^EVs8cZ8N$qNgR5t>B2~BirPMflEDSNTD z+{mjf0jferI=+xf@xvn`6)lm3{{RH=_Z|i%wm^@?tNsp35{-y+R*lSt&$)?+Pw-h+ zb4A#oOJX+twf!y4wl^WYv|rJ#ia}}TlSSBuZpB5r+{^JXm*l#nhAEm$JSRgU^fY!7 z^$ozN!7B?!AqW)(f{7)84s3gy*&X)^VJs8L2AG|t+M@($HiDV%eV6tk*p>DA$lOa|DW0s=3 z9x*Zgic2M_oJsBzxgb*5)yXIz2BiGQkF(bZKsmqg$u6QpYU$f zVQghTv#^+l_IDCg%q{-_f>J28da`AnP5Tp4rSsfXmf?H8!>lS>5`!Xt2D3+4{B{=s zs~V8bp?Zly13idR;zHCcEY*^rz(gd6mMytu2@U&_P#}VYkka0bqhTq2#khpw7!l0J zsMWc{LOhXrBSB+hLLMM^2z^4bWTpEFHUQy39>3dq6L*pB6KL)xR>U6#7>6jH;hfOD z48;sh(Cifu9kinO7;OV2J)%Whpy-mUGO{wA!$Nm|0VwijFe40_lU7QN#QiqC7|U^6 z;F;QbH9M0?fr{xX)23#No<++^g{rAjQoI@v6h|X)ERrn71(szJdDL%U(r=MW<~AGd zFxXk6Hp?K)Ar=Qs_r@I zp{!R#*GYcHaf~S&cJ>%@*+jPwxEdrZbE^_5@84q#Cv#poH`V;S6{pJAz|%FeF%KU*OP^GuJ$%^cBu*O|-N( zgo{x&?t2r+ln6u*PK5Leea4&AQjn${swQJ`%Fw51&`z;|xq-`vxlqLRWY(C|nI-)v zRj{%oPFULuX(NZK=)y%)lTq|UqXOfpWvIfTC6g?(40~!>PV|R2WO2Md;BBe>FGL$BXE@M&7$Sz62dJ#E8y^p>IEogdI zpttw|%9RJEgeRb}NIc@nTwW?Qf>Mf>#zGm7Qyds7;)urLo7cwX6tV71mC&~)w0m_t zh|7g$v3oydMi#?LCmpxYV{+J%&z*`)Ef$3m{ExAO ze2=)3a~Rd=G(YGyr-Ecy&#_!8`pHZ5YBBiJrX3!C^dG_O z)mc4w8MV-EOcEuAVdRPRAL&F^>9O3hdl0*R2a0Ugr;+Jsi6y*_9odnnC34AsOjzSx`VH_mXQAshQ16h&XlhWJA&p)PPFwUeB~-=BG9$W*EpcO$+ahLKi0)j80vHs$ zNkB)D{2_H+60DX=RhQJnswino;9aqS)e9-$ZlsuM$%Uno@*c&($!<*YR>HTB^lu|H zK=#gi;8D6}WK~Z-&@>H2PEiO+VkSyXZN0x_sq6Q0x@8IGx*^lViL%i598@GPaVcl= zLn)@iivIu!$8GfROK#n>iT4a@5{-v!S#JZ?uU5~Eh~ilzZ`)tdr!6e4{kayS*)0xl z-lNZj{5#*(jF++_FJndAZ+yz+{g9KFyJeInuw&7WqYpi7$5bKO8wof!hE9l7EECfq z&bohkZsA?t<3i^u2A{KB5>mDlrLLsC3OFU(HDr53w+ac!L`QxM2}k!f_Xk2E}&z5WL% zNc)Q+z94LtXrxPs6LN^Ww_B&7N=}`v6b>i!C7elO@!``T;Z0@Gvb<}c;=Mi1a`hHU z$a5S}`Ojh$)bv9-dlPHvB%+@sv-vOB%3*AdA!^y){;0P1XL{(->>(z{FWvd+;qW5Bo(9?EL+eBBE-_hsr zpGFe;K{1YtsvMPwyMe{^;9VOZgefKA8p&UN=EJI3MpQYFkdz`Ce%TO_Y?b_M z!7X59nfDuSxY8*RD21R(7D8x<9H((1>n_XWZ;^K%tMoEF4W0&3M9AZju50yRQ*Pp!UhSl$cVP^V?wUd9}-tbaOvEN6gM#-POQh^O-d!|i)t~9 z?MwZToT-&$xUFQJzN2umY3@vusbT4QV0L4_&th&#TUQi}ov2p?1dd3bL@goZehj)X zscshJB$heUNw#v&>de=Lu=`r87%y zSt5)rgo^xDtaLO>3t|iC#@M}*A!iR}i0+vy2QTW0NsSqFcpiH6i4Tv-A2^Ap*%8L< zdJ<`&NoahM-Lemi&)0I4q>`xN(1$jbYyOg2cqZ>U9-`bybZCj3F|nO{5;-VGg8PrX zoE+YRw_;6ED1u_4cP1xM%Zjn{L`GPR@=HT9hX^L{LfB~z+XCcl*wpzVjRwa8ymXNK zj%q!~hAj{1EMz?Qy%@WVK0nz4_Q`OMQY$6&NVp9{X{vPEfFgoQyhoCdt~m5Ny^5B8K~t@)wmWu-KmhZZbHbzfqcbXE6q$h`*u9 zfnj;Tq)8zYPtd)MtaRW?n>duqeNc?rL+Vlddp2e^zA*C8Oooq7RYx z5w}DvTi^IKxK_k`jTd8_+cvwJAuZ!%;f>7~VPIP)4err)8s9CAQLTB@$vE6Jh)NgF zatgt=euoPg{2xVmTjYkcCMAw3vZ@gx6LzsnKBKCGx`dj-$i0PuGqJR=VNS_IQV4XLlP=9@ zJfuV^g6_t|sw9`Vj#$!CB!p`s^%I;GeJpBYz00-SYdzmscoWxA$r#PEb`WY0>3+hI z3{}AM9_QSOW%eH(3-bDwTY^T3Iih=Dfd}h!znHRW0-8q_reoBHTh%C?ayBF^Y?MzV znz6)BM&U{SFWp5t1U<9G0;7+&wWGM7tpWG{9| zryGWm2}1eqQ-OQ3ZW7Q@zqcc8RoNr5{fJ5G?#78i#;1eP}=p2A5`$Zl*| zvPe{T7;L?SW+bh>;b>EoJSR~Iv5q?R5`_B6O$O8wzod>IBJ4=G8dLA_5Y2SRdWvSxZsiMivif<@SivemEQsu1R1UE_0u0+OR7jvRI) z{{R4IbViHg816p(P_@vLZ6u|fvuzB4M3DD=he#vk++#vS;kUkvv1YB2p4(`K?Jb?l z>c%yS?tfN1A@q8Uk7y$*ott+X)i%iG_8~LBcE*WElhuhzmf?NC+i;TXiuzG*NXd^> z4fIFj1X%)_Ep|-G!oZU>id%GvQcg$u9K5(Q10jXMEfH)n8&E3r^eIY5F~fsaGJB1# zNl6?Ep3Brv6_SWaEDV$-u+x!t6Uv6rm$01{_wr{XFAz9yLQymP2GE6a<&sqjb>O;i zbCNtLrorh1lvt$PNYc3*Y=MPvFx43&w24;56C*(rc1)g^+%(5x_9rn?AIO!7QYh1I zhUiXTvQKlkwO^ucWYlsut!uEBoMekqoai=RB9js%U}kOPD^iKqXkwI0w#AMQ9UWN~ zOP_-}z@Z8z(BCGa%<7?z8kDK@mdKv4LOU7gxgnV3oQQn4A2I2+Z8X|x=+$*y403K3 z<3yn&<&=bd%0n7k6=sXwjAVOd~156d?T$W5quOYGrOp@JkXB(B&a{ z9Y6GZqg+0oIwEq>R79m@Iifp`gQwV)I)ghK^H?jcICZQYR8{fv1NVh%=46pXq!BH~9U$kSWnx!n5*D9;@q zpyCO!zDaO$D-bh)w!o4#)2Vbzo|2}68WHs=4qJA2f2%Wv3DJA8yM&Z2i4?j;FqCX0 zyeiv@3ACkmJ%U~U)7E7nfBOYmoT+uFstRA{Wo_aJe+KLKv+OV|SKMVZn1{_BIpbmu1IIS*`}?RQA|u z!@quslWv%%3^4}W7=vE0Be0zHT?u_KHihoTSV|Dz*^Z`vR3zLLvOc0idxpiiJ-?%x zSr4PsZTcLO`mu$Pg)h*%o%3YizhkhCzF8?RbKFc#7s*1aL_em2Yd2c45ek>wl+Jb} zMCPtbS2YCfzvEJ3_BjH;DLUUOSGI}jUIx3 z>z#&}81O`l*P?3$sU>+5WTQaffwsr7j%BU1=*F;1+>JQs)UmRE63Hq)|g>mg_2e__x=nhhV%NNjaemfeNovhSkfaMcVw3^xBd@C7Q``L zpV0YCm4 zB(98&2`HUM?oawLq=}`V*Fsb(T_P4um}EmTxf^UDV)}!y-un^(r*_evE0)U(q$z z80TR@DLFypnjQ(VTaCTRDX}PqWAa6cB6}RDvj|e>mklEpp$ z07cor`kAWqf|;WEB~YnsOK!IfA{4%RlBt+m zmBkm_jdz8(=?a(6a-9!$Pr_OdSKx~mvMU?-2{%6MMWv6DDLw(dQ93jiT$68%TMAA7MUZ!Y_EN?C>`p5sjWWGf3H zZY?Z%)Z$XbCq3J|7dO6g0{{TlYOXzuUKeN)e>qCMw zE25U$D)s1yzTYDe&5UbyBkO+yP}GukJckNpHr7)JYq>VT#j&b?Ie@Mbbwz_@hB*r9aDV!^n|ScHTA{ zZP<(PI2gj2%5XPp6DGpbagIHZku7tD{TOn{gJrcVfln(LO*=M8JSk^^5f)A3fln(N z(SOkjMK??OKs95k^&J!3XxGtaJS@KtG(_HjD zBYDT#_iCKiHN+8OICVV`noAxHlyxyXjl9$8qIuHZN=ivvr)gM@s25m#eyYkQ-Rll& zMu67%shizS2LS6`QcTUINMe)iqi(j%%^sp|6+{n)S24pdcMyUoqM}l9+z{M1KdFDJ zy1wggJ?8q=e6zNRjiR4HhWFv;&1_a}X#CQae=3;& z0Gw0F7wvbO?Fu>XTMOBDM>J$&8eO@M&vjJ=6;QO$GEXs|wN`wI+45Y$E8#r5&HYtp z$t|%@a@fSp=EHw>sz~IG<*#9DhMQehM_k`XSy~xK0;Qz^rOkGn zH{$w>AjcsCkY*FwS71?-(u}O6aBS_rB~td1%qHriC}Sv_@$1viBoi`58#$)oSHl%; z7^sD|=F#Y$E&RskE@+~U1rg0nsoW6U&36-XydtqTnld#9sb|L4j*d3li(6~?tMx9f ziA^^hL*8$#N4HuiTMt2oMx3SxJ#MHzP}B>9IQ8*SR{>EZ*7`?|@$C;#$sMV-iR?O+ z-qHQy6J*=86y{kS8>7X1Jaae7${ATKG04-|dqEz)r9~wzLvYOtBp$4oaTZpr_6zu{BMJsy|o@4Xe5J5(qY}3>& z&8@i4RXDM*aCefj!h*WG3iemr=Z2>|l?QXF=>>Il9P-EAc?vzRQr&Fj2HlR>TpZAK z;uRL(?6r~09uP8U?4M$&+Y(7k=EwAM5#Bm2hgnqcVxKn*!>$Oow)a$Z@YQsf1G^5k zZ6o5YFKh3X7-eB>k+tnS94-}q$o-c-ab5?0&TaM;S@J(>!*d=PoBJlm*i_I@3-HMs znC?y%`0g8^oH2!`ieR}3)K2tG)1z}vVO)%@kPq&>gwwIu~}bndC+R578>TRsn20u3zQgED-H#G)#tie z?;@WQ+1_a$d#7ezr{zX&m-J3R&;XIrygV^@nrr}*n98$9kjBDG;!sFhm{z`0_ zu2|PXe_-Y8Dy!sv(2nL8vQf*r?|xqOQ%O}$Z8R^B@e6Keb;>HqTdrx9P`@W-=ATH% zX40gmm72FMYTbBUhE^U<3u>~$6Zn^VVY|qDR>Q+2bHsnm@kp4)USIdWKl;=&^(%18OhkcFU>!_y67RIk=J=wLp1kWv&mgw zs1?=K&;{IV6Qu(t`%yV7s#DnYrnc)70>|SZrPze(S+u_oq@+4Db>;jQ$G6Y^qkv zDRZ=CvgWpjbSvSK{EBnW0=h%VM_A~b9KZ=22)IzzOxrnYBP}1Df30 z?(<3x`b8shx+Qui)VLKznVG!U?{+dC91awV3 zl8Ma&*mtR2QRc$2D|1D_ENQN;W}O?q3-F)qSxGJ|=c94)F^N6m>9TC+9>j%*_GBa6 z>#Q&IYH#@1552bdE|RP^-G`1r{{WNhrHTz-MTho-3rDs*li;S4yCc11JS~)P%_DE9 z_$tA8%6_UhU4a#4VuRhyLtOe((hK^Hl|Ra+Wr_a)#6Lv_-KUU$%5IrkO#LJtyp|*< z==PK&gsgLYZOkTWsK>)hD(E{qp<)!G&RiUaA0gdy0_LGrvk=@xti776>{A4#e_%L&v}? z%nxREA23*b*p#zWp8ehX;oj|4H__G%h`&6P`NmYEH3PdSD{r6d&fww>sx6~g9+H#i zny2|kR}gULZz1YKZ_-ToT;f{j+#D~KicgeSp~Vgmc#>`{>QtL{f^!>0%*0u9+8G{6 z)l|)P6EgwiTrRE(tfHf+xSAF}#Eku_tK_94@6gKV&m-CjtK{y}hU7=m4s6fes&ih~ zSb_*lshPZQ=Du+g#h%1rW4o?SjlniEF2HtLM}Ofz;J!&`9YFoBii$GLyMvWU{_^=@ z{{VSJf7p+S+;`nwU0q!ehN=hH;Zw5hx*U9yPQFSfS?xYbwBAQL*15X6x}*Ctd{tBL zExB%QVk_}}L(7ilvF<&VljNMgO2Pa293iI z(5BT@H%VRXP1#76*LpT{4z%7JCPPx-OX%vbejTk{?0-1UnJzUMMHd(vTIAtMvyx@L*`?ss`{ zK53^Xd2XUTPxO zj4XKZQXDd$G}kA?41=`5RYfZ!?J$<8Honq?_G%n_^Z2IxsFcR$O#n9*yG@lh%DM-| z5X5n09gDN;n7(xrefQKD^Z)C;X}}dn{VqR=xcTFO;)3U9mnFcWlw>2;Cd2tMy3S z)5BEpnN;tK@RB+R}&r0PVI3+5n2u$#9&SCa zQ-*7$rF$IG%NGk713+_p?XR9iRnfL40vyxDt{3eEPHz&bYG-&RW)ltZ9yxw{hJ`&+ zT*dtvwxPoQ%!(iSC-+@{a3|o5t!MPFi}4Kq0O3Wfalg?!#J9EZKNMr+(mxCD@g+!P zXw~3yBmV$}PU}6w{FO_sB8p6qa8nFI$@?Or*I{eNFn?7oB}9Gb+&smRX-%ghApB%q zL@OFD$y9r8zAvX3MLt*?a>{1go?oInLSfwv2(&-X z;<`Ji!3?oZr!IBCUp7U*PTwWb=7?AnFMD03)DMb#XJ;N9RCLzMGQMpg;G0Hxs+wsU z$G;G;8e26}(R|UQ5p!arcX)WGcTbWBG=DV@a4EQMmr>-8ecoxDe78||%5HpF`X-Aj^Ghv6cmQ?A@zv7*;D;hf$ zEdC-BsjEr1OB}zx96z+G;hrg?FM*(wYXHy>@TtX2QIPl?BPTYNkT2kP6-e&#sm-g3 zVaTDUv*3Uv8*olG04FK~CNqVtBy%8qQFERr*EbUR^iCb+GsEc>MX}|? zDRl}Hs9$pEaZoSybZ!s4iF9Sn_; znSwZ`Z5dp^LqK0SsVf~p^p!x9qJ7=LN&MI-&(gchqoF|N5kD1A_I|ZL4byraN#AJY z9;e9eq5D-1#;S8hEG_k^W9a02FMK4AX8IkWMI$5;OVVNO93uAiU4JQ4$ql+zhaWX~ zkM!|>5d72-Lm_TA0vh1vkpBQkWx)8Yu^Vb4I39Zh-!5Dcjpf8+XudZH~6`~{gxa0F`pt^{{Sjw*1L zorxRX9%^?R1r&ugUj*kc@KG}E?Oxt|k)>`Qx4$JL3uPk}&pcOBHdyg6>I#AH1$53F z6m=JjU70PS);bv60QV-t(hwAsPiZ;vO~>;h2BBjmfDoo`K{Heu2wYIXk1N{TRWQV= zoJV8YfU>e@rMTQ*#3;VZDKBuWwcO$WB&pxh1DLSUZ1Q>%)9976q{P=U`Gjhq9+EZj z4I1L4uEx^b7Qc!u#X@`&f1EqaqTbQ+UXs4PYM1QKnyB=5O{9KF-)Q9?r^u>c)3f6^ zMxUCY`$XC1`RfNr>tnUAlB@o8H{(0@cG%dIRV|u0D%#t$(`Ly|hs_A2rreB$D7CJI+8+Tj} zP%1s2SjleMh=YcA$8U|6*nrIFT@_4sEbYc+Z)L-XBF+&P6*L#T#lv%@v9i23EYw5h3JE;ot3%X}48 zJW)Ff21foX`a4-va6#_KUnCxy?$(PJYWA|HkhND#;S-<5Lpx+>=f`mGyLFUuWn^=4 zM{!jWTy|vw+NYKo0ruHYV8AR7q{h?9!y+u)f-OTm;RTtWz;q%u20O|!<{OWJSce`e70j(6{{{U^ur)AGe z98a2=?GG0R5VyEwkIlSCy%xp5l>_T2aJ{(W>06O3$-~0v?cJsQXZAv_sI0BSB!ebp zh&L?xs(qVPJ7jchC9#XS+5ziSQ#G_xMS>D>Um0)?ABOtqn0oOFMP^bU>Q zVDdYxLb7CSzjy$Rj?DqL!pHIEQ?p`jyMRza`7zk(!#~MLUfGStqtPkyyB;Blqbpm% zOmRH8Hr@k*9qrLN*F@f$QIgru!&?#YPwx!9#4VzGOex-%nwv0!1#v+}b5+eIpH3l0 z+%g;ROpum~+UR%vTkm3q@~W6yBa_<|uG4=DEt?qu?mTk#C~woV@dY>9zs##ED8X|` z@gY>Sva#Usa*4Q&ycGTSek9&1m$-b?{n6*TZ?cU$?FwMcQtb$1r0Cqketxw{_Q5l0 z(lxF63%Ct(Q_0dy_uANw_U#R#&h1RC@_7htt+GGyvws!-DcfIaxb~`^eQCR2X#w_t z`LUG(Ec3^B*?SDtQ*gf4&3Z_69-@`j{&Ohs z;qh3@wU{5oF09*qq#y480O?p;_v!xt-Twg6oW|e1AE*)eE}lUfme$pPHRQ8x9=a#Z2|OcH@<1ql{Y^| zRMEYr&S?k7eq>(9PFtbxg<97dXWLPlhBrL8ljc<`ev<3c8_FF?;=X9hotiUIvjYQE zcMb;j@n1BB_BrMTicOu#{?zIzyb6Zl&&x-ZTyI?S%-uxd+$B|gte=TR>3JwSg-zVu zFWEEAAG9f>P1L_?6MCHo$?yEiPyYa1{;83CmHD)db!>)1+>@ek!|LL`X|ewR^KbtE zgV)VAPyT8@+&xDJ$*}~1`$gtM+;d@dhnV7-@_HAst`&~40FR(NeQFJXK^x1bo=3(S zZW|sG=c!Rmu&GIqI!d>L*kNZz&)(*#>T0L1#L?S%Hfvj!d;nVqrfceBk+vY+z~plU zZMdAu!+brVe6-}9st2g$5zD_De-m%{6?6S;AMXDE=wH80{_p;UVfU%O-Twg4kK%*) zKj}nxv(zp5Et42`ObbEDN-)Ub$e>lzO`VwLgqOZaxK65_tZ~dOr5jmPaN5KJl5UD7 z;*bt#QaU@5NBQ-tYIeFBzr4%@W#XRx&c5|48!KGXpJmqKqc0(b`mimgz+iKT?FS6} zH^E}&0tJZxo+-k_g$H~hrG=5nptl7RMea0C?=@R3(u*4)Rr;72d*K(~9YsSN3uS&@ zDuTbKyI)o9%*c z=68jJEFobF2v|a8ZBPJw=1>M%IR2_ix0dBqQ0s$x<^KQ?SX;?b_9YK#!dcn}$xyhu zrk-3z^$_;S?Oe2Hs`)~~=SMaoL+@*gTG`v)pQ>*D2(6sfy^^-s-RMWk6%n>lSu7esLb z+0MzDDBWoqc@*b2q<8uJl`TUeo}dwYY#>jRcA1y1KW7|(^Z zQ>g00c$aaFwsOnNTnNM?h6q6rLgvD}L3qP9$VQRPupmahwGv^J;IqWd=aD)Ii2C#gUQ zC&^Qu$EYcLhjgD{Jkz`BhcUXJ?V5)lGrWFVuC5i-amgdB#yQ8pT}ceFn>~fTV#qb; zMw9AY1xNJ~e*r>4YXG)%m!hLM@Rn3yCrbx$TY4N+ZIrp$5ZgQa>ZP^vO(<_GXCJ0; z=EK71G&W~6w?`3stWE&wbZaQEf?NT@E&xB0uc~B|0>ggD;JQ){oR?0($2SoNf~k?x z%IBR*Xu5PIz2^XVaZbyMe!b*PL*6n$vL}#{Lq%}IcS3>eqn^%c&RAs zBb2$&JPTaqdn##hz{7tI3UhW5ejLMS?Qq-Jwv;cPrxPu2A2l0rumizuALkD83Noo( zmAn+a(wpUVCAuuRsXNLu+8dzzR;%WG&s#Nx+?LIlQke_kB;+$@A#OxmDR#9}X2i;7 zfHLhi);`rI$_=C4!4M-Y59*(`_Jf-ClFK^x%bF3MYLa|3lSW-@xvm4^pDd396Wnd3 z%9m8t6jD9Gt!X6i2*Ad7>YvfyE33Jt;H)jXPi@x#lVs@Z z0m5}R%^vAFYB=*<9FB~T+v%N&MkiR@hUJhT=A8?*yM0AHLqA1}EXc4tlZ%TxCp0ku z+CiZDUcq&M554TNawjd%4CvSP*BBi5)KOm4P7Kb6$o-tptbU~ze97qL%i zX&{XE9TaZ1EU$G*lMk8-N04$s_g7LL$_~F&jFuI2O0g-eu~>?^S3BP81zK-0@=a8y zw0x6}N<1HQDx5ZTxrHvFGrTuJ_Msh5pIkk_5p!dz#(Ar&?g$~GeyO^uikQQCI5K61 zBgMz`O&G49TTmQFWM|1y$t&5~%-@oE1cv8#@__%@jlmG9m8-^_d?ljj7&q`8VKWm$y8O)!f@TsJ_86d#%{Zny^+0xe#oQZn^++R_v=MD`c779eL-5HujIH8rj@Fjw^VoewU%` zl3YpMbmL_NxULSPeL?N2j_~r`2Y-9M%&B-Fj-^BPfjD-Rm6elWk;fr)Hf?GCf=@go zA+x38V;*ueerx7Y4QeAF#C`#M-i&KA&*6^$01_4&A?V8Q-VOYU#{fA~wEia(@Ujf5 zIkuDm&!f2h)z#`pq}KTsx7-tlH-qg%MD6xgKyHLH=Ee9yAn_mKjPDF2TlhdunrDM| zlb1WFqMA$LW1W`(tmPfmXzWM0ZWi$MqVfJkkvoD4%0bvs8#pQ{>6uB1<+_lnYbhmY zYj3N(v&~ZwA{GhVb&7Q>UhevJ6i(A3jXUCSCqy?pxaB7e$5NvEK%K7ZZh!KV zvV5-|50?+bqfRf{o+0Zl{^%>Cry9%uShsZ?^ti-S4j=*DcXG8m$$)zVBmyRQET7e>HP-Q9)T-eH+I?I}P&y zg^^>p7fhOYNHDSP&rsbj@Lw~R5i-jhdFOYF`vz%|QL|ZI{{X_qNogGx(#l_IsG?M`L4axAafW z>Cv+eY*rdFOL!Fa%cI+I%qK=~l7?A&29uW!H{^j*w7H|rTE}y{rAeY2qwj?6M^mDN zYfwLRr2K5Z(8u+z_A~xuWv3cbkKS=V-NK`NL;lK{zNoQ$Zn0Nd#(V>pNE`0DF-Lp4 zcHh7fefD4Iqx#4D8UFw>5=Kp#1|#cn_gOsDebCP^)}MWo$>SyWlsdaH?G+j5T2Or> zZDzgsdQP{&HgePSaj~9Fx%Rs1N#~QlE1lq7`6&CvV=8M9ZKcseb4A=V^{$2tJ@+{H z3k1nxrY(MD;>2f?s;qrW%b%Nxxhl!1xV0SaoZAu?L`rioh0wnI`SSLndq&B!G}v)d za7~WT;y%ROP8aTk);EBu6NFUSC0tZ;q;3iRJIeN^Ylno&f`&)H)^|S`mxI zeA$;$fY`uA$UAV{>?crzV*8;~fy;srIzf_r5=Y)`w==#G;DFzbka`k3_kjG6nU}L* zW&LFP%C^wg36IAJi{Fxa9`CiOXRXEm0NuJjHE8d>-c7Lf-5RI|b_@PD6aCyM*2%%F zAG$szKBjl%=^RHQRn;tU$tY{wMfCGh#^~g84xsKRA=GFkzzs>TP3*~Ma5gE#jwn6& z3wRO<^(`eqmps$khV3H9kxmJy4bGB7#aNYw$foF=+p0~QFWN{0V;Z^Ib5-UOPnY*RoQ0Im*2PS7QA9MgN@ecvQE{O;Y@qHpfv{{7d_>a^*d zZa~<30_hFbGL6cySXKnc-1BC%xcFwe7Y$2W>(AM$uc(T16Nh{@&pZy}xOD+7a3fHr zdtWo$&_E{Q>n&`a16gwRXTTej=L}a(GG6$I@>oJCs3H~?tr0~rM(i>)*c%=q;W}En zw>)BWj{1t0(mX66IBIcWdqBo7IVcAsV7O)jj8wAeD1k#wxwC)kV?w?}l-}g~t zxhy4mN>VTsTmrqo-G~OW%%->=xzRXS?r>H^E%v&@yAqb45hNdK5k0RL1#la#u82S& z77)H|HM^D^-Ps=W=-4%tzO4TM!Cn!;$KCGDM+B~JD@7R3H-FW2B(En4d2k+NEcq&XW)P?8I^^+l$;gvT_aIJ4Zt|)n+Z9mGz~a%Qa0R^cTZU?jlwM( zKI3%%02LuRDr_s#x(((PZ&uB62AAl-h-XTOf(eJGkFk zr|x?GqEVIA(K~?Z*;!#gAV^)4;D{kz7)aI4Z?EzfD%9gwx!ywF^ZG89G2MuM@h9PB z^Qnf9!T5VW1&;nJw0JBI;!(M^&Fm~Zm4bHQj^WkWP7SY$`a0nH!oz{iZ{2Yo#9q*! zASk@1p3qhR3ltFD5jtDiyVk@@xqpR`)6|`y3A>7OGdJX^JEcEDj`E>tHUMT-JtsjH zedDT-sS}Y-)G2sw;Et)su-{sz?w>!XaJVS$w({bHoYlg(R|p=UaSO6hC0XkOt*XE| zAl#7x$d!#3qJFAAjO6Q%N0*wBua4Y~glJXNRP{4`oG*8iNmVqH(~u+p6T=WE*KHJ3 zGY}s8M&G>by`0pcqb8eb;mW4R#*xc zhfq7J8lGcgF6RI|aze#{z}y`cB*zXVIpG(TexPYBHfc80{Xg0=@=UvV&Jj#${Hx0nph1xo-llY;o@}NDW_o7y-WXb;k zXG7s}?@w}jl!Z8_CnB7t;kam>s&;`>YJQ?lQUBEfznWODIcD}laVv44tjad0?~mo*<`06sU|Lb9?9aTm5+(_tmNB6(_ueORwS z9Z=j=(Rc~Yx!x^eZ^=gqa|^N!8kOC4NyPwzR>xz2k=DygsS3oNQD>NKXJMKD#M--Af3mRN`0d(qV85G@tCc~rYHII+ zxewY6`2LEYVbU}>%R74`L6`QO`2|T!MLb0O))3Q>%I_b&suU4YRK4=b*)X2NZ*qSU zse9!vbF(LL+&Y|AB2CFfW{-U`Mu&GXcxbLo&%qQo6l69;p{C%N8eN&m93`M|8U-N* z>dCPcLqPhM@?h^D$RftZ(AJ9qpi-1daZV}7qKbZJcLmK96Ajur^Ke*@D*bXd0z%03 zY_O;c<8W|sZSpqC;Tt~_8p1(;WlKoPXO_)y{EiSlH%)WgZ4?99h^ja%AgE;0&C%QM z8uh5X@O@%_dBpLTzXt>;k4N zPDMEsQQUMZiYO?}k6v9uaIxMuUv<^@SS&483d-ArPAQa5BIizLvOT3!(Kb;pM&>xi zFK&KG)5`wCBLe3@E>Eo7sP-jO!F1MR3xWDW%aBi&iv(f!sCxymcW#c+k%O?tv7>`> z_v(You1UUnr%2G9IpB%D;q?p`a2ed_tZqTzRg;l^Nv`cH3kwtw+!})lEjyZqp&bgs zpn_SXY-|~ycZ#N_j+zk0`p+_?a(PlYIIHBGiaVq>2<|!~swtZ8W3D>V>JiK<6fUj0 z-9U#|2vJn*82h(h=oG>*>oLW{ovnVPH29%(h2Q{fZ%Fwp zTsQ>%&D;|h=e?|GyPRwSqN8OjHoe??#XZVLC%HVQZYZLPdApI^^n2TxuI9SOtv;i0 z>XEwJhgi&cCOo@yje;|HE|l2hj_Mtev@31U!b^Vn$-dPV!P!)Hb!}T5@3;+zyE3QS z`!XhUaK`sEdMzCr6#I6q`9{`6KV)3{O2;GHDRH}vyG}Xp9b)GbnO8xj#=u;rQ$qm7 zQF87P48Mr*QqL^b#bD9hXvhyGAS0;KbU`~^%|Q(ZJ^^gao%42MkbNtLf3i6zl=mmO zJf{@mu8JtB4Vrg_Ck2IIO<)InPRiX`#$#~$iz{`oG?U48S`n7%&%L}i07iU<9Ia5(fJ*TxCe%ev9N0Tf{V^ zoF|lh816Uk9)3yAPkMWk%5hF8lzOsefnD#|W9u7i(BZWhTuyVP*Ya6xJHG}YTO7e_TbO_Vo_yI1qz zrQ0l?Aoo;;x9;-ruv7e?!Es9+kM)k%)Ho?6mRFZO#m+jCfKcq}xj=JS*|qT#X8TI$ zwpBiJPG@)Ai%;SzqioO7nBZe+<2)Iu^E#ACKTtY>s^bN`Qxhk98sN|_?6F%rcp|Y_ zB5$~F4Z0QqP|?1!Y$GN&zVFC*rZPCAYZ@%pivXPD_b0hLrxfG3qt!a2xTC3D0y~EG z9QTb>x^-MP4a4djK7s74@=E7IX%h6IHHOu^&O_-f_9s@Be-<|xJPuH5b6PX%Vkn0ALvRDA`A6FDfBa>H> z@?P_}?hX2;tMx5qvUWf(6U;R3H}fc`6ylsyoSac8?kJ*-Yl$0&me*Oqt>$XDEOSyyjQp@?y>`_I=1ZHH8Y_RyttfNHKXEJ z9l%*^jhbqt>0}>7-9K#piov45SvaQ@;+|8AD3nn}6jAECaNSX;QTLJRo$TB~ApLVr z_OzWBWO$!)Jls}iz$@}zNc&3j+y`8BZ;?&j7SEu4Y}U+1-uu5Gn#M-PwW7ca0Etd1 z&Q2-C6y#3xy!A>bZlAyEQFoE*o#T6v`lHkbdvS1C4-$`x^T;dZ$Rc+E^~bu?%x(jU zrKOs#aTI^N@46K=G&ImQIK`S1ig8XU#W-YuavK}kXb(OxqA7D3<1B#A0W|gtVF868rt)D>^92gs!Il0`*aZV}45}f2p zB{-srJC5VJ`;SzqGxn7i6n>~9q8vwD?;U$DaO!){`haRTxBy+m*bZxCY_UlWC+92s z$CrRkPASDWrxfCfDa8_wl-WH|e-_E&ierb`Z8E z$uWIA&H6$+BkqsY5}f4sB2$Vb5{hv}9p~zX=?`R~-9BnQ>O1fCF30G#Hat3;ll4Qb zeb%-4zR;x63Q*&WV2Rf#26gK3Ik}=J2$c1Hd?DKUp8rMV5kwj3a z&U8nG!36P=j`MW8w^D;(6YeR#%r!=+BY2azG9J0o0$CmjhdS=qy@E`Ij z4wtgKdaUzIk~fOAz>f>)umthhDl$t|MRsd;cTr&|1w`zcR8B5$s(|L;np@V9$GNsy z`q=wB?ajYMXP2#redg_9%8hjs>F(-rPQjTq7A7* zlv9r0##A@CjCPl{Z+l)W7K)ubPin#CZga~)I1RU8d$fV;HbO4`VKE{jehik<@& z3X$3z#={==8nwh91!w7dQ!;mKcd%HsPKO@kpA{!Z+N1YsJQi&^+}Am#tt1{{Mbfsm zYD340i>7Su$?bDb#3<&9LD+bAk+`WMY4AoTw?x^9)Fw#Np;g;>t51@$I=KB7oghE1 z3#2O7F~jw70MM!xhs9F) zU}Ti8qe7g)gq0h!0`4JkR=k8~@icsLQErRoG+|M<6TrOYY4KE4x|fEkB8I8sFo4Ms z+i$9(wv7E28?clZi)_x~=J~e2l4~43&T-|756Ge|V|qMZZTc#+j5PlMYrd#58q^mz z&A&w(q|jO`;flZgkbKS55B{pyD#o7`XUWJX4DPMe3RoK}aFr)SavuZrQgn39{$h#N zd7G;Z!I-SFAGqLs4oWlHQQ9Yub>w&``cgr@zpF;(A^xeGd8${&XJM3 z%?IS8w^`rsVe>+T7~9EpZWow(+06Pm1ZyV)ajUE{-RhKceUw`y>AV?jOi^ zG(m7vvn*c~3+U=xLVIe`6_inca#UDVv+V%zR#s9Vd8#AYeK-q=Xw=aIi_7!6w^d^0 zaa3OGP|zoDH_q#RjX-ur$%-THh3wICPO(OOe8ToZH(W=NA$3y^eENV;Y`k#%Kbiup zAvK^H(K8EL02EGeMn>!f*339p#D6-WYv=PHN~YQxf012de3nS)rkks7sn27Z+htRr zvOkhN8gr+jn$09)t{Eoog&Up^WyE!R;Im=6>EVW3V~aDJtr6Y z+J7a}G1dsk;Df*PLVwdE^j!$rSkH^i)b1#TVF6Vm3m;E{iQ(LNCk&HTBY$tYHF%s|%b{7zGoxH;urC z=uXwXcU$)c&$f;uaxn5&FxY~++>*jM!%w``i>rX{&he&>4Dx$wBL2Z$o~N0HROB4)Klo2BZ{^H zUOba?XQFdcUiRonQA+n~nv%@}^@w^S(eZ;;ypJ^gjkE_iRFHvDd5s4mztqYkC_9b!ur{J>}7CEU?E>txv@YAgHOb zFK&TaS&I*b9X&ccS3i&=XJmA_Dh}qSoCrZALYB`Q-lXrpWym0*pKl` ze|WVp7YYfgIb`H);+a*x15bvFtC%<(T6?v~{S;D{IOIn$$v=5Q6SPEjhnjsVDfo}Y z#`dYHk4I<38Y=$)sqqSh+!+4=FX*9B!NbVpv3UGj{{V#CB>ShyOxmd1*-v6DQRGmz z#i0ehVKP|Vqk+2-#Z6PN9q7V+iq7rV(4%6swCEf>6_Jg6P7W2#S}d(Fv4GK`_4wqO zgfushq7-#Ncfm#+24y#9s$5QC4RLAan+>4QI?!r`iZ6NYKBzjBMe0wqs7#c!5K8OH z4*iR~(U&b#O;=f(mk9$kD12kUC)x)Rjjk5)PPemyvf9}BSj;+|YOHRQ+S~{<6#*9H znZDmN!L8GMp-j@~_YazG=uhG2b-V1gU2C<%^V=>OW}@i}Q>3OQ0j^`{g}T+oIM2i* z(uJf4L9Ztv(iU=iP`lw&B(veJ2Q&9b{FD+x_CE}S`kcNgy;F~7L5Oj#fA~_36lR5R zO$V)y{mUvgpLM@H)iYR13ekrSbn{73kIk^~+t zDnme`kWo3!U6O-BHaLLdJin4QP(?$-yuxHtwCnTz788w13U0ZkhA1M8VxUq+3ktxR z+D=eW@lOh_3aY8{VGG)YX+^+PBAn_hXhAfGu*W;dRY_%?qZ64xQ$}gdDyuM?`6%Li zvbC*Nt2Qv`-;o7NU}IiEbC4lG0OVE{WZX|a0-u_*&DOyjl%_U=Pdn{aOHMWyM~1m(SFy^;Y8dJd zCHqhEr2Z1E4{#YY*Tt)&-eQrsnmbhatYVv|F9gR7Z)O@{r_C6mFOwSsz2@^yp^ca-#UT8Ql^^*2!vsA#PjHmc6wf8df~RV+Zd zwW17t0n67V9dKq8Wv*^oBEWf}1Gqe*)etSVKP^JEeVVh#{;j>E$u-2#lyy<+fXz`j zYjn&GYOun`PS#T_)uPVmo_TP&+l@s6sZ?^w1XER#8WWBZ>gZH$c#MyQS4oGM;&Xr$ zBv30uTIfwwCb2X9qT#9y9?}gss(#DT=6;B)U)m2P4;!M@>OH`pr>IuX`KS6TXV{?M z{UrYY*>~i+H2(l#v;P3w7mI6u&mY91=@&@n22KfS%u#z)tC&LBXz)_n8)+C_f!JN8 zbUuhf0RI3Cd`h!U%RO4q-1DVVdQF{VY;^&7qjOy)6aB6?m%L?6BQ~Ej@!BbxE*l|X zII1=YJdhN$)aqOLE-2pj2GKT!xqQ%Qr7|g*3Rd!u{KREQD;ZG^$sQ@1BAcoj+~gFR z6<}}{ooU%LQo9slnmbdfmLME(*AF!9&s*EUJ6poBN*xnIjLcj^o5xzNmy{D52Mo`u zf7!dqLzGPN3aY5M30i!VL^>9pVLag;sxBT$%GbW%B^I|sy!6e~{ncr)OCT^WVtjic$2#O3UgejokE{Zu_23#VzDKr(QV)%?|#&C#$px;kP$^Nml$u9ubV ziH?dswBn%v@KeafJG6CCW)Rb9&Hn&RrD6X7PvBSXKls0z{n!5h7xP2NuC1lP)XXET z^hZmAXwLGa-i-L8p-d*O`UJ#eXWmU9=!UlxHB#iltS)G^vlfRhinKk94FGusZGY^D zU*VhE!8jV7i!9hpDy;UNt{mFkHuy@=v3SJX#f) zQqIB1Q9Yp4C^uRD(4MV3%~5Y%7W@2@HxslsTK>eQ?vE{wqoqIT+5Z5zjhm*r#*qs! z@oq7ns_J@2{+Is%kS?dBKk0w@0(GCJI>wgS+8e|djMJx;t}QXLqhH!`^iP)5-~@eJ zTuxPWe?&Svq@^LQEQP-D2NS&&x;U9(A;+@7Gbfjd=?<*6CpFavN@f*+heMTeJXZ~n zo-^HX@;}W}@Jak%&3^2k#r)Uq$^2iqd#;;st}4wua@neJV;X@)YLL>L zQ+FMFTdI``n;m4BX^7L6?}8V!heq7|6|%+D(aqgIE1Y{abNE7o?9=y;nzMbJeo5-l zt5kc}MfQ*KOx;f1H(I{M$-ARt+UDlHu7ADqUfDn1UfDn1`7djquLYP5ftRBo2g;S-J~yLfmjbYd>PEFzIh*aIp9qUgc^e00sUsyc024jd&@ zG(e%rVQ(Y=$#7PYooEzBvA4@&w?9P%XyHybNHIp?%qZ*Z@%k%7ZhUZ%X@$V3wQfV; zi*YuZdMDDQMg?nd;GMd&%}F4^LPcqGa>*I&nj%RxB{Sagx^HQr>dSpqkji2mX7VP9 z&nEg_5mPg)h3)1tU0l3gz@C3&X8!WvwDWbIdshihF_KxyQ-A~;& zT7JpOJ;AUdm+l{e>L&Yzdo=$5aIa;b?k8ELXzfsWkeOwX`WBy6>~e8)ek<93){o+> z7G9~&B-E~)ZXS`Fu~7H}yvWL*CDF1{a>@+BH6PZChSj5R9z_FIgW-2kHhSyLS>i2a z4>g4r65vxB8qfy|gX^HG1qH5C*mKPj)Ag z2Bld(#h=#{%`;G)=S2;1=6EJ(pSo_eeUVJv8#N0XKJfUix)JZU;;m9eC}XGq2l#ib5{jw-yT;rp%&8SZz3)if~r*z2oMn@jTQxkU~eTsb4F^< zluboiM__rNq{sJ)vR99ak3yZgR|P-Wl+Dq7Rb9^dDk^Lph~jZ}IWu)~9Fv78jiL46 zqj_@)DVVUIGz6SbVdwd-tXKIU-!*}!g2D#`FQVhLRBUD}ZXluD^+iv)>vNk>tor;^d{NM$pi_OBO*ty6tVl2_ zECZUrIjjL-3V{@E-G+Gg=m;=_=tj7;DO z0lzc>8oH^}9Co;>=7O$i`^Mp}&6>?TuIvv~-jB1l?Ip$GqO|p0ckI9o4Hsc30bJFt z0K(Tdb;B#+-BE7(-8SMkN4^w%ZW;wI*^~}Ss;aB3Vm63B%x}eeOFtLzmF*7U8@^wv z>t;US^>VEI+C3}uRFjOq06$uy=(x-O0QFDF9;@z?<_Y}Oe+nJLiR%EZ76JVgg`$)J zJ?(C;1I<-AN2|NG?_B1*yCfLkr5&rls-@xfXEv^As_N_~nDtXc#*>Bkf|GjTvL|?G zqp3keXLmHiP@4>cGSY+CMI1f})@Qg36d?SRmVfNHH6Ub-&8RscX>(h;n{1G4m|oDy z8$qf%T^{gE<8aU^HwqeTG}*?C2g$WfnDC9?m*%lc!GqNODxKGJX!NhqQT${+fGkk) zmcPC9U$OrHD!;q+LGR%mJkQdr_P>OD{)hv_s&^8O+-Jt=SRBF=nG-k5N10uFXfUAwj1{SXg7;(P%8YM$7J^oP~dRD**n`?W#t;TiR!{jcF4KcX#g z-jbgHnj+?1@hEwBGlPTGQe(sqAyw6byyqWUeo3*$-PQr{Vitx&S_D%`yVh zq18sl#=zE!0IG?(x`Y)~Q8yM9X4I)13G9FmG~&jezEkCsLTC>KX^cw?*;-T_)LjD> zvdsBBH$sBRWrOVEYkX;%c8$C=n)eNwM(E;%)Zm4w^wltzN=gkyw;P*Lp+(Yt;;f|C zUMsB(Ii^a-=uuAHE1tn@yukEDhl|hTt7qqn*Ai>M-H7}t`O$vV`=ENRu1+ub53N^H z-X+J?HYnSpK=isoTH1h`;O2rj_XTRy5dyX1xGJjGHCyVkRaLm4RF;-*wytr|1r{s< zx4Tp{PIFomb6}|EpDOEI1p=oi|*EC?a6Tn46#7~xastns~=!P2!I#;JgY zG#zPhH98+Vh#ZmGsn{vPpoGXGg{^f18(max4iy!rj#pJnTqK0YxVKw~MCW!U^9vB> zsufqbqaC1wN~J1?6hSrePHi^AX{fjrRCB(nw+8sA=fVIX85ri85V)O&n~2R(2^r45 zS)yIRrFQ%RbJ{q$-e1WK1KrQbRPtH(FK}%u&X-jSoHE(T83G4kk>^(u_kH4-x)hG0 z4>eB2N&=V3qLtKPO-(}Kx}(u!Rm`T0iW;f+{Wijg=R`sQa73!?RaIg{q84i_gU0|* zqO6BN2bwH#)&Bqm)ia2%o`0IeI4sD~XU$L!@^M(4t56RxsDM;X(`DnuX<7;{I{`*= zQj|5>c~Vc-NtS~|#MCYt6(1Yhx`w4%H;0#S(?Sq8R_Y+ojC_>}p;Rc1g4IGU&3dW? zPG}V2pL**4gA} z!$9H&Nw(Wj_r6Kf%=(8rB^wh$2wXU*xZd5=-Ac6kz0LrRS5U0Ns>BhY*e%+`6>g9w zrwx)oBR=OVyAIDHoXkfz8gf7YC!_!a!{t>vFLTFEJX`uuVxG4DEI3T4J z9mHGP6ehaRA(KI!swbb(RQM5DwG#$-090?lH5vI)$*fl_B;!{P$sqF$W;&cc+ zlbUKF0ENQfo2A@QK@{;v9k_KX3N50h-EJbQF}6cps0CJysXQ}6ZnHL0oYq4($@&Fo zd@@E@)?=?Bl7_)J-k5>NDD1E^M(51ud`4m&ji7doDa*JQcCl{d&i8jRm-ks#Z55N~*2cf-|&b$Ckg6lF@*( z5F-~JUT8I~%((fIjk3>D)}FBIL7M8V7~ES(n(i8*V`1JU$C=Gg05Vi74|Y{{9j47y zR}FwwQ{L7zxyL0d-6QH#8k!SA3BxotBNaEdTY`Rj z{{R*+IVKxpMKD9?N`JLT+$FJ(Qki0oowJMN6Ij*?Gypsi2sdI`;)%6;!$*pfq-K*S zBcS{4{Q|K%Om4*NMDgwjs7%}&63EtyYIzM)3yAFs9$OEPhlYllk*@%#hRJXMl5z@p z%&+j^niB$OO+hmV)e3IAZ2|ZcMO3H;bNpPv{W@7Ga_EV@kQ$zFDBrnZ4Gu|I^27Ts_YA^br86e`+OkL zpR_(2d{zr|=}<_dI?!x+TvGXAK5nr&*va`%tXF^ywDXrS($*AiOV55%L1k=L2jol0|$3!RFY zYMKZGI-YwE#f0G%JQQKrMlE>VR7$qm!vT%E+^F35N4lN7b{lk~ zwdb(iM#D|ScVsJ7vdw25xdndir9Y&tlYNwZTYmHP_#tPK(EN#&++OX`V*OApOVa`H8duK(3%jus%3aN1yOqkWFx#)6)p;@T?N!dqXm3ZH?|sRZZ~V(*3oEd z+Fn{T?B&Byqv-yv^br?3?;oj5=&=0eeSZMc)d-r#(Vzj~sGmGdG$%e>Rx3;QS%7nP zzXctRJXoGgj^^*DdR(}oofvU#YnKik#crNP)~1Benh*#M;A}T^ zE+q?%?-^`HGudgl+jvXiaKpO=tVz&D$S+j9q6 z9C&HrRBGnrY2uj+8f&TMb=e)!J3LAP=SkbbHT;%=4Trg`X>*MLR*9c9n2tk$bIz(3%rRc*RTM-cCJ|t94a&C{u+B7X#dbc?j(jb!(@^P*<5lXrAV%?OQE`4y~Mb zjld>^0%$|JH#1@lN`C3?0+V{+o3C^Y?iW)0)km3B=kT~8uu89XQ?cR{(}~!&+JsHO z)7w8KQtZlTO$a80JJGQQg-N)E$haq}Ab}O`#qJl9vpx&T?>mL$_orgjX>1YxBf1eK z&LoLTI=I z1;8LXifW1*Owmijx}qp*h|u?=`^{EVB$@%Lx7$qeQM--P$Hg@_G|++p0D!o6IDzxH zY=y_sNvQ6~x`P73LqX@>>OZ^yx=P)Q^=v(nspf)cQMCYoKp-vw5AF;dptz|u?^}YM zrzgpZloSI`{{TzUR;RUvjmo;8qM~R_384UhKwJ?1;~wxwP^8}Os;Z5{yE{iw-<^p` zY!VTuX`*}F_Y&h$Zfa_3Y8QkyhS<2>7WhYke)sobwg1EbClCPu0s;a80s;d80RaI3 z000315g{=_QDJd`k)g4{(eUB%5Fr2B00;pA00BP`xL^MO0{&s;1!N-bdmo7M{sZ|U zWYAlKXJn(G^p*RFgiiM=8mr^x0>n`ar4TybP;DbZ111xXIe+BUknlB7b}N@e{&~L0P{v-0q{V40ySe*hw}kjbQs(`rAEZUH2yOg zt!W-^3tcsgCv!1m0l#wNQpU)s=l2T`ZpeqAfFN2ay22t#ZMy4-2vD%wX?()j-uo+F zX0(M6ESCX+CH{BB$I7}aq{2@dR~m-4FR){{+z7`mXNHb78&wJTZZY7`-X*DAx%iR$ z${w4=&Qy>@xpKv%!hhPar7bAxwU;HVyHhy9Oxnl9pb5fd2k*eG^eP7fVdi4N# zL=SC1IYIvb5B_2b*{tvQiq~ziq2v9S*7J_#Zyd|@`MGp3uT=Sz4Yr$$ic7?H`;OSF z;yw%!^%`nr-eNE6I$x++Tj1_njr9}E;hL3qoJ7aG`nhZG6%h(o*UzXhHaD<5zF;UZ znV-0y2rI0z(8bD~4d>KWH-(z%d_O%z?m1o?gUhy34uh6y;_zI5*+sJ&BgN6NqfvnG zCngw%@R`guBHnfD`6c?^l|K=&M5>nt>I@bYwolcs<~S}pC9Q_&$@!OwcP#xez{g2Y zGoMg>+_Uuqf($nuUSL@D4Faym-eaviq`W9yW)jsQgz+(8BcA6z;qcA?6^ELIyBFC~ zxNL008v<*<{l$V=>IrYrmQ9^88;(qI5|7*m%tAj2g?g70e=w4|VB*6Pt_6KZ7OM}$ z$-d)OV@oCSUks{JSD$kN%=xK|P8pDU3Z0+zz%vV(ms%hl!(G6Ayg(XC@WF)F;QN`d zv+^@4I3lb%OYbp;ng0N(O&rzwfQu{l;B2w>iA6^R^g|WPcYnba3MqE>ynI5yf4PHh zdZ?p(I4Oi_Q8FhUXWd66)LlT#|^JpOaL=vw#p#=CP)s&5PqKV-&@H+fQkKQFW=TPIA5N_w~adGt#TCXz0 zs6iN@Y)bW;_5T3VL@53=5;=K@y1}WqxYn9KM0o!IXNk>6>Qi|osLdH;L~=jwAQJ#Y zw){tb&vEW5j2V@i=UDDJHr2~)KkBB+{{RxjVlMCNa+`DR9IJVT4NAuDpuCEz<#Aod zM-kRO{6N4JjX|wUmyJTbTn+#&R)V=$6CA=3{JpW%ZxENYU8?Q!$`cI~exJ;&3s%+2 z`MHCFCO2Lp2xD)_FR{rsbOiF`a4S|!OMTTo6)M|_1rzrQ6hZZxQV?5ZuqcPS%uxy# z!qhIoJTmhM4IyVvLoygmPM%1xir{=MIca=0eK8D=c*FF{0=^&2`XTasPq{J~;}bBy zZZJTtV=oX6)ZLqxKM43@`j(@po>RmE+8kI!i#RDnsO-!&bhUiN<*v7v>R)>UBp6z} zY=-9YOc0xvD;oykVq>MXWGb>e>+u0xxAQ92E{Rws%RNAWAE+9wU-1MpG5o~1a!R)N zf;%c}ad|lXOu_d6i^ap+%njIGLFLoI-`qjKdSmsx-vaRw$scD0^DN~=xBL!ECq+eux)MyQhqfb62MJ~)KnN%ycPz73Yw_REJc_? zHONyA<-j=E@a+Ep;b~>L1u!FN9>Y(`2FpJ}0fcPJH7}ZKs^C<>-oJ5j;`~OX`IuJoiOMUz zDgOY`Q*_J52kHR2>oWB(QpMIAjAaeL{X|80^#dF7$NcU&Gu*Sep_|A;`iz+Ca9|iD5<97wq{J{X@MSoF89A$|^h+t*Zu$a6{HX8Yers(w+E*d)j z0Fw+9%UpCupree+#m@c2uN4mUolEGi#4;-Aa=#&cL2y_LgS2nah+wCHW(x*~0i)k+ zpjNn=Y5Ise`iO>^Kse(+1PoJUd_;>!lHb%~8{$7v_tI964&@7H{P7$b(8hAT5M>JE zb;0)%vG8*HF%5w`OZ<@u1z{az?o-Z;Y)zsWp3<*zF#<9GbeA4V+^l^tH9My$5pU$1 z82O9_n_8~%QA6hQ8dQ|=xWy*w&owFznNBx8yu&}@0J9s^abdV%ItuqV2w^t^#Ps=z zRS8U3xx@x3j=R1%Wfp@K4#S1vl;3>8PbQ;qm|PVDo5T%0!XI*lr|vzo!g&Bap8>QU8>CMHjnl?|ET=28Grpp6_oD##@bab}MUSfy6*XE2hN zwyq=&`M=B;ZCm<6ss4Zys+UoVMxqlZOtxZH=_L~)DufXcN00B}|TWh??yD!*|< zEARS=R}a7ajkN`S)0+GUS09^)91V?(y+XEm^$Mz{Si}JSJjcHHkDPtWrXsnS$-zJY zDx6EdH%71lT}6r-&9csod2z-60O(nRgxk-`74h93$a1>1@U%r!)L-T-lxdY#Zg)Zd z0FB(PoJZPLc>e%|RRyDNg0u~5wK%-Nw4(J7mgrLnV52xYMe1;$rfuDhF44?wEASvh!z*g{Y#opJXA-sQ63>? zJbIqllh)&$s$URQSbLaHQ%|^ESzeFrigWouY5Rrk@!TGD=wtL?a#Z3) zgD};oq6OHHlBL6=P~$vdir?lok35mh#lT>tdlmk__!a0+$bT^x?%+R| zt#ws7?;4jlYs99;xP~#Ghq&$NdmdNZQ4^AKy~NVi+qMgqP{#8pt_2)%2Ol#~4MO$g z@aN14GeaJu4MXFoMrdmOk^C%UMGM}i>$W5GG zJBs7g!ZfxwK=4A;wzc50Cvl+Kip{m_#8uf%%@bcdz()Ci1}6?5O~+ohwSS%_YL(>v za}zy7x>w>gHy<?vESzo9S&EuXDT!X|tKf54OUA6H6E_x2?zA*utW8i;y zMq9}*@S7N096@Wn!@PgkWz1HPgtFhbR3Y)n ziYZ;+1i5%#Q{-a;Cr&wu15G*CaT3*(8fi{1S2_Ox6r>0+IV<$l0 zR@j*5K3JhV(5|VL6s$sc`-x^B!7+pHU0@*90{Idv^`U9bUak z0k5!p#8U%=%o4JCpq5tGoW*l+oHp|tsHq9w6Y&JC#CV0?$U#E){{UgKFgx@&C@ZuN zq~UpfV>vXxm?qGa_*@8c{1BnY9j==$UKE;%$Zy0=IM(9Tf2un8hY6*@J=gOap1;|R zshl;$L=u6f_bNO$l4NH7BTXLqnZDqc{KaFP;-C#;B3wBQJVd{aJq4HFsm`_$y5#rN zxvF9S&Fq3TQg0^5%yO5QRTzU0DRFVpD&9*TM!%R)Q&{@yRX}i_Z~RPIs62s&0)%=s z`D0SuXO~E}p2HH9L5Hz`>I(k=Y7Jk15Y=nHxc9DL8zHBof4}}abk)!@40cyMfx6}@4Vs|sa{J}B7 zDlG?Vdt#>xIF34PKQSIIU=f$PUjG333E2x>LDVs`^8~J3CgLKLI5jWodSGqu3;T$4 zYowRtl)@nqcysdxVvtO;cw1t(m{uuGagtCiL(HoM;|;*6jL$#($NvBlZ4EOcdGbFJ zm*GE>X6p`$eUpI~pA!f55^)2Tb5LR>)Axx*%G6WW{goX=M)QwQp_lxQ=L~(orCZV5 zecruU1gY3Rq-6?xK!>~M@iMAc)5JB8N+_9?tLAUjbYd;rXSjMUHJ8~1WAFa}BGS-XGI*%n zIUjx*QuBIFNkvu}O%QWq%+J01m|3Xis}KUoK?f~cFxbbd%PO+bEkuEE8TfWVVs|$< z!QO%O7aXk~VvO-uEN(|~t!N5D7I6NN{w4%&o6{q=vH1z&DQ}^F66Ngi)v91x(GLFr zl_RpV?^>u@3%O)5Dib}wQ!YHz$-G8Emg`dEtJ}60=s)-%>4Duj{30$HjoX%Y3Hgt0 zjWMlbSdA|kz#Ymv_wrxV4<$#V64#IYhFx(jiZNN4=2DqtMjvl57w#BwDCjRT)mrl% z3tF9m$Q-aa+w_pk)C?8l60u;*zV07NRZt#Ss=gSZ>!Zxf99rxbYUPO?uJ8}wm`|Bn ztgqC#Y5wC~>+vgYF29PIJb#s%D}zxw2j)KR4&cD709w5PxmQ#|ujBc?qb9c?9Ma%= zg@allUSQ|+fCfgpk2jTLQq|ii^hIo7t%v*ExKpkp&KTr_s;`&#FSa2lD@Tc9VTECZ zi%`_RyvHp~^(YqH%)@;`A*BrJTnIIXVm`UFZF=#kNpfB>8=}WfWzTOU=b=ueW;Fu& zh=%mS<*-^(6_n_HLNcuIHhqzz(b}XTPI>9 z-9vDXD~2UZru+JqDUEE*3t*o|%%-ZrnCMWSYuGo?Bs6l^o9Wq{{3US=%ZURjLA6El51$|XNiE4TV8Am8z%}NyP zf07p!gYyqYB>HUlxHb;47hJC#lEr;6hAo&=ME)ZD^{8w%T?WkM;kIFxoh5=~3ZTu5qNTs(AXoqYb z=PWf8FT^Ob?7$ik_m*Z`OZO7$ij)r0I#zx(!T$ixK@sx|s$F`TEQ~8MoZNizij;_B z>|fMc=HKRQV~IyaL=3scKbc5snNj_7se#FeT|tLEzC1+Gh9@dAd`cV?L;;ykO-hk! zXaj61>6?M5OndhWSd<1i{7xIaMgIVU%N|oyUrozSC1)pTqr|y8CZ+rznXFp@`S%3# z5W(w+9cCg}i@HZ~8=%BL zLSsMq2LpMQI%fvYnP?6*)V7YvNVj!<fnWYzw)D^Ez?VNzSSq z{LIVU=z|pxgP5%o>Q>6TPGTIZzcI4B#Mb4Uq!XhkB67vc3*!OC04YGVi-nNXuQ!B8 zf&K8#ApWe$_=i$nF>!TdfME`OLshDv`LE_GWZk~R#$pzFZU)*zD>(x{F-bxPmWVv; zjpTkvD=4Sw*Y^Nf7uI8Ro!I@c9SEa*$1DE;d`q2sfIdh^)q(7MD>pva{M+Uc;RCUy-F#;-ce8D(7i_8KuS&>($sD(w8J`Nx> zU~0#s`H5|xH7i+rx|VRI$l+@8`-xJteM9%J%)Kv4{G*1C@I(!9K%JEG@JofzMjq@Q z;inBRrZ|7BFwCu?@4uN{gtxnbIZtqrW@GTJYrBZH9_;@B5xD(9{@@>6s{GtRFP)1& z5e+K6C4W&RiIe@faD=;vaK{kl$Q0s%y~+s6RA#(5f)*e`4c~G2+%NCUI7ZhDJ+2^q zG39p&TDGbMhNay$9T?^6VS~XhH@eRCBLX3Vptt^ zH3Nr3#_^_75(?DgrX;4fJ6|x>P50a9iGG7$;KZjP(ori%i;?+<11|P|$zBEIXJ(@H zU(9c#a@~1{!u5Z+n>9G&oX0Dvl@r{|g$EC)*%nxFiBa9x@06g;7`$Z~hc!$269s-| zVk2hxF+UQ%D53Pv>RKaTGPv9={lg3cdSBEu6GRkshj6607_z1jq)%;XYR}B1E1+ld z5Dl(^9CM3@v=wL$m zDM0zFn2juWHheI{k#mjqV#$SRBB;7d^uVjTIe|qk_Z^(<{fu#{tFg5z{{Rt+Xj-UW zUBTuMImOS^F2*Q(o5yR5YLh;Rt7AQf{Y;1EJUFRk;}H7p zz=Sk7mGT`Ppb>zg;R7EO;g9Yz)a$VW=HW2-fkx|4FkS$MmgS%xb@qrnr69z<8Cz%d zHg0w5T7$0;F}w%NLv#5)nAhwF;vmrSL^6k_rU0Ib9cRfu4L~+N!V8}nN9lmzJIMPk zU1xmJfA!~yAqpA{?F!VTj1*F72S zl}67USB!p)_brkc3OhKX^nA+V!Ht=F*FHFg@+)A;{yAX-Ew^P`uGyF;P^W@bGt?E! zPCdcn7N!U}j6e7@iE+ei8@%DfMBu%-dV@}H0=MLd7D)o1iH^tAL|2|C-2UdmuVDa} zF3S8&Cyro*4K#`B7io0m^o7`WZ}kQ8n=(H#gfq*7{{Uu{!)(7Zcn1|kDpfV9ey_~E zZm}w%zXUosnV8+1(FG}x3-m?nJYN$qbqAN3KqjTc@REv^*gp*6-??r$_=_L7ABcY) zO1%F7wE>y-i=dA$5Zc2bMz5gT7?6Niu2jIj2vk(n5AHO6SY+`t>l+MTxJU>!g1#fW z#bW&ND9DwQ)Wb##%okE#gcsrS9U^_fV82j{NKwF4$clL}1=(w5%b|8(ivIxSWbRy1pnv(!w+|Q*Lner6A(($pF9DABLsJ)4!4&sOXflHFE-&%mCs* z9lx=@!G?^rWE|b|76+(}?7;6g#_jZ1m|>2P>bfiRW4S|BTCT)DkHl6v+4g^5iA^{8 z{(p=3B^Gf6gRNu#09lG_+Z&+C{&6v-d@BaXwx%|u1xXE`}_DYvpQe4>hQ}#=LNGtcbW&vJ*L8dB4K&SIF*7%F@ zH3yq-xn3*m5vBz#Y4twHDt?#~;ljSq$!_%7@WJUr$bKdTr{w`Mzb5))yFuq6>%G|a6$0D zD8{^ah|s5XaKY?1;-jI6l(eVw6PBe<#{}3_)#ZPNTeyI4*XW;6AF4(7itkQ};~&^n#f zzqtd5pNSYO@t;ydRoR(Gp^BqG_3TV;j5Bd)zg*|sF~SsO=g}W6VybhkN-AEfQGlYH zN=CCpRKq9e($!RUN3pcE-0%bJfjhA`xJ!l`}8IwzQ{4eh6#xpWc6O!9NJ+zj^20dLeyt{?-2gaRHt%V_kLpcQP|okNN)q zKLoXT&40gtmKzhre;@PkOEk0}mvWHX+ zaAEZ_bvAs;l{ay5E@yGTzO0vMZ#()x;eyeBL%Cj%aEkh6H=n4QcMO|R`9zkH8xhP5 zqHy2l6~bfBE@*D%J*fO87kD)thuaJGK}CD`e~C;|^!^#Ri?xLKV)8x3{{XzgDz7=X zS(R-({{ZB3LxkTDo6Bo>-X#FvCGl}!Zt-GHL-!RUc%#+^9!!GIRW3CbgI8jy22DLmb1b z!45pvj1S3*y7w@9^#Q_U{)S*}tqyNZx4R;7hw5z)m@$Y)hbBZJjuq1RFE7L#+Vx^Q zD+kypA27<^V6$IYj?h+ZMm6<2LGa9U(cllvb6UKGeUmHX1{5v_N`6QgMXopZQx)<` z`_y|&s$$H3DXC~%cf|}Mc&*Ygp7*;$w{aRf(^mwpA6e|kRgD-S%a z-Pbwaw;d2L_(k5|&H3BJ2Duvfc*j0B+&0#aqkkWWJ~da*r$3Lu4BN^6`~3F8l=17l zc>X%MTTXwTfA8jM_j~l?{(d9y2bJL<8NGb|pbdlpVq8ND3(|Rst7V)HHP6HZ1~_R^ zR@n}yvHOOC?54k|L|s-*WE2+VhRp#V3@Odao&Nyw2XT-%u`C>BC#zQ_SH(R`RE6#4 zy)`X_G78ho7#afon2y}a0mPtEJ&@7^RZOTUIQ@;wt|@rK`-x*2HQwnZVF9fK1g-^5 z{4v9^QvT3BgQjOTpX4V6RKK{LadGA_p-2<$SK?H-yB{n3$`g?xh0GT zD~3*xaYKkt53VNmGY0MJs8;=cSk2XwzLhd6iZj<{H(n;tZaip>hj+KBAdOI~I}`L4nBRWkypO;n@AfAkS6h zfwHSn=x`ne$f?08FGRq_w%jd1f?~nq2zPCHr|v5pL+@|YS)~qB;tOGBh(WgeOuXGf zR}WJKT`#r?8UTD^5N;UtwqE_>5xqtJ>Zc2`QnOEJa74WCa+7hjVxq0=ZU7}X?PbiL z{z*{fnTs_BE<5^}8Xxu$S#SMf2vLX_@P-CCB3&T@^xmP;%Bp{2 zBRH;!U!?rX?CJMX+1H8@cy=C0;l{|6YY{7}=~({&;Hw{rXP@yTblBehV~}TlAq9mJ zyfK3N>r(g*pTYh}{P{lt11f%R#o+je(cqO+4JI3Wf5=uS>3_LWy<7ZF_4)q*39j#; zABY%y?XS#s7i5?p%~3TbV90#jKNG==zmohl5HzLmKdPF1o?qoIm5f-}_{3$8#0Sr) zZa(2z3%q^I3aw%Il+<%6VyG`Qvp1YXD{nbpeCPJ(qGN*ZSncE6_QVE5BZK~N<6W~B z+kD^Y{+AsRo7HBy+xfrD(|NR?*KV&(GzF zPKzmp8+*K^L~!&m>L+k~rWkQE8u%&Hq{=iAI{u(EwRx2=7}^wmVN7lLh$ZEq>9~Tqbl#+R5u*vS7!Z(q?QtY(C%u0Jkegk$?-| zBf8lhj5`2I)O%~1l`vRW(%bWxg6J>mrg$(FzDk0N#wTkQued@XQz>1TkK3+??i?sa z3tZDsQ@g%kx)GFOqw@=d!g4?g!GnzcW9%iHHaWe@lGVb20}4YLjbqYj%()%~Yu~v+ z*OpZ96#+qZ;nFGniJnPsmwyi*{Hq|>u~dGix|?%cOv$$V*Y_@*c$Si?{{XZ_b0z+g zo;2g{J^aFRvF-Kq$pqgm=Kbaalmyj1Kn=oP&f}WNoAAH>GTdzc0L~&U?(oc-FU%O< zar`{KUI>V?zqp_G)kOZG;SPDmR|hr`Y!S|Nc=rp}Uoe-CQ3ocq?kE_yGS@6uI9}q7}gNDoA>HI^d>TvO^f`;HUJN`+FM?r$UzY!JVqUwO8m05lR zECZ@SABdO#0K!3wyZlB^sAqWm!Tr7}QqzaH!!}1~ofCk57($0$@AnWrvM=#3*P5r{ zg(a)S57JU$I&FV214{WP>HJ9{ ze<;1};!|iIE58-|eX^}RXIeiP`XgiWn!UL9pTAHmUMjKk{{VRHmfoo|=ZE@!h!!?u zKB#lzyW_YKTg~~}ec$KQKP)}}0Ke556G zkIs88Wz`L4u*5FI-lmWz)N;w+#0t~IH>A*|?vdE@D=&ss^PK#|47S1Xwf7ApSlanA z4E)anHCPW!JmGI0^uKX0V2Pk?IJ4Y7-unq*CCVu5g*>$VlI|(1LE*{ygWhEpc|pfIh>D&VR{;8d62P>TZ9sE|&xW7?scaNH#CTneU^7@9 zd4_>K09sS7XS=NCl!#nXtJ;t^aLWD1?!VhrWL3+21}uvBC*h0C5--9-76`AU_$etYIjim_ z&OspX#U!$Q0Dh60qmup~#7U%{OUy?)%tFn}=Hwb$C>5)ko1tYtqs-o4mCqD%`C%cAv{*ck>N2vGWiEltMi87F zLImP9PVry9Ws0bFpGda zEP-h+oZ?t{VwHK)15%nUBx6yRN3Ym`Fh>}JiC4RXaJRT`FC@v+aR>4om6=p7X0uV! zFRM@L3xc|SN@WcFW1Y_IgJ}Dyc9!{E69S0$s0>1%6AJq!6qir#Q+&eQxuceX2wm|R z3`rAIx%B9u?Xs_45aBmRL2BptwKCNyUD|oH#B~J56AJ;rvtmLdEcLW z_b9vFf6w{&f_zi-Hva&=;BRf@=Jo#n0E7k1*w?R!I0ntyzQ`pW4kleyTWA@-#owxp zMG4xM;t|3wtykg-%cxbbdVjdrv>r-<;j_t^oq+nL2o$x{TL6eJOeCs^`skY`XSY|I zm#$)Wd+%WC1vV%E0Pd#SryB%r_2` z`G6zW^Kmd;Y3!`Trp#9ev&6CHwfDedWWa(_b#hsyU+tId~aRg0)2@#5s3W1GxOD+&-=v;@CU-i)=p- z5fMt%lrnj?7#39X141?RGT|}b?pE7^XLFPVi*(-=AC|_hTyx@fT>G`BjPy5VN14B=qq8$7&h8#aizO?~9}< zUmkIC-S}KT6B?=W z95qqB*7qNPcPpw3xJ5hrKBeOPPT}Ebo5o;T2`PFr5Vp_va=!1bB91GmfM|5#^#I0f z?hbu(b0<3ExnjBT1p_=rCY&>y9gm1;p-ZiA9-bwJ?I-av;Y+y03lrhEteZb50@Mmw zrr>#xh(Y;_pj&RaH!CXCuQh+fMmx#?#Hu|gM@lJM*e|Ne6_DkJ4eLG1TzQmT!lnmf ziGyn_drOFhZR^o2OT^iKQt?`#9%0srw*8ECa@1HKxpmfJl8UamdQ0-rR2m*2@q6_d zqM$nv8DTg?vq-q*tIE0?Szi?!9sL*7@XF0yB*v-`)pZtbq*7K~ZD7 zHLdqi%0uGw^)SH7{X|!#kEkup6kgyhA%GlHvzP)YE9!50q_nc*h!s#rF%@>oioSCS z%$A9*^Ig8aiEBZ&!TYe8v6eL&&09GhrSpZhdLe6QgTpw=hmAG8PZ z%)&d$vIAd?{leIz(@=Br4t#11Y(r(=s+XEylk+J3;D00=k>4wRBXOmTt-WumidYu2 zK@78$*Tu?aubFA*UznE{1>=AF1$BodFmO&rGh#i(W1$;R$n}TI~x#sYihg04?hAZC=@wElz2=uIgL2OS;Bm zps5IJf6;m+-VD%ps^9r`#a*2*N=YK5QdBVb> zj;dxKpY$SNl7$w0kMS$qmrFp^R}lMc&-kbbsKCryy_Jb`eaz?A%p9G#fjd6LQtX~H zU(CF>_XS^Dl~+GA9>4Z41)N5xEr1Q5*%6a&fsdasK-wNUlw%Gv7p@?zB>|hnpwU`D zZ1hDKqif8*v?~$a1{z-c!GUcibb|UrAISM2P%3R^g=gv?tpUa{Uvn9M*hHk6WG`5Z z9vZ#)Y8ER#^0VnDDOvib@h>nOMMB8spgHRN%pVAdDQZy`QH;UXnxhGTf@axPu~1^w zCI**6t6f4Fgjmr>?lK_TtlqzIuB$qv%*}02yPUd>jC210V^=ko<_*y?lG(|)DP;V^lWci-(kR6-YDvR$*J0;&H1!Q8?=l;~@iibF%w z`9uK6@XvR{@=uwp#J$8SA;zPJqP%MBVFSD zC5}A_IETW2(mGynjQK`tl5!tuQDsk&Pt)m*4MwBZboZ1o-h~mqPr($%z!8KjT3u0% zz?Lvrxw@JjQ}~&Bv5?j&CBcmDXR&YH`~LvAmIa)}^ex}UAE*h6foKy2@Ahv%00J43}Z2qNs z`M^J!gt0_pYq()>us!iDLb$1Q;9Bbw8LijFkpM=Mk>WTDoY%}@%g3!uM+j>3Vf%v! zVcCyxTPs^XkrG^Z+#2J^Y9dL+>3s4(P_l3pXI~fMC?L=?KhS$45*sc}@=(hPW1Dg! z+iB)onp%8!U*Z;As@n25YuH8?SsL{5xwvIRwQKl_F-ki4fx;Woe&7UPZoM2zXLJBE zz}1aV#CHj=4IT){1!7ieoWvrMwTDP2h~nHrm~juAAUKuQb1Z*C~VJeurnN)CmiAn&VU%+~aVSLtNQo3q9=a(xwm6DwK8>YN5l%R$_ z;t*EE6y%NmrM_7fD60OHm_?FLRJmC>f3bgv3$}W^hbHvQZHqxxsZejZXB8|`yR5pZ z*AnDo3SLdwqpd?R3h)V>v$D#CL;^G$78VB?!yU$Q2dEd6tbD`+XM@D63ah-pkCD3Q z?QQWZD_uSh?pc78n{$1hB?tSjyQQ9(twsvpP-wMl_V&VqaMm7l>$z1lfhMgWi`m^s zwFZwc-g!J<$IKzM71xt{+$*{-d|kryA`%#CKWWF}DJ@|AlKnbYsMsb@_+uNtu>}S8 zfth%RPt?29{YS@?vm* z@mR(sY8vt<<_=I8?O@oj>@i<-;{G6Cvm&csFn@5y8?3*mAEIyexDV+s?xSV=Dt}8C zedf{omjNF#y$kN?BMq0MkB`|0iI7`Ewmq36RYea@Z66i~Bzp;1*d#t;{{Zl4ODw>k zYjrq@3L&y#rkH|lbx_%1u50#sBW4Q@rYEcV>3>$zR&oN zNH5ZSS11!$&txl6WGsa)tyJ0!s5Nj}vVaIEmzztO+kDR zSQ~~l9Ltu&4fJ%_vIuL`*awrC?5^k-#Hj6?FotHw@i>~YRdmf5=283oGJvYH@XMs> zhETuGVu$VgGQRfw%FLyo>$m;kW2N#G=hVq?H@@7xfnr{v{y{Ibdb85a%a3xm+2eayS}|d@2iCr?`Tlr)!t> z>!@(Ms56jzA#QVbDA(l{nf=@_(Rvt%Bk0AoerzfqOl@C*kMk>H7-UXw_Jw}SNL#QN zUTX_6LL(6=J;hW^Y1vYUqQq(0)J+!F&zO-s%a;Bn{$3lukjV3v6*%DSYgu%Z7HDsJWhZhbLuLOXXKg1IpP~z zdx;v`9mc$>zjHgtergN=63lhR8DT97ZRe)q1{16c^97}Mp#4F~YW4pBgDr@fv=6#4 zTWy*0f5>&LCC6%JRlt3h{7b6^vBl=#)Z^d8Mz3^hW*B9HY2-N#KTt8M046&>Na^Mt zK(W=vc`F!ojgj$Tvkij+G?C|l<}D_etewo0tYUC=j^kI5XiL-Q<{9*@ma}A}I`J2% zeipx(Larg1H;3XNrm0`>9uoEg;b(=Zd~q*)rd0`vaMHPBaFf@e;$M<{b%a*ehbCZ*OLnOO7cT7+tU&3Cwi8q$1$xDdFQWG0&daixA1HcjGwB@g0(6BHOgJ0FHr;H&);>(FZzFfhNviA{gAm~K3m`<%yc z6Y48Bv-)Atal8@zrbTxBQw`1xo}#l`{VB5yDKN(*X5GY>s&x=WDOFhZ(=4bByGeFk z>f>Bc9Ahg8+M=J*QCLR-l}RyeRJ%O*F&4~?I+4v*7~3#ng{&=An{&n>ENqs>jx@PN zWtvmPVxt%;gKNSy7^hi5RGW4*>RCc=_$?YoTO2`E-o$;OvrBv8$adH@tydXV2FSpg zymW_qG#w8uNw+8+#g=7t5;eXZ-t^c;|06 zZ~d2yMa)4IunwGH{{U147iw*Tc`cHn=y29=`M>vauQ#jTkALpvi?PM?{{H}ZhF5og z`zqM#hxG<_x_`7bEshuWQT|S4{H92GV3A6md0hF0#bD*Hrw}aDy#-&XMb_niX=0D4 ze>3`$!|ebH4O=x_mljfKxM9VdJu<_pqJZgzclNbxK4r0f?1IZaxAuo!sjxQ^aj)?T zs^uy^fQ84gJzWcZR9GIYcRw-8szkMaP!G}w_#gWj z!LTY;td#uM>5rqHaS~DEjZ6VtCZ!;nmv-CENkAz3{YUGjN80p2yd!IeCBdL^zcEy; zx*lp+m4p=@K45n#n$0!(m0L=8{ltdY)~bdLRs^pDH5V5UXFLX2hyId;>`$_xJm3Y& zgJ9=xjsfuz)#l%lbbL;nyD9v%uTT|?bCo+~N}=my3{{>daE=}mLlELVHIW!qdyR)d zQAfZgVU1L^$muIU&L2*SsBeB*fQSX5noE^($L_AyWV9A#E`w|Df?E1*rR}^|xS*3o z1azis;fpARQ?lJRxG+)yWzl7?P_}`>M znj*nqYBK@J!rq{P-vq{9g3~JMQx+&f$_C6Wbt!#xs@$hdj^U=%p$I6dw+oql2!$*j zNKcpookE4KVKZg~y5sI@KlFw$a}?7-Jx4$B1&sBZ@hR1{{6t}@t?5gRvI2(b#lynd z{Yt^<{j%>@^$YOuHeSQ=8BTNhurnz={<&ww`hij3bkFp9h*IFYn4s2aagOVZ2XU57 zEqZ1SIn;PUlmNbnhj_4!SbVNKg2h&Wb%82K-@BZ_AjOjCC4+@=}MNz~PFc zLsLtxBOJuk!L?|l%oZS`P{u8NM)5`FScs{hm$6`WB>bKcY~&282CP3gF%ZR3h^DF4 zi*6%pDQ-b(D#lmhp`@r~p_^TBnHU3gfVVEOxbWrvqs3RC8dJJ(#tfUF&r-m^-u!=1 zEtGAUHrQgDHyxWbqT11LL`B6{82E_U)b$g-C9&V~m6-2LO9o@o%G%Ied&Du)&e{dS zj*J>HQjf3K`{S8QCl&Mm0Dj;af-<10!P$sYZ1<$LCJP8UH*TfaP(QdE`u>f79}xq3 z{{UhGf~Y0oQCe;J5&>V7{?nG6TSNP(P|YqX_v+<`0XQe_B0IhrX@Sc~ex*%|tdH(c z;kUj&6Oeq5?Qzb0;Qs(HwT6b)n0qmK7EcoWVk*I(ad~-gru{P55)B*hk*XD4ls^n= zfu*|-%}TYP)vZRVuZvgb5da087w!ozj7KB8G4If{6O;e2U3OM{*BK~ z_m9wiBFUVv1mMrGT)myd6C`AH@G(>4C@J_^x zcvQ7_N=`1mFCB9oJHtud;nJDWT6!vP05S3{Mw}!EF4c05FerN^00aD`jhs(0gr!#; z!%7urLCk-chp>*G^c;tia|v4}7*K}R_0+oK7icUk#bJ+ds{op%EVB>HMA&e0?h|EP}Z-s4IU0Y<_ikXY|&J>RaPZ;RKi*YfiXctiW4jW zO{v~(K4vu!72MLO6bjhVjzkrZXXmx8$U@ za|W`MQr>+>4 zw6_jE@c^pm4aW*qCMFh@KK}soGJ-96uk9cTu*20*WAXGu6-Gbb@65y{wwKflq3*%} z5CY&S&vKaO7q6f9^$eWVKkxehY}|5>(irRo_=(6r-sRz} zm&B_?J|C$_QQw>Ul;k?0{ZvWWwE7bkd_+#AezqmdZpD{cE`<9U5kC|Z) zXVl}j8;QqtWtRrye4mJ;ZqC2C2J7&%LUXFrpdrna?vzAp7;!Y!I@2l&L=}sL-V{b0 zTD1YL;L40xa(T0eF=D4R(RAL(TlaRuH7!++@fIovTa0Qbpol_uv$$GC4XAh8Akn-E z-P(Zvi@MD04S&j}tk78uuYs?M|oeK#ljofZRSs}p02~Fy3r8NK+7}c9#s2^?v=j+NASh*6#5i7AcibKeGWP4tJQYygJ75rXwmr~s!1@2$c^N)eCGQt1c?$#_D57H(>v ziju7r$bWa3Q%b7;0K@{kUd9;thAELXQFNUVBT|GXdA+E-J;q3Vpyn zD&KgGls}qXK7jgyo6GemCk*|^L)@znS#~m@=%WLiRQ~`m4j)Xc{{YNJRnv-^?1{eB z5Wd0l3JR@VtUXEDrg*nVb|DSAKZ zBqqv~1o-KR&RrE_{X<<5c8~DPT&(S1nO8rWWoklm{{R9atX->5K4oPwqRTtO_a67% z{_YGp6BDt>dtt<<<>eZCa~WxCxPnV~LBE3GrC?fpjuzi!II#?cW{ks=1Cvh~pK_LV z2&Z)k)eYd$d=LTMXDX&s2u%Pu<`E`hH8D)qo+Ug(z|fE;?$-FC87zG7ulMx|3iJB@ zzaHQm4?h(UDycp{a1ZCl@BQ4_#7*II`~Lucu48;x^)NvjVq03W?<7G06LT;X(;P$| z0mYm$!KQpm`a_iu%mLDT!`wBPFRu{PgHpVl_uK#q(4-!gyu)5)B2qd>>J|8yuEp1R zmytghfNk^l6D2vWrNE|(csBe?DqEIdtGXN!0##@K0Ai$48%^g1H2aSw(Gu^Z1y#0D znR`Ho?5cw-9?dJ&N+>m2#w$K5BG4Pc6y80`PZ}r07KQ?Q1FZOq2dIP~ICBFL2CIKb z8cRn*&d_p;-p8~di*U1o`~fMA^ta}t%vMpgBcSWtP{<9@9St$ctx@I_ASoGgIF18D zi%>l>g9|byrHwGETa97@tqI$fO7RUP^a~;o7%WEr0C1Mo>BMH1(XUpD6`0|3GeWH} zR~z>#3ZMr(lG!H!c!CoxkB1Ov9cu3|$rPKBi0L&P>)QJx*eWx|CBm$4U&d-M1|G^e zuPBd$!R}qA>t_+wJ7g(Np1z5bFi^w0{P8LepsQEW*KsIQXU&n)4u^EO<`F^A0@sU* zZu0X0%LnRSFv<=$a0m{UY9OYR^toVp5Ag`%iL}yXwUNS0vnEb~!`RCf)>!?;;ES@O zlN`sXFAD%#AA+N0kgLYH2Y-ok1gbT$jy3TN0IZHhNFAn&Ameik&Q3v85Vv%l1y#LHl^aU|BLvjeJ*Du3+M>7<1kz?0 znLhAGG@C#8edHgA{lMN@{Y46zx9mb!%&b1AvM<>dyVCyUFXo>Zl^=9S ztEmU#1O+opWA{X7gsL~zVpzxZ78zydpVa>V2N0(^hvm;OSo(j7O)7qbMH3xk6>0kz za+~2Vh%*4R163_6{{Z4{<1mSbywr@{W4%Upm8@5_t;&FsW`>H8t_()CYcv7$GTGed zsJ;t=C6R#kPWsT3X7msn$*JYX2(rVVj4sl0j4;WRfR0pFDo|_N@GJT zZ!Kn4X{zDojlkU!r7H3so>+v0Og1me{6^MQ7zTnOTj}oMl$k}kcg(=TG|Ici#vF{P z?x=M26}C)&yv(n8KHfXe+sqMuZ{9fn0Cfd;!NKVR?jGV*Ku zf4?v>e~N2o{}%4a0WCYI0h2;-~|(*&*p$jRN1(a_%` zMt1q`7Bu!Q>8wU107i-}uY53D3RMOAxk;(Ama7)Z7mV&y!qR&oB(1aJP};x1gdnQY zHP%oOZ??@wIyblVl(znVppv4DRV<*c*qHrP1l9@3;wytyzfmcaETitR3(br$uwkyF zu|!gW7t>utostew%NJ7i3InlrU*=I^18!;9U*fmv5%R(+FGsGXip-Z3%)1U4-T z0o@krGeL2Z)Bv}r0#_g?0Lr-$rd2Hqq01(&87HBI2oG9J;L{=<3v=CRmZi((g833y z`HE3{Ul{MAVA=Hpm8f@E>WCcNZgn8C+T*4;i-aE{Yc`pZuKvAD73jY(t!c?|()Ifx z{G;(J#1u+Luip_zsOgIU>5OLNWkc1z>QmcM@ZdFY3V-6w|~Ulzl~jDSTJ+IdAI!0GVFLoBse3 z-#Zx)smRB0GGFProl4m(KJOGR4x7A}6re1J7^*HsV(~500@le_N^NJHOR&H~JSzg8 zvEosHV^;nybr>b|0BZ z94vV~JNo;8{{VUZ{{TLxIBCE6$NQ*YRs4N%{_0z{_55-Cd`mI9sH*Rm{rHx=r%)Ch zu`m^iEGgY{5wCjNTVY}^xW8ClO#&9pW0#~&(RQ!y;OMFWNtf9v0;{ko;KXhiMg@T~ z3Sc56D}z>_w|N&p+Q7wc1(Wg-ZZ${h5Gyo> zYO6AXnzk|VQ~86<%Aed&QTUnBzxx?r)wL>|!doorRdrxpVMe#gCOU|^z*(+gw`3qv zjniC^RhD`kRp+z986xI7cZ!Lp;VOIt&|RlzHVQD2#wIwn6H&KhMN)<6j;V& zY%aE6xEBy%huHqd@R$rkJRhc*meEyxv`F(ppNjq>4?+MY8~*^ovj% z=d)x695JYu0dguV1ty-V99k2+eW+zS2Iv44cKy57buhqEL9ho*Kf%igOaHrfloomS{mJ4 z9$i*~u;EVpMq>2mWh*{lT=2nH)Er88&-ExL>~NpNzrx>`FM|GJjeN5&aAHL$bX*eG zBjzHj*YAw8sBJtW_bJ6!`$IzVT?M$5HBJw?yJ+QN3`8h+hgiDL%QXWv#2^e+$GFw~ zL|*>@&|&P$mWyc}LFHyAVNKL6wK_TJXXXmkYb8^fh@^|o{{Y^j5fxe+tU8FrVs9~@ z==F^Qg7Dbyb3yuxPY}+hD*@^?7Z|EzfLLp^e=OR*B8Lw8xS+AeKk^`1o2qhea`wos zpVV_?`aw2nxrh7x#Sc{r)1h zxAOt}mz#~l$KUQWi|snxV1O76!Lyg*rKyzlV03LZN zxIf~mHqT-v40Zbv(~p?BW$ZM6F!Gc;YECvBONmj5O;c%qUgl)#5}Oq$wkMdjq;y;x zaUO<^zYwJ~HPQu;8ukr7d6lHtXRX`;f~Y(1{{UbQ+5Vu~-bNykkN=g9a zdho=qRaoAyk?IqdSbUm^Oe-riUD5lB_o?rg zy9`@p!Hm-vn@o=juegV=!2bYShotYgUzo>HDub(0e&%}NOEZLG z&M_^1EVH`Bt?pd=QSQ>Zo9!V*N;IhV z7a)ZtB5}7=Zq3oft--??$WCM+{fU6;q1v3t3XzJzY%2ScN^9y5dkkTqTR6 z;cUTIZU`V?v4V=RgaaaNp>_@`X54RKAjeA4tU>|MYmsUWVkb4+$yY8qa;5t(^q@VV zz6JwkCFLsNgW7MVD_+Oy7XVi`86lM*7CA>zV|```LrF6I!D$xPcUheXDb?nnL9TDu zM;u3;MPcU{&&Tr1?IBs8v-yBeIgMp%jLMf2CPCeF%H{B-RX~8Cs`zmMsaU{Hf`Ye; zq9Q&*i}uD_>X7Lg{F6XOvPP-?|E))+9+g-CZ%H!b%Ha<7ViGe7np zr~;!ZMdUQYQ--#%O`K+aW6$`D8!~0Zr%ocMxC$^$Lw!KqX?Ke;0d9u{c4;dBCL-0? z`JO{x6vMZN@Wv_OeKFhL?f`!TPpo{weR_@Fo^j$*7|;dGF92HlnjK7DpHMfkQ36}j zKde+*0?l&fKUKY1+yvxT5my8nb#Bwd51?wg{{YCS9FBWXxW4WuA5l}aBNAJGEq^3d zC8IFm6EF%DIDy2V6V9Wew{PT^1R6q={=dA3xCnQI**I)BBZcV80LjM&Yw42XRJ*F{kfFQtStm2rc$6x_NMLa@jf~~8nt|Bf<_tYL{5cj$o~1OIxFOm<>wtULw0Nyq6XOQFc)V!(}sz z#pB!tdbLYvrWvA~YnXalII?%b5Cf^7R+g8D8w90zD1;CV0L9zQMX`|x+8GAiP5}%k z5TN0FMolz`DaSrY0A$xd;|=$4W~s~$vY4b?fyK>;&@>wW2s&VtkCcktXoDufgS9v>GS1pFHOLeE{d<|GMU z68``hlJv({!2bX~=5kmmJqUp#z<9VD^4R+#$A9=h)EZCD7;;O)L|l4fo1godo~ z<_^6`;}lCbUBy>Ein=tudx{AM%?MbdhjG9QWDo((GAXD#2BD)I;~ z!-4pU=(%CQ&r~081x$+`qbdbzE6o{GbVM0lG=^-`+!z#Pym|8kAk}l?Ql>e?NkdX# zZ2ig*sIGg{^%-<*W>DLs`$>a7;F!$OEio~E7DfK%O8)>a^#e){Bb!H`C`wyPGrTc& z)r|OY8KgIy44F#sK+P9RvDW7n;im8J5aNI^<5Ihe63P3T-stj~^%hu+4JHqx8kZ^j zn>>a(IiK+!a!#v+mP$*xDg&@hrEYb4I z@FqmYZ;&{eG?YTK!h8s!?Q2}fbuj4#lumeGz_9IAlm$K8jiTksi9`i zJVhqVEnythaJsz59imimo?7A@^aWzql#jLOPO)68Qk)IKU0YU0GNy57xGKTO!`FTFPO#u0O8_Rc;9Z` z6sIeFxr;Xlum_eb3F6D5U8T$BeFc3?AJ~E%-^}0PS)g1@Kn{i;rL2BW|1BZ|k#vs@UoS@?K*5VWJ++stFBp%m#Zm#=o!jWi*wq{G4FK z9~J?mF{^16Sr226aQx4I@;r*Z7x_};(OPA`Ymywo;RxgLV)j;&&fUEmY zn;sAK>R4k}ANhpAXbN-ZF{}jUfmn}=#pqxKaY+3z5uG*Gp_#_VF#eFNv-1-0H~6@K z8rM;=yelIP-xULCLi`hsZadM0Ev|LkvX`}DudA)VinNf^c2^Lag_fe%W+DY77y!Ct zh@}vLM+M$EVs*7bWNA!>bIj7X{6kq<;)qqD@qf%>?@24i^o*QKrD$&~8&;)vc5U&x zmD>c$A$hnfZiF|kpSiP2sPH;IVD^F}QESP3_?jfUy#z!&&buXyXobVqf$`2;erEY- z*Tl>jKr_l1#@UONj-ocgvOZ5x&>f(1EzC0WW9*mX<% zFm)E&ORry0EF_AvE;qnHDZsDw%s~PqDs#kEPFYBH78V&pMjG=dF+?Lp8LKLxOs;@X z7cFAdZBz#)aIYgr=$V1-LsVQ_)6Bpd0Qf)$zbKmgvW_TSp|k2R&+V@G!RUkn6%N{&6{#5J8t(4qOorT!$z|gjHP4p zpuT4AFx72VZ2Ze`*tzrq6kRs$2GykX5MofbxnRom%lIewfh(2%paHFoL^@IY%h~=2 zXr9~rqLn*DCBp248cQ#F{4*~<=#GsYG)asbb+lMvP4YnUN^2xMv~xiI&`^$Yg8oQe-ZySP8hk>@LRDjt*IMQ# zQBh_fE{I>bb_yC;rj)w_#4t_iDw_z&^oY^A%CNtP{@BQX&&#(9U z<_%%|eQ_53SMLy1+y^E2H*o_d%lnPqzuJ#n@AzN^1+tm`C719MT%qL_Lo?V+(KrXo zBp2oX0NNMzKjP*XKf~O?K=P~mX2H-WImhu7D!r=3C~G2e_XJj#+Pc&I!4cDU7T#+= zxGnMV2P>qXYNE)`)M#N~PPb$Pfc?U*302j%{lIc8mL2P5Ot}>jwN-8T--%gCUInWT zGsM008yit$D%l$$jl)A$$i|K`Zh0YW;5~#|;-KbQ;kA{V=*%H2#)bHcmXXL0h_zdV zYpir)X5F+6ilo)<{{WeGe%~OcTDVcQuv6kHf>B(zFF3Xf*KlLQqo6!-mXOjJ}>U54uC@Ay#+X4eAl>-9 z+<8z!T=WQ7IAN?M8ab~kx6BgI<15UtnzYw-rhIwcIMeIz2n8%i{faVg4BoLze#h~E$3%D!Xxnh6Eg3vrI71H!*@^FC4H@g|VkSVZ3 zZk6MjQu%l$UHyfd;SviSdO{_8(3OFmS zCRr%}PSI?9R0miM)0jqf^ua#NY=`-4V8G!k8P8CO3??Hit$dwaaflnZ46qJEWO_!b z;{wV-y8}$Zm&~Y`I779+m{5I$a?dB9U+=^@KVKYwxPWtq#|+ZF{{V0`j(^|p<|>Tm z6;NaYjD0~9TJ_by$4wjfnWPZ`9}^d<`B`d(OCn7PZDAp_2S+lzwGyeNq0daU1&S>1 zBMilAC{_ip#J!`ZkbW>gD4~E0UU4dl3(14`;yaPcPrQ*9kS#z9>;A!@qLvynn+)qv zfx2}L5G#wt$`{o!mD_zI$LbDix&kO<@ig|w zfG-w8h%^lCLBj;cjKVE(P5G{`m@q17n)HGi<9wRze8N~8q}y26h?*q>$bm@umwy8 zu$JRoz^Y=iaPy^zU>2(mas18IISqwX7*66s6gRx#FOHy%gWXZ(Dsguyvp9NP^ z$YAX?q5~mW(Phy|SM< zF1wxuxqLysyAOdP^@>VbsuroDj7uGbkXeEqXbO`H@mA%xIc<$(aO$YzT)@IBX_w%R z{-DdcAt6Ea+_x!?ra*JgQkFxRP{6ggD`+98Sf~1yFyvT{Mm2YFiDzMbt1;jz+Z`Td zhmq}M*T&DNz2EBOiM`(g%i4vR0BFD|vPRki%HTe>$HM?2vIk(-xU9V+^9YOJ5ddDr zd@+6KCFmX7xH+F!$4nb;$Lb>!c>e$?fp&hDEAA*+y0{QWpW_hTEEvr}*s}PEd~5JP zUAb_5K`%8G=jCuT>hJZEgL*$oX;S9V+9}*|jbDmCFdGjis2Xj0bHoc)cU|TJe!F>= zFw>5CfyO-d%LD*8$Ceft^{&|A0M-uL?p_KJ>&;Bq8LqOmEQRk{^9zCv@%>8yefY|s z%%&}+=HMcktf|S5`zT=<75Y6zd@DB~%6q{XJZ|tjRNEd7_bPboA-Ht7T(n?jrbx7) zb7VBXk&H2Cy~0(l5_5{e{$hT@Z^L%ioN+OD;pO<_>+nu=XU_is-|RzQmA~#5xg`T^ z>*YKcuA!?Azu%c-#}x}jX?lmJe}bvNyp;n;loqz+oev4Q6U#r45%vD3&N$tF5jh6~0# zL$prA9~G2>fGZGM{4+6WGAICI%HkGjlAK4@HcLS$wu?^dF^N?L88>_*#>6GKx z*~1$K8@+#?fG!rnd#O^Yw=^Oi<*mbgfibT$FdJg1f|Lj~yIKV=cH3W|+Fr z?gy<_rI7~cZ}^O`u~5*rH+m(S%Q?g91grbvD;YAv+Y=30D0oSKWr)#>yv0FQYaXK$ z%ppdzA{~irw=Kn1{WWr}qq| zNXGC}>56eeS5A*n+c+Q3{{S@Y69^gR{$LOYmFDKI!^8EONO78|Zfc9Q75z70-=XejwIbn&Y z;~yFJ$DKp3cT8js8cz&dDw;o2POEjFnBk`M#bj?xUjV3aXsfsW^Dr$|S6}X7L!!5@ zGQen~b*3mUgI@jni3M6)C6+F5_+_1fsrQH=ugjw%?{{SKe_x!eOOKdjIUPvqjseT{(FoR6xhU>rFrK)Zn z@A;Md;>kGpfG^n;sY8COWAq#J{{XV3LWS!r#UxUzN=FRJyw&LhV#1|-1->qcbS6-x zcB_^16C9;+xm5v{BJYL<+}rmWti3_r#2IhmRYPKYGV4*}5g+u%-FYejvvlC zXDv%~vo>cA_)35i39UzTR|QTY4T>pBRkc?tt*e!bZ~@Ps_?6$dLQ_6tqVj%ND?wTB za@e`QiIk6zQ5Ca%uuGP_8u+LM!S=#-0nKNrdWUE;*@6c@00-(`XdpPp{UCy?P4Grg zDE8x(esc>~wQohULB%od8AjHNO7dqCj+AvT^BUD$7y+ovD0#vcmHn73>rKAnB)V|6 z>op0a;r^rfCH&2Q46Pho1%fzW80`CHxI-wur34IU_Yze2ehi{k4bC8Y8732KmMwdA z4HGvEqRsv#mFg=;i(rF+Vp=d2AX=kZmKJeeGm*RB)GF8i0GvSE^ewWt67vuyVhLB6 z;_7QU{{Rq>pj=`7%IwmDH)wvK%pkDYbMpg7Z~*Vz zyed_d%^WdRPSZ_TW*3?QfZIuEiy^DcGw75+f`yZx6>`?#i#>Eo!{P-kz_Qo_eZ_#V zL|9$6<`7v;fS)jyjsWKobOIz!QNK` zH7##Byt$S;fgY1)qE>zq)!*m+{^B+9e}-79Ijli?b6UjDh$_6W0nk0enu^5mpUh=! zaTG-I3`oj5)y^^6bm53V2dV=}mvONY%Sxv#G#jT4e-JgIq@cz*fFH@fiD-ZtwpDV) zPSY1kh+(i`cDyjjcI6;SR3#1+YW$FxNGw*I_gjLD;{K+p`}{_ud=JwNWM2`KyXZlI z9V#rpch3PH@ccC(4DMD75k2Xe?frlSFEbbP=FXCwW@O2e(> z@6jwzYfQNEvpN@SYXN<^)Ng=Rpq*u_`ilwy^4vv8N_EVofseA^#9J~|=N>0@SRUXm z3K{F_U^l^v{X(n^YyymRSV1Xg$Ybs%#&Qr%Dgs@A;Z_Y~#L6AP1Ief2R8Wz;7fmD% z0aOj&Za*Yi-X8%7j-S-ry9Gsnz8FGyVbFU%U`DVPYz&7jkZ-lE zKra|adfO#qbYC^W+K_?*_A&^K0fVIM!Y!}oD0EuOka0&!%tC529lV#;I z0l%02!Y#tI`HUq&EV0P!NWef(ptU;1fjA(;06cOYp+>8S8EWie5USA_q1;d&^LhQi zmN%aK=4gvh%Xei?P4kxc+!BK4fmk{&Pb{03TK&I*1AcELDtp)BP*#G>9po1>-~>r8 zdpCK^F&r5hIxT?Iu$V72g&9aBY2+*V36vGpkf__ch~m(}Yur+`5h`p(FmoHQyn>wU zg0c;Wnm7#0M!0nN(E@zZ$xbOV#H~)MOd+&HRCYh9l)WDbwjpLkr$n>`J+@YygE!0}U9`l!3L{X_$PHu6<`#;|eMP7=xkZ=U3`(xE zNA6?`>g`_{np7#Tv-_M$oBd1CIX<8<7b;_gO$Af?xPst#UL|M!OsU=69U&D=)pS5) z0@RvUE#ut&oG{bYRL0~(h}37m)K-k_3$q?#!tpxPLVz)gBwgJah0(S=VW`H1@AH&U zS9h`@^uFda{!l-d)Vs&#}AC*N8p5L53TxRp#@k2x{go%g^o>9tRT2Y@*F(TSzq@7%c#0i2RjiNnJzP z3ZWI%c`83J;W-TZIa`Fm@5}lanoz(Iu;hodyR7NI^i3%~pO)b+LxP9mqN=vLTm8%! zVZtI8L2|6!5v#l_`iNETn1WOP0Af4`aI-%dgiDO<1aQ2{zC42RCBs(O+j@;P058g- zeo`hvhkm6*THa+3=5t%m)GRoefYqq=9bj{})Md;0^D7^2pi#mp_+X1ien{eFuW<6j z$IfLt{{XV#mO7VG{I&gLv1=a5b~>-b2P=wvKx+kSF&eUc@el=BZx6bDGE%RpHz2T-8O|rOvS^lS4XRfZKQJZq*;i_-n5Im%z9CoN&_>_P^WQP2FA<&D1!xzP zGR@o+15w#&S2Az`ViJaD^)Se<07F1tU}<51xT?uU@<1_Y1l4aWVC?HqWZCD5b~wAd z!|#b<7{mO`UyCutFx+C+e^RV!*ms_xCqZ--^gnUxRX8gRSXti(nmTcK<|zzYt2*^? z592cY%q#{Rr}K^5_bL4h1Q+5@?j8@+5I|j0uT4U;u>SzxxNgNI_+$uknh5I1ne|%$JAdxkk z237{GUxqE>7qG|&xV1}cA3gF!9-NR36xr?o1h*gd7`CP?vc6(4Q+aXJ8jpUiFjz|J zT4+T%UK~J39)R3yH3Lt$i?{iYH=F$2sjZ4y^up7I0qk|+ST)DuzvdCanCN2sL1NXn z_4|xfLvIU(>8cP_QxK>M*w&z8?yd+$;t1QwZ+8<=&$S}766(X=C6tO)q(!eNdn#2m zRZGdtJf?kIz>t7NV(|M z+j;%NZnY`_RiG%A#mm*r@2F&b>LWOuGcilw@gJs>FI-@W++iB)3*63SmspKg zZ2BW|JE{4dOPAP-f(ElNR^S7}b8eRrqSYKXy-U*n0O%5tx65Csl8R(BP+l>Ky0qe} z{{SZ1hKxlTyWtFIjtWFNfgi3(V4@qaFi4PUMy*vi(167Z777JxM+MOZIR@GVy|v7z z(BK90L~gvFwqdqfzoJoR%KC*EO0B>h6E)Fra~hoS77c1rdZPlzZ4Zc6KiZ7B<_a&t zjjNuImhk5GM2T% z+z_k+24u9`7G&-y(UpGCj6?usCO+h<)2xNGTqD+S?m)H2M4KXd)f!dExBT)APsm_)f$ajiZ)hbNfkg6 z91^oImwEFAc^CB-%j8dqdPi&*j*$Rsf*Xd;G5-J{^%$xatXiY;!>ShrqMa8gYyNpz zNLue|pP@Km`TY|UVd|fj1qsr=vp8G|@hbC%{Sqo2<>nx8g3cc##|?}jD7<{rYwo4} zRB##q<3H4|Ql|(T&S)i^4HjznmjzK!^hQf@SOxvex~jFVI4jg1q(17Z0%$s03XkSg zO*k#KxW1*E1+4!7t}>3#=l-R6e*ne%n+@2^Z(*TDQM&U4akx-cD>)`4*muiZ5L0D} zw{hZDwF(+hJ|1PfQhtFA)Tap%uaa_t^%)l0x7wBc)Y0iZ4% zZK@ecOe;GP$eqSl&|I%2%eohsg1?!6xf*~0S`0KwJ=f+Cj=q1W0iHcX%s@D(4-J&h zL?A)AE7fsQiGA}B9DyE^azRn1vMQZ!5h!RdQD4OP72ojyZCqcd;N%|h9_ZZPlAnbx zJI1A&%<;#VV}B9dLd8ADlUjna=6#WGG3(SAlGPmtiQVB>I0A*0oWs1iBc*zUA8YCb zE{EnAKv}Lnt`LY_VX2t4hZU^FVO{i8wD3N(;?Ul$ zznPvzm++6P5laMbsKD^4xz)c=Y4Dht=UeZ%OV&c?vK~v!4P;qWk@FRRx4vMDDDL1! z&E^3%e88~Z_>>&m)G5#R9e{C9bFL>_GS%W`+C4a)AMN^pRBcTqsJw@QTUk#iexV?9 zkGP3W7?i*bg-zR0TV;DCDd7(O}p{@^HK>YE!R>8!w{QHI4W)@IeQ}G4U>43&L zSxzjx3x5Sp4~~DdEJwsi_=6kzh5gG_;_L1fhXOpG^hL!o=qH;YqTufj%pO|0qQ96m zwR3jbe*|#eTFFw!Pv%m|C6ueO5n*v(6$zwOfL*m0I>2+G(*45f6N9_+1`;+_bB413 z?gd+WFz}S0dBnbg29T+rwv+hYA5e zgU#R6#Sf_HT9PdN!~2BD7UIfE)PQsE$rm^Tb=t9p^Tb6b6lC!nms{dsHcf=Wn!&*W zfI$%XQ}~vGnQyCdlS>4`0YeB74Xjgk&;s6HEK)X)!~kq=W;t>UHG7p5)fs_vN83Nt zS^JEDsbQ9Zk%+7AF{inEXUth58HVP@AqiAZ-eQ!cw;vJ3LCdJAn`k$_>HrqVE@(m? zEXtQW#JYOizOUkF#y!T_X;-iUd$pCqG3Q=Z9rj)(IM5mVzO`Nuh^#7d{!#0?1X#v@&sshy~(aoQLyUlZ5CpI+ul zvSPv01>)iSu`x_PQ9uHv3?=orSj&@niJwiy#ksGzm@m4Hec&>_$P*&Yx|fts#IRKF zGJ`0-E+s}puZOi@CRUs8Lt>m;t_2-%Lk_Yunr4~GaNt< zW-f9?aDT~TF!q(N2Vj2L_5*kE4|tpvsPiwGu-<*bEOI^}Ykpy$Rm>C{@6^5p7dOOu z#;L}2m^M||*6RJkXbve!_~e$gfl39s!e{txe#A=jqbWOz?4}m%xWKyq0N4pPV=S_W z8x#%u(pr;wSF(Ese#vonYwU@ZaJ+Ykc%F^|vOjiimAIy$C-NGzSLR-sb{gfGc@7~= zI5zF#3*>f7Rul*VSPZX28>`h*3i3cqVo=2dML1wOnLn7gE1Q>F#h2;{PBRM&?kW^T zi&9h~%;~N9ges|d#hzHZq-y}z;sYuQ_P-Z3`84ax1H7Kw{19}329Gl@33DN3b@4JK zzqfVV8fUh`VR1{o4X8+d4|vIgHgB>xzB|ZH~`!*IEXpjZ8y+otKitlq>yF_UsP*jt#^`U< zzP!vt7|a};=3%B|2 z#|I{TenCuTAfavNEEk*~+$*pA^#HlL&rvpMYwBI&{qgZCcu(4ut{4qw0QQCXCTOy@ zAT}zM6DM3u5$;+$&&Ly4*1s`jojHMlYYeV?zvP2yLl}lIT;rH@UwD}F+YnUcScWUF zFgChBNr8Wsr9%(6Hk=kC*!2!n#mW1R3_2EFv*#=KD?2^Jsc!4q{-wrO*vdxD;2*e& z#O)ub9En}OV=32oAIwZZy#YHTd4wGN&9TW~)6F&a8$#h`c0jtC8L)6aa@hmK7MQ!3 zj>-u3TsW$ailPq#gt3-qh~SB?6OS=!1p(CtpsKUyh4Bb2*IJfw+@Te_^%PT%p|`gY zF_5Ecsdwcllv&BM&w@5DAiz#lkbOm1kVGq}*gV8T6`<8hftS3*X@FLD0p${xw4h-4 zz(LyK)b)zg^9#VacG%k|#Y}u41B2{w%`&!;=k{!^9G30iGMqVNoXmN zpvJ|BGnhvJSAzshKqb=O;Et3;zD%%0r5KZV@9sWmbc=28<|%+{ajEfpIby5Lc#ONP z<}_89?-7V|+_NtkgcK0ML^mPI-`pDKsJ6PyUpT!&L|IHBIl?C^;~x>qpg;ofr!tLE z!qljU0PsE~LpQ(4taVIUrpzg8ES_7iKq8zJVkbjAQb!XID zfJCZ-&2k^iphRaS%)`6=tvsvLuiS1ZK2J;#ZZr_0y+;ZX_YfR$O9vAk?pX2}Me7o= zfOgRk(JvHU<`eR8PA~IpuE1EJ7Qa&hG&6#594`AZnB%3Rb`V`)6Xsiin!$U-9)=m2 zufdnP+be2o+j;%LSVvMo4#IvJS>XGa;If#`WecgYj2Pj)PZ67kU*NsCjkv=s@fdGE z)UcW0h+V8dnV80|X2qExu)cq&plX-Bz`9mghr++r707G+#3jm1#PYn{8IKv1rgwPg zh_tVWC;r@8XIBJm`X*72Oa|c^@vgt9O!9K9c&A_ON5Xgt{6%Wh+Y}ogEz1gUx`w(J zIpy=O)EvMc;IKb^8MkdU77vHZM>E&91$py%k==D z*Yt_hb|s>yXp?+azdg;hEah$>FMgmZ1{_zY6dqt!>;B5vpXL`0#aah=Eo-7%sMTJM zEme8uUySfe^Pw~gR*ya3=BSW2WltFu^9+jgVyU|^4?}=g>Z8gnBym+=v5t;fDCSrx zVmQntLz7(Jh@zQB02gKnfXdoy39I9n6jBQcY~rUD^D*7B6dZdcC+ZIt-r=P)UFT%Y zgJ3P*+K6Q{JN*#W!y0p_Dmay2!GuI|<2oDphUlu~Z|+|-ivf8Mw=3j}y0#w|nBjOK zjR7)|0?M7Z_nMaUsxKzC(J}i*NxM729kpnV_6bA=&X*%W}xr;M^5y6t%Ks=`hei13Y(6s2R7U8 zF__hlP`SN zJh4A)9lxoUpuIZl;ySrz+u~OGihQbjcs;8YTI2-ZlmJ9IQhvYX`&}+^PQnp$P}@f%kD!*ALW2-FAlSxJ#PzDSc{R zpSo@TE<< z^%YrnQ5^l7n>bG}IA0yc(9Okfwxu5sz;ja0>J2#g@fc3AAg1cZKMXaA7KKeQFzN_x zp;ym&%mCntUP~LR<F^*i(MlgVZc7Xo?sae(WZi4wH-l8pO-HH!kktHr#>2(+>5xYXtDs$^Dk;nZpV zXVeqH2gCtr;Ktal(hn)Rc$Rh5O?JFyCfr_PE7UizQTZUefU~Fl+zC~htO4MITngiG zOuTECsZoKKF6B><(}I5x)8_}mLlH)BPKcH;x;`9#F}R?Qw10=x!lP4uVYFSJsEd#W z;#4TFpOzT0-{v!HG<-~4*NEj2?hbXX$o^v`ikEqHd&I^TtmaZ?PC;jjtN4xg3l1W; zcnHD@8mKt}Gj+7mQ*Mmx)ZJ)HzPdE-Rrzi?f;%11TsUfBCyaE8*Ec2#^(tjXuQHW( z9l{rGpccEzyg~k&I*kN;xrh)0-NKjESl#@}1<|vga^GqX7skk313PhiJ0Jy@Te@qw zK(p8FiZ{4G^7@L9_D7cZer2K+P{`Y(KH^%;V=|RoqE*H(IhRyjVZ^hg_ldPz59VYf z&R;PrpOgWzx`^NTabc_^tNWYp-hVJFTn0b(0fG85JHYVQ>Y|lm`%msE6h}@Y`K)OI z&k%U}2LAvOklV@r)(@EO@8Fa-Z+5Tcl)3rC{mSVd^C-kl*X%d;+-XX^0s0^fSC#%_ zOE>tq9I7>A`XFoo_CID)_=Nr{h9IQD88HLD>Ju>Mm*#Va%}gy;CMs{%brEmwrJQ9| z!hk*M2j`eS%tJY>^#m!Z_YMKs^A#@!;GkmQpvOntedcuDCO@JyBs&-CG&Tr^M*(Xd zA#{tY9A&&h?9*LK_Q7*+2|?(~8W^DlkTctPjTVRta6-jN<^fJcrZe14K^xWA5CsMA z%o#0(l%5ED%D|`z%{exI@^-8^sIFy!bdTy)e=qJ}tIl&dEr7sr2cym)>;6G7L(LZa z!UR&r-mwwj_Qb3fCRsi*$I0;?g~|I9B3p4mar)wGZsU)FET^jZfMB2SujPn3;JSQ< zKG}I#02o$wQ7G-6PU2?dQN??XV0A*NS@!V`&1#keFpY(EoJu-abMoiZtu`E0V+zWF zsH;2*=2&1&UlN*~s&oF_4)*E3VXFTCYz1WYzj4@&YFC7ZI{~7buOrWK#!hPM!^6Y` z3h+$L_~*AZUCNhw3E9L--m?^|D@(&C+*#VOR}zjDyPJ?3^DkD|)w0x1LNsM^gyDkf2f<4q!C)M_dsO@tTlcG z7$(L40NhXoCDLG!Qm`_?&A}l`9zze*#hjo((SEsN^r#dF(UHO2NJB==#3=&St%yYj4u2736?zP2C25GOzNQ@a+!6&?Du$2TcAV+s z?ojPx_>mf^O~eoB`<1()8*SuEU2Y9a$B3Y-Kfy6Os{OGHO(E>HI` zX+6qP-ke-;easB*ru12VxVp?P^1-fpy8i&O>|Hhf^B;@B1nkRyDAX-g45e!oRo^m} zyoZiT{KC#n@PBD=3>5HDTbP30FHAilNS#)kTzsL*Q~Rt-Y%0ln)q*1*lBYns?BD5CZy$5F=jQ z(f2=eVivak<^a6JSK?w9#v}FO6$dC!jFHZE(Z(8x#n1ah6Y(u~98BtHYYey^3Vh2N ztn)?2k+doOB>X^=XBLyij-c&bt&3<6hVcOZ0QlMb!ov+$AaqsKKXd?q+7EK7*xp;@ zepfH;7HdwZ3HFUeZm1q@QSmWaiyVihAJVNLz0uqVU?CBwobPiidXH@#V5lM5yxT*? zl9+>{U$W)fA>WVeh9c)n6=joKN%n#3cYNw+wJ$JFLGki`P`^THqE=xwf7ug>I8VdB zWD9xsL8r{G3U7$T-8_+M@S^FNM#m7utq&2qIM?wVD~jR>7nOH2id`?^6qm{ow=fzk z#u9^j{j7WVjdZhu+#yY722P-A)rqFl2ksil5OOYJebD}(>>kJTL+CgDE_i!leHh!C zJF0&a$5cs>| zK!O3h6^}7xLe&q_TPk~2B}z5*6Oswz;#J3o_YPJSaM8ikEHQO(8NEwWInCL`D!M

(TPYPcBTlkEs8AF@#cPo*iw%vYUp<*2Rdav#(T?pvZgT__a;jj#Rf`{p#VgxNeXX%gw>QZM*WHSE%)sI; z%st-a5}~xC;Zn=m3?MAcd5%rHq@~?ZVY{b@BwRwA1&-3n_=Oaer@-UxT>)j?RI30r z$GDJTpez?wb==?NR*Z?^fKiVliDEqd3U|#T9D8Cn9Pt z1z~0mhJNB%Gs69%?62TC_i&srKrG0Gs#vSSeM_CSm|&|RNz8E8b@+w|Pkv%3;@O9L z6aN5F8BBl9UGb0GGOVlAXyv#Zd@ftF+*I6h=AppxFP>oaZ=b+V!T$h%Y;9vaSLjcj ziyz!k$^4Nmi}wN_V!xh5}o?Qa#VLT_2%Xm^%xb5 zMNjeWU7-0jxT?JMEG3Yu62U^}Q!U${Kg_nWfrV<^f3q9~$F$HwTmi1{iE*)cN~~i8 z`Gxdb<0sAv%xM9%ek=jlR`T2QqxBT(DAJ$CABk8Pfp%V=Ai#vt;oqx`d}NJnuZ8sm zdz+P^cNs_&^3^r)^(Y~B7dh)WS$WC0!p&#%EIu)a-4^Q`7=yxqiAy919>l@Yh~ z7S;x0U?3SICQD{uAh2y-gZ#m}$@q;jA{uN?S&<9YJLZFIxC$~^Ti^^4TW2Rc(ZsNG zO9(Zf{@9LkVDMcvf-B)j4H@EKLZ7g|#JtsjLhr;NDMYO2ABmsaP}6V37eun@!aYM~ zZ%l%52bFWe>9~{70A&TwvIXw0k8yahp$ho2KJX(#^G76ZER>0H^9YOp7my6!Qsx^? z^1Q2l9(f1s!Vi=V_==4Q7CG{%WeJ2ihBfeRHU_&mgM{AUbB%NRkBlB6Q46g`C^qo(Ho~-qAf_^#A`=!qqpL=pe-Nhv zb~uX7#ZW0Ncr!JgrZDzL8u)SUW+YpX_+t#{rhS+&QKeb`0JFr_cepL)qPrMiH@rTg zQU3s&j#o(d-c3dBsm~jiT*X#b^*+XlV@C!)W3}e4e(DveM^XB?Q-<|P^-v&obPM;G zBEp>h6H@58c<=4FmY+b=^n@%uIX`ULU0uLM1IpLjTXY}hW`0`_$f;3y!k77$rMpV} zz#$`o2xeE_A5a)a#BRX76`7Rb4Jrh=dJfx;n*RWbmRnnwu%|KZv;50NeanE?{f_?t zQ9ZrFtBpzuFD2X%_TFa=KbdT>Javk>dQ@+nvBEFO+(z>j=gdZh*CSt6N>c#g{4e=u zCeO#}Y%!ANJOF@jRoKgB;%K{8+u`EF@I_KeHr})C2!#Q2Z3o7WsfMk1EAr7eA_?;J zKMYHF!;8`Mexj}zxsKli2i+a!J1Z!+INwoUadTP5m_H?8uhTIg==A!32T~{; zXGN`X6iexB@DZy)ir|1#by1HAP;Gazh#(rhS!%vsVU{dQm73hSuw6e$VN#{?RWVa) zDjRQq24K`i#Qd3q+v^_R+9B68F(3=Ao>$QPz)3}ibUB#nnGWoA9Ga2D1hDKw601nt zZ-2~dcnvh+scb{FxbKUKk5cm%TOOrK>Me#BkBCk4+fV-h z1IPaW7=QA!{{Ym5N{+XflYuvR&pwEIG)u78iB{CP!zi?L{RfQB8D$Mu5W05GA*?LT z6%^B8#te`l6Ex-!gYa`Im^e{k?x975ShgLHF;)_e!&d|mB2GGbbo+%x`WcmyjAogz zH>TN@Q?7Lyh@MHHqzr!al+hpk4-w|UwUyFGbKqHT(*oHI;(Vl77+W!Z;&8VJWOoK@ zH>ge%b?3~)5!7R{s&|M#XHH@=EQh#)p}|`DiDubZSkUQu%n)*==jMm?6D(ds_HJ7E zgs;U)Bh%(R6Ssc`C4}|J{kIfpaF{REB9>X_FWzHoHrAyP5sGC7GRs2ge-kEK_Ya_c zbuDdww#y^eKQU=t({BF&%}$Iia6L;Wmr$`Ox|b0x^Mj1UAofC(y;Vzl=a{~0Z*tZ9 zmx`-<^E6*l_sr0Kpy5t`s9-9hvYED?rJ5`(x8~5n0nSdm#=u)Jiu+JMGckT;-=IHm z8f+6vkKy+SQUhVt{L;SIm1}%e{{RBc3)?mORld?$vc_8TlkJWG92y-o-^(wGbmEJ~ zzt;$bd9!uyDwyB?%|93Q6-;vR9rL-4uYc;4=bqt&HXFu&E+rzUJZ$-55r#ck9B!5$Z zGBmtOee_EZ9C5z-;tN~URNV$NQMNUDSO<#6U?|BkijR)rKq?29CsDXqN|a;yu~<$eT}MPstH?1G?Tvy%S+Bm(NP;&?bcK$fmpS%#Wx zxITe%6F$$81cfHY6OPrY7%N})h*3j}xPZl6qQ$e^0a(v6hab%4iC;(V0s2w&!O98# zNKEIA0AIEtupS@bQP^wbxAPNZo(Vv!UlP;y%Yp}qzFy5t1-ve=r>SDKdbNm3O=iIS zKv1CaGV0zXE3DO*`}&n}fV%MO?l=lw7u(MIB?aXw!^a1b_zD=h{}`i)-oFg4Bn!k%tfT--8i zT+MT;gW;xE_F5-HpTxM4c8YG8mzGL#<8Y3s>jE!H`XCX*%lo19N0`TML*}Q^3np#1 zKj7e(1VIol@uTQvE}`A?@WBGCtvbg>zYw=52V_=MmoS)hN^XK=mZ$0!8Ruw0$~CAyU=lDx;VDKXR{lw!M={{X-s zJcl<>So#*kHBUCR@h|L6;*F1FdazmI{{X3A_i>{?6~y)ksB6`lz8H#*G43!~Wf}D> zStT|*l^l~p{1Q8$yi=c;XxM(B+y;q{)NscAM1rv5hx?7S(JKC<22HH^+WA3T2;i2q zj-e27BoeyDC6#Z85xiGXz_UC@YK;yi3ceq>6XDnQD2s;E%mK}_{{RrOf@vdyyW=r7 z3!=q-Wnh@@Tzag%k!-i$UZQd+;@>`$xH`6N2zh7tDLT@`GVOLwsP{X zsD%!&FM;oNnM&n;EMwpzJW@5ei3$FIwOg_g~BjMXO~W@6@rE zpq9~_81tUJX_l;{%&m=0YWI49P*LGWY*?s11YL_j__6~LSa!pqYA*&PD!@{k&)hHt z>H+2T5s|qrg8bgK_Z)EUsh=z#Y-XA@JjgZU~}}c^vE1 zKO)T35pQ=J_h%SwjhP+lu?@bd(1^BfkL<{?lE)k2k96#&pcFTaMR3u`0)0I@|_u$=pjB2ekcULj*s$QF9$ z6>*sx_g3tZOhiYIgEm zB#B(5`q8(~+fH;&TEP`|v}p+Y#JqWz0RuD_ekNkD8f~>dM6m@%y52MIh{0m9 zN7jt61j)i0JHs3**8U)sONaSBpn%vG?=Y@{{{TqVJmw+a67UlDU}Wn}%Z-M}w%rSA zog)#YwvYmBgBDjY*#IfBmJsn5!9sii_$MZSJED@ZuwL2dEO~ynq3z6s4u}M{`fCY}P45HM0%qWFM zZ{j+DS@YzV*sX*6m$>#ZC-*wG3s1Vtx(>KOSH2h~U%nH{j`~G)zZWQ!3E~%ZE}&BZ zqvk6Kl?!3Hl&my5xM8$LCGI{U!9fw;<%x^6TI~Lz^#xyRnNSgO23zb2f*{cFyIam> zPYw$ZIK*kA-O_^{R%#@rys2yU%@Oo8A5{dlg`=&@`Xn0$G)~>cZ+61Q4JQ@4+*kw< zDjB0~UZJ}^pVKAOvDp1cRCEbG%>H58(>9;72nj&`W3B!oT_b)Vit=JtH{WuT4+}6` z-rx>1#0tN}y1&FI1uZ4DmA~A&FR#qK9Lu2He=yrlG0e_FW6~6@K$LM#EZhRQ0CHdL z54lT`_OtN;DRx&qx?1`S#jUh0on0wa#tXQJ!+c%e#$Q00Qt_7g&pCjiQye<(tMbb0 zW0u07*Wkr!(T`tOpP-CF;1^fU_WjJBRuLL{H_^?%M`+m>#H^>O${|!CLWZD)+Qp&XB?urGT;e1+R)-kGVip$O-9U4_qTwL50X#(cPcV2OV1~zi;Ei~9 z%yH2g6ag>+%xf8j#UF6qqX^;$o_K{85j=4KJ00p$D$C0pw{I>Ghk;r&6nJ=;dLmeJ1$c5F5#}=P;|k!TbzdnPz7kX^tOlQ z0^swwQVJgb0H|szJiC-+$2<`edM)FB$uOTiCa zN|x`zpHV7{e~4d=)-GXhh3^mxPd5{jK~MpwUL`xDK4R$I8N5nV)4t^jzdi`GOL4{A zu8RX1fn3~8mB!=hBbjs*qpT;KOs%Si2e(7qM6i)t>H90>gCk zL`sNjspcBC(^s^L(>Bg!P3#+^7Jt~pN>hV%%k?rD#}cdte~7BETvxep;^1?u zxop4QpaI+-rRMP8@dYy6+b)gN(1b+^HL~%GQ7_a2N6E;ESxA?!ZO1&uIRr9CY6Di6mDxVkw>Ib7G8u@O% zC6fR4!mI=7DPoL2ir>Uj07P;whdg%;0H6j(NIadhJ)|aRFJ52) z3yp$s*I%d)Q=lO}NE>Gutvf>^f~fv;{vv7e0mF$^ ziqP>w%!A7Sd7ROI5XTnxG*Em+xvyA?rd68LnB!Ky;JI@)Kh&{G`*96wlMQ8oB?mpS zmA{wxmlFAC#JSop7qSSY8xpupQ`ssDIX(vv;m4iJQmMtqt0}sfs}Qm^Vyl=?L9ktZ zCRMMp0^O@CJC`kH@eC_Bqi|$%zEIR4*sRxRyaH;r5rv+zxLu*y38gJP&a%zD1wfjbb9vyqwfEmd7%&5l^4GX}+e9H$1 zY=1^SfxOf_J0k^w*yhS+8&s?8z&HY8H7eo}kk~mk7%p>Wh6SOhi zxAAjJ1UdQCW)dGj{HuFg2-kR;mWP z(fp0RNor2Z?aW?0FFoqkYQ8i+ULY8*cEEP ze-!Z;_9?wx=P@;8vz>>pG0o$B{@Jm+s8v)4tN!~&tXD&vVvq|bBe$$EcTOQH7xTz)`* zIhGV=p>nWONa2WL?E-?l97R*lR~iW5b#WI*d~+<=-z2e?@^8d=AQxrVld@Q%u6ob@ zK-pVZ;sp|$)kZ3Zdf&`ct!NG3nD|&)O+PZDMRvpbB_Kc}bl^uw_Y+GF&{$^yzM;u! z*Z}>{)G$9hM+jF`l>n@j$GDZOT&=4(xWSAn%s#xulD_v5c)`JmxLLZmM!>}w2g&mW zsiy`H{f(0Y1$aHQ%wd)DQNi@FPi`2537qd=2q{W#=klRMV7FqzreQ+yd?KvJDaofklvjAf1`|bIcd|7`#@+O5C z2sv&AyLAIk;QY!C51hqHJ1rWswzY9NtD^z+77B5q8}2(F8jItfxGg+Ozusb_C){f7 zbA<=xid4etquqf~(#G1kt6|WAQeGEqoT5xFk ziR*AynC2xQS9dl_O|fe0&D_e~)CGEkte!>mO~ zU}SaWUU^9G(Hqaf5s(fDNdT{aULc1Ux0lUx>Hv2@?g^!9`HK?bhZoBEge{vc9U6j! z!g%O;cQN2SA=~rJEa+N-gZ`27Y5Ne$@76GSJBZ7YtK;98G%I=k0N6E~GQyIav4#Ht zf93`+@$P9(^?3FWUJzI)J#Ifp`Is|@^%sn)V!QJiwW(lz#G4=~k78jd?=>jkdxhq- zJw{2!A5ksSL0XmNy=0WXSKLj`Se#;hW>R>RwX%}QbSs!lA$&#yZCtJ8xLa(!vk24he{$)k<$_!_ zo5XgP;s+1Rb?q-kcw7pa;P;2g!b@Idk9iAmHUTZNLHi9o0HhGoL->;7_WPrkE-1v+J(Z?g=E94N*Ym@?w-KBHN zc+_J9R;=A{d($p-QzOg(`w!~k4Es?I7Apv3^$$+`Qj^?Hb8j8YFO!H7vF7=ST{Yan z=a^AO{7YLw>xczZahZ!B3(P9EP+G`%ltP=S^$wx3ugSRCnHo#45hH4=puX5j*fut2 zuX4?hzhp}GY1;L;5DFRdY#Rro*DPjsTZ_j8XwzKh8UFy1p=wrzdwyUAXw7ZwP)>_$ zqrWouD1KkWDa!fFUxGLpdMYhSE33;D864*30_AM#i1oOs`KMo~Sl8oQi)DtL?pB)4 zAX@I>T6mVfWKmR@%p3)?7kBEThNVfo=!RO^9eL|;&Yr8xYK-@iS&u_;D)KbJau;Yj zE3tZbgxPp|&Z1#sjvxpQND;nkvFGEtfn+&JwWk< z1mx6eaaU@2RWDc-NGVxm{0V(CUN*auxne6Od`Cy9@UCOL#G~R`$HK#X_?IfaClUdH z=g49NunAaCU*aOACX3;Ff>uFnP;DFm8E?ueyt)s}qhb=+^HIvGmD~c#V?zPpR-po~ z7cPV~T8`Bh`IK=5v}o>}OeW}1TCfnPvJC_Hb*NxO+r_`ccf$dStkErYUn5MG{_!mY zfG#gze9Q(kD{Ghdg>X^>-6+D@mcQ5~XgNB~f7xS^j(^*g0ioHhW+}&a)C~#??p?yY zdyP!GymJbwIc{=kT8-9gKBxCI$7}(RjppS<@^S ziDFxi;#gDpAWaV!h5JFn1`XZN5%}=H)NCkx2Kys}Y)M&Jk=_~Op zW9W_1i>Y;yTu_bKe?&F?(!tTp6G!EiD(}n$vBQ~_;sV0{Vm!eL!M<|~%m!}_)mQK_ z4{9$-UaqBumg?^YHb<+Qhu`iu5n2OOHL5){>G&#?YoGmGozVJL)n)7%OnC_fx{>}-(if)D4{q10FZSrhUwsN z@WD#VWlr~g#v-F_AxZ3uWC4t7XEn18-XUut@l^ayytcLSe^C<`dmm7aeNN^RFPe#5 zu@R*y_blBjYlsU$c@56N@ytn8%K&qjC#5(!)kSbXTX@W@f{S`x;$DL#zE&Gs6jpOA z7D3gpyuKN^+wLa(gXQ$^$FubE!4j9esD)!KoNaGhjlDj#gME&l-6+CHT#4mpK~ zm_;k|3-xl_pqVfJ#cKic16e>h;w}wU6n|3ec6Tj##|hNKC0?0^%v%i&_&|zo?Z=sD z10^l%7##YTQq{{@r?7@GF^n6F7_0oe%hd^?%{)L>h+5ZC!95J1y%}E;l<`Gdu2u*% zg^aI+1Iq?YtFx8${{Rn|$&1Oq4tJk+%pJR;8n}R?%vV#I-NOR+iI*Kl-df^d1X}+9 zAp9lYoj>uPnP;vc7XAa=Z5c6o)p!R|%Ua$o{^BVCTI7{~i4K;g{0aJoC(rI$^~v)v zD%`z6!f#PKZUIizy?$Y6($hSLM@RXH0c(S6{^#l=Md0g0#3$A26PTuk&+`!=tg?Y& zU`#is&LeE-;}sQUN1e+z?ks#W8{8p8&GkUt0FZwfvsXxN@bm8 zDi~RP69vChA?H@bFbDB0+Wk)4{IbfxFavpO2&nDxI6-SH6R0}w2(RuGm;B36QJp*! zWfrP9NBe_dFk5rXK&$X=JjdeXP+bT3nNnPZ_WSVpkM&zDuE&w#jII$Y{>&`{k$LX%l&^@2P#qBWp)Ov@-vS-dgtX(p-4=q6!w#n<6TY-Rr8s#r?xd*$fKRimt9(+x@ zcy$u^Vv2!uRAMD!?L%t!1?gN~En*|43f5odUq-OEN)^_JH&D??6(aPFw+&|P+mZvm zRptuC!rC>{DTJeSt#tN=#xje^)0{xM5}FM^s2g+hEu1Zu4B#lS&+>H*fVVtN(E(>d?znb;Cjq7 z=O&`h%nH1@mJ|7dCq@We7mlxTgQ%~KT>KBjaQr?Z$e|SBur^TQD^%UHW!7~lZ3m(y zHZc^6l%@|lgaX$xfu%v1@tfvr`bLj96%P!;E0DAo#pw8zJ)3erb@WPXK-d)g^n1(Z zBIq*Pm)kE2nQolkv7TZH%5}W-L7nXU!)Fm-&e#w;m_FhxSOeLxsIH(po-T)?Bm1%;N% zbiqnkyHo!Fr-&>(qkb3qB7wAU;~4W8D@p7h?Z6Y9&>cM2Q3;jl>m{}PbsVaIW#+wk zW#s{L00nR?pGwsBW00A?_MnSC3FMSQSN#JQ9% zZ{`a}^*&(JxLu)-a*J4dv{{Wa_>I%LhRJ)C1AG0R?L}Htzg&)*P4NoWJfE32) z_KYK0LCiwroH%`Xf)om62)i8%w+CuG1lFF54pVA$acD~(GFk;n4~P=Kbe zoj=1bDk%~-rt6P9#7m~X+@64V*g1QE@B zK%8MU5IMMFe-NtfDt?|~z7@H6Gd6*VNDFGTD{oNBX9TlL z@d^hlT+`~FswplnD7kW~)7RV)007t&d@jD<5LWW`dB;ak4kDG=S^Zzz3>n7AAlv4p zBTN%YCl!#!@$O`bry=L6hfpIU@c?FS8Le>*30<;uYY`yQ?d_QX9gp&*M5g1nT!CGj4SlXu0OOZbY4S41G71_Oi37zJC+OQX=`#G>+aRrn>3L~sxB z3kZn7%GRa_FKK#$C;-N;T|lf^Ib0+4A1Hi8rC!QUG3Ay-C7cHX$3W(D^F_QaXB3}{>V>K3ZK7c&p+g{dKDjT`KL>>-RAjpuLNrFD6i zen9sI1T}mCEVL<@_gU6sp+(j9qw_0KQ!yBV`tVJ=`G!13uW(D;ey8mQ3kA4VVA(e@ z5qFFscsl!t(eZ7}8xNa+L&s9ojw|=nYn)NK=i&-Z&NTS(8VPh)K-KhXr5ZS*YYvALio*pTucOug*<%|B3d8r8HJLi2->lP1G27;JS7ygs$J34 zNiMky7d7@&E6$Y@!GWm zZJwx!;<(K}mR7{QkGV@Q@4xs8Oc0f(MtG^zQif`L)h zd*{ry$xI#~g($`@Cv38t{q;F>1m@wUy+a3>=gu<9!+25^p?)b{=F=U`g7n|@dYE*#+B zt|NBw8qn%EOM#zL(szz=6+kFc>x0izR>UvJKXDIes|&W=#sGEbPbatmT#E?CBW$W8zx2X9P8#)IbN`Ta@f}R?1+Rr*O_LLIiccRiKmbI8;pmFrf*j>E?j4r zrRv1Z%+=5wO&R9CXI0m^Q2@p96cX&ERC=QNIIkp7ghj_w1DIb*Uc;DEwU+W9?qDnm zM*H}jCQo$gDh+b*jcX~zbuEDI*sE&j74%9_OLNz8;DBkwEP`EFP9kxy-Oh3qMo;bq zHo=;0h!jU8-5;2bcs`;lLO^n;d>$dprEdzP{K1MX@aEu7cvYx;N;Cs^j8216=s{$v zFXk_AIx7aN(=j^+>Soq%bQXs!=OWpiRm>0qU9{gf12KxW8s|T#42G#od>0JNRndxb zjmlcjhubdQw*(F$EmQ|9LI>d4DC=oLXJy5Z`!J^k=hG-|^0*U<;#F1z3)n8hgO*{` z3vw4~^4tFa#5p&!6B)!n!0sEc)BH=foJ-GAmi{Ak^S)(it~WKGt|I4O3`~k*74Yr} zXYS#my)zAPfDJcD77>nYt*?-1kVx$`q&yj=Vurl0Mx`8PpS-i^5D&wZS zVNRa3eHG>tSkD>fABYj)LcF;yp1p1b zX@?`(`i)wunW!(1)D2${GDAVt^AMJj^}bSK0H5gBET?cR76Y5;4RIe{(<>i~qHC24 zs8cAIRfdKT;y-Nj5N)Oj9xLhy{v$H;2hl?31uHL_)N^!eo&NyO#A*n7I()BCZ=_1q zqOrzi_vJO15O(PP;aPXJcmDt&WbNhDQB@I%uZ=*AR@+rQCK3yk+C=4b$5&CJkfEGXDTF!AmVdu*QbC+{*f8 zHF_ZAU*dUp77g-5GS>NYe&SsE>LJ5Vxt_9{vs_?{MV>Trg5mtzh+ z&Tvb9iYM{AjzEcuQK2Ao!2 zcg&y~Sx)<5MpQZe;CPYKXlnx%Wqq@GYMWqIgV#S(eA(Qf8yw^#N{7Z=cp@|iHCTTT zGXfR><)m30`HFzy0y-C7Vh&c zKeiijpD`<~&n7%e-|O76-26fT8ZoujemaX5^IDasWoTCg@F_>2zF>v9<`fvAF=el# zY%~^wl`-JhKCPRg%W!PhZ@1AcfK!^!-bm8eP?m6e#ywFC(uG-korfo!9%W}N+vdJ2 z9;ov&j)sm4tQD_-_X3msm~gYqRq=*G1*4v0%oI~{$U6dZU5;NrmoU@dXvKxq>gzQPdsP1b67#SEwF8TnvvJtm(mBjl>zECDiw+wUIalLHZrGIY zhGcd}5XNgC-w;f|OIeN(`Gc)#cf?%Tf1h!UrJPyMh$5ihTt~UbM-Q| z-x2KmOlvgoKp6A5#y0|d#2(!<1fqg^%uBb-1${?}c%(f?n}~F?R>YXD z9DXq=HQ{mGs+t4#)F>R-ef>gWa0K^_N+8MyH)(z<=cl+m5Mh^BFU}#8)G=^#uM^Ik zFk$W#)K=zP25fz?0J|zWX-bYS3_d8i3Vvo8$mslMsF^`%V+NPZyx0+{f_*?OJO_ls z9Qaf>a+l4ma|!fSgGt$3G4a=33;a#S_l6>uK3R-w^8;nO<_#24aSjf6mtHSg^9MAw z;xIVmbohqE@VW41CIj7dENw}jxW=Tv=lz98$!{R@<^VvuX0=_#V{B}QQZumCAVxE5 z*SMyIkoj19Fw6(AL@niK&iIx=hB(=dDX$)8h~-&0mc@_uJ|=suTmaMW6~A(~4cWnU z^Bn6O<;yH&d|O|F3=BD@4hF=vb>Dtrl7>nO${sqIWFd>#S>`A<{{TX!YpqU4(H#|w z{-SYJZ5|&+zGf90t(bVz?lx9=f59JBV*!Kcsd6mka9UTneP#|dnf5JzQoUXU-d8(( z#!y(MA=p-*QI$2i?EL-B6Bo=yxVTE!scdG(7ZA8yqb*|L#(0BfGc-Thm67IZ&oLG* zh9a%Ia}@bSygSsbQ(Ku+SiQ4Gf6TDa29l05aiis#e^kZ79GZ1GCDO|hno&6y#w~g% zmCb40n0n+sEsaq1Y5d#vz9k`$CRr)r!k(z}0#_Y---F}sRZc_L@+n1|?kHXe3Ze14mI%1xS(!wb8*T0xz$+z}ntb?yO;W3v zP%U)zDOO@rrVmgC&GjwWD&-Y_KOP`Vcf!YZ7>K*b{mZebi(j;^EV*OM4X!dp1x4LQ z(w{quG)?nRl;cT$S|<0J%>Bc=Ssc27D=uz7an`D0zvS6e$F>D48~vrMQV$~JmQBfd zcKU%_z;_Bcb<8W`8G>7wR)$ykDf)s;Vxa3h!~K|i(Ebw9gd1~IY5;I`nqYoI513%H zn>aqFgCjhB!PL8;m3- z{6uOP1s&H!eC1WK%RI%zLuGuW*UY7rHdqUW3Z2gNoqz znQ6RY55i`np5=a+f{DL=rUsraBgI+ymi+VL=2IN9%kSmNtC&se9y=Ib3@9SkN}tSh74{x9H+7m!=q?*9Ovq691x z&^Wtx>zFqcXvF*qh5Lg`=3k$PU1i|&4Yr7J0K$FoD+ItRg+G|LFrejKR2G>R?7lt0 zh1Cs{k>J4H-9-sKhGKR{eQ)%A$%=wg-M-MJx4Ja=Qu<6x)|x z30PB$_+aIFtk;j2cIR8ygZYHz2;HN56s<+$=)#3wo9@fh%Www|BTSjRiXmKP*Q706Dg*a?%Bx<{*~Agm;}xHdyD`2Hf3c09@#O+%=s zBrAjdEKSl;{^DPEiaNEk*|IJcXm->7kf0g@V%+&W$}FM98N1>Np+lUwi8pj!kj%w0 zDq}38t@`^!0J1eLscGcX2JMUISK=-J*Bkx9kx?i?u{l{Eh6adv%0`o+p~C%OJC z3XX-RhEl1$m_KkqX{z-4hTySNFD<4~5_ znwB5#V`dc&%P5rT4J53yRIx$9YImoY5>VM2N?SiOo_E9wg1V%iLx#zNZ$T`niC5p;Pgn1QtOe$Y@>D=pq95Lj3C87xfKi^&LFiQm?t413bV& zwkS#tJCxkfi3XPn&z@pMw|vJbiyvEKtZ!@uUhWp`)WMc3Mdn^D=Mc`d)V%I1t#by0 z%k>I;zZvdlr+@Zw&oE>29=j+dUDjoG(kwVTdVmUx!CM^X7Ybmjy2_qc_Tmy7qgHA4 zS@>hjrk<(#?D;-p#yM4Y@p<)eg6SXjSc=;1s_(`>R{16IUUh(b$3C;$G>y0Fy>%_x zH5_#Nj(liAlt6OI_U0V!$5FpB>CT?u{{U!*^9UN0Wch(dwtiz)cR?}10)@ilz`yD@ zNQ(z6LdNr*N*FPMfk5Ce9YUF}y`NBrJHF%4J6ZUZ!Ik_=*g0Qs2Z&dUUfm|DXebSD zL^^gy7nft2;Ejfc{O_M|UagDcuguWH4l`H$$4crC8&{*20u9Ux@Er#PKvJs3@*>b^ z+eb4Iv=F|YpxEN`N9tTFvrxtaTDSzYU@#>?;M%@cSyBTvo83biyQf{kCC|(JMR^5) zAZ}S1=|p0OY{~WFAqr!j{6|!4eWb|E zc%CS&Z1V?2Gm?uEBPO$kdIq2NU;*1qS>15fptD}%-M01IPS?&Q4mVr6nMe34bO$R~ zhs=sAv60ksF_;1d*D0{qX6Bf)y0H6zHhIM#;Vvi*XD`IG7V8DC!JpJF#f<1Iyj zMP+(!GbwQjSLdjeV8|XjXEqdAlzGmHZf^nkFc_GFPxYF7OKS&fuSSV$hA0+VU(G;4 zpMikRiZ1miokS2Xii85|eGt`C;QrW90=9h{Z#=2<6T87 zSBN&GO1f$WnrNsLwXM0-cCSd3LAo!v0Y)~!2eH8XpAlX0iw|A&dBdVNcIds=ZPq>4 zgak6)ou7<-Mk7~dzBSk2s8~0iT6}i?p;WNCxQmQ7pqdnd_hnvU8G+40lR{*yyv0^+ zFYhp~e^aio%Qug5z=Kh^EdtJlGF;2T0a08~klLDS%tR@XU$$TZhH~ig`F|4U z1vQJ~&$)(R6_u%V+%;OuExZG$logHbP={9$8~KH7 z(}TbE=0FfTQzySHTh{X_$XZ=g%?vcl$iypL#)NS35rYg}R$v0_20u~w>%;{DjOQ@- zH{89qc4`C^?+^?XKzDo+{btNJvgIwi*?wi@I6pAj>h9*rFJkNn>u}@fGSnKH@w`Nc zT`av_{#%&g2rF3&mDIT%J+c*14C-n$6BoUh6clH9z7-)2qGws z5ipOaI?NvsV7*0{d$^W-o!lYJ&aQi$4>p$TQ@V|>ZpA2rV0$(`$eKNou}wT#+_Wp5 zA~o-r@7c(+@XOUxYlq{RQCUaC8Ns3{E<*_wizD7;w>Q@*Vkg4$FPn4YnXqfa%sK-% z_pI&TZ0ah(zhSZd9!M33t??Vf8qw4$r!=D*@hGsc5&TNDJEX4?nUrhzsFlfAvJ3Fd zum-lSP%4+p2vwx|8(qot!y03E?bnyQGF+0@?yP2)&Sr? zWyG*i!4KHNa#ug$m=|q#{{X1g5!ZvF)elvzlY8g+n3n3SH{n6|IPih|As1@=nt}_w z(-c!?$d&~`@-dh>ZruY$MUoWnh zWNHOoaLVQDeq!yvKk`$m&l19~=3pJ&zz!|HNBaoUE0G6?UmX~wVd&8HXUxDnd=JXK zE-=?tT?6^2@6t5!%Z#t4^X0Zy!>eEId1VIPEBc1_oY$Z3JFvFUcZXg#I#$vz3AUQom_MjPBx(2u|1%i$st&Q=A^n*rP z<6|#B+QcX^g5h_hy79!s6@yxf&gdC%KxPG--;@1D-`oR(hzP5ra3xXQfc(ZV5zD$7 z&v437&pUj-iB)XXQIz7n%WKZ72N^IL9$~+l^KnZ=`)-d! z$=1(^RfT_ zZ!}Z&5Gew_iDKJDr^EY|h_(sfamO*6t$}rQsDLIxjPb?#xk^?H_ild@=(b94%mt}m z$qzBc^8r0$q{Hz1{{Znp;}YDM?O~$rgK2)}mI{F9^#P~}IuL&}E?pbfpZNhpo~4?) zK;wEDLk5~hiiJ6TPt>i}4$iYGjcl*|h=9@VZLYgc1jzHvNV!u*^61@>J>a-ewz+lqwx;~QJ^2BNa(+tm@an0#k%nWucr(!Of?KqA3~1*TDMGh3I1_Z)%Jox7K?HaP_Z{Ax2TsaYI{(w?dtf;LqR@zs@&6g=3l@B zFcm9G`JbNyD1bvfukka>ekV(Ybk%UBy!xoX^F&IBb_-KA)XJsgRxbYl%&f|9I}O$F zMMeUG+XjUSA$bC|Z(f_+Y@gH80GaFVFe@xOC5@;|q)+)gcJP-P6# zaB9!Gi*5^fZ}Tt6*#0GI+kT+fTGL+LLG+9lq37p`n^Eym1tGtQLtex6FfIqwy5*zq zK4wHm!~@Xrzqqa|szh~Cn^pBUWK!_Kd9>%^W^H^x>}#lspGGqgS>ySX(9-qdE0DXE z3tKMUJ|ZqXJU*ht@~8Ts*aXSR)WuqauA5=J6Vyef{{ZIy05L=|?A75=%S)iaLoCsB zT~t&OB~q@3xC+bC(J02Lg;bgo!e#l5T>FkVygHR+VfxG&h1Kly?BW$UsqdlTGD|9P ziORk>l_(DpCZD(~MjW3o@OKN~hDEHda@X?%_D2v<<28p7%D`RLAd%&MOX2x|RPf!# z%4leHm)s=iYFlB-sFe}CNKot(T_>XE;;`J~n8^(VLDOapqJqT9F(j zWKI}?Ml`F&O!%hsnp?jjSx7DOsgJYN1Y)sX7~wGZZYZ*iU;afC$N-+dnOrhjPv!!S zDb0o;vCnyhG~G49DU%Hm3A97}N($@Ke-Zxx7ZWgkRia@5THuA50T$@wc%K%qK*Wi_z7IuF`t|PQxq_M^YR>yi0;6aAyJ>QmjiGV&# z>ETBWl}>lr_LmKS1C7%PN(;C+a(ohomZGd`rk1aNxSQSNhdlB|zig1jbBJxC>ahb0 zb9kL${^N@LRBYS+^8;gzYq{i#+Ep<$IKV2F4f!w%p`6te`|a*6(dr7iMbh^uD9d8J z6y{JA18)UH5Z&e5ejyO@A+qFf9$$!$wS@i0{seRKP^_G-rTxQUwH1K9{{H~AjqoGv z#9&>2)lCg3Q8NMc&=c!gSQ?v5|)aSQ-|uP_ueUAaz(`AV)`f2vV#Ka5OMS~w5pG*JWJ5iltE zbpauAz9CJeR~=swH01te=nbn9;D&LSfH|rn<t`hYg56t>*FSD2`*8Zb-Q zNsTZOYB3b6`-}vg-@&$W)#m}|W%+`h02TBgW`fsLW}^XpAGN7hpyjpgU|vTvdDVUf zE2x}bNCHfjJ(X&2hF6N!DC-ek8C*e-6nznWWG_Ov1NO5L+ZkH-7cMo-gD!Yu9wGsr zH4`+%a3B1R_bfWcQ!^|l{g;OID$k0!d!`y*25$x~p|fr}fxFb>%nff$F6#<_v133T z%OvO~@l?QrEt<0XgJ^FYl2eox!sENHWqh#31#5u&jl46UHfs=tuz3_(!bT3(*t=Z$ zKM|t3KC{qp{(VNZ1skoma9k` zRbRMEX}G*J#X`A^HZ}=e60IuHqmPdMrVy|d7z*lfsZKE~KtB2q^RJIih@cVyJ<5+5N<$B~$E*s>-iej>%eeM3?5l#~nDVYN;7;J~LeqcQP1Qz92aH}Xf zAPzG5h0Jj&&m&&UkQ7vNQKO^S&U*QSq$Pr%xpwTycH&qtQfTH?Fa323g9TB$;IozM zTKR;-s`Cn~dU5Vp+53Tvajeyn+u{%ry1X~kviFt0F@{%L&-$V*RloR%pcSLsXtm=v z&-V~DjlZdz`{TKiV7cyHZuv0>HO}6Ng51fVX$%5sJCGs;WQaL3xoje7cr46y(IAVs6|Q{KEsm zc^-|imd}OA2h1?)*Y_QA z!Y3xPMM3I{Bb|N$A85)(tY9EE^#PZ-st5{Rmu118{6S5@>aHxD=W`)LtUz2imIr>| z+EX*LP;Rls#R{p%9%HsQ5pkzN)LP(J164PTjRWe4Laxow$UJz2p}|~ZoBN7vsLp}w z$C#ssE*t{}kIW;bI0p;2>Q&-}(LvL%F(RA14X?-XpCnEd8ZtQF7!Oj$-*S>KMc}hh zY>WPn{esuj85Xw^*PTOJ_?{S@<{SHnvSBHKv{Y!Fk29Q_ik#A>LjM3sgLDBv90P$J z6wO2kU0!Yio#RA8##S|D%voXxQI*|5QgV(v1@iF)O3-z~78<}eUSbm0Cyc}&+l~2+ zU}tqLLX4g95mO!JCwA5=65QIW1l4bjr7p*M{{VP_6p7~HMWyu%m$j$Ah*NlRRgK~q zM~rXoYr7TsLTZOR$^yyp6f|q8RAl?jP0Bta-3=2uhylA?LOG%ft4+g)C^E$~ zBm+)GeatAbj~XB5T$7Ale;>@!2DvV=)M~SFUA*6) z;^k9GKk$}uUGoGrCQap^LY?{{ZB=d)J9d8mr#o(yNcTfK!s7Q3CfZ z3oy_AM?@6o#NApve=^0DHJmU33>;rR;s>`4#?9YGh$uN|taIu(H&htzRY1kucGWnp z<*ZVvs-u{Wl93F2`-OqFkw2~;$!VZobFQ}x4QHCu289A|eE2pW7 zXD;JF+v00BJD>Q3ENUjl+$M+zY#%HOO1{5mY+L~8Ko-9!{8WF~RnB-h{6mQsMOo+} z0u9z*p@M5hPhV@m>dDMu(a8tKpSnb%5kn8EF^`naG5#f-M+7SyGhc>Ydg@g&{KZNY zhAN9_4W$FfgeV3n_}hgKqwX0br7h@6JOD(pI+oPJR^mXq%o=wa^@yg8)akcGRXjh$ zw9POYtrM@un3tRX0O1X15qsj)gbS>xS%cf6%~w8%H4_?^26zJXj6rF@rQkbXQ(OoE z4@eJ+2LrTPj%HVu4hR4#hI$vtd-mg@gECkgIA;0xpF~+{$$GwT74PYb4%FuF=I#5; z7VpM#>(Tj`003&Rf4|oWW)z*A@hHVrSJNxE%YLG+TRFn+M!X@V8Q> z=TS4`b*#7|M#?ms{{WCc0`r2^9}(#l!0R_}Qo*%t?mKA7Lzvd*mA>n8)Zh=o7XIa1 znD=82>ewqjA}Z_h_5L99t`r}{b%5}ElD1t4o??yN;^Gou)A8I4i;KbA1);L{;_+}B z{{VX_U#UkSvCTZ*;d{3%-rq3BsJBi8WvX-KU6S^GZXP$x2+_v`U;UOs)^Qv-Ke=}A zP18N2r5g}mm7l0mKzQK5;06uBG)ldmW(-t&q6WoF;^T*f=02voGlc8kP${<#K-=*c z06wNHE_P^`V=q|v#HkkzwWy2D-*GWReqcbwQ9`xf2T&ytI0^|$rf=hz;3p`TanieP zqNW^Y>MF2ruhgIgSQxl6{LGpke;s&^R|)>8>EYe#W?P>m#^^nSz;f5lVl6EeG$bv% zReg==>O6!U*BrwK3*@*kKPWfNkHpv?iB%d*tiK&au(re@oafKtQ?eXhSI|d~o^5rc zTGQK)L_8Mg%p_CB{`4V(itS$1q7cIb+mHKnML|uLkeF2Ni->U~emg>rD7b(fIw0cR zPZbkCsFz&;OEwEfI)IBLti#ekbL)su2h)GWd(>gp2m#G%^+eGD5qK>9pF&xiI;n08 zX>B{MqBwE-h(BcM)D;{|R`7Euc(ZY8^%bM3eCNb1!OSbJ z;l#nC^8{!Py#Q!D*$R;8G_FH{?VHQWvz_~Y$em4S@1&WVSrNxFegsSglFykpv1kZk6DGv5VpH8tJv@l-kX5R%39qV>tU*Hctwzy}&C2 zPOkcuCC8Uc8}Qy95@ z@=Kj>4PWsn8VxU+V)5i$q*s2=AE{hj@9JGmG`Yot^EMXW51q#V(_Dx8<^yeq63?Go zjt+{v=zy`hgQxX{9$ujI4@p6teqhw2Pcfx0+J;9kU9got)(TslaF9@QbYWco0NHJK zr+Xk;y65pMxYIiQF;vJrgyeiWWk5>vj~(=Z!RpN0Tls=VNEb!ke86^rfb^9UK+3qT z<%^$Id_{g9*g3Dla$*PqdE%QsFM|*(X?0#_)04z;YKy-TjSaPO@EQ=$`?#bpSLwM( zPzvk09&@*})eE2bKO+6A$0VWBCU4QCEMj#K8v4;I|$Ku|R+(R4|j`p zL~LHv#kfax1FII>{I&hYRt*_fMRu_>8C0MfS_h(79~flaTC27$qGDW#_MQ$68Y96D z2WQc?bX;#w!-8Rjs_Sj{5OO?}`G1H|IV;|6+4UN$lsf{H^7_7$)Otox4>F@q-CxI$1tTyl) zjkaFBO%-9lHQYgY9SC&vGXsDS1r?;=&murjMxodUf+@Y&h zpayg90fqQGjMY2uaR{+h`jin73K7*ie8(HO2Z49Y3kWKj-GsjS{BuZ4^tAcTL!8M4$D_lKpAxa2I)o4(cJUOg<-wLurOw~C<*VgwD4Jo z)-$}oI1?efM6Mtdsr^iKS$K7r@wXX-^?k@WehxTK&f%KjyIB}%;k9&9U}jIq`3#^XdJ zYgeaQW$zr$&|8Z2?~a=xaRd;uj+3bN3u5#Sa~pA0UmV4{*>_vaTrJ=m&MqSMiX|Tr zy7If~Ic%`Hyly5O?SJ+tGNW0EXwtI83e!9tL2t-kaTLSj60(QSCz2G0{1$kPwU)B* zMevEm_+n{f^gf6nmklry?p$7>q^qK>Z=i_KyZivb6JW+W`Xy1K?XSNPMlE-{Knk>C@I7#kc_A91ts?ulX8KwoTdbEUYvui4&iQIh}!~S3__;OcYx|Y69Tjh)QTd z6YS1NiAA9-4}{;M6vilG;WU=OqUUUIreBP3sB`xjT^nIS=M4*t4ugFMpoM1`K$wdVP^%YDHVfMx-)Mji`& z_2ZX_(W(P}3O8Ax@AAt==uukEXu)h7k%3%wufRiwysNv{chHy9Fy8aGeHkc~uO6dMNgJ}u?{Sh)>i)A9yj;d|AUNkk#EN`=G<}pCY z)$=K9iz?-;DFdosafr?&Bnez#!r!Vyl-7htcv0fo&7*78H zn1vSJYp-(`Ee+PU;w6!J$No$T>rQQyg3CCdirH5fn+mSK_5oKX`HHPt0K19>zXi;S zc+H+;mz(p%&)@q}jq|Jc#6g>z8^O<$nUo3H{{X&WW|X?SyS_O2iMHM~c9$@m2dD^d zENI;W9pO=ykZ#KH%uG68s$cUJC(fnf3Kg$*mrkz_Bq&N@4o5W@!$(`sWV&09>RJsh zD);IxnQHzFYA{mO{30D!;TaF+TE*r605I0@IP}UDzQ62kJzrwB7&(Jk{{X@sL`R+V znTT0_bNC|VCr&8wo61H}rrQ{Z9E3!qMv;lRYr(dc;udSu-K*4TheGvvT)|%qH6?uYUYP^pol6MR%TnOR<=|asa`#yE70bS7i~5C?F^$}| z?SG6yH9+u$=x3f^66p86!uHe_eGV#K6`tBhBV6|#sjZl^eNle1<_l(3fq*M!8Uw}- zTZNWrZ-Vt-!?%J|2Q4Y)wi(OWe$g7YtorBr;NX}y79Kap_YOh3`saLpJL(PE=WBj9 zzt88W06S&R^RM>nP}H>oETJn@E5FeJQW*noMQp-fJVlS4N=vBbmvqZh>~rc`t7K7b z*;h^e7yucL&#)J`AqrhpQ;T?mFk5ISw@329D$RUHf!AHZ6u=Y%YG1X8>9LU&w-oqh zR9>|Jdy0jAvv7-5Rs;HqU0Eq&heo-WEKTB5l02PD`6;3i*qp`)2GH%--w>D=tavbb zjbLNOo9Yl&i(i=F01oAvES+CboZ}REf2m^I=QRzSx8u zJ@p&S@9W%ewN~B1ShZL^Lg9J;0C59W`?6T0j2}Ce7fKr335+Wu&}LBOnx`;L?6<_z zyw}_of8OHv7Qszf{%VxIxnO);i?}h+C;`Zc!w8kwTMVr2g2T0;?ovAtN_-i8gc(;3 z{RaHD!RrsyP}?%+Kkrk4&!77R{{H|mu%H?izxF&iWUTr&vHD&Dv(>=H)*!Ntm7LZe zisoS`uf3De#M<|ba74*%uDtq`Znnx>J#q6fie25WZ}k^LX}3Y`j>2{5{{V2VGe(2a z6?=me`Va0U3K~{d&mM@*=M+W4L34=E3YqJ09jmyh{dtM26#nJooJF(G6E%V%puZ4- zS>Lp*0*(S*+va0ZzDKm_`C<$qLmmma1#iSeLb&yCSiV(^ zKoz&xZeW6|O-v5*HC@AlWftSDW)$qLZdL|dJYB>_DQ4R8V}prhDb6g;K4C_%ywn1b z!=1rm>`Kz-e01}F2rg-|5Vy@Enf5cJuUmj!nj`bWFr^IMGjy4CD z3LsG_IB$%~8w5dVqmF{1a_{Au;_LM(3SsB{ za7e3Fn&_8Qt`9QdbYX3;TZYWEaGT?V9BHm$r}aRLakvK3PF{Pk17z zRUG0yp1GE-<0V$(qYS;vMw=A?%U)%S7u*7iYCa&YB zDBYAUnI*YsFvYnX0UrZt@DLjZ&fnkACcyobJU8yL`i{NYtuEfzmS%;#!0n0`?u%Z( zv=i04Z|9GQ;6(Mgu!}cf_WuA-nm}ljJ|IY=o-c=fiFfw^_-L37yuJMO0i!1<;o$B7 zZk`QH43KD^%>MwY417Zjcks*Kqss==nux99YVcN9h^^=H`io_{-MqnRUclV0TZRPO zcH3{}#=AoS4@c@^KW>Q^jcfA4I#0mhWC0(3__uC6< zfy-NW{Si1L-lVvA?TrR71_8iE*wy#UdTjb)Syz&#?<=QFP>w+P-?&~8tn;@L^jckg z%VxIm*X9vuKg+_pI{TQ|Xq8^#e3yF4d};x>op#SPCIWdyYY} z>sp4Z2Xz5P^7Z7ZepKQMkT#| zr4FVz?=!4$zlVsiE#L>5l(khCu47(5tab4Q*J%Dv;yn$auhataX~q8l*r+-#T(L@4 zPwH6EZn=~@Ena`9Wk*pwAr+3SA(@xF%+;Q;3e%LM_?D%fR}@?;{%RaNIOUlYO0UF2 zz_ali7W(sXEGqSydF}|v;}wSxYUg@h`yp8dm*)M;6r`AP@yA;uw$K)y5it2Vfpis> zlv-;UaRjb~Xy|!wa>qsVW0R`p79VtH&t-QqrG91D^XL7T0i?{snA>K!S(#8k2M%WE z23N#wa9>H&{?i(<2d%}UmF7~b@bM{a{{Xx~tW&ZnMU>I4!oXvPHwP7qtGde-PR(cd z^)i=?9F-o1Yd5}NYz2$D*Tf1-3LWZDNC;4Fl4v*>lse=T^O&30s5xU2&LP%E=Klb3 zh=YfJ`3{yk+dW$jM>a8@jK31V4a;{PY|lkRVAWM)*iVwC_K->L#fN!ZM(h3~4YiMm z=O8fb{zd?&vF`YAK*6^8U~k%AmqaeDJy-4KE|yl?0quYde3kf%u&e?40F?!ma9`W6 z+ENUP{np$v+x5xmwg6(Ru2=dzMM&2>iaD0&oJ z5!TnLGqx)C$7;Zq*PV{jS*GiJ*64*%?VRA{s`&g7MGjc)rW;jg*Yz!nShjA{JpQE& zR2dw&11jz=1Pv%5;Wj^T5GsMkPcd8n0FeofMq-R&-xKpB9#~#DhM}u3p?|l63LsbJ z&Gx>1@d~4}J)Is2L)4E5>E*J*&h+2L+>w_P|RH*AJuI%9(0`k5NFUk#H?xU0u}7Z~>>+sJBY9 zs;ie}0S|QVVl7&~}s8QSHP5hl=$SwR3$`Rpj4syU6SP^DZlcnuM}E%h6YU^9qT= zkK%veOii7J9QHBBD6#I1r6NPH4*X6>r(2Gq%Ar1Hy&b>%6%)(<0FZJ0oXlg@q{;op zvtsHd5~-?hgNSTsBb)1StTHt&@sQgt5m+ZPn!~pLqUPfHD-AH@2O4#*-Ov}o)$w}d}jZoo)`YnfcORKuWujURw&3~AD zs<00{L}06Hm?p$p-;QNP?R#6qvvgNS?iMDW+zuAk5ugEoiiMQq@CTV<=EY|yrfSS`U^}5OQI3N;MbNwNK)4@I}4zhtl1AMMZ6Z zGgn+rS(aFD26~Nv^~b@?7rb!i^d92zfVY020;hwO@yEUhm_Nr8 zUJi*uREac<{zDL6omqc89-+`w&krJSLoHD9Gk7`qi&V^F%RaRwy0T3PYDk@a8koTvw8{jueTiAB-?sq%`=e&Ii;(JJtJQuPD3cF=);V z*y=i5jrxQGAiO*}l~7H0s1^koZf1&Tn+o6NA~~i?>r*2w97P8!a@;@mQmxKvSU2m4 zt+Tzd&@X2b23>RUxhA zf+*Gb{y3IY1q04(c$oz^c`+`umzL>`*m?oZt}qdB-Ee9;&g?mYf~@sevH&W0m!bJ3 zH-=q>!~2w$-Cb@P?7{wG5`4w)N8!^b)N2|40BT$oF_Sp%8F$ToCjiZ*bcE1NQo0g`B1T0LD=`v&QZqQE6c*PQ*-!!wJ<|t8rsO!N)Vwg~d6l)&!_C<*{<96wO1ygW~b;O1qGJ3#hv5h~F6F(8#K*57Y1H7LNUA24#bbYTAg9L)Xsa`7>*eY~EndLe?NaHxOSU_h%WexcFjgo$00 z=AF5SE}iau9Lwro8FLXr0a}?c&kz*={6Yxw1U2Ahf^pfqn&1q4#@4zxA{k2NtN#Fw z9&UWiSHlJ=$bKURIA>YVekMd|1pX_}!u1y+tqKCD-tq5)n_!l3UF|pTKHa{eB1@ue z{-PbH>lY!**l0m6THOd|X56yOhSomL2ZL)+@13_~YP=jc=FDQ6w((85=iA&6-EcgR z+V5bKhF5LNK~<^q60aZp4-r5!f2gz(vX0N{TlP@kJ6;2r3CGT>{+&c@8+f?uF9!6f z3V~-gqkX~bDygS@W@`#_Ls0^S4%Ytw65#I|h|XRVB^s*m*D==@A!n>aEDC9{{32UH zb-WC&Mh-cHO{TBEsZR$d3&cza2EF-($<8k(y+9jVhVXclX4(g4a~j?;TI=pxj!Ux` zAephr&yV61A+BpOs;^}g{6e6vsEW&Z84Y4wC|7Q?0D#TPk(8?$af~;AhzU)2gQBGRiMO!1;#$gu5%`r6 zMvCWw<|gvr9K~oq8xjw+y`Q)!6;h1qBC2Ijc|YVKyZ(R3nv@%-l|07TYxA6Vi;!e) z_VW}n_51slSeRCY_jNZNnmhL6hFn__IO?TL#BrMDVYI)vsA`J)nJ@_FrgE@YPsK1; ztUn&04Tm|;6#!yt(Bu3?1YwQrjUf0me{!yX<^95=n^fY=Y7Px#;aq%9TIYWJMUWal zUVTE~EK16D4@{s-1Od0n>zLBrwsTo!gIrUFj-$qy6JGRuOw9$$!2Cz4RKZSNSBQJH z7a+oUhvLFP$MqNYF4Qggh}c}3IdaA8uvpI1q3SA7LrBa2048K0QkYK!8%Pw`;}Crv zf5;q*;d_a)E=f%I$J`x|)qk!za!cKiwUy7}V0cmL1B)%joaz7x5DqODM(6?8HvzC; z9iC!sMN+NHDly*&%vqP=ul@+&F4IHX9dp=M#OLM-TAJweaS>mmkB;E~0L*?IL^Ueq zUB|U|BQK;JxKl4L`hte&58F1td8qq&_>|2W1_QuFo5qyC^lgE0hHZZCo(L*2xls1Z zeSQskaqbFbxo&gqT@eR0$JhIQCHNMu{{YySVR@H$_=5LYl!^Bq6}<30MCWgqQD{4h zRawkQ=HHkxzr-#4!uevL#@CPL56_5=ZaleZNp9zhk%O@*RR+ldjowrouDbh%76z*h zHLPQ;5Tcx{Yp80h!N|c>oda)gFNpY5+AUJWxFeteI{B)CFfIhkPoqJ;c<;Vtmt2Xu zyo3mZJKKGhT{OiDtw2KszT&TxHG`kr9pH*<%mM>0<*4LnD76`_vg<-oz;{ZZGUr7Ks*!LO3Ow! zC{$+Evl+}(Y4R1hRcXj7@j6w!&H|qqvT%mlxt-bZ60`0F0+lfZ)6q-&iD*_{b-7An zzZk=L2e#w9V+m88$_kLHnN@$>YS-DlCIHt`&_eCJ%_v$6; zu;knK8v@HQo;#L+sdY(8X`_s0S|x8SD_`!Vs$VVN{f^~Vlcw^2GU_mOW+g^5c(@|d z#9F~$a8jieSL)v+K~%S!@6AdU)Dyq;#6WT=HNK|6ms&fGl-YrEP}&D>o?r?!-nsGa zXto%~7=^TjslkjzT4Ow#_=vSf97E#9av~;vL?;5>rM1%bX^7_-c|IZvtyjfaG#w_Y z>JeI)IzmuraS8(P4*W_%2JXHr3-oV$%Dm*(P3(7lEUMb>JvZdHYv#$O-p!70nWU%0GmaBnZ^kAqxK zb0sPW8ypv+4Z4ym#9kbtQSFVjfy)QGV0e##{msLU4F|-q8F}dT@j7SF(f8b_91RQy zqFW(BwSn_C!N#=obMqN3eDHYc6toW7zv?q0qh1e5VpcHiSyXGh?D=&os_$pE*+_U7 zk7NE|+zyKi{biT@(qiWqlLxW-A()K-Jph6NAuKw*aCu_^U3BNl<%LH%j9;PruxbvV z%k6n$EFRz2a4HnH46_c7Zd*V@$1%FSOp&(mZgBw6YyQB|y@WlHfA13(`<6v+H9f)i zGHY0=O2bINI5Swsag|3RKKJnoErCipjl(L6y$0AhHg5W?TX3cyo`#%MAB?%EgifVy z?I(KmgAolY6cvncdWXiBv$w~B8CP6Htyu4b+a(vLKh6k7T^zR+2rP69=6#0}mzeaA z6}fkC%|Y8v0ynW~qa?@x&J3T3j%uCsc!)DY_XB}g-5G=hZL3ZWB8ACyHrZp2pcrz< zb^ickMO9GN-w?_W(0R<^4Y|@@fVJ)AhQ;w*`STrDnl8D9#6(>jLYyI8pQ%s>US%+e z-!=6RXuVVg3S{SqwuV!Bre$QSGGK)?FFfvK$`0?i(Yj2U`}U)}U6BXct0f|hQ-O$4MN(^we=@_cH-*>!LV#-O*Tf5xYW(6k zQk?g$&t1aBa?kp!5p1A)@bfO$uC1H#0{C&9-Afv!&N!#O;X{C4w@+u7Vyh}}S0Vmp zCD(w(XMb>3ui!Xc%^*;Isx4h6`Nnj>TMa3Q@5IAfR3Z-|w-s+F)_YtM+Hq+TXk zu0ObiEZ_hR?QxO<7_qvyu%~`t0TQ~TPeg20Wr&>lj(F38j)_AwCEvM{fz9h)qh%FZ zmV>TljJL-X?pqXb!5d@q29yvBOYRw=d~XNT>7I>z5Ri+aesOdj-|7&+c$6aPXKx3% z^H)Kq_bP9LK%n$WmNxhgY}oYlf4O>U1`P+IBNMb`^yV8fF5X|y9}tXDpe=m=0NGD4 z`7r&SiGv1!syZXtE2Tb%^9>l6ZQrr{Oh_9By^ry@X(-~?yI$PFAXV6Pr^p?? zSg@_Vtii*fr?;2!8C6~{f)^=i+o*wX`<6!syvvULENN_ch4hUy899Z%qgTZHc>e(B ziq!U&eM{!Kfn!OUjqzw)4Ij*TMV#4K7p{H5LsvFoYjXbp1zPgf)#AF;2w5lS@{LaU zc|fZ+6yGAUx;Q@J2Rz)U?f(FJjHR?7{{V4DMoI@Z_=rhQ;D5P^(0>;fd`ukIa<4vq z^8z1aL~&L1E26T&jpiOzTLWJU>D;0xE!Dp5+bXKy!DjE6@mUx%Gm%AfmJkjF(XXFU zh`fiol%WMXw(6kB(3r$1(PG|GluH8~TQLHLR4+Yyif)OUws-0WaZXXp8bVUEHNO!v6r?W!ZRsVv>Si zij_7K1HDW*SG7h9C=dd!3`Yw{y2N#MWvk!BSH)ndWGj1)K$mU(;v`+X_+#GkZy22q zw@1Xprq{zQZvOx<6s*TNs5w~1VB=>a+ObugNoa9&$Ce?&&oBEB*kXmQ1&S?MF<+nR z4Lw_}#~OI~%%B}5-#>jvg7a^j7Jj8cO3%WgETJyfoX!gj7z)V(D!;EXpyk7tpE99} z(eLU5YpkpFnQ<_W$(n%dTX5-eklbXfsl=lO4-pV;o5XiYFX;aOq67ezXOOPttmwT^ zOs`CDPX0*vE@{-ZSt-Z+Q4mb6mj3`y0u2EHFN4U2ggV`ux!J7U!bkEq^2 z-CrER*=;{upD;9qa2nlCuX82@95L7T0kWbfbX(N0otdav7mbe1Qyex_S->&n)LIdA za-0{>i1IG3_eSlmhulCUQ@jxJ@c~s?01Dm?-rxv|0fx`*D~1Rlr2~U4#*H6b%oGG% zuwc5p-I@&%)-rf;D+6csh>N41EAuQN`=}(!G1sU7g=jLzvHYJe5x5VBg9LaQ3#1t)+Jbk#JIFntkcu|!b0BvG0<$+jR2Zue&6al2Ful6OaVO*CC2D{Jv+@k9g*#7_#y2X>ZRYzN| zxKg2^YjuNjkkd78AaXu#v$$s*wC9;YLhtTai?;IaBXw?%iBv^lqiGQ>UT|~1V&MZW zd3mVR3R=d0+JG<;>YpavOd};`$1#UGC41=4L>%jr-Ef#Gz#L(eMa@Lz5SlWmkQHF9ss+tWu-S!Ba^&Fs?|;S zV%XaA=bZV42st<%{lujiQqDf_m|zRbUVX;WuEpC`eM zuN<=w3gWQ)_W&9lSY9)C*N7Iiq*T^wxn39w2wmFf#cyO`k_8H?F}C)1D@9>yR(MmK zqV)kzMxIYM#4>BB*3E>f%>{4!gIPbShOAs?cTrF{N0ap}RJL?R%5=Q$;JW6Zz+||r ztf0TKe7cMxcsBcBEdZ-AntD0;bsE5Qn|-qH0lg5S=)89Rqw+{L`+0&?x<8n1Yv#w> z%&NQ%7!N@(q#fY@0Ix(zORl{9G(t0eLa@zu*w?YIh(KU;N*^E0qXgGCrr*>;ic*x{ zx!~|i%AhzN$M}wLzGa_>9``TBWjXe*9$;1cl=0i~F{=n4AD8{l#JjI8*XScGp=SR8 zM{z8Nxsxx6l)52GD_O^xu?I2OcX112wiXzQ3`^xGsLf#UEWC%h z!*NnrxAE@f;zWyNEYn6T;vj(Lz9AH@i}84y4wZTgEW#UVw-`8%9X=Y4>1Z6A`X2a2 zNh$(^fR+Fp7|oEIk7Z(w%U}`l{{SVMEmKyiMKEg7qZsNIB1;$xtk>oWxVI;G{{WK} z1!x)9xK=rl#xcw&9J;-0pK%7|ryYDiD9%c)#bi-qe5X*t*9uo2reuYx{@x?GP2&Fm zs3FOEz5ZowqUfx9f{_$)a2IPXN||C#B4@SMS%EM+12TqOjx|-Ah6;#!APtqtfAR?& zOMnbB}1*kcIo`Ij|<(;oi-xaCMHXG@4kL{p#QG|^G) zY0pvRII1!*)LsncYb??;ya%j9gi%zH6yMlsvUrfCybx3n(;l^YI6EOFV zxwFrxW|YXLe^uOFgDyQ*Dgw$@oHbu?J-3ppJwX})#-mu4LWQY=a5UBAdFG$QMZ;uF zIsHYl>@OO>F*1VP*{$(%oU(+=4|7Ee%Z0u>saqf$RvO7!+)Y+tEI4o9G&pB?L5MQWD3Ir4gDL@2RLmCCt@8QAfv;#Ia$9^keN4qC2j1Y1T2 z%b%Ks7J|bUDx>Z;YF`9X^B&^7b1o9Ed;3K>d5tYQ9lnSKc@Hb+iIwG7jH?2QjbSZ& z>Q+PTy}Zk6UXDGyPH5>IJ;J$kV95ZzMtX) zSzWoy?dCiIDQ9%^{{XQv-Q2lfa7uxx$9PG9@&%aACCs7jEVO;faQ4atb4kbh~_5HxvOoPU} z!hp(c)mL#4Hen?{``1$h9$r7}+8ZHYs0iPZBeV^stGGczgT20AFf%Wk#d&z@1{rroRH_+DuQ8&w+rJ7L99C)9sbB*Qvxo+a-8tq}jN!?< z(-1(|3*PzM3pU-knVp&?TP$g16N^vOL;4-@m;eRAH-B?d#8fY(>KvxJ&U4f#-CHLA z08>aKH5e!h>RYWVjc{=qr zFj;HnR2+);V7?=HUbl~~qq^F|?Upfb!MRDTYj=L-#V)a`uU$$@cDy&G=Ms?>vZg{T zvj$JNirF0Miy*9EEB^ptP_!^zZlIYjnRj?k1e8^QCFLLWGK@T-y#8X{Lk9Fd{XiQs zS_<>wfGd5i#HDWmD1Es(jG*ckP`rD2f;k1+Zijbfq0LgGTGm@A;fY`nIS;b2bC9!6 zvM+kaQpL8~HOaRgV$iS}tP$qsbFP`6W+80+7;H?T~% zJ;$7GWdpDylBrCjORg!3ak#WVmGgf@%w?Z zhVJtcxQO%2w5X}^Qj@be-N8g%a?Na@Mxn>&m@vUm7p}7`p~b$g6@jJJ{6g`a=$Y3F zA?o9R(2evH4l2q%Wf$Vz+-?caj}%@m6*b&7-&S|!m$G48e=)n$Rc2ua2HzCz}PQ@(eVUDoJOri3;v-kWFsP` zAH+a?!Y3h&F=JX{s^h|}^A|_fBP<@d`N$ zipz*qusts{Ln9^13tiNq37b}R==gOpBC9T(7599^kYcsucl(A$ zP>N_Q=DbYHH+VvnF0U4$rbqDMf^bVPC9$^Ct{^%KRH2unEusw4IH+f681Vl9=L~V6 zA!9(A59Sm32QfO^D&g$Yab(1!>j(SY2epPqzpwWmw0i#l$Px-R)Fq#TuzE!H9@rI+ zvh?OtM&P}_qtqycs{2@MFR`DO5VHW*AcLU=;18Z(1PI#IR(>3Q;V}{Ru;6;mFPA=u zm=~@Cz;O%~nkf9;KQLJX(be|$`CucUE?)u9y+J#M&A!fkYC6z6dR~VtMPh9H&t3^a z*G@M09)0RpM*OAPyIlUGrO&7IpWM2iy|=aHo#yW1Lo8HRzj0VK7{e6}o9Y8{oBhX4 zN`K5R)cnl3-9qmQj+{q@Xp~p4<`Ao_D_@((+yjwTT6xO1{{X$s4qdue{q^n@5CpuK z0nAE918@7E)Eer**5F5-Yvq@y43@uCN;f@YyXrRWp zm^?R+5pC!lewfPMKwWDT^!J--%p4SRK->_*mIM9!jc~Zxv9IO^+O00LSRe(121AWz zP@BV07ly;krqRR{9oG@Hhtlx^qFW8Xv09dbwEAMyRl>1Rq;gmL>I7BTmR9CPxFM@= zg{T!pueYD(5w=}w0B-8JtuCY73meT}@h_7GF#O^RK?`%Q{WzCZR0>=~Xt%y1P1307 z+$LE-U zy0v47^%e=>j`fRP;tfLBJ`Qtnk%calejJa~0CRw$MXxoyn4&43=_>ccN8c{rNaBX6{ilya41~~Sj_~a;_UF+T9U3$q4kr5=v*;1$@n){4T~dGuMe# z9Mq>58uK?;mH`4}NjFH{F}fG7Qtq?$6JS2SiACAQ zvCO==&7Wjc2GR3Z2SXZy!ln9x)Di8Kas$5Sa9@gy+MsV->J~t2wW}bSO z3IiD8xx`y79Q^+POjVRC19*QV;PvhTzJwNy6IyFXAA zcbi1vAm4(+`+*H|>aBu{ znzEd1cw$vY=)H0C77#BbW%T)&TzMCwa|9w|*RcFQh?y*xWn$QFu>j?4({X4kHFiK_P5R~w zBss#uhY8{!1eDZAIe&yrwA8HEm&~&?O;(cJT)QYV)tmgwKrMWp9ASg!{{Vv;KtL4e zH!B&tAG9FAbZ`-nb+y2IW#a+lVd&;AwD!RZ*01m%%PB{G^Y%Z8iZ?~P0p;RW)3|*{ zFSz9>nR56)CeO<7cpy`$KcW1=ScS#i*DID(76%$H=#&Nn zL85c!SsDKT;cXKz!|EYN%(`cpi)W6ZlqN8%U-o~ftCsEVS09KAhYFA4KRgC-15N_I zWA+YU5W27V{`^EuKkx1e$^rbJ_x-tS92z`-QIWEjOJt;tdNko22S;x@L!K{X_x*FmJ4GH+QP5>#7cazoRlwovZ)1!X#OEGKjqGWnR2ASK5eUs zD>ZA*qbZjVO<**(vkiN9d(Vu;rCGgd6q_pJV8NbVYq;j_tfP~o>{EmqDGpc%3`DL@*%a2ShK&q`a{2cXSVhxTNTw3A3M2n zfO08yp)-OdhR7799(uh(Ko;OL9=eYuT9_#^H`kbhN|RUoO$v*oQ1WujR*a3&C@ju4 z#qhWYKi_b&Fzn*}%6pE6AX-iMCeTwe5%iD#JZ1*`Jq)c7f2r6$1g4XnSG8 z@Ust3<&By=j#dE{*udZ4L>xlV*UxOVN;WDV+u{L*r)k|deqp6A4ZwH&kNJKhwx=)X zlqfff8-0x)Hwdx-P_f_ zW_J2Nu86Na)~hJxrh-ouVAT2j-`_De})kjs5arcgJVZ0b1xh0EW#)B?mT zxPwjo(&vJqsM7xc&r+qSz~x=|Ie}a`&VS?rBsGj5+$>qAySc6ze6P$XGS*j(`-ph1 z$ARL!%nQ3#c8!r#!&#_g)^3eJc?D(t5`v%|EoLjBZSOgMnDs?bqn~qN6uS;%cx-01 zY(@F8Kl=XOgMzHruG8jD~o2Q9GBQ9~K15#g^>A(8+AZQVUhOx3%W zZ2thii0S_TaeHIk0WiMDh-1M(_M~kyY#~;@rIdyM0nl+8wkn+5J+ix*%pi}IpN+;+ z!wYzZ#>5+#vi3Efn9%75Sl5|Kx8L`30&#hK#0Enz>KjhQpDGxZ%G;|rIgb=`#a+q` zx7Rr)4K#djnB4YxzxJitn;r#%{-;9t2cOhzy14582lp$`Iyhf)$Z)|;`s!F>fx(Me zPf*{o-EFoU7pnOn2$~$fxk7x~vc^1w3)145%4xGM|aH&8@y` zJD9O5?K=HHu|Vol8WT@)=N4`H^A{xn*YnlE8nKdtdGK`s(Tgs>#A7fl$-s7&LaG;$ zuWlw9*6d8L$GLt>zegTpOHBa5k$<>^?-{iIViGiI8$5Zqoe4~#2PUOxt6M8nbWxbe z!>YRwFZ`%3RteYz#beYh7NA<8YfjN&n$vnw{`D6sQB2;xcNn6X#%y{20OQZN02)>$ zf|c_&mf(s5UR|$jEC5w-8(3wlcuf4bg#!+;1RZP+kbKINFOoCx;(18pm1mmie{MZY z$)TW7dM8{iT(3@L9YojUOR|{RfcDE9>9L=OAGj#mY_waI$L>7L?G^BNJnmPOg3*BO zO&c_&l>9h9F?GP1bbIsSToPUv{sJm&2PVB8z9j%Dwfv7>Ar2Lh`g%OX0YDr&FZRkD z2AJ#c{YRp=6KMp41$gI)4DeM^$U_p=bPD6EHv`D0z%m90W2N-?T$KN&tEb~+%UQr7dRy_HYjf@IcB z*0ZS4qz5*UbQN6Xl~w#vrtca=XezVGDOJ*%PA&(_el^BmfKoNrUS}RvzcRL7-YNi1 zxZb|t4mWu2CoFYSD!IRKIRx19U+r-O*5`m(cZk$ZFS(h#xVYg5~TyoqJkm^p3NK2P@G7bWX|pHbVYDIDjY6TCXK z(I`hDoL$v0okE3&$#?fIoU{EdGz2Sq&&Om+wyOYo;qw?X z1*pMxk9pAsN?5xX&j+u#WjXRzt#Z~^6iU#li#FT_+W{@9--FDgR8DCDH0HfTBt^6` zG~HJb#fEmY-y^#GOu@&fpdX2B*CW?+ZJ@h~`*aHFn?m85FhXvyF^8f$_ zz~J)s`C*HOkK(-^Vh{t-(dWzMf#fW)$Frl%AT^+1^nq^vElb}=<^xxOVDJ-bqTAcr zeMM+z%Dr~dDN5W2UQqqs?#P|S4!4)m#*v-3GSmn3CZrT+k4C!3khATy0k zgiVLk7}=rujwKmPUZ32nCKisI7}BY~J;gI+t7Twl>y^>`u|=8iRR)4{QXPBEJfIxd z27quvv0Y@YexPZ!uHVR(&}}xY20QRGf#>;wu$A@D#anVN{c)IQj4qAAKwlRx`%tmI zlSW^dIhhhFyqXrS=z3uHMNFyE|?+UqW#MEHaAuG0;2)DA|TJ; zlrj}uDIGIsOaOLXu5lT>^4lm;MVZ7eE<@I4dK|5~#Y_$y+y4L~K`}~%UPBHsaXg#e z_#taJ>iof*6?3^x#Qx$5fI73Tq5-_KJCz<`r;P@)E&G422OSg#R7*6vm-M>JyrqG9 zy-i}*m@CeC;cA>?HRflR{PP$|vrj6RA!BXk{{H~EkZ3O}Veu^^d20EWi~tucHDAnd zSW3gs{y|jZ9zWTNsIT`wF`ujh>tDgxVxH*qz(Y20y?j8ZGIx|C9B-KC{vpw5zV`esZgD`I&Z&pRvqb3!^K5JhJ(W@1xWClvP~6+j>03 znn2~2@OJr>oP4wGxqJck_4fIc65Vfp)^gt?{D;iJS@Y-0uM?3A{U6H{XIWMcEcV0- z4vvozO4t})ub}0IeC@W{a=G=XR*dn<_t6QJYk+Wg@JcsFv7^`b0mVY`nBaq&j^q4e zV6|Q9aE8A={7*eV%%;4?*QteR#N(J@`HiFEC1)bq43@6|=3nr1RiV>e;`o}>vG4c( zX4}BZ=h5Yfr6{Z%5DBZzc6UF=aHbO<$oCPNfx89#LZ=p%tLj}FT4NfGTZ`*e1j6-^ ztow}Hg$gsC{KN{u!rc`dfnxa}@cHf{#?CL+;PQr!O8CT8M);L+YkG7_7s63yr7KGL zD0n=w$Pm2Q;vqEG8O~rd+e+~qKu;}2E#JoB3mDJ)EWm}mYcVj$qMttzY<4g@{J|>5 z&oN51aCkh%%7BS2Q|B#lVa9i z5x4*t@dw|TlxJDI#VFwJT^JFCCErigOL}VP<(EcU9ZRt8pjhK~)CaYi#3{Tn5v{TO z!l+T>%}jk(pN&JYcq7p>1&BW57z=jhtz(T_i@?;Qs?RK7Oegkb0x8mT~9e;g&&5C4R ze^S}Ogv4bms>$)x27ud#kNCmR0bp=-oxlO5hD;HK=D2@wZ#H`v!v6quF+9Y8OJmdc zsZ&6+E0buHWT%56e^QBYx1FzX+w0%MsDuD}cbqNxm6%}7U_AWF3QgS|Sqe7>xeelBse!ib^ZD^In3P|H z;0-4vN-6Ms>Z(HReu-9e&KHSc)?PO^d4}pNdSV2;oy!k0{eZr6US-%UxzM@)00%5i zk{m1Yba;Z)o(QSIec#I#xn5Qcno}K*(dHNcIv>w$WLTZ^=;k7j`!=rEF;NSj=yJij z7i!D}SkZCg#|;f+<;u#v`Q625c0Gd-3xtRb^Oz5z1i7cz?7+P%)r?ZIg7v?G1(Cq|KU@&7ScQ3f z+@;q;%Y`}gMNn)vM|-a__w#@3Ln;)@jbAeYZAuHL7q|g*F1)XY`SB=EC6!9|%vtRv z^YJaJ)`8@LlvRpX*5ILpHK}Bvo~kXT&MnnMAPOA0g41WsW9lP9mom-)tlj?rF~Z#@ zjYLf;>&J-EkfT+P)KK&N%s41HtGP;YdB5sW1`Ds^FgCj4w*)&4c(=r0)k2$WSXnc3 zVZKSx3R~PjD%iR|n1#8FW(L)~F{yw$8f}W$D~B}{*$LwjBDM4ovzqE#RUXpfw&$2d zv{8Q%w~TXXJX`U&A%245mlEBSa={fHm{ArprvnIH4>EL#rcYua%qdiT@djG z1#JrqMJ*gAjyT-4V+sD^P_^bQ?bL`YzC^Ih_#W#zy{nduDn37IK`az3LGz2tL|7<=95{DMGKr(+;;)v z33lGjhRfd)fDD^v($EZ?Hbr);d~+{5{J?H9<$>y8F+e7RaLT7oPks7|s=m5B6XB)( z59VI`QF-=EkTviftg)Hi{14_Gko#B{U$lAc@h;(9AJI3B@h-iO)W0jL0Kgpt9LdA` zxb+MxtO20tA)H&;qW=H`#3g0A0|$U0;Vv!DPxI)WiNFsnN$#4m&1I zW>ORqsAFHtFy<$&VjMf3&%_xzU=5SMQAD!$odL_@Ee$UAy|JAP232^N_ZXdYd>qEM zHsZJHP^@3WnTa59J}qF>P=#1u&C1P?-eJLmp4rMk6mf6)46X5(pI+H%B8HkE)8ez| z1WJt>tq-b!parFmT}PCspxMoPib3S|r*-|xEVLAPa`kWy1ZBMEsirP=MF1jj^X3|g zP0{$&QzzuV;sInAkT(%{y;g5Om>dm`A+~1tu48ust<8Vf4K1YA?g2F9=tZSaPt3p+ zJ;Q}hiBP7G7{n8ArBnfEndhJRD2P~H=ghE(eeQVRkpf8f0#dl3R77#3Pxt)o}4a-&M!&@NfRne)E0YbcCuW>2iUo}N>6(G`YZ-{{4 z1mktgO)fsutbgo5Kh+9S z`nXI5W&k@NkA1^@8+neiS;g`&66FWw+-9RpYUXWsug93>qWN3?qI4^!%zZ|er8UuR zp+hbI0Gaa)Em=@jZ~L4fO`c*e;A+9q?gsi({YA@~Zyfljr~!L~{6b2Cp!nGx-S#$L zn1X{RUB>J*1t`Xau0Uzk`IG^6=i)h2teF1*aT_ygb9;(cV8^c#(QRL8s)C}#>llRr zYuA_hiDXv;a2n!WXUwaSd1I{0H+H^c$aVF^Za8y7=vaq>fkQcsxccL21G4$`y8*Wqb}la;7cYogJ*NPYzcf z=P`;CFZTF_tAV;>{Bsr2tfipxuysn5g*5PVMW}(XqW=J&5k>UvEBa3np_00-_idM!``FA=hL6{U;e(HuZG(#q~z5)sXL;wlDlcIqq*(eYO-3Mt#R0dSzu z4y*WJ3qZ}K;sJ1#lh1-)jhD_Fd-XONQqE2vdtzNhA&TO;Vo>2;Dl9`lFOBsqQmYm& z!_360&W{ljBc5})lF&WY=2-#bV;uzrLM<0hKd6DBj2Q>7yxk*ce@qf5| zL(M?QBKKH=7DecH{YJp>apo!-W}b#Nb6#qtaMg)oS zvZwN0d6WRsfcH{8aoX?Vn>bve1;F##L?lm5c4HSq*X z$^phuoj#Zs(HMYlt>>6j57Dxqt6opk3NnCVo!=0pi_Lf=E#%#KgoM!XC_|Iu?qoPu zD(}Qy-Y~m9WsU;8=Rd@8O0}%vJyca@M&Ev-3ajDg{{UqYV^?3`l;ah_-mVKEq1L=Z z=NSCT>^ya?KmlXR3`LL-urfC&*jFVfTh8M!m2tpLr*P~k0CU1({nRi84G)GnE|)Py zyAF7lo7{<9g^to1te#nf2%xu0GqqF9#DbG7h$ba~bxJRGM48jasIVos4uV&9F)f^|TB5yvc29>yvoSMHF#x}WmcNk%r{%5iN*1@1d|!7>NL1roo=O- zNc>6`Xnfb1R3YE*%o<-5(^bwQ_WKo-e#OD*nwYKs0PvdkDn|!%bDi=Ieab0s>KBNe zoSr+0t7!XJW(%eR`*G?imJJ2~@c?aI2jJ`6CX;L*x3|n7e?KpxJGQjj<$0Bq($cej z2P`{x9Oub$%ZHrv;1R$o*OzO{0R~r9{zsUk0pTM50BhVSCBxIt^nfnY0Dg}UrHV%; zPfp)5+@R&ie{2EYvAxrk12SUH{`x#^l-%*?AHfkf>e)-{qGqlvu*9VUANMlu155S(>Me@syw_f$g<}H05aq$EXfyTu zN9Lm$<`+}){$m8J^HrZ1m|1$WXNbxI#)~EDZRX8Nwveh@(YQ1+-uou}gUD0@Qtj&f zz^c)0X~4k2wP-c&G#FK4>K15d=Gm!8<>`E`zNQPSx0i0 zm$zp(3y(|j1RJQeev>&CgB5nvSkiuysm<^1yO@d_&VC`uF;)^R3v_v?t>vaHcQi|X z5Cmzia1|Iu6O-T6P$|V>t{}uT6nF`5HHGEx)U$fX50v`M8nOjxH5rEKS$4(B(~LQ@ zag_4Eq~Zc=ykD%#2c_N@ZOqJvu-dLf{6>BTGl~pb8^%Fo5T;mq&a7v+J zliAxXV6kI}BTnyMsDc}DXFt@=>fiOjth~7LK4s-C!^YURV|W5LLy?BFQ+W!0Xp{#O z%Dq;l6BXkuQ+e~a0cBBbrsa)g`M*&`oe0FOOT}X`054_@;p*oxO*Gs^mHFyy72msp z8fNp2z&HdA^%k4$$>J#sS1Bn@_(_?Xn`$Rgw_L!Za}u0y=qJ|SFSaz*0kZ+_0_0dx zpXflqrh_XAYh4@mcFO{&_kJc$oHxP#aThQA@Ix@#n6*F9@fsJi8)?S^i z1NaZ(8HLCG1@R8w`g{=FoSZ+$F<>LgG#*x87S7y1(D49S;SKuu^BoEU24L`Z$~@E4 zkM6}97Rmn0Fp4~Ky~Q)jg5%%y0IqphC3$*(Z}l5h+RV#k94U7gKo?BK z&6ZIC@t7RNv>*ue=3;wkUdHR!aPO2N!C&pmV-N<2Iov=DtL>V?>@QW^2x%4GAS>u^ zGWnIHtEdZ0*z92Wg_;KOcK&4m0nHb3oLD-Aiqtqcmex$+n2RS5+vXq$XcIZw`+|kW zF^nHksRwx68B4Ht%rGiwq`I~tXDP+k)VRf4&hBONWgr}u4c5C^>3Z*|w?j*_)2T%u#8nQMxlIiRp5-aZ%@@f6ors*aaRw}wam+Ac@sBXdIm4b} zV$x*ka_l)6wiw^kRZCTVqMqN#MlEb5HB{7Lk-@|*-J_b!dnaAWO3m^v464{SjSGl3 zC~TXz`j**aKbcx(JY_z05U97l;fbg&7S;aZUVim z&1sM3HCl~A+%&xVjH$Zv*!qQY4n_R1fTjrDhHYXog0 zWdv1a@kcyP3#?u1#0C^+38T7{JFgHF)(;#D`j22)6m^M%I9~5DuJzt~<^yDMKKYoh zOxhduGHh3dh$*72=C(r7M$F?MnOiTL)+14U{yt@q&Assqc?M4^;wh~C-np^h}M$7 zaVb=%3(Ei-t*}WIu&p&wP@j(C4A-Oil$X9vqO9uL#{MFaG;5>gB^f|W6@H~ErufA_ zn96dL(%fmfp=&B2yBv6qmW5|7J9IGF^z7GhspF0b4NZ%-JAQO7=NDW@@5 zPg1sCCAM?%3uP*PZdlN&TJ5dM();{GHjT&2sKqb*8M{w+xTB$^%vKy#L}N^c&dQmB zPpr?J-V5X6UD<7R^(i7<7?oWv=t@a`uqC_93C zV!!00B0IKMFN*+&)LIY@;)1cpsjz#9t`)m|MW+XKo0mwX&|WgvO8~&a=)IrR zLO8PvS}_DBkoa>LMZw~u17e@^<%5uJ_Ujm|K(x~xxW+!^7*^Fub2N(Ap8o)dH_VRf z>3h=>uA35_qc<$_C!hA9YX*KM@M#-tiA=VwYO6W%0xc*%-w^xKINF0qS5kLC>f68~_&s`SToE zl9l=nS%h-uIbE;fIyx>_eFrQCmR&o1z$$6N{{Tm9%s$-P>s&8m({zSAgmY>DjZ_x6EfPWi(%Xus~~*&y)Q^l5Njtm!ft=Yd{~OAS1V* z;&guD&~3C~uU#Hv08;1-1Ea@mpo43){yagHC-sVT8d{mGPfmYu@657vsI7h6uI3=Z z3&sAT^Yt>MyOe5HDk;QH*lHb3BEc+7wD4YIpV}zw&7pva*~RLm1)EKg1uaWiuWJ(F zl`ow_;mLzERJb5~?^4DodDgyRQi$b!!q=nDKk_|R8*#YPwRnH*7aOe97pM*~cI^7* zWRYvyReP7ALzgRTK}uVWZxIF}u5JOOB5hyHsHSkO%mAxuW-ki?`+(f8YRBqZSE}`x zutmb^_Qvc1qi=H0Vy_?j1T@xhU3q}E)xKgwHJx|jI|a}=SyVKqXw0uFs&DZER>aed za|mwQ*BO{VZrQS}4^KylG8{a?Fv2%azKw+?3{K0S-#dGm6ej!oh$!FZ&F%{Yc|(4phV#)>AI!pMjk(+qkD6WF zHd-+Q6gm7x3uiZX1Pv>$aVTxe!F=C-d_ks6999W=bY(f}GKKBChq~`ks&kW7&$)XF zy!UWf{pwOE4f7fwugns>LJJ#!FU%F`ISy97;!q2GwQi9u)mNBu1hK>4aoVls;7Hy8 za`}3LFO3`?rchu57WsOGEr6gf2Z+pyRWPqF1V;L*qJZdyanEbt^o*L`{Xn_{aslVd z0SmM;_Qq`Aw)i?HAxDs}5n2L`2Yvg2SOTThIeTR>c9~bWK;9RX+B@;$Q8tbRKTi_U zZ36!Qj|Yg$3-aZ!;!%5@9^@p92Yi&&^Sd+uXEcz5nKWi@#(uBD5WwD|J~ zfK_jqN#HF%)c^|wZ=W!x84U{7UsC~SD9YlDtrpHDDnQ`hj-eJBO(*pd>21EK%z79c zc$ciS9Qcg@zHyHbE;KDT&L<$E#+XSu{9jR0RdZdj!J>xEInJO|UEVq%KW4d@rvPTWOd(nQVmadjhnz*SD)j=;g@)kSrl`Qjmng8KoMK%v7pD@+ zgNpft+4pfEYb{(AE!mogi?NEuxyGSKJL+6ncJ5zgx{X5eKr?e1!OB8o-A_Zq(HKfx zj0nyX#x_E7Geh{yv(CJ=Dk*t8t;3G_{@}H$w_XUsCaYC`Ihh7BABz3WwOdlI^A-cT z8zNc;ca~VI2S4+uTyED}UJ$lMQ;uL3rkTavUOu8F9xVBnOdQo`xB#~mTuTmarC(2Q zEe>^lB_IZ}{lqQh+$1A-A92G<2K>UfG8MRh8)+Bgi-dGmr3*uc>oD|&=&V4Zb`{hZ zw()y{k>Sk%lIXzQK2jQD?5oYf3tX?_4Wh$uIF>5Yb;Jc8UodF$Oz}}5rLA>yBDJj; zl+0zp0$UkhW@;fR1YG`jjTcXmuLRUt& z2}a%u`<9=WaTt_U>LcD)%3Kuwv^|y@nN9B31PU$dFGPjZ7I3~DCI~h0dY7F8A8^ar zgqx$FdOpI(9b#|DQyfzi9-~lL7|r3;Zc$NIH>{DtQV1R3w6{A<)pdrYFc>!}PGPQK zCPGrXj2r-zpQO9arB_WsTOf=ksR}LgdddiFKA62|P#NZD+FWiQsKQ*hF+O@ihX(|2 zD5~-9`0mS7ewI zwWIG(!pB{HuiB8{M-ZfGDDP!TQ>z9DQym;KhND`79ecz%4+KNb`ouADu`MoNALwc+K9w)TFUN5?e%Zg2c7`N8*2e z%|8(@XD8grEx}2u&ZppPmEb(m-|YC68Nz*7%vTJHvUOa_kXLK-lhO`>de-|sCGZJPV62h0LwU`yS4j zT~70Ii+8nmDS)%6!C@5J=3YD`q@o9Ou~1x#bcU*wi@ zddAleUy4+J=4nLN%s0vq``RGVGLBoF$5+}4L#qZI3+&1xMBRSKnQKWe8gUL0t!sU7 zxR^iS4I{=tq5?R>kFGcpr2@q6GQ=h4f2YDJwFTaR@Dou6L&?L{ZXU9t*5_-)tT6ij z0ESb^JwAtW(J|4({z^hUhm<;z3^{SWFk^o$in1s<^i*!IuxLlmlB!TZaX0sxk^ zmXCHEVv1R(#KXMF)V<4$D}_21HP5q6cO{?-6>HKnMQr7Vrk25RGx6i=9_kM^p3eljV2Wklx>y&cK^?!;!VwR7y zho@h8R0YWohw(9I9E%@POALQTK0y4lHMQ86wepqRPa7#xm~6n%*foWFWOj82Qz-r)gnC^Oc*0ydrI(yQ5&QerC_(E#+2OPdEohN4aW4 zpGj}Z9R(O&=adq!qB^v&Ac&5VG1E*Bfl27gldeyqX+C8bpEmyhPlQyo$dr$Ml9a{M zFZ&vxJ$8SyBwjA!n)aJbRivk6W5E9aA;4=D(8J;&@R*~ox_=k6YZ}#ib z3FRu~SpOJ65M}B|! zsi*Ez8zi1-_7)mV>i+<;d?RRKifEIhsPvZ7K+w1^bVj#lDQf_v$9l{I%r#I7CdKuN zQL_cNsq|rGzolN_eV9_AH|)@UQ_++U+By(UNm1)Vtu={6pe;>&!<}jcP2*W33RBHM z`N*5>8uV{Q+;d3qB28E^+<=&4ARG;BVS2|&Xg@;B<1d+sH(xhN?6GOKZ93{-F7U3K zd4#rePL{u>8m=?1eEmVDVJ~|MRBAoik>K28#W)q!tmeMS%mJlxPO+0*m_g=a%2VpD zo)RVWeIsn~`-$t{6Z}BR%nR77zi}wqzj)2qZ=hv;2!%>+NDMhsSc-BTSIv0la#@sO^J{$B#(YS0f(W-(nle9L7EE9@5zEUkP!9b5HoxAFaycS0L@_VeCON z`h!Q$(1Cla-(-u}eHbsiNuTDK_m%36Q>U+4gC*(L+Vu;z>6j&5Ha_He@;Y);dwmxe z<~#yp7i0%!=neYZL&;BjC$d6sDxzqHblj`6;mi@pYGsh$I8~FX%$gbPP-+pfQL})R zAHfMsM>3aT(dvHXUTKvWqW$al3iFnIBMI#GY8=6rp-*-{NaRwCW>OPidP34;^o8xX zdb#$RX3BZtzni~eUX51PUZ`prQ9&6f-A7(z_$>QaZ`S*mj}c>|+(q}1e?Ful$Ni{% zf>f!?!}kw>K4ki!t$WI<`+c#Ko4k*sybO$UMSDMKVj*H@sZ>Q)wM!EBLGY>C0-zLv zZjA$iz@6Y3S47j%EPKjBqLRc{WK^BwP`*ujM`%($YJTS^GIu7d$$-2%JA^CLgsVdN zyU#!f=V`uQvpSBEcSyV0L{ee9E%Z;suDrQL{{TBoXa{I&joy4fPGgP3&Z~DTTRA%P z!F{Ko%9inamYXT|VR2)V{{WCO-CR(hSUQ3z#kBD!|m_8VzFEcPITwSnp$Q@Tcdel=J8o*_)LH&pt9o##MNJ_DzU2GYfz+G2;Jto*l5!IEdiqed4F> z<@qmkK9H%v)mN$u?G^`6T-XKE$(HKxt!AMWI!Yh;8-?B7cPsY=Qkm1ae!@`o)DRHg|(9O6vm1#VMpH3Rb@o|)PXR$M6W$HmsuY$_{ z>U@@e8>FbR^!j5;8aHR+br4Nms^UYRiE}V57;&tcfha|nfQ~MLWma9{zlZiCo&Y8S zFJVg#9&n8GDygf1->f3JWE7qHdclxnE`iPU9n3jO>LYQh*xAT}$1>gB2j&6;ocT|P zm1^EAr36Tp#%t~n0;h7nY*56C;6se_jB={6YNM`G?**XNn?636{p4~_>JhU$XUscb zx;~chsHI&Be^Tjie2HaT7~jGWDOSy^!tqk6uS%`USuX`Q3E3%tVa1j47^)ya#g8h8 z0_7~I7E~E&9iRI!Evt|-m*$4+ov3^9P~Jz`;h&3<<%3ip}AcZsSZ+h<6LkaHcHMju&|?n~X~ zXKJVyHnwmj}IDp*^U_lN-uS#wxGm^65d0dkRH$|L^(AW{fuqU+}6 zr8M3kVcZ4JIE{6n?<|_07+*MsReO_Bj5MuFQ~(Opy^ui3sd;-lmI8_}bLvJDLRggZ zNF{i=LyxKL|eIYw}O;?GL{2-tGF}OK;;4FO%$;*TgsW zL&E^DUh%z|>Dz{fM3Cr3dYt;m8^-t2u>@_js0~_L zzLAxs)H69$Uv0;mWY&6Wiw_~zJ7-Bm9#L@R7aQRi98-aOipHLBFIk+&ZL_d^ zLC1@i1U&*xz4(O%WMx5QS`00)#VIQEPWJ@rus@z-3KvP}HvB+8&6tjpZ9yK;db40u zK0*PfnLP`Ry#D~Xk>X4^==qVnQGDMn0kzQi@o2xv9(^vehgrJfciug zD1(|6`ibun>(#)eLUlr!=0Rb3G40|O_Y9y_ECFw;&L47WM@rcZLd3TE-yGr6XH%NO z%m4dQ^(VS{)RTpSP1;egQyosS1+S*!7TYD!Ab>h2+Uk7Bd>U!LeOPb zc#i|PuyDM}fNEskHbSdx)ls*^6S*7j&Qg?S`#H?e3Kee1V&CZh03h5R2mOZGezS1R zC@^`HjaYN>2p50c7Xw~|%z2eRv&{NZhlAyxabwl{ixv+2GUs2P(*#}C2Gb5;`IeRs zQk`AAps%<@c2L{0MF()A%#F$~#)4}Bbabg|8!$q-VO9kt)h7BgS z01q#J`5wwylSC{{bB4ujyE;rUSI9W`l@>890Jtt%y7-;GQ`#?f0WxAH5Lr#@(qDyg zVx}InRhnw*`^s(=WNNJ<2`MTY;YX}1F_xNOr8;Jw&LWY3SeiaM&a5BkT=bKI{Aeh9 ze^YNfSi^slhw*uvJP@B`W3MUpMAQ6cEN0J#>UJk%hVA4J%uj z&10k)O=DEvHw9@~-DCd%C92EGo99=UhgSDO(j_-spcEibB{NuEN>Cixol@C`CaGG; z>M)qff1M&ahNF_5KJjBaTs*&U8N+$A^rFi+kygUs)E47TDhz8S)TfY(?R`OpTp-ML zDvNV9m|HSa%4XS;AUb*g=Zyv<@GG<5Ki=`Qa&;*83wJcPy#ft zWSy}cq&p_%V&O8P{-T~jiuO4|AuY_k<&9iC8ST|sxeGJhP(2LJ|@1x zJP>fhEV!{QNEb{@Vgd+#Yd6Hj(SxA8@_|OLU22SQuBmFdo{UGC0Ms73d`gqV7qJ7{ zg4Mtlmm74Vn7f!C61+obW?X*Z_dlK1ELb^wKT&37bTDB=*74RY`F;~6MQVaM)TbpX z*E4{jc3yK2Xu|S+R{Ox=5l!eE*O4n-JVGw9LfUdBoDkqqK@dA$xEst!mQBQbx*s^O?&6WaR@##5O>|sbUr9 zF$(Zdi@Kt@`Gs;Rzz^#C_X}^luX0_8k9v;b>i+;%0MPr7 z)Jh#7vcMs33ZSp8!yy!m<$U8B5TG<1)!sJ(tH5DO!+I_#QcM}sQL6)FH|YjGV6dDU zhetR*k>Qm3YA9}q-Xe;POmJO!j|-xOtg#V`a6`K7;CH2)U%)OMOjS~{jND7Bozmr% zXg@^wQG0~Ev3s-SzW6Y^d_PkWWxyd(LBm-u#8&M$yKeXx8)lp;#975sFO}7lvu^0< z^<}Ia3#HT*h$9^yWyq;hOCH3~5``+t!=TJaAZcc-yJ_(@id(x^=gI?t)$-1KkaU{M zaoWfU$aXP$O*i*aidR{<=XAP&S18q|B*}ZnJA*eaTc4;2M)PR~u*MCN*gDss%%u*r z{*n6RHokXIRy05I81IHuDX{el`HER9+}SrfT~*!*jL|VOMCnd2WXug1f|y#}$J<_HMOV_=dL#*tXK6BWCQT4!tE1 zfec`Eu(&f^kXDK3lt?tH5ifk9TX@Di!XA9EMpB#=dNVojF@?XA^$Df)A1%vY^|l$C zL5N__VDMD|DGCamJHd|P4k$Gh28652mvWeTvY>)3Q0C>0!Ini-W0m10d-QAks7Q=; zxUh_?E38fj1;8fRE>^{!#1&f5M}S_pm~zVUGWRqXD(}STwMo1z8CrdyDrtZmE0tRe zZ1s(MN@!D7J)+MC>_dn)L+>*f=}&f;L8#9Kk}24R7SW3gyn zh4rh6XJdkg-#-KZh~R*Y=*#NFfwK=k@@)ZAFmsJKdrrbB?F4lH0J4%?0WlOd9+boB z0HA?%EvKyz#A#v_1~c9&t3^=f1;Fwba8%ei?>=GTz$odiQDPg_pUoc;??YUA8lNYM zaR-RwL85LaXnLT)NN8gEtS%T50m@mgX_g4EVHR?pp#snj(xL$ZJo(&9FLx)_1|gZh zNx-fJ3P)7F!uyOJ8T!b4KtL7M7JNSO+gDZJaDHaVodeKG_>FQAO(Od;cm?@}bQvgk z9ZNuX;*Vh8SU%}J`aZLldmx^vH*piYO` z{?spYyn=W-W*QC)doYR^64*HzBfZs{g9Kf5fD93j52F>jDr)}G4ebncg(e>|<1MMe zzZN=lCA!qnFlqs+fWr^{M&RuzL>UC>xXuS^B5F6u)DHmAHJE+a>%kT{xUfC4j9~LK z;s~;9F5a^h6fPcQj!>Q6$V#|J;e z%osw)CAtwkTeW%TEoFig*y@6>5jjD&n!f@fE^#9Y^hRpMtLY90A762R^rBT& zl?SA#9i}PO{UTT8`LNT>Fx#|30KFy%5P>QpA_eCFDS%4uJ|b{>Pe0Yq& z)yJ6XS%ryh(w#MCqN1%g19Qs$GX+`E{1!N+0kvRZ;~w6`%>EDUKzyS%JiI%Dqw=Wn z^x(OFgDM?JFI$NY3xecE95FCm%o~qd69@+=vz$)88PhBikbJ?cs=CHLpbnSUoarC| z0=KAwB(_@yJP15kQLt*9o(^Y?ca`RlwOJh>vu8N9+)$%$BxAt_`&~dg68fd&9V>2l z0{RI|fYQn7Y&XS9-Fp83(T&!5miIchc6V&=D#eqhJL})i%vwsq73SVvCXN8HPcfD? zqnvtnnRmZr-4P97YYFqGJ&SafvaB(lZvTj*r0|mK!Kpe|LH-Wzyj#&P<^eH)} zaXA}&;&PaZmcU?4K#L1#>ei#chZY%EP8f*YmA1v0y4B_u5mtc8!o%ds+mL?c7&~EG z*yGu8%g)hXnMOgSENpA1XznaVSdLBq09382FJ1g~`GpK?uKxhPwH+s_d__@m+L_9p zb}$2W(b8vGN|&AFH=?PP%(t9;&C&5S^(MZ!5M?)cip!$MLHSf>9zpLIoXLn4W}VBT zDdos}Dho zjwM#pyO)cu(8U%}T`j3m4|rM987D>>N7PuCh@tGN^y&WqJ|Q-~I=|n{&s7~D<_;jZ z$Ut?!{E78ky?Py9((Sj0QC@Q94QgHxfbuBYQ3r9sCDFwf0mCzWa;A*UgZ9k#bA!Zq zx~Er)Ujp3N6m8*YXtsoNk`h?(vUdoBrYjYRfKzm&_z6a3z`%71o}?1HIPssDp+QxQ zV{-3q#r0V&XOfn50^_eM1*WRP9YZYJ2JLF@$C)xnAhG21S2*HZwfzypz;RsVEdp%J zU9hdI)-B#I z+B8_b`FAf3GQ3qoc3> z%Ws0$#;5KJD@R?b7h@xc7XD9JEd`aOLG{TZ4!?{xirNwXn%eM}c3BeGVB97_U{-RpuVN~E<`>MUwW&oF!M2wH(3Z}iOcn>4(05{>o^QX?=zGdxAWb2~ zmzTVC7(z1R8~#=|4Ned}7yFtB%vM!&z0=I~?YQg*1JLJI(G9_j3rg$v8UsLcaTe99 zLGkXyUL`>dkqH*vLgxPfomgWM8g}$MP4g!UGag{#IlmJB0HK3`BB1j$d5Ni~*Ml4H zFozM0E&ZW1fhtsGhkj6O@63&*W+veYa^TB_4p9crSV1M{s1$<8(|qEDi%K5RF}RXI z(}wkL!E761toZ)`;GB+tUeRj+w!ep8Y0MfB&~2r>#g73VH7^LdwYMH)1rUt^m4g9g z<{tcqV=-d1E}vllq@-z_;Tb^-PR_HN&#b36ykg4&u^N+3W!{3?Q zn|N6vx8Sedgt)jSllDrj-#JBdrdlnD9m}lQ9tQX#ouWsWL-4G^QUXD5L z%Pd}qnuem0R0PT^DdRvYg)UyhgDo zU5jJ(6T67M9);YybO=-y)LJ^LE|4x!Z`kVUO-G#Kp-ip`)9#msvr>wr%6~aEw^SD}DZ@W>y|x#92)~C1vvhFlGe3 z9aR}e4c~$%g@i`L+4B&HoGu(HBuFntWpLd1j`rWB$Geir=s#oj4ncu7;{f5mix@8W`fPtxnfZGSVru3@=9FI=k+#Mc02+Rj5RCpL{^wS@eINp zUeVYtbzB0r1Cvuu2?k>x(yV*wHnXS|SUJp4006)TBl6(Yu}6t=_bUK5<}6}04%l>{ zVrY&7UMS?sLAY*O6isfUsP@aHR<~9+`kTEtCkuO`Oc|sbc77STTmlCcqCqB{ylFB; zbq&OyO5LvM&Oo#b3VLFACiYXT>=I#(ZNQ{xz9LQpW~5#4;vIFKlaj0vPmmz^GGNnC1I3&|-Ap}V`Qzgb3Co@=G1NmfAM zz9W8Hj%EBcfU<`Dq9*WiCoYHFUMmgEcKb~nSh@?ppwT)rXRHMjV)(y80#`wx<>4zu zjVWfaFoSSrSub(P;#WWnp`5ceWz}ExfJK)Et!2=T)?7+-O=JN=1#FSYDoGhoL z2!089P2q@xxGT7IWe)!UgDNxvvCCp9Yi|w8B}(8hy&TpQYm=M%5v%I;i=M1wz)OgJ zTx)|0f^wG~C`~AQ_JGs?P$)VFB4`on&VAsg<_2Z9006Zx0oGwa=)iMNH>M$+E!<7w zIL99nv%!Zki7OFQw0V_?H|A6smB52Y!z2*e;Bl@3F)k%=L9!6Ia3dLsxm*s_jQ$w^ z0J7>YCWkKFqENgXiOZq)9dtO|xj<3C{6#HyY-h~-Ob(1Mwr5wY9a^_K7nRp8v6TUK zprE%w(mLpeYaf|{H*6vS z{F81BFVOgI^Mdcj9cY^!>U@PC7?A`B=Hey1f4Ylz&8mb}VY zR^?}jug8&&Or|j&iA9mShf24U1{PZgV~Y8wer}Oh1?49H012E}qKkpx#tW_h%&4&U zhlH*9;2bvKBFDAHCI&0lh;gW#UJI16<*RsKWrV~BE_)C)M=!sFoTn9aTmJxr!Jzs- z?AdPU{{VlnZ{JIpXsU184yjY^5cT&Yl(1UD_!kL_ntlYF@qRy@#P6z>OE1760En#n40HZokzRKlr280?Otoea{d55 z0*WkAj99Yq7_oQ;I4XudCj-FYrvrdy+lLn%z)FE0D;8V`zEi~&zEso-bAkza=MYig zKeGkV_2>TpCqplf`yHyc`*Bxsv?yDHmw4#B9EkjJcoEb;5*AUjpEH;|62ta~Am+LR z2hB*&TfxYk-^pXMfZ~cQQDTb_f@!JBI*T3z)+=9P9sGU+gQ%H@0i2UKRx6tPY7j?) zhXiiV{emEz2mrOCr#3B|zmIu)1IrHEftoVRAHn|sXFjaw)t@gHfw=HB#D?Krx{0qS zc~gKTSOhz%^&I-KO7HMi*rLY}QDTc81`iFK9%I1VI8JTd^9Q*!j^QhVufpbN#ad#s z1m>xJzm-Il2%t)dYrH=Ucrz=kt2z)#(CaS%znuy_ZBK$VUL5?z*N3piJs z7_EZW@TpuA@Ej<35TmJl+{9&@{Le_Yo=p)}mt2YGT_xQ9ufgNWJQct}FD!4-BP>Hl zBKjV`g4g~7ETEa9#lY|}jw2YyiHxhoKC3i&jCfxY2~v`sCKAb%t^q5BM2VSP2Zcrx zq`+GmJuV#ClK6&J`a>}1;&q(3^N#{M;v0lDY_nDcmpozGdrwhjw(ZaD6jCdXk zk15tVj}hM)^NIW&=l1gm>yFTjA;IAOPWZfeIKThI04Naw00IF60|WyB0RaI300001 z01+V&F+ovbaeIoa>n8R7p~_ttwWG%K-?f zMmKehzu8cJ7}%|#2R$nmN3t80bspIfj8}FJ2YCEtbo(;Gfi`JO8u*y0WbEUQZKZG@ zQ}tp|h@2Wd%s>kpBWJ{SO`BG9-p^PK*V%>KkrX#MHK|giN|pFgrGpZb-p$kv4ek)4 zvmJ~Tv7xD@#VaIp)sDf(lZIFwB{ppB=52#V?Jn?NtB-6}%r=RkBB$Pn3Wr!x4hdQ? zrKmO$@%IjJJz|!^GaK!R4;Up%jHQGSLKb3FsZc$m?p%neOkGut`o-16H(98l8Fppr z=ju?!Jrm%^p$h52Ssr2WOC!06q`fU7>XYhFfpq8%CztY0<3|B|63|3weo5L!l`&J4XaUu1ZVQ!a;;>&(5Z%t9}+G)%8`w9S=aQ zzP;e`LFYDMqjUsLc1iHeF6Hvn8X8os0cEv4KgYbPBFfj9L+pIO(YhzVX95KBq5Vy$ks?bw6U;Fy zK~x*RP=J*yQG;PgQl*stqgE;?pMrsLgXs- zGx09L8_?I2jRr}wdnQXS69s3(HdgvDj54J?B}4NG9S|UlQXsqI-UtX&4o8T6(gZTwCj*D7IUDTX>8I zX;P(VM=S)QR%ES+$d}NBL)h`6ukhYD0Xc-_AS#CuV-A47KfxkPH##TD zvK>d%L-Cl^RC)`0GT~e{m(U`+zSAmoAbB8q#f}#$oOg!TST5gp2;O#V_F+sIgH+kt zCGnyHB5aK$s3=?ywjY>bV|gr%@YFut>iQbcybN5vn+=gp_DSkr@o-(odoVAF)Yvd)AVp-?5R#k4e+3C^PZ z!=`3>Pn{JP!V?RD?U-*%2);d#gU2CS5*B@!Ikf&jSg2q0I(J6KC)>$V1+P&0#pbPG-VOA+YP=QViMV6ZRD6w zBsaMe-efzVUs@qFkHq^+@XJ|9DCCRhgVTKtupL&Vt2XWXc#09|f<`XmHRt&_AiD7k zh?d@A_MrD)OsF8%b|uS?5d@^Ux_eQ5k>UQ$*1QG4@tq4*?Yw0U;1Lyzb!4}VrBDaCZ*%!6 zGI_Gka(9SI$=Z(5i}qCgMHpSlH3(sy<_*0`YK*cxfoqD zezC`R-B)Z{mqfs95`-JO%Y_Vts6EynV2AwrGNH-laBA@Y-8_fV76ngI`wzrZ>=5@1 zpx1ym?MYHKvaS7DfKuIy%tXE5up7b}4d{T#3WMl<5Yu$c19)NiVebHK*%#&k?=h%* zt@u;o8y*dZsQ!^pxcL-@U0Kv``4DT7UKnfd4kw7;wlfaSt2g$6)mrR+DhysDLd<(X zfn{vJ2U$ubF@)dDvrvtdN+PFnpK>suS&dbf#5PDan7dUGJuw<`%MU_dwcUK8TV_$` zR+vH>2!{J(-al$CJ@E63{Yrc8d`p?o`;p;<8)d5xtN?>j|F$8=+Vn9u|RM)YUfb9``H7o_U2`76PI#s=l{JZu80oxA5oE2i_XJ*Ja)WTL@ zaMJHAwAh6ZiGlsaKrKc(=30v_o}v-ln~6{nrYSr$jl>*|)t=~m!;22AO)BxN9*TWK zui_rz2EZp({{Yw^YU<`DHkPqS7Hf%e)zLpzPld#5+EDD*h~FR%vhq;uAM$QbcHzM)xq=jK68IyqBAb`V?-}BN9t=R0k1EM1mTjJInPiDk(&b^{3m>>hdSov?GGZer zbVYpJVkxJdkOWg_6C_)s>l{O62Z?2s%|f+I7u<;F%ZXz4BeY;5%>@u~$7sM8sn^V* zWAH!|`KC#(%F;4i6IG z%i3crKviuo%)A^Cz8=o7Mgz0`Bak>C{=3T3TdJ;J@MW26a{|Q$s*OIep3ztXPT{FCUbK`jA5e+~ zqSP#{srGJIcH%cm+}!zQWGhI=a7!dqYd3=nEr?s*?oHM=FBG{51k6c)mA6P zrMg{f9m%@p;VLSD($-}WnN`sQtTpms_^5caR$mf<043!=R|*zBE&f1W)G>$dB>X7# zrVPK$paAIx7$yh8JZRc>{^5q-_{=#e1O?j$(BJ#oCP2cw3tjKL{{U~JRk+&}MA<6y z6u~uT1869#cvX3+E@n1Z>q%s#@BzJv+%wv5W8gjFV=s*>q#$7?=v*;m zy5?0rC1{rH;-6Vm!jWzUV1PJhV*-X3!BudF$v&6ODiT=bN9ZJ#L9Luqs zaawEP11@LvFnB=4UBv=fQ5A$pV~^}O1>U=Ni8nnK0~HW)98Xd@7TA=sCfnWun|wu9 z3xfx7YEipY$;U;^YXy;EbnlLk4B1c4C8w% zO8Uf6lDXxS4r^juV#{t&s-bIFus-p}8>4px(`3EbUz%({!X|YM4Mxrfxh^SEw7Z$fz2_MBmFjT03AQ%G%*vwQ)G+KLGlaNqKv+<9H{~scx{y%tY*2_J9qAJTdI-A6BN$^Yl%x ziI@8CH~boX81eOsFDKkuJr+CQsC^;&i;FaYLBUw!GI)Fyb=nb?-n5^oe*Ta?h!_ZY ze58F+zOTGN7e`=Q#iI^%?MVW3TETB+R7(hNHT~8Uz@bD&{K~yH1IrU{5Mv*}hP!UA z{{T{k)$kw4UHvdGr|4lJ_oMrYg@Fa);;7;z6tL*cLnY{;Hs`CiiEw3T{{Wx3s}mi+ z*!`wZcg^|-Ny`=KQ+H2!P@yk~u<}I9=?>8}o6ksC2E}_}Q!9(YUHfeS+9_$$E%p;z z``LKou9b)}ZQ$*lmRLAx@a{2(jf0?UK#-S{9nGQuL43?&5_ z2J~hxvhtvOT%osy88r^u@prOcZ}mjJ^X4Ifo;Z$H();IWz?%xJFn#AB{x6k75nA~ z-4(xY)H_eTDecqzJ<0d5`#O9F>U>i|FzEe>*n1~s>`(F&{YUax3@WZw-CAJE)Jq{N z(X>DL68=VzDds+rdf^Dj@;2U9fng2W*>6Ry7gKmeTvm_V$8zM~q%~PnUS~jL#8809 zF*n~ADRu;_qQq|(5h^V$Hip`90Os)obSh#43+5BLY3@MrEgfDu!$6R-?5vW9LQUK; zQwtlN3?QrGEM;ZatQ<#viMxw%xy6$|-)8Pd9Wf`q0eQOXxY`ONBd0%qx4ScWwt!51Q zGjC7!_?zBm!E_hmexUYXK#7l1EBc_H#tHub2tWNF*a4=sVlUNzDZ@$^>g8q+mHdFP zMR7Gsg%z+$Uch&hI7^Ljch@)3GfR%q&>;guaCfh$p*xnP4R27q1ulvoxV%Fu&B`Y3 zY?PQzMB$rlY?npA7K5)Ion{T9Gl}8EHh;+MZ|YXjuo%kR_<+ew8ZuLiR3z5hJ4KL`%*{Dh^ZJMdS<{QkIc83$-u-C2k zp&mS+dI7pb_di5;p3(s`ExR}@OA5P-Q6p-|V|OZq3Lu+4OQ=m%8zw1U9<0x(c`N%U zGMgBiRgP;cV6nLXzcYhHA`R7`n-_8WEc{A#PmuS-bS}Q40+3FON=zw0>}?E%n-ogG z%~Quu{>l+199VAzDQ7A;ZvOzt3KVa4LI?|iwCmY}f0>H8iyV8kEcExi2tpPVURV1s zdnH#GgO*zBQ+CH3LQtmWbo^D1S<5WEs@KGL;_ z^<#G0fq`(oU|bTVuxNB&y%$7ty>0h5vmNSEmbD|~LF_Qu;(R#ea^fiReZ{v=mQz(l?6ko3XPdT*9J-HMD9eFSmF=G`3CaIpt%Eu)E~n+xH6J zF8=`3GqIZB{g6!9@YmE)n|aIyfO<`YB@hO{N2B2V&nxQx03i(s2YwrlQU?er;hP8d zDP!Zm+#yUL3TY+01 z2~j$^wgg*2<)~AI@IOgsZzd1ip>!T?`XUuT&s);;ra!Wr9~GfF*BHIkUcw)BEr4}x zF$nvC^%^&t;OQ-v#c4Xizy+4|VAgv$j5fxH&Z}w>$*oHdDWHtoqj87!V*&;Z-R=U~9+c6>$c6FB0>S3znXSSOdD7ub&gygBbx z72Ws4>@$RZhtSKvdHY4hq2NN@D)hBQ&XnqR_lxqlY8^eEC%)Yp44D6e3 z*A>hwG1#yx%S0}_ozJ)R2xxgBN81SR4W-MJi69rV7en12$uN-7>!48A8jS0V`#?d; z#1&$??HUvsIgLjf;~pBC^M{hcUaq0U>SM>u-`uGy}LF3e5ii?;tHlfcd|;jl2YY zES2KEf%x3L2#idDCS~XHFLgenwG`Q-+L6mh{AxHt>QYY-V2eY6@%a+z7%%gMGY{hUcTD2!P z&|_Vv4cmy6X*hvndvbb36o$QQSn134N9M>9>G}_X;dLc+x5qH5^rPbhYtvOssUDjwec*l&6Y}_)?&&q5PLiQP2s6(2Ur>e zM`Uc!vf%_XoV%N)5Y)CDmYU;ZystL;E!S*AdBhp{n?5FUmEtA_BGRoc52aq{W+`o9 ztG?aA5^f+2HW$_|xjaGAaDU)nIh%ED+xVHch~0?EAO0l~Ynf!Rg&B-bHo#$5T${ww zuI1km!=E-LgsCgHqP4+`vu8n9e4R2>)A_nccj=m?f-r5E#DXPQ zd`cT3@DZ|04K;n^^06gAmXU?RQdyv*b0@KJ+pg*d_u^A>mq1^feMG5NG5-L<6cUKd zW$KdMz0G}GH(Gl_8k!nf6G*`2iVo7AQ>PFGXh2Z!F5^zetTyYDf5{$RV88&EQe55Pw0M9AhyDuRQ8p#K z#ZIYX6viyLK?~^;?109+Too!_uLVQhq9EWmR-T9UC6mAd=GfPR%l#l|1+IoY_MIT| z{{SKKJZeAK+0bDiyZ@X1`JU9umA(9xAk26 z*O(RR`TqRFnCtd%8YaZKdJH1fpxt6O^*h0|Itlfqz8V|N_;YxJFa8S$Ms+rKmr<8^ zLXFr}mS#$<&cr$qIU;6XSP+@Q$u6pe@dvc|m2E4*iB}%H%BJE_yUKwBMjwS~a;j>E z>vzM}U)71Zgz`#I`{i4Ehy2P7B_9D;`G9B%+D`#+GiRmBI<2u&lPn|vU@Y)Q=B@d53@pI7EEyU{A9 zHwWRBxIzk}DSR_0iDFJdGUdR75ZwqBE1>(z#7rT6vF&iuZoap1wV|<;y}hMHH|VF-y%7d%#W0E-7|q9-cj${QB-Q7nG&g~0%Za>Pa8~a? z{pe`vaNa5nbd3Ny@OTa2sH65CHb)>`_KYmSy+}BY`VykNN--)m6<`?6&G?oFPOoTW z%n5ZZalM%vNEHr`T>}IfZzdH~E!vaaO}^NE?DCcZ_$6m@^bf&{%;SL4oR@{=?cn za#q&#+-V4CBFQoGgA9)(+IfkaC2Nh(%zFg8$r3g&V;M{?4Z#p@7|d*>vQbtrVhNZq zV8SYavdg6B%xDO#+iyDU7eS^4INDd4mo~vD=fEM4(Z>){e)q#m8qrFZ7!v45(G=T5TmDfVAExpdXMrx zYgtq?k<17oqVWykv8T=_7+aVVIBtFhyFC)Oaseh(9GgqH*tt*+g-KS?>P2qB38vUN~u;k4)PPL5) z{sSxyqI<@@lD(in({vQZW$6N;&^KPVm8t|GDd4yCW)!!Vb}4buX(fGv)>tZq)3<)| z)>EOdjvtH=hLxRMu+UoZKUh1>&rJo-X-W99p7ACQj)N%i$pt-~p#vz9(a|6}z$y#_ zwB91&T42YgBk29h^MOWgv(@G$04t`^ z7S=4oVmdoQ2u+cRk_RoG&k+@V(_>F-NNCtX5R@*gOT@gfX4T(L@!_S%geD$hB97C83?XkYr)Ir42yRa`53H4 z-N2SOV)pff4gqwldeIHEsKbDb7KQH11uhs^Utzi>)Ilwb4n}7OgT9GXemP+c=5Wwq zpzRik%v{0ppCVol8Zl!^gVB1LYh}xTm!(W9soD65jJ*;%j2JH&kE?JEzJw_c$$1gJ z&H77b7Rc_U$6_1`*uZQ>7`xox7hFRYzMcCBzqdR+hQZ_i07V|Lmonjbj10ii;l7m1 zSO)4Ws>7>r6;)NWcUBKcUWFP0h{6m?W!@%)WZ;ilUSqz5Z^;Y~67eQC=29VEmTIFH zWgk-?xv}`5%*c@`-1>r&$#H)HmgbzPg&RzzwV0{p0{iev#2c|w*4_An3N{51QC8(t zEqW1s5_kOr_b5a*$R0KjG5-LNOc6Mg-Qf?9{?0LJq1kMn7p+4uErC+|M{O=No3h}i z)*i27I_fQ3&55`8oimtec#fM54YU*;i2TrP;!zAEEVoSv+9bL7*7UUT7-q~kv)9g- zFY!zV@QP5u-TI2bEz9w4fG?~5voE}?1K5c-{{ZHYi0JJL7tyT-VHa=sOHeJnW458W zu#t{M(3yfb;5Z3wOp$adh74F}Xu$)nVJ|4Q@vih>q(z|5rf$~a2*D+g(+0nCm+QQL z5-PDnm|g*=(l(B^!v5fvJzD<&Rsfwh_pIa5+15xP!Vs6K!om@A4Uc48OUzq%)pFwv<_>hBPO)Py zUV2mZ^=bvp4!30xi?@k-jtv|X`-C7Fx*ars5zpibT&M)Wljb@v40nhq5BT4t zZI^!!`!e=EXFyTbcF&j^0Tk*w)TkPBH$=pRtvVbBc&3(=gD4 zG$EraFeVU=gD-M#dT4m?ej7AY%+Z@JDuA_CCCBm|2BNUnl!HuQ0duW~PGPkSb(kKk zLmQq6o!lKI)25eNH*nUqU&698cDu{g8~*^S+An5Wyt?IDMyf3VDg+Hp7#a|ZK~KzI zEg#R$E*&BxGtVZ`V@FGuE*A$yOdT0hr4oZPEbM&3s*R=R8F^urTVTXmj-&qoK1WR! z@e<|ZnRM~iFP69La2*M20ufNm5LDd;f)LiAT-^vjhJs`2kKBjjyGAi4!vW$v)Y_8e z%lrZcmryg!67`E3H&}%-X$AQ8Cj2LcGfUL7!Z{W2xx-#&_4y6 z;8&p>4H!laVWBG%6d)J8yGMZSUZ0i0)ipgs z9OG2d*25j)jlJeS^CPOp;QZq*$|a{~RuPg5SmsuK6%auPON6*O)UUvdA@|_QO?9gPj6#6L4lnmRbtg6M=uyM=BcUa*#Vgv8-D5a@ot3E75@ jhlqaRqr`cfH52arP5xi0zv&oi{$yeQ0EXyj=+FPzNa*b< literal 0 HcmV?d00001 From c073ccc5eaedd161b7fbdf687cc420d1b9daff5f Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 21 Jul 2023 18:34:38 -0700 Subject: [PATCH 20/62] Add a couple more linters (#3886) Lint for kv-pairs in log calls and things that can be replaced with constants from the standard library Also add the linters that are enabled by default just so things are more consistent and clear going forward --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3654 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/.golangci.yml | 10 ++++++++-- src/internal/m365/graph/http_wrapper_test.go | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/.golangci.yml b/src/.golangci.yml index 06ccaa3dd..da54f6217 100644 --- a/src/.golangci.yml +++ b/src/.golangci.yml @@ -3,14 +3,20 @@ run: linters: enable: + - errcheck + - forbidigo - gci - gofmt - gofumpt - - errcheck - - forbidigo + - gosimple + - govet + - ineffassign - lll + - loggercheck - misspell - revive + - unused + - usestdlibvars - wsl disable: diff --git a/src/internal/m365/graph/http_wrapper_test.go b/src/internal/m365/graph/http_wrapper_test.go index 594eb75cd..19711edc4 100644 --- a/src/internal/m365/graph/http_wrapper_test.go +++ b/src/internal/m365/graph/http_wrapper_test.go @@ -93,7 +93,7 @@ func (suite *HTTPWrapperUnitSuite) TestNewHTTPWrapper_redirectMiddleware() { hdr.Set("Location", "localhost:99999999/smarfs") toResp := &http.Response{ - StatusCode: 302, + StatusCode: http.StatusFound, Header: hdr, } From 0bb2c06a435d32bfa4f68861d4eb3409eb5cc415 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jul 2023 05:32:41 +0000 Subject: [PATCH 21/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20postcss=20fro?= =?UTF-8?q?m=208.4.26=20to=208.4.27=20in=20/website=20(#3888)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [postcss](https://github.com/postcss/postcss) from 8.4.26 to 8.4.27.

Release notes

Sourced from postcss's releases.

8.4.27

  • Fixed Container clone methods types.
Changelog

Sourced from postcss's changelog.

8.4.27

  • Fixed Container clone methods types.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=postcss&package-manager=npm_and_yarn&previous-version=8.4.26&new-version=8.4.27)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 14 +++++++------- website/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index bbf48e4b8..a897f25bc 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -33,7 +33,7 @@ "@docusaurus/module-type-aliases": "2.4.1", "@iconify/react": "^4.1.1", "autoprefixer": "^10.4.14", - "postcss": "^8.4.26", + "postcss": "^8.4.27", "tailwindcss": "^3.3.3" } }, @@ -10656,9 +10656,9 @@ } }, "node_modules/postcss": { - "version": "8.4.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.26.tgz", - "integrity": "sha512-jrXHFF8iTloAenySjM/ob3gSj7pCu0Ji49hnjqzsgSRa50hkWCKD0HQ+gMNJkW38jBI68MpAAg7ZWwHwX8NMMw==", + "version": "8.4.27", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.27.tgz", + "integrity": "sha512-gY/ACJtJPSmUFPDCHtX78+01fHa64FaU4zaaWfuh1MhGJISufJAH4cun6k/8fwsHYeK4UQmENQK+tRLCFJE8JQ==", "funding": [ { "type": "opencollective", @@ -22569,9 +22569,9 @@ "integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==" }, "postcss": { - "version": "8.4.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.26.tgz", - "integrity": "sha512-jrXHFF8iTloAenySjM/ob3gSj7pCu0Ji49hnjqzsgSRa50hkWCKD0HQ+gMNJkW38jBI68MpAAg7ZWwHwX8NMMw==", + "version": "8.4.27", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.27.tgz", + "integrity": "sha512-gY/ACJtJPSmUFPDCHtX78+01fHa64FaU4zaaWfuh1MhGJISufJAH4cun6k/8fwsHYeK4UQmENQK+tRLCFJE8JQ==", "requires": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", diff --git a/website/package.json b/website/package.json index 3e1388b77..58031eb02 100644 --- a/website/package.json +++ b/website/package.json @@ -39,7 +39,7 @@ "@docusaurus/module-type-aliases": "2.4.1", "@iconify/react": "^4.1.1", "autoprefixer": "^10.4.14", - "postcss": "^8.4.26", + "postcss": "^8.4.27", "tailwindcss": "^3.3.3" }, "browserslist": { From ae0d0f6684669636b0928e24f84c5280950bf0e4 Mon Sep 17 00:00:00 2001 From: neha-Gupta1 Date: Mon, 24 Jul 2023 17:32:08 +0530 Subject: [PATCH 22/62] use teams API --- src/pkg/services/m365/api/groups.go | 165 ------------------ src/pkg/services/m365/api/teams.go | 113 ++++++++++++ .../api/{groups_test.go => teams_test.go} | 26 ++- 3 files changed, 124 insertions(+), 180 deletions(-) delete mode 100644 src/pkg/services/m365/api/groups.go create mode 100644 src/pkg/services/m365/api/teams.go rename src/pkg/services/m365/api/{groups_test.go => teams_test.go} (86%) diff --git a/src/pkg/services/m365/api/groups.go b/src/pkg/services/m365/api/groups.go deleted file mode 100644 index 8eef3fba3..000000000 --- a/src/pkg/services/m365/api/groups.go +++ /dev/null @@ -1,165 +0,0 @@ -package api - -import ( - "context" - - "github.com/alcionai/clues" - msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - "github.com/microsoftgraph/msgraph-sdk-go/models" - - "github.com/alcionai/corso/src/internal/common/str" - "github.com/alcionai/corso/src/internal/common/tform" - "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/logger" -) - -const ( - teamsAdditionalDataLabel = "Team" - ResourceProvisioningOptions = "resourceProvisioningOptions" -) - -// --------------------------------------------------------------------------- -// controller -// --------------------------------------------------------------------------- - -func (c Client) Teams() Teams { - return Teams{c} -} - -// On creation of each Teams team a corrsponding group gets created. -// The group acts as the protected resource, and all teams data like events, -// drive and mail messages are owned by that group. - -// Teams is an interface-compliant provider of the client. -type Teams struct { - Client -} - -// GetAllTeams retrieves all groups. -func (c Teams) GetAll( - ctx context.Context, - errs *fault.Bus, -) ([]models.Groupable, error) { - service, err := c.Service() - if err != nil { - return nil, err - } - - return getGroups(ctx, true, errs, service) -} - -// GetAll retrieves all groups. -func getGroups( - ctx context.Context, - getOnlyTeams bool, - errs *fault.Bus, - service graph.Servicer, -) ([]models.Groupable, error) { - resp, err := service.Client().Groups().Get(ctx, nil) - if err != nil { - return nil, graph.Wrap(ctx, err, "getting all groups") - } - - iter, err := msgraphgocore.NewPageIterator[models.Groupable]( - resp, - service.Adapter(), - models.CreateTeamCollectionResponseFromDiscriminatorValue) - if err != nil { - return nil, graph.Wrap(ctx, err, "creating groups iterator") - } - - var ( - groups = make([]models.Groupable, 0) - el = errs.Local() - ) - - iterator := func(item models.Groupable) bool { - if el.Failure() != nil { - return false - } - - err := ValidateGroup(item) - if err != nil { - el.AddRecoverable(ctx, graph.Wrap(ctx, err, "validating groups")) - } else { - isTeam := IsTeam(ctx, item) - if !getOnlyTeams || isTeam { - groups = append(groups, item) - } - } - - return true - } - - if err := iter.Iterate(ctx, iterator); err != nil { - return nil, graph.Wrap(ctx, err, "iterating all groups") - } - - return groups, el.Failure() -} - -func IsTeam(ctx context.Context, g models.Groupable) bool { - log := logger.Ctx(ctx) - - if g.GetAdditionalData()[ResourceProvisioningOptions] != nil { - val, _ := tform.AnyValueToT[[]any](ResourceProvisioningOptions, g.GetAdditionalData()) - for _, v := range val { - s, err := str.AnyToString(v) - if err != nil { - log.Debug("could not be converted to string value: ", ResourceProvisioningOptions) - return false - } - - if s == teamsAdditionalDataLabel { - return true - } - } - } - - return false -} - -// GetID retrieves team by groupID/teamID. -func (c Teams) GetByID( - ctx context.Context, - identifier string, -) (models.Groupable, error) { - service, err := c.Service() - if err != nil { - return nil, err - } - - resp, err := service.Client().Groups().ByGroupId(identifier).Get(ctx, nil) - if err != nil { - err := graph.Wrap(ctx, err, "getting group by id") - - return nil, err - } - - if !IsTeam(ctx, resp) { - err := clues.New("given teamID is not related to any team") - - return nil, err - } - - return resp, graph.Stack(ctx, err).OrNil() -} - -// --------------------------------------------------------------------------- -// helpers -// --------------------------------------------------------------------------- - -// ValidateGroup ensures the item is a Groupable, and contains the necessary -// identifiers that we handle with all groups. -func ValidateGroup(item models.Groupable) error { - if item.GetId() == nil { - return clues.New("missing ID") - } - - if item.GetDisplayName() == nil { - return clues.New("missing display name") - } - - return nil -} diff --git a/src/pkg/services/m365/api/teams.go b/src/pkg/services/m365/api/teams.go new file mode 100644 index 000000000..883ddee01 --- /dev/null +++ b/src/pkg/services/m365/api/teams.go @@ -0,0 +1,113 @@ +package api + +import ( + "context" + + "github.com/alcionai/clues" + msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/pkg/fault" +) + +// --------------------------------------------------------------------------- +// controller +// --------------------------------------------------------------------------- + +func (c Client) Teams() Teams { + return Teams{c} +} + +// Teams is an interface-compliant provider of the client. +type Teams struct { + Client +} + +// GetAllTeams retrieves all teams. +func (c Teams) GetAll( + ctx context.Context, + errs *fault.Bus, +) ([]models.Teamable, error) { + service, err := c.Service() + if err != nil { + return nil, err + } + + resp, err := service.Client().Teams().Get(ctx, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "getting all teams") + } + + iter, err := msgraphgocore.NewPageIterator[models.Teamable]( + resp, + service.Adapter(), + models.CreateTeamCollectionResponseFromDiscriminatorValue) + if err != nil { + return nil, graph.Wrap(ctx, err, "creating teams iterator") + } + + var ( + teams = make([]models.Teamable, 0) + el = errs.Local() + ) + + iterator := func(item models.Teamable) bool { + if el.Failure() != nil { + return false + } + + err := ValidateTeams(item) + if err != nil { + el.AddRecoverable(ctx, graph.Wrap(ctx, err, "validating teams")) + } else { + teams = append(teams, item) + } + + return true + } + + if err := iter.Iterate(ctx, iterator); err != nil { + return nil, graph.Wrap(ctx, err, "iterating all teams") + } + + return teams, el.Failure() +} + +// GetID retrieves team by teamID. +func (c Teams) GetByID( + ctx context.Context, + identifier string, +) (models.Teamable, error) { + service, err := c.Service() + if err != nil { + return nil, err + } + + resp, err := service.Client().Teams().ByTeamId(identifier).Get(ctx, nil) + if err != nil { + err := graph.Wrap(ctx, err, "getting team by id") + + return nil, err + } + + return resp, graph.Stack(ctx, err).OrNil() +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +// ValidateTeams ensures the item is a Teamable, and contains the necessary +// identifiers that we handle with all teams. +func ValidateTeams(item models.Teamable) error { + if item.GetId() == nil { + return clues.New("missing ID") + } + + if item.GetDisplayName() == nil { + return clues.New("missing display name") + } + + return nil +} diff --git a/src/pkg/services/m365/api/groups_test.go b/src/pkg/services/m365/api/teams_test.go similarity index 86% rename from src/pkg/services/m365/api/groups_test.go rename to src/pkg/services/m365/api/teams_test.go index dcb039dc5..89aab93b5 100644 --- a/src/pkg/services/m365/api/groups_test.go +++ b/src/pkg/services/m365/api/teams_test.go @@ -25,21 +25,21 @@ func TestTeamsUnitSuite(t *testing.T) { suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *TeamsUnitSuite) TestValidateGroup() { +func (suite *TeamsUnitSuite) TestValidateTeams() { team := models.NewTeam() - team.SetDisplayName(ptr.To("testgroup")) + team.SetDisplayName(ptr.To("testteam")) team.SetId(ptr.To("testID")) tests := []struct { name string - args models.Groupable + args models.Teamable errCheck assert.ErrorAssertionFunc errIsSkippable bool }{ { - name: "Valid group ", - args: func() *models.Group { - s := models.NewGroup() + name: "Valid Team", + args: func() *models.Team { + s := models.NewTeam() s.SetId(ptr.To("id")) s.SetDisplayName(ptr.To("testTeam")) return s @@ -48,8 +48,8 @@ func (suite *TeamsUnitSuite) TestValidateGroup() { }, { name: "No name", - args: func() *models.Group { - s := models.NewGroup() + args: func() *models.Team { + s := models.NewTeam() s.SetId(ptr.To("id")) return s }(), @@ -57,8 +57,8 @@ func (suite *TeamsUnitSuite) TestValidateGroup() { }, { name: "No ID", - args: func() *models.Group { - s := models.NewGroup() + args: func() *models.Team { + s := models.NewTeam() s.SetDisplayName(ptr.To("testTeam")) return s }(), @@ -70,7 +70,7 @@ func (suite *TeamsUnitSuite) TestValidateGroup() { suite.Run(test.name, func() { t := suite.T() - err := api.ValidateGroup(test.args) + err := api.ValidateTeams(test.args) test.errCheck(t, err, clues.ToCore(err)) if test.errIsSkippable { @@ -108,10 +108,6 @@ func (suite *TeamsIntgSuite) TestGetAllTeams() { GetAll(ctx, fault.New(true)) require.NoError(t, err) require.NotZero(t, len(teams), "must have at least one team") - - for _, team := range teams { - assert.True(t, api.IsTeam(ctx, team), "must not return non teams groups") - } } func (suite *TeamsIntgSuite) TestTeams_GetByID() { From ab344422d68578a539cc4260c5751c09df13ba18 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:43:27 -0700 Subject: [PATCH 23/62] Cleanup figuring out kopia retention parameters (#3857) Create a struct that handles: * initialization from existing kopia config info * in-memory updates to config info * detecting which config info structs from kopia need updated * returning kopia config info structs Overall, this allows us to isolate the logic for calculating the new retention configuration info in kopia Viewing by commit may help. First commit just splits up existing code, moving it into either conn.go (will be used later) or retention/opts.go. Subsequent commits switch to using a struct, add tests, and fixup existing logic --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/conn.go | 122 ++++++++++++- src/internal/kopia/retention/opts.go | 139 +++++++++++++++ src/internal/kopia/retention/opts_test.go | 204 ++++++++++++++++++++++ src/internal/kopia/wrapper.go | 203 +-------------------- 4 files changed, 463 insertions(+), 205 deletions(-) create mode 100644 src/internal/kopia/retention/opts.go create mode 100644 src/internal/kopia/retention/opts_test.go diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index d28001f3f..e9d20918a 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -12,12 +12,16 @@ import ( "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" + "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/kopia/retention" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/storage" ) @@ -326,12 +330,12 @@ func updateCompressionOnPolicy(compressor string, p *policy.Policy) (bool, error return true, nil } -func updateRetentionOnPolicy(retention policy.RetentionPolicy, p *policy.Policy) bool { - if retention == p.RetentionPolicy { +func updateRetentionOnPolicy(retPolicy policy.RetentionPolicy, p *policy.Policy) bool { + if retPolicy == p.RetentionPolicy { return false } - p.RetentionPolicy = retention + p.RetentionPolicy = retPolicy return true } @@ -410,6 +414,118 @@ func checkCompressor(compressor compression.Name) error { return clues.Stack(clues.New("unknown compressor type"), clues.New(string(compressor))) } +func (w *conn) setRetentionParameters( + ctx context.Context, + rrOpts repository.Retention, +) error { + if rrOpts.Mode == nil && rrOpts.Duration == nil && rrOpts.Extend == nil { + return nil + } + + // Somewhat confusing case, when we have no retention but a non-zero duration + // it acts like we passed in only the duration and returns an error about + // having to set both. Return a clearer error here instead. + if ptr.Val(rrOpts.Mode) == repository.NoRetention && ptr.Val(rrOpts.Duration) != 0 { + return clues.New("duration must be 0 if rrOpts is disabled").WithClues(ctx) + } + + dr, ok := w.Repository.(repo.DirectRepository) + if !ok { + return clues.New("getting handle to repo").WithClues(ctx) + } + + blobCfg, params, err := getRetentionConfigs(ctx, dr) + if err != nil { + return clues.Stack(err) + } + + opts := retention.OptsFromConfigs(*blobCfg, *params) + if err := opts.Set(rrOpts); err != nil { + return clues.Stack(err).WithClues(ctx) + } + + return clues.Stack(persistRetentionConfigs(ctx, dr, opts)).OrNil() +} + +func getRetentionConfigs( + ctx context.Context, + dr repo.DirectRepository, +) (*format.BlobStorageConfiguration, *maintenance.Params, error) { + blobCfg, err := dr.FormatManager().BlobCfgBlob() + if err != nil { + return nil, nil, clues.Wrap(err, "getting storage config").WithClues(ctx) + } + + params, err := maintenance.GetParams(ctx, dr) + if err != nil { + return nil, nil, clues.Wrap(err, "getting maintenance config").WithClues(ctx) + } + + return &blobCfg, params, nil +} + +func persistRetentionConfigs( + ctx context.Context, + dr repo.DirectRepository, + opts *retention.Opts, +) error { + // Persist changes. + if !opts.BlobChanged() && !opts.ParamsChanged() { + return nil + } + + blobCfg, params, err := opts.AsConfigs(ctx) + if err != nil { + return clues.Stack(err) + } + + mp, err := dr.FormatManager().GetMutableParameters() + if err != nil { + return clues.Wrap(err, "getting mutable parameters").WithClues(ctx) + } + + requiredFeatures, err := dr.FormatManager().RequiredFeatures() + if err != nil { + return clues.Wrap(err, "getting required features").WithClues(ctx) + } + + // Must be the case that only blob changed. + if !opts.ParamsChanged() { + return clues.Wrap( + dr.FormatManager().SetParameters(ctx, mp, blobCfg, requiredFeatures), + "persisting storage config", + ).WithClues(ctx).OrNil() + } + + // Both blob and maintenance changed. A DirectWriteSession is required to + // update the maintenance config but not the blob config. + err = repo.DirectWriteSession( + ctx, + dr, + repo.WriteSessionOptions{ + Purpose: "Corso immutable backups config", + }, + func(ctx context.Context, dw repo.DirectRepositoryWriter) error { + // Set the maintenance config first as we can bail out of the write + // session later. + if err := maintenance.SetParams(ctx, dw, ¶ms); err != nil { + return clues.Wrap(err, "maintenance config"). + WithClues(ctx) + } + + if !opts.BlobChanged() { + return nil + } + + return clues.Wrap( + dr.FormatManager().SetParameters(ctx, mp, blobCfg, requiredFeatures), + "storage config", + ).WithClues(ctx).OrNil() + }) + + return clues.Wrap(err, "persisting config changes").WithClues(ctx).OrNil() +} + func (w *conn) LoadSnapshot( ctx context.Context, id manifest.ID, diff --git a/src/internal/kopia/retention/opts.go b/src/internal/kopia/retention/opts.go new file mode 100644 index 000000000..b63a6a6a3 --- /dev/null +++ b/src/internal/kopia/retention/opts.go @@ -0,0 +1,139 @@ +package retention + +import ( + "context" + "time" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/format" + "github.com/kopia/kopia/repo/maintenance" + + "github.com/alcionai/corso/src/pkg/control/repository" +) + +type Opts struct { + blobCfg format.BlobStorageConfiguration + params maintenance.Params + + blobChanged bool + paramsChanged bool +} + +func NewOpts() *Opts { + return &Opts{} +} + +func OptsFromConfigs( + blobCfg format.BlobStorageConfiguration, + params maintenance.Params, +) *Opts { + return &Opts{ + blobCfg: blobCfg, + params: params, + } +} + +func (r *Opts) AsConfigs( + ctx context.Context, +) (format.BlobStorageConfiguration, maintenance.Params, error) { + // Check the new config is valid. + if r.blobCfg.IsRetentionEnabled() { + if err := maintenance.CheckExtendRetention(ctx, r.blobCfg, &r.params); err != nil { + return format.BlobStorageConfiguration{}, maintenance.Params{}, clues.Wrap( + err, + "invalid retention config", + ).WithClues(ctx) + } + } + + return r.blobCfg, r.params, nil +} + +func (r *Opts) BlobChanged() bool { + return r.blobChanged +} + +func (r *Opts) ParamsChanged() bool { + return r.paramsChanged +} + +func (r *Opts) Set(opts repository.Retention) error { + r.setMaintenanceParams(opts.Extend) + + return clues.Wrap( + r.setBlobConfigParams(opts.Mode, opts.Duration), + "setting mode or duration", + ).OrNil() +} + +func (r *Opts) setMaintenanceParams(extend *bool) { + if extend != nil && r.params.ExtendObjectLocks != *extend { + r.params.ExtendObjectLocks = *extend + r.paramsChanged = true + } +} + +func (r *Opts) setBlobConfigParams( + mode *repository.RetentionMode, + duration *time.Duration, +) error { + err := r.setBlobConfigMode(mode) + if err != nil { + return clues.Stack(err) + } + + r.setBlobConfigDuration(duration) + + return nil +} + +func (r *Opts) setBlobConfigDuration(duration *time.Duration) { + if duration != nil && r.blobCfg.RetentionPeriod != *duration { + r.blobCfg.RetentionPeriod = *duration + r.blobChanged = true + } +} + +func (r *Opts) setBlobConfigMode( + mode *repository.RetentionMode, +) error { + if mode == nil { + return nil + } + + startMode := r.blobCfg.RetentionMode + + switch *mode { + case repository.NoRetention: + if !r.blobCfg.IsRetentionEnabled() { + return nil + } + + r.blobCfg.RetentionMode = "" + r.blobCfg.RetentionPeriod = 0 + + case repository.GovernanceRetention: + r.blobCfg.RetentionMode = blob.Governance + + case repository.ComplianceRetention: + r.blobCfg.RetentionMode = blob.Compliance + + default: + return clues.New("unknown retention mode"). + With("provided_retention_mode", mode.String()) + } + + // Only check if the retention mode is not empty. IsValid errors out if it's + // empty. + if len(r.blobCfg.RetentionMode) > 0 && !r.blobCfg.RetentionMode.IsValid() { + return clues.New("invalid retention mode"). + With("retention_mode", r.blobCfg.RetentionMode) + } + + // Take into account previous operations on r that could have already updated + // blobChanged. + r.blobChanged = r.blobChanged || startMode != r.blobCfg.RetentionMode + + return nil +} diff --git a/src/internal/kopia/retention/opts_test.go b/src/internal/kopia/retention/opts_test.go new file mode 100644 index 000000000..8b250c79a --- /dev/null +++ b/src/internal/kopia/retention/opts_test.go @@ -0,0 +1,204 @@ +package retention_test + +import ( + "testing" + "time" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/format" + "github.com/kopia/kopia/repo/maintenance" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/kopia/retention" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control/repository" +) + +type OptsUnitSuite struct { + tester.Suite +} + +func TestOptsUnitSuite(t *testing.T) { + suite.Run(t, &OptsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *OptsUnitSuite) TestOptsFromConfigs() { + var ( + t = suite.T() + + mode = blob.Governance + duration = time.Hour * 48 + extend = true + + blobCfgInput = format.BlobStorageConfiguration{ + RetentionMode: mode, + RetentionPeriod: duration, + } + paramsInput = maintenance.Params{ExtendObjectLocks: extend} + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + opts := retention.OptsFromConfigs(blobCfgInput, paramsInput) + + assert.False(t, opts.BlobChanged(), "BlobChanged") + assert.False(t, opts.ParamsChanged(), "ParamsChanged") + + blobCfg, params, err := opts.AsConfigs(ctx) + require.NoError(t, err, "AsConfigs: %v", clues.ToCore(err)) + assert.Equal(t, blobCfgInput, blobCfg) + assert.Equal(t, paramsInput, params) +} + +func (suite *OptsUnitSuite) TestSet() { + var ( + kopiaMode = blob.Governance + mode = repository.GovernanceRetention + duration = time.Hour * 48 + ) + + table := []struct { + name string + inputBlob format.BlobStorageConfiguration + inputParams maintenance.Params + ctrlOpts repository.Retention + setErr require.ErrorAssertionFunc + expectMode blob.RetentionMode + expectDuration time.Duration + expectExtend bool + expectBlobChanged bool + expectParamsChanged bool + }{ + { + name: "All Nils", + setErr: require.NoError, + }, + { + name: "All Off", + ctrlOpts: repository.Retention{ + Mode: ptr.To(repository.NoRetention), + Duration: ptr.To(time.Duration(0)), + Extend: ptr.To(false), + }, + setErr: require.NoError, + }, + { + name: "UnknownRetention", + ctrlOpts: repository.Retention{ + Mode: ptr.To(repository.UnknownRetention), + Duration: ptr.To(duration), + }, + setErr: require.Error, + }, + { + name: "Invalid Retention Mode", + ctrlOpts: repository.Retention{ + Mode: ptr.To(repository.RetentionMode(-1)), + Duration: ptr.To(duration), + }, + setErr: require.Error, + }, + { + name: "Valid Set All", + ctrlOpts: repository.Retention{ + Mode: ptr.To(mode), + Duration: ptr.To(duration), + Extend: ptr.To(true), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration, + expectExtend: true, + expectBlobChanged: true, + expectParamsChanged: true, + }, + { + name: "Valid Set BlobConfig", + ctrlOpts: repository.Retention{ + Mode: ptr.To(mode), + Duration: ptr.To(duration), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration, + expectBlobChanged: true, + }, + { + name: "Valid Set Params", + ctrlOpts: repository.Retention{ + Extend: ptr.To(true), + }, + setErr: require.NoError, + expectExtend: true, + expectParamsChanged: true, + }, + { + name: "Partial BlobConfig Change", + inputBlob: format.BlobStorageConfiguration{ + RetentionMode: kopiaMode, + RetentionPeriod: duration, + }, + ctrlOpts: repository.Retention{ + Duration: ptr.To(duration + time.Hour), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration + time.Hour, + expectBlobChanged: true, + }, + { + name: "No BlobConfig Change", + inputBlob: format.BlobStorageConfiguration{ + RetentionMode: kopiaMode, + RetentionPeriod: duration, + }, + ctrlOpts: repository.Retention{ + Mode: ptr.To(mode), + Duration: ptr.To(duration), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration, + }, + { + name: "No Params Change", + inputParams: maintenance.Params{ExtendObjectLocks: true}, + ctrlOpts: repository.Retention{ + Extend: ptr.To(true), + }, + setErr: require.NoError, + expectExtend: true, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + opts := retention.OptsFromConfigs(test.inputBlob, test.inputParams) + err := opts.Set(test.ctrlOpts) + test.setErr(t, err, "setting params: %v", clues.ToCore(err)) + + if err != nil { + return + } + + blobCfg, params, err := opts.AsConfigs(ctx) + require.NoError(t, err, "getting configs: %v", clues.ToCore(err)) + + assert.Equal(t, test.expectMode, blobCfg.RetentionMode, "mode") + assert.Equal(t, test.expectDuration, blobCfg.RetentionPeriod, "duration") + assert.Equal(t, test.expectExtend, params.ExtendObjectLocks, "extend locks") + assert.Equal(t, test.expectBlobChanged, opts.BlobChanged(), "blob changed") + assert.Equal(t, test.expectParamsChanged, opts.ParamsChanged(), "params changed") + }) + } +} diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 59235cce2..06f81c635 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -4,13 +4,10 @@ import ( "context" "errors" "strings" - "time" "github.com/alcionai/clues" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" @@ -20,7 +17,6 @@ import ( "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/common/prefixmatcher" - "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/observe" @@ -737,202 +733,5 @@ func (w *Wrapper) SetRetentionParameters( ctx context.Context, retention repository.Retention, ) error { - if retention.Mode == nil && retention.Duration == nil && retention.Extend == nil { - return nil - } - - // Somewhat confusing case, when we have no retention but a non-zero duration - // it acts like we passed in only the duration and returns an error about - // having to set both. Return a clearer error here instead. Check if mode is - // set so we still allow changing duration if mode is already set. - if m, ok := ptr.ValOK(retention.Mode); ok && m == repository.NoRetention && ptr.Val(retention.Duration) != 0 { - return clues.New("duration must be 0 if retention is disabled").WithClues(ctx) - } - - dr, ok := w.c.Repository.(repo.DirectRepository) - if !ok { - return clues.New("getting handle to repo").WithClues(ctx) - } - - blobCfg, params, err := getRetentionConfigs(ctx, dr) - if err != nil { - return clues.Stack(err) - } - - // Update blob config information. - blobChanged, err := w.setBlobConfigParams(retention.Mode, retention.Duration, blobCfg) - if err != nil { - return clues.Wrap(err, "setting retention mode or duration").WithClues(ctx) - } - - // Update maintenance config information. - var maintenanceChanged bool - - if retention.Extend != nil && params.ExtendObjectLocks != *retention.Extend { - params.ExtendObjectLocks = *retention.Extend - maintenanceChanged = true - } - - // Check the new config is valid. - if blobCfg.IsRetentionEnabled() { - if err := maintenance.CheckExtendRetention(ctx, *blobCfg, params); err != nil { - return clues.Wrap(err, "invalid retention config").WithClues(ctx) - } - } - - return clues.Stack(persistRetentionConfigs( - ctx, - dr, - blobCfg, - blobChanged, - params, - maintenanceChanged, - )).OrNil() -} - -func getRetentionConfigs( - ctx context.Context, - dr repo.DirectRepository, -) (*format.BlobStorageConfiguration, *maintenance.Params, error) { - blobCfg, err := dr.FormatManager().BlobCfgBlob() - if err != nil { - return nil, nil, clues.Wrap(err, "getting storage config").WithClues(ctx) - } - - params, err := maintenance.GetParams(ctx, dr) - if err != nil { - return nil, nil, clues.Wrap(err, "getting maintenance config").WithClues(ctx) - } - - return &blobCfg, params, nil -} - -func persistRetentionConfigs( - ctx context.Context, - dr repo.DirectRepository, - blobCfg *format.BlobStorageConfiguration, - blobChanged bool, - params *maintenance.Params, - maintenanceChanged bool, -) error { - // Persist changes. - if !blobChanged && !maintenanceChanged { - return nil - } - - mp, err := dr.FormatManager().GetMutableParameters() - if err != nil { - return clues.Wrap(err, "getting mutable parameters") - } - - requiredFeatures, err := dr.FormatManager().RequiredFeatures() - if err != nil { - return clues.Wrap(err, "getting required features").WithClues(ctx) - } - - // Must be the case that only blob changed. - if !maintenanceChanged { - return clues.Wrap( - dr.FormatManager().SetParameters(ctx, mp, *blobCfg, requiredFeatures), - "persisting storage config", - ).WithClues(ctx).OrNil() - } - - // Both blob and maintenance changed. A DirectWriteSession is required to - // update the maintenance config but not the blob config. - err = repo.DirectWriteSession( - ctx, - dr, - repo.WriteSessionOptions{ - Purpose: "Corso immutable backups config", - }, - func(ctx context.Context, dw repo.DirectRepositoryWriter) error { - // Set the maintenance config first as we can bail out of the write - // session later. - if err := maintenance.SetParams(ctx, dw, params); err != nil { - return clues.Wrap(err, "maintenance config"). - WithClues(ctx) - } - - if !blobChanged { - return nil - } - - return clues.Wrap( - dr.FormatManager().SetParameters(ctx, mp, *blobCfg, requiredFeatures), - "storage config", - ).WithClues(ctx).OrNil() - }) - - return clues.Wrap(err, "persisting config changes").WithClues(ctx).OrNil() -} - -func (w Wrapper) setBlobConfigParams( - mode *repository.RetentionMode, - duration *time.Duration, - blobCfg *format.BlobStorageConfiguration, -) (bool, error) { - changed, err := setBlobConfigMode(mode, blobCfg) - if err != nil { - return false, clues.Stack(err) - } - - tmp := setBlobConfigDuration(duration, blobCfg) - changed = changed || tmp - - return changed, nil -} - -func setBlobConfigDuration( - duration *time.Duration, - blobCfg *format.BlobStorageConfiguration, -) bool { - var changed bool - - if duration != nil && blobCfg.RetentionPeriod != *duration { - blobCfg.RetentionPeriod = *duration - changed = true - } - - return changed -} - -func setBlobConfigMode( - mode *repository.RetentionMode, - blobCfg *format.BlobStorageConfiguration, -) (bool, error) { - if mode == nil { - return false, nil - } - - startMode := blobCfg.RetentionMode - - switch *mode { - case repository.NoRetention: - if !blobCfg.IsRetentionEnabled() { - return false, nil - } - - blobCfg.RetentionMode = "" - blobCfg.RetentionPeriod = 0 - - case repository.GovernanceRetention: - blobCfg.RetentionMode = blob.Governance - - case repository.ComplianceRetention: - blobCfg.RetentionMode = blob.Compliance - - default: - return false, clues.New("unknown retention mode"). - With("provided_retention_mode", mode.String()) - } - - // Only check if the retention mode is not empty. IsValid errors out if it's - // empty. - if len(blobCfg.RetentionMode) > 0 && !blobCfg.RetentionMode.IsValid() { - return false, clues.New("invalid retention mode"). - With("retention_mode", blobCfg.RetentionMode) - } - - return startMode != blobCfg.RetentionMode, nil + return clues.Stack(w.c.setRetentionParameters(ctx, retention)).OrNil() } From f7496e52420ab431703140a523e97ce62927b7a3 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:46:21 -0700 Subject: [PATCH 24/62] Require passing AWS bucket for integration tests (#3852) #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/tester/tconfig/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/tester/tconfig/config.go b/src/internal/tester/tconfig/config.go index c6bcd6e4b..963f4f6b4 100644 --- a/src/internal/tester/tconfig/config.go +++ b/src/internal/tester/tconfig/config.go @@ -108,7 +108,7 @@ func ReadTestConfig() (map[string]string, error) { testEnv := map[string]string{} fallbackTo(testEnv, TestCfgStorageProvider, vpr.GetString(TestCfgStorageProvider)) fallbackTo(testEnv, TestCfgAccountProvider, vpr.GetString(TestCfgAccountProvider)) - fallbackTo(testEnv, TestCfgBucket, os.Getenv("S3_BUCKET"), vpr.GetString(TestCfgBucket), "test-corso-repo-init") + fallbackTo(testEnv, TestCfgBucket, os.Getenv("S3_BUCKET"), vpr.GetString(TestCfgBucket)) fallbackTo(testEnv, TestCfgEndpoint, vpr.GetString(TestCfgEndpoint), "s3.amazonaws.com") fallbackTo(testEnv, TestCfgPrefix, vpr.GetString(TestCfgPrefix)) fallbackTo(testEnv, TestCfgAzureTenantID, os.Getenv(account.AzureTenantID), vpr.GetString(TestCfgAzureTenantID)) From d2dda001950ad4211b2e0d4e2fb097be72e223b6 Mon Sep 17 00:00:00 2001 From: Keepers Date: Mon, 24 Jul 2023 15:45:57 -0600 Subject: [PATCH 25/62] update all graph packages (#3879) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup --- src/go.mod | 13 ++++++------- src/go.sum | 26 ++++++++++++-------------- src/internal/m365/graph/errors.go | 2 +- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/go.mod b/src/go.mod index 5cb7b980e..b589ca542 100644 --- a/src/go.mod +++ b/src/go.mod @@ -5,7 +5,7 @@ go 1.20 replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 require ( - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go v1.44.305 @@ -14,12 +14,12 @@ require ( github.com/google/uuid v1.3.0 github.com/h2non/gock v1.2.0 github.com/kopia/kopia v0.12.2-0.20230327171220-747baeebdab1 - github.com/microsoft/kiota-abstractions-go v1.0.0 + github.com/microsoft/kiota-abstractions-go v1.1.0 github.com/microsoft/kiota-authentication-azure-go v1.0.0 github.com/microsoft/kiota-http-go v1.0.0 github.com/microsoft/kiota-serialization-form-go v1.0.0 - github.com/microsoft/kiota-serialization-json-go v1.0.2 - github.com/microsoftgraph/msgraph-sdk-go v1.4.0 + github.com/microsoft/kiota-serialization-json-go v1.0.4 + github.com/microsoftgraph/msgraph-sdk-go v1.12.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 github.com/puzpuzpuz/xsync/v2 v2.4.1 @@ -42,7 +42,6 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect @@ -62,9 +61,9 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/src/go.sum b/src/go.sum index fe18cc952..718084162 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,14 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= @@ -102,7 +102,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= @@ -272,20 +271,20 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microsoft/kiota-abstractions-go v1.0.0 h1:teQS3yOmcTyps+O48AD17LI8TR1B3wCEwGFcwC6K75c= -github.com/microsoft/kiota-abstractions-go v1.0.0/go.mod h1:2yaRQnx2KU7UaenYSApiTT4pf7fFkPV0B71Rm2uYynQ= +github.com/microsoft/kiota-abstractions-go v1.1.0 h1:X1aKlsYCRs/0RSChr/fbq4j/+kxRzbSY5GeWhtHQNYI= +github.com/microsoft/kiota-abstractions-go v1.1.0/go.mod h1:RkxyZ5x87Njik7iVeQY9M2wtrrL1MJZcXiI/BxD/82g= github.com/microsoft/kiota-authentication-azure-go v1.0.0 h1:29FNZZ/4nnCOwFcGWlB/sxPvWz487HA2bXH8jR5k2Rk= github.com/microsoft/kiota-authentication-azure-go v1.0.0/go.mod h1:rnx3PRlkGdXDcA/0lZQTbBwyYGmc+3POt7HpE/e4jGw= github.com/microsoft/kiota-http-go v1.0.0 h1:F1hd6gMlLeEgH2CkRB7z13ow7LxMKMWEmms/t0VfS+k= github.com/microsoft/kiota-http-go v1.0.0/go.mod h1:eujxJliqodotsYepIc6ihhK+vXMMt5Q8YiSNL7+7M7U= github.com/microsoft/kiota-serialization-form-go v1.0.0 h1:UNdrkMnLFqUCccQZerKjblsyVgifS11b3WCx+eFEsAI= github.com/microsoft/kiota-serialization-form-go v1.0.0/go.mod h1:h4mQOO6KVTNciMF6azi1J9QB19ujSw3ULKcSNyXXOMA= -github.com/microsoft/kiota-serialization-json-go v1.0.2 h1:RXan8v7yWBD88XxVZ2W38BBcqu2UqWtgS54nCbOS5ow= -github.com/microsoft/kiota-serialization-json-go v1.0.2/go.mod h1:AUItT9exyxmjZQE8IeFD9ygP77q9GKVb+AQE2V5Ikho= +github.com/microsoft/kiota-serialization-json-go v1.0.4 h1:5TaISWwd2Me8clrK7SqNATo0tv9seOq59y4I5953egQ= +github.com/microsoft/kiota-serialization-json-go v1.0.4/go.mod h1:rM4+FsAY+9AEpBsBzkFFis+b/LZLlNKKewuLwK9Q6Mg= github.com/microsoft/kiota-serialization-text-go v1.0.0 h1:XOaRhAXy+g8ZVpcq7x7a0jlETWnWrEum0RhmbYrTFnA= github.com/microsoft/kiota-serialization-text-go v1.0.0/go.mod h1:sM1/C6ecnQ7IquQOGUrUldaO5wj+9+v7G2W3sQ3fy6M= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0 h1:ibNwMDEZ6HikA9BVXu+TljCzCiE+yFsD6wLpJbTc1tc= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0/go.mod h1:JIDL1xENx92B60NjO2ACyqGeKvtYkdl9rirgajIgryw= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0 h1:/jZJ1KCtVlvxStKq31VsEPOQQ5Iy26R1pgvc+RYt7XI= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0/go.mod h1:ccLv84FJFtwdSzYWM/HlTes5FLzkzzBsYh9kg93/WS8= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 h1:7NWTfyXvOjoizW7PmxNp3+8wCKPgpODs/D1cUZ3fkAY= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0/go.mod h1:tQb4q3YMIj2dWhhXhQSJ4ELpol931ANKzHSYK5kX1qE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -303,7 +302,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= diff --git a/src/internal/m365/graph/errors.go b/src/internal/m365/graph/errors.go index dad2674a4..f35b91385 100644 --- a/src/internal/m365/graph/errors.go +++ b/src/internal/m365/graph/errors.go @@ -365,7 +365,7 @@ func errData(err odataerrors.ODataErrorable) (string, []any, string) { msgConcat += ptr.Val(d.GetMessage()) } - inner := mainErr.GetInnererror() + inner := mainErr.GetInnerError() if inner != nil { data = appendIf(data, "odataerror_inner_cli_req_id", inner.GetClientRequestId()) data = appendIf(data, "odataerror_inner_req_id", inner.GetRequestId()) From 42adc033d9775838396d46ca3a9cf2aff59d6bcc Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:31:54 -0700 Subject: [PATCH 26/62] Pass BackupBases directly to ConsumeBackupCollections (#3876) Shift things so BackupBases is passed directly to the kopia package which then extracts information from it. This allows for fine-grained control over kopia-assisted incremental bases and merge bases. Generating subtree paths from Reasons is also shifted to the kopia package Also expands tests for better coverage of various incremental backup situations Viewing by commit may help and individual commit comments usually contain reasons for changes, especially for test removal --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #2068 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/kopia/inject/inject.go | 2 +- src/internal/kopia/upload.go | 47 ++++- src/internal/kopia/upload_test.go | 39 ++-- src/internal/kopia/wrapper.go | 26 ++- src/internal/kopia/wrapper_test.go | 262 +++++++++++++++++++------ src/internal/operations/backup.go | 100 +--------- src/internal/operations/backup_test.go | 207 ++++++------------- 7 files changed, 337 insertions(+), 346 deletions(-) diff --git a/src/internal/kopia/inject/inject.go b/src/internal/kopia/inject/inject.go index 22ae0d429..5d8dd3bc7 100644 --- a/src/internal/kopia/inject/inject.go +++ b/src/internal/kopia/inject/inject.go @@ -16,7 +16,7 @@ type ( ConsumeBackupCollections( ctx context.Context, backupReasons []kopia.Reasoner, - bases []kopia.IncrementalBase, + bases kopia.BackupBases, cs []data.BackupCollection, pmr prefixmatcher.StringSetReader, tags map[string]string, diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 8a91367c6..8be75009f 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -20,6 +20,7 @@ import ( "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot/snapshotfs" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" @@ -970,10 +971,32 @@ func traverseBaseDir( return nil } +func logBaseInfo(ctx context.Context, m ManifestEntry) { + svcs := map[string]struct{}{} + cats := map[string]struct{}{} + + for _, r := range m.Reasons { + svcs[r.Service().String()] = struct{}{} + cats[r.Category().String()] = struct{}{} + } + + mbID, _ := m.GetTag(TagBackupID) + if len(mbID) == 0 { + mbID = "no_backup_id_tag" + } + + logger.Ctx(ctx).Infow( + "using base for backup", + "base_snapshot_id", m.ID, + "services", maps.Keys(svcs), + "categories", maps.Keys(cats), + "base_backup_id", mbID) +} + func inflateBaseTree( ctx context.Context, loader snapshotLoader, - snap IncrementalBase, + snap ManifestEntry, updatedPaths map[string]path.Path, roots map[string]*treeMap, ) error { @@ -996,13 +1019,25 @@ func inflateBaseTree( return clues.New("snapshot root is not a directory").WithClues(ctx) } + // Some logging to help track things. + logBaseInfo(ctx, snap) + // For each subtree corresponding to the tuple // (resource owner, service, category) merge the directories in the base with // what has been reported in the collections we got. - for _, subtreePath := range snap.SubtreePaths { + for _, r := range snap.Reasons { + ictx := clues.Add( + ctx, + "subtree_service", r.Service().String(), + "subtree_category", r.Category().String()) + + subtreePath, err := r.SubtreePath() + if err != nil { + return clues.Wrap(err, "building subtree path").WithClues(ictx) + } + // We're starting from the root directory so don't need it in the path. pathElems := encodeElements(subtreePath.PopFront().Elements()...) - ictx := clues.Add(ctx, "subtree_path", subtreePath) ent, err := snapshotfs.GetNestedEntry(ictx, dir, pathElems) if err != nil { @@ -1022,7 +1057,7 @@ func inflateBaseTree( // This ensures that a migration on the directory prefix can complete. // The prefix is the tenant/service/owner/category set, which remains // otherwise unchecked in tree inflation below this point. - newSubtreePath := subtreePath + newSubtreePath := subtreePath.ToBuilder() if p, ok := updatedPaths[subtreePath.String()]; ok { newSubtreePath = p.ToBuilder() } @@ -1031,7 +1066,7 @@ func inflateBaseTree( ictx, 0, updatedPaths, - subtreePath.Dir(), + subtreePath.ToBuilder().Dir(), newSubtreePath.Dir(), subtreeDir, roots, @@ -1059,7 +1094,7 @@ func inflateBaseTree( func inflateDirTree( ctx context.Context, loader snapshotLoader, - baseSnaps []IncrementalBase, + baseSnaps []ManifestEntry, collections []data.BackupCollection, globalExcludeSet prefixmatcher.StringSetReader, progress *corsoProgress, diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index 0ac10ec6b..bbdbe9e6f 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -946,21 +946,22 @@ func (msw *mockSnapshotWalker) SnapshotRoot(*snapshot.Manifest) (fs.Entry, error return msw.snapshotRoot, nil } -func mockIncrementalBase( +func makeManifestEntry( id, tenant, resourceOwner string, service path.ServiceType, categories ...path.CategoryType, -) IncrementalBase { - stps := []*path.Builder{} +) ManifestEntry { + var reasons []Reasoner + for _, c := range categories { - stps = append(stps, path.Builder{}.Append(tenant, service.String(), resourceOwner, c.String())) + reasons = append(reasons, NewReason(tenant, resourceOwner, service, c)) } - return IncrementalBase{ + return ManifestEntry{ Manifest: &snapshot.Manifest{ ID: manifest.ID(id), }, - SubtreePaths: stps, + Reasons: reasons, } } @@ -1331,8 +1332,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() { dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, test.inputCollections(), pmMock.NewPrefixMap(nil), @@ -2260,8 +2261,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, test.inputCollections(t), ie, @@ -2425,8 +2426,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, collections, pmMock.NewPrefixMap(nil), @@ -2531,8 +2532,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_HandleEmptyBase() dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, collections, pmMock.NewPrefixMap(nil), @@ -2782,9 +2783,9 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("id1", testTenant, testUser, path.ExchangeService, path.ContactsCategory), - mockIncrementalBase("id2", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("id1", testTenant, testUser, path.ExchangeService, path.ContactsCategory), + makeManifestEntry("id2", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, collections, pmMock.NewPrefixMap(nil), @@ -2948,8 +2949,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsMigrateSubt dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("id1", testTenant, testUser, path.ExchangeService, path.EmailCategory, path.ContactsCategory), + []ManifestEntry{ + makeManifestEntry("id1", testTenant, testUser, path.ExchangeService, path.EmailCategory, path.ContactsCategory), }, []data.BackupCollection{mce, mcc}, pmMock.NewPrefixMap(nil), diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 06f81c635..3c6854ece 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -143,7 +143,7 @@ type IncrementalBase struct { func (w Wrapper) ConsumeBackupCollections( ctx context.Context, backupReasons []Reasoner, - previousSnapshots []IncrementalBase, + bases BackupBases, collections []data.BackupCollection, globalExcludeSet prefixmatcher.StringSetReader, additionalTags map[string]string, @@ -172,15 +172,23 @@ func (w Wrapper) ConsumeBackupCollections( // When running an incremental backup, we need to pass the prior // snapshot bases into inflateDirTree so that the new snapshot // includes historical data. - var base []IncrementalBase - if buildTreeWithBase { - base = previousSnapshots + var ( + mergeBase []ManifestEntry + assistBase []ManifestEntry + ) + + if bases != nil { + if buildTreeWithBase { + mergeBase = bases.MergeBases() + } + + assistBase = bases.AssistBases() } dirTree, err := inflateDirTree( ctx, w.c, - base, + mergeBase, collections, globalExcludeSet, progress) @@ -203,7 +211,7 @@ func (w Wrapper) ConsumeBackupCollections( s, err := w.makeSnapshotWithRoot( ctx, - previousSnapshots, + assistBase, dirTree, tags, progress) @@ -216,7 +224,7 @@ func (w Wrapper) ConsumeBackupCollections( func (w Wrapper) makeSnapshotWithRoot( ctx context.Context, - prevSnapEntries []IncrementalBase, + prevSnapEntries []ManifestEntry, root fs.Directory, addlTags map[string]string, progress *corsoProgress, @@ -236,8 +244,8 @@ func (w Wrapper) makeSnapshotWithRoot( ctx = clues.Add( ctx, - "len_prev_base_snapshots", len(prevSnapEntries), - "assist_snap_ids", snapIDs, + "num_assist_snapshots", len(prevSnapEntries), + "assist_snapshot_ids", snapIDs, "additional_tags", addlTags) if len(snapIDs) > 0 { diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 733cdaadd..5014e07c1 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -696,6 +696,24 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { 42), } + c1 := exchMock.NewCollection( + suite.storePath1, + suite.locPath1, + 0) + c1.ColState = data.NotMovedState + c1.PrevPath = suite.storePath1 + + c2 := exchMock.NewCollection( + suite.storePath2, + suite.locPath2, + 0) + c2.ColState = data.NotMovedState + c2.PrevPath = suite.storePath2 + + // Make empty collections at the same locations to force a backup with no + // changes. Needed to ensure we force a backup even if nothing has changed. + emptyCollections := []data.BackupCollection{c1, c2} + // tags that are supplied by the caller. This includes basic tags to support // lookups and extra tags the caller may want to apply. tags := map[string]string{ @@ -730,86 +748,219 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { expectedTags = normalizeTagKVs(expectedTags) - table := []struct { + type testCase struct { name string + baseBackups func(base ManifestEntry) BackupBases + collections []data.BackupCollection expectedUploadedFiles int expectedCachedFiles int + // We're either going to get details entries or entries in the details + // merger. Details is populated when there's entries in the collection. The + // details merger is populated for cached entries. The details merger + // doesn't count folders, only items. + // + // Setting this to true looks for details merger entries. Setting it to + // false looks for details entries. + expectMerge bool // Whether entries in the resulting details should be marked as updated. - deetsUpdated bool - }{ - { - name: "Uncached", - expectedUploadedFiles: 47, - expectedCachedFiles: 0, - deetsUpdated: true, - }, - { - name: "Cached", - expectedUploadedFiles: 0, - expectedCachedFiles: 47, - deetsUpdated: false, - }, + deetsUpdated assert.BoolAssertionFunc + hashedBytesCheck assert.ValueAssertionFunc + // Range of bytes (inclusive) to expect as uploaded. A little fragile, but + // allows us to differentiate between content that wasn't uploaded due to + // being cached/deduped/skipped due to existing dir entries and stuff that + // was actually pushed to S3. + uploadedBytes []int64 } - prevSnaps := []IncrementalBase{} + // Initial backup. All files should be considered new by kopia. + baseBackupCase := testCase{ + name: "Uncached", + baseBackups: func(ManifestEntry) BackupBases { + return NewMockBackupBases() + }, + collections: collections, + expectedUploadedFiles: 47, + expectedCachedFiles: 0, + deetsUpdated: assert.True, + hashedBytesCheck: assert.NotZero, + uploadedBytes: []int64{8000, 10000}, + } + + runAndTestBackup := func(test testCase, base ManifestEntry) ManifestEntry { + var res ManifestEntry - for _, test := range table { suite.Run(test.name, func() { t := suite.T() - stats, deets, _, err := suite.w.ConsumeBackupCollections( - suite.ctx, + ctx, flush := tester.NewContext(t) + defer flush() + + bbs := test.baseBackups(base) + + stats, deets, deetsMerger, err := suite.w.ConsumeBackupCollections( + ctx, reasons, - prevSnaps, - collections, + bbs, + test.collections, nil, tags, true, fault.New(true)) - assert.NoError(t, err, clues.ToCore(err)) + require.NoError(t, err, clues.ToCore(err)) assert.Equal(t, test.expectedUploadedFiles, stats.TotalFileCount, "total files") assert.Equal(t, test.expectedUploadedFiles, stats.UncachedFileCount, "uncached files") assert.Equal(t, test.expectedCachedFiles, stats.CachedFileCount, "cached files") - assert.Equal(t, 6, stats.TotalDirectoryCount) + assert.Equal(t, 4+len(test.collections), stats.TotalDirectoryCount, "directory count") assert.Equal(t, 0, stats.IgnoredErrorCount) assert.Equal(t, 0, stats.ErrorCount) assert.False(t, stats.Incomplete) - - // 47 file and 2 folder entries. - details := deets.Details().Entries - assert.Len( + test.hashedBytesCheck(t, stats.TotalHashedBytes, "hashed bytes") + assert.LessOrEqual( t, - details, - test.expectedUploadedFiles+test.expectedCachedFiles+2, - ) + test.uploadedBytes[0], + stats.TotalUploadedBytes, + "low end of uploaded bytes") + assert.GreaterOrEqual( + t, + test.uploadedBytes[1], + stats.TotalUploadedBytes, + "high end of uploaded bytes") - for _, entry := range details { - assert.Equal(t, test.deetsUpdated, entry.Updated) + if test.expectMerge { + assert.Empty(t, deets.Details().Entries, "details entries") + assert.Equal( + t, + test.expectedUploadedFiles+test.expectedCachedFiles, + deetsMerger.ItemsToMerge(), + "details merger entries") + } else { + assert.Zero(t, deetsMerger.ItemsToMerge(), "details merger entries") + + details := deets.Details().Entries + assert.Len( + t, + details, + // 47 file and 2 folder entries. + test.expectedUploadedFiles+test.expectedCachedFiles+2, + ) + + for _, entry := range details { + test.deetsUpdated(t, entry.Updated) + } } checkSnapshotTags( t, - suite.ctx, + ctx, suite.w.c, expectedTags, stats.SnapshotID, ) snap, err := snapshot.LoadSnapshot( - suite.ctx, + ctx, suite.w.c, manifest.ID(stats.SnapshotID), ) require.NoError(t, err, clues.ToCore(err)) - prevSnaps = append(prevSnaps, IncrementalBase{ + res = ManifestEntry{ Manifest: snap, - SubtreePaths: []*path.Builder{ - suite.storePath1.ToBuilder().Dir(), - }, - }) + Reasons: reasons, + } }) + + return res + } + + base := runAndTestBackup(baseBackupCase, ManifestEntry{}) + + table := []testCase{ + { + name: "Kopia Assist And Merge All Files Changed", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithMergeBases(base) + }, + collections: collections, + expectedUploadedFiles: 0, + expectedCachedFiles: 47, + deetsUpdated: assert.False, + hashedBytesCheck: assert.Zero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Kopia Assist And Merge No Files Changed", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithMergeBases(base) + }, + // Pass in empty collections to force a backup. Otherwise we'll skip + // actually trying to do anything because we'll see there's nothing that + // changed. The real goal is to get it to deal with the merged collections + // again though. + collections: emptyCollections, + // Should hit cached check prior to dir entry check so we see them as + // cached. + expectedUploadedFiles: 0, + expectedCachedFiles: 47, + // Entries go into the details merger because we never materialize details + // info for the items since they're from the base. + expectMerge: true, + // Not used since there's no details entries. + deetsUpdated: assert.False, + hashedBytesCheck: assert.Zero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Kopia Assist Only", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithAssistBases(base) + }, + collections: collections, + expectedUploadedFiles: 0, + expectedCachedFiles: 47, + deetsUpdated: assert.False, + hashedBytesCheck: assert.Zero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Merge Only", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithMergeBases(base).ClearMockAssistBases() + }, + // Pass in empty collections to force a backup. Otherwise we'll skip + // actually trying to do anything because we'll see there's nothing that + // changed. The real goal is to get it to deal with the merged collections + // again though. + collections: emptyCollections, + expectedUploadedFiles: 47, + expectedCachedFiles: 0, + expectMerge: true, + // Not used since there's no details entries. + deetsUpdated: assert.False, + // Kopia still counts these bytes as "hashed" even though it shouldn't + // read the file data since they already have dir entries it can reuse. + hashedBytesCheck: assert.NotZero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Content Hash Only", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases() + }, + collections: collections, + expectedUploadedFiles: 47, + expectedCachedFiles: 0, + // Marked as updated because we still fall into the uploadFile handler in + // kopia instead of the cachedFile handler. + deetsUpdated: assert.True, + hashedBytesCheck: assert.NotZero, + uploadedBytes: []int64{4000, 6000}, + }, + } + + for _, test := range table { + runAndTestBackup(test, base) } } @@ -938,7 +1089,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { }, } - prevSnaps := []IncrementalBase{} + prevSnaps := NewMockBackupBases() for _, test := range table { suite.Run(test.name, func() { @@ -1000,12 +1151,12 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { manifest.ID(stats.SnapshotID)) require.NoError(t, err, clues.ToCore(err)) - prevSnaps = append(prevSnaps, IncrementalBase{ - Manifest: snap, - SubtreePaths: []*path.Builder{ - storePath.ToBuilder().Dir(), + prevSnaps.WithMergeBases( + ManifestEntry{ + Manifest: snap, + Reasons: reasons, }, - }) + ) }) } } @@ -1424,17 +1575,6 @@ func (c *i64counter) Count(i int64) { func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) - subtreePathTmp, err := path.Build( - testTenant, - testUser, - path.ExchangeService, - path.EmailCategory, - false, - "tmp") - require.NoError(suite.T(), err, clues.ToCore(err)) - - subtreePath := subtreePathTmp.ToBuilder().Dir() - man, err := suite.w.c.LoadSnapshot(suite.ctx, suite.snapshotID) require.NoError(suite.T(), err, "getting base snapshot: %v", clues.ToCore(err)) @@ -1527,14 +1667,12 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { stats, _, _, err := suite.w.ConsumeBackupCollections( suite.ctx, []Reasoner{r}, - []IncrementalBase{ - { + NewMockBackupBases().WithMergeBases( + ManifestEntry{ Manifest: man, - SubtreePaths: []*path.Builder{ - subtreePath, - }, + Reasons: []Reasoner{r}, }, - }, + ), test.cols(), excluded, nil, diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 98ceab012..82ae79fb6 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -6,7 +6,6 @@ import ( "github.com/alcionai/clues" "github.com/google/uuid" - "github.com/kopia/kopia/repo/manifest" "github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/common/dttm" @@ -449,26 +448,6 @@ func selectorToReasons( return reasons } -func builderFromReason(ctx context.Context, tenant string, r kopia.Reasoner) (*path.Builder, error) { - ctx = clues.Add(ctx, "category", r.Category().String()) - - // This is hacky, but we want the path package to format the path the right - // way (e.x. proper order for service, category, etc), but we don't care about - // the folders after the prefix. - p, err := path.Build( - tenant, - r.ProtectedResource(), - r.Service(), - r.Category(), - false, - "tmp") - if err != nil { - return nil, clues.Wrap(err, "building path").WithClues(ctx) - } - - return p.ToBuilder().Dir(), nil -} - // calls kopia to backup the collections of data func consumeBackupCollections( ctx context.Context, @@ -495,85 +474,10 @@ func consumeBackupCollections( kopia.TagBackupCategory: "", } - // AssistBases should be the upper bound for how many snapshots we pass in. - bases := make([]kopia.IncrementalBase, 0, len(bbs.AssistBases())) - // Track IDs we've seen already so we don't accidentally duplicate some - // manifests. This can be removed when we move the code below into the kopia - // package. - ids := map[manifest.ID]struct{}{} - - var mb []kopia.ManifestEntry - - if bbs != nil { - mb = bbs.MergeBases() - } - - // TODO(ashmrtn): Make a wrapper for Reson that allows adding a tenant and - // make a function that will spit out a prefix that includes the tenant. With - // that done this code can be moved to kopia wrapper since it's really more - // specific to that. - for _, m := range mb { - paths := make([]*path.Builder, 0, len(m.Reasons)) - services := map[string]struct{}{} - categories := map[string]struct{}{} - - for _, reason := range m.Reasons { - pb, err := builderFromReason(ctx, tenantID, reason) - if err != nil { - return nil, nil, nil, clues.Wrap(err, "getting subtree paths for bases") - } - - paths = append(paths, pb) - services[reason.Service().String()] = struct{}{} - categories[reason.Category().String()] = struct{}{} - } - - ids[m.ID] = struct{}{} - - bases = append(bases, kopia.IncrementalBase{ - Manifest: m.Manifest, - SubtreePaths: paths, - }) - - svcs := make([]string, 0, len(services)) - for k := range services { - svcs = append(svcs, k) - } - - cats := make([]string, 0, len(categories)) - for k := range categories { - cats = append(cats, k) - } - - mbID, ok := m.GetTag(kopia.TagBackupID) - if !ok { - mbID = "no_backup_id_tag" - } - - logger.Ctx(ctx).Infow( - "using base for backup", - "base_snapshot_id", m.ID, - "services", svcs, - "categories", cats, - "base_backup_id", mbID) - } - - // At the moment kopia assisted snapshots are in the same set as merge bases. - // When we fixup generating subtree paths we can remove this. - if bbs != nil { - for _, ab := range bbs.AssistBases() { - if _, ok := ids[ab.ID]; ok { - continue - } - - bases = append(bases, kopia.IncrementalBase{Manifest: ab.Manifest}) - } - } - kopiaStats, deets, itemsSourcedFromBase, err := bc.ConsumeBackupCollections( ctx, reasons, - bases, + bbs, cs, pmr, tags, @@ -581,7 +485,7 @@ func consumeBackupCollections( errs) if err != nil { if kopiaStats == nil { - return nil, nil, nil, err + return nil, nil, nil, clues.Stack(err) } return nil, nil, nil, clues.Stack(err).With( diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index f15f88f02..3aaeae45c 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -108,7 +108,7 @@ func checkPaths(t *testing.T, expected, got []path.Path) { type mockBackupConsumer struct { checkFunc func( backupReasons []kopia.Reasoner, - bases []kopia.IncrementalBase, + bases kopia.BackupBases, cs []data.BackupCollection, tags map[string]string, buildTreeWithBase bool) @@ -117,7 +117,7 @@ type mockBackupConsumer struct { func (mbu mockBackupConsumer) ConsumeBackupCollections( ctx context.Context, backupReasons []kopia.Reasoner, - bases []kopia.IncrementalBase, + bases kopia.BackupBases, cs []data.BackupCollection, excluded prefixmatcher.StringSetReader, tags map[string]string, @@ -390,179 +390,84 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() { func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections_Paths() { var ( + t = suite.T() + tenant = "a-tenant" resourceOwner = "a-user" - emailBuilder = path.Builder{}.Append( - tenant, - path.ExchangeService.String(), - resourceOwner, - path.EmailCategory.String()) - contactsBuilder = path.Builder{}.Append( - tenant, - path.ExchangeService.String(), - resourceOwner, - path.ContactsCategory.String()) - emailReason = kopia.NewReason( - "", + tenant, resourceOwner, path.ExchangeService, path.EmailCategory) contactsReason = kopia.NewReason( - "", + tenant, resourceOwner, path.ExchangeService, path.ContactsCategory) + reasons = []kopia.Reasoner{ + emailReason, + contactsReason, + } + manifest1 = &snapshot.Manifest{ ID: "id1", } manifest2 = &snapshot.Manifest{ ID: "id2", } + + bases = kopia.NewMockBackupBases().WithMergeBases( + kopia.ManifestEntry{ + Manifest: manifest1, + Reasons: []kopia.Reasoner{ + emailReason, + }, + }).WithAssistBases( + kopia.ManifestEntry{ + Manifest: manifest2, + Reasons: []kopia.Reasoner{ + contactsReason, + }, + }) + + backupID = model.StableID("foo") + expectedTags = map[string]string{ + kopia.TagBackupID: string(backupID), + kopia.TagBackupCategory: "", + } ) - table := []struct { - name string - // Backup model is untouched in this test so there's no need to populate it. - input kopia.BackupBases - expected []kopia.IncrementalBase - }{ - { - name: "SingleManifestSingleReason", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reasoner{ - emailReason, - }, - }).ClearMockAssistBases(), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - }, - }, - }, - }, - { - name: "SingleManifestMultipleReasons", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reasoner{ - emailReason, - contactsReason, - }, - }).ClearMockAssistBases(), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - contactsBuilder, - }, - }, - }, - }, - { - name: "MultipleManifestsMultipleReasons", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reasoner{ - emailReason, - contactsReason, - }, - }, - kopia.ManifestEntry{ - Manifest: manifest2, - Reasons: []kopia.Reasoner{ - emailReason, - contactsReason, - }, - }).ClearMockAssistBases(), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - contactsBuilder, - }, - }, - { - Manifest: manifest2, - SubtreePaths: []*path.Builder{ - emailBuilder, - contactsBuilder, - }, - }, - }, - }, - { - name: "Single Manifest Single Reason With Assist Base", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reasoner{ - emailReason, - }, - }).WithAssistBases( - kopia.ManifestEntry{ - Manifest: manifest2, - Reasons: []kopia.Reasoner{ - contactsReason, - }, - }), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - }, - }, - { - Manifest: manifest2, - }, - }, + mbu := &mockBackupConsumer{ + checkFunc: func( + backupReasons []kopia.Reasoner, + gotBases kopia.BackupBases, + cs []data.BackupCollection, + gotTags map[string]string, + buildTreeWithBase bool, + ) { + kopia.AssertBackupBasesEqual(t, bases, gotBases) + assert.Equal(t, expectedTags, gotTags) + assert.ElementsMatch(t, reasons, backupReasons) }, } - for _, test := range table { - suite.Run(test.name, func() { - t := suite.T() + ctx, flush := tester.NewContext(t) + defer flush() - ctx, flush := tester.NewContext(t) - defer flush() - - mbu := &mockBackupConsumer{ - checkFunc: func( - backupReasons []kopia.Reasoner, - bases []kopia.IncrementalBase, - cs []data.BackupCollection, - tags map[string]string, - buildTreeWithBase bool, - ) { - assert.ElementsMatch(t, test.expected, bases) - }, - } - - //nolint:errcheck - consumeBackupCollections( - ctx, - mbu, - tenant, - nil, - test.input, - nil, - nil, - model.StableID(""), - true, - fault.New(true)) - }) - } + //nolint:errcheck + consumeBackupCollections( + ctx, + mbu, + tenant, + reasons, + bases, + nil, + nil, + backupID, + true, + fault.New(true)) } func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() { From f7acf97489ba3aa69434747c86d06e32b61c87be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 01:46:20 +0000 Subject: [PATCH 27/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.305=20to=201.44.306=20in=20/src=20(#?= =?UTF-8?q?3887)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.305 to 1.44.306.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.306 (2023-07-21)

Service Client Updates

  • service/glue: Updates service API and documentation
    • This release adds support for AWS Glue Crawler with Apache Hudi Tables, allowing Crawlers to discover Hudi Tables in S3 and register them in Glue Data Catalog for query engines to query against.
  • service/mediaconvert: Updates service documentation
    • This release includes improvements to Preserve 444 handling, compatibility of HEVC sources without frame rates, and general improvements to MP4 outputs.
  • service/rds: Updates service API, documentation, waiters, paginators, and examples
    • Adds support for the DBSystemID parameter of CreateDBInstance to RDS Custom for Oracle.
  • service/workspaces: Updates service documentation
    • Fixed VolumeEncryptionKey descriptions

SDK Bugs

  • codegen: Prevent unused imports from being generated for event streams.
    • Potentially-unused "time" import was causing vet failures on generated code.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.305&new-version=1.44.306)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index b589ca542..03cf9efff 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.305 + github.com/aws/aws-sdk-go v1.44.306 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 718084162..40adca2f6 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.305 h1:fU/5lY3WyBjGU9fkmQYd8o4fZu+2RaOv/i+sPaJVvFg= -github.com/aws/aws-sdk-go v1.44.305/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.306 h1:H487V/1N09BDxeGR7oR+LloC2uUpmf4atmqJaBgQOIs= +github.com/aws/aws-sdk-go v1.44.306/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From bc205ec774c8caaeeffc7a4b8f28bef77da7554f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 06:07:14 +0000 Subject: [PATCH 28/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20sass=20from?= =?UTF-8?q?=201.64.0=20to=201.64.1=20in=20/website=20(#3899)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sass](https://github.com/sass/dart-sass) from 1.64.0 to 1.64.1.
Release notes

Sourced from sass's releases.

Dart Sass 1.64.1

To install Sass 1.64.1, download one of the packages below and add it to your PATH, or see the Sass website for full installation instructions.

Changes

Embedded Sass

  • Fix a bug where a valid SassCalculation.clamp() with less than 3 arguments would throw an error.

See the full changelog for changes in earlier releases.

Changelog

Sourced from sass's changelog.

1.64.1

Embedded Sass

  • Fix a bug where a valid SassCalculation.clamp() with less than 3 arguments would throw an error.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sass&package-manager=npm_and_yarn&previous-version=1.64.0&new-version=1.64.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 14 +++++++------- website/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index a897f25bc..54b366dae 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -24,7 +24,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.64.0", + "sass": "^1.64.1", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" @@ -12571,9 +12571,9 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.64.0", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.0.tgz", - "integrity": "sha512-m7YtAGmQta9uANIUJwXesAJMSncqH+3INc8kdVXs6eV6GUC8Qu2IYKQSN8PRLgiQfpca697G94klm2leYMxSHw==", + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.1.tgz", + "integrity": "sha512-16rRACSOFEE8VN7SCgBu1MpYCyN7urj9At898tyzdXFhC+a+yOX5dXwAR7L8/IdPJ1NB8OYoXmD55DM30B2kEQ==", "dependencies": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -23802,9 +23802,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sass": { - "version": "1.64.0", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.0.tgz", - "integrity": "sha512-m7YtAGmQta9uANIUJwXesAJMSncqH+3INc8kdVXs6eV6GUC8Qu2IYKQSN8PRLgiQfpca697G94klm2leYMxSHw==", + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.1.tgz", + "integrity": "sha512-16rRACSOFEE8VN7SCgBu1MpYCyN7urj9At898tyzdXFhC+a+yOX5dXwAR7L8/IdPJ1NB8OYoXmD55DM30B2kEQ==", "requires": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", diff --git a/website/package.json b/website/package.json index 58031eb02..14becab38 100644 --- a/website/package.json +++ b/website/package.json @@ -30,7 +30,7 @@ "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.64.0", + "sass": "^1.64.1", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" From 8fbe4ea32655b4c34f7e72b1ee7cbb8d43027308 Mon Sep 17 00:00:00 2001 From: neha-Gupta1 Date: Tue, 25 Jul 2023 13:29:54 +0530 Subject: [PATCH 29/62] use groups API --- src/pkg/services/m365/api/teams.go | 94 +++++++++++++++++++------ src/pkg/services/m365/api/teams_test.go | 26 ++++--- 2 files changed, 88 insertions(+), 32 deletions(-) diff --git a/src/pkg/services/m365/api/teams.go b/src/pkg/services/m365/api/teams.go index 883ddee01..8eef3fba3 100644 --- a/src/pkg/services/m365/api/teams.go +++ b/src/pkg/services/m365/api/teams.go @@ -7,8 +7,16 @@ import ( msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/str" + "github.com/alcionai/corso/src/internal/common/tform" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" +) + +const ( + teamsAdditionalDataLabel = "Team" + ResourceProvisioningOptions = "resourceProvisioningOptions" ) // --------------------------------------------------------------------------- @@ -19,74 +27,118 @@ func (c Client) Teams() Teams { return Teams{c} } +// On creation of each Teams team a corrsponding group gets created. +// The group acts as the protected resource, and all teams data like events, +// drive and mail messages are owned by that group. + // Teams is an interface-compliant provider of the client. type Teams struct { Client } -// GetAllTeams retrieves all teams. +// GetAllTeams retrieves all groups. func (c Teams) GetAll( ctx context.Context, errs *fault.Bus, -) ([]models.Teamable, error) { +) ([]models.Groupable, error) { service, err := c.Service() if err != nil { return nil, err } - resp, err := service.Client().Teams().Get(ctx, nil) + return getGroups(ctx, true, errs, service) +} + +// GetAll retrieves all groups. +func getGroups( + ctx context.Context, + getOnlyTeams bool, + errs *fault.Bus, + service graph.Servicer, +) ([]models.Groupable, error) { + resp, err := service.Client().Groups().Get(ctx, nil) if err != nil { - return nil, graph.Wrap(ctx, err, "getting all teams") + return nil, graph.Wrap(ctx, err, "getting all groups") } - iter, err := msgraphgocore.NewPageIterator[models.Teamable]( + iter, err := msgraphgocore.NewPageIterator[models.Groupable]( resp, service.Adapter(), models.CreateTeamCollectionResponseFromDiscriminatorValue) if err != nil { - return nil, graph.Wrap(ctx, err, "creating teams iterator") + return nil, graph.Wrap(ctx, err, "creating groups iterator") } var ( - teams = make([]models.Teamable, 0) - el = errs.Local() + groups = make([]models.Groupable, 0) + el = errs.Local() ) - iterator := func(item models.Teamable) bool { + iterator := func(item models.Groupable) bool { if el.Failure() != nil { return false } - err := ValidateTeams(item) + err := ValidateGroup(item) if err != nil { - el.AddRecoverable(ctx, graph.Wrap(ctx, err, "validating teams")) + el.AddRecoverable(ctx, graph.Wrap(ctx, err, "validating groups")) } else { - teams = append(teams, item) + isTeam := IsTeam(ctx, item) + if !getOnlyTeams || isTeam { + groups = append(groups, item) + } } return true } if err := iter.Iterate(ctx, iterator); err != nil { - return nil, graph.Wrap(ctx, err, "iterating all teams") + return nil, graph.Wrap(ctx, err, "iterating all groups") } - return teams, el.Failure() + return groups, el.Failure() } -// GetID retrieves team by teamID. +func IsTeam(ctx context.Context, g models.Groupable) bool { + log := logger.Ctx(ctx) + + if g.GetAdditionalData()[ResourceProvisioningOptions] != nil { + val, _ := tform.AnyValueToT[[]any](ResourceProvisioningOptions, g.GetAdditionalData()) + for _, v := range val { + s, err := str.AnyToString(v) + if err != nil { + log.Debug("could not be converted to string value: ", ResourceProvisioningOptions) + return false + } + + if s == teamsAdditionalDataLabel { + return true + } + } + } + + return false +} + +// GetID retrieves team by groupID/teamID. func (c Teams) GetByID( ctx context.Context, identifier string, -) (models.Teamable, error) { +) (models.Groupable, error) { service, err := c.Service() if err != nil { return nil, err } - resp, err := service.Client().Teams().ByTeamId(identifier).Get(ctx, nil) + resp, err := service.Client().Groups().ByGroupId(identifier).Get(ctx, nil) if err != nil { - err := graph.Wrap(ctx, err, "getting team by id") + err := graph.Wrap(ctx, err, "getting group by id") + + return nil, err + } + + if !IsTeam(ctx, resp) { + err := clues.New("given teamID is not related to any team") return nil, err } @@ -98,9 +150,9 @@ func (c Teams) GetByID( // helpers // --------------------------------------------------------------------------- -// ValidateTeams ensures the item is a Teamable, and contains the necessary -// identifiers that we handle with all teams. -func ValidateTeams(item models.Teamable) error { +// ValidateGroup ensures the item is a Groupable, and contains the necessary +// identifiers that we handle with all groups. +func ValidateGroup(item models.Groupable) error { if item.GetId() == nil { return clues.New("missing ID") } diff --git a/src/pkg/services/m365/api/teams_test.go b/src/pkg/services/m365/api/teams_test.go index 89aab93b5..dcb039dc5 100644 --- a/src/pkg/services/m365/api/teams_test.go +++ b/src/pkg/services/m365/api/teams_test.go @@ -25,21 +25,21 @@ func TestTeamsUnitSuite(t *testing.T) { suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *TeamsUnitSuite) TestValidateTeams() { +func (suite *TeamsUnitSuite) TestValidateGroup() { team := models.NewTeam() - team.SetDisplayName(ptr.To("testteam")) + team.SetDisplayName(ptr.To("testgroup")) team.SetId(ptr.To("testID")) tests := []struct { name string - args models.Teamable + args models.Groupable errCheck assert.ErrorAssertionFunc errIsSkippable bool }{ { - name: "Valid Team", - args: func() *models.Team { - s := models.NewTeam() + name: "Valid group ", + args: func() *models.Group { + s := models.NewGroup() s.SetId(ptr.To("id")) s.SetDisplayName(ptr.To("testTeam")) return s @@ -48,8 +48,8 @@ func (suite *TeamsUnitSuite) TestValidateTeams() { }, { name: "No name", - args: func() *models.Team { - s := models.NewTeam() + args: func() *models.Group { + s := models.NewGroup() s.SetId(ptr.To("id")) return s }(), @@ -57,8 +57,8 @@ func (suite *TeamsUnitSuite) TestValidateTeams() { }, { name: "No ID", - args: func() *models.Team { - s := models.NewTeam() + args: func() *models.Group { + s := models.NewGroup() s.SetDisplayName(ptr.To("testTeam")) return s }(), @@ -70,7 +70,7 @@ func (suite *TeamsUnitSuite) TestValidateTeams() { suite.Run(test.name, func() { t := suite.T() - err := api.ValidateTeams(test.args) + err := api.ValidateGroup(test.args) test.errCheck(t, err, clues.ToCore(err)) if test.errIsSkippable { @@ -108,6 +108,10 @@ func (suite *TeamsIntgSuite) TestGetAllTeams() { GetAll(ctx, fault.New(true)) require.NoError(t, err) require.NotZero(t, len(teams), "must have at least one team") + + for _, team := range teams { + assert.True(t, api.IsTeam(ctx, team), "must not return non teams groups") + } } func (suite *TeamsIntgSuite) TestTeams_GetByID() { From d3dc0e8fdc94f2433013bee278f13811a4f7ce0b Mon Sep 17 00:00:00 2001 From: neha_gupta Date: Tue, 25 Jul 2023 13:32:05 +0530 Subject: [PATCH 30/62] expose bucket name (#3900) Export Bucket name for CI Nightly test. #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [ ] :computer: CI/Deployment #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- .github/workflows/nightly_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index 2ea556099..cc18732c5 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -94,6 +94,7 @@ jobs: CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} CORSO_LOG_FILE: ${{ github.workspace }}/testlog/run-nightly.log LOG_GRAPH_REQUESTS: true + S3_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }} run: | set -euo pipefail go test \ From 0451df12916e519ea4340647078e81cecbc8632d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:55:05 +0000 Subject: [PATCH 31/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.306=20to=201.44.307=20in=20/src=20(#?= =?UTF-8?q?3898)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.306 to 1.44.307.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.307 (2023-07-24)

Service Client Updates

  • service/apigatewayv2: Adds new service
    • Documentation updates for Amazon API Gateway.
  • service/ce: Updates service API and documentation
  • service/chime-sdk-media-pipelines: Updates service API and documentation
  • service/cloudformation: Updates service API and documentation
    • This release supports filtering by DRIFT_STATUS for existing API ListStackInstances and adds support for a new API ListStackInstanceResourceDrifts. Customers can now view resource drift information from their StackSet management accounts.
  • service/ec2: Updates service API
    • Add "disabled" enum value to SpotInstanceState.
  • service/glue: Updates service API and documentation
    • Added support for Data Preparation Recipe node in Glue Studio jobs
  • service/quicksight: Updates service API, documentation, and paginators
    • This release launches new Snapshot APIs for CSV and PDF exports, adds support for info icon for filters and parameters in Exploration APIs, adds modeled exception to the DeleteAccountCustomization API, and introduces AttributeAggregationFunction's ability to add UNIQUE_VALUE aggregation in tooltips.
Commits
  • 0b6a21f Release v1.44.307 (2023-07-24) (#4926)
  • 03554b1 Merge pull request #4911 from khushail/khushail/handleStalediscussions
  • 3520bfe added workflow for handling answerable discussions
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.306&new-version=1.44.307)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 03cf9efff..f55290ae2 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.306 + github.com/aws/aws-sdk-go v1.44.307 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 40adca2f6..08c93b127 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.306 h1:H487V/1N09BDxeGR7oR+LloC2uUpmf4atmqJaBgQOIs= -github.com/aws/aws-sdk-go v1.44.306/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.307 h1:2R0/EPgpZcFSUwZhYImq/srjaOrOfLv5MNRzrFyAM38= +github.com/aws/aws-sdk-go v1.44.307/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From c75ca7e41c7d4c8e5c3930da05e5ef1f50e8974b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 16:45:22 +0000 Subject: [PATCH 32/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/mi?= =?UTF-8?q?crosoft/kiota-http-go=20from=201.0.0=20to=201.0.1=20in=20/src?= =?UTF-8?q?=20(#3897)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/microsoft/kiota-http-go](https://github.com/microsoft/kiota-http-go) from 1.0.0 to 1.0.1.
Release notes

Sourced from github.com/microsoft/kiota-http-go's releases.

v1.0.1

Changed

  • Bug Fix: Update Host for Redirect URL in go client.
Changelog

Sourced from github.com/microsoft/kiota-http-go's changelog.

[1.0.1] - 2023-07-19

Changed

  • Bug Fix: Update Host for Redirect URL in go client.
Commits
  • 03f625b Fixes redirect host URL (#98)
  • 53e6b69 Merge pull request #96 from microsoft/dependabot/go_modules/github.com/micros...
  • d37a8a9 Bump github.com/microsoft/kiota-abstractions-go from 1.0.0 to 1.1.0
  • def53fa Merge pull request #95 from microsoft/dependabot/github_actions/dependabot/fe...
  • 4258e27 Bump dependabot/fetch-metadata from 1.5.1 to 1.6.0
  • a671c2b Merge pull request #94 from microsoft/dependabot/go_modules/github.com/stretc...
  • ecb3cbc Bump github.com/stretchr/testify from 1.8.3 to 1.8.4
  • 5c3d188 Disable sonar cloud checks on forks (#92)
  • 78a52a2 Merge pull request #93 from microsoft/dependabot/github_actions/dependabot/fe...
  • 973aef3 Bump dependabot/fetch-metadata from 1.5.0 to 1.5.1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/microsoft/kiota-http-go&package-manager=go_modules&previous-version=1.0.0&new-version=1.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index f55290ae2..1c6091e91 100644 --- a/src/go.mod +++ b/src/go.mod @@ -16,7 +16,7 @@ require ( github.com/kopia/kopia v0.12.2-0.20230327171220-747baeebdab1 github.com/microsoft/kiota-abstractions-go v1.1.0 github.com/microsoft/kiota-authentication-azure-go v1.0.0 - github.com/microsoft/kiota-http-go v1.0.0 + github.com/microsoft/kiota-http-go v1.0.1 github.com/microsoft/kiota-serialization-form-go v1.0.0 github.com/microsoft/kiota-serialization-json-go v1.0.4 github.com/microsoftgraph/msgraph-sdk-go v1.12.0 diff --git a/src/go.sum b/src/go.sum index 08c93b127..cd66bd27b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -275,8 +275,8 @@ github.com/microsoft/kiota-abstractions-go v1.1.0 h1:X1aKlsYCRs/0RSChr/fbq4j/+kx github.com/microsoft/kiota-abstractions-go v1.1.0/go.mod h1:RkxyZ5x87Njik7iVeQY9M2wtrrL1MJZcXiI/BxD/82g= github.com/microsoft/kiota-authentication-azure-go v1.0.0 h1:29FNZZ/4nnCOwFcGWlB/sxPvWz487HA2bXH8jR5k2Rk= github.com/microsoft/kiota-authentication-azure-go v1.0.0/go.mod h1:rnx3PRlkGdXDcA/0lZQTbBwyYGmc+3POt7HpE/e4jGw= -github.com/microsoft/kiota-http-go v1.0.0 h1:F1hd6gMlLeEgH2CkRB7z13ow7LxMKMWEmms/t0VfS+k= -github.com/microsoft/kiota-http-go v1.0.0/go.mod h1:eujxJliqodotsYepIc6ihhK+vXMMt5Q8YiSNL7+7M7U= +github.com/microsoft/kiota-http-go v1.0.1 h1:818u3aiLpxj35hZgfUSqphQ18IUTK3gVdTE4cQ5vjLw= +github.com/microsoft/kiota-http-go v1.0.1/go.mod h1:H0cg+ly+5ZSR8z4swj5ea9O/GB5ll2YuYeQ0/pJs7AY= github.com/microsoft/kiota-serialization-form-go v1.0.0 h1:UNdrkMnLFqUCccQZerKjblsyVgifS11b3WCx+eFEsAI= github.com/microsoft/kiota-serialization-form-go v1.0.0/go.mod h1:h4mQOO6KVTNciMF6azi1J9QB19ujSw3ULKcSNyXXOMA= github.com/microsoft/kiota-serialization-json-go v1.0.4 h1:5TaISWwd2Me8clrK7SqNATo0tv9seOq59y4I5953egQ= From 92412645ecb644f1c7e4b8926b29d49179a1ae3e Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 25 Jul 2023 11:25:34 -0600 Subject: [PATCH 33/62] pii handling for restore config (#3896) add pii concealer compliance for restore config structs. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature --- src/internal/m365/restore.go | 2 +- src/pkg/control/restore.go | 67 ++++++++++++++++++++++++++++-- src/pkg/services/m365/api/sites.go | 8 ++-- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index 31e36e2bb..5d58fdb26 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -36,7 +36,7 @@ func (ctrl *Controller) ConsumeRestoreCollections( defer end() ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) - ctx = clues.Add(ctx, "restore_config", restoreCfg) // TODO(rkeepers): needs PII control + ctx = clues.Add(ctx, "restore_config", restoreCfg) if len(dcs) == 0 { return nil, clues.New("no data collections to restore") diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 2b4129d9f..79d49ae20 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -2,13 +2,17 @@ package control import ( "context" + "encoding/json" + "fmt" "strings" + "github.com/alcionai/clues" "golang.org/x/exp/maps" "golang.org/x/exp/slices" "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" ) const ( @@ -39,24 +43,24 @@ const RootLocation = "/" type RestoreConfig struct { // Defines the per-item collision handling policy. // Defaults to Skip. - OnCollision CollisionPolicy + OnCollision CollisionPolicy `json:"onCollision"` // ProtectedResource specifies which resource the data will be restored to. // If empty, restores to the same resource that was backed up. // Defaults to empty. - ProtectedResource string + ProtectedResource string `json:"protectedResource"` // Location specifies the container into which the data will be restored. // Only accepts container names, does not accept IDs. // If empty or "/", data will get restored in place, beginning at the root. // Defaults to "Corso_Restore_" - Location string + Location string `json:"location"` // Drive specifies the name of the drive into which the data will be // restored. If empty, data is restored to the same drive that was backed // up. // Defaults to empty. - Drive string + Drive string `json:"drive"` } func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { @@ -90,3 +94,58 @@ func EnsureRestoreConfigDefaults( return rc } + +// --------------------------------------------------------------------------- +// pii control +// --------------------------------------------------------------------------- + +var ( + // interface compliance required for handling PII + _ clues.Concealer = &RestoreConfig{} + _ fmt.Stringer = &RestoreConfig{} + + // interface compliance for the observe package to display + // values without concealing PII. + _ clues.PlainStringer = &RestoreConfig{} +) + +func (rc RestoreConfig) marshal() string { + bs, err := json.Marshal(rc) + if err != nil { + return "err marshalling" + } + + return string(bs) +} + +func (rc RestoreConfig) concealed() RestoreConfig { + return RestoreConfig{ + OnCollision: rc.OnCollision, + ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(), + Location: path.LoggableDir(rc.Location), + Drive: clues.Hide(rc.Drive).Conceal(), + } +} + +// Conceal produces a concealed representation of the config, suitable for +// logging, storing in errors, and other output. +func (rc RestoreConfig) Conceal() string { + return rc.concealed().marshal() +} + +// Format produces a concealed representation of the config, even when +// used within a PrintF, suitable for logging, storing in errors, +// and other output. +func (rc RestoreConfig) Format(fs fmt.State, _ rune) { + fmt.Fprint(fs, rc.concealed()) +} + +// String returns a plain text version of the restoreConfig. +func (rc RestoreConfig) String() string { + return rc.PlainString() +} + +// PlainString returns an unescaped, unmodified string of the restore configuration. +func (rc RestoreConfig) PlainString() string { + return rc.marshal() +} diff --git a/src/pkg/services/m365/api/sites.go b/src/pkg/services/m365/api/sites.go index e573cfc07..4e13ebcfb 100644 --- a/src/pkg/services/m365/api/sites.go +++ b/src/pkg/services/m365/api/sites.go @@ -225,13 +225,13 @@ func ValidateSite(item models.Siteable) error { wURL := ptr.Val(item.GetWebUrl()) if len(wURL) == 0 { - return clues.New("missing webURL").With("site_id", id) // TODO: pii + return clues.New("missing webURL").With("site_id", clues.Hide(id)) } // personal (ie: oneDrive) sites have to be filtered out server-side. if strings.Contains(wURL, PersonalSitePath) { return clues.Stack(ErrKnownSkippableCase). - With("site_id", id, "site_web_url", wURL) // TODO: pii + With("site_id", clues.Hide(id), "site_web_url", clues.Hide(wURL)) } name := ptr.Val(item.GetDisplayName()) @@ -239,10 +239,10 @@ func ValidateSite(item models.Siteable) error { // the built-in site at "https://{tenant-domain}/search" never has a name. if strings.HasSuffix(wURL, "/search") { return clues.Stack(ErrKnownSkippableCase). - With("site_id", id, "site_web_url", wURL) // TODO: pii + With("site_id", clues.Hide(id), "site_web_url", clues.Hide(wURL)) } - return clues.New("missing site display name").With("site_id", id) + return clues.New("missing site display name").With("site_id", clues.Hide(id)) } return nil From dd55744f15a08b54a4b7cfd5c8d62710e676547f Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Tue, 25 Jul 2023 12:38:12 -0700 Subject: [PATCH 34/62] Hide storage path for merge collections (#3902) #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3895 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/merge_collection.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/internal/kopia/merge_collection.go b/src/internal/kopia/merge_collection.go index ff32c4e73..25897fd58 100644 --- a/src/internal/kopia/merge_collection.go +++ b/src/internal/kopia/merge_collection.go @@ -70,7 +70,9 @@ func (mc *mergeCollection) Items( for _, c := range mc.cols { // Unfortunately doesn't seem to be a way right now to see if the // iteration failed and we should be exiting early. - ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + ictx := clues.Add( + ctx, + "merged_collection_storage_path", path.LoggableDir(c.storagePath)) logger.Ctx(ictx).Debug("sending items from merged collection") for item := range c.Items(ictx, errs) { @@ -95,7 +97,9 @@ func (mc *mergeCollection) FetchItemByName( "merged_collection_count", len(mc.cols)) for _, c := range mc.cols { - ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + ictx := clues.Add( + ctx, + "merged_collection_storage_path", path.LoggableDir(c.storagePath)) logger.Ctx(ictx).Debug("looking for item in merged collection") From 3a98bcdcf57a04f6555e9137b8ea9bd0ac445410 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Tue, 25 Jul 2023 13:04:09 -0700 Subject: [PATCH 35/62] Minor code cleanup (#3901) Remove now-unused code --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #2360 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/wrapper.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 3c6854ece..3963b30f6 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -129,11 +129,6 @@ func (w *Wrapper) Close(ctx context.Context) error { return nil } -type IncrementalBase struct { - *snapshot.Manifest - SubtreePaths []*path.Builder -} - // ConsumeBackupCollections takes a set of collections and creates a kopia snapshot // with the data that they contain. previousSnapshots is used for incremental // backups and should represent the base snapshot from which metadata is sourced From 0175033a830eebf82cd729cd2d2be1bec1d1a631 Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 25 Jul 2023 17:56:20 -0600 Subject: [PATCH 36/62] centralize the slack message action (#3614) centralizes the ci action for slack messages, and adds slack messages for per-resource cleanup failures. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Issues - #3503 --- .github/actions/slack-message/action.yml | 57 ++++++++++++++++++++++++ .github/workflows/ci_test_cleanup.yml | 22 ++++++--- .github/workflows/longevity_test.yml | 32 ++----------- .github/workflows/nightly_test.yml | 32 ++----------- .github/workflows/sanity-test.yaml | 32 ++----------- website/docs/developers/linters.md | 10 +++++ 6 files changed, 95 insertions(+), 90 deletions(-) create mode 100644 .github/actions/slack-message/action.yml diff --git a/.github/actions/slack-message/action.yml b/.github/actions/slack-message/action.yml new file mode 100644 index 000000000..1b72d9bee --- /dev/null +++ b/.github/actions/slack-message/action.yml @@ -0,0 +1,57 @@ +name: Send a message to slack + +inputs: + msg: + description: The slack message text + slack_url: + description: passthrough for secrets.SLACK_WEBHOOK_URL + +runs: + using: composite + steps: + - uses: actions/checkout@v3 + + - name: set github ref + shell: bash + run: | + echo "github_reference=${{ github.ref }}" >> $GITHUB_ENV + + - name: trim github ref + shell: bash + run: | + echo "trimmed_ref=${github_reference#refs/}" >> $GITHUB_ENV + + - name: build urls + shell: bash + run: | + echo "logurl=$(printf '' ${{ github.run_id }})" >> $GITHUB_ENV + echo "commiturl=$(printf '' ${{ github.sha }})" >> $GITHUB_ENV + echo "refurl=$(printf '' ${{ env.trimmed_ref }})" >> $GITHUB_ENV + + - name: use url or blank val + shell: bash + run: | + echo "JOB=${{ github.job || '' }}" >> $GITHUB_ENV + echo "LOGS=${{ github.run_id && env.logurl || '-' }}" >> $GITHUB_ENV + echo "COMMIT=${{ github.sha && env.commiturl || '-' }}" >> $GITHUB_ENV + echo "REF=${{ env.trimmed_ref && env.refurl || '-' }}" >> $GITHUB_ENV + + - id: slack-message + uses: slackapi/slack-github-action@v1.24.0 + env: + SLACK_WEBHOOK_URL: ${{ inputs.slack_url }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "text": "${{ inputs.msg }} :: ${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ inputs.msg }} :: ${{ env.JOB }}\n${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}" + } + } + ] + } diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 11ba94097..3687c0e0c 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -18,9 +18,7 @@ jobs: - uses: actions/checkout@v3 # sets the maximum time to now-30m. - # CI test have a 10 minute timeout. - # At 20 minutes ago, we should be safe from conflicts. - # The additional 10 minutes is just to be good citizens. + # CI test have a 20 minute timeout. - name: Set purge boundary run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV @@ -36,6 +34,13 @@ jobs: m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} + - name: Notify failure in slack + if: failure() + uses: ./.github/actions/slack-message + with: + msg: "[FAILED] ${{ vars[matrix.user] }} CI Cleanup" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} + Test-Site-Data-Cleanup: environment: Testing runs-on: ubuntu-latest @@ -48,9 +53,7 @@ jobs: - uses: actions/checkout@v3 # sets the maximum time to now-30m. - # CI test have a 10 minute timeout. - # At 20 minutes ago, we should be safe from conflicts. - # The additional 10 minutes is just to be good citizens. + # CI test have a 20 minute timeout. - name: Set purge boundary run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV @@ -67,3 +70,10 @@ jobs: azure-tenant-id: ${{ secrets.TENANT_ID }} m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} + + - name: Notify failure in slack + if: failure() + uses: ./.github/actions/slack-message + with: + msg: "[FAILED] ${{ vars[matrix.site] }} CI Cleanup" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/longevity_test.yml b/.github/workflows/longevity_test.yml index 02b956be1..8d294107a 100644 --- a/.github/workflows/longevity_test.yml +++ b/.github/workflows/longevity_test.yml @@ -277,33 +277,9 @@ jobs: if-no-files-found: error retention-days: 14 - - name: SHA info - id: sha-info + - name: Notify failure in slack if: failure() - run: | - echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA} - echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT - echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_OUTPUT - echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT - - - name: Send Github Action failure to Slack - id: slack-notification - if: failure() - uses: slackapi/slack-github-action@v1.24.0 + uses: ./.github/actions/slack-message with: - payload: | - { - "text": "Longevity test failure - build: ${{ job.status }} - SHA: ${{ steps.sha-info.outputs.SHA }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "[FAILED] Longevity Checks :: <${{ steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ steps.sha-info.outputs.COMMIT_URL }}|[Base]>\nCommit: <${{ steps.sha-info.outputs.COMMIT_URL }}|${{ steps.sha-info.outputs.SHA }}>" - } - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + msg: "[FAILED] Longevity Test" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index cc18732c5..29b69ad20 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -120,33 +120,9 @@ jobs: if-no-files-found: error retention-days: 14 - - name: SHA info - id: sha-info + - name: Notify failure in slack if: failure() - run: | - echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA} - echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT - echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_OUTPUT - echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT - - - name: Send Github Action failure to Slack - id: slack-notification - if: failure() - uses: slackapi/slack-github-action@v1.24.0 + uses: ./.github/actions/slack-message with: - payload: | - { - "text": "Nightly test failure - build: ${{ job.status }} - SHA: ${{ steps.sha-info.outputs.SHA }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "[FAILED] Nightly Checks :: <${{ steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ steps.sha-info.outputs.COMMIT_URL }}|[Base]>\nCommit: <${{ steps.sha-info.outputs.COMMIT_URL }}|${{ steps.sha-info.outputs.SHA }}>" - } - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + msg: "[FAILED] Nightly Checks" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 9e210778d..292c358dd 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -333,33 +333,9 @@ jobs: if-no-files-found: error retention-days: 14 - - name: SHA info - id: sha-info + - name: Notify failure in slack if: failure() - run: | - echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA} - echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT - echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_OUTPUT - echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT - - - name: Send Github Action failure to Slack - id: slack-notification - if: failure() - uses: slackapi/slack-github-action@v1.24.0 + uses: ./.github/actions/slack-message with: - payload: | - { - "text": "Sanity test failure - build: ${{ job.status }} - SHA: ${{ steps.sha-info.outputs.SHA }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "[FAILED] Sanity Checks :: <${{ steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ github.event.pull_request.html_url || github.event.head_commit.url }}|[Base]>\nCommit: <${{ steps.sha-info.outputs.COMMIT_URL }}|${{ steps.sha-info.outputs.SHA }}>" - } - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK \ No newline at end of file + msg: "[FAILED] Sanity Tests" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/website/docs/developers/linters.md b/website/docs/developers/linters.md index 9cf192ff1..e7416b235 100644 --- a/website/docs/developers/linters.md +++ b/website/docs/developers/linters.md @@ -21,6 +21,16 @@ You can run the linter manually or with the `Makefile` in the repository. Runnin the `Makefile` will also ensure you have the proper version of golangci-lint installed. +### Running the actions linter + +Installation: + +```sh +go install github.com/rhysd/actionlint/cmd/actionlint@latest +``` + +[Instructions for running locally.](https://github.com/rhysd/actionlint/blob/main/docs/usage.md) + ### Running with the `Makefile` There’s a `Makefile` in the corso/src that will automatically check if the proper From 05bd2f5ea330b0c648a5de2b3acbbbded2799d4e Mon Sep 17 00:00:00 2001 From: neha-Gupta1 Date: Wed, 26 Jul 2023 13:13:27 +0530 Subject: [PATCH 37/62] correct test env vars --- src/internal/tester/tconfig/config.go | 6 +++--- src/internal/tester/tconfig/protected_resources.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/internal/tester/tconfig/config.go b/src/internal/tester/tconfig/config.go index 1d9a89310..8660e6135 100644 --- a/src/internal/tester/tconfig/config.go +++ b/src/internal/tester/tconfig/config.go @@ -41,7 +41,7 @@ const ( EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL" EnvCorsoM365TestTeamID = "CORSO_M365_TEST_TEAM_ID" - EnvCorsoM365TestGroupID = "CORSO_M365_TEST_Group_ID" + EnvCorsoM365TestGroupID = "CORSO_M365_TEST_GROUP_ID" EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID" @@ -157,13 +157,13 @@ func ReadTestConfig() (map[string]string, error) { TestCfgTeamID, os.Getenv(EnvCorsoM365TestTeamID), vpr.GetString(TestCfgTeamID), - "d288d6bc-4595-4ff5-87a1-6e7fd750aa42") + "6f24b40d-b13d-4752-980f-f5fb9fba7aa0") fallbackTo( testEnv, TestCfgGroupID, os.Getenv(EnvCorsoM365TestGroupID), vpr.GetString(TestCfgGroupID), - "3d1129b1-52e1-4f49-a47a-a515b14c8a8e") + "6f24b40d-b13d-4752-980f-f5fb9fba7aa0") fallbackTo( testEnv, TestCfgSiteURL, diff --git a/src/internal/tester/tconfig/protected_resources.go b/src/internal/tester/tconfig/protected_resources.go index 13c1c9fb9..6f0587e4d 100644 --- a/src/internal/tester/tconfig/protected_resources.go +++ b/src/internal/tester/tconfig/protected_resources.go @@ -226,7 +226,7 @@ func M365TeamsID(t *testing.T) string { // Groups // M365GroupID returns a groupID string representing the m365GroupID described -// by either the env var CORSO_M365_TEST_Group_ID, the corso_test.toml config +// by either the env var CORSO_M365_TEST_GROUP_ID, the corso_test.toml config // file or the default value (in that order of priority). The default is a // last-attempt fallback that will only work on alcion's testing org. func M365GroupID(t *testing.T) string { From f6d44b3c604181446c1cff251d383f3de8ffa01c Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Wed, 26 Jul 2023 18:00:39 +0530 Subject: [PATCH 38/62] Export data from OneDrive (#3819) Core logic for exporting data from OneDrive Next: https://github.com/alcionai/corso/pull/3820 --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [x] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * https://github.com/alcionai/corso/issues/3670 * https://github.com/alcionai/corso/pull/3797 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/archive/zip.go | 99 +++++ src/internal/data/data_collection.go | 5 + src/internal/events/events.go | 3 + src/internal/m365/controller.go | 1 + src/internal/m365/export.go | 61 +++ src/internal/m365/mock/connector.go | 13 + src/internal/m365/onedrive/export.go | 166 ++++++++ src/internal/m365/onedrive/export_test.go | 463 ++++++++++++++++++++++ src/internal/m365/support/status.go | 1 + src/internal/operations/export.go | 359 +++++++++++++++++ src/internal/operations/export_test.go | 321 +++++++++++++++ src/internal/operations/inject/inject.go | 17 + src/pkg/control/export.go | 22 + src/pkg/export/export.go | 45 +++ src/pkg/fault/fault.go | 26 ++ src/pkg/repository/repository.go | 31 ++ 16 files changed, 1633 insertions(+) create mode 100644 src/internal/archive/zip.go create mode 100644 src/internal/m365/export.go create mode 100644 src/internal/m365/onedrive/export.go create mode 100644 src/internal/m365/onedrive/export_test.go create mode 100644 src/internal/operations/export.go create mode 100644 src/internal/operations/export_test.go create mode 100644 src/pkg/control/export.go create mode 100644 src/pkg/export/export.go diff --git a/src/internal/archive/zip.go b/src/internal/archive/zip.go new file mode 100644 index 000000000..f3e02ad66 --- /dev/null +++ b/src/internal/archive/zip.go @@ -0,0 +1,99 @@ +package archive + +import ( + "archive/zip" + "context" + "io" + "path" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/export" +) + +const ( + // ZipCopyBufferSize is the size of the copy buffer for zip + // write operations + // TODO(meain): tweak this value + ZipCopyBufferSize = 5 * 1024 * 1024 +) + +type zipCollection struct { + reader io.ReadCloser +} + +func (z zipCollection) BasePath() string { + return "" +} + +func (z zipCollection) Items(ctx context.Context) <-chan export.Item { + rc := make(chan export.Item, 1) + defer close(rc) + + rc <- export.Item{ + Data: export.ItemData{ + Name: "Corso_Export_" + dttm.FormatNow(dttm.HumanReadable) + ".zip", + Body: z.reader, + }, + } + + return rc +} + +// ZipExportCollection takes a list of export collections and zips +// them into a single collection. +func ZipExportCollection( + ctx context.Context, + expCollections []export.Collection, +) (export.Collection, error) { + if len(expCollections) == 0 { + return nil, clues.New("no export collections provided") + } + + reader, writer := io.Pipe() + wr := zip.NewWriter(writer) + + go func() { + defer writer.Close() + defer wr.Close() + + buf := make([]byte, ZipCopyBufferSize) + + for _, ec := range expCollections { + folder := ec.BasePath() + items := ec.Items(ctx) + + for item := range items { + err := item.Error + if err != nil { + writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID)) + return + } + + name := item.Data.Name + + // We assume folder and name to not contain any path separators. + // Also, this should always use `/` as this is + // created within a zip file and not written to disk. + // TODO(meain): Exchange paths might contain a path + // separator and will have to have special handling. + + //nolint:forbidigo + f, err := wr.Create(path.Join(folder, name)) + if err != nil { + writer.CloseWithError(clues.Wrap(err, "creating zip entry").With("name", name).With("id", item.ID)) + return + } + + _, err = io.CopyBuffer(f, item.Data.Body, buf) + if err != nil { + writer.CloseWithError(clues.Wrap(err, "writing zip entry").With("name", name).With("id", item.ID)) + return + } + } + } + }() + + return zipCollection{reader}, nil +} diff --git a/src/internal/data/data_collection.go b/src/internal/data/data_collection.go index b85e1e977..cec096783 100644 --- a/src/internal/data/data_collection.go +++ b/src/internal/data/data_collection.go @@ -91,6 +91,11 @@ func (c NoFetchRestoreCollection) FetchItemByName(context.Context, string) (Stre return nil, ErrNotFound } +type FetchRestoreCollection struct { + Collection + FetchItemByNamer +} + // Stream represents a single item within a Collection // that can be consumed as a stream (it embeds io.Reader) type Stream interface { diff --git a/src/internal/events/events.go b/src/internal/events/events.go index 1252052f7..99c1651ac 100644 --- a/src/internal/events/events.go +++ b/src/internal/events/events.go @@ -35,6 +35,8 @@ const ( BackupEnd = "Backup End" RestoreStart = "Restore Start" RestoreEnd = "Restore End" + ExportStart = "Export Start" + ExportEnd = "Export End" MaintenanceStart = "Maintenance Start" MaintenanceEnd = "Maintenance End" @@ -49,6 +51,7 @@ const ( ItemsWritten = "items_written" Resources = "resources" RestoreID = "restore_id" + ExportID = "export_id" Service = "service" StartTime = "start_time" Status = "status" diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 4dd2c19e8..051ab5fb2 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -24,6 +24,7 @@ import ( var ( _ inject.BackupProducer = &Controller{} _ inject.RestoreConsumer = &Controller{} + _ inject.ExportConsumer = &Controller{} ) // Controller is a struct used to wrap the GraphServiceClient and diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go new file mode 100644 index 000000000..3840d377b --- /dev/null +++ b/src/internal/m365/export.go @@ -0,0 +1,61 @@ +package m365 + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/diagnostics" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/onedrive" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/selectors" +) + +// ExportRestoreCollections exports data from the specified collections +func (ctrl *Controller) ExportRestoreCollections( + ctx context.Context, + backupVersion int, + sels selectors.Selector, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + errs *fault.Bus, +) ([]export.Collection, error) { + ctx, end := diagnostics.Span(ctx, "m365:export") + defer end() + + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) + ctx = clues.Add(ctx, "export_config", exportCfg) // TODO(meain): needs PII control + + var ( + expCollections []export.Collection + status *support.ControllerOperationStatus + deets = &details.Builder{} + err error + ) + + switch sels.Service { + case selectors.ServiceOneDrive: + expCollections, err = onedrive.ExportRestoreCollections( + ctx, + backupVersion, + exportCfg, + opts, + dcs, + deets, + errs) + default: + err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") + } + + ctrl.incrementAwaitingMessages() + ctrl.UpdateStatus(status) + + return expCollections, err +} diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 977306883..2c2ece635 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -71,3 +72,15 @@ func (ctrl Controller) ConsumeRestoreCollections( } func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {} + +func (ctrl Controller) ExportRestoreCollections( + _ context.Context, + _ int, + _ selectors.Selector, + _ control.ExportConfig, + _ control.Options, + _ []data.RestoreCollection, + _ *fault.Bus, +) ([]export.Collection, error) { + return nil, ctrl.Err +} diff --git a/src/internal/m365/onedrive/export.go b/src/internal/m365/onedrive/export.go new file mode 100644 index 000000000..bf68ede8d --- /dev/null +++ b/src/internal/m365/onedrive/export.go @@ -0,0 +1,166 @@ +package onedrive + +import ( + "context" + "strings" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +var _ export.Collection = &exportCollection{} + +// exportCollection is the implementation of export.ExportCollection for OneDrive +type exportCollection struct { + // baseDir contains the path of the collection + baseDir string + + // backingCollection is the restore collection from which we will + // create the export collection. + backingCollection data.RestoreCollection + + // backupVersion is the backupVersion of the backup this collection was part + // of. This is required to figure out how to get the name of the + // item. + backupVersion int +} + +func (ec exportCollection) BasePath() string { + return ec.baseDir +} + +func (ec exportCollection) Items(ctx context.Context) <-chan export.Item { + ch := make(chan export.Item) + go items(ctx, ec, ch) + + return ch +} + +// items converts items in backing collection to export items +func items(ctx context.Context, ec exportCollection, ch chan<- export.Item) { + defer close(ch) + + errs := fault.New(false) + + // There will only be a single item in the backingCollections + // for OneDrive + for item := range ec.backingCollection.Items(ctx, errs) { + itemUUID := item.UUID() + if isMetadataFile(itemUUID, ec.backupVersion) { + continue + } + + name, err := getItemName(ctx, itemUUID, ec.backupVersion, ec.backingCollection) + + ch <- export.Item{ + ID: itemUUID, + Data: export.ItemData{ + Name: name, + Body: item.ToReader(), + }, + Error: err, + } + } + + eitems, erecovereable := errs.ItemsAndRecovered() + + // Return all the items that we failed to get from kopia at the end + for _, err := range eitems { + ch <- export.Item{ + ID: err.ID, + Error: &err, + } + } + + for _, ec := range erecovereable { + ch <- export.Item{ + Error: ec, + } + } +} + +// isMetadataFile is used to determine if a path corresponds to a +// metadata file. This is OneDrive specific logic and depends on the +// version of the backup unlike metadata.IsMetadataFile which only has +// to be concerned about the current version. +func isMetadataFile(id string, backupVersion int) bool { + if backupVersion < version.OneDrive1DataAndMetaFiles { + return false + } + + return strings.HasSuffix(id, metadata.MetaFileSuffix) || + strings.HasSuffix(id, metadata.DirMetaFileSuffix) +} + +// getItemName is used to get the name of the item. +// How we get the name depends on the version of the backup. +func getItemName( + ctx context.Context, + id string, + backupVersion int, + fin data.FetchItemByNamer, +) (string, error) { + if backupVersion < version.OneDrive1DataAndMetaFiles { + return id, nil + } + + if backupVersion < version.OneDrive5DirMetaNoName { + return strings.TrimSuffix(id, metadata.DataFileSuffix), nil + } + + if strings.HasSuffix(id, metadata.DataFileSuffix) { + trimmedName := strings.TrimSuffix(id, metadata.DataFileSuffix) + metaName := trimmedName + metadata.MetaFileSuffix + + meta, err := fetchAndReadMetadata(ctx, fin, metaName) + if err != nil { + return "", clues.Wrap(err, "getting metadata").WithClues(ctx) + } + + return meta.FileName, nil + } + + return "", clues.New("invalid item id").WithClues(ctx) +} + +// ExportRestoreCollections will create the export collections for the +// given restore collections. +func ExportRestoreCollections( + ctx context.Context, + backupVersion int, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + deets *details.Builder, + errs *fault.Bus, +) ([]export.Collection, error) { + var ( + el = errs.Local() + ec = make([]export.Collection, 0, len(dcs)) + ) + + for _, dc := range dcs { + drivePath, err := path.ToDrivePath(dc.FullPath()) + if err != nil { + return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx) + } + + baseDir := path.Builder{}.Append(drivePath.Folders...) + + ec = append(ec, exportCollection{ + baseDir: baseDir.String(), + backingCollection: dc, + backupVersion: backupVersion, + }) + } + + return ec, el.Failure() +} diff --git a/src/internal/m365/onedrive/export_test.go b/src/internal/m365/onedrive/export_test.go new file mode 100644 index 000000000..b28b5f3a5 --- /dev/null +++ b/src/internal/m365/onedrive/export_test.go @@ -0,0 +1,463 @@ +package onedrive + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/data" + odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +type ExportUnitSuite struct { + tester.Suite +} + +func TestExportUnitSuite(t *testing.T) { + suite.Run(t, &ExportUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportUnitSuite) TestIsMetadataFile() { + table := []struct { + name string + id string + backupVersion int + isMeta bool + }{ + { + name: "legacy", + backupVersion: version.OneDrive1DataAndMetaFiles, + isMeta: false, + }, + { + name: "metadata file", + backupVersion: version.OneDrive3IsMetaMarker, + id: "name" + metadata.MetaFileSuffix, + isMeta: true, + }, + { + name: "dir metadata file", + backupVersion: version.OneDrive3IsMetaMarker, + id: "name" + metadata.DirMetaFileSuffix, + isMeta: true, + }, + { + name: "non metadata file", + backupVersion: version.OneDrive3IsMetaMarker, + id: "name" + metadata.DataFileSuffix, + isMeta: false, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + assert.Equal(suite.T(), test.isMeta, isMetadataFile(test.id, test.backupVersion), "is metadata") + }) + } +} + +type metadataStream struct { + id string + name string +} + +func (ms metadataStream) ToReader() io.ReadCloser { + return io.NopCloser(bytes.NewBufferString(`{"filename": "` + ms.name + `"}`)) +} +func (ms metadataStream) UUID() string { return ms.id } +func (ms metadataStream) Deleted() bool { return false } + +type finD struct { + id string + name string + err error +} + +func (fd finD) FetchItemByName(ctx context.Context, name string) (data.Stream, error) { + if fd.err != nil { + return nil, fd.err + } + + if name == fd.id { + return metadataStream{id: fd.id, name: fd.name}, nil + } + + return nil, assert.AnError +} + +func (suite *ExportUnitSuite) TestGetItemName() { + table := []struct { + tname string + id string + backupVersion int + name string + fin data.FetchItemByNamer + errFunc assert.ErrorAssertionFunc + }{ + { + tname: "legacy", + id: "name", + backupVersion: version.OneDrive1DataAndMetaFiles, + name: "name", + errFunc: assert.NoError, + }, + { + tname: "name in filename", + id: "name.data", + backupVersion: version.OneDrive4DirIncludesPermissions, + name: "name", + errFunc: assert.NoError, + }, + { + tname: "name in metadata", + id: "id.data", + backupVersion: version.Backup, + name: "name", + fin: finD{id: "id.meta", name: "name"}, + errFunc: assert.NoError, + }, + { + tname: "name in metadata but error", + id: "id.data", + backupVersion: version.Backup, + name: "", + fin: finD{err: assert.AnError}, + errFunc: assert.Error, + }, + } + + for _, test := range table { + suite.Run(test.tname, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + name, err := getItemName( + ctx, + test.id, + test.backupVersion, + test.fin, + ) + test.errFunc(t, err) + + assert.Equal(t, test.name, name, "name") + }) + } +} + +type mockRestoreCollection struct { + path path.Path + items []mockDataStream +} + +func (rc mockRestoreCollection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Stream { + ch := make(chan data.Stream) + + go func() { + defer close(ch) + + el := errs.Local() + + for _, item := range rc.items { + if item.err != nil { + el.AddRecoverable(ctx, item.err) + continue + } + + ch <- item + } + }() + + return ch +} + +func (rc mockRestoreCollection) FullPath() path.Path { + return rc.path +} + +type mockDataStream struct { + id string + data string + err error +} + +func (ms mockDataStream) ToReader() io.ReadCloser { + if ms.data != "" { + return io.NopCloser(bytes.NewBufferString(ms.data)) + } + + return nil +} +func (ms mockDataStream) UUID() string { return ms.id } +func (ms mockDataStream) Deleted() bool { return false } + +func (suite *ExportUnitSuite) TestGetItems() { + table := []struct { + name string + version int + backingCollection data.RestoreCollection + expectedItems []export.Item + }{ + { + name: "single item", + version: 1, + backingCollection: data.NoFetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name1", data: "body1"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name1", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + }, + }, + { + name: "multiple items", + version: 1, + backingCollection: data.NoFetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name1", data: "body1"}, + {id: "name2", data: "body2"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name1", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + { + ID: "name2", + Data: export.ItemData{ + Name: "name2", + Body: io.NopCloser((bytes.NewBufferString("body2"))), + }, + }, + }, + }, + { + name: "single item with data suffix", + version: 2, + backingCollection: data.NoFetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name1.data", data: "body1"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + }, + }, + { + name: "single item name from metadata", + version: version.Backup, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "id1.data", data: "body1"}, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", name: "name1"}, + }, + expectedItems: []export.Item{ + { + ID: "id1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + }, + }, + { + name: "single item name from metadata with error", + version: version.Backup, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "id1.data"}, + }, + }, + FetchItemByNamer: finD{err: assert.AnError}, + }, + expectedItems: []export.Item{ + { + ID: "id1.data", + Error: assert.AnError, + }, + }, + }, + { + name: "items with success and metadata read error", + version: version.Backup, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "missing.data"}, + {id: "id1.data", data: "body1"}, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", name: "name1"}, + }, + expectedItems: []export.Item{ + { + ID: "missing.data", + Error: assert.AnError, + }, + { + ID: "id1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser(bytes.NewBufferString("body1")), + }, + }, + }, + }, + { + name: "items with success and fetch error", + version: version.OneDrive1DataAndMetaFiles, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name0", data: "body0"}, + {id: "name1", err: assert.AnError}, + {id: "name2", data: "body2"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name0", + Data: export.ItemData{ + Name: "name0", + Body: io.NopCloser(bytes.NewBufferString("body0")), + }, + }, + { + ID: "name2", + Data: export.ItemData{ + Name: "name2", + Body: io.NopCloser(bytes.NewBufferString("body2")), + }, + }, + { + ID: "", + Error: assert.AnError, + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ec := exportCollection{ + baseDir: "", + backingCollection: test.backingCollection, + backupVersion: test.version, + } + + items := ec.Items(ctx) + + fitems := []export.Item{} + for item := range items { + fitems = append(fitems, item) + } + + assert.Len(t, fitems, len(test.expectedItems), "num of items") + + // We do not have any grantees about the ordering of the + // items in the SDK, but leaving the test this way for now + // to simplify testing. + for i, item := range fitems { + assert.Equal(t, test.expectedItems[i].ID, item.ID, "id") + assert.Equal(t, test.expectedItems[i].Data.Name, item.Data.Name, "name") + assert.Equal(t, test.expectedItems[i].Data.Body, item.Data.Body, "body") + assert.ErrorIs(t, item.Error, test.expectedItems[i].Error) + } + }) + } +} + +func (suite *ExportUnitSuite) TestExportRestoreCollections() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + dpb := odConsts.DriveFolderPrefixBuilder("driveID1") + + p, err := dpb.ToDataLayerOneDrivePath("t", "u", false) + assert.NoError(t, err, "build path") + + dcs := []data.RestoreCollection{ + data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + path: p, + items: []mockDataStream{ + {id: "id1.data", data: "body1"}, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", name: "name1"}, + }, + } + + expectedItems := []export.Item{ + { + ID: "id1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + } + + exportCfg := control.ExportConfig{} + ecs, err := ExportRestoreCollections(ctx, int(version.Backup), exportCfg, control.Options{}, dcs, nil, fault.New(true)) + assert.NoError(t, err, "export collections error") + + assert.Len(t, ecs, 1, "num of collections") + + items := ecs[0].Items(ctx) + + fitems := []export.Item{} + for item := range items { + fitems = append(fitems, item) + } + + assert.Equal(t, expectedItems, fitems, "items") +} diff --git a/src/internal/m365/support/status.go b/src/internal/m365/support/status.go index f241909fe..b1a7d2449 100644 --- a/src/internal/m365/support/status.go +++ b/src/internal/m365/support/status.go @@ -40,6 +40,7 @@ const ( OpUnknown Operation = iota Backup Restore + Export ) // Constructor for ConnectorOperationStatus. If the counts do not agree, an error is returned. diff --git a/src/internal/operations/export.go b/src/internal/operations/export.go new file mode 100644 index 000000000..dc4823935 --- /dev/null +++ b/src/internal/operations/export.go @@ -0,0 +1,359 @@ +package operations + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/alcionai/clues" + "github.com/google/uuid" + + "github.com/alcionai/corso/src/internal/archive" + "github.com/alcionai/corso/src/internal/common/crash" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/diagnostics" + "github.com/alcionai/corso/src/internal/events" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" +) + +const ( + // CopyBufferSize is the size of the copy buffer for disk + // write operations + // TODO(meain): tweak this value + CopyBufferSize = 5 * 1024 * 1024 +) + +// ExportOperation wraps an operation with export-specific props. +type ExportOperation struct { + operation + + BackupID model.StableID + Results RestoreResults + Selectors selectors.Selector + ExportCfg control.ExportConfig + Version string + + acct account.Account + ec inject.ExportConsumer +} + +// NewExportOperation constructs and validates a export operation. +func NewExportOperation( + ctx context.Context, + opts control.Options, + kw *kopia.Wrapper, + sw *store.Wrapper, + ec inject.ExportConsumer, + acct account.Account, + backupID model.StableID, + sel selectors.Selector, + exportCfg control.ExportConfig, + bus events.Eventer, +) (ExportOperation, error) { + op := ExportOperation{ + operation: newOperation(opts, bus, count.New(), kw, sw), + acct: acct, + BackupID: backupID, + ExportCfg: exportCfg, + Selectors: sel, + Version: "v0", + ec: ec, + } + if err := op.validate(); err != nil { + return ExportOperation{}, err + } + + return op, nil +} + +func (op ExportOperation) validate() error { + if op.ec == nil { + return clues.New("missing export consumer") + } + + return op.operation.validate() +} + +// aggregates stats from the export.Run(). +// primarily used so that the defer can take in a +// pointer wrapping the values, while those values +// get populated asynchronously. +type exportStats struct { + cs []data.RestoreCollection + ctrl *data.CollectionStats + bytesRead *stats.ByteCounter + resourceCount int + + // a transient value only used to pair up start-end events. + exportID string +} + +// Run begins a synchronous export operation. +func (op *ExportOperation) Run(ctx context.Context) ( + expColl []export.Collection, + err error, +) { + defer func() { + if crErr := crash.Recovery(ctx, recover(), "export"); crErr != nil { + err = crErr + } + }() + + var ( + opStats = exportStats{ + bytesRead: &stats.ByteCounter{}, + exportID: uuid.NewString(), + } + start = time.Now() + sstore = streamstore.NewStreamer(op.kopia, op.acct.ID(), op.Selectors.PathService()) + ) + + // ----- + // Setup + // ----- + + ctx, end := diagnostics.Span(ctx, "operations:export:run") + defer func() { + end() + // wait for the progress display to clean up + observe.Complete() + }() + + ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx}) + defer flushMetrics() + + ctx = clues.Add( + ctx, + "tenant_id", clues.Hide(op.acct.ID()), + "backup_id", op.BackupID, + "service", op.Selectors.Service) + + defer func() { + op.bus.Event( + ctx, + events.ExportEnd, + map[string]any{ + events.BackupID: op.BackupID, + events.DataRetrieved: op.Results.BytesRead, + events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), + events.EndTime: dttm.Format(op.Results.CompletedAt), + events.ItemsRead: op.Results.ItemsRead, + events.ItemsWritten: op.Results.ItemsWritten, + events.Resources: op.Results.ResourceOwners, + events.ExportID: opStats.exportID, + events.Service: op.Selectors.Service.String(), + events.StartTime: dttm.Format(op.Results.StartedAt), + events.Status: op.Status.String(), + }) + }() + + // ----- + // Execution + // ----- + + expCollections, err := op.do(ctx, &opStats, sstore, start) + if err != nil { + // No return here! We continue down to persistResults, even in case of failure. + logger.CtxErr(ctx, err).Error("running export") + + if errors.Is(err, kopia.ErrNoRestorePath) { + op.Errors.Fail(clues.New("empty backup or unknown path provided")) + } + + op.Errors.Fail(clues.Wrap(err, "running export")) + } + + finalizeErrorHandling(ctx, op.Options, op.Errors, "running export") + LogFaultErrors(ctx, op.Errors.Errors(), "running export") + + // ----- + // Persistence + // ----- + + err = op.finalizeMetrics(ctx, start, &opStats) + if err != nil { + op.Errors.Fail(clues.Wrap(err, "finalizing export metrics")) + return nil, op.Errors.Failure() + } + + logger.Ctx(ctx).Infow("completed export", "results", op.Results) + + return expCollections, nil +} + +func (op *ExportOperation) do( + ctx context.Context, + opStats *exportStats, + detailsStore streamstore.Reader, + start time.Time, +) ([]export.Collection, error) { + logger.Ctx(ctx). + With("control_options", op.Options, "selectors", op.Selectors). + Info("exporting selection") + + bup, deets, err := getBackupAndDetailsFromID( + ctx, + op.BackupID, + op.store, + detailsStore, + op.Errors) + if err != nil { + return nil, clues.Wrap(err, "getting backup and details") + } + + observe.Message(ctx, "Exporting", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) + + paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.ec, op.Errors) + if err != nil { + return nil, clues.Wrap(err, "formatting paths from details") + } + + ctx = clues.Add( + ctx, + "resource_owner_id", bup.Selector.ID(), + "resource_owner_name", clues.Hide(bup.Selector.Name()), + "details_entries", len(deets.Entries), + "details_paths", len(paths), + "backup_snapshot_id", bup.SnapshotID, + "backup_version", bup.Version) + + op.bus.Event( + ctx, + events.ExportStart, + map[string]any{ + events.StartTime: start, + events.BackupID: op.BackupID, + events.BackupCreateTime: bup.CreationTime, + events.ExportID: opStats.exportID, + }) + + observe.Message(ctx, fmt.Sprintf("Discovered %d items in backup %s to export", len(paths), op.BackupID)) + + kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository") + defer close(kopiaComplete) + + dcs, err := op.kopia.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, opStats.bytesRead, op.Errors) + if err != nil { + return nil, clues.Wrap(err, "producing collections to export") + } + + kopiaComplete <- struct{}{} + + ctx = clues.Add(ctx, "coll_count", len(dcs)) + + // should always be 1, since backups are 1:1 with resourceOwners. + opStats.resourceCount = 1 + opStats.cs = dcs + + expCollections, err := exportRestoreCollections( + ctx, + op.ec, + bup.Version, + op.Selectors, + op.ExportCfg, + op.Options, + dcs, + op.Errors) + if err != nil { + return nil, clues.Wrap(err, "exporting collections") + } + + opStats.ctrl = op.ec.Wait() + + logger.Ctx(ctx).Debug(opStats.ctrl) + + if op.ExportCfg.Archive { + zc, err := archive.ZipExportCollection(ctx, expCollections) + if err != nil { + return nil, clues.Wrap(err, "zipping export collections") + } + + return []export.Collection{zc}, nil + } + + return expCollections, nil +} + +// persists details and statistics about the export operation. +func (op *ExportOperation) finalizeMetrics( + ctx context.Context, + started time.Time, + opStats *exportStats, +) error { + op.Results.StartedAt = started + op.Results.CompletedAt = time.Now() + + op.Status = Completed + + if op.Errors.Failure() != nil { + op.Status = Failed + } + + op.Results.BytesRead = opStats.bytesRead.NumBytes + op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count + op.Results.ResourceOwners = opStats.resourceCount + + if opStats.ctrl == nil { + op.Status = Failed + return clues.New("restoration never completed") + } + + if op.Status != Failed && opStats.ctrl.IsZero() { + op.Status = NoData + } + + // We don't have data on what all items were written + // op.Results.ItemsWritten = opStats.ctrl.Successes + + return op.Errors.Failure() +} + +// --------------------------------------------------------------------------- +// Exporter funcs +// --------------------------------------------------------------------------- + +func exportRestoreCollections( + ctx context.Context, + ec inject.ExportConsumer, + backupVersion int, + sel selectors.Selector, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + errs *fault.Bus, +) ([]export.Collection, error) { + complete := observe.MessageWithCompletion(ctx, "Preparing export") + defer func() { + complete <- struct{}{} + close(complete) + }() + + expCollections, err := ec.ExportRestoreCollections( + ctx, + backupVersion, + sel, + exportCfg, + opts, + dcs, + errs) + if err != nil { + return nil, clues.Wrap(err, "exporting collections") + } + + return expCollections, nil +} diff --git a/src/internal/operations/export_test.go b/src/internal/operations/export_test.go new file mode 100644 index 000000000..c81114da9 --- /dev/null +++ b/src/internal/operations/export_test.go @@ -0,0 +1,321 @@ +package operations + +import ( + "archive/zip" + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/archive" + "github.com/alcionai/corso/src/internal/data" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/mock" + "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" +) + +type ExportOpSuite struct { + tester.Suite +} + +func TestExportOpSuite(t *testing.T) { + suite.Run(t, &ExportOpSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportOpSuite) TestExportOperation_PersistResults() { + var ( + kw = &kopia.Wrapper{} + sw = &store.Wrapper{} + ctrl = &mock.Controller{} + now = time.Now() + exportCfg = control.DefaultExportConfig() + ) + + table := []struct { + expectStatus OpStatus + expectErr assert.ErrorAssertionFunc + stats exportStats + fail error + }{ + { + expectStatus: Completed, + expectErr: assert.NoError, + stats: exportStats{ + resourceCount: 1, + bytesRead: &stats.ByteCounter{ + NumBytes: 42, + }, + cs: []data.RestoreCollection{ + data.NoFetchRestoreCollection{ + Collection: &exchMock.DataCollection{}, + }, + }, + ctrl: &data.CollectionStats{ + Objects: 1, + Successes: 1, + }, + }, + }, + { + expectStatus: Failed, + expectErr: assert.Error, + fail: assert.AnError, + stats: exportStats{ + bytesRead: &stats.ByteCounter{}, + ctrl: &data.CollectionStats{}, + }, + }, + { + expectStatus: NoData, + expectErr: assert.NoError, + stats: exportStats{ + bytesRead: &stats.ByteCounter{}, + cs: []data.RestoreCollection{}, + ctrl: &data.CollectionStats{}, + }, + }, + } + for _, test := range table { + suite.Run(test.expectStatus.String(), func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + op, err := NewExportOperation( + ctx, + control.Defaults(), + kw, + sw, + ctrl, + account.Account{}, + "foo", + selectors.Selector{DiscreteOwner: "test"}, + exportCfg, + evmock.NewBus()) + require.NoError(t, err, clues.ToCore(err)) + + op.Errors.Fail(test.fail) + + err = op.finalizeMetrics(ctx, now, &test.stats) + test.expectErr(t, err, clues.ToCore(err)) + + assert.Equal(t, test.expectStatus.String(), op.Status.String(), "status") + assert.Equal(t, len(test.stats.cs), op.Results.ItemsRead, "items read") + assert.Equal(t, test.stats.bytesRead.NumBytes, op.Results.BytesRead, "resource owners") + assert.Equal(t, test.stats.resourceCount, op.Results.ResourceOwners, "resource owners") + assert.Equal(t, now, op.Results.StartedAt, "started at") + assert.Less(t, now, op.Results.CompletedAt, "completed at") + }) + } +} + +type expCol struct { + base string + items []export.Item +} + +func (ec expCol) BasePath() string { return ec.base } +func (ec expCol) Items(ctx context.Context) <-chan export.Item { + ch := make(chan export.Item) + + go func() { + defer close(ch) + + for _, item := range ec.items { + ch <- item + } + }() + + return ch +} + +// ReadSeekCloser implements io.ReadSeekCloser. +type ReadSeekCloser struct { + *bytes.Reader +} + +// NewReadSeekCloser creates a new ReadSeekCloser from a byte slice. +func NewReadSeekCloser(byts []byte) *ReadSeekCloser { + return &ReadSeekCloser{ + Reader: bytes.NewReader(byts), + } +} + +// Close implements the io.Closer interface. +func (r *ReadSeekCloser) Close() error { + // Nothing to close for a byte slice. + return nil +} + +func (suite *ExportOpSuite) TestZipExports() { + table := []struct { + name string + collection []export.Collection + shouldErr bool + readErr bool + }{ + { + name: "nothing", + collection: []export.Collection{}, + shouldErr: true, + }, + { + name: "empty", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{}, + }, + }, + }, + { + name: "one item", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{ + { + ID: "id1", + Data: export.ItemData{ + Name: "test", + Body: NewReadSeekCloser([]byte("test")), + }, + }, + }, + }, + }, + }, + { + name: "multiple items", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{ + { + ID: "id1", + Data: export.ItemData{ + Name: "test", + Body: NewReadSeekCloser([]byte("test")), + }, + }, + }, + }, + expCol{ + base: "/fold", + items: []export.Item{ + { + ID: "id2", + Data: export.ItemData{ + Name: "test2", + Body: NewReadSeekCloser([]byte("test2")), + }, + }, + }, + }, + }, + }, + { + name: "one item with err", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{ + { + ID: "id3", + Error: assert.AnError, + }, + }, + }, + }, + readErr: true, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + zc, err := archive.ZipExportCollection(ctx, test.collection) + + if test.shouldErr { + assert.Error(t, err, "error") + return + } + + require.NoError(t, err, "error") + assert.Empty(t, zc.BasePath(), "base path") + + zippedItems := []export.ItemData{} + + count := 0 + for item := range zc.Items(ctx) { + assert.True(t, strings.HasPrefix(item.Data.Name, "Corso_Export_"), "name prefix") + assert.True(t, strings.HasSuffix(item.Data.Name, ".zip"), "name suffix") + + data, err := io.ReadAll(item.Data.Body) + if test.readErr { + assert.Error(t, err, "read error") + return + } + + size := int64(len(data)) + + item.Data.Body.Close() + + reader, err := zip.NewReader(bytes.NewReader(data), size) + require.NoError(t, err, "zip reader") + + for _, f := range reader.File { + rc, err := f.Open() + assert.NoError(t, err, "open file in zip") + + data, err := io.ReadAll(rc) + require.NoError(t, err, "read zip file content") + + rc.Close() + + zippedItems = append(zippedItems, export.ItemData{ + Name: f.Name, + Body: NewReadSeekCloser([]byte(data)), + }) + } + + count++ + } + + assert.Equal(t, 1, count, "single item") + + expectedZippedItems := []export.ItemData{} + for _, col := range test.collection { + for item := range col.Items(ctx) { + if col.BasePath() != "" { + item.Data.Name = strings.Join([]string{col.BasePath(), item.Data.Name}, "/") + } + _, err := item.Data.Body.(io.ReadSeeker).Seek(0, io.SeekStart) + require.NoError(t, err, "seek") + expectedZippedItems = append(expectedZippedItems, item.Data) + } + } + assert.Equal(t, expectedZippedItems, zippedItems, "items") + }) + } +} diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index 912b46743..5f6da9230 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -59,6 +60,22 @@ type ( CacheItemInfo(v details.ItemInfo) } + ExportConsumer interface { + ExportRestoreCollections( + ctx context.Context, + backupVersion int, + selector selectors.Selector, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + errs *fault.Bus, + ) ([]export.Collection, error) + + Wait() *data.CollectionStats + + CacheItemInfoer + } + RepoMaintenancer interface { RepoMaintenance(ctx context.Context, opts repository.Maintenance) error } diff --git a/src/pkg/control/export.go b/src/pkg/control/export.go new file mode 100644 index 000000000..e58633a5f --- /dev/null +++ b/src/pkg/control/export.go @@ -0,0 +1,22 @@ +package control + +// ExportConfig contains config for exports +type ExportConfig struct { + // Archive decides if we should create an archive from the data + // instead of just returning all the files. If Archive is set to + // true, we return a single collection with a single file which is + // the archive. + Archive bool + + // DataFormat decides the format in which we return the data. This is + // only useful for outlook exports, for example they can be in eml + // or pst for emails. + // TODO: Enable once we support outlook exports + // DataFormat string +} + +func DefaultExportConfig() ExportConfig { + return ExportConfig{ + Archive: false, + } +} diff --git a/src/pkg/export/export.go b/src/pkg/export/export.go new file mode 100644 index 000000000..76a6b6d8b --- /dev/null +++ b/src/pkg/export/export.go @@ -0,0 +1,45 @@ +package export + +import ( + "context" + "io" +) + +// Collection is the interface that is returned to the SDK consumer +type Collection interface { + // BasePath gets the base path of the collection + BasePath() string + + // Items gets the items within the collection(folder) + Items(context.Context) <-chan Item +} + +// ItemData is the data for an individual item. +type ItemData struct { + // Name is the name of the item. This is the name that the item + // would have had in the service. + Name string + + // Body is the body of the item. This is an io.ReadCloser and the + // SDK consumer is responsible for closing it. + Body io.ReadCloser +} + +// Item is the item that is returned to the SDK consumer +type Item struct { + // ID will be a unique id for the item. This is same as the id + // that is used to store the data. This is not the name and is + // mostly used just for tracking. + ID string + + // Data contains the actual data of the item. It will have both + // the name of the item and an io.ReadCloser which contains the + // body of the item. + Data ItemData + + // Error will contain any error that happened while trying to get + // the item/items like when trying to resolve the name of the item. + // In case we have the error bound to a particular item, we will + // also return the id of the item. + Error error +} diff --git a/src/pkg/fault/fault.go b/src/pkg/fault/fault.go index 97cc0bae3..488656fa4 100644 --- a/src/pkg/fault/fault.go +++ b/src/pkg/fault/fault.go @@ -208,6 +208,32 @@ func (e *Bus) Errors() *Errors { } } +// ItemsAndRecovered returns the items that failed along with other +// recoverable errors +func (e *Bus) ItemsAndRecovered() ([]Item, []error) { + var ( + is = map[string]Item{} + non = []error{} + ) + + for _, err := range e.recoverable { + var ie *Item + if !errors.As(err, &ie) { + non = append(non, err) + continue + } + + is[ie.dedupeID()] = *ie + } + + var ie *Item + if errors.As(e.failure, &ie) { + is[ie.dedupeID()] = *ie + } + + return maps.Values(is), non +} + // --------------------------------------------------------------------------- // Errors Data // --------------------------------------------------------------------------- diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 8a8abb33e..1e344bc7a 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -74,6 +74,12 @@ type Repository interface { sel selectors.Selector, restoreCfg control.RestoreConfig, ) (operations.RestoreOperation, error) + NewExport( + ctx context.Context, + backupID string, + sel selectors.Selector, + exportCfg control.ExportConfig, + ) (operations.ExportOperation, error) NewMaintenance( ctx context.Context, mOpts rep.Maintenance, @@ -349,6 +355,31 @@ func (r repository) NewBackupWithLookup( r.Bus) } +// NewExport generates a exportOperation runner. +func (r repository) NewExport( + ctx context.Context, + backupID string, + sel selectors.Selector, + exportCfg control.ExportConfig, +) (operations.ExportOperation, error) { + ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts) + if err != nil { + return operations.ExportOperation{}, clues.Wrap(err, "connecting to m365") + } + + return operations.NewExportOperation( + ctx, + r.Opts, + r.dataLayer, + store.NewKopiaStore(r.modelStore), + ctrl, + r.Account, + model.StableID(backupID), + sel, + exportCfg, + r.Bus) +} + // NewRestore generates a restoreOperation runner. func (r repository) NewRestore( ctx context.Context, From 7653af5143cba7a9f6692fe5b41a6eda6bb11bb3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jul 2023 16:35:21 +0000 Subject: [PATCH 39/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.307=20to=201.44.308=20in=20/src=20(#?= =?UTF-8?q?3906)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.307 to 1.44.308.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.308 (2023-07-25)

Service Client Updates

  • service/billingconductor: Updates service API and documentation
  • service/customer-profiles: Updates service API and documentation
  • service/datasync: Updates service API and documentation
  • service/dynamodb: Updates service API, documentation, waiters, paginators, and examples
    • Documentation updates for DynamoDB
  • service/ec2: Updates service API and documentation
    • This release adds an instance's peak and baseline network bandwidth as well as the memory sizes of an instance's inference accelerators to DescribeInstanceTypes.
  • service/emr-serverless: Updates service API and documentation
  • service/lambda: Updates service API
    • Add Python 3.11 (python3.11) support to AWS Lambda
  • service/rds: Updates service API, documentation, waiters, paginators, and examples
    • This release adds support for monitoring storage optimization progress on the DescribeDBInstances API.
  • service/sagemaker: Updates service API and documentation
    • Mark ContentColumn and TargetLabelColumn as required Targets in TextClassificationJobConfig in CreateAutoMLJobV2API
  • service/securityhub: Updates service API and documentation
  • service/sts: Updates service API and documentation
    • API updates for the AWS Security Token Service
  • service/transfer: Updates service API and documentation
    • This release adds support for SFTP Connectors.
  • service/wisdom: Updates service API and documentation
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.307&new-version=1.44.308)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 1c6091e91..04a4d8c51 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.307 + github.com/aws/aws-sdk-go v1.44.308 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index cd66bd27b..f40e14356 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.307 h1:2R0/EPgpZcFSUwZhYImq/srjaOrOfLv5MNRzrFyAM38= -github.com/aws/aws-sdk-go v1.44.307/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.308 h1:XKu+76UHsD5LaiU2Zb1q42uWakw80Az7x39jJXXahos= +github.com/aws/aws-sdk-go v1.44.308/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 3613cb2aa03d87572bd42b2ec4a98af2381a3d17 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 26 Jul 2023 11:24:53 -0600 Subject: [PATCH 40/62] look up restore resource if specified (#3853) If the restore configuration specifies a protected resource as a restore target, use that as the destination for the restore. First step is to ensure the provided target can be retrieved and identified. --- #### Does this PR need a docs update or release note? - [x] :clock1: Yes, but in a later PR #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test --- CHANGELOG.md | 4 + src/cli/backup/help_e2e_test.go | 2 +- src/cli/flags/restore_config.go | 7 +- src/cli/repo/s3_e2e_test.go | 2 +- src/cli/restore/exchange_test.go | 2 + src/cli/restore/onedrive_test.go | 6 + src/cli/restore/sharepoint_test.go | 8 + src/cli/utils/options.go | 3 +- src/cli/utils/restore_config.go | 15 +- src/cli/utils/restore_config_test.go | 44 +++- src/cli/utils/testdata/flags.go | 1 + src/cmd/factory/impl/common.go | 32 ++- src/cmd/factory/impl/exchange.go | 6 +- src/internal/common/idname/idname.go | 4 + src/internal/common/idname/idname_test.go | 60 +++++ src/internal/events/events_test.go | 2 +- src/internal/m365/backup.go | 13 +- src/internal/m365/backup_test.go | 16 +- src/internal/m365/controller.go | 19 +- src/internal/m365/controller_test.go | 132 +++++----- src/internal/m365/exchange/backup_test.go | 16 +- src/internal/m365/exchange/collection_test.go | 2 +- .../m365/exchange/container_resolver_test.go | 2 +- src/internal/m365/exchange/helper_test.go | 2 +- .../exchange/mail_container_cache_test.go | 2 +- src/internal/m365/exchange/restore.go | 40 ++- src/internal/m365/exchange/restore_test.go | 2 +- src/internal/m365/helper_test.go | 4 +- src/internal/m365/mock/connector.go | 19 +- src/internal/m365/onedrive/collection_test.go | 2 +- .../m365/onedrive/item_collector_test.go | 2 +- src/internal/m365/onedrive/restore.go | 205 ++++----------- src/internal/m365/onedrive/restore_caches.go | 116 +++++++++ src/internal/m365/onedrive/restore_test.go | 11 +- src/internal/m365/onedrive/service_test.go | 2 +- src/internal/m365/onedrive/url_cache_test.go | 2 +- src/internal/m365/onedrive_test.go | 87 +++---- src/internal/m365/restore.go | 59 +++-- src/internal/m365/sharepoint/backup_test.go | 6 +- .../m365/sharepoint/collection_test.go | 4 +- src/internal/m365/sharepoint/restore.go | 28 +- src/internal/m365/stub/stub.go | 3 +- src/internal/operations/backup_test.go | 6 +- src/internal/operations/export_test.go | 2 +- src/internal/operations/help_test.go | 4 +- src/internal/operations/inject/containers.go | 18 ++ src/internal/operations/inject/inject.go | 25 +- src/internal/operations/maintenance_test.go | 2 +- src/internal/operations/operation_test.go | 4 +- src/internal/operations/restore.go | 64 ++++- src/internal/operations/restore_test.go | 87 ++++++- src/internal/operations/test/exchange_test.go | 241 +++++++++++++++++- src/internal/operations/test/helper_test.go | 113 ++++---- src/internal/operations/test/onedrive_test.go | 204 +++++++++++++-- .../operations/test/sharepoint_test.go | 57 +++-- src/internal/tester/tconfig/config.go | 14 +- .../tester/tconfig/protected_resources.go | 11 + src/pkg/control/options.go | 5 +- src/pkg/control/restore.go | 13 +- src/pkg/repository/repository.go | 2 +- src/pkg/repository/repository_test.go | 28 +- src/pkg/services/m365/api/helper_test.go | 2 +- src/pkg/services/m365/api/sites_test.go | 17 +- src/pkg/services/m365/m365.go | 2 +- website/docs/setup/restore-options.md | 41 ++- 65 files changed, 1365 insertions(+), 591 deletions(-) create mode 100644 src/internal/common/idname/idname_test.go create mode 100644 src/internal/m365/onedrive/restore_caches.go create mode 100644 src/internal/operations/inject/containers.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 296b01a85..8c7123822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +### Added +- Restore commands now accept an optional resource override with the `--to-resource` flag. This allows restores to recreate backup data within different mailboxes, sites, and users. + ### Fixed - SharePoint document libraries deleted after the last backup can now be restored. +- Restore requires the protected resource to have access to the service being restored. ## [v0.11.1] (beta) - 2023-07-20 diff --git a/src/cli/backup/help_e2e_test.go b/src/cli/backup/help_e2e_test.go index d99d3769a..b7100d333 100644 --- a/src/cli/backup/help_e2e_test.go +++ b/src/cli/backup/help_e2e_test.go @@ -47,7 +47,7 @@ func prepM365Test( vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force) ctx = config.SetViper(ctx, vpr) - repo, err := repository.Initialize(ctx, acct, st, control.Defaults()) + repo, err := repository.Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) return acct, st, repo, vpr, recorder, cfgFP diff --git a/src/cli/flags/restore_config.go b/src/cli/flags/restore_config.go index a2b8c3a86..4a1868d01 100644 --- a/src/cli/flags/restore_config.go +++ b/src/cli/flags/restore_config.go @@ -9,11 +9,13 @@ import ( const ( CollisionsFN = "collisions" DestinationFN = "destination" + ToResourceFN = "to-resource" ) var ( CollisionsFV string DestinationFV string + ToResourceFV string ) // AddRestoreConfigFlags adds the restore config flag set. @@ -25,5 +27,8 @@ func AddRestoreConfigFlags(cmd *cobra.Command) { "Sets the behavior for existing item collisions: "+string(control.Skip)+", "+string(control.Copy)+", or "+string(control.Replace)) fs.StringVar( &DestinationFV, DestinationFN, "", - "Overrides the destination where items get restored; '/' places items into their original location") + "Overrides the folder where items get restored; '/' places items into their original location") + fs.StringVar( + &ToResourceFV, ToResourceFN, "", + "Overrides the protected resource (mailbox, site, user, etc) where data gets restored") } diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index 540d08836..4c3af6e5f 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -200,7 +200,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() { ctx = config.SetViper(ctx, vpr) // init the repo first - _, err = repository.Initialize(ctx, account.Account{}, st, control.Defaults()) + _, err = repository.Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) // then test it diff --git a/src/cli/restore/exchange_test.go b/src/cli/restore/exchange_test.go index 0dd022a81..a257fff7a 100644 --- a/src/cli/restore/exchange_test.go +++ b/src/cli/restore/exchange_test.go @@ -84,6 +84,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -125,6 +126,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) diff --git a/src/cli/restore/onedrive_test.go b/src/cli/restore/onedrive_test.go index a21f1191d..8a9fc7a94 100644 --- a/src/cli/restore/onedrive_test.go +++ b/src/cli/restore/onedrive_test.go @@ -70,6 +70,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -80,6 +81,9 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output @@ -99,6 +103,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) @@ -109,6 +114,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + assert.True(t, flags.RestorePermissionsFV) }) } } diff --git a/src/cli/restore/sharepoint_test.go b/src/cli/restore/sharepoint_test.go index b4547077f..6a8de8e57 100644 --- a/src/cli/restore/sharepoint_test.go +++ b/src/cli/restore/sharepoint_test.go @@ -75,6 +75,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -85,6 +86,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output @@ -111,6 +115,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) @@ -121,6 +126,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + + // bool flags + assert.True(t, flags.RestorePermissionsFV) }) } } diff --git a/src/cli/utils/options.go b/src/cli/utils/options.go index 7f9176a90..932c56b6b 100644 --- a/src/cli/utils/options.go +++ b/src/cli/utils/options.go @@ -8,7 +8,7 @@ import ( // Control produces the control options based on the user's flags. func Control() control.Options { - opt := control.Defaults() + opt := control.DefaultOptions() if flags.FailFastFV { opt.FailureHandling = control.FailFast @@ -21,7 +21,6 @@ func Control() control.Options { opt.DeltaPageSize = dps opt.DisableMetrics = flags.NoStatsFV - opt.RestorePermissions = flags.RestorePermissionsFV opt.SkipReduce = flags.SkipReduceFV opt.ToggleFeatures.DisableIncrementals = flags.DisableIncrementalsFV opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV diff --git a/src/cli/utils/restore_config.go b/src/cli/utils/restore_config.go index fa036e3f9..6be54f1ab 100644 --- a/src/cli/utils/restore_config.go +++ b/src/cli/utils/restore_config.go @@ -18,16 +18,20 @@ type RestoreCfgOpts struct { // DTTMFormat is the timestamp format appended // to the default folder name. Defaults to // dttm.HumanReadable. - DTTMFormat dttm.TimeFormat + DTTMFormat dttm.TimeFormat + ProtectedResource string + RestorePermissions bool Populated flags.PopulatedFlags } func makeRestoreCfgOpts(cmd *cobra.Command) RestoreCfgOpts { return RestoreCfgOpts{ - Collisions: flags.CollisionsFV, - Destination: flags.DestinationFV, - DTTMFormat: dttm.HumanReadable, + Collisions: flags.CollisionsFV, + Destination: flags.DestinationFV, + DTTMFormat: dttm.HumanReadable, + ProtectedResource: flags.ToResourceFV, + RestorePermissions: flags.RestorePermissionsFV, // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate @@ -67,6 +71,9 @@ func MakeRestoreConfig( restoreCfg.Location = opts.Destination } + restoreCfg.ProtectedResource = opts.ProtectedResource + restoreCfg.IncludePermissions = opts.RestorePermissions + Infof(ctx, "Restoring to folder %s", restoreCfg.Location) return restoreCfg diff --git a/src/cli/utils/restore_config_test.go b/src/cli/utils/restore_config_test.go index 1324c9571..c3509e360 100644 --- a/src/cli/utils/restore_config_test.go +++ b/src/cli/utils/restore_config_test.go @@ -68,18 +68,18 @@ func (suite *RestoreCfgUnitSuite) TestValidateRestoreConfigFlags() { } func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { - rco := &RestoreCfgOpts{ - Collisions: "collisions", - Destination: "destination", - } - table := []struct { name string + rco *RestoreCfgOpts populated flags.PopulatedFlags expect control.RestoreConfig }{ { - name: "not populated", + name: "not populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{}, expect: control.RestoreConfig{ OnCollision: control.Skip, @@ -88,6 +88,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "collision populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.CollisionsFN: {}, }, @@ -98,6 +102,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "destination populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.DestinationFN: {}, }, @@ -108,6 +116,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "both populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.CollisionsFN: {}, flags.DestinationFN: {}, @@ -117,6 +129,23 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { Location: "destination", }, }, + { + name: "with restore permissions", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + RestorePermissions: true, + }, + populated: flags.PopulatedFlags{ + flags.CollisionsFN: {}, + flags.DestinationFN: {}, + }, + expect: control.RestoreConfig{ + OnCollision: control.CollisionPolicy("collisions"), + Location: "destination", + IncludePermissions: true, + }, + }, } for _, test := range table { suite.Run(test.name, func() { @@ -125,12 +154,13 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { ctx, flush := tester.NewContext(t) defer flush() - opts := *rco + opts := *test.rco opts.Populated = test.populated result := MakeRestoreConfig(ctx, opts) assert.Equal(t, test.expect.OnCollision, result.OnCollision) assert.Contains(t, result.Location, test.expect.Location) + assert.Equal(t, test.expect.IncludePermissions, result.IncludePermissions) }) } } diff --git a/src/cli/utils/testdata/flags.go b/src/cli/utils/testdata/flags.go index d29198072..85131a1e0 100644 --- a/src/cli/utils/testdata/flags.go +++ b/src/cli/utils/testdata/flags.go @@ -46,6 +46,7 @@ var ( Collisions = "collisions" Destination = "destination" + ToResource = "toResource" RestorePermissions = true DeltaPageSize = "deltaPageSize" diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index 04967dc2a..837063b40 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -21,12 +21,12 @@ import ( odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" m365Stub "github.com/alcionai/corso/src/internal/m365/stub" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/fault" @@ -104,7 +104,15 @@ func generateAndRestoreItems( print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination) - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs, ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + + return ctrl.ConsumeRestoreCollections(ctx, rcc, dataColls, errs, ctr) } // ------------------------------------------------------------------------------------------ @@ -144,7 +152,7 @@ func getControllerAndVerifyResourceOwner( return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api") } - id, _, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, resourceOwner, nil) + id, _, err := ctrl.PopulateProtectedResourceIDAndName(ctx, resourceOwner, nil) if err != nil { return nil, account.Account{}, nil, clues.Wrap(err, "verifying user") } @@ -407,10 +415,8 @@ func generateAndRestoreDriveItems( // input, // version.Backup) - opts := control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - } + opts := control.DefaultOptions() + restoreCfg.IncludePermissions = true config := m365Stub.ConfigInfo{ Opts: opts, @@ -418,7 +424,7 @@ func generateAndRestoreDriveItems( Service: service, Tenant: tenantID, ResourceOwners: []string{resourceOwner}, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } _, _, collections, _, err := m365Stub.GetCollectionsAndExpected( @@ -429,5 +435,13 @@ func generateAndRestoreDriveItems( return nil, err } - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs, ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + + return ctrl.ConsumeRestoreCollections(ctx, rcc, collections, errs, ctr) } diff --git a/src/cmd/factory/impl/exchange.go b/src/cmd/factory/impl/exchange.go index eb923969b..b7ad4840d 100644 --- a/src/cmd/factory/impl/exchange.go +++ b/src/cmd/factory/impl/exchange.go @@ -72,7 +72,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error { subject, body, body, now, now, now, now) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { @@ -121,7 +121,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error exchMock.NoAttachments, exchMock.NoCancelledOccurrences, exchMock.NoExceptionOccurrences) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { @@ -172,7 +172,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error { "123-456-7890", ) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { diff --git a/src/internal/common/idname/idname.go b/src/internal/common/idname/idname.go index 0367d954b..e2a48fca3 100644 --- a/src/internal/common/idname/idname.go +++ b/src/internal/common/idname/idname.go @@ -28,6 +28,10 @@ type is struct { name string } +func NewProvider(id, name string) *is { + return &is{id, name} +} + func (is is) ID() string { return is.id } func (is is) Name() string { return is.name } diff --git a/src/internal/common/idname/idname_test.go b/src/internal/common/idname/idname_test.go new file mode 100644 index 000000000..229177d61 --- /dev/null +++ b/src/internal/common/idname/idname_test.go @@ -0,0 +1,60 @@ +package idname + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" +) + +type IDNameUnitSuite struct { + tester.Suite +} + +func TestIDNameUnitSuite(t *testing.T) { + suite.Run(t, &IDNameUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *IDNameUnitSuite) TestAdd() { + table := []struct { + name string + inID string + inName string + searchID string + searchName string + }{ + { + name: "basic", + inID: "foo", + inName: "bar", + searchID: "foo", + searchName: "bar", + }, + { + name: "change casing", + inID: "FNORDS", + inName: "SMARF", + searchID: "fnords", + searchName: "smarf", + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cache := NewCache(nil) + + cache.Add(test.inID, test.inName) + + id, found := cache.IDOf(test.searchName) + assert.True(t, found) + assert.Equal(t, test.inID, id) + + name, found := cache.NameOf(test.searchID) + assert.True(t, found) + assert.Equal(t, test.inName, name) + }) + } +} diff --git a/src/internal/events/events_test.go b/src/internal/events/events_test.go index 7cc47f607..f8b1d6beb 100644 --- a/src/internal/events/events_test.go +++ b/src/internal/events/events_test.go @@ -52,7 +52,7 @@ func (suite *EventsIntegrationSuite) TestNewBus() { ) require.NoError(t, err, clues.ToCore(err)) - b, err := events.NewBus(ctx, s, a.ID(), control.Defaults()) + b, err := events.NewBus(ctx, s, a.ID(), control.DefaultOptions()) require.NotEmpty(t, b) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index 50514e4ad..a5a89f134 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -2,7 +2,6 @@ package m365 import ( "context" - "strings" "github.com/alcionai/clues" @@ -44,7 +43,7 @@ func (ctrl *Controller) ProduceBackupCollections( ctx, end := diagnostics.Span( ctx, "m365:produceBackupCollections", - diagnostics.Index("service", sels.Service.String())) + diagnostics.Index("service", sels.PathService().String())) defer end() ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) @@ -61,8 +60,8 @@ func (ctrl *Controller) ProduceBackupCollections( serviceEnabled, canMakeDeltaQueries, err := checkServiceEnabled( ctx, ctrl.AC.Users(), - path.ServiceType(sels.Service), - sels.DiscreteOwner) + sels.PathService(), + owner.ID()) if err != nil { return nil, nil, false, err } @@ -194,10 +193,8 @@ func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error { ids = siteIDs } - resourceOwner := strings.ToLower(sels.DiscreteOwner) - - if !filters.Equal(ids).Compare(resourceOwner) { - return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_resource_owner", sels.DiscreteOwner) + if !filters.Contains(ids).Compare(sels.ID()) { + return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_protected_resource", sels.DiscreteOwner) } return nil diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index b80bd4ddc..87c9c766d 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -57,7 +57,7 @@ func (suite *DataCollectionIntgSuite) SetupSuite() { suite.tenantID = creds.AzureTenantID - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } @@ -120,7 +120,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() { sel := test.getSelector(t) uidn := inMock.NewProvider(sel.ID(), sel.Name()) - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections( @@ -239,7 +239,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() test.getSelector(t), nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) assert.Error(t, err, clues.ToCore(err)) assert.False(t, canUsePreviousBackup, "can use previous backup") @@ -296,7 +296,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { nil, ctrl.credentials, ctrl, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -367,7 +367,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { siteIDs = []string{siteID} ) - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil) require.NoError(t, err, clues.ToCore(err)) sel := selectors.NewSharePointBackup(siteIDs) @@ -381,7 +381,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { sel.Selector, nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -414,7 +414,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { siteIDs = []string{siteID} ) - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil) require.NoError(t, err, clues.ToCore(err)) sel := selectors.NewSharePointBackup(siteIDs) @@ -428,7 +428,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { sel.Selector, nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 051ab5fb2..174148a76 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -84,10 +84,11 @@ func NewController( AC: ac, IDNameLookup: idname.NewCache(nil), - credentials: creds, - ownerLookup: rCli, - tenant: acct.ID(), - wg: &sync.WaitGroup{}, + credentials: creds, + ownerLookup: rCli, + tenant: acct.ID(), + wg: &sync.WaitGroup{}, + backupDriveIDNames: idname.NewCache(nil), } return &ctrl, nil @@ -150,10 +151,6 @@ func (ctrl *Controller) incrementAwaitingMessages() { } func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { - if ctrl.backupDriveIDNames == nil { - ctrl.backupDriveIDNames = idname.NewCache(map[string]string{}) - } - if dii.SharePoint != nil { ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) } @@ -249,15 +246,15 @@ func (r resourceClient) getOwnerIDAndNameFrom( return id, name, nil } -// PopulateOwnerIDAndNamesFrom takes the provided owner identifier and produces +// PopulateProtectedResourceIDAndName takes the provided owner identifier and produces // the owner's name and ID from that value. Returns an error if the owner is // not recognized by the current tenant. // -// The id-name swapper is optional. Some processes will look up all owners in +// The id-name cacher is optional. Some processes will look up all owners in // the tenant before reaching this step. In that case, the data gets handed // down for this func to consume instead of performing further queries. The // data gets stored inside the controller instance for later re-use. -func (ctrl *Controller) PopulateOwnerIDAndNamesFrom( +func (ctrl *Controller) PopulateProtectedResourceIDAndName( ctx context.Context, owner string, // input value, can be either id or name ins idname.Cacher, diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 487603b39..f4ff3c032 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -12,15 +12,18 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/idname" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" @@ -223,7 +226,7 @@ func (suite *ControllerUnitSuite) TestPopulateOwnerIDAndNamesFrom() { ctrl := &Controller{ownerLookup: test.rc} - rID, rName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, test.owner, test.ins) + rID, rName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, test.owner, test.ins) test.expectErr(t, err, clues.ToCore(err)) assert.Equal(t, test.expectID, rID, "id") assert.Equal(t, test.expectName, rName, "name") @@ -385,20 +388,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { } ) + restoreCfg.IncludePermissions = true + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + deets, err := suite.ctrl.ConsumeRestoreCollections( ctx, - version.Backup, - sel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, []data.RestoreCollection{&dataMock.Collection{}}, fault.New(true), count.New()) - assert.Error(t, err, clues.ToCore(err)) - assert.NotNil(t, deets) + assert.Error(t, err, graph.ErrServiceNotEnabled, clues.ToCore(err)) + assert.Nil(t, deets) status := suite.ctrl.Wait() assert.Equal(t, 0, status.Objects) @@ -408,6 +415,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { func (suite *ControllerIntegrationSuite) TestEmptyCollections() { restoreCfg := testdata.DefaultRestoreConfig("") + restoreCfg.IncludePermissions = true + table := []struct { name string col []data.RestoreCollection @@ -464,15 +473,17 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() { ctx, flush := tester.NewContext(t) defer flush() + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: test.sel, + RestoreConfig: restoreCfg, + Selector: test.sel, + } + deets, err := suite.ctrl.ConsumeRestoreCollections( ctx, - version.Backup, - test.sel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, test.col, fault.New(true), count.New()) @@ -503,12 +514,18 @@ func runRestore( restoreCtrl := newController(ctx, t, sci.Resource, path.ExchangeService) restoreSel := getSelectorWith(t, sci.Service, sci.ResourceOwners, true) + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: control.DefaultOptions(), + ProtectedResource: restoreSel, + RestoreConfig: sci.RestoreCfg, + Selector: restoreSel, + } + deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, - backupVersion, - restoreSel, - sci.RestoreCfg, - sci.Opts, + rcc, collections, fault.New(true), count.New()) @@ -610,6 +627,7 @@ func runRestoreBackupTest( tenant string, resourceOwners []string, opts control.Options, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -620,7 +638,7 @@ func runRestoreBackupTest( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected( @@ -655,6 +673,7 @@ func runRestoreTestWithVersion( tenant string, resourceOwners []string, opts control.Options, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -665,7 +684,7 @@ func runRestoreTestWithVersion( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( @@ -692,7 +711,7 @@ func runRestoreBackupTestVersions( tenant string, resourceOwners []string, opts control.Options, - crc control.RestoreConfig, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -703,7 +722,7 @@ func runRestoreBackupTestVersions( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: crc, + RestoreCfg: restoreCfg, } totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( @@ -737,7 +756,7 @@ func runRestoreBackupTestVersions( test.collectionsLatest) } -func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { +func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_core() { bodyText := "This email has some text. However, all the text is on the same line." subjectText := "Test message for restore" @@ -996,10 +1015,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { test, suite.ctrl.tenant, []string{suite.user}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)) }) } } @@ -1080,6 +1097,8 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { for i, collection := range test.collections { // Get a restoreCfg per collection so they're independent. restoreCfg := testdata.DefaultRestoreConfig("") + restoreCfg.IncludePermissions = true + expectedDests = append(expectedDests, destAndCats{ resourceOwner: suite.user, dest: restoreCfg.Location, @@ -1112,15 +1131,18 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { ) restoreCtrl := newController(ctx, t, test.resourceCat, path.ExchangeService) + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: restoreSel, + RestoreConfig: restoreCfg, + Selector: restoreSel, + } + deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, - version.Backup, - restoreSel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, collections, fault.New(true), count.New()) @@ -1152,10 +1174,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { backupSel, nil, version.NoBackup, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -1164,10 +1183,13 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { t.Log("Backup enumeration complete") + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + ci := stub.ConfigInfo{ - Opts: control.Options{RestorePermissions: true}, + Opts: control.DefaultOptions(), // Alright to be empty, needed for OneDrive. - RestoreCfg: control.RestoreConfig{}, + RestoreCfg: restoreCfg, } // Pull the data prior to waiting for the status as otherwise it will @@ -1205,16 +1227,16 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen }, } + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + runRestoreBackupTest( suite.T(), test, suite.ctrl.tenant, []string{suite.user}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - ) + control.DefaultOptions(), + restoreCfg) } func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { @@ -1233,8 +1255,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { sel.Include( sel.ContactFolders([]string{selectors.NoneTgt}), sel.EventCalendars([]string{selectors.NoneTgt}), - sel.MailFolders([]string{selectors.NoneTgt}), - ) + sel.MailFolders([]string{selectors.NoneTgt})) return sel.Selector }, @@ -1297,23 +1318,20 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { start = time.Now() ) - id, name, err := backupCtrl.PopulateOwnerIDAndNamesFrom(ctx, backupSel.DiscreteOwner, nil) + id, name, err := backupCtrl.PopulateProtectedResourceIDAndName(ctx, backupSel.DiscreteOwner, nil) require.NoError(t, err, clues.ToCore(err)) backupSel.SetDiscreteOwnerIDName(id, name) dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections( ctx, - inMock.NewProvider(id, name), + idname.NewProvider(id, name), backupSel, nil, version.NoBackup, - control.Options{ - RestorePermissions: false, - ToggleFeatures: control.Toggles{}, - }, + control.DefaultOptions(), fault.New(true)) - require.NoError(t, err) + require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") // No excludes yet because this isn't an incremental backup. assert.True(t, excludes.Empty()) diff --git a/src/internal/m365/exchange/backup_test.go b/src/internal/m365/exchange/backup_test.go index 34735eda8..fa4b87d9d 100644 --- a/src/internal/m365/exchange/backup_test.go +++ b/src/internal/m365/exchange/backup_test.go @@ -414,7 +414,7 @@ func (suite *BackupIntgSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) suite.tenantID = creds.AzureTenantID @@ -466,7 +466,7 @@ func (suite *BackupIntgSuite) TestMailFetch() { ctx, flush := tester.NewContext(t) defer flush() - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !test.canMakeDeltaQueries collections, err := createCollections( @@ -554,7 +554,7 @@ func (suite *BackupIntgSuite) TestDelta() { inMock.NewProvider(userID, userID), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), func(status *support.ControllerOperationStatus) {}, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -587,7 +587,7 @@ func (suite *BackupIntgSuite) TestDelta() { inMock.NewProvider(userID, userID), test.scope, dps, - control.Defaults(), + control.DefaultOptions(), func(status *support.ControllerOperationStatus) {}, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -633,7 +633,7 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() { inMock.NewProvider(suite.user, suite.user), sel.Scopes()[0], DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -709,7 +709,7 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() { inMock.NewProvider(suite.user, suite.user), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -834,7 +834,7 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() { inMock.NewProvider(suite.user, suite.user), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -1995,7 +1995,7 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i ctx, flush := tester.NewContext(t) defer flush() - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !deltaAfter getter := test.getter diff --git a/src/internal/m365/exchange/collection_test.go b/src/internal/m365/exchange/collection_test.go index 2c023d703..7c5a4adab 100644 --- a/src/internal/m365/exchange/collection_test.go +++ b/src/internal/m365/exchange/collection_test.go @@ -178,7 +178,7 @@ func (suite *CollectionSuite) TestNewCollection_state() { test.curr, test.prev, test.loc, 0, &mockItemer{}, nil, - control.Defaults(), + control.DefaultOptions(), false) assert.Equal(t, test.expect, c.State(), "collection state") assert.Equal(t, test.curr, c.fullPath, "full path") diff --git a/src/internal/m365/exchange/container_resolver_test.go b/src/internal/m365/exchange/container_resolver_test.go index 54cd23c67..b2ff30830 100644 --- a/src/internal/m365/exchange/container_resolver_test.go +++ b/src/internal/m365/exchange/container_resolver_test.go @@ -699,7 +699,7 @@ func (suite *ContainerResolverSuite) SetupSuite() { } func (suite *ContainerResolverSuite) TestPopulate() { - ac, err := api.NewClient(suite.credentials, control.Defaults()) + ac, err := api.NewClient(suite.credentials, control.DefaultOptions()) require.NoError(suite.T(), err, clues.ToCore(err)) eventFunc := func(t *testing.T) graph.ContainerResolver { diff --git a/src/internal/m365/exchange/helper_test.go b/src/internal/m365/exchange/helper_test.go index f8cadd227..9b1583b9c 100644 --- a/src/internal/m365/exchange/helper_test.go +++ b/src/internal/m365/exchange/helper_test.go @@ -31,7 +31,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.creds = creds - its.ac, err = api.NewClient(creds, control.Defaults()) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.userID = tconfig.GetM365UserID(ctx) diff --git a/src/internal/m365/exchange/mail_container_cache_test.go b/src/internal/m365/exchange/mail_container_cache_test.go index 64f453092..de0694749 100644 --- a/src/internal/m365/exchange/mail_container_cache_test.go +++ b/src/internal/m365/exchange/mail_container_cache_test.go @@ -84,7 +84,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() { ctx, flush := tester.NewContext(t) defer flush() - ac, err := api.NewClient(suite.credentials, control.Defaults()) + ac, err := api.NewClient(suite.credentials, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) acm := ac.Mail() diff --git a/src/internal/m365/exchange/restore.go b/src/internal/m365/exchange/restore.go index 5a5dbfcbc..7871f68d4 100644 --- a/src/internal/m365/exchange/restore.go +++ b/src/internal/m365/exchange/restore.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -28,7 +29,7 @@ import ( func ConsumeRestoreCollections( ctx context.Context, ac api.Client, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, @@ -39,16 +40,13 @@ func ConsumeRestoreCollections( } var ( - userID = dcs[0].FullPath().ResourceOwner() + resourceID = rcc.ProtectedResource.ID() directoryCache = make(map[path.CategoryType]graph.ContainerResolver) handlers = restoreHandlers(ac) metrics support.CollectionMetrics el = errs.Local() ) - // FIXME: should be user name - ctx = clues.Add(ctx, "resource_owner", clues.Hide(userID)) - for _, dc := range dcs { if el.Failure() != nil { break @@ -69,7 +67,7 @@ func ConsumeRestoreCollections( } if directoryCache[category] == nil { - gcr := handler.newContainerCache(userID) + gcr := handler.newContainerCache(resourceID) if err := gcr.Populate(ctx, errs, handler.defaultRootContainer()); err != nil { return nil, clues.Wrap(err, "populating container cache") } @@ -80,8 +78,8 @@ func ConsumeRestoreCollections( containerID, gcc, err := createDestination( ictx, handler, - handler.formatRestoreDestination(restoreCfg.Location, dc.FullPath()), - userID, + handler.formatRestoreDestination(rcc.RestoreConfig.Location, dc.FullPath()), + resourceID, directoryCache[category], errs) if err != nil { @@ -92,7 +90,7 @@ func ConsumeRestoreCollections( directoryCache[category] = gcc ictx = clues.Add(ictx, "restore_destination_id", containerID) - collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, userID, containerID) + collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, resourceID, containerID) if err != nil { el.AddRecoverable(ctx, clues.Wrap(err, "building item collision cache")) continue @@ -102,10 +100,10 @@ func ConsumeRestoreCollections( ictx, handler, dc, - userID, + resourceID, containerID, collisionKeyToItemID, - restoreCfg.OnCollision, + rcc.RestoreConfig.OnCollision, deets, errs, ctr) @@ -126,7 +124,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), metrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } @@ -136,7 +134,7 @@ func restoreCollection( ctx context.Context, ir itemRestorer, dc data.RestoreCollection, - userID, destinationID string, + resourceID, destinationID string, collisionKeyToItemID map[string]string, collisionPolicy control.CollisionPolicy, deets *details.Builder, @@ -187,7 +185,7 @@ func restoreCollection( info, err := ir.restore( ictx, body, - userID, + resourceID, destinationID, collisionKeyToItemID, collisionPolicy, @@ -240,7 +238,7 @@ func createDestination( ctx context.Context, ca containerAPI, destination *path.Builder, - userID string, + resourceID string, gcr graph.ContainerResolver, errs *fault.Bus, ) (string, graph.ContainerResolver, error) { @@ -264,7 +262,7 @@ func createDestination( ca, cache, restoreLoc, - userID, + resourceID, containerParentID, container, errs) @@ -285,7 +283,7 @@ func getOrPopulateContainer( ca containerAPI, gcr graph.ContainerResolver, restoreLoc *path.Builder, - userID, containerParentID, containerName string, + resourceID, containerParentID, containerName string, errs *fault.Bus, ) (string, error) { cached, ok := gcr.LocationInCache(restoreLoc.String()) @@ -293,7 +291,7 @@ func getOrPopulateContainer( return cached, nil } - c, err := ca.CreateContainer(ctx, userID, containerParentID, containerName) + c, err := ca.CreateContainer(ctx, resourceID, containerParentID, containerName) // 409 handling case: // attempt to fetch the container by name and add that result to the cache. @@ -301,7 +299,7 @@ func getOrPopulateContainer( // sometimes the backend will create the folder despite the 5xx response, // leaving our local containerResolver with inconsistent state. if graph.IsErrFolderExists(err) { - cc, e := ca.GetContainerByName(ctx, userID, containerParentID, containerName) + cc, e := ca.GetContainerByName(ctx, resourceID, containerParentID, containerName) if e != nil { err = clues.Stack(err, e) } else { @@ -327,7 +325,7 @@ func uploadAttachments( ctx context.Context, ap attachmentPoster, as []models.Attachmentable, - userID, destinationID, itemID string, + resourceID, destinationID, itemID string, errs *fault.Bus, ) error { el := errs.Local() @@ -340,7 +338,7 @@ func uploadAttachments( err := uploadAttachment( ctx, ap, - userID, + resourceID, destinationID, itemID, a) diff --git a/src/internal/m365/exchange/restore_test.go b/src/internal/m365/exchange/restore_test.go index 42e61a915..a30d56dd0 100644 --- a/src/internal/m365/exchange/restore_test.go +++ b/src/internal/m365/exchange/restore_test.go @@ -44,7 +44,7 @@ func (suite *RestoreIntgSuite) SetupSuite() { require.NoError(t, err, clues.ToCore(err)) suite.credentials = m365 - suite.ac, err = api.NewClient(m365, control.Defaults()) + suite.ac, err = api.NewClient(m365, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/helper_test.go b/src/internal/m365/helper_test.go index 25f4bb18c..a3f1d7d5e 100644 --- a/src/internal/m365/helper_test.go +++ b/src/internal/m365/helper_test.go @@ -796,8 +796,8 @@ func compareDriveItem( assert.Equal(t, expectedMeta.FileName, itemMeta.FileName) } - if !mci.Opts.RestorePermissions { - assert.Equal(t, 0, len(itemMeta.Permissions)) + if !mci.RestoreCfg.IncludePermissions { + assert.Empty(t, itemMeta.Permissions, "no permissions should be included in restore") return true } diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 2c2ece635..64bc8deda 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -27,6 +27,10 @@ type Controller struct { Err error Stats data.CollectionStats + + ProtectedResourceID string + ProtectedResourceName string + ProtectedResourceErr error } func (ctrl Controller) ProduceBackupCollections( @@ -60,10 +64,7 @@ func (ctrl Controller) Wait() *data.CollectionStats { func (ctrl Controller) ConsumeRestoreCollections( _ context.Context, - _ int, - _ selectors.Selector, - _ control.RestoreConfig, - _ control.Options, + _ inject.RestoreConsumerConfig, _ []data.RestoreCollection, _ *fault.Bus, _ *count.Bus, @@ -84,3 +85,13 @@ func (ctrl Controller) ExportRestoreCollections( ) ([]export.Collection, error) { return nil, ctrl.Err } + +func (ctrl Controller) PopulateProtectedResourceIDAndName( + ctx context.Context, + protectedResource string, // input value, can be either id or name + ins idname.Cacher, +) (string, string, error) { + return ctrl.ProtectedResourceID, + ctrl.ProtectedResourceName, + ctrl.ProtectedResourceErr +} diff --git a/src/internal/m365/onedrive/collection_test.go b/src/internal/m365/onedrive/collection_test.go index bcd4da4b6..3c30cac22 100644 --- a/src/internal/m365/onedrive/collection_test.go +++ b/src/internal/m365/onedrive/collection_test.go @@ -945,7 +945,7 @@ func (suite *CollectionUnitTestSuite) TestItemExtensions() { nil, } - opts := control.Defaults() + opts := control.DefaultOptions() opts.ItemExtensionFactory = append( opts.ItemExtensionFactory, test.factories...) diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/onedrive/item_collector_test.go index fc2cccd62..6078517c7 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -313,7 +313,7 @@ func (suite *OneDriveIntgSuite) SetupSuite() { suite.creds = creds - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index f951419be..7c920e6ff 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -23,6 +23,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" @@ -38,114 +39,11 @@ const ( maxUploadRetries = 3 ) -type driveInfo struct { - id string - name string - rootFolderID string -} - -type restoreCaches struct { - BackupDriveIDName idname.Cacher - collisionKeyToItemID map[string]api.DriveItemIDType - DriveIDToDriveInfo map[string]driveInfo - DriveNameToDriveInfo map[string]driveInfo - Folders *folderCache - OldLinkShareIDToNewID map[string]string - OldPermIDToNewID map[string]string - ParentDirToMeta map[string]metadata.Metadata - - pool sync.Pool -} - -func (rc *restoreCaches) AddDrive( - ctx context.Context, - md models.Driveable, - grf GetRootFolderer, -) error { - di := driveInfo{ - id: ptr.Val(md.GetId()), - name: ptr.Val(md.GetName()), - } - - ctx = clues.Add(ctx, "drive_info", di) - - root, err := grf.GetRootFolder(ctx, di.id) - if err != nil { - return clues.Wrap(err, "getting drive root id") - } - - di.rootFolderID = ptr.Val(root.GetId()) - - rc.DriveIDToDriveInfo[di.id] = di - rc.DriveNameToDriveInfo[di.name] = di - - return nil -} - -// Populate looks up drive items available to the protectedResource -// and adds their info to the caches. -func (rc *restoreCaches) Populate( - ctx context.Context, - gdparf GetDrivePagerAndRootFolderer, - protectedResourceID string, -) error { - drives, err := api.GetAllDrives( - ctx, - gdparf.NewDrivePager(protectedResourceID, nil), - true, - maxDrivesRetries) - if err != nil { - return clues.Wrap(err, "getting drives") - } - - for _, md := range drives { - if err := rc.AddDrive(ctx, md, gdparf); err != nil { - return clues.Wrap(err, "caching drive") - } - } - - return nil -} - -type GetDrivePagerAndRootFolderer interface { - GetRootFolderer - NewDrivePagerer -} - -func NewRestoreCaches( - backupDriveIDNames idname.Cacher, -) *restoreCaches { - // avoid nil panics - if backupDriveIDNames == nil { - backupDriveIDNames = idname.NewCache(nil) - } - - return &restoreCaches{ - BackupDriveIDName: backupDriveIDNames, - collisionKeyToItemID: map[string]api.DriveItemIDType{}, - DriveIDToDriveInfo: map[string]driveInfo{}, - DriveNameToDriveInfo: map[string]driveInfo{}, - Folders: NewFolderCache(), - OldLinkShareIDToNewID: map[string]string{}, - OldPermIDToNewID: map[string]string{}, - ParentDirToMeta: map[string]metadata.Metadata{}, - // Buffer pool for uploads - pool: sync.Pool{ - New: func() any { - b := make([]byte, graph.CopyBufferSize) - return &b - }, - }, - } -} - // ConsumeRestoreCollections will restore the specified data collections into OneDrive func ConsumeRestoreCollections( ctx context.Context, rh RestoreHandler, - backupVersion int, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc inject.RestoreConsumerConfig, backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, @@ -153,16 +51,15 @@ func ConsumeRestoreCollections( ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - restoreMetrics support.CollectionMetrics - el = errs.Local() - caches = NewRestoreCaches(backupDriveIDNames) - protectedResourceID = dcs[0].FullPath().ResourceOwner() - fallbackDriveName = restoreCfg.Location + restoreMetrics support.CollectionMetrics + el = errs.Local() + caches = NewRestoreCaches(backupDriveIDNames) + fallbackDriveName = rcc.RestoreConfig.Location ) - ctx = clues.Add(ctx, "backup_version", backupVersion) + ctx = clues.Add(ctx, "backup_version", rcc.BackupVersion) - err := caches.Populate(ctx, rh, protectedResourceID) + err := caches.Populate(ctx, rh, rcc.ProtectedResource.ID()) if err != nil { return nil, clues.Wrap(err, "initializing restore caches") } @@ -183,19 +80,16 @@ func ConsumeRestoreCollections( ictx = clues.Add( ctx, "category", dc.FullPath().Category(), - "resource_owner", clues.Hide(protectedResourceID), "full_path", dc.FullPath()) ) metrics, err = RestoreCollection( ictx, rh, - restoreCfg, - backupVersion, + rcc, dc, caches, deets, - opts.RestorePermissions, fallbackDriveName, errs, ctr.Local()) @@ -215,7 +109,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), restoreMetrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } @@ -228,26 +122,23 @@ func ConsumeRestoreCollections( func RestoreCollection( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, - backupVersion int, + rcc inject.RestoreConsumerConfig, dc data.RestoreCollection, caches *restoreCaches, deets *details.Builder, - restorePerms bool, // TODD: move into restoreConfig fallbackDriveName string, errs *fault.Bus, ctr *count.Bus, ) (support.CollectionMetrics, error) { var ( - metrics = support.CollectionMetrics{} - directory = dc.FullPath() - protectedResourceID = directory.ResourceOwner() - el = errs.Local() - metricsObjects int64 - metricsBytes int64 - metricsSuccess int64 - wg sync.WaitGroup - complete bool + metrics = support.CollectionMetrics{} + directory = dc.FullPath() + el = errs.Local() + metricsObjects int64 + metricsBytes int64 + metricsSuccess int64 + wg sync.WaitGroup + complete bool ) ctx, end := diagnostics.Span(ctx, "gc:drive:restoreCollection", diagnostics.Label("path", directory)) @@ -263,7 +154,7 @@ func RestoreCollection( rh, caches, drivePath, - protectedResourceID, + rcc.ProtectedResource.ID(), fallbackDriveName) if err != nil { return metrics, clues.Wrap(err, "ensuring drive exists") @@ -281,8 +172,8 @@ func RestoreCollection( // the drive into which this folder gets restored is tracked separately in drivePath. restoreDir := &path.Builder{} - if len(restoreCfg.Location) > 0 { - restoreDir = restoreDir.Append(restoreCfg.Location) + if len(rcc.RestoreConfig.Location) > 0 { + restoreDir = restoreDir.Append(rcc.RestoreConfig.Location) } restoreDir = restoreDir.Append(drivePath.Folders...) @@ -301,8 +192,8 @@ func RestoreCollection( drivePath, dc, caches, - backupVersion, - restorePerms) + rcc.BackupVersion, + rcc.RestoreConfig.IncludePermissions) if err != nil { return metrics, clues.Wrap(err, "getting permissions").WithClues(ctx) } @@ -316,7 +207,7 @@ func RestoreCollection( dc.FullPath(), colMeta, caches, - restorePerms) + rcc.RestoreConfig.IncludePermissions) if err != nil { return metrics, clues.Wrap(err, "creating folders for restore") } @@ -390,14 +281,12 @@ func RestoreCollection( itemInfo, skipped, err := restoreItem( ictx, rh, - restoreCfg, + rcc, dc, - backupVersion, drivePath, restoreFolderID, copyBuffer, caches, - restorePerms, itemData, itemPath, ctr) @@ -440,14 +329,12 @@ func RestoreCollection( func restoreItem( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, fibn data.FetchItemByNamer, - backupVersion int, drivePath *path.DrivePath, restoreFolderID string, copyBuffer []byte, caches *restoreCaches, - restorePerms bool, itemData data.Stream, itemPath path.Path, ctr *count.Bus, @@ -455,11 +342,11 @@ func restoreItem( itemUUID := itemData.UUID() ctx = clues.Add(ctx, "item_id", itemUUID) - if backupVersion < version.OneDrive1DataAndMetaFiles { + if rcc.BackupVersion < version.OneDrive1DataAndMetaFiles { itemInfo, err := restoreV0File( ctx, rh, - restoreCfg, + rcc.RestoreConfig, drivePath, fibn, restoreFolderID, @@ -468,7 +355,7 @@ func restoreItem( itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -491,7 +378,7 @@ func restoreItem( // Only the version.OneDrive1DataAndMetaFiles needed to deserialize the // permission for child folders here. Later versions can request // permissions inline when processing the collection. - if !restorePerms || backupVersion >= version.OneDrive4DirIncludesPermissions { + if !rcc.RestoreConfig.IncludePermissions || rcc.BackupVersion >= version.OneDrive4DirIncludesPermissions { return details.ItemInfo{}, true, nil } @@ -511,22 +398,21 @@ func restoreItem( // only items with DataFileSuffix from this point on - if backupVersion < version.OneDrive6NameInMeta { + if rcc.BackupVersion < version.OneDrive6NameInMeta { itemInfo, err := restoreV1File( ctx, rh, - restoreCfg, + rcc, drivePath, fibn, restoreFolderID, copyBuffer, - restorePerms, caches, itemPath, itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -541,18 +427,17 @@ func restoreItem( itemInfo, err := restoreV6File( ctx, rh, - restoreCfg, + rcc, drivePath, fibn, restoreFolderID, copyBuffer, - restorePerms, caches, itemPath, itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -596,12 +481,11 @@ func restoreV0File( func restoreV1File( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, drivePath *path.DrivePath, fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - restorePerms bool, caches *restoreCaches, itemPath path.Path, itemData data.Stream, @@ -611,7 +495,7 @@ func restoreV1File( itemID, itemInfo, err := restoreFile( ctx, - restoreCfg, + rcc.RestoreConfig, rh, fibn, trimmedName, @@ -627,7 +511,7 @@ func restoreV1File( // Mark it as success without processing .meta // file if we are not restoring permissions - if !restorePerms { + if !rcc.RestoreConfig.IncludePermissions { return itemInfo, nil } @@ -657,12 +541,11 @@ func restoreV1File( func restoreV6File( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, drivePath *path.DrivePath, fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - restorePerms bool, caches *restoreCaches, itemPath path.Path, itemData data.Stream, @@ -696,7 +579,7 @@ func restoreV6File( itemID, itemInfo, err := restoreFile( ctx, - restoreCfg, + rcc.RestoreConfig, rh, fibn, meta.FileName, @@ -712,10 +595,12 @@ func restoreV6File( // Mark it as success without processing .meta // file if we are not restoring permissions - if !restorePerms { + if !rcc.RestoreConfig.IncludePermissions { return itemInfo, nil } + fmt.Printf("\n-----\nrestorev6 %+v\n-----\n", rcc.RestoreConfig.IncludePermissions) + err = RestorePermissions( ctx, rh, @@ -765,6 +650,8 @@ func CreateRestoreFolders( return id, nil } + fmt.Printf("\n-----\ncreatefolders %+v\n-----\n", restorePerms) + err = RestorePermissions( ctx, rh, diff --git a/src/internal/m365/onedrive/restore_caches.go b/src/internal/m365/onedrive/restore_caches.go new file mode 100644 index 000000000..6951a8bfe --- /dev/null +++ b/src/internal/m365/onedrive/restore_caches.go @@ -0,0 +1,116 @@ +package onedrive + +import ( + "context" + "sync" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type driveInfo struct { + id string + name string + rootFolderID string +} + +type restoreCaches struct { + BackupDriveIDName idname.Cacher + collisionKeyToItemID map[string]api.DriveItemIDType + DriveIDToDriveInfo map[string]driveInfo + DriveNameToDriveInfo map[string]driveInfo + Folders *folderCache + OldLinkShareIDToNewID map[string]string + OldPermIDToNewID map[string]string + ParentDirToMeta map[string]metadata.Metadata + + pool sync.Pool +} + +func (rc *restoreCaches) AddDrive( + ctx context.Context, + md models.Driveable, + grf GetRootFolderer, +) error { + di := driveInfo{ + id: ptr.Val(md.GetId()), + name: ptr.Val(md.GetName()), + } + + ctx = clues.Add(ctx, "drive_info", di) + + root, err := grf.GetRootFolder(ctx, di.id) + if err != nil { + return clues.Wrap(err, "getting drive root id") + } + + di.rootFolderID = ptr.Val(root.GetId()) + + rc.DriveIDToDriveInfo[di.id] = di + rc.DriveNameToDriveInfo[di.name] = di + + return nil +} + +// Populate looks up drive items available to the protectedResource +// and adds their info to the caches. +func (rc *restoreCaches) Populate( + ctx context.Context, + gdparf GetDrivePagerAndRootFolderer, + protectedResourceID string, +) error { + drives, err := api.GetAllDrives( + ctx, + gdparf.NewDrivePager(protectedResourceID, nil), + true, + maxDrivesRetries) + if err != nil { + return clues.Wrap(err, "getting drives") + } + + for _, md := range drives { + if err := rc.AddDrive(ctx, md, gdparf); err != nil { + return clues.Wrap(err, "caching drive") + } + } + + return nil +} + +type GetDrivePagerAndRootFolderer interface { + GetRootFolderer + NewDrivePagerer +} + +func NewRestoreCaches( + backupDriveIDNames idname.Cacher, +) *restoreCaches { + // avoid nil panics + if backupDriveIDNames == nil { + backupDriveIDNames = idname.NewCache(nil) + } + + return &restoreCaches{ + BackupDriveIDName: backupDriveIDNames, + collisionKeyToItemID: map[string]api.DriveItemIDType{}, + DriveIDToDriveInfo: map[string]driveInfo{}, + DriveNameToDriveInfo: map[string]driveInfo{}, + Folders: NewFolderCache(), + OldLinkShareIDToNewID: map[string]string{}, + OldPermIDToNewID: map[string]string{}, + ParentDirToMeta: map[string]metadata.Metadata{}, + // Buffer pool for uploads + pool: sync.Pool{ + New: func() any { + b := make([]byte, graph.CopyBufferSize) + return &b + }, + }, + } +} diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index dbb7317c9..301a1b01e 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/onedrive/mock" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" @@ -512,21 +513,25 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { ctr := count.New() + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + RestoreConfig: restoreCfg, + } + _, skip, err := restoreItem( ctx, rh, - restoreCfg, + rcc, mock.FetchItemByName{ Item: &mock.Data{ Reader: mock.FileRespReadCloser(mock.DriveFileMetaData), }, }, - version.Backup, dp, "", make([]byte, graph.CopyBufferSize), caches, - false, &mock.Data{ ID: uuid.NewString(), Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData), diff --git a/src/internal/m365/onedrive/service_test.go b/src/internal/m365/onedrive/service_test.go index a39a65a76..a2766b8ee 100644 --- a/src/internal/m365/onedrive/service_test.go +++ b/src/internal/m365/onedrive/service_test.go @@ -21,7 +21,7 @@ type oneDriveService struct { } func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) { - ac, err := api.NewClient(credentials, control.Defaults()) + ac, err := api.NewClient(credentials, control.DefaultOptions()) if err != nil { return nil, err } diff --git a/src/internal/m365/onedrive/url_cache_test.go b/src/internal/m365/onedrive/url_cache_test.go index 7946da840..bf4f25350 100644 --- a/src/internal/m365/onedrive/url_cache_test.go +++ b/src/internal/m365/onedrive/url_cache_test.go @@ -53,7 +53,7 @@ func (suite *URLCacheIntegrationSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user) diff --git a/src/internal/m365/onedrive_test.go b/src/internal/m365/onedrive_test.go index eade30c9d..ba81a477a 100644 --- a/src/internal/m365/onedrive_test.go +++ b/src/internal/m365/onedrive_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" @@ -223,9 +224,9 @@ func (suite *SharePointIntegrationSuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.Backup) } -func (suite *SharePointIntegrationSuite) TestPermissionsBackupAndNoRestore() { +func (suite *SharePointIntegrationSuite) TestRestoreNoPermissionsAndBackup() { suite.T().Skip("Temporarily disabled due to CI issues") - testPermissionsBackupAndNoRestore(suite, version.Backup) + testRestoreNoPermissionsAndBackup(suite, version.Backup) } func (suite *SharePointIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -290,8 +291,8 @@ func (suite *OneDriveIntegrationSuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.Backup) } -func (suite *OneDriveIntegrationSuite) TestPermissionsBackupAndNoRestore() { - testPermissionsBackupAndNoRestore(suite, version.Backup) +func (suite *OneDriveIntegrationSuite) TestRestoreNoPermissionsAndBackup() { + testRestoreNoPermissionsAndBackup(suite, version.Backup) } func (suite *OneDriveIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -354,8 +355,8 @@ func (suite *OneDriveNightlySuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.OneDrive1DataAndMetaFiles) } -func (suite *OneDriveNightlySuite) TestPermissionsBackupAndNoRestore() { - testPermissionsBackupAndNoRestore(suite, version.OneDrive1DataAndMetaFiles) +func (suite *OneDriveNightlySuite) TestRestoreNoPermissionsAndBackup() { + testRestoreNoPermissionsAndBackup(suite, version.OneDrive1DataAndMetaFiles) } func (suite *OneDriveNightlySuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -517,19 +518,17 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("od_restore_and_backup_multi") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("od_restore_and_backup_multi") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -768,24 +767,22 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("perms_restore_and_backup") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("perms_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } -func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { +func testRestoreNoPermissionsAndBackup(suite oneDriveSuite, startVersion int) { t := suite.T() ctx, flush := tester.NewContext(t) @@ -860,19 +857,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("perms_backup_no_restore") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("perms_backup_no_restore") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = false + + fmt.Printf("\n-----\nrcfg %+v\n-----\n", restoreCfg.IncludePermissions) runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: false, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1067,19 +1064,17 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1264,19 +1259,17 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1383,16 +1376,16 @@ func testRestoreFolderNamedFolderRegression( collectionsLatest: expected, } + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + runRestoreTestWithVersion( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - ) + control.DefaultOptions(), + restoreCfg) }) } } diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index 5d58fdb26..de9e0bb13 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -12,11 +12,11 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/sharepoint" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/path" ) // ConsumeRestoreCollections restores data from the specified collections @@ -24,10 +24,7 @@ import ( // SideEffect: status is updated at the completion of operation func (ctrl *Controller) ConsumeRestoreCollections( ctx context.Context, - backupVersion int, - sels selectors.Selector, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc inject.RestoreConsumerConfig, dcs []data.RestoreCollection, errs *fault.Bus, ctr *count.Bus, @@ -35,48 +32,64 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctx, end := diagnostics.Span(ctx, "m365:restore") defer end() - ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) - ctx = clues.Add(ctx, "restore_config", restoreCfg) + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: rcc.Selector.PathService()}) + ctx = clues.Add(ctx, "restore_config", rcc.RestoreConfig) if len(dcs) == 0 { return nil, clues.New("no data collections to restore") } + serviceEnabled, _, err := checkServiceEnabled( + ctx, + ctrl.AC.Users(), + rcc.Selector.PathService(), + rcc.ProtectedResource.ID()) + if err != nil { + return nil, err + } + + if !serviceEnabled { + return nil, clues.Stack(graph.ErrServiceNotEnabled).WithClues(ctx) + } + var ( - status *support.ControllerOperationStatus - deets = &details.Builder{} - err error + service = rcc.Selector.PathService() + status *support.ControllerOperationStatus + deets = &details.Builder{} ) - switch sels.Service { - case selectors.ServiceExchange: - status, err = exchange.ConsumeRestoreCollections(ctx, ctrl.AC, restoreCfg, dcs, deets, errs, ctr) - case selectors.ServiceOneDrive: + switch service { + case path.ExchangeService: + status, err = exchange.ConsumeRestoreCollections( + ctx, + ctrl.AC, + rcc, + dcs, + deets, + errs, + ctr) + case path.OneDriveService: status, err = onedrive.ConsumeRestoreCollections( ctx, onedrive.NewRestoreHandler(ctrl.AC), - backupVersion, - restoreCfg, - opts, + rcc, ctrl.backupDriveIDNames, dcs, deets, errs, ctr) - case selectors.ServiceSharePoint: + case path.SharePointService: status, err = sharepoint.ConsumeRestoreCollections( ctx, - backupVersion, + rcc, ctrl.AC, - restoreCfg, - opts, ctrl.backupDriveIDNames, dcs, deets, errs, ctr) default: - err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") + err = clues.Wrap(clues.New(service.String()), "service not supported") } ctrl.incrementAwaitingMessages() diff --git a/src/internal/m365/sharepoint/backup_test.go b/src/internal/m365/sharepoint/backup_test.go index 973a55670..348b15dfd 100644 --- a/src/internal/m365/sharepoint/backup_test.go +++ b/src/internal/m365/sharepoint/backup_test.go @@ -107,7 +107,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { tenantID, site, nil, - control.Defaults()) + control.DefaultOptions()) c.CollectionMap = collMap @@ -201,7 +201,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds, control.Defaults()) + ac, err := api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) col, err := collectPages( @@ -210,7 +210,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() { ac, mock.NewProvider(siteID, siteID), &MockGraphService{}, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) assert.NotEmpty(t, col) diff --git a/src/internal/m365/sharepoint/collection_test.go b/src/internal/m365/sharepoint/collection_test.go index 42f9ad9a1..0462a5c8e 100644 --- a/src/internal/m365/sharepoint/collection_test.go +++ b/src/internal/m365/sharepoint/collection_test.go @@ -43,7 +43,7 @@ func (suite *SharePointCollectionSuite) SetupSuite() { suite.creds = m365 - ac, err := api.NewClient(m365, control.Defaults()) + ac, err := api.NewClient(m365, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) suite.ac = ac @@ -168,7 +168,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { suite.ac, test.category, nil, - control.Defaults()) + control.DefaultOptions()) col.data <- test.getItem(t, test.itemName) readItems := []data.Stream{} diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index c38b82e08..bb894f5ea 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive" betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -31,10 +32,8 @@ import ( // ConsumeRestoreCollections will restore the specified data collections into OneDrive func ConsumeRestoreCollections( ctx context.Context, - backupVersion int, + rcc inject.RestoreConsumerConfig, ac api.Client, - restoreCfg control.RestoreConfig, - opts control.Options, backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, @@ -42,14 +41,13 @@ func ConsumeRestoreCollections( ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - lrh = libraryRestoreHandler{ac} - protectedResourceID = dcs[0].FullPath().ResourceOwner() - restoreMetrics support.CollectionMetrics - caches = onedrive.NewRestoreCaches(backupDriveIDNames) - el = errs.Local() + lrh = libraryRestoreHandler{ac} + restoreMetrics support.CollectionMetrics + caches = onedrive.NewRestoreCaches(backupDriveIDNames) + el = errs.Local() ) - err := caches.Populate(ctx, lrh, protectedResourceID) + err := caches.Populate(ctx, lrh, rcc.ProtectedResource.ID()) if err != nil { return nil, clues.Wrap(err, "initializing restore caches") } @@ -70,7 +68,7 @@ func ConsumeRestoreCollections( metrics support.CollectionMetrics ictx = clues.Add(ctx, "category", category, - "restore_location", restoreCfg.Location, + "restore_location", clues.Hide(rcc.RestoreConfig.Location), "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), "full_path", dc.FullPath()) ) @@ -80,12 +78,10 @@ func ConsumeRestoreCollections( metrics, err = onedrive.RestoreCollection( ictx, lrh, - restoreCfg, - backupVersion, + rcc, dc, caches, deets, - opts.RestorePermissions, control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem), errs, ctr) @@ -95,7 +91,7 @@ func ConsumeRestoreCollections( ictx, ac.Stable, dc, - restoreCfg.Location, + rcc.RestoreConfig.Location, deets, errs) @@ -104,7 +100,7 @@ func ConsumeRestoreCollections( ictx, ac.Stable, dc, - restoreCfg.Location, + rcc.RestoreConfig.Location, deets, errs) @@ -128,7 +124,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), restoreMetrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } diff --git a/src/internal/m365/stub/stub.go b/src/internal/m365/stub/stub.go index 601e57722..da3340f60 100644 --- a/src/internal/m365/stub/stub.go +++ b/src/internal/m365/stub/stub.go @@ -68,8 +68,7 @@ func GetCollectionsAndExpected( owner, config.RestoreCfg, testCollections, - backupVersion, - ) + backupVersion) if err != nil { return totalItems, totalKopiaItems, collections, expectedData, err } diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 3aaeae45c..a2783e92e 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -362,7 +362,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() { op, err := NewBackupOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, @@ -1137,7 +1137,7 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } @@ -1147,7 +1147,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { sw = &store.Wrapper{} ctrl = &mock.Controller{} acct = tconfig.NewM365Account(suite.T()) - opts = control.Defaults() + opts = control.DefaultOptions() ) table := []struct { diff --git a/src/internal/operations/export_test.go b/src/internal/operations/export_test.go index c81114da9..10dec2ab1 100644 --- a/src/internal/operations/export_test.go +++ b/src/internal/operations/export_test.go @@ -99,7 +99,7 @@ func (suite *ExportOpSuite) TestExportOperation_PersistResults() { op, err := NewExportOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, diff --git a/src/internal/operations/help_test.go b/src/internal/operations/help_test.go index 0951572ba..bd8509a1a 100644 --- a/src/internal/operations/help_test.go +++ b/src/internal/operations/help_test.go @@ -27,7 +27,7 @@ func ControllerWithSelector( ins idname.Cacher, onFail func(), ) (*m365.Controller, selectors.Selector) { - ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) + ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions()) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail() @@ -36,7 +36,7 @@ func ControllerWithSelector( t.FailNow() } - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail() diff --git a/src/internal/operations/inject/containers.go b/src/internal/operations/inject/containers.go new file mode 100644 index 000000000..f44bb7e66 --- /dev/null +++ b/src/internal/operations/inject/containers.go @@ -0,0 +1,18 @@ +package inject + +import ( + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/selectors" +) + +// RestoreConsumerConfig is a container-of-things for holding options and +// configurations from various packages, which are widely used by all +// restore consumers independent of service or data category. +type RestoreConsumerConfig struct { + BackupVersion int + Options control.Options + ProtectedResource idname.Provider + RestoreConfig control.RestoreConfig + Selector selectors.Selector +} diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index 5f6da9230..0a5e8581c 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -37,10 +37,7 @@ type ( RestoreConsumer interface { ConsumeRestoreCollections( ctx context.Context, - backupVersion int, - selector selectors.Selector, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc RestoreConsumerConfig, dcs []data.RestoreCollection, errs *fault.Bus, ctr *count.Bus, @@ -49,6 +46,7 @@ type ( Wait() *data.CollectionStats CacheItemInfoer + PopulateProtectedResourceIDAndNamer } CacheItemInfoer interface { @@ -76,6 +74,25 @@ type ( CacheItemInfoer } + PopulateProtectedResourceIDAndNamer interface { + // PopulateProtectedResourceIDAndName takes the provided owner identifier and produces + // the owner's name and ID from that value. Returns an error if the owner is + // not recognized by the current tenant. + // + // The id-name cacher should be optional. Some processes will look up all owners in + // the tenant before reaching this step. In that case, the data gets handed + // down for this func to consume instead of performing further queries. The + // data gets stored inside the controller instance for later re-use. + PopulateProtectedResourceIDAndName( + ctx context.Context, + owner string, // input value, can be either id or name + ins idname.Cacher, + ) ( + id, name string, + err error, + ) + } + RepoMaintenancer interface { RepoMaintenance(ctx context.Context, opts repository.Maintenance) error } diff --git a/src/internal/operations/maintenance_test.go b/src/internal/operations/maintenance_test.go index a625af482..4ec6a3ee9 100644 --- a/src/internal/operations/maintenance_test.go +++ b/src/internal/operations/maintenance_test.go @@ -54,7 +54,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() { mo, err := NewMaintenanceOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, repository.Maintenance{ Type: repository.MetadataMaintenance, diff --git a/src/internal/operations/operation_test.go b/src/internal/operations/operation_test.go index 4cbbe2a0c..b615a492e 100644 --- a/src/internal/operations/operation_test.go +++ b/src/internal/operations/operation_test.go @@ -26,7 +26,7 @@ func TestOperationSuite(t *testing.T) { func (suite *OperationSuite) TestNewOperation() { t := suite.T() - op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil) + op := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, nil, nil) assert.Greater(t, op.CreatedAt, time.Time{}) } @@ -46,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() { } for _, test := range table { suite.Run(test.name, func() { - err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() + err := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() test.errCheck(suite.T(), err, clues.ToCore(err)) }) } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 0f853a853..141300f6a 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/events" @@ -172,7 +173,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De logger.CtxErr(ctx, err).Error("running restore") if errors.Is(err, kopia.ErrNoRestorePath) { - op.Errors.Fail(clues.New("empty backup or unknown path provided")) + op.Errors.Fail(clues.Wrap(err, "empty backup or unknown path provided")) } op.Errors.Fail(clues.Wrap(err, "running restore")) @@ -217,7 +218,19 @@ func (op *RestoreOperation) do( return nil, clues.Wrap(err, "getting backup and details") } - observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) + restoreToProtectedResource, err := chooseRestoreResource(ctx, op.rc, op.RestoreCfg, bup.Selector) + if err != nil { + return nil, clues.Wrap(err, "getting destination protected resource") + } + + ctx = clues.Add( + ctx, + "backup_protected_resource_id", bup.Selector.ID(), + "backup_protected_resource_name", clues.Hide(bup.Selector.Name()), + "restore_protected_resource_id", restoreToProtectedResource.ID(), + "restore_protected_resource_name", clues.Hide(restoreToProtectedResource.Name())) + + observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(restoreToProtectedResource.Name())) paths, err := formatDetailsForRestoration( ctx, @@ -232,8 +245,6 @@ func (op *RestoreOperation) do( ctx = clues.Add( ctx, - "resource_owner_id", bup.Selector.ID(), - "resource_owner_name", clues.Hide(bup.Selector.Name()), "details_entries", len(deets.Entries), "details_paths", len(paths), "backup_snapshot_id", bup.SnapshotID, @@ -254,7 +265,12 @@ func (op *RestoreOperation) do( kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository") defer close(kopiaComplete) - dcs, err := op.kopia.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, opStats.bytesRead, op.Errors) + dcs, err := op.kopia.ProduceRestoreCollections( + ctx, + bup.SnapshotID, + paths, + opStats.bytesRead, + op.Errors) if err != nil { return nil, clues.Wrap(err, "producing collections to restore") } @@ -271,6 +287,7 @@ func (op *RestoreOperation) do( ctx, op.rc, bup.Version, + restoreToProtectedResource, op.Selectors, op.RestoreCfg, op.Options, @@ -321,6 +338,24 @@ func (op *RestoreOperation) persistResults( return op.Errors.Failure() } +func chooseRestoreResource( + ctx context.Context, + pprian inject.PopulateProtectedResourceIDAndNamer, + restoreCfg control.RestoreConfig, + orig idname.Provider, +) (idname.Provider, error) { + if len(restoreCfg.ProtectedResource) == 0 { + return orig, nil + } + + id, name, err := pprian.PopulateProtectedResourceIDAndName( + ctx, + restoreCfg.ProtectedResource, + nil) + + return idname.NewProvider(id, name), clues.Stack(err).OrNil() +} + // --------------------------------------------------------------------------- // Restorer funcs // --------------------------------------------------------------------------- @@ -329,6 +364,7 @@ func consumeRestoreCollections( ctx context.Context, rc inject.RestoreConsumer, backupVersion int, + toProtectedResource idname.Provider, sel selectors.Selector, restoreCfg control.RestoreConfig, opts control.Options, @@ -342,15 +378,15 @@ func consumeRestoreCollections( close(complete) }() - deets, err := rc.ConsumeRestoreCollections( - ctx, - backupVersion, - sel, - restoreCfg, - opts, - dcs, - errs, - ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: opts, + ProtectedResource: toProtectedResource, + RestoreConfig: restoreCfg, + Selector: sel, + } + + deets, err := rc.ConsumeRestoreCollections(ctx, rcc, dcs, errs, ctr) if err != nil { return nil, clues.Wrap(err, "restoring collections") } diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 5a314aaf4..5b124ee64 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -37,15 +39,15 @@ import ( // unit // --------------------------------------------------------------------------- -type RestoreOpSuite struct { +type RestoreOpUnitSuite struct { tester.Suite } -func TestRestoreOpSuite(t *testing.T) { - suite.Run(t, &RestoreOpSuite{Suite: tester.NewUnitSuite(t)}) +func TestRestoreOpUnitSuite(t *testing.T) { + suite.Run(t, &RestoreOpUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { +func (suite *RestoreOpUnitSuite) TestRestoreOperation_PersistResults() { var ( kw = &kopia.Wrapper{} sw = &store.Wrapper{} @@ -107,7 +109,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { op, err := NewRestoreOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, @@ -135,6 +137,75 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { } } +func (suite *RestoreOpUnitSuite) TestChooseRestoreResource() { + var ( + id = "id" + name = "name" + cfgWithPR = control.DefaultRestoreConfig(dttm.HumanReadable) + ) + + cfgWithPR.ProtectedResource = "cfgid" + + table := []struct { + name string + cfg control.RestoreConfig + ctrl *mock.Controller + orig idname.Provider + expectErr assert.ErrorAssertionFunc + expectProvider assert.ValueAssertionFunc + expectID string + expectName string + }{ + { + name: "use original", + cfg: control.DefaultRestoreConfig(dttm.HumanReadable), + ctrl: &mock.Controller{ + ProtectedResourceID: id, + ProtectedResourceName: name, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.NoError, + expectID: "oid", + expectName: "oname", + }, + { + name: "look up resource with iface", + cfg: cfgWithPR, + ctrl: &mock.Controller{ + ProtectedResourceID: id, + ProtectedResourceName: name, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.NoError, + expectID: id, + expectName: name, + }, + { + name: "error looking up protected resource", + cfg: cfgWithPR, + ctrl: &mock.Controller{ + ProtectedResourceErr: assert.AnError, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.Error, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + result, err := chooseRestoreResource(ctx, test.ctrl, test.cfg, test.orig) + test.expectErr(t, err, clues.ToCore(err)) + require.NotNil(t, result) + assert.Equal(t, test.expectID, result.ID()) + assert.Equal(t, test.expectName, result.Name()) + }) + } +} + // --------------------------------------------------------------------------- // integration // --------------------------------------------------------------------------- @@ -216,7 +287,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { sw = &store.Wrapper{} ctrl = &mock.Controller{} restoreCfg = testdata.DefaultRestoreConfig("") - opts = control.Defaults() + opts = control.DefaultOptions() ) table := []struct { @@ -275,12 +346,12 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { suite.acct, resource.Users, rsel.PathService(), - control.Defaults()) + control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) ro, err := NewRestoreOperation( ctx, - control.Defaults(), + control.DefaultOptions(), suite.kw, suite.sw, ctrl, diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index e33cdd0ae..ddb16216b 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -67,9 +67,9 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Mail", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) - sel.DiscreteOwner = suite.its.userID + sel.DiscreteOwner = suite.its.user.ID return sel }, @@ -79,7 +79,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Contacts", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch())) return sel }, @@ -89,7 +89,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Calendar Events", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) return sel }, @@ -107,7 +107,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { var ( mb = evmock.NewBus() sel = test.selector().Selector - opts = control.Defaults() + opts = control.DefaultOptions() whatSet = deeTD.CategoryFromRepoRef ) @@ -258,9 +258,9 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr // later on during the tests. Putting their identifiers into the selector // at this point is harmless. containers = []string{container1, container2, container3, containerRename} - sel = selectors.NewExchangeBackup([]string{suite.its.userID}) + sel = selectors.NewExchangeBackup([]string{suite.its.user.ID}) whatSet = deeTD.CategoryFromRepoRef - opts = control.Defaults() + opts = control.DefaultOptions() ) opts.ToggleFeatures = toggles @@ -278,7 +278,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds, control.Defaults()) + ac, err := api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. @@ -295,7 +295,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr mailDBF := func(id, timeStamp, subject, body string) []byte { return exchMock.MessageWith( - suite.its.userID, suite.its.userID, suite.its.userID, + suite.its.user.ID, suite.its.user.ID, suite.its.user.ID, subject, body, body, now, now, now, now) } @@ -312,7 +312,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr eventDBF := func(id, timeStamp, subject, body string) []byte { return exchMock.EventWith( - suite.its.userID, subject, body, body, + suite.its.user.ID, subject, body, body, exchMock.NoOriginalStartDate, now, now, exchMock.NoRecurrence, exchMock.NoAttendees, exchMock.NoAttachments, exchMock.NoCancelledOccurrences, @@ -578,7 +578,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr service, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, - creds.AzureTenantID, suite.its.userID, "", container3, + creds.AzureTenantID, suite.its.user.ID, "", container3, 2, version.Backup, gen.dbf) @@ -897,7 +897,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio // a backup is required to run restores - baseSel := selectors.NewExchangeBackup([]string{suite.its.userID}) + baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) baseSel.Include( // events cannot be run, for the same reason as incremental backups: the user needs // to have their account recycled. @@ -905,11 +905,11 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) - baseSel.DiscreteOwner = suite.its.userID + baseSel.DiscreteOwner = suite.its.user.ID var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() ) bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) @@ -1272,3 +1272,216 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio assert.Len(t, result, 0, "no items should have been added as copies") }) } + +func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtectedResource() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) + baseSel.Include( + // events cannot be run, for the same reason as incremental backups: the user needs + // to have their account recycled. + // base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()), + baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), + baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) + + baseSel.DiscreteOwner = suite.its.user.ID + + var ( + mb = evmock.NewBus() + opts = control.DefaultOptions() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + rsel, err := baseSel.ToExchangeRestore() + require.NoError(t, err, clues.ToCore(err)) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_restore_to_user") + sel = rsel.Selector + userID = suite.its.user.ID + secondaryUserID = suite.its.secondaryUser.ID + uid = userID + acCont = suite.its.ac.Contacts() + acMail = suite.its.ac.Mail() + // acEvts = suite.its.ac.Events() + firstCtr = count.New() + ) + + restoreCfg.OnCollision = control.Copy + mb = evmock.NewBus() + + // first restore to the current user + + ro1, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + firstCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro1, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + + var ( + userItemIDs = map[path.CategoryType]map[string]struct{}{} + userCollisionKeys = map[path.CategoryType]map[string]string{} + ) + + // --- contacts + cat := path.ContactsCategory + userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acCont, + uid, + "", + restoreCfg.Location) + + // --- events + // cat = path.EventsCategory + // userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + // t, + // ctx, + // acEvts, + // uid, + // "", + // restoreCfg.Location) + + // --- mail + cat = path.EmailCategory + userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acMail, + uid, + "", + restoreCfg.Location, + api.MailInbox) + + // then restore to the secondary user + + uid = secondaryUserID + mb = evmock.NewBus() + secondCtr := count.New() + restoreCfg.ProtectedResource = uid + + ro2, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + secondCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro2, mb, false) + + var ( + secondaryItemIDs = map[path.CategoryType]map[string]struct{}{} + secondaryCollisionKeys = map[path.CategoryType]map[string]string{} + ) + + // --- contacts + cat = path.ContactsCategory + secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acCont, + uid, + "", + restoreCfg.Location) + + // --- events + // cat = path.EventsCategory + // secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + // t, + // ctx, + // acEvts, + // uid, + // "", + // restoreCfg.Location) + + // --- mail + cat = path.EmailCategory + secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acMail, + uid, + "", + restoreCfg.Location, + api.MailInbox) + + // compare restore results + for _, cat := range []path.CategoryType{path.ContactsCategory, path.EmailCategory, path.EventsCategory} { + assert.Equal(t, len(userItemIDs[cat]), len(secondaryItemIDs[cat])) + assert.ElementsMatch(t, maps.Keys(userCollisionKeys[cat]), maps.Keys(secondaryCollisionKeys[cat])) + } +} + +type GetItemsKeysAndContainerByNameer interface { + GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, + ) (map[string]struct{}, error) + GetContainerByName( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + userID, containerID string, + ) (map[string]string, error) +} + +func getCollKeysAndItemIDs( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + gikacbn GetItemsKeysAndContainerByNameer, + userID, parentContainerID string, + containerNames ...string, +) (map[string]struct{}, map[string]string) { + var ( + c graph.Container + err error + cID string + ) + + for _, cn := range containerNames { + pcid := parentContainerID + + if len(cID) != 0 { + pcid = cID + } + + c, err = gikacbn.GetContainerByName(ctx, userID, pcid, cn) + require.NoError(t, err, clues.ToCore(err)) + + cID = ptr.Val(c.GetId()) + } + + itemIDs, err := gikacbn.GetItemIDsInContainer(ctx, userID, cID) + require.NoError(t, err, clues.ToCore(err)) + + collisionKeys, err := gikacbn.GetItemsInContainerByCollisionKey(ctx, userID, cID) + require.NoError(t, err, clues.ToCore(err)) + + return itemIDs, collisionKeys +} diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index f1da62cbe..f1bf65261 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -25,6 +25,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" @@ -400,6 +401,7 @@ func generateContainerOfItems( restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) restoreCfg.Location = destFldr + restoreCfg.IncludePermissions = true dataColls := buildCollections( t, @@ -408,15 +410,19 @@ func generateContainerOfItems( restoreCfg, collections) - opts := control.Defaults() - opts.RestorePermissions = true + opts := control.DefaultOptions() + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } deets, err := ctrl.ConsumeRestoreCollections( ctx, - backupVersion, - sel, - restoreCfg, - opts, + rcc, dataColls, fault.New(true), count.New()) @@ -535,7 +541,7 @@ func ControllerWithSelector( ins idname.Cacher, onFail func(*testing.T, context.Context), ) (*m365.Controller, selectors.Selector) { - ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) + ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions()) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail(t, ctx) @@ -544,7 +550,7 @@ func ControllerWithSelector( t.FailNow() } - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail(t, ctx) @@ -562,15 +568,19 @@ func ControllerWithSelector( // Suite Setup // --------------------------------------------------------------------------- +type ids struct { + ID string + DriveID string + DriveRootFolderID string +} + type intgTesterSetup struct { - ac api.Client - gockAC api.Client - userID string - userDriveID string - userDriveRootFolderID string - siteID string - siteDriveID string - siteDriveRootFolderID string + ac api.Client + gockAC api.Client + user ids + secondaryUser ids + site ids + secondarySite ids } func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { @@ -585,43 +595,58 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds, control.Defaults()) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) require.NoError(t, err, clues.ToCore(err)) - // user drive - - its.userID = tconfig.M365UserID(t) - - userDrive, err := its.ac.Users().GetDefaultDrive(ctx, its.userID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveID = ptr.Val(userDrive.GetId()) - - userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.userDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveRootFolderID = ptr.Val(userDriveRootFolder.GetId()) - - its.siteID = tconfig.M365SiteID(t) - - // site - - siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, its.siteID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveID = ptr.Val(siteDrive.GetId()) - - siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.siteDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) + its.user = userIDs(t, tconfig.M365UserID(t), its.ac) + its.secondaryUser = userIDs(t, tconfig.SecondaryM365UserID(t), its.ac) + its.site = siteIDs(t, tconfig.M365SiteID(t), its.ac) + its.secondarySite = siteIDs(t, tconfig.SecondaryM365SiteID(t), its.ac) return its } +func userIDs(t *testing.T, id string, ac api.Client) ids { + ctx, flush := tester.NewContext(t) + defer flush() + + r := ids{ID: id} + + drive, err := ac.Users().GetDefaultDrive(ctx, id) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveID = ptr.Val(drive.GetId()) + + driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId()) + + return r +} + +func siteIDs(t *testing.T, id string, ac api.Client) ids { + ctx, flush := tester.NewContext(t) + defer flush() + + r := ids{ID: id} + + drive, err := ac.Sites().GetDefaultDrive(ctx, id) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveID = ptr.Val(drive.GetId()) + + driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId()) + + return r +} + func getTestExtensionFactories() []extensions.CreateItemExtensioner { return []extensions.CreateItemExtensioner{ &extensions.MockItemExtensionFactory{}, diff --git a/src/internal/operations/test/onedrive_test.go b/src/internal/operations/test/onedrive_test.go index 41aab489d..b5057be31 100644 --- a/src/internal/operations/test/onedrive_test.go +++ b/src/internal/operations/test/onedrive_test.go @@ -72,7 +72,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { osel = selectors.NewOneDriveBackup([]string{userID}) ws = deeTD.DriveIDFromRepoRef svc = path.OneDriveService - opts = control.Defaults() + opts = control.DefaultOptions() ) osel.Include(selTD.OneDriveBackupFolderScope(osel)) @@ -106,7 +106,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { } func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { - sel := selectors.NewOneDriveRestore([]string{suite.its.userID}) + sel := selectors.NewOneDriveRestore([]string{suite.its.user.ID}) ic := func(cs []string) selectors.Selector { sel.Include(sel.Folders(cs, selectors.PrefixMatch())) @@ -117,10 +117,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { t *testing.T, ctx context.Context, ) string { - d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.userID) + d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.user.ID) if err != nil { err = graph.Wrap(ctx, err, "retrieving default user drive"). - With("user", suite.its.userID) + With("user", suite.its.user.ID) } require.NoError(t, err, clues.ToCore(err)) @@ -137,8 +137,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { runDriveIncrementalTest( suite, - suite.its.userID, - suite.its.userID, + suite.its.user.ID, + suite.its.user.ID, resource.Users, path.OneDriveService, path.FilesCategory, @@ -166,7 +166,7 @@ func runDriveIncrementalTest( var ( acct = tconfig.NewM365Account(t) - opts = control.Defaults() + opts = control.DefaultOptions() mb = evmock.NewBus() ws = deeTD.DriveIDFromRepoRef @@ -683,7 +683,7 @@ func runDriveIncrementalTest( } for _, test := range table { suite.Run(test.name, func() { - cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.Defaults()) + cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) bod.ctrl = cleanCtrl @@ -785,7 +785,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { var ( acct = tconfig.NewM365Account(t) - opts = control.Defaults() + opts = control.DefaultOptions() mb = evmock.NewBus() categories = map[path.CategoryType][]string{ @@ -801,10 +801,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { acct, resource.Users, path.OneDriveService, - control.Defaults()) + control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) - userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.userID) + userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID) require.NoError(t, err, clues.ToCore(err)) uid := ptr.Val(userable.GetId()) @@ -922,7 +922,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() { osel = selectors.NewOneDriveBackup([]string{userID}) ws = deeTD.DriveIDFromRepoRef svc = path.OneDriveService - opts = control.Defaults() + opts = control.DefaultOptions() ) opts.ItemExtensionFactory = getTestExtensionFactories() @@ -982,17 +982,17 @@ func (suite *OneDriveRestoreIntgSuite) SetupSuite() { } func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() { - sel := selectors.NewOneDriveBackup([]string{suite.its.userID}) + sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID}) sel.Include(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = suite.its.userID + sel.DiscreteOwner = suite.its.user.ID runDriveRestoreWithAdvancedOptions( suite.T(), suite, suite.its.ac, sel.Selector, - suite.its.userDriveID, - suite.its.userDriveRootFolderID) + suite.its.user.DriveID, + suite.its.user.DriveRootFolderID) } func runDriveRestoreWithAdvancedOptions( @@ -1009,7 +1009,7 @@ func runDriveRestoreWithAdvancedOptions( var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() ) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) @@ -1250,3 +1250,173 @@ func runDriveRestoreWithAdvancedOptions( assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy") }) } + +func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveAlternateProtectedResource() { + sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = suite.its.user.ID + + runDriveRestoreToAlternateProtectedResource( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.user, + suite.its.secondaryUser) +} + +func runDriveRestoreToAlternateProtectedResource( + t *testing.T, + suite tester.Suite, + ac api.Client, + sel selectors.Selector, // owner should match 'from', both Restore and Backup types work. + from, to ids, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + var ( + mb = evmock.NewBus() + opts = control.DefaultOptions() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("drive_restore_to_resource") + fromCollisionKeys map[string]api.DriveItemIDType + fromItemIDs map[string]api.DriveItemIDType + acd = ac.Drives() + ) + + // first restore to the 'from' resource + + suite.Run("restore original resource", func() { + mb = evmock.NewBus() + fromCtr := count.New() + driveID := from.DriveID + rootFolderID := from.DriveRootFolderID + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + fromCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + fromItemIDs, fromCollisionKeys = getDriveCollKeysAndItemIDs( + t, + ctx, + acd, + driveID, + rootFolderID, + restoreCfg.Location, + selTD.TestFolderName) + }) + + // then restore to the 'to' resource + var ( + toCollisionKeys map[string]api.DriveItemIDType + toItemIDs map[string]api.DriveItemIDType + ) + + suite.Run("restore to alternate resource", func() { + mb = evmock.NewBus() + toCtr := count.New() + driveID := to.DriveID + rootFolderID := to.DriveRootFolderID + restoreCfg.ProtectedResource = to.ID + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + toCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + toItemIDs, toCollisionKeys = getDriveCollKeysAndItemIDs( + t, + ctx, + acd, + driveID, + rootFolderID, + restoreCfg.Location, + selTD.TestFolderName) + }) + + // compare restore results + assert.Equal(t, len(fromItemIDs), len(toItemIDs)) + assert.ElementsMatch(t, maps.Keys(fromCollisionKeys), maps.Keys(toCollisionKeys)) +} + +type GetItemsKeysAndFolderByNameer interface { + GetItemIDsInContainer( + ctx context.Context, + driveID, containerID string, + ) (map[string]api.DriveItemIDType, error) + GetFolderByName( + ctx context.Context, + driveID, parentFolderID, folderName string, + ) (models.DriveItemable, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + driveID, containerID string, + ) (map[string]api.DriveItemIDType, error) +} + +func getDriveCollKeysAndItemIDs( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + gikafbn GetItemsKeysAndFolderByNameer, + driveID, parentContainerID string, + containerNames ...string, +) (map[string]api.DriveItemIDType, map[string]api.DriveItemIDType) { + var ( + c models.DriveItemable + err error + cID string + ) + + for _, cn := range containerNames { + pcid := parentContainerID + + if len(cID) != 0 { + pcid = cID + } + + c, err = gikafbn.GetFolderByName(ctx, driveID, pcid, cn) + require.NoError(t, err, clues.ToCore(err)) + + cID = ptr.Val(c.GetId()) + } + + itemIDs, err := gikafbn.GetItemIDsInContainer(ctx, driveID, cID) + require.NoError(t, err, clues.ToCore(err)) + + collisionKeys, err := gikafbn.GetItemsInContainerByCollisionKey(ctx, driveID, cID) + require.NoError(t, err, clues.ToCore(err)) + + return itemIDs, collisionKeys +} diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index ad2e5d79a..08cc4cb1c 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -49,7 +49,7 @@ func (suite *SharePointBackupIntgSuite) SetupSuite() { } func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { - sel := selectors.NewSharePointRestore([]string{suite.its.siteID}) + sel := selectors.NewSharePointRestore([]string{suite.its.site.ID}) ic := func(cs []string) selectors.Selector { sel.Include(sel.LibraryFolders(cs, selectors.PrefixMatch())) @@ -60,10 +60,10 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { t *testing.T, ctx context.Context, ) string { - d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.siteID) + d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.site.ID) if err != nil { err = graph.Wrap(ctx, err, "retrieving default site drive"). - With("site", suite.its.siteID) + With("site", suite.its.site.ID) } require.NoError(t, err, clues.ToCore(err)) @@ -80,8 +80,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { runDriveIncrementalTest( suite, - suite.its.siteID, - suite.its.userID, + suite.its.site.ID, + suite.its.user.ID, resource.Sites, path.SharePointService, path.LibrariesCategory, @@ -91,7 +91,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { true) } -func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { +func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -99,8 +99,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { var ( mb = evmock.NewBus() - sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) - opts = control.Defaults() + sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) + opts = control.DefaultOptions() ) sel.Include(selTD.SharePointBackupFolderScope(sel)) @@ -116,7 +116,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { bod.sw, &bo, bod.sel, - suite.its.siteID, + bod.sel.ID(), path.LibrariesCategory) } @@ -128,8 +128,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { var ( mb = evmock.NewBus() - sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) - opts = control.Defaults() + sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) + opts = control.DefaultOptions() tenID = tconfig.M365TenantID(t) svc = path.SharePointService ws = deeTD.DriveIDFromRepoRef @@ -150,7 +150,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { bod.sw, &bo, bod.sel, - suite.its.siteID, + bod.sel.ID(), path.LibrariesCategory) bID := bo.Results.BackupID @@ -201,18 +201,33 @@ func (suite *SharePointRestoreIntgSuite) SetupSuite() { } func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { - sel := selectors.NewSharePointBackup([]string{suite.its.siteID}) + sel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Filter(sel.Library("documents")) - sel.DiscreteOwner = suite.its.siteID + sel.DiscreteOwner = suite.its.site.ID runDriveRestoreWithAdvancedOptions( suite.T(), suite, suite.its.ac, sel.Selector, - suite.its.siteDriveID, - suite.its.siteDriveRootFolderID) + suite.its.site.DriveID, + suite.its.site.DriveRootFolderID) +} + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointAlternateProtectedResource() { + sel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) + sel.Include(selTD.SharePointBackupFolderScope(sel)) + sel.Filter(sel.Library("documents")) + sel.DiscreteOwner = suite.its.site.ID + + runDriveRestoreToAlternateProtectedResource( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.site, + suite.its.secondarySite) } func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives() { @@ -229,7 +244,7 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives rc.OnCollision = control.Copy // create a new drive - md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.siteID, rc.Location) + md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.site.ID, rc.Location) require.NoError(t, err, clues.ToCore(err)) driveID := ptr.Val(md.GetId()) @@ -260,14 +275,14 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives // run a backup var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() graphClient = suite.its.ac.Stable.Client() ) - bsel := selectors.NewSharePointBackup([]string{suite.its.siteID}) + bsel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) bsel.Include(selTD.SharePointBackupFolderScope(bsel)) bsel.Filter(bsel.Library(rc.Location)) - bsel.DiscreteOwner = suite.its.siteID + bsel.DiscreteOwner = suite.its.site.ID bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup) defer bod.close(t, ctx) @@ -367,7 +382,7 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives pgr := suite.its.ac. Drives(). - NewSiteDrivePager(suite.its.siteID, []string{"id", "name"}) + NewSiteDrivePager(suite.its.site.ID, []string{"id", "name"}) drives, err := api.GetAllDrives(ctx, pgr, false, -1) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/tester/tconfig/config.go b/src/internal/tester/tconfig/config.go index 963f4f6b4..fe8597da9 100644 --- a/src/internal/tester/tconfig/config.go +++ b/src/internal/tester/tconfig/config.go @@ -23,6 +23,7 @@ const ( // M365 config TestCfgAzureTenantID = "azure_tenantid" + TestCfgSecondarySiteID = "secondarym365siteid" TestCfgSiteID = "m365siteid" TestCfgSiteURL = "m365siteurl" TestCfgUserID = "m365userid" @@ -36,13 +37,14 @@ const ( // test specific env vars const ( + EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID" + EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS" EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL" EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" + EnvCorsoSecondaryM365TestSiteID = "CORSO_SECONDARY_M365_TEST_SITE_ID" EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID" - EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID" - EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS" EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE" EnvCorsoUnlicensedM365TestUserID = "CORSO_M365_TEST_UNLICENSED_USER" ) @@ -147,13 +149,19 @@ func ReadTestConfig() (map[string]string, error) { TestCfgSiteID, os.Getenv(EnvCorsoM365TestSiteID), vpr.GetString(TestCfgSiteID), - "10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") + "4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") fallbackTo( testEnv, TestCfgSiteURL, os.Getenv(EnvCorsoM365TestSiteURL), vpr.GetString(TestCfgSiteURL), "https://10rqc2.sharepoint.com/sites/CorsoCI") + fallbackTo( + testEnv, + TestCfgSecondarySiteID, + os.Getenv(EnvCorsoSecondaryM365TestSiteID), + vpr.GetString(TestCfgSecondarySiteID), + "053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c") fallbackTo( testEnv, TestCfgUnlicensedUserID, diff --git a/src/internal/tester/tconfig/protected_resources.go b/src/internal/tester/tconfig/protected_resources.go index b9e31ce06..bd2fded46 100644 --- a/src/internal/tester/tconfig/protected_resources.go +++ b/src/internal/tester/tconfig/protected_resources.go @@ -198,6 +198,17 @@ func GetM365SiteID(ctx context.Context) string { return strings.ToLower(cfg[TestCfgSiteID]) } +// SecondaryM365SiteID returns a siteID string representing the secondarym365SiteID described +// by either the env var CORSO_SECONDARY_M365_TEST_SITE_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func SecondaryM365SiteID(t *testing.T) string { + cfg, err := ReadTestConfig() + require.NoError(t, err, "retrieving secondary m365 site id from test configuration: %+v", clues.ToCore(err)) + + return strings.ToLower(cfg[TestCfgSecondarySiteID]) +} + // UnlicensedM365UserID returns an userID string representing the m365UserID // described by either the env var CORSO_M365_TEST_UNLICENSED_USER, the // corso_test.toml config file or the default value (in that order of priority). diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index fbb3d08a9..01c88b5eb 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -15,7 +15,6 @@ type Options struct { ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` Parallelism Parallelism `json:"parallelism"` Repo repository.Options `json:"repo"` - RestorePermissions bool `json:"restorePermissions"` SkipReduce bool `json:"skipReduce"` ToggleFeatures Toggles `json:"toggleFeatures"` } @@ -38,8 +37,8 @@ const ( BestEffort FailurePolicy = "best-effort" ) -// Defaults provides an Options with the default values set. -func Defaults() Options { +// DefaultOptions provides an Options with the default values set. +func DefaultOptions() Options { return Options{ FailureHandling: FailAfterRecovery, DeltaPageSize: 500, diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 79d49ae20..c30b7d177 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -61,6 +61,10 @@ type RestoreConfig struct { // up. // Defaults to empty. Drive string `json:"drive"` + + // IncludePermissions toggles whether the restore will include the original + // folder- and item-level permissions. + IncludePermissions bool `json:"includePermissions"` } func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { @@ -120,10 +124,11 @@ func (rc RestoreConfig) marshal() string { func (rc RestoreConfig) concealed() RestoreConfig { return RestoreConfig{ - OnCollision: rc.OnCollision, - ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(), - Location: path.LoggableDir(rc.Location), - Drive: clues.Hide(rc.Drive).Conceal(), + OnCollision: rc.OnCollision, + ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(), + Location: path.LoggableDir(rc.Location), + Drive: clues.Hide(rc.Drive).Conceal(), + IncludePermissions: rc.IncludePermissions, } } diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 1e344bc7a..b26a6a2ef 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -335,7 +335,7 @@ func (r repository) NewBackupWithLookup( return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365") } - ownerID, ownerName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + ownerID, ownerName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if err != nil { return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details") } diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 80c2b71f6..34c55d4e7 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -60,7 +60,7 @@ func (suite *RepositoryUnitSuite) TestInitialize() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Initialize(ctx, test.account, st, control.Defaults()) + _, err = Initialize(ctx, test.account, st, control.DefaultOptions()) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -94,7 +94,7 @@ func (suite *RepositoryUnitSuite) TestConnect() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Connect(ctx, test.account, st, "not_found", control.Defaults()) + _, err = Connect(ctx, test.account, st, "not_found", control.DefaultOptions()) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -137,7 +137,7 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { defer flush() st := test.storage(t) - r, err := Initialize(ctx, test.account, st, control.Defaults()) + r, err := Initialize(ctx, test.account, st, control.DefaultOptions()) if err == nil { defer func() { err := r.Close(ctx) @@ -186,11 +186,11 @@ func (suite *RepositoryIntegrationSuite) TestConnect() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) // now re-connect - _, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.Defaults()) + _, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.DefaultOptions()) assert.NoError(t, err, clues.ToCore(err)) } @@ -203,7 +203,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + r, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) oldID := r.GetID() @@ -212,7 +212,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { require.NoError(t, err, clues.ToCore(err)) // now re-connect - r, err = Connect(ctx, account.Account{}, st, oldID, control.Defaults()) + r, err = Connect(ctx, account.Account{}, st, oldID, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) assert.Equal(t, oldID, r.GetID()) } @@ -228,7 +228,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) userID := tconfig.M365UserID(t) @@ -250,7 +250,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg) @@ -269,7 +269,7 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{}) @@ -286,7 +286,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err) // now re-connect @@ -308,14 +308,14 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { { name: "default options", opts: func() control.Options { - return control.Defaults() + return control.DefaultOptions() }, expectedLen: 0, }, { name: "options with an extension factory", opts: func() control.Options { - o := control.Defaults() + o := control.DefaultOptions() o.ItemExtensionFactory = append( o.ItemExtensionFactory, &extensions.MockItemExtensionFactory{}) @@ -327,7 +327,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { { name: "options with multiple extension factories", opts: func() control.Options { - o := control.Defaults() + o := control.DefaultOptions() f := []extensions.CreateItemExtensioner{ &extensions.MockItemExtensionFactory{}, &extensions.MockItemExtensionFactory{}, diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index 8a98a5d56..cc97caa45 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -97,7 +97,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds, control.Defaults()) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) diff --git a/src/pkg/services/m365/api/sites_test.go b/src/pkg/services/m365/api/sites_test.go index d8f49614d..8c4ccf17f 100644 --- a/src/pkg/services/m365/api/sites_test.go +++ b/src/pkg/services/m365/api/sites_test.go @@ -143,12 +143,16 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { var ( t = suite.T() siteID = tconfig.M365SiteID(t) - host = strings.Split(siteID, ",")[0] - shortID = strings.TrimPrefix(siteID, host+",") + parts = strings.Split(siteID, ",") + uuids = siteID siteURL = tconfig.M365SiteURL(t) modifiedSiteURL = siteURL + "foo" ) + if len(parts) == 3 { + uuids = strings.Join(parts[1:], ",") + } + sitesAPI := suite.its.ac.Sites() table := []struct { @@ -165,7 +169,7 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { }, { name: "2 part id", - id: shortID, + id: uuids, expectErr: func(t *testing.T, err error) { assert.NoError(t, err, clues.ToCore(err)) }, @@ -191,13 +195,6 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { assert.NoError(t, err, clues.ToCore(err)) }, }, - { - name: "host only", - id: host, - expectErr: func(t *testing.T, err error) { - assert.NoError(t, err, clues.ToCore(err)) - }, - }, { name: "malformed url", id: "barunihlda", diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index 9dd803cf5..5b61885e5 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -329,7 +329,7 @@ func makeAC( return api.Client{}, clues.Wrap(err, "getting m365 account creds") } - cli, err := api.NewClient(creds, control.Defaults()) + cli, err := api.NewClient(creds, control.DefaultOptions()) if err != nil { return api.Client{}, clues.Wrap(err, "constructing api client") } diff --git a/website/docs/setup/restore-options.md b/website/docs/setup/restore-options.md index 9072b2d3f..13265c378 100644 --- a/website/docs/setup/restore-options.md +++ b/website/docs/setup/restore-options.md @@ -10,7 +10,7 @@ manner to a new folder. When you need more control over the results you can use the advanced configuration options to change where and how your data gets restored. -## Destination +## Restore to target folder The `--destination` flag lets you select the top-level folder where Corso will write all of the restored data. @@ -18,7 +18,7 @@ write all of the restored data. ### The default destination { - `corso restore onedrive --backup abcd` + `corso restore onedrive --backup a422895c-c20c-4b06-883d-b866db9f86ef` } If the flag isn't provided, Corso will create a new folder with a standard name: @@ -29,7 +29,7 @@ data integrity then this is always the safest option. ### An alternate destination { - `corso restore onedrive --backup abcd --destination /my-latest-restore` + `corso restore onedrive --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef` } When a destination is manually specified, all restored will appear in that top-level @@ -41,14 +41,14 @@ folder multiple times. ### The original location { - `corso restore onedrive --backup abcd --destination /` + `corso restore onedrive --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } You can restore items back to their original location by setting the destination to `/`. This skips the creation of a top-level folder, and all restored items will appear back in their location at the time of backup. -### Limitations +### Destination Limitations * Destination won't create N-depth folder structures. `--destination a/b/c` doesn't create three folders; it creates a single, top-level folder named `a/b/c`. @@ -79,19 +79,19 @@ it still collides. Collisions can be handled with three different configurations: `Skip`, `Copy`, and `Replace`. -## Skip (default) +### Skip (default) { - `corso restore onedrive --backup abcd --collisions skip --destination /` + `corso restore onedrive --collisions skip --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } When a collision is identified, the item is skipped and no restore is attempted. -## Copy +### Copy { - `corso restore onedrive --backup abcd --collisions copy --destination /my-latest-restore` + `corso restore onedrive --collisions copy --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef` } Item collisions create a copy of the item in the backup. The copy holds the backup @@ -99,12 +99,31 @@ version of the item, leaving the current version unchanged. If necessary, change item properties (such as filenames) to avoid additional collisions. Eg: the copy of`reports.txt` is named `reports 1.txt`. -## Replace +### Replace { - `corso restore onedrive --backup abcd --collisions replace --destination /` + `corso restore onedrive --collisions replace --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } Collisions will entirely replace the current version of the item with the backup version. If multiple existing items collide with the backup item, only one of the existing items is replaced. + +## Restore to target resource + +The `--to-resource` flag lets you select which resource will receive the restored data. +A resource can be a mailbox, user, sharepoint site, or other owner of data. + +When restoring to a target resource, all other restore configuration behaves normally. +Data is restored into the default folder: `Corso_Restore_` (unless a +`--destination` flag is added). When restoring in-place, collision policies are followed. + +{ + `corso restore onedrive --to-resource adelev@alcion.ai --backup a422895c-c20c-4b06-883d-b866db9f86ef` +} + +### Resource Limitations + +* The resource must exist. Corso won't create new mailboxes, users, or sites. +* The resource must have access to the service being restored. No restore will be +performed for an unlicensed resource. From 0c25c568c1a0bc3bd9d89be77181f1ad8c688845 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 26 Jul 2023 14:46:30 -0600 Subject: [PATCH 41/62] use proper resource ids in factory (#3912) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :bug: Bugfix #### Test Plan - [x] :green_heart: E2E --- .github/actions/slack-message/action.yml | 3 ++- src/cmd/factory/impl/common.go | 20 +++++++++++++++----- src/cmd/factory/impl/onedrive.go | 2 +- src/cmd/factory/impl/sharepoint.go | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/actions/slack-message/action.yml b/.github/actions/slack-message/action.yml index 1b72d9bee..57091d430 100644 --- a/.github/actions/slack-message/action.yml +++ b/.github/actions/slack-message/action.yml @@ -31,6 +31,7 @@ runs: - name: use url or blank val shell: bash run: | + echo "STEP=${{ github.action || '' }}" >> $GITHUB_ENV echo "JOB=${{ github.job || '' }}" >> $GITHUB_ENV echo "LOGS=${{ github.run_id && env.logurl || '-' }}" >> $GITHUB_ENV echo "COMMIT=${{ github.sha && env.commiturl || '-' }}" >> $GITHUB_ENV @@ -50,7 +51,7 @@ runs: "type": "section", "text": { "type": "mrkdwn", - "text": "${{ inputs.msg }} :: ${{ env.JOB }}\n${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}" + "text": "${{ inputs.msg }} :: ${{ env.JOB }} - ${{ env.STEP }}\n${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}" } } ] diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index 837063b40..5904e09d4 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -224,7 +224,8 @@ var ( func generateAndRestoreDriveItems( ctrl *m365.Controller, - resourceOwner, secondaryUserID, secondaryUserName string, + protectedResource idname.Provider, + secondaryUserID, secondaryUserName string, acct account.Account, service path.ServiceType, cat path.CategoryType, @@ -248,14 +249,23 @@ func generateAndRestoreDriveItems( switch service { case path.SharePointService: - d, err := ctrl.AC.Stable.Client().Sites().BySiteId(resourceOwner).Drive().Get(ctx, nil) + d, err := ctrl.AC.Stable. + Client(). + Sites(). + BySiteId(protectedResource.ID()). + Drive(). + Get(ctx, nil) if err != nil { return nil, clues.Wrap(err, "getting site's default drive") } driveID = ptr.Val(d.GetId()) default: - d, err := ctrl.AC.Stable.Client().Users().ByUserId(resourceOwner).Drive().Get(ctx, nil) + d, err := ctrl.AC.Stable.Client(). + Users(). + ByUserId(protectedResource.ID()). + Drive(). + Get(ctx, nil) if err != nil { return nil, clues.Wrap(err, "getting user's default drive") } @@ -423,7 +433,7 @@ func generateAndRestoreDriveItems( Resource: resource.Users, Service: service, Tenant: tenantID, - ResourceOwners: []string{resourceOwner}, + ResourceOwners: []string{protectedResource.ID()}, RestoreCfg: restoreCfg, } @@ -438,7 +448,7 @@ func generateAndRestoreDriveItems( rcc := inject.RestoreConsumerConfig{ BackupVersion: version.Backup, Options: opts, - ProtectedResource: sel, + ProtectedResource: protectedResource, RestoreConfig: restoreCfg, Selector: sel, } diff --git a/src/cmd/factory/impl/onedrive.go b/src/cmd/factory/impl/onedrive.go index e647863dd..6ccc98977 100644 --- a/src/cmd/factory/impl/onedrive.go +++ b/src/cmd/factory/impl/onedrive.go @@ -47,7 +47,7 @@ func handleOneDriveFileFactory(cmd *cobra.Command, args []string) error { deets, err := generateAndRestoreDriveItems( ctrl, - inp.ID(), + inp, SecondaryUser, strings.ToLower(SecondaryUser), acct, diff --git a/src/cmd/factory/impl/sharepoint.go b/src/cmd/factory/impl/sharepoint.go index eeec57c95..ab9d3fc92 100644 --- a/src/cmd/factory/impl/sharepoint.go +++ b/src/cmd/factory/impl/sharepoint.go @@ -47,7 +47,7 @@ func handleSharePointLibraryFileFactory(cmd *cobra.Command, args []string) error deets, err := generateAndRestoreDriveItems( ctrl, - inp.ID(), + inp, SecondaryUser, strings.ToLower(SecondaryUser), acct, From 046a471077dfe23b9f0654941310b2c667c8ecae Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Wed, 26 Jul 2023 15:15:37 -0700 Subject: [PATCH 42/62] Delete details snapshot (#3914) Check for a populated StreamStoreID and fallback to the DetailsID if necessary. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * closes #3913 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/pkg/repository/repository.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index b26a6a2ef..1417d872d 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -616,8 +616,13 @@ func deleteBackup( } } - if len(b.DetailsID) > 0 { - if err := kw.DeleteSnapshot(ctx, b.DetailsID); err != nil { + ssid := b.StreamStoreID + if len(ssid) == 0 { + ssid = b.DetailsID + } + + if len(ssid) > 0 { + if err := kw.DeleteSnapshot(ctx, ssid); err != nil { return err } } From 578a720c678187f62dd3db1dbc220b442e8b2b7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 05:15:18 +0000 Subject: [PATCH 43/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20mermaid=20fro?= =?UTF-8?q?m=2010.2.4=20to=2010.3.0=20in=20/website=20(#3917)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [mermaid](https://github.com/mermaid-js/mermaid) from 10.2.4 to 10.3.0.
Release notes

Sourced from mermaid's releases.

10.3.0

What's Changed

Features

Bugfixes

Chores

Documentation

... (truncated)

Commits
  • 165e0b6 Merge pull request #4669 from mermaid-js/release/10.3.0
  • 9c92f77 Updates after testing and new version for mermaid-zenuml
  • aa54127 Set proper release version
  • 21b90cb Add release version
  • e1ca108 Update version
  • 29d078c Merge branch 'master' into release/10.3.0
  • 40d30b7 Merge remote-tracking branch 'origin/release/10.3.2' into develop
  • dbee34a fix: edgeLabel width
  • 68909a4 Update lockfile
  • d178626 Merge branch 'develop' into release/10.4.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=mermaid&package-manager=npm_and_yarn&previous-version=10.2.4&new-version=10.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- website/package-lock.json | 144 ++++++++++++++++++++++++++++++++++---- website/package.json | 2 +- 2 files changed, 130 insertions(+), 16 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 54b366dae..23f62b662 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -20,7 +20,7 @@ "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", - "mermaid": "^10.2.4", + "mermaid": "^10.3.0", "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", @@ -3377,6 +3377,24 @@ "@types/node": "*" } }, + "node_modules/@types/d3-scale": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.3.tgz", + "integrity": "sha512-PATBiMCpvHJSMtZAMEhc2WyL+hnzarKzI6wAHYjhsonjWJYGq5BXTzQjv4l8m2jO183/4wZ90rKvSeT7o72xNQ==", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" + }, + "node_modules/@types/d3-time": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", + "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" + }, "node_modules/@types/debug": { "version": "4.1.8", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.8.tgz", @@ -6079,6 +6097,41 @@ "node": ">=12" } }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==" + }, "node_modules/d3-scale": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", @@ -6612,9 +6665,9 @@ } }, "node_modules/dompurify": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.3.tgz", - "integrity": "sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.5.tgz", + "integrity": "sha512-F9e6wPGtY+8KNMRAVfxeCOHU0/NPWMSENNq4pQctuXRqqdEPW7q3CrLbR5Nse044WwacyjHGOMlvNsBe1y6z9A==" }, "node_modules/domutils": { "version": "2.8.0", @@ -9279,18 +9332,21 @@ } }, "node_modules/mermaid": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.2.4.tgz", - "integrity": "sha512-zHGjEI7lBvWZX+PQYmlhSA2p40OzW6QbGodTCSzDeVpqaTnyAC+2sRGqrpXO+uQk3CnoeClHQPraQUMStdqy2g==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.3.0.tgz", + "integrity": "sha512-H5quxuQjwXC8M1WuuzhAp2TdqGg74t5skfDBrNKJ7dt3z8Wprl5S6h9VJsRhoBUTSs1TMtHEdplLhCqXleZZLw==", "dependencies": { "@braintree/sanitize-url": "^6.0.2", + "@types/d3-scale": "^4.0.3", + "@types/d3-scale-chromatic": "^3.0.0", "cytoscape": "^3.23.0", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.1.0", "d3": "^7.4.0", + "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.10", "dayjs": "^1.11.7", - "dompurify": "3.0.3", + "dompurify": "3.0.5", "elkjs": "^0.8.2", "khroma": "^2.0.0", "lodash-es": "^4.17.21", @@ -17620,6 +17676,24 @@ "@types/node": "*" } }, + "@types/d3-scale": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.3.tgz", + "integrity": "sha512-PATBiMCpvHJSMtZAMEhc2WyL+hnzarKzI6wAHYjhsonjWJYGq5BXTzQjv4l8m2jO183/4wZ90rKvSeT7o72xNQ==", + "requires": { + "@types/d3-time": "*" + } + }, + "@types/d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" + }, + "@types/d3-time": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", + "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" + }, "@types/debug": { "version": "4.1.8", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.8.tgz", @@ -19542,6 +19616,43 @@ "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==" }, + "d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "requires": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + }, + "dependencies": { + "d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "requires": { + "internmap": "^1.0.0" + } + }, + "d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==" + }, + "d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "requires": { + "d3-path": "1" + } + }, + "internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==" + } + } + }, "d3-scale": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", @@ -19913,9 +20024,9 @@ } }, "dompurify": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.3.tgz", - "integrity": "sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.5.tgz", + "integrity": "sha512-F9e6wPGtY+8KNMRAVfxeCOHU0/NPWMSENNq4pQctuXRqqdEPW7q3CrLbR5Nse044WwacyjHGOMlvNsBe1y6z9A==" }, "domutils": { "version": "2.8.0", @@ -21729,18 +21840,21 @@ "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==" }, "mermaid": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.2.4.tgz", - "integrity": "sha512-zHGjEI7lBvWZX+PQYmlhSA2p40OzW6QbGodTCSzDeVpqaTnyAC+2sRGqrpXO+uQk3CnoeClHQPraQUMStdqy2g==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.3.0.tgz", + "integrity": "sha512-H5quxuQjwXC8M1WuuzhAp2TdqGg74t5skfDBrNKJ7dt3z8Wprl5S6h9VJsRhoBUTSs1TMtHEdplLhCqXleZZLw==", "requires": { "@braintree/sanitize-url": "^6.0.2", + "@types/d3-scale": "^4.0.3", + "@types/d3-scale-chromatic": "^3.0.0", "cytoscape": "^3.23.0", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.1.0", "d3": "^7.4.0", + "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.10", "dayjs": "^1.11.7", - "dompurify": "3.0.3", + "dompurify": "3.0.5", "elkjs": "^0.8.2", "khroma": "^2.0.0", "lodash-es": "^4.17.21", diff --git a/website/package.json b/website/package.json index 14becab38..470fbf579 100644 --- a/website/package.json +++ b/website/package.json @@ -26,7 +26,7 @@ "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", - "mermaid": "^10.2.4", + "mermaid": "^10.3.0", "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", From a973f69d182c161d6c30ad3376f43335f20b7c41 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 27 Jul 2023 00:29:56 -0600 Subject: [PATCH 44/62] rename export interface (#3916) to better match current standards #### Type of change - [x] :broom: Tech Debt/Cleanup --- src/internal/m365/export.go | 6 +++--- src/internal/m365/mock/connector.go | 2 +- src/internal/m365/onedrive/export.go | 4 ++-- src/internal/m365/onedrive/export_test.go | 2 +- src/internal/operations/export.go | 2 +- src/internal/operations/inject/inject.go | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index 3840d377b..085881803 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -17,8 +17,8 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" ) -// ExportRestoreCollections exports data from the specified collections -func (ctrl *Controller) ExportRestoreCollections( +// ProduceExportCollections exports data from the specified collections +func (ctrl *Controller) ProduceExportCollections( ctx context.Context, backupVersion int, sels selectors.Selector, @@ -42,7 +42,7 @@ func (ctrl *Controller) ExportRestoreCollections( switch sels.Service { case selectors.ServiceOneDrive: - expCollections, err = onedrive.ExportRestoreCollections( + expCollections, err = onedrive.ProduceExportCollections( ctx, backupVersion, exportCfg, diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 64bc8deda..5510d2f9a 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -74,7 +74,7 @@ func (ctrl Controller) ConsumeRestoreCollections( func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {} -func (ctrl Controller) ExportRestoreCollections( +func (ctrl Controller) ProduceExportCollections( _ context.Context, _ int, _ selectors.Selector, diff --git a/src/internal/m365/onedrive/export.go b/src/internal/m365/onedrive/export.go index bf68ede8d..9868a9b71 100644 --- a/src/internal/m365/onedrive/export.go +++ b/src/internal/m365/onedrive/export.go @@ -131,9 +131,9 @@ func getItemName( return "", clues.New("invalid item id").WithClues(ctx) } -// ExportRestoreCollections will create the export collections for the +// ProduceExportCollections will create the export collections for the // given restore collections. -func ExportRestoreCollections( +func ProduceExportCollections( ctx context.Context, backupVersion int, exportCfg control.ExportConfig, diff --git a/src/internal/m365/onedrive/export_test.go b/src/internal/m365/onedrive/export_test.go index b28b5f3a5..ce707885f 100644 --- a/src/internal/m365/onedrive/export_test.go +++ b/src/internal/m365/onedrive/export_test.go @@ -447,7 +447,7 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { } exportCfg := control.ExportConfig{} - ecs, err := ExportRestoreCollections(ctx, int(version.Backup), exportCfg, control.Options{}, dcs, nil, fault.New(true)) + ecs, err := ProduceExportCollections(ctx, int(version.Backup), exportCfg, control.Options{}, dcs, nil, fault.New(true)) assert.NoError(t, err, "export collections error") assert.Len(t, ecs, 1, "num of collections") diff --git a/src/internal/operations/export.go b/src/internal/operations/export.go index dc4823935..772ba44f2 100644 --- a/src/internal/operations/export.go +++ b/src/internal/operations/export.go @@ -343,7 +343,7 @@ func exportRestoreCollections( close(complete) }() - expCollections, err := ec.ExportRestoreCollections( + expCollections, err := ec.ProduceExportCollections( ctx, backupVersion, sel, diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index 0a5e8581c..5a30fe8a3 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -59,7 +59,7 @@ type ( } ExportConsumer interface { - ExportRestoreCollections( + ProduceExportCollections( ctx context.Context, backupVersion int, selector selectors.Selector, From 34d6c18d74698b3ec8e0318e855dd498d4e93bf7 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 27 Jul 2023 13:16:11 -0600 Subject: [PATCH 45/62] clean up exchange adv restore test (#3870) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/operations/test/exchange_test.go | 351 +++++++----------- 1 file changed, 138 insertions(+), 213 deletions(-) diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index ddb16216b..33bd18a6a 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -889,6 +889,26 @@ func (suite *ExchangeRestoreIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } +type clientItemPager interface { + GetItemsInContainerByCollisionKeyer[string] + GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, + ) (map[string]struct{}, error) + GetContainerByName( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + userID, containerID string, + ) (map[string]string, error) + CreateContainer( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) +} + func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptions() { t := suite.T() @@ -921,25 +941,28 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio require.NoError(t, err, clues.ToCore(err)) var ( - restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") - sel = rsel.Selector - userID = sel.ID() - cIDs = map[path.CategoryType]string{ - path.ContactsCategory: "", - path.EmailCategory: "", - path.EventsCategory: "", + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") + sel = rsel.Selector + userID = sel.ID() + countItemsInRestore int + + itemIDs = map[path.CategoryType]map[string]struct{}{} + collisionKeys = map[path.CategoryType]map[string]string{} + containerIDs = map[path.CategoryType]string{} + parentContainerIDs = map[path.CategoryType]string{ + path.EmailCategory: api.MsgFolderRoot, + } + parentContainerNames = map[path.CategoryType][]string{ + path.EmailCategory: {api.MailInbox}, + path.ContactsCategory: {}, + path.EventsCategory: {}, + } + + testCategories = map[path.CategoryType]clientItemPager{ + path.ContactsCategory: suite.its.ac.Contacts(), + path.EmailCategory: suite.its.ac.Mail(), + // path.EventsCategory: suite.its.ac.Events(), } - collKeys = map[path.CategoryType]map[string]string{} - countContactsInRestore int - acCont = suite.its.ac.Contacts() - contactIDs map[string]struct{} - countEmailsInRestore int - acMail = suite.its.ac.Mail() - mailIDs map[string]struct{} - countItemsInRestore int - // countEventsInRestore int - // acEvts = suite.its.ac.Events() - // eventIDs = []string{} ) // initial restore @@ -971,61 +994,27 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio // get all files in folder, use these as the base // set of files to compare against. - // --- contacts + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - contGC, err := acCont.GetContainerByName(ctx, userID, "", restoreCfg.Location) - require.NoError(t, err, clues.ToCore(err)) + ctx, flush := tester.NewContext(t) + defer flush() - cIDs[path.ContactsCategory] = ptr.Val(contGC.GetId()) + containers := append([]string{restoreCfg.Location}, parentContainerNames[cat]...) - collKeys[path.ContactsCategory], err = acCont.GetItemsInContainerByCollisionKey( - ctx, - userID, - cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) - countContactsInRestore = len(collKeys[path.ContactsCategory]) - t.Log(countContactsInRestore, "contacts restored") + itemIDs[cat], collisionKeys[cat], containerIDs[cat] = getCollKeysAndItemIDs( + t, + ctx, + ac, + userID, + parentContainerIDs[cat], + containers...) - contactIDs, err = acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + countItemsInRestore += len(collisionKeys[cat]) + }) + } - // --- events - - // gc, err = acEvts.GetContainerByName(ctx, userID, "", restoreCfg.Location) - // require.NoError(t, err, clues.ToCore(err)) - - // restoredContainerID[path.EventsCategory] = ptr.Val(gc.GetId()) - - // collKeys[path.EventsCategory], err = acEvts.GetItemsInContainerByCollisionKey( - // ctx, - // userID, - // cIDs[path.EventsCategory]) - // require.NoError(t, err, clues.ToCore(err)) - // countEventsInRestore = len(collKeys[path.EventsCategory]) - // t.Log(countContactsInRestore, "events restored") - - mailGC, err := acMail.GetContainerByName(ctx, userID, api.MsgFolderRoot, restoreCfg.Location) - require.NoError(t, err, clues.ToCore(err)) - - mailGC, err = acMail.GetContainerByName(ctx, userID, ptr.Val(mailGC.GetId()), api.MailInbox) - require.NoError(t, err, clues.ToCore(err)) - - cIDs[path.EmailCategory] = ptr.Val(mailGC.GetId()) - - // --- mail - - collKeys[path.EmailCategory], err = acMail.GetItemsInContainerByCollisionKey( - ctx, - userID, - cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - countEmailsInRestore = len(collKeys[path.EmailCategory]) - t.Log(countContactsInRestore, "emails restored") - - mailIDs, err = acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - countItemsInRestore = countContactsInRestore + countEmailsInRestore // + countEventsInRestore checkRestoreCounts(t, ctr1, 0, 0, countItemsInRestore) }) @@ -1062,43 +1051,30 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio checkRestoreCounts(t, ctr2, countItemsInRestore, 0, 0) - // --- contacts + result := map[string]string{} - // get all files in folder, use these as the base - // set of files to compare against. - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + ctx, flush := tester.NewContext(t) + defer flush() - assert.Equal(t, contactIDs, currentContactIDs, "ids are equal") + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) - // --- events + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) - - // --- mail - - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) - - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, mailIDs, currentMailIDs, "ids are equal") + assert.Equal(t, itemIDs[cat], currentIDs, "ids are equal") + }) + } assert.Len(t, result, 0, "no new items should get added") }) @@ -1136,60 +1112,40 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio } } - assert.Len( - t, - filtEnts, - countItemsInRestore, - "every item should have been replaced") - - // --- contacts - - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) - - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, len(contactIDs), len(currentContactIDs), "count of ids are equal") - for orig := range contactIDs { - assert.NotContains(t, currentContactIDs, orig, "original item should not exist after replacement") - } - - contactIDs = currentContactIDs - - // --- events - - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) - - // --- mail - - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) + assert.Len(t, filtEnts, countItemsInRestore, "every item should have been replaced") checkRestoreCounts(t, ctr3, 0, countItemsInRestore, 0) - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) + result := map[string]string{} - assert.Equal(t, len(mailIDs), len(currentMailIDs), "count of ids are equal") - for orig := range mailIDs { - assert.NotContains(t, currentMailIDs, orig, "original item should not exist after replacement") + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) + + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(itemIDs[cat]), len(currentIDs), "count of ids are equal") + for orig := range itemIDs[cat] { + assert.NotContains(t, currentIDs, orig, "original item should not exist after replacement") + } + + itemIDs[cat] = currentIDs + }) } - mailIDs = currentMailIDs - assert.Len(t, result, 0, "all items should have been replaced") }) @@ -1226,45 +1182,35 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio } } - assert.Len( - t, - filtEnts, - countItemsInRestore, - "every item should have been copied") + assert.Len(t, filtEnts, countItemsInRestore, "every item should have been copied") checkRestoreCounts(t, ctr4, 0, 0, countItemsInRestore) - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) + result := map[string]string{} - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - assert.Equal(t, 2*len(contactIDs), len(currentContactIDs), "count of ids should be double from before") - assert.Subset(t, maps.Keys(currentContactIDs), maps.Keys(contactIDs), "original item should exist after copy") + ctx, flush := tester.NewContext(t) + defer flush() - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, 2*len(mailIDs), len(currentMailIDs), "count of ids should be double from before") - assert.Subset(t, maps.Keys(currentMailIDs), maps.Keys(mailIDs), "original item should exist after copy") + assert.Equal(t, 2*len(itemIDs[cat]), len(currentIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentIDs), maps.Keys(itemIDs[cat]), "original item should exist after copy") + }) + } // TODO: we have the option of modifying copy creations in exchange // so that the results don't collide. But we haven't made that @@ -1344,7 +1290,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- contacts cat := path.ContactsCategory - userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acCont, @@ -1354,7 +1300,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- events // cat = path.EventsCategory - // userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + // userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( // t, // ctx, // acEvts, @@ -1364,7 +1310,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- mail cat = path.EmailCategory - userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acMail, @@ -1400,7 +1346,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- contacts cat = path.ContactsCategory - secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acCont, @@ -1410,7 +1356,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- events // cat = path.EventsCategory - // secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + // secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( // t, // ctx, // acEvts, @@ -1420,7 +1366,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte // --- mail cat = path.EmailCategory - secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( t, ctx, acMail, @@ -1436,52 +1382,31 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtecte } } -type GetItemsKeysAndContainerByNameer interface { - GetItemIDsInContainer( - ctx context.Context, - userID, containerID string, - ) (map[string]struct{}, error) - GetContainerByName( - ctx context.Context, - userID, parentContainerID, containerName string, - ) (graph.Container, error) - GetItemsInContainerByCollisionKey( - ctx context.Context, - userID, containerID string, - ) (map[string]string, error) -} - func getCollKeysAndItemIDs( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument - gikacbn GetItemsKeysAndContainerByNameer, + cip clientItemPager, userID, parentContainerID string, containerNames ...string, -) (map[string]struct{}, map[string]string) { +) (map[string]struct{}, map[string]string, string) { var ( c graph.Container err error - cID string + cID = parentContainerID ) for _, cn := range containerNames { - pcid := parentContainerID - - if len(cID) != 0 { - pcid = cID - } - - c, err = gikacbn.GetContainerByName(ctx, userID, pcid, cn) + c, err = cip.GetContainerByName(ctx, userID, cID, cn) require.NoError(t, err, clues.ToCore(err)) cID = ptr.Val(c.GetId()) } - itemIDs, err := gikacbn.GetItemIDsInContainer(ctx, userID, cID) + itemIDs, err := cip.GetItemIDsInContainer(ctx, userID, cID) require.NoError(t, err, clues.ToCore(err)) - collisionKeys, err := gikacbn.GetItemsInContainerByCollisionKey(ctx, userID, cID) + collisionKeys, err := cip.GetItemsInContainerByCollisionKey(ctx, userID, cID) require.NoError(t, err, clues.ToCore(err)) - return itemIDs, collisionKeys + return itemIDs, collisionKeys, cID } From 4e2ee2484fe5341e718947dc5d0a599c92c27adb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:47:38 +0000 Subject: [PATCH 46/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.308=20to=201.44.309=20in=20/src=20(#?= =?UTF-8?q?3918)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.308 to 1.44.309.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.309 (2023-07-26)

Service Client Updates

  • service/cloudcontrol: Updates service documentation
  • service/entityresolution: Adds new service
  • service/glue: Updates service API and documentation
    • Release Glue Studio Snowflake Connector Node for SDK/CLI
  • service/healthlake: Updates service documentation
  • service/managedblockchain-query: Adds new service
  • service/mediaconvert: Updates service API and documentation
    • This release includes general updates to user documentation.
  • service/omics: Updates service documentation
  • service/opensearchserverless: Updates service API and documentation
  • service/polly: Updates service API
    • Amazon Polly adds 1 new voice - Lisa (nl-BE)
  • service/route53: Updates service documentation
    • Update that corrects the documents for received feedback.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.308&new-version=1.44.309)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 04a4d8c51..992c60f69 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.308 + github.com/aws/aws-sdk-go v1.44.309 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index f40e14356..0c1ea866b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.308 h1:XKu+76UHsD5LaiU2Zb1q42uWakw80Az7x39jJXXahos= -github.com/aws/aws-sdk-go v1.44.308/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.309 h1:IPJOFBzXekakxmEpDwd4RTKmmBR6LIAiXgNsM51bWbU= +github.com/aws/aws-sdk-go v1.44.309/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From d7443c2211a393f0a56892208833a049e8408021 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 13:31:12 +0530 Subject: [PATCH 47/62] CLI for exporting data from OneDrive (#3822) This adds the final piece to enable OneDrive exports. The CLI interface which consumes NewExport interface from Repository and lets the user to export and write the contents in a backup to the local filesystem. Prev: https://github.com/alcionai/corso/pull/3821 Next: https://github.com/alcionai/corso/pull/3824 --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * https://github.com/alcionai/corso/pull/3797 * https://github.com/alcionai/corso/issues/3670 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 3 + src/cli/cli.go | 4 +- src/cli/export/export.go | 39 +++++ src/cli/export/export_test.go | 175 ++++++++++++++++++++++ src/cli/export/onedrive.go | 215 ++++++++++++++++++++++++++++ src/cli/export/onedrive_test.go | 106 ++++++++++++++ src/cli/flags/export.go | 15 ++ src/cli/utils/export_config.go | 38 +++++ src/cli/utils/export_config_test.go | 54 +++++++ src/cli/utils/onedrive.go | 2 + src/cli/utils/testdata/flags.go | 4 + src/internal/observe/observe.go | 46 ++++++ src/pkg/control/restore.go | 6 +- 13 files changed, 703 insertions(+), 4 deletions(-) create mode 100644 src/cli/export/export.go create mode 100644 src/cli/export/export_test.go create mode 100644 src/cli/export/onedrive.go create mode 100644 src/cli/export/onedrive_test.go create mode 100644 src/cli/flags/export.go create mode 100644 src/cli/utils/export_config.go create mode 100644 src/cli/utils/export_config_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7123822..5170dc50f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - SharePoint document libraries deleted after the last backup can now be restored. - Restore requires the protected resource to have access to the service being restored. +### Added +- Added option to export data from OneDrive backups as individual files or as a single zip file. + ## [v0.11.1] (beta) - 2023-07-20 ### Fixed diff --git a/src/cli/cli.go b/src/cli/cli.go index 5f03364a2..68f454407 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/cli/backup" "github.com/alcionai/corso/src/cli/config" + "github.com/alcionai/corso/src/cli/export" "github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/help" "github.com/alcionai/corso/src/cli/print" @@ -53,7 +54,7 @@ func preRun(cc *cobra.Command, args []string) error { } avoidTheseCommands := []string{ - "corso", "env", "help", "backup", "details", "list", "restore", "delete", "repo", "init", "connect", + "corso", "env", "help", "backup", "details", "list", "restore", "export", "delete", "repo", "init", "connect", } if len(logger.ResolvedLogFile) > 0 && !slices.Contains(avoidTheseCommands, cc.Use) { @@ -150,6 +151,7 @@ func BuildCommandTree(cmd *cobra.Command) { repo.AddCommands(cmd) backup.AddCommands(cmd) restore.AddCommands(cmd) + export.AddCommands(cmd) help.AddCommands(cmd) } diff --git a/src/cli/export/export.go b/src/cli/export/export.go new file mode 100644 index 000000000..d3ec09a8a --- /dev/null +++ b/src/cli/export/export.go @@ -0,0 +1,39 @@ +package export + +import ( + "github.com/spf13/cobra" +) + +var exportCommands = []func(cmd *cobra.Command) *cobra.Command{ + addOneDriveCommands, +} + +// AddCommands attaches all `corso export * *` commands to the parent. +func AddCommands(cmd *cobra.Command) { + exportC := exportCmd() + cmd.AddCommand(exportC) + + for _, addExportTo := range exportCommands { + addExportTo(exportC) + } +} + +const exportCommand = "export" + +// The export category of commands. +// `corso export [] [...]` +func exportCmd() *cobra.Command { + return &cobra.Command{ + Use: exportCommand, + Short: "Export your service data", + Long: `Export the data stored in one of your M365 services.`, + RunE: handleExportCmd, + Args: cobra.NoArgs, + } +} + +// Handler for flat calls to `corso export`. +// Produces the same output as `corso export --help`. +func handleExportCmd(cmd *cobra.Command, args []string) error { + return cmd.Help() +} diff --git a/src/cli/export/export_test.go b/src/cli/export/export_test.go new file mode 100644 index 000000000..f3df68177 --- /dev/null +++ b/src/cli/export/export_test.go @@ -0,0 +1,175 @@ +package export + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/export" +) + +type ExportE2ESuite struct { + tester.Suite + called bool +} + +func TestExportE2ESuite(t *testing.T) { + suite.Run(t, &ExportE2ESuite{Suite: tester.NewE2ESuite(t, nil)}) +} + +func (suite *ExportE2ESuite) SetupSuite() { + suite.called = true +} + +type mockExportCollection struct { + path string + items []export.Item +} + +func (mec mockExportCollection) BasePath() string { return mec.path } +func (mec mockExportCollection) Items(context.Context) <-chan export.Item { + ch := make(chan export.Item) + + go func() { + defer close(ch) + + for _, item := range mec.items { + ch <- item + } + }() + + return ch +} + +func (suite *ExportE2ESuite) TestWriteExportCollection() { + type ei struct { + name string + body string + } + + type i struct { + path string + items []ei + } + + table := []struct { + name string + cols []i + }{ + { + name: "single root collection single item", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + }, + }, + }, + }, + { + name: "single root collection multiple items", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + { + name: "name2", + body: "body2", + }, + }, + }, + }, + }, + { + name: "multiple collections multiple items", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + { + name: "name2", + body: "body2", + }, + }, + }, + { + path: "folder", + items: []ei{ + { + name: "name3", + body: "body3", + }, + }, + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ecs := []export.Collection{} + for _, col := range test.cols { + items := []export.Item{} + for _, item := range col.items { + items = append(items, export.Item{ + Data: export.ItemData{ + Name: item.name, + Body: io.NopCloser((bytes.NewBufferString(item.body))), + }, + }) + } + + ecs = append(ecs, mockExportCollection{ + path: col.path, + items: items, + }) + } + + dir, err := os.MkdirTemp("", "export-test") + require.NoError(t, err) + defer os.RemoveAll(dir) + + err = writeExportCollections(ctx, dir, ecs) + require.NoError(t, err, "writing data") + + for _, col := range test.cols { + for _, item := range col.items { + f, err := os.Open(filepath.Join(dir, col.path, item.name)) + require.NoError(t, err, "opening file") + + buf := new(bytes.Buffer) + + _, err = buf.ReadFrom(f) + require.NoError(t, err, "reading file") + + assert.Equal(t, item.body, buf.String(), "file contents") + } + } + }) + } +} diff --git a/src/cli/export/onedrive.go b/src/cli/export/onedrive.go new file mode 100644 index 000000000..6e715153d --- /dev/null +++ b/src/cli/export/onedrive.go @@ -0,0 +1,215 @@ +package export + +import ( + "context" + "io" + "os" + ospath "path" + + "github.com/alcionai/clues" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/repo" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/path" +) + +// called by export.go to map subcommands to provider-specific handling. +func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, oneDriveExportCmd()) + + c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddOneDriveDetailsAndRestoreFlags(c) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +const ( + oneDriveServiceCommand = "onedrive" + oneDriveServiceCommandUseSuffix = "--backup " + + //nolint:lll + oneDriveServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to my-exports directory +corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef + +# Export files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" to current directory +corso export onedrive . --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + +# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to my-exports +corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd + --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` +) + +// `corso export onedrive [...] ` +func oneDriveExportCmd() *cobra.Command { + return &cobra.Command{ + Use: oneDriveServiceCommand, + Short: "Export M365 OneDrive service data", + RunE: exportOneDriveCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing restore destination") + } + + return nil + }, + Example: oneDriveServiceCommandExportExamples, + } +} + +// processes an onedrive service export. +func exportOneDriveCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + opts := utils.MakeOneDriveOpts(cmd) + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := utils.ValidateOneDriveRestoreFlags(flags.BackupIDFV, opts); err != nil { + return err + } + + r, _, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd)) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + exportLocation := args[0] + if exportLocation == "" { + // This is unlikely, but adding it just in case. + exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) + } + + Infof(ctx, "Exporting to folder %s", exportLocation) + + sel := utils.IncludeOneDriveRestoreDataSelectors(opts) + utils.FilterOneDriveRestoreInfoSelectors(sel, opts) + + eo, err := r.NewExport( + ctx, + flags.BackupIDFV, + sel.Selector, + utils.MakeExportConfig(ctx, opts.ExportCfg), + ) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to initialize OneDrive export")) + } + + expColl, err := eo.Run(ctx) + if err != nil { + if errors.Is(err, data.ErrNotFound) { + return Only(ctx, clues.New("Backup or backup details missing for id "+flags.BackupIDFV)) + } + + return Only(ctx, clues.Wrap(err, "Failed to run OneDrive export")) + } + + // It would be better to give a progressbar than a spinner, but we + // have know way of knowing how many files are available as of now. + diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") + defer func() { + diskWriteComplete <- struct{}{} + close(diskWriteComplete) + }() + + err = writeExportCollections(ctx, exportLocation, expColl) + if err != nil { + return err + } + + return nil +} + +func writeExportCollections( + ctx context.Context, + exportLocation string, + expColl []export.Collection, +) error { + for _, col := range expColl { + folder := ospath.Join(exportLocation, col.BasePath()) + + for item := range col.Items(ctx) { + err := item.Error + if err != nil { + return Only(ctx, clues.Wrap(err, "getting item").With("dir_name", folder)) + } + + err = writeExportItem(ctx, item, folder) + if err != nil { + return err + } + } + } + + return nil +} + +// writeExportItem writes an ExportItem to disk in the specified folder. +func writeExportItem(ctx context.Context, item export.Item, folder string) error { + name := item.Data.Name + fpath := ospath.Join(folder, name) + + progReader, pclose := observe.ItemSpinner( + ctx, + item.Data.Body, + observe.ItemExportMsg, + clues.Hide(name)) + + defer item.Data.Body.Close() + defer pclose() + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return Only(ctx, clues.Wrap(err, "creating directory").With("dir_name", folder)) + } + + // In case the user tries to restore to a non-clean + // directory, we might run into collisions an fail. + f, err := os.Create(fpath) + if err != nil { + return Only(ctx, clues.Wrap(err, "creating file").With("file_name", name, "file_dir", folder)) + } + + _, err = io.Copy(f, progReader) + if err != nil { + return Only(ctx, clues.Wrap(err, "writing file").With("file_name", name, "file_dir", folder)) + } + + return nil +} diff --git a/src/cli/export/onedrive_test.go b/src/cli/export/onedrive_test.go new file mode 100644 index 000000000..775dd4a70 --- /dev/null +++ b/src/cli/export/onedrive_test.go @@ -0,0 +1,106 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type OneDriveUnitSuite struct { + tester.Suite +} + +func TestOneDriveUnitSuite(t *testing.T) { + suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { + expectUse := oneDriveServiceCommand + " " + oneDriveServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export onedrive", exportCommand, expectUse, oneDriveExportCmd().Short, exportOneDriveCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addOneDriveCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "onedrive", + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), + "--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput), + "--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput, + "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, + "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, + "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, + + "--" + flags.ArchiveFN, + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + testdata.RestoreDestination, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + assert.NoError(t, err, clues.ToCore(err)) + + opts := utils.MakeOneDriveOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName) + assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath) + assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter) + assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) + assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) + assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/flags/export.go b/src/cli/flags/export.go new file mode 100644 index 000000000..b9af7e141 --- /dev/null +++ b/src/cli/flags/export.go @@ -0,0 +1,15 @@ +package flags + +import ( + "github.com/spf13/cobra" +) + +const ArchiveFN = "archive" + +var ArchiveFV bool + +// AddExportConfigFlags adds the restore config flag set. +func AddExportConfigFlags(cmd *cobra.Command) { + fs := cmd.Flags() + fs.BoolVar(&ArchiveFV, ArchiveFN, false, "Export data as an archive instead of individual files") +} diff --git a/src/cli/utils/export_config.go b/src/cli/utils/export_config.go new file mode 100644 index 000000000..2fd3827bd --- /dev/null +++ b/src/cli/utils/export_config.go @@ -0,0 +1,38 @@ +package utils + +import ( + "context" + + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/pkg/control" +) + +type ExportCfgOpts struct { + Archive bool + + Populated flags.PopulatedFlags +} + +func makeExportCfgOpts(cmd *cobra.Command) ExportCfgOpts { + return ExportCfgOpts{ + Archive: flags.ArchiveFV, + + // populated contains the list of flags that appear in the + // command, according to pflags. Use this to differentiate + // between an "empty" and a "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} + +func MakeExportConfig( + ctx context.Context, + opts ExportCfgOpts, +) control.ExportConfig { + exportCfg := control.DefaultExportConfig() + + exportCfg.Archive = opts.Archive + + return exportCfg +} diff --git a/src/cli/utils/export_config_test.go b/src/cli/utils/export_config_test.go new file mode 100644 index 000000000..d25d6629b --- /dev/null +++ b/src/cli/utils/export_config_test.go @@ -0,0 +1,54 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" +) + +type ExportCfgUnitSuite struct { + tester.Suite +} + +func TestExportCfgUnitSuite(t *testing.T) { + suite.Run(t, &ExportCfgUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportCfgUnitSuite) TestMakeExportConfig() { + rco := &ExportCfgOpts{Archive: true} + + table := []struct { + name string + populated flags.PopulatedFlags + expect control.ExportConfig + }{ + { + name: "archive populated", + populated: flags.PopulatedFlags{ + flags.ArchiveFN: {}, + }, + expect: control.ExportConfig{ + Archive: true, + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + opts := *rco + opts.Populated = test.populated + + result := MakeExportConfig(ctx, opts) + assert.Equal(t, test.expect.Archive, result.Archive) + }) + } +} diff --git a/src/cli/utils/onedrive.go b/src/cli/utils/onedrive.go index 06699adca..e12f35230 100644 --- a/src/cli/utils/onedrive.go +++ b/src/cli/utils/onedrive.go @@ -19,6 +19,7 @@ type OneDriveOpts struct { FileModifiedBefore string RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts Populated flags.PopulatedFlags } @@ -35,6 +36,7 @@ func MakeOneDriveOpts(cmd *cobra.Command) OneDriveOpts { FileModifiedBefore: flags.FileModifiedBeforeFV, RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate diff --git a/src/cli/utils/testdata/flags.go b/src/cli/utils/testdata/flags.go index 85131a1e0..b04a9ab63 100644 --- a/src/cli/utils/testdata/flags.go +++ b/src/cli/utils/testdata/flags.go @@ -51,6 +51,8 @@ var ( DeltaPageSize = "deltaPageSize" + Archive = true + AzureClientID = "testAzureClientId" AzureTenantID = "testAzureTenantId" AzureClientSecret = "testAzureClientSecret" @@ -60,4 +62,6 @@ var ( AWSSessionToken = "testAWSSessionToken" CorsoPassphrase = "testCorsoPassphrase" + + RestoreDestination = "test-restore-destination" ) diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index bd94b6531..c05297587 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -133,6 +133,7 @@ func Complete() { const ( ItemBackupMsg = "Backing up item" ItemRestoreMsg = "Restoring item" + ItemExportMsg = "Exporting item" ItemQueueMsg = "Queuing items" ) @@ -281,6 +282,51 @@ func ItemProgress( return bar.ProxyReader(rc), abort } +// ItemSpinner is similar to ItemProgress, but for use in cases where +// we don't know the file size but want to show progress. +func ItemSpinner( + ctx context.Context, + rc io.ReadCloser, + header string, + iname any, +) (io.ReadCloser, func()) { + plain := plainString(iname) + log := logger.Ctx(ctx).With("item", iname) + log.Debug(header) + + if cfg.hidden() || rc == nil { + defer log.Debug("done - " + header) + return rc, func() {} + } + + wg.Add(1) + + barOpts := []mpb.BarOption{ + mpb.PrependDecorators( + decor.Name(header, decor.WCSyncSpaceR), + decor.Name(plain, decor.WCSyncSpaceR), + decor.CurrentKibiByte(" %.1f", decor.WC{W: 8})), + } + + if !cfg.keepBarsAfterComplete { + barOpts = append(barOpts, mpb.BarRemoveOnComplete()) + } + + bar := progress.New(-1, mpb.NopStyle(), barOpts...) + + go waitAndCloseBar(bar, func() { + // might be overly chatty, we can remove if needed. + log.Debug("done - " + header) + })() + + abort := func() { + bar.SetTotal(-1, true) + bar.Abort(true) + } + + return bar.ProxyReader(rc), abort +} + // ProgressWithCount tracks the display of a bar that tracks the completion // of the specified count. // Each write to the provided channel counts as a single increment. diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index c30b7d177..74cf88093 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -16,7 +16,7 @@ import ( ) const ( - defaultRestoreLocation = "Corso_Restore_" + DefaultRestoreLocation = "Corso_Restore_" ) // CollisionPolicy describes how the datalayer behaves in case of a collision. @@ -70,12 +70,12 @@ type RestoreConfig struct { func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { return RestoreConfig{ OnCollision: Skip, - Location: defaultRestoreLocation + dttm.FormatNow(timeFormat), + Location: DefaultRestoreLocation + dttm.FormatNow(timeFormat), } } func DefaultRestoreContainerName(timeFormat dttm.TimeFormat) string { - return defaultRestoreLocation + dttm.FormatNow(timeFormat) + return DefaultRestoreLocation + dttm.FormatNow(timeFormat) } // EnsureRestoreConfigDefaults sets all non-supported values in the config From c654dfba1b107330aae564647e148dfec02e5ec9 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 14:10:35 +0530 Subject: [PATCH 48/62] Export data from SharePoint (#3824) This borrows a lot of the core logic from OneDrive as the internal structure is mostly the same. Prev: https://github.com/alcionai/corso/pull/3822 --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * fixes https://github.com/alcionai/corso/issues/3823 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 2 +- src/cli/export/export.go | 69 ++++++++++ src/cli/export/onedrive.go | 121 +----------------- src/cli/export/onedrive_test.go | 6 +- src/cli/export/sharepoint.go | 100 +++++++++++++++ src/cli/export/sharepoint_test.go | 118 +++++++++++++++++ src/cli/restore/sharepoint_test.go | 2 +- src/cli/utils/sharepoint.go | 2 + src/internal/m365/export.go | 3 +- src/pkg/export/consume.go | 79 ++++++++++++ .../export/consume_test.go} | 20 +-- src/pkg/export/export.go | 4 +- 12 files changed, 389 insertions(+), 137 deletions(-) create mode 100644 src/cli/export/sharepoint.go create mode 100644 src/cli/export/sharepoint_test.go create mode 100644 src/pkg/export/consume.go rename src/{cli/export/export_test.go => pkg/export/consume_test.go} (86%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5170dc50f..906267535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Restore requires the protected resource to have access to the service being restored. ### Added -- Added option to export data from OneDrive backups as individual files or as a single zip file. +- Added option to export data from OneDrive and SharePoint backups as individual files or as a single zip file. ## [v0.11.1] (beta) - 2023-07-20 diff --git a/src/cli/export/export.go b/src/cli/export/export.go index d3ec09a8a..e0deed014 100644 --- a/src/cli/export/export.go +++ b/src/cli/export/export.go @@ -1,11 +1,26 @@ package export import ( + "context" + "errors" + + "github.com/alcionai/clues" "github.com/spf13/cobra" + + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/repo" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/selectors" ) var exportCommands = []func(cmd *cobra.Command) *cobra.Command{ addOneDriveCommands, + addSharePointCommands, } // AddCommands attaches all `corso export * *` commands to the parent. @@ -37,3 +52,57 @@ func exportCmd() *cobra.Command { func handleExportCmd(cmd *cobra.Command, args []string) error { return cmd.Help() } + +func runExport( + ctx context.Context, + cmd *cobra.Command, + args []string, + ueco utils.ExportCfgOpts, + sel selectors.Selector, + backupID, serviceName string, +) error { + r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd)) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + exportLocation := args[0] + if len(exportLocation) == 0 { + // This should not be possible, but adding it just in case. + exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) + } + + Infof(ctx, "Exporting to folder %s", exportLocation) + + eo, err := r.NewExport( + ctx, + backupID, + sel, + utils.MakeExportConfig(ctx, ueco)) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to initialize "+serviceName+" export")) + } + + expColl, err := eo.Run(ctx) + if err != nil { + if errors.Is(err, data.ErrNotFound) { + return Only(ctx, clues.New("Backup or backup details missing for id "+backupID)) + } + + return Only(ctx, clues.Wrap(err, "Failed to run "+serviceName+" export")) + } + + // It would be better to give a progressbar than a spinner, but we + // have any way of knowing how many files are available as of now. + diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") + defer close(diskWriteComplete) + + err = export.ConsumeExportCollections(ctx, exportLocation, expColl, eo.Errors) + if err != nil { + return Only(ctx, err) + } + + return nil +} diff --git a/src/cli/export/onedrive.go b/src/cli/export/onedrive.go index 6e715153d..593149bd9 100644 --- a/src/cli/export/onedrive.go +++ b/src/cli/export/onedrive.go @@ -1,26 +1,12 @@ package export import ( - "context" - "io" - "os" - ospath "path" - - "github.com/alcionai/clues" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/alcionai/corso/src/cli/flags" - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cli/repo" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/dttm" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/observe" - "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/export" - "github.com/alcionai/corso/src/pkg/path" ) // called by export.go to map subcommands to provider-specific handling. @@ -103,113 +89,8 @@ func exportOneDriveCmd(cmd *cobra.Command, args []string) error { return err } - r, _, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd)) - if err != nil { - return Only(ctx, err) - } - - defer utils.CloseRepo(ctx, r) - - exportLocation := args[0] - if exportLocation == "" { - // This is unlikely, but adding it just in case. - exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) - } - - Infof(ctx, "Exporting to folder %s", exportLocation) - sel := utils.IncludeOneDriveRestoreDataSelectors(opts) utils.FilterOneDriveRestoreInfoSelectors(sel, opts) - eo, err := r.NewExport( - ctx, - flags.BackupIDFV, - sel.Selector, - utils.MakeExportConfig(ctx, opts.ExportCfg), - ) - if err != nil { - return Only(ctx, clues.Wrap(err, "Failed to initialize OneDrive export")) - } - - expColl, err := eo.Run(ctx) - if err != nil { - if errors.Is(err, data.ErrNotFound) { - return Only(ctx, clues.New("Backup or backup details missing for id "+flags.BackupIDFV)) - } - - return Only(ctx, clues.Wrap(err, "Failed to run OneDrive export")) - } - - // It would be better to give a progressbar than a spinner, but we - // have know way of knowing how many files are available as of now. - diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") - defer func() { - diskWriteComplete <- struct{}{} - close(diskWriteComplete) - }() - - err = writeExportCollections(ctx, exportLocation, expColl) - if err != nil { - return err - } - - return nil -} - -func writeExportCollections( - ctx context.Context, - exportLocation string, - expColl []export.Collection, -) error { - for _, col := range expColl { - folder := ospath.Join(exportLocation, col.BasePath()) - - for item := range col.Items(ctx) { - err := item.Error - if err != nil { - return Only(ctx, clues.Wrap(err, "getting item").With("dir_name", folder)) - } - - err = writeExportItem(ctx, item, folder) - if err != nil { - return err - } - } - } - - return nil -} - -// writeExportItem writes an ExportItem to disk in the specified folder. -func writeExportItem(ctx context.Context, item export.Item, folder string) error { - name := item.Data.Name - fpath := ospath.Join(folder, name) - - progReader, pclose := observe.ItemSpinner( - ctx, - item.Data.Body, - observe.ItemExportMsg, - clues.Hide(name)) - - defer item.Data.Body.Close() - defer pclose() - - err := os.MkdirAll(folder, os.ModePerm) - if err != nil { - return Only(ctx, clues.Wrap(err, "creating directory").With("dir_name", folder)) - } - - // In case the user tries to restore to a non-clean - // directory, we might run into collisions an fail. - f, err := os.Create(fpath) - if err != nil { - return Only(ctx, clues.Wrap(err, "creating file").With("file_name", name, "file_dir", folder)) - } - - _, err = io.Copy(f, progReader) - if err != nil { - return Only(ctx, clues.Wrap(err, "writing file").With("file_name", name, "file_dir", folder)) - } - - return nil + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "OneDrive") } diff --git a/src/cli/export/onedrive_test.go b/src/cli/export/onedrive_test.go index 775dd4a70..59ab966e8 100644 --- a/src/cli/export/onedrive_test.go +++ b/src/cli/export/onedrive_test.go @@ -59,6 +59,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { cmd.SetArgs([]string{ "onedrive", + testdata.RestoreDestination, "--" + flags.RunModeFN, flags.RunModeFlagTest, "--" + flags.BackupFN, testdata.BackupInput, "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), @@ -68,15 +69,14 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, - "--" + flags.ArchiveFN, - "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, - testdata.RestoreDestination, + // bool flags + "--" + flags.ArchiveFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output diff --git a/src/cli/export/sharepoint.go b/src/cli/export/sharepoint.go new file mode 100644 index 000000000..ec71a5f2b --- /dev/null +++ b/src/cli/export/sharepoint.go @@ -0,0 +1,100 @@ +package export + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by export.go to map subcommands to provider-specific handling. +func addSharePointCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, sharePointExportCmd()) + + c.Use = c.Use + " " + sharePointServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddSharePointDetailsAndRestoreFlags(c) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +const ( + sharePointServiceCommand = "sharepoint" + sharePointServiceCommandUseSuffix = "--backup " + + //nolint:lll + sharePointServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's latest backup (1234abcd...) to my-exports directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef my-exports + +# Export files named "ServerRenderTemplate.xsl" in the folder "Display Templates/Style Sheets". as archive to current directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets" --archive . + +# Export all files in the folder "Display Templates/Style Sheets" that were created before 2020 to my-exports directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --file-created-before 2020-01-01T00:00:00 --folder "Display Templates/Style Sheets" my-exports + +# Export all files in the "Documents" library to current directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --library Documents --folder "Display Templates/Style Sheets" .` +) + +// `corso export sharepoint [...] ` +func sharePointExportCmd() *cobra.Command { + return &cobra.Command{ + Use: sharePointServiceCommand, + Short: "Export M365 SharePoint service data", + RunE: exportSharePointCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing restore destination") + } + + return nil + }, + Example: sharePointServiceCommandExportExamples, + } +} + +// processes an sharepoint service export. +func exportSharePointCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + opts := utils.MakeSharePointOpts(cmd) + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := utils.ValidateSharePointRestoreFlags(flags.BackupIDFV, opts); err != nil { + return err + } + + sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) + utils.FilterSharePointRestoreInfoSelectors(sel, opts) + + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "SharePoint") +} diff --git a/src/cli/export/sharepoint_test.go b/src/cli/export/sharepoint_test.go new file mode 100644 index 000000000..48ce28f5c --- /dev/null +++ b/src/cli/export/sharepoint_test.go @@ -0,0 +1,118 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type SharePointUnitSuite struct { + tester.Suite +} + +func TestSharePointUnitSuite(t *testing.T) { + suite.Run(t, &SharePointUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *SharePointUnitSuite) TestAddSharePointCommands() { + expectUse := sharePointServiceCommand + " " + sharePointServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export sharepoint", exportCommand, expectUse, sharePointExportCmd().Short, exportSharePointCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addSharePointCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "sharepoint", + testdata.RestoreDestination, + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + "--" + flags.LibraryFN, testdata.LibraryInput, + "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), + "--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput), + "--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput, + "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, + "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, + "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, + "--" + flags.ListItemFN, testdata.FlgInputs(testdata.ListItemInput), + "--" + flags.ListFolderFN, testdata.FlgInputs(testdata.ListFolderInput), + "--" + flags.PageFN, testdata.FlgInputs(testdata.PageInput), + "--" + flags.PageFolderFN, testdata.FlgInputs(testdata.PageFolderInput), + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.ArchiveFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + assert.NoError(t, err, clues.ToCore(err)) + + opts := utils.MakeSharePointOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.LibraryInput, opts.Library) + assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName) + assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath) + assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter) + assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) + assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) + assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) + + assert.ElementsMatch(t, testdata.ListItemInput, opts.ListItem) + assert.ElementsMatch(t, testdata.ListFolderInput, opts.ListFolder) + + assert.ElementsMatch(t, testdata.PageInput, opts.Page) + assert.ElementsMatch(t, testdata.PageFolderInput, opts.PageFolder) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/restore/sharepoint_test.go b/src/cli/restore/sharepoint_test.go index 6a8de8e57..c9bc8277f 100644 --- a/src/cli/restore/sharepoint_test.go +++ b/src/cli/restore/sharepoint_test.go @@ -34,7 +34,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { expectShort string expectRunE func(*cobra.Command, []string) error }{ - {"restore onedrive", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, + {"restore sharepoint", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, } for _, test := range table { suite.Run(test.name, func() { diff --git a/src/cli/utils/sharepoint.go b/src/cli/utils/sharepoint.go index 6672beda1..2ab43d90c 100644 --- a/src/cli/utils/sharepoint.go +++ b/src/cli/utils/sharepoint.go @@ -32,6 +32,7 @@ type SharePointOpts struct { Page []string RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts Populated flags.PopulatedFlags } @@ -56,6 +57,7 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts { PageFolder: flags.PageFolderFV, RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index 085881803..4da037e26 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -41,7 +41,8 @@ func (ctrl *Controller) ProduceExportCollections( ) switch sels.Service { - case selectors.ServiceOneDrive: + case selectors.ServiceOneDrive, selectors.ServiceSharePoint: + // OneDrive and SharePoint can share the code to create collections expCollections, err = onedrive.ProduceExportCollections( ctx, backupVersion, diff --git a/src/pkg/export/consume.go b/src/pkg/export/consume.go new file mode 100644 index 000000000..899f9c3ba --- /dev/null +++ b/src/pkg/export/consume.go @@ -0,0 +1,79 @@ +package export + +import ( + "context" + "io" + "os" + "path/filepath" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/fault" +) + +func ConsumeExportCollections( + ctx context.Context, + exportLocation string, + expColl []Collection, + errs *fault.Bus, +) error { + el := errs.Local() + + for _, col := range expColl { + if el.Failure() != nil { + break + } + + folder := filepath.Join(exportLocation, col.BasePath()) + ictx := clues.Add(ctx, "dir_name", folder) + + for item := range col.Items(ctx) { + if item.Error != nil { + el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item").WithClues(ctx)) + } + + if err := writeItem(ictx, item, folder); err != nil { + el.AddRecoverable( + ictx, + clues.Wrap(err, "writing item").With("file_name", item.Data.Name).WithClues(ctx)) + } + } + } + + return el.Failure() +} + +// writeItem writes an ExportItem to disk in the specified folder. +func writeItem(ctx context.Context, item Item, folder string) error { + name := item.Data.Name + fpath := filepath.Join(folder, name) + + progReader, pclose := observe.ItemSpinner( + ctx, + item.Data.Body, + observe.ItemExportMsg, + clues.Hide(name)) + + defer item.Data.Body.Close() + defer pclose() + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return clues.Wrap(err, "creating directory") + } + + // In case the user tries to restore to a non-clean + // directory, we might run into collisions an fail. + f, err := os.Create(fpath) + if err != nil { + return clues.Wrap(err, "creating file") + } + + _, err = io.Copy(f, progReader) + if err != nil { + return clues.Wrap(err, "writing data") + } + + return nil +} diff --git a/src/cli/export/export_test.go b/src/pkg/export/consume_test.go similarity index 86% rename from src/cli/export/export_test.go rename to src/pkg/export/consume_test.go index f3df68177..7d22dc237 100644 --- a/src/cli/export/export_test.go +++ b/src/pkg/export/consume_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" ) type ExportE2ESuite struct { @@ -31,12 +31,12 @@ func (suite *ExportE2ESuite) SetupSuite() { type mockExportCollection struct { path string - items []export.Item + items []Item } func (mec mockExportCollection) BasePath() string { return mec.path } -func (mec mockExportCollection) Items(context.Context) <-chan export.Item { - ch := make(chan export.Item) +func (mec mockExportCollection) Items(context.Context) <-chan Item { + ch := make(chan Item) go func() { defer close(ch) @@ -49,7 +49,7 @@ func (mec mockExportCollection) Items(context.Context) <-chan export.Item { return ch } -func (suite *ExportE2ESuite) TestWriteExportCollection() { +func (suite *ExportE2ESuite) TestConsumeExportCollection() { type ei struct { name string body string @@ -132,12 +132,12 @@ func (suite *ExportE2ESuite) TestWriteExportCollection() { ctx, flush := tester.NewContext(t) defer flush() - ecs := []export.Collection{} + ecs := []Collection{} for _, col := range test.cols { - items := []export.Item{} + items := []Item{} for _, item := range col.items { - items = append(items, export.Item{ - Data: export.ItemData{ + items = append(items, Item{ + Data: ItemData{ Name: item.name, Body: io.NopCloser((bytes.NewBufferString(item.body))), }, @@ -154,7 +154,7 @@ func (suite *ExportE2ESuite) TestWriteExportCollection() { require.NoError(t, err) defer os.RemoveAll(dir) - err = writeExportCollections(ctx, dir, ecs) + err = ConsumeExportCollections(ctx, dir, ecs, fault.New(true)) require.NoError(t, err, "writing data") for _, col := range test.cols { diff --git a/src/pkg/export/export.go b/src/pkg/export/export.go index 76a6b6d8b..73c173e04 100644 --- a/src/pkg/export/export.go +++ b/src/pkg/export/export.go @@ -7,7 +7,9 @@ import ( // Collection is the interface that is returned to the SDK consumer type Collection interface { - // BasePath gets the base path of the collection + // BasePath gets the base path of the collection. This is derived + // from FullPath, but trim out thing like drive id or any other part + // that is not needed to show the path to the collection. BasePath() string // Items gets the items within the collection(folder) From 26d50832695743077e053338fc14ce3241197dba Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 15:20:38 +0530 Subject: [PATCH 49/62] Separate pieces of sanity test into different package (#3909) Just moving around code, no changes. This just splits the single fine sanity test into multiple files. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/sanity_test/restore/exchange.go | 219 +++++++ src/cmd/sanity_test/restore/onedrive.go | 353 +++++++++++ src/cmd/sanity_test/restore/sharepoint.go | 39 ++ src/cmd/sanity_test/sanity_tests.go | 686 +--------------------- src/cmd/sanity_test/utils/utils.go | 82 +++ 5 files changed, 701 insertions(+), 678 deletions(-) create mode 100644 src/cmd/sanity_test/restore/exchange.go create mode 100644 src/cmd/sanity_test/restore/onedrive.go create mode 100644 src/cmd/sanity_test/restore/sharepoint.go create mode 100644 src/cmd/sanity_test/utils/utils.go diff --git a/src/cmd/sanity_test/restore/exchange.go b/src/cmd/sanity_test/restore/exchange.go new file mode 100644 index 000000000..e484550de --- /dev/null +++ b/src/cmd/sanity_test/restore/exchange.go @@ -0,0 +1,219 @@ +package restore + +import ( + "context" + "fmt" + stdpath "path" + "strings" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-sdk-go/users" + + "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/filters" +) + +// CheckEmailRestoration verifies that the emails count in restored folder is equivalent to +// emails in actual m365 account +func CheckEmailRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + testUser, folderName, dataFolder, baseBackupFolder string, + startTime time.Time, +) { + var ( + restoreFolder models.MailFolderable + itemCount = make(map[string]int32) + restoreItemCount = make(map[string]int32) + builder = client.Users().ByUserId(testUser).MailFolders() + ) + + for { + result, err := builder.Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting mail folders", err) + } + + values := result.GetValue() + + for _, v := range values { + itemName := ptr.Val(v.GetDisplayName()) + + if itemName == folderName { + restoreFolder = v + continue + } + + if itemName == dataFolder || itemName == baseBackupFolder { + // otherwise, recursively aggregate all child folders. + getAllMailSubFolders(ctx, client, testUser, v, itemName, dataFolder, itemCount) + + itemCount[itemName] = ptr.Val(v.GetTotalItemCount()) + } + } + + link, ok := ptr.ValOK(result.GetOdataNextLink()) + if !ok { + break + } + + builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter()) + } + + folderID := ptr.Val(restoreFolder.GetId()) + folderName = ptr.Val(restoreFolder.GetDisplayName()) + ctx = clues.Add( + ctx, + "restore_folder_id", folderID, + "restore_folder_name", folderName) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting restore folder child folders", err) + } + + for _, fld := range childFolder.GetValue() { + restoreDisplayName := ptr.Val(fld.GetDisplayName()) + + // check if folder is the data folder we loaded or the base backup to verify + // the incremental backup worked fine + if strings.EqualFold(restoreDisplayName, dataFolder) || strings.EqualFold(restoreDisplayName, baseBackupFolder) { + count, _ := ptr.ValOK(fld.GetTotalItemCount()) + + restoreItemCount[restoreDisplayName] = count + checkAllSubFolder(ctx, client, fld, testUser, restoreDisplayName, dataFolder, restoreItemCount) + } + } + + verifyEmailData(ctx, restoreItemCount, itemCount) +} + +func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[string]int32) { + for fldName, expected := range messageCount { + got := restoreMessageCount[fldName] + + utils.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("Restore item counts do not match: %s", fldName), + expected, + got) + } +} + +// getAllSubFolder will recursively check for all subfolders and get the corresponding +// email count. +func getAllMailSubFolders( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + testUser string, + r models.MailFolderable, + parentFolder, + dataFolder string, + messageCount map[string]int32, +) { + var ( + folderID = ptr.Val(r.GetId()) + count int32 = 99 + options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ + Top: &count, + }, + } + ) + + ctx = clues.Add(ctx, "parent_folder_id", folderID) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, options) + if err != nil { + utils.Fatal(ctx, "getting mail subfolders", err) + } + + for _, child := range childFolder.GetValue() { + var ( + childDisplayName = ptr.Val(child.GetDisplayName()) + childFolderCount = ptr.Val(child.GetChildFolderCount()) + //nolint:forbidigo + fullFolderName = stdpath.Join(parentFolder, childDisplayName) + ) + + if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { + messageCount[fullFolderName] = ptr.Val(child.GetTotalItemCount()) + // recursively check for subfolders + if childFolderCount > 0 { + parentFolder := fullFolderName + + getAllMailSubFolders(ctx, client, testUser, child, parentFolder, dataFolder, messageCount) + } + } + } +} + +// checkAllSubFolder will recursively traverse inside the restore folder and +// verify that data matched in all subfolders +func checkAllSubFolder( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + r models.MailFolderable, + testUser, + parentFolder, + dataFolder string, + restoreMessageCount map[string]int32, +) { + var ( + folderID = ptr.Val(r.GetId()) + count int32 = 99 + options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ + Top: &count, + }, + } + ) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, options) + if err != nil { + utils.Fatal(ctx, "getting mail subfolders", err) + } + + for _, child := range childFolder.GetValue() { + var ( + childDisplayName = ptr.Val(child.GetDisplayName()) + //nolint:forbidigo + fullFolderName = stdpath.Join(parentFolder, childDisplayName) + ) + + if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { + childTotalCount, _ := ptr.ValOK(child.GetTotalItemCount()) + restoreMessageCount[fullFolderName] = childTotalCount + } + + childFolderCount := ptr.Val(child.GetChildFolderCount()) + + if childFolderCount > 0 { + parentFolder := fullFolderName + checkAllSubFolder(ctx, client, child, testUser, parentFolder, dataFolder, restoreMessageCount) + } + } +} diff --git a/src/cmd/sanity_test/restore/onedrive.go b/src/cmd/sanity_test/restore/onedrive.go new file mode 100644 index 000000000..ca332d608 --- /dev/null +++ b/src/cmd/sanity_test/restore/onedrive.go @@ -0,0 +1,353 @@ +package restore + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/path" +) + +const ( + owner = "owner" +) + +type permissionInfo struct { + entityID string + roles []string +} + +func CheckOneDriveRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + userID, folderName, dataFolder string, + startTime time.Time, +) { + drive, err := client. + Users(). + ByUserId(userID). + Drive(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting the drive:", err) + } + + checkDriveRestoration( + ctx, + client, + path.OneDriveService, + folderName, + ptr.Val(drive.GetId()), + ptr.Val(drive.GetName()), + dataFolder, + startTime, + false) +} + +func checkDriveRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + service path.ServiceType, + folderName, + driveID, + driveName, + dataFolder string, + startTime time.Time, + skipPermissionTest bool, +) { + var ( + // map itemID -> item size + fileSizes = make(map[string]int64) + // map itemID -> permission id -> []permission roles + folderPermissions = make(map[string][]permissionInfo) + restoreFile = make(map[string]int64) + restoredFolderPermissions = make(map[string][]permissionInfo) + ) + + var restoreFolderID string + + ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) + + response, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId("root"). + Children(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting drive by id", err) + } + + for _, driveItem := range response.GetValue() { + var ( + itemID = ptr.Val(driveItem.GetId()) + itemName = ptr.Val(driveItem.GetName()) + ) + + if itemName == folderName { + restoreFolderID = itemID + continue + } + + if itemName != dataFolder { + utils.LogAndPrint(ctx, "test data for folder: %s", dataFolder) + continue + } + + // if it's a file check the size + if driveItem.GetFile() != nil { + fileSizes[itemName] = ptr.Val(driveItem.GetSize()) + } + + if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { + continue + } + + // currently we don't restore blank folders. + // skip permission check for empty folders + if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { + utils.LogAndPrint(ctx, "skipped empty folder: %s", itemName) + continue + } + + folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) + } + + getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) + + checkRestoredDriveItemPermissions( + ctx, + service, + skipPermissionTest, + folderPermissions, + restoredFolderPermissions) + + for fileName, expected := range fileSizes { + utils.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := restoreFile[fileName] + + utils.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} + +func checkRestoredDriveItemPermissions( + ctx context.Context, + service path.ServiceType, + skip bool, + folderPermissions map[string][]permissionInfo, + restoredFolderPermissions map[string][]permissionInfo, +) { + if skip { + return + } + + /** + TODO: replace this check with testElementsMatch + from internal/connecter/graph_connector_helper_test.go + **/ + + for folderName, permissions := range folderPermissions { + utils.LogAndPrint(ctx, "checking for folder: %s", folderName) + + restoreFolderPerm := restoredFolderPermissions[folderName] + + if len(permissions) < 1 { + utils.LogAndPrint(ctx, "no permissions found in: %s", folderName) + continue + } + + permCheck := func() bool { return len(permissions) == len(restoreFolderPerm) } + + if service == path.SharePointService { + permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } + } + + utils.Assert( + ctx, + permCheck, + fmt.Sprintf("wrong number of restored permissions: %s", folderName), + permissions, + restoreFolderPerm) + + for _, perm := range permissions { + eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) } + i := slices.IndexFunc(restoreFolderPerm, eqID) + + utils.Assert( + ctx, + func() bool { return i >= 0 }, + fmt.Sprintf("permission was restored in: %s", folderName), + perm.entityID, + restoreFolderPerm) + + // permissions should be sorted, so a by-index comparison works + restored := restoreFolderPerm[i] + + utils.Assert( + ctx, + func() bool { return slices.Equal(perm.roles, restored.roles) }, + fmt.Sprintf("different roles restored: %s", folderName), + perm.roles, + restored.roles) + } + } +} + +func getOneDriveChildFolder( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, itemID, parentName string, + fileSizes map[string]int64, + folderPermission map[string][]permissionInfo, + startTime time.Time, +) { + response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting child folder", err) + } + + for _, driveItem := range response.GetValue() { + var ( + itemID = ptr.Val(driveItem.GetId()) + itemName = ptr.Val(driveItem.GetName()) + fullName = parentName + "/" + itemName + ) + + folderTime, hasTime := utils.MustGetTimeFromName(ctx, itemName) + if !utils.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) { + continue + } + + // if it's a file check the size + if driveItem.GetFile() != nil { + fileSizes[fullName] = ptr.Val(driveItem.GetSize()) + } + + if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { + continue + } + + // currently we don't restore blank folders. + // skip permission check for empty folders + if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { + utils.LogAndPrint(ctx, "skipped empty folder: %s", fullName) + + continue + } + + folderPermission[fullName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, fullName, fileSizes, folderPermission, startTime) + } +} + +func getRestoredDrive( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, restoreFolderID string, + restoreFile map[string]int64, + restoreFolder map[string][]permissionInfo, + startTime time.Time, +) { + restored, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(restoreFolderID). + Children(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting child folder", err) + } + + for _, item := range restored.GetValue() { + var ( + itemID = ptr.Val(item.GetId()) + itemName = ptr.Val(item.GetName()) + itemSize = ptr.Val(item.GetSize()) + ) + + if item.GetFile() != nil { + restoreFile[itemName] = itemSize + continue + } + + if item.GetFolder() == nil && item.GetPackage() == nil { + continue + } + + restoreFolder[itemName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, restoreFile, restoreFolder, startTime) + } +} + +// --------------------------------------------------------------------------- +// permission helpers +// --------------------------------------------------------------------------- + +func permissionIn( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, itemID string, +) []permissionInfo { + pi := []permissionInfo{} + + pcr, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(itemID). + Permissions(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting permission", err) + } + + for _, perm := range pcr.GetValue() { + if perm.GetGrantedToV2() == nil { + continue + } + + var ( + gv2 = perm.GetGrantedToV2() + permInfo = permissionInfo{} + entityID string + ) + + // TODO: replace with filterUserPermissions in onedrive item.go + if gv2.GetUser() != nil { + entityID = ptr.Val(gv2.GetUser().GetId()) + } else if gv2.GetGroup() != nil { + entityID = ptr.Val(gv2.GetGroup().GetId()) + } + + roles := utils.FilterSlice(perm.GetRoles(), owner) + for _, role := range roles { + permInfo.entityID = entityID + permInfo.roles = append(permInfo.roles, role) + } + + if len(roles) > 0 { + slices.Sort(permInfo.roles) + pi = append(pi, permInfo) + } + } + + return pi +} diff --git a/src/cmd/sanity_test/restore/sharepoint.go b/src/cmd/sanity_test/restore/sharepoint.go new file mode 100644 index 000000000..d84e28f6c --- /dev/null +++ b/src/cmd/sanity_test/restore/sharepoint.go @@ -0,0 +1,39 @@ +package restore + +import ( + "context" + "time" + + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/path" +) + +func CheckSharePointRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + siteID, userID, folderName, dataFolder string, + startTime time.Time, +) { + drive, err := client. + Sites(). + BySiteId(siteID). + Drive(). + Get(ctx, nil) + if err != nil { + utils.Fatal(ctx, "getting the drive:", err) + } + + checkDriveRestoration( + ctx, + client, + path.SharePointService, + folderName, + ptr.Val(drive.GetId()), + ptr.Val(drive.GetName()), + dataFolder, + startTime, + true) +} diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index ab89586b4..a4d455edd 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -2,45 +2,20 @@ package main import ( "context" - "errors" - "fmt" "os" - stdpath "path" "strings" "time" "github.com/alcionai/clues" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/users" - "golang.org/x/exp/slices" - "github.com/alcionai/corso/src/internal/common/dttm" - "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/cmd/sanity_test/utils" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester/tconfig" - "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/logger" - "github.com/alcionai/corso/src/pkg/path" ) -// --------------------------------------------------------------------------- -// types, consts, etc -// --------------------------------------------------------------------------- - -type permissionInfo struct { - entityID string - roles []string -} - -const ( - owner = "owner" -) - -// --------------------------------------------------------------------------- -// main -// --------------------------------------------------------------------------- - func main() { ls := logger.Settings{ File: logger.GetLogFile(""), @@ -60,7 +35,7 @@ func main() { os.Getenv("AZURE_CLIENT_ID"), os.Getenv("AZURE_CLIENT_SECRET")) if err != nil { - fatal(ctx, "creating adapter", err) + utils.Fatal(ctx, "creating adapter", err) } var ( @@ -69,7 +44,7 @@ func main() { testSite = tconfig.GetM365SiteID(ctx) testService = os.Getenv("SANITY_RESTORE_SERVICE") folder = strings.TrimSpace(os.Getenv("SANITY_RESTORE_FOLDER")) - startTime, _ = mustGetTimeFromName(ctx, folder) + startTime, _ = utils.MustGetTimeFromName(ctx, folder) dataFolder = os.Getenv("TEST_DATA") baseBackupFolder = os.Getenv("BASE_BACKUP") ) @@ -85,657 +60,12 @@ func main() { switch testService { case "exchange": - checkEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) + restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) case "onedrive": - checkOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) + restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) case "sharepoint": - checkSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) default: - fatal(ctx, "no service specified", nil) + utils.Fatal(ctx, "unknown service for restore sanity tests", nil) } } - -// --------------------------------------------------------------------------- -// exchange -// --------------------------------------------------------------------------- - -// checkEmailRestoration verifies that the emails count in restored folder is equivalent to -// emails in actual m365 account -func checkEmailRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - testUser, folderName, dataFolder, baseBackupFolder string, - startTime time.Time, -) { - var ( - restoreFolder models.MailFolderable - itemCount = make(map[string]int32) - restoreItemCount = make(map[string]int32) - builder = client.Users().ByUserId(testUser).MailFolders() - ) - - for { - result, err := builder.Get(ctx, nil) - if err != nil { - fatal(ctx, "getting mail folders", err) - } - - values := result.GetValue() - - for _, v := range values { - itemName := ptr.Val(v.GetDisplayName()) - - if itemName == folderName { - restoreFolder = v - continue - } - - if itemName == dataFolder || itemName == baseBackupFolder { - // otherwise, recursively aggregate all child folders. - getAllMailSubFolders(ctx, client, testUser, v, itemName, dataFolder, itemCount) - - itemCount[itemName] = ptr.Val(v.GetTotalItemCount()) - } - } - - link, ok := ptr.ValOK(result.GetOdataNextLink()) - if !ok { - break - } - - builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter()) - } - - folderID := ptr.Val(restoreFolder.GetId()) - folderName = ptr.Val(restoreFolder.GetDisplayName()) - ctx = clues.Add( - ctx, - "restore_folder_id", folderID, - "restore_folder_name", folderName) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting restore folder child folders", err) - } - - for _, fld := range childFolder.GetValue() { - restoreDisplayName := ptr.Val(fld.GetDisplayName()) - - // check if folder is the data folder we loaded or the base backup to verify - // the incremental backup worked fine - if strings.EqualFold(restoreDisplayName, dataFolder) || strings.EqualFold(restoreDisplayName, baseBackupFolder) { - count, _ := ptr.ValOK(fld.GetTotalItemCount()) - - restoreItemCount[restoreDisplayName] = count - checkAllSubFolder(ctx, client, fld, testUser, restoreDisplayName, dataFolder, restoreItemCount) - } - } - - verifyEmailData(ctx, restoreItemCount, itemCount) -} - -func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[string]int32) { - for fldName, expected := range messageCount { - got := restoreMessageCount[fldName] - - assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("Restore item counts do not match: %s", fldName), - expected, - got) - } -} - -// getAllSubFolder will recursively check for all subfolders and get the corresponding -// email count. -func getAllMailSubFolders( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - testUser string, - r models.MailFolderable, - parentFolder, - dataFolder string, - messageCount map[string]int32, -) { - var ( - folderID = ptr.Val(r.GetId()) - count int32 = 99 - options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ - Top: &count, - }, - } - ) - - ctx = clues.Add(ctx, "parent_folder_id", folderID) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, options) - if err != nil { - fatal(ctx, "getting mail subfolders", err) - } - - for _, child := range childFolder.GetValue() { - var ( - childDisplayName = ptr.Val(child.GetDisplayName()) - childFolderCount = ptr.Val(child.GetChildFolderCount()) - //nolint:forbidigo - fullFolderName = stdpath.Join(parentFolder, childDisplayName) - ) - - if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { - messageCount[fullFolderName] = ptr.Val(child.GetTotalItemCount()) - // recursively check for subfolders - if childFolderCount > 0 { - parentFolder := fullFolderName - - getAllMailSubFolders(ctx, client, testUser, child, parentFolder, dataFolder, messageCount) - } - } - } -} - -// checkAllSubFolder will recursively traverse inside the restore folder and -// verify that data matched in all subfolders -func checkAllSubFolder( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - r models.MailFolderable, - testUser, - parentFolder, - dataFolder string, - restoreMessageCount map[string]int32, -) { - var ( - folderID = ptr.Val(r.GetId()) - count int32 = 99 - options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ - Top: &count, - }, - } - ) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, options) - if err != nil { - fatal(ctx, "getting mail subfolders", err) - } - - for _, child := range childFolder.GetValue() { - var ( - childDisplayName = ptr.Val(child.GetDisplayName()) - //nolint:forbidigo - fullFolderName = stdpath.Join(parentFolder, childDisplayName) - ) - - if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { - childTotalCount, _ := ptr.ValOK(child.GetTotalItemCount()) - restoreMessageCount[fullFolderName] = childTotalCount - } - - childFolderCount := ptr.Val(child.GetChildFolderCount()) - - if childFolderCount > 0 { - parentFolder := fullFolderName - checkAllSubFolder(ctx, client, child, testUser, parentFolder, dataFolder, restoreMessageCount) - } - } -} - -// --------------------------------------------------------------------------- -// oneDrive -// --------------------------------------------------------------------------- - -func checkOneDriveRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - userID, folderName, dataFolder string, - startTime time.Time, -) { - drive, err := client. - Users(). - ByUserId(userID). - Drive(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting the drive:", err) - } - - checkDriveRestoration( - ctx, - client, - path.OneDriveService, - folderName, - ptr.Val(drive.GetId()), - ptr.Val(drive.GetName()), - dataFolder, - startTime, - false) -} - -// --------------------------------------------------------------------------- -// sharePoint -// --------------------------------------------------------------------------- - -func checkSharePointRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - siteID, userID, folderName, dataFolder string, - startTime time.Time, -) { - drive, err := client. - Sites(). - BySiteId(siteID). - Drive(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting the drive:", err) - } - - checkDriveRestoration( - ctx, - client, - path.SharePointService, - folderName, - ptr.Val(drive.GetId()), - ptr.Val(drive.GetName()), - dataFolder, - startTime, - true) -} - -// --------------------------------------------------------------------------- -// shared drive tests -// --------------------------------------------------------------------------- - -func checkDriveRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - service path.ServiceType, - folderName, - driveID, - driveName, - dataFolder string, - startTime time.Time, - skipPermissionTest bool, -) { - var ( - // map itemID -> item size - fileSizes = make(map[string]int64) - // map itemID -> permission id -> []permission roles - folderPermissions = make(map[string][]permissionInfo) - restoreFile = make(map[string]int64) - restoredFolderPermissions = make(map[string][]permissionInfo) - ) - - var restoreFolderID string - - ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) - - response, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId("root"). - Children(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting drive by id", err) - } - - for _, driveItem := range response.GetValue() { - var ( - itemID = ptr.Val(driveItem.GetId()) - itemName = ptr.Val(driveItem.GetName()) - ) - - if itemName == folderName { - restoreFolderID = itemID - continue - } - - if itemName != dataFolder { - logAndPrint(ctx, "test data for folder: %s", dataFolder) - continue - } - - // if it's a file check the size - if driveItem.GetFile() != nil { - fileSizes[itemName] = ptr.Val(driveItem.GetSize()) - } - - if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { - continue - } - - // currently we don't restore blank folders. - // skip permission check for empty folders - if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - logAndPrint(ctx, "skipped empty folder: %s", itemName) - continue - } - - folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) - } - - getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) - - checkRestoredDriveItemPermissions( - ctx, - service, - skipPermissionTest, - folderPermissions, - restoredFolderPermissions) - - for fileName, expected := range fileSizes { - logAndPrint(ctx, "checking for file: %s", fileName) - - got := restoreFile[fileName] - - assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("different file size: %s", fileName), - expected, - got) - } - - fmt.Println("Success") -} - -func checkRestoredDriveItemPermissions( - ctx context.Context, - service path.ServiceType, - skip bool, - folderPermissions map[string][]permissionInfo, - restoredFolderPermissions map[string][]permissionInfo, -) { - if skip { - return - } - - /** - TODO: replace this check with testElementsMatch - from internal/connecter/graph_connector_helper_test.go - **/ - - for folderName, permissions := range folderPermissions { - logAndPrint(ctx, "checking for folder: %s", folderName) - - restoreFolderPerm := restoredFolderPermissions[folderName] - - if len(permissions) < 1 { - logAndPrint(ctx, "no permissions found in: %s", folderName) - continue - } - - permCheck := func() bool { return len(permissions) == len(restoreFolderPerm) } - - if service == path.SharePointService { - permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } - } - - assert( - ctx, - permCheck, - fmt.Sprintf("wrong number of restored permissions: %s", folderName), - permissions, - restoreFolderPerm) - - for _, perm := range permissions { - eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) } - i := slices.IndexFunc(restoreFolderPerm, eqID) - - assert( - ctx, - func() bool { return i >= 0 }, - fmt.Sprintf("permission was restored in: %s", folderName), - perm.entityID, - restoreFolderPerm) - - // permissions should be sorted, so a by-index comparison works - restored := restoreFolderPerm[i] - - assert( - ctx, - func() bool { return slices.Equal(perm.roles, restored.roles) }, - fmt.Sprintf("different roles restored: %s", folderName), - perm.roles, - restored.roles) - } - } -} - -func getOneDriveChildFolder( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, itemID, parentName string, - fileSizes map[string]int64, - folderPermission map[string][]permissionInfo, - startTime time.Time, -) { - response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) - if err != nil { - fatal(ctx, "getting child folder", err) - } - - for _, driveItem := range response.GetValue() { - var ( - itemID = ptr.Val(driveItem.GetId()) - itemName = ptr.Val(driveItem.GetName()) - fullName = parentName + "/" + itemName - ) - - folderTime, hasTime := mustGetTimeFromName(ctx, itemName) - if !isWithinTimeBound(ctx, startTime, folderTime, hasTime) { - continue - } - - // if it's a file check the size - if driveItem.GetFile() != nil { - fileSizes[fullName] = ptr.Val(driveItem.GetSize()) - } - - if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { - continue - } - - // currently we don't restore blank folders. - // skip permission check for empty folders - if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - logAndPrint(ctx, "skipped empty folder: %s", fullName) - - continue - } - - folderPermission[fullName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, fullName, fileSizes, folderPermission, startTime) - } -} - -func getRestoredDrive( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, restoreFolderID string, - restoreFile map[string]int64, - restoreFolder map[string][]permissionInfo, - startTime time.Time, -) { - restored, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId(restoreFolderID). - Children(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting child folder", err) - } - - for _, item := range restored.GetValue() { - var ( - itemID = ptr.Val(item.GetId()) - itemName = ptr.Val(item.GetName()) - itemSize = ptr.Val(item.GetSize()) - ) - - if item.GetFile() != nil { - restoreFile[itemName] = itemSize - continue - } - - if item.GetFolder() == nil && item.GetPackage() == nil { - continue - } - - restoreFolder[itemName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, restoreFile, restoreFolder, startTime) - } -} - -// --------------------------------------------------------------------------- -// permission helpers -// --------------------------------------------------------------------------- - -func permissionIn( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, itemID string, -) []permissionInfo { - pi := []permissionInfo{} - - pcr, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId(itemID). - Permissions(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting permission", err) - } - - for _, perm := range pcr.GetValue() { - if perm.GetGrantedToV2() == nil { - continue - } - - var ( - gv2 = perm.GetGrantedToV2() - permInfo = permissionInfo{} - entityID string - ) - - // TODO: replace with filterUserPermissions in onedrive item.go - if gv2.GetUser() != nil { - entityID = ptr.Val(gv2.GetUser().GetId()) - } else if gv2.GetGroup() != nil { - entityID = ptr.Val(gv2.GetGroup().GetId()) - } - - roles := filterSlice(perm.GetRoles(), owner) - for _, role := range roles { - permInfo.entityID = entityID - permInfo.roles = append(permInfo.roles, role) - } - - if len(roles) > 0 { - slices.Sort(permInfo.roles) - pi = append(pi, permInfo) - } - } - - return pi -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -func fatal(ctx context.Context, msg string, err error) { - logger.CtxErr(ctx, err).Error("test failure: " + msg) - fmt.Println(msg+": ", err) - os.Exit(1) -} - -func mustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { - t, err := dttm.ExtractTime(name) - if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { - fatal(ctx, "extracting time from name: "+name, err) - } - - return t, !errors.Is(err, dttm.ErrNoTimeString) -} - -func isWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { - if hasTime { - if bound.Before(check) { - logger.Ctx(ctx). - With("boundary_time", bound, "check_time", check). - Info("skipping restore folder: not older than time bound") - - return false - } - } - - return true -} - -func filterSlice(sl []string, remove string) []string { - r := []string{} - - for _, s := range sl { - if !strings.EqualFold(s, remove) { - r = append(r, s) - } - } - - return r -} - -func assert( - ctx context.Context, - passes func() bool, - header string, - expect, current any, -) { - if passes() { - return - } - - header = "Error: " + header - expected := fmt.Sprintf("* Expected: %+v", expect) - got := fmt.Sprintf("* Current: %+v", current) - - logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) - - fmt.Println(header) - fmt.Println(expected) - fmt.Println(got) - - os.Exit(1) -} - -func logAndPrint(ctx context.Context, tmpl string, vs ...any) { - logger.Ctx(ctx).Infof(tmpl, vs...) - fmt.Printf(tmpl+"\n", vs...) -} diff --git a/src/cmd/sanity_test/utils/utils.go b/src/cmd/sanity_test/utils/utils.go new file mode 100644 index 000000000..d0a877d61 --- /dev/null +++ b/src/cmd/sanity_test/utils/utils.go @@ -0,0 +1,82 @@ +package utils + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/logger" +) + +func Assert( + ctx context.Context, + passes func() bool, + header string, + expect, current any, +) { + if passes() { + return + } + + header = "Error: " + header + expected := fmt.Sprintf("* Expected: %+v", expect) + got := fmt.Sprintf("* Current: %+v", current) + + logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) + + fmt.Println(header) + fmt.Println(expected) + fmt.Println(got) + + os.Exit(1) +} + +func Fatal(ctx context.Context, msg string, err error) { + logger.CtxErr(ctx, err).Error("test failure: " + msg) + fmt.Println(msg+": ", err) + os.Exit(1) +} + +func MustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { + t, err := dttm.ExtractTime(name) + if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { + Fatal(ctx, "extracting time from name: "+name, err) + } + + return t, !errors.Is(err, dttm.ErrNoTimeString) +} + +func IsWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { + if hasTime { + if bound.Before(check) { + logger.Ctx(ctx). + With("boundary_time", bound, "check_time", check). + Info("skipping restore folder: not older than time bound") + + return false + } + } + + return true +} + +func FilterSlice(sl []string, remove string) []string { + r := []string{} + + for _, s := range sl { + if !strings.EqualFold(s, remove) { + r = append(r, s) + } + } + + return r +} + +func LogAndPrint(ctx context.Context, tmpl string, vs ...any) { + logger.Ctx(ctx).Infof(tmpl, vs...) + fmt.Printf(tmpl+"\n", vs...) +} From 8f30db4f6e4b2cd3c764d24f1902551f7f91f847 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Fri, 28 Jul 2023 16:13:59 +0530 Subject: [PATCH 50/62] Sanity tests for OneDrive exports (#3910) Add sanity tests for OneDrive exports (archive and non-archive exports). --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * https://github.com/alcionai/corso/issues/3889 #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- .../actions/backup-restore-test/action.yml | 72 ++++++++- src/cmd/sanity_test/common/common.go | 6 + src/cmd/sanity_test/common/utils.go | 82 ++++++++++ src/cmd/sanity_test/export/onedrive.go | 88 +++++++++++ src/cmd/sanity_test/restore/exchange.go | 12 +- src/cmd/sanity_test/restore/onedrive.go | 144 ++++++++++-------- src/cmd/sanity_test/restore/sharepoint.go | 4 +- src/cmd/sanity_test/sanity_tests.go | 45 ++++-- 8 files changed, 363 insertions(+), 90 deletions(-) create mode 100644 src/cmd/sanity_test/common/common.go create mode 100644 src/cmd/sanity_test/common/utils.go create mode 100644 src/cmd/sanity_test/export/onedrive.go diff --git a/.github/actions/backup-restore-test/action.yml b/.github/actions/backup-restore-test/action.yml index 2d161af63..56213d3e7 100644 --- a/.github/actions/backup-restore-test/action.yml +++ b/.github/actions/backup-restore-test/action.yml @@ -72,12 +72,78 @@ runs: cat /tmp/corsologs - - name: Check ${{ inputs.service }} ${{ inputs.kind }} + - name: Check restore ${{ inputs.service }} ${{ inputs.kind }} shell: bash working-directory: src env: - SANITY_RESTORE_FOLDER: ${{ steps.restore.outputs.result }} - SANITY_RESTORE_SERVICE: ${{ inputs.service }} + SANITY_TEST_KIND: restore + SANITY_TEST_FOLDER: ${{ steps.restore.outputs.result }} + SANITY_TEST_SERVICE: ${{ inputs.service }} + TEST_DATA: ${{ inputs.test-folder }} + BASE_BACKUP: ${{ inputs.base-backup }} + run: | + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log + ./sanity-test + + - name: Export ${{ inputs.service }} ${{ inputs.kind }} + id: export + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive + run: | + set -euo pipefail + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log + ./corso export '${{ inputs.service }}' \ + /tmp/export-${{ inputs.service }}-${{inputs.kind }} \ + --no-stats \ + --hide-progress \ + ${{ inputs.export-args }} \ + --backup '${{ steps.backup.outputs.result }}' + + cat /tmp/corsologs + + - name: Check export ${{ inputs.service }} ${{ inputs.kind }} + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} + env: + SANITY_TEST_KIND: export + SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }} + SANITY_TEST_SERVICE: ${{ inputs.service }} + TEST_DATA: ${{ inputs.test-folder }} + BASE_BACKUP: ${{ inputs.base-backup }} + run: | + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log + ./sanity-test + + - name: Export archive ${{ inputs.service }} ${{ inputs.kind }} + id: export-archive + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive + run: | + set -euo pipefail + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log + ./corso export '${{ inputs.service }}' \ + /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive \ + --no-stats \ + --hide-progress \ + --archive \ + ${{ inputs.export-args }} \ + --backup '${{ steps.backup.outputs.result }}' + + unzip /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive/*.zip \ + -d /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + cat /tmp/corsologs + + - name: Check archive export ${{ inputs.service }} ${{ inputs.kind }} + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} + env: + SANITY_TEST_KIND: export + SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + SANITY_TEST_SERVICE: ${{ inputs.service }} TEST_DATA: ${{ inputs.test-folder }} BASE_BACKUP: ${{ inputs.base-backup }} run: | diff --git a/src/cmd/sanity_test/common/common.go b/src/cmd/sanity_test/common/common.go new file mode 100644 index 000000000..344d6dc19 --- /dev/null +++ b/src/cmd/sanity_test/common/common.go @@ -0,0 +1,6 @@ +package common + +type PermissionInfo struct { + EntityID string + Roles []string +} diff --git a/src/cmd/sanity_test/common/utils.go b/src/cmd/sanity_test/common/utils.go new file mode 100644 index 000000000..e14fa86c6 --- /dev/null +++ b/src/cmd/sanity_test/common/utils.go @@ -0,0 +1,82 @@ +package common + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/logger" +) + +func Assert( + ctx context.Context, + passes func() bool, + header string, + expect, current any, +) { + if passes() { + return + } + + header = "Error: " + header + expected := fmt.Sprintf("* Expected: %+v", expect) + got := fmt.Sprintf("* Current: %+v", current) + + logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) + + fmt.Println(header) + fmt.Println(expected) + fmt.Println(got) + + os.Exit(1) +} + +func Fatal(ctx context.Context, msg string, err error) { + logger.CtxErr(ctx, err).Error("test failure: " + msg) + fmt.Println(msg+": ", err) + os.Exit(1) +} + +func MustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { + t, err := dttm.ExtractTime(name) + if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { + Fatal(ctx, "extracting time from name: "+name, err) + } + + return t, !errors.Is(err, dttm.ErrNoTimeString) +} + +func IsWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { + if hasTime { + if bound.Before(check) { + logger.Ctx(ctx). + With("boundary_time", bound, "check_time", check). + Info("skipping restore folder: not older than time bound") + + return false + } + } + + return true +} + +func FilterSlice(sl []string, remove string) []string { + r := []string{} + + for _, s := range sl { + if !strings.EqualFold(s, remove) { + r = append(r, s) + } + } + + return r +} + +func LogAndPrint(ctx context.Context, tmpl string, vs ...any) { + logger.Ctx(ctx).Infof(tmpl, vs...) + fmt.Printf(tmpl+"\n", vs...) +} diff --git a/src/cmd/sanity_test/export/onedrive.go b/src/cmd/sanity_test/export/onedrive.go new file mode 100644 index 000000000..3d5564bcc --- /dev/null +++ b/src/cmd/sanity_test/export/onedrive.go @@ -0,0 +1,88 @@ +package export + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/internal/common/ptr" +) + +func CheckOneDriveExport( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + userID, folderName, dataFolder string, +) { + drive, err := client. + Users(). + ByUserId(userID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + // map itemID -> item size + var ( + fileSizes = make(map[string]int64) + exportFileSizes = make(map[string]int64) + startTime = time.Now() + ) + + err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error { + if err != nil { + return clues.Stack(err) + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(folderName, path) + if err != nil { + return clues.Stack(err) + } + + exportFileSizes[relPath] = info.Size() + if startTime.After(info.ModTime()) { + startTime = info.ModTime() + } + + return nil + }) + if err != nil { + fmt.Println("Error walking the path:", err) + } + + _ = restore.PopulateDriveDetails( + ctx, + client, + ptr.Val(drive.GetId()), + folderName, + dataFolder, + fileSizes, + map[string][]common.PermissionInfo{}, + startTime) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := exportFileSizes[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} diff --git a/src/cmd/sanity_test/restore/exchange.go b/src/cmd/sanity_test/restore/exchange.go index e484550de..2dc65e6e1 100644 --- a/src/cmd/sanity_test/restore/exchange.go +++ b/src/cmd/sanity_test/restore/exchange.go @@ -12,7 +12,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/users" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/filters" ) @@ -35,7 +35,7 @@ func CheckEmailRestoration( for { result, err := builder.Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting mail folders", err) + common.Fatal(ctx, "getting mail folders", err) } values := result.GetValue() @@ -79,7 +79,7 @@ func CheckEmailRestoration( ChildFolders(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting restore folder child folders", err) + common.Fatal(ctx, "getting restore folder child folders", err) } for _, fld := range childFolder.GetValue() { @@ -102,7 +102,7 @@ func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[ for fldName, expected := range messageCount { got := restoreMessageCount[fldName] - utils.Assert( + common.Assert( ctx, func() bool { return expected == got }, fmt.Sprintf("Restore item counts do not match: %s", fldName), @@ -142,7 +142,7 @@ func getAllMailSubFolders( ChildFolders(). Get(ctx, options) if err != nil { - utils.Fatal(ctx, "getting mail subfolders", err) + common.Fatal(ctx, "getting mail subfolders", err) } for _, child := range childFolder.GetValue() { @@ -194,7 +194,7 @@ func checkAllSubFolder( ChildFolders(). Get(ctx, options) if err != nil { - utils.Fatal(ctx, "getting mail subfolders", err) + common.Fatal(ctx, "getting mail subfolders", err) } for _, child := range childFolder.GetValue() { diff --git a/src/cmd/sanity_test/restore/onedrive.go b/src/cmd/sanity_test/restore/onedrive.go index ca332d608..87ea424c3 100644 --- a/src/cmd/sanity_test/restore/onedrive.go +++ b/src/cmd/sanity_test/restore/onedrive.go @@ -10,7 +10,7 @@ import ( msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" "golang.org/x/exp/slices" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/path" ) @@ -19,11 +19,6 @@ const ( owner = "owner" ) -type permissionInfo struct { - entityID string - roles []string -} - func CheckOneDriveRestoration( ctx context.Context, client *msgraphsdk.GraphServiceClient, @@ -36,7 +31,7 @@ func CheckOneDriveRestoration( Drive(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting the drive:", err) + common.Fatal(ctx, "getting the drive:", err) } checkDriveRestoration( @@ -66,15 +61,58 @@ func checkDriveRestoration( // map itemID -> item size fileSizes = make(map[string]int64) // map itemID -> permission id -> []permission roles - folderPermissions = make(map[string][]permissionInfo) + folderPermissions = make(map[string][]common.PermissionInfo) restoreFile = make(map[string]int64) - restoredFolderPermissions = make(map[string][]permissionInfo) + restoredFolderPermissions = make(map[string][]common.PermissionInfo) ) - var restoreFolderID string - ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) + restoreFolderID := PopulateDriveDetails( + ctx, + client, + driveID, + folderName, + dataFolder, + fileSizes, + folderPermissions, + startTime) + + getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) + + checkRestoredDriveItemPermissions( + ctx, + service, + skipPermissionTest, + folderPermissions, + restoredFolderPermissions) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := restoreFile[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} + +func PopulateDriveDetails( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, folderName, dataFolder string, + fileSizes map[string]int64, + folderPermissions map[string][]common.PermissionInfo, + startTime time.Time, +) string { + var restoreFolderID string + response, err := client. Drives(). ByDriveId(driveID). @@ -83,7 +121,7 @@ func checkDriveRestoration( Children(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting drive by id", err) + common.Fatal(ctx, "getting drive by id", err) } for _, driveItem := range response.GetValue() { @@ -98,7 +136,7 @@ func checkDriveRestoration( } if itemName != dataFolder { - utils.LogAndPrint(ctx, "test data for folder: %s", dataFolder) + common.LogAndPrint(ctx, "test data for folder: %s", dataFolder) continue } @@ -114,7 +152,7 @@ func checkDriveRestoration( // currently we don't restore blank folders. // skip permission check for empty folders if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - utils.LogAndPrint(ctx, "skipped empty folder: %s", itemName) + common.LogAndPrint(ctx, "skipped empty folder: %s", itemName) continue } @@ -122,37 +160,15 @@ func checkDriveRestoration( getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) } - getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) - - checkRestoredDriveItemPermissions( - ctx, - service, - skipPermissionTest, - folderPermissions, - restoredFolderPermissions) - - for fileName, expected := range fileSizes { - utils.LogAndPrint(ctx, "checking for file: %s", fileName) - - got := restoreFile[fileName] - - utils.Assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("different file size: %s", fileName), - expected, - got) - } - - fmt.Println("Success") + return restoreFolderID } func checkRestoredDriveItemPermissions( ctx context.Context, service path.ServiceType, skip bool, - folderPermissions map[string][]permissionInfo, - restoredFolderPermissions map[string][]permissionInfo, + folderPermissions map[string][]common.PermissionInfo, + restoredFolderPermissions map[string][]common.PermissionInfo, ) { if skip { return @@ -164,12 +180,12 @@ func checkRestoredDriveItemPermissions( **/ for folderName, permissions := range folderPermissions { - utils.LogAndPrint(ctx, "checking for folder: %s", folderName) + common.LogAndPrint(ctx, "checking for folder: %s", folderName) restoreFolderPerm := restoredFolderPermissions[folderName] if len(permissions) < 1 { - utils.LogAndPrint(ctx, "no permissions found in: %s", folderName) + common.LogAndPrint(ctx, "no permissions found in: %s", folderName) continue } @@ -179,7 +195,7 @@ func checkRestoredDriveItemPermissions( permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } } - utils.Assert( + common.Assert( ctx, permCheck, fmt.Sprintf("wrong number of restored permissions: %s", folderName), @@ -187,25 +203,25 @@ func checkRestoredDriveItemPermissions( restoreFolderPerm) for _, perm := range permissions { - eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) } + eqID := func(pi common.PermissionInfo) bool { return strings.EqualFold(pi.EntityID, perm.EntityID) } i := slices.IndexFunc(restoreFolderPerm, eqID) - utils.Assert( + common.Assert( ctx, func() bool { return i >= 0 }, fmt.Sprintf("permission was restored in: %s", folderName), - perm.entityID, + perm.EntityID, restoreFolderPerm) // permissions should be sorted, so a by-index comparison works restored := restoreFolderPerm[i] - utils.Assert( + common.Assert( ctx, - func() bool { return slices.Equal(perm.roles, restored.roles) }, + func() bool { return slices.Equal(perm.Roles, restored.Roles) }, fmt.Sprintf("different roles restored: %s", folderName), - perm.roles, - restored.roles) + perm.Roles, + restored.Roles) } } } @@ -215,12 +231,12 @@ func getOneDriveChildFolder( client *msgraphsdk.GraphServiceClient, driveID, itemID, parentName string, fileSizes map[string]int64, - folderPermission map[string][]permissionInfo, + folderPermission map[string][]common.PermissionInfo, startTime time.Time, ) { response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting child folder", err) + common.Fatal(ctx, "getting child folder", err) } for _, driveItem := range response.GetValue() { @@ -230,8 +246,8 @@ func getOneDriveChildFolder( fullName = parentName + "/" + itemName ) - folderTime, hasTime := utils.MustGetTimeFromName(ctx, itemName) - if !utils.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) { + folderTime, hasTime := common.MustGetTimeFromName(ctx, itemName) + if !common.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) { continue } @@ -247,7 +263,7 @@ func getOneDriveChildFolder( // currently we don't restore blank folders. // skip permission check for empty folders if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - utils.LogAndPrint(ctx, "skipped empty folder: %s", fullName) + common.LogAndPrint(ctx, "skipped empty folder: %s", fullName) continue } @@ -262,7 +278,7 @@ func getRestoredDrive( client *msgraphsdk.GraphServiceClient, driveID, restoreFolderID string, restoreFile map[string]int64, - restoreFolder map[string][]permissionInfo, + restoreFolder map[string][]common.PermissionInfo, startTime time.Time, ) { restored, err := client. @@ -273,7 +289,7 @@ func getRestoredDrive( Children(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting child folder", err) + common.Fatal(ctx, "getting child folder", err) } for _, item := range restored.GetValue() { @@ -305,8 +321,8 @@ func permissionIn( ctx context.Context, client *msgraphsdk.GraphServiceClient, driveID, itemID string, -) []permissionInfo { - pi := []permissionInfo{} +) []common.PermissionInfo { + pi := []common.PermissionInfo{} pcr, err := client. Drives(). @@ -316,7 +332,7 @@ func permissionIn( Permissions(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting permission", err) + common.Fatal(ctx, "getting permission", err) } for _, perm := range pcr.GetValue() { @@ -326,7 +342,7 @@ func permissionIn( var ( gv2 = perm.GetGrantedToV2() - permInfo = permissionInfo{} + permInfo = common.PermissionInfo{} entityID string ) @@ -337,14 +353,14 @@ func permissionIn( entityID = ptr.Val(gv2.GetGroup().GetId()) } - roles := utils.FilterSlice(perm.GetRoles(), owner) + roles := common.FilterSlice(perm.GetRoles(), owner) for _, role := range roles { - permInfo.entityID = entityID - permInfo.roles = append(permInfo.roles, role) + permInfo.EntityID = entityID + permInfo.Roles = append(permInfo.Roles, role) } if len(roles) > 0 { - slices.Sort(permInfo.roles) + slices.Sort(permInfo.Roles) pi = append(pi, permInfo) } } diff --git a/src/cmd/sanity_test/restore/sharepoint.go b/src/cmd/sanity_test/restore/sharepoint.go index d84e28f6c..a5146d7a4 100644 --- a/src/cmd/sanity_test/restore/sharepoint.go +++ b/src/cmd/sanity_test/restore/sharepoint.go @@ -6,7 +6,7 @@ import ( msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" + "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/path" ) @@ -23,7 +23,7 @@ func CheckSharePointRestoration( Drive(). Get(ctx, nil) if err != nil { - utils.Fatal(ctx, "getting the drive:", err) + common.Fatal(ctx, "getting the drive:", err) } checkDriveRestoration( diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index a4d455edd..44680ab47 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -9,8 +9,9 @@ import ( "github.com/alcionai/clues" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/export" "github.com/alcionai/corso/src/cmd/sanity_test/restore" - "github.com/alcionai/corso/src/cmd/sanity_test/utils" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/logger" @@ -35,16 +36,16 @@ func main() { os.Getenv("AZURE_CLIENT_ID"), os.Getenv("AZURE_CLIENT_SECRET")) if err != nil { - utils.Fatal(ctx, "creating adapter", err) + common.Fatal(ctx, "creating adapter", err) } var ( client = msgraphsdk.NewGraphServiceClient(adapter) testUser = tconfig.GetM365UserID(ctx) testSite = tconfig.GetM365SiteID(ctx) - testService = os.Getenv("SANITY_RESTORE_SERVICE") - folder = strings.TrimSpace(os.Getenv("SANITY_RESTORE_FOLDER")) - startTime, _ = utils.MustGetTimeFromName(ctx, folder) + testKind = os.Getenv("SANITY_TEST_KIND") // restore or export (cli arg?) + testService = os.Getenv("SANITY_TEST_SERVICE") + folder = strings.TrimSpace(os.Getenv("SANITY_TEST_FOLDER")) dataFolder = os.Getenv("TEST_DATA") baseBackupFolder = os.Getenv("BASE_BACKUP") ) @@ -53,19 +54,33 @@ func main() { ctx, "resource_owner", testUser, "service", testService, - "sanity_restore_folder", folder, - "start_time", startTime.Format(time.RFC3339Nano)) + "sanity_restore_folder", folder) logger.Ctx(ctx).Info("starting sanity test check") - switch testService { - case "exchange": - restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) - case "onedrive": - restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) - case "sharepoint": - restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + switch testKind { + case "restore": + startTime, _ := common.MustGetTimeFromName(ctx, folder) + clues.Add(ctx, "sanity_restore_start_time", startTime.Format(time.RFC3339)) + + switch testService { + case "exchange": + restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) + case "onedrive": + restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) + case "sharepoint": + restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + default: + common.Fatal(ctx, "unknown service for restore sanity tests", nil) + } + case "export": + switch testService { + case "onedrive": + export.CheckOneDriveExport(ctx, client, testUser, folder, dataFolder) + default: + common.Fatal(ctx, "unknown service for export sanity tests", nil) + } default: - utils.Fatal(ctx, "unknown service for restore sanity tests", nil) + common.Fatal(ctx, "unknown test kind (expected restore or export)", nil) } } From 1e126bd2afc14582aa21c2bed389753d5ef78037 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 08:39:17 -0700 Subject: [PATCH 51/62] Allow setting retention parameters on repo init (#3903) Take in information about retention so that when the repo is initialized we can configure retention Not currently exposed by CLI layer **Requires changing the interface for repo init so SDK consumers will require changes** --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3519 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/cli/backup/help_e2e_test.go | 8 +- src/cli/repo/s3.go | 9 +- src/cli/repo/s3_e2e_test.go | 8 +- src/cli/restore/exchange_e2e_test.go | 8 +- src/internal/kopia/conn.go | 30 ++++++- src/internal/kopia/conn_test.go | 82 +++++++++++++++++-- src/internal/kopia/model_store_test.go | 2 +- src/internal/operations/maintenance_test.go | 2 +- src/internal/operations/restore_test.go | 2 +- src/internal/operations/test/helper_test.go | 2 +- src/internal/streamstore/collectables_test.go | 3 +- .../loadtest/repository_load_test.go | 3 +- src/pkg/repository/repository.go | 12 +-- src/pkg/repository/repository_test.go | 60 +++++++++++--- .../repository/repository_unexported_test.go | 9 +- 15 files changed, 203 insertions(+), 37 deletions(-) diff --git a/src/cli/backup/help_e2e_test.go b/src/cli/backup/help_e2e_test.go index b7100d333..f6b4edd83 100644 --- a/src/cli/backup/help_e2e_test.go +++ b/src/cli/backup/help_e2e_test.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -47,7 +48,12 @@ func prepM365Test( vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force) ctx = config.SetViper(ctx, vpr) - repo, err := repository.Initialize(ctx, acct, st, control.DefaultOptions()) + repo, err := repository.Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) return acct, st, repo, vpr, recorder, cfgFP diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 58f04764c..b611e93e5 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -15,6 +15,7 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/pkg/account" + rep "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -158,7 +159,13 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config")) } - r, err := repository.Initialize(ctx, cfg.Account, cfg.Storage, opt) + // TODO(ashmrtn): Wire to flags for retention during repo init. + r, err := repository.Initialize( + ctx, + cfg.Account, + cfg.Storage, + opt, + rep.Retention{}) if err != nil { if succeedIfExists && errors.Is(err, repository.ErrorRepoAlreadyExists) { return nil diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index 4c3af6e5f..4b4975a6a 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" ) @@ -200,7 +201,12 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() { ctx = config.SetViper(ctx, vpr) // init the repo first - _, err = repository.Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + _, err = repository.Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) // then test it diff --git a/src/cli/restore/exchange_e2e_test.go b/src/cli/restore/exchange_e2e_test.go index 56b0e7255..904025116 100644 --- a/src/cli/restore/exchange_e2e_test.go +++ b/src/cli/restore/exchange_e2e_test.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" @@ -83,7 +84,12 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() { ) // init the repo first - suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{}) + suite.repo, err = repository.Initialize( + ctx, + suite.acct, + suite.st, + control.Options{}, + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) suite.backupOps = make(map[path.CategoryType]operations.BackupOperation) diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index e9d20918a..7eac9df5c 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -74,7 +74,11 @@ func NewConn(s storage.Storage) *conn { } } -func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { +func (w *conn) Initialize( + ctx context.Context, + opts repository.Options, + retentionOpts repository.Retention, +) error { bst, err := blobStoreByProvider(ctx, opts, w.storage) if err != nil { return clues.Wrap(err, "initializing storage") @@ -86,8 +90,23 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { return clues.Stack(err).WithClues(ctx) } - // todo - issue #75: nil here should be a storage.NewRepoOptions() - if err = repo.Initialize(ctx, bst, nil, cfg.CorsoPassphrase); err != nil { + rOpts := retention.NewOpts() + if err := rOpts.Set(retentionOpts); err != nil { + return clues.Wrap(err, "setting retention configuration").WithClues(ctx) + } + + blobCfg, _, err := rOpts.AsConfigs(ctx) + if err != nil { + return clues.Stack(err) + } + + // Minimal config for retention if caller requested it. + kopiaOpts := repo.NewRepositoryOptions{ + RetentionMode: blobCfg.RetentionMode, + RetentionPeriod: blobCfg.RetentionPeriod, + } + + if err = repo.Initialize(ctx, bst, &kopiaOpts, cfg.CorsoPassphrase); err != nil { if errors.Is(err, repo.ErrAlreadyInitialized) { return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) } @@ -111,7 +130,10 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { return clues.Stack(err).WithClues(ctx) } - return nil + // Calling with all parameters here will set extend object locks for + // maintenance. Parameters for actual retention should have been set during + // initialization and won't be updated again. + return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil() } func (w *conn) Connect(ctx context.Context, opts repository.Options) error { diff --git a/src/internal/kopia/conn_test.go b/src/internal/kopia/conn_test.go index 899447e00..36a2bcdfd 100644 --- a/src/internal/kopia/conn_test.go +++ b/src/internal/kopia/conn_test.go @@ -7,12 +7,15 @@ import ( "time" "github.com/alcionai/clues" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -26,7 +29,7 @@ func openKopiaRepo( st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - if err := k.Initialize(ctx, repository.Options{}); err != nil { + if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil { return nil, err } @@ -82,13 +85,13 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() { st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) err = k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Initialize(ctx, repository.Options{}) + err = k.Initialize(ctx, repository.Options{}, repository.Retention{}) assert.Error(t, err, clues.ToCore(err)) assert.ErrorIs(t, err, ErrorRepoAlreadyExists) } @@ -103,7 +106,7 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() { st.Provider = storage.ProviderUnknown k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) assert.Error(t, err, clues.ToCore(err)) } @@ -413,7 +416,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Initialize(ctx, opts) + err := k.Initialize(ctx, opts, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) kopiaOpts := k.ClientOptions() @@ -453,3 +456,72 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { err = k.Close(ctx) assert.NoError(t, err, clues.ToCore(err)) } + +// --------------- +// integration tests that require object locking to be enabled on the bucket. +// --------------- +type ConnRetentionIntegrationSuite struct { + tester.Suite +} + +func TestConnRetentionIntegrationSuite(t *testing.T) { + suite.Run(t, &ConnRetentionIntegrationSuite{ + Suite: tester.NewRetentionSuite( + t, + [][]string{storeTD.AWSStorageCredEnvs}, + ), + }) +} + +// Test that providing retention doesn't change anything but retention values +// from the default values that kopia uses. +func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + st1 := storeTD.NewPrefixedS3Storage(t) + + k1 := NewConn(st1) + err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err)) + + st2 := storeTD.NewPrefixedS3Storage(t) + + k2 := NewConn(st2) + err = k2.Initialize( + ctx, + repository.Options{}, + repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }) + require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err)) + + dr1, ok := k1.Repository.(repo.DirectRepository) + require.True(t, ok, "getting direct repo 1") + + dr2, ok := k2.Repository.(repo.DirectRepository) + require.True(t, ok, "getting direct repo 2") + + format1 := dr1.FormatManager().ScrubbedContentFormat() + format2 := dr2.FormatManager().ScrubbedContentFormat() + + assert.Equal(t, format1, format2) + + blobCfg1, err := dr1.FormatManager().BlobCfgBlob() + require.NoError(t, err, "getting blob config 1: %v", clues.ToCore(err)) + + blobCfg2, err := dr2.FormatManager().BlobCfgBlob() + require.NoError(t, err, "getting retention config 2: %v", clues.ToCore(err)) + + assert.NotEqual(t, blobCfg1, blobCfg2) + + // Check to make sure retention not enabled unexpectedly. + checkRetentionParams(t, ctx, k1, blob.RetentionMode(""), 0, assert.False) + + // Some checks to make sure retention was fully initialized as expected. + checkRetentionParams(t, ctx, k2, blob.Governance, time.Hour*48, assert.True) +} diff --git a/src/internal/kopia/model_store_test.go b/src/internal/kopia/model_store_test.go index bd7d651ca..aa929ea9d 100644 --- a/src/internal/kopia/model_store_test.go +++ b/src/internal/kopia/model_store_test.go @@ -808,7 +808,7 @@ func openConnAndModelStore( st := storeTD.NewPrefixedS3Storage(t) c := NewConn(st) - err := c.Initialize(ctx, repository.Options{}) + err := c.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/operations/maintenance_test.go b/src/internal/operations/maintenance_test.go index 4ec6a3ee9..6c822e1cd 100644 --- a/src/internal/operations/maintenance_test.go +++ b/src/internal/operations/maintenance_test.go @@ -40,7 +40,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() { ctx, flush := tester.NewContext(t) defer flush() - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) kw, err := kopia.NewWrapper(k) diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 5b124ee64..c97812a63 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -243,7 +243,7 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { suite.acct = tconfig.NewM365Account(t) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index f1bf65261..c826b3e44 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -102,7 +102,7 @@ func prepNewTestBackupOp( k := kopia.NewConn(bod.st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/streamstore/collectables_test.go b/src/internal/streamstore/collectables_test.go index d8897c146..ccbeab7c2 100644 --- a/src/internal/streamstore/collectables_test.go +++ b/src/internal/streamstore/collectables_test.go @@ -44,7 +44,8 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() { st := storeTD.NewPrefixedS3Storage(t) k := kopia.NewConn(st) - require.NoError(t, k.Initialize(ctx, repository.Options{})) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, clues.ToCore(err)) suite.kcloser = func() { k.Close(ctx) } diff --git a/src/pkg/repository/loadtest/repository_load_test.go b/src/pkg/repository/loadtest/repository_load_test.go index c723955d1..718ac678d 100644 --- a/src/pkg/repository/loadtest/repository_load_test.go +++ b/src/pkg/repository/loadtest/repository_load_test.go @@ -21,6 +21,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -102,7 +103,7 @@ func initM365Repo(t *testing.T) ( FailureHandling: control.FailFast, } - repo, err := repository.Initialize(ctx, ac, st, opts) + repo, err := repository.Initialize(ctx, ac, st, opts, ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) return ctx, repo, ac, st diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 1417d872d..291ef5fd0 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -25,7 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - rep "github.com/alcionai/corso/src/pkg/control/repository" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" @@ -82,7 +82,7 @@ type Repository interface { ) (operations.ExportOperation, error) NewMaintenance( ctx context.Context, - mOpts rep.Maintenance, + mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) DeleteBackup(ctx context.Context, id string) error BackupGetter @@ -117,7 +117,8 @@ func (r repository) GetID() string { // - validate the m365 account & secrets // - connect to the m365 account to ensure communication capability // - validate the provider config & secrets -// - initialize the kopia repo with the provider +// - initialize the kopia repo with the provider and retention parameters +// - update maintenance retention parameters as needed // - store the configuration details // - connect to the provider // - return the connected repository @@ -126,6 +127,7 @@ func Initialize( acct account.Account, s storage.Storage, opts control.Options, + retentionOpts ctrlRepo.Retention, ) (repo Repository, err error) { ctx = clues.Add( ctx, @@ -140,7 +142,7 @@ func Initialize( }() kopiaRef := kopia.NewConn(s) - if err := kopiaRef.Initialize(ctx, opts.Repo); err != nil { + if err := kopiaRef.Initialize(ctx, opts.Repo, retentionOpts); err != nil { // replace common internal errors so that sdk users can check results with errors.Is() if errors.Is(err, kopia.ErrorRepoAlreadyExists) { return nil, clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) @@ -408,7 +410,7 @@ func (r repository) NewRestore( func (r repository) NewMaintenance( ctx context.Context, - mOpts rep.Maintenance, + mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) { return operations.NewMaintenanceOperation( ctx, diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 34c55d4e7..2ee90b5da 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -60,7 +60,12 @@ func (suite *RepositoryUnitSuite) TestInitialize() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Initialize(ctx, test.account, st, control.DefaultOptions()) + _, err = Initialize( + ctx, + test.account, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -137,7 +142,12 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { defer flush() st := test.storage(t) - r, err := Initialize(ctx, test.account, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + test.account, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) if err == nil { defer func() { err := r.Close(ctx) @@ -169,7 +179,7 @@ func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() { st.SessionName = "corso-repository-test" st.SessionDuration = roleDuration.String() - r, err := Initialize(ctx, account.Account{}, st, control.Options{}) + r, err := Initialize(ctx, account.Account{}, st, control.Options{}, ctrlRepo.Retention{}) require.NoError(suite.T(), err) defer func() { @@ -186,7 +196,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + repo, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) // now re-connect @@ -203,7 +218,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) oldID := r.GetID() @@ -228,7 +248,12 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) userID := tconfig.M365UserID(t) @@ -250,7 +275,12 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg) @@ -269,7 +299,12 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.DefaultOptions()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{}) @@ -286,7 +321,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) + repo, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err) // now re-connect @@ -350,7 +390,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { ctx, flush := tester.NewContext(t) defer flush() - repo, err := Initialize(ctx, acct, st, test.opts()) + repo, err := Initialize(ctx, acct, st, test.opts(), ctrlRepo.Retention{}) require.NoError(t, err) r := repo.(*repository) diff --git a/src/pkg/repository/repository_unexported_test.go b/src/pkg/repository/repository_unexported_test.go index e284fad30..0e600157d 100644 --- a/src/pkg/repository/repository_unexported_test.go +++ b/src/pkg/repository/repository_unexported_test.go @@ -240,7 +240,7 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() { require.NotNil(t, k) - err = k.Initialize(ctx, rep.Options{}) + err = k.Initialize(ctx, rep.Options{}, rep.Retention{}) require.NoError(t, err, clues.ToCore(err)) err = k.Connect(ctx, rep.Options{}) @@ -291,8 +291,11 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() { k = kopia.NewConn(s) ) - require.NoError(t, k.Initialize(ctx, rep.Options{})) - require.NoError(t, k.Connect(ctx, rep.Options{})) + err := k.Initialize(ctx, rep.Options{}, rep.Retention{}) + require.NoError(t, err, "initializing repo: %v", clues.ToCore(err)) + + err = k.Connect(ctx, rep.Options{}) + require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err)) defer k.Close(ctx) From b559cba4e70d5c612b151a195327057be2af36e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:09:51 +0000 Subject: [PATCH 52/62] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.309=20to=201.44.311=20in=20/src=20(#?= =?UTF-8?q?3926)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.309 to 1.44.311.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.311 (2023-07-28)

Service Client Updates

  • service/sqs: Updates service documentation
    • Documentation changes related to SQS APIs.

SDK Bugs

  • aws/session: Modify resolving sso credential logic to fix stack overflow bug while configuring shared config profile via env var.

Release v1.44.310 (2023-07-27)

Service Client Updates

  • service/autoscaling: Updates service API, documentation, paginators, and examples
    • This release updates validation for instance types used in the AllowedInstanceTypes and ExcludedInstanceTypes parameters of the InstanceRequirements property of a MixedInstancesPolicy.
  • service/ebs: Updates service API and documentation
  • service/ec2: Updates service API, documentation, and waiters
    • SDK and documentation updates for Amazon Elastic Block Store APIs
  • service/eks: Updates service API
  • service/sagemaker: Updates service API and documentation
    • Expose ProfilerConfig attribute in SageMaker Search API response.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.309&new-version=1.44.311)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 992c60f69..fa4e729e3 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.309 + github.com/aws/aws-sdk-go v1.44.311 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 0c1ea866b..1a2e8345b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.309 h1:IPJOFBzXekakxmEpDwd4RTKmmBR6LIAiXgNsM51bWbU= -github.com/aws/aws-sdk-go v1.44.309/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.311 h1:60i8hyVMOXqabKJQPCq4qKRBQ6hRafI/WOcDxGM+J7Q= +github.com/aws/aws-sdk-go v1.44.311/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From a836e877af340b4e3318849fd5f8eb0d9ccf876f Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 10:47:42 -0700 Subject: [PATCH 53/62] Pull backup deletion logic into function (#3924) Make a separate function to handle backup deletion. Makes repo connection management a little bit easier since a simple defer will work now. Will also help keep things organized as we expand these tests to do other things like use the point-in-time function to check the deleted backups are still accessible, especially since that will require opening another connection to the repo --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3799 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/longevity_test/longevity.go | 71 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index 33de060f8..ef17b7e32 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "github.com/alcionai/clues" "github.com/spf13/cobra" "github.com/alcionai/corso/src/cli/config" @@ -16,6 +17,51 @@ import ( "github.com/alcionai/corso/src/pkg/store" ) +// deleteBackups connects to the repository and deletes all backups for +// service that are at least deletionDays old. Returns the IDs of all backups +// that were deleted. +func deleteBackups( + ctx context.Context, + service path.ServiceType, + deletionDays int, +) ([]string, error) { + ctx = clues.Add(ctx, "cutoff_days", deletionDays) + + r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, nil) + if err != nil { + return nil, clues.Wrap(err, "connecting to account").WithClues(ctx) + } + + defer r.Close(ctx) + + backups, err := r.BackupsByTag(ctx, store.Service(service)) + if err != nil { + return nil, clues.Wrap(err, "listing backups").WithClues(ctx) + } + + var ( + deleted []string + cutoff = time.Now().Add(-time.Hour * 24 * time.Duration(deletionDays)) + ) + + for _, backup := range backups { + if backup.StartAndEndTime.CompletedAt.Before(cutoff) { + if err := r.DeleteBackup(ctx, backup.ID.String()); err != nil { + return nil, clues.Wrap( + err, + "deleting backup"). + With("backup_id", backup.ID). + WithClues(ctx) + } + + deleted = append(deleted, backup.ID.String()) + logAndPrint(ctx, "Deleted backup %s", backup.ID.String()) + } + } + + return deleted, nil +} + func main() { var ( service path.ServiceType @@ -39,31 +85,16 @@ func main() { fatal(cc.Context(), "unknown service", nil) } - r, _, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil) - if err != nil { - fatal(cc.Context(), "unable to connect account", err) - } - - defer r.Close(cc.Context()) - - backups, err := r.BackupsByTag(cc.Context(), store.Service(service)) - if err != nil { - fatal(cc.Context(), "unable to find backups", err) - } + ctx := clues.Add(cc.Context(), "service", service) days, err := strconv.Atoi(os.Getenv("DELETION_DAYS")) if err != nil { - fatal(cc.Context(), "invalid no of days provided", nil) + fatal(ctx, "invalid number of days provided", nil) } - for _, backup := range backups { - if backup.StartAndEndTime.CompletedAt.Before(time.Now().AddDate(0, 0, -days)) { - if err := r.DeleteBackup(cc.Context(), backup.ID.String()); err != nil { - fatal(cc.Context(), "deleting backup", err) - } - - logAndPrint(cc.Context(), "Deleted backup %s", backup.ID.String()) - } + _, err = deleteBackups(ctx, service, days) + if err != nil { + fatal(cc.Context(), "deleting backups", clues.Stack(err)) } } From b12bb50a4aff88dc2d3c9afbc10126805eacc416 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 11:27:54 -0700 Subject: [PATCH 54/62] Retention update op (#3865) Create new operation type to update retention parameters for immutable backups --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/operations/retention_config.go | 77 +++++++++++++++++++ .../operations/retention_config_test.go | 74 ++++++++++++++++++ src/pkg/repository/repository.go | 16 ++++ 3 files changed, 167 insertions(+) create mode 100644 src/internal/operations/retention_config.go create mode 100644 src/internal/operations/retention_config_test.go diff --git a/src/internal/operations/retention_config.go b/src/internal/operations/retention_config.go new file mode 100644 index 000000000..ca8504382 --- /dev/null +++ b/src/internal/operations/retention_config.go @@ -0,0 +1,77 @@ +package operations + +import ( + "context" + "time" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/crash" + "github.com/alcionai/corso/src/internal/events" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" +) + +// RetentionConfigOperation wraps an operation with restore-specific props. +type RetentionConfigOperation struct { + operation + Results RetentionConfigResults + rcOpts repository.Retention +} + +// RetentionConfigResults aggregate the details of the results of the operation. +type RetentionConfigResults struct { + stats.StartAndEndTime +} + +// NewRetentionConfigOperation constructs and validates an operation to change +// retention parameters. +func NewRetentionConfigOperation( + ctx context.Context, + opts control.Options, + kw *kopia.Wrapper, + rcOpts repository.Retention, + bus events.Eventer, +) (RetentionConfigOperation, error) { + op := RetentionConfigOperation{ + operation: newOperation(opts, bus, count.New(), kw, nil), + rcOpts: rcOpts, + } + + // Don't run validation because we don't populate the model store. + + return op, nil +} + +func (op *RetentionConfigOperation) Run(ctx context.Context) (err error) { + defer func() { + if crErr := crash.Recovery(ctx, recover(), "retention_config"); crErr != nil { + err = crErr + } + }() + + op.Results.StartedAt = time.Now() + + // TODO(ashmrtn): Send telemetry? + + return op.do(ctx) +} + +func (op *RetentionConfigOperation) do(ctx context.Context) error { + defer func() { + op.Results.CompletedAt = time.Now() + }() + + err := op.operation.kopia.SetRetentionParameters(ctx, op.rcOpts) + if err != nil { + op.Status = Failed + return clues.Wrap(err, "running retention config operation") + } + + op.Status = Completed + + return nil +} diff --git a/src/internal/operations/retention_config_test.go b/src/internal/operations/retention_config_test.go new file mode 100644 index 000000000..ce57cd879 --- /dev/null +++ b/src/internal/operations/retention_config_test.go @@ -0,0 +1,74 @@ +package operations + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" +) + +type RetentionConfigOpIntegrationSuite struct { + tester.Suite +} + +func TestRetentionConfigOpIntegrationSuite(t *testing.T) { + suite.Run(t, &RetentionConfigOpIntegrationSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *RetentionConfigOpIntegrationSuite) TestRepoRetentionConfig() { + var ( + t = suite.T() + // need to initialize the repository before we can test connecting to it. + st = storeTD.NewPrefixedS3Storage(t) + k = kopia.NewConn(st) + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, clues.ToCore(err)) + + kw, err := kopia.NewWrapper(k) + // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe + // to close here. + k.Close(ctx) + + require.NoError(t, err, clues.ToCore(err)) + + defer kw.Close(ctx) + + // Only set extend locks parameter as other retention options require a bucket + // with object locking enabled. There's more complete tests in the kopia + // package. + rco, err := NewRetentionConfigOperation( + ctx, + control.DefaultOptions(), + kw, + repository.Retention{ + Extend: ptr.To(true), + }, + evmock.NewBus()) + require.NoError(t, err, clues.ToCore(err)) + + err = rco.Run(ctx) + assert.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, Completed, rco.Status) + assert.NotZero(t, rco.Results.StartedAt) + assert.NotZero(t, rco.Results.CompletedAt) + assert.NotEqual(t, rco.Results.StartedAt, rco.Results.CompletedAt) +} diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 291ef5fd0..d4b0f6eaa 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -84,6 +84,10 @@ type Repository interface { ctx context.Context, mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) + NewRetentionConfig( + ctx context.Context, + rcOpts ctrlRepo.Retention, + ) (operations.RetentionConfigOperation, error) DeleteBackup(ctx context.Context, id string) error BackupGetter // ConnectToM365 establishes graph api connections @@ -420,6 +424,18 @@ func (r repository) NewMaintenance( r.Bus) } +func (r repository) NewRetentionConfig( + ctx context.Context, + rcOpts ctrlRepo.Retention, +) (operations.RetentionConfigOperation, error) { + return operations.NewRetentionConfigOperation( + ctx, + r.Opts, + r.dataLayer, + rcOpts, + r.Bus) +} + // Backup retrieves a backup by id. func (r repository) Backup(ctx context.Context, id string) (*backup.Backup, error) { return getBackup(ctx, id, store.NewKopiaStore(r.modelStore)) From e0368ba45af72be2d97e7e82285f76bba84c0636 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:00:53 -0700 Subject: [PATCH 55/62] Track and print stats about merging (#3930) Track and print information about the number of directories that were moved, deleted, recursively moved or deleted, and that didn't move at all This can help shed light on what's happening and generally help with debugging issues that may crop up --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3929 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/upload.go | 52 ++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 8be75009f..ea5a55dbd 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -28,6 +28,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph/metadata" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -852,6 +853,7 @@ func traverseBaseDir( expectedDirPath *path.Builder, dir fs.Directory, roots map[string]*treeMap, + stats *count.Bus, ) error { ctx = clues.Add(ctx, "old_dir_path", oldDirPath, @@ -889,9 +891,27 @@ func traverseBaseDir( // This directory was deleted. if upb == nil { currentPath = nil + + stats.Inc(statDel) } else { - // This directory was moved/renamed and the new location is in upb. + // This directory was explicitly mentioned and the new (possibly + // unchanged) location is in upb. currentPath = upb.ToBuilder() + + if oldDirPath.String() == currentPath.String() { + stats.Inc(statNoMove) + } else { + stats.Inc(statMove) + } + } + } else { + // Just stats tracking stuff. + if currentPath == nil { + stats.Inc(statRecursiveDel) + } else if oldDirPath.String() == currentPath.String() { + stats.Inc(statNoMove) + } else { + stats.Inc(statRecursiveMove) } } @@ -920,7 +940,8 @@ func traverseBaseDir( oldDirPath, currentPath, dEntry, - roots) + roots, + stats) }) if err != nil { return clues.Wrap(err, "traversing base directory") @@ -993,6 +1014,21 @@ func logBaseInfo(ctx context.Context, m ManifestEntry) { "base_backup_id", mbID) } +const ( + // statNoMove denotes an directory that wasn't moved at all. + statNoMove = "directories_not_moved" + // statMove denotes an directory that was explicitly moved. + statMove = "directories_explicitly_moved" + // statRecursiveMove denotes an directory that moved because one or more or + // its ancestors moved and it wasn't explicitly mentioned. + statRecursiveMove = "directories_recursively_moved" + // statDel denotes a directory that was explicitly deleted. + statDel = "directories_explicitly_deleted" + // statRecursiveDel denotes a directory that was deleted because one or more + // of its ancestors was deleted and it wasn't explicitly mentioned. + statRecursiveDel = "directories_recursively_deleted" +) + func inflateBaseTree( ctx context.Context, loader snapshotLoader, @@ -1058,10 +1094,13 @@ func inflateBaseTree( // The prefix is the tenant/service/owner/category set, which remains // otherwise unchecked in tree inflation below this point. newSubtreePath := subtreePath.ToBuilder() + if p, ok := updatedPaths[subtreePath.String()]; ok { newSubtreePath = p.ToBuilder() } + stats := count.New() + if err = traverseBaseDir( ictx, 0, @@ -1070,9 +1109,18 @@ func inflateBaseTree( newSubtreePath.Dir(), subtreeDir, roots, + stats, ); err != nil { return clues.Wrap(err, "traversing base snapshot").WithClues(ictx) } + + logger.Ctx(ctx).Infow( + "merge subtree stats", + statNoMove, stats.Get(statNoMove), + statMove, stats.Get(statMove), + statRecursiveMove, stats.Get(statRecursiveMove), + statDel, stats.Get(statDel), + statRecursiveDel, stats.Get(statRecursiveDel)) } return nil From f672325193679feda6fff0ce30400164c05f1495 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:46:49 -0700 Subject: [PATCH 56/62] Update clues dependency (#3933) Pull in fix for double hashing some values ---- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3895 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/go.mod | 2 +- src/go.sum | 4 ++-- src/internal/common/pii/url.go | 4 ++++ src/internal/observe/observe.go | 4 ++-- src/pkg/control/restore.go | 4 ---- src/pkg/filters/filters.go | 2 +- src/pkg/path/elements.go | 4 ---- src/pkg/path/path.go | 4 ---- src/pkg/selectors/scopes.go | 3 --- src/pkg/selectors/selectors.go | 2 +- 10 files changed, 11 insertions(+), 22 deletions(-) diff --git a/src/go.mod b/src/go.mod index fa4e729e3..9b03cf95b 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.2023071323 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a + github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go v1.44.311 github.com/aws/aws-xray-sdk-go v1.8.1 diff --git a/src/go.sum b/src/go.sum index 1a2e8345b..74f71dadf 100644 --- a/src/go.sum +++ b/src/go.sum @@ -53,8 +53,8 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a h1:mtJyeK/FhArTn06M5Lfgxk/GWnu8yqCGNN1BY16vjaA= -github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a/go.mod h1:MLEWSZ0cjEMg6hiGCRvE7AtrOhs7deBcm7ZrJBpfGRM= +github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 h1:husF7eAYw2HEzgjfAmNy+ZLzyztJV2SyoUngSUo829Y= +github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4/go.mod h1:MLEWSZ0cjEMg6hiGCRvE7AtrOhs7deBcm7ZrJBpfGRM= github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 h1:w50/aVU+zRP5lvE86TSSCCYrrEyuXOlJA06R5RdTS8Y= github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377/go.mod h1:WH725ws0BYpZpTkVh4uqFHHPiiJuirl1Cm73jv5RYyA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/src/internal/common/pii/url.go b/src/internal/common/pii/url.go index 339bee19f..de6e5ed8a 100644 --- a/src/internal/common/pii/url.go +++ b/src/internal/common/pii/url.go @@ -78,6 +78,10 @@ func (u SafeURL) Format(fs fmt.State, _ rune) { fmt.Fprint(fs, u.Conceal()) } +func (u SafeURL) PlainString() string { + return u.URL +} + // String complies with Stringer to ensure the Conceal() version // of the url is printed anytime it gets transformed to a string. func (u SafeURL) String() string { diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index c05297587..50994eacb 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -563,8 +563,8 @@ func (b bulletf) String() string { // observe progress bar. Logged values should only use // the fmt %v to ensure Concealers hide PII. func plainString(v any) string { - if ps, ok := v.(clues.PlainStringer); ok { - return ps.PlainString() + if c, ok := v.(clues.Concealer); ok { + return c.PlainString() } return fmt.Sprintf("%v", v) diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 74cf88093..cce3bea9b 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -107,10 +107,6 @@ var ( // interface compliance required for handling PII _ clues.Concealer = &RestoreConfig{} _ fmt.Stringer = &RestoreConfig{} - - // interface compliance for the observe package to display - // values without concealing PII. - _ clues.PlainStringer = &RestoreConfig{} ) func (rc RestoreConfig) marshal() string { diff --git a/src/pkg/filters/filters.go b/src/pkg/filters/filters.go index 40c12a630..c697203b7 100644 --- a/src/pkg/filters/filters.go +++ b/src/pkg/filters/filters.go @@ -501,7 +501,7 @@ func suffixed(target, input string) bool { // Printers and PII control // ---------------------------------------------------------------------------------------------------- -var _ clues.PlainConcealer = &Filter{} +var _ clues.Concealer = &Filter{} var safeFilterValues = map[string]struct{}{"*": {}} diff --git a/src/pkg/path/elements.go b/src/pkg/path/elements.go index d1ca932dc..5a0802d50 100644 --- a/src/pkg/path/elements.go +++ b/src/pkg/path/elements.go @@ -54,10 +54,6 @@ var ( // interface compliance required for handling PII _ clues.Concealer = &Elements{} _ fmt.Stringer = &Elements{} - - // interface compliance for the observe package to display - // values without concealing PII. - _ clues.PlainStringer = &Elements{} ) // Elements are a PII Concealer-compliant slice of elements within a path. diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index ca63bf8fe..b7cd38da0 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -120,10 +120,6 @@ type Path interface { // is appropriately hidden from logging, errors, and other outputs. clues.Concealer fmt.Stringer - - // In the rare case that the path needs to get printed as a plain string, - // without obscuring values for PII. - clues.PlainStringer } // interface compliance required for handling PII diff --git a/src/pkg/selectors/scopes.go b/src/pkg/selectors/scopes.go index 2a860892b..aebd0f156 100644 --- a/src/pkg/selectors/scopes.go +++ b/src/pkg/selectors/scopes.go @@ -153,9 +153,6 @@ type ( // Primarily to ensure that root- or mid-tier scopes (such as folders) // cascade 'Any' matching to more granular categories. setDefaults() - - // Scopes need to comply with PII printing controls. - clues.PlainConcealer } // scopeT is the generic type interface of a scoper. scopeT interface { diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 1e148e04b..936bc3a32 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -341,7 +341,7 @@ func selectorAsIface[T any](s Selector) (T, error) { // Stringers and Concealers // --------------------------------------------------------------------------- -var _ clues.PlainConcealer = &Selector{} +var _ clues.Concealer = &Selector{} type loggableSelector struct { Service service `json:"service,omitempty"` From f0425a0f9ded79bfc9382dee2a04dd43a77ccd03 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:26:38 -0700 Subject: [PATCH 57/62] Add test to lookup deleted backup by opening old version of repo (#3925) Make helper function that allows * opening the repo at a previous point in time * listing backups * searching the list for backups in a given set This will allow us to open the repo just prior to the point in time at which backups were deleted so we can ensure PITR is working right This PR does not enable this functionality in CI, it only adds the code for it --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3799 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/longevity_test/longevity.go | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index ef17b7e32..8a5c01537 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -9,11 +9,13 @@ import ( "github.com/alcionai/clues" "github.com/spf13/cobra" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/cli/config" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/store" ) @@ -62,6 +64,65 @@ func deleteBackups( return deleted, nil } +// pitrListBackups connects to the repository at the given point in time and +// lists the backups for service. It then checks the list of backups contains +// the backups in backupIDs. +// +//nolint:unused +//lint:ignore U1000 Waiting for full support. +func pitrListBackups( + ctx context.Context, + service path.ServiceType, + pitr time.Time, + backupIDs []string, +) error { + if len(backupIDs) == 0 { + return nil + } + + ctx = clues.Add(ctx, "pitr_time", pitr, "search_backups", backupIDs) + + // TODO(ashmrtn): This may be moved into CLI layer at some point when we add + // flags for opening a repo at a point in time. + cfg, err := config.GetConfigRepoDetails(ctx, true, true, nil) + if err != nil { + return clues.Wrap(err, "getting config info") + } + + opts := utils.ControlWithConfig(cfg) + opts.Repo.ViewTimestamp = &pitr + + r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, cfg.RepoID, opts) + if err != nil { + return clues.Wrap(err, "connecting to repo").WithClues(ctx) + } + + defer r.Close(ctx) + + backups, err := r.BackupsByTag(ctx, store.Service(service)) + if err != nil { + return clues.Wrap(err, "listing backups").WithClues(ctx) + } + + bups := map[string]struct{}{} + + for _, backup := range backups { + bups[backup.ID.String()] = struct{}{} + } + + ctx = clues.Add(ctx, "found_backups", maps.Keys(bups)) + + for _, backupID := range backupIDs { + if _, ok := bups[backupID]; !ok { + return clues.New("looking for backup"). + With("search_backup_id", backupID). + WithClues(ctx) + } + } + + return nil +} + func main() { var ( service path.ServiceType From dfbca1540d875bb4e8fc96c3bb0ee38ac9276b57 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:52:41 -0700 Subject: [PATCH 58/62] Minor refactor for handling of LoggableDir (#3934) Reuse existing Elements code to conceal information. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3895 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/pkg/path/elements.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/pkg/path/elements.go b/src/pkg/path/elements.go index 5a0802d50..838cea114 100644 --- a/src/pkg/path/elements.go +++ b/src/pkg/path/elements.go @@ -2,7 +2,6 @@ package path import ( "fmt" - "strings" "github.com/alcionai/clues" @@ -119,16 +118,8 @@ func (el Elements) Last() string { // LoggableDir takes in a path reference (of any structure) and conceals any // non-standard elements (ids, filenames, foldernames, etc). func LoggableDir(ref string) string { - r := ref - n := strings.TrimSuffix(r, string(PathSeparator)) - - for n != r { - r = n - n = strings.TrimSuffix(r, string(PathSeparator)) - } - - elems := Split(r) - elems = pii.ConcealElements(elems, piiSafePathElems) - - return join(elems) + // Can't directly use Builder since that could return an error. Instead split + // into elements and use that. + split := Split(TrimTrailingSlash(ref)) + return Elements(split).Conceal() } From 5b455bfc4a3c803a984eb75de4c2eb74c7a35cd5 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:00:20 -0700 Subject: [PATCH 59/62] Better handling of clues, error, and log data if it's PII (#3935) Add better handling for * hiding possibly sensitive data when logging * adding possibly sensitive data to context clues * adding context clues to returned errors --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3895 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/upload.go | 44 +++++++++++++++++++++-------------- src/internal/kopia/wrapper.go | 16 ++++++------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index ea5a55dbd..3e954c244 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -252,7 +252,9 @@ func (cp *corsoProgress) FinishedHashingFile(fname string, bs int64) { sl[i] = string(rdt) } - logger.Ctx(context.Background()).Debugw("finished hashing file", "path", sl[2:]) + logger.Ctx(cp.ctx).Debugw( + "finished hashing file", + "path", clues.Hide(path.Elements(sl[2:]))) atomic.AddInt64(&cp.totalBytes, bs) } @@ -442,12 +444,12 @@ func streamBaseEntries( ctx = clues.Add( ctx, - "current_item_path", curPath, - "longest_prefix", longest) + "current_directory_path", curPath, + "longest_prefix", path.LoggableDir(longest)) err := dir.IterateEntries(ctx, func(innerCtx context.Context, entry fs.Entry) error { if err := innerCtx.Err(); err != nil { - return err + return clues.Stack(err).WithClues(ctx) } // Don't walk subdirectories in this function. @@ -464,7 +466,9 @@ func streamBaseEntries( entName, err := decodeElement(entry.Name()) if err != nil { - return clues.Wrap(err, "decoding entry name: "+entry.Name()) + return clues.Wrap(err, "decoding entry name"). + WithClues(ctx). + With("entry_name", entry.Name()) } // This entry was marked as deleted by a service that can't tell us the @@ -476,7 +480,7 @@ func streamBaseEntries( // For now assuming that item IDs don't need escaping. itemPath, err := curPath.AppendItem(entName) if err != nil { - return clues.Wrap(err, "getting full item path for base entry") + return clues.Wrap(err, "getting full item path for base entry").WithClues(ctx) } // We need the previous path so we can find this item in the base snapshot's @@ -485,7 +489,7 @@ func streamBaseEntries( // to look for. prevItemPath, err := prevPath.AppendItem(entName) if err != nil { - return clues.Wrap(err, "getting previous full item path for base entry") + return clues.Wrap(err, "getting previous full item path for base entry").WithClues(ctx) } // Meta files aren't in backup details since it's the set of items the user @@ -509,13 +513,15 @@ func streamBaseEntries( } if err := ctr(ctx, entry); err != nil { - return clues.Wrap(err, "executing callback on item").With("item_path", itemPath) + return clues.Wrap(err, "executing callback on item"). + WithClues(ctx). + With("item_path", itemPath) } return nil }) if err != nil { - return clues.Wrap(err, "traversing items in base snapshot directory") + return clues.Wrap(err, "traversing items in base snapshot directory").WithClues(ctx) } return nil @@ -826,7 +832,9 @@ func inflateCollectionTree( } if node.collection != nil && node.collection.State() == data.NotMovedState { - return nil, nil, clues.New("conflicting states for collection").With("changed_path", p) + return nil, nil, clues.New("conflicting states for collection"). + WithClues(ctx). + With("changed_path", p) } } @@ -860,7 +868,7 @@ func traverseBaseDir( "expected_dir_path", expectedDirPath) if depth >= maxInflateTraversalDepth { - return clues.New("base snapshot tree too tall") + return clues.New("base snapshot tree too tall").WithClues(ctx) } // Wrapper base64 encodes all file and folder names to avoid issues with @@ -868,7 +876,9 @@ func traverseBaseDir( // from kopia we need to do the decoding here. dirName, err := decodeElement(dir.Name()) if err != nil { - return clues.Wrap(err, "decoding base directory name").With("dir_name", dir.Name()) + return clues.Wrap(err, "decoding base directory name"). + WithClues(ctx). + With("dir_name", clues.Hide(dir.Name())) } // Form the path this directory would be at if the hierarchy remained the same @@ -944,7 +954,7 @@ func traverseBaseDir( stats) }) if err != nil { - return clues.Wrap(err, "traversing base directory") + return clues.Wrap(err, "traversing base directory").WithClues(ctx) } // We only need to add this base directory to the tree we're building if it @@ -961,7 +971,7 @@ func traverseBaseDir( // in the if-block though as that is an optimization. node := getTreeNode(roots, currentPath.Elements()) if node == nil { - return clues.New("getting tree node") + return clues.New("getting tree node").WithClues(ctx) } // Now that we have the node we need to check if there is a collection @@ -976,12 +986,12 @@ func traverseBaseDir( curP, err := path.FromDataLayerPath(currentPath.String(), false) if err != nil { - return clues.New("converting current path to path.Path") + return clues.New("converting current path to path.Path").WithClues(ctx) } oldP, err := path.FromDataLayerPath(oldDirPath.String(), false) if err != nil { - return clues.New("converting old path to path.Path") + return clues.New("converting old path to path.Path").WithClues(ctx) } node.baseDir = dir @@ -1172,7 +1182,7 @@ func inflateDirTree( } if len(roots) > 1 { - return nil, clues.New("multiple root directories") + return nil, clues.New("multiple root directories").WithClues(ctx) } var res fs.Directory diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 3963b30f6..7b1feca44 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -324,7 +324,7 @@ func (w Wrapper) makeSnapshotWithRoot( // Telling kopia to always flush may hide other errors if it fails while // flushing the write session (hence logging above). if err != nil { - return nil, clues.Wrap(err, "kopia backup") + return nil, clues.Wrap(err, "kopia backup").WithClues(ctx) } res := manifestToStats(man, progress, bc) @@ -369,7 +369,7 @@ func getDir( encodeElements(dirPath.PopFront().Elements()...)) if err != nil { if isErrEntryNotFound(err) { - err = clues.Stack(data.ErrNotFound, err) + err = clues.Stack(data.ErrNotFound, err).WithClues(ctx) } return nil, clues.Wrap(err, "getting nested object handle").WithClues(ctx) @@ -487,7 +487,7 @@ func (w Wrapper) ProduceRestoreCollections( // load it here. snapshotRoot, err := w.getSnapshotRoot(ctx, snapshotID) if err != nil { - return nil, clues.Wrap(err, "loading snapshot root") + return nil, clues.Wrap(err, "loading snapshot root").WithClues(ctx) } var ( @@ -507,8 +507,8 @@ func (w Wrapper) ProduceRestoreCollections( // items from a single directory instance lower down. ictx := clues.Add( ctx, - "item_path", itemPaths.StoragePath.String(), - "restore_path", itemPaths.RestorePath.String()) + "item_path", itemPaths.StoragePath, + "restore_path", itemPaths.RestorePath) parentStoragePath, err := itemPaths.StoragePath.Dir() if err != nil { @@ -552,7 +552,7 @@ func (w Wrapper) ProduceRestoreCollections( // then load the items from the directory. res, err := loadDirsAndItems(ctx, snapshotRoot, bcounter, dirsToItems, errs) if err != nil { - return nil, clues.Wrap(err, "loading items") + return nil, clues.Wrap(err, "loading items").WithClues(ctx) } return res, el.Failure() @@ -610,12 +610,12 @@ func (w Wrapper) RepoMaintenance( ) error { kopiaSafety, err := translateSafety(opts.Safety) if err != nil { - return clues.Wrap(err, "identifying safety level") + return clues.Wrap(err, "identifying safety level").WithClues(ctx) } mode, err := translateMode(opts.Type) if err != nil { - return clues.Wrap(err, "identifying maintenance mode") + return clues.Wrap(err, "identifying maintenance mode").WithClues(ctx) } currentOwner := w.c.ClientOptions().UsernameAtHost() From 2596fb91041450494be82f78736bc2ef1f62e497 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:36:36 -0700 Subject: [PATCH 60/62] Track when we skip merging due to DoNotMerge (#3938) Add some more nuanced tracking that takes into account the DoNotMerge flag and the New state of collections. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3929 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/upload.go | 40 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 3e954c244..9df1327f9 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -897,6 +897,8 @@ func traverseBaseDir( currentPath = currentPath.Append(dirName) } + var explicitMention bool + if upb, ok := updatedPaths[oldDirPath.String()]; ok { // This directory was deleted. if upb == nil { @@ -908,21 +910,16 @@ func traverseBaseDir( // unchanged) location is in upb. currentPath = upb.ToBuilder() - if oldDirPath.String() == currentPath.String() { - stats.Inc(statNoMove) - } else { - stats.Inc(statMove) - } + // Below we check if the collection was marked as new or DoNotMerge which + // disables merging behavior. That means we can't directly update stats + // here else we'll miss delta token refreshes and whatnot. Instead note + // that we did see the path explicitly so it's not counted as a recursive + // operation. + explicitMention = true } - } else { + } else if currentPath == nil { // Just stats tracking stuff. - if currentPath == nil { - stats.Inc(statRecursiveDel) - } else if oldDirPath.String() == currentPath.String() { - stats.Inc(statNoMove) - } else { - stats.Inc(statRecursiveMove) - } + stats.Inc(statRecursiveDel) } ctx = clues.Add(ctx, "new_path", currentPath) @@ -981,9 +978,20 @@ func traverseBaseDir( // directories. The expected usecase for this is delta token expiry in M365. if node.collection != nil && (node.collection.DoNotMergeItems() || node.collection.State() == data.NewState) { + stats.Inc(statSkipMerge) + return nil } + // Just stats tracking stuff. + if oldDirPath.String() == currentPath.String() { + stats.Inc(statNoMove) + } else if explicitMention { + stats.Inc(statMove) + } else { + stats.Inc(statRecursiveMove) + } + curP, err := path.FromDataLayerPath(currentPath.String(), false) if err != nil { return clues.New("converting current path to path.Path").WithClues(ctx) @@ -1037,6 +1045,9 @@ const ( // statRecursiveDel denotes a directory that was deleted because one or more // of its ancestors was deleted and it wasn't explicitly mentioned. statRecursiveDel = "directories_recursively_deleted" + // statSkipMerge denotes the number of directories that weren't merged because + // they were marked either DoNotMerge or New. + statSkipMerge = "directories_skipped_merging" ) func inflateBaseTree( @@ -1130,7 +1141,8 @@ func inflateBaseTree( statMove, stats.Get(statMove), statRecursiveMove, stats.Get(statRecursiveMove), statDel, stats.Get(statDel), - statRecursiveDel, stats.Get(statRecursiveDel)) + statRecursiveDel, stats.Get(statRecursiveDel), + statSkipMerge, stats.Get(statSkipMerge)) } return nil From 57511635729f798288e272685f3e5ef7b8f32cd8 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Tue, 1 Aug 2023 11:33:51 +0530 Subject: [PATCH 61/62] Sanity tests for SharePoint export (#3928) This extends the existing export sanity tests to also cover SharePoint exports. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * closes https://github.com/alcionai/corso/issues/3889 #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../actions/backup-restore-test/action.yml | 4 +- src/cmd/sanity_test/export/sharepoint.go | 88 +++++++++++++++++++ src/cmd/sanity_test/sanity_tests.go | 2 + 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 src/cmd/sanity_test/export/sharepoint.go diff --git a/.github/actions/backup-restore-test/action.yml b/.github/actions/backup-restore-test/action.yml index 56213d3e7..95e41d1bb 100644 --- a/.github/actions/backup-restore-test/action.yml +++ b/.github/actions/backup-restore-test/action.yml @@ -89,7 +89,7 @@ runs: id: export shell: bash working-directory: src - if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive + if: ${{ inputs.service == 'onedrive' || inputs.service == 'sharepoint' }} run: | set -euo pipefail CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log @@ -105,7 +105,7 @@ runs: - name: Check export ${{ inputs.service }} ${{ inputs.kind }} shell: bash working-directory: src - if: ${{ inputs.service == 'onedrive' }} + if: ${{ inputs.service == 'onedrive' || inputs.service == 'sharepoint' }} env: SANITY_TEST_KIND: export SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }} diff --git a/src/cmd/sanity_test/export/sharepoint.go b/src/cmd/sanity_test/export/sharepoint.go new file mode 100644 index 000000000..55ab8ed5c --- /dev/null +++ b/src/cmd/sanity_test/export/sharepoint.go @@ -0,0 +1,88 @@ +package export + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/internal/common/ptr" +) + +func CheckSharePointExport( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + siteID, folderName, dataFolder string, +) { + drive, err := client. + Sites(). + BySiteId(siteID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + // map itemID -> item size + var ( + fileSizes = make(map[string]int64) + exportFileSizes = make(map[string]int64) + startTime = time.Now() + ) + + err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error { + if err != nil { + return clues.Stack(err) + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(folderName, path) + if err != nil { + return clues.Stack(err) + } + + exportFileSizes[relPath] = info.Size() + if startTime.After(info.ModTime()) { + startTime = info.ModTime() + } + + return nil + }) + if err != nil { + fmt.Println("Error walking the path:", err) + } + + _ = restore.PopulateDriveDetails( + ctx, + client, + ptr.Val(drive.GetId()), + folderName, + dataFolder, + fileSizes, + map[string][]common.PermissionInfo{}, + startTime) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := exportFileSizes[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index 44680ab47..84bce47a0 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -77,6 +77,8 @@ func main() { switch testService { case "onedrive": export.CheckOneDriveExport(ctx, client, testUser, folder, dataFolder) + case "sharepoint": + export.CheckSharePointExport(ctx, client, testSite, folder, dataFolder) default: common.Fatal(ctx, "unknown service for export sanity tests", nil) } From 0f41256010eae3c784fda13e2d6299bbbd892225 Mon Sep 17 00:00:00 2001 From: neha-Gupta1 Date: Tue, 1 Aug 2023 13:47:42 +0530 Subject: [PATCH 62/62] update resource path --- src/pkg/path/resource_path.go | 4 +- src/pkg/path/servicetype_string.go | 6 +- .../services/m365/api/{teams.go => groups.go} | 62 +++++++--- .../api/{teams_test.go => groups_test.go} | 113 ++++++++++++++---- src/pkg/services/m365/api/helper_test.go | 2 +- 5 files changed, 145 insertions(+), 42 deletions(-) rename src/pkg/services/m365/api/{teams.go => groups.go} (72%) rename src/pkg/services/m365/api/{teams_test.go => groups_test.go} (53%) diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index 88f025371..c2702faed 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -30,8 +30,8 @@ const ( ExchangeMetadataService // exchangeMetadata OneDriveMetadataService // onedriveMetadata SharePointMetadataService // sharepointMetadata - TeamsService // teams - TeamsMetadataService // teamsMetadata + GroupsService // groups + GroupsMetadataService // groupsMetadata ) func toServiceType(service string) ServiceType { diff --git a/src/pkg/path/servicetype_string.go b/src/pkg/path/servicetype_string.go index 6d6b960d8..6fa499364 100644 --- a/src/pkg/path/servicetype_string.go +++ b/src/pkg/path/servicetype_string.go @@ -15,11 +15,13 @@ func _() { _ = x[ExchangeMetadataService-4] _ = x[OneDriveMetadataService-5] _ = x[SharePointMetadataService-6] + _ = x[GroupsService-7] + _ = x[GroupsMetadataService-8] } -const _ServiceType_name = "UnknownServiceexchangeonedrivesharepointexchangeMetadataonedriveMetadatasharepointMetadata" +const _ServiceType_name = "UnknownServiceexchangeonedrivesharepointexchangeMetadataonedriveMetadatasharepointMetadatagroupsgroupsMetadata" -var _ServiceType_index = [...]uint8{0, 14, 22, 30, 40, 56, 72, 90} +var _ServiceType_index = [...]uint8{0, 14, 22, 30, 40, 56, 72, 90, 96, 110} func (i ServiceType) String() string { if i < 0 || i >= ServiceType(len(_ServiceType_index)-1) { diff --git a/src/pkg/services/m365/api/teams.go b/src/pkg/services/m365/api/groups.go similarity index 72% rename from src/pkg/services/m365/api/teams.go rename to src/pkg/services/m365/api/groups.go index 8eef3fba3..589da1a45 100644 --- a/src/pkg/services/m365/api/teams.go +++ b/src/pkg/services/m365/api/groups.go @@ -23,21 +23,21 @@ const ( // controller // --------------------------------------------------------------------------- -func (c Client) Teams() Teams { - return Teams{c} +func (c Client) Groups() Groups { + return Groups{c} } // On creation of each Teams team a corrsponding group gets created. // The group acts as the protected resource, and all teams data like events, // drive and mail messages are owned by that group. -// Teams is an interface-compliant provider of the client. -type Teams struct { +// Groups is an interface-compliant provider of the client. +type Groups struct { Client } -// GetAllTeams retrieves all groups. -func (c Teams) GetAll( +// GetAllGroups retrieves all groups. +func (c Groups) GetAll( ctx context.Context, errs *fault.Bus, ) ([]models.Groupable, error) { @@ -46,13 +46,26 @@ func (c Teams) GetAll( return nil, err } - return getGroups(ctx, true, errs, service) + return getGroups(ctx, func(ctx context.Context, g models.Groupable) bool { return true }, errs, service) +} + +// GetTeams retrieves all Teams. +func (c Groups) GetTeams( + ctx context.Context, + errs *fault.Bus, +) ([]models.Groupable, error) { + service, err := c.Service() + if err != nil { + return nil, err + } + + return getGroups(ctx, FetchOnlyTeams, errs, service) } // GetAll retrieves all groups. func getGroups( ctx context.Context, - getOnlyTeams bool, + filterGroupsData func(ctx context.Context, g models.Groupable) bool, errs *fault.Bus, service graph.Servicer, ) ([]models.Groupable, error) { @@ -64,7 +77,7 @@ func getGroups( iter, err := msgraphgocore.NewPageIterator[models.Groupable]( resp, service.Adapter(), - models.CreateTeamCollectionResponseFromDiscriminatorValue) + models.CreateGroupCollectionResponseFromDiscriminatorValue) if err != nil { return nil, graph.Wrap(ctx, err, "creating groups iterator") } @@ -83,8 +96,7 @@ func getGroups( if err != nil { el.AddRecoverable(ctx, graph.Wrap(ctx, err, "validating groups")) } else { - isTeam := IsTeam(ctx, item) - if !getOnlyTeams || isTeam { + if filterGroupsData(ctx, item) { groups = append(groups, item) } } @@ -99,7 +111,7 @@ func getGroups( return groups, el.Failure() } -func IsTeam(ctx context.Context, g models.Groupable) bool { +func FetchOnlyTeams(ctx context.Context, g models.Groupable) bool { log := logger.Ctx(ctx) if g.GetAdditionalData()[ResourceProvisioningOptions] != nil { @@ -120,8 +132,8 @@ func IsTeam(ctx context.Context, g models.Groupable) bool { return false } -// GetID retrieves team by groupID/teamID. -func (c Teams) GetByID( +// GetID retrieves group by groupID. +func (c Groups) GetByID( ctx context.Context, identifier string, ) (models.Groupable, error) { @@ -137,7 +149,27 @@ func (c Teams) GetByID( return nil, err } - if !IsTeam(ctx, resp) { + return resp, graph.Stack(ctx, err).OrNil() +} + +// GetTeamByID retrieves group by groupID. +func (c Groups) GetTeamByID( + ctx context.Context, + identifier string, +) (models.Groupable, error) { + service, err := c.Service() + if err != nil { + return nil, err + } + + resp, err := service.Client().Groups().ByGroupId(identifier).Get(ctx, nil) + if err != nil { + err := graph.Wrap(ctx, err, "getting group by id") + + return nil, err + } + + if !FetchOnlyTeams(ctx, resp) { err := clues.New("given teamID is not related to any team") return nil, err diff --git a/src/pkg/services/m365/api/teams_test.go b/src/pkg/services/m365/api/groups_test.go similarity index 53% rename from src/pkg/services/m365/api/teams_test.go rename to src/pkg/services/m365/api/groups_test.go index dcb039dc5..5d43ea08d 100644 --- a/src/pkg/services/m365/api/teams_test.go +++ b/src/pkg/services/m365/api/groups_test.go @@ -17,18 +17,18 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api" ) -type TeamsUnitSuite struct { +type GroupUnitSuite struct { tester.Suite } -func TestTeamsUnitSuite(t *testing.T) { - suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) +func TestGroupsUnitSuite(t *testing.T) { + suite.Run(t, &GroupUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *TeamsUnitSuite) TestValidateGroup() { - team := models.NewTeam() - team.SetDisplayName(ptr.To("testgroup")) - team.SetId(ptr.To("testID")) +func (suite *GroupUnitSuite) TestValidateGroup() { + group := models.NewGroup() + group.SetDisplayName(ptr.To("testgroup")) + group.SetId(ptr.To("testID")) tests := []struct { name string @@ -41,7 +41,7 @@ func (suite *TeamsUnitSuite) TestValidateGroup() { args: func() *models.Group { s := models.NewGroup() s.SetId(ptr.To("id")) - s.SetDisplayName(ptr.To("testTeam")) + s.SetDisplayName(ptr.To("testgroup")) return s }(), errCheck: assert.NoError, @@ -59,7 +59,7 @@ func (suite *TeamsUnitSuite) TestValidateGroup() { name: "No ID", args: func() *models.Group { s := models.NewGroup() - s.SetDisplayName(ptr.To("testTeam")) + s.SetDisplayName(ptr.To("testgroup")) return s }(), errCheck: assert.Error, @@ -80,47 +80,60 @@ func (suite *TeamsUnitSuite) TestValidateGroup() { } } -type TeamsIntgSuite struct { +type GroupsIntgSuite struct { tester.Suite its intgTesterSetup } -func TestTeamsIntgSuite(t *testing.T) { - suite.Run(t, &TeamsIntgSuite{ +func TestGroupsIntgSuite(t *testing.T) { + suite.Run(t, &GroupsIntgSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tconfig.M365AcctCredEnvs}), }) } -func (suite *TeamsIntgSuite) SetupSuite() { +func (suite *GroupsIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } -func (suite *TeamsIntgSuite) TestGetAllTeams() { +func (suite *GroupsIntgSuite) TestGetAllGroups() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() - teams, err := suite.its.ac. - Teams(). + groups, err := suite.its.ac. + Groups(). GetAll(ctx, fault.New(true)) require.NoError(t, err) - require.NotZero(t, len(teams), "must have at least one team") + require.NotZero(t, len(groups), "must have at least one group") +} - for _, team := range teams { - assert.True(t, api.IsTeam(ctx, team), "must not return non teams groups") +func (suite *GroupsIntgSuite) TestGetAllTeams() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + groups, err := suite.its.ac. + Groups(). + GetTeams(ctx, fault.New(true)) + require.NoError(t, err) + require.NotZero(t, len(groups), "must have at least one group") + + for _, team := range groups { + assert.True(t, api.FetchOnlyTeams(ctx, team), "must not return non groups groups") } } -func (suite *TeamsIntgSuite) TestTeams_GetByID() { +func (suite *GroupsIntgSuite) TestTeams_GetByID() { var ( t = suite.T() teamID = tconfig.M365TeamsID(t) ) - teamsAPI := suite.its.ac.Teams() + teamsAPI := suite.its.ac.Groups() table := []struct { name string @@ -164,7 +177,63 @@ func (suite *TeamsIntgSuite) TestTeams_GetByID() { ctx, flush := tester.NewContext(t) defer flush() - _, err := teamsAPI.GetByID(ctx, test.id) + _, err := teamsAPI.GetTeamByID(ctx, test.id) + test.expectErr(t, err) + }) + } +} + +func (suite *GroupsIntgSuite) TestGroups_GetByID() { + var ( + t = suite.T() + groupID = tconfig.M365GroupID(t) + ) + + groupsAPI := suite.its.ac.Groups() + + table := []struct { + name string + id string + expectErr func(*testing.T, error) + }{ + { + name: "3 part id", + id: groupID, + expectErr: func(t *testing.T, err error) { + assert.NoError(t, err, clues.ToCore(err)) + }, + }, + { + name: "malformed id", + id: uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + { + name: "random id", + id: uuid.NewString() + "," + uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + + { + name: "malformed url", + id: "barunihlda", + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + _, err := groupsAPI.GetByID(ctx, test.id) test.expectErr(t, err) }) } diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index fb5138350..cf61c0277 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -135,7 +135,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { // teams its.teamID = tconfig.M365TeamsID(t) - team, err := its.ac.Teams().GetByID(ctx, its.teamID) + team, err := its.ac.Groups().GetTeamByID(ctx, its.teamID) require.NoError(t, err, clues.ToCore(err)) its.teamID = ptr.Val(team.GetId())