Update README.md (#5466)

<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic
words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E

.github/ISSUE_TEMPLATE/BUG-REPORT.yaml

@@ -35,6 +35,6 @@ body:
     id: logs
     attributes:
       label: Relevant log output
-      description: Please run Corso with `--log-level debug` and attach the log file.
+      description: Please run Corso with `--log-level debug --mask-sensitive-data` and attach the log file.
       placeholder: This will be automatically formatted, so no need for backticks.
       render: shell

.github/actions/backup-restore-test/action.yml

@@ -1,4 +1,5 @@
 name: Backup Restore Test
+description: Run various backup/restore/export tests for a service.
 
 inputs:
   service:
@@ -7,23 +8,38 @@ inputs:
   kind:
     description: Kind of test
     required: true
+  backup-id:
+    description: Backup to retrieve data out of
+    required: false
   backup-args:
     description: Arguments to pass for backup
     required: false
     default: ""
   restore-args:
-    description: Arguments to pass for restore
+    description: Arguments to pass for restore; restore is skipped when missing.
     required: false
     default: ""
-  test-folder:
+  export-args:
+    description: Arguments to pass for export.
+    required: false
+    default: ""
+  restore-container:
     description: Folder to use for testing
     required: true
-  base-backup:
-    description: Base backup to use for testing
-    required: false
   log-dir:
     description: Folder to store test log files
     required: true
+  on-collision:
+    description: Value for the --collisions flag
+    required: false
+    default: "replace"
+  with-export:
+    description: Runs export tests when true
+    required: false
+    default: false
+  category:
+    description: category of data for given service
+    required: false
 
 outputs:
   backup-id:
@@ -37,8 +53,13 @@ runs:
       shell: bash
       working-directory: src
       run: |
+        echo "---------------------------"
+        echo Backup ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
         set -euo pipefail
-        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}-${{inputs.kind }}.log
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-backup-${{inputs.kind }}.log
         ./corso backup create '${{ inputs.service }}' \
             --no-stats --hide-progress --json \
             ${{ inputs.backup-args }} |
@@ -48,15 +69,22 @@ runs:
             tee $GITHUB_OUTPUT
 
     - name: Restore ${{ inputs.service }} ${{ inputs.kind }}
+      if: inputs.restore-args
       id: restore
       shell: bash
       working-directory: src
       run: |
+        echo "---------------------------"
+        echo Restore ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
         set -euo pipefail
-        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
         ./corso restore '${{ inputs.service }}' \
             --no-stats \
             --hide-progress \
+            --collisions ${{ inputs.on-collision }} \
             ${{ inputs.restore-args }} \
             --backup '${{ steps.backup.outputs.result }}' \
             2>&1 |
@@ -67,24 +95,125 @@ runs:
 
             cat /tmp/corsologs
 
-    - name: Check ${{ inputs.service }} ${{ inputs.kind }}
+    - name: Check restore ${{ inputs.service }} ${{ inputs.kind }}
+      if: inputs.restore-args
       shell: bash
       working-directory: src
       env:
-        SANITY_RESTORE_FOLDER: ${{ steps.restore.outputs.result }}
-        SANITY_RESTORE_SERVICE: ${{ inputs.service }}
-        TEST_DATA: ${{ inputs.test-folder }}
-        BASE_BACKUP: ${{ inputs.base-backup }}
+        SANITY_TEST_RESTORE_CONTAINER: ${{ steps.restore.outputs.result }}
+        SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
+        SANITY_BACKUP_ID: ${{ inputs.backup-id }}
+        # lists are not restored to a different folder.  they get created adjacent to their originals
+        # hence SANITY_TEST_RESTORE_CONTAINER_PREFIX is necessary to differentiate restored from original
+        SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
+        SANITY_TEST_CATEGORY: ${{ inputs.category }}
       run: |
-        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log
-        ./sanity-test
+        echo "---------------------------"
+        echo Sanity Test Restore ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
+        ./sanity-test restore ${{ inputs.service }}
+
+    - name: Export ${{ inputs.service }} ${{ inputs.kind }}
+      if: ${{ inputs.with-export == 'true' }}
+      id: export
+      shell: bash
+      working-directory: src
+      run: |
+        echo "---------------------------"
+        echo Export ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
+        set -euo pipefail
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
+        ./corso export '${{ inputs.service }}' \
+            /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }} \
+            --no-stats \
+            --hide-progress \
+            ${{ inputs.export-args }} \
+            --backup '${{ steps.backup.outputs.result }}'
+
+          cat /tmp/corsologs
+
+    - name: Check export ${{ inputs.service }} ${{ inputs.kind }}
+      if: ${{ inputs.with-export == 'true' }}
+      shell: bash
+      working-directory: src
+      env:
+        SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{ inputs.kind }}
+        SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
+        SANITY_BACKUP_ID: ${{ inputs.backup-id }}
+        # applies only for sharepoint lists
+        SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
+        SANITY_TEST_CATEGORY: ${{ inputs.category }}
+      run: |
+        echo "---------------------------"
+        echo Sanity-Test Export ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
+        ./sanity-test export ${{ inputs.service }}
+
+    - name: Export archive ${{ inputs.service }} ${{ inputs.kind }}
+      if: ${{ inputs.with-export == 'true' }}
+      id: export-archive
+      shell: bash
+      working-directory: src
+      run: |
+        echo "---------------------------"
+        echo Export Archive ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
+        set -euo pipefail
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
+        ./corso export '${{ inputs.service }}' \
+            /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive \
+            --no-stats \
+            --hide-progress \
+            --archive \
+            ${{ inputs.export-args }} \
+            --backup '${{ steps.backup.outputs.result }}'
+
+          unzip /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive/*.zip \
+            -d /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-unzipped
+          cat /tmp/corsologs
+
+    - name: Check archive export ${{ inputs.service }} ${{ inputs.kind }}
+      if: ${{ inputs.with-export == 'true' }}
+      shell: bash
+      working-directory: src
+      env:
+        SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{inputs.kind }}-unzipped
+        SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
+        SANITY_BACKUP_ID: ${{ inputs.backup-id }}
+        # applies only for sharepoint lists
+        SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
+        SANITY_TEST_CATEGORY: ${{ inputs.category }}
+      run: |
+        echo "---------------------------"
+        echo Sanity-Test Export Archive ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
+        ./sanity-test export ${{ inputs.service }}
 
     - name: List ${{ inputs.service }} ${{ inputs.kind }}
       shell: bash
       working-directory: src
       run: |
+        echo "---------------------------"
+        echo Backup list ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
         set -euo pipefail
-        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}-${{inputs.kind }}.log
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}${CATEGORY_SUFFIX}-list-${{inputs.kind }}.log
         ./corso backup list ${{ inputs.service }} \
           --no-stats \
           --hide-progress \
@@ -101,8 +230,14 @@ runs:
       shell: bash
       working-directory: src
       run: |
+        echo "---------------------------"
+        echo Backup List w/ Backup ${{ inputs.service }} ${{ inputs.kind }}
+        echo "---------------------------"
         set -euo pipefail
-        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-single-${{ inputs.service }}-${{inputs.kind }}.log
+        # Include category in the log file name if present
+        CATEGORY_SUFFIX=""
+        [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
+        CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}${CATEGORY_SUFFIX}-single-${{inputs.kind }}.log
         ./corso backup list ${{ inputs.service }} \
           --no-stats \
           --hide-progress \
@@ -116,7 +251,13 @@ runs:
           exit 1
         fi
 
-    # Upload the original go test output as an artifact for later review.
+    - if: always()
+      shell: bash
+      run: |
+        echo "---------------------------"
+        echo Logging Results
+        echo "---------------------------"
+
     - name: Upload test log
       if: always()
       uses: actions/upload-artifact@v3
@@ -124,4 +265,4 @@ runs:
         name: "${{ inputs.service }}-${{ inputs.kind }}-logs"
         path: ${{ inputs.log-dir }}/*
         if-no-files-found: error
-        retention-days: 14
+        retention-days: 14

.github/actions/go-setup-cache/action.yml

@@ -1,4 +1,5 @@
 name: Setup and Cache Golang
+description: Build golang binaries for later use in CI.
 
 # clone of: https://github.com/magnetikonline/action-golang-cache/blob/main/action.yaml
 #

.github/actions/publish-binary/action.yml

@@ -1,4 +1,5 @@
 name: Publish Binary
+description: Publish binary artifacts.
 
 inputs:
   version:
@@ -36,7 +37,7 @@ runs:
       uses: goreleaser/goreleaser-action@v4
       with:
         version: latest
-        args: release --rm-dist --timeout 500m --parallelism 1 ${{ env.grflags }}
+        args: release --clean --timeout 500m --parallelism 1 ${{ env.grflags }}
         workdir: src
       env:
         GITHUB_TOKEN: ${{ inputs.github_token }}

.github/actions/publish-website/action.yml

@@ -1,4 +1,5 @@
 name: Publish Website
+description: Publish website artifacts.
 
 inputs:
   aws-iam-role:

.github/actions/purge-m365-data/action.yml

@@ -1,4 +1,5 @@
 name: Purge M365 User Data
+description: Deletes M365 data generated during CI tests.
 
 # Hard deletion of an m365 user's data.  Our CI processes create a lot
 # of data churn (creation and immediate deletion) of files, the likes
@@ -19,7 +20,9 @@ inputs:
   site:
     description: Sharepoint site where data is to be purged.
   libraries:
-    description: List of library names within site where data is to be purged.
+    description: List of library names within the site where data is to be purged.
+  library-prefix:
+    description: List of library names within the site where the library will get deleted entirely.
   folder-prefix:
     description: Name of the folder to be purged.  If falsy, will purge the set of static, well known folders instead.
   older-than:
@@ -28,12 +31,19 @@ inputs:
     description: Secret value of for AZURE_CLIENT_ID
   azure-client-secret:
     description: Secret value of for AZURE_CLIENT_SECRET
+  azure-pnp-client-id:
+    description: Secret value of AZURE_PNP_CLIENT_ID
+  azure-pnp-client-cert:
+    description: Base64 encoded private certificate for the azure-pnp-client-id (Secret value of AZURE_PNP_CLIENT_CERT)
   azure-tenant-id:
-    description: Secret value of for AZURE_TENANT_ID
+    description: Secret value of AZURE_TENANT_ID
   m365-admin-user:
     description: Secret value of for M365_TENANT_ADMIN_USER
   m365-admin-password:
     description: Secret value of for M365_TENANT_ADMIN_PASSWORD
+  tenant-domain:
+    description: The domain of the tenant (ex. 10rqc2.onmicrosft.com)
+    required: true
 
 runs:
   using: composite
@@ -52,17 +62,22 @@ runs:
         AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
         AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
       run: |
-        ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
+        for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
+        {
+          if (./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
+            break
+          }
+        }
 
-    - name: Reset retention for all mailboxes to 0
-      if: ${{ inputs.user == '' }}
-      shell: pwsh
-      working-directory: ./src/cmd/purge/scripts
-      env:
-        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
-        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
-      run: |
-        ./exchangeRetention.ps1
+    # TODO(ashmrtn): Re-enable when we figure out errors we're seeing with Get-Mailbox call.
+    #- name: Reset retention for all mailboxes to 0
+    #  if: ${{ inputs.user == '' }}
+    #  shell: pwsh
+    #  working-directory: ./src/cmd/purge/scripts
+    #  env:
+    #    M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
+    #    M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
+    #  run: ./exchangeRetention.ps1
 
     ################################################################################################################
     # OneDrive
@@ -73,10 +88,16 @@ runs:
       shell: pwsh
       working-directory: ./src/cmd/purge/scripts
       env:
-        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
-        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
+        AZURE_CLIENT_ID: ${{ inputs.azure-pnp-client-id }}
+        AZURE_APP_CERT: ${{ inputs.azure-pnp-client-cert }}
+        TENANT_DOMAIN:  ${{ inputs.tenant-domain }}
       run: |
-        ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
+        for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
+        {
+          if (./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
+            break
+          }
+        }
 
     ################################################################################################################
     # Sharepoint
@@ -87,7 +108,14 @@ runs:
       shell: pwsh
       working-directory: ./src/cmd/purge/scripts
       env:
-        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
-        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
+        AZURE_CLIENT_ID: ${{ inputs.azure-pnp-client-id }}
+        AZURE_APP_CERT: ${{ inputs.azure-pnp-client-cert }}
+        TENANT_DOMAIN:  ${{ inputs.tenant-domain }}
       run: |
-        ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }}
+        for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
+        {
+          if (./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix && inputs.library-prefix || '[]' }} -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
+            break
+          }
+        }
+

.github/actions/teams-message/action.yml

@@ -0,0 +1,92 @@
+name: Send a message to Teams
+description: Send messages to communication apps.
+
+inputs:
+  msg:
+    description: The teams message text
+  teams_url:
+    description: passthrough for secrets.TEAMS_CORSO_CI_WEBHOOK_URL
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/checkout@v3
+
+    - name: set github ref
+      shell: bash
+      run: |
+        echo "github_reference=${{ github.ref }}" >> $GITHUB_ENV
+
+    - name: trim github ref
+      shell: bash
+      run: |
+        echo "trimmed_ref=${github_reference#refs/}" >> $GITHUB_ENV
+
+    - name: build urls
+      shell: bash
+      run: |
+        echo "logurl=$(printf 'https://github.com/alcionai/corso/actions/runs/%s' ${{ github.run_id }})" >> $GITHUB_ENV
+        echo "commiturl=$(printf 'https://github.com/alcionai/corso/commit/%s' ${{ github.sha }})" >> $GITHUB_ENV
+        echo "refurl=$(printf 'https://github.com/alcionai/corso/%s' ${{ env.trimmed_ref }})" >> $GITHUB_ENV
+
+    - name: use url or blank val
+      shell: bash
+      run: |
+        echo "STEP=${{ env.trimmed_ref || '' }}" >> $GITHUB_ENV
+        echo "JOB=${{ github.job || '' }}" >> $GITHUB_ENV
+        echo "LOGS=${{ github.run_id && env.logurl || '-' }}" >> $GITHUB_ENV
+        echo "COMMIT=${{ github.sha && env.commiturl || '-' }}" >> $GITHUB_ENV
+        echo "REF=${{ env.trimmed_ref && env.refurl || '-' }}" >> $GITHUB_ENV
+
+    - name: Send JSON payload to Teams Webhook
+      shell: bash
+      run: |
+        curl -X POST \
+        -H "Content-Type: application/json" \
+        -d '{
+           "type":"message",
+           "attachments":[
+              {
+                 "contentType":"application/vnd.microsoft.card.adaptive",
+                 "contentUrl":null,
+                 "content":{
+                    "$schema":"http://adaptivecards.io/schemas/adaptive-card.json",
+                    "type":"AdaptiveCard",
+                    "body": [
+                        {
+                            "type": "TextBlock",
+                            "size": "Medium",
+                            "weight": "Bolder",
+                            "text": "${{ inputs.msg }}",
+                            "color": "Attention"
+                        },
+                        {
+                            "type": "TextBlock",
+                            "text": "${{ env.JOB }} :: ${{ env.STEP }}",
+                            "wrap": true
+                        }
+                    ],
+                    "actions": [
+                        {
+                            "type": "Action.OpenUrl",
+                            "title": "Action",
+                            "url": "${{ env.LOGS }}"
+                        },
+                        {
+                            "type": "Action.OpenUrl",
+                            "title": "Commit",
+                            "url": "${{ env.COMMIT }}"
+                        },
+                        {
+                            "type": "Action.OpenUrl",
+                            "title": "Ref",
+                            "url": "${{ env.REF }}"
+                        }
+                    ],
+                    "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                    "version": "1.5"
+                 }
+              }
+           ]
+        }' \
+        ${{ inputs.teams_url }}

.github/actions/website-linting/action.yml

@@ -1,4 +1,5 @@
 name: Lint Website
+description: Lint website content.
 
 inputs:
   version:

.github/workflows/_filechange_checker.yml

@@ -24,11 +24,11 @@ jobs:
       websitefileschanged: ${{ steps.websitechecker.outputs.websitefileschanged }}
       actionsfileschanged: ${{ steps.actionschecker.outputs.actionsfileschanged }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # only run CI tests if the src folder or workflow actions have changed
       - name: Check for file changes in src/ or .github/workflows/
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: dornycheck
         with:
           list-files: json

.github/workflows/binary-publish.yml

@@ -9,7 +9,7 @@ jobs:
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Get version string
         id: version
@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Publish Binary
         uses: ./.github/actions/publish-binary
@@ -34,4 +34,11 @@ jobs:
           version: ${{ needs.SetEnv.outputs.version }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           rudderstack_write_key: ${{ secrets.RUDDERSTACK_CORSO_WRITE_KEY }}
-          rudderstack_data_plane_url: ${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }}
+          rudderstack_data_plane_url: ${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }}
+
+      - name: Notify failure in teams
+        if: failure()
+        uses: ./.github/actions/teams-message
+        with:
+          msg: "[CORSO FAILED] Publishing Binary"
+          teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
       run:
         working-directory: src
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # single setup and sum cache handling here.
       # the results will cascade onto both testing and linting.
@@ -63,7 +63,7 @@ jobs:
       website-bucket: ${{ steps.website-bucket.outputs.website-bucket }}
       website-cfid: ${{ steps.website-cfid.outputs.website-cfid }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Figure out environment
         id: environment
@@ -110,10 +110,11 @@ jobs:
     needs: [Precheck, Checkout, SetEnv]
     environment: Testing
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || needs.precheck.outputs.websitefileschanged == 'true' # websitefileschanged also includes srcfileschanged
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint Website
         uses: ./.github/actions/website-linting
@@ -128,6 +129,7 @@ jobs:
     needs: [Precheck, Checkout, SetM365App]
     environment: Testing
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     if: (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main') || (needs.precheck.outputs.srcfileschanged == 'true' && github.event.pull_request.head.repo.full_name == github.repository)
     defaults:
       run:
@@ -140,7 +142,7 @@ jobs:
       CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-ci.log
       LOG_GRAPH_REQUESTS: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Golang with cache
         uses: magnetikonline/action-golang-cache@v4
@@ -155,7 +157,7 @@ jobs:
 
       # AWS creds
       - name: Configure AWS credentials from Test account
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
           role-session-name: integration-testing
@@ -171,6 +173,7 @@ jobs:
           CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }}
           CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }}
           CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
+          S3_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }}
         run: |
           set -euo pipefail
           go test       \
@@ -186,17 +189,89 @@ jobs:
       # Upload the original go test output as an artifact for later review.
       - name: Upload test log
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ci-test-log
           path: src/testlog/*
           if-no-files-found: error
           retention-days: 14
 
+  Retention-Test-Suite-Trusted:
+    needs: [Precheck, Checkout, SetM365App]
+    environment: Testing
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    if: (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main') || (needs.precheck.outputs.srcfileschanged == 'true' && github.event.pull_request.head.repo.full_name == github.repository)
+    defaults:
+      run:
+        working-directory: src
+    env:
+      # Resolve the name of the secret that contains the Azure client ID/secret
+      AZURE_CLIENT_ID_NAME: ${{ needs.SetM365App.outputs.client_id_env }}
+      AZURE_CLIENT_SECRET_NAME: ${{ needs.SetM365App.outputs.client_secret_env }}
+      CLIENT_APP_SLOT: ${{ needs.SetM365App.outputs.client_app_slot }}
+      CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-ci-retention.log
+      LOG_GRAPH_REQUESTS: true
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Golang with cache
+        uses: magnetikonline/action-golang-cache@v4
+        with:
+          go-version-file: src/go.mod
+
+      - run: mkdir testlog
+
+      # Install gotestfmt
+      - name: Set up gotestfmt
+        run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+
+      # AWS creds
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
+          role-session-name: integration-testing
+          aws-region: us-east-1
+
+      # run the tests
+      - name: Retention Tests
+        env:
+          AZURE_CLIENT_ID: ${{ secrets[env.AZURE_CLIENT_ID_NAME] }}
+          AZURE_CLIENT_SECRET: ${{ secrets[env.AZURE_CLIENT_SECRET_NAME] }}
+          AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
+          CORSO_RETENTION_TESTS: true
+          CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }}
+          CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }}
+          CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
+          S3_BUCKET: ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }}
+        run: |
+          set -euo pipefail
+          go test       \
+          -tags testing \
+          -json         \
+          -v            \
+          -failfast     \
+          -p 1          \
+          -timeout 10m  \
+          ./... \
+          2>&1 | tee ./testlog/gotest-ci.log | gotestfmt -hide successful-tests
+
+      # Upload the original go test output as an artifact for later review.
+      - name: Upload test log
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-retention-test-log
+          path: src/testlog/*
+          if-no-files-found: error
+          retention-days: 14
+
   Unit-Test-Suite:
     needs: [Precheck, Checkout]
     environment: Testing
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     if: needs.precheck.outputs.srcfileschanged == 'true'
     defaults:
       run:
@@ -205,7 +280,7 @@ jobs:
       CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-unit.log
       LOG_GRAPH_REQUESTS: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Golang with cache
         uses: magnetikonline/action-golang-cache@v4
@@ -240,7 +315,7 @@ jobs:
       # Upload the original go test output as an artifact for later review.
       - name: Upload test log
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: unit-test-log
           path: src/testlog/*
@@ -285,7 +360,7 @@ jobs:
 
       # Check out merge commit
       - name: Fork based /ok-to-test checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: "refs/pull/${{ github.event.client_payload.pull_request.number }}/merge"
 
@@ -302,7 +377,7 @@ jobs:
 
       # AWS creds
       - name: Configure AWS credentials from Test account
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
           role-session-name: integration-testing
@@ -329,7 +404,7 @@ jobs:
       # Upload the original go test log as an artifact for later review.
       - name: Upload test log
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: fork-test-log
           path: src/testlog/*
@@ -337,7 +412,7 @@ jobs:
           retention-days: 14
 
       # Update check run called "Test-Suite-Fork"
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         id: update-check-run
         if: failure()
         env:
@@ -374,12 +449,13 @@ jobs:
     needs: [Precheck, Checkout]
     environment: Testing
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || needs.precheck.outputs.srcfileschanged == 'true'
     defaults:
       run:
         working-directory: src
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Golang with cache
         uses: magnetikonline/action-golang-cache@v4
@@ -387,11 +463,11 @@ jobs:
           go-version-file: src/go.mod
 
       - name: Go Lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           # Keep pinned to a verson as sometimes updates will add new lint
           # failures in unchanged code.
-          version: v1.52.2
+          version: v1.54.2
           working-directory: src
           skip-pkg-cache: true
           skip-build-cache: true
@@ -409,6 +485,53 @@ jobs:
           install-go: false
           working-directory: src
 
+      - name: Run allowtags
+        run: |
+          go install github.com/ashmrtn/allowtags@latest
+          allowtags --allow-key json --allow-key uriparametername ./...
+
+      # I could not find a way to install tree-grepper without nix
+      # https://github.com/BrianHicks/tree-grepper/issues/293
+      - uses: cachix/install-nix-action@v25
+      - uses: cachix/cachix-action@v14
+        with:
+            name: tree-grepper
+      - run: nix-env -if https://github.com/BrianHicks/tree-grepper/archive/refs/heads/main.tar.gz
+      - name: Run trailing comma lint rule
+        run: |
+          # Using `grep .` as the exit codes are always true for correct grammar
+          if tree-grepper -q go '(argument_list "," @no-trailing-comma .)' | grep .; then
+            echo "No trailing commas for function calls"
+            exit 1
+          fi
+      - name: Check for empty string comparison
+        run: |
+          # Using `grep .` as the exit codes are always true for correct grammar
+          if tree-grepper -q go '((binary_expression (identifier) ["==" "!="] (interpreted_string_literal) @_ri) @exp (#eq? @_ri "\"\""))' | grep .; then
+            echo "Use len check instead of empty string comparison"
+            exit 1
+          fi
+      - name: Check for cases where errors are not propagated
+        run: |
+          # Using `grep .` as the exit codes are always true for correct grammar
+          if tree-grepper -q go '((if_statement (binary_expression) @_if (block (return_statement (expression_list (call_expression (selector_expression) @_fun ) @ret .)))) (#match? @_if "err != nil") (#match? @_fun "clues.NewWC"))' | grep .; then
+            echo "Make sure to propagate errors with clues"
+            exit 1
+          fi
+      - name: Check if clues without context are used when context is passed in
+        run: |
+          # Using `grep .` as the exit codes are always true for correct grammar
+          if tree-grepper -q go '((function_declaration (parameter_list . (parameter_declaration (identifier) @_octx)) body: (block (short_var_declaration left: (expression_list (identifier) @_err . ) right: (expression_list (call_expression (argument_list . (identifier) @_ctx)))) . (if_statement (binary_expression) @_exp consequence: (block (return_statement (expression_list (call_expression (selector_expression (call_expression (selector_expression) @clue)))  . )))))) (#eq? @_err "err") (#eq? @_octx "ctx") (#eq? @_ctx "ctx") (#eq? @_exp "err != nil") (#match? @clue "^clues\.") (#match? @clue "WC$"))' | grep .; then
+            echo "Do not use clues.*WC when context is passed in"
+            exit 1
+          fi
+      - name: Check clues with context is used when context is not passed in
+        run: |
+          # Using `grep .` as the exit codes are always true for correct grammar
+          if tree-grepper -q go '((function_declaration (parameter_list . (parameter_declaration (identifier) @_octx)) body: (block (short_var_declaration left: (expression_list (identifier) @_err . ) right: (expression_list (call_expression (argument_list . (identifier) @_ctx)))) . (if_statement (binary_expression) @_exp consequence: (block (return_statement (expression_list (call_expression (selector_expression (call_expression (selector_expression) @clue)))  . )))))) (#eq? @_err "err") (#eq? @_octx "ctx") (#not-eq? @_ctx "ctx") (#eq? @_exp "err != nil") (#match? @clue "^clues\.") (#not-match? @clue "WC$"))' | grep .; then
+            echo "Use clues.*WC when context is not passed in"
+            exit 1
+          fi
 
   # ----------------------------------------------------------------------------------------------------
   # --- GitHub Actions Linting -------------------------------------------------------------------------
@@ -420,7 +543,7 @@ jobs:
     runs-on: ubuntu-latest
     if: needs.precheck.outputs.actionsfileschanged == 'true'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: actionlint
         uses: raven-actions/actionlint@v1
@@ -445,7 +568,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main'
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Publish Binary
         uses: ./.github/actions/publish-binary
@@ -466,17 +589,17 @@ jobs:
     env:
       PLATFORMS: linux/amd64,linux/arm64
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # Setup buildx
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       # retrieve credentials for ghcr.io
       - name: Login to Github Packages
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -484,7 +607,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.IMAGE_NAME }}
           tags: |
@@ -494,7 +617,7 @@ jobs:
 
       # deploy the image
       - name: Build image and push to GitHub Container Registry
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ./build/Dockerfile
@@ -547,9 +670,9 @@ jobs:
       CORSO_VERSION: ${{ needs.SetEnv.outputs.version }}
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Validate amd64 container images
         run: |
@@ -604,7 +727,7 @@ jobs:
     if: github.ref == 'refs/heads/main'
 
     steps:
-      - uses: actions/checkout@v3 # need to checkout to make the action available
+      - uses: actions/checkout@v4 # need to checkout to make the action available
 
       - name: Publish website
         uses: ./.github/actions/publish-website
@@ -620,7 +743,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')
 
     steps:
-      - uses: actions/checkout@v3 # need to checkout to make the action available
+      - uses: actions/checkout@v4 # need to checkout to make the action available
 
       - name: Publish website
         uses: ./.github/actions/publish-website

.github/workflows/ci_test_cleanup.yml

@@ -12,18 +12,15 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID, '' ]
+        user: [CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, ""]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # sets the maximum time to now-30m.
-      # CI test have a 10 minute timeout.
-      # At 20 minutes ago, we should be safe from conflicts.
-      # The additional 10 minutes is just to be good citizens.
+      # CI test have a 20 minute timeout.
       - name: Set purge boundary
-        run: |
-          echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+        run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
 
       - name: Purge CI-Produced Folders for Users
         uses: ./.github/actions/purge-m365-data
@@ -36,6 +33,16 @@ jobs:
           azure-tenant-id: ${{ secrets.TENANT_ID }}
           m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
           m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
+          azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
+          azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
+          tenant-domain: ${{ vars.TENANT_DOMAIN }}
+
+      - name: Notify failure in teams
+        if: failure()
+        uses: ./.github/actions/teams-message
+        with:
+          msg: "[CORSO FAILED] ${{ vars[matrix.user] }} CI Cleanup"
+          teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
 
   Test-Site-Data-Cleanup:
     environment: Testing
@@ -43,28 +50,36 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        site: [ CORSO_M365_TEST_SITE_URL, EXT_SDK_TEST_SITE_URL ] 
+        site: [CORSO_M365_TEST_SITE_URL, CORSO_M365_TEST_GROUPS_SITE_URL]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # sets the maximum time to now-30m.
-      # CI test have a 10 minute timeout.
-      # At 20 minutes ago, we should be safe from conflicts.
-      # The additional 10 minutes is just to be good citizens.
+      # CI test have a 20 minute timeout.
       - name: Set purge boundary
-        run: |
-          echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+        run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
 
       - name: Purge CI-Produced Folders for Sites
         uses: ./.github/actions/purge-m365-data
         with:
           site: ${{ vars[matrix.site] }}
-          folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} 
-          libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} 
+          folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }}
+          libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }}
+          library-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }}
           older-than: ${{ env.HALF_HOUR_AGO }}
           azure-client-id: ${{ secrets.CLIENT_ID }}
           azure-client-secret: ${{ secrets.CLIENT_SECRET }}
           azure-tenant-id: ${{ secrets.TENANT_ID }}
           m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
           m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
+          azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
+          azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
+          tenant-domain: ${{ vars.TENANT_DOMAIN }}
+
+      - name: Notify failure in teams
+        if: failure()
+        uses: ./.github/actions/teams-message
+        with:
+          msg: "[CORSO FAILED] ${{ vars[matrix.site] }} CI Cleanup"
+          teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

.github/workflows/load_test.yml

@@ -26,7 +26,7 @@ jobs:
       run:
         working-directory: src
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Build the otel-daemon
         run: make build-otel-daemon
@@ -107,7 +107,7 @@ jobs:
       # package all artifacts for later review
       - name: Upload Log, Profilers, Traces
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: load-test-profiling
           path: ${{ github.workspace }}/testlog/*
@@ -120,7 +120,7 @@ jobs:
     outputs:
       matrix: ${{ steps.build.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - id: build
         run: |
           u=$(echo "${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}" | sed 's/\[/["/g' | sed 's/\]/"]/g' | sed 's/|/","/g')
@@ -140,7 +140,7 @@ jobs:
       matrix:
         user: [ CORSO_M365_LOAD_TEST_USER_ID, '' ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set folder boundary datetime
         run: |
           echo "NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
@@ -155,3 +155,6 @@ jobs:
           azure-tenant-id: ${{ secrets.TENANT_ID }}
           m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
           m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
+          azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
+          azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
+          tenant-domain: ${{ vars.TENANT_DOMAIN }}

.github/workflows/longevity_test.yml

@@ -0,0 +1,396 @@
+name: Longevity Testing
+on:
+  schedule:
+    # Run every day at 04:00 GMT (roughly 8pm PST)
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+    inputs:
+      user:
+        description: "User to run longevity test on"
+
+permissions:
+  # required to retrieve AWS credentials
+  id-token: write
+  contents: write
+
+# cancel currently running jobs if a new version of the branch is pushed
+concurrency:
+  group: longevity_testing-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  SetM365App:
+    uses: alcionai/corso/.github/workflows/accSelector.yaml@main
+
+  Longevity-Tests:
+    needs: [SetM365App]
+    environment: Testing
+    runs-on: ubuntu-latest
+    env:
+      # Need these in the local env so that corso can read them
+      AZURE_CLIENT_ID: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
+      AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
+      AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
+      CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
+      # re-used values
+      CORSO_LOG_DIR: ${{ github.workspace }}/src/testlog
+      CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-longevity.log
+      RESTORE_DEST_PFX: Corso_Test_Longevity_
+      TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || vars.CORSO_M365_TEST_USER_ID }}
+      PREFIX: "longevity"
+
+      # Options for retention.
+      RETENTION_MODE: GOVERNANCE
+      # Time to retain blobs for in hours.
+      RETENTION_DURATION: 216
+    defaults:
+      run:
+        working-directory: src
+
+    ############################################################################
+    # setup
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # needed to get latest tag
+
+      - name: Setup Golang with cache
+        uses: magnetikonline/action-golang-cache@v4
+        with:
+          go-version-file: src/go.mod
+
+      - run: |
+          go build -o longevity-test ./cmd/longevity_test
+          go build -o s3checker ./cmd/s3checker
+
+      - name: Get version string
+        id: version
+        run: |
+          echo version=$(git describe --tags --abbrev=0) | tee -a $GITHUB_OUTPUT
+
+      # Checkout the .github directory at the original branch's ref so we have a
+      # stable view of the actions.
+      - name: Code Checkout
+        working-directory: ${{ github.workspace }}
+        run: |
+          git checkout ${{ steps.version.outputs.version }}
+          git checkout ${{ github.ref }} -- .github
+
+      - run: go build -o corso
+        timeout-minutes: 10
+
+      - run: mkdir ${CORSO_LOG_DIR}
+
+        # Use shorter-lived credentials obtained from assume-role since these
+        # runs haven't been taking long.
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v4
+        timeout-minutes: 10
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
+          role-session-name: integration-testing
+          aws-region: us-east-1
+
+      ##########################################################################
+      # Repository commands
+
+      - name: Version Test
+        timeout-minutes: 10
+        run: |
+          ./corso --version | grep -c 'Corso version:'
+
+      - name: Repo init test
+        id: repo-init
+        timeout-minutes: 10
+        run: |
+          set -euo pipefail
+          echo -e "\nRepo init test\n" >> ${{ env.CORSO_LOG_FILE }}
+          ./corso repo init s3 \
+            --no-stats \
+            --hide-progress \
+            --retention-mode $(echo "${{ env.RETENTION_MODE }}" | tr '[:upper:]' '[:lower:]') \
+            --retention-duration "${{ env.RETENTION_DURATION }}h" \
+            --extend-retention \
+            --prefix ${{ env.PREFIX }} \
+            --bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
+            2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
+
+          if  grep -q 'Failed to' ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
+          then
+            echo "Repo could not be initialized"
+            exit 1
+          fi
+
+      - name: Repo connect test
+        timeout-minutes: 10
+        run: |
+          set -euo pipefail
+          echo -e "\nRepo connect test\n" >> ${{ env.CORSO_LOG_FILE }}
+          ./corso repo connect s3 \
+            --no-stats \
+            --hide-progress \
+            --prefix ${{ env.PREFIX }} \
+            --bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
+            2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-connect.log
+
+          if ! grep -q 'Connected to S3 bucket' ${{ env.CORSO_LOG_DIR }}/gotest-repo-connect.log
+          then
+            echo "Repo could not be connected"
+            exit 1
+          fi
+
+      ##########################################################################
+      # Exchange
+
+      - name: Backup exchange test
+        id: exchange-test
+        timeout-minutes: 30
+        run: |
+          echo -e "\nBackup Exchange test\n" >> ${CORSO_LOG_FILE}
+          ./corso backup create exchange \
+          --no-stats \
+          --mailbox "${TEST_USER}" \
+          --hide-progress \
+          --json \
+          2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_exchange.txt 
+
+          resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_exchange.txt ) 
+
+          if  [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
+            echo "backup was not successful"
+            exit 1
+          fi
+
+          data=$( echo $resultjson | jq -r '.[0] | .id' )
+          echo result=$data >> $GITHUB_OUTPUT
+
+      ##########################################################################
+      # Onedrive
+
+      - name: Backup onedrive test
+        id: onedrive-test
+        timeout-minutes: 30
+        run: |
+          set -euo pipefail
+          echo -e "\nBackup OneDrive test\n" >> ${CORSO_LOG_FILE}
+          ./corso backup create onedrive \
+          --no-stats \
+          --hide-progress \
+          --user "${TEST_USER}" \
+          --json \
+          2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_onedrive.txt 
+
+          resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_onedrive.txt ) 
+
+          if  [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
+            echo "backup was not successful"
+            exit 1
+          fi
+
+          data=$( echo $resultjson | jq -r '.[0] | .id' )
+          echo result=$data >> $GITHUB_OUTPUT
+
+      ##########################################################################
+      # Sharepoint test
+      - name: Backup sharepoint test
+        id: sharepoint-test
+        timeout-minutes: 30
+        run: |
+          set -euo pipefail
+          echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          ./corso backup create sharepoint \
+          --no-stats \
+          --hide-progress \
+          --site "${{ vars.CORSO_M365_TEST_SITE_URL }}" \
+          --json \
+          2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_sharepoint.txt 
+
+          resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_sharepoint.txt ) 
+
+          if  [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
+            echo "backup was not successful"
+            exit 1
+          fi
+
+          data=$( echo $resultjson | jq -r '.[0] | .id' )
+          echo result=$data >> $GITHUB_OUTPUT
+
+      ##########################################################################
+      # Backup Exchange Deletion test
+      - name: Backup Delete exchange test
+        id: delete-exchange-test
+        timeout-minutes: 30
+        env:
+          SERVICE: "exchange"
+          DELETION_DAYS: 10
+        run: |
+          set -euo pipefail
+          echo -e "\nDelete Backup exchange \n" >> ${CORSO_LOG_FILE}
+          ./longevity-test
+
+      ##########################################################################
+      # Backup Onedrive Deletion test
+      - name: Backup Delete onedrive test
+        id: delete-onedrive-test
+        timeout-minutes: 30
+        env:
+          SERVICE: "onedrive"
+          DELETION_DAYS: 10
+        run: |
+          set -euo pipefail
+          echo -e "\nDelete Backup onedrive \n" >> ${CORSO_LOG_FILE}
+          ./longevity-test
+
+      ##########################################################################
+      # Backup Sharepoint Deletion test
+      - name: Backup Delete Sharepoint test
+        id: delete-sharepoint-test
+        timeout-minutes: 30
+        env:
+          SERVICE: "sharepoint"
+          DELETION_DAYS: 5
+        run: |
+          set -euo pipefail
+          echo -e "\nDelete Backup sharepoint \n" >> ${CORSO_LOG_FILE}
+          ./longevity-test
+
+      ##########################################################################
+      # Export OneDrive Test
+      - name: OneDrive Export test
+        timeout-minutes: 30
+        run: |
+          set -euo pipefail
+          echo -e "\Export OneDrive test\n" >> ${CORSO_LOG_FILE}
+
+          echo -e "\Export OneDrive test - first entry\n" >> ${CORSO_LOG_FILE}
+          ./corso backup list onedrive 2>/dev/null | tail -n+2 | head -n1 | awk '{print $1}' |
+              while read -r line; do
+                  ./corso export onedrive \
+                    "/tmp/corso-export--$line" \
+                    --no-stats \
+                    --backup "$line" \
+                    2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_onedrive_first.txt
+              done
+
+          echo -e "\Export OneDrive test - last entry\n" >> ${CORSO_LOG_FILE}
+          ./corso backup list onedrive 2>/dev/null | tail -n1 | awk '{print $1}' |
+              while read -r line; do
+                  ./corso export onedrive \
+                    "/tmp/corso-export--$line" \
+                    --no-stats \
+                    --backup "$line" \
+                    2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_onedrive_last.txt
+              done
+
+      ##########################################################################
+      # Export SharePoint Test
+      - name: SharePoint Export test
+        timeout-minutes: 30
+        run: |
+          set -euo pipefail
+          echo -e "\Export SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          echo -e "\Export SharePoint test - first entry\n" >> ${CORSO_LOG_FILE}
+          ./corso backup list sharepoint 2>/dev/null | tail -n+2 | head -n1 | awk '{print $1}' |
+              while read -r line; do
+                  ./corso export sharepoint \
+                    "/tmp/corso-export--$line" \
+                    --no-stats \
+                    --backup "$line" \
+                    2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_sharepoint_first.txt
+              done
+
+          echo -e "\Export SharePoint test - last entry\n" >> ${CORSO_LOG_FILE}
+          ./corso backup list sharepoint 2>/dev/null | tail -n1 | awk '{print $1}' |
+              while read -r line; do
+                  ./corso export sharepoint \
+                    "/tmp/corso-export--$line" \
+                    --no-stats \
+                    --backup "$line" \
+                    2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_sharepoint_last.txt
+              done
+
+      ##########################################################################
+      # Maintenance test
+      - name: Maintenance test Daily
+        id: maintenance-test-daily
+        timeout-minutes: 30
+        run: |
+          set -euo pipefail
+          echo -e "\n Maintenance test Daily\n" >> ${CORSO_LOG_FILE}
+
+          # Run with the force flag so it doesn't fail if the github runner
+          # hostname isn't what's expected. This is only safe because we can
+          # guarantee only one runner will be executing maintenance at a time.
+          ./corso repo  maintenance --mode metadata \
+          --no-stats \
+          --hide-progress \
+          --force \
+          --json \
+          2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_metadata.txt
+
+      - name: Maintenance test Weekly
+        id: maintenance-test-weekly
+        timeout-minutes: 30
+        run: |
+          if [[ $(date +%A) == "Saturday" ]]; then
+            set -euo pipefail
+            echo -e "\n Maintenance test Weekly\n" >> ${CORSO_LOG_FILE}
+
+            ./corso repo  maintenance --mode complete \
+            --no-stats \
+            --hide-progress \
+            --force \
+            --json \
+            2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_complete.txt 
+
+            # TODO(ashmrtn): We can also check that non-current versions of
+            # blobs don't have their retention extended if we want.
+            #
+            # Assuming no failures during full maintenance, current versions of
+            # objects with the below versions should have retention times that
+            # are roughly (now + RETENTION_DURATION). We can explicitly check
+            # for this, but leave a little breathing room since maintenance may
+            # take some time to run.
+            #
+            # If we pick a live-retention-duration that is too small then we'll
+            # start seeing failures. The check for live objects is a lower bound
+            # check.
+            #
+            # Blob prefixes are as follows:
+            #   - kopia.blobcfg - repo-wide config
+            #   - kopia.repository - repo-wide config
+            #   - p - data pack blobs (i.e. file data)
+            #   - q - metadata pack blobs (i.e. manifests, directory listings, etc)
+            #   - x - index blobs
+            ./s3checker \
+              --bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
+              --prefix ${{ env.PREFIX }} \
+              --retention-mode ${{ env.RETENTION_MODE }} \
+              --live-retention-duration "$((${{ env.RETENTION_DURATION }}-1))h" \
+              --object-prefix "kopia.blobcfg" \
+              --object-prefix "kopia.repository" \
+              --object-prefix "p" \
+              --object-prefix "q" \
+              --object-prefix "x"
+          fi
+
+      ##########################################################################
+      # Logging & Notifications
+      # Upload the original go test output as an artifact for later review.
+      - name: Upload test log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: longevity-test-log
+          path: src/testlog/*
+          if-no-files-found: error
+          retention-days: 14
+
+      - name: Notify failure in teams
+        if: failure()
+        uses: ./.github/actions/teams-message
+        with:
+          msg: "[CORSO FAILED] Longevity Test"
+          teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

.github/workflows/nightly_test.yml

@@ -30,7 +30,7 @@ jobs:
       run:
         working-directory: src
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # single setup and sum cache handling here.
       # the results will cascade onto both testing and linting.
@@ -48,7 +48,7 @@ jobs:
   # ----------------------------------------------------------------------------------------------------
 
   Test-Suite-Trusted:
-    needs: [ Checkout, SetM365App] 
+    needs: [Checkout, SetM365App]
     environment: Testing
     runs-on: ubuntu-latest
     defaults:
@@ -60,7 +60,7 @@ jobs:
       AZURE_CLIENT_SECRET_NAME: ${{ needs.SetM365App.outputs.client_secret_env }}
       CLIENT_APP_SLOT: ${{ needs.SetM365App.outputs.client_app_slot }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Golang with cache
         uses: magnetikonline/action-golang-cache@v4
@@ -73,17 +73,12 @@ jobs:
       - name: Set up gotestfmt
         run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
 
-      # AWS creds
-      - name: Configure AWS credentials from Test account
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
-          role-session-name: integration-testing
-          aws-region: us-east-1
-
       # run the tests
       - name: Integration Tests
         env:
+          # Use long-lived AWS credentials.
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
           AZURE_CLIENT_ID: ${{ secrets[env.AZURE_CLIENT_ID_NAME] }}
           AZURE_CLIENT_SECRET: ${{ secrets[env.AZURE_CLIENT_SECRET_NAME] }}
           AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
@@ -92,60 +87,36 @@ jobs:
           CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }}
           CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }}
           CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
-          CORSO_LOG_FILE: ${{ github.workspace }}/testlog/run-nightly.log
+          CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-nightly.log
           LOG_GRAPH_REQUESTS: true
+          S3_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }}
         run: |
           set -euo pipefail
           go test       \
           -tags testing \
           -json         \
           -v            \
-          -failfast     \
           -p 1          \
-          -timeout 15m  \
+          -timeout 2h  \
           ./... 2>&1 | tee ./testlog/gotest-nightly.log | gotestfmt -hide successful-tests
 
-##########################################################################################################################################
+      ##########################################################################################################################################
 
-# Logging & Notifications
+      # Logging & Notifications
 
       # Upload the original go test output as an artifact for later review.
       - name: Upload test log
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: nightly-test-log
           path: src/testlog/*
           if-no-files-found: error
           retention-days: 14
 
-      - name: SHA info
-        id: sha-info
+      - name: Notify failure in teams
         if: failure()
-        run: |
-          echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA}
-          echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT
-          echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}  >> $GITHUB_OUTPUT
-          echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT
-
-      - name: Send Github Action failure to Slack
-        id: slack-notification
-        if: failure()
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: ./.github/actions/teams-message
         with:
-          payload: |
-            {
-              "text": "Nightly test failure - build: ${{ job.status }} - SHA: ${{  steps.sha-info.outputs.SHA }}",
-              "blocks": [
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "[FAILED] Nightly Checks :: <${{  steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ steps.sha-info.outputs.COMMIT_URL }}|[Base]>\nCommit: <${{  steps.sha-info.outputs.COMMIT_URL }}|${{  steps.sha-info.outputs.SHA }}>"
-                  }
-                }
-              ]
-            }
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+          msg: "[COROS FAILED] Nightly Checks"
+          teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

.github/workflows/ok-to-test.yml

@@ -13,13 +13,13 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: tibdex/github-app-token@v1
+        uses: tibdex/github-app-token@v2
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.PRIVATE_KEY }}
 
       - name: Slash Command Dispatch
-        uses: peter-evans/slash-command-dispatch@v3
+        uses: peter-evans/slash-command-dispatch@v4
         env:
           TOKEN: ${{ steps.generate_token.outputs.token }}
         with:

.github/workflows/sanity-test.yaml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       user:
-        description: 'User to run sanity test on'
+        description: "User to run sanity test on"
 
 permissions:
   # required to retrieve AWS credentials
@@ -23,7 +23,7 @@ jobs:
     uses: alcionai/corso/.github/workflows/accSelector.yaml@main
 
   Sanity-Tests:
-    needs: [ SetM365App ]
+    needs: [SetM365App]
     environment: Testing
     runs-on: ubuntu-latest
     env:
@@ -38,18 +38,17 @@ jobs:
       CORSO_LOG_DIR: ${{ github.workspace }}/src/testlog
       CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-sanity.log
       RESTORE_DEST_PFX: Corso_Test_Sanity_
-      TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }}
+      TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || vars.CORSO_M365_TEST_USER_ID }}
 
     defaults:
       run:
         working-directory: src
-        
-##########################################################################################################################################
 
-# setup
+    ##########################################################################################################################################
+
+    # setup
     steps:
-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Golang with cache
         uses: magnetikonline/action-golang-cache@v4
@@ -57,13 +56,16 @@ jobs:
           go-version-file: src/go.mod
 
       - run: go build -o corso
+        timeout-minutes: 10
+
       - run: go build -o sanity-test ./cmd/sanity_test
+        timeout-minutes: 10
 
       - run: mkdir ${CORSO_LOG_DIR}
 
-##########################################################################################################################################
+      ##########################################################################################################################################
 
-# Pre-Run cleanup
+      # Pre-Run cleanup
 
       # unlike CI tests, sanity tests are not expected to run concurrently.
       # however, the sanity yaml concurrency is set to a maximum of 1 run, preferring
@@ -77,6 +79,7 @@ jobs:
           echo "NOW=$(date +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
 
       - name: Purge CI-Produced Folders for Users
+        timeout-minutes: 30
         uses: ./.github/actions/purge-m365-data
         with:
           user: ${{ env.TEST_USER }}
@@ -87,30 +90,39 @@ jobs:
           azure-tenant-id: ${{ secrets.TENANT_ID }}
           m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
           m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
+          azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
+          azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
+          tenant-domain: ${{ vars.TENANT_DOMAIN }}
 
       - name: Purge CI-Produced Folders for Sites
+        timeout-minutes: 30
         if: always()
         uses: ./.github/actions/purge-m365-data
         with:
-          site: ${{ secrets.CORSO_M365_TEST_SITE_URL }}
+          site: ${{ vars.CORSO_M365_TEST_SITE_URL }}
           folder-prefix: ${{ env.RESTORE_DEST_PFX }}
-          libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} 
+          libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }}
           older-than: ${{ env.NOW }}
           azure-client-id: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
           azure-client-secret: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
           azure-tenant-id: ${{ secrets.TENANT_ID }}
           m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
           m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
+          azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
+          azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
+          tenant-domain: ${{ vars.TENANT_DOMAIN }}
 
-##########################################################################################################################################
+      ##########################################################################################################################################
 
-# Repository commands
+      # Repository commands
 
       - name: Version Test
+        timeout-minutes: 10
         run: |
           ./corso --version | grep -c 'Corso version:'
 
       - name: Repo init test
+        timeout-minutes: 10
         id: repo-init
         run: |
           set -euo pipefail
@@ -132,6 +144,7 @@ jobs:
           echo result="$prefix" >> $GITHUB_OUTPUT
 
       - name: Repo connect test
+        timeout-minutes: 10
         run: |
           set -euo pipefail
           echo -e "\nRepo connect test\n" >> ${{ env.CORSO_LOG_FILE }}
@@ -148,13 +161,27 @@ jobs:
             exit 1
           fi
 
-##########################################################################################################################################
+      # Run maintenance on an empty repo just to make sure the command still
+      # works.
+      - name: Repo maintenance test
+        timeout-minutes: 30
+        run: |
+          set -euo pipefail
+          echo -e "\nRepo maintenance test\n" >> ${{ env.CORSO_LOG_FILE }}
+          ./corso repo maintenance \
+            --no-stats \
+            --hide-progress \
+            --mode complete \
+            2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-maintenance.log
 
-# Exchange
+      ##########################################################################################################################################
+
+      # Exchange
 
       # generate new entries to roll into the next load test
       # only runs if the test was successful
       - name: Exchange - Create new data
+        timeout-minutes: 30
         working-directory: ./src/cmd/factory
         run: |
           go run . exchange emails \
@@ -164,60 +191,68 @@ jobs:
           --count 4
 
       - name: Exchange - Backup
+        timeout-minutes: 30
         id: exchange-backup
         uses: ./.github/actions/backup-restore-test
         with:
           service: exchange
-          kind: initial
+          kind: first-backup
           backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
-          restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
+          restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Incremental backup
+        timeout-minutes: 30
         id: exchange-backup-incremental
         uses: ./.github/actions/backup-restore-test
         with:
           service: exchange
           kind: incremental
           backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
-          restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
-          base-backup: ${{ steps.exchange-backup.outputs.backup-id }}
+          restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
+          backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Non delta backup
+        timeout-minutes: 30
         id: exchange-backup-non-delta
         uses: ./.github/actions/backup-restore-test
         with:
           service: exchange
           kind: non-delta
           backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email" --disable-delta'
-          restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
-          base-backup: ${{ steps.exchange-backup.outputs.backup-id }}
+          restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
+          backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Incremental backup after non-delta
+        timeout-minutes: 30
         id: exchange-backup-incremental-after-non-delta
         uses: ./.github/actions/backup-restore-test
         with:
           service: exchange
           kind: non-delta-incremental
           backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
-          restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
-          base-backup: ${{ steps.exchange-backup.outputs.backup-id }}
+          restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
+          backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
+      ##########################################################################################################################################
 
-##########################################################################################################################################
-
-# Onedrive
+      # Onedrive
 
       # generate new entries for test
       - name: OneDrive - Create new data
         id: new-data-creation-onedrive
+        timeout-minutes: 30
         working-directory: ./src/cmd/factory
         run: |
           suffix=$(date +"%Y-%m-%d_%H-%M-%S")
@@ -233,17 +268,20 @@ jobs:
 
       - name: OneDrive - Backup
         id: onedrive-backup
+        timeout-minutes: 30
         uses: ./.github/actions/backup-restore-test
         with:
           service: onedrive
-          kind: initial
+          kind: first-backup
           backup-args: '--user "${{ env.TEST_USER }}"'
-          restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
+          restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       # generate some more enteries for incremental check
       - name: OneDrive - Create new data (for incremental)
+        timeout-minutes: 30
         working-directory: ./src/cmd/factory
         run: |
           go run . onedrive files  \
@@ -255,28 +293,31 @@ jobs:
 
       - name: OneDrive - Incremental backup
         id: onedrive-incremental
+        timeout-minutes: 30
         uses: ./.github/actions/backup-restore-test
         with:
           service: onedrive
           kind: incremental
           backup-args: '--user "${{ env.TEST_USER }}"'
-          restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
+          restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
-##########################################################################################################################################
+      ##########################################################################################################################################
 
-# Sharepoint
+      # Sharepoint Library
 
       # generate new entries for test
       - name: SharePoint - Create new data
         id: new-data-creation-sharepoint
+        timeout-minutes: 30
         working-directory: ./src/cmd/factory
         run: |
           suffix=$(date +"%Y-%m-%d_%H-%M-%S")
 
           go run . sharepoint files  \
-            --site ${{ secrets.CORSO_M365_TEST_SITE_URL }} \
+            --site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
             --user ${{ env.TEST_USER }} \
             --secondaryuser  ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
             --tenant ${{ secrets.TENANT_ID }} \
@@ -287,21 +328,25 @@ jobs:
 
       - name: SharePoint - Backup
         id: sharepoint-backup
+        timeout-minutes: 30
         uses: ./.github/actions/backup-restore-test
         with:
           service: sharepoint
-          kind: initial
-          backup-args: '--site "${{ secrets.CORSO_M365_TEST_SITE_URL }}"'
-          restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
+          kind: first-backup
+          backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries'
+          restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
+          category: libraries
 
       # generate some more enteries for incremental check
       - name: SharePoint - Create new data (for incremental)
+        timeout-minutes: 30
         working-directory: ./src/cmd/factory
         run: |
           go run . sharepoint files  \
-            --site ${{ secrets.CORSO_M365_TEST_SITE_URL }} \
+            --site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
             --user ${{ env.TEST_USER }} \
             --secondaryuser  ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
             --tenant ${{ secrets.TENANT_ID }} \
@@ -310,56 +355,186 @@ jobs:
 
       - name: SharePoint - Incremental backup
         id: sharepoint-incremental
+        timeout-minutes: 30
         uses: ./.github/actions/backup-restore-test
         with:
           service: sharepoint
           kind: incremental
-          backup-args: '--site "${{ secrets.CORSO_M365_TEST_SITE_URL }}"'
-          restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions'
-          test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
+          backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries'
+          restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
+          category: libraries
 
-##########################################################################################################################################
+      ##########################################################################################################################################
 
-# Logging & Notifications
+      # Sharepoint Lists
+
+      # generate new entries for test
+      # The `awk | tr | sed` command chain is used to get a comma separated list of SharePoint list names.
+      - name: SharePoint Lists - Create new data
+        id: new-data-creation-sharepoint-lists
+        timeout-minutes: 30
+        working-directory: ./src/cmd/factory
+        run: |
+          suffix=$(date +"%Y-%m-%d_%H-%M-%S")
+
+          go run . sharepoint lists  \
+            --site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
+            --user ${{ env.TEST_USER }} \
+            --secondaryuser  ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
+            --tenant ${{ secrets.TENANT_ID }} \
+            --destination ${{ env.RESTORE_DEST_PFX }}$suffix \
+            --count 4 |
+            awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' |
+            tee $GITHUB_OUTPUT
+      # Extracts the common prefix for the Sharepoint list names.
+      - name: SharePoint Lists - Store restore container
+        id: sharepoint-lists-store-restore-container
+        run: |
+          echo ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} |
+          cut -d',' -f1 |
+          cut -d'_' -f1,2,3,4,5 |
+          sed -e 's/^/result=/' |
+          tee $GITHUB_OUTPUT
+
+      - name: SharePoint Lists - Backup
+        id: sharepoint-lists-backup
+        timeout-minutes: 30
+        uses: ./.github/actions/backup-restore-test
+        with:
+          service: sharepoint
+          kind: first-backup-lists
+          backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists'
+          restore-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S')"
+          export-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }}"
+          restore-container: "${{ steps.sharepoint-lists-store-restore-container.outputs.result }}"
+          log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
+          category: lists
+          on-collision: copy
+
+      # generate some more enteries for incremental check
+      - name: SharePoint Lists - Create new data (for incremental)
+        id: inc-data-creation-sharepoint-lists
+        timeout-minutes: 30
+        working-directory: ./src/cmd/factory
+        run: |
+          suffix=$(date +"%Y-%m-%d_%H-%M-%S")
+
+          go run . sharepoint lists  \
+            --site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
+            --user ${{ env.TEST_USER }} \
+            --secondaryuser  ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
+            --tenant ${{ secrets.TENANT_ID }} \
+            --destination ${{ env.RESTORE_DEST_PFX }}$suffix \
+            --count 4 |
+            awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' |
+            tee $GITHUB_OUTPUT
+
+      - name: SharePoint Lists - Store restore container (for incremental)
+        id: sharepoint-lists-store-restore-container-inc
+        run: |
+          echo ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }} |
+          cut -d',' -f1 |
+          cut -d'_' -f1,2,3,4,5 |
+          sed -e 's/^/result=/' |
+          tee $GITHUB_OUTPUT
+
+      - name: SharePoint Lists - Incremental backup
+        id: sharepoint-lists-incremental
+        timeout-minutes: 30
+        uses: ./.github/actions/backup-restore-test
+        with:
+          service: sharepoint
+          kind: incremental-lists
+          backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists'
+          restore-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S')"
+          export-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }}"
+          restore-container: "${{ steps.sharepoint-lists-store-restore-container-inc.outputs.result }},${{ steps.sharepoint-lists-store-restore-container.outputs.result }}"
+          log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
+          category: lists
+          on-collision: copy
+
+      ##########################################################################################################################################
+
+      # Groups and Teams
+
+      # generate new entries for test
+      - name: Groups - Create new data
+        id: new-data-creation-groups
+        timeout-minutes: 30
+        working-directory: ./src/cmd/factory
+        run: |
+          suffix=$(date +"%Y-%m-%d_%H-%M-%S")
+
+          go run . sharepoint files  \
+            --site ${{ vars.CORSO_M365_TEST_GROUPS_SITE_URL }} \
+            --user ${{ env.TEST_USER }} \
+            --secondaryuser  ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
+            --tenant ${{ secrets.TENANT_ID }} \
+            --destination ${{ env.RESTORE_DEST_PFX }}$suffix \
+            --count 4
+
+          echo result="${suffix}" >> $GITHUB_OUTPUT
+
+      - name: Groups - Backup
+        id: groups-backup
+        timeout-minutes: 30
+        uses: ./.github/actions/backup-restore-test
+        with:
+          service: groups
+          kind: first-backup
+          backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}" --data messages,libraries'
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}"
+          log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
+
+      # generate some more entries for incremental check
+      - name: Groups - Create new data (for incremental)
+        timeout-minutes: 30
+        working-directory: ./src/cmd/factory
+        run: |
+          go run . sharepoint files  \
+            --site ${{ vars.CORSO_M365_TEST_GROUPS_SITE_URL }} \
+            --user ${{ env.TEST_USER }} \
+            --secondaryuser  ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
+            --tenant ${{ secrets.TENANT_ID }} \
+            --destination ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }} \
+            --count 4
+
+      - name: Groups - Incremental backup
+        id: groups-incremental
+        timeout-minutes: 30
+        uses: ./.github/actions/backup-restore-test
+        with:
+          service: groups
+          kind: incremental
+          backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}" --data messages,libraries'
+          restore-args: '--site "${{ vars.CORSO_M365_TEST_GROUPS_SITE_URL }}" --folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
+          restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}"
+          log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
+
+      ##########################################################################################################################################
+
+      # Logging & Notifications
 
       # Upload the original go test output as an artifact for later review.
       - name: Upload test log
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: sanity-test-log
           path: ${{ env.CORSO_LOG_DIR }}/*
           if-no-files-found: error
           retention-days: 14
 
-      - name: SHA info
-        id: sha-info
+      - name: Notify failure in teams
         if: failure()
-        run: |
-          echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA}
-          echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT
-          echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}  >> $GITHUB_OUTPUT
-          echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT
-
-      - name: Send Github Action failure to Slack
-        id: slack-notification
-        if: failure()
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: ./.github/actions/teams-message
         with:
-          payload: |
-            {
-              "text": "Sanity test failure - build: ${{ job.status }} - SHA: ${{  steps.sha-info.outputs.SHA }}",
-              "blocks": [
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "[FAILED] Sanity Checks :: <${{  steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ github.event.pull_request.html_url || github.event.head_commit.url }}|[Base]>\nCommit: <${{  steps.sha-info.outputs.COMMIT_URL }}|${{  steps.sha-info.outputs.SHA }}>"
-                  }
-                }
-              ]
-            }
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+          msg: "[CORSO FAILED] Sanity Tests"
+          teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

.github/workflows/testnotification.yml

@@ -0,0 +1,23 @@
+name: Manually Test Teams Action
+
+on:
+  workflow_dispatch:
+    inputs:
+      msg:
+        description: 'Message to send:'
+        required: true
+        default: 'This is a test message'
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Send notification
+      uses: ./.github/actions/teams-message
+      with:
+        msg: ${{ github.event.inputs.msg }}
+        teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

.github/workflows/website-publish.yml

@@ -21,7 +21,7 @@ jobs:
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # needed to get latest tag
 
@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint Website
         uses: ./.github/actions/website-linting
@@ -56,7 +56,7 @@ jobs:
         working-directory: website
 
     steps:
-      - uses: actions/checkout@v3 # need to checkout to make the action available
+      - uses: actions/checkout@v4 # need to checkout to make the action available
 
       - name: Publish website
         uses: ./.github/actions/publish-website

.github/workflows/weekly_cleanup.yml

@@ -1,26 +0,0 @@
-name: Weekly S3 Test Bucket Cleanup
-on:
-  schedule:
-    # every saturday at 23:59 (11:59pm)
-    - cron: "59 23 * * 6"
-
-permissions:
-  # required to retrieve AWS credentials
-  id-token: write
-
-jobs:
-  S3-Test-Cleanup:
-    runs-on: ubuntu-latest
-    environment: Testing
-
-    steps:
-      - name: Configure AWS credentials from Test account
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
-          role-session-name: integration-testing
-          aws-region: us-east-1
-
-      - name: Delete all files in the test bucket
-        run: |
-          aws s3 rm s3://${{ secrets.CI_TESTS_S3_BUCKET }} --recursive

CHANGELOG.md

@@ -6,11 +6,201 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased] (beta)
+### Fixed
+- Handle the case where an email or event cannot be retrieved from Exchange due to an `ErrorCorruptData` error. Corso will skip over the item but report it in the backup summary.
+- Emails attached within other emails are now correctly exported
+- Gracefully handle email and post attachments without name when exporting to eml
+- Use correct timezone for event start and end times in Exchange exports (helps fix issues in relative recurrence patterns)
+- Fixed an issue causing exports dealing with calendar data to have high memory usage
+
+## [v0.19.0] (beta) - 2024-02-06
 
 ### Added
+- Events can now be exported from Exchange backups as .ics files.
+- Update repo init configuration to reduce the total number of GET requests sent
+  to the object store when using corso. This affects repos that have many
+  backups created in them per day the most.
+- Feature Preview: Corso now supports backup, export & restore of SharePoint lists. Lists backup can be initiated using `corso backup create sharepoint --site <site-url> --data lists`.
+- Group mailbox(aka conversations) backup and export support is now officially available. Group mailbox posts can be exported as `.eml` files.
+
+### Fixed
+- Retry transient 400 "invalidRequest" errors during onedrive & sharepoint backup.
+- Backup attachments associated with group mailbox items.
+- Groups and Teams backups no longer fail when a resource has no display name.
+- Contacts in-place restore failed if the restore destination was empty.
+- Link shares with external users are now backed up and restored as expected
+- Ensure persistent repo config is populated on repo init if repo init failed partway through during the previous init attempt.
+
+### Changed
+- When running `backup details` on an empty backup returns a more helpful error message.
+- Backup List additionally shows the data category for each backup.
+- Remove hidden `--succeed-if-exists` flag for repo init. Repo init will now succeed without error if run on an existing repo with the same passphrase.
+
+### Known issues
+- Backing up a group mailbox item may fail if it has a very large number of attachments (500+).
+- Event description for exchange exports might look slightly different for certain events.
+- Exchange in-place restore may restore items in well-known folders to different folders if the user has well-known folder names change based on locale and has updated the locale since the backup was created.
+- In-place Exchange contacts restore will merge items in folders named "Contacts" or "contacts" into the default folder.
+- External users with access through shared links will not receive these links as they are not sent via email during restore.
+- Graph API has limited support for certain column types such as `location`, `hyperlink/picture`, and `metadata`. Restoring SharePoint list items containing these columns will result in differences compared to the original items.
+- SharePoint list item attachments are not available due to graph API limitations.
+- Group mailbox restore is not supported due to limited Graph API support for creating mailbox items.
+- Due to Graph API limitations, any group mailbox items present in subfolders other than Inbox aren't backed up.
+
+## [v0.18.0] (beta) - 2024-01-02
+
+### Fixed
+- Handle the case where an email cannot be retrieved from Exchange due to an `ErrorInvalidRecipients` error. In
+this case, Corso will skip over the item but report this in the backup summary.
+- Fix `ErrorItemNotFound` errors when restoring emails with multiple attachments.
+- Avoid Graph SDK `Requests must contain extension changes exclusively.` errors by removing server-populated field from restored event items.
+- Improve Group mailbox(conversations) backup performance by only downloading new items or items with modified content.
+- Handle cases where Exchange backup stored invalid JSON blobs if there were special characters in the user content. These would result in errors during restore.
+
+### Known issues
+- Restoring OneDrive, SharePoint, or Teams & Groups items shared with external users while the tenant or site is configured to not allow sharing with external users will not restore permissions.
+
+### Added
+- Contacts can now be exported from Exchange backups as .vcf files
+
+## [v0.17.0] (beta) - 2023-12-11
+
+### Changed
+- Memory optimizations for large scale OneDrive and Sharepoint backups.
+
+### Fixed
+- Resolved a possible deadlock when backing up Teams Channel Messages.
+- Fixed an attachment download failure(ErrorTooManyObjectsOpened) during exchange backup.
+
+## [v0.16.0] (beta) - 2023-11-28
+
+### Added
+- Export support for emails in exchange backups as `.eml` files.
+- More colorful and informational cli display.
+
+### Changed
+- The file extension in Teams messages exports has switched to json to match the content type.
+- SDK consumption of the /services/m365 package has shifted from independent functions to a client-based api.
+- SDK consumers can now configure the /services/m365 graph api client configuration when constructing a new m365 client.
+- Dynamic api rate limiting allows small-scale Exchange backups to complete more quickly.
+- Kopia's local config files now uses unique filenames that match Corso configurations.  This can protect concurrent Corso operations from mistakenly clobbering storage configs during runtime.
+
+### Fixed
+- Handle OneDrive folders being deleted and recreated midway through a backup.
+- Automatically re-run a full delta query on incremental if the prior backup is found to have malformed prior-state information.
+- Retry drive item permission downloads during long-running backups after the jwt token expires and refreshes.
+- Retry item downloads during connection timeouts.
+
+## [v0.15.0] (beta) - 2023-10-31
+
+### Added
+- Added `corso repo update-passphrase` command to update the passphrase of an existing Corso repository
+- Added Subject and Message preview to channel messages detail entries
+
+### Fixed
+- SharePoint backup would fail if any site had an empty display name
+- Fix a bug with exports hanging post completion
+- Handle 503 errors in nested OneDrive packages
+
+### Changed
+- Item Details formatting in Groups and Teams backups
+
+## [v0.14.2] (beta) - 2023-10-17
+
+### Added
+- Skips graph calls for expired item download URLs.
+- Export operation now shows the stats at the end of the run
+
+### Fixed
+- Catch and report cases where a protected resource is locked out of access.  SDK consumers have a new errs sentinel that allows them to check for this case.
+- Fix a case where missing item LastModifiedTimes could cause incremental backups to fail.
+- Email size metadata was incorrectly set to the size of the last attachment.  Emails will now correctly report the size of the mail content plus the size of all attachments.
+- Improves the filtering capabilities for Groups restore and backup
+- Improve check to skip OneNote files that cannot be downloaded.
+- Fix Groups backup for non Team groups
+
+### Changed
+- Groups restore now expects the site whose backup we should restore
+
+## [v0.14.0] (beta) - 2023-10-09
+
+### Added
+- Enables local or network-attached storage for Corso repositories.
+- Reduce backup runtime for OneDrive and SharePoint incremental backups that have no file changes.
+- Increase Exchange backup performance by lazily fetching data only for items whose content changed.
+- Added `--backups` flag to delete multiple backups in `corso backup delete` command.
+- Backup now includes all sites that belongs to a team, not just the root site.
+
+### Fixed
+- Teams Channels that cannot support delta tokens (those without messages) fall back to non-delta enumeration and no longer fail a backup.
+
+### Known issues
+- Restoring the data into a different Group from the one it was backed up from is not currently supported
+
+### Other
+- Groups and Teams service support is still in feature preview
+
+## [v0.13.0] (beta) - 2023-09-18
+
+### Added
+- Groups and Teams service support available as a feature preview!  Channel messages and Files are now available for backup and restore in the CLI: `corso backup create groups --group '*'`
+  - The cli commands for "groups" and "teams" can be used interchangeably, and will operate on the same backup data.
+  - New permissions are required to backup Channel messages.  See the [Corso Documentation](https://corsobackup.io/docs/setup/m365-access/#configure-required-permissions) for complete details.
+  Even though Channel message restoration is not available, message write permissions are included to cover future integration.
+  - This is a feature preview, and may be subject to breaking changes based on feedback and testing.
+
+### Changed
+- Switched to Go 1.21
+- SharePoint exported libraries are now exported with a `Libraries` prefix.
+
+### Fixed
+- Contacts backups no longer slices root-folder data if outlook is set to languages other than english.
+- Failed backups if the --disable-incrementals flag was passed when there was a valid merge base under some conditions.
+
+## [v0.12.0] (beta) - 2023-08-29
+
+### Added
+- Added `export` command to export data from OneDrive and SharePoint backups as individual files or as a single zip file.
+- Restore commands now accept an optional resource override with the `--to-resource` flag.  This allows restores to recreate backup data within different mailboxes, sites, and users.  
+- Improve `--mask-sensitive-data` logging mode.
+- Reliability: Handle connection cancellation and resets observed when backing up or restoring large data sets.
+- Reliability: Recover from Graph SDK panics when the Graph API returns incomplete responses.
+- Performance: Improve backup delete performance by batching multiple storage operations into a single operation.
+
+### Fixed
+- SharePoint document libraries deleted after the last backup can now be restored.
+- Restore requires the protected resource to have access to the service being restored.
+- SharePoint data from multiple document libraries are not merged in exports
+- `corso backup delete` was not removing the backup details data associated with that snapshot
+- Fix OneDrive restores could fail with a concurrent map write error
+- Fix backup list displaying backups that had errors
+- Fix OneDrive backup could fail if item was deleted during backup
+- Exchange backups would fail attempting to use delta tokens even if the user was over quota
+
+
+## [v0.11.1] (beta) - 2023-07-20
+
+### Fixed
+- Allow repo connect to succeed when a `corso.toml` file was not provided but configuration is specified using environment variables and flags.
+
+## [v0.11.0] (beta) - 2023-07-18
+
+### Added
+- Drive items backup and restore link shares
+- Restore commands now accept an optional top-level restore destination with the `--destination` flag.  Setting the destination to '/' will restore items back into their original location.  
+- Restore commands can specify item collision behavior.  Options are Skip (default), Replace, and Copy.
+- Introduced repository maintenance commands to help optimize the repository as well as unreferenced data.
 
 ### Fixed
 - Return a ServiceNotEnabled error when a tenant has no active SharePoint license.
+- Added retries for http/2 stream connection failures when downloading large item content.
+- SharePoint document libraries that were deleted after the last backup can now be restored.
+
+### Known issues
+- If a link share is created for an item with inheritance disabled
+  (via the Graph API), the link shares restored in that item will
+  not be inheritable by children
+- Link shares with password protection can't be restored
 
 ## [v0.10.0] (beta) - 2023-06-26
 
@@ -24,6 +214,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Handle OLE conversion errors when trying to fetch attachments
 - Fix uploading large attachments for emails and calendar
 - Fixed high memory use in OneDrive backup related to logging
+- Return a ServiceNotEnabled error when a tenant has no active SharePoint license.
 
 ### Changed
 - Switched to Go 1.20
@@ -311,7 +502,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Miscellaneous
   - Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35))
 
-[Unreleased]: https://github.com/alcionai/corso/compare/v0.10.0...HEAD
+[Unreleased]: https://github.com/alcionai/corso/compare/v0.19.0...HEAD
+[v0.19.0]: https://github.com/alcionai/corso/compare/v0.18.0...v0.19.0
+[v0.18.0]: https://github.com/alcionai/corso/compare/v0.17.0...v0.18.0
+[v0.17.0]: https://github.com/alcionai/corso/compare/v0.16.0...v0.17.0
+[v0.16.0]: https://github.com/alcionai/corso/compare/v0.15.0...v0.16.0
+[v0.15.0]: https://github.com/alcionai/corso/compare/v0.14.0...v0.15.0
+[v0.14.0]: https://github.com/alcionai/corso/compare/v0.13.0...v0.14.0
+[v0.13.0]: https://github.com/alcionai/corso/compare/v0.12.0...v0.13.0
+[v0.12.0]: https://github.com/alcionai/corso/compare/v0.11.1...v0.12.0
+[v0.11.1]: https://github.com/alcionai/corso/compare/v0.11.0...v0.11.1
+[v0.11.0]: https://github.com/alcionai/corso/compare/v0.10.0...v0.11.0
 [v0.10.0]: https://github.com/alcionai/corso/compare/v0.9.0...v0.10.0
 [v0.9.0]: https://github.com/alcionai/corso/compare/v0.8.1...v0.9.0
 [v0.8.0]: https://github.com/alcionai/corso/compare/v0.7.1...v0.8.0

README.md

@@ -1,3 +1,6 @@
+> [!NOTE] 
+> **The Corso project is no longer actively maintained and has been archived**.
+
 <p align="center">
     <img src="https://github.com/alcionai/corso/blob/main/website/static/img/corso_logo.svg?raw=true" alt="Corso Logo" width="100" />
 </p>

build/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19-alpine as builder
+FROM golang:1.21-alpine as builder
 
 WORKDIR /go/src/app
 COPY src .

build/build.sh

@@ -12,7 +12,7 @@ usage() {
 }
 
 ROOT=$(dirname $(dirname $(readlink -f $0)))
-GOVER=1.19                           # go version
+GOVER=1.21                           # go version
 CORSO_BUILD_CACHE="/tmp/.corsobuild" # shared persistent cache
 
 # Figure out os and architecture

src/.golangci.yml

@@ -3,14 +3,21 @@ run:
 
 linters:
   enable:
+    - errcheck
+    - exhaustive
+    - forbidigo
     - gci
     - gofmt
     - gofumpt
-    - errcheck
-    - forbidigo
+    - gosimple
+    - govet
+    - ineffassign
     - lll
+    - loggercheck
     - misspell
     - revive
+    - unused
+    - usestdlibvars
     - wsl
 
   disable:
@@ -19,6 +26,11 @@ linters:
     - staticcheck
 
 linters-settings:
+  exhaustive:
+    check:
+      - switch
+    default-signifies-exhaustive: false
+    explicit-exhaustive-switch: true
   gci:
     sections:
       - standard
@@ -43,10 +55,13 @@ linters-settings:
       # String formatting should be avoided in favor of structured errors (ie: err.With(k, v)).
       - '(errors|fmt)\.(New|Stack|Wrap|Error)f?\((# error handling should use clues pkg)?'
       # Avoid Warn-level logging in favor of Info or Error.
-      - 'Warn[wf]?\((# logging should use Info or Error)?'
+      - 'Warnw?f?\((# logging should use Info or Error)?'
       # Prefer suite.Run(name, func() {}) for subtests as testify has it instead
       # of suite.T().Run(name, func(t *testing.T) {}).
       - '(T\(\)|\st[a-zA-Z0-9]*)\.Run(# prefer testify suite.Run(name, func()) )?'
+      # Prefer packing ctx values into the error using NewWC, WrapWC, or StackWC
+      # instead of New|Stack|Wrap().WithClues(ctx)
+      - 'WithClues(# prefer the builderWC variant - ex: StackWC(ctx, ...))?'
   lll:
     line-length: 120
   revive:
@@ -119,7 +134,13 @@ issues:
       linters:
         - forbidigo
       text: "context.(Background|TODO)"
-    - path: internal/m365/graph/betasdk
+    - path: internal/m365/collection/drive/collections_test.go
+      linters:
+        - lll
+    - path: internal/m365/collection/drive/collections_tree_test.go
+      linters:
+        - lll
+    - path: pkg/services/m365/api/graph/betasdk
       linters:
         - wsl
         - revive

src/.goreleaser.yaml

@@ -18,13 +18,15 @@ builds:
           - -X 'github.com/alcionai/corso/src/internal/events.RudderStackDataPlaneURL={{.Env.RUDDERSTACK_CORSO_DATA_PLANE_URL}}'
 
 archives:
-  - name_template: "{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}"
-    replacements:
-      darwin: Darwin
-      linux: Linux
-      windows: Windows
-      386: i386
-      amd64: x86_64
+  # this name template makes the OS and Arch compatible with the results of uname.
+  - name_template: >-
+      {{ .ProjectName }}_
+      {{- .Tag }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
     format: tar.gz
     format_overrides:
       - goos: windows

src/.gotestfmt/github/package.gotpl

@@ -0,0 +1,56 @@
+{{- /*gotype: github.com/gotesttools/gotestfmt/v2/parser.Package*/ -}}
+{{- /*
+This template contains the format for an individual package. GitHub actions does not currently support nested groups so
+we are creating a stylized header for each package.
+*/ -}}
+{{- $settings := .Settings -}}
+{{- if and (or (not $settings.HideSuccessfulPackages) (ne .Result "PASS")) (or (not $settings.HideEmptyPackages) (ne .Result "SKIP") (ne (len .TestCases) 0)) -}}
+    {{- if eq .Result "PASS" -}}
+        {{ "\033" }}[0;32m
+    {{- else if eq .Result "SKIP" -}}
+        {{ "\033" }}[0;33m
+    {{- else -}}
+        {{ "\033" }}[0;31m
+    {{- end -}}
+    📦 {{ .Name }}{{- "\033" }}[0m
+    {{- with .Coverage -}}
+       {{- "\033" -}}[0;37m ({{ . }}% coverage){{- "\033" -}}[0m
+    {{- end -}}
+    {{- " " -}}({{- .Duration -}})
+    {{- "\n" -}}
+    {{- with .Reason -}}
+        {{- "  " -}}🛑 {{ . -}}{{- "\n" -}}
+    {{- end -}}
+    {{- with .Output -}}
+        {{- . -}}{{- "\n" -}}
+    {{- end -}}
+    {{- with .TestCases -}}
+        {{- range . -}}
+            {{- if or (not $settings.HideSuccessfulTests) (ne .Result "PASS") -}}
+                ::group::
+                {{- if eq .Result "PASS" -}}
+                    {{ "\033" }}[0;32m✅
+                {{- else if eq .Result "SKIP" -}}
+                    {{ "\033" }}[0;33m🚧
+                {{- else -}}
+                    {{ "\033" }}[0;31m❌
+                {{- end -}}
+                {{ " " }}{{- .Name -}}
+                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
+                {{- with .Coverage -}}
+                    , coverage: {{ . }}%
+                {{- end -}})
+                {{- "\033" -}}[0m
+                {{- "\n" -}}
+
+                {{- with .Output -}}
+                    {{- formatTestOutput . $settings -}}
+                    {{- "\n" -}}
+                {{- end -}}
+
+                ::endgroup::{{- "\n" -}}
+            {{- end -}}
+        {{- end -}}
+    {{- end -}}
+    {{- "\n" -}}
+{{- end -}}

src/Makefile

@@ -1,5 +1,5 @@
 # This must match the version defined in .github/workflows/lint.yaml.
-WANTED_LINT_VERSION := 1.52.2
+WANTED_LINT_VERSION := 1.54.2
 LINT_VERSION := $(shell golangci-lint version | cut -d' ' -f4)
 HAS_LINT := $(shell which golangci-lint)
 
@@ -18,7 +18,7 @@ lint: check-lint-version
 fmt:
 	gofumpt -w .
 	goimports -w .
-	gci write --skip-generated -s 'standard,default,prefix(github.com/alcionai/corso)' .
+	gci write --skip-generated -s 'standard' -s 'default' -s 'prefix(github.com/alcionai/corso)' .
 
 check-lint-version: check-lint
 	@if [ "$(LINT_VERSION)" != "$(WANTED_LINT_VERSION)" ]; then \

src/cli/backup/backup.go

@@ -11,12 +11,15 @@ import (
 
 	"github.com/alcionai/corso/src/cli/flags"
 	. "github.com/alcionai/corso/src/cli/print"
-	"github.com/alcionai/corso/src/cli/repo"
 	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/common/color"
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/data"
-	"github.com/alcionai/corso/src/internal/m365/graph"
+	"github.com/alcionai/corso/src/internal/observe"
 	"github.com/alcionai/corso/src/pkg/backup"
+	"github.com/alcionai/corso/src/pkg/backup/details"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/errs/core"
 	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/repository"
@@ -24,6 +27,8 @@ import (
 	"github.com/alcionai/corso/src/pkg/store"
 )
 
+var ErrEmptyBackup = clues.New("no items in backup")
+
 // ---------------------------------------------------------------------------
 // adding commands to cobra
 // ---------------------------------------------------------------------------
@@ -39,6 +44,8 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{
 	addExchangeCommands,
 	addOneDriveCommands,
 	addSharePointCommands,
+	addGroupsCommands,
+	addTeamsChatsCommands,
 }
 
 // AddCommands attaches all `corso backup * *` commands to the parent.
@@ -51,33 +58,13 @@ func AddCommands(cmd *cobra.Command) {
 		backupC.AddCommand(subCommand)
 
 		for _, addBackupTo := range serviceCommands {
-			addBackupTo(subCommand)
+			sc := addBackupTo(subCommand)
+			flags.AddAllProviderFlags(sc)
+			flags.AddAllStorageFlags(sc)
 		}
 	}
 }
 
-// ---------------------------------------------------------------------------
-// common flags and flag attachers for commands
-// ---------------------------------------------------------------------------
-
-func addFailedItemsFN(cmd *cobra.Command) {
-	cmd.Flags().StringVar(
-		&flags.ListFailedItemsFV, flags.FailedItemsFN, "show",
-		"Toggles showing or hiding the list of items that failed.")
-}
-
-func addSkippedItemsFN(cmd *cobra.Command) {
-	cmd.Flags().StringVar(
-		&flags.ListSkippedItemsFV, flags.SkippedItemsFN, "show",
-		"Toggles showing or hiding the list of items that were skipped.")
-}
-
-func addRecoveredErrorsFN(cmd *cobra.Command) {
-	cmd.Flags().StringVar(
-		&flags.ListRecoveredErrorsFV, flags.RecoveredErrorsFN, "show",
-		"Toggles showing or hiding the list of errors which corso recovered from.")
-}
-
 // ---------------------------------------------------------------------------
 // commands
 // ---------------------------------------------------------------------------
@@ -183,10 +170,10 @@ func handleDeleteCmd(cmd *cobra.Command, args []string) error {
 // standard set of selector behavior that we want used in the cli
 var defaultSelectorConfig = selectors.Config{OnlyMatchItemNames: true}
 
-func runBackups(
+func genericCreateCommand(
 	ctx context.Context,
-	r repository.Repository,
-	serviceName, resourceOwnerType string,
+	r repository.Repositoryer,
+	serviceName string,
 	selectorSet []selectors.Selector,
 	ins idname.Cacher,
 ) error {
@@ -203,29 +190,47 @@ func runBackups(
 			ictx  = clues.Add(ctx, "resource_owner_selected", owner)
 		)
 
+		logger.Ctx(ictx).Infof("setting up backup")
+
 		bo, err := r.NewBackupWithLookup(ictx, discSel, ins)
 		if err != nil {
-			errs = append(errs, clues.Wrap(err, owner).WithClues(ictx))
-			Errf(ictx, "%v\n", err)
+			cerr := clues.WrapWC(ictx, err, owner)
+			errs = append(errs, cerr)
+
+			Errf(
+				ictx,
+				"%s\nCause: %s",
+				"Unable to initiate backup",
+				err.Error())
 
 			continue
 		}
 
 		ictx = clues.Add(
-			ctx,
+			ictx,
 			"resource_owner_id", bo.ResourceOwner.ID(),
-			"resource_owner_name", bo.ResourceOwner.Name())
+			"resource_owner_name", clues.Hide(bo.ResourceOwner.Name()))
+
+		logger.Ctx(ictx).Infof("running backup")
 
 		err = bo.Run(ictx)
 		if err != nil {
-			if errors.Is(err, graph.ErrServiceNotEnabled) {
-				logger.Ctx(ctx).Infow("service not enabled", "resource_owner_name", bo.ResourceOwner.Name())
+			if errors.Is(err, core.ErrServiceNotEnabled) {
+				logger.Ctx(ictx).Infow("service not enabled",
+					"resource_owner_id", bo.ResourceOwner.ID(),
+					"service", serviceName)
 
 				continue
 			}
 
-			errs = append(errs, clues.Wrap(err, owner).WithClues(ictx))
-			Errf(ictx, "%v\n", err)
+			cerr := clues.Wrap(err, owner)
+			errs = append(errs, cerr)
+
+			Errf(
+				ictx,
+				"%s\nCause: %s",
+				"Unable to complete backup",
+				err.Error())
 
 			continue
 		}
@@ -233,10 +238,10 @@ func runBackups(
 		bIDs = append(bIDs, string(bo.Results.BackupID))
 
 		if !DisplayJSONFormat() {
-			Infof(ctx, "Done\n")
-			printBackupStats(ctx, r, string(bo.Results.BackupID))
+			Infof(ictx, fmt.Sprintf("Backup complete %s %s", observe.Bullet, color.BlueOutput(bo.Results.BackupID)))
+			printBackupStats(ictx, r, string(bo.Results.BackupID))
 		} else {
-			Infof(ctx, "Done - ID: %v\n", bo.Results.BackupID)
+			Infof(ictx, "Backup complete - ID: %v\n", bo.Results.BackupID)
 		}
 	}
 
@@ -245,8 +250,10 @@ func runBackups(
 		return Only(ctx, clues.Wrap(berrs.Failure(), "Unable to retrieve backup results from storage"))
 	}
 
-	Info(ctx, "Completed Backups:")
-	backup.PrintAll(ctx, bups)
+	if len(bups) > 0 {
+		Info(ctx, "\nCompleted Backups:")
+		backup.PrintAll(ctx, bups)
+	}
 
 	if len(errs) > 0 {
 		sb := fmt.Sprintf("%d of %d backups failed:\n", len(errs), len(selectorSet))
@@ -264,35 +271,53 @@ func runBackups(
 
 // genericDeleteCommand is a helper function that all services can use
 // for the removal of an entry from the repository
-func genericDeleteCommand(cmd *cobra.Command, bID, designation string, args []string) error {
+func genericDeleteCommand(
+	cmd *cobra.Command,
+	pst path.ServiceType,
+	designation string,
+	bID, args []string,
+) error {
 	if utils.HasNoFlagsAndShownHelp(cmd) {
 		return nil
 	}
 
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
 	ctx := clues.Add(cmd.Context(), "delete_backup_id", bID)
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
+	r, _, err := utils.GetAccountAndConnect(ctx, cmd, pst)
 	if err != nil {
 		return Only(ctx, err)
 	}
 
 	defer utils.CloseRepo(ctx, r)
 
-	if err := r.DeleteBackup(ctx, bID); err != nil {
-		return Only(ctx, clues.Wrap(err, "Deleting backup "+bID))
+	if err := r.DeleteBackups(ctx, true, bID...); err != nil {
+		return Only(ctx, clues.Wrap(err, fmt.Sprintf("Deleting backup %v", bID)))
 	}
 
-	Infof(ctx, "Deleted %s backup %s", designation, bID)
+	Infof(ctx, "Deleted %s backup %v", designation, bID)
 
 	return nil
 }
 
 // genericListCommand is a helper function that all services can use
 // to display the backup IDs saved within the repository
-func genericListCommand(cmd *cobra.Command, bID string, service path.ServiceType, args []string) error {
+func genericListCommand(
+	cmd *cobra.Command,
+	bID string,
+	service path.ServiceType,
+	args []string,
+) error {
 	ctx := cmd.Context()
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	r, _, err := utils.GetAccountAndConnect(ctx, cmd, service)
 	if err != nil {
 		return Only(ctx, err)
 	}
@@ -312,7 +337,8 @@ func genericListCommand(cmd *cobra.Command, bID string, service path.ServiceType
 		b.Print(ctx)
 		fe.PrintItems(
 			ctx,
-			!ifShow(flags.ListFailedItemsFV),
+			!ifShow(flags.ListAlertsFV),
+			!ifShow(flags.FailedItemsFV),
 			!ifShow(flags.ListSkippedItemsFV),
 			!ifShow(flags.ListRecoveredErrorsFV))
 
@@ -329,16 +355,78 @@ func genericListCommand(cmd *cobra.Command, bID string, service path.ServiceType
 	return nil
 }
 
+func genericDetailsCommand(
+	cmd *cobra.Command,
+	backupID string,
+	sel selectors.Selector,
+) (*details.Details, error) {
+	ctx := cmd.Context()
+
+	r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.OneDriveService)
+	if err != nil {
+		return nil, clues.Stack(err)
+	}
+
+	defer utils.CloseRepo(ctx, r)
+
+	return genericDetailsCore(
+		ctx,
+		r,
+		backupID,
+		sel,
+		rdao.Opts)
+}
+
+func genericDetailsCore(
+	ctx context.Context,
+	bg repository.BackupGetter,
+	backupID string,
+	sel selectors.Selector,
+	opts control.Options,
+) (*details.Details, error) {
+	ctx = clues.Add(ctx, "backup_id", backupID)
+
+	sel.Configure(selectors.Config{OnlyMatchItemNames: true})
+
+	d, _, errs := bg.GetBackupDetails(ctx, backupID)
+	// TODO: log/track recoverable errors
+	if errs.Failure() != nil {
+		if errors.Is(errs.Failure(), data.ErrNotFound) {
+			return nil, clues.New("no backup exists with the id " + backupID)
+		}
+
+		return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
+	}
+
+	if len(d.Entries) == 0 {
+		return nil, ErrEmptyBackup
+	}
+
+	if opts.SkipReduce {
+		return d, nil
+	}
+
+	d, err := sel.Reduce(ctx, d, errs)
+	if err != nil {
+		return nil, clues.Wrap(err, "filtering backup details to selection")
+	}
+
+	return d, nil
+}
+
+// ---------------------------------------------------------------------------
+// helper funcs
+// ---------------------------------------------------------------------------
+
 func ifShow(flag string) bool {
 	return strings.ToLower(strings.TrimSpace(flag)) == "show"
 }
 
-func printBackupStats(ctx context.Context, r repository.Repository, bid string) {
+func printBackupStats(ctx context.Context, r repository.Repositoryer, bid string) {
 	b, err := r.Backup(ctx, bid)
 	if err != nil {
 		logger.CtxErr(ctx, err).Error("finding backup immediately after backup operation completion")
 	}
 
-	b.ToPrintable().Stats.Print(ctx)
-	Info(ctx, " ")
+	b.ToPrintable().Stats.PrintProperties(ctx)
 }

src/cli/backup/backup_test.go

@@ -0,0 +1,97 @@
+package backup
+
+import (
+	"testing"
+
+	"github.com/alcionai/clues"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/utils/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/backup/details"
+	dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+type BackupUnitSuite struct {
+	tester.Suite
+}
+
+func TestBackupUnitSuite(t *testing.T) {
+	suite.Run(t, &BackupUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *BackupUnitSuite) TestGenericDetailsCore() {
+	t := suite.T()
+
+	expected := append(
+		append(
+			dtd.GetItemsForVersion(
+				t,
+				path.ExchangeService,
+				path.EmailCategory,
+				0,
+				-1),
+			dtd.GetItemsForVersion(
+				t,
+				path.ExchangeService,
+				path.EventsCategory,
+				0,
+				-1)...),
+		dtd.GetItemsForVersion(
+			t,
+			path.ExchangeService,
+			path.ContactsCategory,
+			0,
+			-1)...)
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	bg := testdata.VersionedBackupGetter{
+		Details: dtd.GetDetailsSetForVersion(t, 0),
+	}
+
+	sel := selectors.NewExchangeBackup([]string{"user-id"})
+	sel.Include(sel.AllData())
+
+	output, err := genericDetailsCore(
+		ctx,
+		bg,
+		"backup-ID",
+		sel.Selector,
+		control.DefaultOptions())
+	assert.NoError(t, err, clues.ToCore(err))
+	assert.ElementsMatch(t, expected, output.Entries)
+}
+
+func (suite *BackupUnitSuite) TestGenericDetailsCore_empty() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	bg := testdata.VersionedBackupGetter{
+		Details: &details.Details{
+			DetailsModel: details.DetailsModel{
+				Entries: []details.Entry{},
+			},
+		},
+	}
+
+	sel := selectors.NewExchangeBackup([]string{"user-id"})
+	sel.Include(sel.AllData())
+
+	_, err := genericDetailsCore(
+		ctx,
+		bg,
+		"backup-ID",
+		sel.Selector,
+		control.DefaultOptions())
+	require.Error(t, err, "has error")
+	assert.ErrorIs(t, err, ErrEmptyBackup, clues.ToCore(err))
+}

src/cli/backup/exchange.go

@@ -1,22 +1,14 @@
 package backup
 
 import (
-	"context"
-
 	"github.com/alcionai/clues"
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 
 	"github.com/alcionai/corso/src/cli/flags"
 	. "github.com/alcionai/corso/src/cli/print"
-	"github.com/alcionai/corso/src/cli/repo"
 	"github.com/alcionai/corso/src/cli/utils"
-	"github.com/alcionai/corso/src/internal/data"
-	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/fault"
 	"github.com/alcionai/corso/src/pkg/path"
-	"github.com/alcionai/corso/src/pkg/repository"
 	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
@@ -33,7 +25,7 @@ const (
 const (
 	exchangeServiceCommand                 = "exchange"
 	exchangeServiceCommandCreateUseSuffix  = "--mailbox <email> | '" + flags.Wildcard + "'"
-	exchangeServiceCommandDeleteUseSuffix  = "--backup <backupId>"
+	exchangeServiceCommandDeleteUseSuffix  = "--backups <backupId>"
 	exchangeServiceCommandDetailsUseSuffix = "--backup <backupId>"
 )
 
@@ -47,8 +39,9 @@ corso backup create exchange --mailbox alice@example.com,bob@example.com --data
 # Backup all Exchange data for all M365 users 
 corso backup create exchange --mailbox '*'`
 
-	exchangeServiceCommandDeleteExamples = `# Delete Exchange backup with ID 1234abcd-12ab-cd34-56de-1234abcd
-corso backup delete exchange --backup 1234abcd-12ab-cd34-56de-1234abcd`
+	exchangeServiceCommandDeleteExamples = `# Delete Exchange backup with IDs 1234abcd-12ab-cd34-56de-1234abcd \
+and 1234abcd-12ab-cd34-56de-1234abce
+corso backup delete exchange --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce`
 
 	exchangeServiceCommandDetailsExamples = `# Explore items in Alice's latest backup (1234abcd...)
 corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd
@@ -68,15 +61,11 @@ corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \
 
 // called by backup.go to map subcommands to provider-specific handling.
 func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
-	var (
-		c  *cobra.Command
-		fs *pflag.FlagSet
-	)
+	var c *cobra.Command
 
 	switch cmd.Use {
 	case createCommand:
-		c, fs = utils.AddCommand(cmd, exchangeCreateCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, exchangeCreateCmd())
 
 		c.Use = c.Use + " " + exchangeServiceCommandCreateUseSuffix
 		c.Example = exchangeServiceCommandCreateExamples
@@ -85,31 +74,21 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
 		// More generic (ex: --user) and more frequently used flags take precedence.
 		flags.AddMailBoxFlag(c)
 		flags.AddDataFlag(c, []string{dataEmail, dataContacts, dataEvents}, false)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
 		flags.AddFetchParallelismFlag(c)
-		flags.AddFailFastFlag(c)
-		flags.AddDisableIncrementalsFlag(c)
 		flags.AddDisableDeltaFlag(c)
 		flags.AddEnableImmutableIDFlag(c)
-		flags.AddDisableConcurrencyLimiterFlag(c)
+		flags.AddDeltaPageSizeFlag(c)
+		flags.AddGenericBackupFlags(c)
+		flags.AddDisableSlidingWindowLimiterFlag(c)
 
 	case listCommand:
-		c, fs = utils.AddCommand(cmd, exchangeListCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, exchangeListCmd())
 
 		flags.AddBackupIDFlag(c, false)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
-		addFailedItemsFN(c)
-		addSkippedItemsFN(c)
-		addRecoveredErrorsFN(c)
+		flags.AddAllBackupListFlags(c)
 
 	case detailsCommand:
-		c, fs = utils.AddCommand(cmd, exchangeDetailsCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, exchangeDetailsCmd())
 
 		c.Use = c.Use + " " + exchangeServiceCommandDetailsUseSuffix
 		c.Example = exchangeServiceCommandDetailsExamples
@@ -119,22 +98,16 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
 		// Flags addition ordering should follow the order we want them to appear in help and docs:
 		// More generic (ex: --user) and more frequently used flags take precedence.
 		flags.AddBackupIDFlag(c, true)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
-		flags.AddExchangeDetailsAndRestoreFlags(c)
+		flags.AddExchangeDetailsAndRestoreFlags(c, false)
 
 	case deleteCommand:
-		c, fs = utils.AddCommand(cmd, exchangeDeleteCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, exchangeDeleteCmd())
 
 		c.Use = c.Use + " " + exchangeServiceCommandDeleteUseSuffix
 		c.Example = exchangeServiceCommandDeleteExamples
 
-		flags.AddBackupIDFlag(c, true)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
+		flags.AddMultipleBackupIDsFlag(c, false)
+		flags.AddBackupIDFlag(c, false)
 	}
 
 	return c
@@ -162,11 +135,18 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
 	if err := validateExchangeBackupCreateFlags(flags.UserFV, flags.CategoryDataFV); err != nil {
 		return err
 	}
 
-	r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
+	r, acct, err := utils.AccountConnectAndWriteRepoConfig(
+		ctx,
+		cmd,
+		path.ExchangeService)
 	if err != nil {
 		return Only(ctx, err)
 	}
@@ -175,7 +155,12 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
 
 	sel := exchangeBackupCreateSelectors(flags.UserFV, flags.CategoryDataFV)
 
-	ins, err := utils.UsersMap(ctx, *acct, fault.New(true))
+	ins, err := utils.UsersMap(
+		ctx,
+		*acct,
+		utils.Control(),
+		r.Counter(),
+		fault.New(true))
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
 	}
@@ -186,10 +171,10 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
 		selectorSet = append(selectorSet, discSel.Selector)
 	}
 
-	return runBackups(
+	return genericCreateCommand(
 		ctx,
 		r,
-		"Exchange", "user",
+		"Exchange",
 		selectorSet,
 		ins)
 }
@@ -272,69 +257,33 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	ctx := cmd.Context()
-	opts := utils.MakeExchangeOpts(cmd)
-
-	r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
-	if err != nil {
-		return Only(ctx, err)
-	}
-
-	defer utils.CloseRepo(ctx, r)
-
-	ctrlOpts := utils.Control()
-
-	ds, err := runDetailsExchangeCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
-	if err != nil {
-		return Only(ctx, err)
-	}
-
-	if len(ds.Entries) == 0 {
-		Info(ctx, selectors.ErrorNoMatchingItems)
+	if flags.RunModeFV == flags.RunModeFlagTest {
 		return nil
 	}
 
-	ds.PrintEntries(ctx)
-
-	return nil
+	return runDetailsExchangeCmd(cmd)
 }
 
-// runDetailsExchangeCmd actually performs the lookup in backup details.
-// the fault.Errors return is always non-nil.  Callers should check if
-// errs.Failure() == nil.
-func runDetailsExchangeCmd(
-	ctx context.Context,
-	r repository.BackupGetter,
-	backupID string,
-	opts utils.ExchangeOpts,
-	skipReduce bool,
-) (*details.Details, error) {
-	if err := utils.ValidateExchangeRestoreFlags(backupID, opts); err != nil {
-		return nil, err
+func runDetailsExchangeCmd(cmd *cobra.Command) error {
+	ctx := cmd.Context()
+	opts := utils.MakeExchangeOpts(cmd)
+
+	sel := utils.IncludeExchangeRestoreDataSelectors(opts)
+	sel.Configure(selectors.Config{OnlyMatchItemNames: true})
+	utils.FilterExchangeRestoreInfoSelectors(sel, opts)
+
+	ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
+	if err != nil {
+		return Only(ctx, err)
 	}
 
-	ctx = clues.Add(ctx, "backup_id", backupID)
-
-	d, _, errs := r.GetBackupDetails(ctx, backupID)
-	// TODO: log/track recoverable errors
-	if errs.Failure() != nil {
-		if errors.Is(errs.Failure(), data.ErrNotFound) {
-			return nil, clues.New("No backup exists with the id " + backupID)
-		}
-
-		return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
+	if len(ds.Entries) > 0 {
+		ds.PrintEntries(ctx)
+	} else {
+		Info(ctx, selectors.ErrorNoMatchingItems)
 	}
 
-	ctx = clues.Add(ctx, "details_entries", len(d.Entries))
-
-	if !skipReduce {
-		sel := utils.IncludeExchangeRestoreDataSelectors(opts)
-		sel.Configure(selectors.Config{OnlyMatchItemNames: true})
-		utils.FilterExchangeRestoreInfoSelectors(sel, opts)
-		d = sel.Reduce(ctx, d, errs)
-	}
-
-	return d, nil
+	return nil
 }
 
 // ------------------------------------------------------------------------------------------------
@@ -353,5 +302,15 @@ func exchangeDeleteCmd() *cobra.Command {
 
 // deletes an exchange service backup.
 func deleteExchangeCmd(cmd *cobra.Command, args []string) error {
-	return genericDeleteCommand(cmd, flags.BackupIDFV, "Exchange", args)
+	var backupIDValue []string
+
+	if len(flags.BackupIDsFV) > 0 {
+		backupIDValue = flags.BackupIDsFV
+	} else if len(flags.BackupIDFV) > 0 {
+		backupIDValue = append(backupIDValue, flags.BackupIDFV)
+	} else {
+		return clues.New("either --backup or --backups flag is required")
+	}
+
+	return genericDeleteCommand(cmd, path.ExchangeService, "Exchange", backupIDValue, args)
 }

src/cli/backup/exchange_e2e_test.go

@@ -1,32 +1,30 @@
 package backup_test
 
 import (
-	"context"
 	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/alcionai/clues"
 	"github.com/google/uuid"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/cli"
-	"github.com/alcionai/corso/src/cli/config"
 	"github.com/alcionai/corso/src/cli/flags"
 	"github.com/alcionai/corso/src/cli/print"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
 	"github.com/alcionai/corso/src/internal/common/idname"
-	"github.com/alcionai/corso/src/internal/m365/exchange"
 	"github.com/alcionai/corso/src/internal/operations"
 	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/internal/tester/its"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/config"
 	"github.com/alcionai/corso/src/pkg/path"
-	"github.com/alcionai/corso/src/pkg/repository"
 	"github.com/alcionai/corso/src/pkg/selectors"
-	"github.com/alcionai/corso/src/pkg/storage"
+	"github.com/alcionai/corso/src/pkg/services/m365/api"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
 )
 
 var (
@@ -36,203 +34,19 @@ var (
 )
 
 // ---------------------------------------------------------------------------
-// tests with azure flags in exchange create
-// ---------------------------------------------------------------------------
-
-type ExchangeCMDWithFlagsE2ESuite struct {
-	tester.Suite
-	acct       account.Account
-	st         storage.Storage
-	vpr        *viper.Viper
-	cfgFP      string
-	repo       repository.Repository
-	m365UserID string
-	recorder   strings.Builder
-}
-
-func TestExchangeCMDWithFlagsE2ESuite(t *testing.T) {
-	suite.Run(t, &ExchangeCMDWithFlagsE2ESuite{Suite: tester.NewE2ESuite(
-		t,
-		[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-	)})
-}
-
-func (suite *ExchangeCMDWithFlagsE2ESuite) SetupSuite() {
-	t := suite.T()
-
-	ctx, flush := tester.NewContext(t)
-	defer flush()
-
-	acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.vpr = vpr
-	suite.recorder = recorder
-	suite.cfgFP = cfgFilePath
-	suite.m365UserID = tester.M365UserID(t)
-}
-
-func (suite *ExchangeCMDWithFlagsE2ESuite) TestBackupCreateExchange_badAzureClientID() {
-	t := suite.T()
-	ctx, flush := tester.NewContext(t)
-
-	defer flush()
-
-	suite.recorder.Reset()
-
-	cmd := tester.StubRootCmd(
-		"backup", "create", "exchange",
-		"--user", suite.m365UserID,
-		"--azure-client-id", "invalid-value",
-	)
-	cli.BuildCommandTree(cmd)
-
-	cmd.SetErr(&suite.recorder)
-
-	ctx = print.SetRootCmd(ctx, cmd)
-
-	// run the command
-	err := cmd.ExecuteContext(ctx)
-	require.Error(t, err, clues.ToCore(err))
-}
-
-func (suite *ExchangeCMDWithFlagsE2ESuite) TestBackupCreateExchange_azureIDFromConfigFile() {
-	t := suite.T()
-	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
-
-	defer flush()
-
-	suite.recorder.Reset()
-
-	cmd := tester.StubRootCmd(
-		"backup", "create", "exchange",
-		"--user", suite.m365UserID,
-		"--config-file", suite.cfgFP)
-	cli.BuildCommandTree(cmd)
-
-	cmd.SetErr(&suite.recorder)
-
-	ctx = print.SetRootCmd(ctx, cmd)
-
-	// run the command
-	err := cmd.ExecuteContext(ctx)
-	require.NoError(t, err, clues.ToCore(err))
-
-	result := suite.recorder.String()
-	t.Log("backup results", result)
-
-	// as an offhand check: the result should contain the m365 user id
-	assert.Contains(t, result, suite.m365UserID)
-}
-
-func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupValueFromEnvCmd_empty() {
-	t := suite.T()
-	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
-
-	defer flush()
-
-	suite.recorder.Reset()
-
-	cmd := tester.StubRootCmd(
-		"backup", "create", "exchange",
-		"--user", suite.m365UserID)
-	cli.BuildCommandTree(cmd)
-
-	cmd.SetErr(&suite.recorder)
-
-	ctx = print.SetRootCmd(ctx, cmd)
-
-	// run the command
-	err := cmd.ExecuteContext(ctx)
-	require.NoError(t, err, clues.ToCore(err))
-
-	result := suite.recorder.String()
-	t.Log("backup results", result)
-
-	// as an offhand check: the result should contain the m365 user id
-	assert.Contains(t, result, suite.m365UserID)
-}
-
-// AWS flags
-func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupInvalidAWSClientIDCmd_empty() {
-	t := suite.T()
-	ctx, flush := tester.NewContext(t)
-
-	defer flush()
-
-	suite.recorder.Reset()
-
-	cmd := tester.StubRootCmd(
-		"backup", "create", "exchange",
-		"--user", suite.m365UserID,
-		"--aws-access-key", "invalid-value",
-		"--aws-secret-access-key", "some-invalid-value",
-	)
-	cli.BuildCommandTree(cmd)
-
-	cmd.SetErr(&suite.recorder)
-
-	ctx = print.SetRootCmd(ctx, cmd)
-
-	// run the command
-	err := cmd.ExecuteContext(ctx)
-	// since invalid aws creds are explicitly set, should see a failure
-	require.Error(t, err, clues.ToCore(err))
-}
-
-func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupAWSValueFromEnvCmd_empty() {
-	t := suite.T()
-	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
-
-	defer flush()
-
-	suite.recorder.Reset()
-
-	cmd := tester.StubRootCmd(
-		"backup", "create", "exchange",
-		"--user", suite.m365UserID)
-	cli.BuildCommandTree(cmd)
-
-	cmd.SetErr(&suite.recorder)
-
-	ctx = print.SetRootCmd(ctx, cmd)
-
-	// run the command
-	err := cmd.ExecuteContext(ctx)
-	require.NoError(t, err, clues.ToCore(err))
-
-	result := suite.recorder.String()
-	t.Log("backup results", result)
-
-	// as an offhand check: the result should contain the m365 user id
-	assert.Contains(t, result, suite.m365UserID)
-}
-
-// ---------------------------------------------------------------------------
-// tests with no backups
+// tests that depend on no backups existing
 // ---------------------------------------------------------------------------
 
 type NoBackupExchangeE2ESuite struct {
 	tester.Suite
-	acct       account.Account
-	st         storage.Storage
-	vpr        *viper.Viper
-	cfgFP      string
-	repo       repository.Repository
-	m365UserID string
-	recorder   strings.Builder
+	dpnd dependencies
+	m365 its.M365IntgTestSetup
 }
 
 func TestNoBackupExchangeE2ESuite(t *testing.T) {
-	suite.Run(t, &NoBackupExchangeE2ESuite{Suite: tester.NewE2ESuite(
+	suite.Run(t, &BackupExchangeE2ESuite{Suite: tester.NewE2ESuite(
 		t,
-		[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-	)})
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
 }
 
 func (suite *NoBackupExchangeE2ESuite) SetupSuite() {
@@ -241,32 +55,25 @@ func (suite *NoBackupExchangeE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.vpr = vpr
-	suite.recorder = recorder
-	suite.cfgFP = cfgFilePath
-	suite.m365UserID = tester.M365UserID(t)
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
 }
 
-func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_empty() {
+func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_noBackups() {
 	t := suite.T()
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "list", "exchange",
-		"--config-file", suite.cfgFP)
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
 	cli.BuildCommandTree(cmd)
 
-	cmd.SetErr(&suite.recorder)
+	cmd.SetErr(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -274,7 +81,7 @@ func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_empty() {
 	err := cmd.ExecuteContext(ctx)
 	require.NoError(t, err, clues.ToCore(err))
 
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 
 	// as an offhand check: the result should contain the m365 user id
 	assert.True(t, strings.HasSuffix(result, "No backups available\n"))
@@ -286,19 +93,14 @@ func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_empty() {
 
 type BackupExchangeE2ESuite struct {
 	tester.Suite
-	acct       account.Account
-	st         storage.Storage
-	vpr        *viper.Viper
-	cfgFP      string
-	repo       repository.Repository
-	m365UserID string
+	dpnd dependencies
+	m365 its.M365IntgTestSetup
 }
 
 func TestBackupExchangeE2ESuite(t *testing.T) {
 	suite.Run(t, &BackupExchangeE2ESuite{Suite: tester.NewE2ESuite(
 		t,
-		[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-	)})
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
 }
 
 func (suite *BackupExchangeE2ESuite) SetupSuite() {
@@ -307,40 +109,39 @@ func (suite *BackupExchangeE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, _, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.vpr = vpr
-	suite.cfgFP = cfgFilePath
-	suite.m365UserID = tester.M365UserID(t)
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
 }
 
 func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_email() {
-	runExchangeBackupCategoryTest(suite, "email")
+	runExchangeBackupCategoryTest(suite, email)
 }
 
 func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_contacts() {
-	runExchangeBackupCategoryTest(suite, "contacts")
+	runExchangeBackupCategoryTest(suite, contacts)
 }
 
 func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_events() {
-	runExchangeBackupCategoryTest(suite, "events")
+	runExchangeBackupCategoryTest(suite, events)
 }
 
-func runExchangeBackupCategoryTest(suite *BackupExchangeE2ESuite, category string) {
+func runExchangeBackupCategoryTest(suite *BackupExchangeE2ESuite, category path.CategoryType) {
 	recorder := strings.Builder{}
 	recorder.Reset()
 
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd, ctx := buildExchangeBackupCmd(ctx, suite.cfgFP, suite.m365UserID, category, &recorder)
+	cmd, ctx := buildExchangeBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		suite.m365.User.ID,
+		category.String(),
+		&recorder)
 
 	// run the command
 	err := cmd.ExecuteContext(ctx)
@@ -349,22 +150,25 @@ func runExchangeBackupCategoryTest(suite *BackupExchangeE2ESuite, category strin
 	result := recorder.String()
 	t.Log("backup results", result)
 
-	// as an offhand check: the result should contain the m365 user id
-	assert.Contains(t, result, suite.m365UserID)
+	// As an offhand check: the result should contain the m365 user's email.
+	assert.Contains(
+		t,
+		strings.ToLower(result),
+		strings.ToLower(suite.m365.User.Provider.Name()))
 }
 
 func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_ServiceNotEnabled_email() {
-	runExchangeBackupServiceNotEnabledTest(suite, "email")
+	runExchangeBackupServiceNotEnabledTest(suite, email)
 }
 
-func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, category string) {
+func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, category path.CategoryType) {
 	recorder := strings.Builder{}
 	recorder.Reset()
 
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
@@ -372,9 +176,9 @@ func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, categ
 
 	cmd, ctx := buildExchangeBackupCmd(
 		ctx,
-		suite.cfgFP,
-		fmt.Sprintf("%s,%s", tester.UnlicensedM365UserID(suite.T()), suite.m365UserID),
-		category,
+		suite.dpnd.configFilePath,
+		fmt.Sprintf("%s,%s", tconfig.UnlicensedM365UserID(suite.T()), suite.m365.User.ID),
+		category.String(),
 		&recorder)
 	err := cmd.ExecuteContext(ctx)
 	require.NoError(t, err, clues.ToCore(err))
@@ -382,34 +186,42 @@ func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, categ
 	result := recorder.String()
 	t.Log("backup results", result)
 
-	// as an offhand check: the result should contain the m365 user id
-	assert.Contains(t, result, suite.m365UserID)
+	// As an offhand check: the result should contain the m365 user's email.
+	assert.Contains(
+		t,
+		strings.ToLower(result),
+		strings.ToLower(suite.m365.User.Provider.Name()))
 }
 
 func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_userNotFound_email() {
-	runExchangeBackupUserNotFoundTest(suite, "email")
+	runExchangeBackupUserNotFoundTest(suite, email)
 }
 
 func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_userNotFound_contacts() {
-	runExchangeBackupUserNotFoundTest(suite, "contacts")
+	runExchangeBackupUserNotFoundTest(suite, contacts)
 }
 
 func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_userNotFound_events() {
-	runExchangeBackupUserNotFoundTest(suite, "events")
+	runExchangeBackupUserNotFoundTest(suite, events)
 }
 
-func runExchangeBackupUserNotFoundTest(suite *BackupExchangeE2ESuite, category string) {
+func runExchangeBackupUserNotFoundTest(suite *BackupExchangeE2ESuite, category path.CategoryType) {
 	recorder := strings.Builder{}
 	recorder.Reset()
 
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd, ctx := buildExchangeBackupCmd(ctx, suite.cfgFP, "foo@not-there.com", category, &recorder)
+	cmd, ctx := buildExchangeBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		"foo@not-there.com",
+		category.String(),
+		&recorder)
 
 	// run the command
 	err := cmd.ExecuteContext(ctx)
@@ -417,7 +229,8 @@ func runExchangeBackupUserNotFoundTest(suite *BackupExchangeE2ESuite, category s
 	assert.Contains(
 		t,
 		err.Error(),
-		"not found in tenant", "error missing user not found")
+		"not found",
+		"error missing user not found")
 	assert.NotContains(t, err.Error(), "runtime error", "panic happened")
 
 	t.Logf("backup error message: %s", err.Error())
@@ -426,27 +239,105 @@ func runExchangeBackupUserNotFoundTest(suite *BackupExchangeE2ESuite, category s
 	t.Log("backup results", result)
 }
 
+func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_badAzureClientIDFlag() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "exchange",
+		"--user", suite.m365.User.ID,
+		"--azure-client-id", "invalid-value")
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetErr(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_fromConfigFile() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "exchange",
+		"--user", suite.m365.User.ID,
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := suite.dpnd.recorder.String()
+	t.Log("backup results", result)
+
+	// As an offhand check: the result should contain the m365 user's email.
+	assert.Contains(
+		t,
+		strings.ToLower(result),
+		strings.ToLower(suite.m365.User.Provider.Name()))
+}
+
+// AWS flags
+func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_badAWSFlags() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "exchange",
+		"--user", suite.m365.User.ID,
+		"--aws-access-key", "invalid-value",
+		"--aws-secret-access-key", "some-invalid-value")
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	// since invalid aws creds are explicitly set, should see a failure
+	require.Error(t, err, clues.ToCore(err))
+}
+
 // ---------------------------------------------------------------------------
 // tests prepared with a previous backup
 // ---------------------------------------------------------------------------
 
 type PreparedBackupExchangeE2ESuite struct {
 	tester.Suite
-	acct       account.Account
-	st         storage.Storage
-	vpr        *viper.Viper
-	cfgFP      string
-	repo       repository.Repository
-	m365UserID string
-	backupOps  map[path.CategoryType]string
-	recorder   strings.Builder
+	dpnd      dependencies
+	backupOps map[path.CategoryType]string
+	m365      its.M365IntgTestSetup
 }
 
 func TestPreparedBackupExchangeE2ESuite(t *testing.T) {
-	suite.Run(t, &PreparedBackupExchangeE2ESuite{Suite: tester.NewE2ESuite(
-		t,
-		[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-	)})
+	suite.Run(t, &PreparedBackupExchangeE2ESuite{
+		Suite: tester.NewE2ESuite(
+			t,
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
+	})
 }
 
 func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
@@ -455,20 +346,13 @@ func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.vpr = vpr
-	suite.recorder = recorder
-	suite.cfgFP = cfgFilePath
-	suite.m365UserID = tester.M365UserID(t)
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
 	suite.backupOps = make(map[path.CategoryType]string)
 
 	var (
-		users = []string{suite.m365UserID}
-		ins   = idname.NewCache(map[string]string{suite.m365UserID: suite.m365UserID})
+		users = []string{suite.m365.User.ID}
+		ins   = idname.NewCache(map[string]string{suite.m365.User.ID: suite.m365.User.ID})
 	)
 
 	for _, set := range []path.CategoryType{email, contacts, events} {
@@ -479,18 +363,18 @@ func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
 
 		switch set {
 		case email:
-			scopes = sel.MailFolders([]string{exchange.DefaultMailFolder}, selectors.PrefixMatch())
+			scopes = sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())
 
 		case contacts:
-			scopes = sel.ContactFolders([]string{exchange.DefaultContactFolder}, selectors.PrefixMatch())
+			scopes = sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch())
 
 		case events:
-			scopes = sel.EventCalendars([]string{exchange.DefaultCalendar}, selectors.PrefixMatch())
+			scopes = sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())
 		}
 
 		sel.Include(scopes)
 
-		bop, err := suite.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+		bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
 		require.NoError(t, err, clues.ToCore(err))
 
 		err = bop.Run(ctx)
@@ -499,11 +383,11 @@ func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
 		bIDs := string(bop.Results.BackupID)
 
 		// sanity check, ensure we can find the backup and its details immediately
-		b, err := suite.repo.Backup(ctx, string(bop.Results.BackupID))
+		b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
 		require.NoError(t, err, "retrieving recent backup by ID")
 		require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
 
-		_, b, errs := suite.repo.GetBackupDetails(ctx, bIDs)
+		_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
 		require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
 		require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
 		require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
@@ -525,20 +409,20 @@ func (suite *PreparedBackupExchangeE2ESuite) TestExchangeListCmd_events() {
 }
 
 func runExchangeListCmdTest(suite *PreparedBackupExchangeE2ESuite, category path.CategoryType) {
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "list", "exchange",
-		"--config-file", suite.cfgFP)
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
 	cli.BuildCommandTree(cmd)
-	cmd.SetOut(&suite.recorder)
+	cmd.SetOut(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -547,7 +431,7 @@ func runExchangeListCmdTest(suite *PreparedBackupExchangeE2ESuite, category path
 	require.NoError(t, err, clues.ToCore(err))
 
 	// compare the output
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 	assert.Contains(t, result, suite.backupOps[category])
 }
 
@@ -564,24 +448,24 @@ func (suite *PreparedBackupExchangeE2ESuite) TestExchangeListCmd_singleID_events
 }
 
 func runExchangeListSingleCmdTest(suite *PreparedBackupExchangeE2ESuite, category path.CategoryType) {
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
 	bID := suite.backupOps[category]
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "list", "exchange",
-		"--config-file", suite.cfgFP,
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
 		"--backup", string(bID))
 	cli.BuildCommandTree(cmd)
 
-	cmd.SetOut(&suite.recorder)
+	cmd.SetOut(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -590,7 +474,7 @@ func runExchangeListSingleCmdTest(suite *PreparedBackupExchangeE2ESuite, categor
 	require.NoError(t, err, clues.ToCore(err))
 
 	// compare the output
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 	assert.Contains(t, result, bID)
 }
 
@@ -598,13 +482,13 @@ func (suite *PreparedBackupExchangeE2ESuite) TestExchangeListCmd_badID() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "list", "exchange",
-		"--config-file", suite.cfgFP,
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
 		"--backup", "smarfs")
 	cli.BuildCommandTree(cmd)
 
@@ -628,28 +512,28 @@ func (suite *PreparedBackupExchangeE2ESuite) TestExchangeDetailsCmd_events() {
 }
 
 func runExchangeDetailsCmdTest(suite *PreparedBackupExchangeE2ESuite, category path.CategoryType) {
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
 	bID := suite.backupOps[category]
 
 	// fetch the details from the repo first
-	deets, _, errs := suite.repo.GetBackupDetails(ctx, string(bID))
+	deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
 	require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
 	require.Empty(t, errs.Recovered())
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "details", "exchange",
-		"--config-file", suite.cfgFP,
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
 		"--"+flags.BackupFN, string(bID))
 	cli.BuildCommandTree(cmd)
-	cmd.SetOut(&suite.recorder)
+	cmd.SetOut(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -658,7 +542,7 @@ func runExchangeDetailsCmdTest(suite *PreparedBackupExchangeE2ESuite, category p
 	require.NoError(t, err, clues.ToCore(err))
 
 	// compare the output
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 
 	i := 0
 	foundFolders := 0
@@ -688,20 +572,15 @@ func runExchangeDetailsCmdTest(suite *PreparedBackupExchangeE2ESuite, category p
 
 type BackupDeleteExchangeE2ESuite struct {
 	tester.Suite
-	acct     account.Account
-	st       storage.Storage
-	vpr      *viper.Viper
-	cfgFP    string
-	repo     repository.Repository
-	backupOp operations.BackupOperation
+	dpnd      dependencies
+	backupOps [3]operations.BackupOperation
 }
 
 func TestBackupDeleteExchangeE2ESuite(t *testing.T) {
 	suite.Run(t, &BackupDeleteExchangeE2ESuite{
 		Suite: tester.NewE2ESuite(
 			t,
-			[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-		),
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
 	})
 }
 
@@ -711,42 +590,41 @@ func (suite *BackupDeleteExchangeE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, _, cfgFilePath := prepM365Test(t, ctx)
+	suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
 
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.vpr = vpr
-	suite.cfgFP = cfgFilePath
-
-	m365UserID := tester.M365UserID(t)
+	m365UserID := tconfig.M365UserID(t)
 	users := []string{m365UserID}
 
 	// some tests require an existing backup
 	sel := selectors.NewExchangeBackup(users)
-	sel.Include(sel.MailFolders([]string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
+	sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
 
-	backupOp, err := suite.repo.NewBackup(ctx, sel.Selector)
-	require.NoError(t, err, clues.ToCore(err))
+	for i := 0; i < cap(suite.backupOps); i++ {
+		backupOp, err := suite.dpnd.repo.NewBackup(ctx, sel.Selector)
+		require.NoError(t, err, clues.ToCore(err))
 
-	suite.backupOp = backupOp
+		suite.backupOps[i] = backupOp
 
-	err = suite.backupOp.Run(ctx)
-	require.NoError(t, err, clues.ToCore(err))
+		err = suite.backupOps[i].Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+	}
 }
 
 func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "delete", "exchange",
-		"--config-file", suite.cfgFP,
-		"--"+flags.BackupFN, string(suite.backupOp.Results.BackupID))
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN,
+		fmt.Sprintf("%s,%s",
+			string(suite.backupOps[0].Results.BackupID),
+			string(suite.backupOps[1].Results.BackupID)))
 	cli.BuildCommandTree(cmd)
 
 	// run the command
@@ -754,10 +632,50 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
 	require.NoError(t, err, clues.ToCore(err))
 
 	// a follow-up details call should fail, due to the backup ID being deleted
-	cmd = tester.StubRootCmd(
+	cmd = cliTD.StubRootCmd(
 		"backup", "details", "exchange",
-		"--config-file", suite.cfgFP,
-		"--backup", string(suite.backupOp.Results.BackupID))
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(suite.backupOps[0].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+
+	// a follow-up details call should fail, due to the backup ID being deleted
+	cmd = cliTD.StubRootCmd(
+		"backup", "details", "exchange",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(suite.backupOps[1].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_SingleID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "exchange",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupFN,
+		string(suite.backupOps[2].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// a follow-up details call should fail, due to the backup ID being deleted
+	cmd = cliTD.StubRootCmd(
+		"backup", "details", "exchange",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(suite.backupOps[2].Results.BackupID))
 	cli.BuildCommandTree(cmd)
 
 	err = cmd.ExecuteContext(ctx)
@@ -768,14 +686,14 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_UnknownID
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "delete", "exchange",
-		"--config-file", suite.cfgFP,
-		"--"+flags.BackupFN, uuid.NewString())
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN, uuid.NewString())
 	cli.BuildCommandTree(cmd)
 
 	// unknown backupIDs should error since the modelStore can't find the backup
@@ -783,22 +701,20 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_UnknownID
 	require.Error(t, err, clues.ToCore(err))
 }
 
-// ---------------------------------------------------------------------------
-// helpers
-// ---------------------------------------------------------------------------
+func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_NoBackupID() {
+	t := suite.T()
 
-func buildExchangeBackupCmd(
-	ctx context.Context,
-	configFile, user, category string,
-	recorder *strings.Builder,
-) (*cobra.Command, context.Context) {
-	cmd := tester.StubRootCmd(
-		"backup", "create", "exchange",
-		"--config-file", configFile,
-		"--"+flags.UserFN, user,
-		"--"+flags.CategoryDataFN, category)
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "exchange",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
 	cli.BuildCommandTree(cmd)
-	cmd.SetOut(recorder)
 
-	return cmd, print.SetRootCmd(ctx, cmd)
+	// empty backupIDs should error since no data provided
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
 }

src/cli/backup/exchange_test.go

@@ -1,7 +1,7 @@
 package backup
 
 import (
-	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/alcionai/clues"
@@ -11,10 +11,11 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/cli/flags"
-	"github.com/alcionai/corso/src/cli/utils/testdata"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
 	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/internal/version"
-	dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
+	"github.com/alcionai/corso/src/pkg/control"
 )
 
 type ExchangeUnitSuite struct {
@@ -33,72 +34,35 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() {
 		use         string
 		expectUse   string
 		expectShort string
-		flags       []string
 		expectRunE  func(*cobra.Command, []string) error
 	}{
 		{
-			"create exchange",
-			createCommand,
-			expectUse + " " + exchangeServiceCommandCreateUseSuffix,
-			exchangeCreateCmd().Short,
-			[]string{
-				flags.UserFN,
-				flags.CategoryDataFN,
-				flags.DisableIncrementalsFN,
-				flags.DisableDeltaFN,
-				flags.FailFastFN,
-				flags.FetchParallelismFN,
-				flags.SkipReduceFN,
-				flags.NoStatsFN,
-			},
-			createExchangeCmd,
+			name:        "create exchange",
+			use:         createCommand,
+			expectUse:   expectUse + " " + exchangeServiceCommandCreateUseSuffix,
+			expectShort: exchangeCreateCmd().Short,
+			expectRunE:  createExchangeCmd,
 		},
 		{
-			"list exchange",
-			listCommand,
-			expectUse,
-			exchangeListCmd().Short,
-			[]string{
-				flags.BackupFN,
-				flags.FailedItemsFN,
-				flags.SkippedItemsFN,
-				flags.RecoveredErrorsFN,
-			},
-			listExchangeCmd,
+			name:        "list exchange",
+			use:         listCommand,
+			expectUse:   expectUse,
+			expectShort: exchangeListCmd().Short,
+			expectRunE:  listExchangeCmd,
 		},
 		{
-			"details exchange",
-			detailsCommand,
-			expectUse + " " + exchangeServiceCommandDetailsUseSuffix,
-			exchangeDetailsCmd().Short,
-			[]string{
-				flags.BackupFN,
-				flags.ContactFN,
-				flags.ContactFolderFN,
-				flags.ContactNameFN,
-				flags.EmailFN,
-				flags.EmailFolderFN,
-				flags.EmailReceivedAfterFN,
-				flags.EmailReceivedBeforeFN,
-				flags.EmailSenderFN,
-				flags.EmailSubjectFN,
-				flags.EventFN,
-				flags.EventCalendarFN,
-				flags.EventOrganizerFN,
-				flags.EventRecursFN,
-				flags.EventStartsAfterFN,
-				flags.EventStartsBeforeFN,
-				flags.EventSubjectFN,
-			},
-			detailsExchangeCmd,
+			name:        "details exchange",
+			use:         detailsCommand,
+			expectUse:   expectUse + " " + exchangeServiceCommandDetailsUseSuffix,
+			expectShort: exchangeDetailsCmd().Short,
+			expectRunE:  detailsExchangeCmd,
 		},
 		{
-			"delete exchange",
-			deleteCommand,
-			expectUse + " " + exchangeServiceCommandDeleteUseSuffix,
-			exchangeDeleteCmd().Short,
-			[]string{flags.BackupFN},
-			deleteExchangeCmd,
+			name:        "delete exchange",
+			use:         deleteCommand,
+			expectUse:   expectUse + " " + exchangeServiceCommandDeleteUseSuffix,
+			expectShort: exchangeDeleteCmd().Short,
+			expectRunE:  deleteExchangeCmd,
 		},
 	}
 	for _, test := range table {
@@ -121,6 +85,147 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() {
 	}
 }
 
+func (suite *ExchangeUnitSuite) TestBackupCreateFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: createCommand},
+		addExchangeCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			exchangeServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.MailBoxFN, flagsTD.FlgInputs(flagsTD.MailboxInput),
+				"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.ExchangeCategoryDataInput),
+				"--" + flags.FetchParallelismFN, flagsTD.FetchParallelism,
+				"--" + flags.DeltaPageSizeFN, flagsTD.DeltaPageSize,
+
+				// bool flags
+				"--" + flags.DisableDeltaFN,
+				"--" + flags.EnableImmutableIDFN,
+				"--" + flags.DisableSlidingWindowLimiterFN,
+			},
+			flagsTD.PreparedGenericBackupFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	opts := utils.MakeExchangeOpts(cmd)
+	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
+	assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(backupOpts.M365.DeltaPageSize)))
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
+	assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
+	assert.True(t, backupOpts.M365.ExchangeImmutableIDs)
+	assert.True(t, backupOpts.ServiceRateLimiter.DisableSlidingWindowLimiter)
+
+	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
+	assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(co.DeltaPageSize)))
+	assert.Equal(t, control.FailFast, co.FailureHandling)
+	assert.True(t, co.ToggleFeatures.DisableIncrementals)
+	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
+	assert.True(t, co.ToggleFeatures.DisableDelta)
+	assert.True(t, co.ToggleFeatures.ExchangeImmutableIDs)
+	assert.True(t, co.ToggleFeatures.DisableSlidingWindowLimiter)
+
+	assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users)
+	flagsTD.AssertGenericBackupFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *ExchangeUnitSuite) TestBackupListFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: listCommand},
+		addExchangeCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			exchangeServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedBackupListFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertBackupListFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *ExchangeUnitSuite) TestBackupDetailsFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: detailsCommand},
+		addExchangeCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			exchangeServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+				"--" + flags.SkipReduceFN,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	co := utils.Control()
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	assert.True(t, co.SkipReduce)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *ExchangeUnitSuite) TestBackupDeleteFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: deleteCommand},
+		addExchangeCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			exchangeServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
 func (suite *ExchangeUnitSuite) TestValidateBackupCreateFlags() {
 	table := []struct {
 		name       string
@@ -272,51 +377,3 @@ func (suite *ExchangeUnitSuite) TestExchangeBackupCreateSelectors() {
 		})
 	}
 }
-
-func (suite *ExchangeUnitSuite) TestExchangeBackupDetailsSelectors() {
-	for v := 0; v <= version.Backup; v++ {
-		suite.Run(fmt.Sprintf("version%d", v), func() {
-			for _, test := range testdata.ExchangeOptionDetailLookups {
-				suite.Run(test.Name, func() {
-					t := suite.T()
-
-					ctx, flush := tester.NewContext(t)
-					defer flush()
-
-					bg := testdata.VersionedBackupGetter{
-						Details: dtd.GetDetailsSetForVersion(t, v),
-					}
-
-					output, err := runDetailsExchangeCmd(
-						ctx,
-						bg,
-						"backup-ID",
-						test.Opts(t, v),
-						false)
-					assert.NoError(t, err, clues.ToCore(err))
-					assert.ElementsMatch(t, test.Expected(t, v), output.Entries)
-				})
-			}
-		})
-	}
-}
-
-func (suite *ExchangeUnitSuite) TestExchangeBackupDetailsSelectorsBadFormats() {
-	for _, test := range testdata.BadExchangeOptionsFormats {
-		suite.Run(test.Name, func() {
-			t := suite.T()
-
-			ctx, flush := tester.NewContext(t)
-			defer flush()
-
-			output, err := runDetailsExchangeCmd(
-				ctx,
-				test.BackupGetter,
-				"backup-ID",
-				test.Opts(t, version.Backup),
-				false)
-			assert.Error(t, err, clues.ToCore(err))
-			assert.Empty(t, output)
-		})
-	}
-}

src/cli/backup/groups.go

@@ -0,0 +1,329 @@
+package backup
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/alcionai/clues"
+	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	. "github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/common/idname"
+	"github.com/alcionai/corso/src/pkg/fault"
+	"github.com/alcionai/corso/src/pkg/filters"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
+	"github.com/alcionai/corso/src/pkg/services/m365"
+)
+
+// ------------------------------------------------------------------------------------------------
+// setup and globals
+// ------------------------------------------------------------------------------------------------
+
+const (
+	groupsServiceCommand                 = "groups"
+	teamsServiceCommand                  = "teams"
+	groupsServiceCommandCreateUseSuffix  = "--group <groupName> | '" + flags.Wildcard + "'"
+	groupsServiceCommandDeleteUseSuffix  = "--backups <backupId>"
+	groupsServiceCommandDetailsUseSuffix = "--backup <backupId>"
+)
+
+const (
+	groupsServiceCommandCreateExamples = `# Backup all Groups and Teams data for the Marketing group
+corso backup create groups --group Marketing
+
+# Backup only Teams channel messages
+corso backup create groups --group Marketing --data messages
+
+# Backup only group mailbox posts
+corso backup create groups --group Marketing --data conversations
+
+# Backup all Groups and Teams data for all groups
+corso backup create groups --group '*'`
+
+	groupsServiceCommandDeleteExamples = `# Delete Groups backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
+and 1234abcd-12ab-cd34-56de-1234abce
+corso backup delete groups --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce`
+
+	groupsServiceCommandDetailsExamples = `# Explore items in Marketing's latest backup (1234abcd...)
+corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd
+
+# Explore Marketing messages posted after the start of 2022
+corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --last-message-reply-after 2022-01-01T00:00:00
+
+# Explore group mailbox posts with conversation subject "hello world"
+corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world"`
+)
+
+// called by backup.go to map subcommands to provider-specific handling.
+func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case createCommand:
+		c, _ = utils.AddCommand(cmd, groupsCreateCmd(), utils.MarkPreviewCommand())
+
+		c.Use = c.Use + " " + groupsServiceCommandCreateUseSuffix
+		c.Example = groupsServiceCommandCreateExamples
+
+		// Flags addition ordering should follow the order we want them to appear in help and docs:
+		flags.AddGroupFlag(c)
+		flags.AddDataFlag(c, []string{flags.DataLibraries, flags.DataMessages, flags.DataConversations}, false)
+		flags.AddFetchParallelismFlag(c)
+		flags.AddDisableDeltaFlag(c)
+		flags.AddGenericBackupFlags(c)
+		flags.AddDisableLazyItemReader(c)
+
+	case listCommand:
+		c, _ = utils.AddCommand(cmd, groupsListCmd(), utils.MarkPreviewCommand())
+
+		flags.AddBackupIDFlag(c, false)
+		flags.AddAllBackupListFlags(c)
+
+	case detailsCommand:
+		c, _ = utils.AddCommand(cmd, groupsDetailsCmd(), utils.MarkPreviewCommand())
+
+		c.Use = c.Use + " " + groupsServiceCommandDetailsUseSuffix
+		c.Example = groupsServiceCommandDetailsExamples
+
+		flags.AddSkipReduceFlag(c)
+
+		// Flags addition ordering should follow the order we want them to appear in help and docs:
+		// More generic (ex: --user) and more frequently used flags take precedence.
+		flags.AddBackupIDFlag(c, true)
+		flags.AddGroupDetailsAndRestoreFlags(c)
+		flags.AddSharePointDetailsAndRestoreFlags(c)
+
+	case deleteCommand:
+		c, _ = utils.AddCommand(cmd, groupsDeleteCmd(), utils.MarkPreviewCommand())
+
+		c.Use = c.Use + " " + groupsServiceCommandDeleteUseSuffix
+		c.Example = groupsServiceCommandDeleteExamples
+
+		flags.AddMultipleBackupIDsFlag(c, false)
+		flags.AddBackupIDFlag(c, false)
+	}
+
+	return c
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup create
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup create groups [<flag>...]`
+func groupsCreateCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     groupsServiceCommand,
+		Aliases: []string{teamsServiceCommand},
+		Short:   "Backup M365 Groups & Teams service data",
+		RunE:    createGroupsCmd,
+		Args:    cobra.NoArgs,
+	}
+}
+
+// processes a groups service backup.
+func createGroupsCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	if err := validateGroupsBackupCreateFlags(flags.GroupFV, flags.CategoryDataFV); err != nil {
+		return err
+	}
+
+	r, acct, err := utils.AccountConnectAndWriteRepoConfig(
+		ctx,
+		cmd,
+		path.GroupsService)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	defer utils.CloseRepo(ctx, r)
+
+	// TODO: log/print recoverable errors
+	errs := fault.New(false)
+
+	svcCli, err := m365.NewM365Client(ctx, *acct)
+	if err != nil {
+		return Only(ctx, clues.Stack(err))
+	}
+
+	ins, err := svcCli.AC.Groups().GetAllIDsAndNames(ctx, errs)
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 groups"))
+	}
+
+	sel := groupsBackupCreateSelectors(ctx, ins, flags.GroupFV, flags.CategoryDataFV)
+	selectorSet := []selectors.Selector{}
+
+	for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) {
+		selectorSet = append(selectorSet, discSel.Selector)
+	}
+
+	return genericCreateCommand(
+		ctx,
+		r,
+		"Group",
+		selectorSet,
+		ins)
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup list
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup list groups [<flag>...]`
+func groupsListCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   groupsServiceCommand,
+		Short: "List the history of M365 Groups service backups",
+		RunE:  listGroupsCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// lists the history of backup operations
+func listGroupsCmd(cmd *cobra.Command, args []string) error {
+	return genericListCommand(cmd, flags.BackupIDFV, path.GroupsService, args)
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup details
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup details groups [<flag>...]`
+func groupsDetailsCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   groupsServiceCommand,
+		Short: "Shows the details of a M365 Groups service backup",
+		RunE:  detailsGroupsCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// processes a groups service backup.
+func detailsGroupsCmd(cmd *cobra.Command, args []string) error {
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	return runDetailsGroupsCmd(cmd)
+}
+
+func runDetailsGroupsCmd(cmd *cobra.Command) error {
+	ctx := cmd.Context()
+	opts := utils.MakeGroupsOpts(cmd)
+
+	sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts)
+	sel.Configure(selectors.Config{OnlyMatchItemNames: true})
+	utils.FilterGroupsRestoreInfoSelectors(sel, opts)
+
+	ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	if len(ds.Entries) > 0 {
+		ds.PrintEntries(ctx)
+	} else {
+		Info(ctx, selectors.ErrorNoMatchingItems)
+	}
+
+	return nil
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup delete
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup delete groups [<flag>...]`
+func groupsDeleteCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   groupsServiceCommand,
+		Short: "Delete backed-up M365 Groups service data",
+		RunE:  deleteGroupsCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// deletes an groups service backup.
+func deleteGroupsCmd(cmd *cobra.Command, args []string) error {
+	backupIDValue := []string{}
+
+	if len(flags.BackupIDsFV) > 0 {
+		backupIDValue = flags.BackupIDsFV
+	} else if len(flags.BackupIDFV) > 0 {
+		backupIDValue = append(backupIDValue, flags.BackupIDFV)
+	} else {
+		return clues.New("either --backup or --backups flag is required")
+	}
+
+	return genericDeleteCommand(cmd, path.GroupsService, "Groups", backupIDValue, args)
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func validateGroupsBackupCreateFlags(groups, cats []string) error {
+	if len(groups) == 0 {
+		return clues.New(
+			"requires one or more --" +
+				flags.GroupFN + " ids, or the wildcard --" +
+				flags.GroupFN + " *")
+	}
+
+	// TODO(keepers): release conversations support
+
+	msg := fmt.Sprintf(
+		" is an unrecognized data type; only %s and %s are supported",
+		flags.DataLibraries, flags.DataMessages)
+
+	// msg := fmt.Sprintf(
+	// 	" is an unrecognized data type; only %s, %s and %s are supported",
+	// 	flags.DataLibraries, flags.DataMessages, flags.DataConversations)
+
+	allowedCats := utils.GroupsAllowedCategories()
+
+	for _, d := range cats {
+		if _, ok := allowedCats[d]; !ok {
+			return clues.New(d + msg)
+		}
+	}
+
+	return nil
+}
+
+func groupsBackupCreateSelectors(
+	ctx context.Context,
+	ins idname.Cacher,
+	group, cats []string,
+) *selectors.GroupsBackup {
+	if filters.PathContains(group).Compare(flags.Wildcard) {
+		return includeAllGroupsWithCategories(ins, cats)
+	}
+
+	sel := selectors.NewGroupsBackup(slices.Clone(group))
+
+	return utils.AddGroupsCategories(sel, cats)
+}
+
+func includeAllGroupsWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup {
+	return utils.AddGroupsCategories(selectors.NewGroupsBackup(ins.IDs()), categories)
+}

src/cli/backup/groups_e2e_test.go

@@ -0,0 +1,690 @@
+package backup_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/alcionai/clues"
+	"github.com/google/uuid"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/print"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/common/idname"
+	"github.com/alcionai/corso/src/internal/operations"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/internal/tester/its"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/config"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
+	selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
+)
+
+// ---------------------------------------------------------------------------
+// tests that require no existing backups
+// ---------------------------------------------------------------------------
+
+type NoBackupGroupsE2ESuite struct {
+	tester.Suite
+	dpnd dependencies
+	m365 its.M365IntgTestSetup
+}
+
+func TestNoBackupGroupsE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupGroupsE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *NoBackupGroupsE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
+}
+
+func (suite *NoBackupGroupsE2ESuite) TestGroupsBackupListCmd_noBackups() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetErr(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := suite.dpnd.recorder.String()
+
+	// as an offhand check: the result should contain the m365 group id
+	assert.True(t, strings.HasSuffix(result, "No backups available\n"))
+}
+
+// ---------------------------------------------------------------------------
+// tests with no prior backup
+// ---------------------------------------------------------------------------
+
+type BackupGroupsE2ESuite struct {
+	tester.Suite
+	dpnd dependencies
+	m365 its.M365IntgTestSetup
+}
+
+func TestBackupGroupsE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupGroupsE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *BackupGroupsE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
+}
+
+func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_channelMessages() {
+	runGroupsBackupCategoryTest(suite, flags.DataMessages)
+}
+
+func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_conversations() {
+	// skip
+	suite.T().Skip("CorsoCITeam group mailbox backup is broken")
+	runGroupsBackupCategoryTest(suite, flags.DataConversations)
+}
+
+func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_libraries() {
+	runGroupsBackupCategoryTest(suite, flags.DataLibraries)
+}
+
+func runGroupsBackupCategoryTest(suite *BackupGroupsE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildGroupsBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		suite.m365.Group.ID,
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_groupNotFound_channelMessages() {
+	runGroupsBackupGroupNotFoundTest(suite, flags.DataMessages)
+}
+
+func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_groupNotFound_conversations() {
+	runGroupsBackupGroupNotFoundTest(suite, flags.DataConversations)
+}
+
+func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_groupNotFound_libraries() {
+	runGroupsBackupGroupNotFoundTest(suite, flags.DataLibraries)
+}
+
+func runGroupsBackupGroupNotFoundTest(suite *BackupGroupsE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildGroupsBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		"foo@not-there.com",
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+	assert.Contains(
+		t,
+		err.Error(),
+		"not found",
+		"error missing user not found")
+	assert.NotContains(t, err.Error(), "runtime error", "panic happened")
+
+	t.Logf("backup error message: %s", err.Error())
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAzureClientIDFlag() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "groups",
+		"--group", suite.m365.Group.ID,
+		"--azure-client-id", "invalid-value")
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetErr(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_fromConfigFile() {
+	// Skip
+	suite.T().Skip("CorsoCITeam group mailbox backup is broken")
+
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "groups",
+		"--group", suite.m365.Group.ID,
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+}
+
+// AWS flags
+func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAWSFlags() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "groups",
+		"--group", suite.m365.Group.ID,
+		"--aws-access-key", "invalid-value",
+		"--aws-secret-access-key", "some-invalid-value")
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	// since invalid aws creds are explicitly set, should see a failure
+	require.Error(t, err, clues.ToCore(err))
+}
+
+// ---------------------------------------------------------------------------
+// tests prepared with a previous backup
+// ---------------------------------------------------------------------------
+
+type PreparedBackupGroupsE2ESuite struct {
+	tester.Suite
+	dpnd      dependencies
+	backupOps map[path.CategoryType]string
+	m365      its.M365IntgTestSetup
+}
+
+func TestPreparedBackupGroupsE2ESuite(t *testing.T) {
+	suite.Run(t, &PreparedBackupGroupsE2ESuite{
+		Suite: tester.NewE2ESuite(
+			t,
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
+	})
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
+	suite.backupOps = make(map[path.CategoryType]string)
+
+	var (
+		groups = []string{suite.m365.Group.ID}
+		ins    = idname.NewCache(map[string]string{suite.m365.Group.ID: suite.m365.Group.ID})
+		cats   = []path.CategoryType{
+			path.ChannelMessagesCategory,
+			// TODO(pandeyabs): CorsoCITeam group mailbox backup is currently broken because of invalid
+			// odata.NextLink which causes an infinite loop during paging. Disabling conversations tests while
+			// we go fix the group mailbox.
+			// path.ConversationPostsCategory,
+			path.LibrariesCategory,
+		}
+	)
+
+	for _, set := range cats {
+		var (
+			sel    = selectors.NewGroupsBackup(groups)
+			scopes []selectors.GroupsScope
+		)
+
+		switch set {
+		case path.ChannelMessagesCategory:
+			scopes = selTD.GroupsBackupChannelScope(sel)
+
+		case path.ConversationPostsCategory:
+			scopes = selTD.GroupsBackupConversationScope(sel)
+
+		case path.LibrariesCategory:
+			scopes = selTD.GroupsBackupLibraryFolderScope(sel)
+		}
+
+		sel.Include(scopes)
+
+		bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+		require.NoError(t, err, clues.ToCore(err))
+
+		err = bop.Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+
+		bIDs := string(bop.Results.BackupID)
+
+		// sanity check, ensure we can find the backup and its details immediately
+		b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
+		require.NoError(t, err, "retrieving recent backup by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
+
+		_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
+		require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
+		require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
+
+		suite.backupOps[set] = string(b.ID)
+	}
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_channelMessages() {
+	runGroupsListCmdTest(suite, path.ChannelMessagesCategory)
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_conversations() {
+	runGroupsListCmdTest(suite, path.ConversationPostsCategory)
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_libraries() {
+	runGroupsListCmdTest(suite, path.LibrariesCategory)
+}
+
+func runGroupsListCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+	assert.Contains(t, result, suite.backupOps[category])
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_singleID_channelMessages() {
+	runGroupsListSingleCmdTest(suite, path.ChannelMessagesCategory)
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_singleID_conversations() {
+	runGroupsListSingleCmdTest(suite, path.ConversationPostsCategory)
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_singleID_libraries() {
+	runGroupsListSingleCmdTest(suite, path.LibrariesCategory)
+}
+
+func runGroupsListSingleCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	bID := suite.backupOps[category]
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(bID))
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+	assert.Contains(t, result, bID)
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_badID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", "smarfs")
+	cli.BuildCommandTree(cmd)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsDetailsCmd_channelMessages() {
+	runGroupsDetailsCmdTest(suite, path.ChannelMessagesCategory)
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsDetailsCmd_conversations() {
+	// skip
+	suite.T().Skip("CorsoCITeam group mailbox backup is broken")
+	runGroupsDetailsCmdTest(suite, path.ConversationPostsCategory)
+}
+
+func (suite *PreparedBackupGroupsE2ESuite) TestGroupsDetailsCmd_libraries() {
+	runGroupsDetailsCmdTest(suite, path.LibrariesCategory)
+}
+
+func runGroupsDetailsCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	bID := suite.backupOps[category]
+
+	// fetch the details from the repo first
+	deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
+	require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
+	require.Empty(t, errs.Recovered())
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "details", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupFN, string(bID))
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+
+	i := 0
+	foundFolders := 0
+
+	for _, ent := range deets.Entries {
+		// Skip folders as they don't mean anything to the end group.
+		if ent.Folder != nil {
+			foundFolders++
+			continue
+		}
+
+		suite.Run(fmt.Sprintf("detail %d", i), func() {
+			assert.Contains(suite.T(), result, ent.ShortRef)
+		})
+
+		i++
+	}
+
+	// We only backup the default folder for each category so there should be at
+	// least that folder (we don't make details entries for prefix folders).
+	assert.GreaterOrEqual(t, foundFolders, 1)
+}
+
+// ---------------------------------------------------------------------------
+// tests for deleting backups
+// ---------------------------------------------------------------------------
+
+type BackupDeleteGroupsE2ESuite struct {
+	tester.Suite
+	dpnd      dependencies
+	backupOps [3]operations.BackupOperation
+}
+
+func TestBackupDeleteGroupsE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupDeleteGroupsE2ESuite{
+		Suite: tester.NewE2ESuite(
+			t,
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
+	})
+}
+
+func (suite *BackupDeleteGroupsE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
+
+	m365GroupID := tconfig.M365TeamID(t)
+	groups := []string{m365GroupID}
+
+	// some tests require an existing backup
+	sel := selectors.NewGroupsBackup(groups)
+	sel.Include(selTD.GroupsBackupChannelScope(sel))
+
+	for i := 0; i < cap(suite.backupOps); i++ {
+		backupOp, err := suite.dpnd.repo.NewBackup(ctx, sel.Selector)
+		require.NoError(t, err, clues.ToCore(err))
+
+		suite.backupOps[i] = backupOp
+
+		err = suite.backupOps[i].Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+	}
+}
+
+func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN,
+		fmt.Sprintf("%s,%s",
+			string(suite.backupOps[0].Results.BackupID),
+			string(suite.backupOps[1].Results.BackupID)))
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// a follow-up details call should fail, due to the backup ID being deleted
+	cmd = cliTD.StubRootCmd(
+		"backup", "details", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backups", string(suite.backupOps[0].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_SingleID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupFN,
+		string(suite.backupOps[2].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// a follow-up details call should fail, due to the backup ID being deleted
+	cmd = cliTD.StubRootCmd(
+		"backup", "details", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(suite.backupOps[2].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_UnknownID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN, uuid.NewString())
+	cli.BuildCommandTree(cmd)
+
+	// unknown backupIDs should error since the modelStore can't find the backup
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_NoBackupID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	// empty backupIDs should error since no data provided
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func buildGroupsBackupCmd(
+	ctx context.Context,
+	configFile, group, category string,
+	recorder *strings.Builder,
+) (*cobra.Command, context.Context) {
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "groups",
+		"--"+flags.ConfigFileFN, configFile,
+		"--"+flags.GroupFN, group,
+		"--"+flags.CategoryDataFN, category)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(recorder)
+
+	return cmd, print.SetRootCmd(ctx, cmd)
+}

src/cli/backup/groups_test.go

@@ -0,0 +1,273 @@
+package backup
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/alcionai/clues"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/control"
+)
+
+type GroupsUnitSuite struct {
+	tester.Suite
+}
+
+func TestGroupsUnitSuite(t *testing.T) {
+	suite.Run(t, &GroupsUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
+	expectUse := groupsServiceCommand
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{
+			name:        "create groups",
+			use:         createCommand,
+			expectUse:   expectUse + " " + groupsServiceCommandCreateUseSuffix,
+			expectShort: groupsCreateCmd().Short,
+			expectRunE:  createGroupsCmd,
+		},
+		{
+			name:        "list groups",
+			use:         listCommand,
+			expectUse:   expectUse,
+			expectShort: groupsListCmd().Short,
+			expectRunE:  listGroupsCmd,
+		},
+		{
+			name:        "details groups",
+			use:         detailsCommand,
+			expectUse:   expectUse + " " + groupsServiceCommandDetailsUseSuffix,
+			expectShort: groupsDetailsCmd().Short,
+			expectRunE:  detailsGroupsCmd,
+		},
+		{
+			name:        "delete groups",
+			use:         deleteCommand,
+			expectUse:   expectUse + " " + groupsServiceCommandDeleteUseSuffix,
+			expectShort: groupsDeleteCmd().Short,
+			expectRunE:  deleteGroupsCmd,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			cmd := &cobra.Command{Use: test.use}
+
+			c := addGroupsCommands(cmd)
+			require.NotNil(t, c)
+
+			cmds := cmd.Commands()
+			require.Len(t, cmds, 1)
+
+			child := cmds[0]
+			assert.Equal(t, test.expectUse, child.Use)
+			assert.Equal(t, test.expectShort, child.Short)
+			tester.AreSameFunc(t, test.expectRunE, child.RunE)
+		})
+	}
+}
+
+func (suite *GroupsUnitSuite) TestValidateGroupsBackupCreateFlags() {
+	table := []struct {
+		name   string
+		cats   []string
+		expect assert.ErrorAssertionFunc
+	}{
+		{
+			name:   "none",
+			cats:   []string{},
+			expect: assert.NoError,
+		},
+		{
+			name:   "libraries",
+			cats:   []string{flags.DataLibraries},
+			expect: assert.NoError,
+		},
+		{
+			name:   "messages",
+			cats:   []string{flags.DataMessages},
+			expect: assert.NoError,
+		},
+		{
+			name:   "conversations",
+			cats:   []string{flags.DataConversations},
+			expect: assert.NoError,
+		},
+		{
+			name: "all allowed",
+			cats: []string{
+				flags.DataLibraries,
+				flags.DataMessages,
+				flags.DataConversations,
+			},
+			expect: assert.NoError,
+		},
+		{
+			name:   "bad inputs",
+			cats:   []string{"foo"},
+			expect: assert.Error,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			err := validateGroupsBackupCreateFlags([]string{"*"}, test.cats)
+			test.expect(suite.T(), err, clues.ToCore(err))
+		})
+	}
+}
+
+func (suite *GroupsUnitSuite) TestBackupCreateFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: createCommand},
+		addGroupsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			groupsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.GroupFN, flagsTD.FlgInputs(flagsTD.GroupsInput),
+				"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.GroupsCategoryDataInput),
+				"--" + flags.FetchParallelismFN, flagsTD.FetchParallelism,
+				"--" + flags.DisableDeltaFN,
+				"--" + flags.DisableLazyItemReaderFN,
+			},
+			flagsTD.PreparedGenericBackupFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	opts := utils.MakeGroupsOpts(cmd)
+	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
+	assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
+
+	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
+	assert.Equal(t, control.FailFast, co.FailureHandling)
+	assert.True(t, co.ToggleFeatures.DisableIncrementals)
+	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
+	assert.True(t, co.ToggleFeatures.DisableDelta)
+	assert.True(t, co.ToggleFeatures.DisableLazyItemReader)
+
+	assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups)
+	flagsTD.AssertGenericBackupFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *GroupsUnitSuite) TestBackupListFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: listCommand},
+		addGroupsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			groupsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedBackupListFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertBackupListFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *GroupsUnitSuite) TestBackupDetailsFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: detailsCommand},
+		addGroupsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			groupsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+				"--" + flags.SkipReduceFN,
+			},
+			flagsTD.PreparedChannelFlags(),
+			flagsTD.PreparedConversationFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags(),
+			flagsTD.PreparedLibraryFlags()))
+
+	co := utils.Control()
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	assert.True(t, co.SkipReduce)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+	flagsTD.AssertChannelFlags(t, cmd)
+	flagsTD.AssertConversationFlags(t, cmd)
+	flagsTD.AssertLibraryFlags(t, cmd)
+}
+
+func (suite *GroupsUnitSuite) TestBackupDeleteFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: deleteCommand},
+		addGroupsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			groupsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}

src/cli/backup/help_e2e_test.go

@@ -1,53 +0,0 @@
-package backup_test
-
-import (
-	"context"
-	"strings"
-	"testing"
-
-	"github.com/alcionai/clues"
-	"github.com/spf13/viper"
-	"github.com/stretchr/testify/require"
-
-	"github.com/alcionai/corso/src/cli/config"
-	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/pkg/account"
-	"github.com/alcionai/corso/src/pkg/control"
-	"github.com/alcionai/corso/src/pkg/repository"
-	"github.com/alcionai/corso/src/pkg/storage"
-)
-
-func prepM365Test(
-	t *testing.T,
-	ctx context.Context, //revive:disable-line:context-as-argument
-) (
-	account.Account,
-	storage.Storage,
-	repository.Repository,
-	*viper.Viper,
-	strings.Builder,
-	string,
-) {
-	var (
-		acct     = tester.NewM365Account(t)
-		st       = tester.NewPrefixedS3Storage(t)
-		recorder = strings.Builder{}
-	)
-
-	cfg, err := st.S3Config()
-	require.NoError(t, err, clues.ToCore(err))
-
-	force := map[string]string{
-		tester.TestCfgAccountProvider: "M365",
-		tester.TestCfgStorageProvider: "S3",
-		tester.TestCfgPrefix:          cfg.Prefix,
-	}
-
-	vpr, cfgFP := tester.MakeTempTestConfigClone(t, force)
-	ctx = config.SetViper(ctx, vpr)
-
-	repo, err := repository.Initialize(ctx, acct, st, control.Defaults())
-	require.NoError(t, err, clues.ToCore(err))
-
-	return acct, st, repo, vpr, recorder, cfgFP
-}

src/cli/backup/helpers_test.go

@@ -0,0 +1,98 @@
+package backup_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/alcionai/clues"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+
+	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/print"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/config"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/repository"
+	"github.com/alcionai/corso/src/pkg/storage"
+	"github.com/alcionai/corso/src/pkg/storage/testdata"
+)
+
+type dependencies struct {
+	st             storage.Storage
+	repo           repository.Repositoryer
+	vpr            *viper.Viper
+	recorder       strings.Builder
+	configFilePath string
+}
+
+func prepM365Test(
+	t *testing.T,
+	ctx context.Context, //revive:disable-line:context-as-argument
+	pst path.ServiceType,
+) dependencies {
+	var (
+		acct     = tconfig.NewM365Account(t)
+		st       = testdata.NewPrefixedS3Storage(t)
+		recorder = strings.Builder{}
+	)
+
+	cfg, err := st.ToS3Config()
+	require.NoError(t, err, clues.ToCore(err))
+
+	force := map[string]string{
+		tconfig.TestCfgAccountProvider: account.ProviderM365.String(),
+		tconfig.TestCfgStorageProvider: storage.ProviderS3.String(),
+		tconfig.TestCfgPrefix:          cfg.Prefix,
+	}
+
+	vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force)
+	ctx = config.SetViper(ctx, vpr)
+
+	repo, err := repository.New(
+		ctx,
+		acct,
+		st,
+		control.DefaultOptions(),
+		repository.NewRepoID)
+	require.NoError(t, err, clues.ToCore(err))
+
+	err = repo.Initialize(ctx, repository.InitConfig{
+		Service: pst,
+	})
+	require.NoError(t, err, clues.ToCore(err))
+
+	return dependencies{
+		st:             st,
+		repo:           repo,
+		vpr:            vpr,
+		recorder:       recorder,
+		configFilePath: cfgFP,
+	}
+}
+
+// ---------------------------------------------------------------------------
+// funcs
+// ---------------------------------------------------------------------------
+
+func buildExchangeBackupCmd(
+	ctx context.Context,
+	configFile, user, category string,
+	recorder *strings.Builder,
+) (*cobra.Command, context.Context) {
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "exchange",
+		"--"+flags.ConfigFileFN, configFile,
+		"--"+flags.UserFN, user,
+		"--"+flags.CategoryDataFN, category)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(recorder)
+
+	return cmd, print.SetRootCmd(ctx, cmd)
+}

src/cli/backup/onedrive.go

@@ -1,22 +1,15 @@
 package backup
 
 import (
-	"context"
-
 	"github.com/alcionai/clues"
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
 	"github.com/alcionai/corso/src/cli/flags"
 	. "github.com/alcionai/corso/src/cli/print"
-	"github.com/alcionai/corso/src/cli/repo"
 	"github.com/alcionai/corso/src/cli/utils"
-	"github.com/alcionai/corso/src/internal/data"
-	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/fault"
 	"github.com/alcionai/corso/src/pkg/path"
-	"github.com/alcionai/corso/src/pkg/repository"
 	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
@@ -27,7 +20,7 @@ import (
 const (
 	oneDriveServiceCommand                 = "onedrive"
 	oneDriveServiceCommandCreateUseSuffix  = "--user <email> | '" + flags.Wildcard + "'"
-	oneDriveServiceCommandDeleteUseSuffix  = "--backup <backupId>"
+	oneDriveServiceCommandDeleteUseSuffix  = "--backups <backupId>"
 	oneDriveServiceCommandDetailsUseSuffix = "--backup <backupId>"
 )
 
@@ -41,8 +34,9 @@ corso backup create onedrive --user alice@example.com,bob@example.com
 # Backup all OneDrive data for all M365 users 
 corso backup create onedrive --user '*'`
 
-	oneDriveServiceCommandDeleteExamples = `# Delete OneDrive backup with ID 1234abcd-12ab-cd34-56de-1234abcd
-corso backup delete onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd`
+	oneDriveServiceCommandDeleteExamples = `# Delete OneDrive backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
+and 1234abcd-12ab-cd34-56de-1234abce
+corso backup delete onedrive --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce`
 
 	oneDriveServiceCommandDetailsExamples = `# Explore items in Bob's latest backup (1234abcd...)
 corso backup details onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd
@@ -66,56 +60,43 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
 	switch cmd.Use {
 	case createCommand:
 		c, fs = utils.AddCommand(cmd, oneDriveCreateCmd())
-		fs.SortFlags = false
 
 		c.Use = c.Use + " " + oneDriveServiceCommandCreateUseSuffix
 		c.Example = oneDriveServiceCommandCreateExamples
 
 		flags.AddUserFlag(c)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
-
-		flags.AddFailFastFlag(c)
-		flags.AddDisableIncrementalsFlag(c)
+		flags.AddGenericBackupFlags(c)
+		fs.BoolVar(
+			&flags.UseOldDeltaProcessFV,
+			flags.UseOldDeltaProcessFN,
+			false,
+			"process backups using the old delta processor instead of tree-based enumeration")
+		cobra.CheckErr(fs.MarkHidden(flags.UseOldDeltaProcessFN))
 
 	case listCommand:
-		c, fs = utils.AddCommand(cmd, oneDriveListCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, oneDriveListCmd())
 
 		flags.AddBackupIDFlag(c, false)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
-		addFailedItemsFN(c)
-		addSkippedItemsFN(c)
-		addRecoveredErrorsFN(c)
+		flags.AddAllBackupListFlags(c)
 
 	case detailsCommand:
-		c, fs = utils.AddCommand(cmd, oneDriveDetailsCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, oneDriveDetailsCmd())
 
 		c.Use = c.Use + " " + oneDriveServiceCommandDetailsUseSuffix
 		c.Example = oneDriveServiceCommandDetailsExamples
 
 		flags.AddSkipReduceFlag(c)
 		flags.AddBackupIDFlag(c, true)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
 		flags.AddOneDriveDetailsAndRestoreFlags(c)
 
 	case deleteCommand:
-		c, fs = utils.AddCommand(cmd, oneDriveDeleteCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, oneDriveDeleteCmd())
 
 		c.Use = c.Use + " " + oneDriveServiceCommandDeleteUseSuffix
 		c.Example = oneDriveServiceCommandDeleteExamples
 
-		flags.AddBackupIDFlag(c, true)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
+		flags.AddMultipleBackupIDsFlag(c, false)
+		flags.AddBackupIDFlag(c, false)
 	}
 
 	return c
@@ -144,11 +125,18 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
 	if err := validateOneDriveBackupCreateFlags(flags.UserFV); err != nil {
 		return err
 	}
 
-	r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
+	r, acct, err := utils.AccountConnectAndWriteRepoConfig(
+		ctx,
+		cmd,
+		path.OneDriveService)
 	if err != nil {
 		return Only(ctx, err)
 	}
@@ -157,7 +145,12 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
 
 	sel := oneDriveBackupCreateSelectors(flags.UserFV)
 
-	ins, err := utils.UsersMap(ctx, *acct, fault.New(true))
+	ins, err := utils.UsersMap(
+		ctx,
+		*acct,
+		utils.Control(),
+		r.Counter(),
+		fault.New(true))
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
 	}
@@ -168,10 +161,10 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
 		selectorSet = append(selectorSet, discSel.Selector)
 	}
 
-	return runBackups(
+	return genericCreateCommand(
 		ctx,
 		r,
-		"OneDrive", "user",
+		"OneDrive",
 		selectorSet,
 		ins)
 }
@@ -231,69 +224,33 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	ctx := cmd.Context()
-	opts := utils.MakeOneDriveOpts(cmd)
-
-	r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
-	if err != nil {
-		return Only(ctx, err)
-	}
-
-	defer utils.CloseRepo(ctx, r)
-
-	ctrlOpts := utils.Control()
-
-	ds, err := runDetailsOneDriveCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
-	if err != nil {
-		return Only(ctx, err)
-	}
-
-	if len(ds.Entries) == 0 {
-		Info(ctx, selectors.ErrorNoMatchingItems)
+	if flags.RunModeFV == flags.RunModeFlagTest {
 		return nil
 	}
 
-	ds.PrintEntries(ctx)
-
-	return nil
+	return runDetailsOneDriveCmd(cmd)
 }
 
-// runDetailsOneDriveCmd actually performs the lookup in backup details.
-// the fault.Errors return is always non-nil.  Callers should check if
-// errs.Failure() == nil.
-func runDetailsOneDriveCmd(
-	ctx context.Context,
-	r repository.BackupGetter,
-	backupID string,
-	opts utils.OneDriveOpts,
-	skipReduce bool,
-) (*details.Details, error) {
-	if err := utils.ValidateOneDriveRestoreFlags(backupID, opts); err != nil {
-		return nil, err
+func runDetailsOneDriveCmd(cmd *cobra.Command) error {
+	ctx := cmd.Context()
+	opts := utils.MakeOneDriveOpts(cmd)
+
+	sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
+	sel.Configure(selectors.Config{OnlyMatchItemNames: true})
+	utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
+
+	ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
+	if err != nil {
+		return Only(ctx, err)
 	}
 
-	ctx = clues.Add(ctx, "backup_id", backupID)
-
-	d, _, errs := r.GetBackupDetails(ctx, backupID)
-	// TODO: log/track recoverable errors
-	if errs.Failure() != nil {
-		if errors.Is(errs.Failure(), data.ErrNotFound) {
-			return nil, clues.New("no backup exists with the id " + backupID)
-		}
-
-		return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
+	if len(ds.Entries) > 0 {
+		ds.PrintEntries(ctx)
+	} else {
+		Info(ctx, selectors.ErrorNoMatchingItems)
 	}
 
-	ctx = clues.Add(ctx, "details_entries", len(d.Entries))
-
-	if !skipReduce {
-		sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
-		sel.Configure(selectors.Config{OnlyMatchItemNames: true})
-		utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
-		d = sel.Reduce(ctx, d, errs)
-	}
-
-	return d, nil
+	return nil
 }
 
 // `corso backup delete onedrive [<flag>...]`
@@ -309,5 +266,15 @@ func oneDriveDeleteCmd() *cobra.Command {
 
 // deletes a oneDrive service backup.
 func deleteOneDriveCmd(cmd *cobra.Command, args []string) error {
-	return genericDeleteCommand(cmd, flags.BackupIDFV, "OneDrive", args)
+	backupIDValue := []string{}
+
+	if len(flags.BackupIDsFV) > 0 {
+		backupIDValue = flags.BackupIDsFV
+	} else if len(flags.BackupIDFV) > 0 {
+		backupIDValue = append(backupIDValue, flags.BackupIDFV)
+	} else {
+		return clues.New("either --backup or --backups flag is required")
+	}
+
+	return genericDeleteCommand(cmd, path.OneDriveService, "OneDrive", backupIDValue, args)
 }

src/cli/backup/onedrive_e2e_test.go

@@ -7,23 +7,23 @@ import (
 
 	"github.com/alcionai/clues"
 	"github.com/google/uuid"
-	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/cli"
-	"github.com/alcionai/corso/src/cli/config"
 	"github.com/alcionai/corso/src/cli/flags"
 	"github.com/alcionai/corso/src/cli/print"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/operations"
 	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/pkg/account"
-	"github.com/alcionai/corso/src/pkg/repository"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/config"
+	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/selectors"
 	selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
-	"github.com/alcionai/corso/src/pkg/storage"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
 )
 
 // ---------------------------------------------------------------------------
@@ -32,20 +32,14 @@ import (
 
 type NoBackupOneDriveE2ESuite struct {
 	tester.Suite
-	acct       account.Account
-	st         storage.Storage
-	vpr        *viper.Viper
-	cfgFP      string
-	repo       repository.Repository
-	m365UserID string
-	recorder   strings.Builder
+	dpnd dependencies
 }
 
 func TestNoBackupOneDriveE2ESuite(t *testing.T) {
 	suite.Run(t, &NoBackupOneDriveE2ESuite{
 		Suite: tester.NewE2ESuite(
 			t,
-			[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}),
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
 	})
 }
 
@@ -55,33 +49,25 @@ func (suite *NoBackupOneDriveE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.recorder = recorder
-	suite.vpr = vpr
-	suite.cfgFP = cfgFilePath
-	suite.m365UserID = tester.M365UserID(t)
+	suite.dpnd = prepM365Test(t, ctx, path.OneDriveService)
 }
 
 func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupListCmd_empty() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "list", "onedrive",
-		"--config-file", suite.cfgFP)
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
 	cli.BuildCommandTree(cmd)
 
-	cmd.SetErr(&suite.recorder)
+	cmd.SetErr(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -89,13 +75,13 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupListCmd_empty() {
 	err := cmd.ExecuteContext(ctx)
 	require.NoError(t, err, clues.ToCore(err))
 
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 
 	// as an offhand check: the result should contain the m365 user id
 	assert.True(t, strings.HasSuffix(result, "No backups available\n"))
 }
 
-func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_UserNotInTenant() {
+func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_userNotInTenant() {
 	recorder := strings.Builder{}
 
 	t := suite.T()
@@ -103,12 +89,12 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_UserNotInTenant() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "create", "onedrive",
-		"--config-file", suite.cfgFP,
-		"--"+flags.UserFN, "foo@nothere.com")
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.UserFN, "foo@not-there.com")
 	cli.BuildCommandTree(cmd)
 
 	cmd.SetOut(&recorder)
@@ -121,7 +107,8 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_UserNotInTenant() {
 	assert.Contains(
 		t,
 		err.Error(),
-		"not found in tenant", "error missing user not found")
+		"not found",
+		"error missing user not found")
 	assert.NotContains(t, err.Error(), "runtime error", "panic happened")
 
 	t.Logf("backup error message: %s", err.Error())
@@ -136,20 +123,15 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_UserNotInTenant() {
 
 type BackupDeleteOneDriveE2ESuite struct {
 	tester.Suite
-	acct     account.Account
-	st       storage.Storage
-	vpr      *viper.Viper
-	cfgFP    string
-	repo     repository.Repository
-	backupOp operations.BackupOperation
-	recorder strings.Builder
+	dpnd      dependencies
+	backupOps [3]operations.BackupOperation
 }
 
 func TestBackupDeleteOneDriveE2ESuite(t *testing.T) {
 	suite.Run(t, &BackupDeleteOneDriveE2ESuite{
 		Suite: tester.NewE2ESuite(
 			t,
-			[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}),
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
 	})
 }
 
@@ -159,17 +141,10 @@ func (suite *BackupDeleteOneDriveE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.recorder = recorder
-	suite.vpr = vpr
-	suite.cfgFP = cfgFilePath
+	suite.dpnd = prepM365Test(t, ctx, path.OneDriveService)
 
 	var (
-		m365UserID = tester.M365UserID(t)
+		m365UserID = tconfig.M365UserID(t)
 		users      = []string{m365UserID}
 		ins        = idname.NewCache(map[string]string{m365UserID: m365UserID})
 	)
@@ -178,31 +153,36 @@ func (suite *BackupDeleteOneDriveE2ESuite) SetupSuite() {
 	sel := selectors.NewOneDriveBackup(users)
 	sel.Include(selTD.OneDriveBackupFolderScope(sel))
 
-	backupOp, err := suite.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
-	require.NoError(t, err, clues.ToCore(err))
+	for i := 0; i < cap(suite.backupOps); i++ {
+		backupOp, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+		require.NoError(t, err, clues.ToCore(err))
 
-	suite.backupOp = backupOp
+		suite.backupOps[i] = backupOp
 
-	err = suite.backupOp.Run(ctx)
-	require.NoError(t, err, clues.ToCore(err))
+		err = suite.backupOps[i].Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+	}
 }
 
 func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "delete", "onedrive",
-		"--config-file", suite.cfgFP,
-		"--"+flags.BackupFN, string(suite.backupOp.Results.BackupID))
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN,
+		fmt.Sprintf("%s,%s",
+			string(suite.backupOps[0].Results.BackupID),
+			string(suite.backupOps[1].Results.BackupID)))
 	cli.BuildCommandTree(cmd)
-	cmd.SetErr(&suite.recorder)
+	cmd.SetErr(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -210,19 +190,61 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd() {
 	err := cmd.ExecuteContext(ctx)
 	require.NoError(t, err, clues.ToCore(err))
 
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 	assert.True(t,
 		strings.HasSuffix(
 			result,
-			fmt.Sprintf("Deleted OneDrive backup %s\n", string(suite.backupOp.Results.BackupID)),
-		),
-	)
+			fmt.Sprintf("Deleted OneDrive backup [%s %s]\n",
+				string(suite.backupOps[0].Results.BackupID),
+				string(suite.backupOps[1].Results.BackupID))))
 
 	// a follow-up details call should fail, due to the backup ID being deleted
-	cmd = tester.StubRootCmd(
+	cmd = cliTD.StubRootCmd(
 		"backup", "details", "onedrive",
-		"--config-file", suite.cfgFP,
-		"--backup", string(suite.backupOp.Results.BackupID))
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backups", string(suite.backupOps[0].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_SingleID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "onedrive",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupFN,
+		string(suite.backupOps[2].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+	cmd.SetErr(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := suite.dpnd.recorder.String()
+	assert.True(t,
+		strings.HasSuffix(
+			result,
+			fmt.Sprintf("Deleted OneDrive backup [%s]\n",
+				string(suite.backupOps[2].Results.BackupID))))
+
+	// a follow-up details call should fail, due to the backup ID being deleted
+	cmd = cliTD.StubRootCmd(
+		"backup", "details", "onedrive",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(suite.backupOps[0].Results.BackupID))
 	cli.BuildCommandTree(cmd)
 
 	err = cmd.ExecuteContext(ctx)
@@ -233,17 +255,35 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_unknownID
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "delete", "onedrive",
-		"--config-file", suite.cfgFP,
-		"--"+flags.BackupFN, uuid.NewString())
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN, uuid.NewString())
 	cli.BuildCommandTree(cmd)
 
 	// unknown backupIDs should error since the modelStore can't find the backup
 	err := cmd.ExecuteContext(ctx)
 	require.Error(t, err, clues.ToCore(err))
 }
+
+func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_NoBackupID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "onedrive",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	// empty backupIDs should error since no data provided
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}

src/cli/backup/onedrive_test.go

@@ -1,7 +1,6 @@
 package backup
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/alcionai/clues"
@@ -11,10 +10,11 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/cli/flags"
-	"github.com/alcionai/corso/src/cli/utils/testdata"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
 	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/internal/version"
-	dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
+	"github.com/alcionai/corso/src/pkg/control"
 )
 
 type OneDriveUnitSuite struct {
@@ -33,57 +33,35 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
 		use         string
 		expectUse   string
 		expectShort string
-		flags       []string
 		expectRunE  func(*cobra.Command, []string) error
 	}{
 		{
-			"create onedrive",
-			createCommand,
-			expectUse + " " + oneDriveServiceCommandCreateUseSuffix,
-			oneDriveCreateCmd().Short,
-			[]string{
-				flags.UserFN,
-				flags.DisableIncrementalsFN,
-				flags.FailFastFN,
-			},
-			createOneDriveCmd,
+			name:        "create onedrive",
+			use:         createCommand,
+			expectUse:   expectUse + " " + oneDriveServiceCommandCreateUseSuffix,
+			expectShort: oneDriveCreateCmd().Short,
+			expectRunE:  createOneDriveCmd,
 		},
 		{
-			"list onedrive",
-			listCommand,
-			expectUse,
-			oneDriveListCmd().Short,
-			[]string{
-				flags.BackupFN,
-				flags.FailedItemsFN,
-				flags.SkippedItemsFN,
-				flags.RecoveredErrorsFN,
-			},
-			listOneDriveCmd,
+			name:        "list onedrive",
+			use:         listCommand,
+			expectUse:   expectUse,
+			expectShort: oneDriveListCmd().Short,
+			expectRunE:  listOneDriveCmd,
 		},
 		{
-			"details onedrive",
-			detailsCommand,
-			expectUse + " " + oneDriveServiceCommandDetailsUseSuffix,
-			oneDriveDetailsCmd().Short,
-			[]string{
-				flags.BackupFN,
-				flags.FolderFN,
-				flags.FileFN,
-				flags.FileCreatedAfterFN,
-				flags.FileCreatedBeforeFN,
-				flags.FileModifiedAfterFN,
-				flags.FileModifiedBeforeFN,
-			},
-			detailsOneDriveCmd,
+			name:        "details onedrive",
+			use:         detailsCommand,
+			expectUse:   expectUse + " " + oneDriveServiceCommandDetailsUseSuffix,
+			expectShort: oneDriveDetailsCmd().Short,
+			expectRunE:  detailsOneDriveCmd,
 		},
 		{
-			"delete onedrive",
-			deleteCommand,
-			expectUse + " " + oneDriveServiceCommandDeleteUseSuffix,
-			oneDriveDeleteCmd().Short,
-			[]string{flags.BackupFN},
-			deleteOneDriveCmd,
+			name:        "delete onedrive",
+			use:         deleteCommand,
+			expectUse:   expectUse + " " + oneDriveServiceCommandDeleteUseSuffix,
+			expectShort: oneDriveDeleteCmd().Short,
+			expectRunE:  deleteOneDriveCmd,
 		},
 	}
 
@@ -103,14 +81,133 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
 			assert.Equal(t, test.expectUse, child.Use)
 			assert.Equal(t, test.expectShort, child.Short)
 			tester.AreSameFunc(t, test.expectRunE, child.RunE)
-
-			for _, f := range test.flags {
-				assert.NotNil(t, c.Flag(f), f+" flag")
-			}
 		})
 	}
 }
 
+func (suite *OneDriveUnitSuite) TestBackupCreateFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: createCommand},
+		addOneDriveCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			oneDriveServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.UserFN, flagsTD.FlgInputs(flagsTD.UsersInput),
+			},
+			flagsTD.PreparedGenericBackupFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	opts := utils.MakeOneDriveOpts(cmd)
+	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
+
+	assert.Equal(t, control.FailFast, co.FailureHandling)
+	assert.True(t, co.ToggleFeatures.DisableIncrementals)
+	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
+
+	assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
+	flagsTD.AssertGenericBackupFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *OneDriveUnitSuite) TestBackupListFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: listCommand},
+		addOneDriveCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			oneDriveServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedBackupListFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertBackupListFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *OneDriveUnitSuite) TestBackupDetailsFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: detailsCommand},
+		addOneDriveCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			oneDriveServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+				"--" + flags.SkipReduceFN,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	co := utils.Control()
+
+	assert.True(t, co.SkipReduce)
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *OneDriveUnitSuite) TestBackupDeleteFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: deleteCommand},
+		addOneDriveCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			oneDriveServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
 func (suite *OneDriveUnitSuite) TestValidateOneDriveBackupCreateFlags() {
 	table := []struct {
 		name   string
@@ -134,51 +231,3 @@ func (suite *OneDriveUnitSuite) TestValidateOneDriveBackupCreateFlags() {
 		})
 	}
 }
-
-func (suite *OneDriveUnitSuite) TestOneDriveBackupDetailsSelectors() {
-	for v := 0; v <= version.Backup; v++ {
-		suite.Run(fmt.Sprintf("version%d", v), func() {
-			for _, test := range testdata.OneDriveOptionDetailLookups {
-				suite.Run(test.Name, func() {
-					t := suite.T()
-
-					ctx, flush := tester.NewContext(t)
-					defer flush()
-
-					bg := testdata.VersionedBackupGetter{
-						Details: dtd.GetDetailsSetForVersion(t, v),
-					}
-
-					output, err := runDetailsOneDriveCmd(
-						ctx,
-						bg,
-						"backup-ID",
-						test.Opts(t, v),
-						false)
-					assert.NoError(t, err, clues.ToCore(err))
-					assert.ElementsMatch(t, test.Expected(t, v), output.Entries)
-				})
-			}
-		})
-	}
-}
-
-func (suite *OneDriveUnitSuite) TestOneDriveBackupDetailsSelectorsBadFormats() {
-	for _, test := range testdata.BadOneDriveOptionsFormats {
-		suite.Run(test.Name, func() {
-			t := suite.T()
-
-			ctx, flush := tester.NewContext(t)
-			defer flush()
-
-			output, err := runDetailsOneDriveCmd(
-				ctx,
-				test.BackupGetter,
-				"backup-ID",
-				test.Opts(t, version.Backup),
-				false)
-			assert.Error(t, err, clues.ToCore(err))
-			assert.Empty(t, output)
-		})
-	}
-}

src/cli/backup/sharepoint.go

@@ -4,22 +4,16 @@ import (
 	"context"
 
 	"github.com/alcionai/clues"
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 	"golang.org/x/exp/slices"
 
 	"github.com/alcionai/corso/src/cli/flags"
 	. "github.com/alcionai/corso/src/cli/print"
-	"github.com/alcionai/corso/src/cli/repo"
 	"github.com/alcionai/corso/src/cli/utils"
 	"github.com/alcionai/corso/src/internal/common/idname"
-	"github.com/alcionai/corso/src/internal/data"
-	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/fault"
 	"github.com/alcionai/corso/src/pkg/filters"
 	"github.com/alcionai/corso/src/pkg/path"
-	"github.com/alcionai/corso/src/pkg/repository"
 	"github.com/alcionai/corso/src/pkg/selectors"
 	"github.com/alcionai/corso/src/pkg/services/m365"
 )
@@ -28,15 +22,10 @@ import (
 // setup and globals
 // ------------------------------------------------------------------------------------------------
 
-const (
-	dataLibraries = "libraries"
-	dataPages     = "pages"
-)
-
 const (
 	sharePointServiceCommand                 = "sharepoint"
 	sharePointServiceCommandCreateUseSuffix  = "--site <siteURL> | '" + flags.Wildcard + "'"
-	sharePointServiceCommandDeleteUseSuffix  = "--backup <backupId>"
+	sharePointServiceCommandDeleteUseSuffix  = "--backups <backupId>"
 	sharePointServiceCommandDetailsUseSuffix = "--backup <backupId>"
 )
 
@@ -48,10 +37,15 @@ corso backup create sharepoint --site https://example.com/hr
 corso backup create sharepoint --site https://example.com/hr,https://example.com/team
 
 # Backup all SharePoint data for all Sites
-corso backup create sharepoint --site '*'`
+corso backup create sharepoint --site '*'
 
-	sharePointServiceCommandDeleteExamples = `# Delete SharePoint backup with ID 1234abcd-12ab-cd34-56de-1234abcd
-corso backup delete sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd`
+# Backup all SharePoint list data for a Site
+corso backup create sharepoint --site https://example.com/hr --data lists
+`
+
+	sharePointServiceCommandDeleteExamples = `# Delete SharePoint backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
+and 1234abcd-12ab-cd34-56de-1234abce
+corso backup delete sharepoint --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce`
 
 	sharePointServiceCommandDetailsExamples = `# Explore items in the HR site's latest backup (1234abcd...)
 corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd
@@ -67,70 +61,70 @@ corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
 # Explore all files within the document library "Work Documents"
 corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
     --library "Work Documents"
-`
+
+# Explore lists by their name(s)
+corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list "list-name-1,list-name-2"
+
+# Explore lists created after a given time
+corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-created-after 2024-01-01T12:23:34
+
+# Explore lists created before a given time
+corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-created-before 2024-01-01T12:23:34
+
+# Explore lists modified before a given time
+corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-modified-before 2024-01-01T12:23:34
+
+# Explore lists modified after a given time
+corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-modified-after 2024-01-01T12:23:34`
 )
 
 // called by backup.go to map subcommands to provider-specific handling.
 func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
-	var (
-		c  *cobra.Command
-		fs *pflag.FlagSet
-	)
+	var c *cobra.Command
 
 	switch cmd.Use {
 	case createCommand:
-		c, fs = utils.AddCommand(cmd, sharePointCreateCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, sharePointCreateCmd())
 
 		c.Use = c.Use + " " + sharePointServiceCommandCreateUseSuffix
 		c.Example = sharePointServiceCommandCreateExamples
 
-		flags.AddSiteFlag(c)
-		flags.AddSiteIDFlag(c)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
-		flags.AddDataFlag(c, []string{dataLibraries}, true)
-		flags.AddFailFastFlag(c)
-		flags.AddDisableIncrementalsFlag(c)
+		flags.AddSiteFlag(c, true)
+		flags.AddSiteIDFlag(c, true)
+		// [TODO](hitesh) to add lists flag to invoke backup for lists
+		// when explicit invoke is not required anymore
+		flags.AddDataFlag(c, []string{flags.DataLibraries}, true)
+		flags.AddGenericBackupFlags(c)
 
 	case listCommand:
-		c, fs = utils.AddCommand(cmd, sharePointListCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, sharePointListCmd())
 
 		flags.AddBackupIDFlag(c, false)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
-		addFailedItemsFN(c)
-		addSkippedItemsFN(c)
-		addRecoveredErrorsFN(c)
+		flags.AddAllBackupListFlags(c)
 
 	case detailsCommand:
-		c, fs = utils.AddCommand(cmd, sharePointDetailsCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, sharePointDetailsCmd())
 
 		c.Use = c.Use + " " + sharePointServiceCommandDetailsUseSuffix
 		c.Example = sharePointServiceCommandDetailsExamples
 
 		flags.AddSkipReduceFlag(c)
 		flags.AddBackupIDFlag(c, true)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
 		flags.AddSharePointDetailsAndRestoreFlags(c)
 
 	case deleteCommand:
-		c, fs = utils.AddCommand(cmd, sharePointDeleteCmd())
-		fs.SortFlags = false
+		c, _ = utils.AddCommand(cmd, sharePointDeleteCmd())
 
 		c.Use = c.Use + " " + sharePointServiceCommandDeleteUseSuffix
 		c.Example = sharePointServiceCommandDeleteExamples
 
-		flags.AddBackupIDFlag(c, true)
-		flags.AddCorsoPassphaseFlags(c)
-		flags.AddAWSCredsFlags(c)
-		flags.AddAzureCredsFlags(c)
+		flags.AddMultipleBackupIDsFlag(c, false)
+		flags.AddBackupIDFlag(c, false)
 	}
 
 	return c
@@ -159,11 +153,18 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
 	if err := validateSharePointBackupCreateFlags(flags.SiteIDFV, flags.WebURLFV, flags.CategoryDataFV); err != nil {
 		return err
 	}
 
-	r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
+	r, acct, err := utils.AccountConnectAndWriteRepoConfig(
+		ctx,
+		cmd,
+		path.SharePointService)
 	if err != nil {
 		return Only(ctx, err)
 	}
@@ -173,7 +174,12 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
 	// TODO: log/print recoverable errors
 	errs := fault.New(false)
 
-	ins, err := m365.SitesMap(ctx, *acct, errs)
+	svcCli, err := m365.NewM365Client(ctx, *acct)
+	if err != nil {
+		return Only(ctx, clues.Stack(err))
+	}
+
+	ins, err := svcCli.SitesMap(ctx, errs)
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 sites"))
 	}
@@ -189,10 +195,10 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
 		selectorSet = append(selectorSet, discSel.Selector)
 	}
 
-	return runBackups(
+	return genericCreateCommand(
 		ctx,
 		r,
-		"SharePoint", "site",
+		"SharePoint",
 		selectorSet,
 		ins)
 }
@@ -202,15 +208,15 @@ func validateSharePointBackupCreateFlags(sites, weburls, cats []string) error {
 		return clues.New(
 			"requires one or more --" +
 				flags.SiteFN + " urls, or the wildcard --" +
-				flags.SiteFN + " *",
-		)
+				flags.SiteFN + " *")
 	}
 
+	allowedCats := utils.SharePointAllowedCategories()
+
 	for _, d := range cats {
-		if d != dataLibraries && d != dataPages {
+		if _, ok := allowedCats[d]; !ok {
 			return clues.New(
-				d + " is an unrecognized data type; either  " + dataLibraries + "or " + dataPages,
-			)
+				d + " is an unrecognized data type; only  " + flags.DataLibraries + " supported")
 		}
 	}
 
@@ -237,29 +243,11 @@ func sharePointBackupCreateSelectors(
 
 	sel := selectors.NewSharePointBackup(append(slices.Clone(sites), weburls...))
 
-	return addCategories(sel, cats), nil
+	return utils.AddCategories(sel, cats), nil
 }
 
 func includeAllSitesWithCategories(ins idname.Cacher, categories []string) *selectors.SharePointBackup {
-	return addCategories(selectors.NewSharePointBackup(ins.IDs()), categories)
-}
-
-func addCategories(sel *selectors.SharePointBackup, cats []string) *selectors.SharePointBackup {
-	// Issue #2631: Libraries are the only supported feature for SharePoint at this time.
-	if len(cats) == 0 {
-		sel.Include(sel.LibraryFolders(selectors.Any()))
-	}
-
-	for _, d := range cats {
-		switch d {
-		case dataLibraries:
-			sel.Include(sel.LibraryFolders(selectors.Any()))
-		case dataPages:
-			sel.Include(sel.Pages(selectors.Any()))
-		}
-	}
-
-	return sel
+	return utils.AddCategories(selectors.NewSharePointBackup(ins.IDs()), categories)
 }
 
 // ------------------------------------------------------------------------------------------------
@@ -298,14 +286,24 @@ func sharePointDeleteCmd() *cobra.Command {
 
 // deletes a sharePoint service backup.
 func deleteSharePointCmd(cmd *cobra.Command, args []string) error {
-	return genericDeleteCommand(cmd, flags.BackupIDFV, "SharePoint", args)
+	backupIDValue := []string{}
+
+	if len(flags.BackupIDsFV) > 0 {
+		backupIDValue = flags.BackupIDsFV
+	} else if len(flags.BackupIDFV) > 0 {
+		backupIDValue = append(backupIDValue, flags.BackupIDFV)
+	} else {
+		return clues.New("either --backup or --backups flag is required")
+	}
+
+	return genericDeleteCommand(cmd, path.SharePointService, "SharePoint", backupIDValue, args)
 }
 
 // ------------------------------------------------------------------------------------------------
 // backup details
 // ------------------------------------------------------------------------------------------------
 
-// `corso backup details onedrive [<flag>...]`
+// `corso backup details SharePoint [<flag>...]`
 func sharePointDetailsCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:     sharePointServiceCommand,
@@ -322,67 +320,31 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	ctx := cmd.Context()
-	opts := utils.MakeSharePointOpts(cmd)
-
-	r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
-	if err != nil {
-		return Only(ctx, err)
-	}
-
-	defer utils.CloseRepo(ctx, r)
-
-	ctrlOpts := utils.Control()
-
-	ds, err := runDetailsSharePointCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
-	if err != nil {
-		return Only(ctx, err)
-	}
-
-	if len(ds.Entries) == 0 {
-		Info(ctx, selectors.ErrorNoMatchingItems)
+	if flags.RunModeFV == flags.RunModeFlagTest {
 		return nil
 	}
 
-	ds.PrintEntries(ctx)
+	return runDetailsSharePointCmd(cmd)
+}
+
+func runDetailsSharePointCmd(cmd *cobra.Command) error {
+	ctx := cmd.Context()
+	opts := utils.MakeSharePointOpts(cmd)
+
+	sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
+	sel.Configure(selectors.Config{OnlyMatchItemNames: true})
+	utils.FilterSharePointRestoreInfoSelectors(sel, opts)
+
+	ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	if len(ds.Entries) > 0 {
+		ds.PrintEntries(ctx)
+	} else {
+		Info(ctx, selectors.ErrorNoMatchingItems)
+	}
 
 	return nil
 }
-
-// runDetailsSharePointCmd actually performs the lookup in backup details.
-// the fault.Errors return is always non-nil.  Callers should check if
-// errs.Failure() == nil.
-func runDetailsSharePointCmd(
-	ctx context.Context,
-	r repository.BackupGetter,
-	backupID string,
-	opts utils.SharePointOpts,
-	skipReduce bool,
-) (*details.Details, error) {
-	if err := utils.ValidateSharePointRestoreFlags(backupID, opts); err != nil {
-		return nil, err
-	}
-
-	ctx = clues.Add(ctx, "backup_id", backupID)
-
-	d, _, errs := r.GetBackupDetails(ctx, backupID)
-	// TODO: log/track recoverable errors
-	if errs.Failure() != nil {
-		if errors.Is(errs.Failure(), data.ErrNotFound) {
-			return nil, clues.New("no backup exists with the id " + backupID)
-		}
-
-		return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
-	}
-
-	ctx = clues.Add(ctx, "details_entries", len(d.Entries))
-
-	if !skipReduce {
-		sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
-		sel.Configure(selectors.Config{OnlyMatchItemNames: true})
-		utils.FilterSharePointRestoreInfoSelectors(sel, opts)
-		d = sel.Reduce(ctx, d, errs)
-	}
-
-	return d, nil
-}

src/cli/backup/sharepoint_e2e_test.go

@@ -1,51 +1,48 @@
 package backup_test
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/alcionai/clues"
 	"github.com/google/uuid"
-	"github.com/spf13/viper"
+	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/cli"
-	"github.com/alcionai/corso/src/cli/config"
 	"github.com/alcionai/corso/src/cli/flags"
 	"github.com/alcionai/corso/src/cli/print"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/operations"
 	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/pkg/account"
-	"github.com/alcionai/corso/src/pkg/repository"
+	"github.com/alcionai/corso/src/internal/tester/its"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/backup/details"
+	"github.com/alcionai/corso/src/pkg/config"
+	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/selectors"
 	"github.com/alcionai/corso/src/pkg/selectors/testdata"
-	"github.com/alcionai/corso/src/pkg/storage"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
 )
 
 // ---------------------------------------------------------------------------
-// tests with no prior backup
+// tests that require no existing backups
 // ---------------------------------------------------------------------------
 
 type NoBackupSharePointE2ESuite struct {
 	tester.Suite
-	acct       account.Account
-	st         storage.Storage
-	vpr        *viper.Viper
-	cfgFP      string
-	repo       repository.Repository
-	m365SiteID string
-	recorder   strings.Builder
+	dpnd dependencies
 }
 
 func TestNoBackupSharePointE2ESuite(t *testing.T) {
 	suite.Run(t, &NoBackupSharePointE2ESuite{Suite: tester.NewE2ESuite(
 		t,
-		[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-	)})
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
 }
 
 func (suite *NoBackupSharePointE2ESuite) SetupSuite() {
@@ -54,33 +51,25 @@ func (suite *NoBackupSharePointE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.vpr = vpr
-	suite.recorder = recorder
-	suite.cfgFP = cfgFilePath
-	suite.m365SiteID = tester.M365SiteID(t)
+	suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
 }
 
 func (suite *NoBackupSharePointE2ESuite) TestSharePointBackupListCmd_empty() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "list", "sharepoint",
-		"--config-file", suite.cfgFP)
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
 	cli.BuildCommandTree(cmd)
 
-	cmd.SetErr(&suite.recorder)
+	cmd.SetErr(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -88,32 +77,319 @@ func (suite *NoBackupSharePointE2ESuite) TestSharePointBackupListCmd_empty() {
 	err := cmd.ExecuteContext(ctx)
 	require.NoError(t, err, clues.ToCore(err))
 
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 
 	// as an offhand check: the result should contain the m365 sitet id
 	assert.True(t, strings.HasSuffix(result, "No backups available\n"))
 }
 
+// ---------------------------------------------------------------------------
+// tests with no prior backup
+// ---------------------------------------------------------------------------
+
+type BackupSharepointE2ESuite struct {
+	tester.Suite
+	dpnd dependencies
+	m365 its.M365IntgTestSetup
+}
+
+func TestBackupSharepointE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupSharepointE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *BackupSharepointE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
+}
+
+func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_lists() {
+	// Issue: https://github.com/alcionai/corso/issues/4754
+	suite.T().Skip("unskip when sharepoint lists support is enabled")
+	runSharepointBackupCategoryTest(suite, flags.DataLists)
+}
+
+func runSharepointBackupCategoryTest(suite *BackupSharepointE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildSharepointBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		suite.m365.Site.ID,
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_siteNotFound_lists() {
+	// Issue: https://github.com/alcionai/corso/issues/4754
+	suite.T().Skip("un-skip test when lists support is enabled")
+	runSharepointBackupSiteNotFoundTest(suite, flags.DataLists)
+}
+
+func runSharepointBackupSiteNotFoundTest(suite *BackupSharepointE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildSharepointBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		uuid.NewString(),
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+	assert.Contains(
+		t,
+		err.Error(),
+		"Invalid hostname for this tenancy", "error missing site not found")
+	assert.NotContains(t, err.Error(), "runtime error", "panic happened")
+
+	t.Logf("backup error message: %s", err.Error())
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+// ---------------------------------------------------------------------------
+// tests prepared with a previous backup
+// ---------------------------------------------------------------------------
+
+type PreparedBackupSharepointE2ESuite struct {
+	tester.Suite
+	dpnd      dependencies
+	backupOps map[path.CategoryType]string
+	m365      its.M365IntgTestSetup
+}
+
+func TestPreparedBackupSharepointE2ESuite(t *testing.T) {
+	suite.Run(t, &PreparedBackupSharepointE2ESuite{
+		Suite: tester.NewE2ESuite(
+			t,
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
+	})
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
+	suite.backupOps = make(map[path.CategoryType]string)
+
+	var (
+		sites = []string{suite.m365.Site.ID}
+		ins   = idname.NewCache(map[string]string{suite.m365.Site.ID: suite.m365.Site.ID})
+		cats  = []path.CategoryType{
+			path.ListsCategory,
+		}
+	)
+
+	for _, set := range cats {
+		var (
+			sel    = selectors.NewSharePointBackup(sites)
+			scopes []selectors.SharePointScope
+		)
+
+		switch set {
+		case path.ListsCategory:
+			scopes = testdata.SharePointBackupListsScope(sel)
+		}
+
+		sel.Include(scopes)
+
+		bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+		require.NoError(t, err, clues.ToCore(err))
+
+		err = bop.Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+
+		bIDs := string(bop.Results.BackupID)
+
+		// sanity check, ensure we can find the backup and its details immediately
+		b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
+		require.NoError(t, err, "retrieving recent backup by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
+
+		_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
+		require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
+		require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
+
+		suite.backupOps[set] = string(b.ID)
+	}
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_lists() {
+	runSharepointListCmdTest(suite, path.ListsCategory)
+}
+
+func runSharepointListCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "sharepoint",
+		"--config-file", suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+	assert.Contains(t, result, suite.backupOps[category])
+
+	t.Log("backup results", result)
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_badID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "sharepoint",
+		"--config-file", suite.dpnd.configFilePath,
+		"--backup", uuid.NewString())
+	cli.BuildCommandTree(cmd)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) TestSharepointDetailsCmd_lists() {
+	runSharepointDetailsCmdTest(suite, path.ListsCategory)
+}
+
+func runSharepointDetailsCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	bID := suite.backupOps[category]
+
+	// fetch the details from the repo first
+	deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
+	require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
+	require.Empty(t, errs.Recovered())
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "details", "sharepoint",
+		"--config-file", suite.dpnd.configFilePath,
+		"--"+flags.BackupFN, string(bID))
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+
+	i := 0
+	findings := make(map[path.CategoryType]int)
+
+	incrementor := func(cond bool, cat path.CategoryType) {
+		if cond {
+			findings[cat]++
+		}
+	}
+
+	for _, ent := range deets.Entries {
+		if ent.SharePoint == nil {
+			continue
+		}
+
+		isSharePointList := ent.SharePoint.ItemType == details.SharePointList
+		hasListName := isSharePointList && len(ent.SharePoint.List.Name) > 0
+		hasItemName := !isSharePointList && len(ent.SharePoint.ItemName) > 0
+
+		incrementor(hasListName, category)
+		incrementor(hasItemName, category)
+
+		suite.Run(fmt.Sprintf("detail %d", i), func() {
+			assert.Contains(suite.T(), result, ent.ShortRef)
+		})
+
+		i++
+	}
+
+	assert.GreaterOrEqual(t, findings[category], 1)
+}
+
 // ---------------------------------------------------------------------------
 // tests for deleting backups
 // ---------------------------------------------------------------------------
 
 type BackupDeleteSharePointE2ESuite struct {
 	tester.Suite
-	acct     account.Account
-	st       storage.Storage
-	vpr      *viper.Viper
-	cfgFP    string
-	repo     repository.Repository
-	backupOp operations.BackupOperation
-	recorder strings.Builder
+	dpnd              dependencies
+	backupOp          operations.BackupOperation
+	secondaryBackupOp operations.BackupOperation
 }
 
 func TestBackupDeleteSharePointE2ESuite(t *testing.T) {
 	suite.Run(t, &BackupDeleteSharePointE2ESuite{
 		Suite: tester.NewE2ESuite(
 			t,
-			[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}),
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
 	})
 }
 
@@ -123,17 +399,10 @@ func (suite *BackupDeleteSharePointE2ESuite) SetupSuite() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
-
-	suite.acct = acct
-	suite.st = st
-	suite.repo = repo
-	suite.vpr = vpr
-	suite.recorder = recorder
-	suite.cfgFP = cfgFilePath
+	suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
 
 	var (
-		m365SiteID = tester.M365SiteID(t)
+		m365SiteID = tconfig.M365SiteID(t)
 		sites      = []string{m365SiteID}
 		ins        = idname.NewCache(map[string]string{m365SiteID: m365SiteID})
 	)
@@ -142,31 +411,43 @@ func (suite *BackupDeleteSharePointE2ESuite) SetupSuite() {
 	sel := selectors.NewSharePointBackup(sites)
 	sel.Include(testdata.SharePointBackupFolderScope(sel))
 
-	backupOp, err := suite.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+	backupOp, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
 	require.NoError(t, err, clues.ToCore(err))
 
 	suite.backupOp = backupOp
 
 	err = suite.backupOp.Run(ctx)
 	require.NoError(t, err, clues.ToCore(err))
+
+	// secondary backup
+	secondaryBackupOp, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+	require.NoError(t, err, clues.ToCore(err))
+
+	suite.secondaryBackupOp = secondaryBackupOp
+
+	err = suite.secondaryBackupOp.Run(ctx)
+	require.NoError(t, err, clues.ToCore(err))
 }
 
 func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	suite.recorder.Reset()
+	suite.dpnd.recorder.Reset()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "delete", "sharepoint",
-		"--config-file", suite.cfgFP,
-		"--"+flags.BackupFN, string(suite.backupOp.Results.BackupID))
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN,
+		fmt.Sprintf("%s,%s",
+			string(suite.backupOp.Results.BackupID),
+			string(suite.secondaryBackupOp.Results.BackupID)))
 	cli.BuildCommandTree(cmd)
-	cmd.SetErr(&suite.recorder)
+	cmd.SetErr(&suite.dpnd.recorder)
 
 	ctx = print.SetRootCmd(ctx, cmd)
 
@@ -174,20 +455,20 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd() {
 	err := cmd.ExecuteContext(ctx)
 	require.NoError(t, err, clues.ToCore(err))
 
-	result := suite.recorder.String()
+	result := suite.dpnd.recorder.String()
 	assert.True(t,
 		strings.HasSuffix(
 			result,
-			fmt.Sprintf("Deleted SharePoint backup %s\n", string(suite.backupOp.Results.BackupID)),
-		),
-	)
+			fmt.Sprintf("Deleted SharePoint backup [%s %s]\n",
+				string(suite.backupOp.Results.BackupID),
+				string(suite.secondaryBackupOp.Results.BackupID))))
 }
 
 // moved out of the func above to make the linter happy
 // // a follow-up details call should fail, due to the backup ID being deleted
-// cmd = tester.StubRootCmd(
+// cmd = cliTD.StubRootCmd(
 // 	"backup", "details", "sharepoint",
-// 	"--config-file", suite.cfgFP,
+// 	"--"+flags.ConfigFileFN, suite.cfgFP,
 // 	"--backup", string(suite.backupOp.Results.BackupID))
 // cli.BuildCommandTree(cmd)
 
@@ -198,17 +479,55 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd_unkno
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
-	ctx = config.SetViper(ctx, suite.vpr)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
 
 	defer flush()
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"backup", "delete", "sharepoint",
-		"--config-file", suite.cfgFP,
-		"--"+flags.BackupFN, uuid.NewString())
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN, uuid.NewString())
 	cli.BuildCommandTree(cmd)
 
 	// unknown backupIDs should error since the modelStore can't find the backup
 	err := cmd.ExecuteContext(ctx)
 	require.Error(t, err, clues.ToCore(err))
 }
+
+func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd_NoBackupID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "groups",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	// empty backupIDs should error since no data provided
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func buildSharepointBackupCmd(
+	ctx context.Context,
+	configFile, site, category string,
+	recorder *strings.Builder,
+) (*cobra.Command, context.Context) {
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "sharepoint",
+		"--config-file", configFile,
+		"--"+flags.SiteIDFN, site,
+		"--"+flags.CategoryDataFN, category)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(recorder)
+
+	return cmd, print.SetRootCmd(ctx, cmd)
+}

src/cli/backup/sharepoint_test.go

@@ -1,7 +1,7 @@
 package backup
 
 import (
-	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/alcionai/clues"
@@ -11,11 +11,12 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/cli/flags"
-	"github.com/alcionai/corso/src/cli/utils/testdata"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
 	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/internal/version"
-	dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
+	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
@@ -35,58 +36,35 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
 		use         string
 		expectUse   string
 		expectShort string
-		flags       []string
 		expectRunE  func(*cobra.Command, []string) error
 	}{
 		{
-			"create sharepoint",
-			createCommand,
-			expectUse + " " + sharePointServiceCommandCreateUseSuffix,
-			sharePointCreateCmd().Short,
-			[]string{
-				flags.SiteFN,
-				flags.DisableIncrementalsFN,
-				flags.FailFastFN,
-			},
-			createSharePointCmd,
+			name:        "create sharepoint",
+			use:         createCommand,
+			expectUse:   expectUse + " " + sharePointServiceCommandCreateUseSuffix,
+			expectShort: sharePointCreateCmd().Short,
+			expectRunE:  createSharePointCmd,
 		},
 		{
-			"list sharepoint",
-			listCommand,
-			expectUse,
-			sharePointListCmd().Short,
-			[]string{
-				flags.BackupFN,
-				flags.FailedItemsFN,
-				flags.SkippedItemsFN,
-				flags.RecoveredErrorsFN,
-			},
-			listSharePointCmd,
+			name:        "list sharepoint",
+			use:         listCommand,
+			expectUse:   expectUse,
+			expectShort: sharePointListCmd().Short,
+			expectRunE:  listSharePointCmd,
 		},
 		{
-			"details sharepoint",
-			detailsCommand,
-			expectUse + " " + sharePointServiceCommandDetailsUseSuffix,
-			sharePointDetailsCmd().Short,
-			[]string{
-				flags.BackupFN,
-				flags.LibraryFN,
-				flags.FolderFN,
-				flags.FileFN,
-				flags.FileCreatedAfterFN,
-				flags.FileCreatedBeforeFN,
-				flags.FileModifiedAfterFN,
-				flags.FileModifiedBeforeFN,
-			},
-			detailsSharePointCmd,
+			name:        "details sharepoint",
+			use:         detailsCommand,
+			expectUse:   expectUse + " " + sharePointServiceCommandDetailsUseSuffix,
+			expectShort: sharePointDetailsCmd().Short,
+			expectRunE:  detailsSharePointCmd,
 		},
 		{
-			"delete sharepoint",
-			deleteCommand,
-			expectUse + " " + sharePointServiceCommandDeleteUseSuffix,
-			sharePointDeleteCmd().Short,
-			[]string{flags.BackupFN},
-			deleteSharePointCmd,
+			name:        "delete sharepoint",
+			use:         deleteCommand,
+			expectUse:   expectUse + " " + sharePointServiceCommandDeleteUseSuffix,
+			expectShort: sharePointDeleteCmd().Short,
+			expectRunE:  deleteSharePointCmd,
 		},
 	}
 	for _, test := range table {
@@ -105,19 +83,142 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
 			assert.Equal(t, test.expectUse, child.Use)
 			assert.Equal(t, test.expectShort, child.Short)
 			tester.AreSameFunc(t, test.expectRunE, child.RunE)
-
-			for _, f := range test.flags {
-				assert.NotNil(t, c.Flag(f), f+" flag")
-			}
 		})
 	}
 }
 
+func (suite *SharePointUnitSuite) TestBackupCreateFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: createCommand},
+		addSharePointCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			sharePointServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.SiteIDFN, flagsTD.FlgInputs(flagsTD.SiteIDInput),
+				"--" + flags.SiteFN, flagsTD.FlgInputs(flagsTD.WebURLInput),
+				"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.SharepointCategoryDataInput),
+			},
+			flagsTD.PreparedGenericBackupFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	opts := utils.MakeSharePointOpts(cmd)
+	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
+
+	assert.Equal(t, control.FailFast, co.FailureHandling)
+	assert.True(t, co.ToggleFeatures.DisableIncrementals)
+	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
+
+	assert.ElementsMatch(t, []string{strings.Join(flagsTD.SiteIDInput, ",")}, opts.SiteID)
+	assert.ElementsMatch(t, flagsTD.WebURLInput, opts.WebURL)
+	flagsTD.AssertGenericBackupFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *SharePointUnitSuite) TestBackupListFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: listCommand},
+		addSharePointCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			sharePointServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedBackupListFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertBackupListFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *SharePointUnitSuite) TestBackupDetailsFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: detailsCommand},
+		addSharePointCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			sharePointServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+				"--" + flags.SkipReduceFN,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	co := utils.Control()
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	assert.True(t, co.SkipReduce)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *SharePointUnitSuite) TestBackupDeleteFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: deleteCommand},
+		addSharePointCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			sharePointServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
 func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
 	table := []struct {
 		name   string
 		site   []string
 		weburl []string
+		cats   []string
 		expect assert.ErrorAssertionFunc
 	}{
 		{
@@ -125,25 +226,61 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
 			expect: assert.Error,
 		},
 		{
-			name:   "sites",
+			name:   "sites but no category",
 			site:   []string{"smarf"},
 			expect: assert.NoError,
 		},
 		{
-			name:   "urls",
+			name:   "web urls but no category",
 			weburl: []string{"fnord"},
 			expect: assert.NoError,
 		},
 		{
-			name:   "both",
+			name:   "both web urls and sites but no category",
 			site:   []string{"smarf"},
 			weburl: []string{"fnord"},
 			expect: assert.NoError,
 		},
+		{
+			name:   "site with libraries category",
+			site:   []string{"smarf"},
+			cats:   []string{flags.DataLibraries},
+			expect: assert.NoError,
+		},
+		{
+			name:   "site with invalid category",
+			site:   []string{"smarf"},
+			cats:   []string{"invalid category"},
+			expect: assert.Error,
+		},
+		{
+			name:   "site with lists category",
+			site:   []string{"smarf"},
+			cats:   []string{flags.DataLists},
+			expect: assert.NoError,
+		},
+
+		// [TODO]: Uncomment when pages are enabled
+
+		// {
+		// 	name:   "site with pages category",
+		// 	site:   []string{"smarf"},
+		// 	cats:   []string{flags.DataPages},
+		// 	expect: assert.NoError,
+		// },
+
+		// [TODO]: Uncomment when pages & lists are enabled
+
+		// {
+		// 	name:   "site with all categories",
+		// 	site:   []string{"smarf"},
+		// 	cats:   []string{flags.DataLists, flags.DataPages, flags.DataLibraries},
+		// 	expect: assert.NoError,
+		// },
 	}
 	for _, test := range table {
 		suite.Run(test.name, func() {
-			err := validateSharePointBackupCreateFlags(test.site, test.weburl, nil)
+			err := validateSharePointBackupCreateFlags(test.site, test.weburl, test.cats)
 			test.expect(suite.T(), err, clues.ToCore(err))
 		})
 	}
@@ -163,12 +300,11 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() {
 	)
 
 	table := []struct {
-		name            string
-		site            []string
-		weburl          []string
-		data            []string
-		expect          []string
-		expectScopesLen int
+		name   string
+		site   []string
+		weburl []string
+		data   []string
+		expect []string
 	}{
 		{
 			name:   "no sites or urls",
@@ -181,63 +317,60 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() {
 			expect: selectors.None(),
 		},
 		{
-			name:            "site wildcard",
-			site:            []string{flags.Wildcard},
-			expect:          bothIDs,
-			expectScopesLen: 2,
+			name:   "site wildcard",
+			site:   []string{flags.Wildcard},
+			expect: bothIDs,
 		},
 		{
-			name:            "url wildcard",
-			weburl:          []string{flags.Wildcard},
-			expect:          bothIDs,
-			expectScopesLen: 2,
+			name:   "url wildcard",
+			weburl: []string{flags.Wildcard},
+			expect: bothIDs,
 		},
 		{
-			name:            "sites",
-			site:            []string{id1, id2},
-			expect:          []string{id1, id2},
-			expectScopesLen: 2,
+			name:   "sites",
+			site:   []string{id1, id2},
+			expect: []string{id1, id2},
 		},
 		{
-			name:            "urls",
-			weburl:          []string{url1, url2},
-			expect:          []string{url1, url2},
-			expectScopesLen: 2,
+			name:   "urls",
+			weburl: []string{url1, url2},
+			expect: []string{url1, url2},
 		},
 		{
-			name:            "mix sites and urls",
-			site:            []string{id1},
-			weburl:          []string{url2},
-			expect:          []string{id1, url2},
-			expectScopesLen: 2,
+			name:   "mix sites and urls",
+			site:   []string{id1},
+			weburl: []string{url2},
+			expect: []string{id1, url2},
 		},
 		{
-			name:            "duplicate sites and urls",
-			site:            []string{id1, id2},
-			weburl:          []string{url1, url2},
-			expect:          []string{id1, id2, url1, url2},
-			expectScopesLen: 2,
+			name:   "duplicate sites and urls",
+			site:   []string{id1, id2},
+			weburl: []string{url1, url2},
+			expect: []string{id1, id2, url1, url2},
 		},
 		{
-			name:            "unnecessary site wildcard",
-			site:            []string{id1, flags.Wildcard},
-			weburl:          []string{url1, url2},
-			expect:          bothIDs,
-			expectScopesLen: 2,
+			name:   "unnecessary site wildcard",
+			site:   []string{id1, flags.Wildcard},
+			weburl: []string{url1, url2},
+			expect: bothIDs,
 		},
 		{
-			name:            "unnecessary url wildcard",
-			site:            []string{id1},
-			weburl:          []string{url1, flags.Wildcard},
-			expect:          bothIDs,
-			expectScopesLen: 2,
+			name:   "unnecessary url wildcard",
+			site:   []string{id1},
+			weburl: []string{url1, flags.Wildcard},
+			expect: bothIDs,
 		},
 		{
-			name:            "Pages",
-			site:            bothIDs,
-			data:            []string{dataPages},
-			expect:          bothIDs,
-			expectScopesLen: 1,
+			name:   "Pages",
+			site:   bothIDs,
+			data:   []string{flags.DataPages},
+			expect: bothIDs,
+		},
+		{
+			name:   "Lists",
+			site:   bothIDs,
+			data:   []string{flags.DataLists},
+			expect: bothIDs,
 		},
 	}
 	for _, test := range table {
@@ -249,55 +382,7 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() {
 
 			sel, err := sharePointBackupCreateSelectors(ctx, ins, test.site, test.weburl, test.data)
 			require.NoError(t, err, clues.ToCore(err))
-			assert.ElementsMatch(t, test.expect, sel.DiscreteResourceOwners())
-		})
-	}
-}
-
-func (suite *SharePointUnitSuite) TestSharePointBackupDetailsSelectors() {
-	for v := 0; v <= version.Backup; v++ {
-		suite.Run(fmt.Sprintf("version%d", v), func() {
-			for _, test := range testdata.SharePointOptionDetailLookups {
-				suite.Run(test.Name, func() {
-					t := suite.T()
-
-					ctx, flush := tester.NewContext(t)
-					defer flush()
-
-					bg := testdata.VersionedBackupGetter{
-						Details: dtd.GetDetailsSetForVersion(t, v),
-					}
-
-					output, err := runDetailsSharePointCmd(
-						ctx,
-						bg,
-						"backup-ID",
-						test.Opts(t, v),
-						false)
-					assert.NoError(t, err, clues.ToCore(err))
-					assert.ElementsMatch(t, test.Expected(t, v), output.Entries)
-				})
-			}
-		})
-	}
-}
-
-func (suite *SharePointUnitSuite) TestSharePointBackupDetailsSelectorsBadFormats() {
-	for _, test := range testdata.BadSharePointOptionsFormats {
-		suite.Run(test.Name, func() {
-			t := suite.T()
-
-			ctx, flush := tester.NewContext(t)
-			defer flush()
-
-			output, err := runDetailsSharePointCmd(
-				ctx,
-				test.BackupGetter,
-				"backup-ID",
-				test.Opts(t, version.Backup),
-				false)
-			assert.Error(t, err, clues.ToCore(err))
-			assert.Empty(t, output)
+			assert.ElementsMatch(t, test.expect, sel.ResourceOwners.Targets)
 		})
 	}
 }

src/cli/backup/teamschats.go

@@ -0,0 +1,305 @@
+package backup
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/alcionai/clues"
+	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	. "github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/common/idname"
+	"github.com/alcionai/corso/src/pkg/fault"
+	"github.com/alcionai/corso/src/pkg/filters"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
+	"github.com/alcionai/corso/src/pkg/services/m365"
+)
+
+// ------------------------------------------------------------------------------------------------
+// setup and globals
+// ------------------------------------------------------------------------------------------------
+
+const (
+	teamschatsServiceCommand                 = "chats"
+	teamschatsServiceCommandCreateUseSuffix  = "--user <userEmail> | '" + flags.Wildcard + "'"
+	teamschatsServiceCommandDeleteUseSuffix  = "--backups <backupId>"
+	teamschatsServiceCommandDetailsUseSuffix = "--backup <backupId>"
+)
+
+const (
+	teamschatsServiceCommandCreateExamples = `# Backup all chats with bob@company.hr
+corso backup create chats --user bob@company.hr
+
+# Backup all chats for all users
+corso backup create chats --user '*'`
+
+	teamschatsServiceCommandDeleteExamples = `# Delete chats backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
+and 1234abcd-12ab-cd34-56de-1234abce
+corso backup delete chats --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce`
+
+	teamschatsServiceCommandDetailsExamples = `# Explore chats in Bob's latest backup (1234abcd...)
+corso backup details chats --backup 1234abcd-12ab-cd34-56de-1234abcd`
+)
+
+// called by backup.go to map subcommands to provider-specific handling.
+func addTeamsChatsCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case createCommand:
+		c, _ = utils.AddCommand(cmd, teamschatsCreateCmd(), utils.MarkPreReleaseCommand())
+
+		c.Use = c.Use + " " + teamschatsServiceCommandCreateUseSuffix
+		c.Example = teamschatsServiceCommandCreateExamples
+
+		// Flags addition ordering should follow the order we want them to appear in help and docs:
+		flags.AddUserFlag(c)
+		flags.AddDataFlag(c, []string{flags.DataChats}, false)
+		flags.AddGenericBackupFlags(c)
+
+	case listCommand:
+		c, _ = utils.AddCommand(cmd, teamschatsListCmd(), utils.MarkPreReleaseCommand())
+
+		flags.AddBackupIDFlag(c, false)
+		flags.AddAllBackupListFlags(c)
+
+	case detailsCommand:
+		c, _ = utils.AddCommand(cmd, teamschatsDetailsCmd(), utils.MarkPreReleaseCommand())
+
+		c.Use = c.Use + " " + teamschatsServiceCommandDetailsUseSuffix
+		c.Example = teamschatsServiceCommandDetailsExamples
+
+		flags.AddSkipReduceFlag(c)
+
+		// Flags addition ordering should follow the order we want them to appear in help and docs:
+		// More generic (ex: --user) and more frequently used flags take precedence.
+		flags.AddBackupIDFlag(c, true)
+		flags.AddTeamsChatsDetailsAndRestoreFlags(c)
+
+	case deleteCommand:
+		c, _ = utils.AddCommand(cmd, teamschatsDeleteCmd(), utils.MarkPreReleaseCommand())
+
+		c.Use = c.Use + " " + teamschatsServiceCommandDeleteUseSuffix
+		c.Example = teamschatsServiceCommandDeleteExamples
+
+		flags.AddMultipleBackupIDsFlag(c, false)
+		flags.AddBackupIDFlag(c, false)
+	}
+
+	return c
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup create
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup create chats [<flag>...]`
+func teamschatsCreateCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     teamschatsServiceCommand,
+		Aliases: []string{teamsServiceCommand},
+		Short:   "Backup M365 Chats data",
+		RunE:    createTeamsChatsCmd,
+		Args:    cobra.NoArgs,
+	}
+}
+
+// processes a teamschats backup.
+func createTeamsChatsCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	if err := validateTeamsChatsBackupCreateFlags(flags.UserFV, flags.CategoryDataFV); err != nil {
+		return err
+	}
+
+	r, acct, err := utils.AccountConnectAndWriteRepoConfig(
+		ctx,
+		cmd,
+		path.TeamsChatsService)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	defer utils.CloseRepo(ctx, r)
+
+	// TODO: log/print recoverable errors
+	errs := fault.New(false)
+
+	svcCli, err := m365.NewM365Client(ctx, *acct)
+	if err != nil {
+		return Only(ctx, clues.Stack(err))
+	}
+
+	ins, err := svcCli.AC.Users().GetAllIDsAndNames(ctx, errs)
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 teamschats"))
+	}
+
+	sel := teamschatsBackupCreateSelectors(ctx, ins, flags.UserFV, flags.CategoryDataFV)
+	selectorSet := []selectors.Selector{}
+
+	for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) {
+		selectorSet = append(selectorSet, discSel.Selector)
+	}
+
+	return genericCreateCommand(
+		ctx,
+		r,
+		"Chats",
+		selectorSet,
+		ins)
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup list
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup list teamschats [<flag>...]`
+func teamschatsListCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   teamschatsServiceCommand,
+		Short: "List the history of M365 Chats backups",
+		RunE:  listTeamsChatsCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// lists the history of backup operations
+func listTeamsChatsCmd(cmd *cobra.Command, args []string) error {
+	return genericListCommand(cmd, flags.BackupIDFV, path.TeamsChatsService, args)
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup details
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup details teamschats [<flag>...]`
+func teamschatsDetailsCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   teamschatsServiceCommand,
+		Short: "Shows the details of a M365 Chats backup",
+		RunE:  detailsTeamsChatsCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// processes a teamschats backup.
+func detailsTeamsChatsCmd(cmd *cobra.Command, args []string) error {
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	return runDetailsTeamsChatsCmd(cmd)
+}
+
+func runDetailsTeamsChatsCmd(cmd *cobra.Command) error {
+	ctx := cmd.Context()
+	opts := utils.MakeTeamsChatsOpts(cmd)
+
+	sel := utils.IncludeTeamsChatsRestoreDataSelectors(ctx, opts)
+	sel.Configure(selectors.Config{OnlyMatchItemNames: true})
+	utils.FilterTeamsChatsRestoreInfoSelectors(sel, opts)
+
+	ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	if len(ds.Entries) > 0 {
+		ds.PrintEntries(ctx)
+	} else {
+		Info(ctx, selectors.ErrorNoMatchingItems)
+	}
+
+	return nil
+}
+
+// ------------------------------------------------------------------------------------------------
+// backup delete
+// ------------------------------------------------------------------------------------------------
+
+// `corso backup delete teamschats [<flag>...]`
+func teamschatsDeleteCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   teamschatsServiceCommand,
+		Short: "Delete backed-up M365 Chats data",
+		RunE:  deleteTeamsChatsCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// deletes an teamschats backup.
+func deleteTeamsChatsCmd(cmd *cobra.Command, args []string) error {
+	backupIDValue := []string{}
+
+	if len(flags.BackupIDsFV) > 0 {
+		backupIDValue = flags.BackupIDsFV
+	} else if len(flags.BackupIDFV) > 0 {
+		backupIDValue = append(backupIDValue, flags.BackupIDFV)
+	} else {
+		return clues.New("either --backup or --backups flag is required")
+	}
+
+	return genericDeleteCommand(cmd, path.TeamsChatsService, "TeamsChats", backupIDValue, args)
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func validateTeamsChatsBackupCreateFlags(teamschats, cats []string) error {
+	if len(teamschats) == 0 {
+		return clues.New(
+			"requires one or more --" +
+				flags.UserFN + " ids, or the wildcard --" +
+				flags.UserFN + " *")
+	}
+
+	msg := fmt.Sprintf(
+		" is an unrecognized data type; only %s is supported",
+		flags.DataChats)
+
+	allowedCats := utils.TeamsChatsAllowedCategories()
+
+	for _, d := range cats {
+		if _, ok := allowedCats[d]; !ok {
+			return clues.New(d + msg)
+		}
+	}
+
+	return nil
+}
+
+func teamschatsBackupCreateSelectors(
+	ctx context.Context,
+	ins idname.Cacher,
+	users, cats []string,
+) *selectors.TeamsChatsBackup {
+	if filters.PathContains(users).Compare(flags.Wildcard) {
+		return includeAllTeamsChatsWithCategories(ins, cats)
+	}
+
+	sel := selectors.NewTeamsChatsBackup(slices.Clone(users))
+
+	return utils.AddTeamsChatsCategories(sel, cats)
+}
+
+func includeAllTeamsChatsWithCategories(ins idname.Cacher, categories []string) *selectors.TeamsChatsBackup {
+	return utils.AddTeamsChatsCategories(selectors.NewTeamsChatsBackup(ins.IDs()), categories)
+}

src/cli/backup/teamschats_e2e_test.go

@@ -0,0 +1,636 @@
+package backup_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/alcionai/clues"
+	"github.com/google/uuid"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/print"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/common/idname"
+	"github.com/alcionai/corso/src/internal/operations"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/internal/tester/its"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/config"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
+	selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
+)
+
+// ---------------------------------------------------------------------------
+// tests that require no existing backups
+// ---------------------------------------------------------------------------
+
+type NoBackupTeamsChatsE2ESuite struct {
+	tester.Suite
+	dpnd dependencies
+	m365 its.M365IntgTestSetup
+}
+
+func TestNoBackupTeamsChatsE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *NoBackupTeamsChatsE2ESuite) SetupSuite() {
+	t := suite.T()
+	t.Skip("not fully implemented")
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
+}
+
+func (suite *NoBackupTeamsChatsE2ESuite) TestTeamsChatsBackupListCmd_noBackups() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetErr(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := suite.dpnd.recorder.String()
+
+	// as an offhand check: the result should contain the m365 teamschat id
+	assert.True(t, strings.HasSuffix(result, "No backups available\n"))
+}
+
+// ---------------------------------------------------------------------------
+// tests with no prior backup
+// ---------------------------------------------------------------------------
+
+type BackupTeamsChatsE2ESuite struct {
+	tester.Suite
+	dpnd dependencies
+	m365 its.M365IntgTestSetup
+}
+
+func TestBackupTeamsChatsE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *BackupTeamsChatsE2ESuite) SetupSuite() {
+	t := suite.T()
+	t.Skip("not fully implemented")
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
+}
+
+func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_chats() {
+	runTeamsChatsBackupCategoryTest(suite, flags.DataChats)
+}
+
+func runTeamsChatsBackupCategoryTest(suite *BackupTeamsChatsE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildTeamsChatsBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		suite.m365.User.ID,
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_teamschatNotFound_chats() {
+	runTeamsChatsBackupTeamsChatNotFoundTest(suite, flags.DataChats)
+}
+
+func runTeamsChatsBackupTeamsChatNotFoundTest(suite *BackupTeamsChatsE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildTeamsChatsBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		"foo@not-there.com",
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+	assert.Contains(
+		t,
+		err.Error(),
+		"not found",
+		"error missing user not found")
+	assert.NotContains(t, err.Error(), "runtime error", "panic happened")
+
+	t.Logf("backup error message: %s", err.Error())
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAzureClientIDFlag() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "chats",
+		"--teamschat", suite.m365.User.ID,
+		"--azure-client-id", "invalid-value")
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetErr(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_fromConfigFile() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "chats",
+		"--teamschat", suite.m365.User.ID,
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+}
+
+// AWS flags
+func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAWSFlags() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+
+	defer flush()
+
+	suite.dpnd.recorder.Reset()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "chats",
+		"--teamschat", suite.m365.User.ID,
+		"--aws-access-key", "invalid-value",
+		"--aws-secret-access-key", "some-invalid-value")
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	// since invalid aws creds are explicitly set, should see a failure
+	require.Error(t, err, clues.ToCore(err))
+}
+
+// ---------------------------------------------------------------------------
+// tests prepared with a previous backup
+// ---------------------------------------------------------------------------
+
+type PreparedBackupTeamsChatsE2ESuite struct {
+	tester.Suite
+	dpnd      dependencies
+	backupOps map[path.CategoryType]string
+	m365      its.M365IntgTestSetup
+}
+
+func TestPreparedBackupTeamsChatsE2ESuite(t *testing.T) {
+	suite.Run(t, &PreparedBackupTeamsChatsE2ESuite{
+		Suite: tester.NewE2ESuite(
+			t,
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
+	})
+}
+
+func (suite *PreparedBackupTeamsChatsE2ESuite) SetupSuite() {
+	t := suite.T()
+	t.Skip("not fully implemented")
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.m365 = its.GetM365(t)
+	suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
+	suite.backupOps = make(map[path.CategoryType]string)
+
+	var (
+		teamschats = []string{suite.m365.User.ID}
+		ins        = idname.NewCache(map[string]string{suite.m365.User.ID: suite.m365.User.ID})
+		cats       = []path.CategoryType{
+			path.ChatsCategory,
+		}
+	)
+
+	for _, set := range cats {
+		var (
+			sel    = selectors.NewTeamsChatsBackup(teamschats)
+			scopes []selectors.TeamsChatsScope
+		)
+
+		switch set {
+		case path.ChatsCategory:
+			scopes = selTD.TeamsChatsBackupChatScope(sel)
+		}
+
+		sel.Include(scopes)
+
+		bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+		require.NoError(t, err, clues.ToCore(err))
+
+		err = bop.Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+
+		bIDs := string(bop.Results.BackupID)
+
+		// sanity check, ensure we can find the backup and its details immediately
+		b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
+		require.NoError(t, err, "retrieving recent backup by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
+
+		_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
+		require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
+		require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
+
+		suite.backupOps[set] = string(b.ID)
+	}
+}
+
+func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_chats() {
+	runTeamsChatsListCmdTest(suite, path.ChatsCategory)
+}
+
+func runTeamsChatsListCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+	assert.Contains(t, result, suite.backupOps[category])
+}
+
+func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_singleID_chats() {
+	runTeamsChatsListSingleCmdTest(suite, path.ChatsCategory)
+}
+
+func runTeamsChatsListSingleCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	bID := suite.backupOps[category]
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(bID))
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+	assert.Contains(t, result, bID)
+}
+
+func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_badID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", "smarfs")
+	cli.BuildCommandTree(cmd)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsDetailsCmd_chats() {
+	runTeamsChatsDetailsCmdTest(suite, path.ChatsCategory)
+}
+
+func runTeamsChatsDetailsCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	bID := suite.backupOps[category]
+
+	// fetch the details from the repo first
+	deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
+	require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
+	require.Empty(t, errs.Recovered())
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "details", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupFN, string(bID))
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+
+	i := 0
+	foundFolders := 0
+
+	for _, ent := range deets.Entries {
+		// Skip folders as they don't mean anything to the end teamschat.
+		if ent.Folder != nil {
+			foundFolders++
+			continue
+		}
+
+		suite.Run(fmt.Sprintf("detail %d", i), func() {
+			assert.Contains(suite.T(), result, ent.ShortRef)
+		})
+
+		i++
+	}
+
+	// We only backup the default folder for each category so there should be at
+	// least that folder (we don't make details entries for prefix folders).
+	assert.GreaterOrEqual(t, foundFolders, 1)
+}
+
+// ---------------------------------------------------------------------------
+// tests for deleting backups
+// ---------------------------------------------------------------------------
+
+type BackupDeleteTeamsChatsE2ESuite struct {
+	tester.Suite
+	dpnd      dependencies
+	backupOps [3]operations.BackupOperation
+}
+
+func TestBackupDeleteTeamsChatsE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupDeleteTeamsChatsE2ESuite{
+		Suite: tester.NewE2ESuite(
+			t,
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
+	})
+}
+
+func (suite *BackupDeleteTeamsChatsE2ESuite) SetupSuite() {
+	t := suite.T()
+	t.Skip("not fully implemented")
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
+
+	m365TeamsChatID := tconfig.M365TeamID(t)
+	teamschats := []string{m365TeamsChatID}
+
+	// some tests require an existing backup
+	sel := selectors.NewTeamsChatsBackup(teamschats)
+	sel.Include(selTD.TeamsChatsBackupChatScope(sel))
+
+	for i := 0; i < cap(suite.backupOps); i++ {
+		backupOp, err := suite.dpnd.repo.NewBackup(ctx, sel.Selector)
+		require.NoError(t, err, clues.ToCore(err))
+
+		suite.backupOps[i] = backupOp
+
+		err = suite.backupOps[i].Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+	}
+}
+
+func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN,
+		fmt.Sprintf("%s,%s",
+			string(suite.backupOps[0].Results.BackupID),
+			string(suite.backupOps[1].Results.BackupID)))
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// a follow-up details call should fail, due to the backup ID being deleted
+	cmd = cliTD.StubRootCmd(
+		"backup", "details", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backups", string(suite.backupOps[0].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_SingleID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupFN,
+		string(suite.backupOps[2].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// a follow-up details call should fail, due to the backup ID being deleted
+	cmd = cliTD.StubRootCmd(
+		"backup", "details", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--backup", string(suite.backupOps[2].Results.BackupID))
+	cli.BuildCommandTree(cmd)
+
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_UnknownID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
+		"--"+flags.BackupIDsFN, uuid.NewString())
+	cli.BuildCommandTree(cmd)
+
+	// unknown backupIDs should error since the modelStore can't find the backup
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_NoBackupID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "delete", "chats",
+		"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+
+	// empty backupIDs should error since no data provided
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func buildTeamsChatsBackupCmd(
+	ctx context.Context,
+	configFile, resource, category string,
+	recorder *strings.Builder,
+) (*cobra.Command, context.Context) {
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "chats",
+		"--"+flags.ConfigFileFN, configFile,
+		"--"+flags.UserFN, resource,
+		"--"+flags.CategoryDataFN, category)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(recorder)
+
+	return cmd, print.SetRootCmd(ctx, cmd)
+}

src/cli/backup/teamschats_test.go

@@ -0,0 +1,248 @@
+package backup
+
+import (
+	"testing"
+
+	"github.com/alcionai/clues"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/control"
+)
+
+type TeamsChatsUnitSuite struct {
+	tester.Suite
+}
+
+func TestTeamsChatsUnitSuite(t *testing.T) {
+	suite.Run(t, &TeamsChatsUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *TeamsChatsUnitSuite) TestAddTeamsChatsCommands() {
+	expectUse := teamschatsServiceCommand
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{
+			name:        "create teamschats",
+			use:         createCommand,
+			expectUse:   expectUse + " " + teamschatsServiceCommandCreateUseSuffix,
+			expectShort: teamschatsCreateCmd().Short,
+			expectRunE:  createTeamsChatsCmd,
+		},
+		{
+			name:        "list teamschats",
+			use:         listCommand,
+			expectUse:   expectUse,
+			expectShort: teamschatsListCmd().Short,
+			expectRunE:  listTeamsChatsCmd,
+		},
+		{
+			name:        "details teamschats",
+			use:         detailsCommand,
+			expectUse:   expectUse + " " + teamschatsServiceCommandDetailsUseSuffix,
+			expectShort: teamschatsDetailsCmd().Short,
+			expectRunE:  detailsTeamsChatsCmd,
+		},
+		{
+			name:        "delete teamschats",
+			use:         deleteCommand,
+			expectUse:   expectUse + " " + teamschatsServiceCommandDeleteUseSuffix,
+			expectShort: teamschatsDeleteCmd().Short,
+			expectRunE:  deleteTeamsChatsCmd,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			cmd := &cobra.Command{Use: test.use}
+
+			c := addTeamsChatsCommands(cmd)
+			require.NotNil(t, c)
+
+			cmds := cmd.Commands()
+			require.Len(t, cmds, 1)
+
+			child := cmds[0]
+			assert.Equal(t, test.expectUse, child.Use)
+			assert.Equal(t, test.expectShort, child.Short)
+			tester.AreSameFunc(t, test.expectRunE, child.RunE)
+		})
+	}
+}
+
+func (suite *TeamsChatsUnitSuite) TestValidateTeamsChatsBackupCreateFlags() {
+	table := []struct {
+		name   string
+		cats   []string
+		expect assert.ErrorAssertionFunc
+	}{
+		{
+			name:   "none",
+			cats:   []string{},
+			expect: assert.NoError,
+		},
+		{
+			name:   "chats",
+			cats:   []string{flags.DataChats},
+			expect: assert.NoError,
+		},
+		{
+			name: "all allowed",
+			cats: []string{
+				flags.DataChats,
+			},
+			expect: assert.NoError,
+		},
+		{
+			name:   "bad inputs",
+			cats:   []string{"foo"},
+			expect: assert.Error,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			err := validateTeamsChatsBackupCreateFlags([]string{"*"}, test.cats)
+			test.expect(suite.T(), err, clues.ToCore(err))
+		})
+	}
+}
+
+func (suite *TeamsChatsUnitSuite) TestBackupCreateFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: createCommand},
+		addTeamsChatsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			teamschatsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.UserFN, flagsTD.FlgInputs(flagsTD.UsersInput),
+				"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.TeamsChatsCategoryDataInput),
+			},
+			flagsTD.PreparedGenericBackupFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	opts := utils.MakeTeamsChatsOpts(cmd)
+	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
+
+	assert.Equal(t, control.FailFast, co.FailureHandling)
+	assert.True(t, co.ToggleFeatures.DisableIncrementals)
+	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
+
+	assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
+	flagsTD.AssertGenericBackupFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *TeamsChatsUnitSuite) TestBackupListFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: listCommand},
+		addTeamsChatsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			teamschatsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedBackupListFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertBackupListFlags(t, cmd)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}
+
+func (suite *TeamsChatsUnitSuite) TestBackupDetailsFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: detailsCommand},
+		addTeamsChatsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			teamschatsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+				"--" + flags.SkipReduceFN,
+			},
+			flagsTD.PreparedTeamsChatsFlags(),
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	co := utils.Control()
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	assert.True(t, co.SkipReduce)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+	flagsTD.AssertTeamsChatsFlags(t, cmd)
+}
+
+func (suite *TeamsChatsUnitSuite) TestBackupDeleteFlags() {
+	t := suite.T()
+
+	cmd := cliTD.SetUpCmdHasFlags(
+		t,
+		&cobra.Command{Use: deleteCommand},
+		addTeamsChatsCommands,
+		[]cliTD.UseCobraCommandFn{
+			flags.AddAllProviderFlags,
+			flags.AddAllStorageFlags,
+		},
+		flagsTD.WithFlags(
+			teamschatsServiceCommand,
+			[]string{
+				"--" + flags.RunModeFN, flags.RunModeFlagTest,
+				"--" + flags.BackupFN, flagsTD.BackupInput,
+			},
+			flagsTD.PreparedProviderFlags(),
+			flagsTD.PreparedStorageFlags()))
+
+	assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+	flagsTD.AssertProviderFlags(t, cmd)
+	flagsTD.AssertStorageFlags(t, cmd)
+}

src/cli/cli.go

@@ -10,15 +10,16 @@ import (
 	"golang.org/x/exp/slices"
 
 	"github.com/alcionai/corso/src/cli/backup"
-	"github.com/alcionai/corso/src/cli/config"
+	"github.com/alcionai/corso/src/cli/debug"
+	"github.com/alcionai/corso/src/cli/export"
 	"github.com/alcionai/corso/src/cli/flags"
 	"github.com/alcionai/corso/src/cli/help"
 	"github.com/alcionai/corso/src/cli/print"
 	"github.com/alcionai/corso/src/cli/repo"
 	"github.com/alcionai/corso/src/cli/restore"
-	"github.com/alcionai/corso/src/cli/utils"
 	"github.com/alcionai/corso/src/internal/observe"
 	"github.com/alcionai/corso/src/internal/version"
+	"github.com/alcionai/corso/src/pkg/config"
 	"github.com/alcionai/corso/src/pkg/logger"
 )
 
@@ -37,7 +38,7 @@ var corsoCmd = &cobra.Command{
 }
 
 func preRun(cc *cobra.Command, args []string) error {
-	if err := config.InitFunc(cc, args); err != nil {
+	if err := config.InitCmd(cc, args); err != nil {
 		return err
 	}
 
@@ -53,39 +54,13 @@ func preRun(cc *cobra.Command, args []string) error {
 	}
 
 	avoidTheseCommands := []string{
-		"corso", "env", "help", "backup", "details", "list", "restore", "delete", "repo", "init", "connect",
+		"corso", "env", "help", "backup", "details", "list", "restore", "export", "delete", "repo", "init", "connect",
 	}
 
 	if len(logger.ResolvedLogFile) > 0 && !slices.Contains(avoidTheseCommands, cc.Use) {
 		print.Infof(ctx, "Logging to file: %s", logger.ResolvedLogFile)
 	}
 
-	avoidTheseDescription := []string{
-		"Initialize a repository.",
-		"Initialize a S3 repository",
-		"Help about any command",
-		"Free, Secure, Open-Source Backup for M365.",
-		"env var guide",
-	}
-
-	if !slices.Contains(avoidTheseDescription, cc.Short) {
-		overrides := repo.S3Overrides()
-
-		cfg, err := config.GetConfigRepoDetails(ctx, true, false, overrides)
-		if err != nil {
-			log.Error("Error while getting config info to run command: ", cc.Use)
-			return err
-		}
-
-		utils.SendStartCorsoEvent(
-			ctx,
-			cfg.Storage,
-			cfg.Account.ID(),
-			map[string]any{"command": cc.CommandPath()},
-			cfg.RepoID,
-			utils.Control())
-	}
-
 	// handle deprecated user flag in Backup exchange command
 	if cc.CommandPath() == "corso backup create exchange" {
 		handleMailBoxFlag(ctx, cc, flagSl)
@@ -98,7 +73,7 @@ func preRun(cc *cobra.Command, args []string) error {
 
 func handleMailBoxFlag(ctx context.Context, c *cobra.Command, flagNames []string) {
 	if !slices.Contains(flagNames, "user") && !slices.Contains(flagNames, "mailbox") {
-		print.Errf(ctx, "either --user or --mailbox flag is required")
+		print.Err(ctx, "either --user or --mailbox flag is required")
 		os.Exit(1)
 	}
 
@@ -150,6 +125,8 @@ func BuildCommandTree(cmd *cobra.Command) {
 	repo.AddCommands(cmd)
 	backup.AddCommands(cmd)
 	restore.AddCommands(cmd)
+	export.AddCommands(cmd)
+	debug.AddCommands(cmd)
 	help.AddCommands(cmd)
 }
 
@@ -161,15 +138,15 @@ func BuildCommandTree(cmd *cobra.Command) {
 func Handle() {
 	//nolint:forbidigo
 	ctx := config.Seed(context.Background())
+	ctx, log := logger.Seed(ctx, logger.PreloadLoggingFlags(os.Args[1:]))
 	ctx = print.SetRootCmd(ctx, corsoCmd)
-
-	observe.SeedWriter(ctx, print.StderrWriter(ctx), observe.PreloadFlags())
+	ctx = observe.SeedObserver(ctx, print.StderrWriter(ctx), observe.PreloadFlags())
 
 	BuildCommandTree(corsoCmd)
 
-	ctx, log := logger.Seed(ctx, logger.PreloadLoggingFlags(os.Args[1:]))
-
 	defer func() {
+		observe.Flush(ctx) // flush the progress bars
+
 		_ = log.Sync() // flush all logs in the buffer
 	}()
 

src/cli/config/config_test.go

@@ -1,342 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/alcionai/clues"
-	"github.com/spf13/viper"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/stretchr/testify/suite"
-
-	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/pkg/account"
-	"github.com/alcionai/corso/src/pkg/credentials"
-	"github.com/alcionai/corso/src/pkg/storage"
-)
-
-const (
-	configFileTemplate = `
-` + BucketNameKey + ` = '%s'
-` + EndpointKey + ` = 's3.amazonaws.com'
-` + PrefixKey + ` = 'test-prefix/'
-` + StorageProviderTypeKey + ` = 'S3'
-` + AccountProviderTypeKey + ` = 'M365'
-` + AzureTenantIDKey + ` = '%s'
-` + DisableTLSKey + ` = 'false'
-` + DisableTLSVerificationKey + ` = 'false'
-` + AccessKey + ` = '%s'
-` + SecretAccessKey + ` = '%s'
-` + SessionToken + ` = '%s'
-` + CorsoPassphrase + ` = '%s'
-`
-)
-
-type ConfigSuite struct {
-	tester.Suite
-}
-
-func TestConfigSuite(t *testing.T) {
-	suite.Run(t, &ConfigSuite{Suite: tester.NewUnitSuite(t)})
-}
-
-func (suite *ConfigSuite) TestRequireProps() {
-	table := []struct {
-		name     string
-		props    map[string]string
-		errCheck assert.ErrorAssertionFunc
-	}{
-		{
-			props:    map[string]string{"exists": "I have seen the fnords!"},
-			errCheck: assert.NoError,
-		},
-		{
-			props:    map[string]string{"not-exists": ""},
-			errCheck: assert.Error,
-		},
-	}
-	for _, test := range table {
-		err := requireProps(test.props)
-		test.errCheck(suite.T(), err, clues.ToCore(err))
-	}
-}
-
-func (suite *ConfigSuite) TestReadRepoConfigBasic() {
-	var (
-		t   = suite.T()
-		vpr = viper.New()
-	)
-
-	const (
-		b          = "read-repo-config-basic-bucket"
-		tID        = "6f34ac30-8196-469b-bf8f-d83deadbbbba"
-		accKey     = "aws-test-access-key"
-		secret     = "aws-test-secret-key"
-		token      = "aws-test-session-token"
-		passphrase = "passphrase-test"
-	)
-
-	// Generate test config file
-	testConfigData := fmt.Sprintf(configFileTemplate, b, tID, accKey, secret, token, passphrase)
-	testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
-	err := os.WriteFile(testConfigFilePath, []byte(testConfigData), 0o700)
-	require.NoError(t, err, clues.ToCore(err))
-
-	// Configure viper to read test config file
-	vpr.SetConfigFile(testConfigFilePath)
-
-	// Read and validate config
-	err = vpr.ReadInConfig()
-	require.NoError(t, err, "reading repo config", clues.ToCore(err))
-
-	s3Cfg, err := s3ConfigsFromViper(vpr)
-	require.NoError(t, err, clues.ToCore(err))
-	assert.Equal(t, b, s3Cfg.Bucket)
-
-	s3Cfg, err = s3CredsFromViper(vpr, s3Cfg)
-	require.NoError(t, err, clues.ToCore(err))
-	assert.Equal(t, accKey, s3Cfg.AWS.AccessKey)
-	assert.Equal(t, secret, s3Cfg.AWS.SecretKey)
-	assert.Equal(t, token, s3Cfg.AWS.SessionToken)
-
-	m365, err := m365ConfigsFromViper(vpr)
-	require.NoError(t, err, clues.ToCore(err))
-	assert.Equal(t, tID, m365.AzureTenantID)
-}
-
-func (suite *ConfigSuite) TestWriteReadConfig() {
-	var (
-		t   = suite.T()
-		vpr = viper.New()
-		// Configure viper to read test config file
-		testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml")
-	)
-
-	const (
-		bkt = "write-read-config-bucket"
-		tid = "3c0748d2-470e-444c-9064-1268e52609d5"
-	)
-
-	err := initWithViper(vpr, testConfigFilePath)
-	require.NoError(t, err, "initializing repo config", clues.ToCore(err))
-
-	s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true}
-	m365 := account.M365Config{AzureTenantID: tid}
-
-	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
-	require.NoError(t, err, "writing repo config", clues.ToCore(err))
-
-	err = vpr.ReadInConfig()
-	require.NoError(t, err, "reading repo config", clues.ToCore(err))
-
-	readS3Cfg, err := s3ConfigsFromViper(vpr)
-	require.NoError(t, err, clues.ToCore(err))
-	assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
-	assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS)
-	assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
-
-	readM365, err := m365ConfigsFromViper(vpr)
-	require.NoError(t, err, clues.ToCore(err))
-	assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
-}
-
-func (suite *ConfigSuite) TestMustMatchConfig() {
-	var (
-		t   = suite.T()
-		vpr = viper.New()
-		// Configure viper to read test config file
-		testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml")
-	)
-
-	const (
-		bkt = "must-match-config-bucket"
-		tid = "dfb12063-7598-458b-85ab-42352c5c25e2"
-	)
-
-	err := initWithViper(vpr, testConfigFilePath)
-	require.NoError(t, err, "initializing repo config")
-
-	s3Cfg := storage.S3Config{Bucket: bkt}
-	m365 := account.M365Config{AzureTenantID: tid}
-
-	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
-	require.NoError(t, err, "writing repo config", clues.ToCore(err))
-
-	err = vpr.ReadInConfig()
-	require.NoError(t, err, "reading repo config", clues.ToCore(err))
-
-	table := []struct {
-		name     string
-		input    map[string]string
-		errCheck assert.ErrorAssertionFunc
-	}{
-		{
-			name: "full match",
-			input: map[string]string{
-				storage.Bucket:        bkt,
-				account.AzureTenantID: tid,
-			},
-			errCheck: assert.NoError,
-		},
-		{
-			name: "empty values",
-			input: map[string]string{
-				storage.Bucket:        "",
-				account.AzureTenantID: "",
-			},
-			errCheck: assert.NoError,
-		},
-		{
-			name:     "no overrides",
-			input:    map[string]string{},
-			errCheck: assert.NoError,
-		},
-		{
-			name:     "nil map",
-			input:    nil,
-			errCheck: assert.NoError,
-		},
-		{
-			name: "no recognized keys",
-			input: map[string]string{
-				"fnords":   "smurfs",
-				"nonsense": "",
-			},
-			errCheck: assert.NoError,
-		},
-		{
-			name: "mismatch",
-			input: map[string]string{
-				storage.Bucket:        tid,
-				account.AzureTenantID: bkt,
-			},
-			errCheck: assert.Error,
-		},
-	}
-	for _, test := range table {
-		suite.Run(test.name, func() {
-			test.errCheck(suite.T(), mustMatchConfig(vpr, test.input), clues.ToCore(err))
-		})
-	}
-}
-
-// ------------------------------------------------------------
-// integration tests
-// ------------------------------------------------------------
-
-type ConfigIntegrationSuite struct {
-	tester.Suite
-}
-
-func TestConfigIntegrationSuite(t *testing.T) {
-	suite.Run(t, &ConfigIntegrationSuite{Suite: tester.NewIntegrationSuite(
-		t,
-		[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-	)})
-}
-
-func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
-	t := suite.T()
-	vpr := viper.New()
-
-	const (
-		bkt = "get-storage-and-account-bucket"
-		end = "https://get-storage-and-account.com"
-		pfx = "get-storage-and-account-prefix/"
-		tid = "3a2faa4e-a882-445c-9d27-f552ef189381"
-	)
-
-	// Configure viper to read test config file
-	testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
-
-	err := initWithViper(vpr, testConfigFilePath)
-	require.NoError(t, err, "initializing repo config", clues.ToCore(err))
-
-	s3Cfg := storage.S3Config{
-		Bucket:         bkt,
-		Endpoint:       end,
-		Prefix:         pfx,
-		DoNotVerifyTLS: true,
-		DoNotUseTLS:    true,
-	}
-	m365 := account.M365Config{AzureTenantID: tid}
-
-	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
-	require.NoError(t, err, "writing repo config", clues.ToCore(err))
-
-	err = vpr.ReadInConfig()
-	require.NoError(t, err, "reading repo config", clues.ToCore(err))
-
-	config, err := getStorageAndAccountWithViper(vpr, true, false, nil)
-	require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
-
-	readS3Cfg, err := config.Storage.S3Config()
-	require.NoError(t, err, "reading s3 config from storage", clues.ToCore(err))
-	assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
-	assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint)
-	assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix)
-	assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS)
-	assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
-	assert.Equal(t, config.RepoID, "repoid")
-
-	common, err := config.Storage.CommonConfig()
-	require.NoError(t, err, "reading common config from storage", clues.ToCore(err))
-	assert.Equal(t, common.CorsoPassphrase, os.Getenv(credentials.CorsoPassphrase))
-
-	readM365, err := config.Account.M365Config()
-	require.NoError(t, err, "reading m365 config from account", clues.ToCore(err))
-	// Env var gets preference here. Where to get env tenantID from
-	// assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
-	assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID))
-	assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret))
-}
-
-func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverrides() {
-	t := suite.T()
-	vpr := viper.New()
-
-	const (
-		bkt = "get-storage-and-account-no-file-bucket"
-		end = "https://get-storage-and-account.com/no-file"
-		pfx = "get-storage-and-account-no-file-prefix/"
-		tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1"
-	)
-
-	m365 := account.M365Config{AzureTenantID: tid}
-
-	overrides := map[string]string{
-		account.AzureTenantID:  tid,
-		AccountProviderTypeKey: account.ProviderM365.String(),
-		storage.Bucket:         bkt,
-		storage.Endpoint:       end,
-		storage.Prefix:         pfx,
-		storage.DoNotUseTLS:    "true",
-		storage.DoNotVerifyTLS: "true",
-		StorageProviderTypeKey: storage.ProviderS3.String(),
-	}
-
-	config, err := getStorageAndAccountWithViper(vpr, false, false, overrides)
-	require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
-
-	readS3Cfg, err := config.Storage.S3Config()
-	require.NoError(t, err, "reading s3 config from storage", clues.ToCore(err))
-	assert.Equal(t, readS3Cfg.Bucket, bkt)
-	assert.Equal(t, config.RepoID, "")
-	assert.Equal(t, readS3Cfg.Endpoint, end)
-	assert.Equal(t, readS3Cfg.Prefix, pfx)
-	assert.True(t, readS3Cfg.DoNotUseTLS)
-	assert.True(t, readS3Cfg.DoNotVerifyTLS)
-
-	common, err := config.Storage.CommonConfig()
-	require.NoError(t, err, "reading common config from storage", clues.ToCore(err))
-	assert.Equal(t, common.CorsoPassphrase, os.Getenv(credentials.CorsoPassphrase))
-
-	readM365, err := config.Account.M365Config()
-	require.NoError(t, err, "reading m365 config from account", clues.ToCore(err))
-	assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
-	assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID))
-	assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret))
-}

src/cli/config/storage.go

@@ -1,172 +0,0 @@
-package config
-
-import (
-	"os"
-	"path/filepath"
-	"strconv"
-
-	"github.com/alcionai/clues"
-	"github.com/aws/aws-sdk-go/aws/defaults"
-	"github.com/spf13/viper"
-
-	"github.com/alcionai/corso/src/cli/flags"
-	"github.com/alcionai/corso/src/internal/common"
-	"github.com/alcionai/corso/src/internal/common/str"
-	"github.com/alcionai/corso/src/pkg/credentials"
-	"github.com/alcionai/corso/src/pkg/storage"
-)
-
-// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
-func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) {
-	var s3Config storage.S3Config
-
-	providerType := vpr.GetString(StorageProviderTypeKey)
-	if providerType != storage.ProviderS3.String() {
-		return s3Config, clues.New("unsupported storage provider: " + providerType)
-	}
-
-	s3Config.Bucket = vpr.GetString(BucketNameKey)
-	s3Config.Endpoint = vpr.GetString(EndpointKey)
-	s3Config.Prefix = vpr.GetString(PrefixKey)
-	s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey)
-	s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey)
-
-	return s3Config, nil
-}
-
-// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
-func s3CredsFromViper(vpr *viper.Viper, s3Config storage.S3Config) (storage.S3Config, error) {
-	s3Config.AccessKey = vpr.GetString(AccessKey)
-	s3Config.SecretKey = vpr.GetString(SecretAccessKey)
-	s3Config.SessionToken = vpr.GetString(SessionToken)
-
-	return s3Config, nil
-}
-
-func s3Overrides(in map[string]string) map[string]string {
-	return map[string]string{
-		storage.Bucket:         in[storage.Bucket],
-		storage.Endpoint:       in[storage.Endpoint],
-		storage.Prefix:         in[storage.Prefix],
-		storage.DoNotUseTLS:    in[storage.DoNotUseTLS],
-		storage.DoNotVerifyTLS: in[storage.DoNotVerifyTLS],
-		StorageProviderTypeKey: in[StorageProviderTypeKey],
-	}
-}
-
-// configureStorage builds a complete storage configuration from a mix of
-// viper properties and manual overrides.
-func configureStorage(
-	vpr *viper.Viper,
-	readConfigFromViper bool,
-	matchFromConfig bool,
-	overrides map[string]string,
-) (storage.Storage, error) {
-	var (
-		s3Cfg storage.S3Config
-		store storage.Storage
-		err   error
-	)
-
-	if readConfigFromViper {
-		if s3Cfg, err = s3ConfigsFromViper(vpr); err != nil {
-			return store, clues.Wrap(err, "reading s3 configs from corso config file")
-		}
-
-		if b, ok := overrides[storage.Bucket]; ok {
-			overrides[storage.Bucket] = common.NormalizeBucket(b)
-		}
-
-		if p, ok := overrides[storage.Prefix]; ok {
-			overrides[storage.Prefix] = common.NormalizePrefix(p)
-		}
-	}
-
-	if matchFromConfig {
-		if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
-			return store, clues.Wrap(err, "verifying s3 configs in corso config file")
-		}
-	}
-
-	if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil {
-		return store, clues.Wrap(err, "reading s3 configs from corso config file")
-	}
-
-	s3Overrides(overrides)
-	aws := credentials.GetAWS(overrides)
-
-	if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 {
-		_, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get()
-		if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) {
-			aws = credentials.AWS{
-				AccessKey:    s3Cfg.AccessKey,
-				SecretKey:    s3Cfg.SecretKey,
-				SessionToken: s3Cfg.SessionToken,
-			}
-			err = nil
-		}
-
-		if err != nil {
-			return store, clues.Wrap(err, "validating aws credentials")
-		}
-	}
-
-	s3Cfg = storage.S3Config{
-		AWS:      aws,
-		Bucket:   str.First(overrides[storage.Bucket], s3Cfg.Bucket),
-		Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint),
-		Prefix:   str.First(overrides[storage.Prefix], s3Cfg.Prefix),
-		DoNotUseTLS: str.ParseBool(str.First(
-			overrides[storage.DoNotUseTLS],
-			strconv.FormatBool(s3Cfg.DoNotUseTLS),
-		)),
-		DoNotVerifyTLS: str.ParseBool(str.First(
-			overrides[storage.DoNotVerifyTLS],
-			strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
-		)),
-	}
-
-	// compose the common config and credentials
-	corso := GetAndInsertCorso(vpr.GetString(CorsoPassphrase))
-	if err := corso.Validate(); err != nil {
-		return store, clues.Wrap(err, "validating corso credentials")
-	}
-
-	cCfg := storage.CommonConfig{
-		Corso: corso,
-	}
-	// the following is a hack purely for integration testing.
-	// the value is not required, and if empty, kopia will default
-	// to its routine behavior
-	if t, ok := vpr.Get("corso-testing").(bool); t && ok {
-		dir, _ := filepath.Split(vpr.ConfigFileUsed())
-		cCfg.KopiaCfgDir = dir
-	}
-
-	// ensure required properties are present
-	if err := requireProps(map[string]string{
-		storage.Bucket:              s3Cfg.Bucket,
-		credentials.CorsoPassphrase: corso.CorsoPassphrase,
-	}); err != nil {
-		return storage.Storage{}, err
-	}
-
-	// build the storage
-	store, err = storage.NewStorage(storage.ProviderS3, s3Cfg, cCfg)
-	if err != nil {
-		return store, clues.Wrap(err, "configuring repository storage")
-	}
-
-	return store, nil
-}
-
-// GetCorso is a helper for aggregating Corso secrets and credentials.
-func GetAndInsertCorso(passphase string) credentials.Corso {
-	// fetch data from flag, env var or func param giving priority to func param
-	// Func param generally will be value fetched from config file using viper.
-	corsoPassph := str.First(flags.CorsoPassphraseFV, os.Getenv(credentials.CorsoPassphrase), passphase)
-
-	return credentials.Corso{
-		CorsoPassphrase: corsoPassph,
-	}
-}

src/cli/debug/debug.go

@@ -0,0 +1,120 @@
+package debug
+
+import (
+	"context"
+
+	"github.com/alcionai/clues"
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	. "github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/pkg/fault"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+var subCommandFuncs = []func() *cobra.Command{
+	metadataFilesCmd,
+}
+
+var debugCommands = []func(cmd *cobra.Command) *cobra.Command{
+	addOneDriveCommands,
+	addSharePointCommands,
+	addGroupsCommands,
+	addExchangeCommands,
+}
+
+// AddCommands attaches all `corso debug * *` commands to the parent.
+func AddCommands(cmd *cobra.Command) {
+	debugC, _ := utils.AddCommand(cmd, debugCmd(), utils.MarkDebugCommand())
+
+	for _, sc := range subCommandFuncs {
+		subCommand := sc()
+		utils.AddCommand(debugC, subCommand, utils.MarkDebugCommand())
+
+		for _, addTo := range debugCommands {
+			servCmd := addTo(subCommand)
+			flags.AddAllProviderFlags(servCmd)
+			flags.AddAllStorageFlags(servCmd)
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Commands
+// ---------------------------------------------------------------------------
+
+const debugCommand = "debug"
+
+// The debug category of commands.
+// `corso debug [<subcommand>] [<flag>...]`
+func debugCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   debugCommand,
+		Short: "debugging & troubleshooting utilities",
+		Long:  `debug the data stored in corso.`,
+		RunE:  handledebugCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// Handler for flat calls to `corso debug`.
+// Produces the same output as `corso debug --help`.
+func handledebugCmd(cmd *cobra.Command, args []string) error {
+	return cmd.Help()
+}
+
+// The debug metadataFiles subcommand.
+// `corso debug metadata-files <service> [<flag>...]`
+var metadataFilesCommand = "metadata-files"
+
+func metadataFilesCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   metadataFilesCommand,
+		Short: "display all the metadata file contents stored by the service",
+		RunE:  handleMetadataFilesCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// Handler for calls to `corso debug metadata-files`.
+// Produces the same output as `corso debug metadata-files --help`.
+func handleMetadataFilesCmd(cmd *cobra.Command, args []string) error {
+	return cmd.Help()
+}
+
+// ---------------------------------------------------------------------------
+// runners
+// ---------------------------------------------------------------------------
+
+func genericMetadataFiles(
+	ctx context.Context,
+	cmd *cobra.Command,
+	args []string,
+	sel selectors.Selector,
+	backupID string,
+) error {
+	ctx = clues.Add(ctx, "backup_id", backupID)
+
+	r, _, err := utils.GetAccountAndConnect(ctx, cmd, sel.PathService())
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	defer utils.CloseRepo(ctx, r)
+
+	// read metadata
+	files, err := r.GetBackupMetadata(ctx, sel, backupID, fault.New(true))
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "retrieving metadata files"))
+	}
+
+	for _, file := range files {
+		Infof(ctx, "\n------------------------------")
+		Info(ctx, file.Name)
+		Info(ctx, file.Path)
+		Pretty(ctx, file.Data)
+	}
+
+	return nil
+}

src/cli/debug/exchange.go

@@ -0,0 +1,66 @@
+package debug
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+// called by debug.go to map subcommands to provider-specific handling.
+func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case metadataFilesCommand:
+		c, _ = utils.AddCommand(cmd, exchangeMetadataFilesCmd(), utils.MarkDebugCommand())
+		c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+	}
+
+	return c
+}
+
+const (
+	exchangeServiceCommand          = "exchange"
+	exchangeServiceCommandUseSuffix = "--backup <backupId>"
+
+	//nolint:lll
+	exchangeServiceCommandDebugExamples = `# Display file contents for backup 1234abcd
+corso debug metadata-files exchange --backup 1234abcd-12ab-cd34-56de-1234abcd`
+)
+
+// `corso debug metadata-files exchange [<flag>...] <destination>`
+func exchangeMetadataFilesCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     exchangeServiceCommand,
+		Short:   "Display exchange metadata file content",
+		RunE:    metadataFilesExchangeCmd,
+		Args:    cobra.NoArgs,
+		Example: exchangeServiceCommandDebugExamples,
+	}
+}
+
+func metadataFilesExchangeCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	sel := selectors.NewExchangeBackup([]string{"unused-placeholder"})
+	sel.Include(sel.AllData())
+
+	return genericMetadataFiles(
+		ctx,
+		cmd,
+		args,
+		sel.Selector,
+		flags.BackupIDFV)
+}

src/cli/debug/exchange_test.go

@@ -0,0 +1,77 @@
+package debug
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type ExchangeUnitSuite struct {
+	tester.Suite
+}
+
+func TestExchangeUnitSuite(t *testing.T) {
+	suite.Run(t, &ExchangeUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *ExchangeUnitSuite) TestExchangeCommands() {
+	expectUse := exchangeServiceCommand + " " + exchangeServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{
+			name:        "metdata-files exchange",
+			use:         metadataFilesCommand,
+			expectUse:   expectUse,
+			expectShort: exchangeMetadataFilesCmd().Short,
+			expectRunE:  metadataFilesExchangeCmd,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: metadataFilesCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addExchangeCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					exchangeServiceCommand,
+					[]string{
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			flagsTD.AssertProviderFlags(t, cmd)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/debug/groups.go

@@ -0,0 +1,68 @@
+package debug
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+// called by debug.go to map subcommands to provider-specific handling.
+func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case metadataFilesCommand:
+		c, _ = utils.AddCommand(cmd, groupsMetadataFilesCmd(), utils.MarkDebugCommand())
+
+		c.Use = c.Use + " " + groupsServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+	}
+
+	return c
+}
+
+// TODO: correct examples
+const (
+	groupsServiceCommand          = "groups"
+	groupsServiceCommandUseSuffix = "--backup <backupId>"
+
+	//nolint:lll
+	groupsServiceCommandDebugExamples = `# Display file contents for backup 1234abcd
+	corso debug metadata-files groups --backup 1234abcd-12ab-cd34-56de-1234abcd`
+)
+
+// `corso debug metadata-files groups [<flag>...] <destination>`
+func groupsMetadataFilesCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     groupsServiceCommand,
+		Short:   "Display groups metadata file content",
+		RunE:    metadataFilesGroupsCmd,
+		Args:    cobra.NoArgs,
+		Example: groupsServiceCommandDebugExamples,
+	}
+}
+
+func metadataFilesGroupsCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	sel := selectors.NewGroupsBackup([]string{"unused-placeholder"})
+	sel.Include(sel.AllData())
+
+	return genericMetadataFiles(
+		ctx,
+		cmd,
+		args,
+		sel.Selector,
+		flags.BackupIDFV)
+}

src/cli/debug/groups_test.go

@@ -0,0 +1,76 @@
+package debug
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type GroupsUnitSuite struct {
+	tester.Suite
+}
+
+func TestGroupsUnitSuite(t *testing.T) {
+	suite.Run(t, &GroupsUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
+	expectUse := groupsServiceCommand + " " + groupsServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{
+			name:        "metdata-files groups",
+			use:         metadataFilesCommand,
+			expectUse:   expectUse,
+			expectShort: groupsMetadataFilesCmd().Short,
+			expectRunE:  metadataFilesGroupsCmd,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: metadataFilesCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addGroupsCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					groupsServiceCommand,
+					[]string{
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/debug/onedrive.go

@@ -0,0 +1,66 @@
+package debug
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+// called by debug.go to map subcommands to provider-specific handling.
+func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case metadataFilesCommand:
+		c, _ = utils.AddCommand(cmd, oneDriveMetadataFilesCmd(), utils.MarkDebugCommand())
+		c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+	}
+
+	return c
+}
+
+const (
+	oneDriveServiceCommand          = "onedrive"
+	oneDriveServiceCommandUseSuffix = "--backup <backupId>"
+
+	//nolint:lll
+	oneDriveServiceCommandDebugExamples = `# Display file contents for backup 1234abcd
+	corso debug metadata-files onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd`
+)
+
+// `corso debug metadata-files onedrive [<flag>...] <destination>`
+func oneDriveMetadataFilesCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     oneDriveServiceCommand,
+		Short:   "Display onedrive metadata file content",
+		RunE:    metadataFilesOneDriveCmd,
+		Args:    cobra.NoArgs,
+		Example: oneDriveServiceCommandDebugExamples,
+	}
+}
+
+func metadataFilesOneDriveCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	sel := selectors.NewOneDriveBackup([]string{"unused-placeholder"})
+	sel.Include(sel.AllData())
+
+	return genericMetadataFiles(
+		ctx,
+		cmd,
+		args,
+		sel.Selector,
+		flags.BackupIDFV)
+}

src/cli/debug/onedrive_test.go

@@ -0,0 +1,76 @@
+package debug
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type OneDriveUnitSuite struct {
+	tester.Suite
+}
+
+func TestOneDriveUnitSuite(t *testing.T) {
+	suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
+	expectUse := oneDriveServiceCommand + " " + oneDriveServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{
+			name:        "metadata-files onedrive",
+			use:         metadataFilesCommand,
+			expectUse:   expectUse,
+			expectShort: oneDriveMetadataFilesCmd().Short,
+			expectRunE:  metadataFilesOneDriveCmd,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: metadataFilesCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addOneDriveCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					oneDriveServiceCommand,
+					[]string{
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/debug/sharepoint.go

@@ -0,0 +1,66 @@
+package debug
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+// called by debug.go to map subcommands to provider-specific handling.
+func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case metadataFilesCommand:
+		c, _ = utils.AddCommand(cmd, sharePointMetadataFilesCmd(), utils.MarkDebugCommand())
+		c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+	}
+
+	return c
+}
+
+const (
+	sharePointServiceCommand          = "sharepoint"
+	sharePointServiceCommandUseSuffix = "--backup <backupId>"
+
+	//nolint:lll
+	sharePointServiceCommandDebugExamples = `# Display file contents for backup 1234abcd
+	corso debug metadata-files sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd`
+)
+
+// `corso debug metadata-files sharepoint [<flag>...] <destination>`
+func sharePointMetadataFilesCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     sharePointServiceCommand,
+		Short:   "Display sharepoint metadata file content",
+		RunE:    metadataFilesSharePointCmd,
+		Args:    cobra.NoArgs,
+		Example: sharePointServiceCommandDebugExamples,
+	}
+}
+
+func metadataFilesSharePointCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	sel := selectors.NewSharePointBackup([]string{"unused-placeholder"})
+	sel.Include(sel.LibraryFolders(selectors.Any()))
+
+	return genericMetadataFiles(
+		ctx,
+		cmd,
+		args,
+		sel.Selector,
+		flags.BackupIDFV)
+}

src/cli/debug/sharepoint_test.go

@@ -0,0 +1,76 @@
+package debug
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type SharePointUnitSuite struct {
+	tester.Suite
+}
+
+func TestSharePointUnitSuite(t *testing.T) {
+	suite.Run(t, &SharePointUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
+	expectUse := sharePointServiceCommand + " " + sharePointServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{
+			name:        "metdata-files sharepoint",
+			use:         metadataFilesCommand,
+			expectUse:   expectUse,
+			expectShort: sharePointMetadataFilesCmd().Short,
+			expectRunE:  metadataFilesSharePointCmd,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: metadataFilesCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addSharePointCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					sharePointServiceCommand,
+					[]string{
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/export/exchange.go

@@ -0,0 +1,101 @@
+package export
+
+import (
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+)
+
+// called by export.go to map subcommands to provider-specific handling.
+func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case exportCommand:
+		c, _ = utils.AddCommand(cmd, exchangeExportCmd())
+
+		c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+		flags.AddExchangeDetailsAndRestoreFlags(c, true)
+		flags.AddExportConfigFlags(c)
+		flags.AddFailFastFlag(c)
+	}
+
+	return c
+}
+
+const (
+	exchangeServiceCommand          = "exchange"
+	exchangeServiceCommandUseSuffix = "<destination> --backup <backupId>"
+
+	// TODO(meain): remove message about only supporting email exports once others are added
+	//nolint:lll
+	exchangeServiceCommandExportExamples = `> Only email exports are supported as of now.
+
+# Export emails with ID 98765abcdef and 12345abcdef from Alice's last backup (1234abcd...) to my-folder
+corso export exchange my-folder --backup 1234abcd-12ab-cd34-56de-1234abcd --email 98765abcdef,12345abcdef
+
+# Export emails with subject containing "Hello world" in the "Inbox" to my-folder
+corso export exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --email-subject "Hello world" --email-folder Inbox my-folder`
+
+// TODO(meain): Uncomment once support for these are added
+// 		`# Export an entire calendar to my-folder
+// corso export exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \
+//     --event-calendar Calendar my-folder
+
+// # Export the contact with ID abdef0101 to my-folder
+// corso export exchange --backup 1234abcd-12ab-cd34-56de-1234abcd --contact abdef0101 my-folder`
+)
+
+// `corso export exchange [<flag>...] <destination>`
+func exchangeExportCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   exchangeServiceCommand,
+		Short: "Export M365 Exchange service data",
+		RunE:  exportExchangeCmd,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("missing export destination")
+			}
+
+			return nil
+		},
+		Example: exchangeServiceCommandExportExamples,
+	}
+}
+
+// processes an exchange service export.
+func exportExchangeCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	opts := utils.MakeExchangeOpts(cmd)
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	if err := utils.ValidateExchangeRestoreFlags(flags.BackupIDFV, opts); err != nil {
+		return err
+	}
+
+	sel := utils.IncludeExchangeRestoreDataSelectors(opts)
+	utils.FilterExchangeRestoreInfoSelectors(sel, opts)
+
+	return runExport(
+		ctx,
+		cmd,
+		args,
+		opts.ExportCfg,
+		sel.Selector,
+		flags.BackupIDFV,
+		"Exchange",
+		defaultAcceptedFormatTypes)
+}

src/cli/export/exchange_test.go

@@ -0,0 +1,78 @@
+package export
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type ExchangeUnitSuite struct {
+	tester.Suite
+}
+
+func TestExchangeUnitSuite(t *testing.T) {
+	suite.Run(t, &ExchangeUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *ExchangeUnitSuite) TestAddExchangeCommands() {
+	expectUse := exchangeServiceCommand + " " + exchangeServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{"export exchange", exportCommand, expectUse, exchangeExportCmd().Short, exportExchangeCmd},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: exportCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addExchangeCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					exchangeServiceCommand,
+					[]string{
+						flagsTD.RestoreDestination,
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+						"--" + flags.FormatFN, flagsTD.FormatType,
+						"--" + flags.ArchiveFN,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			opts := utils.MakeExchangeOpts(cmd)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			assert.Equal(t, flagsTD.Archive, opts.ExportCfg.Archive)
+			assert.Equal(t, flagsTD.FormatType, opts.ExportCfg.Format)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/export/export.go

@@ -0,0 +1,153 @@
+package export
+
+import (
+	"context"
+	"errors"
+
+	"github.com/alcionai/clues"
+	"github.com/dustin/go-humanize"
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	. "github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/data"
+	"github.com/alcionai/corso/src/internal/observe"
+	"github.com/alcionai/corso/src/internal/operations"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/dttm"
+	"github.com/alcionai/corso/src/pkg/export"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+var exportCommands = []func(cmd *cobra.Command) *cobra.Command{
+	addOneDriveCommands,
+	addSharePointCommands,
+	addGroupsCommands,
+	addExchangeCommands,
+}
+
+var defaultAcceptedFormatTypes = []string{string(control.DefaultFormat)}
+
+// AddCommands attaches all `corso export * *` commands to the parent.
+func AddCommands(cmd *cobra.Command) {
+	subCommand := exportCmd()
+	cmd.AddCommand(subCommand)
+
+	for _, addExportTo := range exportCommands {
+		sc := addExportTo(subCommand)
+		flags.AddAllStorageFlags(sc)
+	}
+}
+
+const exportCommand = "export"
+
+// The export category of commands.
+// `corso export [<subcommand>] [<flag>...]`
+func exportCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   exportCommand,
+		Short: "Export your service data",
+		Long:  `Export the data stored in one of your M365 services.`,
+		RunE:  handleExportCmd,
+		Args:  cobra.NoArgs,
+	}
+}
+
+// Handler for flat calls to `corso export`.
+// Produces the same output as `corso export --help`.
+func handleExportCmd(cmd *cobra.Command, args []string) error {
+	return cmd.Help()
+}
+
+func runExport(
+	ctx context.Context,
+	cmd *cobra.Command,
+	args []string,
+	ueco utils.ExportCfgOpts,
+	sel selectors.Selector,
+	backupID, serviceName string,
+	acceptedFormatTypes []string,
+) error {
+	if err := utils.ValidateExportConfigFlags(&ueco, acceptedFormatTypes); err != nil {
+		return Only(ctx, err)
+	}
+
+	r, _, err := utils.GetAccountAndConnect(ctx, cmd, sel.PathService())
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	defer utils.CloseRepo(ctx, r)
+
+	exportLocation := args[0]
+	if len(exportLocation) == 0 {
+		// This should not be possible, but adding it just in case.
+		exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem)
+	}
+
+	Infof(ctx, "Exporting to folder %s", exportLocation)
+
+	eo, err := r.NewExport(
+		ctx,
+		backupID,
+		sel,
+		utils.MakeExportConfig(ctx, ueco))
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to initialize "+serviceName+" export"))
+	}
+
+	collections, err := eo.Run(ctx)
+	if err != nil {
+		if errors.Is(err, data.ErrNotFound) {
+			return Only(ctx, clues.New("Backup or backup details missing for id "+backupID))
+		}
+
+		return Only(ctx, clues.Wrap(err, "Failed to run "+serviceName+" export"))
+	}
+
+	if err = showExportProgress(ctx, eo, collections, exportLocation); err != nil {
+		return err
+	}
+
+	if len(eo.Errors.Recovered()) > 0 {
+		Infof(ctx, "\nExport failures")
+
+		for _, i := range eo.Errors.Recovered() {
+			Err(ctx, i.Error())
+		}
+
+		return Only(ctx, clues.New("Incomplete export of "+serviceName+" data"))
+	}
+
+	stats := eo.GetStats()
+	if len(stats) > 0 {
+		Infof(ctx, "\nExport details")
+	}
+
+	for k, s := range stats {
+		Infof(ctx, "%s: %d items (%s)", k.HumanString(), s.ResourceCount, humanize.Bytes(uint64(s.BytesRead)))
+	}
+
+	return nil
+}
+
+// slim wrapper that allows us to defer the progress bar closure with the expected scope.
+func showExportProgress(
+	ctx context.Context,
+	op operations.ExportOperation,
+	collections []export.Collectioner,
+	exportLocation string,
+) error {
+	// It would be better to give a progressbar than a spinner, but we
+	// have any way of knowing how many files are available as of now.
+	progressMessage := observe.MessageWithCompletion(ctx, observe.DefaultCfg(), "Writing data to disk")
+	defer close(progressMessage)
+
+	err := export.ConsumeExportCollections(ctx, exportLocation, collections, op.Errors)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	return nil
+}

src/cli/export/groups.go

@@ -0,0 +1,115 @@
+package export
+
+import (
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/pkg/control"
+)
+
+// called by export.go to map subcommands to provider-specific handling.
+func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case exportCommand:
+		c, _ = utils.AddCommand(cmd, groupsExportCmd(), utils.MarkPreviewCommand())
+
+		c.Use = c.Use + " " + groupsServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+		flags.AddSiteFlag(c, false)
+		flags.AddSiteIDFlag(c, false)
+		flags.AddSharePointDetailsAndRestoreFlags(c)
+		flags.AddGroupDetailsAndRestoreFlags(c)
+		flags.AddExportConfigFlags(c)
+		flags.AddFailFastFlag(c)
+	}
+
+	return c
+}
+
+const (
+	groupsServiceCommand          = "groups"
+	teamsServiceCommand           = "teams"
+	groupsServiceCommandUseSuffix = "<destination> --backup <backupId>"
+
+	//nolint:lll
+	groupsServiceCommandExportExamples = `# Export a message in Marketing's last backup (1234abcd...) to /my-exports
+corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --message 98765abcdef
+
+# Export all messages named in channel "Finance Reports" to the current directory
+corso export groups . --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --message '*' --channel "Finance Reports"
+
+# Export all messages in channel "Finance Reports" that were created before 2020 to /my-exports
+corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd
+    --channel "Finance Reports" --message-created-before 2020-01-01T00:00:00
+
+# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to /my-exports
+corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00
+
+# Export all posts from a conversation with topic "hello world" from group mailbox's last backup to /my-exports
+corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world"
+
+# Export post with ID 98765abcdef from a conversation from group mailbox's last backup to /my-exports
+corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world" --post 98765abcdef`
+)
+
+// `corso export groups [<flag>...] <destination>`
+func groupsExportCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     groupsServiceCommand,
+		Aliases: []string{teamsServiceCommand},
+		Short:   "Export M365 Groups service data",
+		RunE:    exportGroupsCmd,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("missing export destination")
+			}
+
+			return nil
+		},
+		Example: groupsServiceCommandExportExamples,
+	}
+}
+
+// processes an groups service export.
+func exportGroupsCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	opts := utils.MakeGroupsOpts(cmd)
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	if err := utils.ValidateGroupsRestoreFlags(flags.BackupIDFV, opts, false); err != nil {
+		return err
+	}
+
+	sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts)
+	utils.FilterGroupsRestoreInfoSelectors(sel, opts)
+
+	acceptedGroupsFormatTypes := []string{
+		string(control.DefaultFormat),
+		string(control.JSONFormat),
+	}
+
+	return runExport(
+		ctx,
+		cmd,
+		args,
+		opts.ExportCfg,
+		sel.Selector,
+		flags.BackupIDFV,
+		"Groups",
+		acceptedGroupsFormatTypes)
+}

src/cli/export/groups_test.go

@@ -0,0 +1,78 @@
+package export
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type GroupsUnitSuite struct {
+	tester.Suite
+}
+
+func TestGroupsUnitSuite(t *testing.T) {
+	suite.Run(t, &GroupsUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
+	expectUse := groupsServiceCommand + " " + groupsServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{"export groups", exportCommand, expectUse, groupsExportCmd().Short, exportGroupsCmd},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: exportCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addGroupsCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					groupsServiceCommand,
+					[]string{
+						flagsTD.RestoreDestination,
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+						"--" + flags.FormatFN, flagsTD.FormatType,
+						"--" + flags.ArchiveFN,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			opts := utils.MakeGroupsOpts(cmd)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			assert.Equal(t, flagsTD.Archive, opts.ExportCfg.Archive)
+			assert.Equal(t, flagsTD.FormatType, opts.ExportCfg.Format)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/export/onedrive.go

@@ -0,0 +1,94 @@
+package export
+
+import (
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+)
+
+// called by export.go to map subcommands to provider-specific handling.
+func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case exportCommand:
+		c, _ = utils.AddCommand(cmd, oneDriveExportCmd())
+
+		c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+		flags.AddOneDriveDetailsAndRestoreFlags(c)
+		flags.AddExportConfigFlags(c)
+		flags.AddFailFastFlag(c)
+	}
+
+	return c
+}
+
+const (
+	oneDriveServiceCommand          = "onedrive"
+	oneDriveServiceCommandUseSuffix = "<destination> --backup <backupId>"
+
+	//nolint:lll
+	oneDriveServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to /my-exports
+corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef
+
+# Export files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" to he current directory
+corso export onedrive . --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports"
+
+# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to /my-exports
+corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00`
+)
+
+// `corso export onedrive [<flag>...] <destination>`
+func oneDriveExportCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   oneDriveServiceCommand,
+		Short: "Export M365 OneDrive service data",
+		RunE:  exportOneDriveCmd,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("missing export destination")
+			}
+
+			return nil
+		},
+		Example: oneDriveServiceCommandExportExamples,
+	}
+}
+
+// processes an onedrive service export.
+func exportOneDriveCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	opts := utils.MakeOneDriveOpts(cmd)
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	if err := utils.ValidateOneDriveRestoreFlags(flags.BackupIDFV, opts); err != nil {
+		return err
+	}
+
+	sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
+	utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
+
+	return runExport(
+		ctx,
+		cmd,
+		args,
+		opts.ExportCfg,
+		sel.Selector,
+		flags.BackupIDFV,
+		"OneDrive",
+		defaultAcceptedFormatTypes)
+}

src/cli/export/onedrive_test.go

@@ -0,0 +1,92 @@
+package export
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type OneDriveUnitSuite struct {
+	tester.Suite
+}
+
+func TestOneDriveUnitSuite(t *testing.T) {
+	suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
+	expectUse := oneDriveServiceCommand + " " + oneDriveServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{"export onedrive", exportCommand, expectUse, oneDriveExportCmd().Short, exportOneDriveCmd},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: exportCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addOneDriveCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					oneDriveServiceCommand,
+					[]string{
+						flagsTD.RestoreDestination,
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+						"--" + flags.FileFN, flagsTD.FlgInputs(flagsTD.FileNameInput),
+						"--" + flags.FolderFN, flagsTD.FlgInputs(flagsTD.FolderPathInput),
+						"--" + flags.FileCreatedAfterFN, flagsTD.FileCreatedAfterInput,
+						"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
+						"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
+						"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
+
+						"--" + flags.FormatFN, flagsTD.FormatType,
+
+						// bool flags
+						"--" + flags.ArchiveFN,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			opts := utils.MakeOneDriveOpts(cmd)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			assert.ElementsMatch(t, flagsTD.FileNameInput, opts.FileName)
+			assert.ElementsMatch(t, flagsTD.FolderPathInput, opts.FolderPath)
+			assert.Equal(t, flagsTD.FileCreatedAfterInput, opts.FileCreatedAfter)
+			assert.Equal(t, flagsTD.FileCreatedBeforeInput, opts.FileCreatedBefore)
+			assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
+			assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
+			assert.Equal(t, flagsTD.CorsoPassphrase, flags.PassphraseFV)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/export/sharepoint.go

@@ -0,0 +1,118 @@
+package export
+
+import (
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/utils"
+)
+
+// called by export.go to map subcommands to provider-specific handling.
+func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case exportCommand:
+		c, _ = utils.AddCommand(cmd, sharePointExportCmd())
+
+		c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
+
+		flags.AddBackupIDFlag(c, true)
+		flags.AddSharePointDetailsAndRestoreFlags(c)
+		flags.AddExportConfigFlags(c)
+		flags.AddFailFastFlag(c)
+	}
+
+	return c
+}
+
+const (
+	sharePointServiceCommand          = "sharepoint"
+	sharePointServiceCommandUseSuffix = "<destination> --backup <backupId>"
+
+	//nolint:lll
+	sharePointServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's latest backup (1234abcd...) to /my-exports
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef my-exports
+
+# Export file "ServerRenderTemplate.xsl" in "Display Templates/Style Sheets" as archive to the current directory
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets" --archive .
+
+# Export all files in the folder "Display Templates/Style Sheets" that were created before 2020 to /my-exports
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --file-created-before 2020-01-01T00:00:00 --folder "Display Templates/Style Sheets" my-exports
+
+# Export all files in the "Documents" library to the current directory.
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --library Documents --folder "Display Templates/Style Sheets" .
+
+# Export lists by their name(s)
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list "list-name-1,list-name-2" .
+
+# Export lists created after a given time
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-created-after 2024-01-01T12:23:34 .
+
+# Export lists created before a given time
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-created-before 2024-01-01T12:23:34 .
+
+# Export lists modified before a given time
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-modified-before 2024-01-01T12:23:34 .
+
+# Export lists modified after a given time
+corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
+    --list-modified-after 2024-01-01T12:23:34 .`
+)
+
+// `corso export sharepoint [<flag>...] <destination>`
+func sharePointExportCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   sharePointServiceCommand,
+		Short: "Export M365 SharePoint service data",
+		RunE:  exportSharePointCmd,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return errors.New("missing export destination")
+			}
+
+			return nil
+		},
+		Example: sharePointServiceCommandExportExamples,
+	}
+}
+
+// processes an sharepoint service export.
+func exportSharePointCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	if utils.HasNoFlagsAndShownHelp(cmd) {
+		return nil
+	}
+
+	opts := utils.MakeSharePointOpts(cmd)
+
+	if flags.RunModeFV == flags.RunModeFlagTest {
+		return nil
+	}
+
+	if err := utils.ValidateSharePointRestoreFlags(flags.BackupIDFV, opts); err != nil {
+		return err
+	}
+
+	sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
+	utils.FilterSharePointRestoreInfoSelectors(sel, opts)
+
+	return runExport(
+		ctx,
+		cmd,
+		args,
+		opts.ExportCfg,
+		sel.Selector,
+		flags.BackupIDFV,
+		"SharePoint",
+		defaultAcceptedFormatTypes)
+}

src/cli/export/sharepoint_test.go

@@ -0,0 +1,106 @@
+package export
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type SharePointUnitSuite struct {
+	tester.Suite
+}
+
+func TestSharePointUnitSuite(t *testing.T) {
+	suite.Run(t, &SharePointUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
+	expectUse := sharePointServiceCommand + " " + sharePointServiceCommandUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{"export sharepoint", exportCommand, expectUse, sharePointExportCmd().Short, exportSharePointCmd},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			parent := &cobra.Command{Use: exportCommand}
+
+			cmd := cliTD.SetUpCmdHasFlags(
+				t,
+				parent,
+				addSharePointCommands,
+				[]cliTD.UseCobraCommandFn{
+					flags.AddAllProviderFlags,
+					flags.AddAllStorageFlags,
+				},
+				flagsTD.WithFlags(
+					sharePointServiceCommand,
+					[]string{
+						flagsTD.RestoreDestination,
+						"--" + flags.RunModeFN, flags.RunModeFlagTest,
+						"--" + flags.BackupFN, flagsTD.BackupInput,
+						"--" + flags.LibraryFN, flagsTD.LibraryInput,
+						"--" + flags.FileFN, flagsTD.FlgInputs(flagsTD.FileNameInput),
+						"--" + flags.FolderFN, flagsTD.FlgInputs(flagsTD.FolderPathInput),
+						"--" + flags.FileCreatedAfterFN, flagsTD.FileCreatedAfterInput,
+						"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
+						"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
+						"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
+						"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
+						"--" + flags.ListCreatedAfterFN, flagsTD.ListCreatedAfterInput,
+						"--" + flags.ListCreatedBeforeFN, flagsTD.ListCreatedBeforeInput,
+						"--" + flags.ListModifiedAfterFN, flagsTD.ListModifiedAfterInput,
+						"--" + flags.ListModifiedBeforeFN, flagsTD.ListModifiedBeforeInput,
+						"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
+						"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
+						"--" + flags.FormatFN, flagsTD.FormatType,
+						"--" + flags.ArchiveFN,
+					},
+					flagsTD.PreparedProviderFlags(),
+					flagsTD.PreparedStorageFlags()))
+
+			cliTD.CheckCmdChild(
+				t,
+				parent,
+				3,
+				test.expectUse,
+				test.expectShort,
+				test.expectRunE)
+
+			opts := utils.MakeSharePointOpts(cmd)
+
+			assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
+			assert.Equal(t, flagsTD.LibraryInput, opts.Library)
+			assert.ElementsMatch(t, flagsTD.FileNameInput, opts.FileName)
+			assert.ElementsMatch(t, flagsTD.FolderPathInput, opts.FolderPath)
+			assert.Equal(t, flagsTD.FileCreatedAfterInput, opts.FileCreatedAfter)
+			assert.Equal(t, flagsTD.FileCreatedBeforeInput, opts.FileCreatedBefore)
+			assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
+			assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
+			assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
+			assert.Equal(t, flagsTD.ListCreatedAfterInput, opts.ListCreatedAfter)
+			assert.Equal(t, flagsTD.ListCreatedBeforeInput, opts.ListCreatedBefore)
+			assert.Equal(t, flagsTD.ListModifiedAfterInput, opts.ListModifiedAfter)
+			assert.Equal(t, flagsTD.ListModifiedBeforeInput, opts.ListModifiedBefore)
+			assert.ElementsMatch(t, flagsTD.PageInput, opts.Page)
+			assert.ElementsMatch(t, flagsTD.PageFolderInput, opts.PageFolder)
+			assert.Equal(t, flagsTD.Archive, opts.ExportCfg.Archive)
+			assert.Equal(t, flagsTD.FormatType, opts.ExportCfg.Format)
+			flagsTD.AssertStorageFlags(t, cmd)
+		})
+	}
+}

src/cli/flags/backup_create.go

@@ -0,0 +1,11 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func AddGenericBackupFlags(cmd *cobra.Command) {
+	AddFailFastFlag(cmd)
+	AddDisableIncrementalsFlag(cmd)
+	AddForceItemDataDownloadFlag(cmd)
+}

src/cli/flags/backup_list.go

@@ -0,0 +1,38 @@
+package flags
+
+import "github.com/spf13/cobra"
+
+const Show = "show"
+
+func AddAllBackupListFlags(cmd *cobra.Command) {
+	AddFailedItemsFN(cmd)
+	AddSkippedItemsFN(cmd)
+	AddRecoveredErrorsFN(cmd)
+	AddAlertsFN(cmd)
+}
+
+func AddFailedItemsFN(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.StringVar(
+		&FailedItemsFV, FailedItemsFN, Show,
+		"Toggles showing or hiding the list of items that failed.")
+	cobra.CheckErr(fs.MarkHidden(FailedItemsFN))
+}
+
+func AddSkippedItemsFN(cmd *cobra.Command) {
+	cmd.Flags().StringVar(
+		&ListSkippedItemsFV, SkippedItemsFN, Show,
+		"Toggles showing or hiding the list of items that were skipped.")
+}
+
+func AddRecoveredErrorsFN(cmd *cobra.Command) {
+	cmd.Flags().StringVar(
+		&ListRecoveredErrorsFV, RecoveredErrorsFN, Show,
+		"Toggles showing or hiding the list of errors which Corso recovered from.")
+}
+
+func AddAlertsFN(cmd *cobra.Command) {
+	cmd.Flags().StringVar(
+		&ListAlertsFV, AlertsFN, Show,
+		"Toggles showing or hiding the list of alerts produced during the operation.")
+}

src/cli/flags/exchange.go

@@ -49,7 +49,7 @@ var (
 
 // AddExchangeDetailsAndRestoreFlags adds flags that are common to both the
 // details and restore commands.
-func AddExchangeDetailsAndRestoreFlags(cmd *cobra.Command) {
+func AddExchangeDetailsAndRestoreFlags(cmd *cobra.Command, emailOnly bool) {
 	fs := cmd.Flags()
 
 	// email flags
@@ -78,6 +78,12 @@ func AddExchangeDetailsAndRestoreFlags(cmd *cobra.Command) {
 		EmailReceivedBeforeFN, "",
 		"Select emails received before this datetime.")
 
+	// NOTE: Only temporary until we add support for exporting the
+	// others as well in exchange.
+	if emailOnly {
+		return
+	}
+
 	// event flags
 	fs.StringSliceVar(
 		&EventFV,

src/cli/flags/export.go

@@ -0,0 +1,23 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+const (
+	ArchiveFN = "archive"
+	FormatFN  = "format"
+)
+
+var (
+	ArchiveFV bool
+	FormatFV  string
+)
+
+// AddExportConfigFlags adds the restore config flag set.
+func AddExportConfigFlags(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.BoolVar(&ArchiveFV, ArchiveFN, false, "Export data as an archive instead of individual files")
+	fs.StringVar(&FormatFV, FormatFN, "", "Specify the export file format")
+	cobra.CheckErr(fs.MarkHidden(FormatFN))
+}

src/cli/flags/filesystem.go

@@ -0,0 +1,48 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/pkg/storage"
+)
+
+// filesystem flag names
+const (
+	FilesystemPathFN = "path"
+)
+
+// filesystem flag values
+var (
+	FilesystemPathFV string
+)
+
+func AddFilesystemFlags(cmd *cobra.Command) {
+	fs := cmd.Flags()
+
+	AddAzureCredsFlags(cmd)
+	AddCorsoPassphaseFlags(cmd)
+
+	fs.StringVar(
+		&FilesystemPathFV,
+		FilesystemPathFN,
+		"",
+		"path to local or network storage")
+	cobra.CheckErr(cmd.MarkFlagRequired(FilesystemPathFN))
+}
+
+func FilesystemFlagOverrides(cmd *cobra.Command) map[string]string {
+	fs := GetPopulatedFlags(cmd)
+	return PopulateFilesystemFlags(fs)
+}
+
+func PopulateFilesystemFlags(flagset PopulatedFlags) map[string]string {
+	fsOverrides := map[string]string{
+		storage.StorageProviderTypeKey: storage.ProviderFilesystem.String(),
+	}
+
+	if _, ok := flagset[FilesystemPathFN]; ok {
+		fsOverrides[FilesystemPathFN] = FilesystemPathFV
+	}
+
+	return fsOverrides
+}

src/cli/flags/groups.go

@@ -0,0 +1,93 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+const (
+	DataMessages      = "messages"
+	DataConversations = "conversations"
+)
+
+const (
+	ChannelFN      = "channel"
+	ConversationFN = "conversation"
+	GroupFN        = "group"
+	MessageFN      = "message"
+	PostFN         = "post"
+
+	MessageCreatedAfterFN    = "message-created-after"
+	MessageCreatedBeforeFN   = "message-created-before"
+	MessageLastReplyAfterFN  = "message-last-reply-after"
+	MessageLastReplyBeforeFN = "message-last-reply-before"
+)
+
+var (
+	ChannelFV      []string
+	ConversationFV []string
+	GroupFV        []string
+	MessageFV      []string
+	PostFV         []string
+
+	MessageCreatedAfterFV    string
+	MessageCreatedBeforeFV   string
+	MessageLastReplyAfterFV  string
+	MessageLastReplyBeforeFV string
+)
+
+func AddGroupDetailsAndRestoreFlags(cmd *cobra.Command) {
+	fs := cmd.Flags()
+
+	fs.StringSliceVar(
+		&ChannelFV,
+		ChannelFN, nil,
+		"Select data within a Team's Channel.")
+
+	fs.StringSliceVar(
+		&MessageFV,
+		MessageFN, nil,
+		"Select messages by reference.")
+
+	fs.StringVar(
+		&MessageCreatedAfterFV,
+		MessageCreatedAfterFN, "",
+		"Select messages created after this datetime.")
+
+	fs.StringVar(
+		&MessageCreatedBeforeFV,
+		MessageCreatedBeforeFN, "",
+		"Select messages created before this datetime.")
+
+	fs.StringVar(
+		&MessageLastReplyAfterFV,
+		MessageLastReplyAfterFN, "",
+		"Select messages with replies after this datetime.")
+
+	fs.StringVar(
+		&MessageLastReplyBeforeFV,
+		MessageLastReplyBeforeFN, "",
+		"Select messages with replies before this datetime.")
+
+	fs.StringSliceVar(
+		&ConversationFV,
+		ConversationFN, nil,
+		"Select data within a Group's Conversation.")
+
+	fs.StringSliceVar(
+		&PostFV,
+		PostFN, nil,
+		"Select Conversation Posts by reference.")
+}
+
+// AddGroupFlag adds the --group flag, which accepts either the id,
+// the display name, or the mailbox address as its values.  Users are
+// expected to supply the display name.  The ID is supported becase, well,
+// IDs.  The mailbox address is supported as a lookup fallback for certain
+// SDK cases, therefore it's also supported here, though that support
+// isn't exposed to end users.
+func AddGroupFlag(cmd *cobra.Command) {
+	cmd.Flags().StringSliceVar(
+		&GroupFV,
+		GroupFN, nil,
+		"Backup data by group; accepts '"+Wildcard+"' to select all groups.")
+}

src/cli/flags/maintenance.go

@@ -7,13 +7,17 @@ import (
 )
 
 const (
-	MaintenanceModeFN  = "mode"
-	ForceMaintenanceFN = "force"
+	MaintenanceModeFN     = "mode"
+	ForceMaintenanceFN    = "force"
+	UserMaintenanceFN     = "user"
+	HostnameMaintenanceFN = "host"
 )
 
 var (
-	MaintenanceModeFV  string
-	ForceMaintenanceFV bool
+	MaintenanceModeFV     string
+	ForceMaintenanceFV    bool
+	UserMaintenanceFV     string
+	HostnameMaintenanceFV string
 )
 
 func AddMaintenanceModeFlag(cmd *cobra.Command) {
@@ -22,12 +26,9 @@ func AddMaintenanceModeFlag(cmd *cobra.Command) {
 		&MaintenanceModeFV,
 		MaintenanceModeFN,
 		repository.CompleteMaintenance.String(),
-		"Type of maintenance operation to run. Pass '"+
-			repository.MetadataMaintenance.String()+"' to run a faster maintenance "+
-			"that does minimal clean-up and optimization. Pass '"+
-			repository.CompleteMaintenance.String()+"' to fully compact existing "+
-			"data and delete unused data.")
-	cobra.CheckErr(fs.MarkHidden(MaintenanceModeFN))
+		"Type of maintenance operation to run ('"+
+			repository.MetadataMaintenance.String()+"' | '"+
+			repository.CompleteMaintenance.String()+"' )")
 }
 
 func AddForceMaintenanceFlag(cmd *cobra.Command) {
@@ -39,3 +40,21 @@ func AddForceMaintenanceFlag(cmd *cobra.Command) {
 		"Force maintenance. Caution: user must ensure this is not run concurrently on a single repo")
 	cobra.CheckErr(fs.MarkHidden(ForceMaintenanceFN))
 }
+
+func AddMaintenanceUserFlag(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.StringVar(
+		&UserMaintenanceFV,
+		UserMaintenanceFN,
+		"",
+		"Attempt to run maintenance as the specified user for the repo owner user")
+}
+
+func AddMaintenanceHostnameFlag(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.StringVar(
+		&HostnameMaintenanceFV,
+		HostnameMaintenanceFN,
+		"",
+		"Attempt to run maintenance with the specified hostname for the repo owner hostname")
+}

src/cli/flags/onedrive.go

@@ -12,6 +12,8 @@ const (
 	FileCreatedBeforeFN  = "file-created-before"
 	FileModifiedAfterFN  = "file-modified-after"
 	FileModifiedBeforeFN = "file-modified-before"
+
+	UseOldDeltaProcessFN = "use-old-delta-process"
 )
 
 var (
@@ -22,6 +24,8 @@ var (
 	FileCreatedBeforeFV  string
 	FileModifiedAfterFV  string
 	FileModifiedBeforeFV string
+
+	UseOldDeltaProcessFV bool
 )
 
 // AddOneDriveDetailsAndRestoreFlags adds flags that are common to both the
@@ -43,6 +47,7 @@ func AddOneDriveDetailsAndRestoreFlags(cmd *cobra.Command) {
 		&FileCreatedAfterFV,
 		FileCreatedAfterFN, "",
 		"Select files created after this datetime.")
+
 	fs.StringVar(
 		&FileCreatedBeforeFV,
 		FileCreatedBeforeFN, "",

src/cli/flags/options.go

@@ -5,37 +5,47 @@ import (
 )
 
 const (
-	DisableConcurrencyLimiterFN = "disable-concurrency-limiter"
-	DisableDeltaFN              = "disable-delta"
-	DisableIncrementalsFN       = "disable-incrementals"
-	EnableImmutableIDFN         = "enable-immutable-id"
-	FailFastFN                  = "fail-fast"
-	FailedItemsFN               = "failed-items"
-	FetchParallelismFN          = "fetch-parallelism"
-	NoStatsFN                   = "no-stats"
-	RecoveredErrorsFN           = "recovered-errors"
-	RestorePermissionsFN        = "restore-permissions"
-	RunModeFN                   = "run-mode"
-	SkippedItemsFN              = "skipped-items"
-	SkipReduceFN                = "skip-reduce"
+	AlertsFN                      = "alerts"
+	ConfigFileFN                  = "config-file"
+	DeltaPageSizeFN               = "delta-page-size"
+	DisableDeltaFN                = "disable-delta"
+	DisableIncrementalsFN         = "disable-incrementals"
+	DisableLazyItemReaderFN       = "disable-lazy-item-reader"
+	DisableSlidingWindowLimiterFN = "disable-sliding-window-limiter"
+	ForceItemDataDownloadFN       = "force-item-data-download"
+	EnableImmutableIDFN           = "enable-immutable-id"
+	FailFastFN                    = "fail-fast"
+	FailedItemsFN                 = "failed-items"
+	FetchParallelismFN            = "fetch-parallelism"
+	NoPermissionsFN               = "no-permissions"
+	NoStatsFN                     = "no-stats"
+	RecoveredErrorsFN             = "recovered-errors"
+	RunModeFN                     = "run-mode"
+	SkippedItemsFN                = "skipped-items"
+	SkipReduceFN                  = "skip-reduce"
 )
 
 var (
-	DisableConcurrencyLimiterFV bool
-	DisableDeltaFV              bool
-	DisableIncrementalsFV       bool
-	EnableImmutableIDFV         bool
-	FailFastFV                  bool
-	FetchParallelismFV          int
-	ListFailedItemsFV           string
-	ListSkippedItemsFV          string
-	ListRecoveredErrorsFV       string
-	NoStatsFV                   bool
+	ConfigFileFV                  string
+	DeltaPageSizeFV               int
+	DisableDeltaFV                bool
+	DisableIncrementalsFV         bool
+	DisableLazyItemReaderFV       bool
+	DisableSlidingWindowLimiterFV bool
+	ForceItemDataDownloadFV       bool
+	EnableImmutableIDFV           bool
+	FailFastFV                    bool
+	FailedItemsFV                 string
+	FetchParallelismFV            int
+	ListAlertsFV                  string
+	ListSkippedItemsFV            string
+	ListRecoveredErrorsFV         string
+	NoPermissionsFV               bool
+	NoStatsFV                     bool
 	// RunMode describes the type of run, such as:
 	// flagtest, dry, run.  Should default to 'run'.
-	RunModeFV            string
-	RestorePermissionsFV bool
-	SkipReduceFV         bool
+	RunModeFV    string
+	SkipReduceFV bool
 )
 
 // well-known flag values
@@ -58,10 +68,10 @@ func AddFailFastFlag(cmd *cobra.Command) {
 	cobra.CheckErr(fs.MarkHidden(FailFastFN))
 }
 
-// AddRestorePermissionsFlag adds OneDrive flag for restoring permissions
-func AddRestorePermissionsFlag(cmd *cobra.Command) {
+// AddNoPermissionsFlag adds OneDrive flag for skipping restoring permissions
+func AddNoPermissionsFlag(cmd *cobra.Command) {
 	fs := cmd.Flags()
-	fs.BoolVar(&RestorePermissionsFV, RestorePermissionsFN, false, "Restore permissions for files and folders")
+	fs.BoolVar(&NoPermissionsFV, NoPermissionsFN, false, "don't restore file and folder permissions")
 }
 
 // AddSkipReduceFlag adds a hidden flag that allows callers to skip the selector
@@ -72,6 +82,18 @@ func AddSkipReduceFlag(cmd *cobra.Command) {
 	cobra.CheckErr(fs.MarkHidden(SkipReduceFN))
 }
 
+// AddDeltaPageSizeFlag adds a hidden flag that allows callers to reduce delta
+// query page sizes below 500.
+func AddDeltaPageSizeFlag(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.IntVar(
+		&DeltaPageSizeFV,
+		DeltaPageSizeFN,
+		500,
+		"Control quantity of items returned in paged queries. Valid range is [1-500]. Default: 500")
+	cobra.CheckErr(fs.MarkHidden(DeltaPageSizeFN))
+}
+
 // AddFetchParallelismFlag adds a hidden flag that allows callers to reduce call
 // paralellism (ie, the corso worker pool size) from 4 to as low as 1.
 func AddFetchParallelismFlag(cmd *cobra.Command) {
@@ -96,6 +118,19 @@ func AddDisableIncrementalsFlag(cmd *cobra.Command) {
 	cobra.CheckErr(fs.MarkHidden(DisableIncrementalsFN))
 }
 
+// Adds the hidden '--force-item-data-download' cli flag which, when set,
+// disables kopia-assisted incremental backups.
+func AddForceItemDataDownloadFlag(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.BoolVar(
+		&ForceItemDataDownloadFV,
+		ForceItemDataDownloadFN,
+		false,
+		"Disable cached data checks in backups to force item redownloads for "+
+			"items changed since the last successful backup.")
+	cobra.CheckErr(fs.MarkHidden(ForceItemDataDownloadFN))
+}
+
 // Adds the hidden '--disable-delta' cli flag which, when set, disables
 // delta based backups.
 func AddDisableDeltaFlag(cmd *cobra.Command) {
@@ -120,19 +155,6 @@ func AddEnableImmutableIDFlag(cmd *cobra.Command) {
 	cobra.CheckErr(fs.MarkHidden(EnableImmutableIDFN))
 }
 
-// AddDisableConcurrencyLimiterFlag adds a hidden cli flag which, when set,
-// removes concurrency limits when communicating with graph API. This
-// flag is only relevant for exchange backups for now
-func AddDisableConcurrencyLimiterFlag(cmd *cobra.Command) {
-	fs := cmd.Flags()
-	fs.BoolVar(
-		&DisableConcurrencyLimiterFV,
-		DisableConcurrencyLimiterFN,
-		false,
-		"Disable concurrency limiter middleware. Default: false")
-	cobra.CheckErr(fs.MarkHidden(DisableConcurrencyLimiterFN))
-}
-
 // AddRunModeFlag adds the hidden --run-mode flag.
 func AddRunModeFlag(cmd *cobra.Command, persistent bool) {
 	fs := cmd.Flags()
@@ -143,3 +165,34 @@ func AddRunModeFlag(cmd *cobra.Command, persistent bool) {
 	fs.StringVar(&RunModeFV, RunModeFN, "run", "What mode to run: dry, test, run.  Defaults to run.")
 	cobra.CheckErr(fs.MarkHidden(RunModeFN))
 }
+
+// AddDisableSlidingWindowLimiterFN disables the experimental sliding window rate
+// limiter for graph API requests. This is only relevant for exchange backups.
+// Exchange restores continue to use the default token bucket rate limiter.
+// Setting this flag switches exchange backups to use the default token bucket
+// rate limiter.
+func AddDisableSlidingWindowLimiterFlag(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.BoolVar(
+		&DisableSlidingWindowLimiterFV,
+		DisableSlidingWindowLimiterFN,
+		false,
+		"Disable sliding window rate limiter.")
+	cobra.CheckErr(fs.MarkHidden(DisableSlidingWindowLimiterFN))
+}
+
+// AddDisableLazyItemReader disables lazy item reader, such that we fall back to
+// prefetch reader. This flag is currently only meant for groups conversations
+// backup. Although it can be utilized for other services in future.
+//
+// This flag should only be used if lazy item reader is the default choice and
+// we want to fallback to prefetch reader.
+func AddDisableLazyItemReader(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.BoolVar(
+		&DisableLazyItemReaderFV,
+		DisableLazyItemReaderFN,
+		false,
+		"Disable lazy item reader.")
+	cobra.CheckErr(fs.MarkHidden(DisableLazyItemReaderFN))
+}

src/cli/flags/repo.go

@@ -6,22 +6,38 @@ import (
 
 const (
 	BackupFN             = "backup"
+	BackupIDsFN          = "backups"
 	AWSAccessKeyFN       = "aws-access-key"
 	AWSSecretAccessKeyFN = "aws-secret-access-key"
 	AWSSessionTokenFN    = "aws-session-token"
 
 	// Corso Flags
-	CorsoPassphraseFN = "passphrase"
+	PassphraseFN    = "passphrase"
+	NewPassphraseFN = "new-passphrase"
 )
 
 var (
 	BackupIDFV           string
+	BackupIDsFV          []string
 	AWSAccessKeyFV       string
 	AWSSecretAccessKeyFV string
 	AWSSessionTokenFV    string
-	CorsoPassphraseFV    string
+	PassphraseFV         string
+	NewPhasephraseFV     string
 )
 
+// AddMultipleBackupIDsFlag adds the --backups flag.
+func AddMultipleBackupIDsFlag(cmd *cobra.Command, require bool) {
+	cmd.Flags().StringSliceVar(
+		&BackupIDsFV,
+		BackupIDsFN, nil,
+		"',' separated IDs of the backup to retrieve")
+
+	if require {
+		cobra.CheckErr(cmd.MarkFlagRequired(BackupIDsFN))
+	}
+}
+
 // AddBackupIDFlag adds the --backup flag.
 func AddBackupIDFlag(cmd *cobra.Command, require bool) {
 	cmd.Flags().StringVar(&BackupIDFV, BackupFN, "", "ID of the backup to retrieve.")
@@ -31,6 +47,16 @@ func AddBackupIDFlag(cmd *cobra.Command, require bool) {
 	}
 }
 
+// ---------------------------------------------------------------------------
+// storage
+// ---------------------------------------------------------------------------
+
+func AddAllStorageFlags(cmd *cobra.Command) {
+	AddCorsoPassphaseFlags(cmd)
+	// AddAzureCredsFlags is added by ProviderFlags
+	AddAWSCredsFlags(cmd)
+}
+
 func AddAWSCredsFlags(cmd *cobra.Command) {
 	fs := cmd.Flags()
 	fs.StringVar(&AWSAccessKeyFV, AWSAccessKeyFN, "", "S3 access key")
@@ -41,8 +67,31 @@ func AddAWSCredsFlags(cmd *cobra.Command) {
 // M365 flags
 func AddCorsoPassphaseFlags(cmd *cobra.Command) {
 	fs := cmd.Flags()
-	fs.StringVar(&CorsoPassphraseFV,
-		CorsoPassphraseFN,
+	fs.StringVar(
+		&PassphraseFV,
+		PassphraseFN,
 		"",
 		"Passphrase to protect encrypted repository contents")
 }
+
+// M365 flags
+func AddUpdatePassphraseFlags(cmd *cobra.Command, require bool) {
+	fs := cmd.Flags()
+	fs.StringVar(
+		&NewPhasephraseFV,
+		NewPassphraseFN,
+		"",
+		"update Corso passphrase for repo")
+
+	if require {
+		cobra.CheckErr(cmd.MarkFlagRequired(NewPassphraseFN))
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Provider
+// ---------------------------------------------------------------------------
+
+func AddAllProviderFlags(cmd *cobra.Command) {
+	AddAzureCredsFlags(cmd)
+}

src/cli/flags/restore_config.go

@@ -0,0 +1,37 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/pkg/control"
+)
+
+const (
+	CollisionsFN  = "collisions"
+	DestinationFN = "destination"
+	ToResourceFN  = "to-resource"
+)
+
+var (
+	CollisionsFV  string
+	DestinationFV string
+	ToResourceFV  string
+)
+
+// AddRestoreConfigFlags adds the restore config flag set.
+func AddRestoreConfigFlags(cmd *cobra.Command, canRestoreToAlternate bool) {
+	fs := cmd.Flags()
+	fs.StringVar(
+		&CollisionsFV, CollisionsFN, string(control.Skip),
+		//nolint:lll
+		"Sets the behavior for existing item collisions: "+string(control.Skip)+", "+string(control.Copy)+", or "+string(control.Replace))
+	fs.StringVar(
+		&DestinationFV, DestinationFN, "",
+		"Overrides the folder where items get restored; '/' places items into their original location")
+
+	if canRestoreToAlternate {
+		fs.StringVar(
+			&ToResourceFV, ToResourceFN, "",
+			"Overrides the protected resource (mailbox, site, user, etc) where data gets restored")
+	}
+}

src/cli/flags/retention.go

@@ -0,0 +1,50 @@
+package flags
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/pkg/control/repository"
+)
+
+const (
+	RetentionModeFN     = "retention-mode"
+	RetentionDurationFN = "retention-duration"
+	ExtendRetentionFN   = "extend-retention"
+)
+
+var (
+	RetentionModeFV     string
+	RetentionDurationFV time.Duration
+	ExtendRetentionFV   bool
+)
+
+// AddRetentionConfigFlags adds the retention config flag set.
+func AddRetentionConfigFlags(cmd *cobra.Command) {
+	fs := cmd.Flags()
+	fs.StringVar(
+		&RetentionModeFV,
+		RetentionModeFN,
+		repository.NoRetention.String(),
+		"Sets object locking mode (if any) to use in remote storage: "+
+			repository.NoRetention.String()+", "+
+			repository.GovernanceRetention.String()+", or "+
+			repository.ComplianceRetention.String())
+	cobra.CheckErr(fs.MarkHidden(RetentionModeFN))
+
+	fs.DurationVar(
+		&RetentionDurationFV,
+		RetentionDurationFN,
+		time.Duration(0),
+		"Set the amount of time to lock individual objects in remote storage")
+	cobra.CheckErr(fs.MarkHidden(RetentionDurationFN))
+
+	fs.BoolVar(
+		&ExtendRetentionFV,
+		ExtendRetentionFN,
+		false,
+		"Extends object locks during maintenance. "+
+			"Extends locks by the most recently set value of "+RetentionDurationFN)
+	cobra.CheckErr(fs.MarkHidden(ExtendRetentionFN))
+}

src/cli/flags/s3.go

@@ -0,0 +1,86 @@
+package flags
+
+import (
+	"strconv"
+
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/pkg/credentials"
+	"github.com/alcionai/corso/src/pkg/storage"
+)
+
+// S3 bucket flags
+const (
+	BucketFN         = "bucket"
+	EndpointFN       = "endpoint"
+	PrefixFN         = "prefix"
+	DoNotUseTLSFN    = "disable-tls"
+	DoNotVerifyTLSFN = "disable-tls-verification"
+)
+
+// S3 bucket flag values
+var (
+	BucketFV         string
+	EndpointFV       string
+	PrefixFV         string
+	DoNotUseTLSFV    bool
+	DoNotVerifyTLSFV bool
+)
+
+// S3 bucket flags
+func AddS3BucketFlags(cmd *cobra.Command) {
+	fs := cmd.Flags()
+
+	// Flags addition ordering should follow the order we want them to appear in help and docs:
+	// More generic and more frequently used flags take precedence.
+	fs.StringVar(&BucketFV, BucketFN, "", "Name of S3 bucket for repo. (required)")
+	fs.StringVar(&PrefixFV, PrefixFN, "", "Repo prefix within bucket.")
+	fs.StringVar(&EndpointFV, EndpointFN, "", "S3 service endpoint.")
+	fs.BoolVar(&DoNotUseTLSFV, DoNotUseTLSFN, false, "Disable TLS (HTTPS)")
+	fs.BoolVar(&DoNotVerifyTLSFV, DoNotVerifyTLSFN, false, "Disable TLS (HTTPS) certificate verification.")
+}
+
+func S3FlagOverrides(cmd *cobra.Command) map[string]string {
+	fs := GetPopulatedFlags(cmd)
+	return PopulateS3Flags(fs)
+}
+
+func PopulateS3Flags(flagset PopulatedFlags) map[string]string {
+	s3Overrides := map[string]string{
+		storage.StorageProviderTypeKey: storage.ProviderS3.String(),
+	}
+
+	if _, ok := flagset[AWSAccessKeyFN]; ok {
+		s3Overrides[credentials.AWSAccessKeyID] = AWSAccessKeyFV
+	}
+
+	if _, ok := flagset[AWSSecretAccessKeyFN]; ok {
+		s3Overrides[credentials.AWSSecretAccessKey] = AWSSecretAccessKeyFV
+	}
+
+	if _, ok := flagset[AWSSessionTokenFN]; ok {
+		s3Overrides[credentials.AWSSessionToken] = AWSSessionTokenFV
+	}
+
+	if _, ok := flagset[BucketFN]; ok {
+		s3Overrides[storage.Bucket] = BucketFV
+	}
+
+	if _, ok := flagset[PrefixFN]; ok {
+		s3Overrides[storage.Prefix] = PrefixFV
+	}
+
+	if _, ok := flagset[DoNotUseTLSFN]; ok {
+		s3Overrides[storage.DoNotUseTLS] = strconv.FormatBool(DoNotUseTLSFV)
+	}
+
+	if _, ok := flagset[DoNotVerifyTLSFN]; ok {
+		s3Overrides[storage.DoNotVerifyTLS] = strconv.FormatBool(DoNotVerifyTLSFV)
+	}
+
+	if _, ok := flagset[EndpointFN]; ok {
+		s3Overrides[storage.Endpoint] = EndpointFV
+	}
+
+	return s3Overrides
+}

src/cli/flags/sharepoint.go

@@ -5,23 +5,41 @@ import (
 )
 
 const (
-	LibraryFN    = "library"
-	ListFolderFN = "list"
-	ListItemFN   = "list-item"
+	DataLibraries = "libraries"
+	DataPages     = "pages"
+	DataLists     = "lists"
+)
+
+const (
+	LibraryFN = "library"
+
+	ListFN               = "list"
+	ListModifiedAfterFN  = "list-modified-after"
+	ListModifiedBeforeFN = "list-modified-before"
+	ListCreatedAfterFN   = "list-created-after"
+	ListCreatedBeforeFN  = "list-created-before"
+
 	PageFolderFN = "page-folder"
 	PageFN       = "page"
-	SiteFN       = "site"    // site only accepts WebURL values
-	SiteIDFN     = "site-id" // site-id accepts actual site ids
+
+	SiteFN   = "site"    // site only accepts WebURL values
+	SiteIDFN = "site-id" // site-id accepts actual site ids
 )
 
 var (
-	LibraryFV    string
-	ListFolderFV []string
-	ListItemFV   []string
+	LibraryFV string
+
+	ListFV               []string
+	ListModifiedAfterFV  string
+	ListModifiedBeforeFV string
+	ListCreatedAfterFV   string
+	ListCreatedBeforeFV  string
+
 	PageFolderFV []string
 	PageFV       []string
-	SiteIDFV     []string
-	WebURLFV     []string
+
+	SiteIDFV []string
+	WebURLFV []string
 )
 
 // AddSharePointDetailsAndRestoreFlags adds flags that are common to both the
@@ -61,17 +79,26 @@ func AddSharePointDetailsAndRestoreFlags(cmd *cobra.Command) {
 		"Select files modified before this datetime.")
 
 	// lists
-
 	fs.StringSliceVar(
-		&ListFolderFV,
-		ListFolderFN, nil,
-		"Select lists by name; accepts '"+Wildcard+"' to select all lists.")
-	cobra.CheckErr(fs.MarkHidden(ListFolderFN))
-	fs.StringSliceVar(
-		&ListItemFV,
-		ListItemFN, nil,
-		"Select lists by item name; accepts '"+Wildcard+"' to select all lists.")
-	cobra.CheckErr(fs.MarkHidden(ListItemFN))
+		&ListFV,
+		ListFN, nil,
+		"Select lists by name.")
+	fs.StringVar(
+		&ListModifiedAfterFV,
+		ListModifiedAfterFN, "",
+		"Select lists modified after this datetime.")
+	fs.StringVar(
+		&ListModifiedBeforeFV,
+		ListModifiedBeforeFN, "",
+		"Select lists modified before this datetime.")
+	fs.StringVar(
+		&ListCreatedAfterFV,
+		ListCreatedAfterFN, "",
+		"Select lists created after this datetime.")
+	fs.StringVar(
+		&ListCreatedBeforeFV,
+		ListCreatedBeforeFN, "",
+		"Select lists created before this datetime.")
 
 	// pages
 
@@ -90,24 +117,28 @@ func AddSharePointDetailsAndRestoreFlags(cmd *cobra.Command) {
 // AddSiteIDFlag adds the --site-id flag, which accepts site ID values.
 // This flag is hidden, since we expect users to prefer the --site url
 // and do not want to encourage confusion.
-func AddSiteIDFlag(cmd *cobra.Command) {
+func AddSiteIDFlag(cmd *cobra.Command, multiple bool) {
 	fs := cmd.Flags()
 
+	message := "ID of the site to operate on"
+	if multiple {
+		//nolint:lll
+		message += "; accepts '" + Wildcard + "' to select all sites.  Args cannot be comma-delimited and must use multiple flags."
+	}
+
 	// note string ARRAY var.  IDs naturally contain commas, so we cannot accept
 	// duplicate values within a flag declaration.  ie: --site-id a,b,c does not
 	// work.  Users must call --site-id a --site-id b --site-id c.
-	fs.StringArrayVar(
-		&SiteIDFV,
-		SiteIDFN, nil,
-		//nolint:lll
-		"Backup data by site ID; accepts '"+Wildcard+"' to select all sites.  Args cannot be comma-delimited and must use multiple flags.")
+	fs.StringArrayVar(&SiteIDFV, SiteIDFN, nil, message)
 	cobra.CheckErr(fs.MarkHidden(SiteIDFN))
 }
 
 // AddSiteFlag adds the --site flag, which accepts webURL values.
-func AddSiteFlag(cmd *cobra.Command) {
-	cmd.Flags().StringSliceVar(
-		&WebURLFV,
-		SiteFN, nil,
-		"Backup data by site URL; accepts '"+Wildcard+"' to select all sites.")
+func AddSiteFlag(cmd *cobra.Command, multiple bool) {
+	message := "Web URL of the site to operate on"
+	if multiple {
+		message += "; accepts '" + Wildcard + "' to select all sites."
+	}
+
+	cmd.Flags().StringSliceVar(&WebURLFV, SiteFN, nil, message)
 }

src/cli/flags/teams.go

@@ -0,0 +1,28 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+const (
+	TeamFN = "team"
+)
+
+var TeamFV []string
+
+func AddTeamDetailsAndRestoreFlags(cmd *cobra.Command) {
+	// TODO: implement flags
+}
+
+// AddTeamFlag adds the --team flag, which accepts id or name values.
+// TODO: need to decide what the appropriate "name" to accept here is.
+// keepers thinks its either DisplayName or MailNickname or Mail
+// Mail is most accurate, MailNickame is accurate and shorter, but the end user
+// may not see either one visibly.
+// https://learn.microsoft.com/en-us/graph/api/team-list?view=graph-rest-1.0&tabs=http
+func AddTeamFlag(cmd *cobra.Command) {
+	cmd.Flags().StringSliceVar(
+		&TeamFV,
+		TeamFN, nil,
+		"Backup data by team; accepts '"+Wildcard+"' to select all teams.")
+}

src/cli/flags/teamschats.go

@@ -0,0 +1,13 @@
+package flags
+
+import (
+	"github.com/spf13/cobra"
+)
+
+const (
+	DataChats = "chats"
+)
+
+func AddTeamsChatsDetailsAndRestoreFlags(cmd *cobra.Command) {
+	// TODO: add details flags
+}

src/cli/flags/testdata/backup_list.go

@@ -0,0 +1,26 @@
+package testdata
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"gotest.tools/v3/assert"
+
+	"github.com/alcionai/corso/src/cli/flags"
+)
+
+func PreparedBackupListFlags() []string {
+	return []string{
+		"--" + flags.AlertsFN, flags.Show,
+		"--" + flags.FailedItemsFN, flags.Show,
+		"--" + flags.SkippedItemsFN, flags.Show,
+		"--" + flags.RecoveredErrorsFN, flags.Show,
+	}
+}
+
+func AssertBackupListFlags(t *testing.T, cmd *cobra.Command) {
+	assert.Equal(t, flags.Show, flags.ListAlertsFV)
+	assert.Equal(t, flags.Show, flags.FailedItemsFV)
+	assert.Equal(t, flags.Show, flags.ListSkippedItemsFV)
+	assert.Equal(t, flags.Show, flags.ListRecoveredErrorsFV)
+}

src/cli/flags/testdata/flags.go

@@ -0,0 +1,130 @@
+package testdata
+
+import (
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+func FlgInputs(in []string) string { return strings.Join(in, ",") }
+
+var (
+	BackupInput = "backup-id"
+	SiteInput   = "site-id"
+
+	GroupsInput  = []string{"team1", "group2"}
+	MailboxInput = []string{"mailbox1", "mailbox2"}
+	UsersInput   = []string{"users1", "users2"}
+	SiteIDInput  = []string{"siteID1", "siteID2"}
+	WebURLInput  = []string{"webURL1", "webURL2"}
+
+	ExchangeCategoryDataInput   = []string{"email", "events", "contacts"}
+	SharepointCategoryDataInput = []string{"files", "lists", "pages"}
+	GroupsCategoryDataInput     = []string{"files", "lists", "pages", "messages"}
+	TeamsChatsCategoryDataInput = []string{"chats"}
+
+	ChannelInput                = []string{"channel1", "channel2"}
+	MessageInput                = []string{"message1", "message2"}
+	MessageCreatedAfterInput    = "messageCreatedAfter"
+	MessageCreatedBeforeInput   = "messageCreatedBefore"
+	MessageLastReplyAfterInput  = "messageLastReplyAfter"
+	MessageLastReplyBeforeInput = "messageLastReplyBefore"
+
+	ContactInput     = []string{"contact1", "contact2"}
+	ContactFldInput  = []string{"contactFld1", "contactFld2"}
+	ContactNameInput = "contactName"
+
+	ConversationInput = []string{"conversation1", "conversation2"}
+	PostInput         = []string{"post1", "post2"}
+
+	EmailInput               = []string{"mail1", "mail2"}
+	EmailFldInput            = []string{"mailFld1", "mailFld2"}
+	EmailReceivedAfterInput  = "mailReceivedAfter"
+	EmailReceivedBeforeInput = "mailReceivedBefore"
+	EmailSenderInput         = "mailSender"
+	EmailSubjectInput        = "mailSubject"
+
+	EventInput             = []string{"event1", "event2"}
+	EventCalInput          = []string{"eventCal1", "eventCal2"}
+	EventOrganizerInput    = "eventOrganizer"
+	EventRecursInput       = "eventRecurs"
+	EventStartsAfterInput  = "eventStartsAfter"
+	EventStartsBeforeInput = "eventStartsBefore"
+	EventSubjectInput      = "eventSubject"
+
+	LibraryInput            = "library"
+	FileNameInput           = []string{"fileName1", "fileName2"}
+	FolderPathInput         = []string{"folderPath1", "folderPath2"}
+	FileCreatedAfterInput   = "fileCreatedAfter"
+	FileCreatedBeforeInput  = "fileCreatedBefore"
+	FileModifiedAfterInput  = "fileModifiedAfter"
+	FileModifiedBeforeInput = "fileModifiedBefore"
+
+	ListsInput              = []string{"listName1", "listName2"}
+	ListCreatedAfterInput   = "listCreatedAfter"
+	ListCreatedBeforeInput  = "listCreatedBefore"
+	ListModifiedAfterInput  = "listModifiedAfter"
+	ListModifiedBeforeInput = "listModifiedBefore"
+
+	PageFolderInput = []string{"pageFolder1", "pageFolder2"}
+	PageInput       = []string{"page1", "page2"}
+
+	Collisions      = "collisions"
+	Destination     = "destination"
+	ToResource      = "toResource"
+	SkipPermissions = false
+
+	DeltaPageSize = "7"
+
+	Archive    = true
+	FormatType = "json"
+
+	AzureClientID     = "testAzureClientId"
+	AzureTenantID     = "testAzureTenantId"
+	AzureClientSecret = "testAzureClientSecret"
+
+	AWSAccessKeyID     = "testAWSAccessKeyID"
+	AWSSecretAccessKey = "testAWSSecretAccessKey"
+	AWSSessionToken    = "testAWSSessionToken"
+
+	CorsoPassphrase = "testCorsoPassphrase"
+
+	RestoreDestination = "test-restore-destination"
+
+	FetchParallelism = "3"
+
+	FailFast              = true
+	DisableIncrementals   = true
+	ForceItemDataDownload = true
+	DisableDelta          = true
+	EnableImmutableID     = true
+)
+
+func WithFlags2(
+	cc *cobra.Command,
+	command string,
+	flagSets ...[]string,
+) {
+	args := []string{command}
+
+	for _, sl := range flagSets {
+		args = append(args, sl...)
+	}
+
+	cc.SetArgs(args)
+}
+
+func WithFlags(
+	command string,
+	flagSets ...[]string,
+) func(*cobra.Command) {
+	return func(cc *cobra.Command) {
+		args := []string{command}
+
+		for _, sl := range flagSets {
+			args = append(args, sl...)
+		}
+
+		cc.SetArgs(args)
+	}
+}

src/cli/flags/testdata/groups.go

@@ -0,0 +1,42 @@
+package testdata
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/alcionai/corso/src/cli/flags"
+)
+
+func PreparedChannelFlags() []string {
+	return []string{
+		"--" + flags.ChannelFN, FlgInputs(ChannelInput),
+		"--" + flags.MessageFN, FlgInputs(MessageInput),
+		"--" + flags.MessageCreatedAfterFN, MessageCreatedAfterInput,
+		"--" + flags.MessageCreatedBeforeFN, MessageCreatedBeforeInput,
+		"--" + flags.MessageLastReplyAfterFN, MessageLastReplyAfterInput,
+		"--" + flags.MessageLastReplyBeforeFN, MessageLastReplyBeforeInput,
+	}
+}
+
+func AssertChannelFlags(t *testing.T, cmd *cobra.Command) {
+	assert.ElementsMatch(t, ChannelInput, flags.ChannelFV)
+	assert.ElementsMatch(t, MessageInput, flags.MessageFV)
+	assert.Equal(t, MessageCreatedAfterInput, flags.MessageCreatedAfterFV)
+	assert.Equal(t, MessageCreatedBeforeInput, flags.MessageCreatedBeforeFV)
+	assert.Equal(t, MessageLastReplyAfterInput, flags.MessageLastReplyAfterFV)
+	assert.Equal(t, MessageLastReplyBeforeInput, flags.MessageLastReplyBeforeFV)
+}
+
+func PreparedConversationFlags() []string {
+	return []string{
+		"--" + flags.ConversationFN, FlgInputs(ConversationInput),
+		"--" + flags.PostFN, FlgInputs(PostInput),
+	}
+}
+
+func AssertConversationFlags(t *testing.T, cmd *cobra.Command) {
+	assert.Equal(t, ConversationInput, flags.ConversationFV)
+	assert.Equal(t, PostInput, flags.PostFV)
+}

src/cli/flags/testdata/repo.go

@@ -0,0 +1,56 @@
+package testdata
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/alcionai/corso/src/cli/flags"
+)
+
+func PreparedStorageFlags() []string {
+	return []string{
+		"--" + flags.AWSAccessKeyFN, AWSAccessKeyID,
+		"--" + flags.AWSSecretAccessKeyFN, AWSSecretAccessKey,
+		"--" + flags.AWSSessionTokenFN, AWSSessionToken,
+
+		"--" + flags.PassphraseFN, CorsoPassphrase,
+	}
+}
+
+func AssertStorageFlags(t *testing.T, cmd *cobra.Command) {
+	assert.Equal(t, AWSAccessKeyID, flags.AWSAccessKeyFV)
+	assert.Equal(t, AWSSecretAccessKey, flags.AWSSecretAccessKeyFV)
+	assert.Equal(t, AWSSessionToken, flags.AWSSessionTokenFV)
+
+	assert.Equal(t, CorsoPassphrase, flags.PassphraseFV)
+}
+
+func PreparedProviderFlags() []string {
+	return []string{
+		"--" + flags.AzureClientIDFN, AzureClientID,
+		"--" + flags.AzureClientTenantFN, AzureTenantID,
+		"--" + flags.AzureClientSecretFN, AzureClientSecret,
+	}
+}
+
+func AssertProviderFlags(t *testing.T, cmd *cobra.Command) {
+	assert.Equal(t, AzureClientID, flags.AzureClientIDFV)
+	assert.Equal(t, AzureTenantID, flags.AzureClientTenantFV)
+	assert.Equal(t, AzureClientSecret, flags.AzureClientSecretFV)
+}
+
+func PreparedGenericBackupFlags() []string {
+	return []string{
+		"--" + flags.FailFastFN,
+		"--" + flags.DisableIncrementalsFN,
+		"--" + flags.ForceItemDataDownloadFN,
+	}
+}
+
+func AssertGenericBackupFlags(t *testing.T, cmd *cobra.Command) {
+	assert.True(t, flags.FailFastFV, "fail fast flag")
+	assert.True(t, flags.DisableIncrementalsFV, "disable incrementals flag")
+	assert.True(t, flags.ForceItemDataDownloadFV, "force item data download flag")
+}

src/cli/flags/testdata/sharepoint.go

@@ -0,0 +1,32 @@
+package testdata
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/alcionai/corso/src/cli/flags"
+)
+
+func PreparedLibraryFlags() []string {
+	return []string{
+		"--" + flags.LibraryFN, LibraryInput,
+		"--" + flags.FolderFN, FlgInputs(FolderPathInput),
+		"--" + flags.FileFN, FlgInputs(FileNameInput),
+		"--" + flags.FileCreatedAfterFN, FileCreatedAfterInput,
+		"--" + flags.FileCreatedBeforeFN, FileCreatedBeforeInput,
+		"--" + flags.FileModifiedAfterFN, FileModifiedAfterInput,
+		"--" + flags.FileModifiedBeforeFN, FileModifiedBeforeInput,
+	}
+}
+
+func AssertLibraryFlags(t *testing.T, cmd *cobra.Command) {
+	assert.Equal(t, LibraryInput, flags.LibraryFV)
+	assert.Equal(t, FolderPathInput, flags.FolderPathFV)
+	assert.Equal(t, FileNameInput, flags.FileNameFV)
+	assert.Equal(t, FileCreatedAfterInput, flags.FileCreatedAfterFV)
+	assert.Equal(t, FileCreatedBeforeInput, flags.FileCreatedBeforeFV)
+	assert.Equal(t, FileModifiedAfterInput, flags.FileModifiedAfterFV)
+	assert.Equal(t, FileModifiedBeforeInput, flags.FileModifiedBeforeFV)
+}

src/cli/flags/testdata/teamschats.go

@@ -0,0 +1,25 @@
+package testdata
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+)
+
+func PreparedTeamsChatsFlags() []string {
+	return []string{
+		// FIXME: populate when adding filters
+		// "--" + flags.ChatCreatedAfterFN, ChatCreatedAfterInput,
+		// "--" + flags.ChatCreatedBeforeFN, ChatCreatedBeforeInput,
+		// "--" + flags.ChatLastMessageAfterFN, ChatLastMessageAfterInput,
+		// "--" + flags.ChatLastMessageBeforeFN, ChatLastMessageBeforeInput,
+	}
+}
+
+func AssertTeamsChatsFlags(t *testing.T, cmd *cobra.Command) {
+	// FIXME: populate when adding filters
+	// assert.Equal(t, ChatCreatedAfterInput, flags.ChatCreatedAfterFV)
+	// assert.Equal(t, ChatCreatedBeforeInput, flags.ChatCreatedBeforeFV)
+	// assert.Equal(t, ChatLastMessageAfterInput, flags.ChatLastMessageAfterFV)
+	// assert.Equal(t, ChatLastMessageBeforeInput, flags.ChatLastMessageBeforeFV)
+}

src/cli/help/env.go

@@ -46,11 +46,12 @@ func (ev envVar) MinimumPrintable() any {
 	return ev
 }
 
-func (ev envVar) Headers() []string {
+func (ev envVar) Headers(bool) []string {
+	// NOTE: skipID does not make sense in this context
 	return []string{ev.category, " "}
 }
 
-func (ev envVar) Values() []string {
+func (ev envVar) Values(bool) []string {
 	return []string{ev.name, ev.description}
 }
 
@@ -93,8 +94,7 @@ func envGuide(cmd *cobra.Command, args []string) {
 	Info(ctx,
 		"\n--- Environment Variable Guide ---\n",
 		"As a best practice, Corso retrieves credentials and sensitive information from environment variables.\n ",
-		"\n",
-	)
+		"\n")
 	Table(ctx, toPrintable(corsoEVs))
 	Info(ctx, "\n")
 	Table(ctx, toPrintable(azureEVs))

src/cli/print/print.go

@@ -5,10 +5,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/tidwall/pretty"
 	"github.com/tomlazar/table"
+
+	"github.com/alcionai/corso/src/internal/common/color"
+	"github.com/alcionai/corso/src/internal/observe"
 )
 
 var (
@@ -81,67 +85,89 @@ func Only(ctx context.Context, e error) error {
 
 // Err prints the params to cobra's error writer (stdErr by default)
 // if s is nil, prints nothing.
-// Prepends the message with "Error: "
 func Err(ctx context.Context, s ...any) {
-	out(getRootCmd(ctx).ErrOrStderr(), s...)
+	cw := color.NewColorableWriter(color.Red, getRootCmd(ctx).ErrOrStderr())
+
+	s = append([]any{"Error:"}, s...)
+
+	out(ctx, cw, s...)
 }
 
 // Errf prints the params to cobra's error writer (stdErr by default)
 // if s is nil, prints nothing.
-// Prepends the message with "Error: "
+// You should ideally be using SimpleError or OperationError.
 func Errf(ctx context.Context, tmpl string, s ...any) {
-	outf(getRootCmd(ctx).ErrOrStderr(), "Error: "+tmpl, s...)
+	cw := color.NewColorableWriter(color.Red, getRootCmd(ctx).ErrOrStderr())
+	tmpl = "Error: " + tmpl
+	outf(ctx, cw, tmpl, s...)
 }
 
 // Out prints the params to cobra's output writer (stdOut by default)
 // if s is nil, prints nothing.
 func Out(ctx context.Context, s ...any) {
-	out(getRootCmd(ctx).OutOrStdout(), s...)
+	out(ctx, getRootCmd(ctx).OutOrStdout(), s...)
 }
 
 // Out prints the formatted strings to cobra's output writer (stdOut by default)
 // if t is empty, prints nothing.
 func Outf(ctx context.Context, t string, s ...any) {
-	outf(getRootCmd(ctx).OutOrStdout(), t, s...)
+	outf(ctx, getRootCmd(ctx).OutOrStdout(), t, s...)
 }
 
 // Info prints the params to cobra's error writer (stdErr by default)
 // if s is nil, prints nothing.
 func Info(ctx context.Context, s ...any) {
-	out(getRootCmd(ctx).ErrOrStderr(), s...)
+	out(ctx, getRootCmd(ctx).ErrOrStderr(), s...)
 }
 
 // Info prints the formatted strings to cobra's error writer (stdErr by default)
 // if t is empty, prints nothing.
 func Infof(ctx context.Context, t string, s ...any) {
-	outf(getRootCmd(ctx).ErrOrStderr(), t, s...)
+	outf(ctx, getRootCmd(ctx).ErrOrStderr(), t, s...)
+}
+
+// Pretty prettifies and prints the value.
+func Pretty(ctx context.Context, a any) {
+	if a == nil {
+		Err(ctx, "<nil>")
+		return
+	}
+
+	printPrettyJSON(ctx, getRootCmd(ctx).ErrOrStderr(), a)
 }
 
 // PrettyJSON prettifies and prints the value.
 func PrettyJSON(ctx context.Context, p minimumPrintabler) {
 	if p == nil {
 		Err(ctx, "<nil>")
+		return
 	}
 
-	outputJSON(getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug)
+	outputJSON(ctx, getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug)
 }
 
 // out is the testable core of exported print funcs
-func out(w io.Writer, s ...any) {
+func out(ctx context.Context, w io.Writer, s ...any) {
 	if len(s) == 0 {
 		return
 	}
 
+	// observe bars needs to be flushed before printing
+	observe.Flush(ctx)
+
 	fmt.Fprint(w, s...)
 	fmt.Fprintf(w, "\n")
 }
 
 // outf is the testable core of exported print funcs
-func outf(w io.Writer, t string, s ...any) {
+func outf(ctx context.Context, w io.Writer, t string, s ...any) {
 	if len(t) == 0 {
 		return
 	}
 
+	// observe bars needs to be flushed before printing
+	observe.Flush(ctx)
+
 	fmt.Fprintf(w, t, s...)
 	fmt.Fprintf(w, "\n")
 }
@@ -153,11 +179,11 @@ func outf(w io.Writer, t string, s ...any) {
 type Printable interface {
 	minimumPrintabler
 	// should list the property names of the values surfaced in Values()
-	Headers() []string
+	Headers(skipID bool) []string
 	// list of values for tabular or csv formatting
 	// if the backing data is nil or otherwise missing,
 	// values should provide an empty string as opposed to skipping entries
-	Values() []string
+	Values(skipID bool) []string
 }
 
 type minimumPrintabler interface {
@@ -167,39 +193,56 @@ type minimumPrintabler interface {
 
 // Item prints the printable, according to the caller's requested format.
 func Item(ctx context.Context, p Printable) {
-	printItem(getRootCmd(ctx).OutOrStdout(), p)
+	printItem(ctx, getRootCmd(ctx).OutOrStdout(), p)
 }
 
 // print prints the printable items,
 // according to the caller's requested format.
-func printItem(w io.Writer, p Printable) {
+func printItem(ctx context.Context, w io.Writer, p Printable) {
 	if outputAsJSON || outputAsJSONDebug {
-		outputJSON(w, p, outputAsJSONDebug)
+		outputJSON(ctx, w, p, outputAsJSONDebug)
 		return
 	}
 
-	outputTable(w, []Printable{p})
+	outputTable(ctx, w, []Printable{p})
+}
+
+// ItemProperties prints the printable either as in a single line or a json
+// The difference between this and Item is that this one does not print the ID
+func ItemProperties(ctx context.Context, p Printable) {
+	printItemProperties(ctx, getRootCmd(ctx).OutOrStdout(), p)
+}
+
+// print prints the printable items,
+// according to the caller's requested format.
+func printItemProperties(ctx context.Context, w io.Writer, p Printable) {
+	if outputAsJSON || outputAsJSONDebug {
+		outputJSON(ctx, w, p, outputAsJSONDebug)
+		return
+	}
+
+	outputOneLine(ctx, w, []Printable{p})
 }
 
 // All prints the slice of printable items,
 // according to the caller's requested format.
 func All(ctx context.Context, ps ...Printable) {
-	printAll(getRootCmd(ctx).OutOrStdout(), ps)
+	printAll(ctx, getRootCmd(ctx).OutOrStdout(), ps)
 }
 
 // printAll prints the slice of printable items,
 // according to the caller's requested format.
-func printAll(w io.Writer, ps []Printable) {
+func printAll(ctx context.Context, w io.Writer, ps []Printable) {
 	if len(ps) == 0 {
 		return
 	}
 
 	if outputAsJSON || outputAsJSONDebug {
-		outputJSONArr(w, ps, outputAsJSONDebug)
+		outputJSONArr(ctx, w, ps, outputAsJSONDebug)
 		return
 	}
 
-	outputTable(w, ps)
+	outputTable(ctx, w, ps)
 }
 
 // ------------------------------------------------------------------------------------------
@@ -209,20 +252,23 @@ func printAll(w io.Writer, ps []Printable) {
 // Table writes the printables in a tabular format.  Takes headers from
 // the 0th printable only.
 func Table(ctx context.Context, ps []Printable) {
-	outputTable(getRootCmd(ctx).OutOrStdout(), ps)
+	outputTable(ctx, getRootCmd(ctx).OutOrStdout(), ps)
 }
 
 // output to stdout the list of printable structs in a table
-func outputTable(w io.Writer, ps []Printable) {
+func outputTable(ctx context.Context, w io.Writer, ps []Printable) {
 	t := table.Table{
-		Headers: ps[0].Headers(),
+		Headers: ps[0].Headers(false),
 		Rows:    [][]string{},
 	}
 
 	for _, p := range ps {
-		t.Rows = append(t.Rows, p.Values())
+		t.Rows = append(t.Rows, p.Values(false))
 	}
 
+	// observe bars needs to be flushed before printing
+	observe.Flush(ctx)
+
 	_ = t.WriteTable(
 		w,
 		&table.Config{
@@ -236,20 +282,20 @@ func outputTable(w io.Writer, ps []Printable) {
 // JSON
 // ------------------------------------------------------------------------------------------
 
-func outputJSON(w io.Writer, p minimumPrintabler, debug bool) {
+func outputJSON(ctx context.Context, w io.Writer, p minimumPrintabler, debug bool) {
 	if debug {
-		printJSON(w, p)
+		printJSON(ctx, w, p)
 		return
 	}
 
 	if debug {
-		printJSON(w, p)
+		printJSON(ctx, w, p)
 	} else {
-		printJSON(w, p.MinimumPrintable())
+		printJSON(ctx, w, p.MinimumPrintable())
 	}
 }
 
-func outputJSONArr(w io.Writer, ps []Printable, debug bool) {
+func outputJSONArr(ctx context.Context, w io.Writer, ps []Printable, debug bool) {
 	sl := make([]any, 0, len(ps))
 
 	for _, p := range ps {
@@ -260,11 +306,14 @@ func outputJSONArr(w io.Writer, ps []Printable, debug bool) {
 		}
 	}
 
-	printJSON(w, sl)
+	printJSON(ctx, w, sl)
 }
 
 // output to stdout the list of printable structs as json.
-func printJSON(w io.Writer, a any) {
+func printJSON(ctx context.Context, w io.Writer, a any) {
+	// observe bars needs to be flushed before printing
+	observe.Flush(ctx)
+
 	bs, err := json.Marshal(a)
 	if err != nil {
 		fmt.Fprintf(w, "error formatting results to json: %v\n", err)
@@ -273,3 +322,45 @@ func printJSON(w io.Writer, a any) {
 
 	fmt.Fprintln(w, string(pretty.Pretty(bs)))
 }
+
+// output to stdout the list of printable structs as prettified json.
+func printPrettyJSON(ctx context.Context, w io.Writer, a any) {
+	// observe bars needs to be flushed before printing
+	observe.Flush(ctx)
+
+	bs, err := json.MarshalIndent(a, "", "  ")
+	if err != nil {
+		fmt.Fprintf(w, "error formatting results to json: %v\n", err)
+		return
+	}
+
+	fmt.Fprintln(w, string(pretty.Pretty(bs)))
+}
+
+// -------------------------------------------------------------------------------------------
+// One line
+// -------------------------------------------------------------------------------------------
+
+// Output in the following format:
+// Bytes Uploaded: 401 kB | Items Uploaded: 59 | Items Skipped: 0 | Errors: 0
+func outputOneLine(ctx context.Context, w io.Writer, ps []Printable) {
+	// observe bars needs to be flushed before printing
+	observe.Flush(ctx)
+
+	headers := ps[0].Headers(true)
+	rows := [][]string{}
+
+	for _, p := range ps {
+		rows = append(rows, p.Values(true))
+	}
+
+	printables := []string{}
+
+	for _, row := range rows {
+		for i, col := range row {
+			printables = append(printables, fmt.Sprintf("%s: %s", headers[i], col))
+		}
+	}
+
+	fmt.Fprintln(w, strings.Join(printables, " | "))
+}

src/cli/print/print_test.go

@@ -36,20 +36,28 @@ func (suite *PrintUnitSuite) TestOnly() {
 
 func (suite *PrintUnitSuite) TestOut() {
 	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
 	b := bytes.Buffer{}
 	msg := "I have seen the fnords!"
 
-	out(&b, msg)
+	out(ctx, &b, msg)
 	assert.Contains(t, b.String(), msg)
 }
 
 func (suite *PrintUnitSuite) TestOutf() {
 	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
 	b := bytes.Buffer{}
 	msg := "I have seen the fnords!"
 	msg2 := "smarf"
 
-	outf(&b, msg, msg2)
+	outf(ctx, &b, msg, msg2)
 	bs := b.String()
 	assert.Contains(t, bs, msg)
 	assert.Contains(t, bs, msg2)

src/cli/repo/filesystem.go

@@ -0,0 +1,217 @@
+package repo
+
+import (
+	"github.com/alcionai/clues"
+	"github.com/spf13/cobra"
+
+	"github.com/alcionai/corso/src/cli/flags"
+	. "github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/events"
+	"github.com/alcionai/corso/src/pkg/config"
+	ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
+	"github.com/alcionai/corso/src/pkg/repository"
+	"github.com/alcionai/corso/src/pkg/storage"
+)
+
+const (
+	fsProviderCommand      = "filesystem"
+	fsProviderCmdUseSuffix = "--path <path>"
+)
+
+const (
+	fsProviderCmdInitExamples = `# Create a new Corso repository on local or network attached storage
+corso repo init filesystem --path /tmp/corso-repo`
+
+	fsProviderCmdConnectExamples = `# Connect to a Corso repository on local or network attached storage
+corso repo connect filesystem --path /tmp/corso-repo`
+)
+
+func addFilesystemCommands(cmd *cobra.Command) *cobra.Command {
+	var c *cobra.Command
+
+	switch cmd.Use {
+	case initCommand:
+		init := filesystemInitCmd()
+		c, _ = utils.AddCommand(cmd, init)
+
+	case connectCommand:
+		c, _ = utils.AddCommand(cmd, filesystemConnectCmd())
+	}
+
+	c.Use = c.Use + " " + fsProviderCmdUseSuffix
+	c.SetUsageTemplate(cmd.UsageTemplate())
+
+	flags.AddFilesystemFlags(c)
+
+	return c
+}
+
+// `corso repo init filesystem [<flag>...]`
+func filesystemInitCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     fsProviderCommand,
+		Short:   "Initialize a repository on local or network storage.",
+		Long:    `Bootstraps a new repository on local or network storage and connects it to your m365 account.`,
+		RunE:    initFilesystemCmd,
+		Args:    cobra.NoArgs,
+		Example: fsProviderCmdInitExamples,
+	}
+}
+
+// initializes a filesystem repo.
+func initFilesystemCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+	overrides := flags.FilesystemFlagOverrides(cmd)
+
+	// TODO(pandeyabs): Move filepath conversion to FilesystemConfig scope.
+	abs, err := utils.MakeAbsoluteFilePath(overrides[flags.FilesystemPathFN])
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "getting absolute repo path"))
+	}
+
+	overrides[flags.FilesystemPathFN] = abs
+
+	cfg, err := config.ReadCorsoConfig(
+		ctx,
+		storage.ProviderFilesystem,
+		true,
+		false,
+		flags.FilesystemFlagOverrides(cmd))
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	opt := utils.ControlWithConfig(cfg)
+	// Retention is not supported for filesystem repos.
+	retentionOpts := ctrlRepo.Retention{}
+
+	storageCfg, err := cfg.Storage.ToFilesystemConfig()
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Retrieving filesystem configuration"))
+	}
+
+	m365, err := cfg.Account.M365Config()
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config"))
+	}
+
+	r, err := repository.New(
+		ctx,
+		cfg.Account,
+		cfg.Storage,
+		opt,
+		repository.NewRepoID)
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to construct the repository controller"))
+	}
+
+	ric := repository.InitConfig{RetentionOpts: retentionOpts}
+
+	if err = r.Initialize(ctx, ric); err != nil {
+		return Only(ctx, clues.Stack(ErrInitializingRepo, err))
+	}
+
+	defer utils.CloseRepo(ctx, r)
+
+	Infof(ctx, "Initialized a repository at path %s", storageCfg.Path)
+
+	err = config.WriteRepoConfig(
+		ctx,
+		storageCfg,
+		m365,
+		opt.Repo,
+		r.GetID())
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
+	}
+
+	return nil
+}
+
+// ---------------------------------------------------------------------------------------------------------
+// Connect
+// ---------------------------------------------------------------------------------------------------------
+
+// `corso repo connect filesystem [<flag>...]`
+func filesystemConnectCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     fsProviderCommand,
+		Short:   "Connect to a repository on local or network storage.",
+		Long:    `Ensures a connection to an existing repository on local or network storage.`,
+		RunE:    connectFilesystemCmd,
+		Args:    cobra.NoArgs,
+		Example: fsProviderCmdConnectExamples,
+	}
+}
+
+// connects to an existing filesystem repo.
+func connectFilesystemCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+	overrides := flags.FilesystemFlagOverrides(cmd)
+
+	// TODO(pandeyabs): Move filepath conversion to FilesystemConfig scope.
+	abs, err := utils.MakeAbsoluteFilePath(overrides[flags.FilesystemPathFN])
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "getting absolute repo path"))
+	}
+
+	overrides[flags.FilesystemPathFN] = abs
+
+	cfg, err := config.ReadCorsoConfig(
+		ctx,
+		storage.ProviderFilesystem,
+		true,
+		true,
+		overrides)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	repoID := cfg.RepoID
+	if len(repoID) == 0 {
+		repoID = events.RepoIDNotFound
+	}
+
+	storageCfg, err := cfg.Storage.ToFilesystemConfig()
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Retrieving filesystem configuration"))
+	}
+
+	m365, err := cfg.Account.M365Config()
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config"))
+	}
+
+	opts := utils.ControlWithConfig(cfg)
+
+	r, err := repository.New(
+		ctx,
+		cfg.Account,
+		cfg.Storage,
+		opts,
+		repoID)
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to create a repository controller"))
+	}
+
+	if err := r.Connect(ctx, repository.ConnConfig{}); err != nil {
+		return Only(ctx, clues.Stack(ErrConnectingRepo, err))
+	}
+
+	defer utils.CloseRepo(ctx, r)
+
+	Infof(ctx, "Connected to repository at path %s", storageCfg.Path)
+
+	err = config.WriteRepoConfig(
+		ctx,
+		storageCfg,
+		m365,
+		opts.Repo,
+		r.GetID())
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
+	}
+
+	return nil
+}

src/cli/repo/filesystem_e2e_test.go

@@ -0,0 +1,155 @@
+package repo_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/alcionai/clues"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/cli/flags"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/config"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/repository"
+	"github.com/alcionai/corso/src/pkg/storage"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
+)
+
+type FilesystemE2ESuite struct {
+	tester.Suite
+}
+
+func TestFilesystemE2ESuite(t *testing.T) {
+	suite.Run(t, &FilesystemE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *FilesystemE2ESuite) TestInitFilesystemCmd() {
+	table := []struct {
+		name          string
+		hasConfigFile bool
+	}{
+		{
+			name:          "NoConfigFile",
+			hasConfigFile: false,
+		},
+		{
+			name:          "hasConfigFile",
+			hasConfigFile: true,
+		},
+	}
+
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			ctx, flush := tester.NewContext(t)
+			defer flush()
+
+			st := storeTD.NewFilesystemStorage(t)
+
+			cfg, err := st.ToFilesystemConfig()
+			require.NoError(t, err, clues.ToCore(err))
+
+			force := map[string]string{
+				tconfig.TestCfgStorageProvider: storage.ProviderFilesystem.String(),
+			}
+
+			vpr, configFP := tconfig.MakeTempTestConfigClone(t, force)
+			if !test.hasConfigFile {
+				// Ideally we could use `/dev/null`, but you need a
+				// toml file plus this works cross platform
+				os.Remove(configFP)
+			}
+
+			ctx = config.SetViper(ctx, vpr)
+
+			cmd := cliTD.StubRootCmd(
+				"repo", "init", "filesystem",
+				"--"+flags.ConfigFileFN, configFP,
+				"--path", cfg.Path)
+			cli.BuildCommandTree(cmd)
+
+			// run the command
+			err = cmd.ExecuteContext(ctx)
+			require.NoError(t, err, clues.ToCore(err))
+
+			// noop
+			err = cmd.ExecuteContext(ctx)
+			require.NoError(t, err, clues.ToCore(err))
+		})
+	}
+}
+
+func (suite *FilesystemE2ESuite) TestConnectFilesystemCmd() {
+	table := []struct {
+		name          string
+		hasConfigFile bool
+	}{
+		{
+			name:          "NoConfigFile",
+			hasConfigFile: false,
+		},
+		{
+			name:          "HasConfigFile",
+			hasConfigFile: true,
+		},
+	}
+
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			ctx, flush := tester.NewContext(t)
+			defer flush()
+
+			st := storeTD.NewFilesystemStorage(t)
+			cfg, err := st.ToFilesystemConfig()
+			require.NoError(t, err, clues.ToCore(err))
+
+			force := map[string]string{
+				tconfig.TestCfgAccountProvider: account.ProviderM365.String(),
+				tconfig.TestCfgStorageProvider: storage.ProviderFilesystem.String(),
+				tconfig.TestCfgFilesystemPath:  cfg.Path,
+			}
+			vpr, configFP := tconfig.MakeTempTestConfigClone(t, force)
+			if !test.hasConfigFile {
+				// Ideally we could use `/dev/null`, but you need a
+				// toml file plus this works cross platform
+				os.Remove(configFP)
+			}
+
+			ctx = config.SetViper(ctx, vpr)
+
+			// init the repo first
+			r, err := repository.New(
+				ctx,
+				tconfig.NewM365Account(t),
+				st,
+				control.DefaultOptions(),
+				repository.NewRepoID)
+			require.NoError(t, err, clues.ToCore(err))
+
+			err = r.Initialize(ctx, repository.InitConfig{})
+			require.NoError(t, err, clues.ToCore(err))
+
+			// then test it
+			cmd := cliTD.StubRootCmd(
+				"repo", "connect", "filesystem",
+				"--"+flags.ConfigFileFN, configFP,
+				"--path", cfg.Path)
+			cli.BuildCommandTree(cmd)
+
+			// run the command
+			err = cmd.ExecuteContext(ctx)
+			require.NoError(t, err, clues.ToCore(err))
+		})
+	}
+}

src/cli/repo/filesystem_test.go

@@ -0,0 +1,53 @@
+package repo
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type FilesystemSuite struct {
+	tester.Suite
+}
+
+func TestFilesystemSuite(t *testing.T) {
+	suite.Run(t, &FilesystemSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *FilesystemSuite) TestAddFilesystemCommands() {
+	expectUse := fsProviderCommand + " " + fsProviderCmdUseSuffix
+
+	table := []struct {
+		name        string
+		use         string
+		expectUse   string
+		expectShort string
+		expectRunE  func(*cobra.Command, []string) error
+	}{
+		{"init filesystem", initCommand, expectUse, filesystemInitCmd().Short, initFilesystemCmd},
+		{"connect filesystem", connectCommand, expectUse, filesystemConnectCmd().Short, connectFilesystemCmd},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			cmd := &cobra.Command{Use: test.use}
+
+			c := addFilesystemCommands(cmd)
+			require.NotNil(t, c)
+
+			cmds := cmd.Commands()
+			require.Len(t, cmds, 1)
+
+			child := cmds[0]
+			assert.Equal(t, test.expectUse, child.Use)
+			assert.Equal(t, test.expectShort, child.Short)
+			tester.AreSameFunc(t, test.expectRunE, child.RunE)
+		})
+	}
+}

src/cli/repo/repo.go

@@ -8,19 +8,34 @@ import (
 	"golang.org/x/exp/maps"
 
 	"github.com/alcionai/corso/src/cli/flags"
-	"github.com/alcionai/corso/src/cli/print"
+	. "github.com/alcionai/corso/src/cli/print"
 	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/events"
 	"github.com/alcionai/corso/src/pkg/control/repository"
+	"github.com/alcionai/corso/src/pkg/path"
+	repo "github.com/alcionai/corso/src/pkg/repository"
 )
 
 const (
-	initCommand        = "init"
-	connectCommand     = "connect"
-	maintenanceCommand = "maintenance"
+	initCommand             = "init"
+	connectCommand          = "connect"
+	updatePassphraseCommand = "update-passphrase"
+	MaintenanceCommand      = "maintenance"
+)
+
+const (
+	providerCommandUpdatePhasephraseExamples = `# Update the Corso repository passphrase"
+corso repo update-passphrase --new-passphrase 'newpass'`
+)
+
+var (
+	ErrConnectingRepo   = clues.New("connecting repository")
+	ErrInitializingRepo = clues.New("initializing repository")
 )
 
 var repoCommands = []func(cmd *cobra.Command) *cobra.Command{
 	addS3Commands,
+	addFilesystemCommands,
 }
 
 // AddCommands attaches all `corso repo * *` commands to the parent.
@@ -28,23 +43,25 @@ func AddCommands(cmd *cobra.Command) {
 	var (
 		// Get new instances so that setting the context during tests works
 		// properly.
-		repoCmd        = repoCmd()
-		initCmd        = initCmd()
-		connectCmd     = connectCmd()
-		maintenanceCmd = maintenanceCmd()
+		repoCmd             = repoCmd()
+		initCmd             = initCmd()
+		connectCmd          = connectCmd()
+		maintenanceCmd      = maintenanceCmd()
+		updatePassphraseCmd = updatePassphraseCmd()
 	)
 
 	cmd.AddCommand(repoCmd)
 	repoCmd.AddCommand(initCmd)
 	repoCmd.AddCommand(connectCmd)
+	repoCmd.AddCommand(maintenanceCmd)
+	repoCmd.AddCommand(updatePassphraseCmd)
 
-	utils.AddCommand(
-		repoCmd,
-		maintenanceCmd,
-		utils.HideCommand(),
-		utils.MarkPreReleaseCommand())
 	flags.AddMaintenanceModeFlag(maintenanceCmd)
 	flags.AddForceMaintenanceFlag(maintenanceCmd)
+	flags.AddMaintenanceUserFlag(maintenanceCmd)
+	flags.AddMaintenanceHostnameFlag(maintenanceCmd)
+
+	flags.AddUpdatePassphraseFlags(updatePassphraseCmd, true)
 
 	for _, addRepoTo := range repoCommands {
 		addRepoTo(initCmd)
@@ -58,7 +75,7 @@ func repoCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "repo",
 		Short: "Manage your repositories",
-		Long:  `Initialize, configure, and connect to your account backup repositories.`,
+		Long:  `Initialize, configure, connect and update to your account backup repositories`,
 		RunE:  handleRepoCmd,
 		Args:  cobra.NoArgs,
 	}
@@ -106,7 +123,7 @@ func handleConnectCmd(cmd *cobra.Command, args []string) error {
 
 func maintenanceCmd() *cobra.Command {
 	return &cobra.Command{
-		Use:   maintenanceCommand,
+		Use:   MaintenanceCommand,
 		Short: "Run maintenance on an existing repository",
 		Long:  `Run maintenance on an existing repository to optimize performance and storage use`,
 		RunE:  handleMaintenanceCmd,
@@ -122,9 +139,14 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, S3Overrides())
+	r, _, err := utils.AccountConnectAndWriteRepoConfig(
+		ctx,
+		cmd,
+		// Need to give it a valid service so it won't error out on us even though
+		// we don't need the graph client.
+		path.OneDriveService)
 	if err != nil {
-		return print.Only(ctx, err)
+		return Only(ctx, err)
 	}
 
 	defer utils.CloseRepo(ctx, r)
@@ -137,12 +159,12 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error {
 			Force:  flags.ForceMaintenanceFV,
 		})
 	if err != nil {
-		return print.Only(ctx, err)
+		return Only(ctx, err)
 	}
 
 	err = m.Run(ctx)
 	if err != nil {
-		return print.Only(ctx, err)
+		return Only(ctx, err)
 	}
 
 	return nil
@@ -165,3 +187,55 @@ func getMaintenanceType(t string) (repository.MaintenanceType, error) {
 
 	return res, nil
 }
+
+// The repo update subcommand.
+// `corso repo update-passphrase [<flag>...]`
+func updatePassphraseCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     updatePassphraseCommand,
+		Short:   "Update the repository passphrase",
+		Long:    `Update the repository passphrase`,
+		RunE:    handleUpdateCmd,
+		Args:    cobra.NoArgs,
+		Example: providerCommandUpdatePhasephraseExamples,
+	}
+}
+
+// Handler for calls to `corso repo update-password`.
+func handleUpdateCmd(cmd *cobra.Command, args []string) error {
+	ctx := cmd.Context()
+
+	// Need to give it a valid service so it won't error out on us even though
+	// we don't need the graph client.
+	repos, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.OneDriveService)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	opts := rdao.Opts
+
+	defer utils.CloseRepo(ctx, repos)
+
+	repoID := repos.GetID()
+	if len(repoID) == 0 {
+		repoID = events.RepoIDNotFound
+	}
+
+	r, err := repo.New(
+		ctx,
+		rdao.Repo.Account,
+		rdao.Repo.Storage,
+		opts,
+		repoID)
+	if err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to create a repository controller"))
+	}
+
+	if err := r.UpdatePassword(ctx, flags.NewPhasephraseFV); err != nil {
+		return Only(ctx, clues.Wrap(err, "Failed to update s3"))
+	}
+
+	Infof(ctx, "Updated repo password.")
+
+	return nil
+}

src/cli/repo/repo_test.go

@@ -1,14 +1,23 @@
-package repo
+package repo_test
 
 import (
 	"testing"
 
+	"github.com/alcionai/clues"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
+	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/cli/flags"
+	"github.com/alcionai/corso/src/cli/repo"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
 	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
+	"github.com/alcionai/corso/src/pkg/config"
+	"github.com/alcionai/corso/src/pkg/storage"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
 )
 
 type RepoUnitSuite struct {
@@ -23,7 +32,7 @@ func (suite *RepoUnitSuite) TestAddRepoCommands() {
 	t := suite.T()
 	cmd := &cobra.Command{}
 
-	AddCommands(cmd)
+	repo.AddCommands(cmd)
 
 	var found bool
 
@@ -32,10 +41,95 @@ func (suite *RepoUnitSuite) TestAddRepoCommands() {
 	require.Len(t, repoCmds, 1)
 
 	for _, c := range repoCmds[0].Commands() {
-		if c.Use == maintenanceCommand {
+		if c.Use == repo.MaintenanceCommand {
 			found = true
 		}
 	}
 
 	assert.True(t, found, "looking for maintenance command")
 }
+
+type RepoE2ESuite struct {
+	tester.Suite
+}
+
+func TestRepoE2ESuite(t *testing.T) {
+	suite.Run(t, &RepoE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
+	t := suite.T()
+	ctx, flush := tester.NewContext(t)
+
+	defer flush()
+
+	st := storeTD.NewPrefixedS3Storage(t)
+	sc, err := st.StorageConfig()
+	require.NoError(t, err, clues.ToCore(err))
+
+	cfg := sc.(*storage.S3Config)
+
+	vpr, configFP := tconfig.MakeTempTestConfigClone(t, nil)
+
+	ctx = config.SetViper(ctx, vpr)
+
+	cmd := cliTD.StubRootCmd(
+		"repo", "init", "s3",
+		"--"+flags.ConfigFileFN, configFP,
+		"--prefix", cfg.Prefix)
+
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err = cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// connect with old passphrase
+	cmd = cliTD.StubRootCmd(
+		"repo", "connect", "s3",
+		"--"+flags.ConfigFileFN, configFP,
+		"--bucket", cfg.Bucket,
+		"--prefix", cfg.Prefix)
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err = cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	cmd = cliTD.StubRootCmd(
+		"repo", "update-passphrase",
+		"--"+flags.ConfigFileFN, configFP,
+		"--new-passphrase", "newpass")
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err = cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// connect again with new passphrase
+	cmd = cliTD.StubRootCmd(
+		"repo", "connect", "s3",
+		"--"+flags.ConfigFileFN, configFP,
+		"--bucket", cfg.Bucket,
+		"--prefix", cfg.Prefix,
+		"--passphrase", "newpass")
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err = cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// connect with old passphrase - it will fail
+	cmd = cliTD.StubRootCmd(
+		"repo", "connect", "s3",
+		"--"+flags.ConfigFileFN, configFP,
+		"--bucket", cfg.Bucket,
+		"--prefix", cfg.Prefix)
+	cli.BuildCommandTree(cmd)
+
+	// run the command
+	err = cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}

src/cli/repo/s3.go

@@ -1,70 +1,40 @@
 package repo
 
 import (
-	"os"
-	"strconv"
 	"strings"
 
 	"github.com/alcionai/clues"
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 
-	"github.com/alcionai/corso/src/cli/config"
 	"github.com/alcionai/corso/src/cli/flags"
 	. "github.com/alcionai/corso/src/cli/print"
 	"github.com/alcionai/corso/src/cli/utils"
-	"github.com/alcionai/corso/src/internal/common/str"
 	"github.com/alcionai/corso/src/internal/events"
-	"github.com/alcionai/corso/src/pkg/account"
-	"github.com/alcionai/corso/src/pkg/credentials"
+	"github.com/alcionai/corso/src/pkg/config"
 	"github.com/alcionai/corso/src/pkg/repository"
 	"github.com/alcionai/corso/src/pkg/storage"
 )
 
-// s3 bucket info from flags
-var (
-	succeedIfExists bool
-	bucket          string
-	endpoint        string
-	prefix          string
-	doNotUseTLS     bool
-	doNotVerifyTLS  bool
-)
-
 // called by repo.go to map subcommands to provider-specific handling.
 func addS3Commands(cmd *cobra.Command) *cobra.Command {
-	var (
-		c  *cobra.Command
-		fs *pflag.FlagSet
-	)
+	var c *cobra.Command
 
 	switch cmd.Use {
 	case initCommand:
-		c, fs = utils.AddCommand(cmd, s3InitCmd())
+		init := s3InitCmd()
+		flags.AddRetentionConfigFlags(init)
+		c, _ = utils.AddCommand(cmd, init)
+
 	case connectCommand:
-		c, fs = utils.AddCommand(cmd, s3ConnectCmd())
+		c, _ = utils.AddCommand(cmd, s3ConnectCmd())
 	}
 
 	c.Use = c.Use + " " + s3ProviderCommandUseSuffix
 	c.SetUsageTemplate(cmd.UsageTemplate())
 
-	flags.AddAWSCredsFlags(c)
-	flags.AddAzureCredsFlags(c)
 	flags.AddCorsoPassphaseFlags(c)
-
-	// Flags addition ordering should follow the order we want them to appear in help and docs:
-	// More generic and more frequently used flags take precedence.
-	fs.StringVar(&bucket, "bucket", "", "Name of S3 bucket for repo. (required)")
-	fs.StringVar(&prefix, "prefix", "", "Repo prefix within bucket.")
-	fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "S3 service endpoint.")
-	fs.BoolVar(&doNotUseTLS, "disable-tls", false, "Disable TLS (HTTPS)")
-	fs.BoolVar(&doNotVerifyTLS, "disable-tls-verification", false, "Disable TLS (HTTPS) certificate verification.")
-
-	// In general, we don't want to expose this flag to users and have them mistake it
-	// for a broad-scale idempotency solution.  We can un-hide it later the need arises.
-	fs.BoolVar(&succeedIfExists, "succeed-if-exists", false, "Exit with success if the repo has already been initialized.")
-	cobra.CheckErr(fs.MarkHidden("succeed-if-exists"))
+	flags.AddAWSCredsFlags(c)
+	flags.AddS3BucketFlags(c)
 
 	return c
 }
@@ -114,26 +84,24 @@ func s3InitCmd() *cobra.Command {
 func initS3Cmd(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 
-	// s3 values from flags
-	s3Override := S3Overrides()
-	// s3 values from envs
-	s3Override = S3UpdateFromEnvVar(s3Override)
-
-	cfg, err := config.GetConfigRepoDetails(ctx, true, false, s3Override)
+	cfg, err := config.ReadCorsoConfig(
+		ctx,
+		storage.ProviderS3,
+		true,
+		false,
+		flags.S3FlagOverrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}
 
-	// SendStartCorsoEvent uses distict ID as tenant ID because repoID is still not generated
-	utils.SendStartCorsoEvent(
-		ctx,
-		cfg.Storage,
-		cfg.Account.ID(),
-		map[string]any{"command": "init repo"},
-		cfg.Account.ID(),
-		utils.Control())
+	opt := utils.ControlWithConfig(cfg)
 
-	s3Cfg, err := cfg.Storage.S3Config()
+	retentionOpts, err := utils.MakeRetentionOpts(cmd)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	s3Cfg, err := cfg.Storage.ToS3Config()
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration"))
 	}
@@ -150,20 +118,27 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
 		return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config"))
 	}
 
-	r, err := repository.Initialize(ctx, cfg.Account, cfg.Storage, utils.Control())
+	r, err := repository.New(
+		ctx,
+		cfg.Account,
+		cfg.Storage,
+		opt,
+		repository.NewRepoID)
 	if err != nil {
-		if succeedIfExists && errors.Is(err, repository.ErrorRepoAlreadyExists) {
-			return nil
-		}
+		return Only(ctx, clues.Wrap(err, "Failed to construct the repository controller"))
+	}
 
-		return Only(ctx, clues.Wrap(err, "Failed to initialize a new S3 repository"))
+	ric := repository.InitConfig{RetentionOpts: retentionOpts}
+
+	if err = r.Initialize(ctx, ric); err != nil {
+		return Only(ctx, clues.Stack(ErrInitializingRepo, err))
 	}
 
 	defer utils.CloseRepo(ctx, r)
 
 	Infof(ctx, "Initialized a S3 repository within bucket %s.", s3Cfg.Bucket)
 
-	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil {
+	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opt.Repo, r.GetID()); err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
 	}
 
@@ -190,12 +165,12 @@ func s3ConnectCmd() *cobra.Command {
 func connectS3Cmd(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 
-	// s3 values from flags
-	s3Override := S3Overrides()
-	// s3 values from envs
-	s3Override = S3UpdateFromEnvVar(s3Override)
-
-	cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override)
+	cfg, err := config.ReadCorsoConfig(
+		ctx,
+		storage.ProviderS3,
+		true,
+		true,
+		flags.S3FlagOverrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}
@@ -205,7 +180,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 		repoID = events.RepoIDNotFound
 	}
 
-	s3Cfg, err := cfg.Storage.S3Config()
+	s3Cfg, err := cfg.Storage.ToS3Config()
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration"))
 	}
@@ -222,45 +197,29 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 		return Only(ctx, clues.New(invalidEndpointErr))
 	}
 
-	r, err := repository.ConnectAndSendConnectEvent(ctx, cfg.Account, cfg.Storage, repoID, utils.Control())
+	opts := utils.ControlWithConfig(cfg)
+
+	r, err := repository.New(
+		ctx,
+		cfg.Account,
+		cfg.Storage,
+		opts,
+		repoID)
 	if err != nil {
-		return Only(ctx, clues.Wrap(err, "Failed to connect to the S3 repository"))
+		return Only(ctx, clues.Wrap(err, "Failed to create a repository controller"))
+	}
+
+	if err := r.Connect(ctx, repository.ConnConfig{}); err != nil {
+		return Only(ctx, clues.Stack(ErrConnectingRepo, err))
 	}
 
 	defer utils.CloseRepo(ctx, r)
 
 	Infof(ctx, "Connected to S3 bucket %s.", s3Cfg.Bucket)
 
-	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil {
+	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opts.Repo, r.GetID()); err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
 	}
 
 	return nil
 }
-
-func S3Overrides() map[string]string {
-	return map[string]string{
-		config.AccountProviderTypeKey:  account.ProviderM365.String(),
-		config.StorageProviderTypeKey:  storage.ProviderS3.String(),
-		credentials.AWSAccessKeyID:     flags.AWSAccessKeyFV,
-		credentials.AWSSecretAccessKey: flags.AWSSecretAccessKeyFV,
-		credentials.AWSSessionToken:    flags.AWSSessionTokenFV,
-		storage.Bucket:                 bucket,
-		storage.Endpoint:               endpoint,
-		storage.Prefix:                 prefix,
-		storage.DoNotUseTLS:            strconv.FormatBool(doNotUseTLS),
-		storage.DoNotVerifyTLS:         strconv.FormatBool(doNotVerifyTLS),
-	}
-}
-
-func S3UpdateFromEnvVar(s3Flag map[string]string) map[string]string {
-	s3Flag[storage.Bucket] = str.First(s3Flag[storage.Bucket], os.Getenv(storage.BucketKey))
-	s3Flag[storage.Endpoint] = str.First(s3Flag[storage.Endpoint], os.Getenv(storage.EndpointKey))
-	s3Flag[storage.Prefix] = str.First(s3Flag[storage.Prefix], os.Getenv(storage.PrefixKey))
-	s3Flag[storage.DoNotUseTLS] = str.First(s3Flag[storage.DoNotUseTLS], os.Getenv(storage.DisableTLSKey))
-	s3Flag[storage.DoNotVerifyTLS] = str.First(
-		s3Flag[storage.DoNotVerifyTLS],
-		os.Getenv(storage.DisableTLSVerificationKey))
-
-	return s3Flag
-}

src/cli/repo/s3_e2e_test.go

@@ -8,13 +8,20 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	"golang.org/x/exp/maps"
 
 	"github.com/alcionai/corso/src/cli"
-	"github.com/alcionai/corso/src/cli/config"
+	"github.com/alcionai/corso/src/cli/flags"
+	cliTD "github.com/alcionai/corso/src/cli/testdata"
+	"github.com/alcionai/corso/src/internal/common/str"
 	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/internal/tester/tconfig"
 	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/config"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/repository"
+	"github.com/alcionai/corso/src/pkg/storage"
+	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
 )
 
 type S3E2ESuite struct {
@@ -24,8 +31,7 @@ type S3E2ESuite struct {
 func TestS3E2ESuite(t *testing.T) {
 	suite.Run(t, &S3E2ESuite{Suite: tester.NewE2ESuite(
 		t,
-		[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
-	)})
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
 }
 
 func (suite *S3E2ESuite) TestInitS3Cmd() {
@@ -58,11 +64,12 @@ func (suite *S3E2ESuite) TestInitS3Cmd() {
 			ctx, flush := tester.NewContext(t)
 			defer flush()
 
-			st := tester.NewPrefixedS3Storage(t)
-			cfg, err := st.S3Config()
+			st := storeTD.NewPrefixedS3Storage(t)
+
+			cfg, err := st.ToS3Config()
 			require.NoError(t, err, clues.ToCore(err))
 
-			vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
+			vpr, configFP := tconfig.MakeTempTestConfigClone(t, nil)
 			if !test.hasConfigFile {
 				// Ideally we could use `/dev/null`, but you need a
 				// toml file plus this works cross platform
@@ -71,9 +78,9 @@ func (suite *S3E2ESuite) TestInitS3Cmd() {
 
 			ctx = config.SetViper(ctx, vpr)
 
-			cmd := tester.StubRootCmd(
+			cmd := cliTD.StubRootCmd(
 				"repo", "init", "s3",
-				"--config-file", configFP,
+				"--"+flags.ConfigFileFN, configFP,
 				"--bucket", test.bucketPrefix+cfg.Bucket,
 				"--prefix", cfg.Prefix)
 			cli.BuildCommandTree(cmd)
@@ -82,9 +89,9 @@ func (suite *S3E2ESuite) TestInitS3Cmd() {
 			err = cmd.ExecuteContext(ctx)
 			require.NoError(t, err, clues.ToCore(err))
 
-			// a second initialization should result in an error
+			// noop
 			err = cmd.ExecuteContext(ctx)
-			assert.ErrorIs(t, err, repository.ErrorRepoAlreadyExists, clues.ToCore(err))
+			require.NoError(t, err, clues.ToCore(err))
 		})
 	}
 }
@@ -95,22 +102,21 @@ func (suite *S3E2ESuite) TestInitMultipleTimes() {
 
 	defer flush()
 
-	st := tester.NewPrefixedS3Storage(t)
-	cfg, err := st.S3Config()
+	st := storeTD.NewPrefixedS3Storage(t)
+
+	cfg, err := st.ToS3Config()
 	require.NoError(t, err, clues.ToCore(err))
 
-	vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
+	vpr, configFP := tconfig.MakeTempTestConfigClone(t, nil)
 
 	ctx = config.SetViper(ctx, vpr)
 
 	for i := 0; i < 2; i++ {
-		cmd := tester.StubRootCmd(
+		cmd := cliTD.StubRootCmd(
 			"repo", "init", "s3",
-			"--config-file", configFP,
+			"--"+flags.ConfigFileFN, configFP,
 			"--bucket", cfg.Bucket,
-			"--prefix", cfg.Prefix,
-			"--succeed-if-exists",
-		)
+			"--prefix", cfg.Prefix)
 		cli.BuildCommandTree(cmd)
 
 		// run the command
@@ -125,17 +131,22 @@ func (suite *S3E2ESuite) TestInitS3Cmd_missingBucket() {
 
 	defer flush()
 
-	st := tester.NewPrefixedS3Storage(t)
-	cfg, err := st.S3Config()
+	st := storeTD.NewPrefixedS3Storage(t)
+
+	cfg, err := st.ToS3Config()
 	require.NoError(t, err, clues.ToCore(err))
 
-	vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
+	force := map[string]string{
+		tconfig.TestCfgBucket: "",
+	}
+
+	vpr, configFP := tconfig.MakeTempTestConfigClone(t, force)
 
 	ctx = config.SetViper(ctx, vpr)
 
-	cmd := tester.StubRootCmd(
+	cmd := cliTD.StubRootCmd(
 		"repo", "init", "s3",
-		"--config-file", configFP,
+		"--"+flags.ConfigFileFN, configFP,
 		"--prefix", cfg.Prefix)
 	cli.BuildCommandTree(cmd)
 
@@ -174,16 +185,17 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
 			ctx, flush := tester.NewContext(t)
 			defer flush()
 
-			st := tester.NewPrefixedS3Storage(t)
-			cfg, err := st.S3Config()
+			st := storeTD.NewPrefixedS3Storage(t)
+
+			cfg, err := st.ToS3Config()
 			require.NoError(t, err, clues.ToCore(err))
 
 			force := map[string]string{
-				tester.TestCfgAccountProvider: "M365",
-				tester.TestCfgStorageProvider: "S3",
-				tester.TestCfgPrefix:          cfg.Prefix,
+				tconfig.TestCfgAccountProvider: account.ProviderM365.String(),
+				tconfig.TestCfgStorageProvider: storage.ProviderS3.String(),
+				tconfig.TestCfgPrefix:          cfg.Prefix,
 			}
-			vpr, configFP := tester.MakeTempTestConfigClone(t, force)
+			vpr, configFP := tconfig.MakeTempTestConfigClone(t, force)
 			if !test.hasConfigFile {
 				// Ideally we could use `/dev/null`, but you need a
 				// toml file plus this works cross platform
@@ -193,16 +205,23 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
 			ctx = config.SetViper(ctx, vpr)
 
 			// init the repo first
-			_, err = repository.Initialize(ctx, account.Account{}, st, control.Defaults())
+			r, err := repository.New(
+				ctx,
+				tconfig.NewM365Account(t),
+				st,
+				control.DefaultOptions(),
+				repository.NewRepoID)
+			require.NoError(t, err, clues.ToCore(err))
+
+			err = r.Initialize(ctx, repository.InitConfig{})
 			require.NoError(t, err, clues.ToCore(err))
 
 			// then test it
-			cmd := tester.StubRootCmd(
+			cmd := cliTD.StubRootCmd(
 				"repo", "connect", "s3",
-				"--config-file", configFP,
+				"--"+flags.ConfigFileFN, configFP,
 				"--bucket", test.bucketPrefix+cfg.Bucket,
-				"--prefix", cfg.Prefix,
-			)
+				"--prefix", cfg.Prefix)
 			cli.BuildCommandTree(cmd)
 
 			// run the command
@@ -212,54 +231,63 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
 	}
 }
 
-func (suite *S3E2ESuite) TestConnectS3Cmd_BadBucket() {
-	t := suite.T()
-	ctx, flush := tester.NewContext(t)
+func (suite *S3E2ESuite) TestConnectS3Cmd_badInputs() {
+	table := []struct {
+		name      string
+		bucket    string
+		prefix    string
+		expectErr func(t *testing.T, err error)
+	}{
+		{
+			name:   "bucket",
+			bucket: "wrong",
+			expectErr: func(t *testing.T, err error) {
+				assert.ErrorIs(t, err, storage.ErrVerifyingConfigStorage, clues.ToCore(err))
+			},
+		},
+		{
+			name:   "prefix",
+			prefix: "wrong",
+			expectErr: func(t *testing.T, err error) {
+				assert.ErrorIs(t, err, storage.ErrVerifyingConfigStorage, clues.ToCore(err))
+			},
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
 
-	defer flush()
+			ctx, flush := tester.NewContext(t)
+			defer flush()
 
-	st := tester.NewPrefixedS3Storage(t)
-	cfg, err := st.S3Config()
-	require.NoError(t, err, clues.ToCore(err))
+			st := storeTD.NewPrefixedS3Storage(t)
+			cfg, err := st.ToS3Config()
+			require.NoError(t, err, clues.ToCore(err))
 
-	vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
+			bucket := str.First(test.bucket, cfg.Bucket)
+			prefix := str.First(test.prefix, cfg.Prefix)
 
-	ctx = config.SetViper(ctx, vpr)
+			over := map[string]string{}
+			acct := tconfig.NewM365Account(t)
 
-	cmd := tester.StubRootCmd(
-		"repo", "connect", "s3",
-		"--config-file", configFP,
-		"--bucket", "wrong",
-		"--prefix", cfg.Prefix)
-	cli.BuildCommandTree(cmd)
+			maps.Copy(over, acct.Config)
+			over[account.AccountProviderTypeKey] = account.ProviderM365.String()
+			over[storage.StorageProviderTypeKey] = storage.ProviderS3.String()
 
-	// run the command
-	err = cmd.ExecuteContext(ctx)
-	require.Error(t, err, clues.ToCore(err))
-}
-
-func (suite *S3E2ESuite) TestConnectS3Cmd_BadPrefix() {
-	t := suite.T()
-	ctx, flush := tester.NewContext(t)
-
-	defer flush()
-
-	st := tester.NewPrefixedS3Storage(t)
-	cfg, err := st.S3Config()
-	require.NoError(t, err, clues.ToCore(err))
-
-	vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
-
-	ctx = config.SetViper(ctx, vpr)
-
-	cmd := tester.StubRootCmd(
-		"repo", "connect", "s3",
-		"--config-file", configFP,
-		"--bucket", cfg.Bucket,
-		"--prefix", "wrong")
-	cli.BuildCommandTree(cmd)
-
-	// run the command
-	err = cmd.ExecuteContext(ctx)
-	require.Error(t, err, clues.ToCore(err))
+			vpr, configFP := tconfig.MakeTempTestConfigClone(t, over)
+			ctx = config.SetViper(ctx, vpr)
+
+			cmd := cliTD.StubRootCmd(
+				"repo", "connect", "s3",
+				"--"+flags.ConfigFileFN, configFP,
+				"--bucket", bucket,
+				"--prefix", prefix)
+			cli.BuildCommandTree(cmd)
+
+			// run the command
+			err = cmd.ExecuteContext(ctx)
+			require.Error(t, err, clues.ToCore(err))
+			test.expectErr(t, err)
+		})
+	}
 }