adds purely informational alerts to the fault bus (#4434)

Adds a new type of entry to the fault bus: Alerts. These values are for non-erroneous messages that we want 1/ clearly displayed to the user at the end of an operation and 2/ persisted with the backup for later review.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #4264

#### Test Plan

- [x] ⚡ Unit test

src/cli/backup/backup.go

@@ -317,6 +317,7 @@ func genericListCommand(
 		b.Print(ctx)
 		fe.PrintItems(
 			ctx,
+			!ifShow(flags.ListAlertsFV),
 			!ifShow(flags.ListFailedItemsFV),
 			!ifShow(flags.ListSkippedItemsFV),
 			!ifShow(flags.ListRecoveredErrorsFV))

src/cli/flags/backup_list.go

@@ -8,6 +8,7 @@ func AddAllBackupListFlags(cmd *cobra.Command) {
 	AddFailedItemsFN(cmd)
 	AddSkippedItemsFN(cmd)
 	AddRecoveredErrorsFN(cmd)
+	AddAlertsFN(cmd)
 }
 
 func AddFailedItemsFN(cmd *cobra.Command) {
@@ -27,3 +28,9 @@ func AddRecoveredErrorsFN(cmd *cobra.Command) {
 		&ListRecoveredErrorsFV, RecoveredErrorsFN, Show,
 		"Toggles showing or hiding the list of errors which Corso recovered from.")
 }
+
+func AddAlertsFN(cmd *cobra.Command) {
+	cmd.Flags().StringVar(
+		&ListAlertsFV, AlertsFN, Show,
+		"Toggles showing or hiding the list of alerts produced during the operation.")
+}

src/cli/flags/options.go

@@ -5,6 +5,7 @@ import (
 )
 
 const (
+	AlertsFN                    = "alerts"
 	DeltaPageSizeFN             = "delta-page-size"
 	DisableConcurrencyLimiterFN = "disable-concurrency-limiter"
 	DisableDeltaFN              = "disable-delta"
@@ -31,6 +32,7 @@ var (
 	EnableImmutableIDFV         bool
 	FailFastFV                  bool
 	FetchParallelismFV          int
+	ListAlertsFV                string
 	ListFailedItemsFV           string
 	ListSkippedItemsFV          string
 	ListRecoveredErrorsFV       string

src/cli/flags/testdata/backup_list.go

@@ -11,6 +11,7 @@ import (
 
 func PreparedBackupListFlags() []string {
 	return []string{
+		"--" + flags.AlertsFN, flags.Show,
 		"--" + flags.FailedItemsFN, flags.Show,
 		"--" + flags.SkippedItemsFN, flags.Show,
 		"--" + flags.RecoveredErrorsFN, flags.Show,
@@ -18,6 +19,7 @@ func PreparedBackupListFlags() []string {
 }
 
 func AssertBackupListFlags(t *testing.T, cmd *cobra.Command) {
+	assert.Equal(t, flags.Show, flags.ListAlertsFV)
 	assert.Equal(t, flags.Show, flags.ListFailedItemsFV)
 	assert.Equal(t, flags.Show, flags.ListSkippedItemsFV)
 	assert.Equal(t, flags.Show, flags.ListRecoveredErrorsFV)

src/internal/operations/helpers.go

@@ -48,9 +48,9 @@ func LogFaultErrors(ctx context.Context, fe *fault.Errors, prefix string) {
 	}
 
 	var (
-		log        = logger.Ctx(ctx)
-		pfxMsg     = prefix + ":"
-		li, ls, lr = len(fe.Items), len(fe.Skipped), len(fe.Recovered)
+		log            = logger.Ctx(ctx)
+		pfxMsg         = prefix + ":"
+		li, ls, lr, la = len(fe.Items), len(fe.Skipped), len(fe.Recovered), len(fe.Alerts)
 	)
 
 	if fe.Failure == nil && li+ls+lr == 0 {
@@ -73,4 +73,8 @@ func LogFaultErrors(ctx context.Context, fe *fault.Errors, prefix string) {
 	for i, err := range fe.Recovered {
 		log.With("recovered_error", err).Errorf("%s recoverable error %d of %d: %s", pfxMsg, i+1, lr, err.Msg)
 	}
+
+	for i, alert := range fe.Alerts {
+		log.With("alert", alert).Infof("%s alert %d of %d: %s", pfxMsg, i+1, la, alert.Message)
+	}
 }

src/pkg/fault/fault.go

@@ -36,6 +36,12 @@ type Bus struct {
 	// inability to process an item, due to a well-known cause.
 	skipped []Skipped
 
+	// alerts contain purely informational messages and data.  They
+	// represent situations where the end user should be aware of some
+	// occurrence that is not an error, exception, skipped data, or
+	// other runtime/persistence impacting issue.
+	alerts []Alert
+
 	// if failFast is true, the first errs addition will
 	// get promoted to the err value.  This signifies a
 	// non-recoverable processing state, causing any running
@@ -77,6 +83,11 @@ func (e *Bus) Skipped() []Skipped {
 	return slices.Clone(e.skipped)
 }
 
+// Alerts returns the slice of alerts generated during runtime.
+func (e *Bus) Alerts() []Alert {
+	return slices.Clone(e.alerts)
+}
+
 // Fail sets the non-recoverable error (ie: bus.failure)
 // in the bus.  If a failure error is already present,
 // the error gets added to the recoverable slice for
@@ -182,10 +193,10 @@ func (e *Bus) AddSkip(ctx context.Context, s *Skipped) {
 }
 
 // logs the error and adds a skipped item.
-func (e *Bus) logAndAddSkip(ctx context.Context, s *Skipped, skip int) {
-	logger.CtxStack(ctx, skip+1).
+func (e *Bus) logAndAddSkip(ctx context.Context, s *Skipped, trace int) {
+	logger.CtxStack(ctx, trace+1).
 		With("skipped", s).
-		Info("recoverable error")
+		Info("skipped item")
 	e.addSkip(s)
 }
 
@@ -194,6 +205,35 @@ func (e *Bus) addSkip(s *Skipped) *Bus {
 	return e
 }
 
+// AddAlert appends a record of an Alert message to the fault bus.
+// Importantly, alerts are not errors, exceptions, or skipped items.
+// An alert should only be generated if no other fault functionality
+// is in use, but that we still want the end user to clearly and
+// plainly receive a notification about a runtime event.
+func (e *Bus) AddAlert(ctx context.Context, a *Alert) {
+	if a == nil {
+		return
+	}
+
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.logAndAddAlert(ctx, a, 1)
+}
+
+// logs the error and adds an alert.
+func (e *Bus) logAndAddAlert(ctx context.Context, a *Alert, trace int) {
+	logger.CtxStack(ctx, trace+1).
+		With("alert", a).
+		Info("alert: " + a.Message)
+	e.addAlert(a)
+}
+
+func (e *Bus) addAlert(a *Alert) *Bus {
+	e.alerts = append(e.alerts, *a)
+	return e
+}
+
 // Errors returns the plain record of errors that were aggregated
 // within a fult Bus.
 func (e *Bus) Errors() *Errors {
@@ -204,6 +244,7 @@ func (e *Bus) Errors() *Errors {
 		Recovered: nonItems,
 		Items:     items,
 		Skipped:   slices.Clone(e.skipped),
+		Alerts:    slices.Clone(e.alerts),
 		FailFast:  e.failFast,
 	}
 }
@@ -265,6 +306,12 @@ type Errors struct {
 	// inability to process an item, due to a well-known cause.
 	Skipped []Skipped `json:"skipped"`
 
+	// Alerts contain purely informational messages and data.  They
+	// represent situations where the end user should be aware of some
+	// occurrence that is not an error, exception, skipped data, or
+	// other runtime/persistence impacting issue.
+	Alerts []Alert
+
 	// If FailFast is true, then the first Recoverable error will
 	// promote to the Failure spot, causing processing to exit.
 	FailFast bool `json:"failFast"`
@@ -315,14 +362,23 @@ func UnmarshalErrorsTo(e *Errors) func(io.ReadCloser) error {
 
 // Print writes the DetailModel Entries to StdOut, in the format
 // requested by the caller.
-func (e *Errors) PrintItems(ctx context.Context, ignoreErrors, ignoreSkips, ignoreRecovered bool) {
-	if len(e.Items)+len(e.Skipped)+len(e.Recovered) == 0 ||
-		ignoreErrors && ignoreSkips && ignoreRecovered {
+func (e *Errors) PrintItems(
+	ctx context.Context,
+	ignoreAlerts, ignoreErrors, ignoreSkips, ignoreRecovered bool,
+) {
+	if len(e.Alerts)+len(e.Items)+len(e.Skipped)+len(e.Recovered) == 0 ||
+		(ignoreAlerts && ignoreErrors && ignoreSkips && ignoreRecovered) {
 		return
 	}
 
 	sl := make([]print.Printable, 0)
 
+	if !ignoreAlerts {
+		for _, a := range e.Alerts {
+			sl = append(sl, print.Printable(a))
+		}
+	}
+
 	if !ignoreSkips {
 		for _, s := range e.Skipped {
 			sl = append(sl, print.Printable(s))

src/pkg/fault/item.go

@@ -264,3 +264,67 @@ func itemSkip(t itemType, cause skipCause, namespace, id, name string, addtl map
 		},
 	}
 }
+
+// ---------------------------------------------------------------------------
+// Alerts
+// ---------------------------------------------------------------------------
+
+var _ print.Printable = &Alert{}
+
+// Alerts are informational-only notifications.  The purpose of alerts is to
+// provide a means of end-user communication about important events without
+// needing to generate runtime failures or recoverable errors. When generating
+// an alert, no other fault feature (failure, recoverable, skip, etc) should
+// be in use.  IE: Errors do not also get alerts, since the error itself is a
+// form of end-user communication already.
+type Alert struct {
+	Item    Item   `json:"item"`
+	Message string `json:"message"`
+}
+
+// String complies with the stringer interface.
+func (a Alert) String() string {
+	msg := a.Message
+	if len(msg) == 0 {
+		msg = "<nil>"
+	}
+
+	return "Alert: " + msg
+}
+
+func (a Alert) MinimumPrintable() any {
+	return a
+}
+
+// Headers returns the human-readable names of properties of a skipped Item
+// for printing out to a terminal.
+func (a Alert) Headers() []string {
+	return []string{"Action", "Message", "Container", "Name", "ID"}
+}
+
+// Values populates the printable values matching the Headers list.
+func (a Alert) Values() []string {
+	var cn string
+
+	acn, ok := a.Item.Additional[AddtlContainerName]
+	if ok {
+		str, ok := acn.(string)
+		if ok {
+			cn = str
+		}
+	}
+
+	return []string{"Alert", a.Message, cn, a.Item.Name, a.Item.ID}
+}
+
+func NewAlert(message, namespace, itemID, name string, addtl map[string]any) *Alert {
+	return &Alert{
+		Message: message,
+		Item: Item{
+			Namespace:  namespace,
+			ID:         itemID,
+			Name:       name,
+			Additional: addtl,
+		},
+	}
+}

src/pkg/fault/item_test.go

@@ -256,3 +256,72 @@ func (suite *ItemUnitSuite) TestSkipped_HeadersValues() {
 		})
 	}
 }
+
+func (suite *ItemUnitSuite) TestAlert_String() {
+	var (
+		t = suite.T()
+		a Alert
+	)
+
+	assert.Contains(t, a.String(), "Alert: <nil>")
+
+	a = Alert{
+		Item:    Item{},
+		Message: "",
+	}
+	assert.Contains(t, a.String(), "Alert: <nil>")
+
+	a = Alert{
+		Item: Item{
+			ID: "item_id",
+		},
+		Message: "msg",
+	}
+	assert.NotContains(t, a.String(), "item_id")
+	assert.Contains(t, a.String(), "Alert: msg")
+}
+
+func (suite *ItemUnitSuite) TestNewAlert() {
+	t := suite.T()
+	addtl := map[string]any{"foo": "bar"}
+	a := NewAlert("message-to-show", "ns", "item_id", "item_name", addtl)
+
+	expect := Alert{
+		Item: Item{
+			Namespace:  "ns",
+			ID:         "item_id",
+			Name:       "item_name",
+			Additional: addtl,
+		},
+		Message: "message-to-show",
+	}
+
+	assert.Equal(t, expect, *a)
+}
+
+func (suite *ItemUnitSuite) TestAlert_HeadersValues() {
+	addtl := map[string]any{
+		AddtlContainerID:   "cid",
+		AddtlContainerName: "cname",
+	}
+
+	table := []struct {
+		name   string
+		alert  *Alert
+		expect []string
+	}{
+		{
+			name:   "new alert",
+			alert:  NewAlert("message-to-show", "ns", "id", "name", addtl),
+			expect: []string{"Alert", "message-to-show", "cname", "name", "id"},
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			assert.Equal(t, []string{"Action", "Message", "Container", "Name", "ID"}, test.alert.Headers())
+			assert.Equal(t, test.expect, test.alert.Values())
+		})
+	}
+}