Added integration test.

server/contexts/ctxerr/ctxerr.go

@@ -273,19 +273,19 @@ type StoredError struct {
 	Chain json.RawMessage `json:"chain"`
 }
 
-type handler interface {
+type Handler interface {
 	Store(error)
 	Retrieve(flush bool) ([]*StoredError, error)
 }
 
 // NewContext returns a context derived from ctx that contains the provided
 // error handler.
-func NewContext(ctx context.Context, eh handler) context.Context {
+func NewContext(ctx context.Context, eh Handler) context.Context {
 	return context.WithValue(ctx, errHandlerKey, eh)
 }
 
-func fromContext(ctx context.Context) handler {
-	v, _ := ctx.Value(errHandlerKey).(handler)
+func FromContext(ctx context.Context) Handler {
+	v, _ := ctx.Value(errHandlerKey).(Handler)
 	return v
 }
 
@@ -337,14 +337,14 @@ func Handle(ctx context.Context, err error) {
 		}
 	}
 
-	if eh := fromContext(ctx); eh != nil {
+	if eh := FromContext(ctx); eh != nil {
 		eh.Store(err)
 	}
 }
 
 // Retrieve retrieves an error from the registered error handler
 func Retrieve(ctx context.Context) ([]*StoredError, error) {
-	eh := fromContext(ctx)
+	eh := FromContext(ctx)
 	if eh == nil {
 		return nil, New(ctx, "missing handler in context")
 	}

server/errorstore/errors.go

@@ -270,6 +270,11 @@ func (h *Handler) Store(err error) {
 // ServeHTTP implements an http.Handler that retrieves the errors stored
 // by the Handler and returns them in the response as JSON.
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if h == nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+
 	var flush bool
 	opts := r.URL.Query()
 
@@ -294,5 +299,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
 	w.Write(bytes) //nolint:errcheck
 }

server/service/integration_desktop_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/ptr"
 	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -307,9 +308,13 @@ func (s *integrationTestSuite) TestErrorReporting() {
 	res = s.DoRawNoAuth("POST", "/api/latest/fleet/device/"+token+"/debug/errors", []byte("{}"), http.StatusOK)
 	res.Body.Close()
 
+	// Clear errors in error store
+	s.Do("GET", "/debug/errors", nil, http.StatusOK, "flush", "true")
+
 	testTime, err := time.Parse(time.RFC3339, "1969-06-19T21:44:05Z")
 	require.NoError(t, err)
 	ferr := fleet.FleetdError{
+		Vital:               true,
 		ErrorSource:         "orbit",
 		ErrorSourceVersion:  "1.1.1",
 		ErrorTimestamp:      testTime,
@@ -320,4 +325,75 @@ func (s *integrationTestSuite) TestErrorReporting() {
 	require.NoError(t, err)
 	res = s.DoRawNoAuth("POST", "/api/latest/fleet/device/"+token+"/debug/errors", errBytes, http.StatusOK)
 	res.Body.Close()
+
+	// Check that error was logged.
+	var errors []map[string]interface{}
+	s.DoJSON("GET", "/debug/errors", nil, http.StatusOK, &errors)
+	require.Len(t, errors, 1)
+	expectedCount := 1
+
+	checkError := func(errorItem map[string]interface{}, expectedCount int) {
+		assert.EqualValues(t, expectedCount, errorItem["count"])
+		errChain, ok := errorItem["chain"].([]interface{})
+		require.True(t, ok, fmt.Sprintf("%T", errorItem["chain"]))
+		require.Len(t, errChain, 2)
+		errChain0, ok := errChain[0].(map[string]interface{})
+		require.True(t, ok, fmt.Sprintf("%T", errChain[0]))
+		assert.EqualValues(t, "test message", errChain0["message"])
+		errChain1, ok := errChain[1].(map[string]interface{})
+		require.True(t, ok, fmt.Sprintf("%T", errChain[1]))
+
+		// Check that the exact fleetd error can be retrieved.
+		b, err := json.Marshal(errChain1["data"])
+		require.NoError(t, err)
+		var receivedErr fleet.FleetdError
+		require.NoError(t, json.Unmarshal(b, &receivedErr))
+		assert.EqualValues(t, ferr, receivedErr)
+	}
+	checkError(errors[0], expectedCount)
+
+	// Sending error again should increment the count.
+	res = s.DoRawNoAuth("POST", "/api/latest/fleet/device/"+token+"/debug/errors", errBytes, http.StatusOK)
+	res.Body.Close()
+	expectedCount++
+
+	// Changing the timestamp should only increment the count.
+	testTime2, err := time.Parse(time.RFC3339, "2024-10-30T09:44:05Z")
+	require.NoError(t, err)
+	ferr = fleet.FleetdError{
+		Vital:               true,
+		ErrorSource:         "orbit",
+		ErrorSourceVersion:  "1.1.1",
+		ErrorTimestamp:      testTime2,
+		ErrorMessage:        "test message",
+		ErrorAdditionalInfo: map[string]any{"foo": "bar"},
+	}
+	errBytes, err = json.Marshal(ferr)
+	require.NoError(t, err)
+	res = s.DoRawNoAuth("POST", "/api/latest/fleet/device/"+token+"/debug/errors", errBytes, http.StatusOK)
+	res.Body.Close()
+	expectedCount++
+
+	// Check that error was logged.
+	s.DoJSON("GET", "/debug/errors", nil, http.StatusOK, &errors)
+	require.Len(t, errors, 1)
+	checkError(errors[0], expectedCount)
+
+	// Changing additional info should create a new error
+	ferr = fleet.FleetdError{
+		Vital:               true,
+		ErrorSource:         "orbit",
+		ErrorSourceVersion:  "1.1.1",
+		ErrorTimestamp:      testTime,
+		ErrorMessage:        "test message",
+		ErrorAdditionalInfo: map[string]any{"foo": "bar2"},
+	}
+	errBytes, err = json.Marshal(ferr)
+	require.NoError(t, err)
+	res = s.DoRawNoAuth("POST", "/api/latest/fleet/device/"+token+"/debug/errors", errBytes, http.StatusOK)
+	res.Body.Close()
+
+	s.DoJSON("GET", "/debug/errors", nil, http.StatusOK, &errors)
+	require.Len(t, errors, 2)
+
 }

server/service/testing_client.go

@@ -21,6 +21,7 @@ import (
 	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
 	"github.com/fleetdm/fleet/v4/server/config"
 	"github.com/fleetdm/fleet/v4/server/datastore/mysql"
+	"github.com/fleetdm/fleet/v4/server/datastore/redis/redistest"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/live_query/live_query_mock"
 	"github.com/fleetdm/fleet/v4/server/pubsub"
@@ -86,6 +87,7 @@ func (ts *withServer) SetupSuite(dbName string) {
 		Rs:          rs,
 		Lq:          ts.lq,
 		FleetConfig: &cfg,
+		Pool:        redistest.SetupRedis(ts.s.T(), "integration_core", false, false, false),
 	}
 	if os.Getenv("FLEET_INTEGRATION_TESTS_DISABLE_LOG") != "" {
 		opts.Logger = kitlog.NewNopLogger()

server/service/testing_utils.go

@@ -13,13 +13,16 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/WatchBeam/clock"
 	eeservice "github.com/fleetdm/fleet/v4/ee/server/service"
 	"github.com/fleetdm/fleet/v4/server/config"
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	"github.com/fleetdm/fleet/v4/server/contexts/license"
 	"github.com/fleetdm/fleet/v4/server/datastore/cached_mysql"
 	"github.com/fleetdm/fleet/v4/server/datastore/filesystem"
+	"github.com/fleetdm/fleet/v4/server/errorstore"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/logging"
 	"github.com/fleetdm/fleet/v4/server/mail"
@@ -143,10 +146,17 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf
 
 	cronSchedulesService := fleet.NewCronSchedules()
 
-	if len(opts) > 0 && opts[0].StartCronSchedules != nil {
-		for _, fn := range opts[0].StartCronSchedules {
-			err = cronSchedulesService.StartCronSchedule(fn(ctx, ds))
-			require.NoError(t, err)
+	var eh *errorstore.Handler
+	if len(opts) > 0 {
+		if opts[0].Pool != nil {
+			eh = errorstore.NewHandler(ctx, opts[0].Pool, logger, time.Minute*10)
+			ctx = ctxerr.NewContext(ctx, eh)
+		}
+		if opts[0].StartCronSchedules != nil {
+			for _, fn := range opts[0].StartCronSchedules {
+				err = cronSchedulesService.StartCronSchedule(fn(ctx, ds))
+				require.NoError(t, err)
+			}
 		}
 	}
 
@@ -405,7 +415,8 @@ func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...*TestServ
 
 	apiHandler := MakeHandler(svc, cfg, logger, limitStore, WithLoginRateLimit(throttled.PerMin(1000)))
 	rootMux.Handle("/api/", apiHandler)
-	debugHandler := MakeDebugHandler(svc, cfg, logger, nil, ds)
+	errHandler := ctxerr.FromContext(ctx).(*errorstore.Handler)
+	debugHandler := MakeDebugHandler(svc, cfg, logger, errHandler, ds)
 	rootMux.Handle("/debug/", debugHandler)
 
 	server := httptest.NewUnstartedServer(rootMux)