Merge pull request #667 from forta-network/caner/forta-625-mention-au…

…to-update-config-in-the

Improve auto-updates

Makefile

@@ -37,8 +37,8 @@ perf-test:
 
 MOCKREG = $$(pwd)/tests/e2e/misccontracts/contract_mock_registry
 
-.PHONY: e2e-test-recompile-mock
-e2e-test-recompile-mock:
+.PHONY: e2e-test-mock
+e2e-test-mock:
  solc --bin --abi -o $(MOCKREG) --include-path . --base-path $(MOCKREG) --overwrite --input-file $(MOCKREG)/MockRegistry.sol
  abigen --out $(MOCKREG)/mock_registry.go --pkg contract_mock_registry --type MockRegistry --abi $(MOCKREG)/MockRegistry.abi --bin $(MOCKREG)/MockRegistry.bin
 

cmd/updater/updater.go

@@ -96,7 +96,7 @@ func initServices(ctx context.Context, cfg config.Config) ([]services.Service, e
 
  updaterService := updater.NewUpdaterService(
  ctx, registryClient, releaseClient, config.DefaultContainerPort,
- developmentMode, cfg.AutoUpdate.TrackPrereleases, updateDelay, 0,
+ developmentMode, cfg.AutoUpdate.TrackPrereleases, updateDelay, cfg.AutoUpdate.CheckIntervalSeconds,
  )
 
  return []services.Service{

config/config.go

@@ -96,9 +96,10 @@ type TelemetryConfig struct {
 }
 
 type AutoUpdateConfig struct {
- Disable bool `yaml:"disable" json:"disable"`
- UpdateDelay *int `yaml:"updateDelay" json:"updateDelay"`
- TrackPrereleases bool `yaml:"trackPrereleases" json:"trackPrereleases"`
+ Disable bool `yaml:"disable" json:"disable"`
+ UpdateDelay *int `yaml:"updateDelay" json:"updateDelay"`
+ TrackPrereleases bool `yaml:"trackPrereleases" json:"trackPrereleases"`
+ CheckIntervalSeconds int `yaml:"checkIntervalSeconds" json:"checkIntervalSeconds" default:"60"` // 1m
 }
 
 type AgentLogsConfig struct {

go.mod

@@ -32,7 +32,7 @@ require (
 )
 
 require (
- github.com/forta-network/forta-core-go v0.0.0-20230301144359-910b79dcd336
+ github.com/forta-network/forta-core-go v0.0.0-20230302095116-b0f98655df8a
  github.com/libp2p/go-libp2p v0.23.2
  github.com/patrickmn/go-cache v2.1.0+incompatible
  github.com/rs/cors v1.7.0

go.sum

@@ -325,8 +325,8 @@ github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe
 github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
 github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
-github.com/forta-network/forta-core-go v0.0.0-20230301144359-910b79dcd336 h1:avtTwbM+M/lRSPd1h98rwc21yB1ESGyGLPvmRNV/E8M=
-github.com/forta-network/forta-core-go v0.0.0-20230301144359-910b79dcd336/go.mod h1:gffFqv24ErxEILzjvhXCaVHa2dljzdILlaJyUlkDnfw=
+github.com/forta-network/forta-core-go v0.0.0-20230302095116-b0f98655df8a h1:VISkcUsebS9E12aent55EsP0lbXIJKLb8TspL0GPlFg=
+github.com/forta-network/forta-core-go v0.0.0-20230302095116-b0f98655df8a/go.mod h1:gffFqv24ErxEILzjvhXCaVHa2dljzdILlaJyUlkDnfw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=

nodeutils/error_counter.go

@@ -0,0 +1,31 @@
+package nodeutils
+
+import "sync"
+
+// ErrorCounter checks incoming errors and tells if we are over
+// the max amount of consecutive errors.
+type ErrorCounter struct {
+ max uint
+ isCriticalError func(error) bool
+ count uint
+ mu sync.Mutex
+}
+
+// NewErrorCounter creates a new error counter.
+func NewErrorCounter(max uint, isCriticalError func(error) bool) *ErrorCounter {
+ return &ErrorCounter{
+ max: max,
+ isCriticalError: isCriticalError,
+ }
+}
+
+func (ec *ErrorCounter) TooManyErrs(err error) bool {
+ ec.mu.Lock()
+ defer ec.mu.Unlock()
+ if err == nil || !ec.isCriticalError(err) {
+ ec.count = 0 // reset if other errors or no errors
+ return false
+ }
+ ec.count++
+ return ec.count >= ec.max
+}

nodeutils/error_counter_test.go

@@ -0,0 +1,22 @@
+package nodeutils
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestErrorCounter(t *testing.T) {
+ r := require.New(t)
+
+ // max two consecutive non-nil errors means a red flag
+ counter := NewErrorCounter(2, func(err error) bool {
+ return err != nil
+ })
+
+ r.False(counter.TooManyErrs(errors.New("error 1")))
+ r.False(counter.TooManyErrs(nil)) // no error - back to zero
+ r.False(counter.TooManyErrs(errors.New("error 1")))
+ r.True(counter.TooManyErrs(errors.New("error 2"))) // hit the limit - we're done
+}

services/publisher/publisher.go

@@ -152,9 +152,10 @@ func (pub *Publisher) publishNextBatch(batch *protocol.AlertBatch) (published bo
  // add release info if it's available
  if pub.cfg.ReleaseSummary != nil {
  batch.ScannerVersion = &protocol.ScannerVersion{
- Commit: pub.cfg.ReleaseSummary.Commit,
- Ipfs: pub.cfg.ReleaseSummary.IPFS,
- Version: pub.cfg.ReleaseSummary.Version,
+ Commit: pub.cfg.ReleaseSummary.Commit,
+ Ipfs: pub.cfg.ReleaseSummary.IPFS,
+ Version: pub.cfg.ReleaseSummary.Version,
+ AutoUpdates: !pub.cfg.Config.AutoUpdate.Disable,
  }
  }
  lastBatchRef, err := pub.batchRefStore.Get()

services/scanner/agentpool/poolagent/agent.go

@@ -13,6 +13,7 @@ import (
 
  "github.com/ethereum/go-ethereum/common/hexutil"
  "github.com/forta-network/forta-node/metrics"
+ "github.com/forta-network/forta-node/nodeutils"
  "google.golang.org/grpc"
  "google.golang.org/grpc/codes"
  "google.golang.org/grpc/status"
@@ -47,7 +48,7 @@ type Agent struct {
  combinationRequests chan *CombinationRequest // never closed - deallocated when agent is discarded
  combinationResults chan<- *scanner.CombinationAlertResult
 
- errCounter *errorCounter
+ errCounter *nodeutils.ErrorCounter
  msgClient clients.MessageClient
 
  client clients.AgentClient
@@ -113,7 +114,7 @@ func New(ctx context.Context, agentCfg config.AgentConfig, msgClient clients.Mes
  blockResults: blockResults,
  combinationRequests: make(chan *CombinationRequest, DefaultBufferSize),
  combinationResults: alertResults,
- errCounter: NewErrorCounter(3, isCriticalErr),
+ errCounter: nodeutils.NewErrorCounter(3, isCriticalErr),
  msgClient: msgClient,
  ready: make(chan struct{}),
  closed: make(chan struct{}),

services/updater/updater.go

@@ -18,13 +18,15 @@ import (
  "github.com/forta-network/forta-core-go/utils"
 
  "github.com/forta-network/forta-node/config"
+ "github.com/forta-network/forta-node/nodeutils"
  log "github.com/sirupsen/logrus"
 )
 
 var (
  errNotAvailable = errors.New("new release not available")
 
  defaultUpdateCheckIntervalSeconds = 60
+ maxConsecutiveUpdateErrors = 60
 )
 
 // UpdaterService receives the release updates.
@@ -46,6 +48,8 @@ type UpdaterService struct {
  updateDelay time.Duration
  updateCheckInterval time.Duration
 
+ errCounter *nodeutils.ErrorCounter
+
  lastChecked health.TimeTracker
  lastErr health.ErrorTracker
  latestVersion health.MessageTracker
@@ -69,6 +73,9 @@ func NewUpdaterService(ctx context.Context, registryClient registry.Client, rele
  trackPrereleases: trackPrereleases,
  updateDelay: time.Duration(updateDelaySeconds) * time.Second,
  updateCheckInterval: time.Duration(updateCheckIntervalSeconds) * time.Second,
+ errCounter: nodeutils.NewErrorCounter(uint(maxConsecutiveUpdateErrors), func(err error) bool {
+ return err != nil // all non-nil errors are critical errors
+ }),
  }
 }
 
@@ -116,6 +123,9 @@ func (updater *UpdaterService) Start() error {
  return
  case <-t.C:
  err := updater.updateLatestReleaseWithDelay(updater.updateDelay)
+ if updater.errCounter.TooManyErrs(err) {
+ log.WithError(err).Panic("too many update errors - exiting")
+ }
  updater.lastErr.Set(err)
  updater.lastChecked.Set()
  if err != nil {

store/ens_override.go

@@ -5,18 +5,15 @@ import (
  "path"
 
  "github.com/ethereum/go-ethereum/common"
- "github.com/forta-network/forta-core-go/domain/registry"
- "github.com/forta-network/forta-core-go/ens"
  "github.com/forta-network/forta-node/config"
  "github.com/goccy/go-json"
 )
 
 type ensOverrideStore struct {
- contracts registry.RegistryContracts
  contractsMap map[string]string
 }
 
-func NewENSOverrideStore(cfg config.Config) (*ensOverrideStore, error) {
+func NewENSOverrideResolver(cfg config.Config) (*ensOverrideStore, error) {
  var store ensOverrideStore
  b, err := ioutil.ReadFile(path.Join(cfg.FortaDir, "ens-override.json"))
  if err != nil {
@@ -25,18 +22,9 @@ func NewENSOverrideStore(cfg config.Config) (*ensOverrideStore, error) {
  if err := json.Unmarshal(b, &store.contractsMap); err != nil {
  return nil, err
  }
- store.contracts.Dispatch = common.HexToAddress(store.contractsMap[ens.DispatchContract])
- store.contracts.AgentRegistry = common.HexToAddress(store.contractsMap[ens.AgentRegistryContract])
- store.contracts.ScannerRegistry = common.HexToAddress(store.contractsMap[ens.ScannerRegistryContract])
- store.contracts.ScannerNodeVersion = common.HexToAddress(store.contractsMap[ens.ScannerNodeVersionContract])
- store.contracts.FortaStaking = common.HexToAddress(store.contractsMap[ens.StakingContract])
  return &store, nil
 }
 
 func (store *ensOverrideStore) Resolve(input string) (common.Address, error) {
  return common.HexToAddress(store.contractsMap[input]), nil
 }
-
-func (store *ensOverrideStore) ResolveRegistryContracts() (*registry.RegistryContracts, error) {
- return &store.contracts, nil
-}

store/registry.go

@@ -12,6 +12,7 @@ import (
  "github.com/ipfs/go-cid"
  log "github.com/sirupsen/logrus"
 
+ "github.com/forta-network/forta-core-go/ens"
  "github.com/forta-network/forta-core-go/ethereum"
  "github.com/forta-network/forta-core-go/feeds"
  "github.com/forta-network/forta-core-go/manifest"
@@ -431,10 +432,11 @@ func NewPrivateRegistryStore(ctx context.Context, cfg config.Config) (*privateRe
 // GetRegistryClient checks the config and returns the suitaable registry.
 func GetRegistryClient(ctx context.Context, cfg config.Config, registryClientCfg registry.ClientConfig) (registry.Client, error) {
  if cfg.ENSConfig.Override {
- ensStore, err := NewENSOverrideStore(cfg)
+ ensResolver, err := NewENSOverrideResolver(cfg)
  if err != nil {
- return nil, fmt.Errorf("failed to create ens override store: %v", err)
+ return nil, fmt.Errorf("failed to create ens override resolver: %v", err)
  }
+ ensStore := ens.NewENStoreWithResolver(ensResolver)
  return registry.NewClientWithENSStore(ctx, registryClientCfg, ensStore)
  }
  return registry.NewClient(ctx, registryClientCfg)

tests/e2e/e2e_suite_test.go

@@ -219,11 +219,15 @@ func (s *Suite) SetupTest() {
 
  // point ENS to mock
  ensOverrides := map[string]string{
- ens.DispatchContract: mockRegistryAddr.Hex(),
- ens.AgentRegistryContract: mockRegistryAddr.Hex(),
- ens.ScannerRegistryContract: mockRegistryAddr.Hex(),
- ens.ScannerNodeVersionContract: mockRegistryAddr.Hex(),
- ens.StakingContract: mockRegistryAddr.Hex(),
+ ens.DispatchContract: mockRegistryAddr.Hex(),
+ ens.AgentRegistryContract: mockRegistryAddr.Hex(),
+ ens.ScannerRegistryContract: mockRegistryAddr.Hex(),
+ ens.ScannerPoolRegistryContract: mockRegistryAddr.Hex(),
+ ens.ScannerNodeVersionContract: mockRegistryAddr.Hex(),
+ ens.StakingContract: mockRegistryAddr.Hex(),
+ ens.RewardsContract: mockRegistryAddr.Hex(),
+ ens.StakeAllocatorContract: mockRegistryAddr.Hex(),
+ ens.MigrationContract: mockRegistryAddr.Hex(),
  }
  b, _ := json.MarshalIndent(ensOverrides, "", " ")
  s.r.NoError(ioutil.WriteFile(".forta/ens-override.json", b, 0644))