Jobs are crashing, trying to diagnose.

monetr · Sep 20, 2021 · eefa414 · eefa414
1 parent 5f4b832
commit eefa414
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 9 deletions.
diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go
@@ -2,10 +2,11 @@ package jobs
 
 import (
 	"context"
-	"github.com/monetr/rest-api/pkg/internal/platypus"
 	"math"
 	"time"
 
+	"github.com/monetr/rest-api/pkg/internal/platypus"
+
 	"github.com/go-pg/pg/v10"
 	"github.com/gocraft/work"
 	"github.com/gomodule/redigo/redis"
@@ -104,8 +105,9 @@ func NewJobManager(
 	manager.work.PeriodicallyEnqueue("0 0 * * * *", EnqueueProcessFundingSchedules)
 
 	// Once a day. But also can be triggered by a webhook.
-	manager.work.PeriodicallyEnqueue("0 0 0 * * *", EnqueuePullAccountBalances)
-	manager.work.PeriodicallyEnqueue("0 0 0 * * *", EnqueuePullLatestTransactions)
+	// Once A day. 0 0 0 * * *
+	manager.work.PeriodicallyEnqueue("0 0 * * * *", EnqueuePullAccountBalances)
+	manager.work.PeriodicallyEnqueue("0 0 * * * *", EnqueuePullLatestTransactions)
 	//manager.work.PeriodicallyEnqueue("0 0 0 * * *", UpdateInstitutions)
 
 	manager.work.Start()

diff --git a/pkg/jobs/pull_account_balances.go b/pkg/jobs/pull_account_balances.go
@@ -3,11 +3,12 @@ package jobs
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"time"
+
 	"github.com/getsentry/sentry-go"
 	"github.com/monetr/rest-api/pkg/crumbs"
 	"github.com/sirupsen/logrus"
-	"strconv"
-	"time"
 
 	"github.com/gocraft/work"
 	"github.com/monetr/rest-api/pkg/models"
@@ -80,6 +81,12 @@ func (j *jobManagerBase) enqueuePullAccountBalances(job *work.Job) error {
 }
 
 func (j *jobManagerBase) pullAccountBalances(job *work.Job) (err error) {
+	defer func() {
+		if err := recover(); err != nil {
+			sentry.CaptureException(errors.Errorf("pull account balances failure: %+v", err))
+		}
+	}()
+
 	hub := sentry.CurrentHub().Clone()
 	ctx := sentry.SetHubOnContext(context.Background(), hub)
 	span := sentry.StartSpan(ctx, "Job", sentry.TransactionName("Pull Account Balances"))

diff --git a/pkg/jobs/pull_latest_transactions.go b/pkg/jobs/pull_latest_transactions.go
@@ -3,10 +3,11 @@ package jobs
 import (
 	"context"
 	"fmt"
-	"github.com/monetr/rest-api/pkg/crumbs"
 	"strconv"
 	"time"
 
+	"github.com/monetr/rest-api/pkg/crumbs"
+
 	"github.com/getsentry/sentry-go"
 	"github.com/gocraft/work"
 	"github.com/monetr/rest-api/pkg/internal/myownsanity"
@@ -87,6 +88,12 @@ func (j *jobManagerBase) enqueuePullLatestTransactions(job *work.Job) error {
 }
 
 func (j *jobManagerBase) pullLatestTransactions(job *work.Job) (err error) {
+	defer func() {
+		if err := recover(); err != nil {
+			sentry.CaptureException(errors.Errorf("pull latest transactions failure: %+v", err))
+		}
+	}()
+
 	log := j.getLogForJob(job)
 	log.Infof("pulling account balances")
 

diff --git a/pkg/jobs/transactions_removed.go b/pkg/jobs/transactions_removed.go
@@ -3,14 +3,15 @@ package jobs
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/getsentry/sentry-go"
 	"github.com/gocraft/work"
 	"github.com/monetr/rest-api/pkg/internal/myownsanity"
 	"github.com/monetr/rest-api/pkg/repository"
 	"github.com/pkg/errors"
-	"strconv"
-	"strings"
-	"time"
 )
 
 const (
@@ -31,6 +32,12 @@ func (j *jobManagerBase) TriggerRemoveTransactions(accountId, linkId uint64, rem
 }
 
 func (j *jobManagerBase) removeTransactions(job *work.Job) error {
+	defer func() {
+		if err := recover(); err != nil {
+			sentry.CaptureException(errors.Errorf("remove transactions failure: %+v", err))
+		}
+	}()
+
 	hub := sentry.CurrentHub().Clone()
 	ctx := sentry.SetHubOnContext(context.Background(), hub)
 	span := sentry.StartSpan(ctx, "Job", sentry.TransactionName("Remove Transactions"))