-
Notifications
You must be signed in to change notification settings - Fork 7
/
errors.go
82 lines (64 loc) · 1.99 KB
/
errors.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
package lukai
import (
"context"
"fmt"
"math/rand"
"github.com/luk-ai/lukai/protobuf/aggregatorpb"
"github.com/pkg/errors"
)
// logError just logs an error without reporting it.
func (mt *ModelType) logError(id aggregatorpb.ModelID, typ aggregatorpb.ErrorType, err error) {
mt.errors.Lock()
defer mt.errors.Unlock()
if err == nil {
return
}
pberr := aggregatorpb.Error{
Id: id,
Type: typ,
Error: err.Error(),
Stacktrace: fmt.Sprintf("%+v", err),
}
if len(mt.errors.errors) < MaxQueuedErrors {
mt.errors.errors = append(mt.errors.errors, pberr)
} else {
// If we've hit the max queued errors, randomly replace one of the earlier
// errors.
mt.errors.errors[rand.Intn(len(mt.errors.errors))] = pberr
}
}
// reportError reports an error to the edge client.
func (mt *ModelType) reportError(ctx context.Context, edge aggregatorpb.EdgeClient, id aggregatorpb.ModelID, typ aggregatorpb.ErrorType, err error) error {
mt.logError(id, typ, err)
if !mt.errorLimiter.Allow() {
return nil
}
return mt.reportErrorsInternal(ctx, edge)
}
// reportErrorDial reports an error by dialing a new edge client.
func (mt *ModelType) reportErrorDial(ctx context.Context, id aggregatorpb.ModelID, typ aggregatorpb.ErrorType, err error) error {
mt.logError(id, typ, err)
if !mt.errorLimiter.Allow() {
return nil
}
conn, err := dial(ctx, EdgeAddress)
if err != nil {
return err
}
defer conn.Close()
edge := aggregatorpb.NewEdgeClient(conn)
return mt.reportErrorsInternal(ctx, edge)
}
// reportErrorsInternal actually sends the errors to the server. Should use
// reportError or reportErrorDial instead.
func (mt *ModelType) reportErrorsInternal(ctx context.Context, edge aggregatorpb.EdgeClient) error {
mt.errors.Lock()
defer mt.errors.Unlock()
if _, err := edge.ReportError(ctx, &aggregatorpb.ReportErrorRequest{
Errors: mt.errors.errors,
}); err != nil {
return errors.Wrapf(err, "failed to report error")
}
mt.errors.errors = mt.errors.errors[:0]
return nil
}