Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v2] Blob Metadata Store #818

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 71 additions & 22 deletions core/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@ import (
"errors"
"fmt"
"math/big"
"strconv"

"github.com/Layr-Labs/eigenda/common"
"github.com/Layr-Labs/eigenda/encoding"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/consensys/gnark-crypto/ecc/bn254"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/accounts/abi"
"golang.org/x/crypto/sha3"
)

type AccountID = string
Expand Down Expand Up @@ -483,34 +486,80 @@ func (cb Bundles) FromEncodedBundles(eb EncodedBundles) (Bundles, error) {

// PaymentMetadata represents the header information for a blob
type PaymentMetadata struct {
// Existing fields
AccountID string
// AccountID is the ETH account address for the payer
AccountID string `json:"account_id"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is the json serialization used?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's used when we marshal/unmarshal the struct to/from json, i.e. in dataapi.
I've removed these for now


// New fields
BinIndex uint32
// BinIndex represents the range of time at which the dispersal is made
BinIndex uint32 `json:"bin_index"`
// TODO: we are thinking the contract can use uint128 for cumulative payment,
// but the definition on v2 uses uint64. Double check with team.
CumulativePayment *big.Int
CumulativePayment *big.Int `json:"cumulative_payment"`
}

// Hash returns the Keccak256 hash of the PaymentMetadata
func (pm *PaymentMetadata) Hash() []byte {
// Create a byte slice to hold the serialized data
data := make([]byte, 0, len(pm.AccountID)+4+pm.CumulativePayment.BitLen()/8+1)

// Append AccountID
data = append(data, []byte(pm.AccountID)...)

// Append BinIndex
binIndexBytes := make([]byte, 4)
binary.BigEndian.PutUint32(binIndexBytes, pm.BinIndex)
data = append(data, binIndexBytes...)

// Append CumulativePayment
paymentBytes := pm.CumulativePayment.Bytes()
data = append(data, paymentBytes...)
func (pm *PaymentMetadata) Hash() ([32]byte, error) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updated this hash function to make it compatible with solidity
cc @hopeyen

blobHeaderType, err := abi.NewType("tuple", "", []abi.ArgumentMarshaling{
{
Name: "accountID",
Type: "string",
},
{
Name: "binIndex",
Type: "uint32",
},
{
Name: "cumulativePayment",
Type: "uint256",
},
})
if err != nil {
return [32]byte{}, err
}

arguments := abi.Arguments{
{
Type: blobHeaderType,
},
}

bytes, err := arguments.Pack(pm)
if err != nil {
return [32]byte{}, err
}

var hash [32]byte
hasher := sha3.NewLegacyKeccak256()
hasher.Write(bytes)
copy(hash[:], hasher.Sum(nil)[:32])

return hash, nil
}

func (pm *PaymentMetadata) MarshalDynamoDBAttributeValue() (types.AttributeValue, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this belong in core? Is there any way to have this code in the blobstore?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They need to belong in the same package as where the type is defined. We can wrap and clone these types in blobstore but not sure if that's worth it

return &types.AttributeValueMemberM{
Value: map[string]types.AttributeValue{
"AccountID": &types.AttributeValueMemberS{Value: pm.AccountID},
"BinIndex": &types.AttributeValueMemberN{Value: fmt.Sprintf("%d", pm.BinIndex)},
"CumulativePayment": &types.AttributeValueMemberN{
Value: pm.CumulativePayment.String(),
},
},
}, nil
}

return crypto.Keccak256(data)
func (pm *PaymentMetadata) UnmarshalDynamoDBAttributeValue(av types.AttributeValue) error {
m, ok := av.(*types.AttributeValueMemberM)
if !ok {
return fmt.Errorf("expected *types.AttributeValueMemberM, got %T", av)
}
pm.AccountID = m.Value["AccountID"].(*types.AttributeValueMemberS).Value
binIndex, err := strconv.ParseUint(m.Value["BinIndex"].(*types.AttributeValueMemberN).Value, 10, 32)
if err != nil {
return fmt.Errorf("failed to parse BinIndex: %w", err)
}
pm.BinIndex = uint32(binIndex)
pm.CumulativePayment, _ = new(big.Int).SetString(m.Value["CumulativePayment"].(*types.AttributeValueMemberN).Value, 10)
return nil
}

// OperatorInfo contains information about an operator which is stored on the blockchain state,
Expand Down
6 changes: 3 additions & 3 deletions core/v2/assignment.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"github.com/Layr-Labs/eigenda/core"
)

func GetAssignments(state *core.OperatorState, blobVersion byte, quorum uint8) (map[core.OperatorID]Assignment, error) {
func GetAssignments(state *core.OperatorState, blobVersion BlobVersion, quorum uint8) (map[core.OperatorID]Assignment, error) {

params, ok := ParametersMap[blobVersion]
if !ok {
Expand Down Expand Up @@ -81,7 +81,7 @@ func GetAssignments(state *core.OperatorState, blobVersion byte, quorum uint8) (

}

func GetAssignment(state *core.OperatorState, blobVersion byte, quorum core.QuorumID, id core.OperatorID) (Assignment, error) {
func GetAssignment(state *core.OperatorState, blobVersion BlobVersion, quorum core.QuorumID, id core.OperatorID) (Assignment, error) {

assignments, err := GetAssignments(state, blobVersion, quorum)
if err != nil {
Expand All @@ -96,7 +96,7 @@ func GetAssignment(state *core.OperatorState, blobVersion byte, quorum core.Quor
return assignment, nil
}

func GetChunkLength(blobVersion byte, blobLength uint32) (uint32, error) {
func GetChunkLength(blobVersion BlobVersion, blobLength uint32) (uint32, error) {

if blobLength == 0 {
return 0, fmt.Errorf("blob length must be greater than 0")
Expand Down
10 changes: 5 additions & 5 deletions core/v2/assignment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func TestOperatorAssignmentsV2(t *testing.T) {
state := dat.GetTotalOperatorState(context.Background(), 0)
operatorState := state.OperatorState

blobVersion := byte(0)
blobVersion := corev2.BlobVersion(0)

assignments, err := corev2.GetAssignments(operatorState, blobVersion, 0)
assert.NoError(t, err)
Expand Down Expand Up @@ -90,7 +90,7 @@ func TestAssignmentWithTooManyOperators(t *testing.T) {

assert.Equal(t, len(state.Operators[0]), numOperators)

blobVersion := byte(0)
blobVersion := corev2.BlobVersion(0)

_, err = corev2.GetAssignments(state.OperatorState, blobVersion, 0)
assert.Error(t, err)
Expand Down Expand Up @@ -131,7 +131,7 @@ func FuzzOperatorAssignmentsV2(f *testing.F) {

state := dat.GetTotalOperatorState(context.Background(), 0)

blobVersion := byte(0)
blobVersion := corev2.BlobVersion(0)

assignments, err := corev2.GetAssignments(state.OperatorState, blobVersion, 0)
assert.NoError(t, err)
Expand Down Expand Up @@ -162,7 +162,7 @@ func FuzzOperatorAssignmentsV2(f *testing.F) {

func TestChunkLength(t *testing.T) {

blobVersion := byte(0)
blobVersion := corev2.BlobVersion(0)

pairs := []struct {
blobLength uint32
Expand All @@ -188,7 +188,7 @@ func TestChunkLength(t *testing.T) {

func TestInvalidChunkLength(t *testing.T) {

blobVersion := byte(0)
blobVersion := corev2.BlobVersion(0)

invalidLengths := []uint32{
0,
Expand Down
8 changes: 4 additions & 4 deletions core/v2/core_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func makeTestComponents() (encoding.Prover, encoding.Verifier, error) {
return p, v, nil
}

func makeTestBlob(t *testing.T, p encoding.Prover, version uint8, refBlockNumber uint64, length int, quorums []core.QuorumID) (corev2.BlobCertificate, []byte) {
func makeTestBlob(t *testing.T, p encoding.Prover, version corev2.BlobVersion, refBlockNumber uint64, length int, quorums []core.QuorumID) (corev2.BlobCertificate, []byte) {

data := make([]byte, length*31)
_, err := rand.Read(data)
Expand All @@ -101,7 +101,7 @@ func makeTestBlob(t *testing.T, p encoding.Prover, version uint8, refBlockNumber

header := corev2.BlobCertificate{
BlobHeader: corev2.BlobHeader{
Version: version,
BlobVersion: version,
QuorumNumbers: quorums,
BlobCommitments: commitments,
},
Expand Down Expand Up @@ -148,7 +148,7 @@ func prepareBlobs(t *testing.T, operatorCount uint, headers []corev2.BlobCertifi

for _, quorum := range header.QuorumNumbers {

assignments, err := corev2.GetAssignments(state, header.Version, quorum)
assignments, err := corev2.GetAssignments(state, header.BlobVersion, quorum)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -238,7 +238,7 @@ func TestValidationSucceeds(t *testing.T) {

bn := uint64(0)

version := uint8(0)
version := corev2.BlobVersion(0)

pool := workerpool.New(1)

Expand Down
Loading
Loading