bump

go/protopace/Makefile

@@ -82,11 +82,25 @@ run/live:
 # OPERATIONS
 # ==================================================================================== #
 
-## releast: cross-compile to karapace build dir
+## release: cross-compile to build dir on mac
 .PHONY: release
 release:
- CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -ldflags='-s' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
- #upx -5 ${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so
-
- CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags='-s' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
- #upx -5 ${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so
+ CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
+ CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
+
+ docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/amd64 golang:1.22 env GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-amd64.so
+ cp ${BINARY_NAME}-linux-amd64.so ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.so
+ cp ${BINARY_NAME}-linux-amd64.h ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.h
+ rm ${BINARY_NAME}-linux-amd64.so
+ rm ${BINARY_NAME}-linux-amd64.h
+
+ docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/arm64 golang:1.22 env GOOS=linux GOARCH=arm64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-arm64.so
+ cp ${BINARY_NAME}-linux-arm64.so ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.so
+ cp ${BINARY_NAME}-linux-arm64.h ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.h
+ rm ${BINARY_NAME}-linux-arm64.so
+ rm ${BINARY_NAME}-linux-arm64.h
+
+.PHONY: foo
+foo:
+ dir=$(realpath ${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so)
+ echo ${dir}

go/protopace/compatibility.go

@@ -30,6 +30,7 @@ func Check(schema schema.Schema, previousSchema schema.Schema) error {
  "FILE_SAME_PACKAGE",
  "FIELD_SAME_NAME",
  "FIELD_SAME_JSON_NAME",
+ "FILE_NO_DELETE",
  },
  nil,
  nil,

go/protopace/compatibility_test.go

@@ -30,5 +30,5 @@ func TestCompatibility(t *testing.T) {
  assert.NoError(err)
 
  err = Check(*testSchema, *previousSchema)
- assert.ErrorContains(err, "Previously present field \"5\" with name \"local_nested_value\" on message \"EventValue\" was deleted.")
+ assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".")
 }

go/protopace/fixtures/test.proto

@@ -26,4 +26,5 @@ message EventValue {
  google.protobuf.Timestamp created_at = 2;
  Status status = 3;
  Local.NestedValue local_nested_value = 4;
+ int32 foo = 5;
 }

go/protopace/fixtures/test_previous.proto

@@ -25,5 +25,6 @@ message EventValue {
  NestedValue nested_value = 1;
  google.protobuf.Timestamp created_at = 2;
  Status status = 3;
- Local.NestedValue local_nested_value = 5;
+ Local.NestedValue local_nested_value = 4;
+ string foo = 5;
 }

go/protopace/formatter.go

@@ -19,10 +19,11 @@ import (
 )
 
 func Format(schema s.Schema) (s.Schema, error) {
- res, err := schema.Compile()
+ all, err := schema.Compile()
  if err != nil {
  return schema, err
  }
+ res := all[0]
 
  astNodeMapping := map[ast.Node]protoreflect.FullName{}
 

go/protopace/schema/schema.go

@@ -34,15 +34,22 @@ func FromString(name string, proto string, dependencies []Schema) (*Schema, erro
  return &Schema{Schema: proto, Name: name, ParserResult: result, Dependencies: dependencies}, nil
 }
 
-func (s Schema) Compile() (linker.Result, error) {
+func (s Schema) Compile() ([]linker.Result, error) {
  resolver := NewSchemaResolver(append(s.Dependencies, s))
  compiler := NewCompiler(resolver)
  ctx := context.Background()
- files, err := compiler.Compile(ctx, s.Name)
+ schemas := []string{s.Name}
+ for _, dep := range s.Dependencies {
+ schemas = append(schemas, dep.Name)
+ }
+ files, err := compiler.Compile(ctx, schemas...)
  if err != nil {
  return nil, err
  }
- res := files[0].(linker.Result)
+ res := make([]linker.Result, len(files))
+ for i, f := range files {
+ res[i] = f.(linker.Result)
+ }
  return res, nil
 }
 
@@ -51,19 +58,24 @@ func (s Schema) CompileBufImage() (bufimage.Image, error) {
  if err != nil {
  return nil, err
  }
- imageFile, err := bufimage.NewImageFile(
- res.FileDescriptorProto(),
- nil,
- uuid.Nil,
- "",
- "",
- false,
- false,
- nil,
- )
- if err != nil {
- return nil, err
+ files := make([]bufimage.ImageFile, len(res))
+ for i, r := range res {
+ file, err := bufimage.NewImageFile(
+ r.FileDescriptorProto(),
+ nil,
+ uuid.Nil,
+ "",
+ "",
+ false,
+ false,
+ nil,
+ )
+ if err != nil {
+ return nil, err
+ }
+ files[i] = file
  }
- image, err := bufimage.NewImage([]bufimage.ImageFile{imageFile})
+
+ image, err := bufimage.NewImage(files)
  return image, err
 }

karapace/compatibility/protobuf/checks.py

@@ -11,26 +11,31 @@
 def check_protobuf_schema_compatibility(
  reader: ProtobufSchema, writer: ProtobufSchema, use_protopace: bool = False
 ) -> SchemaCompatibilityResult:
- result = CompareResult()
-
  if use_protopace:
  old_deps = []
- for _, val in reader.dependencies.items():
- old_deps.append(Proto(val.name, val.get_schema().schema_str))
- old_proto = Proto(reader.record_name, reader.to_schema(), old_deps)
+ # TODO: get dependencies recursive
+ if reader.dependencies:
+ for _, val in reader.dependencies.items():
+ old_deps.append(Proto(val.name, val.get_schema().schema_str))
+ old_proto = Proto("test.proto", reader.to_schema(), old_deps)
 
  new_deps = []
- for _, val in writer.dependencies.items():
- new_deps.append(Proto(val.name, val.get_schema().schema_str))
- new_proto = Proto(writer.record_name, writer.to_schema(), old_deps)
+ if writer.dependencies:
+ for _, val in writer.dependencies.items():
+ new_deps.append(Proto(val.name, val.get_schema().schema_str))
+ new_proto = Proto("test.proto", writer.to_schema(), new_deps)
 
  try:
  check_compatibility(new_proto, old_proto)
- except IncompatibleError:
- result.add_modification(Modification.PROTOPACE)
+ except IncompatibleError as err:
+ return SchemaCompatibilityResult(
+ compatibility=SchemaCompatibilityType.incompatible,
+ messages=set([str(err)]),
+ )
 
- return result
+ return SchemaCompatibilityResult(SchemaCompatibilityType.compatible)
 
+ result = CompareResult()
  writer.compare(reader, result)
  if result.is_compatible():
  return SchemaCompatibilityResult(SchemaCompatibilityType.compatible)

karapace/protobuf/compare_result.py

@@ -34,7 +34,6 @@ class Modification(Enum):
  ONE_OF_FIELD_DROP = auto()
  ONE_OF_FIELD_MOVE = auto()
  FEW_FIELDS_CONVERTED_TO_ONE_OF = auto()
- PROTOPACE = auto()
 
  # protobuf compatibility issues is described in at
  # https://yokota.blog/2021/08/26/understanding-protobuf-compatibility/
@@ -48,7 +47,6 @@ def is_compatible(self) -> bool:
  self.FIELD_TAG_ALTER,
  self.ONE_OF_FIELD_DROP,
  self.FEW_FIELDS_CONVERTED_TO_ONE_OF,
- self.PROTOPACE,
  ] # type: ignore[comparison-overlap]
 
 

karapace/protobuf/protopace/protopace.py

@@ -17,6 +17,8 @@
 
 if arch == "x86_64":
  arch = "amd64"
+elif arch == "aarch64":
+ arch = "arm64"
 
 lib_file = os.path.join(file_dir, f"bin/protopace-{system}-{arch}.so")
 
@@ -75,9 +77,9 @@ def check_compatibility(proto: Proto, prev_proto: Proto) -> None:
  c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.dependencies])
  c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.dependencies])
 
- prev_length = len(proto.dependencies)
- prev_c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in prev_proto.dependencies])
- prev_c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in prev_proto.dependencies])
+ prev_length = len(prev_proto.dependencies)
+ prev_c_dependencies = (ctypes.c_char_p * prev_length)(*[d.schema.encode() for d in prev_proto.dependencies])
+ prev_c_dependency_names = (ctypes.c_char_p * prev_length)(*[d.name.encode() for d in prev_proto.dependencies])
 
  err = lib.CheckCompatibility(
  proto.name.encode(),

karapace/schema_registry_apis.py

@@ -452,6 +452,7 @@ async def compatibility_check(
  use_protopace=self.config["use_protopace"],
  )
  if is_incompatible(result):
+ self.log.info(f'incompatible: {result.messages}')
  self.r({"is_compatible": False}, content_type)
  self.r({"is_compatible": True}, content_type)
 

tests/integration/test_dependencies_compatibility_protobuf.py

@@ -323,10 +323,10 @@ async def test_protobuf_schema_compatibility_dependencies1g(registry_async_clien
  evolved_schema = """
 syntax = "proto3";
 package a1;
-import "google/type/postal_address.proto";
+import "google/protobuf/duration.proto";
 message TestMessage {
  message V {
- google.type.PostalAddress h = 1;
+ google.protobuf.Duration h = 1;
  int32 x = 2;
  }
  string t = 1;
@@ -369,10 +369,10 @@ async def test_protobuf_schema_compatibility_dependencies1g_otherway(registry_as
  original_schema = """
 syntax = "proto3";
 package a1;
-import "google/type/postal_address.proto";
+import "google/protobuf/duration.proto";
 message TestMessage {
  message V {
- google.type.PostalAddress h = 1;
+ google.protobuf.Duration h = 1;
  int32 x = 2;
  }
  string t = 1;
@@ -617,13 +617,13 @@ async def test_protobuf_customer_update_when_having_references(registry_async_cl
 syntax = "proto3";
 package a1;
 import "place.proto";
-import "google/type/postal_address.proto";
+import "google/protobuf/duration.proto";
 // @producer: another comment
 message Customer {
  string name = 1;
  int32 code = 2;
  Place place = 3;
- google.type.PostalAddress address = 4;
+ google.protobuf.Duration address = 4;
 }
 """
  body = {
@@ -645,13 +645,13 @@ async def test_protobuf_customer_update_when_having_references(registry_async_cl
 syntax = "proto3";
 package a1;
 import "place.proto";
-import "google/type/postal_address.proto";
+import "google/protobuf/duration.proto";
 // @consumer: the comment was incorrect, updating it now
 message Customer {
  string name = 1;
  int32 code = 2;
  Place place = 3;
- google.type.PostalAddress address = 4;
+ google.protobuf.Duration address = 4;
 }
 """