poc: use Buf and protocompile for protobuf schemas

MANIFEST.in

@@ -9,5 +9,6 @@ include setup.py
 include setup.cfg
 include LICENSE
 include MANIFEST.in
+include *.so
 
 recursive-exclude examples *~ *.pyc \.*

go/protopace/.gitignore

@@ -0,0 +1,25 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env

go/protopace/Makefile

@@ -0,0 +1,92 @@
+# Change these variables as necessary.
+MAIN_PACKAGE_PATH := .
+BINARY_NAME := protopace
+BUILD_DIR := ../../karapace/protobuf/protopace/bin
+
+# ==================================================================================== #
+# HELPERS
+# ==================================================================================== #
+
+## help: print this help message
+.PHONY: help
+help:
+ @echo 'Usage:'
+ @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /'
+
+.PHONY: confirm
+confirm:
+ @echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ]
+
+.PHONY: no-dirty
+no-dirty:
+ git diff --exit-code
+
+
+# ==================================================================================== #
+# QUALITY CONTROL
+# ==================================================================================== #
+
+## tidy: format code and tidy modfile
+.PHONY: tidy
+tidy:
+ go fmt ./...
+ go mod tidy -v
+
+## audit: run quality control checks
+.PHONY: audit
+audit:
+ go mod verify
+ go vet ./...
+ go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./...
+ go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+ go test -race -buildvcs -vet=off ./...
+
+
+# ==================================================================================== #
+# DEVELOPMENT
+# ==================================================================================== #
+
+## test: run all tests
+.PHONY: test
+test:
+ go test -v -race -buildvcs ./...
+
+## test/cover: run all tests and display coverage
+.PHONY: test/cover
+test/cover:
+ go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./...
+ go tool cover -html=/tmp/coverage.out
+
+## build: build the application
+.PHONY: build
+build:
+ # Include additional build steps, like TypeScript, SCSS or Tailwind compilation here...
+ go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH}
+
+## run: run the application
+.PHONY: run
+run: build
+ /tmp/bin/${BINARY_NAME}
+
+## run/live: run the application with reloading on file changes
+.PHONY: run/live
+run/live:
+ go run github.com/cosmtrek/air@v1.43.0 \
+ --build.cmd "make build" --build.bin "/tmp/bin/${BINARY_NAME}" --build.delay "100" \
+ --build.exclude_dir "" \
+ --build.include_ext "go, tpl, tmpl, html, css, scss, js, ts, sql, jpeg, jpg, gif, png, bmp, svg, webp, ico" \
+ --misc.clean_on_exit "true"
+
+
+# ==================================================================================== #
+# OPERATIONS
+# ==================================================================================== #
+
+## releast: cross-compile to karapace build dir
+.PHONY: release
+release:
+ CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -ldflags='-s' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
+ #upx -5 ${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so
+
+ CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags='-s' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH}
+ #upx -5 ${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so

go/protopace/compatibility.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+ "context"
+
+ "github.com/Aiven-Open/karapace/go/protopace/schema"
+
+ "github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking"
+ "github.com/bufbuild/buf/private/bufpkg/bufconfig"
+ "github.com/bufbuild/buf/private/pkg/tracing"
+ "go.uber.org/zap"
+)
+
+func Check(schema schema.Schema, previousSchema schema.Schema) error {
+ handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer)
+ ctx := context.Background()
+ image, err := schema.CompileBufImage()
+ if err != nil {
+ return err
+ }
+ previousImage, err := previousSchema.CompileBufImage()
+ if err != nil {
+ return err
+ }
+ checkConfig, _ := bufconfig.NewEnabledCheckConfig(
+ bufconfig.FileVersionV2,
+ nil,
+ []string{
+ "FIELD_NO_DELETE",
+ "FILE_SAME_PACKAGE",
+ "FIELD_SAME_NAME",
+ "FIELD_SAME_JSON_NAME",
+ },
+ nil,
+ nil,
+ )
+ config := bufconfig.NewBreakingConfig(checkConfig, false)
+ return handler.Check(ctx, config, previousImage, image)
+}

go/protopace/compatibility_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+ "os"
+ "testing"
+
+ s "github.com/Aiven-Open/karapace/go/protopace/schema"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCompatibility(t *testing.T) {
+ assert := assert.New(t)
+
+ data, _ := os.ReadFile("./fixtures/dependency.proto")
+ dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil)
+ assert.NoError(err)
+ assert.NotNil(dependencySchema)
+
+ data, _ = os.ReadFile("./fixtures/test.proto")
+ testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+ assert.NoError(err)
+ assert.NotNil(testSchema)
+
+ data, _ = os.ReadFile("./fixtures/test_previous.proto")
+ previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema})
+ assert.NoError(err)
+ assert.NotNil(previousSchema)
+
+ err = Check(*testSchema, *testSchema)
+ assert.NoError(err)
+
+ err = Check(*testSchema, *previousSchema)
+ assert.ErrorContains(err, "Previously present field \"5\" with name \"local_nested_value\" on message \"EventValue\" was deleted.")
+}

go/protopace/fixtures/dependency.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+package my.awesome.customer.v1;
+
+message NestedValue {
+ string value = 1;
+}
+
+enum Status {
+ UNKNOWN = 0;
+ ACTIVE = 1;
+ INACTIVE = 2;
+}

go/protopace/fixtures/test.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+ message NestedValue {
+ string foo = 1;
+ }
+}
+
+message EventValue {
+ NestedValue nested_value = 1;
+ google.protobuf.Timestamp created_at = 2;
+ Status status = 3;
+ Local.NestedValue local_nested_value = 4;
+}

go/protopace/fixtures/test_previous.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package my.awesome.customer.v1;
+
+import "my/awesome/customer/v1/nested_value.proto";
+import "google/protobuf/timestamp.proto";
+
+option ruby_package = "My::Awesome::Customer::V1";
+option csharp_namespace = "my.awesome.customer.V1";
+option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1";
+option java_multiple_files = true;
+option java_outer_classname = "EventValueProto";
+option java_package = "com.my.awesome.customer.v1";
+option objc_class_prefix = "TDD";
+option php_metadata_namespace = "My\\Awesome\\Customer\\V1";
+option php_namespace = "My\\Awesome\\Customer\\V1";
+
+message Local {
+ message NestedValue {
+ string foo = 1;
+ }
+}
+
+message EventValue {
+ NestedValue nested_value = 1;
+ google.protobuf.Timestamp created_at = 2;
+ Status status = 3;
+ Local.NestedValue local_nested_value = 5;
+}