diff --git a/MANIFEST.in b/MANIFEST.in index fa15133f4..0b55e5636 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,5 +9,6 @@ include setup.py include setup.cfg include LICENSE include MANIFEST.in +include *.so recursive-exclude examples *~ *.pyc \.* diff --git a/go/protopace/.gitignore b/go/protopace/.gitignore new file mode 100644 index 000000000..6f72f8926 --- /dev/null +++ b/go/protopace/.gitignore @@ -0,0 +1,25 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env diff --git a/go/protopace/Makefile b/go/protopace/Makefile new file mode 100644 index 000000000..e8f8f02fb --- /dev/null +++ b/go/protopace/Makefile @@ -0,0 +1,92 @@ +# Change these variables as necessary. +MAIN_PACKAGE_PATH := . +BINARY_NAME := protopace +BUILD_DIR := ../../karapace/protobuf/protopace/bin + +# ==================================================================================== # +# HELPERS +# ==================================================================================== # + +## help: print this help message +.PHONY: help +help: + @echo 'Usage:' + @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /' + +.PHONY: confirm +confirm: + @echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ] + +.PHONY: no-dirty +no-dirty: + git diff --exit-code + + +# ==================================================================================== # +# QUALITY CONTROL +# ==================================================================================== # + +## tidy: format code and tidy modfile +.PHONY: tidy +tidy: + go fmt ./... + go mod tidy -v + +## audit: run quality control checks +.PHONY: audit +audit: + go mod verify + go vet ./... + go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./... + go run golang.org/x/vuln/cmd/govulncheck@latest ./... + go test -race -buildvcs -vet=off ./... + + +# ==================================================================================== # +# DEVELOPMENT +# ==================================================================================== # + +## test: run all tests +.PHONY: test +test: + go test -v -race -buildvcs ./... + +## test/cover: run all tests and display coverage +.PHONY: test/cover +test/cover: + go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./... + go tool cover -html=/tmp/coverage.out + +## build: build the application +.PHONY: build +build: + # Include additional build steps, like TypeScript, SCSS or Tailwind compilation here... + go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH} + +## run: run the application +.PHONY: run +run: build + /tmp/bin/${BINARY_NAME} + +## run/live: run the application with reloading on file changes +.PHONY: run/live +run/live: + go run github.com/cosmtrek/air@v1.43.0 \ + --build.cmd "make build" --build.bin "/tmp/bin/${BINARY_NAME}" --build.delay "100" \ + --build.exclude_dir "" \ + --build.include_ext "go, tpl, tmpl, html, css, scss, js, ts, sql, jpeg, jpg, gif, png, bmp, svg, webp, ico" \ + --misc.clean_on_exit "true" + + +# ==================================================================================== # +# OPERATIONS +# ==================================================================================== # + +## releast: cross-compile to karapace build dir +.PHONY: release +release: + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -ldflags='-s' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH} + #upx -5 ${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so + + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags='-s' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH} + #upx -5 ${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so diff --git a/go/protopace/compatibility.go b/go/protopace/compatibility.go new file mode 100644 index 000000000..065c102c8 --- /dev/null +++ b/go/protopace/compatibility.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + + "github.com/Aiven-Open/karapace/go/protopace/schema" + + "github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/pkg/tracing" + "go.uber.org/zap" +) + +func Check(schema schema.Schema, previousSchema schema.Schema) error { + handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer) + ctx := context.Background() + image, err := schema.CompileBufImage() + if err != nil { + return err + } + previousImage, err := previousSchema.CompileBufImage() + if err != nil { + return err + } + checkConfig, _ := bufconfig.NewEnabledCheckConfig( + bufconfig.FileVersionV2, + nil, + []string{ + "FIELD_NO_DELETE", + "FILE_SAME_PACKAGE", + "FIELD_SAME_NAME", + "FIELD_SAME_JSON_NAME", + }, + nil, + nil, + ) + config := bufconfig.NewBreakingConfig(checkConfig, false) + return handler.Check(ctx, config, previousImage, image) +} diff --git a/go/protopace/compatibility_test.go b/go/protopace/compatibility_test.go new file mode 100644 index 000000000..9efe0d112 --- /dev/null +++ b/go/protopace/compatibility_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "os" + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestCompatibility(t *testing.T) { + assert := assert.New(t) + + data, _ := os.ReadFile("./fixtures/dependency.proto") + dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil) + assert.NoError(err) + assert.NotNil(dependencySchema) + + data, _ = os.ReadFile("./fixtures/test.proto") + testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(testSchema) + + data, _ = os.ReadFile("./fixtures/test_previous.proto") + previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(previousSchema) + + err = Check(*testSchema, *testSchema) + assert.NoError(err) + + err = Check(*testSchema, *previousSchema) + assert.ErrorContains(err, "Previously present field \"5\" with name \"local_nested_value\" on message \"EventValue\" was deleted.") +} diff --git a/go/protopace/fixtures/dependency.proto b/go/protopace/fixtures/dependency.proto new file mode 100644 index 000000000..45b94a1de --- /dev/null +++ b/go/protopace/fixtures/dependency.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} diff --git a/go/protopace/fixtures/test.proto b/go/protopace/fixtures/test.proto new file mode 100644 index 000000000..838d45ad0 --- /dev/null +++ b/go/protopace/fixtures/test.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; +} diff --git a/go/protopace/fixtures/test_previous.proto b/go/protopace/fixtures/test_previous.proto new file mode 100644 index 000000000..3a5ac0bde --- /dev/null +++ b/go/protopace/fixtures/test_previous.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 5; +} diff --git a/go/protopace/formatter.go b/go/protopace/formatter.go new file mode 100644 index 000000000..abf39d6c4 --- /dev/null +++ b/go/protopace/formatter.go @@ -0,0 +1,2499 @@ +package main + +import ( + "errors" + "fmt" + "io" + "reflect" + "sort" + "strings" + "unicode" + "unicode/utf8" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/bufbuild/protocompile/ast" + "github.com/bufbuild/protocompile/walk" + "go.uber.org/multierr" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +func Format(schema s.Schema) (s.Schema, error) { + res, err := schema.Compile() + if err != nil { + return schema, err + } + + astNodeMapping := map[ast.Node]protoreflect.FullName{} + + walk.DescriptorProtos(res.FileDescriptorProto(), func(fn protoreflect.FullName, m proto.Message) error { + astNode := res.Node(m) + astNodeMapping[astNode] = fn + return nil + }) + + fieldTypeMapping := map[protoreflect.FullName]string{} + + walk.Descriptors(res, func(d protoreflect.Descriptor) error { + fd, ok := d.(protoreflect.FieldDescriptor) + if ok { + message := fd.Message() + if message != nil { + fieldTypeMapping[fd.FullName()] = string(message.FullName()) + return nil + } + enum := fd.Enum() + if enum != nil { + fieldTypeMapping[fd.FullName()] = string(enum.FullName()) + return nil + } + } + return nil + }) + writer := strings.Builder{} + fileNode := res.FileNode().(*ast.FileNode) + f := newFormatter(&writer, fileNode, astNodeMapping, fieldTypeMapping) + f.Run() + newSchema := schema + newSchema.Schema = writer.String() + return newSchema, nil +} + +type formatter struct { + writer io.Writer + fileNode *ast.FileNode + astNodeMapping map[ast.Node]protoreflect.FullName + fieldTypeMapping map[protoreflect.FullName]string + + // Used to adjust comments when we remove superfluous + // separators tp canonicalize message literals + overrideTrailingComments map[ast.Node]ast.Comments + + // Current level of indentation. + indent int + // The last character written to writer. + lastWritten rune + + // The last node written. This must be updated from all functions + // that write comments with a node. This flag informs how the next + // node's leading comments and whitespace should be written. + previousNode ast.Node + + // If true, a space will be written to the output unless the next character + // written is a newline (don't wait errant trailing spaces). + pendingSpace bool + // If true, the formatter is in the middle of printing compact options. + inCompactOptions bool + + // Track runes that open blocks/scopes and are expected to increase indention + // level. For example, when runes "{" "[" "(" ")" are written, the pending + // value is 2 (increment three times for "{" "[" "("; decrement once for ")"). + // If it's greater than zero at the end of a line, we call In() so that + // subsequent lines are indented. If it's less than zero at the end of a line, + // we call Out(). This minimizes the amount of explicit indent/unindent code + // that is needed and makes it less error-prone. + pendingIndent int + // If true, an inline node/sequence is being written. We treat whitespace a + // little differently for when blocks are printed inline vs. across multiple + // lines. So this flag informs the logic that makes those whitespace decisions. + inline bool + + // Records all errors that occur during the formatting process. Nearly any + // non-nil error represents a bug in the implementation. + err error +} + +// newFormatter returns a new formatter for the given file. +func newFormatter( + writer io.Writer, + fileNode *ast.FileNode, + astNodeMapping map[ast.Node]protoreflect.FullName, + fieldTypeMapping map[protoreflect.FullName]string, +) *formatter { + return &formatter{ + writer: writer, + fileNode: fileNode, + astNodeMapping: astNodeMapping, + fieldTypeMapping: fieldTypeMapping, + overrideTrailingComments: map[ast.Node]ast.Comments{}, + } +} + +// Run runs the formatter and writes the file's content to the formatter's writer. +func (f *formatter) Run() error { + f.writeFile() + return f.err +} + +// P prints a line to the generated output. +// +// This will emit a newline and proper indentation. If you do not +// want to emit a newline and want to write a raw string, use +// WriteString (which P calls). +// +// If strings.TrimSpace(elem) is empty, no indentation is produced. +func (f *formatter) P(elem string) { + if len(strings.TrimSpace(elem)) > 0 { + // We only want to write an indent if we're + // writing a non-empty string (not just a newline). + f.Indent(nil) + f.WriteString(elem) + } + f.WriteString("\n") + + if f.pendingIndent > 0 { + f.In() + } else if f.pendingIndent < 0 { + f.Out() + } + f.pendingIndent = 0 +} + +// Space adds a space to the generated output. +func (f *formatter) Space() { + f.pendingSpace = true +} + +// In increases the current level of indentation. +func (f *formatter) In() { + f.indent++ +} + +// Out reduces the current level of indentation. +func (f *formatter) Out() { + if f.indent <= 0 { + // Unreachable. + f.err = multierr.Append( + f.err, + errors.New("internal error: attempted to decrement indentation at zero"), + ) + return + } + f.indent-- +} + +// Indent writes the number of spaces associated +// with the current level of indentation. +func (f *formatter) Indent(nextNode ast.Node) { + // only indent at beginning of line + if f.lastWritten != '\n' { + return + } + indent := f.indent + if rn, ok := nextNode.(*ast.RuneNode); ok && indent > 0 { + if strings.ContainsRune("}])>", rn.Rune) { + indent-- + } + } + f.WriteString(strings.Repeat(" ", indent)) +} + +// WriteString writes the given element to the generated output. +// +// This will not write indentation or newlines. Use P if you +// want to emit identation or newlines. +func (f *formatter) WriteString(elem string) { + if f.pendingSpace { + f.pendingSpace = false + first, _ := utf8.DecodeRuneInString(elem) + + // We don't want "dangling spaces" before certain characters: + // newlines, commas, and semicolons. Also, when writing + // elements inline, we don't want spaces before close parens + // and braces. Similarly, we don't want extra/doubled spaces + // or dangling spaces after certain characters when printing + // inline, like open parens/braces. So only print the space + // if the previous and next character don't match above + // conditions. + + prevBlockList := "\x00 \t\n" + nextBlockList := "\n;," + if f.inline { + prevBlockList = "\x00 \t\n<[{(" + nextBlockList = "\n;,)]}>" + } + + if !strings.ContainsRune(prevBlockList, f.lastWritten) && + !strings.ContainsRune(nextBlockList, first) { + if _, err := f.writer.Write([]byte{' '}); err != nil { + f.err = multierr.Append(f.err, err) + return + } + } + } + if len(elem) == 0 { + return + } + f.lastWritten, _ = utf8.DecodeLastRuneInString(elem) + if _, err := f.writer.Write([]byte(elem)); err != nil { + f.err = multierr.Append(f.err, err) + } +} + +// SetPreviousNode sets the previously written node. This should +// be called in all of the comment writing functions. +func (f *formatter) SetPreviousNode(node ast.Node) { + f.previousNode = node +} + +// writeFile writes the file node. +func (f *formatter) writeFile() { + f.writeFileHeader() + f.writeFileTypes() + if f.fileNode.EOF != nil { + info := f.nodeInfo(f.fileNode.EOF) + f.writeMultilineComments(info.LeadingComments()) + } + if f.lastWritten != 0 && f.lastWritten != '\n' { + // If anything was written, we always conclude with + // a newline. + f.P("") + } +} + +// writeFileHeader writes the header of a .proto file. This includes the syntax, +// package, imports, and options (in that order). The imports and options are +// sorted. All other file elements are handled by f.writeFileTypes. +// +// For example, +// +// syntax = "proto3"; +// +// package acme.v1.weather; +// +// import "acme/payment/v1/payment.proto"; +// import "google/type/datetime.proto"; +// +// option cc_enable_arenas = true; +// option optimize_for = SPEED; +func (f *formatter) writeFileHeader() { + var ( + packageNode *ast.PackageNode + importNodes []*ast.ImportNode + optionNodes []*ast.OptionNode + ) + for _, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode: + packageNode = node + case *ast.ImportNode: + importNodes = append(importNodes, node) + case *ast.OptionNode: + optionNodes = append(optionNodes, node) + default: + continue + } + } + if f.fileNode.Syntax == nil && f.fileNode.Edition == nil && + packageNode == nil && importNodes == nil && optionNodes == nil { + // There aren't any header values, so we can return early. + return + } + editionNode := f.fileNode.Edition + if editionNode != nil { + f.writeEdition(editionNode) + } + if syntaxNode := f.fileNode.Syntax; syntaxNode != nil && editionNode == nil { + f.writeSyntax(syntaxNode) + } + if packageNode != nil { + f.writePackage(packageNode) + } + sort.Slice(importNodes, func(i, j int) bool { + iName := importNodes[i].Name.AsString() + jName := importNodes[j].Name.AsString() + // sort by public > None > weak + iOrder := importSortOrder(importNodes[i]) + jOrder := importSortOrder(importNodes[j]) + + if iName < jName { + return true + } + if iName > jName { + return false + } + if iOrder > jOrder { + return true + } + if iOrder < jOrder { + return false + } + + // put commented import first + return !f.importHasComment(importNodes[j]) + }) + for i, importNode := range importNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(importNode) { + f.P("") + } + + // since the imports are sorted, this will skip write imports + // if they have appear before and dont have comment + if i > 0 && importNode.Name.AsString() == importNodes[i-1].Name.AsString() && + !f.importHasComment(importNode) { + continue + } + + f.writeImport(importNode, i > 0) + } + sort.Slice(optionNodes, func(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(optionNodes[i].Name) + right := stringForOptionName(optionNodes[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right + }) + for i, optionNode := range optionNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(optionNode) { + f.P("") + } + f.writeFileOption(optionNode, i > 0) + } +} + +// writeFileTypes writes the types defined in a .proto file. This includes the messages, enums, +// services, etc. All other elements are ignored since they are handled by f.writeFileHeader. +func (f *formatter) writeFileTypes() { + for i, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode, *ast.OptionNode, *ast.ImportNode, *ast.EmptyDeclNode: + // These elements have already been written by f.writeFileHeader. + continue + default: + info := f.nodeInfo(node) + wantNewline := f.previousNode != nil && (i == 0 || info.LeadingComments().Len() > 0) + if wantNewline && !f.leadingCommentsContainBlankLine(node) { + f.P("") + } + f.writeNode(node) + } + } +} + +// writeSyntax writes the syntax. +// +// For example, +// +// syntax = "proto3"; +func (f *formatter) writeSyntax(syntaxNode *ast.SyntaxNode) { + f.writeStart(syntaxNode.Keyword) + f.Space() + f.writeInline(syntaxNode.Equals) + f.Space() + f.writeInline(syntaxNode.Syntax) + f.writeLineEnd(syntaxNode.Semicolon) +} + +// writeEdition writes the edition. +// +// For example, +// +// edition = "2023"; +func (f *formatter) writeEdition(editionNode *ast.EditionNode) { + f.writeStart(editionNode.Keyword) + f.Space() + f.writeInline(editionNode.Equals) + f.Space() + f.writeInline(editionNode.Edition) + f.writeLineEnd(editionNode.Semicolon) +} + +// writePackage writes the package. +// +// For example, +// +// package acme.weather.v1; +func (f *formatter) writePackage(packageNode *ast.PackageNode) { + f.writeStart(packageNode.Keyword) + f.Space() + f.writeInline(packageNode.Name) + f.writeLineEnd(packageNode.Semicolon) +} + +// writeImport writes an import statement. +// +// For example, +// +// import "google/protobuf/descriptor.proto"; +func (f *formatter) writeImport(importNode *ast.ImportNode, forceCompact bool) { + f.writeStartMaybeCompact(importNode.Keyword, forceCompact) + f.Space() + // We don't want to write the "public" and "weak" nodes + // if they aren't defined. One could be set, but never both. + switch { + case importNode.Public != nil: + f.writeInline(importNode.Public) + f.Space() + case importNode.Weak != nil: + f.writeInline(importNode.Weak) + f.Space() + } + f.writeInline(importNode.Name) + f.writeLineEnd(importNode.Semicolon) +} + +// writeFileOption writes a file option. This function is slightly +// different than f.writeOption because file options are sorted at +// the top of the file, and leading comments are adjusted accordingly. +func (f *formatter) writeFileOption(optionNode *ast.OptionNode, forceCompact bool) { + f.writeStartMaybeCompact(optionNode.Keyword, forceCompact) + f.Space() + f.writeNode(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) +} + +// writeOption writes an option. +// +// For example, +// +// option go_package = "github.com/foo/bar"; +func (f *formatter) writeOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + if optionNode.Semicolon != nil { + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) + return + } + + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(node) + return + } + f.writeInline(optionNode.Val) +} + +// writeLastCompactOption writes a compact option but preserves its the +// trailing end comments. This is only used for the last compact option +// since it's the only time a trailing ',' will be omitted. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" // Trailing comment on the last element. +// ] +func (f *formatter) writeLastCompactOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + f.writeLineEnd(optionNode.Val) +} + +// writeOptionValue writes the option prefix, which makes up all of the +// option's definition, excluding the final token(s). +// +// For example, +// +// deprecated = +func (f *formatter) writeOptionPrefix(optionNode *ast.OptionNode) { + if optionNode.Keyword != nil { + // Compact options don't have the keyword. + f.writeStart(optionNode.Keyword) + f.Space() + f.writeNode(optionNode.Name) + } else { + f.writeStart(optionNode.Name) + } + f.Space() + f.writeInline(optionNode.Equals) + f.Space() +} + +// writeOptionName writes an option name. +// +// For example, +// +// go_package +// (custom.thing) +// (custom.thing).bridge.(another.thing) +func (f *formatter) writeOptionName(optionNameNode *ast.OptionNameNode) { + for i := 0; i < len(optionNameNode.Parts); i++ { + if f.inCompactOptions && i == 0 { + // The leading comments of the first token (either open rune or the + // name) will have already been written, so we need to handle this + // case specially. + fieldReferenceNode := optionNameNode.Parts[0] + if fieldReferenceNode.Open != nil { + f.writeNode(fieldReferenceNode.Open) + if info := f.nodeInfo(fieldReferenceNode.Open); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeNode(fieldReferenceNode.Name) + if info := f.nodeInfo(fieldReferenceNode.Name); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + continue + } + if i > 0 { + // The length of this slice must be exactly len(Parts)-1. + f.writeInline(optionNameNode.Dots[i-1]) + } + f.writeNode(optionNameNode.Parts[i]) + } +} + +// writeMessage writes the message node. +// +// For example, +// +// message Foo { +// option deprecated = true; +// reserved 50 to 100; +// extensions 150 to 200; +// +// message Bar { +// string name = 1; +// } +// enum Baz { +// BAZ_UNSPECIFIED = 0; +// } +// extend Bar { +// string value = 2; +// } +// +// Bar bar = 1; +// Baz baz = 2; +// } +func (f *formatter) writeMessage(messageNode *ast.MessageNode) { + var elementWriterFunc func() + if len(messageNode.Decls) != 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(messageNode.Decls)) + for i, n := range messageNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(messageNode.Keyword) + f.Space() + f.writeInline(messageNode.Name) + f.Space() + f.writeCompositeTypeBody( + messageNode.OpenBrace, + messageNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal. +// +// For example, +// +// { +// foo: 1 +// foo: 2 +// foo: 3 +// bar: < +// name:"abc" +// id:123 +// > +// } +func (f *formatter) writeMessageLiteral(messageLiteralNode *ast.MessageLiteralNode) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, false) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + f.writeCompositeValueBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal suitable for +// an element in an array literal. +func (f *formatter) writeMessageLiteralForArray( + messageLiteralNode *ast.MessageLiteralNode, + lastElement bool, +) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, true) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + closeWriter := f.writeBodyEndInline + if lastElement { + closeWriter = f.writeBodyEnd + } + f.writeBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + f.writeOpenBracePrefixForArray, + closeWriter, + ) +} + +func (f *formatter) maybeWriteCompactMessageLiteral( + messageLiteralNode *ast.MessageLiteralNode, + inArrayLiteral bool, +) bool { + if len(messageLiteralNode.Elements) == 0 || len(messageLiteralNode.Elements) > 1 || + f.hasInteriorComments(messageLiteralNode.Children()...) || + messageLiteralHasNestedMessageOrArray(messageLiteralNode) { + return false + } + // messages with a single scalar field and no comments can be + // printed all on one line + openNode := messageLiteralOpen(messageLiteralNode) + closeNode := messageLiteralClose(messageLiteralNode) + if inArrayLiteral { + f.Indent(openNode) + } + f.writeInline(openNode) + fieldNode := messageLiteralNode.Elements[0] + f.writeInline(fieldNode.Name) + if fieldNode.Sep != nil { + f.writeInline(fieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() + if messageLiteralNode.Seps[0] != nil { + // We are dropping the optional trailing separator. If it had + // trailing comments and the value does not, move the separator's + // trailing comment to the value. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[0]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(fieldNode.Val).TrailingComments().Len() == 0 { + f.setTrailingComments(fieldNode.Val, sepTrailingComments) + } + } + f.writeInline(fieldNode.Val) + f.writeInline(closeNode) + return true +} + +func messageLiteralHasNestedMessageOrArray(messageLiteralNode *ast.MessageLiteralNode) bool { + for _, elem := range messageLiteralNode.Elements { + switch elem.Val.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +func arrayLiteralHasNestedMessageOrArray(arrayLiteralNode *ast.ArrayLiteralNode) bool { + for _, elem := range arrayLiteralNode.Elements { + switch elem.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +// writeMessageLiteralElements writes the message literal's elements. +// +// For example, +// +// foo: 1 +// foo: 2 +func (f *formatter) writeMessageLiteralElements(messageLiteralNode *ast.MessageLiteralNode) { + for i := 0; i < len(messageLiteralNode.Elements); i++ { + // Separators ("," or ";") are optional. To avoid inconsistent formatted output, + // we suppress them, since they aren't needed. So we just write the element and + // ignore any optional separator in the AST. + if messageLiteralNode.Seps[i] != nil { + // Since we are dropping the optional trailing separator, we should + // possibly move its trailing comment to the element value so we don't + // lose it. Skip this step if the value already has a trailing comment. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[i]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(messageLiteralNode.Elements[i].Val).TrailingComments().Len() == 0 { + f.setTrailingComments(messageLiteralNode.Elements[i].Val, sepTrailingComments) + } + } + f.writeNode(messageLiteralNode.Elements[i]) + } +} + +// writeMessageField writes the message field node, and concludes the +// line without leaving room for a trailing separator in the parent +// message literal. +func (f *formatter) writeMessageField(messageFieldNode *ast.MessageFieldNode) { + f.writeMessageFieldPrefix(messageFieldNode) + if compoundStringLiteral, ok := messageFieldNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(compoundStringLiteral) + return + } + f.writeLineEnd(messageFieldNode.Val) +} + +// writeMessageFieldPrefix writes the message field node as a single line. +// +// For example, +// +// foo:"bar" +func (f *formatter) writeMessageFieldPrefix(messageFieldNode *ast.MessageFieldNode) { + // The comments need to be written as a multiline comment above + // the message field name. + // + // Note that this is different than how field reference nodes are + // normally formatted in-line (i.e. as option name components). + fieldReferenceNode := messageFieldNode.Name + if fieldReferenceNode.Open != nil { + f.writeStart(fieldReferenceNode.Open) + if fieldReferenceNode.URLPrefix != nil { + f.writeInline(fieldReferenceNode.URLPrefix) + f.writeInline(fieldReferenceNode.Slash) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeStart(fieldReferenceNode.Name) + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + // The colon separator is optional sometimes, but we don't have enough + // information here to know whether it's necessary. For more consistent + // output, just always include it. + if messageFieldNode.Sep != nil { + f.writeInline(messageFieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() +} + +// writeEnum writes the enum node. +// +// For example, +// +// enum Foo { +// option deprecated = true; +// reserved 1 to 5; +// +// FOO_UNSPECIFIED = 0; +// } +func (f *formatter) writeEnum(enumNode *ast.EnumNode) { + var elementWriterFunc func() + if len(enumNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(enumNode.Decls)) + for i, n := range enumNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(enumNode.Keyword) + f.Space() + f.writeInline(enumNode.Name) + f.Space() + f.writeCompositeTypeBody( + enumNode.OpenBrace, + enumNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeEnumValue writes the enum value as a single line. If the enum has +// compact options, it will be written across multiple lines. +// +// For example, +// +// FOO_UNSPECIFIED = 1 [ +// deprecated = true +// ]; +func (f *formatter) writeEnumValue(enumValueNode *ast.EnumValueNode) { + f.writeStart(enumValueNode.Name) + f.Space() + f.writeInline(enumValueNode.Equals) + f.Space() + f.writeInline(enumValueNode.Number) + if enumValueNode.Options != nil { + f.Space() + f.writeNode(enumValueNode.Options) + } + f.writeLineEnd(enumValueNode.Semicolon) +} + +// writeField writes the field node as a single line. If the field has +// compact options, it will be written across multiple lines. +// +// For example, +// +// repeated string name = 1 [ +// deprecated = true, +// json_name = "name" +// ]; +func (f *formatter) writeField(fieldNode *ast.FieldNode) { + // We need to handle the comments for the field label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + n := f.astNodeMapping[fieldNode] + fullType := f.fieldTypeMapping[n] + t := ast.NewIdentNode(fullType, fieldNode.FldType.Start()) + + if fieldNode.Label.KeywordNode != nil { + f.writeStart(fieldNode.Label) + f.Space() + //f.writeInline(fieldNode.FldType) + f.writeInline(t) + } else { + // If a label was not written, the multiline comments will be + // attached to the type. + if compoundIdentNode, ok := fieldNode.FldType.(*ast.CompoundIdentNode); ok { + f.writeCompountIdentForFieldName(compoundIdentNode) + } else { + //f.writeStart(fieldNode.FldType) + f.writeStart(t) + } + } + f.Space() + f.writeInline(fieldNode.Name) + f.Space() + f.writeInline(fieldNode.Equals) + f.Space() + f.writeInline(fieldNode.Tag) + if fieldNode.Options != nil { + f.Space() + f.writeNode(fieldNode.Options) + } + f.writeLineEnd(fieldNode.Semicolon) +} + +// writeMapField writes a map field (e.g. 'map pairs = 1;'). +func (f *formatter) writeMapField(mapFieldNode *ast.MapFieldNode) { + f.writeNode(mapFieldNode.MapType) + f.Space() + f.writeInline(mapFieldNode.Name) + f.Space() + f.writeInline(mapFieldNode.Equals) + f.Space() + f.writeInline(mapFieldNode.Tag) + if mapFieldNode.Options != nil { + f.Space() + f.writeNode(mapFieldNode.Options) + } + f.writeLineEnd(mapFieldNode.Semicolon) +} + +// writeMapType writes a map type (e.g. 'map'). +func (f *formatter) writeMapType(mapTypeNode *ast.MapTypeNode) { + f.writeStart(mapTypeNode.Keyword) + f.writeInline(mapTypeNode.OpenAngle) + f.writeInline(mapTypeNode.KeyType) + f.writeInline(mapTypeNode.Comma) + f.Space() + f.writeInline(mapTypeNode.ValueType) + f.writeInline(mapTypeNode.CloseAngle) +} + +// writeFieldReference writes a field reference (e.g. '(foo.bar)'). +func (f *formatter) writeFieldReference(fieldReferenceNode *ast.FieldReferenceNode) { + if fieldReferenceNode.Open != nil { + f.writeInline(fieldReferenceNode.Open) + } + f.writeInline(fieldReferenceNode.Name) + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } +} + +// writeExtend writes the extend node. +// +// For example, +// +// extend google.protobuf.FieldOptions { +// bool redacted = 33333; +// } +func (f *formatter) writeExtend(extendNode *ast.ExtendNode) { + var elementWriterFunc func() + if len(extendNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(extendNode.Decls)) + for i, n := range extendNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(extendNode.Keyword) + f.Space() + f.writeInline(extendNode.Extendee) + f.Space() + f.writeCompositeTypeBody( + extendNode.OpenBrace, + extendNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeService writes the service node. +// +// For example, +// +// service FooService { +// option deprecated = true; +// +// rpc Foo(FooRequest) returns (FooResponse) {}; +func (f *formatter) writeService(serviceNode *ast.ServiceNode) { + var elementWriterFunc func() + if len(serviceNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(serviceNode.Decls)) + for i, n := range serviceNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(serviceNode.Keyword) + f.Space() + f.writeInline(serviceNode.Name) + f.Space() + f.writeCompositeTypeBody( + serviceNode.OpenBrace, + serviceNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPC writes the RPC node. RPCs are formatted in +// the following order: +// +// For example, +// +// rpc Foo(FooRequest) returns (FooResponse) { +// option deprecated = true; +// }; +func (f *formatter) writeRPC(rpcNode *ast.RPCNode) { + var elementWriterFunc func() + if len(rpcNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(rpcNode.Decls)) + for i, n := range rpcNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(rpcNode.Keyword) + f.Space() + f.writeInline(rpcNode.Name) + f.writeInline(rpcNode.Input) + f.Space() + f.writeInline(rpcNode.Returns) + f.Space() + f.writeInline(rpcNode.Output) + if rpcNode.OpenBrace == nil { + // This RPC doesn't have any elements, so we prefer the + // ';' form. + // + // rpc Ping(PingRequest) returns (PingResponse); + // + f.writeLineEnd(rpcNode.Semicolon) + return + } + f.Space() + f.writeCompositeTypeBody( + rpcNode.OpenBrace, + rpcNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPCType writes the RPC type node (e.g. (stream foo.Bar)). +func (f *formatter) writeRPCType(rpcTypeNode *ast.RPCTypeNode) { + f.writeInline(rpcTypeNode.OpenParen) + if rpcTypeNode.Stream != nil { + f.writeInline(rpcTypeNode.Stream) + f.Space() + } + f.writeInline(rpcTypeNode.MessageType) + f.writeInline(rpcTypeNode.CloseParen) +} + +// writeOneOf writes the oneof node. +// +// For example, +// +// oneof foo { +// option deprecated = true; +// +// string name = 1; +// int number = 2; +// } +func (f *formatter) writeOneOf(oneOfNode *ast.OneofNode) { + var elementWriterFunc func() + if len(oneOfNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(oneOfNode.Decls)) + for i, n := range oneOfNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(oneOfNode.Keyword) + f.Space() + f.writeInline(oneOfNode.Name) + f.Space() + f.writeCompositeTypeBody( + oneOfNode.OpenBrace, + oneOfNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeGroup writes the group node. +// +// For example, +// +// optional group Key = 4 [ +// deprecated = true, +// json_name = "key" +// ] { +// optional uint64 id = 1; +// optional string name = 2; +// } +func (f *formatter) writeGroup(groupNode *ast.GroupNode) { + var elementWriterFunc func() + if len(groupNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(groupNode.Decls)) + for i, n := range groupNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + // We need to handle the comments for the group label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + if groupNode.Label.KeywordNode != nil { + f.writeStart(groupNode.Label) + f.Space() + f.writeInline(groupNode.Keyword) + } else { + // If a label was not written, the multiline comments will be + // attached to the keyword. + f.writeStart(groupNode.Keyword) + } + f.Space() + f.writeInline(groupNode.Name) + f.Space() + f.writeInline(groupNode.Equals) + f.Space() + f.writeInline(groupNode.Tag) + if groupNode.Options != nil { + f.Space() + f.writeNode(groupNode.Options) + } + f.Space() + f.writeCompositeTypeBody( + groupNode.OpenBrace, + groupNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeExtensionRange writes the extension range node. +// +// For example, +// +// extensions 5-10, 100 to max [ +// deprecated = true +// ]; +func (f *formatter) writeExtensionRange(extensionRangeNode *ast.ExtensionRangeNode) { + f.writeStart(extensionRangeNode.Keyword) + f.Space() + for i := 0; i < len(extensionRangeNode.Ranges); i++ { + if i > 0 { + // The length of this slice must be exactly len(Ranges)-1. + f.writeInline(extensionRangeNode.Commas[i-1]) + f.Space() + } + f.writeNode(extensionRangeNode.Ranges[i]) + } + if extensionRangeNode.Options != nil { + f.Space() + f.writeNode(extensionRangeNode.Options) + } + f.writeLineEnd(extensionRangeNode.Semicolon) +} + +// writeReserved writes a reserved node. +// +// For example, +// +// reserved 5-10, 100 to max; +func (f *formatter) writeReserved(reservedNode *ast.ReservedNode) { + f.writeStart(reservedNode.Keyword) + // Either names or ranges will be set, but never both. + elements := make([]ast.Node, 0, len(reservedNode.Names)+len(reservedNode.Ranges)) + switch { + case reservedNode.Names != nil: + for _, nameNode := range reservedNode.Names { + elements = append(elements, nameNode) + } + case reservedNode.Identifiers != nil: + for _, identNode := range reservedNode.Identifiers { + elements = append(elements, identNode) + } + case reservedNode.Ranges != nil: + for _, rangeNode := range reservedNode.Ranges { + elements = append(elements, rangeNode) + } + } + f.Space() + for i := 0; i < len(elements); i++ { + if i > 0 { + // The length of this slice must be exactly len({Names,Ranges})-1. + f.writeInline(reservedNode.Commas[i-1]) + f.Space() + } + f.writeInline(elements[i]) + } + f.writeLineEnd(reservedNode.Semicolon) +} + +// writeRange writes the given range node (e.g. '1 to max'). +func (f *formatter) writeRange(rangeNode *ast.RangeNode) { + f.writeInline(rangeNode.StartVal) + if rangeNode.To != nil { + f.Space() + f.writeInline(rangeNode.To) + } + // Either EndVal or Max will be set, but never both. + switch { + case rangeNode.EndVal != nil: + f.Space() + f.writeInline(rangeNode.EndVal) + case rangeNode.Max != nil: + f.Space() + f.writeInline(rangeNode.Max) + } +} + +// writeCompactOptions writes a compact options node. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" +// ] +func (f *formatter) writeCompactOptions(compactOptionsNode *ast.CompactOptionsNode) { + f.inCompactOptions = true + defer func() { + f.inCompactOptions = false + }() + if len(compactOptionsNode.Options) == 1 && + !f.hasInteriorComments(compactOptionsNode.OpenBracket, compactOptionsNode.Options[0].Name) { + // If there's only a single compact scalar option without comments, we can write it + // in-line. For example: + // + // [deprecated = true] + // + // However, this does not include the case when the '[' has trailing comments, + // or the option name has leading comments. In those cases, we write the option + // across multiple lines. For example: + // + // [ + // // This type is deprecated. + // deprecated = true + // ] + // + optionNode := compactOptionsNode.Options[0] + f.writeInline(compactOptionsNode.OpenBracket) + f.writeInline(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // If there's only a single compact option, the value needs to + // write its comments (if any) in a way that preserves the closing ']'. + f.writeCompoundStringLiteralNoIndentEndInline(node) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + var elementWriterFunc func() + if len(compactOptionsNode.Options) > 0 { + elementWriterFunc = func() { + sort.Slice(compactOptionsNode.Options, func(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(compactOptionsNode.Options[i].Name) + right := stringForOptionName(compactOptionsNode.Options[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right + }) + for i, opt := range compactOptionsNode.Options { + if i == len(compactOptionsNode.Options)-1 { + // The last element won't have a trailing comma. + f.writeLastCompactOption(opt) + return + } + f.writeNode(opt) + f.writeLineEnd(compactOptionsNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + compactOptionsNode.OpenBracket, + compactOptionsNode.CloseBracket, + elementWriterFunc, + ) +} + +func (f *formatter) hasInteriorComments(nodes ...ast.Node) bool { + for i, n := range nodes { + // interior comments mean we ignore leading comments on first + // token and trailing comments on the last one + info := f.nodeInfo(n) + if i > 0 && info.LeadingComments().Len() > 0 { + return true + } + if i < len(nodes)-1 && info.TrailingComments().Len() > 0 { + return true + } + } + return false +} + +// writeArrayLiteral writes an array literal across multiple lines. +// +// For example, +// +// [ +// "foo", +// "bar" +// ] +func (f *formatter) writeArrayLiteral(arrayLiteralNode *ast.ArrayLiteralNode) { + if len(arrayLiteralNode.Elements) == 1 && + !f.hasInteriorComments(arrayLiteralNode.Children()...) && + !arrayLiteralHasNestedMessageOrArray(arrayLiteralNode) { + // arrays with a single scalar value and no comments can be + // printed all on one line + valueNode := arrayLiteralNode.Elements[0] + f.writeInline(arrayLiteralNode.OpenBracket) + f.writeInline(valueNode) + f.writeInline(arrayLiteralNode.CloseBracket) + return + } + + var elementWriterFunc func() + if len(arrayLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + for i := 0; i < len(arrayLiteralNode.Elements); i++ { + lastElement := i == len(arrayLiteralNode.Elements)-1 + if compositeNode, ok := arrayLiteralNode.Elements[i].(ast.CompositeNode); ok { + f.writeCompositeValueForArrayLiteral(compositeNode, lastElement) + if !lastElement { + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + continue + } + if lastElement { + // The last element won't have a trailing comma. + f.writeLineElement(arrayLiteralNode.Elements[i]) + return + } + f.writeStart(arrayLiteralNode.Elements[i]) + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + arrayLiteralNode.OpenBracket, + arrayLiteralNode.CloseBracket, + elementWriterFunc, + ) +} + +// writeCompositeForArrayLiteral writes the composite node in a way that's suitable +// for array literals. In general, signed integers and compound strings should have their +// comments written in-line because they are one of many components in a single line. +// +// However, each of these composite types occupy a single line in an array literal, +// so they need their comments to be formatted like a standalone node. +// +// For example, +// +// option (value) = /* In-line comment for '-42' */ -42; +// +// option (thing) = { +// values: [ +// // Leading comment on -42. +// -42, // Trailing comment on -42. +// ] +// } +// +// The lastElement boolean is used to signal whether or not the composite value +// should be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompositeValueForArrayLiteral( + compositeNode ast.CompositeNode, + lastElement bool, +) { + switch node := compositeNode.(type) { + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralForArray(node, lastElement) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteralForArray(node, lastElement) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteralForArray(node, lastElement) + case *ast.MessageLiteralNode: + f.writeMessageLiteralForArray(node, lastElement) + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected array value node %T", node)) + } +} + +// writeCompositeTypeBody writes the body of a composite type, e.g. message, enum, extend, oneof, etc. +func (f *formatter) writeCompositeTypeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEnd, + ) +} + +// writeCompositeValueBody writes the body of a composite value, e.g. compact options, +// array literal, etc. We need to handle the ']' different than composite types because +// there could be more tokens following the final ']'. +func (f *formatter) writeCompositeValueBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEndInline, + ) +} + +// writeBody writes the body of a type or value, e.g. message, enum, compact options, etc. +// The elementWriterFunc is used to write the declarations within the composite type (e.g. +// fields in a message). The openBraceWriterFunc and closeBraceWriterFunc functions are used +// to customize how the '{' and '} nodes are written, respectively. +func (f *formatter) writeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), + openBraceWriterFunc func(ast.Node), + closeBraceWriterFunc func(ast.Node, bool), +) { + if elementWriterFunc == nil && !f.hasInteriorComments(openBrace, closeBrace) { + // completely empty body + f.writeInline(openBrace) + closeBraceWriterFunc(closeBrace, true) + return + } + + openBraceWriterFunc(openBrace) + if elementWriterFunc != nil { + elementWriterFunc() + } + closeBraceWriterFunc(closeBrace, false) +} + +// writeOpenBracePrefix writes the open brace with its leading comments in-line. +// This is used for nearly every use case of f.writeBody, excluding the instances +// in array literals. +func (f *formatter) writeOpenBracePrefix(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeOpenBracePrefixForArray writes the open brace with its leading comments +// on multiple lines. This is only used for message literals in arrays. +func (f *formatter) writeOpenBracePrefixForArray(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeMultilineComments(info.LeadingComments()) + } + f.Indent(openBrace) + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeCompoundIdent writes a compound identifier (e.g. '.com.foo.Bar'). +func (f *formatter) writeCompoundIdent(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeInline(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeCompountIdentForFieldName writes a compound identifier, but handles comments +// specially for field names. +// +// For example, +// +// message Foo { +// // These are comments attached to bar. +// bar.v1.Bar bar = 1; +// } +func (f *formatter) writeCompountIdentForFieldName(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeStart(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i == 0 && compoundIdentNode.LeadingDot == nil { + f.writeStart(compoundIdentNode.Components[i]) + continue + } + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeFieldLabel writes the field label node. +// +// For example, +// +// optional +// repeated +// required +func (f *formatter) writeFieldLabel(fieldLabel ast.FieldLabel) { + f.WriteString(fieldLabel.Val) +} + +// writeCompoundStringLiteral writes a compound string literal value. +// +// For example, +// +// "one," +// "two," +// "three" +func (f *formatter) writeCompoundStringLiteral( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + needsIndent bool, + hasTrailingPunctuation bool, +) { + f.P("") + if needsIndent { + f.In() + } + for i, child := range compoundStringLiteralNode.Children() { + if hasTrailingPunctuation && i == len(compoundStringLiteralNode.Children())-1 { + // inline because there may be a subsequent comma or punctuation from enclosing element + f.writeStart(child) + break + } + f.writeLineElement(child) + } + if needsIndent { + f.Out() + } +} + +func (f *formatter) writeCompoundStringLiteralIndent( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, false) +} + +func (f *formatter) writeCompoundStringLiteralIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, true) +} + +func (f *formatter) writeCompoundStringLiteralNoIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, false, true) +} + +// writeCompoundStringLiteralForArray writes a compound string literal value, +// but writes its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompoundStringLiteralForArray( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + lastElement bool, +) { + for i, child := range compoundStringLiteralNode.Children() { + if !lastElement && i == len(compoundStringLiteralNode.Children())-1 { + f.writeStart(child) + return + } + f.writeLineElement(child) + } +} + +// writeFloatLiteral writes a float literal value (e.g. '42.2'). +func (f *formatter) writeFloatLiteral(floatLiteralNode *ast.FloatLiteralNode) { + f.writeRaw(floatLiteralNode) +} + +// writeSignedFloatLiteral writes a signed float literal value (e.g. '-42.2'). +func (f *formatter) writeSignedFloatLiteral(signedFloatLiteralNode *ast.SignedFloatLiteralNode) { + f.writeInline(signedFloatLiteralNode.Sign) + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSignedFloatLiteralForArray writes a signed float literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeSignedFloatLiteralForArray( + signedFloatLiteralNode *ast.SignedFloatLiteralNode, + lastElement bool, +) { + f.writeStart(signedFloatLiteralNode.Sign) + if lastElement { + f.writeLineEnd(signedFloatLiteralNode.Float) + return + } + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSpecialFloatLiteral writes a special float literal value (e.g. "nan" or "inf"). +func (f *formatter) writeSpecialFloatLiteral(specialFloatLiteralNode *ast.SpecialFloatLiteralNode) { + f.WriteString(specialFloatLiteralNode.KeywordNode.Val) +} + +// writeStringLiteral writes a string literal value (e.g. "foo"). +// Note that the raw string is written as-is so that it preserves +// the quote style used in the original source. +func (f *formatter) writeStringLiteral(stringLiteralNode *ast.StringLiteralNode) { + f.writeRaw(stringLiteralNode) +} + +// writeUintLiteral writes a uint literal (e.g. '42'). +func (f *formatter) writeUintLiteral(uintLiteralNode *ast.UintLiteralNode) { + f.writeRaw(uintLiteralNode) +} + +// writeNegativeIntLiteral writes a negative int literal (e.g. '-42'). +func (f *formatter) writeNegativeIntLiteral(negativeIntLiteralNode *ast.NegativeIntLiteralNode) { + f.writeInline(negativeIntLiteralNode.Minus) + f.writeInline(negativeIntLiteralNode.Uint) +} + +func (f *formatter) writeRaw(n ast.Node) { + info := f.nodeInfo(n) + f.WriteString(info.RawText()) +} + +// writeNegativeIntLiteralForArray writes a negative int literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeNegativeIntLiteralForArray( + negativeIntLiteralNode *ast.NegativeIntLiteralNode, + lastElement bool, +) { + f.writeStart(negativeIntLiteralNode.Minus) + if lastElement { + f.writeLineEnd(negativeIntLiteralNode.Uint) + return + } + f.writeInline(negativeIntLiteralNode.Uint) +} + +// writeIdent writes an identifier (e.g. 'foo'). +func (f *formatter) writeIdent(identNode *ast.IdentNode) { + f.WriteString(identNode.Val) +} + +// writeKeyword writes a keyword (e.g. 'syntax'). +func (f *formatter) writeKeyword(keywordNode *ast.KeywordNode) { + f.WriteString(keywordNode.Val) +} + +// writeRune writes a rune (e.g. '='). +func (f *formatter) writeRune(runeNode *ast.RuneNode) { + if strings.ContainsRune("{[(<", runeNode.Rune) { + f.pendingIndent++ + } else if strings.ContainsRune("}])>", runeNode.Rune) { + f.pendingIndent-- + } + f.WriteString(string(runeNode.Rune)) +} + +// writeNodes writes nodes with sorted options. +func (f *formatter) writeNodes(nodes []ast.Node) { + optionNodes := []*ast.OptionNode{} + for _, node := range nodes { + if option, ok := node.(*ast.OptionNode); ok { + optionNodes = append(optionNodes, option) + } + } + + sort.Slice(optionNodes, func(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(optionNodes[i].Name) + right := stringForOptionName(optionNodes[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right + }) + + for _, node := range optionNodes { + f.writeNode(node) + } + + for _, node := range nodes { + if _, ok := node.(*ast.OptionNode); !ok { + f.writeNode(node) + } + } +} + +// writeNode writes the node by dispatching to a function tailored to its concrete type. +// +// Comments are handled in each respective write function so that it can determine whether +// to write the comments in-line or not. +func (f *formatter) writeNode(node ast.Node) { + switch element := node.(type) { + case *ast.ArrayLiteralNode: + f.writeArrayLiteral(element) + case *ast.CompactOptionsNode: + f.writeCompactOptions(element) + case *ast.CompoundIdentNode: + f.writeCompoundIdent(element) + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralIndent(element) + case *ast.EnumNode: + f.writeEnum(element) + case *ast.EnumValueNode: + f.writeEnumValue(element) + case *ast.ExtendNode: + f.writeExtend(element) + case *ast.ExtensionRangeNode: + f.writeExtensionRange(element) + case ast.FieldLabel: + f.writeFieldLabel(element) + case *ast.FieldNode: + f.writeField(element) + case *ast.FieldReferenceNode: + f.writeFieldReference(element) + case *ast.FloatLiteralNode: + f.writeFloatLiteral(element) + case *ast.GroupNode: + f.writeGroup(element) + case *ast.IdentNode: + f.writeIdent(element) + case *ast.ImportNode: + f.writeImport(element, false) + case *ast.KeywordNode: + f.writeKeyword(element) + case *ast.MapFieldNode: + f.writeMapField(element) + case *ast.MapTypeNode: + f.writeMapType(element) + case *ast.MessageNode: + f.writeMessage(element) + case *ast.MessageFieldNode: + f.writeMessageField(element) + case *ast.MessageLiteralNode: + f.writeMessageLiteral(element) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteral(element) + case *ast.OneofNode: + f.writeOneOf(element) + case *ast.OptionNode: + f.writeOption(element) + case *ast.OptionNameNode: + f.writeOptionName(element) + case *ast.PackageNode: + f.writePackage(element) + case *ast.RangeNode: + f.writeRange(element) + case *ast.ReservedNode: + f.writeReserved(element) + case *ast.RPCNode: + f.writeRPC(element) + case *ast.RPCTypeNode: + f.writeRPCType(element) + case *ast.RuneNode: + f.writeRune(element) + case *ast.ServiceNode: + f.writeService(element) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteral(element) + case *ast.SpecialFloatLiteralNode: + f.writeSpecialFloatLiteral(element) + case *ast.StringLiteralNode: + f.writeStringLiteral(element) + case *ast.SyntaxNode: + f.writeSyntax(element) + case *ast.UintLiteralNode: + f.writeUintLiteral(element) + case *ast.EmptyDeclNode: + // Nothing to do here. + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected node: %T", node)) + } +} + +// writeStart writes the node across as the start of a line. +// Start nodes have their leading comments written across +// multiple lines, but their trailing comments must be written +// in-line to preserve the line structure. +// +// For example, +// +// // Leading comment on 'message'. +// // Spread across multiple lines. +// message /* This is a trailing comment on 'message' */ Foo {} +// +// Newlines are preserved, so that any logical grouping of elements +// is maintained in the formatted result. +// +// For example, +// +// // Type represents a set of different types. +// enum Type { +// // Unspecified is the naming convention for default enum values. +// TYPE_UNSPECIFIED = 0; +// +// // The following elements are the real values. +// TYPE_ONE = 1; +// TYPE_TWO = 2; +// } +// +// Start nodes are always indented according to the formatter's +// current level of indentation (e.g. nested messages, fields, etc). +// +// Note that this is one of the most complex component of the formatter - it +// controls how each node should be separated from one another and preserves +// newlines in the original source. +func (f *formatter) writeStart(node ast.Node) { + f.writeStartMaybeCompact(node, true) +} + +func (f *formatter) writeStartMaybeCompact(node ast.Node, forceCompact bool) { + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + var ( + nodeNewlineCount = newlineCount(info.LeadingWhitespace()) + compact = forceCompact || isOpenBrace(f.previousNode) + ) + if length := info.LeadingComments().Len(); length > 0 { + // If leading comments are defined, the whitespace we care about + // is attached to the first comment. + f.writeMultilineCommentsMaybeCompact(info.LeadingComments(), forceCompact) + if !forceCompact && nodeNewlineCount > 1 { + // At this point, we're looking at the lines between + // a comment and the node its attached to. + // + // If the last comment is a standard comment, a single newline + // character is sufficient to warrant a separation of the + // two. + // + // If the last comment is a C-style comment, multiple newline + // characters are required because C-style comments don't consume + // a newline. + f.P("") + } + } else if !compact && nodeNewlineCount > 1 { + // If the previous node is an open brace, this is the first element + // in the body of a composite type, so we don't want to write a + // newline. This makes it so that trailing newlines are removed. + // + // For example, + // + // message Foo { + // + // string bar = 1; + // } + // + // Is formatted into the following: + // + // message Foo { + // string bar = 1; + // } + f.P("") + } + f.Indent(node) + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeInline writes the node and its surrounding comments in-line. +// +// This is useful for writing individual nodes like keywords, runes, +// string literals, etc. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = /* This is a leading comment on 'proto3' */" proto3"; +func (f *formatter) writeInline(node ast.Node) { + f.inline = true + defer func() { + f.inline = false + }() + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.writeInlineComments(info.TrailingComments()) +} + +// writeBodyEnd writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and preserve their format. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes a +// composite node: ']', '}', '>', etc. +// +// For example, +// +// message Foo { +// string bar = 1; +// // Leading comment on '}'. +// } // Trailing comment on '}. +func (f *formatter) writeBodyEnd(node ast.Node, leadingEndline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingEndline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + f.writeTrailingEndComments(info.TrailingComments()) +} + +func (f *formatter) writeLineElement(node ast.Node) { + f.writeBodyEnd(node, false) +} + +// writeBodyEndInline writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and adapt their comment style if they +// exist. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes either +// compact options or an array literal. +// +// This is behaviorally similar to f.writeStart, but it ignores +// the preceding newline logic because these body ends should +// always be compact. +// +// For example, +// +// message Foo { +// string bar = 1 [ +// deprecated = true +// +// // Leading comment on ']'. +// ] /* Trailing comment on ']' */ ; +// } +func (f *formatter) writeBodyEndInline(node ast.Node, leadingInline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingInline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeLineEnd writes the node so that it ends a line. +// +// This is useful for writing individual nodes like ';' and other +// tokens that conclude the end of a single line. In this case, we +// don't want to transform the trailing comment's from '//' to C-style +// because it's not necessary. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = " proto3" /* This is a leading comment on the ';'; // This is a trailing comment on the ';'. +func (f *formatter) writeLineEnd(node ast.Node) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.Space() + f.writeTrailingEndComments(info.TrailingComments()) +} + +// writeMultilineComments writes the given comments as a newline-delimited block. +// This is useful for both the beginning of a type (e.g. message, field, etc), as +// well as the trailing comments attached to the beginning of a body block (e.g. +// '{', '[', '<', etc). +// +// For example, +// +// // This is a comment spread across +// // multiple lines. +// message Foo {} +func (f *formatter) writeMultilineComments(comments ast.Comments) { + f.writeMultilineCommentsMaybeCompact(comments, false) +} + +func (f *formatter) writeMultilineCommentsMaybeCompact(comments ast.Comments, forceCompact bool) { + compact := forceCompact || isOpenBrace(f.previousNode) + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if !compact && newlineCount(comment.LeadingWhitespace()) > 1 { + // Newlines between blocks of comments should be preserved. + // + // For example, + // + // // This is a license header + // // spread across multiple lines. + // + // // Package pet.v1 defines a PetStore API. + // package pet.v1; + // + f.P("") + } + compact = false + f.writeComment(comment.RawText()) + f.WriteString("\n") + } +} + +// writeInlineComments writes the given comments in-line. Standard comments are +// transformed to C-style comments so that we can safely write the comment in-line. +// +// Nearly all of these comments will already be C-style comments. The only cases we're +// preventing are when the type is defined across multiple lines. +// +// For example, given the following: +// +// extend . google. // in-line comment +// protobuf . +// ExtensionRangeOptions { +// optional string label = 20000; +// } +// +// The formatted result is shown below: +// +// extend .google.protobuf./* in-line comment */ExtensionRangeOptions { +// optional string label = 20000; +// } +func (f *formatter) writeInlineComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + if i > 0 || comments.Index(i).LeadingWhitespace() != "" || f.lastWritten == ';' || f.lastWritten == '}' { + f.Space() + } + text := comments.Index(i).RawText() + if strings.HasPrefix(text, "//") { + text = strings.TrimSpace(strings.TrimPrefix(text, "//")) + text = "/* " + text + " */" + } else { + // no multi-line comments + lines := strings.Split(text, "\n") + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + text = strings.Join(lines, " ") + } + f.WriteString(text) + } +} + +// writeTrailingEndComments writes the given comments at the end of a line and +// preserves the comment style. This is useful or writing comments attached to +// things like ';' and other tokens that conclude a type definition on a single +// line. +// +// If there is a newline between this trailing comment and the previous node, the +// comments are written immediately underneath the node on a newline. +// +// For example, +// +// enum Type { +// TYPE_UNSPECIFIED = 0; +// } +// // This comment is attached to the '}' +// // So is this one. +func (f *formatter) writeTrailingEndComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if i > 0 || comment.LeadingWhitespace() != "" { + f.Space() + } + f.writeComment(comment.RawText()) + } + f.P("") +} + +func (f *formatter) writeComment(comment string) { + if strings.HasPrefix(comment, "/*") && newlineCount(comment) > 0 { + lines := strings.Split(comment, "\n") + // find minimum indent, so we can make all other lines relative to that + minIndent := -1 // sentinel that means unset + // start at 1 because line at index zero starts with "/*", not whitespace + var prefix string + for i := 1; i < len(lines); i++ { + indent, ok := computeIndent(lines[i]) + if ok && (minIndent == -1 || indent < minIndent) { + minIndent = indent + } + if i > 1 && len(prefix) == 0 { + // no shared prefix + continue + } + line := strings.TrimSpace(lines[i]) + if line == "*/" { + continue + } + var linePrefix string + if len(line) > 0 && isCommentPrefix(line[0]) { + linePrefix = line[:1] + } + if i == 1 { + prefix = linePrefix + } else if linePrefix != prefix { + // they do not share prefix + prefix = "" + } + } + if minIndent < 0 { + // This shouldn't be necessary. + // But we do it just in case, to avoid possible panic + minIndent = 0 + } + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" || trimmedLine == "*/" || len(prefix) > 0 { + line = trimmedLine + } else { + // we only trim space from the right; for the left, + // we unindent based on indentation found above. + line = unindent(line, minIndent) + line = strings.TrimRightFunc(line, unicode.IsSpace) + } + // If we have a block comment with no prefix, we'll format + // like so: + + /* + This is a multi-line comment example. + It has no comment prefix on each line. + */ + + // But if there IS a prefix, "|" for example, we'll left-align + // the prefix symbol under the asterisk of the comment start + // like this: + + /* + | This comment has a prefix before each line. + | Usually the prefix is asterisk, but it's a + | pipe in this example. + */ + + // Finally, if the comment prefix is an asterisk, we'll left-align + // the comment end so its asterisk also aligns, like so: + + /* + * This comment has a prefix before each line. + * Usually the prefix is asterisk, which is the + * case in this example. + */ + + if i > 0 && line != "*/" { + if len(prefix) == 0 { + line = " " + line + } else { + line = " " + line + } + } + if line == "*/" && prefix == "*" { + // align the comment end with the other asterisks + line = " " + line + } + + if i != len(lines)-1 { + f.P(line) + } else { + // for last line, we don't use P because we don't + // want to print a trailing newline + f.Indent(nil) + f.WriteString(line) + } + } + } else { + f.Indent(nil) + f.WriteString(strings.TrimSpace(comment)) + } +} + +func isCommentPrefix(ch byte) bool { + r := rune(ch) + // A multi-line comment prefix is *usually* an asterisk, like in the following + /* + * Foo + * Bar + * Baz + */ + // But we'll allow other prefixes. But if it's a letter or number, it's not a prefix. + return !unicode.IsLetter(r) && !unicode.IsNumber(r) +} + +func unindent(s string, unindent int) string { + pos := 0 + for i, r := range s { + if pos == unindent { + return s[i:] + } + if pos > unindent { + // removing tab-stop unindented too far, so we + // add back some spaces to compensate + return strings.Repeat(" ", pos-unindent) + s[i:] + } + + switch r { + case ' ': + pos++ + case '\t': + // jump to next tab stop + pos += 8 - (pos % 8) + default: + return s[i:] + } + } + // nothing but whitespace... + return "" +} + +func computeIndent(s string) (int, bool) { + if strings.TrimSpace(s) == "*/" { + return 0, false + } + indent := 0 + for _, r := range s { + switch r { + case ' ': + indent++ + case '\t': + // jump to next tab stop + indent += 8 - (indent % 8) + default: + return indent, true + } + } + // if we get here, line is nothing but whitespace + return 0, false +} + +func (f *formatter) leadingCommentsContainBlankLine(n ast.Node) bool { + info := f.nodeInfo(n) + comments := info.LeadingComments() + for i := 0; i < comments.Len(); i++ { + if newlineCount(comments.Index(i).LeadingWhitespace()) > 1 { + return true + } + } + return newlineCount(info.LeadingWhitespace()) > 1 +} + +func (f *formatter) importHasComment(importNode *ast.ImportNode) bool { + if f.nodeHasComment(importNode) { + return true + } + if importNode == nil { + return false + } + + return f.nodeHasComment(importNode.Keyword) || + f.nodeHasComment(importNode.Name) || + f.nodeHasComment(importNode.Semicolon) || + f.nodeHasComment(importNode.Public) || + f.nodeHasComment(importNode.Weak) +} + +func (f *formatter) nodeHasComment(node ast.Node) bool { + // when node != nil, node's value could be nil, see: https://go.dev/doc/faq#nil_error + if node == nil || reflect.ValueOf(node).IsNil() { + return false + } + + nodeinfo := f.nodeInfo(node) + return nodeinfo.LeadingComments().Len() > 0 || + nodeinfo.TrailingComments().Len() > 0 +} + +func (f *formatter) setTrailingComments(node ast.Node, comments ast.Comments) { + f.overrideTrailingComments[node] = comments +} + +func (f *formatter) nodeInfo(node ast.Node) nodeInfo { + info := f.fileNode.NodeInfo(node) + if trailingComments, ok := f.overrideTrailingComments[node]; ok { + return infoWithTrailingComments{info, trailingComments} + } + return info +} + +type nodeInfo interface { + Start() ast.SourcePos + End() ast.SourcePos + LeadingComments() ast.Comments + TrailingComments() ast.Comments + LeadingWhitespace() string + RawText() string +} + +type infoWithTrailingComments struct { + ast.NodeInfo + trailing ast.Comments +} + +func (n infoWithTrailingComments) TrailingComments() ast.Comments { + return n.trailing +} + +// importSortOrder maps import types to a sort order number, so it can be compared and sorted. +// `import`=3, `import public`=2, `import weak`=1 +func importSortOrder(node *ast.ImportNode) int { + switch { + case node.Public != nil: + return 2 + case node.Weak != nil: + return 1 + default: + return 3 + } +} + +// stringForOptionName returns the string representation of the given option name node. +// This is used for sorting file-level options. +func stringForOptionName(optionNameNode *ast.OptionNameNode) string { + var result string + for j, part := range optionNameNode.Parts { + if j > 0 { + // Add a dot between each of the parts. + result += "." + } + result += stringForFieldReference(part) + } + return result +} + +// stringForFieldReference returns the string representation of the given field reference. +// This is used for sorting file-level options. +func stringForFieldReference(fieldReference *ast.FieldReferenceNode) string { + var result string + if fieldReference.Open != nil { + result += "(" + } + result += string(fieldReference.Name.AsIdentifier()) + if fieldReference.Close != nil { + result += ")" + } + return result +} + +// isOpenBrace returns true if the given node represents one of the +// possible open brace tokens, namely '{', '[', or '<'. +func isOpenBrace(node ast.Node) bool { + if node == nil { + return false + } + runeNode, ok := node.(*ast.RuneNode) + if !ok { + return false + } + return runeNode.Rune == '{' || runeNode.Rune == '[' || runeNode.Rune == '<' +} + +// newlineCount returns the number of newlines in the given value. +// This is useful for determining whether or not we should preserve +// the newline between nodes. +// +// The newlines don't need to be adjacent to each other - all of the +// tokens between them are other whitespace characters, so we can +// safely ignore them. +func newlineCount(value string) int { + return strings.Count(value, "\n") +} + +func messageLiteralOpen(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Open + if node.Rune == '{' { + return node + } + // If it's not "{" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "{". + return ast.NewRuneNode('{', node.Token()) +} + +func messageLiteralClose(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Close + if node.Rune == '}' { + return node + } + // If it's not "}" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "}". + return ast.NewRuneNode('}', node.Token()) +} diff --git a/go/protopace/formatter_test.go b/go/protopace/formatter_test.go new file mode 100644 index 000000000..4ec402c2f --- /dev/null +++ b/go/protopace/formatter_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestFormat(t *testing.T) { + assert := assert.New(t) + + data, _ := os.ReadFile("./fixtures/dependency.proto") + dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil) + assert.NoError(err) + assert.NotNil(dependencySchema) + + data, _ = os.ReadFile("./fixtures/test.proto") + testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(testSchema) + + _, err = Format(*testSchema) + assert.NoError(err) +} diff --git a/go/protopace/go.mod b/go/protopace/go.mod new file mode 100644 index 000000000..b24f9651a --- /dev/null +++ b/go/protopace/go.mod @@ -0,0 +1,40 @@ +module github.com/Aiven-Open/karapace/go/protopace + +go 1.22 + +replace github.com/bufbuild/buf => github.com/keejon/buf v0.0.0-20240709110257-34d8b868af21 + +require ( + github.com/bufbuild/buf v1.34.0 + github.com/bufbuild/protocompile v0.14.0 + github.com/gofrs/uuid/v5 v5.2.0 + github.com/stretchr/testify v1.9.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee // indirect + github.com/bufbuild/protovalidate-go v0.6.3 // indirect + github.com/bufbuild/protoyaml-go v0.1.9 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/cel-go v0.20.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240708141625-4ad9e859172b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go/protopace/go.sum b/go/protopace/go.sum new file mode 100644 index 000000000..a7959b131 --- /dev/null +++ b/go/protopace/go.sum @@ -0,0 +1,83 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 h1:cFrEG/pJch6t62+jqndcPXeTNkYcztS4tBRgNkR+drw= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/bufbuild/protocompile v0.14.0 h1:z3DW4IvXE5G/uTOnSQn+qwQQxvhckkTWLS/0No/o7KU= +github.com/bufbuild/protocompile v0.14.0/go.mod h1:N6J1NYzkspJo3ZwyL4Xjvli86XOj1xq4qAasUFxGups= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee h1:E6ET8YUcYJ1lAe6ctR3as7yqzW2BNItDFnaB5zQq/8M= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee/go.mod h1:HjGFxsck9RObrTJp2hXQZfWhPgZqnR6sR1U5fCA/Kus= +github.com/bufbuild/protovalidate-go v0.6.3 h1:wxQyzW035zM16Binbaz/nWAzS12dRIXhZdSUWRY7Fv0= +github.com/bufbuild/protovalidate-go v0.6.3/go.mod h1:J4PtwP9Z2YAGgB0+o+tTWEDtLtXvz/gfhFZD8pbzM/U= +github.com/bufbuild/protoyaml-go v0.1.9 h1:anV5UtF1Mlvkkgp4NWA6U/zOnJFng8Orq4Vf3ZUQHBU= +github.com/bufbuild/protoyaml-go v0.1.9/go.mod h1:KCBItkvZOK/zwGueLdH1Wx1RLyFn5rCH7YjQrdty2Wc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid/v5 v5.2.0 h1:qw1GMx6/y8vhVsx626ImfKMuS5CvJmhIKKtuyvfajMM= +github.com/gofrs/uuid/v5 v5.2.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/keejon/buf v0.0.0-20240709110257-34d8b868af21 h1:9vFdXQGe/i72nN0UAF4gx3TlGuUzHouayOYB1WLW8C0= +github.com/keejon/buf v0.0.0-20240709110257-34d8b868af21/go.mod h1:hgAhSorSpAZGiF0Sqgqc9nhKsrq2bZn8A1mdeSNsVSk= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/genproto/googleapis/api v0.0.0-20240708141625-4ad9e859172b h1:y/kpOWeX2pWERnbsvh/hF+Zmo69wVmjyZhstreXQQeA= +google.golang.org/genproto/googleapis/api v0.0.0-20240708141625-4ad9e859172b/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b h1:04+jVzTs2XBnOZcPsLnmrTGqltqJbZQ1Ey26hjYdQQ0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/protopace/main.go b/go/protopace/main.go new file mode 100644 index 000000000..b39c86fbc --- /dev/null +++ b/go/protopace/main.go @@ -0,0 +1,88 @@ +package main + +/* +#include + +struct result { + char* res; + char* err; +}; + +*/ +import "C" +import ( + "fmt" + "unsafe" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" +) + +var all string = "" + +//export SayHello +func SayHello(name *C.char) { + all = all + " " + C.GoString(name) + fmt.Printf("Hello, %s!\n", all) +} + +func result(schema string, err error) *C.struct_result { + res := (*C.struct_result)(C.malloc(C.size_t(unsafe.Sizeof(C.struct_result{})))) + res.res = C.CString(schema) + if err != nil { + res.err = C.CString(err.Error()) + } + return res +} + +func createSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) (*s.Schema, error) { + depArray := unsafe.Slice(cDependencies, depsLenght) + depNamesArray := unsafe.Slice(cDependencyNames, depsLenght) + dependencies := []s.Schema{} + for i, dep := range depArray { + dependency, err := s.FromString(C.GoString(depNamesArray[i]), C.GoString(dep), []s.Schema{}) + if err != nil { + return nil, err + } + dependencies = append(dependencies, *dependency) + } + + schema, err := s.FromString(C.GoString(cSchemaName), C.GoString(cSchema), dependencies) + return schema, err +} + +//export FormatSchema +func FormatSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) *C.struct_result { + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return result("", err) + } + + res, err := Format(*schema) + if err != nil { + return result("", err) + } + return result(res.Schema, err) +} + +//export CheckCompatibility +func CheckCompatibility( + cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int, + cSchemaNamePrev *C.char, cSchemaPrev *C.char, cDependencyNamesPrev **C.char, cDependenciesPrev **C.char, depsLenghtPrev C.int) *C.char { + + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return C.CString(err.Error()) + } + prevSchema, err := createSchema(cSchemaNamePrev, cSchemaPrev, cDependencyNamesPrev, cDependenciesPrev, depsLenghtPrev) + if err != nil { + return C.CString(err.Error()) + } + + err = Check(*schema, *prevSchema) + if err != nil { + return C.CString(err.Error()) + } + return nil +} + +func main() {} diff --git a/go/protopace/schema/compiler.go b/go/protopace/schema/compiler.go new file mode 100644 index 000000000..5311d022c --- /dev/null +++ b/go/protopace/schema/compiler.go @@ -0,0 +1,10 @@ +package schema + +import ( + "github.com/bufbuild/protocompile" +) + +func NewCompiler(resolver protocompile.Resolver) *protocompile.Compiler { + compiler := protocompile.Compiler{Resolver: resolver, RetainASTs: true} + return &compiler +} diff --git a/go/protopace/schema/resolver.go b/go/protopace/schema/resolver.go new file mode 100644 index 000000000..1d76c4f92 --- /dev/null +++ b/go/protopace/schema/resolver.go @@ -0,0 +1,39 @@ +package schema + +import ( + "fmt" + "strings" + + "github.com/bufbuild/protocompile" +) + +type SchemaResolver struct { + schemas map[string]Schema +} + +func NewSchemaResolver(schemas []Schema) protocompile.Resolver { + schemasIndex := map[string]Schema{} + for _, schema := range schemas { + schemasIndex[schema.Name] = schema + } + resolver := SchemaResolver{schemas: schemasIndex} + return protocompile.WithStandardImports(&resolver) +} + +func (s *SchemaResolver) AddSchema(schema Schema) { + s.schemas[schema.Name] = schema +} + +// FindFileByPath implements protocompile.Resolver. +func (s *SchemaResolver) FindFileByPath(path string) (protocompile.SearchResult, error) { + searchResult := protocompile.SearchResult{} + schema, ok := s.schemas[path] + if !ok { + return searchResult, fmt.Errorf("schema not found: %s", path) + } + searchResult.Source = strings.NewReader(schema.Schema) + searchResult.ParseResult = schema.ParserResult + return searchResult, nil +} + +var _ protocompile.Resolver = (*SchemaResolver)(nil) diff --git a/go/protopace/schema/schema.go b/go/protopace/schema/schema.go new file mode 100644 index 000000000..5fb8eaddb --- /dev/null +++ b/go/protopace/schema/schema.go @@ -0,0 +1,69 @@ +package schema + +import ( + "context" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/protocompile/linker" + "github.com/bufbuild/protocompile/parser" + "github.com/bufbuild/protocompile/reporter" + "github.com/gofrs/uuid/v5" +) + +var ( + handler = reporter.NewHandler(nil) +) + +type Schema struct { + Schema string + Name string + ParserResult parser.Result + Dependencies []Schema +} + +func FromString(name string, proto string, dependencies []Schema) (*Schema, error) { + fileNode, err := parser.Parse(name, strings.NewReader(proto), handler) + if err != nil { + return nil, err + } + result, err := parser.ResultFromAST(fileNode, true, handler) + if err != nil { + return nil, err + } + return &Schema{Schema: proto, Name: name, ParserResult: result, Dependencies: dependencies}, nil +} + +func (s Schema) Compile() (linker.Result, error) { + resolver := NewSchemaResolver(append(s.Dependencies, s)) + compiler := NewCompiler(resolver) + ctx := context.Background() + files, err := compiler.Compile(ctx, s.Name) + if err != nil { + return nil, err + } + res := files[0].(linker.Result) + return res, nil +} + +func (s Schema) CompileBufImage() (bufimage.Image, error) { + res, err := s.Compile() + if err != nil { + return nil, err + } + imageFile, err := bufimage.NewImageFile( + res.FileDescriptorProto(), + nil, + uuid.Nil, + "", + "", + false, + false, + nil, + ) + if err != nil { + return nil, err + } + image, err := bufimage.NewImage([]bufimage.ImageFile{imageFile}) + return image, err +} diff --git a/karapace/protobuf/protopace/__init__.py b/karapace/protobuf/protopace/__init__.py new file mode 100644 index 000000000..3d1ce1a24 --- /dev/null +++ b/karapace/protobuf/protopace/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from .protopace import check_compatibility, format_proto, Proto # noqa: F401 diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-amd64.h b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-amd64.so b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.so new file mode 100644 index 000000000..fe3c0221a Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.so differ diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-arm64.h b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-arm64.so b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.so new file mode 100644 index 000000000..272806f7e Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.so differ diff --git a/karapace/protobuf/protopace/bin/protopace-linux-amd64.h b/karapace/protobuf/protopace/bin/protopace-linux-amd64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-linux-amd64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-linux-amd64.so b/karapace/protobuf/protopace/bin/protopace-linux-amd64.so new file mode 100644 index 000000000..ef5bcdea6 Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-linux-amd64.so differ diff --git a/karapace/protobuf/protopace/bin/protopace-linux-arm64.h b/karapace/protobuf/protopace/bin/protopace-linux-arm64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-linux-arm64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-linux-arm64.so b/karapace/protobuf/protopace/bin/protopace-linux-arm64.so new file mode 100644 index 000000000..75a61342a Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-linux-arm64.so differ diff --git a/karapace/protobuf/protopace/protopace.py b/karapace/protobuf/protopace/protopace.py new file mode 100644 index 000000000..6ccb22fa2 --- /dev/null +++ b/karapace/protobuf/protopace/protopace.py @@ -0,0 +1,229 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from dataclasses import dataclass, field +from typing import List + +import ctypes +import os +import platform +import timeit + +system = platform.system().lower() +arch = platform.machine().lower() +file_dir = os.path.dirname(os.path.abspath(__file__)) + +if arch == "x86_64": + arch = "amd64" + +lib_file = os.path.join(file_dir, f"bin/protopace-{system}-{arch}.so") + +if not os.path.exists(lib_file): + raise RuntimeError(f"Unsupported platform: {system}-{arch}") + +lib = ctypes.CDLL(lib_file) + +lib.FormatSchema.restype = ctypes.c_void_p +lib.CheckCompatibility.restype = ctypes.c_char_p + + +class FormatResult(ctypes.Structure): + _fields_ = [ + ("res", ctypes.c_char_p), + ("err", ctypes.c_char_p), + ] + + +@dataclass +class Proto: + name: str + schema: str + dependencies: List["Proto"] = field(default_factory=list) + + +class CompileError(Exception): + pass + + +class IncompatibleError(Exception): + pass + + +def format_proto(proto: Proto) -> str: + length = len(proto.dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.dependencies]) + + res_ptr = lib.FormatSchema(proto.name.encode(), proto.schema.encode(), c_dependency_names, c_dependencies, length) + res = FormatResult.from_address(res_ptr) + + if res.err: + err = res.err + msg = err.decode() + lib.free(ctypes.c_void_p(res_ptr)) + raise CompileError(msg) + + result = res.res.decode() + lib.free(ctypes.c_void_p(res_ptr)) + return result + + +def check_compatibility(proto: Proto, prev_proto: Proto) -> None: + length = len(proto.dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.dependencies]) + + prev_length = len(proto.dependencies) + prev_c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in prev_proto.dependencies]) + prev_c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in prev_proto.dependencies]) + + err = lib.CheckCompatibility( + proto.name.encode(), + proto.schema.encode(), + c_dependency_names, + c_dependencies, + length, + prev_proto.name.encode(), + prev_proto.schema.encode(), + prev_c_dependency_names, + prev_c_dependencies, + prev_length, + ) + + if err is not None: + msg = err.decode() + raise IncompatibleError(msg) + + +SCHEMA = """ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; +} +""" + +PREV_SCHEMA = """ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 5; +} +""" + +DEPENDENCY = """ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} +""" + + +def _format_time(seconds: float) -> None: + units = [("s", 1), ("ms", 1e-3), ("µs", 1e-6), ("ns", 1e-9)] + for unit, factor in units: + if seconds >= factor: + return f"{seconds / factor:.3f} {unit}" + return f"{seconds:.3f} s" + + +def _test_format() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + res = format_proto(proto) + print(res) + + +def _test_check_compatibility() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + prev_proto = Proto("test.proto", PREV_SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + try: + check_compatibility(proto, prev_proto) + except IncompatibleError as err: + print(err) + + +def _time_format() -> None: + def test(): + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + format_proto(proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Format -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +def _time_check_compatibility() -> None: + def test(): + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + check_compatibility(proto, proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Compatibility Check -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +if __name__ == "__main__": + _test_format() + _test_check_compatibility() + _time_format() + _time_check_compatibility() diff --git a/setup.py b/setup.py index 3b50270d1..fa8f24d3b 100644 --- a/setup.py +++ b/setup.py @@ -70,4 +70,5 @@ "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries", ], + include_package_data=True, ) diff --git a/tests/unit/protobuf/test_protobuf_compatibility.py b/tests/unit/protobuf/test_protobuf_compatibility.py index eaf9b98a5..eb3df5542 100644 --- a/tests/unit/protobuf/test_protobuf_compatibility.py +++ b/tests/unit/protobuf/test_protobuf_compatibility.py @@ -7,6 +7,9 @@ from karapace.protobuf.location import Location from karapace.protobuf.proto_file_element import ProtoFileElement from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.protopace import check_compatibility, Proto + +import pytest location: Location = Location("some/folder", "file.proto") @@ -45,6 +48,39 @@ def test_compatibility_package(): assert result.is_compatible() +def test_compatibility_package_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a2; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a2.TestMessage.Value val = 2; + |} + |""" + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_field_add(): self_schema = """ |syntax = "proto3"; @@ -80,6 +116,39 @@ def test_compatibility_field_add(): assert result.is_compatible() +def test_compatibility_field_add_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | string str2 = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + check_compatibility(proto, prev_proto) + + def test_compatibility_field_drop(): self_schema = """ |syntax = "proto3"; @@ -115,6 +184,41 @@ def test_compatibility_field_drop(): assert result.is_compatible() +def test_compatibility_field_drop_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | string str2 = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_field_add_drop(): self_schema = """ |syntax = "proto3"; @@ -149,6 +253,40 @@ def test_compatibility_field_add_drop(): assert result.is_compatible() +def test_compatibility_field_add_drop_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str2 = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_enum_add(): self_schema = """ |syntax = "proto3"; @@ -190,6 +328,49 @@ def test_compatibility_enum_add(): assert result.is_compatible() +def test_compatibility_enum_add_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str2 = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str2 = 1; + | Enu x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + | enum Enu { + | A = 0; + | B = 1; + | } + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + # Note: This will be interpreted as a type change and will be regognized as breaking with vanilla buf + with pytest.raises(Exception) as e: + check_compatibility(proto, prev_proto) + assert 'Field "2" with name "x" on message "Value" changed type from "int32" to "enum".' in str(e) + + def test_compatibility_ordering_change_msg(): self_schema = """\ syntax = "proto3"; @@ -226,6 +407,39 @@ def test_compatibility_ordering_change_msg(): assert len(result.result) == 0 +def test_compatibility_ordering_change_msg_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + int32 fredfield = 1; +} + +message HodoCode { + int32 hodofield = 1; +} +""" + + other_schema = """\ +syntax = "proto3"; +package tc4; + +message HodoCode { + int32 hodofield = 1; +} + +message Fred { + int32 fredfield = 1; +} +""" + # Note: tag number must be > 0. Had to be changed from original test. + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_ordering_change(): self_schema = """\ syntax = "proto3"; @@ -264,6 +478,40 @@ def test_compatibility_ordering_change(): assert result.result[0].modification == Modification.FIELD_ADD +def test_compatibility_ordering_change_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + HodoCode hodecode = 1; +} + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} +""" + + other_schema = """\ +syntax = "proto3"; +package tc4; + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} + +message Fred { + HodoCode hodecode = 1; + string id = 2; +} +""" + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_ordering_change2(): self_schema = """\ syntax = "proto3"; @@ -302,6 +550,40 @@ def test_compatibility_ordering_change2(): assert result.result[0].modification == Modification.FIELD_ADD +def test_compatibility_ordering_change2_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + HodoCode hodecode = 1; +} + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} +""" + + other_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + HodoCode hodecode = 1; + string id = 2; +} + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} +""" + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_field_tag_change(): self_schema = """\ syntax = "proto3"; @@ -336,3 +618,33 @@ def test_compatibility_field_tag_change(): (Modification.FIELD_DROP, "Foo.4"), (Modification.FIELD_ADD, "Foo.5"), } + + +def test_compatibility_field_tag_change_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package pkg; +message Foo { + string fieldA = 1; + string fieldB = 2; + string fieldC = 3; + string fieldX = 4; +} +""" + + other_schema = """\ +syntax = "proto3"; +package pkg; +message Foo { + string fieldA = 1; + string fieldB = 2; + string fieldC = 3; + string fieldX = 5; +} +""" + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + # Note: because we allow field name change this is not recognized as incompatible + check_compatibility(proto, prev_proto) diff --git a/tests/unit/protobuf/test_protobuf_normalization.py b/tests/unit/protobuf/test_protobuf_normalization.py index b772b293c..d2d132cf8 100644 --- a/tests/unit/protobuf/test_protobuf_normalization.py +++ b/tests/unit/protobuf/test_protobuf_normalization.py @@ -6,6 +6,7 @@ from karapace.protobuf.location import Location from karapace.protobuf.proto_normalizations import normalize from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.protopace import check_compatibility, format_proto, Proto import pytest @@ -63,10 +64,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } """ @@ -75,10 +86,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } """ @@ -87,6 +108,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option) = "my_value"; option (my_option2) = "my_value2"; @@ -99,6 +128,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option3) = "my_value3"; option (my_option) = "my_value"; @@ -111,6 +148,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option) = "my_value"; @@ -125,6 +174,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option3) = "my_value3"; @@ -139,6 +200,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MessageOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1; } @@ -155,6 +224,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MessageOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1; } @@ -171,11 +248,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { oneof my_oneof { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + string test = 1; } } """ @@ -185,11 +272,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { oneof my_oneof { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + string test = 1; } } """ @@ -199,6 +296,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -209,6 +314,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -219,6 +332,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -229,6 +350,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -239,11 +368,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } } """ @@ -253,11 +392,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } } """ @@ -267,6 +416,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; @@ -279,6 +436,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; @@ -292,6 +457,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -306,6 +479,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -320,6 +501,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -334,6 +523,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -348,6 +545,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_generate_equals_and_hash = true; option java_generic_services = true; @@ -368,47 +609,45 @@ message NestedFoo { string fieldA = 1; - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; + option (my_option_Message) = "my_value"; + option (my_option2_Message) = "my_value2"; } - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Message) = "my_value3"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + option (my_option_Oneof) = "my_value"; + + string test = 5; } enum MyEnum { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; + option (my_option_Enum) = "my_value"; + + ACTIVE = 0; } } -extend Foo { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; -} - service MyService { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Service) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; rpc MyRpc (Foo) returns (Foo) { - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; + option (my_option_Method) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; } } @@ -420,6 +659,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_outer_classname = "FooProto"; option optimize_for = SPEED; @@ -440,45 +723,42 @@ message NestedFoo { string fieldA = 1; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; } - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option3_Message) = "my_value3"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; + option (my_option_Oneof) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + + string test = 5; } enum MyEnum { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - } -} - + option (my_option_Enum) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; -extend Foo { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + ACTIVE = 0; + } } service MyService { - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; + option (my_option3_Service) = "my_value3"; rpc MyRpc (Foo) returns (Foo) { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; + option (my_option_Method) = "my_value"; } } @@ -518,3 +798,39 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno normalize(ordered_proto).compare(normalize(unordered_proto), result) assert result.is_compatible() assert normalize(ordered_proto).to_schema() == normalize(unordered_proto).to_schema() + + +@pytest.mark.parametrize( + ("ordered_schema", "unordered_schema"), + ( + (PROTO_WITH_OPTIONS_ORDERED, PROTO_WITH_OPTIONS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_UNORDERED), + (PROTO_WITH_OPTIONS_IN_SERVICE_ORDERED, PROTO_WITH_OPTIONS_IN_SERVICE_UNORDERED), + (PROTO_WITH_OPTIONS_IN_RPC_ORDERED, PROTO_WITH_OPTIONS_IN_RPC_UNORDERED), + # Note: This does not work. Is it valid proto? + # (PROTO_WITH_OPTIONS_IN_EXTEND_ORDERED, PROTO_WITH_OPTIONS_IN_EXTEND_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED, PROTO_WITH_OPTIONS_IN_ONEOF_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_UNORDERED), + (PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_ORDERED, PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_COMPLEX_SCHEMA_ORDERED, PROTO_WITH_COMPLEX_SCHEMA_UNORDERED), + ), +) +def test_differently_ordered_options_normalizes_equally_with_protopace(ordered_schema: str, unordered_schema: str) -> None: + ordered_proto = Proto("test.proto", ordered_schema) + unordered_proto = Proto("test.proto", unordered_schema) + + ordered_str = format_proto(ordered_proto) + unordered_str = format_proto(unordered_proto) + + assert ordered_str == unordered_str + check_compatibility(ordered_proto, unordered_proto)