From 11b3468006dcf147fbb276fdd169068436fb36bf Mon Sep 17 00:00:00 2001 From: Wei Lim Date: Fri, 18 Oct 2024 14:08:39 -0700 Subject: [PATCH 1/2] yamldot: Dotted-path YAML manipulation Add functionality for YAML node manipulation using dotted-path syntax. This will be used in upcoming work that is aimed to manipulate YAML nodes. --- cli/azd/pkg/yamlnode/yamlnode.go | 203 +++++++++++++++++++++++++ cli/azd/pkg/yamlnode/yamlnode_test.go | 204 ++++++++++++++++++++++++++ 2 files changed, 407 insertions(+) create mode 100644 cli/azd/pkg/yamlnode/yamlnode.go create mode 100644 cli/azd/pkg/yamlnode/yamlnode_test.go diff --git a/cli/azd/pkg/yamlnode/yamlnode.go b/cli/azd/pkg/yamlnode/yamlnode.go new file mode 100644 index 00000000000..03ac28a8551 --- /dev/null +++ b/cli/azd/pkg/yamlnode/yamlnode.go @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// yamlnode allows for manipulation of YAML nodes using a dotted-path syntax. +// +// Examples of dotted-path syntax: +// - a.object.key +// - b.item_list[1] +package yamlnode + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/braydonk/yaml" +) + +var ErrNodeNotFound = errors.New("node not found") + +// ErrNodeWrongKind is returned when the node kind is not as expected. +// +// This error may be useful for nodes that have multiple possible kinds. +var ErrNodeWrongKind = errors.New("unexpected node kind") + +// Find retrieves a node at the given path. +func Find(root *yaml.Node, path string) (*yaml.Node, error) { + parts, err := parsePath(path) + if err != nil { + return nil, err + } + + found := find(root, parts) + if found == nil { + return nil, fmt.Errorf("%w: %s", ErrNodeNotFound, path) + } + + return found, nil +} + +// Set sets the node at the given path to the provided value. +func Set(root *yaml.Node, path string, value *yaml.Node) error { + parts, err := parsePath(path) + if err != nil { + return err + } + + // find the anchor node + anchor := find(root, parts[:len(parts)-1]) + if anchor == nil { + return fmt.Errorf("%w: %s", ErrNodeNotFound, path) + } + + // set the node + seek, isKey := parts[len(parts)-1].(string) + idx, isSequence := parts[len(parts)-1].(int) + + if isKey { + if anchor.Kind != yaml.MappingNode { + return fmt.Errorf("%w: %s is not a mapping node", ErrNodeWrongKind, parts[len(parts)-1]) + } + + for i := 0; i < len(anchor.Content); i += 2 { + if anchor.Content[i].Value == seek { + anchor.Content[i+1] = value + return nil + } + } + + anchor.Content = append(anchor.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: seek}) + anchor.Content = append(anchor.Content, value) + } else if isSequence { + if anchor.Kind != yaml.SequenceNode { + return fmt.Errorf("%w: %s is not a sequence node", ErrNodeWrongKind, parts[len(parts)-1]) + } + + if idx < 0 || idx > len(anchor.Content) { + return fmt.Errorf("array index out of bounds: %d", idx) + } + + anchor.Content[idx] = value + } + + return nil +} + +// Append appends a node to the sequence (array) node at the given path. +// +// If the node at the path is not a sequence node, ErrNodeWrongKind is returned. +func Append(root *yaml.Node, path string, node *yaml.Node) error { + parts, err := parsePath(path) + if err != nil { + return err + } + + // find the anchor node + found := find(root, parts) + if found == nil { + return fmt.Errorf("%w: %s", ErrNodeNotFound, path) + } + + if found.Kind != yaml.SequenceNode { + return fmt.Errorf("%w %d for append", ErrNodeWrongKind, found.Kind) + } + + found.Content = append(found.Content, node) + return nil +} + +// Encode encodes a value into a YAML node. +func Encode(value interface{}) (*yaml.Node, error) { + var node yaml.Node + err := node.Encode(value) + if err != nil { + return nil, fmt.Errorf("encoding yaml node: %w", err) + } + + return &node, nil +} + +func find(current *yaml.Node, parts []any) *yaml.Node { + if len(parts) == 0 { + // we automatically skip the document node to avoid having to specify it in the path + if current.Kind == yaml.DocumentNode { + return current.Content[0] + } + + return current + } + + seek, _ := parts[0].(string) + idx, isArray := parts[0].(int) + + switch current.Kind { + case yaml.DocumentNode: + // we automatically skip the document node to avoid having to specify it in the path + return find(current.Content[0], parts) + case yaml.MappingNode: + for i := 0; i < len(current.Content); i += 2 { + if current.Content[i].Value == seek { + return find(current.Content[i+1], parts[1:]) + } + } + case yaml.SequenceNode: + if isArray && idx < len(current.Content) { + return find(current.Content[idx], parts[1:]) + } + } + + return nil +} + +// parsePath parses a dotted path into a slice of parts, where each part is either a string or an integer. +// The integer parts represent array indexes, and the string parts represent keys. +func parsePath(path string) ([]any, error) { + if path == "" { + return nil, errors.New("empty path") + } + + // future: support escaping dots + parts := strings.Split(path, ".") + expanded, err := expandArrays(parts) + if err != nil { + return nil, err + } + + return expanded, nil +} + +// expandArrays expands array indexing into individual elements. +func expandArrays(parts []string) (expanded []any, err error) { + expanded = make([]interface{}, 0, len(parts)) + for _, s := range parts { + before, after := cutBrackets(s) + expanded = append(expanded, before) + + if len(after) > 0 { + content := after[1 : len(after)-1] + idx, err := strconv.Atoi(content) + if err != nil { + return nil, fmt.Errorf("invalid array index: %s in %s", content, after) + } + + expanded = append(expanded, idx) + } + } + + return expanded, nil +} + +// cutBrackets splits a string into two parts, before the brackets, and after the brackets. +func cutBrackets(s string) (before string, after string) { + if len(s) > 0 && s[len(s)-1] == ']' { // reverse check for faster exit + for i := len(s) - 1; i >= 0; i-- { + if s[i] == '[' { + return s[:i], s[i:] + } + } + } + + return s, "" +} diff --git a/cli/azd/pkg/yamlnode/yamlnode_test.go b/cli/azd/pkg/yamlnode/yamlnode_test.go new file mode 100644 index 00000000000..6aeed2cf6bd --- /dev/null +++ b/cli/azd/pkg/yamlnode/yamlnode_test.go @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package yamlnode + +import ( + "testing" + + "github.com/braydonk/yaml" +) + +const doc = ` +root: + nested: + key: value + array: + - item1 + - item2 + - item3 + empty: [] + mixedArray: + - stringItem + - nestedObj: + deepKey: deepValue + - nestedArr: + - item1 + - item2 +` + +func TestFind(t *testing.T) { + var root yaml.Node + err := yaml.Unmarshal([]byte(doc), &root) + if err != nil { + t.Fatalf("Failed to unmarshal YAML: %v", err) + } + + tests := []struct { + name string + path string + expected interface{} + wantErr bool + }{ + {"Simple path", "root.nested.key", "value", false}, + {"Array index", "root.array[1]", "item2", false}, + {"Nested array object", "root.mixedArray[1].nestedObj.deepKey", "deepValue", false}, + + {"Map", "root.nested", map[string]string{"key": "value"}, false}, + {"Array", "root.array", []string{"item1", "item2", "item3"}, false}, + + {"Non-existent path", "root.nonexistent", "", true}, + {"Invalid array index", "root.array[3]", "", true}, + {"Invalid path format", "root.array.[1]", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node, err := Find(&root, tt.path) + + if (err != nil) != tt.wantErr { + t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + assertNodeEquals(t, "Get()", node, tt.expected) + } + }) + } +} + +func TestSet(t *testing.T) { + tests := []struct { + name string + path string + value interface{} + wantErr bool + }{ + {"root", "root", "new_value", false}, + + {"Update", "root.nested.key", "new_value", false}, + {"Update object", "root.nested", map[string]string{"new_key": "new_value"}, false}, + {"Update array", "root.array[1]", "new_item2", false}, + {"Update nested array object", "root.mixedArray[1].nestedObj.deepKey", "new_deep_value", false}, + + {"Create", "root.nested.new_key", "brand_new", false}, + {"Create array", "root.new_array", []string{"first_item"}, false}, + {"Create object", "root.nested.new_object", map[string]string{"key": "value"}, false}, + {"Create nested array object", "root.mixedArray[1].nestedObj.newKey", "new_deep_value", false}, + + {"Invalid path", "root.nonexistent.key", "value", true}, + {"Invalid array index", "root.array[10]", "value", true}, + {"Invalid path format", "root.array.[1]", "value", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var root yaml.Node + err := yaml.Unmarshal([]byte(doc), &root) + if err != nil { + t.Fatalf("Failed to unmarshal YAML: %v", err) + } + + valueNode, err := Encode(tt.value) + if err != nil { + t.Fatalf("Failed to encode value: %v", err) + } + + err = Set(&root, tt.path, valueNode) + + if (err != nil) != tt.wantErr { + t.Errorf("Set() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + // Verify the set value + node, err := Find(&root, tt.path) + if err != nil { + t.Errorf("Failed to get set value: %v", err) + return + } + + assertNodeEquals(t, "Set()", node, tt.value) + } + }) + } +} + +func TestAppend(t *testing.T) { + tests := []struct { + name string + path string + value interface{} + wantErr bool + checkLen int // Expected length after append + }{ + {"Append to array", "root.array", "item4", false, 4}, + {"Append to empty array", "root.empty", "item1", false, 1}, + {"Append object to mixed array", "root.mixedArray", map[string]string{"key": "value"}, false, 4}, + {"Append to nested array", "root.mixedArray[2].nestedArr", "item3", false, 3}, + + {"Invalid path (not an array)", "root.nested.key", "invalid", true, 0}, + {"Non-existent path", "root.nonexistent", "value", true, 0}, + {"Invalid path format", "root.array.[1]", "invalid", true, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var root yaml.Node + err := yaml.Unmarshal([]byte(doc), &root) + if err != nil { + t.Fatalf("Failed to unmarshal YAML: %v", err) + } + + valueNode, err := Encode(tt.value) + if err != nil { + t.Fatalf("Failed to encode value: %v", err) + } + + err = Append(&root, tt.path, valueNode) + + if (err != nil) != tt.wantErr { + t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + // Verify the append operation + node, err := Find(&root, tt.path) + if err != nil { + t.Errorf("Failed to get appended value: %v", err) + return + } + if node.Kind != yaml.SequenceNode { + t.Errorf("Append() did not result in a sequence node at path %s", tt.path) + return + } + if len(node.Content) != tt.checkLen { + t.Errorf("Append() resulted in wrong length = %d, want %d", len(node.Content), tt.checkLen) + return + } + lastNode := node.Content[len(node.Content)-1] + assertNodeEquals(t, "Append()", lastNode, tt.value) + } + }) + } +} + +func assertNodeEquals(t *testing.T, funcName string, node *yaml.Node, expected interface{}) { + t.Helper() + wantStr, err := yaml.Marshal(expected) + if err != nil { + t.Fatalf("Failed to marshal expected: %v", err) + } + + gotStr, err := yaml.Marshal(node) + if err != nil { + t.Fatalf("Failed to marshal node: %v", err) + } + + if string(gotStr) != string(wantStr) { + t.Errorf("%s = %v, want %v", funcName, string(gotStr), string(wantStr)) + } +} From 2408013b15ef05b92915c20488d9d657b0399503 Mon Sep 17 00:00:00 2001 From: Wei Lim Date: Fri, 18 Oct 2024 15:49:27 -0700 Subject: [PATCH 2/2] update cspell --- cli/azd/.vscode/cspell-azd-dictionary.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/azd/.vscode/cspell-azd-dictionary.txt b/cli/azd/.vscode/cspell-azd-dictionary.txt index ae552f0dbed..7ec07bd1992 100644 --- a/cli/azd/.vscode/cspell-azd-dictionary.txt +++ b/cli/azd/.vscode/cspell-azd-dictionary.txt @@ -227,5 +227,6 @@ webfrontend westus2 wireinject yacspin +yamlnode ymlt zerr