From d0783263f5ea99bf77398b031529be4d85e8d7ea Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 18 Sep 2024 11:08:02 -0400 Subject: [PATCH 01/21] Modify lexer to support modes for string templates. --- runtime/parser/lexer/lexer.go | 15 ++++ runtime/parser/lexer/lexer_test.go | 114 +++++++++++++++++++++++++++++ runtime/parser/lexer/state.go | 10 +++ runtime/parser/lexer/tokentype.go | 3 + 4 files changed, 142 insertions(+) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 7b69245ce2..5e9caf2f79 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -49,6 +49,14 @@ type position struct { column int } +type LexerMode int + +const ( + NORMAL = iota + STR_IDENTIFIER + STR_EXPRESSION +) + type lexer struct { // memoryGauge is used for metering memory usage memoryGauge common.MemoryGauge @@ -74,6 +82,8 @@ type lexer struct { prev rune // canBackup indicates whether stepping back is allowed canBackup bool + // lexer mode is used for string templates + mode LexerMode } var _ TokenStream = &lexer{} @@ -414,6 +424,11 @@ func (l *lexer) scanString(quote rune) { l.backupOne() return } + case '$': + // string template, stop and set mode + l.backupOne() + l.mode = STR_IDENTIFIER + return } r = l.next() } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 51f8f53f34..4206edd22e 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -1014,6 +1014,120 @@ func TestLexString(t *testing.T) { ) }) + t.Run("valid, string template", func(t *testing.T) { + testLex(t, + `"$abc.length"`, + []token{ + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenStringTemplate, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + Source: `$`, + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + Source: `abc`, + }, + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 12, Offset: 12}, + }, + }, + Source: `.length"`, + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 13, Offset: 13}, + EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, + }, + }, + }, + }, + ) + }) + + t.Run("invalid, string template", func(t *testing.T) { + testLex(t, + `"$1"`, + []token{ + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenStringTemplate, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + Source: `$`, + }, + { + Token: Token{ + Type: TokenDecimalIntegerLiteral, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + Source: `1`, + }, + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + }, + }, + ) + }) + t.Run("invalid, empty, not terminated at line end", func(t *testing.T) { testLex(t, "\"\n", diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 4b252d6f09..7e9561ba32 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -40,6 +40,12 @@ func rootState(l *lexer) stateFn { switch r { case EOF: return nil + case '$': + if l.mode == STR_EXPRESSION || l.mode == STR_IDENTIFIER { + l.emitType(TokenStringTemplate) + } else { + return l.error(fmt.Errorf("unrecognized character: %#U", r)) + } case '+': l.emitType(TokenPlus) case '-': @@ -296,6 +302,10 @@ func identifierState(l *lexer) stateFn { } } l.emitType(TokenIdentifier) + if l.mode == STR_IDENTIFIER { + l.mode = NORMAL + return stringState + } return rootState } diff --git a/runtime/parser/lexer/tokentype.go b/runtime/parser/lexer/tokentype.go index 0a15c19b6f..a7b3cb2f92 100644 --- a/runtime/parser/lexer/tokentype.go +++ b/runtime/parser/lexer/tokentype.go @@ -82,6 +82,7 @@ const ( TokenAsExclamationMark TokenAsQuestionMark TokenPragma + TokenStringTemplate // NOTE: not an actual token, must be last item TokenMax ) @@ -205,6 +206,8 @@ func (t TokenType) String() string { return `'as?'` case TokenPragma: return `'#'` + case TokenStringTemplate: + return `'$'` default: panic(errors.NewUnreachableError()) } From 7ebe45bee7b77f940b46920a39bbe9f864f6e404 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 18 Sep 2024 12:53:33 -0400 Subject: [PATCH 02/21] Add parsing support for string templates. --- runtime/ast/expression.go | 57 ++++++++++++++++ runtime/ast/precedence.go | 1 + runtime/parser/expression.go | 106 ++++++++++++++++++++++++++---- runtime/parser/expression_test.go | 64 ++++++++++++++++++ 4 files changed, 215 insertions(+), 13 deletions(-) diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 0142b92b20..800cf44668 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -220,6 +220,63 @@ func (*StringExpression) precedence() precedence { return precedenceLiteral } +// StringTemplateExpression + +type StringTemplateExpression struct { + Values []string + Expressions []Expression + Range +} + +var _ Expression = &StringTemplateExpression{} + +func NewStringTemplateExpression(gauge common.MemoryGauge, values []string, exprs []Expression, exprRange Range) *StringTemplateExpression { + common.UseMemory(gauge, common.StringExpressionMemoryUsage) + return &StringTemplateExpression{ + Values: values, + Expressions: exprs, + Range: exprRange, + } +} + +var _ Element = &StringExpression{} +var _ Expression = &StringExpression{} + +func (*StringTemplateExpression) ElementType() ElementType { + return ElementTypeStringExpression +} + +func (*StringTemplateExpression) isExpression() {} + +func (*StringTemplateExpression) isIfStatementTest() {} + +func (*StringTemplateExpression) Walk(_ func(Element)) { + // NO-OP +} + +func (e *StringTemplateExpression) String() string { + return Prettier(e) +} + +func (e *StringTemplateExpression) Doc() prettier.Doc { + return prettier.Text(QuoteString("String template")) +} + +func (e *StringTemplateExpression) MarshalJSON() ([]byte, error) { + type Alias StringTemplateExpression + return json.Marshal(&struct { + *Alias + Type string + }{ + Type: "StringTemplateExpression", + Alias: (*Alias)(e), + }) +} + +func (*StringTemplateExpression) precedence() precedence { + return precedenceLiteral +} + // IntegerExpression type IntegerExpression struct { diff --git a/runtime/ast/precedence.go b/runtime/ast/precedence.go index 3e42f6a8f1..fcc78d259f 100644 --- a/runtime/ast/precedence.go +++ b/runtime/ast/precedence.go @@ -83,6 +83,7 @@ const ( // - BoolExpression // - NilExpression // - StringExpression + // - StringTemplateExpression // - IntegerExpression // - FixedPointExpression // - ArrayExpression diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 41f44df7a7..0cf9186704 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -433,19 +433,6 @@ func init() { }, }) - defineExpr(literalExpr{ - tokenType: lexer.TokenString, - nullDenotation: func(p *parser, token lexer.Token) (ast.Expression, error) { - literal := p.tokenSource(token) - parsedString := parseStringLiteral(p, literal) - return ast.NewStringExpression( - p.memoryGauge, - parsedString, - token.Range, - ), nil - }, - }) - defineExpr(prefixExpr{ tokenType: lexer.TokenMinus, bindingPower: exprLeftBindingPowerUnaryPrefix, @@ -510,6 +497,7 @@ func init() { defineNestedExpression() defineInvocationExpression() defineArrayExpression() + defineStringExpression() defineDictionaryExpression() defineIndexExpression() definePathExpression() @@ -1144,6 +1132,98 @@ func defineNestedExpression() { ) } +func defineStringExpression() { + setExprNullDenotation( + lexer.TokenString, + func(p *parser, startToken lexer.Token) (ast.Expression, error) { + var literals []string + var values []ast.Expression + curToken := startToken + endToken := startToken + + // early check for start " of string literal because of string templates + literal := p.tokenSource(curToken) + length := len(literal) + if length == 0 { + p.reportSyntaxError("invalid end of string literal: missing '\"'") + return ast.NewStringExpression( + p.memoryGauge, + "", + startToken.Range, + ), nil + } + + if length >= 1 { + first := literal[0] + if first != '"' { + p.reportSyntaxError("invalid start of string literal: expected '\"', got %q", first) + } + } + + // flag for late end " check + missingEnd := true + + for curToken.Is(lexer.TokenString) { + literal = p.tokenSource(curToken) + length = len(literal) + + if length >= 1 && literal[0] == '"' { + literal = literal[1:] + length = len(literal) + } + + if length >= 1 && literal[length-1] == '"' { + literal = literal[:length-1] + missingEnd = false + } + + parsedString := parseStringLiteralContent(p, literal) + literals = append(literals, parsedString) + endToken = curToken + + // parser already points to next token + curToken = p.current + if curToken.Is(lexer.TokenStringTemplate) { + p.next() + // advance to the expression + value, err := parseExpression(p, lowestBindingPower) + if err != nil { + return nil, err + } + values = append(values, value) + // parser already points to next token + curToken = p.current + // safely call next because this should always be a string + p.next() + missingEnd = true + } + } + + // late check for end " of string literal because of string templates + if missingEnd { + p.reportSyntaxError("invalid end of string literal: missing '\"'") + } + + if len(values) == 0 { + return ast.NewStringExpression( + p.memoryGauge, + literals[0], // must exist + startToken.Range, + ), nil + } else { + return ast.NewStringTemplateExpression( + p.memoryGauge, + literals, values, + ast.NewRange(p.memoryGauge, + startToken.StartPos, + endToken.EndPos), + ), nil + } + + }, + ) +} + func defineArrayExpression() { setExprNullDenotation( lexer.TokenBracketOpen, diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index eb8b348c45..9f12f47cec 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6055,6 +6055,70 @@ func TestParseStringWithUnicode(t *testing.T) { utils.AssertEqualWithDiff(t, expected, actual) } +func TestParseStringTemplates(t *testing.T) { + + t.Parallel() + + actual, errs := testParseExpression(` + "this is a test $abc $def test" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.NoError(t, err) + + expected := &ast.StringTemplateExpression{ + Values: []string{ + "this is a test ", + " ", + " test", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "abc", + Pos: ast.Position{Offset: 24, Line: 2, Column: 23}, + }, + }, + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "def", + Pos: ast.Position{Offset: 29, Line: 2, Column: 28}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 7, Line: 2, Column: 6}, + EndPos: ast.Position{Offset: 37, Line: 2, Column: 36}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) +} + +func TestParseStringTemplateFail(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "this is a test $FOO + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.Error(t, err) +} + func TestParseNilCoalescing(t *testing.T) { t.Parallel() From 0795b4c02b55b5eb59e5b7fbedb64367750f4319 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 18 Sep 2024 15:58:10 -0400 Subject: [PATCH 03/21] Checking and interpreting for string templates. --- runtime/ast/elementtype.go | 1 + runtime/ast/expression.go | 2 +- runtime/ast/expression_extractor.go | 84 +++++++++++++------ runtime/ast/visitor.go | 4 + runtime/interpreter/interpreter_expression.go | 28 +++++++ runtime/parser/expression.go | 11 +-- runtime/parser/expression_test.go | 41 ++++++++- .../sema/check_string_template_expression.go | 52 ++++++++++++ runtime/sema/elaboration.go | 20 +++++ runtime/tests/checker/string_test.go | 42 ++++++++++ 10 files changed, 253 insertions(+), 32 deletions(-) create mode 100644 runtime/sema/check_string_template_expression.go diff --git a/runtime/ast/elementtype.go b/runtime/ast/elementtype.go index 1ceee39f3e..2369cf5080 100644 --- a/runtime/ast/elementtype.go +++ b/runtime/ast/elementtype.go @@ -85,4 +85,5 @@ const ( ElementTypeForceExpression ElementTypePathExpression ElementTypeAttachExpression + ElementTypeStringTemplateExpression ) diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 800cf44668..2e2ee2f7f2 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -243,7 +243,7 @@ var _ Element = &StringExpression{} var _ Expression = &StringExpression{} func (*StringTemplateExpression) ElementType() ElementType { - return ElementTypeStringExpression + return ElementTypeStringTemplateExpression } func (*StringTemplateExpression) isExpression() {} diff --git a/runtime/ast/expression_extractor.go b/runtime/ast/expression_extractor.go index 777dc8e4d3..4039b0370b 100644 --- a/runtime/ast/expression_extractor.go +++ b/runtime/ast/expression_extractor.go @@ -48,6 +48,10 @@ type StringExtractor interface { ExtractString(extractor *ExpressionExtractor, expression *StringExpression) ExpressionExtraction } +type StringTemplateExtractor interface { + ExtractStringTemplate(extractor *ExpressionExtractor, expression *StringTemplateExpression) ExpressionExtraction +} + type ArrayExtractor interface { ExtractArray(extractor *ExpressionExtractor, expression *ArrayExpression) ExpressionExtraction } @@ -117,31 +121,32 @@ type AttachExtractor interface { } type ExpressionExtractor struct { - IndexExtractor IndexExtractor - ForceExtractor ForceExtractor - BoolExtractor BoolExtractor - NilExtractor NilExtractor - IntExtractor IntExtractor - FixedPointExtractor FixedPointExtractor - StringExtractor StringExtractor - ArrayExtractor ArrayExtractor - DictionaryExtractor DictionaryExtractor - IdentifierExtractor IdentifierExtractor - AttachExtractor AttachExtractor - MemoryGauge common.MemoryGauge - VoidExtractor VoidExtractor - UnaryExtractor UnaryExtractor - ConditionalExtractor ConditionalExtractor - InvocationExtractor InvocationExtractor - BinaryExtractor BinaryExtractor - FunctionExtractor FunctionExtractor - CastingExtractor CastingExtractor - CreateExtractor CreateExtractor - DestroyExtractor DestroyExtractor - ReferenceExtractor ReferenceExtractor - MemberExtractor MemberExtractor - PathExtractor PathExtractor - nextIdentifier int + IndexExtractor IndexExtractor + ForceExtractor ForceExtractor + BoolExtractor BoolExtractor + NilExtractor NilExtractor + IntExtractor IntExtractor + FixedPointExtractor FixedPointExtractor + StringExtractor StringExtractor + StringTemplateExtractor StringTemplateExtractor + ArrayExtractor ArrayExtractor + DictionaryExtractor DictionaryExtractor + IdentifierExtractor IdentifierExtractor + AttachExtractor AttachExtractor + MemoryGauge common.MemoryGauge + VoidExtractor VoidExtractor + UnaryExtractor UnaryExtractor + ConditionalExtractor ConditionalExtractor + InvocationExtractor InvocationExtractor + BinaryExtractor BinaryExtractor + FunctionExtractor FunctionExtractor + CastingExtractor CastingExtractor + CreateExtractor CreateExtractor + DestroyExtractor DestroyExtractor + ReferenceExtractor ReferenceExtractor + MemberExtractor MemberExtractor + PathExtractor PathExtractor + nextIdentifier int } var _ ExpressionVisitor[ExpressionExtraction] = &ExpressionExtractor{} @@ -271,6 +276,35 @@ func (extractor *ExpressionExtractor) ExtractString(expression *StringExpression return rewriteExpressionAsIs(expression) } +func (extractor *ExpressionExtractor) VisitStringTemplateExpression(expression *StringTemplateExpression) ExpressionExtraction { + + // delegate to child extractor, if any, + // or call default implementation + + if extractor.StringTemplateExtractor != nil { + return extractor.StringTemplateExtractor.ExtractStringTemplate(extractor, expression) + } + return extractor.ExtractStringTemplate(expression) +} + +func (extractor *ExpressionExtractor) ExtractStringTemplate(expression *StringTemplateExpression) ExpressionExtraction { + + // copy the expression + newExpression := *expression + + // rewrite all value expressions + + rewrittenExpressions, extractedExpressions := + extractor.VisitExpressions(expression.Expressions) + + newExpression.Expressions = rewrittenExpressions + + return ExpressionExtraction{ + RewrittenExpression: &newExpression, + ExtractedExpressions: extractedExpressions, + } +} + func (extractor *ExpressionExtractor) VisitArrayExpression(expression *ArrayExpression) ExpressionExtraction { // delegate to child extractor, if any, diff --git a/runtime/ast/visitor.go b/runtime/ast/visitor.go index c18fdd797f..a2fea22db0 100644 --- a/runtime/ast/visitor.go +++ b/runtime/ast/visitor.go @@ -183,6 +183,7 @@ type ExpressionVisitor[T any] interface { VisitNilExpression(*NilExpression) T VisitBoolExpression(*BoolExpression) T VisitStringExpression(*StringExpression) T + VisitStringTemplateExpression(*StringTemplateExpression) T VisitIntegerExpression(*IntegerExpression) T VisitFixedPointExpression(*FixedPointExpression) T VisitDictionaryExpression(*DictionaryExpression) T @@ -219,6 +220,9 @@ func AcceptExpression[T any](expression Expression, visitor ExpressionVisitor[T] case ElementTypeStringExpression: return visitor.VisitStringExpression(expression.(*StringExpression)) + case ElementTypeStringTemplateExpression: + return visitor.VisitStringTemplateExpression(expression.(*StringTemplateExpression)) + case ElementTypeIntegerExpression: return visitor.VisitIntegerExpression(expression.(*IntegerExpression)) diff --git a/runtime/interpreter/interpreter_expression.go b/runtime/interpreter/interpreter_expression.go index d2894749bf..4d25ab9c11 100644 --- a/runtime/interpreter/interpreter_expression.go +++ b/runtime/interpreter/interpreter_expression.go @@ -957,6 +957,34 @@ func (interpreter *Interpreter) VisitStringExpression(expression *ast.StringExpr return NewUnmeteredStringValue(expression.Value) } +func (interpreter *Interpreter) VisitStringTemplateExpression(expression *ast.StringTemplateExpression) Value { + values := interpreter.visitExpressionsNonCopying(expression.Expressions) + + templateExpressionTypes := interpreter.Program.Elaboration.StringTemplateExpressionTypes(expression) + argumentTypes := templateExpressionTypes.ArgumentTypes + + var copies []Value + + count := len(values) + if count > 0 { + copies = make([]Value, count) + for i, argument := range values { + argumentType := argumentTypes[i] + argumentExpression := expression.Expressions[i] + locationRange := LocationRange{ + Location: interpreter.Location, + HasPosition: argumentExpression, + } + copies[i] = interpreter.transferAndConvert(argument, argumentType, sema.StringType, locationRange) + } + } + + result := "" + + // NOTE: already metered in lexer/parser + return NewUnmeteredStringValue(result) +} + func (interpreter *Interpreter) VisitArrayExpression(expression *ast.ArrayExpression) Value { values := interpreter.visitExpressionsNonCopying(expression.Values) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 0cf9186704..45290a3093 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1167,16 +1167,17 @@ func defineStringExpression() { literal = p.tokenSource(curToken) length = len(literal) - if length >= 1 && literal[0] == '"' { - literal = literal[1:] - length = len(literal) - } - + // this order of removal matters in case the token is the single quotation " if length >= 1 && literal[length-1] == '"' { literal = literal[:length-1] + length = len(literal) missingEnd = false } + if length >= 1 && literal[0] == '"' { + literal = literal[1:] + } + parsedString := parseStringLiteralContent(p, literal) literals = append(literals, parsedString) endToken = curToken diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index 9f12f47cec..95ea2cb792 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6055,7 +6055,46 @@ func TestParseStringWithUnicode(t *testing.T) { utils.AssertEqualWithDiff(t, expected, actual) } -func TestParseStringTemplates(t *testing.T) { +func TestParseStringTemplateSimple(t *testing.T) { + + t.Parallel() + + actual, errs := testParseExpression(` + "$test" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.NoError(t, err) + + expected := &ast.StringTemplateExpression{ + Values: []string{ + "", + "", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "test", + Pos: ast.Position{Offset: 9, Line: 2, Column: 8}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 7, Line: 2, Column: 6}, + EndPos: ast.Position{Offset: 13, Line: 2, Column: 12}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) +} + +func TestParseStringTemplateMulti(t *testing.T) { t.Parallel() diff --git a/runtime/sema/check_string_template_expression.go b/runtime/sema/check_string_template_expression.go new file mode 100644 index 0000000000..276b8c0f14 --- /dev/null +++ b/runtime/sema/check_string_template_expression.go @@ -0,0 +1,52 @@ +/* + * Cadence - The resource-oriented smart contract programming language + * + * Copyright Flow Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sema + +import "github.com/onflow/cadence/runtime/ast" + +func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression *ast.StringTemplateExpression) Type { + + // visit all elements + + var elementType Type + + elementCount := len(stringTemplateExpression.Expressions) + + var argumentTypes []Type + if elementCount > 0 { + argumentTypes = make([]Type, elementCount) + + for i, element := range stringTemplateExpression.Expressions { + valueType := checker.VisitExpression(element, stringTemplateExpression, elementType) + + argumentTypes[i] = valueType + + checker.checkResourceMoveOperation(element, valueType) + } + } + + checker.Elaboration.SetStringTemplateExpressionTypes( + stringTemplateExpression, + StringTemplateExpressionTypes{ + ArgumentTypes: argumentTypes, + }, + ) + + return StringType +} diff --git a/runtime/sema/elaboration.go b/runtime/sema/elaboration.go index b6b025eef0..d2ef948a78 100644 --- a/runtime/sema/elaboration.go +++ b/runtime/sema/elaboration.go @@ -79,6 +79,10 @@ type ArrayExpressionTypes struct { ArgumentTypes []Type } +type StringTemplateExpressionTypes struct { + ArgumentTypes []Type +} + type DictionaryExpressionTypes struct { DictionaryType *DictionaryType EntryTypes []DictionaryEntryType @@ -140,6 +144,7 @@ type Elaboration struct { dictionaryExpressionTypes map[*ast.DictionaryExpression]DictionaryExpressionTypes integerExpressionTypes map[*ast.IntegerExpression]Type stringExpressionTypes map[*ast.StringExpression]Type + stringTemplateExpressionTypes map[*ast.StringTemplateExpression]StringTemplateExpressionTypes returnStatementTypes map[*ast.ReturnStatement]ReturnStatementTypes functionDeclarationFunctionTypes map[*ast.FunctionDeclaration]*FunctionType variableDeclarationTypes map[*ast.VariableDeclaration]VariableDeclarationTypes @@ -480,6 +485,21 @@ func (e *Elaboration) SetStringExpressionType(expression *ast.StringExpression, e.stringExpressionTypes[expression] = ty } +func (e *Elaboration) StringTemplateExpressionTypes(expression *ast.StringTemplateExpression) (types StringTemplateExpressionTypes) { + if e.stringTemplateExpressionTypes == nil { + return + } + // default, Elaboration.SetStringExpressionType + return e.stringTemplateExpressionTypes[expression] +} + +func (e *Elaboration) SetStringTemplateExpressionTypes(expression *ast.StringTemplateExpression, types StringTemplateExpressionTypes) { + if e.stringTemplateExpressionTypes == nil { + e.stringTemplateExpressionTypes = map[*ast.StringTemplateExpression]StringTemplateExpressionTypes{} + } + e.stringTemplateExpressionTypes[expression] = types +} + func (e *Elaboration) ReturnStatementTypes(statement *ast.ReturnStatement) (types ReturnStatementTypes) { if e.returnStatementTypes == nil { return diff --git a/runtime/tests/checker/string_test.go b/runtime/tests/checker/string_test.go index d857185350..de16b37bcc 100644 --- a/runtime/tests/checker/string_test.go +++ b/runtime/tests/checker/string_test.go @@ -697,3 +697,45 @@ func TestCheckStringCount(t *testing.T) { require.NoError(t, err) }) } + +func TestCheckStringTemplate(t *testing.T) { + + t.Parallel() + + t.Run("valid, int", func(t *testing.T) { + + t.Parallel() + + _, err := ParseAndCheck(t, ` + let a = 1 + let x: String = "The value of a is: $a" + `) + + require.NoError(t, err) + }) + + t.Run("valid, string", func(t *testing.T) { + + t.Parallel() + + _, err := ParseAndCheck(t, ` + let a = "abc def" + let x: String = "$a ghi" + `) + + require.NoError(t, err) + }) + + t.Run("invalid, missing variable", func(t *testing.T) { + + t.Parallel() + + _, err := ParseAndCheck(t, ` + let x: String = "$a" + `) + + errs := RequireCheckerErrors(t, err, 1) + + assert.IsType(t, &sema.NotDeclaredError{}, errs[0]) + }) +} From 32f1c1d734adfd3ea7b4201933155c84cd305ba1 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 19 Sep 2024 11:13:05 -0400 Subject: [PATCH 04/21] Interpret string templates with basic tests --- runtime/ast/expression.go | 1 + runtime/interpreter/interpreter_expression.go | 38 ++++++------ runtime/tests/interpreter/interpreter_test.go | 61 +++++++++++++++++++ 3 files changed, 81 insertions(+), 19 deletions(-) diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 2e2ee2f7f2..2efc783155 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -231,6 +231,7 @@ type StringTemplateExpression struct { var _ Expression = &StringTemplateExpression{} func NewStringTemplateExpression(gauge common.MemoryGauge, values []string, exprs []Expression, exprRange Range) *StringTemplateExpression { + // STRINGTODO: change to be similar to array memory usage? common.UseMemory(gauge, common.StringExpressionMemoryUsage) return &StringTemplateExpression{ Values: values, diff --git a/runtime/interpreter/interpreter_expression.go b/runtime/interpreter/interpreter_expression.go index 4d25ab9c11..264b39edf8 100644 --- a/runtime/interpreter/interpreter_expression.go +++ b/runtime/interpreter/interpreter_expression.go @@ -20,6 +20,7 @@ package interpreter import ( "math/big" + "strings" "time" "github.com/onflow/atree" @@ -960,29 +961,28 @@ func (interpreter *Interpreter) VisitStringExpression(expression *ast.StringExpr func (interpreter *Interpreter) VisitStringTemplateExpression(expression *ast.StringTemplateExpression) Value { values := interpreter.visitExpressionsNonCopying(expression.Expressions) - templateExpressionTypes := interpreter.Program.Elaboration.StringTemplateExpressionTypes(expression) - argumentTypes := templateExpressionTypes.ArgumentTypes - - var copies []Value - - count := len(values) - if count > 0 { - copies = make([]Value, count) - for i, argument := range values { - argumentType := argumentTypes[i] - argumentExpression := expression.Expressions[i] - locationRange := LocationRange{ - Location: interpreter.Location, - HasPosition: argumentExpression, + templatesType := interpreter.Program.Elaboration.StringTemplateExpressionTypes(expression) + argumentTypes := templatesType.ArgumentTypes + + var builder strings.Builder + for i, str := range expression.Values { + builder.WriteString(str) + if i < len(values) { + // STRINGTODO: is this how the conversion should happen? + s := values[i].String() + switch argumentTypes[i] { + case sema.StringType: + // remove quotations + s = s[1 : len(s)-1] + builder.WriteString(s) + default: + builder.WriteString(s) } - copies[i] = interpreter.transferAndConvert(argument, argumentType, sema.StringType, locationRange) } } - result := "" - - // NOTE: already metered in lexer/parser - return NewUnmeteredStringValue(result) + // STRINGTODO: already metered as a string constant in parser? + return NewUnmeteredStringValue(builder.String()) } func (interpreter *Interpreter) VisitArrayExpression(expression *ast.ArrayExpression) Value { diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index f525e8d6ff..86e89ac73b 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12283,3 +12283,64 @@ func TestInterpretOptionalAddressInConditional(t *testing.T) { value, ) } + +func TestInterpretStringTemplates(t *testing.T) { + + t.Parallel() + + t.Run("int", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let x = 123 + let y = "x = $x" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredIntValueFromInt64(123), + inter.Globals.Get("x").GetValue(inter), + ) + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("x = 123"), + inter.Globals.Get("y").GetValue(inter), + ) + }) + + t.Run("multiple", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let x = 123.321 + let y = "abc" + let z = "$y and $x" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("abc and 123.32100000"), + inter.Globals.Get("z").GetValue(inter), + ) + }) + + t.Run("nested template", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let x = "{}" + let y = "[$x]" + let z = "($y)" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("([{}])"), + inter.Globals.Get("z").GetValue(inter), + ) + }) +} From 6a36212d1a3f71280eb15684fb2f66d623fa5db1 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 19 Sep 2024 11:34:12 -0400 Subject: [PATCH 05/21] Add placeholder for compiler linting. --- runtime/compiler/compiler.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/compiler/compiler.go b/runtime/compiler/compiler.go index 342333adaa..6e052f4a17 100644 --- a/runtime/compiler/compiler.go +++ b/runtime/compiler/compiler.go @@ -246,6 +246,11 @@ func (compiler *Compiler) VisitFunctionExpression(_ *ast.FunctionExpression) ir. panic(errors.NewUnreachableError()) } +func (compiler *Compiler) VisitStringTemplateExpression(e *ast.StringTemplateExpression) ir.Expr { + // TODO + panic(errors.NewUnreachableError()) +} + func (compiler *Compiler) VisitStringExpression(e *ast.StringExpression) ir.Expr { return &ir.Const{ Constant: ir.String{ From d90bbf689b8f88367180088fc3c0ce9a5cb58270 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 19 Sep 2024 11:54:20 -0400 Subject: [PATCH 06/21] Fix parsing of invalid strings. --- runtime/parser/expression.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 45290a3093..e3b6bfface 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1167,15 +1167,14 @@ func defineStringExpression() { literal = p.tokenSource(curToken) length = len(literal) - // this order of removal matters in case the token is the single quotation " - if length >= 1 && literal[length-1] == '"' { - literal = literal[:length-1] + if curToken == startToken { + literal = literal[1:] length = len(literal) - missingEnd = false } - if length >= 1 && literal[0] == '"' { - literal = literal[1:] + if length >= 1 && literal[length-1] == '"' { + literal = literal[:length-1] + missingEnd = false } parsedString := parseStringLiteralContent(p, literal) From 81c300460114580adaaaec9eb4aaf82c95236f96 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 19 Sep 2024 12:38:39 -0400 Subject: [PATCH 07/21] Run stringer to update elementtype file. --- runtime/ast/elementtype_string.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/runtime/ast/elementtype_string.go b/runtime/ast/elementtype_string.go index 9f2e36d996..07044923bb 100644 --- a/runtime/ast/elementtype_string.go +++ b/runtime/ast/elementtype_string.go @@ -60,11 +60,12 @@ func _() { _ = x[ElementTypeForceExpression-49] _ = x[ElementTypePathExpression-50] _ = x[ElementTypeAttachExpression-51] + _ = x[ElementTypeStringTemplateExpression-52] } -const _ElementType_name = "ElementTypeUnknownElementTypeProgramElementTypeBlockElementTypeFunctionBlockElementTypeFunctionDeclarationElementTypeSpecialFunctionDeclarationElementTypeCompositeDeclarationElementTypeInterfaceDeclarationElementTypeEntitlementDeclarationElementTypeEntitlementMappingDeclarationElementTypeAttachmentDeclarationElementTypeFieldDeclarationElementTypeEnumCaseDeclarationElementTypePragmaDeclarationElementTypeImportDeclarationElementTypeTransactionDeclarationElementTypeReturnStatementElementTypeBreakStatementElementTypeContinueStatementElementTypeIfStatementElementTypeSwitchStatementElementTypeWhileStatementElementTypeForStatementElementTypeEmitStatementElementTypeVariableDeclarationElementTypeAssignmentStatementElementTypeSwapStatementElementTypeExpressionStatementElementTypeRemoveStatementElementTypeVoidExpressionElementTypeBoolExpressionElementTypeNilExpressionElementTypeIntegerExpressionElementTypeFixedPointExpressionElementTypeArrayExpressionElementTypeDictionaryExpressionElementTypeIdentifierExpressionElementTypeInvocationExpressionElementTypeMemberExpressionElementTypeIndexExpressionElementTypeConditionalExpressionElementTypeUnaryExpressionElementTypeBinaryExpressionElementTypeFunctionExpressionElementTypeStringExpressionElementTypeCastingExpressionElementTypeCreateExpressionElementTypeDestroyExpressionElementTypeReferenceExpressionElementTypeForceExpressionElementTypePathExpressionElementTypeAttachExpression" +const _ElementType_name = "ElementTypeUnknownElementTypeProgramElementTypeBlockElementTypeFunctionBlockElementTypeFunctionDeclarationElementTypeSpecialFunctionDeclarationElementTypeCompositeDeclarationElementTypeInterfaceDeclarationElementTypeEntitlementDeclarationElementTypeEntitlementMappingDeclarationElementTypeAttachmentDeclarationElementTypeFieldDeclarationElementTypeEnumCaseDeclarationElementTypePragmaDeclarationElementTypeImportDeclarationElementTypeTransactionDeclarationElementTypeReturnStatementElementTypeBreakStatementElementTypeContinueStatementElementTypeIfStatementElementTypeSwitchStatementElementTypeWhileStatementElementTypeForStatementElementTypeEmitStatementElementTypeVariableDeclarationElementTypeAssignmentStatementElementTypeSwapStatementElementTypeExpressionStatementElementTypeRemoveStatementElementTypeVoidExpressionElementTypeBoolExpressionElementTypeNilExpressionElementTypeIntegerExpressionElementTypeFixedPointExpressionElementTypeArrayExpressionElementTypeDictionaryExpressionElementTypeIdentifierExpressionElementTypeInvocationExpressionElementTypeMemberExpressionElementTypeIndexExpressionElementTypeConditionalExpressionElementTypeUnaryExpressionElementTypeBinaryExpressionElementTypeFunctionExpressionElementTypeStringExpressionElementTypeCastingExpressionElementTypeCreateExpressionElementTypeDestroyExpressionElementTypeReferenceExpressionElementTypeForceExpressionElementTypePathExpressionElementTypeAttachExpressionElementTypeStringTemplateExpression" -var _ElementType_index = [...]uint16{0, 18, 36, 52, 76, 106, 143, 174, 205, 238, 278, 310, 337, 367, 395, 423, 456, 482, 507, 535, 557, 583, 608, 631, 655, 685, 715, 739, 769, 795, 820, 845, 869, 897, 928, 954, 985, 1016, 1047, 1074, 1100, 1132, 1158, 1185, 1214, 1241, 1269, 1296, 1324, 1354, 1380, 1405, 1432} +var _ElementType_index = [...]uint16{0, 18, 36, 52, 76, 106, 143, 174, 205, 238, 278, 310, 337, 367, 395, 423, 456, 482, 507, 535, 557, 583, 608, 631, 655, 685, 715, 739, 769, 795, 820, 845, 869, 897, 928, 954, 985, 1016, 1047, 1074, 1100, 1132, 1158, 1185, 1214, 1241, 1269, 1296, 1324, 1354, 1380, 1405, 1432, 1467} func (i ElementType) String() string { if i >= ElementType(len(_ElementType_index)-1) { From 99f4ae46cc3371e7f82561062de8d8113ea4f78f Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Fri, 20 Sep 2024 13:59:56 -0400 Subject: [PATCH 08/21] Improve error messages and tests --- runtime/parser/expression.go | 3 + runtime/parser/expression_test.go | 214 ++++++++++++------ runtime/parser/lexer/lexer_test.go | 146 +++++++++++- runtime/tests/checker/string_test.go | 14 ++ runtime/tests/interpreter/interpreter_test.go | 55 +++++ 5 files changed, 358 insertions(+), 74 deletions(-) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index e3b6bfface..01d9dba5d1 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1186,6 +1186,9 @@ func defineStringExpression() { if curToken.Is(lexer.TokenStringTemplate) { p.next() // advance to the expression + if !p.current.Is(lexer.TokenIdentifier) { + return nil, p.syntaxError("expected an identifier got: %s", p.currentTokenSource()) + } value, err := parseExpression(p, lowestBindingPower) if err != nil { return nil, err diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index 95ea2cb792..dabaff5057 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6055,107 +6055,175 @@ func TestParseStringWithUnicode(t *testing.T) { utils.AssertEqualWithDiff(t, expected, actual) } -func TestParseStringTemplateSimple(t *testing.T) { +func TestParseStringTemplate(t *testing.T) { t.Parallel() - actual, errs := testParseExpression(` - "$test" - `) + t.Run("simple", func(t *testing.T) { - var err error - if len(errs) > 0 { - err = Error{ - Errors: errs, + t.Parallel() + + actual, errs := testParseExpression(` + "$test" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } } - } - require.NoError(t, err) + require.NoError(t, err) - expected := &ast.StringTemplateExpression{ - Values: []string{ - "", - "", - }, - Expressions: []ast.Expression{ - &ast.IdentifierExpression{ - Identifier: ast.Identifier{ - Identifier: "test", - Pos: ast.Position{Offset: 9, Line: 2, Column: 8}, + expected := &ast.StringTemplateExpression{ + Values: []string{ + "", + "", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "test", + Pos: ast.Position{Offset: 5, Line: 2, Column: 4}, + }, }, }, - }, - Range: ast.Range{ - StartPos: ast.Position{Offset: 7, Line: 2, Column: 6}, - EndPos: ast.Position{Offset: 13, Line: 2, Column: 12}, - }, - } + Range: ast.Range{ + StartPos: ast.Position{Offset: 3, Line: 2, Column: 2}, + EndPos: ast.Position{Offset: 9, Line: 2, Column: 8}, + }, + } - utils.AssertEqualWithDiff(t, expected, actual) -} + utils.AssertEqualWithDiff(t, expected, actual) + }) -func TestParseStringTemplateMulti(t *testing.T) { + t.Run("multi", func(t *testing.T) { - t.Parallel() + t.Parallel() - actual, errs := testParseExpression(` - "this is a test $abc $def test" - `) + actual, errs := testParseExpression(` + "this is a test $abc$def test" + `) - var err error - if len(errs) > 0 { - err = Error{ - Errors: errs, + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } } - } - require.NoError(t, err) + require.NoError(t, err) - expected := &ast.StringTemplateExpression{ - Values: []string{ - "this is a test ", - " ", - " test", - }, - Expressions: []ast.Expression{ - &ast.IdentifierExpression{ - Identifier: ast.Identifier{ - Identifier: "abc", - Pos: ast.Position{Offset: 24, Line: 2, Column: 23}, + expected := &ast.StringTemplateExpression{ + Values: []string{ + "this is a test ", + "", + " test", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "abc", + Pos: ast.Position{Offset: 20, Line: 2, Column: 19}, + }, + }, + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "def", + Pos: ast.Position{Offset: 24, Line: 2, Column: 24}, + }, }, }, - &ast.IdentifierExpression{ - Identifier: ast.Identifier{ - Identifier: "def", - Pos: ast.Position{Offset: 29, Line: 2, Column: 28}, + Range: ast.Range{ + StartPos: ast.Position{Offset: 3, Line: 2, Column: 2}, + EndPos: ast.Position{Offset: 32, Line: 2, Column: 32}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) + }) + + t.Run("missing end", func(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "this is a test $FOO + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.Error(t, err) + utils.AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "invalid end of string literal: missing '\"'", + Pos: ast.Position{Offset: 25, Line: 2, Column: 25}, }, }, - }, - Range: ast.Range{ - StartPos: ast.Position{Offset: 7, Line: 2, Column: 6}, - EndPos: ast.Position{Offset: 37, Line: 2, Column: 36}, - }, - } + errs, + ) + }) - utils.AssertEqualWithDiff(t, expected, actual) -} + t.Run("invalid identifier", func(t *testing.T) { -func TestParseStringTemplateFail(t *testing.T) { + t.Parallel() - t.Parallel() + _, errs := testParseExpression(` + "$$" + `) - _, errs := testParseExpression(` - "this is a test $FOO - `) + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } - var err error - if len(errs) > 0 { - err = Error{ - Errors: errs, + require.Error(t, err) + utils.AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "expected an identifier got: $", + Pos: ast.Position{Offset: 7, Line: 2, Column: 6}, + }, + }, + errs, + ) + }) + + t.Run("invalid, num", func(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "$(2 + 2) is a" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } } - } - require.Error(t, err) + require.Error(t, err) + utils.AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "expected an identifier got: (", + Pos: ast.Position{Offset: 7, Line: 2, Column: 6}, + }, + }, + errs, + ) + }) } func TestParseNilCoalescing(t *testing.T) { diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 4206edd22e..643f57e902 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -1071,7 +1071,7 @@ func TestLexString(t *testing.T) { ) }) - t.Run("invalid, string template", func(t *testing.T) { + t.Run("invalid, number string template", func(t *testing.T) { testLex(t, `"$1"`, []token{ @@ -1128,6 +1128,150 @@ func TestLexString(t *testing.T) { ) }) + t.Run("invalid, string template", func(t *testing.T) { + testLex(t, + `"$a + 2`, + []token{ + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenStringTemplate, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + Source: `$`, + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + Source: `a`, + }, + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, + }, + }, + Source: ` + 2`, + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, + }, + }, + ) + }) + + t.Run("valid, multi string template", func(t *testing.T) { + testLex(t, + `"$a$b"`, + []token{ + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenStringTemplate, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + Source: `$`, + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + Source: `a`, + }, + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 2}, + }, + }, + Source: ``, + }, + { + Token: Token{ + Type: TokenStringTemplate, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 3}, + }, + }, + Source: `$`, + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 4}, + }, + }, + Source: `b`, + }, + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 6, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 6, Offset: 5}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 6}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 6}, + }, + }, + }, + }, + ) + }) + t.Run("invalid, empty, not terminated at line end", func(t *testing.T) { testLex(t, "\"\n", diff --git a/runtime/tests/checker/string_test.go b/runtime/tests/checker/string_test.go index de16b37bcc..ee2609758f 100644 --- a/runtime/tests/checker/string_test.go +++ b/runtime/tests/checker/string_test.go @@ -726,6 +726,20 @@ func TestCheckStringTemplate(t *testing.T) { require.NoError(t, err) }) + t.Run("valid, struct", func(t *testing.T) { + + t.Parallel() + + _, err := ParseAndCheck(t, ` + access(all) + struct SomeStruct {} + let a = SomeStruct() + let x: String = "$a" + `) + + require.NoError(t, err) + }) + t.Run("invalid, missing variable", func(t *testing.T) { t.Parallel() diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index 86e89ac73b..3669f17960 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12343,4 +12343,59 @@ func TestInterpretStringTemplates(t *testing.T) { inter.Globals.Get("z").GetValue(inter), ) }) + + t.Run("struct", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + access(all) + struct SomeStruct {} + let a = SomeStruct() + let x: String = "$a" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("S.test.SomeStruct()"), + inter.Globals.Get("x").GetValue(inter), + ) + }) + + t.Run("func", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let add = fun(): Int { + return 2+2 + } + let x: String = "$add()" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("fun(): Int()"), + inter.Globals.Get("x").GetValue(inter), + ) + }) + + t.Run("func", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let add = fun(): Int { + return 2+2 + } + let y = add() + let x: String = "$y" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("4"), + inter.Globals.Get("x").GetValue(inter), + ) + }) } From cfa540ecd8299df8190919ae76669b6361353473 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Mon, 23 Sep 2024 12:07:45 -0400 Subject: [PATCH 09/21] Allow escaping of string template $ and add tests. --- runtime/parser/expression.go | 2 ++ runtime/parser/expression_test.go | 28 +++++++++++++++++++ runtime/parser/lexer/lexer_test.go | 27 ++++++++++++++++++ runtime/tests/interpreter/interpreter_test.go | 16 +++++++++++ 4 files changed, 73 insertions(+) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 01d9dba5d1..9e93990e0b 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1735,6 +1735,8 @@ func parseStringLiteralContent(p *parser, s []byte) (result string) { builder.WriteByte('\t') case '"': builder.WriteByte('"') + case '$': + builder.WriteByte('$') case '\'': builder.WriteByte('\'') case '\\': diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index dabaff5057..b7b8489a5f 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6098,6 +6098,34 @@ func TestParseStringTemplate(t *testing.T) { utils.AssertEqualWithDiff(t, expected, actual) }) + t.Run("escaped", func(t *testing.T) { + + t.Parallel() + + actual, errs := testParseExpression(` + "\$1.00" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.NoError(t, err) + + expected := &ast.StringExpression{ + Value: "$1.00", + Range: ast.Range{ + StartPos: ast.Position{Offset: 3, Line: 2, Column: 2}, + EndPos: ast.Position{Offset: 10, Line: 2, Column: 9}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) + }) + t.Run("multi", func(t *testing.T) { t.Parallel() diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 643f57e902..fddebcf064 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -1071,6 +1071,33 @@ func TestLexString(t *testing.T) { ) }) + t.Run("valid, escaped string template", func(t *testing.T) { + testLex(t, + `"\$1.00"`, + []token{ + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, + Source: `"\$1.00"`, + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, + EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, + }, + }, + }, + }, + ) + }) + t.Run("invalid, number string template", func(t *testing.T) { testLex(t, `"$1"`, diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index 3669f17960..74e49cf968 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12398,4 +12398,20 @@ func TestInterpretStringTemplates(t *testing.T) { inter.Globals.Get("x").GetValue(inter), ) }) + + t.Run("escaped", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let x = 123 + let y = "x is worth \$$x" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("x is worth $123"), + inter.Globals.Get("y").GetValue(inter), + ) + }) } From 0cc3a365ae7f3022506f81c8be315881e6e1c9bf Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Mon, 23 Sep 2024 12:41:54 -0400 Subject: [PATCH 10/21] Reset lexer state. --- runtime/parser/lexer/lexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 5e9caf2f79..e5e224f850 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -140,6 +140,7 @@ func (l *lexer) clear() { l.cursor = 0 l.tokens = l.tokens[:0] l.tokenCount = 0 + l.mode = NORMAL } func (l *lexer) Reclaim() { From 942eb034392aa100179fb3d77d403badd5f96419 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Tue, 24 Sep 2024 11:40:27 -0400 Subject: [PATCH 11/21] Change string template token to \( for backwards compatibility. --- runtime/parser/expression.go | 4 + runtime/parser/expression_test.go | 109 +++++--- runtime/parser/lexer/lexer.go | 20 +- runtime/parser/lexer/lexer_test.go | 253 ++++++++++++++---- runtime/parser/lexer/state.go | 32 ++- runtime/parser/lexer/tokentype.go | 2 +- runtime/tests/checker/string_test.go | 27 +- runtime/tests/interpreter/interpreter_test.go | 64 +++-- 8 files changed, 373 insertions(+), 138 deletions(-) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 9e93990e0b..3537de9b93 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1193,6 +1193,10 @@ func defineStringExpression() { if err != nil { return nil, err } + _, err = p.mustOne(lexer.TokenParenClose) + if err != nil { + return nil, err + } values = append(values, value) // parser already points to next token curToken = p.current diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index b7b8489a5f..2f78ddfe59 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6064,7 +6064,7 @@ func TestParseStringTemplate(t *testing.T) { t.Parallel() actual, errs := testParseExpression(` - "$test" + "\(test)" `) var err error @@ -6085,41 +6085,13 @@ func TestParseStringTemplate(t *testing.T) { &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "test", - Pos: ast.Position{Offset: 5, Line: 2, Column: 4}, + Pos: ast.Position{Offset: 6, Line: 2, Column: 5}, }, }, }, Range: ast.Range{ StartPos: ast.Position{Offset: 3, Line: 2, Column: 2}, - EndPos: ast.Position{Offset: 9, Line: 2, Column: 8}, - }, - } - - utils.AssertEqualWithDiff(t, expected, actual) - }) - - t.Run("escaped", func(t *testing.T) { - - t.Parallel() - - actual, errs := testParseExpression(` - "\$1.00" - `) - - var err error - if len(errs) > 0 { - err = Error{ - Errors: errs, - } - } - - require.NoError(t, err) - - expected := &ast.StringExpression{ - Value: "$1.00", - Range: ast.Range{ - StartPos: ast.Position{Offset: 3, Line: 2, Column: 2}, - EndPos: ast.Position{Offset: 10, Line: 2, Column: 9}, + EndPos: ast.Position{Offset: 11, Line: 2, Column: 10}, }, } @@ -6131,7 +6103,7 @@ func TestParseStringTemplate(t *testing.T) { t.Parallel() actual, errs := testParseExpression(` - "this is a test $abc$def test" + "this is a test \(abc)\(def) test" `) var err error @@ -6153,19 +6125,19 @@ func TestParseStringTemplate(t *testing.T) { &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "abc", - Pos: ast.Position{Offset: 20, Line: 2, Column: 19}, + Pos: ast.Position{Offset: 21, Line: 2, Column: 20}, }, }, &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "def", - Pos: ast.Position{Offset: 24, Line: 2, Column: 24}, + Pos: ast.Position{Offset: 27, Line: 2, Column: 27}, }, }, }, Range: ast.Range{ StartPos: ast.Position{Offset: 3, Line: 2, Column: 2}, - EndPos: ast.Position{Offset: 32, Line: 2, Column: 32}, + EndPos: ast.Position{Offset: 36, Line: 2, Column: 36}, }, } @@ -6177,7 +6149,7 @@ func TestParseStringTemplate(t *testing.T) { t.Parallel() _, errs := testParseExpression(` - "this is a test $FOO + "this is a test \(FOO) `) var err error @@ -6192,7 +6164,7 @@ func TestParseStringTemplate(t *testing.T) { []error{ &SyntaxError{ Message: "invalid end of string literal: missing '\"'", - Pos: ast.Position{Offset: 25, Line: 2, Column: 25}, + Pos: ast.Position{Offset: 27, Line: 2, Column: 27}, }, }, errs, @@ -6204,7 +6176,7 @@ func TestParseStringTemplate(t *testing.T) { t.Parallel() _, errs := testParseExpression(` - "$$" + "\(.)" `) var err error @@ -6218,8 +6190,8 @@ func TestParseStringTemplate(t *testing.T) { utils.AssertEqualWithDiff(t, []error{ &SyntaxError{ - Message: "expected an identifier got: $", - Pos: ast.Position{Offset: 7, Line: 2, Column: 6}, + Message: "expected an identifier got: .", + Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, }, }, errs, @@ -6231,7 +6203,34 @@ func TestParseStringTemplate(t *testing.T) { t.Parallel() _, errs := testParseExpression(` - "$(2 + 2) is a" + "\(2 + 2) is a" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.Error(t, err) + utils.AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "expected an identifier got: 2", + Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, + }, + }, + errs, + ) + }) + + t.Run("invalid, nested identifier", func(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "\((a))" `) var err error @@ -6246,7 +6245,33 @@ func TestParseStringTemplate(t *testing.T) { []error{ &SyntaxError{ Message: "expected an identifier got: (", - Pos: ast.Position{Offset: 7, Line: 2, Column: 6}, + Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, + }, + }, + errs, + ) + }) + t.Run("invalid, empty", func(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "\()" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.Error(t, err) + utils.AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "expected an identifier got: )", + Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, }, }, errs, diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index e5e224f850..6b9c0714b8 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -54,7 +54,6 @@ type LexerMode int const ( NORMAL = iota STR_IDENTIFIER - STR_EXPRESSION ) type lexer struct { @@ -84,6 +83,8 @@ type lexer struct { canBackup bool // lexer mode is used for string templates mode LexerMode + // counts the number of unclosed brackets for string templates \((())) + openBrackets int } var _ TokenStream = &lexer{} @@ -141,6 +142,7 @@ func (l *lexer) clear() { l.tokens = l.tokens[:0] l.tokenCount = 0 l.mode = NORMAL + l.openBrackets = 0 } func (l *lexer) Reclaim() { @@ -418,18 +420,24 @@ func (l *lexer) scanString(quote rune) { l.backupOne() return case '\\': + // might have to backup twice due to string template + tmpBackupOffset := l.prevEndOffset + tmpBackup := l.prev r = l.next() switch r { + case '(': + // string template, stop and set mode + l.mode = STR_IDENTIFIER + // no need to update prev values because these next tokens will not backup + l.endOffset = tmpBackupOffset + l.current = tmpBackup + l.canBackup = false + return case '\n', EOF: // NOTE: invalid end of string handled by parser l.backupOne() return } - case '$': - // string template, stop and set mode - l.backupOne() - l.mode = STR_IDENTIFIER - return } r = l.next() } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index fddebcf064..f2a77db4b2 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -1016,7 +1016,7 @@ func TestLexString(t *testing.T) { t.Run("valid, string template", func(t *testing.T) { testLex(t, - `"$abc.length"`, + `"\(abc).length"`, []token{ { Token: Token{ @@ -1033,27 +1033,37 @@ func TestLexString(t *testing.T) { Type: TokenStringTemplate, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, }, - Source: `$`, + Source: `\(`, }, { Token: Token{ Type: TokenIdentifier, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, }, Source: `abc`, }, + { + Token: Token{ + Type: TokenParenClose, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, + EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, + }, + }, + Source: `)`, + }, { Token: Token{ Type: TokenString, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 12, Offset: 12}, + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 14, Offset: 14}, }, }, Source: `.length"`, @@ -1062,8 +1072,8 @@ func TestLexString(t *testing.T) { Token: Token{ Type: TokenEOF, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 13, Offset: 13}, - EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, + StartPos: ast.Position{Line: 1, Column: 15, Offset: 15}, + EndPos: ast.Position{Line: 1, Column: 15, Offset: 15}, }, }, }, @@ -1071,9 +1081,9 @@ func TestLexString(t *testing.T) { ) }) - t.Run("valid, escaped string template", func(t *testing.T) { + t.Run("valid, not a string template", func(t *testing.T) { testLex(t, - `"\$1.00"`, + `"(1.00)"`, []token{ { Token: Token{ @@ -1083,7 +1093,7 @@ func TestLexString(t *testing.T) { EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, }, }, - Source: `"\$1.00"`, + Source: `"(1.00)"`, }, { Token: Token{ @@ -1100,7 +1110,7 @@ func TestLexString(t *testing.T) { t.Run("invalid, number string template", func(t *testing.T) { testLex(t, - `"$1"`, + `"\(7)"`, []token{ { Token: Token{ @@ -1117,27 +1127,37 @@ func TestLexString(t *testing.T) { Type: TokenStringTemplate, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, }, - Source: `$`, + Source: `\(`, }, { Token: Token{ Type: TokenDecimalIntegerLiteral, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, + Source: `7`, + }, + { + Token: Token{ + Type: TokenParenClose, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, }, }, - Source: `1`, + Source: `)`, }, { Token: Token{ Type: TokenString, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, }, Source: `"`, @@ -1146,8 +1166,8 @@ func TestLexString(t *testing.T) { Token: Token{ Type: TokenEOF, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, + EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, }, }, @@ -1155,9 +1175,9 @@ func TestLexString(t *testing.T) { ) }) - t.Run("invalid, string template", func(t *testing.T) { + t.Run("invalid identifier string template", func(t *testing.T) { testLex(t, - `"$a + 2`, + `"\(a+2)"`, []token{ { Token: Token{ @@ -1174,39 +1194,69 @@ func TestLexString(t *testing.T) { Type: TokenStringTemplate, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, }, - Source: `$`, + Source: `\(`, }, { Token: Token{ Type: TokenIdentifier, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, }, Source: `a`, }, { Token: Token{ - Type: TokenString, + Type: TokenPlus, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + Source: `+`, + }, + { + Token: Token{ + Type: TokenDecimalIntegerLiteral, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + }, + }, + Source: `2`, + }, + { + Token: Token{ + Type: TokenParenClose, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, }, - Source: ` + 2`, + Source: `)`, }, { Token: Token{ - Type: TokenEOF, + Type: TokenString, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, }, }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, + EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, + }, + }, }, }, ) @@ -1214,7 +1264,7 @@ func TestLexString(t *testing.T) { t.Run("valid, multi string template", func(t *testing.T) { testLex(t, - `"$a$b"`, + `"\(a)\(b)"`, []token{ { Token: Token{ @@ -1231,27 +1281,37 @@ func TestLexString(t *testing.T) { Type: TokenStringTemplate, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, }, - Source: `$`, + Source: `\(`, }, { Token: Token{ Type: TokenIdentifier, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, }, Source: `a`, }, + { + Token: Token{ + Type: TokenParenClose, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + Source: `)`, + }, { Token: Token{ Type: TokenString, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 2}, + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 4}, }, }, Source: ``, @@ -1260,28 +1320,38 @@ func TestLexString(t *testing.T) { Token: Token{ Type: TokenStringTemplate, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 3}, + StartPos: ast.Position{Line: 1, Column: 6, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 6}, }, }, - Source: `$`, + Source: `\(`, }, { Token: Token{ Type: TokenIdentifier, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 5, Offset: 4}, + StartPos: ast.Position{Line: 1, Column: 8, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 8, Offset: 7}, }, }, Source: `b`, }, + { + Token: Token{ + Type: TokenParenClose, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 9, Offset: 8}, + EndPos: ast.Position{Line: 1, Column: 9, Offset: 8}, + }, + }, + Source: `)`, + }, { Token: Token{ Type: TokenString, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 6, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 6, Offset: 5}, + StartPos: ast.Position{Line: 1, Column: 10, Offset: 9}, + EndPos: ast.Position{Line: 1, Column: 10, Offset: 9}, }, }, Source: `"`, @@ -1290,8 +1360,95 @@ func TestLexString(t *testing.T) { Token: Token{ Type: TokenEOF, Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 7, Offset: 6}, - EndPos: ast.Position{Line: 1, Column: 7, Offset: 6}, + StartPos: ast.Position{Line: 1, Column: 11, Offset: 10}, + EndPos: ast.Position{Line: 1, Column: 11, Offset: 10}, + }, + }, + }, + }, + ) + }) + + t.Run("valid, nested brackets", func(t *testing.T) { + testLex(t, + `"\((a))"`, + []token{ + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenStringTemplate, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + Source: `\(`, + }, + { + Token: Token{ + Type: TokenParenOpen, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, + Source: `(`, + }, + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + Source: `a`, + }, + { + Token: Token{ + Type: TokenParenClose, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + }, + }, + Source: `)`, + }, + { + Token: Token{ + Type: TokenParenClose, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, + EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, + }, + }, + Source: `)`, + }, + { + Token: Token{ + Type: TokenString, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, + Source: `"`, + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, + EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, }, }, }, diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 7e9561ba32..f2775b81dc 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -40,12 +40,6 @@ func rootState(l *lexer) stateFn { switch r { case EOF: return nil - case '$': - if l.mode == STR_EXPRESSION || l.mode == STR_IDENTIFIER { - l.emitType(TokenStringTemplate) - } else { - return l.error(fmt.Errorf("unrecognized character: %#U", r)) - } case '+': l.emitType(TokenPlus) case '-': @@ -62,9 +56,20 @@ func rootState(l *lexer) stateFn { case '%': l.emitType(TokenPercent) case '(': + if l.mode == STR_IDENTIFIER { + // it is necessary to balance brackets when generating tokens for string templates to know when to change modes + l.openBrackets++ + } l.emitType(TokenParenOpen) case ')': l.emitType(TokenParenClose) + if l.mode == STR_IDENTIFIER { + l.openBrackets-- + if l.openBrackets == 0 { + l.mode = NORMAL + return stringState + } + } case '{': l.emitType(TokenBraceOpen) case '}': @@ -124,6 +129,17 @@ func rootState(l *lexer) stateFn { return numberState case '"': return stringState + case '\\': + if l.mode == STR_IDENTIFIER { + r = l.next() + switch r { + case '(': + l.emitType(TokenStringTemplate) + l.openBrackets++ + } + } else { + return l.error(fmt.Errorf("unrecognized character: %#U", r)) + } case '/': r = l.next() switch r { @@ -302,10 +318,6 @@ func identifierState(l *lexer) stateFn { } } l.emitType(TokenIdentifier) - if l.mode == STR_IDENTIFIER { - l.mode = NORMAL - return stringState - } return rootState } diff --git a/runtime/parser/lexer/tokentype.go b/runtime/parser/lexer/tokentype.go index a7b3cb2f92..d2aa45fa8e 100644 --- a/runtime/parser/lexer/tokentype.go +++ b/runtime/parser/lexer/tokentype.go @@ -207,7 +207,7 @@ func (t TokenType) String() string { case TokenPragma: return `'#'` case TokenStringTemplate: - return `'$'` + return `'\('` default: panic(errors.NewUnreachableError()) } diff --git a/runtime/tests/checker/string_test.go b/runtime/tests/checker/string_test.go index ee2609758f..572c4ebc20 100644 --- a/runtime/tests/checker/string_test.go +++ b/runtime/tests/checker/string_test.go @@ -708,7 +708,7 @@ func TestCheckStringTemplate(t *testing.T) { _, err := ParseAndCheck(t, ` let a = 1 - let x: String = "The value of a is: $a" + let x: String = "The value of a is: \(a)" `) require.NoError(t, err) @@ -720,7 +720,7 @@ func TestCheckStringTemplate(t *testing.T) { _, err := ParseAndCheck(t, ` let a = "abc def" - let x: String = "$a ghi" + let x: String = "\(a) ghi" `) require.NoError(t, err) @@ -734,7 +734,7 @@ func TestCheckStringTemplate(t *testing.T) { access(all) struct SomeStruct {} let a = SomeStruct() - let x: String = "$a" + let x: String = "\(a)" `) require.NoError(t, err) @@ -745,11 +745,30 @@ func TestCheckStringTemplate(t *testing.T) { t.Parallel() _, err := ParseAndCheck(t, ` - let x: String = "$a" + let x: String = "\(a)" `) errs := RequireCheckerErrors(t, err, 1) assert.IsType(t, &sema.NotDeclaredError{}, errs[0]) }) + + t.Run("invalid, resource", func(t *testing.T) { + + t.Parallel() + + _, err := ParseAndCheck(t, ` + access(all) resource TestResource {} + fun test(): String { + var x <- create TestResource() + var y = "\(x)" + destroy x + return y + } + `) + + errs := RequireCheckerErrors(t, err, 1) + + assert.IsType(t, &sema.MissingMoveOperationError{}, errs[0]) + }) } diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index 74e49cf968..3c24c70571 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12292,8 +12292,8 @@ func TestInterpretStringTemplates(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - let x = 123 - let y = "x = $x" + let x = 123 + let y = "x = \(x)" `) AssertValuesEqual( @@ -12314,9 +12314,9 @@ func TestInterpretStringTemplates(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - let x = 123.321 - let y = "abc" - let z = "$y and $x" + let x = 123.321 + let y = "abc" + let z = "\(y) and \(x)" `) AssertValuesEqual( @@ -12331,9 +12331,9 @@ func TestInterpretStringTemplates(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - let x = "{}" - let y = "[$x]" - let z = "($y)" + let x = "{}" + let y = "[\(x)]" + let z = "(\(y))" `) AssertValuesEqual( @@ -12348,10 +12348,10 @@ func TestInterpretStringTemplates(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - access(all) - struct SomeStruct {} - let a = SomeStruct() - let x: String = "$a" + access(all) + struct SomeStruct {} + let a = SomeStruct() + let x: String = "\(a)" `) AssertValuesEqual( @@ -12366,16 +12366,16 @@ func TestInterpretStringTemplates(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - let add = fun(): Int { - return 2+2 - } - let x: String = "$add()" + let add = fun(): Int { + return 2+2 + } + let x: String = "\(add())" `) AssertValuesEqual( t, inter, - interpreter.NewUnmeteredStringValue("fun(): Int()"), + interpreter.NewUnmeteredStringValue("4"), inter.Globals.Get("x").GetValue(inter), ) }) @@ -12384,11 +12384,11 @@ func TestInterpretStringTemplates(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - let add = fun(): Int { - return 2+2 - } - let y = add() - let x: String = "$y" + let add = fun(): Int { + return 2+2 + } + let y = add() + let x: String = "\(y)" `) AssertValuesEqual( @@ -12399,19 +12399,29 @@ func TestInterpretStringTemplates(t *testing.T) { ) }) - t.Run("escaped", func(t *testing.T) { + t.Run("resource reference", func(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - let x = 123 - let y = "x is worth \$$x" + resource R {} + + fun test(): String { + let r <- create R() + let ref = &r as &R + let y = "\(ref)" + destroy r + return y + } `) + value, err := inter.Invoke("test") + require.NoError(t, err) + AssertValuesEqual( t, inter, - interpreter.NewUnmeteredStringValue("x is worth $123"), - inter.Globals.Get("y").GetValue(inter), + interpreter.NewUnmeteredStringValue("S.test.R(uuid: 1)"), + value, ) }) } From e3d152d10540198f04fb77a6b857e7efcfabd099 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Tue, 24 Sep 2024 17:21:17 -0400 Subject: [PATCH 12/21] Restrict supported types in template. --- .../sema/check_string_template_expression.go | 12 +++++ runtime/tests/checker/string_test.go | 45 +++++++++++++------ runtime/tests/interpreter/interpreter_test.go | 38 +++------------- 3 files changed, 48 insertions(+), 47 deletions(-) diff --git a/runtime/sema/check_string_template_expression.go b/runtime/sema/check_string_template_expression.go index 276b8c0f14..e2f865391f 100644 --- a/runtime/sema/check_string_template_expression.go +++ b/runtime/sema/check_string_template_expression.go @@ -37,6 +37,18 @@ func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression * argumentTypes[i] = valueType + // All number types, addresses, path types, bool and strings are supported in string template + if !(IsSubType(valueType, NumberType) || IsSubType(valueType, TheAddressType) || + IsSubType(valueType, PathType) || IsSubType(valueType, StringType) || IsSubType(valueType, BoolType)) { + checker.report( + &TypeMismatchWithDescriptionError{ + ActualType: valueType, + ExpectedTypeDescription: "a type with built-in toString() or bool", + Range: ast.NewRangeFromPositioned(checker.memoryGauge, element), + }, + ) + } + checker.checkResourceMoveOperation(element, valueType) } } diff --git a/runtime/tests/checker/string_test.go b/runtime/tests/checker/string_test.go index 572c4ebc20..313d8f08ee 100644 --- a/runtime/tests/checker/string_test.go +++ b/runtime/tests/checker/string_test.go @@ -707,8 +707,8 @@ func TestCheckStringTemplate(t *testing.T) { t.Parallel() _, err := ParseAndCheck(t, ` - let a = 1 - let x: String = "The value of a is: \(a)" + let a = 1 + let x: String = "The value of a is: \(a)" `) require.NoError(t, err) @@ -719,25 +719,40 @@ func TestCheckStringTemplate(t *testing.T) { t.Parallel() _, err := ParseAndCheck(t, ` - let a = "abc def" - let x: String = "\(a) ghi" + let a = "abc def" + let x: String = "\(a) ghi" `) require.NoError(t, err) }) - t.Run("valid, struct", func(t *testing.T) { + t.Run("invalid, struct", func(t *testing.T) { t.Parallel() _, err := ParseAndCheck(t, ` - access(all) - struct SomeStruct {} - let a = SomeStruct() - let x: String = "\(a)" + access(all) + struct SomeStruct {} + let a = SomeStruct() + let x: String = "\(a)" `) - require.NoError(t, err) + errs := RequireCheckerErrors(t, err, 1) + + assert.IsType(t, &sema.TypeMismatchWithDescriptionError{}, errs[0]) + }) + + t.Run("invalid, array", func(t *testing.T) { + t.Parallel() + + _, err := ParseAndCheck(t, ` + let x :[AnyStruct] = ["tmp", 1] + let y = "\(x)" + `) + + errs := RequireCheckerErrors(t, err, 1) + + assert.IsType(t, &sema.TypeMismatchWithDescriptionError{}, errs[0]) }) t.Run("invalid, missing variable", func(t *testing.T) { @@ -745,12 +760,13 @@ func TestCheckStringTemplate(t *testing.T) { t.Parallel() _, err := ParseAndCheck(t, ` - let x: String = "\(a)" + let x: String = "\(a)" `) - errs := RequireCheckerErrors(t, err, 1) + errs := RequireCheckerErrors(t, err, 2) assert.IsType(t, &sema.NotDeclaredError{}, errs[0]) + assert.IsType(t, &sema.TypeMismatchWithDescriptionError{}, errs[1]) }) t.Run("invalid, resource", func(t *testing.T) { @@ -767,8 +783,9 @@ func TestCheckStringTemplate(t *testing.T) { } `) - errs := RequireCheckerErrors(t, err, 1) + errs := RequireCheckerErrors(t, err, 2) - assert.IsType(t, &sema.MissingMoveOperationError{}, errs[0]) + assert.IsType(t, &sema.TypeMismatchWithDescriptionError{}, errs[0]) + assert.IsType(t, &sema.MissingMoveOperationError{}, errs[1]) }) } diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index 3c24c70571..69f203652f 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12344,21 +12344,19 @@ func TestInterpretStringTemplates(t *testing.T) { ) }) - t.Run("struct", func(t *testing.T) { + t.Run("boolean", func(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` - access(all) - struct SomeStruct {} - let a = SomeStruct() - let x: String = "\(a)" + let x = false + let y = "\(x)" `) AssertValuesEqual( t, inter, - interpreter.NewUnmeteredStringValue("S.test.SomeStruct()"), - inter.Globals.Get("x").GetValue(inter), + interpreter.NewUnmeteredStringValue("false"), + inter.Globals.Get("y").GetValue(inter), ) }) @@ -12398,30 +12396,4 @@ func TestInterpretStringTemplates(t *testing.T) { inter.Globals.Get("x").GetValue(inter), ) }) - - t.Run("resource reference", func(t *testing.T) { - t.Parallel() - - inter := parseCheckAndInterpret(t, ` - resource R {} - - fun test(): String { - let r <- create R() - let ref = &r as &R - let y = "\(ref)" - destroy r - return y - } - `) - - value, err := inter.Invoke("test") - require.NoError(t, err) - - AssertValuesEqual( - t, - inter, - interpreter.NewUnmeteredStringValue("S.test.R(uuid: 1)"), - value, - ) - }) } From 1ec5b7eb7c230831d7927cf1476e583945bae4a4 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 25 Sep 2024 10:26:04 -0400 Subject: [PATCH 13/21] Limit string templates to identifiers only. --- runtime/parser/expression.go | 10 +-- runtime/parser/expression_test.go | 64 +++++++++++++++---- runtime/tests/interpreter/interpreter_test.go | 20 +----- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 3537de9b93..395a6befa3 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1184,15 +1184,17 @@ func defineStringExpression() { // parser already points to next token curToken = p.current if curToken.Is(lexer.TokenStringTemplate) { - p.next() // advance to the expression - if !p.current.Is(lexer.TokenIdentifier) { - return nil, p.syntaxError("expected an identifier got: %s", p.currentTokenSource()) - } + p.next() value, err := parseExpression(p, lowestBindingPower) + // consider invalid expression first if err != nil { return nil, err } + // limit string templates to identifiers only + if _, ok := value.(*ast.IdentifierExpression); !ok { + return nil, p.syntaxError("expected identifier got: %s", value.String()) + } _, err = p.mustOne(lexer.TokenParenClose) if err != nil { return nil, err diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index 2f78ddfe59..3804c13ce6 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6190,8 +6190,8 @@ func TestParseStringTemplate(t *testing.T) { utils.AssertEqualWithDiff(t, []error{ &SyntaxError{ - Message: "expected an identifier got: .", - Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, + Message: "unexpected token in expression: '.'", + Pos: ast.Position{Offset: 9, Line: 2, Column: 8}, }, }, errs, @@ -6217,19 +6217,19 @@ func TestParseStringTemplate(t *testing.T) { utils.AssertEqualWithDiff(t, []error{ &SyntaxError{ - Message: "expected an identifier got: 2", - Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, + Message: "expected identifier got: 2 + 2", + Pos: ast.Position{Offset: 13, Line: 2, Column: 12}, }, }, errs, ) }) - t.Run("invalid, nested identifier", func(t *testing.T) { + t.Run("valid, nested identifier", func(t *testing.T) { t.Parallel() - _, errs := testParseExpression(` + actual, errs := testParseExpression(` "\((a))" `) @@ -6240,23 +6240,63 @@ func TestParseStringTemplate(t *testing.T) { } } + require.NoError(t, err) + + expected := &ast.StringTemplateExpression{ + Values: []string{ + "", + "", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{Offset: 9, Line: 2, Column: 8}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 5, Line: 2, Column: 4}, + EndPos: ast.Position{Offset: 12, Line: 2, Column: 11}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) + + }) + t.Run("invalid, empty", func(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "\()" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + require.Error(t, err) utils.AssertEqualWithDiff(t, []error{ &SyntaxError{ - Message: "expected an identifier got: (", - Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, + Message: "unexpected token in expression: ')'", + Pos: ast.Position{Offset: 9, Line: 2, Column: 8}, }, }, errs, ) }) - t.Run("invalid, empty", func(t *testing.T) { + + t.Run("invalid, function identifier", func(t *testing.T) { t.Parallel() _, errs := testParseExpression(` - "\()" + "\(add())" `) var err error @@ -6270,8 +6310,8 @@ func TestParseStringTemplate(t *testing.T) { utils.AssertEqualWithDiff(t, []error{ &SyntaxError{ - Message: "expected an identifier got: )", - Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, + Message: "expected identifier got: add()", + Pos: ast.Position{Offset: 12, Line: 2, Column: 11}, }, }, errs, diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index 69f203652f..728011fbd7 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12360,25 +12360,7 @@ func TestInterpretStringTemplates(t *testing.T) { ) }) - t.Run("func", func(t *testing.T) { - t.Parallel() - - inter := parseCheckAndInterpret(t, ` - let add = fun(): Int { - return 2+2 - } - let x: String = "\(add())" - `) - - AssertValuesEqual( - t, - inter, - interpreter.NewUnmeteredStringValue("4"), - inter.Globals.Get("x").GetValue(inter), - ) - }) - - t.Run("func", func(t *testing.T) { + t.Run("func extracted", func(t *testing.T) { t.Parallel() inter := parseCheckAndInterpret(t, ` From 1ae7fe9123e2c2160abcba72a19fd9f04e1ed661 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 25 Sep 2024 13:25:20 -0400 Subject: [PATCH 14/21] Clean up changes. --- runtime/ast/expression.go | 1 - runtime/interpreter/interpreter_expression.go | 3 +-- runtime/parser/expression.go | 9 ++++----- runtime/sema/check_string_template_expression.go | 8 ++++---- runtime/tests/interpreter/interpreter_test.go | 16 ++++++++++++++++ 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 2efc783155..2e2ee2f7f2 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -231,7 +231,6 @@ type StringTemplateExpression struct { var _ Expression = &StringTemplateExpression{} func NewStringTemplateExpression(gauge common.MemoryGauge, values []string, exprs []Expression, exprRange Range) *StringTemplateExpression { - // STRINGTODO: change to be similar to array memory usage? common.UseMemory(gauge, common.StringExpressionMemoryUsage) return &StringTemplateExpression{ Values: values, diff --git a/runtime/interpreter/interpreter_expression.go b/runtime/interpreter/interpreter_expression.go index 264b39edf8..05272cb606 100644 --- a/runtime/interpreter/interpreter_expression.go +++ b/runtime/interpreter/interpreter_expression.go @@ -968,7 +968,7 @@ func (interpreter *Interpreter) VisitStringTemplateExpression(expression *ast.St for i, str := range expression.Values { builder.WriteString(str) if i < len(values) { - // STRINGTODO: is this how the conversion should happen? + // this is equivalent to toString() for supported types s := values[i].String() switch argumentTypes[i] { case sema.StringType: @@ -981,7 +981,6 @@ func (interpreter *Interpreter) VisitStringTemplateExpression(expression *ast.St } } - // STRINGTODO: already metered as a string constant in parser? return NewUnmeteredStringValue(builder.String()) } diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 395a6befa3..78a53c77ad 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1141,7 +1141,7 @@ func defineStringExpression() { curToken := startToken endToken := startToken - // early check for start " of string literal because of string templates + // check for start " of string literal literal := p.tokenSource(curToken) length := len(literal) if length == 0 { @@ -1160,13 +1160,14 @@ func defineStringExpression() { } } - // flag for late end " check + // flag for ending " check missingEnd := true for curToken.Is(lexer.TokenString) { literal = p.tokenSource(curToken) length = len(literal) + // remove quotation marks if they exist if curToken == startToken { literal = literal[1:] length = len(literal) @@ -1208,7 +1209,7 @@ func defineStringExpression() { } } - // late check for end " of string literal because of string templates + // check for end " of string literal if missingEnd { p.reportSyntaxError("invalid end of string literal: missing '\"'") } @@ -1741,8 +1742,6 @@ func parseStringLiteralContent(p *parser, s []byte) (result string) { builder.WriteByte('\t') case '"': builder.WriteByte('"') - case '$': - builder.WriteByte('$') case '\'': builder.WriteByte('\'') case '\\': diff --git a/runtime/sema/check_string_template_expression.go b/runtime/sema/check_string_template_expression.go index e2f865391f..7a7eac8b3d 100644 --- a/runtime/sema/check_string_template_expression.go +++ b/runtime/sema/check_string_template_expression.go @@ -38,8 +38,10 @@ func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression * argumentTypes[i] = valueType // All number types, addresses, path types, bool and strings are supported in string template - if !(IsSubType(valueType, NumberType) || IsSubType(valueType, TheAddressType) || - IsSubType(valueType, PathType) || IsSubType(valueType, StringType) || IsSubType(valueType, BoolType)) { + if IsSubType(valueType, NumberType) || IsSubType(valueType, TheAddressType) || + IsSubType(valueType, PathType) || IsSubType(valueType, StringType) || IsSubType(valueType, BoolType) { + checker.checkResourceMoveOperation(element, valueType) + } else { checker.report( &TypeMismatchWithDescriptionError{ ActualType: valueType, @@ -48,8 +50,6 @@ func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression * }, ) } - - checker.checkResourceMoveOperation(element, valueType) } } diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index 728011fbd7..855888eea0 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12378,4 +12378,20 @@ func TestInterpretStringTemplates(t *testing.T) { inter.Globals.Get("x").GetValue(inter), ) }) + + t.Run("path expr", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let a = /public/foo + let x = "file at \(a)" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("file at /public/foo"), + inter.Globals.Get("x").GetValue(inter), + ) + }) } From 8799cac377654e613af872b7ed8e043cfa0f7e06 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 25 Sep 2024 13:39:43 -0400 Subject: [PATCH 15/21] Fix checker test. --- runtime/tests/checker/string_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runtime/tests/checker/string_test.go b/runtime/tests/checker/string_test.go index 313d8f08ee..4bea07a7dd 100644 --- a/runtime/tests/checker/string_test.go +++ b/runtime/tests/checker/string_test.go @@ -783,9 +783,8 @@ func TestCheckStringTemplate(t *testing.T) { } `) - errs := RequireCheckerErrors(t, err, 2) + errs := RequireCheckerErrors(t, err, 1) assert.IsType(t, &sema.TypeMismatchWithDescriptionError{}, errs[0]) - assert.IsType(t, &sema.MissingMoveOperationError{}, errs[1]) }) } From bda6b28fb29cff50e635a3e9d94b926290001b33 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 2 Oct 2024 10:58:51 -0400 Subject: [PATCH 16/21] Refactor code from review. --- runtime/ast/expression.go | 9 +- runtime/common/memorykind.go | 1 + runtime/common/memorykind_string.go | 95 ++++++++++--------- runtime/common/metering.go | 7 ++ runtime/interpreter/interpreter_expression.go | 18 ++-- runtime/parser/expression_test.go | 54 +++++++++++ runtime/parser/lexer/lexer.go | 10 +- runtime/parser/lexer/state.go | 6 +- .../sema/check_string_template_expression.go | 12 ++- runtime/tests/interpreter/interpreter_test.go | 18 ++++ 10 files changed, 157 insertions(+), 73 deletions(-) diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 2e2ee2f7f2..b8bf93e86d 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -230,8 +230,13 @@ type StringTemplateExpression struct { var _ Expression = &StringTemplateExpression{} -func NewStringTemplateExpression(gauge common.MemoryGauge, values []string, exprs []Expression, exprRange Range) *StringTemplateExpression { - common.UseMemory(gauge, common.StringExpressionMemoryUsage) +func NewStringTemplateExpression( + gauge common.MemoryGauge, + values []string, + exprs []Expression, + exprRange Range, +) *StringTemplateExpression { + common.UseMemory(gauge, common.NewStringTemplateExpressionMemoryUsage(len(values)+len(exprs))) return &StringTemplateExpression{ Values: values, Expressions: exprs, diff --git a/runtime/common/memorykind.go b/runtime/common/memorykind.go index 06e6748e9a..0754c397f5 100644 --- a/runtime/common/memorykind.go +++ b/runtime/common/memorykind.go @@ -204,6 +204,7 @@ const ( MemoryKindIntegerExpression MemoryKindFixedPointExpression MemoryKindArrayExpression + MemoryKindStringTemplateExpression MemoryKindDictionaryExpression MemoryKindIdentifierExpression MemoryKindInvocationExpression diff --git a/runtime/common/memorykind_string.go b/runtime/common/memorykind_string.go index 9b022e9495..67d6a0b8f8 100644 --- a/runtime/common/memorykind_string.go +++ b/runtime/common/memorykind_string.go @@ -165,56 +165,57 @@ func _() { _ = x[MemoryKindIntegerExpression-154] _ = x[MemoryKindFixedPointExpression-155] _ = x[MemoryKindArrayExpression-156] - _ = x[MemoryKindDictionaryExpression-157] - _ = x[MemoryKindIdentifierExpression-158] - _ = x[MemoryKindInvocationExpression-159] - _ = x[MemoryKindMemberExpression-160] - _ = x[MemoryKindIndexExpression-161] - _ = x[MemoryKindConditionalExpression-162] - _ = x[MemoryKindUnaryExpression-163] - _ = x[MemoryKindBinaryExpression-164] - _ = x[MemoryKindFunctionExpression-165] - _ = x[MemoryKindCastingExpression-166] - _ = x[MemoryKindCreateExpression-167] - _ = x[MemoryKindDestroyExpression-168] - _ = x[MemoryKindReferenceExpression-169] - _ = x[MemoryKindForceExpression-170] - _ = x[MemoryKindPathExpression-171] - _ = x[MemoryKindAttachExpression-172] - _ = x[MemoryKindConstantSizedType-173] - _ = x[MemoryKindDictionaryType-174] - _ = x[MemoryKindFunctionType-175] - _ = x[MemoryKindInstantiationType-176] - _ = x[MemoryKindNominalType-177] - _ = x[MemoryKindOptionalType-178] - _ = x[MemoryKindReferenceType-179] - _ = x[MemoryKindIntersectionType-180] - _ = x[MemoryKindVariableSizedType-181] - _ = x[MemoryKindPosition-182] - _ = x[MemoryKindRange-183] - _ = x[MemoryKindElaboration-184] - _ = x[MemoryKindActivation-185] - _ = x[MemoryKindActivationEntries-186] - _ = x[MemoryKindVariableSizedSemaType-187] - _ = x[MemoryKindConstantSizedSemaType-188] - _ = x[MemoryKindDictionarySemaType-189] - _ = x[MemoryKindOptionalSemaType-190] - _ = x[MemoryKindIntersectionSemaType-191] - _ = x[MemoryKindReferenceSemaType-192] - _ = x[MemoryKindEntitlementSemaType-193] - _ = x[MemoryKindEntitlementMapSemaType-194] - _ = x[MemoryKindEntitlementRelationSemaType-195] - _ = x[MemoryKindCapabilitySemaType-196] - _ = x[MemoryKindInclusiveRangeSemaType-197] - _ = x[MemoryKindOrderedMap-198] - _ = x[MemoryKindOrderedMapEntryList-199] - _ = x[MemoryKindOrderedMapEntry-200] - _ = x[MemoryKindLast-201] + _ = x[MemoryKindStringTemplateExpression-157] + _ = x[MemoryKindDictionaryExpression-158] + _ = x[MemoryKindIdentifierExpression-159] + _ = x[MemoryKindInvocationExpression-160] + _ = x[MemoryKindMemberExpression-161] + _ = x[MemoryKindIndexExpression-162] + _ = x[MemoryKindConditionalExpression-163] + _ = x[MemoryKindUnaryExpression-164] + _ = x[MemoryKindBinaryExpression-165] + _ = x[MemoryKindFunctionExpression-166] + _ = x[MemoryKindCastingExpression-167] + _ = x[MemoryKindCreateExpression-168] + _ = x[MemoryKindDestroyExpression-169] + _ = x[MemoryKindReferenceExpression-170] + _ = x[MemoryKindForceExpression-171] + _ = x[MemoryKindPathExpression-172] + _ = x[MemoryKindAttachExpression-173] + _ = x[MemoryKindConstantSizedType-174] + _ = x[MemoryKindDictionaryType-175] + _ = x[MemoryKindFunctionType-176] + _ = x[MemoryKindInstantiationType-177] + _ = x[MemoryKindNominalType-178] + _ = x[MemoryKindOptionalType-179] + _ = x[MemoryKindReferenceType-180] + _ = x[MemoryKindIntersectionType-181] + _ = x[MemoryKindVariableSizedType-182] + _ = x[MemoryKindPosition-183] + _ = x[MemoryKindRange-184] + _ = x[MemoryKindElaboration-185] + _ = x[MemoryKindActivation-186] + _ = x[MemoryKindActivationEntries-187] + _ = x[MemoryKindVariableSizedSemaType-188] + _ = x[MemoryKindConstantSizedSemaType-189] + _ = x[MemoryKindDictionarySemaType-190] + _ = x[MemoryKindOptionalSemaType-191] + _ = x[MemoryKindIntersectionSemaType-192] + _ = x[MemoryKindReferenceSemaType-193] + _ = x[MemoryKindEntitlementSemaType-194] + _ = x[MemoryKindEntitlementMapSemaType-195] + _ = x[MemoryKindEntitlementRelationSemaType-196] + _ = x[MemoryKindCapabilitySemaType-197] + _ = x[MemoryKindInclusiveRangeSemaType-198] + _ = x[MemoryKindOrderedMap-199] + _ = x[MemoryKindOrderedMapEntryList-200] + _ = x[MemoryKindOrderedMapEntry-201] + _ = x[MemoryKindLast-202] } -const _MemoryKind_name = "UnknownAddressValueStringValueCharacterValueNumberValueArrayValueBaseDictionaryValueBaseCompositeValueBaseSimpleCompositeValueBaseOptionalValueTypeValuePathValueCapabilityValueStorageReferenceValueEphemeralReferenceValueInterpretedFunctionValueHostFunctionValueBoundFunctionValueBigIntSimpleCompositeValuePublishedValueStorageCapabilityControllerValueAccountCapabilityControllerValueAtreeArrayDataSlabAtreeArrayMetaDataSlabAtreeArrayElementOverheadAtreeMapDataSlabAtreeMapMetaDataSlabAtreeMapElementOverheadAtreeMapPreAllocatedElementAtreeEncodedSlabPrimitiveStaticTypeCompositeStaticTypeInterfaceStaticTypeVariableSizedStaticTypeConstantSizedStaticTypeDictionaryStaticTypeInclusiveRangeStaticTypeOptionalStaticTypeIntersectionStaticTypeEntitlementSetStaticAccessEntitlementMapStaticAccessReferenceStaticTypeCapabilityStaticTypeFunctionStaticTypeCadenceVoidValueCadenceOptionalValueCadenceBoolValueCadenceStringValueCadenceCharacterValueCadenceAddressValueCadenceIntValueCadenceNumberValueCadenceArrayValueBaseCadenceArrayValueLengthCadenceDictionaryValueCadenceInclusiveRangeValueCadenceKeyValuePairCadenceStructValueBaseCadenceStructValueSizeCadenceResourceValueBaseCadenceAttachmentValueBaseCadenceResourceValueSizeCadenceAttachmentValueSizeCadenceEventValueBaseCadenceEventValueSizeCadenceContractValueBaseCadenceContractValueSizeCadenceEnumValueBaseCadenceEnumValueSizeCadencePathValueCadenceTypeValueCadenceCapabilityValueCadenceDeprecatedPathCapabilityTypeCadenceFunctionValueCadenceOptionalTypeCadenceDeprecatedRestrictedTypeCadenceVariableSizedArrayTypeCadenceConstantSizedArrayTypeCadenceDictionaryTypeCadenceInclusiveRangeTypeCadenceFieldCadenceParameterCadenceTypeParameterCadenceStructTypeCadenceResourceTypeCadenceAttachmentTypeCadenceEventTypeCadenceContractTypeCadenceStructInterfaceTypeCadenceResourceInterfaceTypeCadenceContractInterfaceTypeCadenceFunctionTypeCadenceEntitlementSetAccessCadenceEntitlementMapAccessCadenceReferenceTypeCadenceIntersectionTypeCadenceCapabilityTypeCadenceEnumTypeRawStringAddressLocationBytesVariableCompositeTypeInfoCompositeFieldInvocationStorageMapStorageKeyTypeTokenErrorTokenSpaceTokenProgramIdentifierArgumentBlockFunctionBlockParameterParameterListTypeParameterTypeParameterListTransferMembersTypeAnnotationDictionaryEntryFunctionDeclarationCompositeDeclarationAttachmentDeclarationInterfaceDeclarationEntitlementDeclarationEntitlementMappingElementEntitlementMappingDeclarationEnumCaseDeclarationFieldDeclarationTransactionDeclarationImportDeclarationVariableDeclarationSpecialFunctionDeclarationPragmaDeclarationAssignmentStatementBreakStatementContinueStatementEmitStatementExpressionStatementForStatementIfStatementReturnStatementSwapStatementSwitchStatementWhileStatementRemoveStatementBooleanExpressionVoidExpressionNilExpressionStringExpressionIntegerExpressionFixedPointExpressionArrayExpressionDictionaryExpressionIdentifierExpressionInvocationExpressionMemberExpressionIndexExpressionConditionalExpressionUnaryExpressionBinaryExpressionFunctionExpressionCastingExpressionCreateExpressionDestroyExpressionReferenceExpressionForceExpressionPathExpressionAttachExpressionConstantSizedTypeDictionaryTypeFunctionTypeInstantiationTypeNominalTypeOptionalTypeReferenceTypeIntersectionTypeVariableSizedTypePositionRangeElaborationActivationActivationEntriesVariableSizedSemaTypeConstantSizedSemaTypeDictionarySemaTypeOptionalSemaTypeIntersectionSemaTypeReferenceSemaTypeEntitlementSemaTypeEntitlementMapSemaTypeEntitlementRelationSemaTypeCapabilitySemaTypeInclusiveRangeSemaTypeOrderedMapOrderedMapEntryListOrderedMapEntryLast" +const _MemoryKind_name = "UnknownAddressValueStringValueCharacterValueNumberValueArrayValueBaseDictionaryValueBaseCompositeValueBaseSimpleCompositeValueBaseOptionalValueTypeValuePathValueCapabilityValueStorageReferenceValueEphemeralReferenceValueInterpretedFunctionValueHostFunctionValueBoundFunctionValueBigIntSimpleCompositeValuePublishedValueStorageCapabilityControllerValueAccountCapabilityControllerValueAtreeArrayDataSlabAtreeArrayMetaDataSlabAtreeArrayElementOverheadAtreeMapDataSlabAtreeMapMetaDataSlabAtreeMapElementOverheadAtreeMapPreAllocatedElementAtreeEncodedSlabPrimitiveStaticTypeCompositeStaticTypeInterfaceStaticTypeVariableSizedStaticTypeConstantSizedStaticTypeDictionaryStaticTypeInclusiveRangeStaticTypeOptionalStaticTypeIntersectionStaticTypeEntitlementSetStaticAccessEntitlementMapStaticAccessReferenceStaticTypeCapabilityStaticTypeFunctionStaticTypeCadenceVoidValueCadenceOptionalValueCadenceBoolValueCadenceStringValueCadenceCharacterValueCadenceAddressValueCadenceIntValueCadenceNumberValueCadenceArrayValueBaseCadenceArrayValueLengthCadenceDictionaryValueCadenceInclusiveRangeValueCadenceKeyValuePairCadenceStructValueBaseCadenceStructValueSizeCadenceResourceValueBaseCadenceAttachmentValueBaseCadenceResourceValueSizeCadenceAttachmentValueSizeCadenceEventValueBaseCadenceEventValueSizeCadenceContractValueBaseCadenceContractValueSizeCadenceEnumValueBaseCadenceEnumValueSizeCadencePathValueCadenceTypeValueCadenceCapabilityValueCadenceDeprecatedPathCapabilityTypeCadenceFunctionValueCadenceOptionalTypeCadenceDeprecatedRestrictedTypeCadenceVariableSizedArrayTypeCadenceConstantSizedArrayTypeCadenceDictionaryTypeCadenceInclusiveRangeTypeCadenceFieldCadenceParameterCadenceTypeParameterCadenceStructTypeCadenceResourceTypeCadenceAttachmentTypeCadenceEventTypeCadenceContractTypeCadenceStructInterfaceTypeCadenceResourceInterfaceTypeCadenceContractInterfaceTypeCadenceFunctionTypeCadenceEntitlementSetAccessCadenceEntitlementMapAccessCadenceReferenceTypeCadenceIntersectionTypeCadenceCapabilityTypeCadenceEnumTypeRawStringAddressLocationBytesVariableCompositeTypeInfoCompositeFieldInvocationStorageMapStorageKeyTypeTokenErrorTokenSpaceTokenProgramIdentifierArgumentBlockFunctionBlockParameterParameterListTypeParameterTypeParameterListTransferMembersTypeAnnotationDictionaryEntryFunctionDeclarationCompositeDeclarationAttachmentDeclarationInterfaceDeclarationEntitlementDeclarationEntitlementMappingElementEntitlementMappingDeclarationEnumCaseDeclarationFieldDeclarationTransactionDeclarationImportDeclarationVariableDeclarationSpecialFunctionDeclarationPragmaDeclarationAssignmentStatementBreakStatementContinueStatementEmitStatementExpressionStatementForStatementIfStatementReturnStatementSwapStatementSwitchStatementWhileStatementRemoveStatementBooleanExpressionVoidExpressionNilExpressionStringExpressionIntegerExpressionFixedPointExpressionArrayExpressionStringTemplateExpressionDictionaryExpressionIdentifierExpressionInvocationExpressionMemberExpressionIndexExpressionConditionalExpressionUnaryExpressionBinaryExpressionFunctionExpressionCastingExpressionCreateExpressionDestroyExpressionReferenceExpressionForceExpressionPathExpressionAttachExpressionConstantSizedTypeDictionaryTypeFunctionTypeInstantiationTypeNominalTypeOptionalTypeReferenceTypeIntersectionTypeVariableSizedTypePositionRangeElaborationActivationActivationEntriesVariableSizedSemaTypeConstantSizedSemaTypeDictionarySemaTypeOptionalSemaTypeIntersectionSemaTypeReferenceSemaTypeEntitlementSemaTypeEntitlementMapSemaTypeEntitlementRelationSemaTypeCapabilitySemaTypeInclusiveRangeSemaTypeOrderedMapOrderedMapEntryListOrderedMapEntryLast" -var _MemoryKind_index = [...]uint16{0, 7, 19, 30, 44, 55, 69, 88, 106, 130, 143, 152, 161, 176, 197, 220, 244, 261, 279, 285, 305, 319, 351, 383, 401, 423, 448, 464, 484, 507, 534, 550, 569, 588, 607, 630, 653, 673, 697, 715, 737, 763, 789, 808, 828, 846, 862, 882, 898, 916, 937, 956, 971, 989, 1010, 1033, 1055, 1081, 1100, 1122, 1144, 1168, 1194, 1218, 1244, 1265, 1286, 1310, 1334, 1354, 1374, 1390, 1406, 1428, 1463, 1483, 1502, 1533, 1562, 1591, 1612, 1637, 1649, 1665, 1685, 1702, 1721, 1742, 1758, 1777, 1803, 1831, 1859, 1878, 1905, 1932, 1952, 1975, 1996, 2011, 2020, 2035, 2040, 2048, 2065, 2079, 2089, 2099, 2109, 2118, 2128, 2138, 2145, 2155, 2163, 2168, 2181, 2190, 2203, 2216, 2233, 2241, 2248, 2262, 2277, 2296, 2316, 2337, 2357, 2379, 2404, 2433, 2452, 2468, 2490, 2507, 2526, 2552, 2569, 2588, 2602, 2619, 2632, 2651, 2663, 2674, 2689, 2702, 2717, 2731, 2746, 2763, 2777, 2790, 2806, 2823, 2843, 2858, 2878, 2898, 2918, 2934, 2949, 2970, 2985, 3001, 3019, 3036, 3052, 3069, 3088, 3103, 3117, 3133, 3150, 3164, 3176, 3193, 3204, 3216, 3229, 3245, 3262, 3270, 3275, 3286, 3296, 3313, 3334, 3355, 3373, 3389, 3409, 3426, 3445, 3467, 3494, 3512, 3534, 3544, 3563, 3578, 3582} +var _MemoryKind_index = [...]uint16{0, 7, 19, 30, 44, 55, 69, 88, 106, 130, 143, 152, 161, 176, 197, 220, 244, 261, 279, 285, 305, 319, 351, 383, 401, 423, 448, 464, 484, 507, 534, 550, 569, 588, 607, 630, 653, 673, 697, 715, 737, 763, 789, 808, 828, 846, 862, 882, 898, 916, 937, 956, 971, 989, 1010, 1033, 1055, 1081, 1100, 1122, 1144, 1168, 1194, 1218, 1244, 1265, 1286, 1310, 1334, 1354, 1374, 1390, 1406, 1428, 1463, 1483, 1502, 1533, 1562, 1591, 1612, 1637, 1649, 1665, 1685, 1702, 1721, 1742, 1758, 1777, 1803, 1831, 1859, 1878, 1905, 1932, 1952, 1975, 1996, 2011, 2020, 2035, 2040, 2048, 2065, 2079, 2089, 2099, 2109, 2118, 2128, 2138, 2145, 2155, 2163, 2168, 2181, 2190, 2203, 2216, 2233, 2241, 2248, 2262, 2277, 2296, 2316, 2337, 2357, 2379, 2404, 2433, 2452, 2468, 2490, 2507, 2526, 2552, 2569, 2588, 2602, 2619, 2632, 2651, 2663, 2674, 2689, 2702, 2717, 2731, 2746, 2763, 2777, 2790, 2806, 2823, 2843, 2858, 2882, 2902, 2922, 2942, 2958, 2973, 2994, 3009, 3025, 3043, 3060, 3076, 3093, 3112, 3127, 3141, 3157, 3174, 3188, 3200, 3217, 3228, 3240, 3253, 3269, 3286, 3294, 3299, 3310, 3320, 3337, 3358, 3379, 3397, 3413, 3433, 3450, 3469, 3491, 3518, 3536, 3558, 3568, 3587, 3602, 3606} func (i MemoryKind) String() string { if i >= MemoryKind(len(_MemoryKind_index)-1) { diff --git a/runtime/common/metering.go b/runtime/common/metering.go index a4b4afc2dd..e6e6249571 100644 --- a/runtime/common/metering.go +++ b/runtime/common/metering.go @@ -795,6 +795,13 @@ func NewArrayExpressionMemoryUsage(length int) MemoryUsage { } } +func NewStringTemplateExpressionMemoryUsage(length int) MemoryUsage { + return MemoryUsage{ + Kind: MemoryKindStringTemplateExpression, + Amount: uint64(length), + } +} + func NewDictionaryExpressionMemoryUsage(length int) MemoryUsage { return MemoryUsage{ Kind: MemoryKindDictionaryExpression, diff --git a/runtime/interpreter/interpreter_expression.go b/runtime/interpreter/interpreter_expression.go index 05272cb606..13e2f192e3 100644 --- a/runtime/interpreter/interpreter_expression.go +++ b/runtime/interpreter/interpreter_expression.go @@ -961,22 +961,18 @@ func (interpreter *Interpreter) VisitStringExpression(expression *ast.StringExpr func (interpreter *Interpreter) VisitStringTemplateExpression(expression *ast.StringTemplateExpression) Value { values := interpreter.visitExpressionsNonCopying(expression.Expressions) - templatesType := interpreter.Program.Elaboration.StringTemplateExpressionTypes(expression) - argumentTypes := templatesType.ArgumentTypes - var builder strings.Builder for i, str := range expression.Values { builder.WriteString(str) if i < len(values) { - // this is equivalent to toString() for supported types - s := values[i].String() - switch argumentTypes[i] { - case sema.StringType: - // remove quotations - s = s[1 : len(s)-1] - builder.WriteString(s) + // switch on value instead of type + switch value := values[i].(type) { + case *StringValue: + builder.WriteString(value.Str) + case CharacterValue: + builder.WriteString(value.Str) default: - builder.WriteString(s) + builder.WriteString(value.String()) } } } diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index 3804c13ce6..8f68a7b468 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6317,6 +6317,60 @@ func TestParseStringTemplate(t *testing.T) { errs, ) }) + + t.Run("unbalanced paren", func(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "\(add" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.Error(t, err) + utils.AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "expected token ')'", + Pos: ast.Position{Offset: 10, Line: 2, Column: 9}, + }, + }, + errs, + ) + }) + + t.Run("nested templates", func(t *testing.T) { + + t.Parallel() + + _, errs := testParseExpression(` + "outer string \( "\(inner template)" )" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.Error(t, err) + utils.AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "expected token ')'", + Pos: ast.Position{Offset: 30, Line: 2, Column: 29}, + }, + }, + errs, + ) + }) } func TestParseNilCoalescing(t *testing.T) { diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 6b9c0714b8..25e7635594 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -49,11 +49,11 @@ type position struct { column int } -type LexerMode int +type lexerMode uint8 const ( - NORMAL = iota - STR_IDENTIFIER + NORMAL lexerMode = iota + STR_INTERPOLATION ) type lexer struct { @@ -82,7 +82,7 @@ type lexer struct { // canBackup indicates whether stepping back is allowed canBackup bool // lexer mode is used for string templates - mode LexerMode + mode lexerMode // counts the number of unclosed brackets for string templates \((())) openBrackets int } @@ -427,7 +427,7 @@ func (l *lexer) scanString(quote rune) { switch r { case '(': // string template, stop and set mode - l.mode = STR_IDENTIFIER + l.mode = STR_INTERPOLATION // no need to update prev values because these next tokens will not backup l.endOffset = tmpBackupOffset l.current = tmpBackup diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index f2775b81dc..b536111c86 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -56,14 +56,14 @@ func rootState(l *lexer) stateFn { case '%': l.emitType(TokenPercent) case '(': - if l.mode == STR_IDENTIFIER { + if l.mode == STR_INTERPOLATION { // it is necessary to balance brackets when generating tokens for string templates to know when to change modes l.openBrackets++ } l.emitType(TokenParenOpen) case ')': l.emitType(TokenParenClose) - if l.mode == STR_IDENTIFIER { + if l.mode == STR_INTERPOLATION { l.openBrackets-- if l.openBrackets == 0 { l.mode = NORMAL @@ -130,7 +130,7 @@ func rootState(l *lexer) stateFn { case '"': return stringState case '\\': - if l.mode == STR_IDENTIFIER { + if l.mode == STR_INTERPOLATION { r = l.next() switch r { case '(': diff --git a/runtime/sema/check_string_template_expression.go b/runtime/sema/check_string_template_expression.go index 7a7eac8b3d..ddb0d6ed0d 100644 --- a/runtime/sema/check_string_template_expression.go +++ b/runtime/sema/check_string_template_expression.go @@ -20,6 +20,12 @@ package sema import "github.com/onflow/cadence/runtime/ast" +// All number types, addresses, path types, bool, strings and characters are supported in string template +func isValidStringTemplateValue(valueType Type) bool { + return valueType == TheAddressType || valueType == StringType || valueType == BoolType || valueType == CharacterType || + IsSubType(valueType, NumberType) || IsSubType(valueType, PathType) +} + func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression *ast.StringTemplateExpression) Type { // visit all elements @@ -37,11 +43,7 @@ func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression * argumentTypes[i] = valueType - // All number types, addresses, path types, bool and strings are supported in string template - if IsSubType(valueType, NumberType) || IsSubType(valueType, TheAddressType) || - IsSubType(valueType, PathType) || IsSubType(valueType, StringType) || IsSubType(valueType, BoolType) { - checker.checkResourceMoveOperation(element, valueType) - } else { + if !isValidStringTemplateValue(valueType) { checker.report( &TypeMismatchWithDescriptionError{ ActualType: valueType, diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index 855888eea0..bf1f3a5c30 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -12394,4 +12394,22 @@ func TestInterpretStringTemplates(t *testing.T) { inter.Globals.Get("x").GetValue(inter), ) }) + + t.Run("consecutive", func(t *testing.T) { + t.Parallel() + + inter := parseCheckAndInterpret(t, ` + let c = "C" + let a: Character = "A" + let n = "N" + let x = "\(c)\(a)\(n)" + `) + + AssertValuesEqual( + t, + inter, + interpreter.NewUnmeteredStringValue("CAN"), + inter.Globals.Get("x").GetValue(inter), + ) + }) } From 060945d93dd7152538f39201156e32e842930014 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 2 Oct 2024 16:41:10 -0400 Subject: [PATCH 17/21] Code cleanup. --- runtime/parser/expression.go | 3 +-- runtime/parser/lexer/lexer.go | 8 ++++---- runtime/parser/lexer/state.go | 8 ++++---- .../sema/check_string_template_expression.go | 19 ++++++++++--------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 78a53c77ad..4d2dbbeef1 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -1165,14 +1165,13 @@ func defineStringExpression() { for curToken.Is(lexer.TokenString) { literal = p.tokenSource(curToken) - length = len(literal) // remove quotation marks if they exist if curToken == startToken { literal = literal[1:] - length = len(literal) } + length = len(literal) if length >= 1 && literal[length-1] == '"' { literal = literal[:length-1] missingEnd = false diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 25e7635594..4bd89aabf6 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -52,8 +52,8 @@ type position struct { type lexerMode uint8 const ( - NORMAL lexerMode = iota - STR_INTERPOLATION + normal lexerMode = iota + stringInterpolation ) type lexer struct { @@ -141,7 +141,7 @@ func (l *lexer) clear() { l.cursor = 0 l.tokens = l.tokens[:0] l.tokenCount = 0 - l.mode = NORMAL + l.mode = normal l.openBrackets = 0 } @@ -427,7 +427,7 @@ func (l *lexer) scanString(quote rune) { switch r { case '(': // string template, stop and set mode - l.mode = STR_INTERPOLATION + l.mode = stringInterpolation // no need to update prev values because these next tokens will not backup l.endOffset = tmpBackupOffset l.current = tmpBackup diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index b536111c86..0a436760af 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -56,17 +56,17 @@ func rootState(l *lexer) stateFn { case '%': l.emitType(TokenPercent) case '(': - if l.mode == STR_INTERPOLATION { + if l.mode == stringInterpolation { // it is necessary to balance brackets when generating tokens for string templates to know when to change modes l.openBrackets++ } l.emitType(TokenParenOpen) case ')': l.emitType(TokenParenClose) - if l.mode == STR_INTERPOLATION { + if l.mode == stringInterpolation { l.openBrackets-- if l.openBrackets == 0 { - l.mode = NORMAL + l.mode = normal return stringState } } @@ -130,7 +130,7 @@ func rootState(l *lexer) stateFn { case '"': return stringState case '\\': - if l.mode == STR_INTERPOLATION { + if l.mode == stringInterpolation { r = l.next() switch r { case '(': diff --git a/runtime/sema/check_string_template_expression.go b/runtime/sema/check_string_template_expression.go index ddb0d6ed0d..5baf38cd83 100644 --- a/runtime/sema/check_string_template_expression.go +++ b/runtime/sema/check_string_template_expression.go @@ -22,8 +22,16 @@ import "github.com/onflow/cadence/runtime/ast" // All number types, addresses, path types, bool, strings and characters are supported in string template func isValidStringTemplateValue(valueType Type) bool { - return valueType == TheAddressType || valueType == StringType || valueType == BoolType || valueType == CharacterType || - IsSubType(valueType, NumberType) || IsSubType(valueType, PathType) + switch valueType { + case TheAddressType, + StringType, + BoolType, + CharacterType: + return true + default: + return IsSubType(valueType, NumberType) || + IsSubType(valueType, PathType) + } } func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression *ast.StringTemplateExpression) Type { @@ -55,12 +63,5 @@ func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression * } } - checker.Elaboration.SetStringTemplateExpressionTypes( - stringTemplateExpression, - StringTemplateExpressionTypes{ - ArgumentTypes: argumentTypes, - }, - ) - return StringType } From 447f05565ac9c6d4092edd117822407b5c02e069 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Thu, 3 Oct 2024 16:02:36 -0400 Subject: [PATCH 18/21] Remove unused code. --- runtime/sema/elaboration.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/runtime/sema/elaboration.go b/runtime/sema/elaboration.go index d2ef948a78..b6b025eef0 100644 --- a/runtime/sema/elaboration.go +++ b/runtime/sema/elaboration.go @@ -79,10 +79,6 @@ type ArrayExpressionTypes struct { ArgumentTypes []Type } -type StringTemplateExpressionTypes struct { - ArgumentTypes []Type -} - type DictionaryExpressionTypes struct { DictionaryType *DictionaryType EntryTypes []DictionaryEntryType @@ -144,7 +140,6 @@ type Elaboration struct { dictionaryExpressionTypes map[*ast.DictionaryExpression]DictionaryExpressionTypes integerExpressionTypes map[*ast.IntegerExpression]Type stringExpressionTypes map[*ast.StringExpression]Type - stringTemplateExpressionTypes map[*ast.StringTemplateExpression]StringTemplateExpressionTypes returnStatementTypes map[*ast.ReturnStatement]ReturnStatementTypes functionDeclarationFunctionTypes map[*ast.FunctionDeclaration]*FunctionType variableDeclarationTypes map[*ast.VariableDeclaration]VariableDeclarationTypes @@ -485,21 +480,6 @@ func (e *Elaboration) SetStringExpressionType(expression *ast.StringExpression, e.stringExpressionTypes[expression] = ty } -func (e *Elaboration) StringTemplateExpressionTypes(expression *ast.StringTemplateExpression) (types StringTemplateExpressionTypes) { - if e.stringTemplateExpressionTypes == nil { - return - } - // default, Elaboration.SetStringExpressionType - return e.stringTemplateExpressionTypes[expression] -} - -func (e *Elaboration) SetStringTemplateExpressionTypes(expression *ast.StringTemplateExpression, types StringTemplateExpressionTypes) { - if e.stringTemplateExpressionTypes == nil { - e.stringTemplateExpressionTypes = map[*ast.StringTemplateExpression]StringTemplateExpressionTypes{} - } - e.stringTemplateExpressionTypes[expression] = types -} - func (e *Elaboration) ReturnStatementTypes(statement *ast.ReturnStatement) (types ReturnStatementTypes) { if e.returnStatementTypes == nil { return From 57805ee7d985eeddd7590e295ff123292df4914e Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 16 Oct 2024 10:45:53 -0400 Subject: [PATCH 19/21] Update code after review. --- runtime/ast/expression.go | 7 +- runtime/ast/string_template_test.go | 48 ++++++ runtime/parser/expression_test.go | 142 +++++++++++++++++- runtime/parser/lexer/lexer.go | 8 +- runtime/parser/lexer/state.go | 8 +- .../sema/check_string_template_expression.go | 3 +- 6 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 runtime/ast/string_template_test.go diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index b8bf93e86d..8cf54fdce5 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -264,7 +264,12 @@ func (e *StringTemplateExpression) String() string { } func (e *StringTemplateExpression) Doc() prettier.Doc { - return prettier.Text(QuoteString("String template")) + if len(e.Expressions) == 0 { + return prettier.Text(e.Values[0]) + } + + // TODO: must reproduce expressions as literals + panic("not implemented") } func (e *StringTemplateExpression) MarshalJSON() ([]byte, error) { diff --git a/runtime/ast/string_template_test.go b/runtime/ast/string_template_test.go new file mode 100644 index 0000000000..d6aecb0299 --- /dev/null +++ b/runtime/ast/string_template_test.go @@ -0,0 +1,48 @@ +/* + * Cadence - The resource-oriented smart contract programming language + * + * Copyright Flow Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ast_test + +import ( + "testing" + + "github.com/onflow/cadence/runtime/ast" + "github.com/stretchr/testify/assert" + "github.com/turbolent/prettier" +) + +func TestStringTemplate_Doc(t *testing.T) { + + t.Parallel() + + stmt := &ast.StringTemplateExpression{ + Values: []string{ + "abc", + }, + Expressions: []ast.Expression{}, + Range: ast.Range{ + StartPos: ast.Position{Offset: 4, Line: 2, Column: 3}, + EndPos: ast.Position{Offset: 11, Line: 2, Column: 10}, + }, + } + + assert.Equal(t, + prettier.Text("abc"), + stmt.Doc(), + ) +} diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index 8f68a7b468..e2377cf934 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -6318,7 +6318,7 @@ func TestParseStringTemplate(t *testing.T) { ) }) - t.Run("unbalanced paren", func(t *testing.T) { + t.Run("invalid, unbalanced paren", func(t *testing.T) { t.Parallel() @@ -6345,7 +6345,7 @@ func TestParseStringTemplate(t *testing.T) { ) }) - t.Run("nested templates", func(t *testing.T) { + t.Run("invalid, nested templates", func(t *testing.T) { t.Parallel() @@ -6371,6 +6371,144 @@ func TestParseStringTemplate(t *testing.T) { errs, ) }) + + t.Run("valid, alternating", func(t *testing.T) { + + t.Parallel() + + actual, errs := testParseExpression(` + "a\(b)c" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.NoError(t, err) + + expected := &ast.StringTemplateExpression{ + Values: []string{ + "a", + "c", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "b", + Pos: ast.Position{Offset: 8, Line: 2, Column: 7}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 4, Line: 2, Column: 3}, + EndPos: ast.Position{Offset: 11, Line: 2, Column: 10}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) + }) + + t.Run("valid, surrounded", func(t *testing.T) { + + t.Parallel() + + actual, errs := testParseExpression(` + "\(a)b\(c)" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.NoError(t, err) + + expected := &ast.StringTemplateExpression{ + Values: []string{ + "", + "b", + "", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{Offset: 7, Line: 2, Column: 6}, + }, + }, + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "c", + Pos: ast.Position{Offset: 12, Line: 2, Column: 11}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 4, Line: 2, Column: 3}, + EndPos: ast.Position{Offset: 14, Line: 2, Column: 13}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) + }) + + t.Run("valid, adjacent", func(t *testing.T) { + + t.Parallel() + + actual, errs := testParseExpression(` + "\(a)\(b)\(c)" + `) + + var err error + if len(errs) > 0 { + err = Error{ + Errors: errs, + } + } + + require.NoError(t, err) + + expected := &ast.StringTemplateExpression{ + Values: []string{ + "", + "", + "", + "", + }, + Expressions: []ast.Expression{ + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{Offset: 7, Line: 2, Column: 6}, + }, + }, + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "b", + Pos: ast.Position{Offset: 11, Line: 2, Column: 11}, + }, + }, + &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "c", + Pos: ast.Position{Offset: 15, Line: 2, Column: 16}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 4, Line: 2, Column: 3}, + EndPos: ast.Position{Offset: 17, Line: 2, Column: 18}, + }, + } + + utils.AssertEqualWithDiff(t, expected, actual) + }) } func TestParseNilCoalescing(t *testing.T) { diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 4bd89aabf6..08f689be0e 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -52,8 +52,8 @@ type position struct { type lexerMode uint8 const ( - normal lexerMode = iota - stringInterpolation + lexerModeNormal lexerMode = iota + lexerModeStringInterpolation ) type lexer struct { @@ -141,7 +141,7 @@ func (l *lexer) clear() { l.cursor = 0 l.tokens = l.tokens[:0] l.tokenCount = 0 - l.mode = normal + l.mode = lexerModeNormal l.openBrackets = 0 } @@ -427,7 +427,7 @@ func (l *lexer) scanString(quote rune) { switch r { case '(': // string template, stop and set mode - l.mode = stringInterpolation + l.mode = lexerModeStringInterpolation // no need to update prev values because these next tokens will not backup l.endOffset = tmpBackupOffset l.current = tmpBackup diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 0a436760af..08558c6b78 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -56,17 +56,17 @@ func rootState(l *lexer) stateFn { case '%': l.emitType(TokenPercent) case '(': - if l.mode == stringInterpolation { + if l.mode == lexerModeStringInterpolation { // it is necessary to balance brackets when generating tokens for string templates to know when to change modes l.openBrackets++ } l.emitType(TokenParenOpen) case ')': l.emitType(TokenParenClose) - if l.mode == stringInterpolation { + if l.mode == lexerModeStringInterpolation { l.openBrackets-- if l.openBrackets == 0 { - l.mode = normal + l.mode = lexerModeNormal return stringState } } @@ -130,7 +130,7 @@ func rootState(l *lexer) stateFn { case '"': return stringState case '\\': - if l.mode == stringInterpolation { + if l.mode == lexerModeStringInterpolation { r = l.next() switch r { case '(': diff --git a/runtime/sema/check_string_template_expression.go b/runtime/sema/check_string_template_expression.go index 5baf38cd83..8b4be45b48 100644 --- a/runtime/sema/check_string_template_expression.go +++ b/runtime/sema/check_string_template_expression.go @@ -42,9 +42,8 @@ func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression * elementCount := len(stringTemplateExpression.Expressions) - var argumentTypes []Type if elementCount > 0 { - argumentTypes = make([]Type, elementCount) + argumentTypes := make([]Type, elementCount) for i, element := range stringTemplateExpression.Expressions { valueType := checker.VisitExpression(element, stringTemplateExpression, elementType) From cd83793f8d65f9a5dcf67c09370891c536c496f8 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 16 Oct 2024 10:53:56 -0400 Subject: [PATCH 20/21] Fix linting. --- runtime/ast/string_template_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/runtime/ast/string_template_test.go b/runtime/ast/string_template_test.go index d6aecb0299..a19a399bb9 100644 --- a/runtime/ast/string_template_test.go +++ b/runtime/ast/string_template_test.go @@ -16,12 +16,11 @@ * limitations under the License. */ -package ast_test +package ast import ( "testing" - "github.com/onflow/cadence/runtime/ast" "github.com/stretchr/testify/assert" "github.com/turbolent/prettier" ) @@ -30,14 +29,14 @@ func TestStringTemplate_Doc(t *testing.T) { t.Parallel() - stmt := &ast.StringTemplateExpression{ + stmt := &StringTemplateExpression{ Values: []string{ "abc", }, - Expressions: []ast.Expression{}, - Range: ast.Range{ - StartPos: ast.Position{Offset: 4, Line: 2, Column: 3}, - EndPos: ast.Position{Offset: 11, Line: 2, Column: 10}, + Expressions: []Expression{}, + Range: Range{ + StartPos: Position{Offset: 4, Line: 2, Column: 3}, + EndPos: Position{Offset: 11, Line: 2, Column: 10}, }, } From 698328897d5e97ae72d98825bab583e9f329b290 Mon Sep 17 00:00:00 2001 From: Raymond Zhang Date: Wed, 16 Oct 2024 15:22:20 -0400 Subject: [PATCH 21/21] Code cleanup. --- runtime/ast/expression.go | 14 +++++++++++--- runtime/ast/string_template_test.go | 2 +- runtime/sema/check_string_template_expression.go | 6 +----- runtime/tests/checker/string_test.go | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 8cf54fdce5..7f6fe4b502 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -223,6 +223,10 @@ func (*StringExpression) precedence() precedence { // StringTemplateExpression type StringTemplateExpression struct { + // Values and Expressions are assumed to be interleaved, V[0] + E[0] + V[1] + ... + E[n-1] + V[n] + // this is enforced in the parser e.g. "a\(b)c" will be parsed as follows + // Values: []string{"a","c"} + // Expressions: []Expression{b} Values []string Expressions []Expression Range @@ -237,6 +241,10 @@ func NewStringTemplateExpression( exprRange Range, ) *StringTemplateExpression { common.UseMemory(gauge, common.NewStringTemplateExpressionMemoryUsage(len(values)+len(exprs))) + if len(values) != len(exprs)+1 { + // assert string template alternating structure + panic(errors.NewUnreachableError()) + } return &StringTemplateExpression{ Values: values, Expressions: exprs, @@ -255,8 +263,8 @@ func (*StringTemplateExpression) isExpression() {} func (*StringTemplateExpression) isIfStatementTest() {} -func (*StringTemplateExpression) Walk(_ func(Element)) { - // NO-OP +func (e *StringTemplateExpression) Walk(walkChild func(Element)) { + walkExpressions(walkChild, e.Expressions) } func (e *StringTemplateExpression) String() string { @@ -265,7 +273,7 @@ func (e *StringTemplateExpression) String() string { func (e *StringTemplateExpression) Doc() prettier.Doc { if len(e.Expressions) == 0 { - return prettier.Text(e.Values[0]) + return prettier.Text(QuoteString(e.Values[0])) } // TODO: must reproduce expressions as literals diff --git a/runtime/ast/string_template_test.go b/runtime/ast/string_template_test.go index a19a399bb9..c6167aaffb 100644 --- a/runtime/ast/string_template_test.go +++ b/runtime/ast/string_template_test.go @@ -41,7 +41,7 @@ func TestStringTemplate_Doc(t *testing.T) { } assert.Equal(t, - prettier.Text("abc"), + prettier.Text("\"abc\""), stmt.Doc(), ) } diff --git a/runtime/sema/check_string_template_expression.go b/runtime/sema/check_string_template_expression.go index 8b4be45b48..6cbd8d80ec 100644 --- a/runtime/sema/check_string_template_expression.go +++ b/runtime/sema/check_string_template_expression.go @@ -43,13 +43,9 @@ func (checker *Checker) VisitStringTemplateExpression(stringTemplateExpression * elementCount := len(stringTemplateExpression.Expressions) if elementCount > 0 { - argumentTypes := make([]Type, elementCount) - - for i, element := range stringTemplateExpression.Expressions { + for _, element := range stringTemplateExpression.Expressions { valueType := checker.VisitExpression(element, stringTemplateExpression, elementType) - argumentTypes[i] = valueType - if !isValidStringTemplateValue(valueType) { checker.report( &TypeMismatchWithDescriptionError{ diff --git a/runtime/tests/checker/string_test.go b/runtime/tests/checker/string_test.go index 4bea07a7dd..84e305d3b3 100644 --- a/runtime/tests/checker/string_test.go +++ b/runtime/tests/checker/string_test.go @@ -746,7 +746,7 @@ func TestCheckStringTemplate(t *testing.T) { t.Parallel() _, err := ParseAndCheck(t, ` - let x :[AnyStruct] = ["tmp", 1] + let x: [AnyStruct] = ["tmp", 1] let y = "\(x)" `)