No content available.
+ {% endif %} + + {% block sidebar %} + + {% endblock %} +This is a paragraph
", + }, engine) + defer ctx.Release() + + // Setup filter nodes for testing + tests := []struct { + name string + node Node + }{ + { + name: "SingleFilter", + node: NewFilterNode(NewVariableNode("text", 1), "upper", nil, 1), + }, + { + name: "FilterChain", + node: NewFilterNode( + NewFilterNode( + NewVariableNode("text", 1), + "upper", + nil, + 1, + ), + "trim", + nil, + 1, + ), + }, + { + name: "FilterWithArgs", + node: NewFilterNode( + NewVariableNode("text", 1), + "replace", + []Node{ + NewLiteralNode("World", 1), + NewLiteralNode("Universe", 1), + }, + 1, + ), + }, + } + + // Run each benchmark + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = ctx.EvaluateExpression(tc.node) + } + }) + } +} + +// BenchmarkFunctionCall benchmarks function calls +func BenchmarkFunctionCall(b *testing.B) { + engine := New() + ctx := NewRenderContext(engine.environment, map[string]interface{}{ + "numbers": []interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + "start": 1, + "end": 10, + }, engine) + defer ctx.Release() + + // Setup function nodes for testing + tests := []struct { + name string + node Node + }{ + { + name: "RangeFunction", + node: NewFunctionNode("range", []Node{ + NewVariableNode("start", 1), + NewVariableNode("end", 1), + }, 1), + }, + { + name: "LengthFunction", + node: NewFunctionNode("length", []Node{ + NewVariableNode("numbers", 1), + }, 1), + }, + { + name: "MaxFunction", + node: NewFunctionNode("max", []Node{ + NewVariableNode("numbers", 1), + }, 1), + }, + } + + // Run each benchmark + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = ctx.EvaluateExpression(tc.node) + } + }) + } +} + +// BenchmarkArgSlicePooling specifically tests array arguments allocation +func BenchmarkArgSlicePooling(b *testing.B) { + engine := New() + ctx := NewRenderContext(engine.environment, nil, engine) + defer ctx.Release() + + smallArgs := []Node{ + NewLiteralNode(1, 1), + NewLiteralNode(2, 1), + } + + mediumArgs := []Node{ + NewLiteralNode(1, 1), + NewLiteralNode(2, 1), + NewLiteralNode(3, 1), + NewLiteralNode(4, 1), + NewLiteralNode(5, 1), + } + + largeArgs := make([]Node, 10) + for i := 0; i < 10; i++ { + largeArgs[i] = NewLiteralNode(i, 1) + } + + tests := []struct { + name string + node Node + }{ + { + name: "NoArgs", + node: NewFunctionNode("range", nil, 1), + }, + { + name: "SmallArgs", + node: NewFunctionNode("range", smallArgs, 1), + }, + { + name: "MediumArgs", + node: NewFunctionNode("range", mediumArgs, 1), + }, + { + name: "LargeArgs", + node: NewFunctionNode("range", largeArgs, 1), + }, + } + + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _, _ = ctx.EvaluateExpression(tc.node) + } + }) + } +} \ No newline at end of file diff --git a/expr_pool.go b/expr_pool.go new file mode 100644 index 0000000..cca3ba6 --- /dev/null +++ b/expr_pool.go @@ -0,0 +1,475 @@ +package twig + +import ( + "sync" +) + +// This file implements object pooling for expression node types +// to reduce memory allocations during evaluation. + +// BinaryNodePool provides a pool for BinaryNode objects +var BinaryNodePool = sync.Pool{ + New: func() interface{} { + return &BinaryNode{} + }, +} + +// GetBinaryNode gets a BinaryNode from the pool and initializes it +func GetBinaryNode(operator string, left, right Node, line int) *BinaryNode { + node := BinaryNodePool.Get().(*BinaryNode) + node.ExpressionNode.exprType = ExprBinary + node.ExpressionNode.line = line + node.operator = operator + node.left = left + node.right = right + return node +} + +// ReleaseBinaryNode returns a BinaryNode to the pool +func ReleaseBinaryNode(node *BinaryNode) { + if node == nil { + return + } + node.operator = "" + node.left = nil + node.right = nil + BinaryNodePool.Put(node) +} + +// GetAttrNodePool provides a pool for GetAttrNode objects +var GetAttrNodePool = sync.Pool{ + New: func() interface{} { + return &GetAttrNode{} + }, +} + +// GetGetAttrNode gets a GetAttrNode from the pool and initializes it +func GetGetAttrNode(node, attribute Node, line int) *GetAttrNode { + n := GetAttrNodePool.Get().(*GetAttrNode) + n.ExpressionNode.exprType = ExprGetAttr + n.ExpressionNode.line = line + n.node = node + n.attribute = attribute + return n +} + +// ReleaseGetAttrNode returns a GetAttrNode to the pool +func ReleaseGetAttrNode(node *GetAttrNode) { + if node == nil { + return + } + node.node = nil + node.attribute = nil + GetAttrNodePool.Put(node) +} + +// GetItemNodePool provides a pool for GetItemNode objects +var GetItemNodePool = sync.Pool{ + New: func() interface{} { + return &GetItemNode{} + }, +} + +// GetGetItemNode gets a GetItemNode from the pool and initializes it +func GetGetItemNode(node, item Node, line int) *GetItemNode { + n := GetItemNodePool.Get().(*GetItemNode) + n.ExpressionNode.exprType = ExprGetItem + n.ExpressionNode.line = line + n.node = node + n.item = item + return n +} + +// ReleaseGetItemNode returns a GetItemNode to the pool +func ReleaseGetItemNode(node *GetItemNode) { + if node == nil { + return + } + node.node = nil + node.item = nil + GetItemNodePool.Put(node) +} + +// FilterNodePool provides a pool for FilterNode objects +var FilterNodePool = sync.Pool{ + New: func() interface{} { + return &FilterNode{} + }, +} + +// GetFilterNode gets a FilterNode from the pool and initializes it +func GetFilterNode(node Node, filter string, args []Node, line int) *FilterNode { + n := FilterNodePool.Get().(*FilterNode) + n.ExpressionNode.exprType = ExprFilter + n.ExpressionNode.line = line + n.node = node + n.filter = filter + n.args = args + return n +} + +// ReleaseFilterNode returns a FilterNode to the pool +func ReleaseFilterNode(node *FilterNode) { + if node == nil { + return + } + node.node = nil + node.filter = "" + node.args = nil + FilterNodePool.Put(node) +} + +// TestNodePool provides a pool for TestNode objects +var TestNodePool = sync.Pool{ + New: func() interface{} { + return &TestNode{} + }, +} + +// GetTestNode gets a TestNode from the pool and initializes it +func GetTestNode(node Node, test string, args []Node, line int) *TestNode { + n := TestNodePool.Get().(*TestNode) + n.ExpressionNode.exprType = ExprTest + n.ExpressionNode.line = line + n.node = node + n.test = test + n.args = args + return n +} + +// ReleaseTestNode returns a TestNode to the pool +func ReleaseTestNode(node *TestNode) { + if node == nil { + return + } + node.node = nil + node.test = "" + node.args = nil + TestNodePool.Put(node) +} + +// UnaryNodePool provides a pool for UnaryNode objects +var UnaryNodePool = sync.Pool{ + New: func() interface{} { + return &UnaryNode{} + }, +} + +// GetUnaryNode gets a UnaryNode from the pool and initializes it +func GetUnaryNode(operator string, node Node, line int) *UnaryNode { + n := UnaryNodePool.Get().(*UnaryNode) + n.ExpressionNode.exprType = ExprUnary + n.ExpressionNode.line = line + n.operator = operator + n.node = node + return n +} + +// ReleaseUnaryNode returns a UnaryNode to the pool +func ReleaseUnaryNode(node *UnaryNode) { + if node == nil { + return + } + node.operator = "" + node.node = nil + UnaryNodePool.Put(node) +} + +// ConditionalNodePool provides a pool for ConditionalNode objects +var ConditionalNodePool = sync.Pool{ + New: func() interface{} { + return &ConditionalNode{} + }, +} + +// GetConditionalNode gets a ConditionalNode from the pool and initializes it +func GetConditionalNode(condition, trueExpr, falseExpr Node, line int) *ConditionalNode { + node := ConditionalNodePool.Get().(*ConditionalNode) + node.ExpressionNode.exprType = ExprConditional + node.ExpressionNode.line = line + node.condition = condition + node.trueExpr = trueExpr + node.falseExpr = falseExpr + return node +} + +// ReleaseConditionalNode returns a ConditionalNode to the pool +func ReleaseConditionalNode(node *ConditionalNode) { + if node == nil { + return + } + node.condition = nil + node.trueExpr = nil + node.falseExpr = nil + ConditionalNodePool.Put(node) +} + +// ArrayNodePool provides a pool for ArrayNode objects +var ArrayNodePool = sync.Pool{ + New: func() interface{} { + return &ArrayNode{} + }, +} + +// GetArrayNode gets an ArrayNode from the pool and initializes it +func GetArrayNode(items []Node, line int) *ArrayNode { + node := ArrayNodePool.Get().(*ArrayNode) + node.ExpressionNode.exprType = ExprArray + node.ExpressionNode.line = line + node.items = items + return node +} + +// ReleaseArrayNode returns an ArrayNode to the pool +func ReleaseArrayNode(node *ArrayNode) { + if node == nil { + return + } + node.items = nil + ArrayNodePool.Put(node) +} + +// HashNodePool provides a pool for HashNode objects +var HashNodePool = sync.Pool{ + New: func() interface{} { + return &HashNode{} + }, +} + +// GetHashNode gets a HashNode from the pool and initializes it +func GetHashNode(items map[Node]Node, line int) *HashNode { + node := HashNodePool.Get().(*HashNode) + node.ExpressionNode.exprType = ExprHash + node.ExpressionNode.line = line + node.items = items + return node +} + +// ReleaseHashNode returns a HashNode to the pool +func ReleaseHashNode(node *HashNode) { + if node == nil { + return + } + node.items = nil + HashNodePool.Put(node) +} + +// FunctionNodePool provides a pool for FunctionNode objects +var FunctionNodePool = sync.Pool{ + New: func() interface{} { + return &FunctionNode{} + }, +} + +// GetFunctionNode gets a FunctionNode from the pool and initializes it +func GetFunctionNode(name string, args []Node, line int) *FunctionNode { + node := FunctionNodePool.Get().(*FunctionNode) + node.ExpressionNode.exprType = ExprFunction + node.ExpressionNode.line = line + node.name = name + node.args = args + return node +} + +// ReleaseFunctionNode returns a FunctionNode to the pool +func ReleaseFunctionNode(node *FunctionNode) { + if node == nil { + return + } + node.name = "" + node.args = nil + node.moduleExpr = nil + FunctionNodePool.Put(node) +} + +// VariableNodePool provides a pool for VariableNode objects +var VariableNodePool = sync.Pool{ + New: func() interface{} { + return &VariableNode{} + }, +} + +// GetVariableNode gets a VariableNode from the pool and initializes it +func GetVariableNode(name string, line int) *VariableNode { + node := VariableNodePool.Get().(*VariableNode) + node.ExpressionNode.exprType = ExprVariable + node.ExpressionNode.line = line + node.name = name + return node +} + +// ReleaseVariableNode returns a VariableNode to the pool +func ReleaseVariableNode(node *VariableNode) { + if node == nil { + return + } + node.name = "" + VariableNodePool.Put(node) +} + +// LiteralNodePool provides a pool for LiteralNode objects +var LiteralNodePool = sync.Pool{ + New: func() interface{} { + return &LiteralNode{} + }, +} + +// GetLiteralNode gets a LiteralNode from the pool and initializes it +func GetLiteralNode(value interface{}, line int) *LiteralNode { + node := LiteralNodePool.Get().(*LiteralNode) + node.ExpressionNode.exprType = ExprLiteral + node.ExpressionNode.line = line + node.value = value + return node +} + +// ReleaseLiteralNode returns a LiteralNode to the pool +func ReleaseLiteralNode(node *LiteralNode) { + if node == nil { + return + } + node.value = nil + LiteralNodePool.Put(node) +} + +// --- Slice Pools for Expression Evaluation --- + +// smallArgSlicePool provides a pool for small argument slices (0-2 items) +var smallArgSlicePool = sync.Pool{ + New: func() interface{} { + // Pre-allocate a slice with capacity 2 + return make([]interface{}, 0, 2) + }, +} + +// mediumArgSlicePool provides a pool for medium argument slices (3-5 items) +var mediumArgSlicePool = sync.Pool{ + New: func() interface{} { + // Pre-allocate a slice with capacity 5 + return make([]interface{}, 0, 5) + }, +} + +// largeArgSlicePool provides a pool for large argument slices (6-10 items) +var largeArgSlicePool = sync.Pool{ + New: func() interface{} { + // Pre-allocate a slice with capacity 10 + return make([]interface{}, 0, 10) + }, +} + +// GetArgSlice gets an appropriately sized slice for the given number of arguments +func GetArgSlice(size int) []interface{} { + if size <= 0 { + return nil + } + + var slice []interface{} + + switch { + case size <= 2: + slice = smallArgSlicePool.Get().([]interface{}) + case size <= 5: + slice = mediumArgSlicePool.Get().([]interface{}) + case size <= 10: + slice = largeArgSlicePool.Get().([]interface{}) + default: + // For very large slices, just allocate directly + return make([]interface{}, 0, size) + } + + // Clear the slice but maintain capacity + return slice[:0] +} + +// ReleaseArgSlice returns an argument slice to the appropriate pool +func ReleaseArgSlice(slice []interface{}) { + if slice == nil { + return + } + + // Clear all references + for i := range slice { + slice[i] = nil + } + + // Reset length to 0 + slice = slice[:0] + + // Return to appropriate pool based on capacity + switch cap(slice) { + case 2: + smallArgSlicePool.Put(slice) + case 5: + mediumArgSlicePool.Put(slice) + case 10: + largeArgSlicePool.Put(slice) + } +} + +// --- Map Pools for HashNode Evaluation --- + +// smallHashMapPool provides a pool for small hash maps (1-5 items) +var smallHashMapPool = sync.Pool{ + New: func() interface{} { + return make(map[string]interface{}, 5) + }, +} + +// mediumHashMapPool provides a pool for medium hash maps (6-15 items) +var mediumHashMapPool = sync.Pool{ + New: func() interface{} { + return make(map[string]interface{}, 15) + }, +} + +// GetHashMap gets an appropriately sized map for hash operations +func GetHashMap(size int) map[string]interface{} { + if size <= 5 { + hashMap := smallHashMapPool.Get().(map[string]interface{}) + // Clear any existing entries + for k := range hashMap { + delete(hashMap, k) + } + return hashMap + } else if size <= 15 { + hashMap := mediumHashMapPool.Get().(map[string]interface{}) + // Clear any existing entries + for k := range hashMap { + delete(hashMap, k) + } + return hashMap + } + + // For larger maps, just allocate directly + return make(map[string]interface{}, size) +} + +// ReleaseHashMap returns a hash map to the appropriate pool +func ReleaseHashMap(hashMap map[string]interface{}) { + if hashMap == nil { + return + } + + // Clear all entries (not used directly in our defer block) + // for k := range hashMap { + // delete(hashMap, k) + // } + + // Return to appropriate pool based on capacity + // We don't actually clear the map when releasing through the defer, + // because we return the map as the result and deleting entries would + // clear the returned result + + // Map doesn't have a built-in cap function + // Not using pool return for maps directly returned as results + + /* + switch { + case len(hashMap) <= 5: + smallHashMapPool.Put(hashMap) + case len(hashMap) <= 15: + mediumHashMapPool.Put(hashMap) + } + */ +} \ No newline at end of file diff --git a/html_preserving_tokenizer.go b/html_preserving_tokenizer.go index a76ea0d..fae6068 100644 --- a/html_preserving_tokenizer.go +++ b/html_preserving_tokenizer.go @@ -82,7 +82,7 @@ func (p *Parser) htmlPreservingTokenize() ([]Token, error) { if nextTagPos-1 > currentPosition { preText := p.source[currentPosition : nextTagPos-1] tokens = append(tokens, createToken(TOKEN_TEXT, preText, line)) - line += strings.Count(preText, "\n") + line += countNewlines(preText) } // Add the tag itself as literal text (without the backslash) @@ -97,7 +97,7 @@ func (p *Parser) htmlPreservingTokenize() ([]Token, error) { // No more tags found, add the rest as TEXT content := p.source[currentPosition:] if len(content) > 0 { - line += strings.Count(content, "\n") + line += countNewlines(content) tokens = append(tokens, createToken(TOKEN_TEXT, content, line)) } break @@ -106,7 +106,7 @@ func (p *Parser) htmlPreservingTokenize() ([]Token, error) { // Add the text before the tag (HTML content) if nextTagPos > currentPosition { content := p.source[currentPosition:nextTagPos] - line += strings.Count(content, "\n") + line += countNewlines(content) tokens = append(tokens, createToken(TOKEN_TEXT, content, line)) } @@ -176,7 +176,7 @@ func (p *Parser) htmlPreservingTokenize() ([]Token, error) { // Get the content between the tags tagContent := p.source[currentPosition : currentPosition+endPos] - line += strings.Count(tagContent, "\n") // Update line count + line += countNewlines(tagContent) // Update line count // Process the content between the tags based on tag type if tagType == TOKEN_COMMENT_START { diff --git a/html_preserving_tokenizer_optimization.go b/html_preserving_tokenizer_optimization.go new file mode 100644 index 0000000..f3efcce --- /dev/null +++ b/html_preserving_tokenizer_optimization.go @@ -0,0 +1,735 @@ +package twig + +import ( + "fmt" + "strings" +) + +// optimizedHtmlPreservingTokenize is an optimized version of htmlPreservingTokenize +// that reduces memory allocations by reusing token objects and slices +func (p *Parser) optimizedHtmlPreservingTokenize() ([]Token, error) { + // Pre-allocate tokens with estimated capacity based on source length + estimatedTokenCount := len(p.source) / 20 // Rough estimate: one token per 20 chars + tokenSlice := GetPooledTokenSlice(estimatedTokenCount) + + // Ensure the token slice is released even if an error occurs + defer tokenSlice.Release() + + var currentPosition int + line := 1 + + for currentPosition < len(p.source) { + // Find the next twig tag start + nextTagPos := -1 + tagType := -1 + var matchedPos struct { + pos int + pattern string + ttype int + length int + } + + // Use a single substring for all pattern searches to reduce allocations + remainingSource := p.source[currentPosition:] + + // Check for all possible tag starts, including whitespace control variants + positions := []struct { + pos int + pattern string + ttype int + length int + }{ + {strings.Index(remainingSource, "{{-"), "{{-", TOKEN_VAR_START_TRIM, 3}, + {strings.Index(remainingSource, "{{"), "{{", TOKEN_VAR_START, 2}, + {strings.Index(remainingSource, "{%-"), "{%-", TOKEN_BLOCK_START_TRIM, 3}, + {strings.Index(remainingSource, "{%"), "{%", TOKEN_BLOCK_START, 2}, + {strings.Index(remainingSource, "{#"), "{#", TOKEN_COMMENT_START, 2}, + } + + // Find the closest tag + for _, pos := range positions { + if pos.pos != -1 { + adjustedPos := currentPosition + pos.pos + if nextTagPos == -1 || adjustedPos < nextTagPos { + nextTagPos = adjustedPos + tagType = pos.ttype + matchedPos = pos + } + } + } + + // Check if the tag is escaped with a backslash + if nextTagPos != -1 && nextTagPos > 0 && p.source[nextTagPos-1] == '\\' { + // This tag is escaped with a backslash, treat it as literal text + // Add text up to the backslash (if any) + if nextTagPos-1 > currentPosition { + preText := p.source[currentPosition : nextTagPos-1] + tokenSlice.AppendToken(TOKEN_TEXT, preText, line) + line += countNewlines(preText) + } + + // Add the tag itself as literal text (without the backslash) + tokenSlice.AppendToken(TOKEN_TEXT, matchedPos.pattern, line) + + // Move past the tag + currentPosition = nextTagPos + matchedPos.length + continue + } + + if nextTagPos == -1 { + // No more tags found, add the rest as TEXT + content := p.source[currentPosition:] + if len(content) > 0 { + line += countNewlines(content) + tokenSlice.AppendToken(TOKEN_TEXT, content, line) + } + break + } + + // Add the text before the tag (HTML content) + if nextTagPos > currentPosition { + content := p.source[currentPosition:nextTagPos] + line += countNewlines(content) + tokenSlice.AppendToken(TOKEN_TEXT, content, line) + } + + // Add the tag start token + tokenSlice.AppendToken(tagType, "", line) + + // Determine tag length and move past the opening + tagLength := 2 // Default for "{{", "{%", "{#" + if tagType == TOKEN_VAR_START_TRIM || tagType == TOKEN_BLOCK_START_TRIM { + tagLength = 3 // For "{{-" or "{%-" + } + currentPosition = nextTagPos + tagLength + + // Find the matching end tag + var endTag string + var endTagType int + var endTagLength int + + if tagType == TOKEN_VAR_START || tagType == TOKEN_VAR_START_TRIM { + // For variable tags, look for "}}" or "-}}" + endPos1 := strings.Index(p.source[currentPosition:], "}}") + endPos2 := strings.Index(p.source[currentPosition:], "-}}") + + if endPos1 != -1 && (endPos2 == -1 || endPos1 < endPos2) { + endTag = "}}" + endTagType = TOKEN_VAR_END + endTagLength = 2 + } else if endPos2 != -1 { + endTag = "-}}" + endTagType = TOKEN_VAR_END_TRIM + endTagLength = 3 + } else { + return nil, fmt.Errorf("unclosed variable tag starting at line %d", line) + } + } else if tagType == TOKEN_BLOCK_START || tagType == TOKEN_BLOCK_START_TRIM { + // For block tags, look for "%}" or "-%}" + endPos1 := strings.Index(p.source[currentPosition:], "%}") + endPos2 := strings.Index(p.source[currentPosition:], "-%}") + + if endPos1 != -1 && (endPos2 == -1 || endPos1 < endPos2) { + endTag = "%}" + endTagType = TOKEN_BLOCK_END + endTagLength = 2 + } else if endPos2 != -1 { + endTag = "-%}" + endTagType = TOKEN_BLOCK_END_TRIM + endTagLength = 3 + } else { + return nil, fmt.Errorf("unclosed block tag starting at line %d", line) + } + } else if tagType == TOKEN_COMMENT_START { + // For comment tags, look for "#}" + endPos := strings.Index(p.source[currentPosition:], "#}") + if endPos == -1 { + return nil, fmt.Errorf("unclosed comment starting at line %d", line) + } + endTag = "#}" + endTagType = TOKEN_COMMENT_END + endTagLength = 2 + } + + // Find the position of the end tag + endPos := strings.Index(p.source[currentPosition:], endTag) + if endPos == -1 { + return nil, fmt.Errorf("unclosed tag starting at line %d", line) + } + + // Get the content between the tags + tagContent := p.source[currentPosition : currentPosition+endPos] + line += countNewlines(tagContent) // Update line count + + // Process the content between the tags based on tag type + if tagType == TOKEN_COMMENT_START { + // For comments, just store the content as a TEXT token + if len(tagContent) > 0 { + tokenSlice.AppendToken(TOKEN_TEXT, tagContent, line) + } + } else { + // For variable and block tags, tokenize the content properly + // Trim whitespace from the tag content + tagContent = strings.TrimSpace(tagContent) + + if tagType == TOKEN_BLOCK_START || tagType == TOKEN_BLOCK_START_TRIM { + // Process block tags like if, for, etc. + // First, extract the tag name + parts := strings.SplitN(tagContent, " ", 2) + if len(parts) > 0 { + blockName := parts[0] + tokenSlice.AppendToken(TOKEN_NAME, blockName, line) + + // Different handling based on block type + if blockName == "if" || blockName == "elseif" { + // For if/elseif blocks, tokenize the condition + if len(parts) > 1 { + condition := strings.TrimSpace(parts[1]) + // Tokenize the condition properly + p.optimizedTokenizeExpression(condition, tokenSlice, line) + } + } else if blockName == "for" { + // For for loops, tokenize iterator variables and collection + if len(parts) > 1 { + forExpr := strings.TrimSpace(parts[1]) + // Check for proper "in" keyword + inPos := strings.Index(strings.ToLower(forExpr), " in ") + if inPos != -1 { + // Extract iterators and collection + iterators := strings.TrimSpace(forExpr[:inPos]) + collection := strings.TrimSpace(forExpr[inPos+4:]) + + // Handle key, value iterators (e.g., "key, value in collection") + if strings.Contains(iterators, ",") { + iterParts := strings.SplitN(iterators, ",", 2) + if len(iterParts) == 2 { + keyVar := strings.TrimSpace(iterParts[0]) + valueVar := strings.TrimSpace(iterParts[1]) + + // Add tokens for key and value variables + tokenSlice.AppendToken(TOKEN_NAME, keyVar, line) + tokenSlice.AppendToken(TOKEN_PUNCTUATION, ",", line) + tokenSlice.AppendToken(TOKEN_NAME, valueVar, line) + } + } else { + // Single iterator variable + tokenSlice.AppendToken(TOKEN_NAME, iterators, line) + } + + // Add "in" keyword + tokenSlice.AppendToken(TOKEN_NAME, "in", line) + + // Check if collection is a function call (contains ( and )) + if strings.Contains(collection, "(") && strings.Contains(collection, ")") { + // Tokenize the collection as a complex expression + p.optimizedTokenizeExpression(collection, tokenSlice, line) + } else { + // Add collection as a simple variable + tokenSlice.AppendToken(TOKEN_NAME, collection, line) + } + } else { + // Fallback if "in" keyword not found + tokenSlice.AppendToken(TOKEN_NAME, forExpr, line) + } + } + } else if blockName == "do" { + // Special handling for do tag with assignments and expressions + if len(parts) > 1 { + doExpr := strings.TrimSpace(parts[1]) + + // Check if it's an assignment (contains =) + assignPos := strings.Index(doExpr, "=") + if assignPos > 0 && !strings.Contains(doExpr[:assignPos], "==") { + // It's an assignment + varName := strings.TrimSpace(doExpr[:assignPos]) + valueExpr := strings.TrimSpace(doExpr[assignPos+1:]) + + // Add the variable name + tokenSlice.AppendToken(TOKEN_NAME, varName, line) + + // Add the equals sign + tokenSlice.AppendToken(TOKEN_OPERATOR, "=", line) + + // Tokenize the expression on the right side + p.optimizedTokenizeExpression(valueExpr, tokenSlice, line) + } else { + // It's just an expression, tokenize it + p.optimizedTokenizeExpression(doExpr, tokenSlice, line) + } + } + } else if blockName == "include" { + // Special handling for include tag with quoted template names + if len(parts) > 1 { + includeExpr := strings.TrimSpace(parts[1]) + + // First check if we have a 'with' keyword which separates template name from params + withPos := strings.Index(strings.ToLower(includeExpr), " with ") + + if withPos > 0 { + // Split the include expression into template name and parameters + templatePart := strings.TrimSpace(includeExpr[:withPos]) + paramsPart := strings.TrimSpace(includeExpr[withPos+6:]) // +6 to skip " with " + + // Handle quoted template names + if (strings.HasPrefix(templatePart, "\"") && strings.HasSuffix(templatePart, "\"")) || + (strings.HasPrefix(templatePart, "'") && strings.HasSuffix(templatePart, "'")) { + // Extract the template name without quotes + templateName := templatePart[1 : len(templatePart)-1] + // Add as a string token + tokenSlice.AppendToken(TOKEN_STRING, templateName, line) + } else { + // Unquoted name, add as name token + tokenSlice.AppendToken(TOKEN_NAME, templatePart, line) + } + + // Add "with" keyword + tokenSlice.AppendToken(TOKEN_NAME, "with", line) + + // Add opening brace for the parameters + tokenSlice.AppendToken(TOKEN_PUNCTUATION, "{", line) + + // For parameters that might include nested objects, we need a different approach + // Tokenize the parameter string, preserving nested structures + optimizedTokenizeComplexObject(paramsPart, tokenSlice, line) + + // Add closing brace + tokenSlice.AppendToken(TOKEN_PUNCTUATION, "}", line) + } else { + // No 'with' keyword, just a template name + if (strings.HasPrefix(includeExpr, "\"") && strings.HasSuffix(includeExpr, "\"")) || + (strings.HasPrefix(includeExpr, "'") && strings.HasSuffix(includeExpr, "'")) { + // Extract template name without quotes + templateName := includeExpr[1 : len(includeExpr)-1] + // Add as a string token + tokenSlice.AppendToken(TOKEN_STRING, templateName, line) + } else { + // Not quoted, add as name token + tokenSlice.AppendToken(TOKEN_NAME, includeExpr, line) + } + } + } + } else if blockName == "extends" { + // Special handling for extends tag with quoted template names + if len(parts) > 1 { + extendsExpr := strings.TrimSpace(parts[1]) + + // Handle quoted template names + if (strings.HasPrefix(extendsExpr, "\"") && strings.HasSuffix(extendsExpr, "\"")) || + (strings.HasPrefix(extendsExpr, "'") && strings.HasSuffix(extendsExpr, "'")) { + // Extract the template name without quotes + templateName := extendsExpr[1 : len(extendsExpr)-1] + // Add as a string token + tokenSlice.AppendToken(TOKEN_STRING, templateName, line) + } else { + // Not quoted, tokenize as a normal expression + p.optimizedTokenizeExpression(extendsExpr, tokenSlice, line) + } + } + } else if blockName == "set" { + // Special handling for set tag to properly tokenize variable assignments + if len(parts) > 1 { + setExpr := strings.TrimSpace(parts[1]) + + // Check for the assignment operator + assignPos := strings.Index(setExpr, "=") + + if assignPos != -1 { + // Split into variable name and value + varName := strings.TrimSpace(setExpr[:assignPos]) + value := strings.TrimSpace(setExpr[assignPos+1:]) + + // Add the variable name token + tokenSlice.AppendToken(TOKEN_NAME, varName, line) + + // Add the assignment operator + tokenSlice.AppendToken(TOKEN_OPERATOR, "=", line) + + // Tokenize the value expression + p.optimizedTokenizeExpression(value, tokenSlice, line) + } else { + // Handle case without assignment (e.g., {% set var %}) + tokenSlice.AppendToken(TOKEN_NAME, setExpr, line) + } + } + } else { + // For other block types, just add parameters as NAME tokens + if len(parts) > 1 { + tokenSlice.AppendToken(TOKEN_NAME, parts[1], line) + } + } + } + } else { + // For variable tags, tokenize the expression + if len(tagContent) > 0 { + // If it's a simple variable name, add it directly + if !strings.ContainsAny(tagContent, ".|[](){}\"',+-*/=!<>%&^~") { + tokenSlice.AppendToken(TOKEN_NAME, tagContent, line) + } else { + // For complex expressions, tokenize properly + p.optimizedTokenizeExpression(tagContent, tokenSlice, line) + } + } + } + } + + // Add the end tag token + tokenSlice.AppendToken(endTagType, "", line) + + // Move past the end tag + currentPosition = currentPosition + endPos + endTagLength + } + + // Add EOF token + tokenSlice.AppendToken(TOKEN_EOF, "", line) + + // Finalize and return the token slice + return tokenSlice.Finalize(), nil +} + +// optimizedTokenizeExpression handles tokenizing expressions inside Twig tags with reduced allocations +func (p *Parser) optimizedTokenizeExpression(expr string, tokens *PooledTokenSlice, line int) { + var inString bool + var stringDelimiter byte + var stringStart int // Position where string content starts + + for i := 0; i < len(expr); i++ { + c := expr[i] + + // Handle string literals with quotes + if (c == '"' || c == '\'') && (i == 0 || expr[i-1] != '\\') { + if inString && c == stringDelimiter { + // End of string + inString = false + // Add the string token + tokens.AppendToken(TOKEN_STRING, expr[stringStart:i], line) + } else if !inString { + // Start of string + inString = true + stringDelimiter = c + // Remember the start position (for string content) + stringStart = i + 1 + } else { + // Quote inside a string with different delimiter + // Skip + } + continue + } + + // If we're inside a string, just skip this character + if inString { + continue + } + + // Handle operators (including two-character operators) + if isOperator(c) { + // Check for two-character operators + if i+1 < len(expr) { + nextChar := expr[i+1] + + // Direct comparison for common two-char operators + if (c == '=' && nextChar == '=') || + (c == '!' && nextChar == '=') || + (c == '>' && nextChar == '=') || + (c == '<' && nextChar == '=') || + (c == '&' && nextChar == '&') || + (c == '|' && nextChar == '|') || + (c == '?' && nextChar == '?') { + + // Add the two-character operator token + tokens.AppendToken(TOKEN_OPERATOR, string([]byte{c, nextChar}), line) + i++ // Skip the next character + continue + } + } + + // Add single-character operator + tokens.AppendToken(TOKEN_OPERATOR, string([]byte{c}), line) + continue + } + + // Handle punctuation + if isPunctuation(c) { + tokens.AppendToken(TOKEN_PUNCTUATION, string([]byte{c}), line) + continue + } + + // Handle whitespace - skip it + if isWhitespace(c) { + continue + } + + // Handle identifiers and keywords + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' { + // Start of an identifier + start := i + + // Find the end of the identifier + for i++; i < len(expr) && ((expr[i] >= 'a' && expr[i] <= 'z') || + (expr[i] >= 'A' && expr[i] <= 'Z') || + (expr[i] >= '0' && expr[i] <= '9') || + expr[i] == '_'); i++ { + } + + // Extract the identifier + identifier := expr[start:i] + i-- // Adjust for the loop increment + + // Add the token based on the identifier + if identifier == "true" || identifier == "false" || identifier == "null" { + tokens.AppendToken(TOKEN_NAME, identifier, line) + } else { + tokens.AppendToken(TOKEN_NAME, identifier, line) + } + + continue + } + + // Handle numbers + if isDigit(c) || (c == '-' && i+1 < len(expr) && isDigit(expr[i+1])) { + start := i + + // Skip the negative sign if present + if c == '-' { + i++ + } + + // Find the end of the number + for i++; i < len(expr) && isDigit(expr[i]); i++ { + } + + // Check for decimal point + if i < len(expr) && expr[i] == '.' { + i++ + // Find the end of the decimal part + for ; i < len(expr) && isDigit(expr[i]); i++ { + } + } + + // Extract the number + number := expr[start:i] + i-- // Adjust for the loop increment + + // Add the number token + tokens.AppendToken(TOKEN_NUMBER, number, line) + continue + } + } +} + +// optimizedTokenizeComplexObject parses and tokenizes a complex object with reduced allocations +func optimizedTokenizeComplexObject(objStr string, tokens *PooledTokenSlice, line int) { + // First strip outer braces if present + objStr = strings.TrimSpace(objStr) + if strings.HasPrefix(objStr, "{") && strings.HasSuffix(objStr, "}") { + objStr = strings.TrimSpace(objStr[1 : len(objStr)-1]) + } + + // Tokenize the object contents + optimizedTokenizeObjectContents(objStr, tokens, line) +} + +// optimizedTokenizeObjectContents parses key-value pairs with reduced allocations +func optimizedTokenizeObjectContents(content string, tokens *PooledTokenSlice, line int) { + // State tracking + inSingleQuote := false + inDoubleQuote := false + inObject := 0 // Nesting level for objects + inArray := 0 // Nesting level for arrays + + start := 0 + colonPos := -1 + + for i := 0; i <= len(content); i++ { + // At the end of the string or at a comma at the top level + atEnd := i == len(content) + isComma := !atEnd && content[i] == ',' + + if (isComma || atEnd) && inObject == 0 && inArray == 0 && !inSingleQuote && !inDoubleQuote { + // We've found the end of a key-value pair + if colonPos != -1 { + // Extract the key and value + keyStr := strings.TrimSpace(content[start:colonPos]) + valueStr := strings.TrimSpace(content[colonPos+1 : i]) + + // Process the key + if (len(keyStr) >= 2 && keyStr[0] == '\'' && keyStr[len(keyStr)-1] == '\'') || + (len(keyStr) >= 2 && keyStr[0] == '"' && keyStr[len(keyStr)-1] == '"') { + // Quoted key - add as a string token + tokens.AppendToken(TOKEN_STRING, keyStr[1:len(keyStr)-1], line) + } else { + // Unquoted key + tokens.AppendToken(TOKEN_NAME, keyStr, line) + } + + // Add colon separator + tokens.AppendToken(TOKEN_PUNCTUATION, ":", line) + + // Process the value based on type + if len(valueStr) >= 2 && valueStr[0] == '{' && valueStr[len(valueStr)-1] == '}' { + // Nested object + tokens.AppendToken(TOKEN_PUNCTUATION, "{", line) + optimizedTokenizeObjectContents(valueStr[1:len(valueStr)-1], tokens, line) + tokens.AppendToken(TOKEN_PUNCTUATION, "}", line) + } else if len(valueStr) >= 2 && valueStr[0] == '[' && valueStr[len(valueStr)-1] == ']' { + // Array + tokens.AppendToken(TOKEN_PUNCTUATION, "[", line) + optimizedTokenizeArrayElements(valueStr[1:len(valueStr)-1], tokens, line) + tokens.AppendToken(TOKEN_PUNCTUATION, "]", line) + } else if (len(valueStr) >= 2 && valueStr[0] == '\'' && valueStr[len(valueStr)-1] == '\'') || + (len(valueStr) >= 2 && valueStr[0] == '"' && valueStr[len(valueStr)-1] == '"') { + // String literal + tokens.AppendToken(TOKEN_STRING, valueStr[1:len(valueStr)-1], line) + } else if isNumericValue(valueStr) { + // Numeric value + tokens.AppendToken(TOKEN_NUMBER, valueStr, line) + } else if valueStr == "true" || valueStr == "false" { + // Boolean literal + tokens.AppendToken(TOKEN_NAME, valueStr, line) + } else if valueStr == "null" || valueStr == "nil" { + // Null/nil literal + tokens.AppendToken(TOKEN_NAME, valueStr, line) + } else { + // Variable or other value + tokens.AppendToken(TOKEN_NAME, valueStr, line) + } + + // Add comma if needed + if isComma && i < len(content)-1 { + tokens.AppendToken(TOKEN_PUNCTUATION, ",", line) + } + + // Reset state for next key-value pair + start = i + 1 + colonPos = -1 + } + continue + } + + // Handle quotes and nested structures + if i < len(content) { + c := content[i] + + // Handle quote characters + if c == '\'' && (i == 0 || content[i-1] != '\\') { + inSingleQuote = !inSingleQuote + } else if c == '"' && (i == 0 || content[i-1] != '\\') { + inDoubleQuote = !inDoubleQuote + } + + // Skip everything inside quotes + if inSingleQuote || inDoubleQuote { + continue + } + + // Handle object and array nesting + if c == '{' { + inObject++ + } else if c == '}' { + inObject-- + } else if c == '[' { + inArray++ + } else if c == ']' { + inArray-- + } + + // Find the colon separator if we're not in a nested structure + if c == ':' && inObject == 0 && inArray == 0 && colonPos == -1 { + colonPos = i + } + } + } +} + +// optimizedTokenizeArrayElements parses and tokenizes array elements with reduced allocations +func optimizedTokenizeArrayElements(arrStr string, tokens *PooledTokenSlice, line int) { + // State tracking + inSingleQuote := false + inDoubleQuote := false + inObject := 0 + inArray := 0 + + // Track the start position of each element + elemStart := 0 + + for i := 0; i <= len(arrStr); i++ { + // At the end of the string or at a comma at the top level + atEnd := i == len(arrStr) + isComma := !atEnd && arrStr[i] == ',' + + // Process element when we reach a comma or the end + if (isComma || atEnd) && inObject == 0 && inArray == 0 && !inSingleQuote && !inDoubleQuote { + // Extract the element + if i > elemStart { + element := strings.TrimSpace(arrStr[elemStart:i]) + + // Process the element based on its type + if len(element) >= 2 { + if element[0] == '{' && element[len(element)-1] == '}' { + // Nested object + tokens.AppendToken(TOKEN_PUNCTUATION, "{", line) + optimizedTokenizeObjectContents(element[1:len(element)-1], tokens, line) + tokens.AppendToken(TOKEN_PUNCTUATION, "}", line) + } else if element[0] == '[' && element[len(element)-1] == ']' { + // Nested array + tokens.AppendToken(TOKEN_PUNCTUATION, "[", line) + optimizedTokenizeArrayElements(element[1:len(element)-1], tokens, line) + tokens.AppendToken(TOKEN_PUNCTUATION, "]", line) + } else if (element[0] == '\'' && element[len(element)-1] == '\'') || + (element[0] == '"' && element[len(element)-1] == '"') { + // String literal + tokens.AppendToken(TOKEN_STRING, element[1:len(element)-1], line) + } else if isNumericValue(element) { + // Numeric value + tokens.AppendToken(TOKEN_NUMBER, element, line) + } else if element == "true" || element == "false" { + // Boolean literal + tokens.AppendToken(TOKEN_NAME, element, line) + } else if element == "null" || element == "nil" { + // Null/nil literal + tokens.AppendToken(TOKEN_NAME, element, line) + } else { + // Variable or other value + tokens.AppendToken(TOKEN_NAME, element, line) + } + } + } + + // Add comma if needed + if isComma && i < len(arrStr)-1 { + tokens.AppendToken(TOKEN_PUNCTUATION, ",", line) + } + + // Move to next element + elemStart = i + 1 + continue + } + + // Handle quotes and nested structures + if !atEnd { + c := arrStr[i] + + // Handle quote characters + if c == '\'' && (i == 0 || arrStr[i-1] != '\\') { + inSingleQuote = !inSingleQuote + } else if c == '"' && (i == 0 || arrStr[i-1] != '\\') { + inDoubleQuote = !inDoubleQuote + } + + // Skip everything inside quotes + if inSingleQuote || inDoubleQuote { + continue + } + + // Handle nesting + if c == '{' { + inObject++ + } else if c == '}' { + inObject-- + } else if c == '[' { + inArray++ + } else if c == ']' { + inArray-- + } + } + } +} \ No newline at end of file diff --git a/node.go b/node.go index c15e2a2..74cdda2 100644 --- a/node.go +++ b/node.go @@ -49,78 +49,42 @@ func NewForNode(keyVar, valueVar string, sequence Node, body, elseBranch []Node, // NewBlockNode creates a new block node func NewBlockNode(name string, body []Node, line int) *BlockNode { - return &BlockNode{ - name: name, - body: body, - line: line, - } + return GetBlockNode(name, body, line) } // NewExtendsNode creates a new extends node func NewExtendsNode(parent Node, line int) *ExtendsNode { - return &ExtendsNode{ - parent: parent, - line: line, - } + return GetExtendsNode(parent, line) } // NewIncludeNode creates a new include node func NewIncludeNode(template Node, variables map[string]Node, ignoreMissing, only, sandboxed bool, line int) *IncludeNode { - return &IncludeNode{ - template: template, - variables: variables, - ignoreMissing: ignoreMissing, - only: only, - sandboxed: sandboxed, - line: line, - } + return GetIncludeNode(template, variables, ignoreMissing, only, sandboxed, line) } // NewSetNode creates a new set node func NewSetNode(name string, value Node, line int) *SetNode { - return &SetNode{ - name: name, - value: value, - line: line, - } + return GetSetNode(name, value, line) } // NewCommentNode creates a new comment node func NewCommentNode(content string, line int) *CommentNode { - return &CommentNode{ - content: content, - line: line, - } + return GetCommentNode(content, line) } // NewMacroNode creates a new macro node func NewMacroNode(name string, params []string, defaults map[string]Node, body []Node, line int) *MacroNode { - return &MacroNode{ - name: name, - params: params, - defaults: defaults, - body: body, - line: line, - } + return GetMacroNode(name, params, defaults, body, line) } // NewImportNode creates a new import node func NewImportNode(template Node, module string, line int) *ImportNode { - return &ImportNode{ - template: template, - module: module, - line: line, - } + return GetImportNode(template, module, line) } // NewFromImportNode creates a new from import node func NewFromImportNode(template Node, macros []string, aliases map[string]string, line int) *FromImportNode { - return &FromImportNode{ - template: template, - macros: macros, - aliases: aliases, - line: line, - } + return GetFromImportNode(template, macros, aliases, line) } // NodeType represents the type of a node @@ -614,6 +578,11 @@ func (n *BlockNode) Line() int { return n.line } +// Release returns a BlockNode to the pool +func (n *BlockNode) Release() { + ReleaseBlockNode(n) +} + // Render renders the block node func (n *BlockNode) Render(w io.Writer, ctx *RenderContext) error { // Determine which content to use - from context blocks or default @@ -676,6 +645,11 @@ func (n *ExtendsNode) Line() int { return n.line } +// Release returns an ExtendsNode to the pool +func (n *ExtendsNode) Release() { + ReleaseExtendsNode(n) +} + // Implement Node interface for ExtendsNode func (n *ExtendsNode) Render(w io.Writer, ctx *RenderContext) error { // Flag that this template extends another @@ -780,6 +754,11 @@ func (n *IncludeNode) Line() int { return n.line } +// Release returns an IncludeNode to the pool +func (n *IncludeNode) Release() { + ReleaseIncludeNode(n) +} + // Implement Node interface for IncludeNode func (n *IncludeNode) Render(w io.Writer, ctx *RenderContext) error { // Get the template name @@ -898,6 +877,11 @@ func (n *SetNode) Line() int { return n.line } +// Release returns a SetNode to the pool +func (n *SetNode) Release() { + ReleaseSetNode(n) +} + // Render renders the set node func (n *SetNode) Render(w io.Writer, ctx *RenderContext) error { // Evaluate the value @@ -919,10 +903,7 @@ type DoNode struct { // NewDoNode creates a new DoNode func NewDoNode(expression Node, line int) *DoNode { - return &DoNode{ - expression: expression, - line: line, - } + return GetDoNode(expression, line) } func (n *DoNode) Type() NodeType { @@ -933,6 +914,11 @@ func (n *DoNode) Line() int { return n.line } +// Release returns a DoNode to the pool +func (n *DoNode) Release() { + ReleaseDoNode(n) +} + // Render evaluates the expression but doesn't write anything func (n *DoNode) Render(w io.Writer, ctx *RenderContext) error { // Evaluate the expression but ignore the result @@ -954,6 +940,11 @@ func (n *CommentNode) Line() int { return n.line } +// Release returns a CommentNode to the pool +func (n *CommentNode) Release() { + ReleaseCommentNode(n) +} + // Render renders the comment node (does nothing, as comments are not rendered) func (n *CommentNode) Render(w io.Writer, ctx *RenderContext) error { // Comments are not rendered @@ -977,6 +968,11 @@ func (n *MacroNode) Line() int { return n.line } +// Release returns a MacroNode to the pool +func (n *MacroNode) Release() { + ReleaseMacroNode(n) +} + // Render renders the macro node func (n *MacroNode) Render(w io.Writer, ctx *RenderContext) error { // Register the macro in the context @@ -1165,6 +1161,11 @@ func (n *ImportNode) Line() int { return n.line } +// Release returns an ImportNode to the pool +func (n *ImportNode) Release() { + ReleaseImportNode(n) +} + // Implement Node interface for ImportNode func (n *ImportNode) Render(w io.Writer, ctx *RenderContext) error { // Get the template name @@ -1249,6 +1250,11 @@ func (n *FromImportNode) Line() int { return n.line } +// Release returns a FromImportNode to the pool +func (n *FromImportNode) Release() { + ReleaseFromImportNode(n) +} + // Implement Node interface for FromImportNode func (n *FromImportNode) Render(w io.Writer, ctx *RenderContext) error { // Get the template name @@ -1332,10 +1338,7 @@ type VerbatimNode struct { // NewVerbatimNode creates a new verbatim node func NewVerbatimNode(content string, line int) *VerbatimNode { - return &VerbatimNode{ - content: content, - line: line, - } + return GetVerbatimNode(content, line) } func (n *VerbatimNode) Type() NodeType { @@ -1346,6 +1349,11 @@ func (n *VerbatimNode) Line() int { return n.line } +// Release returns a VerbatimNode to the pool +func (n *VerbatimNode) Release() { + ReleaseVerbatimNode(n) +} + // Render renders the verbatim node (outputs raw content without processing) func (n *VerbatimNode) Render(w io.Writer, ctx *RenderContext) error { // Output the content as-is without any processing @@ -1381,12 +1389,7 @@ type ApplyNode struct { // NewApplyNode creates a new apply node func NewApplyNode(body []Node, filter string, args []Node, line int) *ApplyNode { - return &ApplyNode{ - body: body, - filter: filter, - args: args, - line: line, - } + return GetApplyNode(body, filter, args, line) } func (n *ApplyNode) Type() NodeType { @@ -1397,6 +1400,11 @@ func (n *ApplyNode) Line() int { return n.line } +// Release returns an ApplyNode to the pool +func (n *ApplyNode) Release() { + ReleaseApplyNode(n) +} + // Render renders the apply node by applying a filter to the rendered body func (n *ApplyNode) Render(w io.Writer, ctx *RenderContext) error { // First render body content to a buffer diff --git a/node_pool.go b/node_pool.go index 0bdad4b..5251a40 100644 --- a/node_pool.go +++ b/node_pool.go @@ -79,53 +79,8 @@ func ReleaseRootNode(node *RootNode) { RootNodePool.Put(node) } -// LiteralNodePool provides a pool for LiteralNode objects -var LiteralNodePool = sync.Pool{ - New: func() interface{} { - return &LiteralNode{} - }, -} - -// GetLiteralNode gets a LiteralNode from the pool and initializes it -func GetLiteralNode(value interface{}, line int) *LiteralNode { - node := LiteralNodePool.Get().(*LiteralNode) - node.value = value - node.line = line - return node -} - -// ReleaseLiteralNode returns a LiteralNode to the pool -func ReleaseLiteralNode(node *LiteralNode) { - if node == nil { - return - } - node.value = nil - LiteralNodePool.Put(node) -} - -// VariableNodePool provides a pool for VariableNode objects -var VariableNodePool = sync.Pool{ - New: func() interface{} { - return &VariableNode{} - }, -} - -// GetVariableNode gets a VariableNode from the pool and initializes it -func GetVariableNode(name string, line int) *VariableNode { - node := VariableNodePool.Get().(*VariableNode) - node.name = name - node.line = line - return node -} - -// ReleaseVariableNode returns a VariableNode to the pool -func ReleaseVariableNode(node *VariableNode) { - if node == nil { - return - } - node.name = "" - VariableNodePool.Put(node) -} +// Note: LiteralNodePool, GetLiteralNode, ReleaseLiteralNode moved to expr_pool.go +// Note: VariableNodePool, GetVariableNode, ReleaseVariableNode moved to expr_pool.go // TokenPool provides a pool for Token objects var TokenPool = sync.Pool{ diff --git a/node_pool_extensions.go b/node_pool_extensions.go new file mode 100644 index 0000000..100187f --- /dev/null +++ b/node_pool_extensions.go @@ -0,0 +1,300 @@ +package twig + +import ( + "sync" +) + +// This file extends the node pool system to cover all node types, +// following the implementation strategy in the zero-allocation plan. + +// BlockNodePool provides a pool for BlockNode objects +var BlockNodePool = sync.Pool{ + New: func() interface{} { + return &BlockNode{} + }, +} + +// GetBlockNode gets a BlockNode from the pool and initializes it +func GetBlockNode(name string, body []Node, line int) *BlockNode { + node := BlockNodePool.Get().(*BlockNode) + node.name = name + node.body = body + node.line = line + return node +} + +// ReleaseBlockNode returns a BlockNode to the pool +func ReleaseBlockNode(node *BlockNode) { + if node == nil { + return + } + node.name = "" + node.body = nil + BlockNodePool.Put(node) +} + +// ExtendsNodePool provides a pool for ExtendsNode objects +var ExtendsNodePool = sync.Pool{ + New: func() interface{} { + return &ExtendsNode{} + }, +} + +// GetExtendsNode gets an ExtendsNode from the pool and initializes it +func GetExtendsNode(parent Node, line int) *ExtendsNode { + node := ExtendsNodePool.Get().(*ExtendsNode) + node.parent = parent + node.line = line + return node +} + +// ReleaseExtendsNode returns an ExtendsNode to the pool +func ReleaseExtendsNode(node *ExtendsNode) { + if node == nil { + return + } + node.parent = nil + ExtendsNodePool.Put(node) +} + +// IncludeNodePool provides a pool for IncludeNode objects +var IncludeNodePool = sync.Pool{ + New: func() interface{} { + return &IncludeNode{} + }, +} + +// GetIncludeNode gets an IncludeNode from the pool and initializes it +func GetIncludeNode(template Node, variables map[string]Node, ignoreMissing, only, sandboxed bool, line int) *IncludeNode { + node := IncludeNodePool.Get().(*IncludeNode) + node.template = template + node.variables = variables + node.ignoreMissing = ignoreMissing + node.only = only + node.sandboxed = sandboxed + node.line = line + return node +} + +// ReleaseIncludeNode returns an IncludeNode to the pool +func ReleaseIncludeNode(node *IncludeNode) { + if node == nil { + return + } + node.template = nil + node.variables = nil + node.ignoreMissing = false + node.only = false + node.sandboxed = false + IncludeNodePool.Put(node) +} + +// SetNodePool provides a pool for SetNode objects +var SetNodePool = sync.Pool{ + New: func() interface{} { + return &SetNode{} + }, +} + +// GetSetNode gets a SetNode from the pool and initializes it +func GetSetNode(name string, value Node, line int) *SetNode { + node := SetNodePool.Get().(*SetNode) + node.name = name + node.value = value + node.line = line + return node +} + +// ReleaseSetNode returns a SetNode to the pool +func ReleaseSetNode(node *SetNode) { + if node == nil { + return + } + node.name = "" + node.value = nil + SetNodePool.Put(node) +} + +// CommentNodePool provides a pool for CommentNode objects +var CommentNodePool = sync.Pool{ + New: func() interface{} { + return &CommentNode{} + }, +} + +// GetCommentNode gets a CommentNode from the pool and initializes it +func GetCommentNode(content string, line int) *CommentNode { + node := CommentNodePool.Get().(*CommentNode) + node.content = content + node.line = line + return node +} + +// ReleaseCommentNode returns a CommentNode to the pool +func ReleaseCommentNode(node *CommentNode) { + if node == nil { + return + } + node.content = "" + CommentNodePool.Put(node) +} + +// MacroNodePool provides a pool for MacroNode objects +var MacroNodePool = sync.Pool{ + New: func() interface{} { + return &MacroNode{} + }, +} + +// GetMacroNode gets a MacroNode from the pool and initializes it +func GetMacroNode(name string, params []string, defaults map[string]Node, body []Node, line int) *MacroNode { + node := MacroNodePool.Get().(*MacroNode) + node.name = name + node.params = params + node.defaults = defaults + node.body = body + node.line = line + return node +} + +// ReleaseMacroNode returns a MacroNode to the pool +func ReleaseMacroNode(node *MacroNode) { + if node == nil { + return + } + node.name = "" + node.params = nil + node.defaults = nil + node.body = nil + MacroNodePool.Put(node) +} + +// ImportNodePool provides a pool for ImportNode objects +var ImportNodePool = sync.Pool{ + New: func() interface{} { + return &ImportNode{} + }, +} + +// GetImportNode gets an ImportNode from the pool and initializes it +func GetImportNode(template Node, module string, line int) *ImportNode { + node := ImportNodePool.Get().(*ImportNode) + node.template = template + node.module = module + node.line = line + return node +} + +// ReleaseImportNode returns an ImportNode to the pool +func ReleaseImportNode(node *ImportNode) { + if node == nil { + return + } + node.template = nil + node.module = "" + ImportNodePool.Put(node) +} + +// FromImportNodePool provides a pool for FromImportNode objects +var FromImportNodePool = sync.Pool{ + New: func() interface{} { + return &FromImportNode{} + }, +} + +// GetFromImportNode gets a FromImportNode from the pool and initializes it +func GetFromImportNode(template Node, macros []string, aliases map[string]string, line int) *FromImportNode { + node := FromImportNodePool.Get().(*FromImportNode) + node.template = template + node.macros = macros + node.aliases = aliases + node.line = line + return node +} + +// ReleaseFromImportNode returns a FromImportNode to the pool +func ReleaseFromImportNode(node *FromImportNode) { + if node == nil { + return + } + node.template = nil + node.macros = nil + node.aliases = nil + FromImportNodePool.Put(node) +} + +// VerbatimNodePool provides a pool for VerbatimNode objects +var VerbatimNodePool = sync.Pool{ + New: func() interface{} { + return &VerbatimNode{} + }, +} + +// GetVerbatimNode gets a VerbatimNode from the pool and initializes it +func GetVerbatimNode(content string, line int) *VerbatimNode { + node := VerbatimNodePool.Get().(*VerbatimNode) + node.content = content + node.line = line + return node +} + +// ReleaseVerbatimNode returns a VerbatimNode to the pool +func ReleaseVerbatimNode(node *VerbatimNode) { + if node == nil { + return + } + node.content = "" + VerbatimNodePool.Put(node) +} + +// DoNodePool provides a pool for DoNode objects +var DoNodePool = sync.Pool{ + New: func() interface{} { + return &DoNode{} + }, +} + +// GetDoNode gets a DoNode from the pool and initializes it +func GetDoNode(expression Node, line int) *DoNode { + node := DoNodePool.Get().(*DoNode) + node.expression = expression + node.line = line + return node +} + +// ReleaseDoNode returns a DoNode to the pool +func ReleaseDoNode(node *DoNode) { + if node == nil { + return + } + node.expression = nil + DoNodePool.Put(node) +} + +// ApplyNodePool provides a pool for ApplyNode objects +var ApplyNodePool = sync.Pool{ + New: func() interface{} { + return &ApplyNode{} + }, +} + +// GetApplyNode gets an ApplyNode from the pool and initializes it +func GetApplyNode(body []Node, filter string, args []Node, line int) *ApplyNode { + node := ApplyNodePool.Get().(*ApplyNode) + node.body = body + node.filter = filter + node.args = args + node.line = line + return node +} + +// ReleaseApplyNode returns an ApplyNode to the pool +func ReleaseApplyNode(node *ApplyNode) { + if node == nil { + return + } + node.body = nil + node.filter = "" + node.args = nil + ApplyNodePool.Put(node) +} \ No newline at end of file diff --git a/parse_do.go b/parse_do.go index be2b8bf..b2b975a 100644 --- a/parse_do.go +++ b/parse_do.go @@ -9,48 +9,82 @@ func (p *Parser) parseDo(parser *Parser) (Node, error) { // Get the line number for error reporting doLine := parser.tokens[parser.tokenIndex-2].Line + // Check if we have an empty do tag ({% do %}) + if parser.tokenIndex < len(parser.tokens) && parser.tokens[parser.tokenIndex].Type == TOKEN_BLOCK_END { + // Empty do tag is not valid + return nil, fmt.Errorf("do tag cannot be empty at line %d", doLine) + } + // Check for special case: assignment expressions // These need to be handled specially since they're not normal expressions - if parser.tokenIndex < len(parser.tokens) && - parser.tokens[parser.tokenIndex].Type == TOKEN_NAME { - - varName := parser.tokens[parser.tokenIndex].Value - parser.tokenIndex++ - - if parser.tokenIndex < len(parser.tokens) && - parser.tokens[parser.tokenIndex].Type == TOKEN_OPERATOR && - parser.tokens[parser.tokenIndex].Value == "=" { - - // Skip the equals sign - parser.tokenIndex++ - - // Parse the right side expression - expr, err := parser.parseExpression() - if err != nil { - return nil, fmt.Errorf("error parsing expression in do assignment at line %d: %w", doLine, err) + if parser.tokenIndex < len(parser.tokens) { + // Look ahead to find possible assignment patterns + // We need to check for NUMBER = EXPR which is invalid + // as well as NAME = EXPR which is valid + + // Check if we have an equals sign in the next few tokens + hasAssignment := false + equalsPosition := -1 + + // Scan ahead a bit to find possible equals sign + for i := 0; i < 3 && parser.tokenIndex+i < len(parser.tokens); i++ { + token := parser.tokens[parser.tokenIndex+i] + if token.Type == TOKEN_OPERATOR && token.Value == "=" { + hasAssignment = true + equalsPosition = i + break } - - // Make sure we have the closing tag - if parser.tokenIndex >= len(parser.tokens) || parser.tokens[parser.tokenIndex].Type != TOKEN_BLOCK_END { - return nil, fmt.Errorf("expecting end of do tag at line %d", doLine) + + // Stop scanning if we hit the end of the block + if token.Type == TOKEN_BLOCK_END { + break } - parser.tokenIndex++ - - // Validate the variable name - it should not be a numeric literal - if _, err := strconv.Atoi(varName); err == nil { - return nil, fmt.Errorf("invalid variable name %q in do tag assignment at line %d", varName, doLine) - } - - // Create a SetNode instead of DoNode for assignments - return &SetNode{ - name: varName, - value: expr, - line: doLine, - }, nil } + + // If we found an equals sign, analyze the left-hand side + if hasAssignment && equalsPosition > 0 { + firstToken := parser.tokens[parser.tokenIndex] + + // Check if the left-hand side is a valid variable name + isValidVariableName := firstToken.Type == TOKEN_NAME + + // If the left-hand side is a number or literal, that's an error + if !isValidVariableName { + return nil, fmt.Errorf("invalid variable name %q in do tag assignment at line %d", firstToken.Value, doLine) + } + + // Handle assignment case + if isValidVariableName && hasAssignment { + varName := parser.tokens[parser.tokenIndex].Value + + // Skip tokens up to and including the equals sign + parser.tokenIndex += equalsPosition + 1 + + // Parse the right side expression + expr, err := parser.parseExpression() + if err != nil { + return nil, fmt.Errorf("error parsing expression in do assignment at line %d: %w", doLine, err) + } + + // Make sure we have the closing tag + if parser.tokenIndex >= len(parser.tokens) || parser.tokens[parser.tokenIndex].Type != TOKEN_BLOCK_END { + return nil, fmt.Errorf("expecting end of do tag at line %d", doLine) + } + parser.tokenIndex++ + + // Additional validation for variable name + if _, err := strconv.Atoi(varName); err == nil { + return nil, fmt.Errorf("invalid variable name %q in do tag assignment at line %d", varName, doLine) + } - // If it wasn't an assignment, backtrack to parse it as a normal expression - parser.tokenIndex -= 1 + // Create a SetNode instead of DoNode for assignments + return &SetNode{ + name: varName, + value: expr, + line: doLine, + }, nil + } + } } // Parse the expression to be executed diff --git a/parse_from.go b/parse_from.go index 51d2bef..ac92efa 100644 --- a/parse_from.go +++ b/parse_from.go @@ -11,9 +11,108 @@ import ( func (p *Parser) parseFrom(parser *Parser) (Node, error) { // Get the line number of the from token fromLine := parser.tokens[parser.tokenIndex-1].Line - - // We need to manually extract the template path, import keyword, and macro(s) from - // the current token. The tokenizer seems to be combining them. + + // Debugging: Print out tokens for debugging purposes + if IsDebugEnabled() { + LogDebug("Parsing from tag. Next tokens (up to 10):") + for i := 0; i < 10 && i+parser.tokenIndex < len(parser.tokens); i++ { + if parser.tokenIndex+i < len(parser.tokens) { + token := parser.tokens[parser.tokenIndex+i] + LogDebug(" Token %d: Type=%d, Value=%q", i, token.Type, token.Value) + } + } + } + + // First try to parse in token-by-token format (from zero-allocation tokenizer) + // Check if we have a proper sequence: [STRING/NAME, NAME="import", NAME="macroname", ...] + if parser.tokenIndex+1 < len(parser.tokens) { + // Check for (template path) followed by "import" keyword + firstToken := parser.tokens[parser.tokenIndex] + secondToken := parser.tokens[parser.tokenIndex+1] + + isTemplatePath := firstToken.Type == TOKEN_STRING || firstToken.Type == TOKEN_NAME + isImportKeyword := secondToken.Type == TOKEN_NAME && secondToken.Value == "import" + + if isTemplatePath && isImportKeyword { + LogDebug("Found tokenized from...import pattern") + + // Get template path + templatePath := firstToken.Value + if firstToken.Type == TOKEN_NAME { + // For paths like ./file.twig, just use the value + templatePath = strings.Trim(templatePath, "\"'") + } + + // Create template expression + templateExpr := &LiteralNode{ + ExpressionNode: ExpressionNode{ + exprType: ExprLiteral, + line: fromLine, + }, + value: templatePath, + } + + // Skip past the template path and import keyword + parser.tokenIndex += 2 + + // Parse macros and aliases + macros := []string{} + aliases := map[string]string{} + + // Process tokens until end of block + for parser.tokenIndex < len(parser.tokens) { + token := parser.tokens[parser.tokenIndex] + + // Stop at block end + if token.Type == TOKEN_BLOCK_END || token.Type == TOKEN_BLOCK_END_TRIM { + parser.tokenIndex++ + break + } + + // Skip punctuation (commas) + if token.Type == TOKEN_PUNCTUATION { + parser.tokenIndex++ + continue + } + + // Handle macro name + if token.Type == TOKEN_NAME { + macroName := token.Value + + // Add to macros list + macros = append(macros, macroName) + + // Check for alias + parser.tokenIndex++ + if parser.tokenIndex < len(parser.tokens) && + parser.tokens[parser.tokenIndex].Type == TOKEN_NAME && + parser.tokens[parser.tokenIndex].Value == "as" { + + // Skip 'as' keyword + parser.tokenIndex++ + + // Get alias + if parser.tokenIndex < len(parser.tokens) && parser.tokens[parser.tokenIndex].Type == TOKEN_NAME { + aliases[macroName] = parser.tokens[parser.tokenIndex].Value + parser.tokenIndex++ + } + } + } else { + // Skip any other token + parser.tokenIndex++ + } + } + + // If we found macros, return a FromImportNode + if len(macros) > 0 { + return NewFromImportNode(templateExpr, macros, aliases, fromLine), nil + } + } + } + + // Fall back to the original approach (for backward compatibility) + // We need to extract the template path, import keyword, and macro(s) from + // the current token. The tokenizer may be combining them. if parser.tokenIndex < len(parser.tokens) && parser.tokens[parser.tokenIndex].Type == TOKEN_NAME { // Extract parts from the combined token value tokenValue := parser.tokens[parser.tokenIndex].Value diff --git a/parse_macro.go b/parse_macro.go index c4865ae..07e36fd 100644 --- a/parse_macro.go +++ b/parse_macro.go @@ -162,7 +162,6 @@ func (p *Parser) parseMacro(parser *Parser) (Node, error) { } paramName := parser.tokens[parser.tokenIndex].Value - fmt.Println("DEBUG: Parameter name:", paramName) params = append(params, paramName) parser.tokenIndex++ diff --git a/parser.go b/parser.go index 86a9517..31cf468 100644 --- a/parser.go +++ b/parser.go @@ -59,19 +59,28 @@ func (p *Parser) Parse(source string) (Node, error) { // Initialize default block handlers p.initBlockHandlers() - // Use the HTML-preserving tokenizer to preserve HTML content exactly + // Use the zero-allocation tokenizer for maximum performance and minimal allocations // This will treat everything outside twig tags as TEXT tokens var err error - p.tokens, err = p.htmlPreservingTokenize() + + // Use the zero-allocation tokenizer to achieve minimal memory usage and high performance + tokenizer := GetTokenizer(p.source, 0) + p.tokens, err = tokenizer.TokenizeHtmlPreserving() + + // Apply whitespace control to handle whitespace trimming directives + if err == nil { + tokenizer.ApplyWhitespaceControl() + } + + // Release the tokenizer back to the pool + ReleaseTokenizer(tokenizer) + if err != nil { return nil, fmt.Errorf("tokenization error: %w", err) } // Template tokenization complete - - // Apply whitespace control processing to the tokens to handle - // the whitespace trimming between template elements - p.tokens = processWhitespaceControl(p.tokens) + // Whitespace control has already been applied by the tokenizer // Parse tokens into nodes nodes, err := p.parseOuterTemplate() diff --git a/relative_path_test.go b/relative_path_test.go index 0ddb9b1..cc439b7 100644 --- a/relative_path_test.go +++ b/relative_path_test.go @@ -80,7 +80,13 @@ func TestRelativePathsWithFromImport(t *testing.T) { // Create simple macro templates macrosTemplate := `{% macro simple() %}Macro output{% endmacro %}` + + // Print the template content for debugging + t.Logf("Simple template content: %s", macrosTemplate) + + // Note: The template needs to be in the format: {% from "template" import macro %} useTemplate := `{% from "./simple.twig" import simple %}{{ simple() }}` + t.Logf("Use template content: %s", useTemplate) // Write templates to files err = os.WriteFile(filepath.Join(macrosDir, "simple.twig"), []byte(macrosTemplate), 0644) diff --git a/render.go b/render.go index b37073e..657b9cb 100644 --- a/render.go +++ b/render.go @@ -30,14 +30,35 @@ type RenderContext struct { sandboxed bool // Flag indicating if this context is sandboxed } +// contextMapPool is a pool for the maps used in RenderContext +var contextMapPool = sync.Pool{ + New: func() interface{} { + return make(map[string]interface{}, 16) // Pre-allocate with reasonable size + }, +} + +// blocksMapPool is a pool for the blocks map used in RenderContext +var blocksMapPool = sync.Pool{ + New: func() interface{} { + return make(map[string][]Node, 8) // Pre-allocate with reasonable size + }, +} + +// macrosMapPool is a pool for the macros map used in RenderContext +var macrosMapPool = sync.Pool{ + New: func() interface{} { + return make(map[string]Node, 8) // Pre-allocate with reasonable size + }, +} + // renderContextPool is a sync.Pool for RenderContext objects var renderContextPool = sync.Pool{ New: func() interface{} { return &RenderContext{ - context: make(map[string]interface{}), - blocks: make(map[string][]Node), - parentBlocks: make(map[string][]Node), - macros: make(map[string]Node), + context: contextMapPool.Get().(map[string]interface{}), + blocks: blocksMapPool.Get().(map[string][]Node), + parentBlocks: blocksMapPool.Get().(map[string][]Node), + macros: macrosMapPool.Get().(map[string]Node), } }, } @@ -46,17 +67,44 @@ var renderContextPool = sync.Pool{ func NewRenderContext(env *Environment, context map[string]interface{}, engine *Engine) *RenderContext { ctx := renderContextPool.Get().(*RenderContext) - // Reset and initialize the context - for k := range ctx.context { - delete(ctx.context, k) - } - for k := range ctx.blocks { - delete(ctx.blocks, k) - } - for k := range ctx.macros { - delete(ctx.macros, k) + // Ensure all maps are initialized (should be from the pool) + if ctx.context == nil { + ctx.context = contextMapPool.Get().(map[string]interface{}) + } else { + // Clear any existing data + for k := range ctx.context { + delete(ctx.context, k) + } } + if ctx.blocks == nil { + ctx.blocks = blocksMapPool.Get().(map[string][]Node) + } else { + // Clear any existing data + for k := range ctx.blocks { + delete(ctx.blocks, k) + } + } + + if ctx.parentBlocks == nil { + ctx.parentBlocks = blocksMapPool.Get().(map[string][]Node) + } else { + // Clear any existing data + for k := range ctx.parentBlocks { + delete(ctx.parentBlocks, k) + } + } + + if ctx.macros == nil { + ctx.macros = macrosMapPool.Get().(map[string]Node) + } else { + // Clear any existing data + for k := range ctx.macros { + delete(ctx.macros, k) + } + } + + // Set basic properties ctx.env = env ctx.engine = engine ctx.extending = false @@ -65,7 +113,7 @@ func NewRenderContext(env *Environment, context map[string]interface{}, engine * ctx.inParentCall = false ctx.sandboxed = false - // Copy the context values + // Copy the context values directly if context != nil { for k, v := range context { ctx.context[k] = v @@ -82,25 +130,52 @@ func (ctx *RenderContext) Release() { ctx.engine = nil ctx.currentBlock = nil - // Clear map contents but keep allocated maps - for k := range ctx.context { - delete(ctx.context, k) - } - for k := range ctx.blocks { - delete(ctx.blocks, k) - } - for k := range ctx.parentBlocks { - delete(ctx.parentBlocks, k) - } - for k := range ctx.macros { - delete(ctx.macros, k) - } + // Save the maps so we can return them to their respective pools + contextMap := ctx.context + blocksMap := ctx.blocks + parentBlocksMap := ctx.parentBlocks + macrosMap := ctx.macros + + // Clear the maps from the context + ctx.context = nil + ctx.blocks = nil + ctx.parentBlocks = nil + ctx.macros = nil // Don't release parent contexts - they'll be released separately ctx.parent = nil // Return to pool renderContextPool.Put(ctx) + + // Clear map contents and return them to their pools + if contextMap != nil { + for k := range contextMap { + delete(contextMap, k) + } + contextMapPool.Put(contextMap) + } + + if blocksMap != nil { + for k := range blocksMap { + delete(blocksMap, k) + } + blocksMapPool.Put(blocksMap) + } + + if parentBlocksMap != nil { + for k := range parentBlocksMap { + delete(parentBlocksMap, k) + } + blocksMapPool.Put(parentBlocksMap) + } + + if macrosMap != nil { + for k := range macrosMap { + delete(macrosMap, k) + } + macrosMapPool.Put(macrosMap) + } } // Error types @@ -242,21 +317,63 @@ func (ctx *RenderContext) IsSandboxed() bool { // Clone creates a new context as a child of the current context func (ctx *RenderContext) Clone() *RenderContext { - // Create a new context - newCtx := NewRenderContext(ctx.env, make(map[string]interface{}), ctx.engine) + // Get a new context from the pool with empty maps + newCtx := renderContextPool.Get().(*RenderContext) - // Set parent relationship + // Initialize the context + newCtx.env = ctx.env + newCtx.engine = ctx.engine + newCtx.extending = false + newCtx.currentBlock = nil newCtx.parent = ctx + newCtx.inParentCall = false // Inherit sandbox state newCtx.sandboxed = ctx.sandboxed - // Copy blocks + // Ensure maps are initialized (they should be from the pool already) + if newCtx.context == nil { + newCtx.context = contextMapPool.Get().(map[string]interface{}) + } else { + // Clear any existing data + for k := range newCtx.context { + delete(newCtx.context, k) + } + } + + if newCtx.blocks == nil { + newCtx.blocks = blocksMapPool.Get().(map[string][]Node) + } else { + // Clear any existing data + for k := range newCtx.blocks { + delete(newCtx.blocks, k) + } + } + + if newCtx.macros == nil { + newCtx.macros = macrosMapPool.Get().(map[string]Node) + } else { + // Clear any existing data + for k := range newCtx.macros { + delete(newCtx.macros, k) + } + } + + if newCtx.parentBlocks == nil { + newCtx.parentBlocks = blocksMapPool.Get().(map[string][]Node) + } else { + // Clear any existing data + for k := range newCtx.parentBlocks { + delete(newCtx.parentBlocks, k) + } + } + + // Copy blocks by reference (no need to deep copy) for name, nodes := range ctx.blocks { newCtx.blocks[name] = nodes } - // Copy macros + // Copy macros by reference (no need to deep copy) for name, macro := range ctx.macros { newCtx.macros[name] = macro } @@ -287,14 +404,14 @@ func (ctx *RenderContext) GetMacros() map[string]Node { // InitMacros initializes the macros map if it's nil func (ctx *RenderContext) InitMacros() { if ctx.macros == nil { - ctx.macros = make(map[string]Node) + ctx.macros = macrosMapPool.Get().(map[string]Node) } } // SetMacro sets a macro in the context func (ctx *RenderContext) SetMacro(name string, macro Node) { if ctx.macros == nil { - ctx.macros = make(map[string]Node) + ctx.macros = macrosMapPool.Get().(map[string]Node) } ctx.macros[name] = macro } @@ -658,14 +775,16 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { } case *ArrayNode: - // Evaluate each item in the array - items := make([]interface{}, len(n.items)) - for i, item := range n.items { - val, err := ctx.EvaluateExpression(item) + // We need to avoid pooling for arrays that might be directly used by filters like merge + // as those filters return the slice directly to the user + items := make([]interface{}, 0, len(n.items)) + + for i := 0; i < len(n.items); i++ { + val, err := ctx.EvaluateExpression(n.items[i]) if err != nil { return nil, err } - items[i] = val + items = append(items, val) } // Always return a non-nil slice, even if empty @@ -676,8 +795,10 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { return items, nil case *HashNode: - // Evaluate each key-value pair in the hash - result := make(map[string]interface{}) + // Evaluate each key-value pair in the hash using a new map + // We can't use pooling with defer here because the map is returned directly + result := make(map[string]interface{}, len(n.items)) + for k, v := range n.items { // Evaluate the key keyVal, err := ctx.EvaluateExpression(k) @@ -712,10 +833,11 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { return nil, err } - // Evaluate all arguments + // Evaluate all arguments - need direct allocation args := make([]interface{}, len(n.args)) - for i, arg := range n.args { - val, err := ctx.EvaluateExpression(arg) + + for i := 0; i < len(n.args); i++ { + val, err := ctx.EvaluateExpression(n.args[i]) if err != nil { return nil, err } @@ -753,10 +875,12 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { // Check if it's a macro call if macro, ok := ctx.GetMacro(n.name); ok { - // Evaluate arguments + // Evaluate arguments - need direct allocation for macro calls args := make([]interface{}, len(n.args)) - for i, arg := range n.args { - val, err := ctx.EvaluateExpression(arg) + + // Evaluate arguments + for i := 0; i < len(n.args); i++ { + val, err := ctx.EvaluateExpression(n.args[i]) if err != nil { return nil, err } @@ -774,10 +898,12 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { } // Otherwise, it's a regular function call - // Evaluate arguments + // Evaluate arguments - need direct allocation for function calls args := make([]interface{}, len(n.args)) - for i, arg := range n.args { - val, err := ctx.EvaluateExpression(arg) + + // Evaluate arguments + for i := 0; i < len(n.args); i++ { + val, err := ctx.EvaluateExpression(n.args[i]) if err != nil { return nil, err } @@ -894,10 +1020,12 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { return nil, err } - // Evaluate test arguments + // Evaluate test arguments - need direct allocation args := make([]interface{}, len(n.args)) - for i, arg := range n.args { - val, err := ctx.EvaluateExpression(arg) + + // Evaluate arguments + for i := 0; i < len(n.args); i++ { + val, err := ctx.EvaluateExpression(n.args[i]) if err != nil { return nil, err } diff --git a/render_context_benchmark_test.go b/render_context_benchmark_test.go new file mode 100644 index 0000000..9fdd3a3 --- /dev/null +++ b/render_context_benchmark_test.go @@ -0,0 +1,129 @@ +package twig + +import ( + "testing" +) + +func BenchmarkRenderContextCreation(b *testing.B) { + engine := New() + + // Create a simple context with a few variables + contextVars := map[string]interface{}{ + "user": map[string]interface{}{ + "name": "John Doe", + "email": "john@example.com", + "age": 30, + }, + "items": []string{"item1", "item2", "item3"}, + "count": 42, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ctx := NewRenderContext(engine.environment, contextVars, engine) + ctx.Release() // Return to pool after use + } +} + +func BenchmarkRenderContextCloning(b *testing.B) { + engine := New() + + // Create a parent context with some variables, blocks, and macros + parentContext := map[string]interface{}{ + "user": map[string]interface{}{ + "name": "John Doe", + "email": "john@example.com", + }, + "items": []string{"item1", "item2", "item3"}, + } + + // Create the parent context + parent := NewRenderContext(engine.environment, parentContext, engine) + + // Add some blocks + header := &TextNode{content: "Header Content", line: 1} + footer := &TextNode{content: "Footer Content", line: 2} + parent.blocks["header"] = []Node{header} + parent.blocks["footer"] = []Node{footer} + + // Add a simple macro + macroNode := &MacroNode{ + name: "format", + line: 3, + } + parent.macros["format"] = macroNode + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Clone the context - this should reuse memory from pools + child := parent.Clone() + + // Do some operations on the child context + child.SetVariable("newVar", "test value") + + // Release the child context + child.Release() + } + + // Clean up parent context + parent.Release() +} + +func BenchmarkNestedContextCreation(b *testing.B) { + engine := New() + baseContext := map[string]interface{}{"base": "value"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Create a chain of 5 nested contexts + ctx1 := NewRenderContext(engine.environment, baseContext, engine) + ctx2 := ctx1.Clone() + ctx3 := ctx2.Clone() + ctx4 := ctx3.Clone() + ctx5 := ctx4.Clone() + + // Make some changes to test variable lookup + ctx5.SetVariable("level5", "value5") + ctx3.SetVariable("level3", "value3") + + // Release in reverse order + ctx5.Release() + ctx4.Release() + ctx3.Release() + ctx2.Release() + ctx1.Release() + } +} + +func BenchmarkContextVariableLookup(b *testing.B) { + engine := New() + + // Create a chain of contexts with variables at different levels + rootCtx := NewRenderContext(engine.environment, map[string]interface{}{ + "rootVar": "root value", + "shared": "root version", + }, engine) + + level1 := rootCtx.Clone() + level1.SetVariable("level1Var", "level1 value") + level1.SetVariable("shared", "level1 version") + + level2 := level1.Clone() + level2.SetVariable("level2Var", "level2 value") + + // Setup complete, start benchmark + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Test variable lookup at different levels + level2.GetVariable("level2Var") // Local var + level2.GetVariable("level1Var") // Parent var + level2.GetVariable("rootVar") // Root var + level2.GetVariable("shared") // Shadowed var + level2.GetVariable("nonExistentVar") // Missing var + } + + // Clean up + level2.Release() + level1.Release() + rootCtx.Release() +} \ No newline at end of file diff --git a/token_pool_improved.go b/token_pool_improved.go new file mode 100644 index 0000000..cb94401 --- /dev/null +++ b/token_pool_improved.go @@ -0,0 +1,521 @@ +package twig + +import ( + "fmt" + "strings" + "sync" +) + +// ImprovedTokenSlice is a more efficient implementation of a token slice pool +// that truly minimizes allocations during tokenization +type ImprovedTokenSlice struct { + tokens []Token // The actual token slice + capacity int // Capacity hint for the token slice + used bool // Whether this slice has been used +} + +// global pool for ImprovedTokenSlice objects +var improvedTokenSlicePool = sync.Pool{ + New: func() interface{} { + // Start with a reasonably sized token slice + tokens := make([]Token, 0, 64) + return &ImprovedTokenSlice{ + tokens: tokens, + capacity: 64, + used: false, + } + }, +} + +// Global token object pool +var tokenObjectPool = sync.Pool{ + New: func() interface{} { + return &Token{} + }, +} + +// GetImprovedTokenSlice gets a token slice from the pool +func GetImprovedTokenSlice(capacityHint int) *ImprovedTokenSlice { + slice := improvedTokenSlicePool.Get().(*ImprovedTokenSlice) + + // Reset the slice but keep capacity + if cap(slice.tokens) < capacityHint { + // Need to allocate a larger slice + slice.tokens = make([]Token, 0, capacityHint) + slice.capacity = capacityHint + } else { + // Reuse existing slice + slice.tokens = slice.tokens[:0] + } + + slice.used = false + return slice +} + +// AppendToken adds a token to the slice +func (s *ImprovedTokenSlice) AppendToken(tokenType int, value string, line int) { + if s.used { + return // Already finalized + } + + // Create a token and add it to the slice + token := Token{ + Type: tokenType, + Value: value, + Line: line, + } + + s.tokens = append(s.tokens, token) +} + +// Finalize returns the token slice +func (s *ImprovedTokenSlice) Finalize() []Token { + if s.used { + return s.tokens + } + + s.used = true + return s.tokens +} + +// Release returns the token slice to the pool +func (s *ImprovedTokenSlice) Release() { + if s.used && cap(s.tokens) <= 1024 { // Don't pool very large slices + // Only return reasonably sized slices to the pool + improvedTokenSlicePool.Put(s) + } +} + +// optimizedTokenizeExpressionImproved is a minimal allocation version of tokenizeExpression +func (p *Parser) optimizedTokenizeExpressionImproved(expr string, tokens *ImprovedTokenSlice, line int) { + var inString bool + var stringDelimiter byte + var stringStart int + + // Preallocate a buffer for building tokens + buffer := make([]byte, 0, 64) + + for i := 0; i < len(expr); i++ { + c := expr[i] + + // Handle string literals + if (c == '"' || c == '\'') && (i == 0 || expr[i-1] != '\\') { + if inString && c == stringDelimiter { + // End of string, add the string token + tokens.AppendToken(TOKEN_STRING, expr[stringStart:i], line) + inString = false + } else if !inString { + // Start of string + inString = true + stringDelimiter = c + stringStart = i + 1 + } + continue + } + + // Skip chars inside strings + if inString { + continue + } + + // Handle operators + if isCharOperator(c) { + // Check for two-character operators + if i+1 < len(expr) { + nextChar := expr[i+1] + + if (c == '=' && nextChar == '=') || + (c == '!' && nextChar == '=') || + (c == '>' && nextChar == '=') || + (c == '<' && nextChar == '=') || + (c == '&' && nextChar == '&') || + (c == '|' && nextChar == '|') || + (c == '?' && nextChar == '?') { + + // Two-char operator + buffer = buffer[:0] + buffer = append(buffer, c, nextChar) + tokens.AppendToken(TOKEN_OPERATOR, string(buffer), line) + i++ + continue + } + } + + // Single-char operator + tokens.AppendToken(TOKEN_OPERATOR, string([]byte{c}), line) + continue + } + + // Handle punctuation + if isCharPunctuation(c) { + tokens.AppendToken(TOKEN_PUNCTUATION, string([]byte{c}), line) + continue + } + + // Skip whitespace + if isCharWhitespace(c) { + continue + } + + // Handle identifiers, literals, etc. + if isCharAlpha(c) || c == '_' { + // Start of an identifier + start := i + + // Find the end + for i++; i < len(expr) && (isCharAlpha(expr[i]) || isCharDigit(expr[i]) || expr[i] == '_'); i++ { + } + + // Extract the identifier + identifier := expr[start:i] + i-- // Adjust for loop increment + + // Add token + tokens.AppendToken(TOKEN_NAME, identifier, line) + continue + } + + // Handle numbers + if isCharDigit(c) || (c == '-' && i+1 < len(expr) && isCharDigit(expr[i+1])) { + start := i + + // Skip negative sign if present + if c == '-' { + i++ + } + + // Find end of number + for i++; i < len(expr) && isCharDigit(expr[i]); i++ { + } + + // Check for decimal point + if i < len(expr) && expr[i] == '.' { + i++ + for ; i < len(expr) && isCharDigit(expr[i]); i++ { + } + } + + // Extract the number + number := expr[start:i] + i-- // Adjust for loop increment + + tokens.AppendToken(TOKEN_NUMBER, number, line) + continue + } + } +} + +// Helper functions to reduce allocations for character checks - inlined to avoid naming conflicts + +// isCharAlpha checks if a character is alphabetic +func isCharAlpha(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +// isCharDigit checks if a character is a digit +func isCharDigit(c byte) bool { + return c >= '0' && c <= '9' +} + +// isCharOperator checks if a character is an operator +func isCharOperator(c byte) bool { + return c == '=' || c == '+' || c == '-' || c == '*' || c == '/' || + c == '%' || c == '&' || c == '|' || c == '^' || c == '~' || + c == '<' || c == '>' || c == '!' || c == '?' +} + +// isCharPunctuation checks if a character is punctuation +func isCharPunctuation(c byte) bool { + return c == '(' || c == ')' || c == '[' || c == ']' || c == '{' || c == '}' || + c == '.' || c == ',' || c == ':' || c == ';' +} + +// isCharWhitespace checks if a character is whitespace +func isCharWhitespace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' +} + +// improvedHtmlPreservingTokenize is a zero-allocation version of the HTML preserving tokenizer +func (p *Parser) improvedHtmlPreservingTokenize() ([]Token, error) { + // Estimate token count based on source length + estimatedTokens := len(p.source) / 20 // Rough estimate + tokens := GetImprovedTokenSlice(estimatedTokens) + defer tokens.Release() + + var currentPosition int + line := 1 + + // Reusable buffers to avoid allocations + tagPatterns := [5]string{"{{-", "{{", "{%-", "{%", "{#"} + tagTypes := [5]int{TOKEN_VAR_START_TRIM, TOKEN_VAR_START, TOKEN_BLOCK_START_TRIM, TOKEN_BLOCK_START, TOKEN_COMMENT_START} + tagLengths := [5]int{3, 2, 3, 2, 2} + + for currentPosition < len(p.source) { + // Find the next tag + nextTagPos := -1 + tagType := -1 + tagLength := 0 + + // Check for all possible tag patterns + for i := 0; i < 5; i++ { + pos := strings.Index(p.source[currentPosition:], tagPatterns[i]) + if pos != -1 { + // Adjust position relative to current position + pos += currentPosition + + // If this is the first tag found or it's closer than previous ones + if nextTagPos == -1 || pos < nextTagPos { + nextTagPos = pos + tagType = tagTypes[i] + tagLength = tagLengths[i] + } + } + } + + // Check if the tag is escaped + if nextTagPos != -1 && nextTagPos > 0 && p.source[nextTagPos-1] == '\\' { + // Add text up to the backslash + if nextTagPos-1 > currentPosition { + preText := p.source[currentPosition:nextTagPos-1] + tokens.AppendToken(TOKEN_TEXT, preText, line) + line += countNewlines(preText) + } + + // Add the tag as literal text (without the backslash) + // Find which pattern was matched + for i := 0; i < 5; i++ { + if tagType == tagTypes[i] { + tokens.AppendToken(TOKEN_TEXT, tagPatterns[i], line) + break + } + } + + // Move past this tag + currentPosition = nextTagPos + tagLength + continue + } + + // No more tags found - add the rest as TEXT + if nextTagPos == -1 { + remainingText := p.source[currentPosition:] + if len(remainingText) > 0 { + tokens.AppendToken(TOKEN_TEXT, remainingText, line) + line += countNewlines(remainingText) + } + break + } + + // Add text before the tag + if nextTagPos > currentPosition { + textContent := p.source[currentPosition:nextTagPos] + tokens.AppendToken(TOKEN_TEXT, textContent, line) + line += countNewlines(textContent) + } + + // Add the tag start token + tokens.AppendToken(tagType, "", line) + + // Move past opening tag + currentPosition = nextTagPos + tagLength + + // Find matching end tag + var endTag string + var endTagType int + var endTagLength int + + if tagType == TOKEN_VAR_START || tagType == TOKEN_VAR_START_TRIM { + // Look for "}}" or "-}}" + endPos1 := strings.Index(p.source[currentPosition:], "}}") + endPos2 := strings.Index(p.source[currentPosition:], "-}}") + + if endPos1 != -1 && (endPos2 == -1 || endPos1 < endPos2) { + endTag = "}}" + endTagType = TOKEN_VAR_END + endTagLength = 2 + } else if endPos2 != -1 { + endTag = "-}}" + endTagType = TOKEN_VAR_END_TRIM + endTagLength = 3 + } else { + return nil, fmt.Errorf("unclosed variable tag at line %d", line) + } + } else if tagType == TOKEN_BLOCK_START || tagType == TOKEN_BLOCK_START_TRIM { + // Look for "%}" or "-%}" + endPos1 := strings.Index(p.source[currentPosition:], "%}") + endPos2 := strings.Index(p.source[currentPosition:], "-%}") + + if endPos1 != -1 && (endPos2 == -1 || endPos1 < endPos2) { + endTag = "%}" + endTagType = TOKEN_BLOCK_END + endTagLength = 2 + } else if endPos2 != -1 { + endTag = "-%}" + endTagType = TOKEN_BLOCK_END_TRIM + endTagLength = 3 + } else { + return nil, fmt.Errorf("unclosed block tag at line %d", line) + } + } else if tagType == TOKEN_COMMENT_START { + // Look for "#}" + endPos := strings.Index(p.source[currentPosition:], "#}") + if endPos == -1 { + return nil, fmt.Errorf("unclosed comment at line %d", line) + } + endTag = "#}" + endTagType = TOKEN_COMMENT_END + endTagLength = 2 + } + + // Find position of the end tag + endPos := strings.Index(p.source[currentPosition:], endTag) + if endPos == -1 { + return nil, fmt.Errorf("unclosed tag at line %d", line) + } + + // Get content between tags + tagContent := p.source[currentPosition:currentPosition+endPos] + line += countNewlines(tagContent) + + // Process tag content based on type + if tagType == TOKEN_COMMENT_START { + // Store comments as TEXT tokens + if len(tagContent) > 0 { + tokens.AppendToken(TOKEN_TEXT, tagContent, line) + } + } else { + // For variable and block tags, tokenize the content + tagContent = strings.TrimSpace(tagContent) + + if tagType == TOKEN_BLOCK_START || tagType == TOKEN_BLOCK_START_TRIM { + // Process block tags with optimized tokenization + processBlockTag(tagContent, tokens, line, p) + } else { + // Process variable tags with optimized tokenization + if len(tagContent) > 0 { + if !strings.ContainsAny(tagContent, ".|[](){}\"',+-*/=!<>%&^~") { + // Simple variable name + tokens.AppendToken(TOKEN_NAME, tagContent, line) + } else { + // Complex expression + expressionTokens := GetImprovedTokenSlice(len(tagContent) / 4) + p.optimizedTokenizeExpressionImproved(tagContent, expressionTokens, line) + + // Copy tokens + for _, token := range expressionTokens.tokens { + tokens.AppendToken(token.Type, token.Value, token.Line) + } + + expressionTokens.Release() + } + } + } + } + + // Add the end tag token + tokens.AppendToken(endTagType, "", line) + + // Move past the end tag + currentPosition = currentPosition + endPos + endTagLength + } + + // Add EOF token + tokens.AppendToken(TOKEN_EOF, "", line) + + return tokens.Finalize(), nil +} + +// Helper function to process block tags +func processBlockTag(content string, tokens *ImprovedTokenSlice, line int, p *Parser) { + // Extract the tag name + parts := strings.SplitN(content, " ", 2) + if len(parts) > 0 { + blockName := parts[0] + tokens.AppendToken(TOKEN_NAME, blockName, line) + + // Process rest of the block content + if len(parts) > 1 { + blockContent := strings.TrimSpace(parts[1]) + + switch blockName { + case "if", "elseif": + // For conditional blocks, tokenize expression + exprTokens := GetImprovedTokenSlice(len(blockContent) / 4) + p.optimizedTokenizeExpressionImproved(blockContent, exprTokens, line) + + // Copy tokens + for _, token := range exprTokens.tokens { + tokens.AppendToken(token.Type, token.Value, token.Line) + } + + exprTokens.Release() + + case "for": + // Process for loop with iterator(s) and collection + inPos := strings.Index(strings.ToLower(blockContent), " in ") + if inPos != -1 { + iterators := strings.TrimSpace(blockContent[:inPos]) + collection := strings.TrimSpace(blockContent[inPos+4:]) + + // Handle key, value iterator syntax + if strings.Contains(iterators, ",") { + iterParts := strings.SplitN(iterators, ",", 2) + if len(iterParts) == 2 { + tokens.AppendToken(TOKEN_NAME, strings.TrimSpace(iterParts[0]), line) + tokens.AppendToken(TOKEN_PUNCTUATION, ",", line) + tokens.AppendToken(TOKEN_NAME, strings.TrimSpace(iterParts[1]), line) + } + } else { + // Single iterator + tokens.AppendToken(TOKEN_NAME, iterators, line) + } + + // Add 'in' keyword + tokens.AppendToken(TOKEN_NAME, "in", line) + + // Process collection expression + collectionTokens := GetImprovedTokenSlice(len(collection) / 4) + p.optimizedTokenizeExpressionImproved(collection, collectionTokens, line) + + // Copy tokens + for _, token := range collectionTokens.tokens { + tokens.AppendToken(token.Type, token.Value, token.Line) + } + + collectionTokens.Release() + } else { + // Fallback for malformed for loops + tokens.AppendToken(TOKEN_NAME, blockContent, line) + } + + case "set": + // Handle variable assignment + assignPos := strings.Index(blockContent, "=") + if assignPos != -1 { + varName := strings.TrimSpace(blockContent[:assignPos]) + value := strings.TrimSpace(blockContent[assignPos+1:]) + + tokens.AppendToken(TOKEN_NAME, varName, line) + tokens.AppendToken(TOKEN_OPERATOR, "=", line) + + // Tokenize value expression + valueTokens := GetImprovedTokenSlice(len(value) / 4) + p.optimizedTokenizeExpressionImproved(value, valueTokens, line) + + // Copy tokens + for _, token := range valueTokens.tokens { + tokens.AppendToken(token.Type, token.Value, token.Line) + } + + valueTokens.Release() + } else { + // Simple set without assignment + tokens.AppendToken(TOKEN_NAME, blockContent, line) + } + + default: + // Other block types + tokens.AppendToken(TOKEN_NAME, blockContent, line) + } + } + } +} \ No newline at end of file diff --git a/token_pool_optimization.go b/token_pool_optimization.go new file mode 100644 index 0000000..17dc02c --- /dev/null +++ b/token_pool_optimization.go @@ -0,0 +1,165 @@ +package twig + +import ( + "sync" +) + +// This file implements optimized token handling functions to reduce allocations +// during the tokenization process. + +// PooledToken represents a token from the token pool +// We use a separate struct to avoid accidentally returning the same instance +type PooledToken struct { + token *Token // Reference to the token from the pool +} + +// PooledTokenSlice is a slice of tokens with a reference to the original pooled slice +type PooledTokenSlice struct { + tokens []Token // The token slice + poolRef *[]Token // Reference to the original slice from the pool + used bool // Whether this slice has been used + tmpPool sync.Pool // Pool for temporary token objects + scratch []*Token // Scratch space for temporary tokens +} + +// GetPooledTokenSlice gets a token slice from the pool with the given capacity hint +func GetPooledTokenSlice(capacityHint int) *PooledTokenSlice { + slice := &PooledTokenSlice{ + tmpPool: sync.Pool{ + New: func() interface{} { + return &Token{} + }, + }, + scratch: make([]*Token, 0, 16), // Pre-allocate scratch space + used: false, + } + + // Get a token slice from the pool + pooledSlice := GetTokenSlice(capacityHint) + slice.tokens = pooledSlice + slice.poolRef = &pooledSlice + + return slice +} + +// AppendToken adds a token to the slice using pooled tokens +func (s *PooledTokenSlice) AppendToken(tokenType int, value string, line int) { + if s.used { + // This slice has already been finalized, can't append anymore + return + } + + // Get a token from the pool + token := s.tmpPool.Get().(*Token) + token.Type = tokenType + token.Value = value + token.Line = line + + // Keep a reference to this token so we can clean it up later + s.scratch = append(s.scratch, token) + + // Add a copy of the token to the slice + s.tokens = append(s.tokens, *token) +} + +// Finalize returns the token slice and cleans up temporary tokens +func (s *PooledTokenSlice) Finalize() []Token { + if s.used { + // Already finalized + return s.tokens + } + + // Mark as used so we don't accidentally use it again + s.used = true + + // Clean up temporary tokens + for _, token := range s.scratch { + token.Value = "" + s.tmpPool.Put(token) + } + + // Clear scratch slice but keep capacity + s.scratch = s.scratch[:0] + + return s.tokens +} + +// Release returns the token slice to the pool +func (s *PooledTokenSlice) Release() { + if s.poolRef != nil { + ReleaseTokenSlice(*s.poolRef) + s.poolRef = nil + } + + // Clean up any remaining temporary tokens + for _, token := range s.scratch { + token.Value = "" + s.tmpPool.Put(token) + } + + // Clear references + s.scratch = nil + s.tokens = nil + s.used = true +} + +// getPooledToken gets a token from the pool (for internal use) +func getPooledToken() *Token { + return TokenPool.Get().(*Token) +} + +// releasePooledToken returns a token to the pool (for internal use) +func releasePooledToken(token *Token) { + if token == nil { + return + } + token.Value = "" + TokenPool.Put(token) +} + +// TOKEN SLICES - additional optimization for token slice reuse + +// TokenNodePool provides a pool for pre-sized token node arrays +var TokenNodePool = sync.Pool{ + New: func() interface{} { + // Default capacity that covers most cases + slice := make([]Node, 0, 32) + return &slice + }, +} + +// GetTokenNodeSlice gets a slice of Node from the pool +func GetTokenNodeSlice(capacityHint int) *[]Node { + slice := TokenNodePool.Get().(*[]Node) + + // If the capacity is too small, allocate a new slice + if cap(*slice) < capacityHint { + *slice = make([]Node, 0, capacityHint) + } else { + // Otherwise, clear the slice but keep capacity + *slice = (*slice)[:0] + } + + return slice +} + +// ReleaseTokenNodeSlice returns a slice of Node to the pool +func ReleaseTokenNodeSlice(slice *[]Node) { + if slice == nil { + return + } + + // Only pool reasonably sized slices + if cap(*slice) > 1000 || cap(*slice) < 32 { + return + } + + // Clear references to help GC + for i := range *slice { + (*slice)[i] = nil + } + + // Clear slice but keep capacity + *slice = (*slice)[:0] + TokenNodePool.Put(slice) +} \ No newline at end of file diff --git a/tokenizer.go b/tokenizer.go index 48b636c..18e8402 100644 --- a/tokenizer.go +++ b/tokenizer.go @@ -8,29 +8,29 @@ func processWhitespaceControl(tokens []Token) []Token { return tokens } - var result []Token = make([]Token, len(tokens)) - copy(result, tokens) - + // Modify tokens in-place to avoid allocation + // This works because we're only changing token values, not adding/removing tokens + // Process each token to apply whitespace trimming - for i := 0; i < len(result); i++ { - token := result[i] + for i := 0; i < len(tokens); i++ { + token := tokens[i] // Handle opening tags that trim whitespace before them if token.Type == TOKEN_VAR_START_TRIM || token.Type == TOKEN_BLOCK_START_TRIM { // If there's a text token before this, trim its trailing whitespace - if i > 0 && result[i-1].Type == TOKEN_TEXT { - result[i-1].Value = trimTrailingWhitespace(result[i-1].Value) + if i > 0 && tokens[i-1].Type == TOKEN_TEXT { + tokens[i-1].Value = trimTrailingWhitespace(tokens[i-1].Value) } } // Handle closing tags that trim whitespace after them if token.Type == TOKEN_VAR_END_TRIM || token.Type == TOKEN_BLOCK_END_TRIM { // If there's a text token after this, trim its leading whitespace - if i+1 < len(result) && result[i+1].Type == TOKEN_TEXT { - result[i+1].Value = trimLeadingWhitespace(result[i+1].Value) + if i+1 < len(tokens) && tokens[i+1].Type == TOKEN_TEXT { + tokens[i+1].Value = trimLeadingWhitespace(tokens[i+1].Value) } } } - return result + return tokens } diff --git a/tokenizer_benchmark_test.go b/tokenizer_benchmark_test.go new file mode 100644 index 0000000..f45a361 --- /dev/null +++ b/tokenizer_benchmark_test.go @@ -0,0 +1,333 @@ +package twig + +import ( + "testing" +) + +func BenchmarkHtmlPreservingTokenize(b *testing.B) { + // A sample template with HTML and Twig tags + source := ` + + +No content available.
+ {% endif %} + + {% block sidebar %} + + {% endblock %} +No content available.
+ {% endif %} + + {% block sidebar %} + + {% endblock %} +No content available.
+ {% endif %} + + {% block sidebar %} + + {% endblock %} +No content available.
+ {% endif %} + + {% block sidebar %} + + {% endblock %} +{{ user.bio|striptags|truncate(100) }}
+ + {% if user.permissions is defined and 'admin' in user.permissions %} + Admin + {% endif %} + +