diff --git a/buffer_pool.go b/buffer_pool.go index 1f4748b..90ecb1c 100644 --- a/buffer_pool.go +++ b/buffer_pool.go @@ -160,13 +160,13 @@ func (b *Buffer) WriteInt(i int) (n int, err error) { } else if i > -100 && i < 0 { return b.WriteString(smallNegIntStrings[-i]) } - + // Optimization: manual integer formatting for common sizes // Avoid the allocations in strconv.Itoa for numbers we can handle directly if i >= -999999 && i <= 999999 { return b.formatInt(int64(i)) } - + // For larger integers, fallback to standard formatting // This still allocates, but is rare enough to be acceptable s := strconv.FormatInt(int64(i), 10) @@ -181,7 +181,7 @@ func (b *Buffer) formatInt(i int64) (int, error) { b.WriteByte('-') i = -i } - + // Count digits to determine buffer size var digits int if i < 10 { @@ -197,7 +197,7 @@ func (b *Buffer) formatInt(i int64) (int, error) { } else { digits = 6 } - + // Reserve space for the digits // Compute in reverse order, then reverse the result start := len(b.buf) @@ -206,13 +206,13 @@ func (b *Buffer) formatInt(i int64) (int, error) { b.buf = append(b.buf, digit) i /= 10 } - + // Reverse the digits end := len(b.buf) - 1 for j := 0; j < digits/2; j++ { b.buf[start+j], b.buf[end-j] = b.buf[end-j], b.buf[start+j] } - + return digits, nil } @@ -226,12 +226,12 @@ func (b *Buffer) WriteFloat(f float64, fmt byte, prec int) (n int, err error) { return b.formatInt(int64(f)) } } - + // Special case for small, common floating-point values with 1-2 decimal places if fmt == 'f' && f >= 0 && f < 1000 && (prec == 1 || prec == 2 || prec == -1) { // Try to format common floats manually without allocation intPart := int64(f) - + // Get the fractional part based on precision var fracFactor int64 var fracPrec int @@ -243,7 +243,7 @@ func (b *Buffer) WriteFloat(f float64, fmt byte, prec int) (n int, err error) { // It's a whole number return b.formatInt(intPart) } - + // Test if 1-2 decimal places is enough if fracPart*100 == float64(int64(fracPart*100)) { // Two decimal places is sufficient @@ -264,20 +264,20 @@ func (b *Buffer) WriteFloat(f float64, fmt byte, prec int) (n int, err error) { fracFactor = 100 fracPrec = 2 } - + // Format integer part first intLen, err := b.formatInt(intPart) if err != nil { return intLen, err } - + // Add decimal point if err := b.WriteByte('.'); err != nil { return intLen, err } - + // Format fractional part, ensuring proper padding with zeros - fracPart := int64((f - float64(intPart)) * float64(fracFactor) + 0.5) // Round + fracPart := int64((f-float64(intPart))*float64(fracFactor) + 0.5) // Round if fracPart >= fracFactor { // Rounding caused carry fracPart = 0 @@ -291,22 +291,22 @@ func (b *Buffer) WriteFloat(f float64, fmt byte, prec int) (n int, err error) { return intLen, err } } - + // Write fractional part with leading zeros if needed if fracPrec == 2 && fracPart < 10 { if err := b.WriteByte('0'); err != nil { return intLen + 1, err } } - + fracLen, err := b.formatInt(fracPart) if err != nil { return intLen + 1, err } - + return intLen + 1 + fracLen, nil } - + useStrconv: // Fallback to standard formatting for complex or unusual cases s := strconv.FormatFloat(f, fmt, prec, 64) @@ -321,30 +321,30 @@ func (b *Buffer) WriteFormat(format string, args ...interface{}) (n int, err err if len(args) == 0 { return b.WriteString(format) } - + startIdx := 0 argIdx := 0 totalWritten := 0 - + // Scan the format string for format specifiers for i := 0; i < len(format); i++ { if format[i] != '%' { continue } - + // Found a potential format specifier if i+1 >= len(format) { // % at the end of the string is invalid break } - + // Check next character next := format[i+1] if next == '%' { // It's an escaped % // Write everything up to and including the first % if i > startIdx { - written, err := b.WriteString(format[startIdx:i+1]) + written, err := b.WriteString(format[startIdx : i+1]) totalWritten += written if err != nil { return totalWritten, err @@ -352,10 +352,10 @@ func (b *Buffer) WriteFormat(format string, args ...interface{}) (n int, err err } // Skip the second % i++ - startIdx = i+1 + startIdx = i + 1 continue } - + // Write the part before the format specifier if i > startIdx { written, err := b.WriteString(format[startIdx:i]) @@ -364,17 +364,17 @@ func (b *Buffer) WriteFormat(format string, args ...interface{}) (n int, err err return totalWritten, err } } - + // Make sure we have an argument for this specifier if argIdx >= len(args) { // More specifiers than arguments, skip startIdx = i continue } - + arg := args[argIdx] argIdx++ - + // Process the format specifier switch next { case 's': @@ -420,12 +420,12 @@ func (b *Buffer) WriteFormat(format string, args ...interface{}) (n int, err err } totalWritten++ } - + // Move past the format specifier i++ - startIdx = i+1 + startIdx = i + 1 } - + // Write any remaining part of the format string if startIdx < len(format) { written, err := b.WriteString(format[startIdx:]) @@ -434,7 +434,7 @@ func (b *Buffer) WriteFormat(format string, args ...interface{}) (n int, err err return totalWritten, err } } - + return totalWritten, nil } @@ -446,19 +446,19 @@ func (b *Buffer) Grow(n int) { if cap(b.buf) >= needed { return // Already have enough capacity } - + // Grow capacity with a smart algorithm that avoids frequent resizing // Double the capacity until we have enough, but with some optimizations: // - For small buffers (<1KB), grow more aggressively (2x) // - For medium buffers (1KB-64KB), grow at 1.5x // - For large buffers (>64KB), grow at 1.25x to avoid excessive memory usage - + newCap := cap(b.buf) const ( smallBuffer = 1024 // 1KB mediumBuffer = 64 * 1024 // 64KB ) - + for newCap < needed { if newCap < smallBuffer { newCap *= 2 // Double small buffers @@ -468,7 +468,7 @@ func (b *Buffer) Grow(n int) { newCap = newCap + newCap/4 // Grow large buffers by 1.25x } } - + // Create new buffer with the calculated capacity newBuf := make([]byte, len(b.buf), newCap) copy(newBuf, b.buf) @@ -517,16 +517,16 @@ func WriteValue(w io.Writer, val interface{}) (n int, err error) { if bw, ok := w.(*Buffer); ok { return writeValueToBuffer(bw, val) } - + // If writer is a StringWriter, we can optimize some cases if sw, ok := w.(io.StringWriter); ok { return writeValueToStringWriter(sw, val) } - + // Fallback path - use temp buffer for conversion to avoid allocating strings buf := GetBuffer() defer buf.Release() - + _, _ = writeValueToBuffer(buf, val) return w.Write(buf.Bytes()) } @@ -536,7 +536,7 @@ func writeValueToBuffer(b *Buffer, val interface{}) (n int, err error) { if val == nil { return 0, nil } - + switch v := val.(type) { case string: return b.WriteString(v) @@ -561,7 +561,7 @@ func writeValueToStringWriter(w io.StringWriter, val interface{}) (n int, err er if val == nil { return 0, nil } - + switch v := val.(type) { case string: return w.WriteString(v) @@ -600,7 +600,7 @@ func stringify(val interface{}) string { if val == nil { return "" } - + // Use type switch for efficient handling of common types switch v := val.(type) { case string: @@ -622,7 +622,7 @@ func stringify(val interface{}) string { case []byte: return string(v) } - + // Fall back to fmt.Sprintf for complex types return fmt.Sprintf("%v", val) } @@ -634,9 +634,9 @@ func GetTokenBuffer(templateSize int) *[]Token { // For medium templates, use the medium pool // For large templates, use the large pool // For extremely large templates, allocate directly - + var buffer *[]Token - + if templateSize < 4*1024 { // Small template buffer = SmallTokenBufferPool.Get().(*[]Token) @@ -664,10 +664,10 @@ func GetTokenBuffer(templateSize int) *[]Token { newBuffer := make([]Token, 0, estimateTokenCount(templateSize)) buffer = &newBuffer } - + // Clear the buffer in case it contains old tokens *buffer = (*buffer)[:0] - + return buffer } @@ -676,10 +676,10 @@ func ReleaseTokenBuffer(buffer *[]Token) { if buffer == nil { return } - + // Clear the buffer to prevent memory leaks *buffer = (*buffer)[:0] - + // Put back in the appropriate pool based on capacity cap := cap(*buffer) if cap <= 64 { @@ -699,7 +699,7 @@ func GetTokenBufferWithCapacity(capacity int) *[]Token { // For large capacity requests, use the large pool // For very large capacity requests, allocate directly var buffer *[]Token - + if capacity <= 64 { buffer = SmallTokenBufferPool.Get().(*[]Token) if cap(*buffer) < capacity { @@ -720,10 +720,10 @@ func GetTokenBufferWithCapacity(capacity int) *[]Token { newBuffer := make([]Token, 0, capacity) buffer = &newBuffer } - + // Clear the buffer *buffer = (*buffer)[:0] - + return buffer } @@ -734,7 +734,7 @@ func RecycleTokens(tokens []Token) []Token { if len(tokens) == 0 { return []Token{} } - + // Create a new slice with the same backing array recycled := tokens[:0] return recycled @@ -747,7 +747,7 @@ func estimateTokenCount(templateSize int) int { // - Small templates: ~1 token per 12 bytes // - Medium templates: ~1 token per 15 bytes // - Large templates: ~1 token per 20 bytes - + if templateSize < 4*1024 { // Small template return max(64, templateSize/12+16) @@ -758,4 +758,4 @@ func estimateTokenCount(templateSize int) int { // Large template return max(1024, templateSize/20+64) } -} \ No newline at end of file +} diff --git a/buffer_pool_benchmark_test.go b/buffer_pool_benchmark_test.go index fdbd032..a65c98b 100644 --- a/buffer_pool_benchmark_test.go +++ b/buffer_pool_benchmark_test.go @@ -34,9 +34,9 @@ func BenchmarkStandardBufferWrite(b *testing.B) { func BenchmarkBufferIntegerFormatting(b *testing.B) { buf := GetBuffer() defer buf.Release() - + vals := []int{0, 5, -5, 123, -123, 9999, -9999, 123456789, -123456789} - + b.ResetTimer() for i := 0; i < b.N; i++ { buf.Reset() @@ -48,9 +48,9 @@ func BenchmarkBufferIntegerFormatting(b *testing.B) { func BenchmarkStandardIntegerFormatting(b *testing.B) { buf := &bytes.Buffer{} - + vals := []int{0, 5, -5, 123, -123, 9999, -9999, 123456789, -123456789} - + b.ResetTimer() for i := 0; i < b.N; i++ { buf.Reset() @@ -65,7 +65,7 @@ func BenchmarkSmallIntegerFormatting(b *testing.B) { // pre-computed string table buf := GetBuffer() defer buf.Release() - + b.Run("Optimized_Small_Ints", func(b *testing.B) { for i := 0; i < b.N; i++ { buf.Reset() @@ -74,7 +74,7 @@ func BenchmarkSmallIntegerFormatting(b *testing.B) { } } }) - + b.Run("Standard_Small_Ints", func(b *testing.B) { sbuf := &bytes.Buffer{} for i := 0; i < b.N; i++ { @@ -89,15 +89,15 @@ func BenchmarkSmallIntegerFormatting(b *testing.B) { func BenchmarkFloatFormatting(b *testing.B) { buf := GetBuffer() defer buf.Release() - + vals := []float64{ - 0.0, 1.0, -1.0, // Whole numbers - 3.14, -2.718, // Common constants - 123.456, -789.012, // Medium floats - 0.123, 0.001, 9.999, // Small decimals - 1234567.89, -9876543.21, // Large numbers + 0.0, 1.0, -1.0, // Whole numbers + 3.14, -2.718, // Common constants + 123.456, -789.012, // Medium floats + 0.123, 0.001, 9.999, // Small decimals + 1234567.89, -9876543.21, // Large numbers } - + b.Run("OptimizedFloat", func(b *testing.B) { for i := 0; i < b.N; i++ { buf.Reset() @@ -106,7 +106,7 @@ func BenchmarkFloatFormatting(b *testing.B) { } } }) - + b.Run("StandardFloat", func(b *testing.B) { sbuf := &bytes.Buffer{} for i := 0; i < b.N; i++ { @@ -121,19 +121,19 @@ func BenchmarkFloatFormatting(b *testing.B) { func BenchmarkFormatString(b *testing.B) { buf := GetBuffer() defer buf.Release() - + format := "Hello, %s! Count: %d, Value: %v" name := "World" count := 42 value := true - + b.Run("BufferFormat", func(b *testing.B) { for i := 0; i < b.N; i++ { buf.Reset() buf.WriteFormat(format, name, count, value) } }) - + b.Run("FmtSprintf", func(b *testing.B) { for i := 0; i < b.N; i++ { // Each fmt.Sprintf creates a new string @@ -148,19 +148,19 @@ func BenchmarkFormatInt(b *testing.B) { _ = FormatInt(42) } }) - + b.Run("SmallInt_Standard", func(b *testing.B) { for i := 0; i < b.N; i++ { _ = strconv.Itoa(42) } }) - + b.Run("LargeInt_Optimized", func(b *testing.B) { for i := 0; i < b.N; i++ { _ = FormatInt(12345678) } }) - + b.Run("LargeInt_Standard", func(b *testing.B) { for i := 0; i < b.N; i++ { _ = strconv.Itoa(12345678) @@ -171,7 +171,7 @@ func BenchmarkFormatInt(b *testing.B) { func BenchmarkWriteValue(b *testing.B) { buf := GetBuffer() defer buf.Release() - + values := []interface{}{ "string value", 123, @@ -180,7 +180,7 @@ func BenchmarkWriteValue(b *testing.B) { true, []byte("byte slice"), } - + b.ResetTimer() for i := 0; i < b.N; i++ { buf.Reset() @@ -199,7 +199,7 @@ func BenchmarkStringifyValues(b *testing.B) { true, []byte("byte slice"), } - + b.ResetTimer() for i := 0; i < b.N; i++ { for _, v := range values { @@ -217,7 +217,7 @@ func BenchmarkBufferGrowth(b *testing.B) { buf.Release() } }) - + b.Run("Medium", func(b *testing.B) { mediumStr := "medium string that is longer than the small one but still reasonable" for i := 0; i < b.N; i++ { @@ -226,7 +226,7 @@ func BenchmarkBufferGrowth(b *testing.B) { buf.Release() } }) - + b.Run("Large", func(b *testing.B) { largeStr := string(make([]byte, 2048)) // 2KB string for i := 0; i < b.N; i++ { @@ -240,13 +240,13 @@ func BenchmarkBufferGrowth(b *testing.B) { func BenchmarkBufferToWriter(b *testing.B) { buf := GetBuffer() defer buf.Release() - + str := "This is a test string that will be written to a discard writer" - + b.ResetTimer() for i := 0; i < b.N; i++ { buf.Reset() buf.WriteString(str) buf.WriteTo(io.Discard) } -} \ No newline at end of file +} diff --git a/buffer_pool_test.go b/buffer_pool_test.go index 2eb81b3..1aaee07 100644 --- a/buffer_pool_test.go +++ b/buffer_pool_test.go @@ -11,7 +11,7 @@ func TestBufferPool(t *testing.T) { if buf == nil { t.Fatal("GetBuffer() returned nil") } - + // Test writing to the buffer str := "Hello, world!" n, err := buf.WriteString(str) @@ -21,28 +21,28 @@ func TestBufferPool(t *testing.T) { if n != len(str) { t.Fatalf("WriteString returned wrong length: got %d, want %d", n, len(str)) } - + // Test getting the string back if buf.String() != str { t.Fatalf("String() returned wrong value: got %q, want %q", buf.String(), str) } - + // Test resetting the buffer buf.Reset() if buf.Len() != 0 { t.Fatalf("Reset() didn't clear the buffer: length = %d", buf.Len()) } - + // Test writing a different string after reset str2 := "Another string" buf.WriteString(str2) if buf.String() != str2 { t.Fatalf("String() after reset returned wrong value: got %q, want %q", buf.String(), str2) } - + // Test releasing the buffer buf.Release() - + // Getting a new buffer should not have any content buf2 := GetBuffer() if buf2.Len() != 0 { @@ -54,7 +54,7 @@ func TestBufferPool(t *testing.T) { func TestWriteValue(t *testing.T) { buf := GetBuffer() defer buf.Release() - + tests := []struct { value interface{} expected string @@ -68,7 +68,7 @@ func TestWriteValue(t *testing.T) { {false, "false"}, {[]byte("bytes"), "bytes"}, } - + for _, test := range tests { buf.Reset() _, err := WriteValue(buf, test.value) @@ -76,7 +76,7 @@ func TestWriteValue(t *testing.T) { t.Errorf("WriteValue(%v) error: %v", test.value, err) continue } - + if buf.String() != test.expected { t.Errorf("WriteValue(%v) = %q, want %q", test.value, buf.String(), test.expected) } @@ -86,7 +86,7 @@ func TestWriteValue(t *testing.T) { func TestWriteInt(t *testing.T) { buf := GetBuffer() defer buf.Release() - + tests := []struct { value int expected string @@ -101,7 +101,7 @@ func TestWriteInt(t *testing.T) { {123456789, "123456789"}, {-123456789, "-123456789"}, } - + for _, test := range tests { buf.Reset() _, err := buf.WriteInt(test.value) @@ -109,7 +109,7 @@ func TestWriteInt(t *testing.T) { t.Errorf("WriteInt(%d) error: %v", test.value, err) continue } - + if buf.String() != test.expected { t.Errorf("WriteInt(%d) = %q, want %q", test.value, buf.String(), test.expected) } @@ -119,13 +119,13 @@ func TestWriteInt(t *testing.T) { func TestBufferWriteTo(t *testing.T) { buf := GetBuffer() defer buf.Release() - + testStr := "This is a test string" buf.WriteString(testStr) - + // Create a destination buffer to write to var dest strings.Builder - + n, err := buf.WriteTo(&dest) if err != nil { t.Fatalf("WriteTo failed: %v", err) @@ -133,7 +133,7 @@ func TestBufferWriteTo(t *testing.T) { if n != int64(len(testStr)) { t.Fatalf("WriteTo returned wrong length: got %d, want %d", n, len(testStr)) } - + if dest.String() != testStr { t.Fatalf("WriteTo output mismatch: got %q, want %q", dest.String(), testStr) } @@ -142,25 +142,25 @@ func TestBufferWriteTo(t *testing.T) { func TestBufferGrowCapacity(t *testing.T) { buf := GetBuffer() defer buf.Release() - + // Start with small string initialStr := "small" buf.WriteString(initialStr) initialCapacity := cap(buf.buf) - + // Write a larger string that should cause a grow largeStr := strings.Repeat("abcdefghijklmnopqrstuvwxyz", 100) // 2600 bytes buf.WriteString(largeStr) - + // Verify capacity increased if cap(buf.buf) <= initialCapacity { - t.Fatalf("Buffer didn't grow capacity: initial=%d, after=%d", + t.Fatalf("Buffer didn't grow capacity: initial=%d, after=%d", initialCapacity, cap(buf.buf)) } - + // Verify content is correct expected := initialStr + largeStr if buf.String() != expected { t.Fatalf("Buffer content incorrect after growth") } -} \ No newline at end of file +} diff --git a/expr.go b/expr.go index 9ed1317..653ade5 100644 --- a/expr.go +++ b/expr.go @@ -466,22 +466,22 @@ func NewFunctionNode(name string, args []Node, line int) *FunctionNode { func ParseExpressionOptimized(expr string) (Node, bool) { // Trim whitespace expr = strings.TrimSpace(expr) - + // Quick empty check if len(expr) == 0 { return nil, false } - + // Try parsing as literal if val, ok := ParseLiteralOptimized(expr); ok { return NewLiteralNode(val, 0), true } - + // Check for simple variable references if IsValidVariableName(expr) { return NewVariableNode(expr, 0), true } - + // More complex expression - will need the full parser return nil, false } @@ -492,27 +492,27 @@ func ParseLiteralOptimized(expr string) (interface{}, bool) { if len(expr) == 0 { return nil, false } - + // Check for string literals - if len(expr) >= 2 && ((expr[0] == '"' && expr[len(expr)-1] == '"') || - (expr[0] == '\'' && expr[len(expr)-1] == '\'')) { + if len(expr) >= 2 && ((expr[0] == '"' && expr[len(expr)-1] == '"') || + (expr[0] == '\'' && expr[len(expr)-1] == '\'')) { // String literal return expr[1 : len(expr)-1], true } - + // Check for number literals if isDigit(expr[0]) || (expr[0] == '-' && len(expr) > 1 && isDigit(expr[1])) { // Try parsing as integer first (most common case) if i, err := strconv.Atoi(expr); err == nil { return i, true } - + // Try parsing as float if f, err := strconv.ParseFloat(expr, 64); err == nil { return f, true } } - + // Check for boolean literals if expr == "true" { return true, true @@ -523,7 +523,7 @@ func ParseLiteralOptimized(expr string) (interface{}, bool) { if expr == "null" || expr == "nil" { return nil, true } - + // Not a simple literal return nil, false } @@ -533,25 +533,25 @@ func IsValidVariableName(name string) bool { if len(name) == 0 { return false } - + // First character must be a letter or underscore if !isAlpha(name[0]) { return false } - + // Rest can be letters, digits, or underscores for i := 1; i < len(name); i++ { if !isNameChar(name[i]) { return false } } - + // Check for reserved keywords switch name { case "true", "false", "null", "nil", "not", "and", "or", "in", "is": return false } - + return true } @@ -566,22 +566,22 @@ func ProcessStringEscapes(text string) (string, bool) { break } } - + // If no escapes, return the original string if !hasEscape { return text, false } - + // Process escapes var sb strings.Builder sb.Grow(len(text)) - + i := 0 for i < len(text) { if text[i] == '\\' && i+1 < len(text) { // Escape sequence found i++ - + // Process the escape sequence switch text[i] { case 'n': @@ -611,7 +611,7 @@ func ProcessStringEscapes(text string) (string, bool) { } i++ } - + // Return the processed string return sb.String(), true } @@ -623,81 +623,81 @@ func ParseNumberOptimized(text string) (interface{}, bool) { if len(text) == 0 { return nil, false } - + // Check for negative sign isNegative := false pos := 0 - + if text[pos] == '-' { isNegative = true pos++ - + // Just a minus sign is not a number if pos >= len(text) { return nil, false } } - + // Parse the integer part var intValue int64 hasDigits := false - + for pos < len(text) && isDigit(text[pos]) { digit := int64(text[pos] - '0') intValue = intValue*10 + digit hasDigits = true pos++ } - + if !hasDigits { return nil, false } - + // Check for decimal point hasDecimal := false var floatValue float64 - + if pos < len(text) && text[pos] == '.' { hasDecimal = true pos++ - + // Need at least one digit after decimal point if pos >= len(text) || !isDigit(text[pos]) { return nil, false } - + // Parse the fractional part floatValue = float64(intValue) fraction := 0.0 multiplier := 0.1 - + for pos < len(text) && isDigit(text[pos]) { digit := float64(text[pos] - '0') fraction += digit * multiplier multiplier *= 0.1 pos++ } - + floatValue += fraction } - + // Check for exponent if pos < len(text) && (text[pos] == 'e' || text[pos] == 'E') { hasDecimal = true pos++ - + // Parse exponent sign expNegative := false if pos < len(text) && (text[pos] == '+' || text[pos] == '-') { expNegative = text[pos] == '-' pos++ } - + // Need at least one digit in exponent if pos >= len(text) || !isDigit(text[pos]) { return nil, false } - + // Parse exponent value exp := 0 for pos < len(text) && isDigit(text[pos]) { @@ -705,14 +705,14 @@ func ParseNumberOptimized(text string) (interface{}, bool) { exp = exp*10 + digit pos++ } - + // Apply exponent // Convert to float regardless of hasDecimal hasDecimal = true if floatValue == 0 { // Not set yet floatValue = float64(intValue) } - + // Apply exponent using a more efficient approach if expNegative { // For negative exponents, divide by 10^exp @@ -730,12 +730,12 @@ func ParseNumberOptimized(text string) (interface{}, bool) { floatValue *= multiplier } } - + // Check that we consumed the whole string if pos < len(text) { return nil, false } - + // Return the appropriate value if hasDecimal { if isNegative { @@ -768,10 +768,10 @@ func isNameChar(c byte) bool { // This is useful for consistent hashing of variable names or strings func calcStringHash(s string) uint32 { var h uint32 - + for i := 0; i < len(s); i++ { h = 31*h + uint32(s[i]) } - + return h } diff --git a/expr_benchmark_test.go b/expr_benchmark_test.go index 20d552f..7301258 100644 --- a/expr_benchmark_test.go +++ b/expr_benchmark_test.go @@ -139,7 +139,7 @@ func BenchmarkExpressionRender(b *testing.B) { // Create a buffer for testing buf := bytes.NewBuffer(nil) - + // Run each benchmark for _, tc := range tests { b.Run(tc.name, func(b *testing.B) { @@ -157,7 +157,7 @@ func BenchmarkExpressionRender(b *testing.B) { func BenchmarkFilterChain(b *testing.B) { engine := New() ctx := NewRenderContext(engine.environment, map[string]interface{}{ - "a": 10, + "a": 10, "text": "Hello, World!", "html": "

This is a paragraph

", }, engine) @@ -177,12 +177,12 @@ func BenchmarkFilterChain(b *testing.B) { node: NewFilterNode( NewFilterNode( NewVariableNode("text", 1), - "upper", - nil, + "upper", + nil, 1, ), - "trim", - nil, + "trim", + nil, 1, ), }, @@ -190,11 +190,11 @@ func BenchmarkFilterChain(b *testing.B) { name: "FilterWithArgs", node: NewFilterNode( NewVariableNode("text", 1), - "replace", + "replace", []Node{ NewLiteralNode("World", 1), NewLiteralNode("Universe", 1), - }, + }, 1, ), }, @@ -217,8 +217,8 @@ func BenchmarkFunctionCall(b *testing.B) { engine := New() ctx := NewRenderContext(engine.environment, map[string]interface{}{ "numbers": []interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - "start": 1, - "end": 10, + "start": 1, + "end": 10, }, engine) defer ctx.Release() @@ -265,12 +265,12 @@ func BenchmarkArgSlicePooling(b *testing.B) { engine := New() ctx := NewRenderContext(engine.environment, nil, engine) defer ctx.Release() - + smallArgs := []Node{ NewLiteralNode(1, 1), NewLiteralNode(2, 1), } - + mediumArgs := []Node{ NewLiteralNode(1, 1), NewLiteralNode(2, 1), @@ -278,12 +278,12 @@ func BenchmarkArgSlicePooling(b *testing.B) { NewLiteralNode(4, 1), NewLiteralNode(5, 1), } - + largeArgs := make([]Node, 10) for i := 0; i < 10; i++ { largeArgs[i] = NewLiteralNode(i, 1) } - + tests := []struct { name string node Node @@ -305,7 +305,7 @@ func BenchmarkArgSlicePooling(b *testing.B) { node: NewFunctionNode("range", largeArgs, 1), }, } - + for _, tc := range tests { b.Run(tc.name, func(b *testing.B) { b.ReportAllocs() @@ -314,4 +314,4 @@ func BenchmarkArgSlicePooling(b *testing.B) { } }) } -} \ No newline at end of file +} diff --git a/expr_pool.go b/expr_pool.go index cca3ba6..96924dc 100644 --- a/expr_pool.go +++ b/expr_pool.go @@ -363,9 +363,9 @@ func GetArgSlice(size int) []interface{} { if size <= 0 { return nil } - + var slice []interface{} - + switch { case size <= 2: slice = smallArgSlicePool.Get().([]interface{}) @@ -377,7 +377,7 @@ func GetArgSlice(size int) []interface{} { // For very large slices, just allocate directly return make([]interface{}, 0, size) } - + // Clear the slice but maintain capacity return slice[:0] } @@ -387,15 +387,15 @@ func ReleaseArgSlice(slice []interface{}) { if slice == nil { return } - + // Clear all references for i := range slice { slice[i] = nil } - + // Reset length to 0 slice = slice[:0] - + // Return to appropriate pool based on capacity switch cap(slice) { case 2: @@ -440,7 +440,7 @@ func GetHashMap(size int) map[string]interface{} { } return hashMap } - + // For larger maps, just allocate directly return make(map[string]interface{}, size) } @@ -450,26 +450,26 @@ func ReleaseHashMap(hashMap map[string]interface{}) { if hashMap == nil { return } - + // Clear all entries (not used directly in our defer block) // for k := range hashMap { // delete(hashMap, k) // } - + // Return to appropriate pool based on capacity - // We don't actually clear the map when releasing through the defer, + // We don't actually clear the map when releasing through the defer, // because we return the map as the result and deleting entries would // clear the returned result - + // Map doesn't have a built-in cap function // Not using pool return for maps directly returned as results - - /* - switch { - case len(hashMap) <= 5: - smallHashMapPool.Put(hashMap) - case len(hashMap) <= 15: - mediumHashMapPool.Put(hashMap) - } + + /* + switch { + case len(hashMap) <= 5: + smallHashMapPool.Put(hashMap) + case len(hashMap) <= 15: + mediumHashMapPool.Put(hashMap) + } */ -} \ No newline at end of file +} diff --git a/node_pool_extensions.go b/node_pool_extensions.go index 100187f..e6e4cad 100644 --- a/node_pool_extensions.go +++ b/node_pool_extensions.go @@ -297,4 +297,4 @@ func ReleaseApplyNode(node *ApplyNode) { node.filter = "" node.args = nil ApplyNodePool.Put(node) -} \ No newline at end of file +} diff --git a/parse_do.go b/parse_do.go index b2b975a..41b1988 100644 --- a/parse_do.go +++ b/parse_do.go @@ -21,11 +21,11 @@ func (p *Parser) parseDo(parser *Parser) (Node, error) { // Look ahead to find possible assignment patterns // We need to check for NUMBER = EXPR which is invalid // as well as NAME = EXPR which is valid - + // Check if we have an equals sign in the next few tokens hasAssignment := false equalsPosition := -1 - + // Scan ahead a bit to find possible equals sign for i := 0; i < 3 && parser.tokenIndex+i < len(parser.tokens); i++ { token := parser.tokens[parser.tokenIndex+i] @@ -34,44 +34,44 @@ func (p *Parser) parseDo(parser *Parser) (Node, error) { equalsPosition = i break } - + // Stop scanning if we hit the end of the block if token.Type == TOKEN_BLOCK_END { break } } - + // If we found an equals sign, analyze the left-hand side if hasAssignment && equalsPosition > 0 { firstToken := parser.tokens[parser.tokenIndex] - + // Check if the left-hand side is a valid variable name isValidVariableName := firstToken.Type == TOKEN_NAME - + // If the left-hand side is a number or literal, that's an error if !isValidVariableName { return nil, fmt.Errorf("invalid variable name %q in do tag assignment at line %d", firstToken.Value, doLine) } - + // Handle assignment case if isValidVariableName && hasAssignment { varName := parser.tokens[parser.tokenIndex].Value - + // Skip tokens up to and including the equals sign parser.tokenIndex += equalsPosition + 1 - + // Parse the right side expression expr, err := parser.parseExpression() if err != nil { return nil, fmt.Errorf("error parsing expression in do assignment at line %d: %w", doLine, err) } - + // Make sure we have the closing tag if parser.tokenIndex >= len(parser.tokens) || parser.tokens[parser.tokenIndex].Type != TOKEN_BLOCK_END { return nil, fmt.Errorf("expecting end of do tag at line %d", doLine) } parser.tokenIndex++ - + // Additional validation for variable name if _, err := strconv.Atoi(varName); err == nil { return nil, fmt.Errorf("invalid variable name %q in do tag assignment at line %d", varName, doLine) diff --git a/parse_from.go b/parse_from.go index ac92efa..d8479df 100644 --- a/parse_from.go +++ b/parse_from.go @@ -11,7 +11,7 @@ import ( func (p *Parser) parseFrom(parser *Parser) (Node, error) { // Get the line number of the from token fromLine := parser.tokens[parser.tokenIndex-1].Line - + // Debugging: Print out tokens for debugging purposes if IsDebugEnabled() { LogDebug("Parsing from tag. Next tokens (up to 10):") @@ -22,27 +22,27 @@ func (p *Parser) parseFrom(parser *Parser) (Node, error) { } } } - + // First try to parse in token-by-token format (from zero-allocation tokenizer) // Check if we have a proper sequence: [STRING/NAME, NAME="import", NAME="macroname", ...] if parser.tokenIndex+1 < len(parser.tokens) { // Check for (template path) followed by "import" keyword firstToken := parser.tokens[parser.tokenIndex] secondToken := parser.tokens[parser.tokenIndex+1] - + isTemplatePath := firstToken.Type == TOKEN_STRING || firstToken.Type == TOKEN_NAME isImportKeyword := secondToken.Type == TOKEN_NAME && secondToken.Value == "import" - + if isTemplatePath && isImportKeyword { LogDebug("Found tokenized from...import pattern") - + // Get template path templatePath := firstToken.Value if firstToken.Type == TOKEN_NAME { // For paths like ./file.twig, just use the value templatePath = strings.Trim(templatePath, "\"'") } - + // Create template expression templateExpr := &LiteralNode{ ExpressionNode: ExpressionNode{ @@ -51,46 +51,46 @@ func (p *Parser) parseFrom(parser *Parser) (Node, error) { }, value: templatePath, } - + // Skip past the template path and import keyword parser.tokenIndex += 2 - + // Parse macros and aliases macros := []string{} aliases := map[string]string{} - + // Process tokens until end of block for parser.tokenIndex < len(parser.tokens) { token := parser.tokens[parser.tokenIndex] - + // Stop at block end if token.Type == TOKEN_BLOCK_END || token.Type == TOKEN_BLOCK_END_TRIM { parser.tokenIndex++ break } - + // Skip punctuation (commas) if token.Type == TOKEN_PUNCTUATION { parser.tokenIndex++ continue } - + // Handle macro name if token.Type == TOKEN_NAME { macroName := token.Value - + // Add to macros list macros = append(macros, macroName) - + // Check for alias parser.tokenIndex++ - if parser.tokenIndex < len(parser.tokens) && - parser.tokens[parser.tokenIndex].Type == TOKEN_NAME && - parser.tokens[parser.tokenIndex].Value == "as" { - + if parser.tokenIndex < len(parser.tokens) && + parser.tokens[parser.tokenIndex].Type == TOKEN_NAME && + parser.tokens[parser.tokenIndex].Value == "as" { + // Skip 'as' keyword parser.tokenIndex++ - + // Get alias if parser.tokenIndex < len(parser.tokens) && parser.tokens[parser.tokenIndex].Type == TOKEN_NAME { aliases[macroName] = parser.tokens[parser.tokenIndex].Value @@ -102,14 +102,14 @@ func (p *Parser) parseFrom(parser *Parser) (Node, error) { parser.tokenIndex++ } } - + // If we found macros, return a FromImportNode if len(macros) > 0 { return NewFromImportNode(templateExpr, macros, aliases, fromLine), nil } } } - + // Fall back to the original approach (for backward compatibility) // We need to extract the template path, import keyword, and macro(s) from // the current token. The tokenizer may be combining them. diff --git a/parser.go b/parser.go index d066e73..21b9903 100644 --- a/parser.go +++ b/parser.go @@ -62,10 +62,10 @@ func (p *Parser) Parse(source string) (Node, error) { // Use the optimized tokenizer for maximum performance and minimal allocations // This will treat everything outside twig tags as TEXT tokens var err error - + // Use zero allocation tokenizer for optimal performance tokenizer := GetTokenizer(p.source, 0) - + // Use optimized version for larger templates if len(p.source) > 4096 { // Use the optimized tag detection for large templates @@ -74,15 +74,15 @@ func (p *Parser) Parse(source string) (Node, error) { // Use regular tokenization for smaller templates p.tokens, err = tokenizer.TokenizeHtmlPreserving() } - + // Apply whitespace control to handle whitespace trimming directives if err == nil { tokenizer.ApplyWhitespaceControl() } - + // Return the tokenizer to the pool ReleaseTokenizer(tokenizer) - + if err != nil { return nil, fmt.Errorf("tokenization error: %w", err) } diff --git a/render.go b/render.go index 0cdd37b..e770d7d 100644 --- a/render.go +++ b/render.go @@ -17,17 +17,17 @@ import ( // RenderContext holds the state during template rendering type RenderContext struct { - env *Environment - context map[string]interface{} - blocks map[string][]Node - parentBlocks map[string][]Node // Original block content from parent templates - macros map[string]Node - parent *RenderContext - engine *Engine // Reference to engine for loading templates - extending bool // Whether this template extends another - currentBlock *BlockNode // Current block being rendered (for parent() function) - inParentCall bool // Flag to indicate if we're currently rendering a parent() call - sandboxed bool // Flag indicating if this context is sandboxed + env *Environment + context map[string]interface{} + blocks map[string][]Node + parentBlocks map[string][]Node // Original block content from parent templates + macros map[string]Node + parent *RenderContext + engine *Engine // Reference to engine for loading templates + extending bool // Whether this template extends another + currentBlock *BlockNode // Current block being rendered (for parent() function) + inParentCall bool // Flag to indicate if we're currently rendering a parent() call + sandboxed bool // Flag indicating if this context is sandboxed lastLoadedTemplate *Template // The template that created this context (for resolving relative paths) } @@ -331,7 +331,7 @@ func (ctx *RenderContext) Clone() *RenderContext { // Inherit sandbox state newCtx.sandboxed = ctx.sandboxed - + // Copy the lastLoadedTemplate reference (crucial for relative path resolution) newCtx.lastLoadedTemplate = ctx.lastLoadedTemplate @@ -782,7 +782,7 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { // We need to avoid pooling for arrays that might be directly used by filters like merge // as those filters return the slice directly to the user items := make([]interface{}, 0, len(n.items)) - + for i := 0; i < len(n.items); i++ { val, err := ctx.EvaluateExpression(n.items[i]) if err != nil { @@ -802,7 +802,7 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { // Evaluate each key-value pair in the hash using a new map // We can't use pooling with defer here because the map is returned directly result := make(map[string]interface{}, len(n.items)) - + for k, v := range n.items { // Evaluate the key keyVal, err := ctx.EvaluateExpression(k) @@ -839,7 +839,7 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { // Evaluate all arguments - need direct allocation args := make([]interface{}, len(n.args)) - + for i := 0; i < len(n.args); i++ { val, err := ctx.EvaluateExpression(n.args[i]) if err != nil { @@ -881,7 +881,7 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { if macro, ok := ctx.GetMacro(n.name); ok { // Evaluate arguments - need direct allocation for macro calls args := make([]interface{}, len(n.args)) - + // Evaluate arguments for i := 0; i < len(n.args); i++ { val, err := ctx.EvaluateExpression(n.args[i]) @@ -904,7 +904,7 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { // Otherwise, it's a regular function call // Evaluate arguments - need direct allocation for function calls args := make([]interface{}, len(n.args)) - + // Evaluate arguments for i := 0; i < len(n.args); i++ { val, err := ctx.EvaluateExpression(n.args[i]) @@ -1026,7 +1026,7 @@ func (ctx *RenderContext) EvaluateExpression(node Node) (interface{}, error) { // Evaluate test arguments - need direct allocation args := make([]interface{}, len(n.args)) - + // Evaluate arguments for i := 0; i < len(n.args); i++ { val, err := ctx.EvaluateExpression(n.args[i]) diff --git a/render_context_benchmark_test.go b/render_context_benchmark_test.go index 9fdd3a3..bc63f97 100644 --- a/render_context_benchmark_test.go +++ b/render_context_benchmark_test.go @@ -57,10 +57,10 @@ func BenchmarkRenderContextCloning(b *testing.B) { for i := 0; i < b.N; i++ { // Clone the context - this should reuse memory from pools child := parent.Clone() - + // Do some operations on the child context child.SetVariable("newVar", "test value") - + // Release the child context child.Release() } @@ -115,10 +115,10 @@ func BenchmarkContextVariableLookup(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { // Test variable lookup at different levels - level2.GetVariable("level2Var") // Local var - level2.GetVariable("level1Var") // Parent var - level2.GetVariable("rootVar") // Root var - level2.GetVariable("shared") // Shadowed var + level2.GetVariable("level2Var") // Local var + level2.GetVariable("level1Var") // Parent var + level2.GetVariable("rootVar") // Root var + level2.GetVariable("shared") // Shadowed var level2.GetVariable("nonExistentVar") // Missing var } @@ -126,4 +126,4 @@ func BenchmarkContextVariableLookup(b *testing.B) { level2.Release() level1.Release() rootCtx.Release() -} \ No newline at end of file +} diff --git a/tokenizer.go b/tokenizer.go index 18e8402..ed55240 100644 --- a/tokenizer.go +++ b/tokenizer.go @@ -10,7 +10,7 @@ func processWhitespaceControl(tokens []Token) []Token { // Modify tokens in-place to avoid allocation // This works because we're only changing token values, not adding/removing tokens - + // Process each token to apply whitespace trimming for i := 0; i < len(tokens); i++ { token := tokens[i] diff --git a/twig.go b/twig.go index 7cd7d33..078ba7b 100644 --- a/twig.go +++ b/twig.go @@ -607,7 +607,7 @@ func (t *Template) Render(context map[string]interface{}) (string, error) { func (t *Template) RenderTo(w io.Writer, context map[string]interface{}) error { // Get a render context from the pool ctx := NewRenderContext(t.env, context, t.engine) - + // Set the template as the lastLoadedTemplate for relative path resolution ctx.lastLoadedTemplate = t diff --git a/utility.go b/utility.go index c167e5a..bea362d 100644 --- a/utility.go +++ b/utility.go @@ -25,7 +25,7 @@ func WriteString(w io.Writer, s string) (int, error) { if sw, ok := w.(io.StringWriter); ok { return sw.WriteString(s) } - + // Fast path for our own Buffer type if buf, ok := w.(*Buffer); ok { return buf.WriteString(s) @@ -46,14 +46,14 @@ func WriteFormat(w io.Writer, format string, args ...interface{}) (int, error) { if buf, ok := w.(*Buffer); ok { return buf.WriteFormat(format, args...) } - + // Use a pooled buffer for other writer types buf := GetBuffer() defer buf.Release() - + // Write the formatted string to the buffer buf.WriteFormat(format, args...) - + // Write the buffer to the writer return w.Write(buf.Bytes()) } @@ -67,7 +67,7 @@ func FormatInt(i int) string { } else if i > -100 && i < 0 { return smallNegIntStrings[-i] } - + // Fall back to standard formatting return strconv.Itoa(i) -} \ No newline at end of file +} diff --git a/zero_alloc_tokenizer.go b/zero_alloc_tokenizer.go index 46ec34a..5837e30 100644 --- a/zero_alloc_tokenizer.go +++ b/zero_alloc_tokenizer.go @@ -10,39 +10,39 @@ import ( const ( // Common HTML/Twig strings to pre-cache maxCacheableLength = 64 // Only cache strings shorter than this to avoid memory bloat - + // Common HTML tags - stringDiv = "div" - stringSpan = "span" - stringP = "p" - stringA = "a" - stringImg = "img" - stringHref = "href" - stringClass = "class" - stringId = "id" - stringStyle = "style" - + stringDiv = "div" + stringSpan = "span" + stringP = "p" + stringA = "a" + stringImg = "img" + stringHref = "href" + stringClass = "class" + stringId = "id" + stringStyle = "style" + // Common Twig syntax - stringIf = "if" - stringFor = "for" - stringEnd = "end" - stringEndif = "endif" - stringEndfor = "endfor" - stringElse = "else" - stringBlock = "block" - stringSet = "set" + stringIf = "if" + stringFor = "for" + stringEnd = "end" + stringEndif = "endif" + stringEndfor = "endfor" + stringElse = "else" + stringBlock = "block" + stringSet = "set" stringInclude = "include" stringExtends = "extends" - stringMacro = "macro" - + stringMacro = "macro" + // Common operators - stringEquals = "==" + stringEquals = "==" stringNotEquals = "!=" - stringAnd = "and" - stringOr = "or" - stringNot = "not" - stringIn = "in" - stringIs = "is" + stringAnd = "and" + stringOr = "or" + stringNot = "not" + stringIn = "in" + stringIs = "is" ) // GlobalStringCache provides a centralized cache for string interning @@ -78,12 +78,12 @@ type TagLocation struct { // ZeroAllocTokenizer is an allocation-free tokenizer // It uses a pre-allocated token buffer for all token operations type ZeroAllocTokenizer struct { - tokenBuffer []Token // Pre-allocated buffer of tokens - source string // Source string being tokenized - position int // Current position in source - line int // Current line - result []Token // Slice of actually used tokens - tempStrings []string // String constants that we can reuse + tokenBuffer []Token // Pre-allocated buffer of tokens + source string // Source string being tokenized + position int // Current position in source + line int // Current line + result []Token // Slice of actually used tokens + tempStrings []string // String constants that we can reuse } // This array contains commonly used strings in tokenization to avoid allocations @@ -92,17 +92,17 @@ var commonStrings = []string{ "if", "else", "elseif", "endif", "for", "endfor", "in", "block", "endblock", "extends", "include", "with", "set", "macro", "endmacro", "import", "from", "as", "do", - + // Common operators "+", "-", "*", "/", "=", "==", "!=", ">", "<", ">=", "<=", "and", "or", "not", "~", "%", "?", ":", "??", - + // Common punctuation "(", ")", "[", "]", "{", "}", ".", ",", "|", ";", - + // Common literals "true", "false", "null", - + // Empty string "", } @@ -119,7 +119,7 @@ var tokenizerPool = sync.Pool{ // Create a pre-allocated tokenizer with reasonable defaults return &TokenizerPooled{ tokenizer: ZeroAllocTokenizer{ - tokenBuffer: make([]Token, 0, 256), // Buffer for tokens + tokenBuffer: make([]Token, 0, 256), // Buffer for tokens tempStrings: append([]string{}, commonStrings...), result: nil, }, @@ -131,13 +131,13 @@ var tokenizerPool = sync.Pool{ // GetTokenizer gets a tokenizer from the pool func GetTokenizer(source string, capacityHint int) *ZeroAllocTokenizer { pooled := tokenizerPool.Get().(*TokenizerPooled) - + // Reset the tokenizer tokenizer := &pooled.tokenizer tokenizer.source = source tokenizer.position = 0 tokenizer.line = 1 - + // Ensure token buffer has enough capacity neededCapacity := capacityHint if neededCapacity <= 0 { @@ -147,20 +147,20 @@ func GetTokenizer(source string, capacityHint int) *ZeroAllocTokenizer { neededCapacity = 32 } } - + // Resize token buffer if needed if cap(tokenizer.tokenBuffer) < neededCapacity { tokenizer.tokenBuffer = make([]Token, 0, neededCapacity) } else { tokenizer.tokenBuffer = tokenizer.tokenBuffer[:0] } - + // Reset result tokenizer.result = nil - + // Mark as used pooled.used = true - + return tokenizer } @@ -169,14 +169,14 @@ func ReleaseTokenizer(tokenizer *ZeroAllocTokenizer) { // Get the parent pooled struct pooled := (*TokenizerPooled)(unsafe.Pointer( uintptr(unsafe.Pointer(tokenizer)) - unsafe.Offsetof(TokenizerPooled{}.tokenizer))) - + // Only return to pool if it's used if pooled.used { // Mark as not used and clear references that might prevent GC pooled.used = false tokenizer.source = "" tokenizer.result = nil - + // Return to pool tokenizerPool.Put(pooled) } @@ -189,7 +189,7 @@ func (t *ZeroAllocTokenizer) AddToken(tokenType int, value string, line int) { token.Type = tokenType token.Value = value token.Line = line - + // Add to buffer t.tokenBuffer = append(t.tokenBuffer, token) } @@ -203,12 +203,12 @@ func (t *ZeroAllocTokenizer) GetStringConstant(s string) string { return constant } } - + // Add to temp strings if it's a short string that might be reused if len(s) <= 20 { t.tempStrings = append(t.tempStrings, s) } - + return s } @@ -218,23 +218,23 @@ func (t *ZeroAllocTokenizer) TokenizeExpression(expr string) []Token { savedSource := t.source savedPosition := t.position savedLine := t.line - + t.source = expr t.position = 0 startTokenCount := len(t.tokenBuffer) - + var inString bool var stringDelimiter byte var stringStart int - + for t.position < len(t.source) { c := t.source[t.position] - + // Handle string literals if (c == '"' || c == '\'') && (t.position == 0 || t.source[t.position-1] != '\\') { if inString && c == stringDelimiter { // End of string, add the string token - value := t.source[stringStart:t.position] + value := t.source[stringStart:t.position] t.AddToken(TOKEN_STRING, value, t.line) inString = false } else if !inString { @@ -246,23 +246,23 @@ func (t *ZeroAllocTokenizer) TokenizeExpression(expr string) []Token { t.position++ continue } - + // Skip chars inside strings if inString { t.position++ continue } - + // Handle operators (includes multi-char operators like ==, !=, etc.) if isOperator(c) { op := string(c) t.position++ - + // Check for two-character operators if t.position < len(t.source) { nextChar := t.source[t.position] twoCharOp := string([]byte{c, nextChar}) - + // Check common two-char operators if (c == '=' && nextChar == '=') || (c == '!' && nextChar == '=') || @@ -271,18 +271,18 @@ func (t *ZeroAllocTokenizer) TokenizeExpression(expr string) []Token { (c == '&' && nextChar == '&') || (c == '|' && nextChar == '|') || (c == '?' && nextChar == '?') { - + op = twoCharOp t.position++ } } - + // Use constant version of the operator string if possible op = t.GetStringConstant(op) t.AddToken(TOKEN_OPERATOR, op, t.line) continue } - + // Handle punctuation if isPunctuation(c) { // Use constant version of punctuation @@ -291,7 +291,7 @@ func (t *ZeroAllocTokenizer) TokenizeExpression(expr string) []Token { t.position++ continue } - + // Skip whitespace if isWhitespace(c) { t.position++ @@ -300,79 +300,79 @@ func (t *ZeroAllocTokenizer) TokenizeExpression(expr string) []Token { } continue } - + // Handle identifiers, literals, etc. if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' { // Start of an identifier start := t.position - + // Find the end t.position++ - for t.position < len(t.source) && - ((t.source[t.position] >= 'a' && t.source[t.position] <= 'z') || - (t.source[t.position] >= 'A' && t.source[t.position] <= 'Z') || - (t.source[t.position] >= '0' && t.source[t.position] <= '9') || - t.source[t.position] == '_') { + for t.position < len(t.source) && + ((t.source[t.position] >= 'a' && t.source[t.position] <= 'z') || + (t.source[t.position] >= 'A' && t.source[t.position] <= 'Z') || + (t.source[t.position] >= '0' && t.source[t.position] <= '9') || + t.source[t.position] == '_') { t.position++ } - + // Extract the identifier identifier := t.source[start:t.position] - + // Try to use a canonical string identifier = t.GetStringConstant(identifier) - + // Keywords/literals get special token types if identifier == "true" || identifier == "false" || identifier == "null" { t.AddToken(TOKEN_NAME, identifier, t.line) } else { t.AddToken(TOKEN_NAME, identifier, t.line) } - + continue } - + // Handle numbers if (c >= '0' && c <= '9') || (c == '-' && t.position+1 < len(t.source) && t.source[t.position+1] >= '0' && t.source[t.position+1] <= '9') { start := t.position - + // Skip the negative sign if present if c == '-' { t.position++ } - + // Consume digits for t.position < len(t.source) && t.source[t.position] >= '0' && t.source[t.position] <= '9' { t.position++ } - + // Handle decimal point if t.position < len(t.source) && t.source[t.position] == '.' { t.position++ - + // Consume fractional digits for t.position < len(t.source) && t.source[t.position] >= '0' && t.source[t.position] <= '9' { t.position++ } } - + // Add the number token t.AddToken(TOKEN_NUMBER, t.source[start:t.position], t.line) continue } - + // Unrecognized character t.position++ } - + // Create slice of tokens tokens := t.tokenBuffer[startTokenCount:] - + // Restore original context t.source = savedSource t.position = savedPosition t.line = savedLine - + return tokens } @@ -381,34 +381,34 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { // Reset position and line t.position = 0 t.line = 1 - + // Clear token buffer t.tokenBuffer = t.tokenBuffer[:0] - + tagPatterns := [5]string{"{{-", "{{", "{%-", "{%", "{#"} tagTypes := [5]int{TOKEN_VAR_START_TRIM, TOKEN_VAR_START, TOKEN_BLOCK_START_TRIM, TOKEN_BLOCK_START, TOKEN_COMMENT_START} tagLengths := [5]int{3, 2, 3, 2, 2} - + for t.position < len(t.source) { // Find the next tag nextTagPos := -1 tagType := -1 tagLength := 0 - + // Check for all possible tag patterns // This loop avoids allocations by manually checking prefixes remainingSource := t.source[t.position:] for i := 0; i < 5; i++ { pattern := tagPatterns[i] - if len(remainingSource) >= len(pattern) && - remainingSource[:len(pattern)] == pattern { + if len(remainingSource) >= len(pattern) && + remainingSource[:len(pattern)] == pattern { // Tag found at current position nextTagPos = t.position tagType = tagTypes[i] tagLength = tagLengths[i] break } - + // If not found at current position, find it in the remainder patternPos := strings.Index(remainingSource, pattern) if patternPos != -1 { @@ -420,16 +420,16 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { } } } - + // Check if the tag is escaped if nextTagPos != -1 && nextTagPos > 0 && t.source[nextTagPos-1] == '\\' { // Add text up to the backslash if nextTagPos-1 > t.position { - preText := t.source[t.position:nextTagPos-1] + preText := t.source[t.position : nextTagPos-1] t.AddToken(TOKEN_TEXT, preText, t.line) t.line += countNewlines(preText) } - + // Add the tag as literal text (without the backslash) // Find which pattern was matched for i := 0; i < 5; i++ { @@ -438,12 +438,12 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { break } } - + // Move past this tag t.position = nextTagPos + tagLength continue } - + // No more tags found - add the rest as TEXT if nextTagPos == -1 { if t.position < len(t.source) { @@ -453,30 +453,30 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { } break } - + // Add text before the tag if nextTagPos > t.position { textContent := t.source[t.position:nextTagPos] t.AddToken(TOKEN_TEXT, textContent, t.line) t.line += countNewlines(textContent) } - + // Add the tag start token t.AddToken(tagType, "", t.line) - + // Move past opening tag t.position = nextTagPos + tagLength - + // Find matching end tag var endTag string var endTagType int var endTagLength int - + if tagType == TOKEN_VAR_START || tagType == TOKEN_VAR_START_TRIM { // Look for "}}" or "-}}" endPos1 := strings.Index(t.source[t.position:], "}}") endPos2 := strings.Index(t.source[t.position:], "-}}") - + if endPos1 != -1 && (endPos2 == -1 || endPos1 < endPos2) { endTag = "}}" endTagType = TOKEN_VAR_END @@ -492,7 +492,7 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { // Look for "%}" or "-%}" endPos1 := strings.Index(t.source[t.position:], "%}") endPos2 := strings.Index(t.source[t.position:], "-%}") - + if endPos1 != -1 && (endPos2 == -1 || endPos1 < endPos2) { endTag = "%}" endTagType = TOKEN_BLOCK_END @@ -514,17 +514,17 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { endTagType = TOKEN_COMMENT_END endTagLength = 2 } - + // Find position of the end tag endPos := strings.Index(t.source[t.position:], endTag) if endPos == -1 { return nil, fmt.Errorf("unclosed tag at line %d", t.line) } - + // Get content between tags - tagContent := t.source[t.position:t.position+endPos] + tagContent := t.source[t.position : t.position+endPos] t.line += countNewlines(tagContent) - + // Process tag content based on type if tagType == TOKEN_COMMENT_START { // Store comments as TEXT tokens @@ -534,7 +534,7 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { } else { // For variable and block tags, tokenize the content tagContent = strings.TrimSpace(tagContent) - + if tagType == TOKEN_BLOCK_START || tagType == TOKEN_BLOCK_START_TRIM { // Process block tags with specialized tokenization t.processBlockTag(tagContent) @@ -552,17 +552,17 @@ func (t *ZeroAllocTokenizer) TokenizeHtmlPreserving() ([]Token, error) { } } } - + // Add the end tag token t.AddToken(endTagType, "", t.line) - + // Move past the end tag t.position = t.position + endPos + endTagLength } - + // Add EOF token t.AddToken(TOKEN_EOF, "", t.line) - + // Save the token buffer to result t.result = t.tokenBuffer return t.result, nil @@ -574,7 +574,7 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { spacePos := strings.IndexByte(content, ' ') var blockName string var blockContent string - + if spacePos == -1 { // No space found, the whole content is the tag name blockName = content @@ -583,29 +583,29 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { blockName = content[:spacePos] blockContent = strings.TrimSpace(content[spacePos+1:]) } - + // Use canonical string for block name blockName = t.GetStringConstant(blockName) t.AddToken(TOKEN_NAME, blockName, t.line) - + // If there's no content, we're done if blockContent == "" { return } - + // Process based on block type switch blockName { case "if", "elseif": // For conditional blocks, tokenize expression t.TokenizeExpression(blockContent) - + case "for": // Process for loop with iterator(s) and collection inPos := strings.Index(strings.ToLower(blockContent), " in ") if inPos != -1 { iterators := strings.TrimSpace(blockContent[:inPos]) collection := strings.TrimSpace(blockContent[inPos+4:]) - + // Handle key, value iterator syntax if strings.Contains(iterators, ",") { iterParts := strings.SplitN(iterators, ",", 2) @@ -613,7 +613,7 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // Process iterator variables keyVar := t.GetStringConstant(strings.TrimSpace(iterParts[0])) valueVar := t.GetStringConstant(strings.TrimSpace(iterParts[1])) - + t.AddToken(TOKEN_NAME, keyVar, t.line) t.AddToken(TOKEN_PUNCTUATION, ",", t.line) t.AddToken(TOKEN_NAME, valueVar, t.line) @@ -623,31 +623,31 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { iterator := t.GetStringConstant(iterators) t.AddToken(TOKEN_NAME, iterator, t.line) } - + // Add 'in' keyword t.AddToken(TOKEN_NAME, "in", t.line) - + // Process collection expression t.TokenizeExpression(collection) } else { // Fallback for malformed for loops t.AddToken(TOKEN_NAME, blockContent, t.line) } - + case "set": // Handle variable assignment assignPos := strings.Index(blockContent, "=") if assignPos != -1 { varName := strings.TrimSpace(blockContent[:assignPos]) value := strings.TrimSpace(blockContent[assignPos+1:]) - + // Add the variable name token varName = t.GetStringConstant(varName) t.AddToken(TOKEN_NAME, varName, t.line) - + // Add the assignment operator t.AddToken(TOKEN_OPERATOR, "=", t.line) - + // Tokenize the value expression t.TokenizeExpression(value) } else { @@ -655,24 +655,24 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { blockContent = t.GetStringConstant(blockContent) t.AddToken(TOKEN_NAME, blockContent, t.line) } - + case "do": // Handle variable assignment similar to set tag assignPos := strings.Index(blockContent, "=") if assignPos != -1 { varName := strings.TrimSpace(blockContent[:assignPos]) value := strings.TrimSpace(blockContent[assignPos+1:]) - + // Check if varName is valid (should be a variable name) // In Twig, variable names must start with a letter or underscore if len(varName) > 0 && (isCharAlpha(varName[0]) || varName[0] == '_') { // Add the variable name token varName = t.GetStringConstant(varName) t.AddToken(TOKEN_NAME, varName, t.line) - + // Add the assignment operator t.AddToken(TOKEN_OPERATOR, "=", t.line) - + // Tokenize the value expression if len(value) > 0 { t.TokenizeExpression(value) @@ -692,25 +692,25 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // No assignment, just an expression to evaluate t.TokenizeExpression(blockContent) } - + case "include": // Handle include with template path and optional context withPos := strings.Index(strings.ToLower(blockContent), " with ") if withPos != -1 { templatePath := strings.TrimSpace(blockContent[:withPos]) contextExpr := strings.TrimSpace(blockContent[withPos+6:]) - + // Process template path t.tokenizeTemplatePath(templatePath) - + // Add 'with' keyword t.AddToken(TOKEN_NAME, "with", t.line) - + // Process context expression as object if strings.HasPrefix(contextExpr, "{") && strings.HasSuffix(contextExpr, "}") { // Context is an object literal t.AddToken(TOKEN_PUNCTUATION, "{", t.line) - objectContent := contextExpr[1:len(contextExpr)-1] + objectContent := contextExpr[1 : len(contextExpr)-1] t.tokenizeObjectContents(objectContent) t.AddToken(TOKEN_PUNCTUATION, "}", t.line) } else { @@ -721,11 +721,11 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // Just a template path t.tokenizeTemplatePath(blockContent) } - + case "extends": // Handle extends tag (similar to include template path) t.tokenizeTemplatePath(blockContent) - + case "from": // Handle from tag which has a special format: // {% from "template.twig" import macro1, macro2 as alias %} @@ -734,32 +734,32 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // Extract template path and macros list templatePath := strings.TrimSpace(blockContent[:importPos]) macrosStr := strings.TrimSpace(blockContent[importPos+8:]) // 8 = len(" import ") - + // Process template path t.tokenizeTemplatePath(templatePath) - + // Add 'import' keyword t.AddToken(TOKEN_NAME, "import", t.line) - + // Process macro imports macros := strings.Split(macrosStr, ",") for i, macro := range macros { macro = strings.TrimSpace(macro) - + // Check for "as" alias asPos := strings.Index(strings.ToLower(macro), " as ") if asPos != -1 { // Extract macro name and alias macroName := strings.TrimSpace(macro[:asPos]) alias := strings.TrimSpace(macro[asPos+4:]) - + // Add macro name macroName = t.GetStringConstant(macroName) t.AddToken(TOKEN_NAME, macroName, t.line) - + // Add 'as' keyword t.AddToken(TOKEN_NAME, "as", t.line) - + // Add alias alias = t.GetStringConstant(alias) t.AddToken(TOKEN_NAME, alias, t.line) @@ -768,7 +768,7 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { macro = t.GetStringConstant(macro) t.AddToken(TOKEN_NAME, macro, t.line) } - + // Add comma if not the last macro if i < len(macros)-1 { t.AddToken(TOKEN_PUNCTUATION, ",", t.line) @@ -778,7 +778,7 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // Malformed from tag, just tokenize as expression t.TokenizeExpression(blockContent) } - + case "import": // Handle import tag which allows importing entire templates // {% import "template.twig" as alias %} @@ -787,13 +787,13 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // Extract template path and alias templatePath := strings.TrimSpace(blockContent[:asPos]) alias := strings.TrimSpace(blockContent[asPos+4:]) - + // Process template path t.tokenizeTemplatePath(templatePath) - + // Add 'as' keyword t.AddToken(TOKEN_NAME, "as", t.line) - + // Add alias alias = t.GetStringConstant(alias) t.AddToken(TOKEN_NAME, alias, t.line) @@ -801,7 +801,7 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // Simple import without alias t.TokenizeExpression(blockContent) } - + default: // Other block types - tokenize as expression t.TokenizeExpression(blockContent) @@ -813,12 +813,12 @@ func (t *ZeroAllocTokenizer) processBlockTag(content string) { // tokenizeTemplatePath handles template paths in extends/include tags func (t *ZeroAllocTokenizer) tokenizeTemplatePath(path string) { path = strings.TrimSpace(path) - + // If it's a quoted string if (strings.HasPrefix(path, "\"") && strings.HasSuffix(path, "\"")) || - (strings.HasPrefix(path, "'") && strings.HasSuffix(path, "'")) { + (strings.HasPrefix(path, "'") && strings.HasSuffix(path, "'")) { // Extract content without quotes - content := path[1:len(path)-1] + content := path[1 : len(path)-1] t.AddToken(TOKEN_STRING, content, t.line) } else { // Otherwise tokenize as expression @@ -831,29 +831,29 @@ func isCharAlpha(c byte) bool { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') } -// tokenizeObjectContents handles object literal contents +// tokenizeObjectContents handles object literal contents func (t *ZeroAllocTokenizer) tokenizeObjectContents(content string) { // Track state for nested structures inString := false stringDelim := byte(0) inObject := 0 inArray := 0 - + start := 0 colonPos := -1 - + for i := 0; i <= len(content); i++ { // At end of string or at a comma at the top level atEnd := i == len(content) isComma := !atEnd && content[i] == ',' - + // Process key-value pair when we find a comma or reach the end if (isComma || atEnd) && inObject == 0 && inArray == 0 && !inString { if colonPos != -1 { // We have a key-value pair keyStr := strings.TrimSpace(content[start:colonPos]) - valueStr := strings.TrimSpace(content[colonPos+1:i]) - + valueStr := strings.TrimSpace(content[colonPos+1 : i]) + // Process key if (len(keyStr) >= 2 && keyStr[0] == '"' && keyStr[len(keyStr)-1] == '"') || (len(keyStr) >= 2 && keyStr[0] == '\'' && keyStr[len(keyStr)-1] == '\'') { @@ -864,33 +864,33 @@ func (t *ZeroAllocTokenizer) tokenizeObjectContents(content string) { keyStr = t.GetStringConstant(keyStr) t.AddToken(TOKEN_NAME, keyStr, t.line) } - + // Add colon t.AddToken(TOKEN_PUNCTUATION, ":", t.line) - + // Process value t.TokenizeExpression(valueStr) - + // Add comma if needed if isComma && i < len(content)-1 { t.AddToken(TOKEN_PUNCTUATION, ",", t.line) } - + // Reset for next pair start = i + 1 colonPos = -1 } continue } - + // Skip end of string case if atEnd { continue } - + // Current character c := content[i] - + // Handle string literals if (c == '"' || c == '\'') && (i == 0 || content[i-1] != '\\') { if inString && c == stringDelim { @@ -901,12 +901,12 @@ func (t *ZeroAllocTokenizer) tokenizeObjectContents(content string) { } continue } - + // Skip processing inside strings if inString { continue } - + // Handle object and array nesting if c == '{' { inObject++ @@ -917,7 +917,7 @@ func (t *ZeroAllocTokenizer) tokenizeObjectContents(content string) { } else if c == ']' { inArray-- } - + // Track colon position for key-value separator if c == ':' && inObject == 0 && inArray == 0 && colonPos == -1 { colonPos = i @@ -928,10 +928,10 @@ func (t *ZeroAllocTokenizer) tokenizeObjectContents(content string) { // ApplyWhitespaceControl applies whitespace control to the tokenized result func (t *ZeroAllocTokenizer) ApplyWhitespaceControl() { tokens := t.result - + for i := 0; i < len(tokens); i++ { token := tokens[i] - + // Handle opening tags that trim whitespace before them if token.Type == TOKEN_VAR_START_TRIM || token.Type == TOKEN_BLOCK_START_TRIM { // If there's a text token before this, trim its trailing whitespace @@ -939,7 +939,7 @@ func (t *ZeroAllocTokenizer) ApplyWhitespaceControl() { tokens[i-1].Value = trimTrailingWhitespace(tokens[i-1].Value) } } - + // Handle closing tags that trim whitespace after them if token.Type == TOKEN_VAR_END_TRIM || token.Type == TOKEN_BLOCK_END_TRIM { // If there's a text token after this, trim its leading whitespace @@ -957,23 +957,23 @@ func newGlobalStringCache() *GlobalStringCache { cache := &GlobalStringCache{ strings: make(map[string]string, 64), // Pre-allocate capacity } - + // Pre-populate with common strings commonStrings := []string{ stringDiv, stringSpan, stringP, stringA, stringImg, stringHref, stringClass, stringId, stringStyle, stringIf, stringFor, stringEnd, stringEndif, stringEndfor, stringElse, stringBlock, stringSet, stringInclude, stringExtends, - stringMacro, stringEquals, stringNotEquals, stringAnd, + stringMacro, stringEquals, stringNotEquals, stringAnd, stringOr, stringNot, stringIn, stringIs, // Add empty string as well "", } - + for _, s := range commonStrings { cache.strings[s] = s } - + return cache } @@ -983,35 +983,35 @@ func newGlobalStringCache() *GlobalStringCache { func Intern(s string) string { // Fast path for very common strings to avoid lock contention switch s { - case stringDiv, stringSpan, stringP, stringA, stringImg, - stringIf, stringFor, stringEnd, stringEndif, stringEndfor, - stringElse, "": + case stringDiv, stringSpan, stringP, stringA, stringImg, + stringIf, stringFor, stringEnd, stringEndif, stringEndfor, + stringElse, "": return s } - + // Don't intern strings that are too long if len(s) > maxCacheableLength { return s } - + // Use read lock for lookup first (less contention) globalCache.RLock() cached, exists := globalCache.strings[s] globalCache.RUnlock() - + if exists { return cached } - + // Not found with read lock, acquire write lock to add globalCache.Lock() defer globalCache.Unlock() - + // Check again after acquiring write lock (double-checked locking) if cached, exists := globalCache.strings[s]; exists { return cached } - + // Add to cache and return globalCache.strings[s] = s return s @@ -1037,7 +1037,7 @@ func FindNextTag(source string, startPos int) TagLocation { // Direct byte comparison for opening characters // This avoids string allocations and uses pointer arithmetic srcPtr := unsafe.Pointer(unsafe.StringData(remainingSource)) - + // Quick check for potential tag start with { character for i := 0; i < remainingLen-1; i++ { if *(*byte)(unsafe.Add(srcPtr, i)) != '{' { @@ -1046,7 +1046,7 @@ func FindNextTag(source string, startPos int) TagLocation { // We found a '{', check next character secondChar := *(*byte)(unsafe.Add(srcPtr, i+1)) - + // Check for start of blocks tagPosition := startPos + i @@ -1057,13 +1057,13 @@ func FindNextTag(source string, startPos int) TagLocation { return TagLocation{TAG_VAR_TRIM, tagPosition, 3} } return TagLocation{TAG_VAR, tagPosition, 2} - + case '%': // Potential block tag {% if i+2 < remainingLen && *(*byte)(unsafe.Add(srcPtr, i+2)) == '-' { return TagLocation{TAG_BLOCK_TRIM, tagPosition, 3} } return TagLocation{TAG_BLOCK, tagPosition, 2} - + case '#': // Comment tag {# return TagLocation{TAG_COMMENT, tagPosition, 2} } @@ -1102,7 +1102,7 @@ func FindTagEnd(source string, startPos int, tagType TagType) int { } } } - + return -1 } @@ -1113,17 +1113,17 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { // Reset position and line t.position = 0 t.line = 1 - + // Clear token buffer t.tokenBuffer = t.tokenBuffer[:0] - + // Process the template content pos := 0 - + for pos < len(t.source) { // Find the next tag using optimized detection tagLoc := FindNextTag(t.source, pos) - + // Check if no more tags found if tagLoc.Position == -1 { // Add remaining text as TOKEN_TEXT @@ -1134,16 +1134,16 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { } break } - + // Check if the tag is escaped with a backslash if tagLoc.Position > 0 && t.source[tagLoc.Position-1] == '\\' { // Add text up to the backslash if tagLoc.Position-1 > pos { - preText := t.source[pos:tagLoc.Position-1] + preText := t.source[pos : tagLoc.Position-1] t.AddToken(TOKEN_TEXT, preText, t.line) t.line += countNewlines(preText) } - + // Add the tag as literal text (without the backslash) var tagText string switch tagLoc.Type { @@ -1158,21 +1158,21 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { case TAG_COMMENT: tagText = "{#" } - + t.AddToken(TOKEN_TEXT, tagText, t.line) - + // Move past this tag pos = tagLoc.Position + tagLoc.Length continue } - + // Add text before the tag if tagLoc.Position > pos { textContent := t.source[pos:tagLoc.Position] t.AddToken(TOKEN_TEXT, textContent, t.line) t.line += countNewlines(textContent) } - + // Add the tag start token var startTokenType int switch tagLoc.Type { @@ -1187,12 +1187,12 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { case TAG_COMMENT: startTokenType = TOKEN_COMMENT_START } - + t.AddToken(startTokenType, "", t.line) - + // Move past the tag's opening characters tagContentStart := tagLoc.Position + tagLoc.Length - + // Find the end of the tag tagEndPos := FindTagEnd(t.source, tagContentStart, tagLoc.Type) if tagEndPos == -1 { @@ -1207,15 +1207,15 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { } return nil, fmt.Errorf("unclosed %s tag at line %d", unclosedType, t.line) } - + // Get tag content tagContent := t.source[tagContentStart:tagEndPos] t.line += countNewlines(tagContent) - + // Determine the end token type and length var endTokenType int var endLength int - + switch tagLoc.Type { case TAG_VAR: endTokenType = TOKEN_VAR_END @@ -1249,7 +1249,7 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { endTokenType = TOKEN_COMMENT_END endLength = 2 // #} } - + // Process tag content based on tag type if tagLoc.Type == TAG_COMMENT { // Store comments as TEXT tokens @@ -1259,7 +1259,7 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { } else { // For variable and block tags, tokenize the content tagContent = strings.TrimSpace(tagContent) - + if tagLoc.Type == TAG_BLOCK || tagLoc.Type == TAG_BLOCK_TRIM { // Process block tags using specialized tokenization if len(tagContent) > 0 { @@ -1280,18 +1280,18 @@ func (t *ZeroAllocTokenizer) TokenizeOptimized() ([]Token, error) { } } } - + // Add end token t.AddToken(endTokenType, "", t.line) - + // Move past the end tag pos = tagEndPos + endLength } - + // Add EOF token t.AddToken(TOKEN_EOF, "", t.line) - + // Save and return result t.result = t.tokenBuffer return t.result, nil -} \ No newline at end of file +}