diff --git a/.gitignore b/.gitignore index 4b64668..0cd4911 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ coverage.out SOLUTION.md FINDINGS.md PROGRESS.md +ZERO_ALLOCATION_PLAN.md CLAUDE.md diff --git a/BENCHMARKING.md b/BENCHMARKING.md deleted file mode 100644 index 0dfdc7e..0000000 --- a/BENCHMARKING.md +++ /dev/null @@ -1,88 +0,0 @@ -# Memory Allocation Analysis for Zero-Allocation Rendering - -This guide explains how to use the memory profiling tools provided to identify and optimize memory allocation hotspots in the Twig template engine. - -## Running the Memory Analysis - -To run a comprehensive memory analysis and generate reports: - -```bash -./scripts/analyze_memory.sh -``` - -This script will: -1. Run benchmarks with memory allocation tracking -2. Generate heap allocation profiles -3. Analyze the profiles to identify allocation hotspots -4. Create a comprehensive report in the `reports/` directory - -## Interpreting the Results - -The analysis generates several files: - -- `reports/memory_optimization_report.md` - Main report with allocation analysis and recommendations -- `reports/benchmark_results.txt` - Raw benchmark results with memory statistics -- `reports/top_allocations.txt` - Top memory allocation sources from pprof -- `reports/heap.prof` - Heap allocation profile that can be analyzed with `go tool pprof` -- `reports/simple_profile.txt`, `reports/medium_profile.txt`, `reports/complex_profile.txt` - Profile results by template complexity - -## Using Individual Profiling Tools - -### Running Benchmarks with Memory Stats - -```bash -go test -run=^$ -bench=. -benchmem ./memory_profile_test.go -``` - -This command runs all benchmarks and reports allocations per operation. - -### Generating a Heap Profile - -```bash -go test -run=^$ -bench=BenchmarkRenderComplexTemplate -benchmem -memprofile=heap.prof ./memory_profile_test.go -``` - -### Analyzing the Heap Profile - -```bash -go tool pprof -alloc_space heap.prof -``` - -Common pprof commands: -- `top` - Show top allocation sources -- `list FunctionName` - Show line-by-line allocations in a function -- `web` - Open a web visualization of the profile - -### Using the Profiling Tool - -For targeted profiling of specific template complexity: - -```bash -go run cmd/profile/main.go -complexity=3 -iterations=1000 -memprofile=complex.prof -``` - -Options: -- `-complexity` - Template complexity level (1=simple, 2=medium, 3=complex) -- `-iterations` - Number of template renders to perform -- `-memprofile` - Output file for memory profile -- `-cpuprofile` - Output file for CPU profile (optional) - -## Zero-Allocation Implementation Strategy - -Based on the profile results, implement optimizations in this order: - -1. **Object Pooling** - Implement pools for all temporary objects -2. **String Operations** - Optimize string handling to avoid allocations -3. **Context Management** - Improve context creation, cloning, and cleanup -4. **Expression Evaluation** - Minimize allocations in expression execution -5. **Buffer Management** - Reuse output buffers with proper pooling - -## Testing Your Optimizations - -After each optimization, run the memory benchmarks again to verify the reduction in allocations: - -```bash -go test -run=^$ -bench=BenchmarkRender -benchmem ./memory_profile_test.go -``` - -The goal is to see zero (or minimal) allocations per operation in the `allocs/op` column. \ No newline at end of file diff --git a/cmd/profile/main.go b/cmd/profile/main.go deleted file mode 100644 index 89d7a92..0000000 --- a/cmd/profile/main.go +++ /dev/null @@ -1,302 +0,0 @@ -// Package main provides a simple utility to run memory profiling on the Twig template engine -package main - -import ( - "bytes" - "flag" - "fmt" - "log" - "os" - "runtime" - "runtime/pprof" - "strconv" - "time" - - "github.com/semihalev/twig" -) - -func main() { - // Command-line flags - cpuProfile := flag.String("cpuprofile", "", "write cpu profile to file") - memProfile := flag.String("memprofile", "", "write memory profile to file") - complexityLevel := flag.Int("complexity", 2, "template complexity level (1-3)") - iterations := flag.Int("iterations", 1000, "number of template renders to perform") - flag.Parse() - - // CPU profiling if requested - if *cpuProfile != "" { - f, err := os.Create(*cpuProfile) - if err != nil { - log.Fatal("could not create CPU profile: ", err) - } - defer f.Close() - if err := pprof.StartCPUProfile(f); err != nil { - log.Fatal("could not start CPU profile: ", err) - } - defer pprof.StopCPUProfile() - } - - // Run the appropriate benchmark based on complexity level - var totalTime time.Duration - var totalAllocBytes uint64 - var numGC uint32 - - // Record memory stats before - var memStatsBefore runtime.MemStats - runtime.ReadMemStats(&memStatsBefore) - - switch *complexityLevel { - case 1: - totalTime = runSimpleTemplateBenchmark(*iterations) - case 2: - totalTime = runMediumTemplateBenchmark(*iterations) - case 3: - totalTime = runComplexTemplateBenchmark(*iterations) - default: - log.Fatalf("Invalid complexity level: %d (must be 1-3)", *complexityLevel) - } - - // Record memory stats after - var memStatsAfter runtime.MemStats - runtime.ReadMemStats(&memStatsAfter) - - // Calculate allocation statistics - totalAllocBytes = memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc - numGC = memStatsAfter.NumGC - memStatsBefore.NumGC - - // Report results - fmt.Printf("=== Twig Memory Profiling Results ===\n") - fmt.Printf("Complexity Level: %d\n", *complexityLevel) - fmt.Printf("Total Iterations: %d\n", *iterations) - fmt.Printf("Total Time: %v\n", totalTime) - fmt.Printf("Time per Iteration: %v\n", totalTime/time.Duration(*iterations)) - fmt.Printf("Total Memory Allocated: %d bytes\n", totalAllocBytes) - fmt.Printf("Memory per Iteration: %d bytes\n", totalAllocBytes/uint64(*iterations)) - fmt.Printf("Number of GCs: %d\n", numGC) - - // Memory profiling if requested - if *memProfile != "" { - f, err := os.Create(*memProfile) - if err != nil { - log.Fatal("could not create memory profile: ", err) - } - defer f.Close() - runtime.GC() // Get up-to-date statistics - if err := pprof.WriteHeapProfile(f); err != nil { - log.Fatal("could not write memory profile: ", err) - } - } -} - -// runSimpleTemplateBenchmark runs a benchmark with a simple template -func runSimpleTemplateBenchmark(iterations int) time.Duration { - engine := twig.New() - err := engine.RegisterString("simple", "Hello, {{ name }}!") - if err != nil { - log.Fatalf("Error registering template: %v", err) - } - - context := map[string]interface{}{ - "name": "World", - } - - startTime := time.Now() - for i := 0; i < iterations; i++ { - var buf bytes.Buffer - template, _ := engine.Load("simple") - err := template.RenderTo(&buf, context) - if err != nil { - log.Fatalf("Error rendering template: %v", err) - } - } - return time.Since(startTime) -} - -// runMediumTemplateBenchmark runs a benchmark with a medium complexity template -func runMediumTemplateBenchmark(iterations int) time.Duration { - engine := twig.New() - templateContent := ` -
-

{{ user.name }}

-

Age: {{ user.age }}

- {% if user.bio %} -
{{ user.bio|capitalize }}
- {% else %} -
No bio available
- {% endif %} - -
-` - err := engine.RegisterString("medium", templateContent) - if err != nil { - log.Fatalf("Error registering template: %v", err) - } - - context := map[string]interface{}{ - "user": map[string]interface{}{ - "name": "John Doe", - "age": 30, - "bio": "web developer and open source enthusiast", - "skills": []map[string]interface{}{ - {"name": "Go", "level": "Advanced"}, - {"name": "JavaScript", "level": "Intermediate"}, - {"name": "CSS", "level": "Beginner"}, - {"name": "HTML", "level": "Advanced"}, - }, - }, - } - - startTime := time.Now() - for i := 0; i < iterations; i++ { - var buf bytes.Buffer - template, _ := engine.Load("medium") - err := template.RenderTo(&buf, context) - if err != nil { - log.Fatalf("Error rendering template: %v", err) - } - } - return time.Since(startTime) -} - -// runComplexTemplateBenchmark runs a benchmark with a complex template -func runComplexTemplateBenchmark(iterations int) time.Duration { - engine := twig.New() - - // Base template - baseTemplate := ` - - - - {% block title %}Default Title{% endblock %} - {% block styles %} - - {% endblock %} - - -
{% block header %}Default Header{% endblock %}
-
{% block content %}Default Content{% endblock %}
- - - -` - err := engine.RegisterString("base", baseTemplate) - if err != nil { - log.Fatalf("Error registering template: %v", err) - } - - // Macro template - macroTemplate := ` -{% macro renderProduct(product) %} -
-

{{ product.name }}

-

{{ product.description|capitalize }}

-
{{ product.price|format("$%.2f") }}
- {% if product.tags %} -
- {% for tag in product.tags %} - {{ tag }} - {% endfor %} -
- {% endif %} -
-{% endmacro %} -` - err = engine.RegisterString("macros", macroTemplate) - if err != nil { - log.Fatalf("Error registering template: %v", err) - } - - // Page template - pageTemplate := ` -{% extends "base" %} -{% import "macros" as components %} - -{% block title %}{{ page.title }} - {{ parent() }}{% endblock %} - -{% block styles %} - {{ parent() }} - -{% endblock %} - -{% block header %} -

{{ page.title }}

- -{% endblock %} - -{% block content %} -
-

Product List ({{ products|length }} items)

- - {% for product in products %} - {{ components.renderProduct(product) }} - {% endfor %} - - {% set total = 0 %} - {% for product in products %} - {% set total = total + product.price %} - {% endfor %} - -
-

Total products: {{ products|length }}

-

Average price: {{ (total / products|length)|format("$%.2f") }}

-

Price range: {{ products|map(p => p.price)|sort|first|format("$%.2f") }} - {{ products|map(p => p.price)|sort|last|format("$%.2f") }}

-
-
-{% endblock %} -` - err = engine.RegisterString("page", pageTemplate) - if err != nil { - log.Fatalf("Error registering template: %v", err) - } - - // Create a complex context with various data types - products := make([]map[string]interface{}, 20) - for i := 0; i < 20; i++ { - products[i] = map[string]interface{}{ - "id": i + 1, - "name": "Product " + strconv.Itoa(i+1), - "description": "This is product " + strconv.Itoa(i+1) + " with detailed information.", - "price": 15.0 + float64(i)*1.5, - "tags": []string{"tag1", "tag2", "tag3"}[0:1+(i%3)], - } - } - - context := map[string]interface{}{ - "page": map[string]interface{}{ - "title": "Product Catalog", - }, - "navigation": []map[string]interface{}{ - {"url": "/", "text": "Home"}, - {"url": "/products", "text": "Products"}, - {"url": "/about", "text": "About"}, - {"url": "/contact", "text": "Contact"}, - }, - "products": products, - } - - startTime := time.Now() - for i := 0; i < iterations; i++ { - var buf bytes.Buffer - template, _ := engine.Load("page") - err := template.RenderTo(&buf, context) - if err != nil { - log.Fatalf("Error rendering template: %v", err) - } - } - return time.Since(startTime) -} \ No newline at end of file diff --git a/scripts/analyze_memory.sh b/scripts/analyze_memory.sh deleted file mode 100755 index d024f20..0000000 --- a/scripts/analyze_memory.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash - -# Create output directory -mkdir -p reports - -echo "=== Running Memory Allocation Analysis for Twig Template Engine ===" -echo "" - -# Run standard benchmarks with memory statistics -echo "Step 1: Running benchmarks with memory allocation reporting..." -go test -run=^$ -bench=BenchmarkRender -benchmem ./memory_profile_test.go | tee reports/benchmark_results.txt - -# Generate allocation profile -echo "" -echo "Step 2: Generating heap allocation profile..." -go test -run=^$ -bench=BenchmarkRenderComplexTemplate -benchmem -memprofile=reports/heap.prof ./memory_profile_test.go - -# Run heap profile analysis and save top allocations -echo "" -echo "Step 3: Analyzing allocation profile..." -go tool pprof -text -alloc_space reports/heap.prof > reports/top_allocations.txt - -echo "" -echo "Step 4: Generating detailed memory profile report..." -# Run with different template complexities -echo " - Profiling simple templates..." -go run cmd/profile/main.go -complexity=1 -iterations=1000 -memprofile=reports/simple.prof > reports/simple_profile.txt - -echo " - Profiling medium templates..." -go run cmd/profile/main.go -complexity=2 -iterations=1000 -memprofile=reports/medium.prof > reports/medium_profile.txt - -echo " - Profiling complex templates..." -go run cmd/profile/main.go -complexity=3 -iterations=1000 -memprofile=reports/complex.prof > reports/complex_profile.txt - -# Generate flamegraph (requires go-torch if available) -if command -v go-torch &> /dev/null -then - echo "" - echo "Step 5: Generating flamegraph visualization..." - go-torch -alloc_space reports/heap.prof -file reports/allocations_flamegraph.svg -fi - -# Compile the comprehensive report -echo "" -echo "Step 6: Compiling final report..." - -cat > reports/memory_optimization_report.md << 'EOF' -# Twig Template Engine Memory Optimization Report - -## Summary - -This report analyzes memory allocation patterns in the Twig template engine to identify areas for implementing a zero-allocation rendering path. - -## Benchmark Results - -``` -EOF - -cat reports/benchmark_results.txt >> reports/memory_optimization_report.md - -cat >> reports/memory_optimization_report.md << 'EOF' -``` - -## Top Allocation Sources - -The following are the top functions allocating memory during template rendering: - -``` -EOF - -head -20 reports/top_allocations.txt >> reports/memory_optimization_report.md - -cat >> reports/memory_optimization_report.md << 'EOF' -``` - -## Memory Profile by Template Complexity - -### Simple Templates - -EOF - -grep -A 10 "Memory" reports/simple_profile.txt >> reports/memory_optimization_report.md - -cat >> reports/memory_optimization_report.md << 'EOF' - -### Medium Templates - -EOF - -grep -A 10 "Memory" reports/medium_profile.txt >> reports/memory_optimization_report.md - -cat >> reports/memory_optimization_report.md << 'EOF' - -### Complex Templates - -EOF - -grep -A 10 "Memory" reports/complex_profile.txt >> reports/memory_optimization_report.md - -cat >> reports/memory_optimization_report.md << 'EOF' - -## Key Allocation Hotspots - -Based on the profiling data, these areas should be prioritized for optimization: - -1. **String Operations** - String concatenation, substring operations, and conversions -2. **Context Creation** - Creating and copying RenderContext objects -3. **Map Allocations** - Temporary maps created during rendering -4. **Slice Allocations** - Dynamic arrays for node collections -5. **Expression Evaluation** - Temporary objects created during expression processing -6. **Buffer Management** - Output buffer allocations -7. **Function/Filter Calls** - Parameter passing and result handling - -## Optimization Strategies - -### String Operations - -- Replace string concatenation with direct writes to io.Writer -- Use pooled byte buffers instead of creating new strings -- Implement specialized ToString methods to avoid allocations for common types - -### Context Handling - -- Implement pooling for RenderContext objects -- Create a linked-context mechanism instead of copying values -- Preallocate and reuse context maps - -### Map and Slice Allocations - -- Preallocate maps and slices with known capacities -- Reuse map and slice objects from pools -- Avoid unnecessary copying of collections - -### Expression Evaluation - -- Pool expression evaluation result objects -- Optimize common expression patterns with specialized handlers -- Reduce intermediate allocations in expression trees - -### Implementation Plan - -1. Start with the highest allocation areas first -2. Implement object pooling for all major components -3. Create specialized non-allocating paths for common operations -4. Revise string handling to minimize allocations -5. Optimize hot spots in critical rendering code paths - -## Next Steps - -1. Implement object pools for all identified allocation sources -2. Create benchmarks to validate each optimization -3. Develop specialized string handling utilities -4. Optimize context handling and cloning -5. Enhance expression evaluation to minimize allocations - -EOF - -echo "" -echo "Report generation complete. See reports/memory_optimization_report.md for results." -echo "" \ No newline at end of file