lexer: restart after consumed comments and verbatim delimiters

lexer.go

@@ -409,9 +409,9 @@ func (l *lexer) emitRemainingHTML() {
 // ignoreSingleLineComment skips over a single-line comment {# ... #}.
 // Comments are not emitted as tokens; they are completely discarded.
 // Reports an error if the comment is not closed or contains a newline.
-func (l *lexer) ignoreSingleLineComment() {
+func (l *lexer) ignoreSingleLineComment() bool {
 	if !strings.HasPrefix(l.input[l.pos:], "{#") {
-		return
+		return false
 	}
 
 	l.emitRemainingHTML()
@@ -423,10 +423,10 @@ func (l *lexer) ignoreSingleLineComment() {
 		switch l.peek() {
 		case EOF:
 			l.errorf("Single-line comment not closed.")
-			return
+			return true
 		case '\n':
 			l.errorf("Newline not permitted in a single-line comment.")
-			return
+			return true
 		}
 
 		if strings.HasPrefix(l.input[l.pos:], "#}") {
@@ -438,6 +438,7 @@ func (l *lexer) ignoreSingleLineComment() {
 		l.next()
 	}
 	l.ignore() // ignore whole comment
+	return true
 }
 
 // processVerbatimTag handles {% verbatim %} and {% endverbatim %} tags.
@@ -446,7 +447,7 @@ func (l *lexer) ignoreSingleLineComment() {
 //
 // TODO: Support verbatim tag names as per Django docs:
 // https://docs.djangoproject.com/en/dev/ref/templates/builtins/#verbatim
-func (l *lexer) processVerbatimTag() {
+func (l *lexer) processVerbatimTag() bool {
 	if l.inVerbatim {
 		// end verbatim
 		if strings.HasPrefix(l.input[l.pos:], "{% endverbatim %}") {
@@ -456,6 +457,7 @@ func (l *lexer) processVerbatimTag() {
 			l.col += w
 			l.ignore()
 			l.inVerbatim = false
+			return true
 		}
 	} else if strings.HasPrefix(l.input[l.pos:], "{% verbatim %}") { // tag
 		l.emitRemainingHTML()
@@ -464,7 +466,9 @@ func (l *lexer) processVerbatimTag() {
 		l.pos += w
 		l.col += w
 		l.ignore()
+		return true
 	}
+	return false
 }
 
 // run is the main lexer loop that processes the entire input.
@@ -475,13 +479,21 @@ func (l *lexer) processVerbatimTag() {
 // The loop terminates when EOF is reached or an error occurs.
 func (l *lexer) run() {
 	for {
-		l.processVerbatimTag()
+		// A consumed delimiter leaves the cursor at a byte that may start the
+		// next comment or verbatim region. Re-run the recognition order before
+		// consuming that byte as ordinary text. The consumed guard makes the
+		// restart incapable of spinning without progress.
+		if l.processVerbatimTag() {
+			continue
+		}
 
 		if !l.inVerbatim {
 			// Ignore single-line comments {# ... #}
-			l.ignoreSingleLineComment()
-			if l.errored {
-				return
+			if l.ignoreSingleLineComment() {
+				if l.errored {
+					return
+				}
+				continue
 			}
 
 			if strings.HasPrefix(l.input[l.pos:], "{{") || // variable

lexer_restart_test.go

@@ -0,0 +1,103 @@
+package pongo2_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/flosch/pongo2/v7"
+)
+
+func TestLexerRestartsAfterCommentAndVerbatim(t *testing.T) {
+	t.Parallel()
+	tests := map[string]struct {
+		source string
+		want   string
+	}{
+		"adjacent comments": {
+			source: "{# a #}{# b #}X",
+			want:   "X",
+		},
+		"verbatim after comment": {
+			source: "{# a #}{% verbatim %}{{ raw }}{% endverbatim %}",
+			want:   "{{ raw }}",
+		},
+		"adjacent verbatim regions": {
+			source: "{% verbatim %}a{% endverbatim %}{% verbatim %}b{% endverbatim %}",
+			want:   "ab",
+		},
+		"comment after verbatim": {
+			source: "{% verbatim %}{# literal #}{% endverbatim %}{# drop #}X",
+			want:   "{# literal #}X",
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			tpl, err := pongo2.FromString(test.source)
+			if err != nil {
+				t.Fatalf("FromString(%q): %v", test.source, err)
+			}
+			got, err := tpl.Execute(nil)
+			if err != nil {
+				t.Fatalf("Execute(%q): %v", test.source, err)
+			}
+			if got != test.want {
+				t.Fatalf("Execute(%q) = %q, want %q", test.source, got, test.want)
+			}
+		})
+	}
+}
+
+// This differential oracle checks the Django comment contract: a short
+// comment produces nothing and every other fragment produces its own output.
+func TestTemplateCommentSemanticsMatchDjangoOracle(t *testing.T) {
+	t.Parallel()
+	fragments := []struct{ source, output string }{
+		{"A", "A"},
+		{"{# c #}", ""},
+		{"{#x#}", ""},
+		{" ", " "},
+		{`{{ "v" }}`, "v"},
+		{"{% if true %}T{% endif %}", "T"},
+		{"{% verbatim %}{{ raw }}{% endverbatim %}", "{{ raw }}"},
+	}
+
+	sequence := make([]int, 0, 3)
+	cases := 0
+	var walk func(int)
+	walk = func(depth int) {
+		if depth != 0 {
+			var source, want strings.Builder
+			for _, index := range sequence {
+				source.WriteString(fragments[index].source)
+				want.WriteString(fragments[index].output)
+			}
+			cases++
+			tpl, err := pongo2.FromString(source.String())
+			if err != nil {
+				t.Fatalf("FromString(%q): %v", source.String(), err)
+			}
+			got, err := tpl.Execute(nil)
+			if err != nil {
+				t.Fatalf("Execute(%q): %v", source.String(), err)
+			}
+			if got != want.String() {
+				t.Fatalf("%q rendered %q, Django renders %q",
+					source.String(), got, want.String())
+			}
+		}
+		if depth == cap(sequence) {
+			return
+		}
+		for index := range fragments {
+			sequence = append(sequence, index)
+			walk(depth + 1)
+			sequence = sequence[:len(sequence)-1]
+		}
+	}
+	walk(0)
+	wantCases := len(fragments) + len(fragments)*len(fragments) +
+		len(fragments)*len(fragments)*len(fragments)
+	if cases != wantCases {
+		t.Fatalf("generated %d sources, want %d", cases, wantCases)
+	}
+}