Profile the parser, and make the token stream cheap to build

README.md

@@ -10,20 +10,83 @@ meyer is being built to replace the ANTLR-generated SQLite parser in
 [sqlc](https://github.com/sqlc-dev/sqlc). See [PLAN.md](PLAN.md) for the
 architecture and [CLAUDE.md](CLAUDE.md) for the development workflow.
 
+[![Go Reference](https://pkg.go.dev/badge/github.com/sqlc-dev/meyer.svg)](https://pkg.go.dev/github.com/sqlc-dev/meyer)
+
 ## Usage
 
+```sh
+go get github.com/sqlc-dev/meyer
+```
+
 ```go
-stmts, err := parser.Parse(ctx, strings.NewReader("SELECT id FROM users WHERE id = ?"))
+package main
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/sqlc-dev/meyer/ast"
+	"github.com/sqlc-dev/meyer/parser"
+)
+
+func main() {
+	const src = `SELECT u.name, count(o.id) FROM users u
+	  JOIN orders o ON o.user_id = u.id
+	 WHERE u.created_at > :since`
+
+	stmts, err := parser.ParseString(src)
+	if err != nil {
+		var perr *parser.Error
+		if errors.As(err, &perr) {
+			fmt.Printf("%s at byte %d\n", perr.Message, perr.Offset)
+		}
+		return
+	}
+
+	for _, stmt := range stmts {
+		ast.Walk(stmt, func(n ast.Node) bool {
+			switch n := n.(type) {
+			case *ast.TableRef:
+				if n.Name != nil {
+					fmt.Println("table:", n.Name.Name.Name)
+				}
+			case *ast.BindParam:
+				fmt.Printf("param: %s at %d:%d\n", n.Raw, n.Pos(), n.End())
+			}
+			return true
+		})
+	}
+}
 ```
 
-`ParseString` and `ParseStatement` take a string; `ParseExpr` parses a single
-expression. A rejected input returns a `*parser.Error` carrying SQLite's
-exact message and the byte offset of the fault.
+```
+table: users
+table: orders
+param: :since at 100:106
+```
+
+`ParseString` and `ParseStatement` take a string, `Parse` takes an
+`io.Reader`, and `ParseExpr` parses a single expression. A rejected input
+returns a `*parser.Error` carrying SQLite's exact message and the byte offset
+of the fault; its `Error()` renders as `line:column: message`.
+
+```go
+_, err := parser.ParseString("SELECT FROM t")
+fmt.Println(err) // 1:8: near "FROM": syntax error
+```
 
 Every node embeds `ast.Span`, so `Pos()` and `End()` give byte offsets into
 the original input — sqlc slices the source with them to find `-- name:`
 comments and to report errors, so they are load-bearing rather than
-diagnostic.
+diagnostic. `ast.String` and `ast.Statements` render a tree back to SQL,
+which is a re-parseable rendering rather than a formatter.
+
+The full API is on
+[pkg.go.dev](https://pkg.go.dev/github.com/sqlc-dev/meyer): `parser` for the
+entry points, [`ast`](https://pkg.go.dev/github.com/sqlc-dev/meyer/ast) for
+the node set, and [`lexer`](https://pkg.go.dev/github.com/sqlc-dev/meyer/lexer)
+and [`token`](https://pkg.go.dev/github.com/sqlc-dev/meyer/token) if you want
+the token stream on its own.
 
 To see what the parser did with something:
 
@@ -36,7 +99,7 @@ go run ./cmd/debug-parse -render -f query.sql            # re-rendered SQL
 ## Status
 
 The parser covers the whole grammar of the pinned SQLite release and passes
-the full corpus: **20,971 of 20,971 cases**, extracted from every test script
+the full corpus: **21,326 of 21,326 cases**, extracted from every test script
 in SQLite's own test suite.
 
 Still to come, from [PLAN.md](PLAN.md): the sqlc integration itself, and the

ast/render.go

@@ -16,20 +16,38 @@ import "strings"
 // String renders n as SQL.
 func String(n Node) string {
 	var b strings.Builder
+	b.Grow(sizeHint(n))
 	write(&b, n)
 	return b.String()
 }
 
 // Statements renders a whole script, one statement per line.
 func Statements(stmts []Stmt) string {
 	var b strings.Builder
+	n := 0
+	for _, s := range stmts {
+		n += sizeHint(s) + 2 // ";\n"
+	}
+	b.Grow(n)
 	for _, s := range stmts {
 		write(&b, s)
 		b.WriteString(";\n")
 	}
 	return b.String()
 }
 
+// sizeHint estimates the rendered length of n from the source it was parsed
+// from, which it is within a few bytes of: rendering rewrites the whitespace
+// and nothing else. It saves a strings.Builder the four doublings a short
+// statement would otherwise cost. A node built by hand rather than parsed
+// has no span, and gets no hint.
+func sizeHint(n Node) int {
+	if size := n.End() - n.Pos(); size > 0 {
+		return size
+	}
+	return 0
+}
+
 func (n *SelectStmt) String() string             { return String(n) }
 func (n *InsertStmt) String() string             { return String(n) }
 func (n *UpdateStmt) String() string             { return String(n) }

cmd/debug-parse/main.go

@@ -46,7 +46,7 @@ func main() {
 
 	if *tokens {
 		for _, t := range lexer.Lex(src) {
-			fmt.Printf("%5d-%-5d %-10s %q\n", t.Pos, t.End, t.Kind, t.Text)
+			fmt.Printf("%5d-%-5d %-10s %q\n", t.Pos, t.End, t.Kind, t.Text(src))
 		}
 		return
 	}

cmd/difftest/main.go

@@ -240,7 +240,7 @@ func mutations(src string, n int, rng *rand.Rand) []string {
 		case 1: // delete a token
 			add(src[:t.Pos] + src[t.End:])
 		case 2: // duplicate a token
-			add(src[:t.End] + " " + t.Text + src[t.End:])
+			add(src[:t.End] + " " + t.Text(src) + src[t.End:])
 		case 3: // replace a token
 			add(src[:t.Pos] + injections[rng.Intn(len(injections))] + src[t.End:])
 		case 4: // insert a token

internal/sqlitesrc/sqlitesrc.go

@@ -323,7 +323,7 @@ func SplitDiffers(sql string) bool {
 				return true
 			}
 		default:
-			if strings.Contains(t.Text, ";") && !completeUnderstands(t) {
+			if text := t.Text(sql); strings.Contains(text, ";") && !completeUnderstands(t, text) {
 				return true
 			}
 		}
@@ -337,12 +337,12 @@ func SplitDiffers(sql string) bool {
 // An unterminated one it does not: in ":v('(%d)',changes());" the tokenizer
 // swallows the first quote into the bind parameter and finds a string
 // starting at the second, while sqlite3_complete pairs the two.
-func completeUnderstands(t token.Token) bool {
+func completeUnderstands(t token.Token, text string) bool {
 	switch t.Kind {
 	case token.STRING, token.BLOB:
 		return true
 	case token.ID:
-		switch t.Text[0] {
+		switch text[0] {
 		case '"', '`', '[':
 			return true
 		}

lexer/lexer.go

@@ -23,10 +23,14 @@ import (
 // A NUL byte terminates the input, matching SQLite, whose tokenizer walks a
 // NUL-terminated buffer.
 func Lex(src string) []token.Token {
-	// SQL averages a little over four bytes per token including the
-	// separators, so this usually gets the whole stream in one allocation.
-	toks := make([]token.Token, 0, len(src)/4+8)
+	// Across the corpus SQL runs to 3.7 bytes per token including the
+	// separators, but the mean is the wrong statistic to size from: what
+	// costs is the tail that has to grow and copy. A divisor of three
+	// covers 93% of cases in one allocation where four covers 80%, for
+	// about seventeen tokens of slack apiece.
+	toks := make([]token.Token, 0, len(src)/3+8)
 	i := 0
+	ambiguous := false
 	for i < len(src) {
 		kind, n := scan(src, i)
 		if kind == token.EOF { // NUL byte: end of input
@@ -36,11 +40,20 @@ func Lex(src string) []token.Token {
 			n = 1
 		}
 		if kind != token.SPACE && kind != token.COMMENT {
-			toks = append(toks, token.Token{Kind: kind, Text: src[i : i+n], Pos: i, End: i + n})
+			switch kind {
+			case token.WINDOW, token.OVER, token.FILTER:
+				ambiguous = true
+			}
+			toks = append(toks, token.Token{Kind: kind, Pos: i, End: i + n})
 		}
 		i += n
 	}
-	resolveWindowKeywords(toks)
+	// Most SQL contains none of the three context-dependent keywords, and
+	// noting whether any turned up costs a comparison per token against a
+	// second pass over all of them.
+	if ambiguous {
+		resolveWindowKeywords(toks)
+	}
 	toks = append(toks, token.Token{Kind: token.EOF, Pos: i, End: i})
 	return toks
 }

lexer/lexer_test.go

@@ -27,7 +27,7 @@ func texts(src string) []string {
 	toks := Lex(src)
 	out := make([]string, 0, len(toks))
 	for _, t := range toks[:len(toks)-1] {
-		out = append(out, t.Text)
+		out = append(out, t.Text(src))
 	}
 	return out
 }
@@ -86,7 +86,7 @@ func TestQuoting(t *testing.T) {
 	}
 	for _, tt := range tests {
 		toks := Lex(tt.src)
-		if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text != tt.text {
+		if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text(tt.src) != tt.text {
 			t.Errorf("Lex(%q) = %v, want one %v %q", tt.src, toks[:len(toks)-1], tt.kind, tt.text)
 		}
 	}
@@ -182,7 +182,7 @@ func TestVariables(t *testing.T) {
 	}
 	for _, tt := range tests {
 		toks := Lex(tt.src)
-		if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text != tt.text {
+		if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text(tt.src) != tt.text {
 			t.Errorf("Lex(%q) = %v, want one %v %q", tt.src, toks[:len(toks)-1], tt.kind, tt.text)
 		}
 	}
@@ -274,7 +274,7 @@ func TestWindowKeywordResolution(t *testing.T) {
 		}
 		if toks[tt.at].Kind != tt.want {
 			t.Errorf("Lex(%q)[%d] = %v %q, want %v",
-				tt.src, tt.at, toks[tt.at].Kind, toks[tt.at].Text, tt.want)
+				tt.src, tt.at, toks[tt.at].Kind, toks[tt.at].Text(tt.src), tt.want)
 		}
 	}
 }
@@ -286,8 +286,8 @@ func TestSpansCoverTheInput(t *testing.T) {
 		if tk.Kind == token.EOF {
 			continue
 		}
-		if got := src[tk.Pos:tk.End]; got != tk.Text {
-			t.Errorf("token %d: span %d:%d is %q, but Text is %q", i, tk.Pos, tk.End, got, tk.Text)
+		if tk.Pos < 0 || tk.Pos >= tk.End || tk.End > len(src) {
+			t.Errorf("token %d: span %d:%d is not a range within %d bytes", i, tk.Pos, tk.End, len(src))
 		}
 		if i > 0 && tk.Pos < toks[i-1].End {
 			t.Errorf("token %d starts at %d, before the previous token ended at %d",

parser/parse_ddl.go

@@ -288,7 +288,7 @@ func (p *parser) parseDefaultValue() ast.Expr {
 	if !token.IsID(p.cur().Kind) {
 		p.syntaxError()
 	}
-	return identOf(p.advance())
+	return p.identOf(p.advance())
 }
 
 // atTerm reports whether the lookahead begins a "term": a literal constant.
@@ -312,7 +312,7 @@ func (p *parser) parseGenerated() (ast.Expr, *ast.Ident) {
 	x := p.parseExpr(precLowest)
 	p.expect(token.RP)
 	if token.IsPlainID(p.cur().Kind) {
-		return x, identOf(p.advance())
+		return x, p.identOf(p.advance())
 	}
 	return x, nil
 }
@@ -381,7 +381,7 @@ func (p *parser) parseReferences() *ast.ForeignKeyClause {
 			arg := &ast.ForeignKeyArg{}
 			switch p.cur().Kind {
 			case token.INSERT, token.DELETE, token.UPDATE:
-				arg.Event = p.advance().Text
+				arg.Event = p.text(p.advance())
 			default:
 				p.syntaxError()
 			}