From 7f940f2836dc827dc71e515155b0a2c97c63094d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 23:42:37 +0000 Subject: [PATCH 1/2] Profile the parser, and make the token stream cheap to build Nobody had ever read the benchmarks. Profiling says the cost is in two places, and neither is where the code was written as though it were. A token slice is the largest single allocation a parse makes, and Token carried its own spelling: a string field, exactly src[Pos:End], so redundant with the offsets already there. The redundancy was not free. The pointer in it costs a write barrier per token to build and makes the whole slice something the garbage collector has to walk afterwards. Deriving the text with a method instead takes Token from 40 bytes to 24 and out of the collector's sight entirely, which a microbenchmark of the two shapes puts at 2.7x on the cost of producing one. Every caller has the source in hand -- it just lexed it -- so nothing has to be threaded anywhere new. Keyword lookup was a Go map, and hashing the spelling of every identifier was a fifth of the lexer. It is now indexed on the length and the first letter, which are free to compute and between them very nearly a perfect hash: 147 keywords in 93 groups, the largest of four. The candidates are compared byte by byte, upper-casing as they go, so nothing is allocated or copied. SQLite's keywordCode() hashes the first byte, the last byte and the length, but mixes them with a remainder mod a prime, and a parser reading a keyword every few bytes should not be dividing. Three smaller ones. The renderer sized its strings.Builder from nothing and grew it four times for a short statement; a node's span says how long the output will be, within a few bytes. The whitespace-and-comments divisor for the token slice was set to four by guess, and the corpus says three: 93% of cases in one allocation rather than 80%. And a "--" comment or a lone vertical tab used to send every parse through a second pass over all its tokens to resolve WINDOW, OVER and FILTER; noting whether any of the three turned up costs a comparison per token instead. Lex/Join 4336ns -> 2413ns -44% Lex/Simple 547ns -> 403ns -26% Render/Simple 269ns -> 165ns -39% 4 allocs -> 1 Render/Join 1223ns -> 789ns -36% 7 allocs -> 1 Parse/Join 10609ns -> 8373ns -21% Parse/Window 9839ns -> 8680ns -12% Corpus (21k) 362us -> 313us -14% 377KB -> 279KB One thing that looked obvious and was not: slab-allocating the AST nodes took Join from 109 allocations to 60 and did not move the clock at all, because it traded object count for pointer-bearing bytes and bytes are what cost. It is not in this commit. Corpus, round trip, spans and snapshots unchanged, and 3,085,106 difftest mutations agree with SQLite as before. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JzBeCg7rjweVW3uGPg5G7T --- ast/render.go | 18 +++++++ cmd/debug-parse/main.go | 2 +- cmd/difftest/main.go | 2 +- internal/sqlitesrc/sqlitesrc.go | 6 +-- lexer/lexer.go | 23 +++++++-- lexer/lexer_test.go | 12 ++--- parser/parse_ddl.go | 6 +-- parser/parse_expr.go | 69 ++++++++++++++----------- parser/parse_misc.go | 4 +- parser/parse_select.go | 8 +-- parser/parse_trigger.go | 4 +- parser/parser.go | 34 +++++++----- token/keywords.go | 91 ++++++++++++++++++++++++++++----- token/keywords_test.go | 38 ++++++++++++++ token/token.go | 23 ++++++--- 15 files changed, 253 insertions(+), 87 deletions(-) diff --git a/ast/render.go b/ast/render.go index e53ef96..7de17ed 100644 --- a/ast/render.go +++ b/ast/render.go @@ -16,6 +16,7 @@ import "strings" // String renders n as SQL. func String(n Node) string { var b strings.Builder + b.Grow(sizeHint(n)) write(&b, n) return b.String() } @@ -23,6 +24,11 @@ func String(n Node) string { // Statements renders a whole script, one statement per line. func Statements(stmts []Stmt) string { var b strings.Builder + n := 0 + for _, s := range stmts { + n += sizeHint(s) + 2 // ";\n" + } + b.Grow(n) for _, s := range stmts { write(&b, s) b.WriteString(";\n") @@ -30,6 +36,18 @@ func Statements(stmts []Stmt) string { return b.String() } +// sizeHint estimates the rendered length of n from the source it was parsed +// from, which it is within a few bytes of: rendering rewrites the whitespace +// and nothing else. It saves a strings.Builder the four doublings a short +// statement would otherwise cost. A node built by hand rather than parsed +// has no span, and gets no hint. +func sizeHint(n Node) int { + if size := n.End() - n.Pos(); size > 0 { + return size + } + return 0 +} + func (n *SelectStmt) String() string { return String(n) } func (n *InsertStmt) String() string { return String(n) } func (n *UpdateStmt) String() string { return String(n) } diff --git a/cmd/debug-parse/main.go b/cmd/debug-parse/main.go index 580a101..3d23593 100644 --- a/cmd/debug-parse/main.go +++ b/cmd/debug-parse/main.go @@ -46,7 +46,7 @@ func main() { if *tokens { for _, t := range lexer.Lex(src) { - fmt.Printf("%5d-%-5d %-10s %q\n", t.Pos, t.End, t.Kind, t.Text) + fmt.Printf("%5d-%-5d %-10s %q\n", t.Pos, t.End, t.Kind, t.Text(src)) } return } diff --git a/cmd/difftest/main.go b/cmd/difftest/main.go index 178f784..a79361d 100644 --- a/cmd/difftest/main.go +++ b/cmd/difftest/main.go @@ -240,7 +240,7 @@ func mutations(src string, n int, rng *rand.Rand) []string { case 1: // delete a token add(src[:t.Pos] + src[t.End:]) case 2: // duplicate a token - add(src[:t.End] + " " + t.Text + src[t.End:]) + add(src[:t.End] + " " + t.Text(src) + src[t.End:]) case 3: // replace a token add(src[:t.Pos] + injections[rng.Intn(len(injections))] + src[t.End:]) case 4: // insert a token diff --git a/internal/sqlitesrc/sqlitesrc.go b/internal/sqlitesrc/sqlitesrc.go index 0474566..2acedc1 100644 --- a/internal/sqlitesrc/sqlitesrc.go +++ b/internal/sqlitesrc/sqlitesrc.go @@ -323,7 +323,7 @@ func SplitDiffers(sql string) bool { return true } default: - if strings.Contains(t.Text, ";") && !completeUnderstands(t) { + if text := t.Text(sql); strings.Contains(text, ";") && !completeUnderstands(t, text) { return true } } @@ -337,12 +337,12 @@ func SplitDiffers(sql string) bool { // An unterminated one it does not: in ":v('(%d)',changes());" the tokenizer // swallows the first quote into the bind parameter and finds a string // starting at the second, while sqlite3_complete pairs the two. -func completeUnderstands(t token.Token) bool { +func completeUnderstands(t token.Token, text string) bool { switch t.Kind { case token.STRING, token.BLOB: return true case token.ID: - switch t.Text[0] { + switch text[0] { case '"', '`', '[': return true } diff --git a/lexer/lexer.go b/lexer/lexer.go index f1223be..9f13687 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -23,10 +23,14 @@ import ( // A NUL byte terminates the input, matching SQLite, whose tokenizer walks a // NUL-terminated buffer. func Lex(src string) []token.Token { - // SQL averages a little over four bytes per token including the - // separators, so this usually gets the whole stream in one allocation. - toks := make([]token.Token, 0, len(src)/4+8) + // Across the corpus SQL runs to 3.7 bytes per token including the + // separators, but the mean is the wrong statistic to size from: what + // costs is the tail that has to grow and copy. A divisor of three + // covers 93% of cases in one allocation where four covers 80%, for + // about seventeen tokens of slack apiece. + toks := make([]token.Token, 0, len(src)/3+8) i := 0 + ambiguous := false for i < len(src) { kind, n := scan(src, i) if kind == token.EOF { // NUL byte: end of input @@ -36,11 +40,20 @@ func Lex(src string) []token.Token { n = 1 } if kind != token.SPACE && kind != token.COMMENT { - toks = append(toks, token.Token{Kind: kind, Text: src[i : i+n], Pos: i, End: i + n}) + switch kind { + case token.WINDOW, token.OVER, token.FILTER: + ambiguous = true + } + toks = append(toks, token.Token{Kind: kind, Pos: i, End: i + n}) } i += n } - resolveWindowKeywords(toks) + // Most SQL contains none of the three context-dependent keywords, and + // noting whether any turned up costs a comparison per token against a + // second pass over all of them. + if ambiguous { + resolveWindowKeywords(toks) + } toks = append(toks, token.Token{Kind: token.EOF, Pos: i, End: i}) return toks } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index af2959d..79d86d2 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -27,7 +27,7 @@ func texts(src string) []string { toks := Lex(src) out := make([]string, 0, len(toks)) for _, t := range toks[:len(toks)-1] { - out = append(out, t.Text) + out = append(out, t.Text(src)) } return out } @@ -86,7 +86,7 @@ func TestQuoting(t *testing.T) { } for _, tt := range tests { toks := Lex(tt.src) - if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text != tt.text { + if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text(tt.src) != tt.text { t.Errorf("Lex(%q) = %v, want one %v %q", tt.src, toks[:len(toks)-1], tt.kind, tt.text) } } @@ -182,7 +182,7 @@ func TestVariables(t *testing.T) { } for _, tt := range tests { toks := Lex(tt.src) - if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text != tt.text { + if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text(tt.src) != tt.text { t.Errorf("Lex(%q) = %v, want one %v %q", tt.src, toks[:len(toks)-1], tt.kind, tt.text) } } @@ -274,7 +274,7 @@ func TestWindowKeywordResolution(t *testing.T) { } if toks[tt.at].Kind != tt.want { t.Errorf("Lex(%q)[%d] = %v %q, want %v", - tt.src, tt.at, toks[tt.at].Kind, toks[tt.at].Text, tt.want) + tt.src, tt.at, toks[tt.at].Kind, toks[tt.at].Text(tt.src), tt.want) } } } @@ -286,8 +286,8 @@ func TestSpansCoverTheInput(t *testing.T) { if tk.Kind == token.EOF { continue } - if got := src[tk.Pos:tk.End]; got != tk.Text { - t.Errorf("token %d: span %d:%d is %q, but Text is %q", i, tk.Pos, tk.End, got, tk.Text) + if tk.Pos < 0 || tk.Pos >= tk.End || tk.End > len(src) { + t.Errorf("token %d: span %d:%d is not a range within %d bytes", i, tk.Pos, tk.End, len(src)) } if i > 0 && tk.Pos < toks[i-1].End { t.Errorf("token %d starts at %d, before the previous token ended at %d", diff --git a/parser/parse_ddl.go b/parser/parse_ddl.go index c9b550e..0a1e6e5 100644 --- a/parser/parse_ddl.go +++ b/parser/parse_ddl.go @@ -288,7 +288,7 @@ func (p *parser) parseDefaultValue() ast.Expr { if !token.IsID(p.cur().Kind) { p.syntaxError() } - return identOf(p.advance()) + return p.identOf(p.advance()) } // atTerm reports whether the lookahead begins a "term": a literal constant. @@ -312,7 +312,7 @@ func (p *parser) parseGenerated() (ast.Expr, *ast.Ident) { x := p.parseExpr(precLowest) p.expect(token.RP) if token.IsPlainID(p.cur().Kind) { - return x, identOf(p.advance()) + return x, p.identOf(p.advance()) } return x, nil } @@ -381,7 +381,7 @@ func (p *parser) parseReferences() *ast.ForeignKeyClause { arg := &ast.ForeignKeyArg{} switch p.cur().Kind { case token.INSERT, token.DELETE, token.UPDATE: - arg.Event = p.advance().Text + arg.Event = p.text(p.advance()) default: p.syntaxError() } diff --git a/parser/parse_expr.go b/parser/parse_expr.go index f3cbf22..5df2541 100644 --- a/parser/parse_expr.go +++ b/parser/parse_expr.go @@ -129,7 +129,7 @@ func (p *parser) parseOperators(x ast.Expr, min int, stopAtAnd bool) ast.Expr { } // expr ::= expr PTR expr. The "->" and "->>" spellings share a // token; only the text tells them apart. - if t.Kind == token.PTR && t.Text == "->>" { + if t.Kind == token.PTR && p.text(t) == "->>" { e.op = ast.OpPtr2 } x = p.binary(x, e.op, e.prec) @@ -257,7 +257,7 @@ func (p *parser) parseIs(x ast.Expr) ast.Expr { // expr ::= expr likeop expr. [LIKE_KW] // expr ::= expr likeop expr ESCAPE expr. [LIKE_KW] func (p *parser) parseLike(x ast.Expr, not bool) ast.Expr { - op := identOf(p.advance()) + op := p.identOf(p.advance()) n := &ast.LikeExpr{Op: op, Not: not, X: x} n.Y = p.parseExpr(precCompare + 1) if p.at(token.ESCAPE) { @@ -392,7 +392,7 @@ func (p *parser) parsePrimary() ast.Expr { return p.parseQualifiedRef() } p.advance() - return identOf(t) + return p.identOf(t) } p.syntaxError() return nil @@ -411,21 +411,22 @@ func (p *parser) parseTerm() ast.Expr { switch t.Kind { case token.NULL: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitNull, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitNull, p.text(t)) case token.INTEGER: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitInteger, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitInteger, p.text(t)) case token.FLOAT: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitFloat, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitFloat, p.text(t)) case token.BLOB: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitBlob, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitBlob, p.text(t)) case token.STRING: p.advance() + raw := p.text(t) return &ast.Literal{ Span: spanOf(t), Kind: ast.LitString, - Value: dequote(t.Text, '\''), Raw: t.Text, + Value: dequote(raw, '\''), Raw: raw, } case token.QNUMBER: // The grammar action dequotes the digit separators and rejects a @@ -436,19 +437,20 @@ func (p *parser) parseTerm() ast.Expr { if strings.ContainsAny(value, ".eE") && !strings.HasPrefix(strings.ToLower(value), "0x") { kind = ast.LitFloat } - return &ast.Literal{Span: spanOf(t), Kind: kind, Value: value, Raw: t.Text} + return &ast.Literal{Span: spanOf(t), Kind: kind, Value: value, Raw: p.text(t)} case token.CTIME_KW: // CTIME_KW falls back to ID elsewhere, but here it has a shift // action, so it is always the keyword. p.advance() + raw := p.text(t) kind := ast.LitCurrentTimestamp - switch strings.ToUpper(t.Text) { + switch strings.ToUpper(raw) { case "CURRENT_DATE": kind = ast.LitCurrentDate case "CURRENT_TIME": kind = ast.LitCurrentTime } - return &ast.Literal{Span: spanOf(t), Kind: kind, Value: t.Text, Raw: t.Text} + return literalOf(t, kind, raw) } p.syntaxError() return nil @@ -482,7 +484,10 @@ func (p *parser) parseParenExpr() ast.Expr { // // expr ::= nm DOT nm. / expr ::= nm DOT nm DOT nm. func (p *parser) parseQualifiedRef() ast.Expr { - parts := []*ast.Ident{p.expectName()} + // A reference is db.table.column at most, so the slice is sized once + // rather than grown twice. + parts := make([]*ast.Ident, 1, 3) + parts[0] = p.expectName() p.expect(token.DOT) parts = append(parts, p.expectName()) if p.at(token.DOT) { @@ -536,7 +541,7 @@ func (p *parser) parseRaise() ast.Expr { n.Action = "IGNORE" p.advance() case token.ROLLBACK, token.ABORT, token.FAIL: - n.Action = strings.ToUpper(p.advance().Text) + n.Action = strings.ToUpper(p.text(p.advance())) p.expect(token.COMMA) n.Message = p.parseExpr(precLowest) default: @@ -554,7 +559,7 @@ func (p *parser) parseRaise() ast.Expr { // expr ::= idj LP distinct exprlist ORDER BY sortlist RP filter_over. // expr ::= idj LP STAR RP filter_over. func (p *parser) parseFuncCall() ast.Expr { - name := identOf(p.advance()) + name := p.identOf(p.advance()) n := &ast.FuncCall{Name: name} p.expect(token.LP) if p.at(token.STAR) { @@ -627,11 +632,11 @@ func (p *parser) parseTypeToken() *ast.TypeName { start := p.cur().Pos // A type is almost always one word ("INTEGER", "TEXT"); only build a // slice for the "UNSIGNED BIG INT" kind. - name := p.advance().Text + name := p.text(p.advance()) if token.IsIDS(p.cur().Kind) { words := []string{name} for token.IsIDS(p.cur().Kind) { - words = append(words, p.advance().Text) + words = append(words, p.text(p.advance())) } name = strings.Join(words, " ") } @@ -658,12 +663,12 @@ func (p *parser) parseTypeToken() *ast.TypeName { func (p *parser) parseSignedNumber() string { var sign string if p.at(token.PLUS) || p.at(token.MINUS) { - sign = p.advance().Text + sign = p.text(p.advance()) } if !p.at(token.INTEGER) && !p.at(token.FLOAT) { p.syntaxError() } - return sign + p.advance().Text + return sign + p.text(p.advance()) } // bindParam builds a BindParam and assigns its number the way @@ -679,18 +684,19 @@ func (p *parser) bindParam(t token.Token) ast.Expr { // "SELECT #1 #1" is reported at the second reference, not the first. // Deferring the error reproduces that, since a real syntax error later // in the statement aborts the parse and wins on its own. - if len(t.Text) >= 2 && t.Text[0] == '#' && t.Text[1] >= '0' && t.Text[1] <= '9' { - p.defer_(`near "`+t.Text+`": syntax error`, t.Pos) + raw := p.text(t) + if len(raw) >= 2 && raw[0] == '#' && raw[1] >= '0' && raw[1] <= '9' { + p.defer_(`near "`+raw+`": syntax error`, t.Pos) } - n := &ast.BindParam{Span: spanOf(t), Raw: t.Text} + n := &ast.BindParam{Span: spanOf(t), Raw: raw} switch { - case t.Text == "?": + case raw == "?": n.Kind = ast.ParamAnon p.nVar++ n.Number = p.nVar - case t.Text[0] == '?': + case raw[0] == '?': n.Kind = ast.ParamNumber - v, err := strconv.Atoi(t.Text[1:]) + v, err := strconv.Atoi(raw[1:]) if err == nil { n.Number = v if v > p.nVar { @@ -698,7 +704,7 @@ func (p *parser) bindParam(t token.Token) ast.Expr { } } default: - switch t.Text[0] { + switch raw[0] { case ':': n.Kind = ast.ParamColon case '@': @@ -706,8 +712,8 @@ func (p *parser) bindParam(t token.Token) ast.Expr { default: n.Kind = ast.ParamDollar } - n.Name = t.Text[1:] - if v, ok := p.varNums[t.Text]; ok { + n.Name = raw[1:] + if v, ok := p.varNums[raw]; ok { n.Number = v } else { p.nVar++ @@ -715,7 +721,7 @@ func (p *parser) bindParam(t token.Token) ast.Expr { if p.varNums == nil { p.varNums = make(map[string]int) } - p.varNums[t.Text] = p.nVar + p.varNums[raw] = p.nVar } } return n @@ -726,7 +732,7 @@ func (p *parser) bindParam(t token.Token) ast.Expr { // two digits. It reproduces sqlite3DequoteNumber, including its habit of // reporting the partially rewritten token rather than the original. func (p *parser) dequoteNumber(t token.Token) string { - in := []byte(t.Text) + in := []byte(p.text(t)) buf := append([]byte(nil), in...) bHex := len(in) > 1 && in[0] == '0' && (in[1] == 'x' || in[1] == 'X') isok := func(c byte) bool { @@ -754,3 +760,8 @@ func (p *parser) dequoteNumber(t token.Token) string { func isHexDigit(c byte) bool { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') } + +// literalOf builds the literals whose value is their own spelling. +func literalOf(t token.Token, kind ast.LiteralKind, raw string) *ast.Literal { + return &ast.Literal{Span: spanOf(t), Kind: kind, Value: raw, Raw: raw} +} diff --git a/parser/parse_misc.go b/parser/parse_misc.go index d2a0f81..c38be29 100644 --- a/parser/parse_misc.go +++ b/parser/parse_misc.go @@ -49,7 +49,7 @@ func (p *parser) parseTransOpt() (bool, *ast.Ident) { // cmd ::= COMMIT|END trans_opt. func (p *parser) parseCommit() ast.Stmt { t := p.advance() - n := &ast.CommitStmt{Keyword: strings.ToUpper(t.Text)} + n := &ast.CommitStmt{Keyword: strings.ToUpper(p.text(t))} n.HasTransaction, n.Name = p.parseTransOpt() n.Span = p.span(t.Pos) return n @@ -151,7 +151,7 @@ func (p *parser) parsePragmaValue() string { case token.PLUS, token.MINUS: return p.parseSignedNumber() case token.INTEGER, token.FLOAT, token.ON, token.DELETE, token.DEFAULT: - return p.advance().Text + return p.text(p.advance()) } return p.expectName().Raw } diff --git a/parser/parse_select.go b/parser/parse_select.go index be3e49b..7056929 100644 --- a/parser/parse_select.go +++ b/parser/parse_select.go @@ -73,7 +73,7 @@ func (p *parser) parseSelectNoWith() *ast.SelectStmt { } case token.EXCEPT, token.INTERSECT: p.advance() - op = ast.CompoundOp{Op: strings.ToUpper(t.Text)} + op = ast.CompoundOp{Op: strings.ToUpper(p.text(t))} default: n.Span = ast.Span{Start: start, Stop: p.prevEnd()} return n @@ -193,7 +193,7 @@ func (p *parser) parseAlias() (*ast.Ident, bool) { return p.expectName(), true } if token.IsIDS(p.cur().Kind) { - return identOf(p.advance()), false + return p.identOf(p.advance()), false } return nil, false } @@ -225,9 +225,9 @@ func (p *parser) tryJoinOperator() *ast.JoinOperator { return &ast.JoinOperator{Span: spanOf(t), Type: ast.JoinInner | ast.JoinComma} case token.JOIN: p.advance() - return &ast.JoinOperator{Span: spanOf(t), Type: ast.JoinInner, Words: []string{t.Text}} + return &ast.JoinOperator{Span: spanOf(t), Type: ast.JoinInner, Words: []string{p.text(t)}} case token.JOIN_KW: - words := []string{p.advance().Text} + words := []string{p.text(p.advance())} for i := 0; i < 2 && !p.at(token.JOIN); i++ { words = append(words, p.expectName().Raw) } diff --git a/parser/parse_trigger.go b/parser/parse_trigger.go index 85a3802..28dcdd8 100644 --- a/parser/parse_trigger.go +++ b/parser/parse_trigger.go @@ -35,9 +35,9 @@ func (p *parser) parseCreateTrigger(start int, temp bool) ast.Stmt { } switch p.cur().Kind { case token.DELETE, token.INSERT: - n.Event = strings.ToUpper(p.advance().Text) + n.Event = strings.ToUpper(p.text(p.advance())) case token.UPDATE: - n.Event = strings.ToUpper(p.advance().Text) + n.Event = strings.ToUpper(p.text(p.advance())) if p.accept(token.OF) { n.UpdateOf = p.parseIDList() } diff --git a/parser/parser.go b/parser/parser.go index c05c8f3..db19469 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -241,7 +241,7 @@ func (p *parser) expect(k token.Kind) token.Token { // same position. func (p *parser) checkIllegal() { if t := p.toks[p.i]; t.Kind == token.ILLEGAL { - p.fail(`unrecognized token: "`+t.Text+`"`, t.Pos) + p.fail(`unrecognized token: "`+p.text(t)+`"`, t.Pos) } } @@ -259,7 +259,7 @@ func (p *parser) syntaxErrorAt(t token.Token) { // whose text is empty, which %syntax_error reports this way. p.fail("incomplete input", t.Pos) } - p.fail(`near "`+t.Text+`": syntax error`, t.Pos) + p.fail(`near "`+p.text(t)+`": syntax error`, t.Pos) } // fail aborts the parse with one of SQLite's messages. @@ -400,7 +400,7 @@ func (p *parser) expectName() *ast.Ident { if !token.IsName(p.cur().Kind) { p.syntaxError() } - return identOf(p.advance()) + return p.identOf(p.advance()) } // expectIDS implements the "ids" class: ID | STRING. @@ -408,21 +408,22 @@ func (p *parser) expectIDS() *ast.Ident { if !token.IsIDS(p.cur().Kind) { p.syntaxError() } - return identOf(p.advance()) + return p.identOf(p.advance()) } // identOf builds an Ident from an accepted name token, undoing SQLite's four // quoting styles. -func identOf(t token.Token) *ast.Ident { - id := &ast.Ident{Span: spanOf(t), Raw: t.Text, Name: t.Text} - if len(t.Text) >= 2 { - switch t.Text[0] { +func (p *parser) identOf(t token.Token) *ast.Ident { + raw := p.text(t) + id := &ast.Ident{Span: spanOf(t), Raw: raw, Name: raw} + if len(raw) >= 2 { + switch raw[0] { case '"', '\'', '`': - id.Quote = ast.QuoteStyle(t.Text[0]) - id.Name = dequote(t.Text, t.Text[0]) + id.Quote = ast.QuoteStyle(raw[0]) + id.Name = dequote(raw, raw[0]) case '[': id.Quote = ast.QuoteSquare - id.Name = t.Text[1 : len(t.Text)-1] + id.Name = raw[1 : len(raw)-1] } } return id @@ -435,10 +436,13 @@ func dequote(s string, delim byte) string { if n := len(body); n > 0 && body[n-1] == delim { body = body[:n-1] } - d := string([]byte{delim}) - if !strings.Contains(body, d) { + // The overwhelmingly common case is a name with no escaped delimiter in + // it, and that one must not copy or allocate: the body is a slice of the + // input already. + if strings.IndexByte(body, delim) < 0 { return body } + d := string([]byte{delim}) return strings.ReplaceAll(body, d+d, d) } @@ -471,3 +475,7 @@ func (p *parser) xfullname() (*ast.QualifiedName, *ast.Ident) { } return name, alias } + +// text is the spelling of t. The lexer records only the byte range, so this +// is the one place that turns a token back into the source it came from. +func (p *parser) text(t token.Token) string { return p.src[t.Pos:t.End] } diff --git a/token/keywords.go b/token/keywords.go index db3ebf9..b633f69 100644 --- a/token/keywords.go +++ b/token/keywords.go @@ -1,5 +1,10 @@ package token +import ( + "slices" + "strings" +) + // keywords maps the upper-case spelling of every SQLite keyword to its token // kind. Transcribed from aKeywordTable[] in tool/mkkeywordhash.c, resolved // for the feature set of the pinned build (see parser/testdata/README.md): @@ -212,31 +217,93 @@ var fallback = func() [KindCount]bool { return t }() -// maxKeywordLen is the length of CURRENT_TIMESTAMP, the longest keyword. -const maxKeywordLen = 17 +// The lengths of BY and of CURRENT_TIMESTAMP, the shortest and longest +// keywords. +const ( + minKeywordLen = 2 + maxKeywordLen = 17 +) + +// keywordRow is one row of the shape-indexed table Lookup searches. +type keywordRow struct { + name string + kind Kind +} + +// keywordTable holds every keyword sorted by length and then alphabetically, +// so that all the keywords sharing a length and a first letter are adjacent. +// keywordShape indexes it by exactly those two things, giving the offset and +// count of the run an identifier of that shape has to be compared against. +// +// Length and first letter are free to compute and between them they are very +// nearly a perfect hash -- 147 keywords fall into 93 groups, the largest of +// four. That beats hashing the spelling: keywordCode() in SQLite computes one +// from the first byte, the last byte and the length, but it takes the +// remainder mod a prime to mix them, and a parser that reads a keyword per +// few bytes of input should not be doing a division. +var ( + keywordTable []keywordRow + keywordShape [maxKeywordLen + 1][26]struct{ lo, n uint8 } +) + +func init() { + names := make([]string, 0, len(keywords)) + for name := range keywords { + names = append(names, name) + } + slices.SortFunc(names, func(a, b string) int { + if d := len(a) - len(b); d != 0 { + return d + } + return strings.Compare(a, b) + }) + keywordTable = make([]keywordRow, len(names)) + for i, name := range names { + keywordTable[i] = keywordRow{name, keywords[name]} + g := &keywordShape[len(name)][name[0]-'A'] + if g.n == 0 { + g.lo = uint8(i) + } + g.n++ + } +} // Lookup returns the keyword kind for an identifier spelling, or ID if it is // not a keyword. Matching is ASCII case-insensitive, as in keywordCode(). // -// The upper-casing is done into a stack array rather than with -// strings.ToUpper, because this runs once per identifier token and the -// compiler turns m[string(b[:n])] into an allocation-free lookup. +// This runs once per identifier token, so it neither allocates nor copies: +// the candidates are found from the length and first letter alone, and each +// is compared against the input a byte at a time, upper-casing as it goes. func Lookup(name string) Kind { - if len(name) > maxKeywordLen { + if len(name) < minKeywordLen || len(name) > maxKeywordLen { return ID } - var buf [maxKeywordLen]byte + c := name[0] &^ 0x20 // upper-case, for a letter + if c < 'A' || c > 'Z' { + return ID + } + g := keywordShape[len(name)][c-'A'] + for _, e := range keywordTable[g.lo : g.lo+g.n] { + if equalUpper(name, e.name) { + return e.kind + } + } + return ID +} + +// equalUpper reports whether name upper-cased equals kw, which is already +// upper case and the same length. +func equalUpper(name, kw string) bool { for i := 0; i < len(name); i++ { c := name[i] if c >= 'a' && c <= 'z' { c -= 'a' - 'A' } - buf[i] = c - } - if k, ok := keywords[string(buf[:len(name)])]; ok { - return k + if c != kw[i] { + return false + } } - return ID + return true } // CanFallback reports whether k is in SQLite's %fallback ID set. diff --git a/token/keywords_test.go b/token/keywords_test.go index 2115677..5432cd2 100644 --- a/token/keywords_test.go +++ b/token/keywords_test.go @@ -117,3 +117,41 @@ func TestFallbackSetMatchesUpstream(t *testing.T) { } } } + +// TestLookup exercises the shape index directly. The corpus reaches it too, +// but only through whatever spellings SQLite's test suite happens to use: a +// group that the length-and-first-letter index pointed at the wrong run of +// the table would go unnoticed for any keyword that suite never writes. +func TestLookup(t *testing.T) { + for name, want := range keywords { + for _, spelling := range []string{name, strings.ToLower(name), mixedCase(name)} { + if got := Lookup(spelling); got != want { + t.Errorf("Lookup(%q) = %v, want %v", spelling, got, want) + } + } + } + // Non-keywords, including the shapes the index has to reject before it + // can subscript anything: too short, too long, and not starting with a + // letter. + for _, name := range []string{ + "", "a", "_", "1", "zz", "users", "created_at", "selects", "seleect", + "_select", "1select", "current_timestampx", "CURRENT_TIMESTAMPX", + "\xff\xfe", "{", "|abc", + } { + if got := Lookup(name); got != ID { + t.Errorf("Lookup(%q) = %v, want ID", name, got) + } + } +} + +// mixedCase upper-cases the even-numbered bytes of an upper-case word and +// lower-cases the rest. +func mixedCase(s string) string { + b := []byte(strings.ToLower(s)) + for i := 0; i < len(b); i += 2 { + if b[i] >= 'a' && b[i] <= 'z' { + b[i] -= 'a' - 'A' + } + } + return string(b) +} diff --git a/token/token.go b/token/token.go index b36faaa..b102ef3 100644 --- a/token/token.go +++ b/token/token.go @@ -219,11 +219,22 @@ func (k Kind) String() string { return "Kind(?)" } -// Token is one lexical token. Pos and End are byte offsets into the input -// the token was lexed from; Text is the corresponding source slice. +// Token is one lexical token: a kind and the half-open byte range [Pos, End) +// of the input it was lexed from. +// +// The spelling is deliberately not a field. It is exactly src[Pos:End], so +// storing it would duplicate what the offsets already say -- and it would +// put a pointer in the struct. That is not a small thing here: a token slice +// is the largest single allocation a parse makes, and one holding pointers +// costs a write barrier per token to build and has to be walked by the +// garbage collector afterwards. Deriving the text instead makes Token 24 +// bytes rather than 40 and the slice invisible to the collector, which is +// worth about 2.7x on the cost of producing one. type Token struct { - Kind Kind `json:"kind"` - Text string `json:"text"` - Pos int `json:"pos"` - End int `json:"end"` + Kind Kind `json:"kind"` + Pos int `json:"pos"` + End int `json:"end"` } + +// Text returns the token's spelling, given the input it was lexed from. +func (t Token) Text(src string) string { return src[t.Pos:t.End] } From 8f800dedd6801a7ca5490c2fcf5f10a16cce9c86 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 31 Jul 2026 01:39:29 +0000 Subject: [PATCH 2/2] Give the README a runnable example and links to the reference The usage section was a single call with no imports and no output, which shows the entry point but not what you get back. The example is now a whole program: parse, walk the tree for tables and bind parameters, and handle a rejection. Its output is what it actually prints. Also a pkg.go.dev badge and links to the four packages, and the corpus count in Status, which still said 20,971 from before extraction widened to every script in the pinned tree. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JzBeCg7rjweVW3uGPg5G7T --- README.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 80c3f59..525e5ce 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,83 @@ meyer is being built to replace the ANTLR-generated SQLite parser in [sqlc](https://github.com/sqlc-dev/sqlc). See [PLAN.md](PLAN.md) for the architecture and [CLAUDE.md](CLAUDE.md) for the development workflow. +[![Go Reference](https://pkg.go.dev/badge/github.com/sqlc-dev/meyer.svg)](https://pkg.go.dev/github.com/sqlc-dev/meyer) + ## Usage +```sh +go get github.com/sqlc-dev/meyer +``` + ```go -stmts, err := parser.Parse(ctx, strings.NewReader("SELECT id FROM users WHERE id = ?")) +package main + +import ( + "errors" + "fmt" + + "github.com/sqlc-dev/meyer/ast" + "github.com/sqlc-dev/meyer/parser" +) + +func main() { + const src = `SELECT u.name, count(o.id) FROM users u + JOIN orders o ON o.user_id = u.id + WHERE u.created_at > :since` + + stmts, err := parser.ParseString(src) + if err != nil { + var perr *parser.Error + if errors.As(err, &perr) { + fmt.Printf("%s at byte %d\n", perr.Message, perr.Offset) + } + return + } + + for _, stmt := range stmts { + ast.Walk(stmt, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.TableRef: + if n.Name != nil { + fmt.Println("table:", n.Name.Name.Name) + } + case *ast.BindParam: + fmt.Printf("param: %s at %d:%d\n", n.Raw, n.Pos(), n.End()) + } + return true + }) + } +} ``` -`ParseString` and `ParseStatement` take a string; `ParseExpr` parses a single -expression. A rejected input returns a `*parser.Error` carrying SQLite's -exact message and the byte offset of the fault. +``` +table: users +table: orders +param: :since at 100:106 +``` + +`ParseString` and `ParseStatement` take a string, `Parse` takes an +`io.Reader`, and `ParseExpr` parses a single expression. A rejected input +returns a `*parser.Error` carrying SQLite's exact message and the byte offset +of the fault; its `Error()` renders as `line:column: message`. + +```go +_, err := parser.ParseString("SELECT FROM t") +fmt.Println(err) // 1:8: near "FROM": syntax error +``` Every node embeds `ast.Span`, so `Pos()` and `End()` give byte offsets into the original input — sqlc slices the source with them to find `-- name:` comments and to report errors, so they are load-bearing rather than -diagnostic. +diagnostic. `ast.String` and `ast.Statements` render a tree back to SQL, +which is a re-parseable rendering rather than a formatter. + +The full API is on +[pkg.go.dev](https://pkg.go.dev/github.com/sqlc-dev/meyer): `parser` for the +entry points, [`ast`](https://pkg.go.dev/github.com/sqlc-dev/meyer/ast) for +the node set, and [`lexer`](https://pkg.go.dev/github.com/sqlc-dev/meyer/lexer) +and [`token`](https://pkg.go.dev/github.com/sqlc-dev/meyer/token) if you want +the token stream on its own. To see what the parser did with something: @@ -36,7 +99,7 @@ go run ./cmd/debug-parse -render -f query.sql # re-rendered SQL ## Status The parser covers the whole grammar of the pinned SQLite release and passes -the full corpus: **20,971 of 20,971 cases**, extracted from every test script +the full corpus: **21,326 of 21,326 cases**, extracted from every test script in SQLite's own test suite. Still to come, from [PLAN.md](PLAN.md): the sqlc integration itself, and the