diff --git a/README.md b/README.md index 80c3f59..525e5ce 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,83 @@ meyer is being built to replace the ANTLR-generated SQLite parser in [sqlc](https://github.com/sqlc-dev/sqlc). See [PLAN.md](PLAN.md) for the architecture and [CLAUDE.md](CLAUDE.md) for the development workflow. +[![Go Reference](https://pkg.go.dev/badge/github.com/sqlc-dev/meyer.svg)](https://pkg.go.dev/github.com/sqlc-dev/meyer) + ## Usage +```sh +go get github.com/sqlc-dev/meyer +``` + ```go -stmts, err := parser.Parse(ctx, strings.NewReader("SELECT id FROM users WHERE id = ?")) +package main + +import ( + "errors" + "fmt" + + "github.com/sqlc-dev/meyer/ast" + "github.com/sqlc-dev/meyer/parser" +) + +func main() { + const src = `SELECT u.name, count(o.id) FROM users u + JOIN orders o ON o.user_id = u.id + WHERE u.created_at > :since` + + stmts, err := parser.ParseString(src) + if err != nil { + var perr *parser.Error + if errors.As(err, &perr) { + fmt.Printf("%s at byte %d\n", perr.Message, perr.Offset) + } + return + } + + for _, stmt := range stmts { + ast.Walk(stmt, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.TableRef: + if n.Name != nil { + fmt.Println("table:", n.Name.Name.Name) + } + case *ast.BindParam: + fmt.Printf("param: %s at %d:%d\n", n.Raw, n.Pos(), n.End()) + } + return true + }) + } +} ``` -`ParseString` and `ParseStatement` take a string; `ParseExpr` parses a single -expression. A rejected input returns a `*parser.Error` carrying SQLite's -exact message and the byte offset of the fault. +``` +table: users +table: orders +param: :since at 100:106 +``` + +`ParseString` and `ParseStatement` take a string, `Parse` takes an +`io.Reader`, and `ParseExpr` parses a single expression. A rejected input +returns a `*parser.Error` carrying SQLite's exact message and the byte offset +of the fault; its `Error()` renders as `line:column: message`. + +```go +_, err := parser.ParseString("SELECT FROM t") +fmt.Println(err) // 1:8: near "FROM": syntax error +``` Every node embeds `ast.Span`, so `Pos()` and `End()` give byte offsets into the original input — sqlc slices the source with them to find `-- name:` comments and to report errors, so they are load-bearing rather than -diagnostic. +diagnostic. `ast.String` and `ast.Statements` render a tree back to SQL, +which is a re-parseable rendering rather than a formatter. + +The full API is on +[pkg.go.dev](https://pkg.go.dev/github.com/sqlc-dev/meyer): `parser` for the +entry points, [`ast`](https://pkg.go.dev/github.com/sqlc-dev/meyer/ast) for +the node set, and [`lexer`](https://pkg.go.dev/github.com/sqlc-dev/meyer/lexer) +and [`token`](https://pkg.go.dev/github.com/sqlc-dev/meyer/token) if you want +the token stream on its own. To see what the parser did with something: @@ -36,7 +99,7 @@ go run ./cmd/debug-parse -render -f query.sql # re-rendered SQL ## Status The parser covers the whole grammar of the pinned SQLite release and passes -the full corpus: **20,971 of 20,971 cases**, extracted from every test script +the full corpus: **21,326 of 21,326 cases**, extracted from every test script in SQLite's own test suite. Still to come, from [PLAN.md](PLAN.md): the sqlc integration itself, and the diff --git a/ast/render.go b/ast/render.go index e53ef96..7de17ed 100644 --- a/ast/render.go +++ b/ast/render.go @@ -16,6 +16,7 @@ import "strings" // String renders n as SQL. func String(n Node) string { var b strings.Builder + b.Grow(sizeHint(n)) write(&b, n) return b.String() } @@ -23,6 +24,11 @@ func String(n Node) string { // Statements renders a whole script, one statement per line. func Statements(stmts []Stmt) string { var b strings.Builder + n := 0 + for _, s := range stmts { + n += sizeHint(s) + 2 // ";\n" + } + b.Grow(n) for _, s := range stmts { write(&b, s) b.WriteString(";\n") @@ -30,6 +36,18 @@ func Statements(stmts []Stmt) string { return b.String() } +// sizeHint estimates the rendered length of n from the source it was parsed +// from, which it is within a few bytes of: rendering rewrites the whitespace +// and nothing else. It saves a strings.Builder the four doublings a short +// statement would otherwise cost. A node built by hand rather than parsed +// has no span, and gets no hint. +func sizeHint(n Node) int { + if size := n.End() - n.Pos(); size > 0 { + return size + } + return 0 +} + func (n *SelectStmt) String() string { return String(n) } func (n *InsertStmt) String() string { return String(n) } func (n *UpdateStmt) String() string { return String(n) } diff --git a/cmd/debug-parse/main.go b/cmd/debug-parse/main.go index 580a101..3d23593 100644 --- a/cmd/debug-parse/main.go +++ b/cmd/debug-parse/main.go @@ -46,7 +46,7 @@ func main() { if *tokens { for _, t := range lexer.Lex(src) { - fmt.Printf("%5d-%-5d %-10s %q\n", t.Pos, t.End, t.Kind, t.Text) + fmt.Printf("%5d-%-5d %-10s %q\n", t.Pos, t.End, t.Kind, t.Text(src)) } return } diff --git a/cmd/difftest/main.go b/cmd/difftest/main.go index 178f784..a79361d 100644 --- a/cmd/difftest/main.go +++ b/cmd/difftest/main.go @@ -240,7 +240,7 @@ func mutations(src string, n int, rng *rand.Rand) []string { case 1: // delete a token add(src[:t.Pos] + src[t.End:]) case 2: // duplicate a token - add(src[:t.End] + " " + t.Text + src[t.End:]) + add(src[:t.End] + " " + t.Text(src) + src[t.End:]) case 3: // replace a token add(src[:t.Pos] + injections[rng.Intn(len(injections))] + src[t.End:]) case 4: // insert a token diff --git a/internal/sqlitesrc/sqlitesrc.go b/internal/sqlitesrc/sqlitesrc.go index 0474566..2acedc1 100644 --- a/internal/sqlitesrc/sqlitesrc.go +++ b/internal/sqlitesrc/sqlitesrc.go @@ -323,7 +323,7 @@ func SplitDiffers(sql string) bool { return true } default: - if strings.Contains(t.Text, ";") && !completeUnderstands(t) { + if text := t.Text(sql); strings.Contains(text, ";") && !completeUnderstands(t, text) { return true } } @@ -337,12 +337,12 @@ func SplitDiffers(sql string) bool { // An unterminated one it does not: in ":v('(%d)',changes());" the tokenizer // swallows the first quote into the bind parameter and finds a string // starting at the second, while sqlite3_complete pairs the two. -func completeUnderstands(t token.Token) bool { +func completeUnderstands(t token.Token, text string) bool { switch t.Kind { case token.STRING, token.BLOB: return true case token.ID: - switch t.Text[0] { + switch text[0] { case '"', '`', '[': return true } diff --git a/lexer/lexer.go b/lexer/lexer.go index f1223be..9f13687 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -23,10 +23,14 @@ import ( // A NUL byte terminates the input, matching SQLite, whose tokenizer walks a // NUL-terminated buffer. func Lex(src string) []token.Token { - // SQL averages a little over four bytes per token including the - // separators, so this usually gets the whole stream in one allocation. - toks := make([]token.Token, 0, len(src)/4+8) + // Across the corpus SQL runs to 3.7 bytes per token including the + // separators, but the mean is the wrong statistic to size from: what + // costs is the tail that has to grow and copy. A divisor of three + // covers 93% of cases in one allocation where four covers 80%, for + // about seventeen tokens of slack apiece. + toks := make([]token.Token, 0, len(src)/3+8) i := 0 + ambiguous := false for i < len(src) { kind, n := scan(src, i) if kind == token.EOF { // NUL byte: end of input @@ -36,11 +40,20 @@ func Lex(src string) []token.Token { n = 1 } if kind != token.SPACE && kind != token.COMMENT { - toks = append(toks, token.Token{Kind: kind, Text: src[i : i+n], Pos: i, End: i + n}) + switch kind { + case token.WINDOW, token.OVER, token.FILTER: + ambiguous = true + } + toks = append(toks, token.Token{Kind: kind, Pos: i, End: i + n}) } i += n } - resolveWindowKeywords(toks) + // Most SQL contains none of the three context-dependent keywords, and + // noting whether any turned up costs a comparison per token against a + // second pass over all of them. + if ambiguous { + resolveWindowKeywords(toks) + } toks = append(toks, token.Token{Kind: token.EOF, Pos: i, End: i}) return toks } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index af2959d..79d86d2 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -27,7 +27,7 @@ func texts(src string) []string { toks := Lex(src) out := make([]string, 0, len(toks)) for _, t := range toks[:len(toks)-1] { - out = append(out, t.Text) + out = append(out, t.Text(src)) } return out } @@ -86,7 +86,7 @@ func TestQuoting(t *testing.T) { } for _, tt := range tests { toks := Lex(tt.src) - if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text != tt.text { + if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text(tt.src) != tt.text { t.Errorf("Lex(%q) = %v, want one %v %q", tt.src, toks[:len(toks)-1], tt.kind, tt.text) } } @@ -182,7 +182,7 @@ func TestVariables(t *testing.T) { } for _, tt := range tests { toks := Lex(tt.src) - if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text != tt.text { + if len(toks) != 2 || toks[0].Kind != tt.kind || toks[0].Text(tt.src) != tt.text { t.Errorf("Lex(%q) = %v, want one %v %q", tt.src, toks[:len(toks)-1], tt.kind, tt.text) } } @@ -274,7 +274,7 @@ func TestWindowKeywordResolution(t *testing.T) { } if toks[tt.at].Kind != tt.want { t.Errorf("Lex(%q)[%d] = %v %q, want %v", - tt.src, tt.at, toks[tt.at].Kind, toks[tt.at].Text, tt.want) + tt.src, tt.at, toks[tt.at].Kind, toks[tt.at].Text(tt.src), tt.want) } } } @@ -286,8 +286,8 @@ func TestSpansCoverTheInput(t *testing.T) { if tk.Kind == token.EOF { continue } - if got := src[tk.Pos:tk.End]; got != tk.Text { - t.Errorf("token %d: span %d:%d is %q, but Text is %q", i, tk.Pos, tk.End, got, tk.Text) + if tk.Pos < 0 || tk.Pos >= tk.End || tk.End > len(src) { + t.Errorf("token %d: span %d:%d is not a range within %d bytes", i, tk.Pos, tk.End, len(src)) } if i > 0 && tk.Pos < toks[i-1].End { t.Errorf("token %d starts at %d, before the previous token ended at %d", diff --git a/parser/parse_ddl.go b/parser/parse_ddl.go index c9b550e..0a1e6e5 100644 --- a/parser/parse_ddl.go +++ b/parser/parse_ddl.go @@ -288,7 +288,7 @@ func (p *parser) parseDefaultValue() ast.Expr { if !token.IsID(p.cur().Kind) { p.syntaxError() } - return identOf(p.advance()) + return p.identOf(p.advance()) } // atTerm reports whether the lookahead begins a "term": a literal constant. @@ -312,7 +312,7 @@ func (p *parser) parseGenerated() (ast.Expr, *ast.Ident) { x := p.parseExpr(precLowest) p.expect(token.RP) if token.IsPlainID(p.cur().Kind) { - return x, identOf(p.advance()) + return x, p.identOf(p.advance()) } return x, nil } @@ -381,7 +381,7 @@ func (p *parser) parseReferences() *ast.ForeignKeyClause { arg := &ast.ForeignKeyArg{} switch p.cur().Kind { case token.INSERT, token.DELETE, token.UPDATE: - arg.Event = p.advance().Text + arg.Event = p.text(p.advance()) default: p.syntaxError() } diff --git a/parser/parse_expr.go b/parser/parse_expr.go index f3cbf22..5df2541 100644 --- a/parser/parse_expr.go +++ b/parser/parse_expr.go @@ -129,7 +129,7 @@ func (p *parser) parseOperators(x ast.Expr, min int, stopAtAnd bool) ast.Expr { } // expr ::= expr PTR expr. The "->" and "->>" spellings share a // token; only the text tells them apart. - if t.Kind == token.PTR && t.Text == "->>" { + if t.Kind == token.PTR && p.text(t) == "->>" { e.op = ast.OpPtr2 } x = p.binary(x, e.op, e.prec) @@ -257,7 +257,7 @@ func (p *parser) parseIs(x ast.Expr) ast.Expr { // expr ::= expr likeop expr. [LIKE_KW] // expr ::= expr likeop expr ESCAPE expr. [LIKE_KW] func (p *parser) parseLike(x ast.Expr, not bool) ast.Expr { - op := identOf(p.advance()) + op := p.identOf(p.advance()) n := &ast.LikeExpr{Op: op, Not: not, X: x} n.Y = p.parseExpr(precCompare + 1) if p.at(token.ESCAPE) { @@ -392,7 +392,7 @@ func (p *parser) parsePrimary() ast.Expr { return p.parseQualifiedRef() } p.advance() - return identOf(t) + return p.identOf(t) } p.syntaxError() return nil @@ -411,21 +411,22 @@ func (p *parser) parseTerm() ast.Expr { switch t.Kind { case token.NULL: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitNull, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitNull, p.text(t)) case token.INTEGER: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitInteger, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitInteger, p.text(t)) case token.FLOAT: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitFloat, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitFloat, p.text(t)) case token.BLOB: p.advance() - return &ast.Literal{Span: spanOf(t), Kind: ast.LitBlob, Value: t.Text, Raw: t.Text} + return literalOf(t, ast.LitBlob, p.text(t)) case token.STRING: p.advance() + raw := p.text(t) return &ast.Literal{ Span: spanOf(t), Kind: ast.LitString, - Value: dequote(t.Text, '\''), Raw: t.Text, + Value: dequote(raw, '\''), Raw: raw, } case token.QNUMBER: // The grammar action dequotes the digit separators and rejects a @@ -436,19 +437,20 @@ func (p *parser) parseTerm() ast.Expr { if strings.ContainsAny(value, ".eE") && !strings.HasPrefix(strings.ToLower(value), "0x") { kind = ast.LitFloat } - return &ast.Literal{Span: spanOf(t), Kind: kind, Value: value, Raw: t.Text} + return &ast.Literal{Span: spanOf(t), Kind: kind, Value: value, Raw: p.text(t)} case token.CTIME_KW: // CTIME_KW falls back to ID elsewhere, but here it has a shift // action, so it is always the keyword. p.advance() + raw := p.text(t) kind := ast.LitCurrentTimestamp - switch strings.ToUpper(t.Text) { + switch strings.ToUpper(raw) { case "CURRENT_DATE": kind = ast.LitCurrentDate case "CURRENT_TIME": kind = ast.LitCurrentTime } - return &ast.Literal{Span: spanOf(t), Kind: kind, Value: t.Text, Raw: t.Text} + return literalOf(t, kind, raw) } p.syntaxError() return nil @@ -482,7 +484,10 @@ func (p *parser) parseParenExpr() ast.Expr { // // expr ::= nm DOT nm. / expr ::= nm DOT nm DOT nm. func (p *parser) parseQualifiedRef() ast.Expr { - parts := []*ast.Ident{p.expectName()} + // A reference is db.table.column at most, so the slice is sized once + // rather than grown twice. + parts := make([]*ast.Ident, 1, 3) + parts[0] = p.expectName() p.expect(token.DOT) parts = append(parts, p.expectName()) if p.at(token.DOT) { @@ -536,7 +541,7 @@ func (p *parser) parseRaise() ast.Expr { n.Action = "IGNORE" p.advance() case token.ROLLBACK, token.ABORT, token.FAIL: - n.Action = strings.ToUpper(p.advance().Text) + n.Action = strings.ToUpper(p.text(p.advance())) p.expect(token.COMMA) n.Message = p.parseExpr(precLowest) default: @@ -554,7 +559,7 @@ func (p *parser) parseRaise() ast.Expr { // expr ::= idj LP distinct exprlist ORDER BY sortlist RP filter_over. // expr ::= idj LP STAR RP filter_over. func (p *parser) parseFuncCall() ast.Expr { - name := identOf(p.advance()) + name := p.identOf(p.advance()) n := &ast.FuncCall{Name: name} p.expect(token.LP) if p.at(token.STAR) { @@ -627,11 +632,11 @@ func (p *parser) parseTypeToken() *ast.TypeName { start := p.cur().Pos // A type is almost always one word ("INTEGER", "TEXT"); only build a // slice for the "UNSIGNED BIG INT" kind. - name := p.advance().Text + name := p.text(p.advance()) if token.IsIDS(p.cur().Kind) { words := []string{name} for token.IsIDS(p.cur().Kind) { - words = append(words, p.advance().Text) + words = append(words, p.text(p.advance())) } name = strings.Join(words, " ") } @@ -658,12 +663,12 @@ func (p *parser) parseTypeToken() *ast.TypeName { func (p *parser) parseSignedNumber() string { var sign string if p.at(token.PLUS) || p.at(token.MINUS) { - sign = p.advance().Text + sign = p.text(p.advance()) } if !p.at(token.INTEGER) && !p.at(token.FLOAT) { p.syntaxError() } - return sign + p.advance().Text + return sign + p.text(p.advance()) } // bindParam builds a BindParam and assigns its number the way @@ -679,18 +684,19 @@ func (p *parser) bindParam(t token.Token) ast.Expr { // "SELECT #1 #1" is reported at the second reference, not the first. // Deferring the error reproduces that, since a real syntax error later // in the statement aborts the parse and wins on its own. - if len(t.Text) >= 2 && t.Text[0] == '#' && t.Text[1] >= '0' && t.Text[1] <= '9' { - p.defer_(`near "`+t.Text+`": syntax error`, t.Pos) + raw := p.text(t) + if len(raw) >= 2 && raw[0] == '#' && raw[1] >= '0' && raw[1] <= '9' { + p.defer_(`near "`+raw+`": syntax error`, t.Pos) } - n := &ast.BindParam{Span: spanOf(t), Raw: t.Text} + n := &ast.BindParam{Span: spanOf(t), Raw: raw} switch { - case t.Text == "?": + case raw == "?": n.Kind = ast.ParamAnon p.nVar++ n.Number = p.nVar - case t.Text[0] == '?': + case raw[0] == '?': n.Kind = ast.ParamNumber - v, err := strconv.Atoi(t.Text[1:]) + v, err := strconv.Atoi(raw[1:]) if err == nil { n.Number = v if v > p.nVar { @@ -698,7 +704,7 @@ func (p *parser) bindParam(t token.Token) ast.Expr { } } default: - switch t.Text[0] { + switch raw[0] { case ':': n.Kind = ast.ParamColon case '@': @@ -706,8 +712,8 @@ func (p *parser) bindParam(t token.Token) ast.Expr { default: n.Kind = ast.ParamDollar } - n.Name = t.Text[1:] - if v, ok := p.varNums[t.Text]; ok { + n.Name = raw[1:] + if v, ok := p.varNums[raw]; ok { n.Number = v } else { p.nVar++ @@ -715,7 +721,7 @@ func (p *parser) bindParam(t token.Token) ast.Expr { if p.varNums == nil { p.varNums = make(map[string]int) } - p.varNums[t.Text] = p.nVar + p.varNums[raw] = p.nVar } } return n @@ -726,7 +732,7 @@ func (p *parser) bindParam(t token.Token) ast.Expr { // two digits. It reproduces sqlite3DequoteNumber, including its habit of // reporting the partially rewritten token rather than the original. func (p *parser) dequoteNumber(t token.Token) string { - in := []byte(t.Text) + in := []byte(p.text(t)) buf := append([]byte(nil), in...) bHex := len(in) > 1 && in[0] == '0' && (in[1] == 'x' || in[1] == 'X') isok := func(c byte) bool { @@ -754,3 +760,8 @@ func (p *parser) dequoteNumber(t token.Token) string { func isHexDigit(c byte) bool { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') } + +// literalOf builds the literals whose value is their own spelling. +func literalOf(t token.Token, kind ast.LiteralKind, raw string) *ast.Literal { + return &ast.Literal{Span: spanOf(t), Kind: kind, Value: raw, Raw: raw} +} diff --git a/parser/parse_misc.go b/parser/parse_misc.go index d2a0f81..c38be29 100644 --- a/parser/parse_misc.go +++ b/parser/parse_misc.go @@ -49,7 +49,7 @@ func (p *parser) parseTransOpt() (bool, *ast.Ident) { // cmd ::= COMMIT|END trans_opt. func (p *parser) parseCommit() ast.Stmt { t := p.advance() - n := &ast.CommitStmt{Keyword: strings.ToUpper(t.Text)} + n := &ast.CommitStmt{Keyword: strings.ToUpper(p.text(t))} n.HasTransaction, n.Name = p.parseTransOpt() n.Span = p.span(t.Pos) return n @@ -151,7 +151,7 @@ func (p *parser) parsePragmaValue() string { case token.PLUS, token.MINUS: return p.parseSignedNumber() case token.INTEGER, token.FLOAT, token.ON, token.DELETE, token.DEFAULT: - return p.advance().Text + return p.text(p.advance()) } return p.expectName().Raw } diff --git a/parser/parse_select.go b/parser/parse_select.go index be3e49b..7056929 100644 --- a/parser/parse_select.go +++ b/parser/parse_select.go @@ -73,7 +73,7 @@ func (p *parser) parseSelectNoWith() *ast.SelectStmt { } case token.EXCEPT, token.INTERSECT: p.advance() - op = ast.CompoundOp{Op: strings.ToUpper(t.Text)} + op = ast.CompoundOp{Op: strings.ToUpper(p.text(t))} default: n.Span = ast.Span{Start: start, Stop: p.prevEnd()} return n @@ -193,7 +193,7 @@ func (p *parser) parseAlias() (*ast.Ident, bool) { return p.expectName(), true } if token.IsIDS(p.cur().Kind) { - return identOf(p.advance()), false + return p.identOf(p.advance()), false } return nil, false } @@ -225,9 +225,9 @@ func (p *parser) tryJoinOperator() *ast.JoinOperator { return &ast.JoinOperator{Span: spanOf(t), Type: ast.JoinInner | ast.JoinComma} case token.JOIN: p.advance() - return &ast.JoinOperator{Span: spanOf(t), Type: ast.JoinInner, Words: []string{t.Text}} + return &ast.JoinOperator{Span: spanOf(t), Type: ast.JoinInner, Words: []string{p.text(t)}} case token.JOIN_KW: - words := []string{p.advance().Text} + words := []string{p.text(p.advance())} for i := 0; i < 2 && !p.at(token.JOIN); i++ { words = append(words, p.expectName().Raw) } diff --git a/parser/parse_trigger.go b/parser/parse_trigger.go index 85a3802..28dcdd8 100644 --- a/parser/parse_trigger.go +++ b/parser/parse_trigger.go @@ -35,9 +35,9 @@ func (p *parser) parseCreateTrigger(start int, temp bool) ast.Stmt { } switch p.cur().Kind { case token.DELETE, token.INSERT: - n.Event = strings.ToUpper(p.advance().Text) + n.Event = strings.ToUpper(p.text(p.advance())) case token.UPDATE: - n.Event = strings.ToUpper(p.advance().Text) + n.Event = strings.ToUpper(p.text(p.advance())) if p.accept(token.OF) { n.UpdateOf = p.parseIDList() } diff --git a/parser/parser.go b/parser/parser.go index c05c8f3..db19469 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -241,7 +241,7 @@ func (p *parser) expect(k token.Kind) token.Token { // same position. func (p *parser) checkIllegal() { if t := p.toks[p.i]; t.Kind == token.ILLEGAL { - p.fail(`unrecognized token: "`+t.Text+`"`, t.Pos) + p.fail(`unrecognized token: "`+p.text(t)+`"`, t.Pos) } } @@ -259,7 +259,7 @@ func (p *parser) syntaxErrorAt(t token.Token) { // whose text is empty, which %syntax_error reports this way. p.fail("incomplete input", t.Pos) } - p.fail(`near "`+t.Text+`": syntax error`, t.Pos) + p.fail(`near "`+p.text(t)+`": syntax error`, t.Pos) } // fail aborts the parse with one of SQLite's messages. @@ -400,7 +400,7 @@ func (p *parser) expectName() *ast.Ident { if !token.IsName(p.cur().Kind) { p.syntaxError() } - return identOf(p.advance()) + return p.identOf(p.advance()) } // expectIDS implements the "ids" class: ID | STRING. @@ -408,21 +408,22 @@ func (p *parser) expectIDS() *ast.Ident { if !token.IsIDS(p.cur().Kind) { p.syntaxError() } - return identOf(p.advance()) + return p.identOf(p.advance()) } // identOf builds an Ident from an accepted name token, undoing SQLite's four // quoting styles. -func identOf(t token.Token) *ast.Ident { - id := &ast.Ident{Span: spanOf(t), Raw: t.Text, Name: t.Text} - if len(t.Text) >= 2 { - switch t.Text[0] { +func (p *parser) identOf(t token.Token) *ast.Ident { + raw := p.text(t) + id := &ast.Ident{Span: spanOf(t), Raw: raw, Name: raw} + if len(raw) >= 2 { + switch raw[0] { case '"', '\'', '`': - id.Quote = ast.QuoteStyle(t.Text[0]) - id.Name = dequote(t.Text, t.Text[0]) + id.Quote = ast.QuoteStyle(raw[0]) + id.Name = dequote(raw, raw[0]) case '[': id.Quote = ast.QuoteSquare - id.Name = t.Text[1 : len(t.Text)-1] + id.Name = raw[1 : len(raw)-1] } } return id @@ -435,10 +436,13 @@ func dequote(s string, delim byte) string { if n := len(body); n > 0 && body[n-1] == delim { body = body[:n-1] } - d := string([]byte{delim}) - if !strings.Contains(body, d) { + // The overwhelmingly common case is a name with no escaped delimiter in + // it, and that one must not copy or allocate: the body is a slice of the + // input already. + if strings.IndexByte(body, delim) < 0 { return body } + d := string([]byte{delim}) return strings.ReplaceAll(body, d+d, d) } @@ -471,3 +475,7 @@ func (p *parser) xfullname() (*ast.QualifiedName, *ast.Ident) { } return name, alias } + +// text is the spelling of t. The lexer records only the byte range, so this +// is the one place that turns a token back into the source it came from. +func (p *parser) text(t token.Token) string { return p.src[t.Pos:t.End] } diff --git a/token/keywords.go b/token/keywords.go index db3ebf9..b633f69 100644 --- a/token/keywords.go +++ b/token/keywords.go @@ -1,5 +1,10 @@ package token +import ( + "slices" + "strings" +) + // keywords maps the upper-case spelling of every SQLite keyword to its token // kind. Transcribed from aKeywordTable[] in tool/mkkeywordhash.c, resolved // for the feature set of the pinned build (see parser/testdata/README.md): @@ -212,31 +217,93 @@ var fallback = func() [KindCount]bool { return t }() -// maxKeywordLen is the length of CURRENT_TIMESTAMP, the longest keyword. -const maxKeywordLen = 17 +// The lengths of BY and of CURRENT_TIMESTAMP, the shortest and longest +// keywords. +const ( + minKeywordLen = 2 + maxKeywordLen = 17 +) + +// keywordRow is one row of the shape-indexed table Lookup searches. +type keywordRow struct { + name string + kind Kind +} + +// keywordTable holds every keyword sorted by length and then alphabetically, +// so that all the keywords sharing a length and a first letter are adjacent. +// keywordShape indexes it by exactly those two things, giving the offset and +// count of the run an identifier of that shape has to be compared against. +// +// Length and first letter are free to compute and between them they are very +// nearly a perfect hash -- 147 keywords fall into 93 groups, the largest of +// four. That beats hashing the spelling: keywordCode() in SQLite computes one +// from the first byte, the last byte and the length, but it takes the +// remainder mod a prime to mix them, and a parser that reads a keyword per +// few bytes of input should not be doing a division. +var ( + keywordTable []keywordRow + keywordShape [maxKeywordLen + 1][26]struct{ lo, n uint8 } +) + +func init() { + names := make([]string, 0, len(keywords)) + for name := range keywords { + names = append(names, name) + } + slices.SortFunc(names, func(a, b string) int { + if d := len(a) - len(b); d != 0 { + return d + } + return strings.Compare(a, b) + }) + keywordTable = make([]keywordRow, len(names)) + for i, name := range names { + keywordTable[i] = keywordRow{name, keywords[name]} + g := &keywordShape[len(name)][name[0]-'A'] + if g.n == 0 { + g.lo = uint8(i) + } + g.n++ + } +} // Lookup returns the keyword kind for an identifier spelling, or ID if it is // not a keyword. Matching is ASCII case-insensitive, as in keywordCode(). // -// The upper-casing is done into a stack array rather than with -// strings.ToUpper, because this runs once per identifier token and the -// compiler turns m[string(b[:n])] into an allocation-free lookup. +// This runs once per identifier token, so it neither allocates nor copies: +// the candidates are found from the length and first letter alone, and each +// is compared against the input a byte at a time, upper-casing as it goes. func Lookup(name string) Kind { - if len(name) > maxKeywordLen { + if len(name) < minKeywordLen || len(name) > maxKeywordLen { return ID } - var buf [maxKeywordLen]byte + c := name[0] &^ 0x20 // upper-case, for a letter + if c < 'A' || c > 'Z' { + return ID + } + g := keywordShape[len(name)][c-'A'] + for _, e := range keywordTable[g.lo : g.lo+g.n] { + if equalUpper(name, e.name) { + return e.kind + } + } + return ID +} + +// equalUpper reports whether name upper-cased equals kw, which is already +// upper case and the same length. +func equalUpper(name, kw string) bool { for i := 0; i < len(name); i++ { c := name[i] if c >= 'a' && c <= 'z' { c -= 'a' - 'A' } - buf[i] = c - } - if k, ok := keywords[string(buf[:len(name)])]; ok { - return k + if c != kw[i] { + return false + } } - return ID + return true } // CanFallback reports whether k is in SQLite's %fallback ID set. diff --git a/token/keywords_test.go b/token/keywords_test.go index 2115677..5432cd2 100644 --- a/token/keywords_test.go +++ b/token/keywords_test.go @@ -117,3 +117,41 @@ func TestFallbackSetMatchesUpstream(t *testing.T) { } } } + +// TestLookup exercises the shape index directly. The corpus reaches it too, +// but only through whatever spellings SQLite's test suite happens to use: a +// group that the length-and-first-letter index pointed at the wrong run of +// the table would go unnoticed for any keyword that suite never writes. +func TestLookup(t *testing.T) { + for name, want := range keywords { + for _, spelling := range []string{name, strings.ToLower(name), mixedCase(name)} { + if got := Lookup(spelling); got != want { + t.Errorf("Lookup(%q) = %v, want %v", spelling, got, want) + } + } + } + // Non-keywords, including the shapes the index has to reject before it + // can subscript anything: too short, too long, and not starting with a + // letter. + for _, name := range []string{ + "", "a", "_", "1", "zz", "users", "created_at", "selects", "seleect", + "_select", "1select", "current_timestampx", "CURRENT_TIMESTAMPX", + "\xff\xfe", "{", "|abc", + } { + if got := Lookup(name); got != ID { + t.Errorf("Lookup(%q) = %v, want ID", name, got) + } + } +} + +// mixedCase upper-cases the even-numbered bytes of an upper-case word and +// lower-cases the rest. +func mixedCase(s string) string { + b := []byte(strings.ToLower(s)) + for i := 0; i < len(b); i += 2 { + if b[i] >= 'a' && b[i] <= 'z' { + b[i] -= 'a' - 'A' + } + } + return string(b) +} diff --git a/token/token.go b/token/token.go index b36faaa..b102ef3 100644 --- a/token/token.go +++ b/token/token.go @@ -219,11 +219,22 @@ func (k Kind) String() string { return "Kind(?)" } -// Token is one lexical token. Pos and End are byte offsets into the input -// the token was lexed from; Text is the corresponding source slice. +// Token is one lexical token: a kind and the half-open byte range [Pos, End) +// of the input it was lexed from. +// +// The spelling is deliberately not a field. It is exactly src[Pos:End], so +// storing it would duplicate what the offsets already say -- and it would +// put a pointer in the struct. That is not a small thing here: a token slice +// is the largest single allocation a parse makes, and one holding pointers +// costs a write barrier per token to build and has to be walked by the +// garbage collector afterwards. Deriving the text instead makes Token 24 +// bytes rather than 40 and the slice invisible to the collector, which is +// worth about 2.7x on the cost of producing one. type Token struct { - Kind Kind `json:"kind"` - Text string `json:"text"` - Pos int `json:"pos"` - End int `json:"end"` + Kind Kind `json:"kind"` + Pos int `json:"pos"` + End int `json:"end"` } + +// Text returns the token's spelling, given the input it was lexed from. +func (t Token) Text(src string) string { return src[t.Pos:t.End] }