pqarrays

Paddy 2016-02-26 Parent:ce9c92fc81ab

2:9a415db0346a Go to Latest

pqarrays/parser.go

Fix whitespace lexing/parsing. Our consumeWhitespace method had a bug where it wouldn't emit the whitespace token, meaning we weren't actually skipping the whitespace. Oops. While in there, I removed an outdated TODO (we already supplied a default state). I updated the lexing of unquoted strings to take into account the rules about spaces and unquoted strings; basically, spaces before or after are ignored as whitespace, spaces in the middle ocunt, and empty strings are not allowed. I removed an extra case when detecting what to do when lexing an unquoted string; we already consumed all the whitespace, so the next character shouldn't be whitespace, so no need to test for it. We need to consume whitespace before we start lexing the separator character. I updated the token debugging to be a bit more useful, by defining a String() method on the token type itself, so it'll expose both the type and the value. This makes unexpected errors easier to deal with, and is used in all the errors raised by the parser now. I added a bunch of whitespace tests for lexing and parsing.

Download raw file
View source Diff to previous Annotate
History
1 package pqarrays
2
3 import (
4 "errors"
5 )
6
7 func parse(l *lexer) ([]*string, error) {
8 var parsed []*string
9 pchan := make(chan *string)
10 errchan := make(chan error)
11 done := make(chan struct{})
12 go runParse(l, pchan, errchan, done)
13 for {
14 select {
15 case err := <-errchan:
16 return parsed, err
17 case item := <-pchan:
18 parsed = append(parsed, item)
19 case <-done:
20 return parsed, nil
21 }
22 }
23 }
24
25 func runParse(l *lexer, parsed chan *string, err chan error, done chan struct{}) {
26 var state parseFunc = parseStart
27 for {
28 var e error
29 state, e = state(l, parsed)
30 if e != nil {
31 err <- e
32 break
33 }
34 if state == nil {
35 break
36 }
37 }
38 close(done)
39 }
40
41 type parseFunc func(*lexer, chan *string) (parseFunc, error)
42
43 func parseEOF(l *lexer, parsed chan *string) (parseFunc, error) {
44 tok := l.nextToken()
45 if tok.typ == tokenWhitespace {
46 return parseEOF, nil
47 }
48 if tok.typ != tokenEOF {
49 return nil, errors.New("expected EOF, got " + tok.String())
50 }
51 return nil, nil
52 }
53
54 func parseStringOrNull(l *lexer, parsed chan *string) (parseFunc, error) {
55 tok := l.nextToken()
56 if tok.typ == tokenWhitespace {
57 return parseStringOrNull, nil
58 } else if tok.typ == tokenString {
59 parsed <- &tok.val
60 return parseSeparatorOrDelim, nil
61 } else if tok.typ == tokenNull {
62 parsed <- nil
63 return parseSeparatorOrDelim, nil
64 }
65 return nil, errors.New("expected string, got " + tok.String())
66 }
67
68 func parseStringOrNullOrEnd(l *lexer, parsed chan *string) (parseFunc, error) {
69 tok := l.nextToken()
70 if tok.typ == tokenWhitespace {
71 return parseStringOrNullOrEnd, nil
72 } else if tok.typ == tokenString {
73 parsed <- &tok.val
74 return parseSeparatorOrDelim, nil
75 } else if tok.typ == tokenNull {
76 parsed <- nil
77 return parseSeparatorOrDelim, nil
78 } else if tok.typ == tokenArrayEnd {
79 return parseEOF, nil
80 }
81 return nil, errors.New("Expected string or end, got " + tok.String())
82 }
83
84 func parseSeparatorOrDelim(l *lexer, parsed chan *string) (parseFunc, error) {
85 tok := l.nextToken()
86 if tok.typ == tokenWhitespace {
87 return parseSeparatorOrDelim, nil
88 } else if tok.typ == tokenSeparator {
89 return parseStringOrNull, nil
90 } else if tok.typ == tokenArrayEnd {
91 return parseEOF, nil
92 }
93 return nil, errors.New("expected separator or delim, got " + tok.String())
94 }
95
96 func parseStart(l *lexer, parsed chan *string) (parseFunc, error) {
97 tok := l.nextToken()
98 if tok.typ == tokenWhitespace {
99 return parseStart, nil
100 } else if tok.typ == tokenArrayStart {
101 return parseStringOrNullOrEnd, nil
102 }
103 return nil, errors.New("expected separator or delim, got " + tok.String())
104 }