pqarrays

Paddy 2016-02-25 Parent:bfe2a4af6bdf Child:9a415db0346a

1:ce9c92fc81ab Go to Latest

pqarrays/lexer.go

Fix bug parsing empty arrays, make golint and go vet happy. Add comments to make golint happy. Also, because comments are a good thing to have. Turn += 1 and -= 1 into ++ and --, respectively, so golint will be happy. Fix an improperly formated errorf, where a rune was being treated as a string. Thanks, go vet! Fix whitespace parsing, returning the parse functions again instead of just skipping the one character. Now if we have more than one whitespace character in a row, they'll all be skipped. Add a parseStringOrNullOrEnd parse function that will be called after the tokenArrayStart character, to fix a bug where empty arrays were expecting a string or null and getting the array end character. This is only valid after tokenArrayStart, however; in other places where parseSeparatorOrDelim is used, it wouldn't be appropriate. Add a parser test for an empty array.

Download raw file
View source Diff to previous Annotate
History
paddy@0 1 package pqarrays
paddy@0 2
paddy@0 3 import (
paddy@0 4 "fmt"
paddy@0 5 "strings"
paddy@0 6 "unicode"
paddy@0 7 "unicode/utf8"
paddy@0 8 )
paddy@0 9
paddy@0 10 const (
paddy@0 11 eof = -1
paddy@0 12 leftDelim = "{"
paddy@0 13 rightDelim = "}"
paddy@0 14 separator = ','
paddy@0 15 )
paddy@0 16
paddy@0 17 type tokenType int
paddy@0 18
paddy@0 19 const (
paddy@0 20 tokenError tokenType = iota
paddy@0 21 tokenWhitespace
paddy@0 22 tokenArrayStart
paddy@0 23 tokenString
paddy@0 24 tokenNull
paddy@0 25 tokenSeparator
paddy@0 26 tokenArrayEnd
paddy@0 27 tokenEOF
paddy@0 28 )
paddy@0 29
paddy@0 30 func (t tokenType) String() string {
paddy@0 31 switch t {
paddy@0 32 case tokenError:
paddy@0 33 return "error"
paddy@0 34 case tokenWhitespace:
paddy@0 35 return "whitespace"
paddy@0 36 case tokenArrayStart:
paddy@0 37 return "array start"
paddy@0 38 case tokenString:
paddy@0 39 return "string"
paddy@0 40 case tokenNull:
paddy@0 41 return "null"
paddy@0 42 case tokenSeparator:
paddy@0 43 return "separator"
paddy@0 44 case tokenArrayEnd:
paddy@0 45 return "array end"
paddy@0 46 case tokenEOF:
paddy@0 47 return "eof"
paddy@0 48 default:
paddy@0 49 return "unknown token"
paddy@0 50 }
paddy@0 51 }
paddy@0 52
paddy@0 53 type stateFunc func(*lexer) stateFunc
paddy@0 54
paddy@0 55 type lexer struct {
paddy@0 56 tokens chan token
paddy@0 57 input string
paddy@0 58 start int
paddy@0 59 pos int
paddy@0 60 omitted []int
paddy@0 61 width int
paddy@0 62 state stateFunc
paddy@0 63 arrayDepth int
paddy@0 64 }
paddy@0 65
paddy@0 66 type token struct {
paddy@0 67 typ tokenType
paddy@0 68 val string
paddy@0 69 }
paddy@0 70
paddy@0 71 func lex(input string) *lexer {
paddy@0 72 l := &lexer{
paddy@0 73 input: input,
paddy@0 74 tokens: make(chan token),
paddy@0 75 }
paddy@0 76 go l.run()
paddy@0 77 return l
paddy@0 78 }
paddy@0 79
paddy@0 80 func (l *lexer) nextToken() token {
paddy@0 81 return <-l.tokens
paddy@0 82 }
paddy@0 83
paddy@0 84 func (l *lexer) run() {
paddy@0 85 for l.state = lexStart; l.state != nil; { // TODO(paddy): default state
paddy@0 86 l.state = l.state(l)
paddy@0 87 }
paddy@0 88 }
paddy@0 89
paddy@0 90 func (l *lexer) emit(t tokenType) {
paddy@0 91 var val string
paddy@0 92 if len(l.omitted) < 1 {
paddy@0 93 val = l.input[l.start:l.pos]
paddy@0 94 } else {
paddy@0 95 start := l.start
paddy@0 96 for _, pos := range l.omitted {
paddy@0 97 val += l.input[start:pos]
paddy@0 98 start = pos + 1
paddy@0 99 }
paddy@0 100 if l.pos > start {
paddy@0 101 val += l.input[start:l.pos]
paddy@0 102 }
paddy@0 103 }
paddy@0 104 l.tokens <- token{typ: t, val: val}
paddy@0 105 l.start = l.pos
paddy@0 106 l.omitted = l.omitted[0:0]
paddy@0 107 }
paddy@0 108
paddy@0 109 func (l *lexer) next() rune {
paddy@0 110 if l.pos >= len(l.input) {
paddy@0 111 l.width = 0
paddy@0 112 return eof
paddy@0 113 }
paddy@0 114 var r rune
paddy@0 115 r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
paddy@0 116 l.pos += l.width
paddy@0 117 return r
paddy@0 118 }
paddy@0 119
paddy@0 120 func (l *lexer) omit() {
paddy@0 121 l.omitted = append(l.omitted, l.pos-1)
paddy@0 122 }
paddy@0 123
paddy@0 124 func (l *lexer) ignore() {
paddy@0 125 l.start = l.pos
paddy@0 126 }
paddy@0 127
paddy@0 128 func (l *lexer) backup() {
paddy@0 129 l.pos -= l.width
paddy@0 130 }
paddy@0 131
paddy@0 132 func (l *lexer) peek() rune {
paddy@0 133 r := l.next()
paddy@0 134 l.backup()
paddy@0 135 return r
paddy@0 136 }
paddy@0 137
paddy@0 138 func (l *lexer) accept(valid string) bool {
paddy@0 139 if strings.IndexRune(valid, l.next()) >= 0 {
paddy@0 140 return true
paddy@0 141 }
paddy@0 142 l.backup()
paddy@0 143 return false
paddy@0 144 }
paddy@0 145
paddy@0 146 func (l *lexer) acceptRun(valid string) {
paddy@0 147 for strings.IndexRune(valid, l.next()) >= 0 {
paddy@0 148 }
paddy@0 149 l.backup()
paddy@0 150 }
paddy@0 151
paddy@0 152 func (l *lexer) errorf(format string, args ...interface{}) stateFunc {
paddy@0 153 l.tokens <- token{tokenError, fmt.Sprintf(format, args...)}
paddy@0 154 return nil
paddy@0 155 }
paddy@0 156
paddy@0 157 func (l *lexer) consumeWhitespace() {
paddy@0 158 for unicode.IsSpace(l.peek()) {
paddy@0 159 l.next()
paddy@0 160 }
paddy@0 161 if l.start > l.pos {
paddy@0 162 l.emit(tokenWhitespace)
paddy@0 163 }
paddy@0 164 }
paddy@0 165
paddy@0 166 func lexStart(l *lexer) stateFunc {
paddy@0 167 l.consumeWhitespace()
paddy@0 168 return lexArrayStart
paddy@0 169 }
paddy@0 170
paddy@0 171 func lexArrayStart(l *lexer) stateFunc {
paddy@0 172 if strings.HasPrefix(l.input[l.pos:], leftDelim) {
paddy@0 173 return lexLeftDelim
paddy@0 174 }
paddy@0 175 return l.errorf("expected array to start before %s", l.input[l.pos:])
paddy@0 176 }
paddy@0 177
paddy@0 178 func lexLeftDelim(l *lexer) stateFunc {
paddy@0 179 l.pos += len(leftDelim)
paddy@0 180 l.emit(tokenArrayStart)
paddy@1 181 l.arrayDepth++
paddy@0 182 return lexItem
paddy@0 183 }
paddy@0 184
paddy@0 185 func lexRightDelim(l *lexer) stateFunc {
paddy@0 186 l.pos += len(rightDelim)
paddy@0 187 l.emit(tokenArrayEnd)
paddy@1 188 l.arrayDepth--
paddy@0 189 return lexSeparator
paddy@0 190 }
paddy@0 191
paddy@0 192 func lexItem(l *lexer) stateFunc {
paddy@0 193 l.consumeWhitespace()
paddy@0 194 if strings.HasPrefix(l.input[l.pos:], rightDelim) {
paddy@0 195 return lexRightDelim
paddy@0 196 }
paddy@0 197 if strings.HasPrefix(l.input[l.pos:], leftDelim) {
paddy@0 198 return lexLeftDelim
paddy@0 199 }
paddy@0 200 switch r := l.peek(); {
paddy@0 201 case r == eof:
paddy@0 202 return l.errorf("unclosed array")
paddy@0 203 case r == separator:
paddy@0 204 return l.errorf("empty item in array")
paddy@0 205 case unicode.IsSpace(r):
paddy@0 206 l.consumeWhitespace()
paddy@0 207 return lexItem
paddy@0 208 case r == '"':
paddy@0 209 return lexQuotedString
paddy@0 210 default:
paddy@0 211 return lexString
paddy@0 212 }
paddy@0 213 }
paddy@0 214
paddy@0 215 func lexQuotedString(l *lexer) stateFunc {
paddy@0 216 l.next()
paddy@0 217 l.ignore() // ignore the open quote
paddy@0 218 for {
paddy@0 219 switch r := l.next(); {
paddy@0 220 case r == eof:
paddy@0 221 return l.errorf("unclosed quoted string")
paddy@0 222 case r == '"':
paddy@0 223 l.backup()
paddy@0 224 l.emit(tokenString)
paddy@0 225 l.next()
paddy@0 226 l.ignore()
paddy@0 227 return lexSeparator
paddy@0 228 case r == '\\':
paddy@0 229 // omit the \ itself
paddy@0 230 l.omit()
paddy@0 231 // always skip over the character following a \
paddy@0 232 l.next()
paddy@0 233 if r == eof {
paddy@0 234 return l.errorf("unclosed quoted string")
paddy@0 235 }
paddy@0 236 }
paddy@0 237 }
paddy@0 238 }
paddy@0 239
paddy@0 240 func lexString(l *lexer) stateFunc {
paddy@0 241 for {
paddy@0 242 if strings.HasPrefix(l.input[l.pos:], leftDelim) {
paddy@0 243 return l.errorf(leftDelim + " in unquoted string")
paddy@0 244 }
paddy@0 245 if strings.HasPrefix(l.input[l.pos:], rightDelim) {
paddy@0 246 if l.pos <= l.start {
paddy@0 247 return l.errorf(rightDelim + " in unquoted string")
paddy@0 248 }
paddy@0 249 if string(l.input[l.start:l.pos]) == "NULL" {
paddy@0 250 l.emit(tokenNull)
paddy@0 251 } else {
paddy@0 252 l.emit(tokenString)
paddy@0 253 }
paddy@0 254 return lexRightDelim
paddy@0 255 }
paddy@0 256 switch r := l.next(); {
paddy@0 257 case r == eof:
paddy@0 258 return l.errorf("eof while parsing string")
paddy@0 259 case r == '"':
paddy@0 260 return l.errorf("\" in unquoted string")
paddy@0 261 case unicode.IsSpace(r):
paddy@0 262 return l.errorf("unquoted empty string")
paddy@0 263 case r == '\\':
paddy@0 264 return l.errorf("\\ in unquoted string")
paddy@0 265 case r == separator:
paddy@0 266 l.backup()
paddy@0 267 if l.pos <= l.start {
paddy@0 268 return l.errorf("unquoted empty string")
paddy@0 269 }
paddy@0 270 if string(l.input[l.start:l.pos]) == "NULL" {
paddy@0 271 l.emit(tokenNull)
paddy@0 272 } else {
paddy@0 273 l.emit(tokenString)
paddy@0 274 }
paddy@0 275 return lexSeparator
paddy@0 276 }
paddy@0 277 }
paddy@0 278 }
paddy@0 279
paddy@0 280 func lexSeparator(l *lexer) stateFunc {
paddy@0 281 if strings.HasPrefix(l.input[l.pos:], rightDelim) {
paddy@0 282 return lexRightDelim
paddy@0 283 }
paddy@0 284 r := l.next()
paddy@0 285 if r == separator {
paddy@0 286 l.emit(tokenSeparator)
paddy@0 287 return lexItem
paddy@0 288 } else if r == eof {
paddy@0 289 if l.arrayDepth > 0 {
paddy@0 290 return l.errorf("unclosed array")
paddy@0 291 }
paddy@0 292 l.emit(tokenEOF)
paddy@0 293 return nil
paddy@0 294 } else {
paddy@0 295 l.backup()
paddy@1 296 return l.errorf("expected %s, none found before %s\n", string(separator), l.input[l.pos:])
paddy@0 297 }
paddy@0 298 }