package lexer import ( "fmt" "strings" "unicode" "unicode/utf8" ) func IsSpace(r rune) bool { return unicode.IsSpace(r) } func IsAlphaNumeric(r rune) bool { return IsAlphabetic(r) || unicode.IsDigit(r) } func IsAlphabetic(r rune) bool { return r == '_' || r == '$' || unicode.IsLetter(r) } var ( newlineNormalizer = strings.NewReplacer("\r\n", "\n", "\r", "\n") ) // Unescape takes a quoted string, unquotes, and unescapes it. func unescape(value string) (string, error) { // All strings normalize newlines to the \n representation. value = newlineNormalizer.Replace(value) n := len(value) // Nothing to unescape / decode. if n < 2 { return value, fmt.Errorf("unable to unescape string") } // Quoted string of some form, must have same first and last char. if value[0] != value[n-1] || (value[0] != '"' && value[0] != '\'') { return value, fmt.Errorf("unable to unescape string") } value = value[1 : n-1] // The string contains escape characters. // The following logic is adapted from `strconv/quote.go` var runeTmp [utf8.UTFMax]byte buf := make([]byte, 0, 3*n/2) for len(value) > 0 { c, multibyte, rest, err := unescapeChar(value) if err != nil { return "", err } value = rest if c < utf8.RuneSelf || !multibyte { buf = append(buf, byte(c)) } else { n := utf8.EncodeRune(runeTmp[:], c) buf = append(buf, runeTmp[:n]...) } } return string(buf), nil } // unescapeChar takes a string input and returns the following info: // // value - the escaped unicode rune at the front of the string. // multibyte - whether the rune value might require multiple bytes to represent. // tail - the remainder of the input string. // err - error value, if the character could not be unescaped. // // When multibyte is true the return value may still fit within a single byte, // but a multibyte conversion is attempted which is more expensive than when the // value is known to fit within one byte. func unescapeChar(s string) (value rune, multibyte bool, tail string, err error) { // 1. Character is not an escape sequence. switch c := s[0]; { case c >= utf8.RuneSelf: r, size := utf8.DecodeRuneInString(s) return r, true, s[size:], nil case c != '\\': return rune(s[0]), false, s[1:], nil } // 2. Last character is the start of an escape sequence. if len(s) <= 1 { err = fmt.Errorf("unable to unescape string, found '\\' as last character") return } c := s[1] s = s[2:] // 3. Common escape sequences shared with Google SQL switch c { case 'a': value = '\a' case 'b': value = '\b' case 'f': value = '\f' case 'n': value = '\n' case 'r': value = '\r' case 't': value = '\t' case 'v': value = '\v' case '\\': value = '\\' case '\'': value = '\'' case '"': value = '"' case '`': value = '`' case '?': value = '?' // 4. Unicode escape sequences, reproduced from `strconv/quote.go` case 'x', 'X', 'u', 'U': n := 0 switch c { case 'x', 'X': n = 2 case 'u': n = 4 case 'U': n = 8 } var v rune if len(s) < n { err = fmt.Errorf("unable to unescape string") return } for j := 0; j < n; j++ { x, ok := unhex(s[j]) if !ok { err = fmt.Errorf("unable to unescape string") return } v = v<<4 | x } s = s[n:] if v > utf8.MaxRune { err = fmt.Errorf("unable to unescape string") return } value = v multibyte = true // 5. Octal escape sequences, must be three digits \[0-3][0-7][0-7] case '0', '1', '2', '3': if len(s) < 2 { err = fmt.Errorf("unable to unescape octal sequence in string") return } v := rune(c - '0') for j := 0; j < 2; j++ { x := s[j] if x < '0' || x > '7' { err = fmt.Errorf("unable to unescape octal sequence in string") return } v = v*8 + rune(x-'0') } if v > utf8.MaxRune { err = fmt.Errorf("unable to unescape string") return } value = v s = s[2:] multibyte = true // Unknown escape sequence. default: err = fmt.Errorf("unable to unescape string") } tail = s return } func unhex(b byte) (rune, bool) { c := rune(b) switch { case '0' <= c && c <= '9': return c - '0', true case 'a' <= c && c <= 'f': return c - 'a' + 10, true case 'A' <= c && c <= 'F': return c - 'A' + 10, true } return 0, false }