You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
357 lines
10 KiB
357 lines
10 KiB
// Copyright 2012 The Gorilla Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package scanner
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// tokenType identifies the type of lexical tokens.
|
|
type tokenType int
|
|
|
|
// String returns a string representation of the token type.
|
|
func (t tokenType) String() string {
|
|
return tokenNames[t]
|
|
}
|
|
|
|
// Token represents a token and the corresponding string.
|
|
type Token struct {
|
|
Type tokenType
|
|
Value string
|
|
Line int
|
|
Column int
|
|
}
|
|
|
|
// String returns a string representation of the token.
|
|
func (t *Token) String() string {
|
|
if len(t.Value) > 10 {
|
|
return fmt.Sprintf("%s (line: %d, column: %d): %.10q...",
|
|
t.Type, t.Line, t.Column, t.Value)
|
|
}
|
|
return fmt.Sprintf("%s (line: %d, column: %d): %q",
|
|
t.Type, t.Line, t.Column, t.Value)
|
|
}
|
|
|
|
// All tokens -----------------------------------------------------------------
|
|
|
|
// The complete list of tokens in CSS3.
|
|
const (
|
|
// Scanner flags.
|
|
TokenError tokenType = iota
|
|
TokenEOF
|
|
// From now on, only tokens from the CSS specification.
|
|
TokenIdent
|
|
TokenAtKeyword
|
|
TokenString
|
|
TokenHash
|
|
TokenNumber
|
|
TokenPercentage
|
|
TokenDimension
|
|
TokenURI
|
|
TokenUnicodeRange
|
|
TokenCDO
|
|
TokenCDC
|
|
TokenS
|
|
TokenComment
|
|
TokenFunction
|
|
TokenIncludes
|
|
TokenDashMatch
|
|
TokenPrefixMatch
|
|
TokenSuffixMatch
|
|
TokenSubstringMatch
|
|
TokenChar
|
|
TokenBOM
|
|
)
|
|
|
|
// tokenNames maps tokenType's to their names. Used for conversion to string.
|
|
var tokenNames = map[tokenType]string{
|
|
TokenError: "error",
|
|
TokenEOF: "EOF",
|
|
TokenIdent: "IDENT",
|
|
TokenAtKeyword: "ATKEYWORD",
|
|
TokenString: "STRING",
|
|
TokenHash: "HASH",
|
|
TokenNumber: "NUMBER",
|
|
TokenPercentage: "PERCENTAGE",
|
|
TokenDimension: "DIMENSION",
|
|
TokenURI: "URI",
|
|
TokenUnicodeRange: "UNICODE-RANGE",
|
|
TokenCDO: "CDO",
|
|
TokenCDC: "CDC",
|
|
TokenS: "S",
|
|
TokenComment: "COMMENT",
|
|
TokenFunction: "FUNCTION",
|
|
TokenIncludes: "INCLUDES",
|
|
TokenDashMatch: "DASHMATCH",
|
|
TokenPrefixMatch: "PREFIXMATCH",
|
|
TokenSuffixMatch: "SUFFIXMATCH",
|
|
TokenSubstringMatch: "SUBSTRINGMATCH",
|
|
TokenChar: "CHAR",
|
|
TokenBOM: "BOM",
|
|
}
|
|
|
|
// Macros and productions -----------------------------------------------------
|
|
// http://www.w3.org/TR/css3-syntax/#tokenization
|
|
|
|
var macroRegexp = regexp.MustCompile(`\{[a-z]+\}`)
|
|
|
|
// macros maps macro names to patterns to be expanded.
|
|
var macros = map[string]string{
|
|
// must be escaped: `\.+*?()|[]{}^$`
|
|
"ident": `-?{nmstart}{nmchar}*`,
|
|
"name": `{nmchar}+`,
|
|
"nmstart": `[a-zA-Z_]|{nonascii}|{escape}`,
|
|
"nonascii": "[\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]",
|
|
"unicode": `\\[0-9a-fA-F]{1,6}{wc}?`,
|
|
"escape": "{unicode}|\\\\[\u0020-\u007E\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]",
|
|
"nmchar": `[a-zA-Z0-9_-]|{nonascii}|{escape}`,
|
|
"num": `[0-9]*\.[0-9]+|[0-9]+`,
|
|
"string": `"(?:{stringchar}|')*"|'(?:{stringchar}|")*'`,
|
|
"stringchar": `{urlchar}|[ ]|\\{nl}`,
|
|
"nl": `[\n\r\f]|\r\n`,
|
|
"w": `{wc}*`,
|
|
"wc": `[\t\n\f\r ]`,
|
|
|
|
// urlchar should accept [(ascii characters minus those that need escaping)|{nonascii}|{escape}]
|
|
// ASCII characters range = `[\u0020-\u007e]`
|
|
// Skip space \u0020 = `[\u0021-\u007e]`
|
|
// Skip quotation mark \0022 = `[\u0021\u0023-\u007e]`
|
|
// Skip apostrophe \u0027 = `[\u0021\u0023-\u0026\u0028-\u007e]`
|
|
// Skip reverse solidus \u005c = `[\u0021\u0023-\u0026\u0028-\u005b\u005d\u007e]`
|
|
// Finally, the left square bracket (\u005b) and right (\u005d) needs escaping themselves
|
|
"urlchar": "[\u0021\u0023-\u0026\u0028-\\\u005b\\\u005d-\u007E]|{nonascii}|{escape}",
|
|
}
|
|
|
|
// productions maps the list of tokens to patterns to be expanded.
|
|
var productions = map[tokenType]string{
|
|
// Unused regexps (matched using other methods) are commented out.
|
|
TokenIdent: `{ident}`,
|
|
TokenAtKeyword: `@{ident}`,
|
|
TokenString: `{string}`,
|
|
TokenHash: `#{name}`,
|
|
TokenNumber: `{num}`,
|
|
TokenPercentage: `{num}%`,
|
|
TokenDimension: `{num}{ident}`,
|
|
TokenURI: `url\({w}(?:{string}|{urlchar}*?){w}\)`,
|
|
TokenUnicodeRange: `U\+[0-9A-F\?]{1,6}(?:-[0-9A-F]{1,6})?`,
|
|
//TokenCDO: `<!--`,
|
|
TokenCDC: `-->`,
|
|
TokenS: `{wc}+`,
|
|
TokenComment: `/\*[^\*]*[\*]+(?:[^/][^\*]*[\*]+)*/`,
|
|
TokenFunction: `{ident}\(`,
|
|
//TokenIncludes: `~=`,
|
|
//TokenDashMatch: `\|=`,
|
|
//TokenPrefixMatch: `\^=`,
|
|
//TokenSuffixMatch: `\$=`,
|
|
//TokenSubstringMatch: `\*=`,
|
|
//TokenChar: `[^"']`,
|
|
//TokenBOM: "\uFEFF",
|
|
}
|
|
|
|
// matchers maps the list of tokens to compiled regular expressions.
|
|
//
|
|
// The map is filled on init() using the macros and productions defined in
|
|
// the CSS specification.
|
|
var matchers = map[tokenType]*regexp.Regexp{}
|
|
|
|
// matchOrder is the order to test regexps when first-char shortcuts
|
|
// can't be used.
|
|
var matchOrder = []tokenType{
|
|
TokenURI,
|
|
TokenFunction,
|
|
TokenUnicodeRange,
|
|
TokenIdent,
|
|
TokenDimension,
|
|
TokenPercentage,
|
|
TokenNumber,
|
|
TokenCDC,
|
|
}
|
|
|
|
func init() {
|
|
// replace macros and compile regexps for productions.
|
|
replaceMacro := func(s string) string {
|
|
return "(?:" + macros[s[1:len(s)-1]] + ")"
|
|
}
|
|
for t, s := range productions {
|
|
for macroRegexp.MatchString(s) {
|
|
s = macroRegexp.ReplaceAllStringFunc(s, replaceMacro)
|
|
}
|
|
matchers[t] = regexp.MustCompile("^(?:" + s + ")")
|
|
}
|
|
}
|
|
|
|
// Scanner --------------------------------------------------------------------
|
|
|
|
// New returns a new CSS scanner for the given input.
|
|
func New(input string) *Scanner {
|
|
// Normalize newlines.
|
|
input = strings.Replace(input, "\r\n", "\n", -1)
|
|
return &Scanner{
|
|
input: input,
|
|
row: 1,
|
|
col: 1,
|
|
}
|
|
}
|
|
|
|
// Scanner scans an input and emits tokens following the CSS3 specification.
|
|
type Scanner struct {
|
|
input string
|
|
pos int
|
|
row int
|
|
col int
|
|
err *Token
|
|
}
|
|
|
|
// Next returns the next token from the input.
|
|
//
|
|
// At the end of the input the token type is TokenEOF.
|
|
//
|
|
// If the input can't be tokenized the token type is TokenError. This occurs
|
|
// in case of unclosed quotation marks or comments.
|
|
func (s *Scanner) Next() *Token {
|
|
if s.err != nil {
|
|
return s.err
|
|
}
|
|
if s.pos >= len(s.input) {
|
|
s.err = &Token{TokenEOF, "", s.row, s.col}
|
|
return s.err
|
|
}
|
|
if s.pos == 0 {
|
|
// Test BOM only once, at the beginning of the file.
|
|
if strings.HasPrefix(s.input, "\uFEFF") {
|
|
return s.emitSimple(TokenBOM, "\uFEFF")
|
|
}
|
|
}
|
|
// There's a lot we can guess based on the first byte so we'll take a
|
|
// shortcut before testing multiple regexps.
|
|
input := s.input[s.pos:]
|
|
switch input[0] {
|
|
case '\t', '\n', '\f', '\r', ' ':
|
|
// Whitespace.
|
|
return s.emitToken(TokenS, matchers[TokenS].FindString(input))
|
|
case '.':
|
|
// Dot is too common to not have a quick check.
|
|
// We'll test if this is a Char; if it is followed by a number it is a
|
|
// dimension/percentage/number, and this will be matched later.
|
|
if len(input) > 1 && !unicode.IsDigit(rune(input[1])) {
|
|
return s.emitSimple(TokenChar, ".")
|
|
}
|
|
case '#':
|
|
// Another common one: Hash or Char.
|
|
if match := matchers[TokenHash].FindString(input); match != "" {
|
|
return s.emitToken(TokenHash, match)
|
|
}
|
|
return s.emitSimple(TokenChar, "#")
|
|
case '@':
|
|
// Another common one: AtKeyword or Char.
|
|
if match := matchers[TokenAtKeyword].FindString(input); match != "" {
|
|
return s.emitSimple(TokenAtKeyword, match)
|
|
}
|
|
return s.emitSimple(TokenChar, "@")
|
|
case ':', ',', ';', '%', '&', '+', '=', '>', '(', ')', '[', ']', '{', '}':
|
|
// More common chars.
|
|
return s.emitSimple(TokenChar, string(input[0]))
|
|
case '"', '\'':
|
|
// String or error.
|
|
match := matchers[TokenString].FindString(input)
|
|
if match != "" {
|
|
return s.emitToken(TokenString, match)
|
|
}
|
|
|
|
s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col}
|
|
return s.err
|
|
case '/':
|
|
// Comment, error or Char.
|
|
if len(input) > 1 && input[1] == '*' {
|
|
match := matchers[TokenComment].FindString(input)
|
|
if match != "" {
|
|
return s.emitToken(TokenComment, match)
|
|
} else {
|
|
s.err = &Token{TokenError, "unclosed comment", s.row, s.col}
|
|
return s.err
|
|
}
|
|
}
|
|
return s.emitSimple(TokenChar, "/")
|
|
case '~':
|
|
// Includes or Char.
|
|
return s.emitPrefixOrChar(TokenIncludes, "~=")
|
|
case '|':
|
|
// DashMatch or Char.
|
|
return s.emitPrefixOrChar(TokenDashMatch, "|=")
|
|
case '^':
|
|
// PrefixMatch or Char.
|
|
return s.emitPrefixOrChar(TokenPrefixMatch, "^=")
|
|
case '$':
|
|
// SuffixMatch or Char.
|
|
return s.emitPrefixOrChar(TokenSuffixMatch, "$=")
|
|
case '*':
|
|
// SubstringMatch or Char.
|
|
return s.emitPrefixOrChar(TokenSubstringMatch, "*=")
|
|
case '<':
|
|
// CDO or Char.
|
|
return s.emitPrefixOrChar(TokenCDO, "<!--")
|
|
}
|
|
// Test all regexps, in order.
|
|
for _, token := range matchOrder {
|
|
if match := matchers[token].FindString(input); match != "" {
|
|
return s.emitToken(token, match)
|
|
}
|
|
}
|
|
// We already handled unclosed quotation marks and comments,
|
|
// so this can only be a Char.
|
|
r, width := utf8.DecodeRuneInString(input)
|
|
token := &Token{TokenChar, string(r), s.row, s.col}
|
|
s.col += width
|
|
s.pos += width
|
|
return token
|
|
}
|
|
|
|
// updatePosition updates input coordinates based on the consumed text.
|
|
func (s *Scanner) updatePosition(text string) {
|
|
width := utf8.RuneCountInString(text)
|
|
lines := strings.Count(text, "\n")
|
|
s.row += lines
|
|
if lines == 0 {
|
|
s.col += width
|
|
} else {
|
|
s.col = utf8.RuneCountInString(text[strings.LastIndex(text, "\n"):])
|
|
}
|
|
s.pos += len(text) // while col is a rune index, pos is a byte index
|
|
}
|
|
|
|
// emitToken returns a Token for the string v and updates the scanner position.
|
|
func (s *Scanner) emitToken(t tokenType, v string) *Token {
|
|
token := &Token{t, v, s.row, s.col}
|
|
s.updatePosition(v)
|
|
return token
|
|
}
|
|
|
|
// emitSimple returns a Token for the string v and updates the scanner
|
|
// position in a simplified manner.
|
|
//
|
|
// The string is known to have only ASCII characters and to not have a newline.
|
|
func (s *Scanner) emitSimple(t tokenType, v string) *Token {
|
|
token := &Token{t, v, s.row, s.col}
|
|
s.col += len(v)
|
|
s.pos += len(v)
|
|
return token
|
|
}
|
|
|
|
// emitPrefixOrChar returns a Token for type t if the current position
|
|
// matches the given prefix. Otherwise it returns a Char token using the
|
|
// first character from the prefix.
|
|
//
|
|
// The prefix is known to have only ASCII characters and to not have a newline.
|
|
func (s *Scanner) emitPrefixOrChar(t tokenType, prefix string) *Token {
|
|
if strings.HasPrefix(s.input[s.pos:], prefix) {
|
|
return s.emitSimple(t, prefix)
|
|
}
|
|
return s.emitSimple(TokenChar, string(prefix[0]))
|
|
}
|