diff options
Diffstat (limited to 'node_modules/css-tree/lib/tokenizer/TokenStream.js')
| -rw-r--r-- | node_modules/css-tree/lib/tokenizer/TokenStream.js | 272 |
1 files changed, 0 insertions, 272 deletions
diff --git a/node_modules/css-tree/lib/tokenizer/TokenStream.js b/node_modules/css-tree/lib/tokenizer/TokenStream.js deleted file mode 100644 index 5dd52755..00000000 --- a/node_modules/css-tree/lib/tokenizer/TokenStream.js +++ /dev/null @@ -1,272 +0,0 @@ -import { adoptBuffer } from './adopt-buffer.js'; -import { cmpStr } from './utils.js'; -import tokenNames from './names.js'; -import { - WhiteSpace, - Comment, - Delim, - EOF, - Function as FunctionToken, - LeftParenthesis, - RightParenthesis, - LeftSquareBracket, - RightSquareBracket, - LeftCurlyBracket, - RightCurlyBracket -} from './types.js'; - -const OFFSET_MASK = 0x00FFFFFF; -const TYPE_SHIFT = 24; -const balancePair = new Map([ - [FunctionToken, RightParenthesis], - [LeftParenthesis, RightParenthesis], - [LeftSquareBracket, RightSquareBracket], - [LeftCurlyBracket, RightCurlyBracket] -]); - -export class TokenStream { - constructor(source, tokenize) { - this.setSource(source, tokenize); - } - reset() { - this.eof = false; - this.tokenIndex = -1; - this.tokenType = 0; - this.tokenStart = this.firstCharOffset; - this.tokenEnd = this.firstCharOffset; - } - setSource(source = '', tokenize = () => {}) { - source = String(source || ''); - - const sourceLength = source.length; - const offsetAndType = adoptBuffer(this.offsetAndType, source.length + 1); // +1 because of eof-token - const balance = adoptBuffer(this.balance, source.length + 1); - let tokenCount = 0; - let balanceCloseType = 0; - let balanceStart = 0; - let firstCharOffset = -1; - - // capture buffers - this.offsetAndType = null; - this.balance = null; - - tokenize(source, (type, start, end) => { - switch (type) { - default: - balance[tokenCount] = sourceLength; - break; - - case balanceCloseType: { - let balancePrev = balanceStart & OFFSET_MASK; - balanceStart = balance[balancePrev]; - balanceCloseType = balanceStart >> TYPE_SHIFT; - balance[tokenCount] = balancePrev; - balance[balancePrev++] = tokenCount; - for (; balancePrev < tokenCount; balancePrev++) { - if (balance[balancePrev] === sourceLength) { - balance[balancePrev] = tokenCount; - } - } - break; - } - - case LeftParenthesis: - case FunctionToken: - case LeftSquareBracket: - case LeftCurlyBracket: - balance[tokenCount] = balanceStart; - balanceCloseType = balancePair.get(type); - balanceStart = (balanceCloseType << TYPE_SHIFT) | tokenCount; - break; - } - - offsetAndType[tokenCount++] = (type << TYPE_SHIFT) | end; - if (firstCharOffset === -1) { - firstCharOffset = start; - } - }); - - // finalize buffers - offsetAndType[tokenCount] = (EOF << TYPE_SHIFT) | sourceLength; // <EOF-token> - balance[tokenCount] = sourceLength; - balance[sourceLength] = sourceLength; // prevents false positive balance match with any token - while (balanceStart !== 0) { - const balancePrev = balanceStart & OFFSET_MASK; - balanceStart = balance[balancePrev]; - balance[balancePrev] = sourceLength; - } - - this.source = source; - this.firstCharOffset = firstCharOffset === -1 ? 0 : firstCharOffset; - this.tokenCount = tokenCount; - this.offsetAndType = offsetAndType; - this.balance = balance; - - this.reset(); - this.next(); - } - - lookupType(offset) { - offset += this.tokenIndex; - - if (offset < this.tokenCount) { - return this.offsetAndType[offset] >> TYPE_SHIFT; - } - - return EOF; - } - lookupOffset(offset) { - offset += this.tokenIndex; - - if (offset < this.tokenCount) { - return this.offsetAndType[offset - 1] & OFFSET_MASK; - } - - return this.source.length; - } - lookupValue(offset, referenceStr) { - offset += this.tokenIndex; - - if (offset < this.tokenCount) { - return cmpStr( - this.source, - this.offsetAndType[offset - 1] & OFFSET_MASK, - this.offsetAndType[offset] & OFFSET_MASK, - referenceStr - ); - } - - return false; - } - getTokenStart(tokenIndex) { - if (tokenIndex === this.tokenIndex) { - return this.tokenStart; - } - - if (tokenIndex > 0) { - return tokenIndex < this.tokenCount - ? this.offsetAndType[tokenIndex - 1] & OFFSET_MASK - : this.offsetAndType[this.tokenCount] & OFFSET_MASK; - } - - return this.firstCharOffset; - } - substrToCursor(start) { - return this.source.substring(start, this.tokenStart); - } - - isBalanceEdge(pos) { - return this.balance[this.tokenIndex] < pos; - } - isDelim(code, offset) { - if (offset) { - return ( - this.lookupType(offset) === Delim && - this.source.charCodeAt(this.lookupOffset(offset)) === code - ); - } - - return ( - this.tokenType === Delim && - this.source.charCodeAt(this.tokenStart) === code - ); - } - - skip(tokenCount) { - let next = this.tokenIndex + tokenCount; - - if (next < this.tokenCount) { - this.tokenIndex = next; - this.tokenStart = this.offsetAndType[next - 1] & OFFSET_MASK; - next = this.offsetAndType[next]; - this.tokenType = next >> TYPE_SHIFT; - this.tokenEnd = next & OFFSET_MASK; - } else { - this.tokenIndex = this.tokenCount; - this.next(); - } - } - next() { - let next = this.tokenIndex + 1; - - if (next < this.tokenCount) { - this.tokenIndex = next; - this.tokenStart = this.tokenEnd; - next = this.offsetAndType[next]; - this.tokenType = next >> TYPE_SHIFT; - this.tokenEnd = next & OFFSET_MASK; - } else { - this.eof = true; - this.tokenIndex = this.tokenCount; - this.tokenType = EOF; - this.tokenStart = this.tokenEnd = this.source.length; - } - } - skipSC() { - while (this.tokenType === WhiteSpace || this.tokenType === Comment) { - this.next(); - } - } - skipUntilBalanced(startToken, stopConsume) { - let cursor = startToken; - let balanceEnd; - let offset; - - loop: - for (; cursor < this.tokenCount; cursor++) { - balanceEnd = this.balance[cursor]; - - // stop scanning on balance edge that points to offset before start token - if (balanceEnd < startToken) { - break loop; - } - - offset = cursor > 0 ? this.offsetAndType[cursor - 1] & OFFSET_MASK : this.firstCharOffset; - - // check stop condition - switch (stopConsume(this.source.charCodeAt(offset))) { - case 1: // just stop - break loop; - - case 2: // stop & included - cursor++; - break loop; - - default: - // fast forward to the end of balanced block - if (this.balance[balanceEnd] === cursor) { - cursor = balanceEnd; - } - } - } - - this.skip(cursor - this.tokenIndex); - } - - forEachToken(fn) { - for (let i = 0, offset = this.firstCharOffset; i < this.tokenCount; i++) { - const start = offset; - const item = this.offsetAndType[i]; - const end = item & OFFSET_MASK; - const type = item >> TYPE_SHIFT; - - offset = end; - - fn(type, start, end, i); - } - } - dump() { - const tokens = new Array(this.tokenCount); - - this.forEachToken((type, start, end, index) => { - tokens[index] = { - idx: index, - type: tokenNames[type], - chunk: this.source.substring(start, end), - balance: this.balance[index] - }; - }); - - return tokens; - } -}; |