1 files changed, 272 insertions, 0 deletions
diff --git a/node_modules/css-tree/lib/tokenizer/TokenStream.js b/node_modules/css-tree/lib/tokenizer/TokenStream.js
new file mode 100644
index 00000000..5dd52755
--- /dev/null
+++ b/node_modules/css-tree/lib/tokenizer/TokenStream.js
@@ -0,0 +1,272 @@
+import { adoptBuffer } from './adopt-buffer.js';
+import { cmpStr } from './utils.js';
+import tokenNames from './names.js';
+import {
+    WhiteSpace,
+    Comment,
+    Delim,
+    EOF,
+    Function as FunctionToken,
+    LeftParenthesis,
+    RightParenthesis,
+    LeftSquareBracket,
+    RightSquareBracket,
+    LeftCurlyBracket,
+    RightCurlyBracket
+} from './types.js';
+
+const OFFSET_MASK = 0x00FFFFFF;
+const TYPE_SHIFT = 24;
+const balancePair = new Map([
+    [FunctionToken, RightParenthesis],
+    [LeftParenthesis, RightParenthesis],
+    [LeftSquareBracket, RightSquareBracket],
+    [LeftCurlyBracket, RightCurlyBracket]
+]);
+
+export class TokenStream {
+    constructor(source, tokenize) {
+        this.setSource(source, tokenize);
+    }
+    reset() {
+        this.eof = false;
+        this.tokenIndex = -1;
+        this.tokenType = 0;
+        this.tokenStart = this.firstCharOffset;
+        this.tokenEnd = this.firstCharOffset;
+    }
+    setSource(source = '', tokenize = () => {}) {
+        source = String(source || '');
+
+        const sourceLength = source.length;
+        const offsetAndType = adoptBuffer(this.offsetAndType, source.length + 1); // +1 because of eof-token
+        const balance = adoptBuffer(this.balance, source.length + 1);
+        let tokenCount = 0;
+        let balanceCloseType = 0;
+        let balanceStart = 0;
+        let firstCharOffset = -1;
+
+        // capture buffers
+        this.offsetAndType = null;
+        this.balance = null;
+
+        tokenize(source, (type, start, end) => {
+            switch (type) {
+                default:
+                    balance[tokenCount] = sourceLength;
+                    break;
+
+                case balanceCloseType: {
+                    let balancePrev = balanceStart & OFFSET_MASK;
+                    balanceStart = balance[balancePrev];
+                    balanceCloseType = balanceStart >> TYPE_SHIFT;
+                    balance[tokenCount] = balancePrev;
+                    balance[balancePrev++] = tokenCount;
+                    for (; balancePrev < tokenCount; balancePrev++) {
+                        if (balance[balancePrev] === sourceLength) {
+                            balance[balancePrev] = tokenCount;
+                        }
+                    }
+                    break;
+                }
+
+                case LeftParenthesis:
+                case FunctionToken:
+                case LeftSquareBracket:
+                case LeftCurlyBracket:
+                    balance[tokenCount] = balanceStart;
+                    balanceCloseType = balancePair.get(type);
+                    balanceStart = (balanceCloseType << TYPE_SHIFT) | tokenCount;
+                    break;
+            }
+
+            offsetAndType[tokenCount++] = (type << TYPE_SHIFT) | end;
+            if (firstCharOffset === -1) {
+                firstCharOffset = start;
+            }
+        });
+
+        // finalize buffers
+        offsetAndType[tokenCount] = (EOF << TYPE_SHIFT) | sourceLength; // <EOF-token>
+        balance[tokenCount] = sourceLength;
+        balance[sourceLength] = sourceLength; // prevents false positive balance match with any token
+        while (balanceStart !== 0) {
+            const balancePrev = balanceStart & OFFSET_MASK;
+            balanceStart = balance[balancePrev];
+            balance[balancePrev] = sourceLength;
+        }
+
+        this.source = source;
+        this.firstCharOffset = firstCharOffset === -1 ? 0 : firstCharOffset;
+        this.tokenCount = tokenCount;
+        this.offsetAndType = offsetAndType;
+        this.balance = balance;
+
+        this.reset();
+        this.next();
+    }
+
+    lookupType(offset) {
+        offset += this.tokenIndex;
+
+        if (offset < this.tokenCount) {
+            return this.offsetAndType[offset] >> TYPE_SHIFT;
+        }
+
+        return EOF;
+    }
+    lookupOffset(offset) {
+        offset += this.tokenIndex;
+
+        if (offset < this.tokenCount) {
+            return this.offsetAndType[offset - 1] & OFFSET_MASK;
+        }
+
+        return this.source.length;
+    }
+    lookupValue(offset, referenceStr) {
+        offset += this.tokenIndex;
+
+        if (offset < this.tokenCount) {
+            return cmpStr(
+                this.source,
+                this.offsetAndType[offset - 1] & OFFSET_MASK,
+                this.offsetAndType[offset] & OFFSET_MASK,
+                referenceStr
+            );
+        }
+
+        return false;
+    }
+    getTokenStart(tokenIndex) {
+        if (tokenIndex === this.tokenIndex) {
+            return this.tokenStart;
+        }
+
+        if (tokenIndex > 0) {
+            return tokenIndex < this.tokenCount
+                ? this.offsetAndType[tokenIndex - 1] & OFFSET_MASK
+                : this.offsetAndType[this.tokenCount] & OFFSET_MASK;
+        }
+
+        return this.firstCharOffset;
+    }
+    substrToCursor(start) {
+        return this.source.substring(start, this.tokenStart);
+    }
+
+    isBalanceEdge(pos) {
+        return this.balance[this.tokenIndex] < pos;
+    }
+    isDelim(code, offset) {
+        if (offset) {
+            return (
+                this.lookupType(offset) === Delim &&
+                this.source.charCodeAt(this.lookupOffset(offset)) === code
+            );
+        }
+
+        return (
+            this.tokenType === Delim &&
+            this.source.charCodeAt(this.tokenStart) === code
+        );
+    }
+
+    skip(tokenCount) {
+        let next = this.tokenIndex + tokenCount;
+
+        if (next < this.tokenCount) {
+            this.tokenIndex = next;
+            this.tokenStart = this.offsetAndType[next - 1] & OFFSET_MASK;
+            next = this.offsetAndType[next];
+            this.tokenType = next >> TYPE_SHIFT;
+            this.tokenEnd = next & OFFSET_MASK;
+        } else {
+            this.tokenIndex = this.tokenCount;
+            this.next();
+        }
+    }
+    next() {
+        let next = this.tokenIndex + 1;
+
+        if (next < this.tokenCount) {
+            this.tokenIndex = next;
+            this.tokenStart = this.tokenEnd;
+            next = this.offsetAndType[next];
+            this.tokenType = next >> TYPE_SHIFT;
+            this.tokenEnd = next & OFFSET_MASK;
+        } else {
+            this.eof = true;
+            this.tokenIndex = this.tokenCount;
+            this.tokenType = EOF;
+            this.tokenStart = this.tokenEnd = this.source.length;
+        }
+    }
+    skipSC() {
+        while (this.tokenType === WhiteSpace || this.tokenType === Comment) {
+            this.next();
+        }
+    }
+    skipUntilBalanced(startToken, stopConsume) {
+        let cursor = startToken;
+        let balanceEnd;
+        let offset;
+
+        loop:
+        for (; cursor < this.tokenCount; cursor++) {
+            balanceEnd = this.balance[cursor];
+
+            // stop scanning on balance edge that points to offset before start token
+            if (balanceEnd < startToken) {
+                break loop;
+            }
+
+            offset = cursor > 0 ? this.offsetAndType[cursor - 1] & OFFSET_MASK : this.firstCharOffset;
+
+            // check stop condition
+            switch (stopConsume(this.source.charCodeAt(offset))) {
+                case 1: // just stop
+                    break loop;
+
+                case 2: // stop & included
+                    cursor++;
+                    break loop;
+
+                default:
+                    // fast forward to the end of balanced block
+                    if (this.balance[balanceEnd] === cursor) {
+                        cursor = balanceEnd;
+                    }
+            }
+        }
+
+        this.skip(cursor - this.tokenIndex);
+    }
+
+    forEachToken(fn) {
+        for (let i = 0, offset = this.firstCharOffset; i < this.tokenCount; i++) {
+            const start = offset;
+            const item = this.offsetAndType[i];
+            const end = item & OFFSET_MASK;
+            const type = item >> TYPE_SHIFT;
+
+            offset = end;
+
+            fn(type, start, end, i);
+        }
+    }
+    dump() {
+        const tokens = new Array(this.tokenCount);
+
+        this.forEachToken((type, start, end, index) => {
+            tokens[index] = {
+                idx: index,
+                type: tokenNames[type],
+                chunk: this.source.substring(start, end),
+                balance: this.balance[index]
+            };
+        });
+
+        return tokens;
+    }
+};