1 files changed, 52 insertions, 0 deletions
diff --git a/node_modules/css-tree/lib/definition-syntax/tokenizer.js b/node_modules/css-tree/lib/definition-syntax/tokenizer.js
new file mode 100644
index 00000000..8c392627
--- /dev/null
+++ b/node_modules/css-tree/lib/definition-syntax/tokenizer.js
@@ -0,0 +1,52 @@
+import { SyntaxError } from './SyntaxError.js';
+
+const TAB = 9;
+const N = 10;
+const F = 12;
+const R = 13;
+const SPACE = 32;
+
+export class Tokenizer {
+    constructor(str) {
+        this.str = str;
+        this.pos = 0;
+    }
+    charCodeAt(pos) {
+        return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
+    }
+    charCode() {
+        return this.charCodeAt(this.pos);
+    }
+    nextCharCode() {
+        return this.charCodeAt(this.pos + 1);
+    }
+    nextNonWsCode(pos) {
+        return this.charCodeAt(this.findWsEnd(pos));
+    }
+    findWsEnd(pos) {
+        for (; pos < this.str.length; pos++) {
+            const code = this.str.charCodeAt(pos);
+            if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
+                break;
+            }
+        }
+
+        return pos;
+    }
+    substringToPos(end) {
+        return this.str.substring(this.pos, this.pos = end);
+    }
+    eat(code) {
+        if (this.charCode() !== code) {
+            this.error('Expect `' + String.fromCharCode(code) + '`');
+        }
+
+        this.pos++;
+    }
+    peek() {
+        return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
+    }
+    error(message) {
+        throw new SyntaxError(message, this.str, this.pos);
+    }
+};