diff options
| author | littleblack111 <littleblack11111@gmail.com> | 2023-10-31 15:37:35 +0800 |
|---|---|---|
| committer | littleblack111 <littleblack11111@gmail.com> | 2023-10-31 15:37:35 +0800 |
| commit | d5f14083061f2ce6a42313188443a42e30796320 (patch) | |
| tree | cc6a9889e08b29595a76a33b9e7b4ef84866468b /node_modules/webpack/lib/cache | |
| parent | b1a6ea021f29f182b62f9f81024a3d8ac8621b7f (diff) | |
added make browserlist(autoprefixer) work fixed
Diffstat (limited to 'node_modules/webpack/lib/cache')
| -rw-r--r-- | node_modules/webpack/lib/cache/AddBuildDependenciesPlugin.js | 33 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/AddManagedPathsPlugin.js | 35 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/IdleFileCachePlugin.js | 228 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/MemoryCachePlugin.js | 57 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/MemoryWithGcCachePlugin.js | 131 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/PackFileCacheStrategy.js | 1467 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/ResolverCachePlugin.js | 357 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/getLazyHashedEtag.js | 81 | ||||
| -rw-r--r-- | node_modules/webpack/lib/cache/mergeEtags.js | 74 |
9 files changed, 2463 insertions, 0 deletions
diff --git a/node_modules/webpack/lib/cache/AddBuildDependenciesPlugin.js b/node_modules/webpack/lib/cache/AddBuildDependenciesPlugin.js new file mode 100644 index 00000000..f2283ffc --- /dev/null +++ b/node_modules/webpack/lib/cache/AddBuildDependenciesPlugin.js @@ -0,0 +1,33 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +/** @typedef {import("../Compiler")} Compiler */ + +class AddBuildDependenciesPlugin { + /** + * @param {Iterable<string>} buildDependencies list of build dependencies + */ + constructor(buildDependencies) { + this.buildDependencies = new Set(buildDependencies); + } + + /** + * Apply the plugin + * @param {Compiler} compiler the compiler instance + * @returns {void} + */ + apply(compiler) { + compiler.hooks.compilation.tap( + "AddBuildDependenciesPlugin", + compilation => { + compilation.buildDependencies.addAll(this.buildDependencies); + } + ); + } +} + +module.exports = AddBuildDependenciesPlugin; diff --git a/node_modules/webpack/lib/cache/AddManagedPathsPlugin.js b/node_modules/webpack/lib/cache/AddManagedPathsPlugin.js new file mode 100644 index 00000000..702aa6c6 --- /dev/null +++ b/node_modules/webpack/lib/cache/AddManagedPathsPlugin.js @@ -0,0 +1,35 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +/** @typedef {import("../Compiler")} Compiler */ + +class AddManagedPathsPlugin { + /** + * @param {Iterable<string | RegExp>} managedPaths list of managed paths + * @param {Iterable<string | RegExp>} immutablePaths list of immutable paths + */ + constructor(managedPaths, immutablePaths) { + this.managedPaths = new Set(managedPaths); + this.immutablePaths = new Set(immutablePaths); + } + + /** + * Apply the plugin + * @param {Compiler} compiler the compiler instance + * @returns {void} + */ + apply(compiler) { + for (const managedPath of this.managedPaths) { + compiler.managedPaths.add(managedPath); + } + for (const immutablePath of this.immutablePaths) { + compiler.immutablePaths.add(immutablePath); + } + } +} + +module.exports = AddManagedPathsPlugin; diff --git a/node_modules/webpack/lib/cache/IdleFileCachePlugin.js b/node_modules/webpack/lib/cache/IdleFileCachePlugin.js new file mode 100644 index 00000000..8a82644f --- /dev/null +++ b/node_modules/webpack/lib/cache/IdleFileCachePlugin.js @@ -0,0 +1,228 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const Cache = require("../Cache"); +const ProgressPlugin = require("../ProgressPlugin"); + +/** @typedef {import("../Compiler")} Compiler */ + +const BUILD_DEPENDENCIES_KEY = Symbol(); + +class IdleFileCachePlugin { + /** + * @param {TODO} strategy cache strategy + * @param {number} idleTimeout timeout + * @param {number} idleTimeoutForInitialStore initial timeout + * @param {number} idleTimeoutAfterLargeChanges timeout after changes + */ + constructor( + strategy, + idleTimeout, + idleTimeoutForInitialStore, + idleTimeoutAfterLargeChanges + ) { + this.strategy = strategy; + this.idleTimeout = idleTimeout; + this.idleTimeoutForInitialStore = idleTimeoutForInitialStore; + this.idleTimeoutAfterLargeChanges = idleTimeoutAfterLargeChanges; + } + + /** + * Apply the plugin + * @param {Compiler} compiler the compiler instance + * @returns {void} + */ + apply(compiler) { + let strategy = this.strategy; + const idleTimeout = this.idleTimeout; + const idleTimeoutForInitialStore = Math.min( + idleTimeout, + this.idleTimeoutForInitialStore + ); + const idleTimeoutAfterLargeChanges = this.idleTimeoutAfterLargeChanges; + const resolvedPromise = Promise.resolve(); + + let timeSpendInBuild = 0; + let timeSpendInStore = 0; + let avgTimeSpendInStore = 0; + + /** @type {Map<string | typeof BUILD_DEPENDENCIES_KEY, () => Promise<void>>} */ + const pendingIdleTasks = new Map(); + + compiler.cache.hooks.store.tap( + { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK }, + (identifier, etag, data) => { + pendingIdleTasks.set(identifier, () => + strategy.store(identifier, etag, data) + ); + } + ); + + compiler.cache.hooks.get.tapPromise( + { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK }, + (identifier, etag, gotHandlers) => { + const restore = () => + strategy.restore(identifier, etag).then(cacheEntry => { + if (cacheEntry === undefined) { + gotHandlers.push((result, callback) => { + if (result !== undefined) { + pendingIdleTasks.set(identifier, () => + strategy.store(identifier, etag, result) + ); + } + callback(); + }); + } else { + return cacheEntry; + } + }); + const pendingTask = pendingIdleTasks.get(identifier); + if (pendingTask !== undefined) { + pendingIdleTasks.delete(identifier); + return pendingTask().then(restore); + } + return restore(); + } + ); + + compiler.cache.hooks.storeBuildDependencies.tap( + { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK }, + dependencies => { + pendingIdleTasks.set(BUILD_DEPENDENCIES_KEY, () => + strategy.storeBuildDependencies(dependencies) + ); + } + ); + + compiler.cache.hooks.shutdown.tapPromise( + { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK }, + () => { + if (idleTimer) { + clearTimeout(idleTimer); + idleTimer = undefined; + } + isIdle = false; + const reportProgress = ProgressPlugin.getReporter(compiler); + const jobs = Array.from(pendingIdleTasks.values()); + if (reportProgress) reportProgress(0, "process pending cache items"); + const promises = jobs.map(fn => fn()); + pendingIdleTasks.clear(); + promises.push(currentIdlePromise); + const promise = Promise.all(promises); + currentIdlePromise = promise.then(() => strategy.afterAllStored()); + if (reportProgress) { + currentIdlePromise = currentIdlePromise.then(() => { + reportProgress(1, `stored`); + }); + } + return currentIdlePromise.then(() => { + // Reset strategy + if (strategy.clear) strategy.clear(); + }); + } + ); + + /** @type {Promise<any>} */ + let currentIdlePromise = resolvedPromise; + let isIdle = false; + let isInitialStore = true; + const processIdleTasks = () => { + if (isIdle) { + const startTime = Date.now(); + if (pendingIdleTasks.size > 0) { + const promises = [currentIdlePromise]; + const maxTime = startTime + 100; + let maxCount = 100; + for (const [filename, factory] of pendingIdleTasks) { + pendingIdleTasks.delete(filename); + promises.push(factory()); + if (maxCount-- <= 0 || Date.now() > maxTime) break; + } + currentIdlePromise = Promise.all(promises); + currentIdlePromise.then(() => { + timeSpendInStore += Date.now() - startTime; + // Allow to exit the process between + idleTimer = setTimeout(processIdleTasks, 0); + idleTimer.unref(); + }); + return; + } + currentIdlePromise = currentIdlePromise + .then(async () => { + await strategy.afterAllStored(); + timeSpendInStore += Date.now() - startTime; + avgTimeSpendInStore = + Math.max(avgTimeSpendInStore, timeSpendInStore) * 0.9 + + timeSpendInStore * 0.1; + timeSpendInStore = 0; + timeSpendInBuild = 0; + }) + .catch(err => { + const logger = compiler.getInfrastructureLogger( + "IdleFileCachePlugin" + ); + logger.warn(`Background tasks during idle failed: ${err.message}`); + logger.debug(err.stack); + }); + isInitialStore = false; + } + }; + /** @type {ReturnType<typeof setTimeout> | undefined} */ + let idleTimer = undefined; + compiler.cache.hooks.beginIdle.tap( + { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK }, + () => { + const isLargeChange = timeSpendInBuild > avgTimeSpendInStore * 2; + if (isInitialStore && idleTimeoutForInitialStore < idleTimeout) { + compiler + .getInfrastructureLogger("IdleFileCachePlugin") + .log( + `Initial cache was generated and cache will be persisted in ${ + idleTimeoutForInitialStore / 1000 + }s.` + ); + } else if ( + isLargeChange && + idleTimeoutAfterLargeChanges < idleTimeout + ) { + compiler + .getInfrastructureLogger("IdleFileCachePlugin") + .log( + `Spend ${Math.round(timeSpendInBuild) / 1000}s in build and ${ + Math.round(avgTimeSpendInStore) / 1000 + }s in average in cache store. This is considered as large change and cache will be persisted in ${ + idleTimeoutAfterLargeChanges / 1000 + }s.` + ); + } + idleTimer = setTimeout(() => { + idleTimer = undefined; + isIdle = true; + resolvedPromise.then(processIdleTasks); + }, Math.min(isInitialStore ? idleTimeoutForInitialStore : Infinity, isLargeChange ? idleTimeoutAfterLargeChanges : Infinity, idleTimeout)); + idleTimer.unref(); + } + ); + compiler.cache.hooks.endIdle.tap( + { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK }, + () => { + if (idleTimer) { + clearTimeout(idleTimer); + idleTimer = undefined; + } + isIdle = false; + } + ); + compiler.hooks.done.tap("IdleFileCachePlugin", stats => { + // 10% build overhead is ignored, as it's not cacheable + timeSpendInBuild *= 0.9; + timeSpendInBuild += stats.endTime - stats.startTime; + }); + } +} + +module.exports = IdleFileCachePlugin; diff --git a/node_modules/webpack/lib/cache/MemoryCachePlugin.js b/node_modules/webpack/lib/cache/MemoryCachePlugin.js new file mode 100644 index 00000000..cec61acd --- /dev/null +++ b/node_modules/webpack/lib/cache/MemoryCachePlugin.js @@ -0,0 +1,57 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const Cache = require("../Cache"); + +/** @typedef {import("webpack-sources").Source} Source */ +/** @typedef {import("../Cache").Etag} Etag */ +/** @typedef {import("../Compiler")} Compiler */ +/** @typedef {import("../Module")} Module */ + +class MemoryCachePlugin { + /** + * Apply the plugin + * @param {Compiler} compiler the compiler instance + * @returns {void} + */ + apply(compiler) { + /** @type {Map<string, { etag: Etag | null, data: any }>} */ + const cache = new Map(); + compiler.cache.hooks.store.tap( + { name: "MemoryCachePlugin", stage: Cache.STAGE_MEMORY }, + (identifier, etag, data) => { + cache.set(identifier, { etag, data }); + } + ); + compiler.cache.hooks.get.tap( + { name: "MemoryCachePlugin", stage: Cache.STAGE_MEMORY }, + (identifier, etag, gotHandlers) => { + const cacheEntry = cache.get(identifier); + if (cacheEntry === null) { + return null; + } else if (cacheEntry !== undefined) { + return cacheEntry.etag === etag ? cacheEntry.data : null; + } + gotHandlers.push((result, callback) => { + if (result === undefined) { + cache.set(identifier, null); + } else { + cache.set(identifier, { etag, data: result }); + } + return callback(); + }); + } + ); + compiler.cache.hooks.shutdown.tap( + { name: "MemoryCachePlugin", stage: Cache.STAGE_MEMORY }, + () => { + cache.clear(); + } + ); + } +} +module.exports = MemoryCachePlugin; diff --git a/node_modules/webpack/lib/cache/MemoryWithGcCachePlugin.js b/node_modules/webpack/lib/cache/MemoryWithGcCachePlugin.js new file mode 100644 index 00000000..d441f07f --- /dev/null +++ b/node_modules/webpack/lib/cache/MemoryWithGcCachePlugin.js @@ -0,0 +1,131 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const Cache = require("../Cache"); + +/** @typedef {import("webpack-sources").Source} Source */ +/** @typedef {import("../Cache").Etag} Etag */ +/** @typedef {import("../Compiler")} Compiler */ +/** @typedef {import("../Module")} Module */ + +class MemoryWithGcCachePlugin { + constructor({ maxGenerations }) { + this._maxGenerations = maxGenerations; + } + /** + * Apply the plugin + * @param {Compiler} compiler the compiler instance + * @returns {void} + */ + apply(compiler) { + const maxGenerations = this._maxGenerations; + /** @type {Map<string, { etag: Etag | null, data: any }>} */ + const cache = new Map(); + /** @type {Map<string, { entry: { etag: Etag | null, data: any }, until: number }>} */ + const oldCache = new Map(); + let generation = 0; + let cachePosition = 0; + const logger = compiler.getInfrastructureLogger("MemoryWithGcCachePlugin"); + compiler.hooks.afterDone.tap("MemoryWithGcCachePlugin", () => { + generation++; + let clearedEntries = 0; + let lastClearedIdentifier; + // Avoid coverage problems due indirect changes + /* istanbul ignore next */ + for (const [identifier, entry] of oldCache) { + if (entry.until > generation) break; + + oldCache.delete(identifier); + if (cache.get(identifier) === undefined) { + cache.delete(identifier); + clearedEntries++; + lastClearedIdentifier = identifier; + } + } + if (clearedEntries > 0 || oldCache.size > 0) { + logger.log( + `${cache.size - oldCache.size} active entries, ${ + oldCache.size + } recently unused cached entries${ + clearedEntries > 0 + ? `, ${clearedEntries} old unused cache entries removed e. g. ${lastClearedIdentifier}` + : "" + }` + ); + } + let i = (cache.size / maxGenerations) | 0; + let j = cachePosition >= cache.size ? 0 : cachePosition; + cachePosition = j + i; + for (const [identifier, entry] of cache) { + if (j !== 0) { + j--; + continue; + } + if (entry !== undefined) { + // We don't delete the cache entry, but set it to undefined instead + // This reserves the location in the data table and avoids rehashing + // when constantly adding and removing entries. + // It will be deleted when removed from oldCache. + cache.set(identifier, undefined); + oldCache.delete(identifier); + oldCache.set(identifier, { + entry, + until: generation + maxGenerations + }); + if (i-- === 0) break; + } + } + }); + compiler.cache.hooks.store.tap( + { name: "MemoryWithGcCachePlugin", stage: Cache.STAGE_MEMORY }, + (identifier, etag, data) => { + cache.set(identifier, { etag, data }); + } + ); + compiler.cache.hooks.get.tap( + { name: "MemoryWithGcCachePlugin", stage: Cache.STAGE_MEMORY }, + (identifier, etag, gotHandlers) => { + const cacheEntry = cache.get(identifier); + if (cacheEntry === null) { + return null; + } else if (cacheEntry !== undefined) { + return cacheEntry.etag === etag ? cacheEntry.data : null; + } + const oldCacheEntry = oldCache.get(identifier); + if (oldCacheEntry !== undefined) { + const cacheEntry = oldCacheEntry.entry; + if (cacheEntry === null) { + oldCache.delete(identifier); + cache.set(identifier, cacheEntry); + return null; + } else { + if (cacheEntry.etag !== etag) return null; + oldCache.delete(identifier); + cache.set(identifier, cacheEntry); + return cacheEntry.data; + } + } + gotHandlers.push((result, callback) => { + if (result === undefined) { + cache.set(identifier, null); + } else { + cache.set(identifier, { etag, data: result }); + } + return callback(); + }); + } + ); + compiler.cache.hooks.shutdown.tap( + { name: "MemoryWithGcCachePlugin", stage: Cache.STAGE_MEMORY }, + () => { + cache.clear(); + oldCache.clear(); + } + ); + } +} +module.exports = MemoryWithGcCachePlugin; diff --git a/node_modules/webpack/lib/cache/PackFileCacheStrategy.js b/node_modules/webpack/lib/cache/PackFileCacheStrategy.js new file mode 100644 index 00000000..3462e914 --- /dev/null +++ b/node_modules/webpack/lib/cache/PackFileCacheStrategy.js @@ -0,0 +1,1467 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const FileSystemInfo = require("../FileSystemInfo"); +const ProgressPlugin = require("../ProgressPlugin"); +const { formatSize } = require("../SizeFormatHelpers"); +const SerializerMiddleware = require("../serialization/SerializerMiddleware"); +const LazySet = require("../util/LazySet"); +const makeSerializable = require("../util/makeSerializable"); +const memoize = require("../util/memoize"); +const { + createFileSerializer, + NOT_SERIALIZABLE +} = require("../util/serialization"); + +/** @typedef {import("../../declarations/WebpackOptions").SnapshotOptions} SnapshotOptions */ +/** @typedef {import("../Cache").Etag} Etag */ +/** @typedef {import("../Compiler")} Compiler */ +/** @typedef {import("../FileSystemInfo").ResolveBuildDependenciesResult} ResolveBuildDependenciesResult */ +/** @typedef {import("../FileSystemInfo").Snapshot} Snapshot */ +/** @typedef {import("../logging/Logger").Logger} Logger */ +/** @typedef {import("../serialization/ObjectMiddleware").ObjectDeserializerContext} ObjectDeserializerContext */ +/** @typedef {import("../serialization/ObjectMiddleware").ObjectSerializerContext} ObjectSerializerContext */ +/** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */ + +/** @typedef {Map<string, string | false>} ResolveResults */ + +class PackContainer { + /** + * @param {Object} data stored data + * @param {string} version version identifier + * @param {Snapshot} buildSnapshot snapshot of all build dependencies + * @param {Set<string>} buildDependencies list of all unresolved build dependencies captured + * @param {ResolveResults} resolveResults result of the resolved build dependencies + * @param {Snapshot} resolveBuildDependenciesSnapshot snapshot of the dependencies of the build dependencies resolving + */ + constructor( + data, + version, + buildSnapshot, + buildDependencies, + resolveResults, + resolveBuildDependenciesSnapshot + ) { + this.data = data; + this.version = version; + this.buildSnapshot = buildSnapshot; + this.buildDependencies = buildDependencies; + this.resolveResults = resolveResults; + this.resolveBuildDependenciesSnapshot = resolveBuildDependenciesSnapshot; + } + + serialize({ write, writeLazy }) { + write(this.version); + write(this.buildSnapshot); + write(this.buildDependencies); + write(this.resolveResults); + write(this.resolveBuildDependenciesSnapshot); + writeLazy(this.data); + } + + /** + * @param {ObjectDeserializerContext} context context + */ + deserialize({ read }) { + this.version = read(); + this.buildSnapshot = read(); + this.buildDependencies = read(); + this.resolveResults = read(); + this.resolveBuildDependenciesSnapshot = read(); + this.data = read(); + } +} + +makeSerializable( + PackContainer, + "webpack/lib/cache/PackFileCacheStrategy", + "PackContainer" +); + +const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB +const CONTENT_COUNT_TO_MERGE = 10; +const MIN_ITEMS_IN_FRESH_PACK = 100; +const MAX_ITEMS_IN_FRESH_PACK = 50000; +const MAX_TIME_IN_FRESH_PACK = 1 * 60 * 1000; // 1 min + +class PackItemInfo { + /** + * @param {string} identifier identifier of item + * @param {string | null} etag etag of item + * @param {any} value fresh value of item + */ + constructor(identifier, etag, value) { + this.identifier = identifier; + this.etag = etag; + this.location = -1; + this.lastAccess = Date.now(); + this.freshValue = value; + } +} + +class Pack { + constructor(logger, maxAge) { + /** @type {Map<string, PackItemInfo>} */ + this.itemInfo = new Map(); + /** @type {(string | undefined)[]} */ + this.requests = []; + this.requestsTimeout = undefined; + /** @type {Map<string, PackItemInfo>} */ + this.freshContent = new Map(); + /** @type {(undefined | PackContent)[]} */ + this.content = []; + this.invalid = false; + this.logger = logger; + this.maxAge = maxAge; + } + + /** + * @param {string} identifier identifier + */ + _addRequest(identifier) { + this.requests.push(identifier); + if (this.requestsTimeout === undefined) { + this.requestsTimeout = setTimeout(() => { + this.requests.push(undefined); + this.requestsTimeout = undefined; + }, MAX_TIME_IN_FRESH_PACK); + if (this.requestsTimeout.unref) this.requestsTimeout.unref(); + } + } + + stopCapturingRequests() { + if (this.requestsTimeout !== undefined) { + clearTimeout(this.requestsTimeout); + this.requestsTimeout = undefined; + } + } + + /** + * @param {string} identifier unique name for the resource + * @param {string | null} etag etag of the resource + * @returns {any} cached content + */ + get(identifier, etag) { + const info = this.itemInfo.get(identifier); + this._addRequest(identifier); + if (info === undefined) { + return undefined; + } + if (info.etag !== etag) return null; + info.lastAccess = Date.now(); + const loc = info.location; + if (loc === -1) { + return info.freshValue; + } else { + if (!this.content[loc]) { + return undefined; + } + return /** @type {PackContent} */ (this.content[loc]).get(identifier); + } + } + + /** + * @param {string} identifier unique name for the resource + * @param {string | null} etag etag of the resource + * @param {any} data cached content + * @returns {void} + */ + set(identifier, etag, data) { + if (!this.invalid) { + this.invalid = true; + this.logger.log(`Pack got invalid because of write to: ${identifier}`); + } + const info = this.itemInfo.get(identifier); + if (info === undefined) { + const newInfo = new PackItemInfo(identifier, etag, data); + this.itemInfo.set(identifier, newInfo); + this._addRequest(identifier); + this.freshContent.set(identifier, newInfo); + } else { + const loc = info.location; + if (loc >= 0) { + this._addRequest(identifier); + this.freshContent.set(identifier, info); + const content = /** @type {PackContent} */ (this.content[loc]); + content.delete(identifier); + if (content.items.size === 0) { + this.content[loc] = undefined; + this.logger.debug("Pack %d got empty and is removed", loc); + } + } + info.freshValue = data; + info.lastAccess = Date.now(); + info.etag = etag; + info.location = -1; + } + } + + getContentStats() { + let count = 0; + let size = 0; + for (const content of this.content) { + if (content !== undefined) { + count++; + const s = content.getSize(); + if (s > 0) { + size += s; + } + } + } + return { count, size }; + } + + /** + * @returns {number} new location of data entries + */ + _findLocation() { + let i; + for (i = 0; i < this.content.length && this.content[i] !== undefined; i++); + return i; + } + + _gcAndUpdateLocation(items, usedItems, newLoc) { + let count = 0; + let lastGC; + const now = Date.now(); + for (const identifier of items) { + const info = this.itemInfo.get(identifier); + if (now - info.lastAccess > this.maxAge) { + this.itemInfo.delete(identifier); + items.delete(identifier); + usedItems.delete(identifier); + count++; + lastGC = identifier; + } else { + info.location = newLoc; + } + } + if (count > 0) { + this.logger.log( + "Garbage Collected %d old items at pack %d (%d items remaining) e. g. %s", + count, + newLoc, + items.size, + lastGC + ); + } + } + + _persistFreshContent() { + const itemsCount = this.freshContent.size; + if (itemsCount > 0) { + const packCount = Math.ceil(itemsCount / MAX_ITEMS_IN_FRESH_PACK); + const itemsPerPack = Math.ceil(itemsCount / packCount); + const packs = []; + let i = 0; + let ignoreNextTimeTick = false; + const createNextPack = () => { + const loc = this._findLocation(); + this.content[loc] = null; // reserve + const pack = { + /** @type {Set<string>} */ + items: new Set(), + /** @type {Map<string, any>} */ + map: new Map(), + loc + }; + packs.push(pack); + return pack; + }; + let pack = createNextPack(); + if (this.requestsTimeout !== undefined) + clearTimeout(this.requestsTimeout); + for (const identifier of this.requests) { + if (identifier === undefined) { + if (ignoreNextTimeTick) { + ignoreNextTimeTick = false; + } else if (pack.items.size >= MIN_ITEMS_IN_FRESH_PACK) { + i = 0; + pack = createNextPack(); + } + continue; + } + const info = this.freshContent.get(identifier); + if (info === undefined) continue; + pack.items.add(identifier); + pack.map.set(identifier, info.freshValue); + info.location = pack.loc; + info.freshValue = undefined; + this.freshContent.delete(identifier); + if (++i > itemsPerPack) { + i = 0; + pack = createNextPack(); + ignoreNextTimeTick = true; + } + } + this.requests.length = 0; + for (const pack of packs) { + this.content[pack.loc] = new PackContent( + pack.items, + new Set(pack.items), + new PackContentItems(pack.map) + ); + } + this.logger.log( + `${itemsCount} fresh items in cache put into pack ${ + packs.length > 1 + ? packs + .map(pack => `${pack.loc} (${pack.items.size} items)`) + .join(", ") + : packs[0].loc + }` + ); + } + } + + /** + * Merges small content files to a single content file + */ + _optimizeSmallContent() { + // 1. Find all small content files + // Treat unused content files separately to avoid + // a merge-split cycle + /** @type {number[]} */ + const smallUsedContents = []; + /** @type {number} */ + let smallUsedContentSize = 0; + /** @type {number[]} */ + const smallUnusedContents = []; + /** @type {number} */ + let smallUnusedContentSize = 0; + for (let i = 0; i < this.content.length; i++) { + const content = this.content[i]; + if (content === undefined) continue; + if (content.outdated) continue; + const size = content.getSize(); + if (size < 0 || size > MIN_CONTENT_SIZE) continue; + if (content.used.size > 0) { + smallUsedContents.push(i); + smallUsedContentSize += size; + } else { + smallUnusedContents.push(i); + smallUnusedContentSize += size; + } + } + + // 2. Check if minimum number is reached + let mergedIndices; + if ( + smallUsedContents.length >= CONTENT_COUNT_TO_MERGE || + smallUsedContentSize > MIN_CONTENT_SIZE + ) { + mergedIndices = smallUsedContents; + } else if ( + smallUnusedContents.length >= CONTENT_COUNT_TO_MERGE || + smallUnusedContentSize > MIN_CONTENT_SIZE + ) { + mergedIndices = smallUnusedContents; + } else return; + + /** @type {PackContent[] } */ + const mergedContent = []; + + // 3. Remove old content entries + for (const i of mergedIndices) { + mergedContent.push(/** @type {PackContent} */ (this.content[i])); + this.content[i] = undefined; + } + + // 4. Determine merged items + /** @type {Set<string>} */ + const mergedItems = new Set(); + /** @type {Set<string>} */ + const mergedUsedItems = new Set(); + /** @type {(function(Map<string, any>): Promise<void>)[]} */ + const addToMergedMap = []; + for (const content of mergedContent) { + for (const identifier of content.items) { + mergedItems.add(identifier); + } + for (const identifier of content.used) { + mergedUsedItems.add(identifier); + } + addToMergedMap.push(async map => { + // unpack existing content + // after that values are accessible in .content + await content.unpack( + "it should be merged with other small pack contents" + ); + for (const [identifier, value] of content.content) { + map.set(identifier, value); + } + }); + } + + // 5. GC and update location of merged items + const newLoc = this._findLocation(); + this._gcAndUpdateLocation(mergedItems, mergedUsedItems, newLoc); + + // 6. If not empty, store content somewhere + if (mergedItems.size > 0) { + this.content[newLoc] = new PackContent( + mergedItems, + mergedUsedItems, + memoize(async () => { + /** @type {Map<string, any>} */ + const map = new Map(); + await Promise.all(addToMergedMap.map(fn => fn(map))); + return new PackContentItems(map); + }) + ); + this.logger.log( + "Merged %d small files with %d cache items into pack %d", + mergedContent.length, + mergedItems.size, + newLoc + ); + } + } + + /** + * Split large content files with used and unused items + * into two parts to separate used from unused items + */ + _optimizeUnusedContent() { + // 1. Find a large content file with used and unused items + for (let i = 0; i < this.content.length; i++) { + const content = this.content[i]; + if (content === undefined) continue; + const size = content.getSize(); + if (size < MIN_CONTENT_SIZE) continue; + const used = content.used.size; + const total = content.items.size; + if (used > 0 && used < total) { + // 2. Remove this content + this.content[i] = undefined; + + // 3. Determine items for the used content file + const usedItems = new Set(content.used); + const newLoc = this._findLocation(); + this._gcAndUpdateLocation(usedItems, usedItems, newLoc); + + // 4. Create content file for used items + if (usedItems.size > 0) { + this.content[newLoc] = new PackContent( + usedItems, + new Set(usedItems), + async () => { + await content.unpack( + "it should be splitted into used and unused items" + ); + const map = new Map(); + for (const identifier of usedItems) { + map.set(identifier, content.content.get(identifier)); + } + return new PackContentItems(map); + } + ); + } + + // 5. Determine items for the unused content file + const unusedItems = new Set(content.items); + const usedOfUnusedItems = new Set(); + for (const identifier of usedItems) { + unusedItems.delete(identifier); + } + const newUnusedLoc = this._findLocation(); + this._gcAndUpdateLocation(unusedItems, usedOfUnusedItems, newUnusedLoc); + + // 6. Create content file for unused items + if (unusedItems.size > 0) { + this.content[newUnusedLoc] = new PackContent( + unusedItems, + usedOfUnusedItems, + async () => { + await content.unpack( + "it should be splitted into used and unused items" + ); + const map = new Map(); + for (const identifier of unusedItems) { + map.set(identifier, content.content.get(identifier)); + } + return new PackContentItems(map); + } + ); + } + + this.logger.log( + "Split pack %d into pack %d with %d used items and pack %d with %d unused items", + i, + newLoc, + usedItems.size, + newUnusedLoc, + unusedItems.size + ); + + // optimizing only one of them is good enough and + // reduces the amount of serialization needed + return; + } + } + } + + /** + * Find the content with the oldest item and run GC on that. + * Only runs for one content to avoid large invalidation. + */ + _gcOldestContent() { + /** @type {PackItemInfo | undefined} */ + let oldest = undefined; + for (const info of this.itemInfo.values()) { + if (oldest === undefined || info.lastAccess < oldest.lastAccess) { + oldest = info; + } + } + if ( + Date.now() - /** @type {PackItemInfo} */ (oldest).lastAccess > + this.maxAge + ) { + const loc = /** @type {PackItemInfo} */ (oldest).location; + if (loc < 0) return; + const content = /** @type {PackContent} */ (this.content[loc]); + const items = new Set(content.items); + const usedItems = new Set(content.used); + this._gcAndUpdateLocation(items, usedItems, loc); + + this.content[loc] = + items.size > 0 + ? new PackContent(items, usedItems, async () => { + await content.unpack( + "it contains old items that should be garbage collected" + ); + const map = new Map(); + for (const identifier of items) { + map.set(identifier, content.content.get(identifier)); + } + return new PackContentItems(map); + }) + : undefined; + } + } + + serialize({ write, writeSeparate }) { + this._persistFreshContent(); + this._optimizeSmallContent(); + this._optimizeUnusedContent(); + this._gcOldestContent(); + for (const identifier of this.itemInfo.keys()) { + write(identifier); + } + write(null); // null as marker of the end of keys + for (const info of this.itemInfo.values()) { + write(info.etag); + } + for (const info of this.itemInfo.values()) { + write(info.lastAccess); + } + for (let i = 0; i < this.content.length; i++) { + const content = this.content[i]; + if (content !== undefined) { + write(content.items); + content.writeLazy(lazy => writeSeparate(lazy, { name: `${i}` })); + } else { + write(undefined); // undefined marks an empty content slot + } + } + write(null); // null as marker of the end of items + } + + deserialize({ read, logger }) { + this.logger = logger; + { + const items = []; + let item = read(); + while (item !== null) { + items.push(item); + item = read(); + } + this.itemInfo.clear(); + const infoItems = items.map(identifier => { + const info = new PackItemInfo(identifier, undefined, undefined); + this.itemInfo.set(identifier, info); + return info; + }); + for (const info of infoItems) { + info.etag = read(); + } + for (const info of infoItems) { + info.lastAccess = read(); + } + } + this.content.length = 0; + let items = read(); + while (items !== null) { + if (items === undefined) { + this.content.push(items); + } else { + const idx = this.content.length; + const lazy = read(); + this.content.push( + new PackContent( + items, + new Set(), + lazy, + logger, + `${this.content.length}` + ) + ); + for (const identifier of items) { + this.itemInfo.get(identifier).location = idx; + } + } + items = read(); + } + } +} + +makeSerializable(Pack, "webpack/lib/cache/PackFileCacheStrategy", "Pack"); + +class PackContentItems { + /** + * @param {Map<string, any>} map items + */ + constructor(map) { + this.map = map; + } + + serialize({ write, snapshot, rollback, logger, profile }) { + if (profile) { + write(false); + for (const [key, value] of this.map) { + const s = snapshot(); + try { + write(key); + const start = process.hrtime(); + write(value); + const durationHr = process.hrtime(start); + const duration = durationHr[0] * 1000 + durationHr[1] / 1e6; + if (duration > 1) { + if (duration > 500) + logger.error(`Serialization of '${key}': ${duration} ms`); + else if (duration > 50) + logger.warn(`Serialization of '${key}': ${duration} ms`); + else if (duration > 10) + logger.info(`Serialization of '${key}': ${duration} ms`); + else if (duration > 5) + logger.log(`Serialization of '${key}': ${duration} ms`); + else logger.debug(`Serialization of '${key}': ${duration} ms`); + } + } catch (e) { + rollback(s); + if (e === NOT_SERIALIZABLE) continue; + const msg = "Skipped not serializable cache item"; + if (e.message.includes("ModuleBuildError")) { + logger.log(`${msg} (in build error): ${e.message}`); + logger.debug(`${msg} '${key}' (in build error): ${e.stack}`); + } else { + logger.warn(`${msg}: ${e.message}`); + logger.debug(`${msg} '${key}': ${e.stack}`); + } + } + } + write(null); + return; + } + // Try to serialize all at once + const s = snapshot(); + try { + write(true); + write(this.map); + } catch (e) { + rollback(s); + + // Try to serialize each item on it's own + write(false); + for (const [key, value] of this.map) { + const s = snapshot(); + try { + write(key); + write(value); + } catch (e) { + rollback(s); + if (e === NOT_SERIALIZABLE) continue; + logger.warn( + `Skipped not serializable cache item '${key}': ${e.message}` + ); + logger.debug(e.stack); + } + } + write(null); + } + } + + deserialize({ read, logger, profile }) { + if (read()) { + this.map = read(); + } else if (profile) { + const map = new Map(); + let key = read(); + while (key !== null) { + const start = process.hrtime(); + const value = read(); + const durationHr = process.hrtime(start); + const duration = durationHr[0] * 1000 + durationHr[1] / 1e6; + if (duration > 1) { + if (duration > 100) + logger.error(`Deserialization of '${key}': ${duration} ms`); + else if (duration > 20) + logger.warn(`Deserialization of '${key}': ${duration} ms`); + else if (duration > 5) + logger.info(`Deserialization of '${key}': ${duration} ms`); + else if (duration > 2) + logger.log(`Deserialization of '${key}': ${duration} ms`); + else logger.debug(`Deserialization of '${key}': ${duration} ms`); + } + map.set(key, value); + key = read(); + } + this.map = map; + } else { + const map = new Map(); + let key = read(); + while (key !== null) { + map.set(key, read()); + key = read(); + } + this.map = map; + } + } +} + +makeSerializable( + PackContentItems, + "webpack/lib/cache/PackFileCacheStrategy", + "PackContentItems" +); + +class PackContent { + /* + This class can be in these states: + | this.lazy | this.content | this.outdated | state + A1 | undefined | Map | false | fresh content + A2 | undefined | Map | true | (will not happen) + B1 | lazy () => {} | undefined | false | not deserialized + B2 | lazy () => {} | undefined | true | not deserialized, but some items has been removed + C1 | lazy* () => {} | Map | false | deserialized + C2 | lazy* () => {} | Map | true | deserialized, and some items has been removed + + this.used is a subset of this.items. + this.items is a subset of this.content.keys() resp. this.lazy().map.keys() + When this.outdated === false, this.items === this.content.keys() resp. this.lazy().map.keys() + When this.outdated === true, this.items should be used to recreated this.lazy/this.content. + When this.lazy and this.content is set, they contain the same data. + this.get must only be called with a valid item from this.items. + In state C this.lazy is unMemoized + */ + + /** + * @param {Set<string>} items keys + * @param {Set<string>} usedItems used keys + * @param {PackContentItems | function(): Promise<PackContentItems>} dataOrFn sync or async content + * @param {Logger=} logger logger for logging + * @param {string=} lazyName name of dataOrFn for logging + */ + constructor(items, usedItems, dataOrFn, logger, lazyName) { + this.items = items; + /** @type {function(): Promise<PackContentItems> | PackContentItems} */ + this.lazy = typeof dataOrFn === "function" ? dataOrFn : undefined; + /** @type {Map<string, any>} */ + this.content = typeof dataOrFn === "function" ? undefined : dataOrFn.map; + this.outdated = false; + this.used = usedItems; + this.logger = logger; + this.lazyName = lazyName; + } + + get(identifier) { + this.used.add(identifier); + if (this.content) { + return this.content.get(identifier); + } + + // We are in state B + const { lazyName } = this; + /** @type {string | undefined} */ + let timeMessage; + if (lazyName) { + // only log once + this.lazyName = undefined; + timeMessage = `restore cache content ${lazyName} (${formatSize( + this.getSize() + )})`; + this.logger.log( + `starting to restore cache content ${lazyName} (${formatSize( + this.getSize() + )}) because of request to: ${identifier}` + ); + this.logger.time(timeMessage); + } + const value = this.lazy(); + if ("then" in value) { + return value.then(data => { + const map = data.map; + if (timeMessage) { + this.logger.timeEnd(timeMessage); + } + // Move to state C + this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); + return map.get(identifier); + }); + } else { + const map = value.map; + if (timeMessage) { + this.logger.timeEnd(timeMessage); + } + // Move to state C + this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); + return map.get(identifier); + } + } + + /** + * @param {string} reason explanation why unpack is necessary + * @returns {void | Promise<void>} maybe a promise if lazy + */ + unpack(reason) { + if (this.content) return; + + // Move from state B to C + if (this.lazy) { + const { lazyName } = this; + /** @type {string | undefined} */ + let timeMessage; + if (lazyName) { + // only log once + this.lazyName = undefined; + timeMessage = `unpack cache content ${lazyName} (${formatSize( + this.getSize() + )})`; + this.logger.log( + `starting to unpack cache content ${lazyName} (${formatSize( + this.getSize() + )}) because ${reason}` + ); + this.logger.time(timeMessage); + } + const value = this.lazy(); + if ("then" in value) { + return value.then(data => { + if (timeMessage) { + this.logger.timeEnd(timeMessage); + } + this.content = data.map; + }); + } else { + if (timeMessage) { + this.logger.timeEnd(timeMessage); + } + this.content = value.map; + } + } + } + + /** + * @returns {number} size of the content or -1 if not known + */ + getSize() { + if (!this.lazy) return -1; + const options = /** @type {any} */ (this.lazy).options; + if (!options) return -1; + const size = options.size; + if (typeof size !== "number") return -1; + return size; + } + + /** + * @param {string} identifier identifier + */ + delete(identifier) { + this.items.delete(identifier); + this.used.delete(identifier); + this.outdated = true; + } + + /** + * @template T + * @param {function(any): function(): Promise<PackContentItems> | PackContentItems} write write function + * @returns {void} + */ + writeLazy(write) { + if (!this.outdated && this.lazy) { + // State B1 or C1 + // this.lazy is still the valid deserialized version + write(this.lazy); + return; + } + if (!this.outdated && this.content) { + // State A1 + const map = new Map(this.content); + // Move to state C1 + this.lazy = SerializerMiddleware.unMemoizeLazy( + write(() => new PackContentItems(map)) + ); + return; + } + if (this.content) { + // State A2 or C2 + /** @type {Map<string, any>} */ + const map = new Map(); + for (const item of this.items) { + map.set(item, this.content.get(item)); + } + // Move to state C1 + this.outdated = false; + this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy( + write(() => new PackContentItems(map)) + ); + return; + } + // State B2 + const { lazyName } = this; + /** @type {string | undefined} */ + let timeMessage; + if (lazyName) { + // only log once + this.lazyName = undefined; + timeMessage = `unpack cache content ${lazyName} (${formatSize( + this.getSize() + )})`; + this.logger.log( + `starting to unpack cache content ${lazyName} (${formatSize( + this.getSize() + )}) because it's outdated and need to be serialized` + ); + this.logger.time(timeMessage); + } + const value = this.lazy(); + this.outdated = false; + if ("then" in value) { + // Move to state B1 + this.lazy = write(() => + value.then(data => { + if (timeMessage) { + this.logger.timeEnd(timeMessage); + } + const oldMap = data.map; + /** @type {Map<string, any>} */ + const map = new Map(); + for (const item of this.items) { + map.set(item, oldMap.get(item)); + } + // Move to state C1 (or maybe C2) + this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); + + return new PackContentItems(map); + }) + ); + } else { + // Move to state C1 + if (timeMessage) { + this.logger.timeEnd(timeMessage); + } + const oldMap = value.map; + /** @type {Map<string, any>} */ + const map = new Map(); + for (const item of this.items) { + map.set(item, oldMap.get(item)); + } + this.content = map; + this.lazy = write(() => new PackContentItems(map)); + } + } +} + +/** + * @param {Buffer} buf buffer + * @returns {Buffer} buffer that can be collected + */ +const allowCollectingMemory = buf => { + const wasted = buf.buffer.byteLength - buf.byteLength; + if (wasted > 8192 && (wasted > 1048576 || wasted > buf.byteLength)) { + return Buffer.from(buf); + } + return buf; +}; + +class PackFileCacheStrategy { + /** + * @param {Object} options options + * @param {Compiler} options.compiler the compiler + * @param {IntermediateFileSystem} options.fs the filesystem + * @param {string} options.context the context directory + * @param {string} options.cacheLocation the location of the cache data + * @param {string} options.version version identifier + * @param {Logger} options.logger a logger + * @param {SnapshotOptions} options.snapshot options regarding snapshotting + * @param {number} options.maxAge max age of cache items + * @param {boolean | undefined} options.profile track and log detailed timing information for individual cache items + * @param {boolean | undefined} options.allowCollectingMemory allow to collect unused memory created during deserialization + * @param {false | "gzip" | "brotli" | undefined} options.compression compression used + * @param {boolean | undefined} options.readonly disable storing cache into filesystem + */ + constructor({ + compiler, + fs, + context, + cacheLocation, + version, + logger, + snapshot, + maxAge, + profile, + allowCollectingMemory, + compression, + readonly + }) { + this.fileSerializer = createFileSerializer( + fs, + compiler.options.output.hashFunction + ); + this.fileSystemInfo = new FileSystemInfo(fs, { + managedPaths: snapshot.managedPaths, + immutablePaths: snapshot.immutablePaths, + logger: logger.getChildLogger("webpack.FileSystemInfo"), + hashFunction: compiler.options.output.hashFunction + }); + this.compiler = compiler; + this.context = context; + this.cacheLocation = cacheLocation; + this.version = version; + this.logger = logger; + this.maxAge = maxAge; + this.profile = profile; + this.readonly = readonly; + this.allowCollectingMemory = allowCollectingMemory; + this.compression = compression; + this._extension = + compression === "brotli" + ? ".pack.br" + : compression === "gzip" + ? ".pack.gz" + : ".pack"; + this.snapshot = snapshot; + /** @type {Set<string>} */ + this.buildDependencies = new Set(); + /** @type {LazySet<string>} */ + this.newBuildDependencies = new LazySet(); + /** @type {Snapshot | undefined} */ + this.resolveBuildDependenciesSnapshot = undefined; + /** @type {ResolveResults | undefined} */ + this.resolveResults = undefined; + /** @type {Snapshot | undefined} */ + this.buildSnapshot = undefined; + /** @type {Promise<Pack> | undefined} */ + this.packPromise = this._openPack(); + this.storePromise = Promise.resolve(); + } + + /** + * @returns {Promise<Pack>} pack + */ + _getPack() { + if (this.packPromise === undefined) { + this.packPromise = this.storePromise.then(() => this._openPack()); + } + return this.packPromise; + } + + /** + * @returns {Promise<Pack>} the pack + */ + _openPack() { + const { logger, profile, cacheLocation, version } = this; + /** @type {Snapshot} */ + let buildSnapshot; + /** @type {Set<string>} */ + let buildDependencies; + /** @type {Set<string>} */ + let newBuildDependencies; + /** @type {Snapshot} */ + let resolveBuildDependenciesSnapshot; + /** @type {ResolveResults | undefined} */ + let resolveResults; + logger.time("restore cache container"); + return this.fileSerializer + .deserialize(null, { + filename: `${cacheLocation}/index${this._extension}`, + extension: `${this._extension}`, + logger, + profile, + retainedBuffer: this.allowCollectingMemory + ? allowCollectingMemory + : undefined + }) + .catch(err => { + if (err.code !== "ENOENT") { + logger.warn( + `Restoring pack failed from ${cacheLocation}${this._extension}: ${err}` + ); + logger.debug(err.stack); + } else { + logger.debug( + `No pack exists at ${cacheLocation}${this._extension}: ${err}` + ); + } + return undefined; + }) + .then(packContainer => { + logger.timeEnd("restore cache container"); + if (!packContainer) return undefined; + if (!(packContainer instanceof PackContainer)) { + logger.warn( + `Restored pack from ${cacheLocation}${this._extension}, but contained content is unexpected.`, + packContainer + ); + return undefined; + } + if (packContainer.version !== version) { + logger.log( + `Restored pack from ${cacheLocation}${this._extension}, but version doesn't match.` + ); + return undefined; + } + logger.time("check build dependencies"); + return Promise.all([ + new Promise((resolve, reject) => { + this.fileSystemInfo.checkSnapshotValid( + packContainer.buildSnapshot, + (err, valid) => { + if (err) { + logger.log( + `Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of build dependencies errored: ${err}.` + ); + logger.debug(err.stack); + return resolve(false); + } + if (!valid) { + logger.log( + `Restored pack from ${cacheLocation}${this._extension}, but build dependencies have changed.` + ); + return resolve(false); + } + buildSnapshot = packContainer.buildSnapshot; + return resolve(true); + } + ); + }), + new Promise((resolve, reject) => { + this.fileSystemInfo.checkSnapshotValid( + packContainer.resolveBuildDependenciesSnapshot, + (err, valid) => { + if (err) { + logger.log( + `Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of resolving of build dependencies errored: ${err}.` + ); + logger.debug(err.stack); + return resolve(false); + } + if (valid) { + resolveBuildDependenciesSnapshot = + packContainer.resolveBuildDependenciesSnapshot; + buildDependencies = packContainer.buildDependencies; + resolveResults = packContainer.resolveResults; + return resolve(true); + } + logger.log( + "resolving of build dependencies is invalid, will re-resolve build dependencies" + ); + this.fileSystemInfo.checkResolveResultsValid( + packContainer.resolveResults, + (err, valid) => { + if (err) { + logger.log( + `Restored pack from ${cacheLocation}${this._extension}, but resolving of build dependencies errored: ${err}.` + ); + logger.debug(err.stack); + return resolve(false); + } + if (valid) { + newBuildDependencies = packContainer.buildDependencies; + resolveResults = packContainer.resolveResults; + return resolve(true); + } + logger.log( + `Restored pack from ${cacheLocation}${this._extension}, but build dependencies resolve to different locations.` + ); + return resolve(false); + } + ); + } + ); + }) + ]) + .catch(err => { + logger.timeEnd("check build dependencies"); + throw err; + }) + .then(([buildSnapshotValid, resolveValid]) => { + logger.timeEnd("check build dependencies"); + if (buildSnapshotValid && resolveValid) { + logger.time("restore cache content metadata"); + const d = packContainer.data(); + logger.timeEnd("restore cache content metadata"); + return d; + } + return undefined; + }); + }) + .then(pack => { + if (pack) { + pack.maxAge = this.maxAge; + this.buildSnapshot = buildSnapshot; + if (buildDependencies) this.buildDependencies = buildDependencies; + if (newBuildDependencies) + this.newBuildDependencies.addAll(newBuildDependencies); + this.resolveResults = resolveResults; + this.resolveBuildDependenciesSnapshot = + resolveBuildDependenciesSnapshot; + return pack; + } + return new Pack(logger, this.maxAge); + }) + .catch(err => { + this.logger.warn( + `Restoring pack from ${cacheLocation}${this._extension} failed: ${err}` + ); + this.logger.debug(err.stack); + return new Pack(logger, this.maxAge); + }); + } + + /** + * @param {string} identifier unique name for the resource + * @param {Etag | null} etag etag of the resource + * @param {any} data cached content + * @returns {Promise<void>} promise + */ + store(identifier, etag, data) { + if (this.readonly) return Promise.resolve(); + + return this._getPack().then(pack => { + pack.set(identifier, etag === null ? null : etag.toString(), data); + }); + } + + /** + * @param {string} identifier unique name for the resource + * @param {Etag | null} etag etag of the resource + * @returns {Promise<any>} promise to the cached content + */ + restore(identifier, etag) { + return this._getPack() + .then(pack => + pack.get(identifier, etag === null ? null : etag.toString()) + ) + .catch(err => { + if (err && err.code !== "ENOENT") { + this.logger.warn( + `Restoring failed for ${identifier} from pack: ${err}` + ); + this.logger.debug(err.stack); + } + }); + } + + /** + * @param {LazySet<string>} dependencies dependencies to store + */ + storeBuildDependencies(dependencies) { + if (this.readonly) return; + this.newBuildDependencies.addAll(dependencies); + } + + afterAllStored() { + const packPromise = this.packPromise; + if (packPromise === undefined) return Promise.resolve(); + const reportProgress = ProgressPlugin.getReporter(this.compiler); + return (this.storePromise = packPromise + .then(pack => { + pack.stopCapturingRequests(); + if (!pack.invalid) return; + this.packPromise = undefined; + this.logger.log(`Storing pack...`); + let promise; + const newBuildDependencies = new Set(); + for (const dep of this.newBuildDependencies) { + if (!this.buildDependencies.has(dep)) { + newBuildDependencies.add(dep); + } + } + if (newBuildDependencies.size > 0 || !this.buildSnapshot) { + if (reportProgress) reportProgress(0.5, "resolve build dependencies"); + this.logger.debug( + `Capturing build dependencies... (${Array.from( + newBuildDependencies + ).join(", ")})` + ); + promise = new Promise((resolve, reject) => { + this.logger.time("resolve build dependencies"); + this.fileSystemInfo.resolveBuildDependencies( + this.context, + newBuildDependencies, + (err, result) => { + this.logger.timeEnd("resolve build dependencies"); + if (err) return reject(err); + + this.logger.time("snapshot build dependencies"); + const { + files, + directories, + missing, + resolveResults, + resolveDependencies + } = /** @type {ResolveBuildDependenciesResult} */ (result); + if (this.resolveResults) { + for (const [key, value] of resolveResults) { + this.resolveResults.set(key, value); + } + } else { + this.resolveResults = resolveResults; + } + if (reportProgress) { + reportProgress( + 0.6, + "snapshot build dependencies", + "resolving" + ); + } + this.fileSystemInfo.createSnapshot( + undefined, + resolveDependencies.files, + resolveDependencies.directories, + resolveDependencies.missing, + this.snapshot.resolveBuildDependencies, + (err, snapshot) => { + if (err) { + this.logger.timeEnd("snapshot build dependencies"); + return reject(err); + } + if (!snapshot) { + this.logger.timeEnd("snapshot build dependencies"); + return reject( + new Error("Unable to snapshot resolve dependencies") + ); + } + if (this.resolveBuildDependenciesSnapshot) { + this.resolveBuildDependenciesSnapshot = + this.fileSystemInfo.mergeSnapshots( + this.resolveBuildDependenciesSnapshot, + snapshot + ); + } else { + this.resolveBuildDependenciesSnapshot = snapshot; + } + if (reportProgress) { + reportProgress( + 0.7, + "snapshot build dependencies", + "modules" + ); + } + this.fileSystemInfo.createSnapshot( + undefined, + files, + directories, + missing, + this.snapshot.buildDependencies, + (err, snapshot) => { + this.logger.timeEnd("snapshot build dependencies"); + if (err) return reject(err); + if (!snapshot) { + return reject( + new Error("Unable to snapshot build dependencies") + ); + } + this.logger.debug("Captured build dependencies"); + + if (this.buildSnapshot) { + this.buildSnapshot = + this.fileSystemInfo.mergeSnapshots( + this.buildSnapshot, + snapshot + ); + } else { + this.buildSnapshot = snapshot; + } + + resolve(); + } + ); + } + ); + } + ); + }); + } else { + promise = Promise.resolve(); + } + return promise.then(() => { + if (reportProgress) reportProgress(0.8, "serialize pack"); + this.logger.time(`store pack`); + const updatedBuildDependencies = new Set(this.buildDependencies); + for (const dep of newBuildDependencies) { + updatedBuildDependencies.add(dep); + } + const content = new PackContainer( + pack, + this.version, + /** @type {Snapshot} */ (this.buildSnapshot), + updatedBuildDependencies, + this.resolveResults, + this.resolveBuildDependenciesSnapshot + ); + return this.fileSerializer + .serialize(content, { + filename: `${this.cacheLocation}/index${this._extension}`, + extension: `${this._extension}`, + logger: this.logger, + profile: this.profile + }) + .then(() => { + for (const dep of newBuildDependencies) { + this.buildDependencies.add(dep); + } + this.newBuildDependencies.clear(); + this.logger.timeEnd(`store pack`); + const stats = pack.getContentStats(); + this.logger.log( + "Stored pack (%d items, %d files, %d MiB)", + pack.itemInfo.size, + stats.count, + Math.round(stats.size / 1024 / 1024) + ); + }) + .catch(err => { + this.logger.timeEnd(`store pack`); + this.logger.warn(`Caching failed for pack: ${err}`); + this.logger.debug(err.stack); + }); + }); + }) + .catch(err => { + this.logger.warn(`Caching failed for pack: ${err}`); + this.logger.debug(err.stack); + })); + } + + clear() { + this.fileSystemInfo.clear(); + this.buildDependencies.clear(); + this.newBuildDependencies.clear(); + this.resolveBuildDependenciesSnapshot = undefined; + this.resolveResults = undefined; + this.buildSnapshot = undefined; + this.packPromise = undefined; + } +} + +module.exports = PackFileCacheStrategy; diff --git a/node_modules/webpack/lib/cache/ResolverCachePlugin.js b/node_modules/webpack/lib/cache/ResolverCachePlugin.js new file mode 100644 index 00000000..4659d617 --- /dev/null +++ b/node_modules/webpack/lib/cache/ResolverCachePlugin.js @@ -0,0 +1,357 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const LazySet = require("../util/LazySet"); +const makeSerializable = require("../util/makeSerializable"); + +/** @typedef {import("enhanced-resolve/lib/Resolver")} Resolver */ +/** @typedef {import("../CacheFacade").ItemCacheFacade} ItemCacheFacade */ +/** @typedef {import("../Compiler")} Compiler */ +/** @typedef {import("../FileSystemInfo")} FileSystemInfo */ +/** @typedef {import("../FileSystemInfo").Snapshot} Snapshot */ + +class CacheEntry { + constructor(result, snapshot) { + this.result = result; + this.snapshot = snapshot; + } + + serialize({ write }) { + write(this.result); + write(this.snapshot); + } + + deserialize({ read }) { + this.result = read(); + this.snapshot = read(); + } +} + +makeSerializable(CacheEntry, "webpack/lib/cache/ResolverCachePlugin"); + +/** + * @template T + * @param {Set<T> | LazySet<T>} set set to add items to + * @param {Set<T> | LazySet<T>} otherSet set to add items from + * @returns {void} + */ +const addAllToSet = (set, otherSet) => { + if (set instanceof LazySet) { + set.addAll(otherSet); + } else { + for (const item of otherSet) { + set.add(item); + } + } +}; + +/** + * @param {Object} object an object + * @param {boolean} excludeContext if true, context is not included in string + * @returns {string} stringified version + */ +const objectToString = (object, excludeContext) => { + let str = ""; + for (const key in object) { + if (excludeContext && key === "context") continue; + const value = object[key]; + if (typeof value === "object" && value !== null) { + str += `|${key}=[${objectToString(value, false)}|]`; + } else { + str += `|${key}=|${value}`; + } + } + return str; +}; + +class ResolverCachePlugin { + /** + * Apply the plugin + * @param {Compiler} compiler the compiler instance + * @returns {void} + */ + apply(compiler) { + const cache = compiler.getCache("ResolverCachePlugin"); + /** @type {FileSystemInfo} */ + let fileSystemInfo; + let snapshotOptions; + let realResolves = 0; + let cachedResolves = 0; + let cacheInvalidResolves = 0; + let concurrentResolves = 0; + compiler.hooks.thisCompilation.tap("ResolverCachePlugin", compilation => { + snapshotOptions = compilation.options.snapshot.resolve; + fileSystemInfo = compilation.fileSystemInfo; + compilation.hooks.finishModules.tap("ResolverCachePlugin", () => { + if (realResolves + cachedResolves > 0) { + const logger = compilation.getLogger("webpack.ResolverCachePlugin"); + logger.log( + `${Math.round( + (100 * realResolves) / (realResolves + cachedResolves) + )}% really resolved (${realResolves} real resolves with ${cacheInvalidResolves} cached but invalid, ${cachedResolves} cached valid, ${concurrentResolves} concurrent)` + ); + realResolves = 0; + cachedResolves = 0; + cacheInvalidResolves = 0; + concurrentResolves = 0; + } + }); + }); + /** + * @param {ItemCacheFacade} itemCache cache + * @param {Resolver} resolver the resolver + * @param {Object} resolveContext context for resolving meta info + * @param {Object} request the request info object + * @param {function((Error | null)=, Object=): void} callback callback function + * @returns {void} + */ + const doRealResolve = ( + itemCache, + resolver, + resolveContext, + request, + callback + ) => { + realResolves++; + const newRequest = { + _ResolverCachePluginCacheMiss: true, + ...request + }; + const newResolveContext = { + ...resolveContext, + stack: new Set(), + /** @type {LazySet<string>} */ + missingDependencies: new LazySet(), + /** @type {LazySet<string>} */ + fileDependencies: new LazySet(), + /** @type {LazySet<string>} */ + contextDependencies: new LazySet() + }; + let yieldResult; + let withYield = false; + if (typeof newResolveContext.yield === "function") { + yieldResult = []; + withYield = true; + newResolveContext.yield = obj => yieldResult.push(obj); + } + const propagate = key => { + if (resolveContext[key]) { + addAllToSet(resolveContext[key], newResolveContext[key]); + } + }; + const resolveTime = Date.now(); + resolver.doResolve( + resolver.hooks.resolve, + newRequest, + "Cache miss", + newResolveContext, + (err, result) => { + propagate("fileDependencies"); + propagate("contextDependencies"); + propagate("missingDependencies"); + if (err) return callback(err); + const fileDependencies = newResolveContext.fileDependencies; + const contextDependencies = newResolveContext.contextDependencies; + const missingDependencies = newResolveContext.missingDependencies; + fileSystemInfo.createSnapshot( + resolveTime, + fileDependencies, + contextDependencies, + missingDependencies, + snapshotOptions, + (err, snapshot) => { + if (err) return callback(err); + const resolveResult = withYield ? yieldResult : result; + // since we intercept resolve hook + // we still can get result in callback + if (withYield && result) yieldResult.push(result); + if (!snapshot) { + if (resolveResult) return callback(null, resolveResult); + return callback(); + } + itemCache.store( + new CacheEntry(resolveResult, snapshot), + storeErr => { + if (storeErr) return callback(storeErr); + if (resolveResult) return callback(null, resolveResult); + callback(); + } + ); + } + ); + } + ); + }; + compiler.resolverFactory.hooks.resolver.intercept({ + factory(type, hook) { + /** @type {Map<string, (function(Error=, Object=): void)[]>} */ + const activeRequests = new Map(); + /** @type {Map<string, [function(Error=, Object=): void, function(Error=, Object=): void][]>} */ + const activeRequestsWithYield = new Map(); + hook.tap( + "ResolverCachePlugin", + /** + * @param {Resolver} resolver the resolver + * @param {Object} options resolve options + * @param {Object} userOptions resolve options passed by the user + * @returns {void} + */ + (resolver, options, userOptions) => { + if (options.cache !== true) return; + const optionsIdent = objectToString(userOptions, false); + const cacheWithContext = + options.cacheWithContext !== undefined + ? options.cacheWithContext + : false; + resolver.hooks.resolve.tapAsync( + { + name: "ResolverCachePlugin", + stage: -100 + }, + (request, resolveContext, callback) => { + if (request._ResolverCachePluginCacheMiss || !fileSystemInfo) { + return callback(); + } + const withYield = typeof resolveContext.yield === "function"; + const identifier = `${type}${ + withYield ? "|yield" : "|default" + }${optionsIdent}${objectToString(request, !cacheWithContext)}`; + + if (withYield) { + const activeRequest = activeRequestsWithYield.get(identifier); + if (activeRequest) { + activeRequest[0].push(callback); + activeRequest[1].push(resolveContext.yield); + return; + } + } else { + const activeRequest = activeRequests.get(identifier); + if (activeRequest) { + activeRequest.push(callback); + return; + } + } + const itemCache = cache.getItemCache(identifier, null); + let callbacks, yields; + const done = withYield + ? (err, result) => { + if (callbacks === undefined) { + if (err) { + callback(err); + } else { + if (result) + for (const r of result) resolveContext.yield(r); + callback(null, null); + } + yields = undefined; + callbacks = false; + } else { + if (err) { + for (const cb of callbacks) cb(err); + } else { + for (let i = 0; i < callbacks.length; i++) { + const cb = callbacks[i]; + const yield_ = yields[i]; + if (result) for (const r of result) yield_(r); + cb(null, null); + } + } + activeRequestsWithYield.delete(identifier); + yields = undefined; + callbacks = false; + } + } + : (err, result) => { + if (callbacks === undefined) { + callback(err, result); + callbacks = false; + } else { + for (const callback of callbacks) { + callback(err, result); + } + activeRequests.delete(identifier); + callbacks = false; + } + }; + /** + * @param {Error=} err error if any + * @param {CacheEntry=} cacheEntry cache entry + * @returns {void} + */ + const processCacheResult = (err, cacheEntry) => { + if (err) return done(err); + + if (cacheEntry) { + const { snapshot, result } = cacheEntry; + fileSystemInfo.checkSnapshotValid( + snapshot, + (err, valid) => { + if (err || !valid) { + cacheInvalidResolves++; + return doRealResolve( + itemCache, + resolver, + resolveContext, + request, + done + ); + } + cachedResolves++; + if (resolveContext.missingDependencies) { + addAllToSet( + resolveContext.missingDependencies, + snapshot.getMissingIterable() + ); + } + if (resolveContext.fileDependencies) { + addAllToSet( + resolveContext.fileDependencies, + snapshot.getFileIterable() + ); + } + if (resolveContext.contextDependencies) { + addAllToSet( + resolveContext.contextDependencies, + snapshot.getContextIterable() + ); + } + done(null, result); + } + ); + } else { + doRealResolve( + itemCache, + resolver, + resolveContext, + request, + done + ); + } + }; + itemCache.get(processCacheResult); + if (withYield && callbacks === undefined) { + callbacks = [callback]; + yields = [resolveContext.yield]; + activeRequestsWithYield.set( + identifier, + /** @type {[any, any]} */ ([callbacks, yields]) + ); + } else if (callbacks === undefined) { + callbacks = [callback]; + activeRequests.set(identifier, callbacks); + } + } + ); + } + ); + return hook; + } + }); + } +} + +module.exports = ResolverCachePlugin; diff --git a/node_modules/webpack/lib/cache/getLazyHashedEtag.js b/node_modules/webpack/lib/cache/getLazyHashedEtag.js new file mode 100644 index 00000000..6cdf6c3a --- /dev/null +++ b/node_modules/webpack/lib/cache/getLazyHashedEtag.js @@ -0,0 +1,81 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const createHash = require("../util/createHash"); + +/** @typedef {import("../util/Hash")} Hash */ +/** @typedef {typeof import("../util/Hash")} HashConstructor */ + +/** + * @typedef {Object} HashableObject + * @property {function(Hash): void} updateHash + */ + +class LazyHashedEtag { + /** + * @param {HashableObject} obj object with updateHash method + * @param {string | HashConstructor} hashFunction the hash function to use + */ + constructor(obj, hashFunction = "md4") { + this._obj = obj; + this._hash = undefined; + this._hashFunction = hashFunction; + } + + /** + * @returns {string} hash of object + */ + toString() { + if (this._hash === undefined) { + const hash = createHash(this._hashFunction); + this._obj.updateHash(hash); + this._hash = /** @type {string} */ (hash.digest("base64")); + } + return this._hash; + } +} + +/** @type {Map<string | HashConstructor, WeakMap<HashableObject, LazyHashedEtag>>} */ +const mapStrings = new Map(); + +/** @type {WeakMap<HashConstructor, WeakMap<HashableObject, LazyHashedEtag>>} */ +const mapObjects = new WeakMap(); + +/** + * @param {HashableObject} obj object with updateHash method + * @param {string | HashConstructor} hashFunction the hash function to use + * @returns {LazyHashedEtag} etag + */ +const getter = (obj, hashFunction = "md4") => { + let innerMap; + if (typeof hashFunction === "string") { + innerMap = mapStrings.get(hashFunction); + if (innerMap === undefined) { + const newHash = new LazyHashedEtag(obj, hashFunction); + innerMap = new WeakMap(); + innerMap.set(obj, newHash); + mapStrings.set(hashFunction, innerMap); + return newHash; + } + } else { + innerMap = mapObjects.get(hashFunction); + if (innerMap === undefined) { + const newHash = new LazyHashedEtag(obj, hashFunction); + innerMap = new WeakMap(); + innerMap.set(obj, newHash); + mapObjects.set(hashFunction, innerMap); + return newHash; + } + } + const hash = innerMap.get(obj); + if (hash !== undefined) return hash; + const newHash = new LazyHashedEtag(obj, hashFunction); + innerMap.set(obj, newHash); + return newHash; +}; + +module.exports = getter; diff --git a/node_modules/webpack/lib/cache/mergeEtags.js b/node_modules/webpack/lib/cache/mergeEtags.js new file mode 100644 index 00000000..8c6af34a --- /dev/null +++ b/node_modules/webpack/lib/cache/mergeEtags.js @@ -0,0 +1,74 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +/** @typedef {import("../Cache").Etag} Etag */ + +class MergedEtag { + /** + * @param {Etag} a first + * @param {Etag} b second + */ + constructor(a, b) { + this.a = a; + this.b = b; + } + + toString() { + return `${this.a.toString()}|${this.b.toString()}`; + } +} + +const dualObjectMap = new WeakMap(); +const objectStringMap = new WeakMap(); + +/** + * @param {Etag} a first + * @param {Etag} b second + * @returns {Etag} result + */ +const mergeEtags = (a, b) => { + if (typeof a === "string") { + if (typeof b === "string") { + return `${a}|${b}`; + } else { + const temp = b; + b = a; + a = temp; + } + } else { + if (typeof b !== "string") { + // both a and b are objects + let map = dualObjectMap.get(a); + if (map === undefined) { + dualObjectMap.set(a, (map = new WeakMap())); + } + const mergedEtag = map.get(b); + if (mergedEtag === undefined) { + const newMergedEtag = new MergedEtag(a, b); + map.set(b, newMergedEtag); + return newMergedEtag; + } else { + return mergedEtag; + } + } + } + // a is object, b is string + let map = objectStringMap.get(a); + if (map === undefined) { + objectStringMap.set(a, (map = new Map())); + } + const mergedEtag = map.get(b); + if (mergedEtag === undefined) { + const newMergedEtag = new MergedEtag(a, b); + map.set(b, newMergedEtag); + return newMergedEtag; + } else { + return mergedEtag; + } +}; + +module.exports = mergeEtags; |
