packages/jest-transform/src/ScriptTransformer.ts (807 lines of code) (raw):
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import {createHash} from 'crypto';
import * as path from 'path';
import {transformSync as babelTransform} from '@babel/core';
// @ts-expect-error: should just be `require.resolve`, but the tests mess that up
import babelPluginIstanbul from 'babel-plugin-istanbul';
import {fromSource as sourcemapFromSource} from 'convert-source-map';
import stableStringify = require('fast-json-stable-stringify');
import * as fs from 'graceful-fs';
import {addHook} from 'pirates';
import slash = require('slash');
import {sync as writeFileAtomic} from 'write-file-atomic';
import type {Config} from '@jest/types';
import HasteMap from 'jest-haste-map';
import {
createDirectory,
isPromise,
requireOrImportModule,
tryRealpath,
} from 'jest-util';
import handlePotentialSyntaxError from './enhanceUnexpectedTokenMessage';
import {
makeInvalidReturnValueError,
makeInvalidSourceMapWarning,
makeInvalidSyncTransformerError,
makeInvalidTransformerError,
} from './runtimeErrorsAndWarnings';
import shouldInstrument from './shouldInstrument';
import type {
Options,
ReducedTransformOptions,
RequireAndTranspileModuleOptions,
StringMap,
SyncTransformer,
TransformOptions,
TransformResult,
TransformedSource,
Transformer,
} from './types';
// Use `require` to avoid TS rootDir
const {version: VERSION} = require('../package.json');
type ProjectCache = {
configString: string;
ignorePatternsRegExp?: RegExp;
transformRegExp?: Array<[RegExp, string, Record<string, unknown>]>;
transformedFiles: Map<string, TransformResult>;
};
// This data structure is used to avoid recalculating some data every time that
// we need to transform a file. Since ScriptTransformer is instantiated for each
// file we need to keep this object in the local scope of this module.
const projectCaches = new Map<string, ProjectCache>();
// To reset the cache for specific changesets (rather than package version).
const CACHE_VERSION = '1';
async function waitForPromiseWithCleanup(
promise: Promise<unknown>,
cleanup: () => void,
) {
try {
await promise;
} finally {
cleanup();
}
}
class ScriptTransformer {
private readonly _cache: ProjectCache;
private readonly _transformCache = new Map<
string,
{transformer: Transformer; transformerConfig: unknown}
>();
private _transformsAreLoaded = false;
constructor(
private readonly _config: Config.ProjectConfig,
private readonly _cacheFS: StringMap,
) {
const configString = stableStringify(this._config);
let projectCache = projectCaches.get(configString);
if (!projectCache) {
projectCache = {
configString,
ignorePatternsRegExp: calcIgnorePatternRegExp(this._config),
transformRegExp: calcTransformRegExp(this._config),
transformedFiles: new Map(),
};
projectCaches.set(configString, projectCache);
}
this._cache = projectCache;
}
private _buildCacheKeyFromFileInfo(
fileData: string,
filename: string,
transformOptions: TransformOptions,
transformerCacheKey: string | undefined,
): string {
if (transformerCacheKey) {
return createHash('md5')
.update(transformerCacheKey)
.update(CACHE_VERSION)
.digest('hex');
}
return createHash('md5')
.update(fileData)
.update(transformOptions.configString)
.update(transformOptions.instrument ? 'instrument' : '')
.update(filename)
.update(CACHE_VERSION)
.digest('hex');
}
private _getCacheKey(
fileData: string,
filename: string,
options: ReducedTransformOptions,
): string {
const configString = this._cache.configString;
const {transformer, transformerConfig = {}} =
this._getTransformer(filename) || {};
let transformerCacheKey = undefined;
const transformOptions: TransformOptions = {
...options,
cacheFS: this._cacheFS,
config: this._config,
configString,
transformerConfig,
};
if (typeof transformer?.getCacheKey === 'function') {
transformerCacheKey = transformer.getCacheKey(
fileData,
filename,
transformOptions,
);
}
return this._buildCacheKeyFromFileInfo(
fileData,
filename,
transformOptions,
transformerCacheKey,
);
}
private async _getCacheKeyAsync(
fileData: string,
filename: string,
options: ReducedTransformOptions,
): Promise<string> {
const configString = this._cache.configString;
const {transformer, transformerConfig = {}} =
this._getTransformer(filename) || {};
let transformerCacheKey = undefined;
const transformOptions: TransformOptions = {
...options,
cacheFS: this._cacheFS,
config: this._config,
configString,
transformerConfig,
};
if (transformer) {
const getCacheKey =
transformer.getCacheKeyAsync || transformer.getCacheKey;
if (typeof getCacheKey === 'function') {
transformerCacheKey = await getCacheKey(
fileData,
filename,
transformOptions,
);
}
}
return this._buildCacheKeyFromFileInfo(
fileData,
filename,
transformOptions,
transformerCacheKey,
);
}
private _createFolderFromCacheKey(
filename: string,
cacheKey: string,
): string {
const HasteMapClass = HasteMap.getStatic(this._config);
const baseCacheDir = HasteMapClass.getCacheFilePath(
this._config.cacheDirectory,
`jest-transform-cache-${this._config.name}`,
VERSION,
);
// Create sub folders based on the cacheKey to avoid creating one
// directory with many files.
const cacheDir = path.join(baseCacheDir, cacheKey[0] + cacheKey[1]);
const cacheFilenamePrefix = path
.basename(filename, path.extname(filename))
.replace(/\W/g, '');
const cachePath = slash(
path.join(cacheDir, `${cacheFilenamePrefix}_${cacheKey}`),
);
createDirectory(cacheDir);
return cachePath;
}
private _getFileCachePath(
filename: string,
content: string,
options: ReducedTransformOptions,
): string {
const cacheKey = this._getCacheKey(content, filename, options);
return this._createFolderFromCacheKey(filename, cacheKey);
}
private async _getFileCachePathAsync(
filename: string,
content: string,
options: ReducedTransformOptions,
): Promise<string> {
const cacheKey = await this._getCacheKeyAsync(content, filename, options);
return this._createFolderFromCacheKey(filename, cacheKey);
}
private _getTransformPath(filename: string) {
const transformRegExp = this._cache.transformRegExp;
if (!transformRegExp) {
return undefined;
}
for (let i = 0; i < transformRegExp.length; i++) {
if (transformRegExp[i][0].test(filename)) {
return transformRegExp[i][1];
}
}
return undefined;
}
async loadTransformers(): Promise<void> {
await Promise.all(
this._config.transform.map(
async ([, transformPath, transformerConfig]) => {
let transformer: Transformer = await requireOrImportModule(
transformPath,
);
if (!transformer) {
throw new Error(makeInvalidTransformerError(transformPath));
}
if (typeof transformer.createTransformer === 'function') {
transformer = transformer.createTransformer(transformerConfig);
}
if (
typeof transformer.process !== 'function' &&
typeof transformer.processAsync !== 'function'
) {
throw new Error(makeInvalidTransformerError(transformPath));
}
const res = {transformer, transformerConfig};
this._transformCache.set(transformPath, res);
},
),
);
this._transformsAreLoaded = true;
}
private _getTransformer(filename: string) {
if (!this._transformsAreLoaded) {
throw new Error(
'Jest: Transformers have not been loaded yet - make sure to run `loadTransformers` and wait for it to complete before starting to transform files',
);
}
if (this._config.transform.length === 0) {
return null;
}
const transformPath = this._getTransformPath(filename);
if (!transformPath) {
return null;
}
const cached = this._transformCache.get(transformPath);
if (cached) {
return cached;
}
throw new Error(
`Jest was unable to load the transformer defined for ${filename}. This is a bug in Jest, please open up an issue`,
);
}
private _instrumentFile(
filename: string,
input: TransformedSource,
canMapToInput: boolean,
options: ReducedTransformOptions,
): TransformedSource {
const inputCode = typeof input === 'string' ? input : input.code;
const inputMap = typeof input === 'string' ? null : input.map;
const result = babelTransform(inputCode, {
auxiliaryCommentBefore: ' istanbul ignore next ',
babelrc: false,
caller: {
name: '@jest/transform',
supportsDynamicImport: options.supportsDynamicImport,
supportsExportNamespaceFrom: options.supportsExportNamespaceFrom,
supportsStaticESM: options.supportsStaticESM,
supportsTopLevelAwait: options.supportsTopLevelAwait,
},
configFile: false,
filename,
plugins: [
[
babelPluginIstanbul,
{
compact: false,
// files outside `cwd` will not be instrumented
cwd: this._config.rootDir,
exclude: [],
extension: false,
inputSourceMap: inputMap,
useInlineSourceMaps: false,
},
],
],
sourceMaps: canMapToInput ? 'both' : false,
});
if (result && result.code) {
return result as TransformResult;
}
return input;
}
private _buildTransformResult(
filename: string,
cacheFilePath: string,
content: string,
transformer: Transformer | undefined,
shouldCallTransform: boolean,
options: ReducedTransformOptions,
processed: TransformedSource | null,
sourceMapPath: string | null,
): TransformResult {
let transformed: TransformedSource = {
code: content,
map: null,
};
if (transformer && shouldCallTransform) {
if (typeof processed === 'string') {
transformed.code = processed;
} else if (processed != null && typeof processed.code === 'string') {
transformed = processed;
} else {
throw new Error(makeInvalidReturnValueError());
}
}
if (!transformed.map) {
try {
//Could be a potential freeze here.
//See: https://github.com/facebook/jest/pull/5177#discussion_r158883570
const inlineSourceMap = sourcemapFromSource(transformed.code);
if (inlineSourceMap) {
transformed.map = inlineSourceMap.toObject();
}
} catch {
const transformPath = this._getTransformPath(filename);
invariant(transformPath);
console.warn(makeInvalidSourceMapWarning(filename, transformPath));
}
}
// That means that the transform has a custom instrumentation
// logic and will handle it based on `config.collectCoverage` option
const transformWillInstrument =
shouldCallTransform && transformer && transformer.canInstrument;
// Apply instrumentation to the code if necessary, keeping the instrumented code and new map
let map = transformed.map;
let code;
if (!transformWillInstrument && options.instrument) {
/**
* We can map the original source code to the instrumented code ONLY if
* - the process of transforming the code produced a source map e.g. ts-jest
* - we did not transform the source code
*
* Otherwise we cannot make any statements about how the instrumented code corresponds to the original code,
* and we should NOT emit any source maps
*
*/
const shouldEmitSourceMaps =
(transformer != null && map != null) || transformer == null;
const instrumented = this._instrumentFile(
filename,
transformed,
shouldEmitSourceMaps,
options,
);
code =
typeof instrumented === 'string' ? instrumented : instrumented.code;
map = typeof instrumented === 'string' ? null : instrumented.map;
} else {
code = transformed.code;
}
if (map) {
const sourceMapContent =
typeof map === 'string' ? map : JSON.stringify(map);
invariant(sourceMapPath, 'We should always have default sourceMapPath');
writeCacheFile(sourceMapPath, sourceMapContent);
} else {
sourceMapPath = null;
}
writeCodeCacheFile(cacheFilePath, code);
return {
code,
originalCode: content,
sourceMapPath,
};
}
transformSource(
filepath: string,
content: string,
options: ReducedTransformOptions,
): TransformResult {
const filename = tryRealpath(filepath);
const {transformer, transformerConfig = {}} =
this._getTransformer(filename) || {};
const cacheFilePath = this._getFileCachePath(filename, content, options);
const sourceMapPath: string = `${cacheFilePath}.map`;
// Ignore cache if `config.cache` is set (--no-cache)
const code = this._config.cache ? readCodeCacheFile(cacheFilePath) : null;
if (code) {
// This is broken: we return the code, and a path for the source map
// directly from the cache. But, nothing ensures the source map actually
// matches that source code. They could have gotten out-of-sync in case
// two separate processes write concurrently to the same cache files.
return {
code,
originalCode: content,
sourceMapPath,
};
}
let processed = null;
let shouldCallTransform = false;
if (transformer && this.shouldTransform(filename)) {
shouldCallTransform = true;
assertSyncTransformer(transformer, this._getTransformPath(filename));
processed = transformer.process(content, filename, {
...options,
cacheFS: this._cacheFS,
config: this._config,
configString: this._cache.configString,
transformerConfig,
});
}
return this._buildTransformResult(
filename,
cacheFilePath,
content,
transformer,
shouldCallTransform,
options,
processed,
sourceMapPath,
);
}
async transformSourceAsync(
filepath: string,
content: string,
options: ReducedTransformOptions,
): Promise<TransformResult> {
const filename = tryRealpath(filepath);
const {transformer, transformerConfig = {}} =
this._getTransformer(filename) || {};
const cacheFilePath = await this._getFileCachePathAsync(
filename,
content,
options,
);
const sourceMapPath: string = `${cacheFilePath}.map`;
// Ignore cache if `config.cache` is set (--no-cache)
const code = this._config.cache ? readCodeCacheFile(cacheFilePath) : null;
if (code) {
// This is broken: we return the code, and a path for the source map
// directly from the cache. But, nothing ensures the source map actually
// matches that source code. They could have gotten out-of-sync in case
// two separate processes write concurrently to the same cache files.
return {
code,
originalCode: content,
sourceMapPath,
};
}
let processed = null;
let shouldCallTransform = false;
if (transformer && this.shouldTransform(filename)) {
shouldCallTransform = true;
const process = transformer.processAsync || transformer.process;
// This is probably dead code since `_getTransformerAsync` already asserts this
invariant(
typeof process === 'function',
'A transformer must always export either a `process` or `processAsync`',
);
processed = await process(content, filename, {
...options,
cacheFS: this._cacheFS,
config: this._config,
configString: this._cache.configString,
transformerConfig,
});
}
return this._buildTransformResult(
filename,
cacheFilePath,
content,
transformer,
shouldCallTransform,
options,
processed,
sourceMapPath,
);
}
private async _transformAndBuildScriptAsync(
filename: string,
options: Options,
transformOptions: ReducedTransformOptions,
fileSource?: string,
): Promise<TransformResult> {
const {isInternalModule} = options;
let fileContent = fileSource ?? this._cacheFS.get(filename);
if (!fileContent) {
fileContent = fs.readFileSync(filename, 'utf8');
this._cacheFS.set(filename, fileContent);
}
const content = stripShebang(fileContent);
let code = content;
let sourceMapPath: string | null = null;
const willTransform =
!isInternalModule &&
(transformOptions.instrument || this.shouldTransform(filename));
try {
if (willTransform) {
const transformedSource = await this.transformSourceAsync(
filename,
content,
transformOptions,
);
code = transformedSource.code;
sourceMapPath = transformedSource.sourceMapPath;
}
return {
code,
originalCode: content,
sourceMapPath,
};
} catch (e: any) {
throw handlePotentialSyntaxError(e);
}
}
private _transformAndBuildScript(
filename: string,
options: Options,
transformOptions: ReducedTransformOptions,
fileSource?: string,
): TransformResult {
const {isInternalModule} = options;
let fileContent = fileSource ?? this._cacheFS.get(filename);
if (!fileContent) {
fileContent = fs.readFileSync(filename, 'utf8');
this._cacheFS.set(filename, fileContent);
}
const content = stripShebang(fileContent);
let code = content;
let sourceMapPath: string | null = null;
const willTransform =
!isInternalModule &&
(transformOptions.instrument || this.shouldTransform(filename));
try {
if (willTransform) {
const transformedSource = this.transformSource(
filename,
content,
transformOptions,
);
code = transformedSource.code;
sourceMapPath = transformedSource.sourceMapPath;
}
return {
code,
originalCode: content,
sourceMapPath,
};
} catch (e: any) {
throw handlePotentialSyntaxError(e);
}
}
async transformAsync(
filename: string,
options: Options,
fileSource?: string,
): Promise<TransformResult> {
const instrument =
options.coverageProvider === 'babel' &&
shouldInstrument(filename, options, this._config);
const scriptCacheKey = getScriptCacheKey(filename, instrument);
let result = this._cache.transformedFiles.get(scriptCacheKey);
if (result) {
return result;
}
result = await this._transformAndBuildScriptAsync(
filename,
options,
{...options, instrument},
fileSource,
);
if (scriptCacheKey) {
this._cache.transformedFiles.set(scriptCacheKey, result);
}
return result;
}
transform(
filename: string,
options: Options,
fileSource?: string,
): TransformResult {
const instrument =
options.coverageProvider === 'babel' &&
shouldInstrument(filename, options, this._config);
const scriptCacheKey = getScriptCacheKey(filename, instrument);
let result = this._cache.transformedFiles.get(scriptCacheKey);
if (result) {
return result;
}
result = this._transformAndBuildScript(
filename,
options,
{...options, instrument},
fileSource,
);
if (scriptCacheKey) {
this._cache.transformedFiles.set(scriptCacheKey, result);
}
return result;
}
transformJson(
filename: string,
options: Options,
fileSource: string,
): string {
const {isInternalModule} = options;
const willTransform = !isInternalModule && this.shouldTransform(filename);
if (willTransform) {
const {code: transformedJsonSource} = this.transformSource(
filename,
fileSource,
{...options, instrument: false},
);
return transformedJsonSource;
}
return fileSource;
}
async requireAndTranspileModule<ModuleType = unknown>(
moduleName: string,
callback?: (module: ModuleType) => void | Promise<void>,
options: RequireAndTranspileModuleOptions = {
applyInteropRequireDefault: true,
instrument: false,
supportsDynamicImport: false,
supportsExportNamespaceFrom: false,
supportsStaticESM: false,
supportsTopLevelAwait: false,
},
): Promise<ModuleType> {
let transforming = false;
const {applyInteropRequireDefault, ...transformOptions} = options;
const revertHook = addHook(
(code, filename) => {
try {
transforming = true;
return (
this.transformSource(filename, code, transformOptions).code || code
);
} finally {
transforming = false;
}
},
{
// Exclude `mjs` extension when addHook because pirates don't support hijack es module
exts: this._config.moduleFileExtensions
.filter(ext => ext !== 'mjs')
.map(ext => `.${ext}`),
ignoreNodeModules: false,
matcher: filename => {
if (transforming) {
// Don't transform any dependency required by the transformer itself
return false;
}
return this.shouldTransform(filename);
},
},
);
try {
const module: ModuleType = await requireOrImportModule(
moduleName,
applyInteropRequireDefault,
);
if (!callback) {
revertHook();
return module;
}
const cbResult = callback(module);
if (isPromise(cbResult)) {
return waitForPromiseWithCleanup(cbResult, revertHook).then(
() => module,
);
}
return module;
} finally {
revertHook();
}
}
shouldTransform(filename: string): boolean {
const ignoreRegexp = this._cache.ignorePatternsRegExp;
const isIgnored = ignoreRegexp ? ignoreRegexp.test(filename) : false;
return this._config.transform.length !== 0 && !isIgnored;
}
}
// TODO: do we need to define the generics twice?
export async function createTranspilingRequire(
config: Config.ProjectConfig,
): Promise<
<TModuleType = unknown>(
resolverPath: string,
applyInteropRequireDefault?: boolean,
) => Promise<TModuleType>
> {
const transformer = await createScriptTransformer(config);
return async function requireAndTranspileModule<TModuleType = unknown>(
resolverPath: string,
applyInteropRequireDefault: boolean = false,
) {
const transpiledModule =
await transformer.requireAndTranspileModule<TModuleType>(
resolverPath,
() => {},
{
applyInteropRequireDefault,
instrument: false,
supportsDynamicImport: false, // this might be true, depending on node version.
supportsExportNamespaceFrom: false,
supportsStaticESM: false,
supportsTopLevelAwait: false,
},
);
return transpiledModule;
};
}
const removeFile = (path: string) => {
try {
fs.unlinkSync(path);
} catch {}
};
const stripShebang = (content: string) => {
// If the file data starts with a shebang remove it. Leaves the empty line
// to keep stack trace line numbers correct.
if (content.startsWith('#!')) {
return content.replace(/^#!.*/, '');
} else {
return content;
}
};
/**
* This is like `writeCacheFile` but with an additional sanity checksum. We
* cannot use the same technique for source maps because we expose source map
* cache file paths directly to callsites, with the expectation they can read
* it right away. This is not a great system, because source map cache file
* could get corrupted, out-of-sync, etc.
*/
function writeCodeCacheFile(cachePath: string, code: string) {
const checksum = createHash('md5').update(code).digest('hex');
writeCacheFile(cachePath, `${checksum}\n${code}`);
}
/**
* Read counterpart of `writeCodeCacheFile`. We verify that the content of the
* file matches the checksum, in case some kind of corruption happened. This
* could happen if an older version of `jest-runtime` writes non-atomically to
* the same cache, for example.
*/
function readCodeCacheFile(cachePath: string): string | null {
const content = readCacheFile(cachePath);
if (content == null) {
return null;
}
const code = content.substring(33);
const checksum = createHash('md5').update(code).digest('hex');
if (checksum === content.substring(0, 32)) {
return code;
}
return null;
}
/**
* Writing to the cache atomically relies on 'rename' being atomic on most
* file systems. Doing atomic write reduces the risk of corruption by avoiding
* two processes to write to the same file at the same time. It also reduces
* the risk of reading a file that's being overwritten at the same time.
*/
const writeCacheFile = (cachePath: string, fileData: string) => {
try {
writeFileAtomic(cachePath, fileData, {encoding: 'utf8', fsync: false});
} catch (e: any) {
if (cacheWriteErrorSafeToIgnore(e, cachePath)) {
return;
}
e.message = `jest: failed to cache transform results in: ${cachePath}\nFailure message: ${e.message}`;
removeFile(cachePath);
throw e;
}
};
/**
* On Windows, renames are not atomic, leading to EPERM exceptions when two
* processes attempt to rename to the same target file at the same time.
* If the target file exists we can be reasonably sure another process has
* legitimately won a cache write race and ignore the error.
*/
const cacheWriteErrorSafeToIgnore = (
e: Error & {code: string},
cachePath: string,
) =>
process.platform === 'win32' &&
e.code === 'EPERM' &&
fs.existsSync(cachePath);
const readCacheFile = (cachePath: string): string | null => {
if (!fs.existsSync(cachePath)) {
return null;
}
let fileData;
try {
fileData = fs.readFileSync(cachePath, 'utf8');
} catch (e: any) {
e.message = `jest: failed to read cache file: ${cachePath}\nFailure message: ${e.message}`;
removeFile(cachePath);
throw e;
}
if (fileData == null) {
// We must have somehow created the file but failed to write to it,
// let's delete it and retry.
removeFile(cachePath);
}
return fileData;
};
const getScriptCacheKey = (filename: string, instrument: boolean) => {
const mtime = fs.statSync(filename).mtime;
return `${filename}_${mtime.getTime()}${instrument ? '_instrumented' : ''}`;
};
const calcIgnorePatternRegExp = (config: Config.ProjectConfig) => {
if (
!config.transformIgnorePatterns ||
config.transformIgnorePatterns.length === 0
) {
return undefined;
}
return new RegExp(config.transformIgnorePatterns.join('|'));
};
const calcTransformRegExp = (config: Config.ProjectConfig) => {
if (!config.transform.length) {
return undefined;
}
const transformRegexp: Array<[RegExp, string, Record<string, unknown>]> = [];
for (let i = 0; i < config.transform.length; i++) {
transformRegexp.push([
new RegExp(config.transform[i][0]),
config.transform[i][1],
config.transform[i][2],
]);
}
return transformRegexp;
};
function invariant(condition: unknown, message?: string): asserts condition {
if (!condition) {
throw new Error(message);
}
}
function assertSyncTransformer(
transformer: Transformer,
name: string | undefined,
): asserts transformer is SyncTransformer {
invariant(name);
invariant(
typeof transformer.process === 'function',
makeInvalidSyncTransformerError(name),
);
}
export type TransformerType = ScriptTransformer;
export async function createScriptTransformer(
config: Config.ProjectConfig,
cacheFS: StringMap = new Map(),
): Promise<TransformerType> {
const transformer = new ScriptTransformer(config, cacheFS);
await transformer.loadTransformers();
return transformer;
}