in internal/linker/link_node_modules.ts [404:732]
export async function main(args: string[], runfiles: Runfiles) {
if (!args || args.length < 1) throw new Error('requires one argument: modulesManifest path');
const [modulesManifest] = args;
log_verbose('manifest file:', modulesManifest);
let {workspace, bin, roots, module_sets} = JSON.parse(fs.readFileSync(modulesManifest));
log_verbose('manifest contents:', JSON.stringify({workspace, bin, roots, module_sets}, null, 2));
roots = roots || {};
module_sets = module_sets || {};
// Bazel starts actions with pwd=execroot/my_wksp when under execroot or pwd=runfiles/my_wksp
// when under runfiles.
// Normalize the slashes in startCwd for easier matching and manipulation.
const startCwd = process.cwd().replace(/\\/g, '/');
log_verbose('startCwd:', startCwd);
const execroot = findExecroot(startCwd);
log_verbose('execroot:', execroot ? execroot : 'not found');
const isExecroot = startCwd == execroot;
log_verbose('isExecroot:', isExecroot.toString());
const isBazelRun = !!process.env['BUILD_WORKSPACE_DIRECTORY']
log_verbose('isBazelRun:', isBazelRun.toString());
if (!isExecroot && execroot) {
// If we're not in the execroot and we've found one then change to the execroot
// directory to create the node_modules symlinks
process.chdir(execroot);
log_verbose('changed directory to execroot', execroot);
}
async function symlinkWithUnlink(
target: string, p: string, stats: fs.Stats|null = null): Promise<boolean> {
if (!path.isAbsolute(target)) {
target = path.resolve(process.cwd(), target);
}
if (stats === null) {
stats = await gracefulLstat(p);
}
// Check if this an an old out-of-date symlink
// If we are running without a runfiles manifest (i.e. in sandbox or with symlinked runfiles),
// then this is guaranteed to be not an artifact from a previous linker run. If not we need to
// check.
if (runfiles.manifest && execroot && stats !== null && stats.isSymbolicLink()) {
// Although `stats` suggests that the file exists as a symlink, it may have been deleted by
// another process. Only proceed unlinking if the file actually still exists.
const symlinkPathRaw = gracefulReadlink(p);
if (symlinkPathRaw !== null) {
const symlinkPath = symlinkPathRaw.replace(/\\/g, '/');
if (path.relative(symlinkPath, target) != '' &&
!path.relative(execroot, symlinkPath).startsWith('..')) {
// Left-over out-of-date symlink from previous run. This can happen if switching between
// root configuration options such as `--noenable_runfiles` and/or
// `--spawn_strategy=standalone`. It can also happen if two different targets link the
// same module name to different targets in a non-sandboxed environment. The latter will
// lead to undeterministic behavior.
// TODO: can we detect the latter case and throw an apprioriate error?
log_verbose(`Out-of-date symlink for ${p} to ${symlinkPath} detected. Target should be ${
target}. Unlinking.`);
await unlink(p);
} else {
log_verbose(`The symlink at ${p} no longer exists, so no need to unlink it.`);
}
}
}
return symlink(target, p);
}
// Symlink all node_modules roots defined. These are 3rd party deps in external npm workspaces
// lined to node_modules folders at the root or in sub-directories
for (const packagePath of Object.keys(roots)) {
const externalWorkspace = roots[packagePath];
let workspaceNodeModules: string | undefined = await resolveWorkspaceNodeModules(
externalWorkspace, startCwd, isExecroot, execroot, runfiles);
if (await exists(workspaceNodeModules)) {
log_verbose(`resolved ${externalWorkspace} external workspace node modules path to ${workspaceNodeModules}`);
} else {
// There are no third party node_modules to symlink to
workspaceNodeModules = undefined;
}
let primaryNodeModules;
if (packagePath) {
const binNodeModules = path.posix.join(bin, packagePath, 'node_modules');
await mkdirp(path.dirname(binNodeModules));
// Create bin/<package_path>/node_modules symlink
// (or empty directory if there are no 3rd party deps to symlink to)
if (workspaceNodeModules) {
await symlinkWithUnlink(workspaceNodeModules, binNodeModules);
primaryNodeModules = workspaceNodeModules;
} else {
await mkdirp(binNodeModules);
primaryNodeModules = binNodeModules;
}
if (!isBazelRun) {
// Special case under bazel run where we don't want to create node_modules
// in an execroot under a package path as this will end up in the user's
// workspace via the package path folder symlink
const execrootNodeModules = path.posix.join(packagePath, 'node_modules');
await mkdirp(path.dirname(execrootNodeModules));
await symlinkWithUnlink(primaryNodeModules, execrootNodeModules);
}
} else {
const execrootNodeModules = 'node_modules';
// Create execroot/node_modules symlink (or empty directory if there are
// no 3rd party deps to symlink to)
if (workspaceNodeModules) {
await symlinkWithUnlink(workspaceNodeModules, execrootNodeModules);
primaryNodeModules = workspaceNodeModules;
} else {
await mkdirp(execrootNodeModules);
primaryNodeModules = execrootNodeModules;
}
// NB: Don't create a bin/node_modules since standard node_modules
// resolution will fall back to the execroot node_modules naturally. See
// https://github.com/bazelbuild/rules_nodejs/issues/3054
}
// If start cwd was in runfiles then create
// start/cwd
if (!isExecroot) {
const runfilesNodeModules = path.posix.join(startCwd, packagePath, 'node_modules');
await mkdirp(path.dirname(runfilesNodeModules));
// Don't link to the root execroot node_modules if there is a workspace node_modules.
// Bazel will delete that symlink on rebuild in the ibazel run context.
await symlinkWithUnlink(primaryNodeModules, runfilesNodeModules);
}
// RUNFILES symlink -> execroot node_modules
if (process.env['RUNFILES']) {
const stat = await gracefulLstat(process.env['RUNFILES']);
if (stat && stat.isDirectory()) {
const runfilesNodeModules = path.posix.join(process.env['RUNFILES'], workspace, 'node_modules');
await mkdirp(path.dirname(runfilesNodeModules));
// Don't link to the root execroot node_modules if there is a workspace node_modules.
// Bazel will delete that symlink on rebuild in the ibazel run context.
await symlinkWithUnlink(primaryNodeModules, runfilesNodeModules);
}
}
}
/**
* Whether the given module resolves to a directory that has been created by a previous linker
* run purely to make space for deep module links. e.g. consider a mapping for `my-pkg/a11y`.
* The linker will create folders like `node_modules/my-pkg/` so that the `a11y` symbolic
* junction can be created. The `my-pkg` folder is then considered a leftover from a previous
* linker run as it only contains symbolic links and no actual source files.
*/
async function isLeftoverDirectoryFromLinker(stats: fs.Stats, modulePath: string) {
// If we are running without a runfiles manifest (i.e. in sandbox or with symlinked runfiles),
// then this is guaranteed to be not an artifact from a previous linker run.
if (runfiles.manifest === undefined) {
return false;
}
if (!stats.isDirectory()) {
return false;
}
let isLeftoverFromPreviousLink = true;
// If the directory contains actual files, this cannot be a leftover from a previous
// linker run. The linker only creates directories in the node modules that hold
// symbolic links for configured module mappings.
await visitDirectoryPreserveLinks(modulePath, async (childPath, childStats) => {
if (!childStats.isSymbolicLink()) {
isLeftoverFromPreviousLink = false;
}
});
return isLeftoverFromPreviousLink;
}
/**
* Creates a symlink for the given module. Existing child symlinks which are part of
* the module are preserved in order to not cause race conditions in non-sandbox
* environments where multiple actions rely on the same node modules root.
*
* To avoid unexpected resource removal, a new temporary link for the target is created.
* Then all symlinks from the existing module are cloned. Once done, the existing module
* is unlinked while the temporary link takes place for the given module. This ensures
* that the module link is never removed at any time (causing race condition failures).
*/
async function createSymlinkAndPreserveContents(stats: fs.Stats, modulePath: string,
target: string) {
const tmpPath = `${modulePath}__linker_tmp`;
log_verbose(`createSymlinkAndPreserveContents( ${modulePath} )`);
await symlink(target, tmpPath);
await visitDirectoryPreserveLinks(modulePath, async (childPath, stat) => {
if (stat.isSymbolicLink()) {
const targetPath = path.join(tmpPath, path.relative(modulePath, childPath));
log_verbose(`Cloning symlink into temporary created link ( ${childPath} )`);
await mkdirp(path.dirname(targetPath));
await symlink(targetPath, await fs.promises.realpath(childPath));
}
});
log_verbose(`Removing existing module so that new link can take place ( ${modulePath} )`);
await unlink(modulePath);
await fs.promises.rename(tmpPath, modulePath);
}
async function linkModules(package_path: string, m: LinkerTreeElement) {
const symlinkIn = package_path ?
path.posix.join(bin, package_path, 'node_modules') :
'node_modules';
// ensure the parent directory exist
if (path.dirname(m.name)) {
await mkdirp(`${symlinkIn}/${path.dirname(m.name)}`);
}
if (m.link) {
const modulePath = m.link;
let target: string|undefined;
if (isExecroot) {
// If we're running out of the execroot, try the execroot path first.
// If the dependency came in exclusively from a transitive binary target
// then the module won't be at this path but in the runfiles of the binary.
// In that case we'll fallback to resolving via runfiles below.
target = `${startCwd}/${modulePath}`;
}
if (!isExecroot || !existsSync(target)) {
// Transform execroot path to the runfiles manifest path so that
// it can be resolved with runfiles.resolve()
let runfilesPath = modulePath;
if (runfilesPath.startsWith(`${bin}/`)) {
runfilesPath = runfilesPath.slice(bin.length + 1);
} else if (runfilesPath === bin) {
runfilesPath = '';
}
const externalPrefix = 'external/';
if (runfilesPath.startsWith(externalPrefix)) {
runfilesPath = runfilesPath.slice(externalPrefix.length);
} else {
runfilesPath = path.posix.join(workspace, runfilesPath);
}
try {
target = runfiles.resolve(runfilesPath);
// if we're resolving from a manifest then make sure we don't resolve
// into the source tree when we are expecting the output tree
if (runfiles.manifest && modulePath.startsWith(`${bin}/`)) {
// Check for BAZEL_OUT_REGEX and not /${bin}/ since resolution
// may be in the `/bazel-out/host` if cfg = "host"
if (!target.match(_BAZEL_OUT_REGEX)) {
const e = new Error(`could not resolve module ${runfilesPath} in output tree`);
(e as any).code = 'MODULE_NOT_FOUND';
throw e;
}
}
} catch (err) {
target = undefined;
log_verbose(`runfiles resolve failed for module '${m.name}': ${err.message}`);
}
}
// Ensure target path absolute for consistency
if (target && !path.isAbsolute(target)) {
target = path.resolve(process.cwd(), target);
}
const symlinkFile = `${symlinkIn}/${m.name}`;
// In environments where runfiles are not symlinked (e.g. Windows), existing linked
// modules are preserved. This could cause issues when a link is created at higher level
// as a conflicting directory is already on disk. e.g. consider in a previous run, we
// linked the modules `my-pkg/overlay`. Later on, in another run, we have a module mapping
// for `my-pkg` itself. The linker cannot create `my-pkg` because the directory `my-pkg`
// already exists. To ensure that the desired link is generated, we create the new desired
// link and move all previous nested links from the old module into the new link. Read more
// about this in the description of `createSymlinkAndPreserveContents`.
const stats = await gracefulLstat(symlinkFile);
const isLeftOver =
(stats !== null && await isLeftoverDirectoryFromLinker(stats, symlinkFile));
// Check if the target exists before creating the symlink.
// This is an extra filesystem access on top of the symlink but
// it is necessary for the time being.
if (target && await exists(target)) {
if (stats !== null && isLeftOver) {
await createSymlinkAndPreserveContents(stats, symlinkFile, target);
} else {
await symlinkWithUnlink(target, symlinkFile, stats);
}
} else {
if (!target) {
log_verbose(`no symlink target found for module ${m.name}`);
} else {
// This can happen if a module mapping is propogated from a dependency
// but the target that generated the mapping in not in the deps. We don't
// want to create symlinks to non-existant targets as this will
// break any nested symlinks that may be created under the module name
// after this.
log_verbose(`potential target ${target} does not exists for module ${m.name}`);
}
if (isLeftOver) {
// Remove left over directory if it exists
await unlink(symlinkFile);
}
}
}
// Process each child branch concurrently
if (m.children) {
await Promise.all(m.children.map(m => linkModules(package_path, m)));
}
}
const links = [];
for (const package_path of Object.keys(module_sets)) {
const modules = module_sets[package_path]
log_verbose(`modules for package path '${package_path}':\n${JSON.stringify(modules, null, 2)}`);
const moduleHierarchy = reduceModules(modules);
log_verbose(`mapping hierarchy for package path '${package_path}':\n${
JSON.stringify(moduleHierarchy)}`);
// Process each root branch concurrently
links.push(...moduleHierarchy.map(m => linkModules(package_path, m)))
}
let code = 0;
await Promise.all(links).catch(e => {
log_error(e);
code = 1;
});
return code;
}