in src/com/facebook/buck/features/python/PythonUtil.java [281:479]
static PythonPackageComponents getAllComponents(
CellPathResolver cellPathResolver,
BuildTarget buildTarget,
ProjectFilesystem projectFilesystem,
BuildRuleParams params,
ActionGraphBuilder graphBuilder,
PythonPackagable binary,
PythonPlatform pythonPlatform,
CxxBuckConfig cxxBuckConfig,
CxxPlatform cxxPlatform,
ImmutableList<? extends Arg> extraLdflags,
NativeLinkStrategy nativeLinkStrategy,
ImmutableSet<BuildTarget> preloadDeps,
boolean compile) {
PythonPackageComponents.Builder allComponents = new PythonPackageComponents.Builder();
Map<BuildTarget, CxxPythonExtension> extensions = new LinkedHashMap<>();
Map<BuildTarget, NativeLinkable> nativeLinkableRoots = new LinkedHashMap<>();
OmnibusRoots.Builder omnibusRoots = OmnibusRoots.builder(preloadDeps, graphBuilder);
// Walk all our transitive deps to build our complete package that we'll
// turn into an executable.
new AbstractBreadthFirstTraversal<Object>(
Iterables.concat(ImmutableList.of(binary), graphBuilder.getAllRules(preloadDeps))) {
private final ImmutableList<BuildRule> empty = ImmutableList.of();
@Override
public Iterable<?> visit(Object node) {
Iterable<?> deps = empty;
if (node instanceof CxxPythonExtension) {
CxxPythonExtension extension = (CxxPythonExtension) node;
NativeLinkTarget target =
extension.getNativeLinkTarget(pythonPlatform, cxxPlatform, graphBuilder, false);
extensions.put(target.getBuildTarget(), extension);
omnibusRoots.addIncludedRoot(target);
List<BuildRule> cxxpydeps = new ArrayList<>();
for (BuildRule dep :
extension.getPythonPackageDeps(pythonPlatform, cxxPlatform, graphBuilder)) {
if (dep instanceof PythonPackagable) {
cxxpydeps.add(dep);
}
}
deps = cxxpydeps;
} else if (node instanceof PythonPackagable) {
PythonPackagable packagable = (PythonPackagable) node;
packagable
.getPythonModules(pythonPlatform, cxxPlatform, graphBuilder)
.ifPresent(modules -> allComponents.putModules(packagable.getBuildTarget(), modules));
if (compile) {
packagable
.getPythonBytecode(pythonPlatform, cxxPlatform, graphBuilder)
.ifPresent(
bytecode -> allComponents.putModules(packagable.getBuildTarget(), bytecode));
}
packagable
.getPythonResources(pythonPlatform, cxxPlatform, graphBuilder)
.ifPresent(
resources -> allComponents.putResources(packagable.getBuildTarget(), resources));
allComponents.addZipSafe(packagable.isPythonZipSafe());
Iterable<BuildRule> packagableDeps =
packagable.getPythonPackageDeps(pythonPlatform, cxxPlatform, graphBuilder);
if (nativeLinkStrategy == NativeLinkStrategy.MERGED
&& packagable.doesPythonPackageDisallowOmnibus(
pythonPlatform, cxxPlatform, graphBuilder)) {
for (BuildRule dep : packagableDeps) {
if (dep instanceof NativeLinkableGroup) {
NativeLinkable linkable =
((NativeLinkableGroup) dep).getNativeLinkable(cxxPlatform, graphBuilder);
nativeLinkableRoots.put(linkable.getBuildTarget(), linkable);
omnibusRoots.addExcludedRoot(linkable);
}
}
}
deps = packagableDeps;
} else if (node instanceof NativeLinkableGroup) {
NativeLinkable linkable =
((NativeLinkableGroup) node).getNativeLinkable(cxxPlatform, graphBuilder);
nativeLinkableRoots.put(linkable.getBuildTarget(), linkable);
omnibusRoots.addPotentialRoot(linkable, false);
}
return deps;
}
}.start();
// For the merged strategy, build up the lists of included native linkable roots, and the
// excluded native linkable roots.
if (nativeLinkStrategy == NativeLinkStrategy.MERGED) {
OmnibusRoots roots = omnibusRoots.build();
Omnibus.OmnibusLibraries libraries =
Omnibus.getSharedLibraries(
buildTarget,
projectFilesystem,
params,
cellPathResolver,
graphBuilder,
cxxBuckConfig,
cxxPlatform,
extraLdflags,
roots.getIncludedRoots().values(),
roots.getExcludedRoots().values());
// Add all the roots from the omnibus link. If it's an extension, add it as a module.
// Otherwise, add it as a native library.
for (Map.Entry<BuildTarget, Omnibus.OmnibusRoot> root : libraries.getRoots().entrySet()) {
CxxPythonExtension extension = extensions.get(root.getKey());
if (extension != null) {
allComponents.putModules(
root.getKey(),
PythonMappedComponents.of(
ImmutableSortedMap.of(extension.getModule(), root.getValue().getPath())));
} else {
NativeLinkTarget target =
Preconditions.checkNotNull(
roots.getIncludedRoots().get(root.getKey()),
"%s: linked unexpected omnibus root: %s",
buildTarget,
root.getKey());
NativeLinkTargetMode mode = target.getNativeLinkTargetMode();
String soname =
Preconditions.checkNotNull(
mode.getLibraryName().orElse(null),
"%s: omnibus library for %s was built without soname",
buildTarget,
root.getKey());
allComponents.putNativeLibraries(
root.getKey(),
PythonMappedComponents.of(
ImmutableSortedMap.of(Paths.get(soname), root.getValue().getPath())));
}
}
// Add all remaining libraries as native libraries.
if (!libraries.getLibraries().isEmpty()) {
libraries.getLibraries().stream()
.forEach(
lib ->
allComponents.putNativeLibraries(
buildTarget,
PythonMappedComponents.of(
ImmutableSortedMap.of(Paths.get(lib.getSoname()), lib.getPath()))));
}
} else {
// For regular linking, add all extensions via the package components interface.
Map<BuildTarget, NativeLinkable> extensionNativeDeps = new LinkedHashMap<>();
for (Map.Entry<BuildTarget, CxxPythonExtension> entry : extensions.entrySet()) {
entry
.getValue()
.getPythonModules(pythonPlatform, cxxPlatform, graphBuilder)
.ifPresent(
modules -> allComponents.putModules(entry.getValue().getBuildTarget(), modules));
entry
.getValue()
.getPythonResources(pythonPlatform, cxxPlatform, graphBuilder)
.ifPresent(
resources ->
allComponents.putResources(entry.getValue().getBuildTarget(), resources));
allComponents.addZipSafe(entry.getValue().isPythonZipSafe());
extensionNativeDeps.putAll(
Maps.uniqueIndex(
entry
.getValue()
.getNativeLinkTarget(pythonPlatform, cxxPlatform, graphBuilder, false)
.getNativeLinkTargetDeps(graphBuilder),
NativeLinkable::getBuildTarget));
}
// Add all the native libraries.
ImmutableList<? extends NativeLinkable> nativeLinkables =
NativeLinkables.getTransitiveNativeLinkables(
graphBuilder,
Iterables.concat(nativeLinkableRoots.values(), extensionNativeDeps.values()));
graphBuilder
.getParallelizer()
.maybeParallelizeTransform(
nativeLinkables.stream()
.filter(
nativeLinkable -> {
NativeLinkableGroup.Linkage linkage = nativeLinkable.getPreferredLinkage();
return nativeLinkableRoots.containsKey(nativeLinkable.getBuildTarget())
|| linkage != NativeLinkableGroup.Linkage.STATIC;
})
.collect(Collectors.toList()),
linkable ->
new AbstractMap.SimpleEntry<BuildTarget, PythonComponents>(
linkable.getBuildTarget(),
PythonMappedComponents.of(
ImmutableSortedMap.copyOf(
MoreMaps.transformKeys(
linkable.getSharedLibraries(graphBuilder), Paths::get)))))
.forEach(e -> allComponents.putNativeLibraries(e.getKey(), e.getValue()));
}
return allComponents.build();
}