in magefile.go [1201:1354]
func collectPackageDependencies(platforms []string, packageVersion string, packageTypes []devtools.PackageType, dependencies []packaging.BinarySpec) (archivePath, dropPath string, d []packaging.BinarySpec) {
dropPath, found := os.LookupEnv(agentDropPath)
// try not to shadow too many variables
var err error
// build deps only when drop is not provided
if !found || len(dropPath) == 0 {
// prepare new drop
dropPath = filepath.Join("build", "distributions", "elastic-agent-drop")
dropPath, err = filepath.Abs(dropPath)
if err != nil {
panic(fmt.Errorf("obtaining absolute path for default drop path: %w", err))
}
if mg.Verbose() {
log.Printf(">> Creating drop-in folder %+v \n", dropPath)
}
archivePath = movePackagesToArchive(dropPath, platforms, packageVersion, dependencies)
os.Setenv(agentDropPath, dropPath)
if devtools.ExternalBuild == true {
if mg.Verbose() {
log.Print(">>> Using external builds to collect components")
}
// Only log fatal logs for logs produced. This is the global logger
// used by github.com/elastic/elastic-agent/dev-tools/mage/downloads which can only be configured globally like this.
//
// Using FatalLevel avoids filling the build log with scary looking errors when we attempt to
// download artifacts on unsupported platforms and choose to ignore the errors.
//
// Change this to InfoLevel to see exactly what the downloader is doing.
downloads.LogLevel.Set(downloads.FatalLevel)
errGroup, ctx := errgroup.WithContext(context.Background())
completedDownloads := &atomic.Int32{}
for _, spec := range dependencies {
for _, platform := range platforms {
if !spec.SupportsPlatform(platform) {
log.Printf(">>> Binary %s does not support %s, download skipped\n", spec.BinaryName, platform)
continue
}
if mg.Verbose() {
log.Printf(">>> Looking for component %s/%s", spec.BinaryName, platform)
}
for _, pkgType := range packageTypes {
if mg.Verbose() {
log.Printf(">>> Evaluating pkgType %v for component %s/%s", pkgType, spec.BinaryName, platform)
}
if !spec.SupportsPackageType(pkgcommon.PackageType(pkgType)) {
log.Printf(">>> PkgType %v for component %s/%s not supported. Skipping...", pkgType, spec.BinaryName, platform)
continue
}
targetPath := filepath.Join(archivePath, manifest.PlatformPackages[platform])
os.MkdirAll(targetPath, 0o755)
packageName := spec.GetPackageName(packageVersion, platform)
if mg.Verbose() {
log.Printf(">>> Downloading package %s component %s/%s", packageName, spec.BinaryName, platform)
}
errGroup.Go(downloadBinary(ctx, spec.ProjectName, packageName, spec.BinaryName, platform, packageVersion, targetPath, completedDownloads))
}
}
}
err = errGroup.Wait()
if err != nil {
panic(err)
}
if completedDownloads.Load() == 0 {
panic(fmt.Sprintf("No packages were successfully downloaded. You may be building against an invalid or unreleased version. version=%s. If this is an unreleased version, try SNAPSHOT=true or EXTERNAL=false", packageVersion))
}
} else {
packedBeats := []string{"agentbeat"}
// restrict the dependency list only to agentbeat in this case
dependencies = packaging.FilterComponents(dependencies, packaging.WithBinaryName("agentbeat"))
if mg.Verbose() {
log.Printf("Packaging using a beats repository, reducing dependendencies to %v", dependencies)
}
// build from local repo, will assume beats repo is located on the same root level
for _, b := range packedBeats {
pwd, err := filepath.Abs(filepath.Join("../beats/x-pack", b))
if err != nil {
panic(err)
}
packagesCopied := 0
if !requiredPackagesPresent(pwd, b, packageVersion, platforms) {
fmt.Printf("--- Package %s\n", pwd)
cmd := exec.Command("mage", "package")
cmd.Dir = pwd
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Env = append(os.Environ(),
fmt.Sprintf("PWD=%s", pwd),
"AGENT_PACKAGING=on",
fmt.Sprintf("FIPS=%v", devtools.FIPSBuild),
)
if envVar := selectedPackageTypes(); envVar != "" {
cmd.Env = append(cmd.Env, envVar)
}
if err := cmd.Run(); err != nil {
panic(err)
}
}
// copy to new drop
sourcePath := filepath.Join(pwd, "build", "distributions")
for _, pltf := range platforms {
rp := manifest.PlatformPackages[pltf]
files, err := filepath.Glob(filepath.Join(sourcePath, "*"+rp+"*"))
if err != nil {
panic(err)
}
targetPath := filepath.Join(archivePath, rp)
os.MkdirAll(targetPath, 0o755)
for _, f := range files {
// safety check; if the user has an older version of the beats repo,
// for example right after a release where you've `git pulled` from on repo and not the other,
// they might end up with a mishmash of packages from different versions.
// check to see if we have mismatched versions.
if !strings.Contains(f, packageVersion) {
// if this panic hits weird edge cases where we don't want actual failures, revert to a printf statement.
panic(fmt.Sprintf("the file %s doesn't match agent version %s, beats repo might be out of date", f, packageVersion))
}
targetFile := filepath.Join(targetPath, filepath.Base(f))
packagesCopied += 1
if err := sh.Copy(targetFile, f); err != nil {
panic(err)
}
}
}
// a very basic footcannon protector; if packages are missing and we need to rebuild them, check to see if those files were copied
// if we needed to repackage beats but still somehow copied nothing, could indicate an issue. Usually due to beats and agent being at different versions.
if packagesCopied == 0 {
fmt.Println(">>> WARNING: no packages were copied, but we repackaged beats anyway. Check binary to see if intended beats are there.")
}
}
}
} else {
archivePath = movePackagesToArchive(dropPath, platforms, packageVersion, dependencies)
}
return archivePath, dropPath, dependencies
}