resources/services/table.go (181 lines of code) (raw):

package services import ( "context" "fmt" "log" "regexp" "strconv" "strings" "time" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/guardian/cq-source-image-packages/client" "github.com/guardian/cq-source-image-packages/store" ) type AmigoBakePackage struct { BaseName string `json:"base_name"` BaseAmiId string `json:"base_ami_id"` // The 'SourceAMI' field in Amiable BaseEolDate time.Time `json:"base_eol_date"` RecipeId string `json:"recipe_id"` BakeNumber int `json:"bake_number"` SourceAmiId string `json:"source_ami_id"` // The 'CopiedFromAMI' field in Amiable StartedAt time.Time `json:"started_at"` StartedBy string `json:"started_by"` PackageName string `json:"package_name"` PackageVersion string `json:"package_version"` } type AmigoRecipe struct { RecipeId string BaseName string } type AmigoBaseImage struct { BaseName string BaseAmiId string BaseEolDate time.Time } type OsPackage struct { PackageName string PackageVersion string } func AmigoBakePackages() *schema.Table { return &schema.Table{ Name: "amigo_bake_packages", Resolver: FetchAmigoBakePackages, Transform: transformers.TransformWithStruct(AmigoBakePackage{}), } } func FetchAmigoBakePackages(_ context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { cl := meta.(*client.Client) s3Store := cl.S3Store bakesTable := cl.BakesTable recipesTable := cl.RecipesTable baseImagesTable := cl.BaseImagesTable allBakes := bakesTable.ListAll() allRecipes := map[string]AmigoRecipe{} for _, recipe := range recipesTable.ListAll() { id := recipe["id"].(*types.AttributeValueMemberS).Value baseName := recipe["baseImageId"].(*types.AttributeValueMemberS).Value allRecipes[id] = AmigoRecipe{ RecipeId: id, BaseName: baseName, } } allBaseImages := map[string]AmigoBaseImage{} for _, baseImgAttribs := range baseImagesTable.ListAll() { baseImage := toAmigoBaseImage(baseImgAttribs) allBaseImages[baseImage.BaseName] = baseImage } records := map[string]AmigoBakePackage{} for _, bake := range allBakes { recipeId := bake["recipeId"].(*types.AttributeValueMemberS).Value bakeNumber, err := strconv.Atoi(bake["buildNumber"].(*types.AttributeValueMemberN).Value) if err != nil { log.Printf("Failed to convert buildNumber to int: %v", err) continue } status := bake["status"].(*types.AttributeValueMemberS).Value startedAt := bake["startedAt"].(*types.AttributeValueMemberS).Value startedBy := bake["startedBy"].(*types.AttributeValueMemberS).Value // Skip if bake doesn't have an AMI ID var amiId string if amiIdAttrib, ok := bake["amiId"]; ok { amiId = amiIdAttrib.(*types.AttributeValueMemberS).Value } else { log.Printf("Skipping bake without AMI ID: recipeId: '%v', bakeId: '%v', status: '%v'\n", recipeId, bakeNumber, status) continue } recipe := allRecipes[recipeId] baseImage := allBaseImages[recipe.BaseName] // Fetch corresponding package file from S3 packages, err := fetchBakePackages(recipeId, bakeNumber, s3Store) if err != nil { log.Fatalf("Error fetching bake packages for recipe '%s', bake '%d': %v", recipeId, bakeNumber, err) } // Create a record for each package for _, pkg := range packages { key := fmt.Sprintf("%s--%d--%s--%s", recipeId, bakeNumber, amiId, pkg.PackageName) bakePackage := AmigoBakePackage{ BaseName: recipe.BaseName, BaseAmiId: baseImage.BaseAmiId, BaseEolDate: baseImage.BaseEolDate, RecipeId: recipeId, BakeNumber: bakeNumber, SourceAmiId: amiId, StartedBy: startedBy, PackageName: pkg.PackageName, PackageVersion: pkg.PackageVersion, } bakePackage.SetStartedAt(startedAt) records[key] = bakePackage } } for _, record := range records { res <- record } return nil } func toAmigoBaseImage(baseImgAttribs map[string]types.AttributeValue) AmigoBaseImage { baseImage := AmigoBaseImage{ BaseName: baseImgAttribs["id"].(*types.AttributeValueMemberS).Value, BaseAmiId: baseImgAttribs["amiId"].(*types.AttributeValueMemberS).Value, } if eolDateAttrib, ok := baseImgAttribs["eolDate"]; ok && eolDateAttrib != nil { baseEolDate := eolDateAttrib.(*types.AttributeValueMemberS).Value baseImage.SetBaseEolDate(baseEolDate) } else { baseImage.BaseEolDate = time.Time{} } return baseImage } func toTime(s string) (time.Time, error) { t, err := time.Parse(time.RFC3339, s) if err != nil { log.Printf("Failed to parse time: %v", err) } return t, err } func (a *AmigoBaseImage) SetBaseEolDate(s string) { t, err := toTime(s) if err == nil { a.BaseEolDate = t } } func (a *AmigoBakePackage) SetStartedAt(s string) { t, err := toTime(s) if err == nil { a.StartedAt = t } } // given a recipe ID and bake number, fetch corresponding packages file from S3 // and extract its contents into a list of OsPackages. func fetchBakePackages(recipeId string, bakeNumber int, s3Store store.S3) ([]OsPackage, error) { key := fmt.Sprintf("%s--%d.txt", recipeId, bakeNumber) data, err := s3Store.Get(key) if err != nil { // If file doesn't exist, return an empty slice if strings.Contains(err.Error(), "NoSuchKey") { return []OsPackage{}, nil } return nil, err } return parseSpaceSeparated(key, data), nil } // parseSpaceSeparated parses a space-separated list of package names and versions func parseSpaceSeparated(listName string, packageData []byte) []OsPackage { fieldSplitter := regexp.MustCompile(`\s+`) lines := strings.Split(string(packageData), "\n") var packages []OsPackage for i, line := range lines { pkg, err := parseLine(line, fieldSplitter) if err != nil { log.Printf("List %v line %d skipped: %v", listName, i+1, err) continue } packages = append(packages, pkg) } return packages } func parseLine(line string, fieldSplitter *regexp.Regexp) (OsPackage, error) { fields := fieldSplitter.Split(line, 2) if len(fields) != 2 { return OsPackage{}, fmt.Errorf("invalid package line: '%s'", line) } return OsPackage{ PackageName: fields[0], PackageVersion: fields[1], }, nil }