Skip to content

Commit

Permalink
feat: allow dup pkgs in a bundle (#533)
Browse files Browse the repository at this point in the history
  • Loading branch information
UncleGedd authored Apr 2, 2024
1 parent d24d9f9 commit 393954a
Show file tree
Hide file tree
Showing 20 changed files with 154 additions and 165 deletions.
6 changes: 0 additions & 6 deletions src/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,6 @@ const (
// EnvVarPrefix is the prefix for environment variables to override bundle helm variables
EnvVarPrefix = "UDS_"

// ZarfPackageNameAnnotation is the annotation key for the value that specifies the zarf package name
ZarfPackageNameAnnotation = "zarf.package.name"

// UDSPackageNameAnnotation is the annotation key for the value that specifies the name given to a zarf package in the uds-bundle.yaml
UDSPackageNameAnnotation = "uds.package.name"

// CachedLogs is a file containing cached logs
CachedLogs = "recent-logs"
)
Expand Down
32 changes: 21 additions & 11 deletions src/pkg/bundle/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ func (b *Bundle) ClearPaths() {
}

// ValidateBundleResources validates the bundle's metadata and package references
func (b *Bundle) ValidateBundleResources(bundle *types.UDSBundle, spinner *message.Spinner) error {
// TODO: need to validate arch of local OS
func (b *Bundle) ValidateBundleResources(spinner *message.Spinner) error {
bundle := &b.bundle
if bundle.Metadata.Architecture == "" {
// ValidateBundle was erroneously called before CalculateBuildInfo
if err := b.CalculateBuildInfo(); err != nil {
Expand Down Expand Up @@ -157,15 +157,7 @@ func (b *Bundle) ValidateBundleResources(bundle *types.UDSBundle, spinner *messa
if b.cfg.CreateOpts.Output != "" {
return fmt.Errorf("detected local Zarf package: %s, outputting to an OCI registry is not supported when using local Zarf packages", pkg.Name)
}
var fullPkgName string
if pkg.Name == "init" {
fullPkgName = fmt.Sprintf("zarf-%s-%s-%s.tar.zst", pkg.Name, bundle.Metadata.Architecture, pkg.Ref)
} else {
// For local zarf packages, we get the package name using the package name provided in the bundle, since the zarf package artifact
// uses the actual zarf package name, these names must match
fullPkgName = fmt.Sprintf("zarf-package-%s-%s-%s.tar.zst", pkg.Name, bundle.Metadata.Architecture, pkg.Ref)
}
path := filepath.Join(pkg.Path, fullPkgName)
path := getPkgPath(pkg, bundle.Metadata.Architecture)
bundle.Packages[idx].Path = path
}

Expand Down Expand Up @@ -219,6 +211,24 @@ func (b *Bundle) ValidateBundleResources(bundle *types.UDSBundle, spinner *messa
return nil
}

func getPkgPath(pkg types.Package, arch string) string {
var fullPkgName string
var path string
if strings.HasSuffix(pkg.Path, ".tar.zst") {
// use the provided pkg tarball
path = pkg.Path
} else if pkg.Name == "init" {
// Zarf init pkgs have a specific naming convention
fullPkgName = fmt.Sprintf("zarf-%s-%s-%s.tar.zst", pkg.Name, arch, pkg.Ref)
path = filepath.Join(pkg.Path, fullPkgName)
} else {
// infer the name of the local Zarf pkg
fullPkgName = fmt.Sprintf("zarf-package-%s-%s-%s.tar.zst", pkg.Name, arch, pkg.Ref)
path = filepath.Join(pkg.Path, fullPkgName)
}
return path
}

// CalculateBuildInfo calculates the build info for the bundle
func (b *Bundle) CalculateBuildInfo() error {
now := time.Now()
Expand Down
52 changes: 52 additions & 0 deletions src/pkg/bundle/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/defenseunicorns/uds-cli/src/types"
zarfTypes "github.com/defenseunicorns/zarf/src/types"
"github.com/stretchr/testify/require"
)

func Test_validateBundleVars(t *testing.T) {
Expand Down Expand Up @@ -145,3 +146,54 @@ func Test_validateOverrides(t *testing.T) {
})
}
}

func Test_getPkgPath(t *testing.T) {
type args struct {
pkg types.Package
arch string
}
tests := []struct {
name string
args args
want string
}{
{
name: "init full path",
args: args{
pkg: types.Package{Name: "init", Ref: "0.0.1", Path: "../fake/path/custom-init.tar.zst"},
arch: "fake64",
},
want: "../fake/path/custom-init.tar.zst",
},
{
name: "init directory only path",
args: args{
pkg: types.Package{Name: "init", Ref: "0.0.1", Path: "../fake/path"},
arch: "fake64",
},
want: "../fake/path/zarf-init-fake64-0.0.1.tar.zst",
},
{
name: "full path",
args: args{
pkg: types.Package{Name: "nginx", Ref: "0.0.1", Path: "./fake/zarf-package-nginx-fake64-0.0.1.tar.zst"},
arch: "fake64",
},
want: "./fake/zarf-package-nginx-fake64-0.0.1.tar.zst",
},
{
name: "directory only path",
args: args{
pkg: types.Package{Name: "nginx", Ref: "0.0.1", Path: "/fake"},
arch: "fake64",
},
want: "/fake/zarf-package-nginx-fake64-0.0.1.tar.zst",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
path := getPkgPath(tt.args.pkg, tt.args.arch)
require.Equal(t, tt.want, path)
})
}
}
2 changes: 1 addition & 1 deletion src/pkg/bundle/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (b *Bundle) Create() error {
defer validateSpinner.Stop()

// validate bundle / verify access to all repositories
if err := b.ValidateBundleResources(&b.bundle, validateSpinner); err != nil {
if err := b.ValidateBundleResources(validateSpinner); err != nil {
return err
}

Expand Down
8 changes: 1 addition & 7 deletions src/pkg/bundle/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func deployPackages(packages []types.Package, resume bool, b *Bundle) error {
// Automatically confirm the package deployment
zarfConfig.CommonOptions.Confirm = true

source, err := sources.New(b.cfg.DeployOpts.Source, b.cfg.DeployOpts.ZarfPackageNameMap[pkg.Name], opts, sha, nsOverrides)
source, err := sources.New(b.cfg.DeployOpts.Source, pkg.Name, opts, sha, nsOverrides)
if err != nil {
return err
}
Expand Down Expand Up @@ -328,12 +328,6 @@ func (b *Bundle) PreDeployValidation() (string, string, string, error) {
return "", "", "", err
}

// Maps name given to zarf package in the bundle to the actual name of the zarf package
zarfPackageNameMap, err := provider.ZarfPackageNameMap()
if err != nil {
return "", "", "", err
}
b.cfg.DeployOpts.ZarfPackageNameMap = zarfPackageNameMap
bundleName := b.bundle.Metadata.Name
return bundleName, string(bundleYAML), source, err
}
Expand Down
3 changes: 0 additions & 3 deletions src/pkg/bundle/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,6 @@ type Provider interface {

// getBundleManifest gets the bundle's root manifest
getBundleManifest() (*oci.Manifest, error)

// ZarfPackageNameMap returns a map of the zarf package name specified in the uds-bundle.yaml to the actual zarf package name
ZarfPackageNameMap() (map[string]string, error)
}

// NewBundleProvider returns a new bundler Provider based on the source type
Expand Down
32 changes: 0 additions & 32 deletions src/pkg/bundle/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils"
"github.com/defenseunicorns/zarf/src/pkg/utils/helpers"
"github.com/defenseunicorns/zarf/src/pkg/zoci"
goyaml "github.com/goccy/go-yaml"
"github.com/mholt/archiver/v4"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2"
Expand Down Expand Up @@ -335,34 +334,3 @@ func CheckOCISourcePath(source string) (string, error) {
}
return source, nil
}

// ZarfPackageNameMap returns the uds bundle zarf package name to actual zarf package name mappings from the oci provider
func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) {
rootManifest, err := op.getBundleManifest()
if err != nil {
return nil, err
}

loaded, err := op.LoadBundleMetadata()
if err != nil {
return nil, err
}

b, err := os.ReadFile(loaded[config.BundleYAML])
if err != nil {
return nil, err
}

var bundle types.UDSBundle
if err := goyaml.Unmarshal(b, &bundle); err != nil {
return nil, err
}

nameMap := make(map[string]string)
for _, pkg := range bundle.Packages {
sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle
manifestDesc := rootManifest.Locate(sha)
nameMap[manifestDesc.Annotations[config.UDSPackageNameAnnotation]] = manifestDesc.Annotations[config.ZarfPackageNameAnnotation]
}
return nameMap, nil
}
21 changes: 5 additions & 16 deletions src/pkg/bundle/remove.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,6 @@ func (b *Bundle) Remove() error {
return err
}

// Maps name given to zarf package in the bundle to the actual name of the zarf package
zarfPackageNameMap, err := provider.ZarfPackageNameMap()
if err != nil {
return err
}

// Check if --packages flag is set and zarf packages have been specified
var packagesToRemove []types.Package

Expand All @@ -73,25 +67,20 @@ func (b *Bundle) Remove() error {
if len(userSpecifiedPackages) != len(packagesToRemove) {
return fmt.Errorf("invalid zarf packages specified by --packages")
}
return removePackages(packagesToRemove, b, zarfPackageNameMap)
return removePackages(packagesToRemove, b)
}
return removePackages(b.bundle.Packages, b, zarfPackageNameMap)
return removePackages(b.bundle.Packages, b)
}

func removePackages(packagesToRemove []types.Package, b *Bundle, zarfPackageNameMap map[string]string) error {
func removePackages(packagesToRemove []types.Package, b *Bundle) error {
// Get deployed packages
deployedPackageNames := GetDeployedPackageNames()

for i := len(packagesToRemove) - 1; i >= 0; i-- {

pkg := packagesToRemove[i]
zarfPackageName := pkg.Name
// use the name map if it has been set (remote pkgs where the pkg name isn't consistent)
if _, ok := zarfPackageNameMap[pkg.Name]; ok {
zarfPackageName = zarfPackageNameMap[pkg.Name]
}

if slices.Contains(deployedPackageNames, zarfPackageName) {
if slices.Contains(deployedPackageNames, pkg.Name) {
opts := zarfTypes.ZarfPackageOptions{
PackageSource: b.cfg.RemoveOpts.Source,
}
Expand All @@ -104,7 +93,7 @@ func removePackages(packagesToRemove []types.Package, b *Bundle, zarfPackageName
}

sha := strings.Split(pkg.Ref, "sha256:")[1]
source, err := sources.New(b.cfg.RemoveOpts.Source, zarfPackageName, opts, sha, nil)
source, err := sources.New(b.cfg.RemoveOpts.Source, pkg.Name, opts, sha, nil)
if err != nil {
return err
}
Expand Down
20 changes: 0 additions & 20 deletions src/pkg/bundle/tarball.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"github.com/defenseunicorns/zarf/src/pkg/oci"
zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils"
"github.com/defenseunicorns/zarf/src/pkg/utils/helpers"
"github.com/defenseunicorns/zarf/src/pkg/zoci"
av3 "github.com/mholt/archiver/v3"
av4 "github.com/mholt/archiver/v4"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -327,22 +326,3 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o
progressBar.Successf("Published %s", remote.Repo().Reference)
return nil
}

// ZarfPackageNameMap gets zarf package name mappings from tarball provider
func (tp *tarballBundleProvider) ZarfPackageNameMap() (map[string]string, error) {
bundleRootManifest, err := tp.getBundleManifest()
if err != nil {
return nil, err
}

nameMap := make(map[string]string)
for _, layer := range bundleRootManifest.Layers {
if layer.MediaType == zoci.ZarfLayerMediaTypeBlob {
// only the uds bundle layer will have AnnotationTitle set
if layer.Annotations[ocispec.AnnotationTitle] != config.BundleYAML {
nameMap[layer.Annotations[config.UDSPackageNameAnnotation]] = layer.Annotations[config.ZarfPackageNameAnnotation]
}
}
}
return nameMap, nil
}
2 changes: 1 addition & 1 deletion src/pkg/bundler/fetcher/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ type localFetcher struct {
extractDst string
}

// Fetch fetches a Zarf pkg and puts it into a local bundle
// Fetch fetches a local Zarf pkg and puts it into a local bundle
func (f *localFetcher) Fetch() ([]ocispec.Descriptor, error) {
fetchSpinner := message.NewProgressSpinner("Fetching package %s", f.pkg.Name)
defer fetchSpinner.Stop()
Expand Down
25 changes: 4 additions & 21 deletions src/pkg/bundler/fetcher/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@ type remoteFetcher struct {
// Fetch fetches a Zarf pkg and puts it into a local bundle
func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) {
fetchSpinner := message.NewProgressSpinner("Fetching package %s", f.pkg.Name)
zarfPackageName := ""
zarfRootLayerAdded := false
defer fetchSpinner.Stop()

layerDescs, err := f.layersToLocalBundle(fetchSpinner, f.cfg.PkgIter+1, f.cfg.NumPkgs)
Expand All @@ -58,22 +56,12 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) {
if err != nil {
return nil, err
}

// ensure media type is Zarf blob for layers in the bundle's root manifest
layerDesc.MediaType = zoci.ZarfLayerMediaTypeBlob

// add package name annotations
annotations := make(map[string]string)
layerDesc.Annotations = annotations
layerDesc.Annotations[config.UDSPackageNameAnnotation] = f.pkg.Name

// If zarf package name has been obtained from zarf config, set the zarf package name annotation
// This block of code will only be triggered if the zarf config is processed before the zarf image manifest
if zarfPackageName != "" {
layerDesc.Annotations[config.ZarfPackageNameAnnotation] = zarfPackageName
}

// add layer to bundle's root manifest
f.cfg.BundleRootManifest.Layers = append(f.cfg.BundleRootManifest.Layers, layerDesc)
zarfRootLayerAdded = true
} else if layerDesc.MediaType == zoci.ZarfConfigMediaType {
// read in and unmarshal zarf config
jsonData, err := os.ReadFile(filepath.Join(f.cfg.TmpDstDir, config.BlobsDir, layerDesc.Digest.Encoded()))
Expand All @@ -85,14 +73,9 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) {
if err != nil {
return nil, err
}
zarfPackageName = zarfConfigData.Annotations[ocispec.AnnotationTitle]
// Check if zarf image manifest has been added to root manifest already, if so add zarfPackageName annotation
// This block of code will only be triggered if the zarf image manifest is processed before the zarf config
if zarfRootLayerAdded {
f.cfg.BundleRootManifest.Layers[f.cfg.PkgIter].Annotations[config.ZarfPackageNameAnnotation] = zarfPackageName
}
}
}

fetchSpinner.Successf("Fetched package: %s", f.pkg.Name)
return layerDescs, nil
}
Expand Down Expand Up @@ -178,7 +161,7 @@ func (f *remoteFetcher) remoteToLocal(layersToCopy []ocispec.Descriptor) ([]ocis
}
}
} else {
// need to grab pkg root manifest manually bc we didn't use oras.Copy()
// need to grab pkg root manifest and config manually bc we didn't use oras.Copy()
pkgManifestDesc, err := utils.ToOCIStore(f.pkgRootManifest, ocispec.MediaTypeImageManifest, f.cfg.Store)
if err != nil {
return nil, err
Expand Down
10 changes: 0 additions & 10 deletions src/pkg/bundler/pusher/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ func NewPkgPusher(pkg types.Package, cfg Config) RemotePusher {

// Push pushes a Zarf pkg to a remote bundle
func (p *RemotePusher) Push() (ocispec.Descriptor, error) {
ctx := context.TODO()
zarfManifestDesc, err := p.PushManifest()
if err != nil {
return ocispec.Descriptor{}, err
Expand All @@ -52,15 +51,6 @@ func (p *RemotePusher) Push() (ocispec.Descriptor, error) {
url := fmt.Sprintf("%s:%s", p.pkg.Repository, p.pkg.Ref)
message.Debugf("Pushed %s sub-manifest into %s: %s", url, p.cfg.RemoteDst.Repo().Reference, message.JSONValue(zarfManifestDesc))

// add package name annotations to zarf manifest
zarfYamlFile, err := p.cfg.RemoteSrc.FetchZarfYAML(ctx)
if err != nil {
return ocispec.Descriptor{}, err
}
zarfManifestDesc.Annotations = make(map[string]string)
zarfManifestDesc.Annotations[config.UDSPackageNameAnnotation] = p.pkg.Name
zarfManifestDesc.Annotations[config.ZarfPackageNameAnnotation] = zarfYamlFile.Metadata.Name

pushSpinner := message.NewProgressSpinner("")
defer pushSpinner.Stop()

Expand Down
Loading

0 comments on commit 393954a

Please sign in to comment.