e2etest/newe2e_resource_managers_file.go (594 lines of code) (raw):
package e2etest
import (
"bytes"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/fileerror"
filesas "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/service"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/share"
"github.com/Azure/azure-storage-azcopy/v10/cmd"
"github.com/Azure/azure-storage-azcopy/v10/common"
"github.com/Azure/azure-storage-azcopy/v10/sddl"
"github.com/Azure/azure-storage-azcopy/v10/ste"
"io"
"path"
"runtime"
"strings"
)
// check that everything complies with interfaces
func init() {
void := func(_ ...any) {} // prevent go from erroring from unused vars
void(
ServiceResourceManager(&FileServiceResourceManager{}),
ContainerResourceManager(&FileShareResourceManager{}),
ObjectResourceManager(&FileObjectResourceManager{}),
RemoteResourceManager(&FileServiceResourceManager{}),
RemoteResourceManager(&FileShareResourceManager{}),
RemoteResourceManager(&FileObjectResourceManager{}),
)
}
func fileStripSAS(uri string) string {
parts, err := filesas.ParseURL(uri)
common.PanicIfErr(err)
parts.SAS = filesas.QueryParameters{} // remove SAS
return parts.String()
}
// ==================== SERVICE ====================
type FileServiceResourceManager struct {
InternalAccount *AzureAccountResourceManager
InternalClient *service.Client
}
func (s *FileServiceResourceManager) DefaultAuthType() ExplicitCredentialTypes {
return EExplicitCredentialType.SASToken()
}
func (s *FileServiceResourceManager) ValidAuthTypes() ExplicitCredentialTypes {
return EExplicitCredentialType.With(EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth())
}
func (s *FileServiceResourceManager) WithSpecificAuthType(cred ExplicitCredentialTypes, a Asserter, opts ...CreateAzCopyTargetOptions) AzCopyTarget {
return CreateAzCopyTarget(s, cred, a, opts...)
}
func (s *FileServiceResourceManager) Canon() string {
return buildCanonForAzureResourceManager(s)
}
func (s *FileServiceResourceManager) Account() AccountResourceManager {
return s.InternalAccount
}
func (s *FileServiceResourceManager) Parent() ResourceManager {
return nil
}
func (s *FileServiceResourceManager) Location() common.Location {
return common.ELocation.File()
}
func (s *FileServiceResourceManager) Level() cmd.LocationLevel {
return cmd.ELocationLevel.Service()
}
func (s *FileServiceResourceManager) URI(opts ...GetURIOptions) string {
base := fileStripSAS(s.InternalClient.URL())
base = s.InternalAccount.ApplySAS(base, s.Location(), opts...)
base = addWildCard(base, opts...)
return base
}
func (s *FileServiceResourceManager) ResourceClient() any {
return s.InternalClient
}
func (s *FileServiceResourceManager) ListContainers(a Asserter) []string {
a.HelperMarker().Helper()
pager := s.InternalClient.NewListSharesPager(nil)
out := make([]string, 0)
for pager.More() {
page, err := pager.NextPage(ctx)
a.NoError("get page", err)
for _, shareEntry := range page.Shares {
if shareEntry == nil || shareEntry.Name == nil {
continue
}
out = append(out, *shareEntry.Name)
}
}
return out
}
func (s *FileServiceResourceManager) GetContainer(container string) ContainerResourceManager {
return &FileShareResourceManager{
InternalAccount: s.InternalAccount,
Service: s,
InternalContainerName: container,
InternalClient: s.InternalClient.NewShareClient(container),
}
}
func (s *FileServiceResourceManager) IsHierarchical() bool {
return true
}
// ==================== CONTAINER ====================
type FileShareResourceManager struct {
InternalAccount *AzureAccountResourceManager
Service *FileServiceResourceManager
InternalContainerName string
InternalClient *share.Client
}
func (s *FileShareResourceManager) DefaultAuthType() ExplicitCredentialTypes {
return (&FileServiceResourceManager{}).DefaultAuthType()
}
func (s *FileShareResourceManager) ValidAuthTypes() ExplicitCredentialTypes {
return (&FileServiceResourceManager{}).ValidAuthTypes()
}
func (s *FileShareResourceManager) WithSpecificAuthType(cred ExplicitCredentialTypes, a Asserter, opts ...CreateAzCopyTargetOptions) AzCopyTarget {
return CreateAzCopyTarget(s, cred, a, opts...)
}
func (s *FileShareResourceManager) Canon() string {
return buildCanonForAzureResourceManager(s)
}
func (s *FileShareResourceManager) Exists() bool {
_, err := s.InternalClient.GetProperties(ctx, nil)
return err == nil || !fileerror.HasCode(err, fileerror.ShareNotFound, fileerror.ShareBeingDeleted, fileerror.ResourceNotFound)
}
func (s *FileShareResourceManager) Parent() ResourceManager {
return s.Service
}
func (s *FileShareResourceManager) Account() AccountResourceManager {
return s.InternalAccount
}
func (s *FileShareResourceManager) ResourceClient() any {
return s.InternalClient
}
func (s *FileShareResourceManager) Location() common.Location {
return s.Service.Location()
}
func (s *FileShareResourceManager) Level() cmd.LocationLevel {
return cmd.ELocationLevel.Container()
}
func (s *FileShareResourceManager) URI(opts ...GetURIOptions) string {
base := fileStripSAS(s.InternalClient.URL())
base = s.InternalAccount.ApplySAS(base, s.Location(), opts...)
base = addWildCard(base, opts...)
return base
}
func (s *FileShareResourceManager) ContainerName() string {
return s.InternalContainerName
}
func (s *FileShareResourceManager) GetProperties(a Asserter) ContainerProperties {
a.HelperMarker().Helper()
resp, err := s.InternalClient.GetProperties(ctx, nil)
a.NoError("get share properties", err)
return ContainerProperties{
Metadata: resp.Metadata,
FileContainerProperties: FileContainerProperties{
AccessTier: (*share.AccessTier)(resp.AccessTier),
EnabledProtocols: resp.EnabledProtocols,
Quota: resp.Quota,
RootSquash: resp.RootSquash,
},
}
}
// SetProperties Sets the quota of a file share
func (s *FileShareResourceManager) SetProperties(a Asserter, properties *ContainerProperties) {
a.HelperMarker().Helper()
props := DerefOrZero(properties)
_, err := s.InternalClient.SetProperties(ctx, &share.SetPropertiesOptions{
Quota: props.FileContainerProperties.Quota})
a.NoError("set share properties", err)
return
}
func (s *FileShareResourceManager) Create(a Asserter, props ContainerProperties) {
a.HelperMarker().Helper()
s.CreateWithOptions(a, &FileShareCreateOptions{
AccessTier: props.FileContainerProperties.AccessTier,
EnabledProtocols: props.FileContainerProperties.EnabledProtocols,
Metadata: props.Metadata,
Quota: props.FileContainerProperties.Quota,
RootSquash: props.FileContainerProperties.RootSquash,
})
}
type FileShareCreateOptions = share.CreateOptions
func (s *FileShareResourceManager) CreateWithOptions(a Asserter, options *FileShareCreateOptions) {
a.HelperMarker().Helper()
_, err := s.InternalClient.Create(ctx, options)
created := true
if fileerror.HasCode(err, fileerror.ShareAlreadyExists) {
created = false
err = nil
}
a.NoError("Create container", err)
if created {
TrackResourceCreation(a, s)
}
}
func (s *FileShareResourceManager) Delete(a Asserter) {
s.DeleteWithOptions(a, nil)
}
type FileShareDeleteOptions = share.DeleteOptions
func (s *FileShareResourceManager) DeleteWithOptions(a Asserter, options *FileShareDeleteOptions) {
a.HelperMarker().Helper()
_, err := s.InternalClient.Delete(ctx, options)
a.NoError("delete share", err)
}
func (s *FileShareResourceManager) ListObjects(a Asserter, targetDir string, recursive bool) map[string]ObjectProperties {
a.HelperMarker().Helper()
queue := []string{targetDir}
out := make(map[string]ObjectProperties)
for len(queue) > 0 {
parent := queue[0] // pop from queue
queue = queue[1:]
dirClient := s.InternalClient.NewDirectoryClient(parent)
pager := dirClient.NewListFilesAndDirectoriesPager(&directory.ListFilesAndDirectoriesOptions{
Include: directory.ListFilesInclude{Timestamps: true, Attributes: true, PermissionKey: true},
IncludeExtendedInfo: pointerTo(true),
})
for pager.More() {
page, err := pager.NextPage(ctx)
a.NoError("Get page", err)
// List directories and add to queue
for _, v := range page.Segment.Directories {
fullPath := path.Join(parent, *v.Name)
if recursive {
queue = append(queue, fullPath)
}
subdirClient := s.InternalClient.NewDirectoryClient(fullPath)
resp, err := subdirClient.GetProperties(ctx, &directory.GetPropertiesOptions{})
a.NoError("Get dir properties", err)
var permissions *string
if resp.FilePermissionKey != nil {
permResp, err := s.InternalClient.GetPermission(ctx, *v.PermissionKey, nil)
a.NoError("Get permissions", err)
permissions = permResp.Permission
}
out[fullPath] = ObjectProperties{
EntityType: common.EEntityType.Folder(),
Metadata: resp.Metadata,
LastModifiedTime: v.Properties.LastModified,
FileProperties: FileProperties{
FileAttributes: v.Attributes,
FileCreationTime: v.Properties.CreationTime,
FileLastWriteTime: v.Properties.LastWriteTime,
FilePermissions: permissions,
LastModifiedTime: v.Properties.LastModified,
},
}
}
// List files
for _, v := range page.Segment.Files {
fullPath := path.Join(parent, *v.Name)
fileClient := s.InternalClient.NewRootDirectoryClient().NewFileClient(fullPath)
resp, err := fileClient.GetProperties(ctx, &file.GetPropertiesOptions{})
a.NoError("Get file properties", err)
var permissions *string
if resp.FilePermissionKey != nil {
permResp, err := s.InternalClient.GetPermission(ctx, *v.PermissionKey, nil)
a.NoError("Get permissions", err)
permissions = permResp.Permission
}
out[fullPath] = ObjectProperties{
EntityType: common.EEntityType.Folder(),
HTTPHeaders: contentHeaders{
cacheControl: resp.CacheControl,
contentDisposition: resp.ContentDisposition,
contentEncoding: resp.ContentEncoding,
contentLanguage: resp.ContentLanguage,
contentType: resp.ContentType,
contentMD5: resp.ContentMD5,
},
Metadata: resp.Metadata,
LastModifiedTime: v.Properties.LastModified,
FileProperties: FileProperties{
FileAttributes: v.Attributes,
FileCreationTime: v.Properties.CreationTime,
FileLastWriteTime: v.Properties.LastWriteTime,
FilePermissions: permissions,
LastModifiedTime: v.Properties.LastModified,
},
}
}
}
}
return out
}
func (s *FileShareResourceManager) GetObject(a Asserter, path string, eType common.EntityType) ObjectResourceManager {
return &FileObjectResourceManager{
internalAccount: s.InternalAccount,
Service: s.Service,
Share: s,
path: path,
entityType: eType,
}
}
// ==================== FILE ====================
type FileObjectResourceManager struct {
internalAccount *AzureAccountResourceManager
Service *FileServiceResourceManager
Share *FileShareResourceManager
path string
entityType common.EntityType
}
func (f *FileObjectResourceManager) DefaultAuthType() ExplicitCredentialTypes {
return (&FileServiceResourceManager{}).DefaultAuthType()
}
func (f *FileObjectResourceManager) WithSpecificAuthType(cred ExplicitCredentialTypes, a Asserter, opts ...CreateAzCopyTargetOptions) AzCopyTarget {
return CreateAzCopyTarget(f, cred, a, opts...)
}
func (f *FileObjectResourceManager) ValidAuthTypes() ExplicitCredentialTypes {
return (&FileServiceResourceManager{}).ValidAuthTypes()
}
func (f *FileObjectResourceManager) ResourceClient() any {
switch f.entityType {
case common.EEntityType.Folder():
return f.getDirClient()
default: // For now, bundle up other entity types as files. That's how they should be implemented in AzCopy, at least.
return f.getFileClient()
}
}
func (f *FileObjectResourceManager) Canon() string {
return buildCanonForAzureResourceManager(f)
}
func (f *FileObjectResourceManager) Parent() ResourceManager {
return f.Share
}
func (f *FileObjectResourceManager) Account() AccountResourceManager {
return f.internalAccount
}
func (f *FileObjectResourceManager) Location() common.Location {
return f.Service.Location()
}
func (f *FileObjectResourceManager) Level() cmd.LocationLevel {
return cmd.ELocationLevel.Object()
}
func (f *FileObjectResourceManager) URI(opts ...GetURIOptions) string {
base := fileStripSAS(f.getFileClient().URL()) // restype doesn't matter here, same URL under the hood
base = f.internalAccount.ApplySAS(base, f.Location(), opts...)
base = addWildCard(base, opts...)
return base
}
func (f *FileObjectResourceManager) EntityType() common.EntityType {
return f.entityType
}
func (f *FileObjectResourceManager) ContainerName() string {
return f.Share.ContainerName()
}
func (f *FileObjectResourceManager) ObjectName() string {
return f.path
}
func (f *FileObjectResourceManager) PreparePermissions(a Asserter, p *string) *file.Permissions {
a.HelperMarker().Helper()
if p == nil {
return nil
}
perm := *p
fSDDL, err := sddl.ParseSDDL(perm)
a.NoError("parse input SDDL", err)
a.AssertNow("parsed string equivalence sanity check", Equal{}, fSDDL.String(), perm)
perm = fSDDL.PortableString()
if len(perm) >= ste.FilesServiceMaxSDDLSize {
resp, err := f.Share.InternalClient.CreatePermission(ctx, perm, nil)
a.NoError("Create share permission", err)
return &file.Permissions{PermissionKey: resp.FilePermissionKey}
}
return &file.Permissions{Permission: &perm}
}
func (f *FileObjectResourceManager) CreateParents(a Asserter) {
if !f.Share.Exists() {
f.Share.Create(a, ContainerProperties{})
}
dir, _ := path.Split(strings.TrimSuffix(f.path, "/"))
if dir != "" {
obj := f.Share.GetObject(a, dir, common.EEntityType.Folder()).(*FileObjectResourceManager)
// Create recursively calls this function.
if !obj.Exists() {
obj.Create(a, nil, ObjectProperties{})
}
}
}
func (f *FileObjectResourceManager) Create(a Asserter, body ObjectContentContainer, props ObjectProperties) {
a.HelperMarker().Helper()
var attr *file.NTFSFileAttributes
if DerefOrZero(props.FileProperties.FileAttributes) != "" {
var err error
attr, err = file.ParseNTFSFileAttributes(props.FileProperties.FileAttributes)
a.NoError("Parse attributes", err)
}
perms := f.PreparePermissions(a, props.FileProperties.FilePermissions)
f.CreateParents(a)
switch f.entityType {
case common.EEntityType.File():
client := f.getFileClient()
_, err := client.Create(ctx, body.Size(), &file.CreateOptions{
SMBProperties: &file.SMBProperties{
Attributes: attr,
CreationTime: props.FileProperties.FileCreationTime,
LastWriteTime: props.FileProperties.FileLastWriteTime,
},
Permissions: perms,
HTTPHeaders: props.HTTPHeaders.ToFile(),
Metadata: props.Metadata,
})
a.NoError("Create file", err)
err = client.UploadStream(ctx, body.Reader(), &file.UploadStreamOptions{
Concurrency: runtime.NumCPU(),
})
a.NoError("Upload Stream", err)
case common.EEntityType.Folder():
client := f.getDirClient()
_, err := client.Create(ctx, &directory.CreateOptions{
FileSMBProperties: &file.SMBProperties{
Attributes: attr,
CreationTime: props.FileProperties.FileCreationTime,
LastWriteTime: props.FileProperties.FileLastWriteTime,
},
FilePermissions: perms,
Metadata: props.Metadata,
})
// This is fine. Instead let's set properties.
if fileerror.HasCode(err, fileerror.ResourceAlreadyExists) {
err = nil
f.SetObjectProperties(a, props)
}
a.NoError("Create directory", err)
default:
a.Error("File Objects only support Files and Folders")
}
TrackResourceCreation(a, f)
}
func (f *FileObjectResourceManager) Delete(a Asserter) {
a.HelperMarker().Helper()
var err error
switch f.entityType {
case common.EEntityType.File():
_, err = f.getFileClient().Delete(ctx, nil)
case common.EEntityType.Folder():
_, err = f.getDirClient().Delete(ctx, nil)
default:
a.Error(fmt.Sprintf("entity type %s is not currently supported", f.entityType))
}
if fileerror.HasCode(err, fileerror.ResourceNotFound, fileerror.ShareNotFound, fileerror.ParentNotFound) {
err = nil
}
a.NoError("delete path", err)
}
func (f *FileObjectResourceManager) ListChildren(a Asserter, recursive bool) map[string]ObjectProperties {
a.HelperMarker().Helper()
a.AssertNow("must be folder to list children", Equal{}, f.entityType, common.EEntityType.Folder())
return f.Share.ListObjects(a, f.path, recursive)
}
func (f *FileObjectResourceManager) GetProperties(a Asserter) (out ObjectProperties) {
a.HelperMarker().Helper()
switch f.entityType {
case common.EEntityType.Folder():
resp, err := f.getDirClient().GetProperties(ctx, &directory.GetPropertiesOptions{})
a.NoError("Get directory properties", err)
var permissions *string
if pkey := DerefOrZero(resp.FilePermissionKey); pkey != "" {
permResp, err := f.Share.InternalClient.GetPermission(ctx, pkey, nil)
a.NoError("Get file permissions", err)
permissions = permResp.Permission
}
out = ObjectProperties{
EntityType: f.entityType, // It should be OK to just return entity type, getproperties should fail with the wrong restype
Metadata: resp.Metadata,
LastModifiedTime: resp.LastModified,
FileProperties: FileProperties{
FileAttributes: resp.FileAttributes,
FileCreationTime: resp.FileCreationTime,
FileLastWriteTime: resp.FileLastWriteTime,
FilePermissions: permissions,
LastModifiedTime: resp.LastModified,
},
}
case common.EEntityType.File():
resp, err := f.getFileClient().GetProperties(ctx, &file.GetPropertiesOptions{})
a.NoError("Get file properties", err)
var permissions *string
if pkey := DerefOrZero(resp.FilePermissionKey); pkey != "" {
permResp, err := f.Share.InternalClient.GetPermission(ctx, pkey, nil)
a.NoError("Get file permissions", err)
permissions = permResp.Permission
}
out = ObjectProperties{
EntityType: f.entityType,
HTTPHeaders: contentHeaders{
cacheControl: resp.CacheControl,
contentDisposition: resp.ContentDisposition,
contentEncoding: resp.ContentEncoding,
contentLanguage: resp.ContentLanguage,
contentType: resp.ContentType,
contentMD5: resp.ContentMD5,
},
Metadata: resp.Metadata,
LastModifiedTime: resp.LastModified,
FileProperties: FileProperties{
FileAttributes: resp.FileAttributes,
FileCreationTime: resp.FileCreationTime,
FileLastWriteTime: resp.FileLastWriteTime,
FilePermissions: permissions,
LastModifiedTime: resp.LastModified,
},
}
default:
a.Error("EntityType must be Folder or File. Currently: " + f.entityType.String())
}
return
}
func (f *FileObjectResourceManager) SetHTTPHeaders(a Asserter, h contentHeaders) {
a.HelperMarker().Helper()
a.AssertNow("HTTP headers are only available on files", Equal{}, f.entityType, common.EEntityType.File())
client := f.getFileClient()
_, err := client.SetHTTPHeaders(ctx, &file.SetHTTPHeadersOptions{
HTTPHeaders: &file.HTTPHeaders{
CacheControl: h.cacheControl,
ContentDisposition: h.contentDisposition,
ContentEncoding: h.contentEncoding,
ContentLanguage: h.contentLanguage,
ContentMD5: h.contentMD5,
ContentType: h.contentType,
},
})
a.NoError("Set HTTP Headers", err)
}
func (f *FileObjectResourceManager) SetMetadata(a Asserter, metadata common.Metadata) {
a.HelperMarker().Helper()
switch f.entityType {
case common.EEntityType.File():
_, err := f.getFileClient().SetMetadata(ctx, &file.SetMetadataOptions{Metadata: metadata})
a.NoError("Set file metadata", err)
case common.EEntityType.Folder():
_, err := f.getDirClient().SetMetadata(ctx, &directory.SetMetadataOptions{Metadata: metadata})
a.NoError("Set directory metadata", err)
default:
a.Error("EntityType must be Folder or File. Currently: " + f.entityType.String())
}
}
func (f *FileObjectResourceManager) SetObjectProperties(a Asserter, props ObjectProperties) {
a.HelperMarker().Helper()
var attr *file.NTFSFileAttributes
if DerefOrZero(props.FileProperties.FileAttributes) != "" {
var err error
attr, err = file.ParseNTFSFileAttributes(props.FileProperties.FileAttributes)
a.NoError("Parse attributes", err)
}
perms := f.PreparePermissions(a, props.FileProperties.FilePermissions)
switch f.entityType {
case common.EEntityType.File():
client := f.getFileClient()
var _, err = client.SetHTTPHeaders(ctx, &file.SetHTTPHeadersOptions{
SMBProperties: &file.SMBProperties{
Attributes: attr,
CreationTime: props.FileProperties.FileCreationTime,
LastWriteTime: props.FileProperties.FileLastWriteTime,
},
Permissions: perms,
HTTPHeaders: props.HTTPHeaders.ToFile(),
})
a.NoError("Set file HTTP headers", err)
_, err = client.SetMetadata(ctx, &file.SetMetadataOptions{Metadata: props.Metadata})
a.NoError("Set file metadata", err)
case common.EEntityType.Folder():
client := f.getDirClient()
var _, err = client.SetProperties(ctx, &directory.SetPropertiesOptions{
FileSMBProperties: &file.SMBProperties{
Attributes: attr,
CreationTime: props.FileProperties.FileCreationTime,
LastWriteTime: props.FileProperties.FileLastWriteTime,
},
FilePermissions: perms,
})
a.NoError("Set folder properties", err)
_, err = f.getDirClient().SetMetadata(ctx, &directory.SetMetadataOptions{Metadata: props.Metadata})
a.NoError("Set folder metadata", err)
}
}
func (f *FileObjectResourceManager) getFileClient() *file.Client {
return f.Share.InternalClient.NewRootDirectoryClient().NewFileClient(f.path)
}
func (f *FileObjectResourceManager) getDirClient() *directory.Client {
return f.Share.InternalClient.NewDirectoryClient(f.path)
}
func (f *FileObjectResourceManager) Download(a Asserter) io.ReadSeeker {
a.HelperMarker().Helper()
a.Assert("Entity type must be file", Equal{}, f.entityType, common.EEntityType.File())
resp, err := f.getFileClient().DownloadStream(ctx, nil)
a.NoError("Download stream", err)
buf := &bytes.Buffer{}
if err == nil && resp.Body != nil {
_, err = io.Copy(buf, resp.Body)
a.NoError("Read body", err)
}
return bytes.NewReader(buf.Bytes())
}
func (f *FileObjectResourceManager) Exists() bool {
var err error
if f.entityType != common.EEntityType.Folder() {
_, err = f.getFileClient().GetProperties(ctx, nil)
} else {
_, err = f.getDirClient().GetProperties(ctx, nil)
}
return err == nil || !fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ShareNotFound, fileerror.ShareBeingDeleted, fileerror.ResourceNotFound)
}