gslib/commands/mb.py

# -*- coding: utf-8 -*-
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of mb command for creating cloud storage buckets."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import re
import textwrap

from gslib.cloud_api import AccessDeniedException, BadRequestException
from gslib.command import Command
from gslib.command_argument import CommandArgument
from gslib.commands.rpo import VALID_RPO_VALUES
from gslib.commands.rpo import VALID_RPO_VALUES_STRING
from gslib.cs_api_map import ApiSelector
from gslib.exception import CommandException
from gslib.exception import InvalidUrlError
from gslib.storage_url import StorageUrlFromString
from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
from gslib.utils.constants import NO_MAX
from gslib.utils.retention_util import RetentionInSeconds
from gslib.utils.shim_util import GcloudStorageFlag
from gslib.utils.shim_util import GcloudStorageMap
from gslib.utils.text_util import InsistAscii
from gslib.utils.text_util import InsistOnOrOff
from gslib.utils.text_util import NormalizeStorageClass
from gslib.utils.encryption_helper import ValidateCMEK

_SYNOPSIS = """
  gsutil mb [-b (on|off)] [-c <class>] [-k <key>] [-l <location>] [-p <project>]
            [--autoclass] [--retention <time>] [--pap <setting>]
            [--placement <region1>,<region2>]
            [--rpo {}] gs://<bucket_name>...
""".format(VALID_RPO_VALUES_STRING)

_DETAILED_HELP_TEXT = ("""
<B>SYNOPSIS</B>
""" + _SYNOPSIS + """


<B>DESCRIPTION</B>
  Create one or more new buckets. Google Cloud Storage has a single namespace,
  so you are not allowed to create a bucket with a name already in use by
  another user. You can, however, carve out parts of the bucket name space
  corresponding to your company's domain name (see "gsutil help naming").

  If you don't specify a project ID or project number using the -p option, the
  buckets are created using the default project ID specified in your `gsutil
  configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_.

  The -l option specifies the location for the buckets. Once a bucket is created
  in a given location, it cannot be moved to a different location. Instead, you
  need to create a new bucket, move the data over, and then delete the original
  bucket.

<B>BUCKET STORAGE CLASSES</B>
  You can specify one of the `storage classes
  <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket
  with the -c option.

  Example:

    gsutil mb -c nearline gs://some-bucket

  See online documentation for
  `pricing <https://cloud.google.com/storage/pricing>`_ and
  `SLA <https://cloud.google.com/storage/sla>`_ details.

  If you don't specify a -c option, the bucket is created with the
  default storage class Standard Storage.

<B>BUCKET LOCATIONS</B>
  You can specify one of the `available locations
  <https://cloud.google.com/storage/docs/locations>`_ for a bucket
  with the -l option.

  Examples:

    gsutil mb -l asia gs://some-bucket

    gsutil mb -c standard -l us-east1 gs://some-bucket

  If you don't specify a -l option, the bucket is created in the default
  location (US).

<B>Retention Policy</B>
  You can specify retention period in one of the following formats:

  --retention <number>s
      Specifies retention period of <number> seconds for objects in this bucket.

  --retention <number>d
      Specifies retention period of <number> days for objects in this bucket.

  --retention <number>m
      Specifies retention period of <number> months for objects in this bucket.

  --retention <number>y
      Specifies retention period of <number> years for objects in this bucket.

  Examples:

    gsutil mb --retention 1y gs://some-bucket

    gsutil mb --retention 36m gs://some-bucket

  If you don't specify a --retention option, the bucket is created with no
  retention policy.

<B>OPTIONS</B>
  --autoclass            Enables the Autoclass feature that automatically
                         sets object storage classes.

  -b <on|off>            Specifies the uniform bucket-level access setting.
                         When "on", ACLs assigned to objects in the bucket are
                         not evaluated. Consequently, only IAM policies grant
                         access to objects in these buckets. Default is "off".

  -c class               Specifies the default storage class. Default is
                         ``Standard``. See `Available storage classes
                         <https://cloud.google.com/storage/docs/storage-classes#classes>`_
                         for a list of possible values.

  -k <key>               Set the default KMS key using the full path to the key,
                         which has the following form:
                         ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]``

  -l location            Can be any supported location. See
                         https://cloud.google.com/storage/docs/locations
                         for a discussion of this distinction. Default is US.
                         Locations are case insensitive.

  -p project             Specifies the project ID or project number to create
                         the bucket under.

  -s class               Same as -c.

  --retention time       Specifies the retention policy. Default is no retention
                         policy. This can only be set on gs:// buckets and
                         requires using the JSON API. For more details about
                         retention policy see "gsutil help retention"

  --pap setting          Specifies the public access prevention setting. Valid
                         values are "enforced" or "inherited". When
                         "enforced", objects in this bucket cannot be made
                         publicly accessible. Default is "inherited".

  --placement reg1,reg2  Two regions that form the custom dual-region.
                         Only regions within the same continent are or will ever
                         be valid. Invalid location pairs (such as
                         mixed-continent, or with unsupported regions)
                         will return an error.

  --rpo setting          Specifies the `replication setting
                         <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_.
                         This flag is not valid for single-region buckets,
                         and multi-region buckets only accept a value of
                         DEFAULT. Valid values for dual region buckets
                         are {rpo_values}. If unspecified, DEFAULT is applied
                         for dual-region and multi-region buckets.

""".format(rpo_values=VALID_RPO_VALUES_STRING))

# Regex to disallow buckets violating charset or not [3..255] chars total.
BUCKET_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$')
# Regex to disallow buckets with individual DNS labels longer than 63.
TOO_LONG_DNS_NAME_COMP = re.compile(r'[-_a-z0-9]{64}')
_RETENTION_FLAG = '--retention'

IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue
BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue


class MbCommand(Command):
  """Implementation of gsutil mb command."""

  # Command specification. See base class for documentation.
  command_spec = Command.CreateCommandSpec(
      'mb',
      command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'],
      usage_synopsis=_SYNOPSIS,
      min_args=1,
      max_args=NO_MAX,
      supported_sub_args='b:c:l:p:s:k:',
      supported_private_args=[
          'autoclass', 'retention=', 'pap=', 'placement=', 'rpo='
      ],
      file_url_ok=False,
      provider_url_ok=False,
      urls_start_arg=0,
      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
      gs_default_api=ApiSelector.JSON,
      argparse_arguments=[
          CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
      ],
  )
  # Help specification. See help_provider.py for documentation.
  help_spec = Command.HelpSpec(
      help_name='mb',
      help_name_aliases=[
          'createbucket',
          'makebucket',
          'md',
          'mkdir',
          'location',
          'dra',
          'dras',
          'reduced_availability',
          'durable_reduced_availability',
          'rr',
          'reduced_redundancy',
          'standard',
          'storage class',
          'nearline',
          'nl',
      ],
      help_type='command_help',
      help_one_line_summary='Make buckets',
      help_text=_DETAILED_HELP_TEXT,
      subcommand_help_text={},
  )

  gcloud_storage_map = GcloudStorageMap(
      gcloud_command=['storage', 'buckets', 'create'],
      flag_map={
          '--autoclass':
              GcloudStorageFlag('--enable-autoclass'),
          '-b':
              GcloudStorageFlag({
                  'on': '--uniform-bucket-level-access',
                  'off': None,
              }),
          '-c':
              GcloudStorageFlag('--default-storage-class'),
          '-s':
              GcloudStorageFlag('--default-storage-class'),
          '-k':
              GcloudStorageFlag('--default-encryption-key'),
          '-l':
              GcloudStorageFlag('--location'),
          '-p':
              GcloudStorageFlag('--project'),
          '--pap':
              GcloudStorageFlag({
                  'enforced': '--public-access-prevention',
                  'inherited': None,
              }),
          '--placement':
              GcloudStorageFlag('--placement'),
          _RETENTION_FLAG:
              GcloudStorageFlag('--retention-period'),
          '--rpo':
              GcloudStorageFlag('--recovery-point-objective')
      },
  )

  def get_gcloud_storage_args(self):
    retention_arg_idx = 0
    while retention_arg_idx < len(self.sub_opts):
      if self.sub_opts[retention_arg_idx][0] == _RETENTION_FLAG:
        break
      retention_arg_idx += 1
    if retention_arg_idx < len(self.sub_opts):
      # Convert retention time to seconds, which gcloud knows how to handle.
      self.sub_opts[retention_arg_idx] = (
          _RETENTION_FLAG,
          str(RetentionInSeconds(self.sub_opts[retention_arg_idx][1])) + 's')
    return super().get_gcloud_storage_args(MbCommand.gcloud_storage_map)

  def RunCommand(self):
    """Command entry point for the mb command."""
    autoclass = False
    bucket_policy_only = None
    kms_key = None
    location = None
    storage_class = None
    seconds = None
    placements = None
    public_access_prevention = None
    rpo = None
    json_only_flags_in_command = []
    if self.sub_opts:
      for o, a in self.sub_opts:
        if o == '--autoclass':
          autoclass = True
          json_only_flags_in_command.append(o)
        elif o == '-k':
          kms_key = a
          ValidateCMEK(kms_key)
          json_only_flags_in_command.append(o)
        elif o == '-l':
          location = a
        elif o == '-p':
          # Project IDs are sent as header values when using gs and s3 XML APIs.
          InsistAscii(a, 'Invalid non-ASCII character found in project ID')
          self.project_id = a
        elif o == '-c' or o == '-s':
          storage_class = NormalizeStorageClass(a)
        elif o == _RETENTION_FLAG:
          seconds = RetentionInSeconds(a)
        elif o == '--rpo':
          rpo = a.strip()
          if rpo not in VALID_RPO_VALUES:
            raise CommandException(
                'Invalid value for --rpo. Must be one of: {},'
                ' provided: {}'.format(VALID_RPO_VALUES_STRING, a))
          json_only_flags_in_command.append(o)
        elif o == '-b':
          InsistOnOrOff(a, 'Only on and off values allowed for -b option')
          bucket_policy_only = (a == 'on')
          json_only_flags_in_command.append(o)
        elif o == '--pap':
          public_access_prevention = a
          json_only_flags_in_command.append(o)
        elif o == '--placement':
          placements = a.split(',')
          if len(placements) != 2:
            raise CommandException(
                'Please specify two regions separated by comma without space.'
                ' Specified: {}'.format(a))
          json_only_flags_in_command.append(o)

    bucket_metadata = apitools_messages.Bucket(location=location,
                                               rpo=rpo,
                                               storageClass=storage_class)
    if autoclass:
      bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue(
          enabled=autoclass)
    if bucket_policy_only or public_access_prevention:
      bucket_metadata.iamConfiguration = IamConfigurationValue()
      iam_config = bucket_metadata.iamConfiguration
      if bucket_policy_only:
        iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
        iam_config.bucketPolicyOnly.enabled = bucket_policy_only
      if public_access_prevention:
        iam_config.publicAccessPrevention = public_access_prevention

    if kms_key:
      encryption = apitools_messages.Bucket.EncryptionValue()
      encryption.defaultKmsKeyName = kms_key
      bucket_metadata.encryption = encryption

    if placements:
      placement_config = apitools_messages.Bucket.CustomPlacementConfigValue()
      placement_config.dataLocations = placements
      bucket_metadata.customPlacementConfig = placement_config

    for bucket_url_str in self.args:
      bucket_url = StorageUrlFromString(bucket_url_str)
      if seconds is not None:
        if bucket_url.scheme != 'gs':
          raise CommandException('Retention policy can only be specified for '
                                 'GCS buckets.')
        retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
            retentionPeriod=seconds))
        bucket_metadata.retentionPolicy = retention_policy

      if json_only_flags_in_command and self.gsutil_api.GetApiSelector(
          bucket_url.scheme) != ApiSelector.JSON:
        raise CommandException('The {} option(s) can only be used for GCS'
                               ' Buckets with the JSON API'.format(
                                   ', '.join(json_only_flags_in_command)))

      if not bucket_url.IsBucket():
        raise CommandException('The mb command requires a URL that specifies a '
                               'bucket.\n"%s" is not valid.' % bucket_url)
      if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or
          TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
        raise InvalidUrlError('Invalid bucket name in URL "%s"' %
                              bucket_url.bucket_name)

      self.logger.info('Creating %s...', bucket_url)
      # Pass storage_class param only if this is a GCS bucket. (In S3 the
      # storage class is specified on the key object.)
      try:
        self.gsutil_api.CreateBucket(bucket_url.bucket_name,
                                     project_id=self.project_id,
                                     metadata=bucket_metadata,
                                     provider=bucket_url.scheme)
      except AccessDeniedException as e:
        message = e.reason
        if 'key' in message:
          # This will print the error reason and append the following as a
          # suggested next step:
          #
          # To authorize, run:
          #   gsutil kms authorize \
          #     -k <kms_key> \
          #     -p <project_id>
          message += ' To authorize, run:\n  gsutil kms authorize'
          message += ' \\\n    -k %s' % kms_key
          if (self.project_id):
            message += ' \\\n    -p %s' % self.project_id
          raise CommandException(message)
        else:
          raise

      except BadRequestException as e:
        if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and
            bucket_url.scheme == 'gs'):
          bucket_name = bucket_url.bucket_name
          final_comp = bucket_name[bucket_name.rfind('.') + 1:]
          raise CommandException('\n'.join(
              textwrap.wrap(
                  'Buckets with "." in the name must be valid DNS names. The bucket'
                  ' you are attempting to create (%s) is not a valid DNS name,'
                  ' because the final component (%s) is not currently a valid part'
                  ' of the top-level DNS tree.' % (bucket_name, final_comp))))
        else:
          raise

    return 0