tools/gcs-bucket-mover/gcs_bucket_mover/cli.py

#!/usr/bin/env python2
# Copyright 2018 Google LLC. All rights reserved. Licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
#
# Any software provided by Google hereunder is distributed "AS IS", WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, and is not intended for production use.
"""Parses the command line and starts the script"""

import textwrap

import configargparse
import yaml

from gcs_bucket_mover import bucket_mover_service
from gcs_bucket_mover import bucket_mover_tester
from gcs_bucket_mover import configuration


def _get_parsed_args():
  """Parses command line arguments and the config file.

  Order of precedence for config values is: command line > config file values > defaults

  Returns:
      A "Namespace" object. See argparse.ArgumentParser.parse_args() for more details.
  """

  parser = configargparse.ArgumentParser(
    description=
    "Moves a GCS bucket from one project to another, along with all objects and optionally"
    " copying all other bucket settings.",
    config_file_parser_class=configargparse.YAMLConfigFileParser,
    formatter_class=configargparse.RawTextHelpFormatter,
  )
  parser.add_argument("--config",
                      is_config_file=True,
                      help="The path to the local config file")

  parser.add_argument("bucket_name",
                      help="The name of the bucket to be moved.")
  parser.add_argument("source_project",
                      help="The project id that the bucket is currently in.")
  parser.add_argument("target_project",
                      help="The project id that the bucket will be moved to.")
  parser.add_argument(
    "--gcp_source_project_service_account_key",
    help=
    "The location on disk for service account key json file from the source project",
  )
  parser.add_argument(
    "--gcp_target_project_service_account_key",
    help=
    "The location on disk for service account key json file from the target project",
  )
  parser.add_argument(
    "--test",
    action="store_true",
    help=textwrap.dedent("""\
        This will run a test of the tool to ensure all permissions are set up correctly and
        buckets can be moved between the two projects, using a randomly generated bucket.
        A fake bucket name will still need to be specified."""),
  )
  parser.add_argument(
    "--disable_bucket_lock",
    action="store_true",
    help=textwrap.dedent("""\
        Disabling the bucket lock option means that the mover will not look for a lock file
        before starting the move, and it will not lock down permissions on the source bucket
        before starting the move."""),
  )
  parser.add_argument("--lock_file_name",
                      help="The name of the lock file in the bucket")
  parser.add_argument(
    "--rename_bucket_to",
    help=textwrap.dedent("""\
        Specifying a target bucket name allows the tool to perform a bucket rename. Note that
        the original source bucket will be deleted, so that bucket name can then potentially be
        used by someone else."""),
  )
  parser.add_argument(
    "--temp_bucket_name",
    help="The temporary bucket name to use in the target project.",
  )
  parser.add_argument(
    "--location",
    help="Specify a different location for the target bucket.")
  parser.add_argument(
    "--storage_class",
    choices=[
      "MULTI_REGIONAL",
      "REGIONAL",
      "STANDARD",
      "NEARLINE",
      "COLDLINE",
      "DURABLE_REDUCED_AVAILABILITY",
    ],
    help="Specify a different storage class for the target bucket.",
  )
  parser.add_argument(
    "--skip_everything",
    action="store_true",
    help=
    "Only copies the bucket's storage class and location. Equivalent to setting every other"
    " --skip parameter to True.",
  )
  parser.add_argument(
    "--skip_acl",
    action="store_true",
    help="Don't replicate the ACLs from the source bucket.",
  )
  parser.add_argument(
    "--skip_cors",
    action="store_true",
    help="Don't copy the CORS settings from the source bucket.",
  )
  parser.add_argument(
    "--skip_default_obj_acl",
    action="store_true",
    help="Don't copy the Default Object ACL from the source bucket.",
  )
  parser.add_argument(
    "--skip_iam",
    action="store_true",
    help="Don't replicate the IAM policies from the source bucket.",
  )
  parser.add_argument(
    "--skip_kms_key",
    action="store_true",
    help="Don't copy the Default KMS Key from the source bucket.",
  )
  parser.add_argument(
    "--skip_labels",
    action="store_true",
    help="Don't copy the Labels from the source bucket.",
  )
  parser.add_argument(
    "--skip_logging",
    action="store_true",
    help="Don't copy the Logging settings from the source bucket.",
  )
  parser.add_argument(
    "--skip_lifecycle_rules",
    action="store_true",
    help="Don't copy the Lifecycle Rules from the source bucket.",
  )
  parser.add_argument(
    "--skip_notifications",
    action="store_true",
    help=
    "Don't copy the Cloud Pub/Sub notification setting from the source bucket.",
  )
  parser.add_argument(
    "--skip_requester_pays",
    action="store_true",
    help="Don't copy the Requester Pays setting from the source bucket.",
  )
  parser.add_argument(
    "--skip_versioning",
    action="store_true",
    help="Don't copy the Versioning setting from the source bucket.",
  )

  # Variables set in the config file for running different bucket tests with the --test option
  parser.add_argument("--test_bucket_location",
                      help="The location to create the test bucket in")
  parser.add_argument(
    "--test_default_kms_key_name",
    help="A custom KSM key to assign to the test bucket",
  )
  parser.add_argument(
    "--test_email_for_iam",
    help="An IAM email to use for testing permissions on the test bucket",
  )
  parser.add_argument(
    "--test_logging_bucket",
    help="An existing bucket to set up logging on the test bucket",
  )
  parser.add_argument(
    "--test_logging_prefix",
    help="A prefix to use for the logging on the test bucket",
  )
  parser.add_argument("--test_storage_class",
                      help="The storage class to use for the test bucket")
  parser.add_argument(
    "--test_topic_name",
    help="A topic name to set up a notification for on the test bucket",
  )
  parser.add_argument(
    "--preserve_custom_time",
    help=
    "Preserves the source objects time created metadata in the custom time field in the destination object",
  )
  parser.add_argument("--log_action",
                      help="A log action to see log on google log  console")
  parser.add_argument(
    "--log_action_state",
    help=
    "A log action state for successfull and failed to see log on google log  console",
  )

  return parser.parse_args()


def _parse_yaml_file(path):
  """Load and parse local YAML file

  Args:
      path: a local file system path

  Returns:
      a Python object representing the parsed YAML data
  """

  with open(path, "r") as stream:
    try:
      return yaml.safe_load(stream)
    except yaml.YAMLError as ex:
      print(ex)


def main():
  """Get passed in args and run either a test run or an actual move"""
  parsed_args = _get_parsed_args()

  # Load the config values set in the config file and create the storage clients.
  config = configuration.Configuration.from_conf(parsed_args)

  # Create the cloud logging client that will be passed to all other modules.
  cloud_logger = config.target_logging_client.logger("gcs-bucket-mover")  # pylint: disable=no-member

  if parsed_args.test:
    test_bucket_name = bucket_mover_tester.set_up_test_bucket(
      config, parsed_args)
    config.bucket_name = test_bucket_name
    config.target_bucket_name = test_bucket_name

  bucket_mover_service.main(config, parsed_args, cloud_logger)