appengine/integration_tests/testsuite/deploy_app.py

#!/usr/bin/python

# Copyright 2017 Google Inc. All rights reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import json
import logging
import os
from retrying import retry
import subprocess
import test_util
import time

from google.cloud import bigquery

import constants
import template

DATASET_NAME = 'cloudperf'
DEPLOY_LATENCY_PROJECT_ENV = 'DEPLOY_LATENCY_PROJECT'
TABLE_NAME = 'deploy_latency'


def _cleanup_files(appdir):
    try:
        os.remove(os.path.join(appdir, 'Dockerfile'))
        os.remove(os.path.join(appdir, 'test.yaml'))
    except Exception:
        pass


def _set_base_image(image):
    # substitute vars in Dockerfile (equivalent of envsubst)
    with open('Dockerfile.in', 'r') as fin:
        with open('Dockerfile', 'w') as fout:
            for line in fin:
                fout.write(line.replace('${STAGING_IMAGE}', image))


def _set_builder_image(builder):
    with open('test.yaml.in', 'r') as fin:
        with open('test.yaml', 'w') as fout:
            for line in fin:
                fout.write(line.replace('${STAGING_BUILDER_IMAGE}', builder))


def _record_latency_to_bigquery(deploy_latency, language, is_xrt):
    current_date = datetime.datetime.now()
    row = [(language, current_date, deploy_latency, is_xrt)]

    project = os.environ.get(DEPLOY_LATENCY_PROJECT_ENV)
    if not project:
        logging.warn('No project specified to record deployment latency!')
        logging.warn('If you wish to record deployment latency, \
                     please set %s env var and try again.',
                     DEPLOY_LATENCY_PROJECT_ENV)
        return 0
    logging.debug('Fetching bigquery client for project %s', project)
    client = bigquery.Client(project=project)
    dataset = client.dataset(DATASET_NAME)
    logging.debug('Writing bigquery data to table %s in dataset %s',
                  TABLE_NAME, dataset)
    table_ref = bigquery.TableReference(dataset_ref=dataset,
                                        table_id=TABLE_NAME)
    table = client.get_table(table_ref)
    return client.create_rows(table, row)


def deploy_app_and_record_latency(appdir, language, is_xrt):
    start_time = time.time()

    version, url = deploy_app(appdir=appdir, environment=constants.GAE)

    # Latency is in seconds round up to 2 decimals
    deploy_latency = round(time.time() - start_time, 2)

    try:
        # Store the deploy latency data to bigquery
        _record_latency_to_bigquery(deploy_latency, language, is_xrt)
    except Exception as e:
        # log error, but ignore and try and cleanup version anyway
        logging.error("Error writing latency to bigquery: %s", e)
    return version, url


def deploy_app_gae(yaml):
    logging.debug('Starting deploy to GAE')

    deployed_version = test_util.generate_version()

    # TODO: once sdk driver is published, use it here
    deploy_command = ['gcloud', 'app', 'deploy', '--no-promote',
                      '--version', deployed_version, '-q']
    if yaml:
        logging.info(yaml)
        deploy_command.append(yaml)

    try:
        test_util.execute_command(deploy_command, True)
        return (deployed_version,
                test_util.retrieve_url_for_version(deployed_version))
    except subprocess.CalledProcessError as e:
        try:
            stop_version(deployed_version)
        except Exception:
            pass
        raise e


def deploy_app_gke(yaml):
    logging.debug('Starting deploy to GKE')

    image_name = test_util.generate_gke_image_name()
    service_name = test_util.generate_gke_service_name()
    namespace = test_util.generate_namespace()

    build_command = ['docker', 'build', '-t', image_name, '.']
    test_util.execute_command(build_command, True)

    push_command = ['gcloud', 'docker', '--', 'push', image_name]
    test_util.execute_command(push_command, True)

    # This command updates the kubeconfig file with credentials for the given
    # GKE cluster. Essentially, points kubectl to our preconfigured cluster.
    cred_command = ['gcloud', 'container', 'clusters',
                    'get-credentials', constants.CLUSTER_NAME]
    test_util.execute_command(cred_command, True)

    namespace_command = ['kubectl', 'create', 'namespace', namespace]
    test_util.execute_command(namespace_command, True)

    deploy_command = ['kubectl', 'apply', '-f', '-',
                      '--namespace', namespace]

    try:
        test_util.execute_command(deploy_command, True,
                                  template.GKE_TEMPLATE.format(
                                    service_name=service_name,
                                    test_image=image_name))

        return namespace, test_util.get_external_ip_for_cluster(service_name,
                                                                namespace)
    except subprocess.CalledProcessError as e:
        try:
            stop_deployment(namespace)
        except Exception:
            pass
        raise e


def deploy_app(appdir, environment, base_image=None,
               builder_image=None, yaml=None):
    try:
        if yaml:
            # convert yaml to absolute path before changing directory
            yaml = os.path.abspath(yaml)

        # change to app directory (and remember original directory)
        owd = os.getcwd()
        os.chdir(appdir)

        # fills in image field in templated Dockerfile and/or builder yaml
        if base_image:
            _set_base_image(base_image)
        if builder_image:
            _set_builder_image(builder_image)

        if environment == constants.GAE:
            return deploy_app_gae(yaml)
        elif environment == constants.GKE:
            return deploy_app_gke(yaml)
        else:
            raise Exception('Invalid environment provided: %s', environment)

    except subprocess.CalledProcessError as cpe:
        logging.error('Error encountered when deploying application! %s',
                      cpe.output)
        raise
    except Exception as e:
        logging.error('Error encountered when deploying application! %s', e)
        raise
    finally:
        _cleanup_files(appdir)
        os.chdir(owd)


@retry(wait_fixed=4000, stop_max_attempt_number=8)
def stop_version(version):
    logging.debug('Removing application version %s', version)
    try:
        delete_command = ['gcloud', 'app', 'services', 'delete',
                          'default', '--version', version, '-q']

        subprocess.check_output(delete_command)
    except subprocess.CalledProcessError as cpe:
        logging.error('Error encountered when deleting app version! %s',
                      cpe.output)
        raise


@retry(wait_fixed=4000, stop_max_attempt_number=8)
def stop_deployment(namespace):
    logging.debug('Removing namespace %s', namespace)
    try:
        service_command = ['kubectl', 'get', 'services', '--namespace',
                           namespace, '--output=json']
        output = test_util.execute_command(service_command, True)
        logging.info(output)
        services = json.loads(output)['items']
        for service in services:
            name = service['metadata']['name']
            delete_service_cmd = ['kubectl', 'delete', 'service', name,
                                  '--namespace', namespace]
            test_util.execute_command(delete_service_cmd, True)
        delete_command = ['kubectl', 'delete', 'namespace', namespace]
        test_util.execute_command(delete_command, True)
    except subprocess.CalledProcessError as cpe:
        logging.error('Error encountered when deleting namespace! '
                      'Manual cleanup may be necessary. %s', cpe.output)
    except Exception as e:
        logging.error('Error encountered when deleting services! '
                      'Manual cleanup may be necessary. %s', e)