# Copyright (c) 2018 Uber Technologies, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import, print_function import argparse import datetime import logging import shutil import subprocess import sys import tempfile import time from os.path import join from . import commandlineutil, projectgen from .cpulogger import CPULogger from .moduletree import ModuleGenType from .statemanagement import SettingsState, XcodeManager from .util import check_dependent_commands, grab_mac_marketing_name, makedir, sudo_enabled class CommandLineMultisuite(object): @staticmethod def parse_config(args): tmp_root = join(tempfile.gettempdir(), 'mock_app_gen_out') log_root = join(tmp_root, 'logs') parser = argparse.ArgumentParser( description='Build all mock app types and log times & traces to a log file. ' 'Useful as a benchmark suite to compare build performance of different computers.') parser.add_argument('--log_dir', default=log_root, help="Where logs such as build times should exist."), parser.add_argument( '--app_gen_output_dir', default=tmp_root, help="Where generated mock apps should be outputted to."), parser.add_argument('--buck_command', default='buck', help="The path to the buck binary. Defaults to `buck`."), commandlineutil.AppGenerationConfig.add_app_gen_options(parser) actions = parser.add_argument_group('Extra Actions') actions.add_argument( '--trace_cpu', action='store_true', default=False, help="If we should add cpu utilization to build traces."), actions.add_argument( '--switch_xcode_versions', action='store_true', default=False, help="Switch xcode verions as part of the full multisuite test. This will search your " "`/Applications` directory for xcode.app bundles to build with. Requires sudo."), actions.add_argument( '--full_clean', action='store_true', default=False, help="Clean all default xcode cache directories to prevent cache effects changing test results."), testing = parser.add_argument_group('Testing Shortcuts') testing.add_argument( '--skip_xcode_build', action='store_true', default=False, help="Skips building the mock apps. Useful for speeding up testing and making " "integration tests independent of non-python dependencies."), testing.add_argument( '--test_build_only', action='store_true', default=False, help="Only builds a small flat build type to create short testing loops."), out = parser.parse_args(args) commandlineutil.AppGenerationConfig.validate_app_gen_options(out) return out # noinspection PyAttributeOutsideInit def config_to_vars(self, config): self.app_gen_options = commandlineutil.AppGenerationConfig() self.app_gen_options.pull_from_args(config) self.log_dir = config.log_dir self.output_dir = config.app_gen_output_dir self.buck_binary = config.buck_command logging.info('Log output directory: %s', self.log_dir) logging.info('Mock app gen output directory: %s', self.output_dir) self.trace_cpu = config.trace_cpu self.switch_xcode_versions = config.switch_xcode_versions self.full_clean = config.full_clean self.run_xcodebuild = (not config.skip_xcode_build) self.test_build_only = config.test_build_only def main(self, args=sys.argv[1:]): logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(funcName)s: %(message)s') args = self.parse_config(args) self.config_to_vars(args) try: self.run_multisuite() except Exception as e: print(e) finally: self.multisuite_cleanup() logging.info("Done") # noinspection PyAttributeOutsideInit def make_context(self, log_dir, output_dir, test_build): self.cpu_logger = CPULogger() self.xcode_manager = XcodeManager() self.settings_state = SettingsState(output_dir) self.log_dir = log_dir self.output_dir = output_dir self.mock_output_dir = join(output_dir, 'apps', 'mockapp') self.mock_app_workspace = join(self.mock_output_dir, 'App', 'MockApp.xcworkspace') self.buckconfig_path = join(output_dir, '.buckconfig.local') self.sys_info_path = join(log_dir, 'system_info.txt') self.build_time_path = join(log_dir, 'build_times.txt') self.build_time_csv_path = join(log_dir, 'build_times.csv') self.build_trace_path = join(log_dir, 'build_traces') self.buck_path = '/apps/mockapp' self.app_buck_path = "/{}/App:MockApp".format(self.buck_path) has_dot = self.app_gen_options.dot_file_path and self.app_gen_options.dot_root_node_name if test_build: logging.info("Using test build settings") self.app_gen_options.lines_of_code = 100000 self.wmo_modes = [False] self.type_list = [ModuleGenType.dot] if has_dot else [ModuleGenType.flat] else: self.wmo_modes = [True, False] self.type_list = ModuleGenType.enum_list() if not has_dot: logging.warning("Removing dot mock app type due to lack of dot file to read. " "Specify one in the command line options.") self.type_list.remove(ModuleGenType.dot) def build_app_type(self, gen_type, wmo_enabled): xcode_version, xcode_build_id = XcodeManager.get_current_xcode_version() xcode_name = '{}_'.format(xcode_version.replace('.', '_')) build_log_path = join(self.log_dir, '{}{}_mockapp_build_log.txt'.format(xcode_name, gen_type)) gen_info = '{} (wmo_enabled: {}, xcode_version: {} {})'.format(gen_type, wmo_enabled, xcode_version, xcode_build_id) logging.info('##### Generating %s', gen_info) self.project_generator.use_wmo = wmo_enabled commandlineutil.del_old_output_dir(self.mock_output_dir) logging.info('Generating mock app') app_node, node_list = commandlineutil.gen_graph(gen_type, self.app_gen_options) self.project_generator.gen_app(app_node, node_list, self.app_gen_options.lines_of_code) swift_loc = commandlineutil.count_loc(self.mock_output_dir) logging.info('App type "%s" generated %d loc', gen_type, swift_loc) # Build App total_time = 0 if self.run_xcodebuild: logging.info('Generate xcode workspace & clean') derived_data_path = join(tempfile.gettempdir(), 'ub_mockapp_derived_data') shutil.rmtree(derived_data_path, ignore_errors=True) makedir(derived_data_path) subprocess.check_call([self.buck_binary, 'project', self.app_buck_path, '-d']) if self.full_clean: self.xcode_manager.clean_caches() logging.info('Start xcodebuild') start = time.time() with open(build_log_path, 'w') as build_log_file: subprocess.check_call([ 'xcodebuild', 'build', '-scheme', 'MockApp', '-sdk', 'iphonesimulator', '-workspace', self.mock_app_workspace, '-derivedDataPath', derived_data_path ], stdout=build_log_file, stderr=build_log_file) end = time.time() total_time = int(end - start) else: logging.info('Skipping xcodebuild & buck project') # Log Results build_end = str(datetime.datetime.now()) log_statement = '{} w/ {} (loc: {}) modules took {} s\n'.format(gen_info, len(node_list), swift_loc, total_time) logging.info(log_statement) self.build_time_file.write(log_statement) self.build_time_file.flush() full_xcode_version = xcode_version + " " + xcode_build_id self.build_time_csv_file.write('{}, {}, {}, {}, {}, {}, {}\n'.format( build_end, gen_type, full_xcode_version, wmo_enabled, total_time, len(node_list), swift_loc)) self.build_time_csv_file.flush() def verify_dependencies(self): if not self.run_xcodebuild: return # We don't need these binaries if we are not going to use them. xcode = ['xcodebuild', 'xcode-select'] local = [self.buck_binary] missing = check_dependent_commands(xcode + local) if missing == xcode: logging.error("Xcode command line tools do not seem to be installed / are missing in your path.") elif missing == local: logging.error("Specified buck command not available. Did you install it in your path?") if missing: logging.error("Missing required binaries: %s", str(missing)) raise OSError("Missing required binaries: {}".format(missing)) def dump_system_info(self): logging.info('Recording device info') with open(self.sys_info_path, 'w') as info_file: info_file.write(grab_mac_marketing_name()) info_file.write('\n') info_file.flush() subprocess.check_call(['system_profiler', 'SPHardwareDataType', '-detailLevel', 'mini'], stdout=info_file) subprocess.check_call(['sw_vers'], stdout=info_file) # noinspection PyAttributeOutsideInit def multisuite_setup(self): self.make_context(self.log_dir, self.output_dir, self.test_build_only) self.settings_state.save_buckconfig_local() self.settings_state.save_xcode_select() if self.switch_xcode_versions: self.sudo_warning() self.xcode_paths = self.xcode_manager.discover_xcode_versions() self.xcode_versions = self.xcode_paths.keys() logging.info("Discovered xcode verions: %s", str(self.xcode_versions)) else: self.xcode_paths = {} self.xcode_versions = [None] for path in [self.log_dir, self.build_trace_path, self.output_dir]: makedir(path) logging.info('Starting build session') self.build_time_file = open(self.build_time_path, 'a') self.build_time_csv_file = open(self.build_time_csv_path, 'a') now = str(datetime.datetime.now()) self.build_time_file.write('Build session started at {}\n'.format(now)) self.build_time_file.flush() self.dump_system_info() self.project_generator = projectgen.BuckProjectGenerator(self.mock_output_dir, self.buck_path) self.verify_dependencies() def multisuite_cleanup(self): logging.info("Cleaning up multisuite build test") self.settings_state.restore_buckconfig_local() if self.switch_xcode_versions: self.settings_state.restore_xcode_select() if self.build_time_file: self.build_time_file.close() if self.build_time_csv_file: self.build_time_csv_file.close() if self.trace_cpu: self.cpu_logger.kill() @staticmethod def sudo_warning(): if not sudo_enabled(): logging.warning('I would suggest executing this in a bash script and adding a sudo keep alive so you') logging.warning('can run this fully unattended: https://gist.github.com/cowboy/3118588') logging.warning("If you don't, then half way through the build suite, it will stall asking for a password.") logging.warning("I don't know how to make xcode-select -s not require sudo unfortunately.") def switch_xcode_version(self, xcode_version): logging.warning('Switching to xcode version %s', str(xcode_version)) self.xcode_manager.switch_xcode_version(self.xcode_paths[xcode_version]) def run_multisuite(self): self.multisuite_setup() start_time = time.time() if self.trace_cpu: self.cpu_logger.start() commandlineutil.make_custom_buckconfig_local(self.buckconfig_path) for xcode_version in self.xcode_versions: if self.switch_xcode_versions: self.switch_xcode_version(xcode_version) for wmo_enabled in self.wmo_modes: logging.info('Swift WMO Enabled: {}'.format(wmo_enabled)) for gen_type in self.type_list: self.build_app_type(gen_type, wmo_enabled) if self.trace_cpu: self.cpu_logger.stop() commandlineutil.apply_cpu_to_traces(self.build_trace_path, self.cpu_logger, start_time) def main(): CommandLineMultisuite().main() if __name__ == '__main__': main()