# # All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or # its licensors. # # For complete copyright and license terms please see the LICENSE at the root of this # distribution (the "License"). All use of this software is governed by the License, # or, if provided, by the license below or the license accompanying this file. Do not # remove or modify any license notices. This file is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # # System Imports import multiprocessing import os import string import subprocess import sys # Attempt to import the winregistry module. try: import winreg WINREG_SUPPORTED = True except ImportError: WINREG_SUPPORTED = False pass # waflib imports from waflib import Logs, Utils, Options, Errors from waflib.Configure import conf # lmbrwaflib imports from lmbrwaflib import utils from lmbrwaflib.cry_utils import WAF_EXECUTABLE from lmbrwaflib.lumberyard import multi_conf, CURRENT_WAF_EXECUTABLE IB_REGISTRY_PATH = "Software\\Wow6432Node\\Xoreax\\Incredibuild\\Builder" ######################################################################################################## # Validate the Incredibuild Registry settings def internal_validate_incredibuild_registry_settings(ctx): """ Helper function to verify the correct incredibuild settings """ if Utils.unversioned_sys_platform() != 'win32': # Check windows registry only return False import winreg if not ctx.is_option_true('use_incredibuild'): # No need to check IB settings if there is no IB return False allow_reg_updated = ctx.is_option_true('auto_update_incredibuild_settings') and \ not ctx.is_option_true('internal_dont_check_recursive_execution') and \ not Options.options.execsolution # Open the incredibuild settings registry key to validate if IB is installed properly try: ib_settings_read_only = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, IB_REGISTRY_PATH, 0, winreg.KEY_READ) except: Logs.debug('lumberyard: Cannot open registry entry "HKEY_LOCAL_MACHINE\\{}"'.format(IB_REGISTRY_PATH)) Logs.warn('[WARNING] Incredibuild does not appear to be correctly installed on your machine. Disabling Incredibuild.') return False def _read_ib_reg_setting(reg_key, setting_name, setting_path, expected_value): try: reg_data, reg_type = winreg.QueryValueEx(reg_key, setting_name) return reg_data == expected_value except: Logs.debug('lumberyard: Cannot find a registry entry for "HKEY_LOCAL_MACHINE\\{}\\{}"'.format(setting_path,setting_name)) return False def _write_ib_reg_setting(reg_key, setting_name, setting_path, value): try: winreg.SetValueEx(reg_key, setting_name, 0, winreg.REG_SZ, str(value)) return True except WindowsError as e: Logs.warn('lumberyard: Unable write to HKEY_LOCAL_MACHINE\\{}\\{} : {}'.format(setting_path,setting_name,e.strerror)) return False valid_ib_reg_key_values = [('MaxConcurrentPDBs', '0')] is_ib_ready = True for settings_name, expected_value in valid_ib_reg_key_values: if is_ib_ready and not _read_ib_reg_setting(ib_settings_read_only, settings_name, IB_REGISTRY_PATH, expected_value): is_ib_ready = False # If we are IB ready, short-circuit out if is_ib_ready: return True # If we need updates, check if we have 'auto auto-update-incredibuild-settings' set or not if not allow_reg_updated: Logs.warn('[WARNING] The required settings for incredibuild is not properly configured. ') if not ctx.is_option_true('auto_update_incredibuild_settings'): Logs.warn("[WARNING]: Set the '--auto-update-incredibuild-settings' to True if you want to attempt to automatically update the settings") return False # if auto-update-incredibuild-settings is true, then attempt to update the values automatically try: ib_settings_writing = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, IB_REGISTRY_PATH, 0, winreg.KEY_SET_VALUE | winreg.KEY_READ) except: Logs.warn('[WARNING] Cannot access a registry entry "HKEY_LOCAL_MACHINE\\{}" for writing.'.format(IB_REGISTRY_PATH)) Logs.warn('[WARNING] Please run "{0}" as an administrator or change the value to "0" in the registry to ensure a correct operation of WAF'.format(WAF_EXECUTABLE) ) return False # Once we get the key, attempt to update the values is_ib_updated = True for settings_name, set_value in valid_ib_reg_key_values: if is_ib_updated and not _write_ib_reg_setting(ib_settings_writing, settings_name, IB_REGISTRY_PATH, set_value): is_ib_updated = False if not is_ib_updated: Logs.warn('[WARNING] Unable to update registry settings for incredibuild') return False Logs.info('[INFO] Registry values updated for incredibuild') return True INTERNAL_INCREDIBUILD_LICENSE_CHECK = None ######################################################################################################## def internal_use_incredibuild(ctx, section_name, option_name, value, verification_fn): """ If Incredibuild should be used, check for required packages """ # GUI if not ctx.is_option_true('console_mode'): return ctx.gui_get_attribute(section_name, option_name, value) if not value or value != 'True': return value if not Utils.unversioned_sys_platform() == 'win32': return value _incredibuild_disclaimer(ctx) if Logs.verbose > 1: ctx.start_msg('Incredibuild Licence Check') (res, warning, error) = verification_fn(ctx, option_name, value) if not res: if warning: Logs.warn(warning) if error: if Logs.verbose > 1: ctx.end_msg(error, color='YELLOW') return 'False' if Logs.verbose > 1: ctx.end_msg('ok') return value ######################################################################################################## def _incredibuild_disclaimer(ctx): """ Helper function to show a disclaimer over incredibuild before asking for settings """ if getattr(ctx, 'incredibuild_disclaimer_shown', False): return Logs.debug('incredibuild:\nWAF is using Incredibuild for distributed Builds') Logs.debug('incredibuild:To be able to compile with WAF, various licenses are required:') Logs.debug('incredibuild:The "IncrediBuild for Make && Build Tools Package" is always needed') Logs.debug('incredibuild:If some packages are missing, please ask IT') Logs.debug('incredibuild:to assign the needed ones to your machine') ctx.incredibuild_disclaimer_shown = True ######################################################################################################## def internal_verify_incredibuild_license(licence_name, platform_name): """ Helper function to check if user has a incredibuild licence """ try: result = subprocess.check_output(['xgconsole.exe', '/QUERYLICENSE']) result = str(result).strip() except: error = '[ERROR] Incredibuild not found on system' return False, "", error if not licence_name in result: error = '[ERROR] Incredibuild on "%s" Disabled - Missing IB licence: "%s"' % (platform_name, licence_name) return False, "", error return True,"", "" def clean_arg_for_subprocess_call(arg): # incredibuild will invoke with a command line like this: # ['xgconsole.exe', '/command="program" "arg1" "arg2" ...', 'other incredibuild params go here'] # it is important that each arg is encapsulated in quotes when it has spaces in it # the input in this case is a single arg from our own command line, which we assume has no quotes but may contain spaces # the most infamous example of this might be something like this: # arg[0] = '"d:\\ly engine\\tools\\python\\python.cmd" "d:\\ly engine\\tools\\build\waf.py"' # arg[1] = '--bootstrap-tool-param=--3rdpartypath="d:/ly engine/3rdParty" --none' # we want to transform this into returning: # '"d:\\ly engine\\tools\\python\\python.cmd" "d:\\ly engine\\tools\\build\waf.py"' (unchanged) # '"--bootstrap-tool-param=--3rdpartypath=""d:/ly engine/3rdParty"" --none"' (wrapped with quotes) # notice we have to double escape existing quotes... # if it starts and ends with a quote, we do nothing to it as we assume its already escaped. if not arg: return '' if arg[:1] == arg[-1:] == '"': return arg arg = arg.replace('"', '""') if (' ' in arg): return '"' + arg + '"' return arg @conf def invoke_waf_recursively(bld, build_metrics_supported=False, metrics_namespace=None): """ Check the incredibuild parameters and environment to see if we need to invoke waf through incredibuild :param bld: The BuildContext :return: True to short circuit the current build flow (because an incredibuild command has been invoked), False to continue the flow """ if not WINREG_SUPPORTED: return False # We can't run incredibuild on systems that don't support windows registry if not Utils.unversioned_sys_platform() == 'win32': return False # Don't use recursive execution on non-windows hosts if bld.is_option_true('internal_dont_check_recursive_execution'): return False # Skip clean_ commands if bld.cmd.startswith('clean_'): return False # Skip non-build commands if getattr(bld, 'is_project_generator', False): return False # Don't use IB for special single file operations if bld.is_option_true('show_includes'): return False if bld.is_option_true('show_preprocessed_file'): return False if bld.is_option_true('show_disassembly'): return False if bld.options.file_filter != "": return False # Skip if incredibuild is disabled if not bld.is_option_true('use_incredibuild'): Logs.warn('[WARNING] Incredibuild disabled by build option') return False try: # Get correct incredibuild installation folder to not depend on PATH IB_settings = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, IB_REGISTRY_PATH, 0, winreg.KEY_READ) (ib_folder, type) = winreg.QueryValueEx(IB_settings, 'Folder') except: Logs.warn('[WARNING] Incredibuild disabled. Cannot find incredibuild installation.') return False # Get the incredibuild profile file ib_profile_xml_check = getattr(bld.options, 'incredibuild_profile', '').replace("Code/Tools/waf-1.7.13/profile.xml", "Tools/build/waf-1.7.13/profile.xml") if os.path.exists(ib_profile_xml_check): ib_profile_xml = ib_profile_xml_check else: # If the profile doesnt exist, then attempt to use the engine as the base ib_profile_xml = os.path.join(bld.get_bintemp_folder_node().abspath(), 'profile.xml') if not os.path.isfile(ib_profile_xml): Logs.warn('[WARN] Unable to locate default incredibuild profile ({}). Run configure to regenerate.') Logs.warn('[WARN] Incredibuild will be disabled.') return False result = subprocess.check_output([str(ib_folder) + '/xgconsole.exe', '/QUERYLICENSE']) result = str(result).strip() # Make & Build Tools is required if not 'Make && Build Tools' in result: Logs.warn('Required Make && Build Tools Package not found. Build will not be accelerated through Incredibuild') return False # Determine if the Dev Tool Acceleration package is available. This package is required for consoles dev_tool_accelerated = 'IncrediBuild for Dev Tool Acceleration' in result ib_flag_check_results = bld.check_ib_flag(bld.cmd, result, dev_tool_accelerated) if not all(ib_flag_check_results): return False # Windows builds can be run without the Dev Tools Acceleration Package, but won't distribute Qt tasks. if not dev_tool_accelerated: Logs.warn('Dev Tool Acceleration Package not found. Qt tasks will not be distributed through Incredibuild') # Get all specific commands, but keep msvs to execute after IB has finished bExecuteMSVS = False if 'msvs' in Options.commands: bExecuteMSVS = True Options.commands = [] cmd_line_args = [] for arg in sys.argv[1:]: if arg == 'generate_uber_files': continue if arg == 'msvs': bExecuteMSVS = True continue if arg == 'configure': Logs.warn( '[WARNING] Incredibuild disabled, running configure and build in one line is not supported with incredibuild. To build with incredibuild, run the build without the configure command on the same line') return False arg = clean_arg_for_subprocess_call(arg) cmd_line_args += [arg] if bExecuteMSVS: # Execute MSVS without IB Options.commands += ['msvs'] command_line_options = ' '.join(cmd_line_args) # Recreate command line # Add special option to not start IB from within IB command_line_options += ' --internal-dont-check-recursive-execution=True' num_jobs = bld.options.incredibuild_max_cores # Build Command Line Logs.debug("incredibuild: Current WAF: {} ".format(CURRENT_WAF_EXECUTABLE)) command = clean_arg_for_subprocess_call(CURRENT_WAF_EXECUTABLE) + ' --jobs=' + str(num_jobs) + ' ' + command_line_options if build_metrics_supported: command += ' --enable-build-metrics' if metrics_namespace is not None: command += ' --metrics-namespace {0}'.format(metrics_namespace) sys.stdout.write('[WAF] Starting Incredibuild: ') process_call = [] if dev_tool_accelerated: process_call.append(str(ib_folder) + '/xgconsole.exe') # If the IB profile is not blank, then attempt to use it if len(ib_profile_xml) > 0: ib_profile_xml_file = os.path.abspath(ib_profile_xml) # Set the profile for incredibuild only if it exists if os.path.exists(ib_profile_xml_file): process_call.append('/profile={}'.format(ib_profile_xml_file)) else: Logs.warn('[WARN] Incredibuild profile file "{}" does not exist. Using default incredibuild settings'.format(ib_profile_xml)) else: process_call.append(str(ib_folder) + '/buildconsole.exe') # using a profile overrides the handling of max link tasks. Unfortunately, the make&build tool doesn't support # the profile, so we must check the registry settings to ensure that they allow parallel linking up to the # count specified in waf. Incredibuild suggests adding an override parameter to the msbuild command to override # this, but since we aren't using this, we warn instead try: # grab the limitor for number of local jobs that incredibuild will use (ib_max_local_cpu,type) = winreg.QueryValueEx(IB_settings, 'ForceCPUCount_WhenInitiator') # grab the limitor that incredibuild will use if a profile is not specified (ib_max_link,type) = winreg.QueryValueEx(IB_settings, 'MaxParallelLinkTargets') except: Logs.warn('[WARNING] unable to query Incredibuild registry, parallel linking may be sub-optimal') else: ib_max_local_cpu = int(ib_max_local_cpu) if (ib_max_local_cpu == 0): ib_max_local_cpu = multiprocessing.cpu_count() # executable links are limited to max_parallel_link using a semaphore. lib/dll links are limited to number # of cores since they are generally single threaded min_setting_needed = min(ib_max_local_cpu, int(bld.options.max_parallel_link)) ib_max_link = int(ib_max_link) if ib_max_link < min_setting_needed: Logs.warn('[WARNING] Incredibuild configuration \'MaxParallelLinkTargets\' limits link tasks to %d, increasing to %d will improve link throughput' % (ib_max_link, min_setting_needed)) process_call.append('/command=' + command) process_call.append('/useidemonitor') process_call.append('/nologo') Logs.debug('incredibuild: process_call: ' + str(process_call)) if subprocess.call(process_call, env=os.environ.copy()): bld.fatal("[ERROR] Build Failed") return True @conf def generate_ib_profile_xml(ctx): """ Make sure that we have a generated profile.xml for the incredibuild commands :param ctx: Context """ tab2_fmt = '\t{}' tab4_fmt = '\t' * 2 + '{}' profile_lines = ['', '', '', tab2_fmt.format('')] tools_list_list = ctx.generate_ib_profile_tool_elements() for tool_element_list in tools_list_list: for tool_element in tool_element_list: profile_lines.append(tab4_fmt.format(tool_element)) profile_lines.append(tab2_fmt.format('')) profile_lines.append('') profile_content = '\n'.join(profile_lines) + '\n' profile_xml_target = os.path.join(ctx.get_bintemp_folder_node().abspath(), 'profile.xml') if not os.path.isfile(profile_xml_target): write_profile = True else: current_content_hash = utils.calculate_string_hash(profile_content) existing_file_hash = utils.calculate_file_hash(profile_xml_target) write_profile = current_content_hash != existing_file_hash if write_profile: try: Logs.debug('lumberyard: Updating incredibuild profile file: {}'.format(profile_xml_target)) with open(profile_xml_target, "w") as profile_file: profile_file.write(profile_content) except Exception as e: Logs.warn("[WARN] Unable to write to incredibuild profile file '{}':{}".format(profile_xml_target, e)) @multi_conf def generate_ib_profile_tool_elements(ctx): """ Generate a list of Tool XML Elements to inject into the incredibuild profile.xml configuration for when we invoke through IB :param ctx: Context :return: List of tool elements """ bullseye_tool_elements = [ '', '' ] return bullseye_tool_elements