#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# System Imports
import multiprocessing
import os
import string
import subprocess
import sys
# Attempt to import the winregistry module.
try:
import winreg
WINREG_SUPPORTED = True
except ImportError:
WINREG_SUPPORTED = False
pass
# waflib imports
from waflib import Logs, Utils, Options, Errors
from waflib.Configure import conf
# lmbrwaflib imports
from lmbrwaflib import utils
from lmbrwaflib.cry_utils import WAF_EXECUTABLE
from lmbrwaflib.lumberyard import multi_conf, CURRENT_WAF_EXECUTABLE
IB_REGISTRY_PATH = "Software\\Wow6432Node\\Xoreax\\Incredibuild\\Builder"
########################################################################################################
# Validate the Incredibuild Registry settings
def internal_validate_incredibuild_registry_settings(ctx):
""" Helper function to verify the correct incredibuild settings """
if Utils.unversioned_sys_platform() != 'win32':
# Check windows registry only
return False
import winreg
if not ctx.is_option_true('use_incredibuild'):
# No need to check IB settings if there is no IB
return False
allow_reg_updated = ctx.is_option_true('auto_update_incredibuild_settings') and \
not ctx.is_option_true('internal_dont_check_recursive_execution') and \
not Options.options.execsolution
# Open the incredibuild settings registry key to validate if IB is installed properly
try:
ib_settings_read_only = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, IB_REGISTRY_PATH, 0, winreg.KEY_READ)
except:
Logs.debug('lumberyard: Cannot open registry entry "HKEY_LOCAL_MACHINE\\{}"'.format(IB_REGISTRY_PATH))
Logs.warn('[WARNING] Incredibuild does not appear to be correctly installed on your machine. Disabling Incredibuild.')
return False
def _read_ib_reg_setting(reg_key, setting_name, setting_path, expected_value):
try:
reg_data, reg_type = winreg.QueryValueEx(reg_key, setting_name)
return reg_data == expected_value
except:
Logs.debug('lumberyard: Cannot find a registry entry for "HKEY_LOCAL_MACHINE\\{}\\{}"'.format(setting_path,setting_name))
return False
def _write_ib_reg_setting(reg_key, setting_name, setting_path, value):
try:
winreg.SetValueEx(reg_key, setting_name, 0, winreg.REG_SZ, str(value))
return True
except WindowsError as e:
Logs.warn('lumberyard: Unable write to HKEY_LOCAL_MACHINE\\{}\\{} : {}'.format(setting_path,setting_name,e.strerror))
return False
valid_ib_reg_key_values = [('MaxConcurrentPDBs', '0')]
is_ib_ready = True
for settings_name, expected_value in valid_ib_reg_key_values:
if is_ib_ready and not _read_ib_reg_setting(ib_settings_read_only, settings_name, IB_REGISTRY_PATH, expected_value):
is_ib_ready = False
# If we are IB ready, short-circuit out
if is_ib_ready:
return True
# If we need updates, check if we have 'auto auto-update-incredibuild-settings' set or not
if not allow_reg_updated:
Logs.warn('[WARNING] The required settings for incredibuild is not properly configured. ')
if not ctx.is_option_true('auto_update_incredibuild_settings'):
Logs.warn("[WARNING]: Set the '--auto-update-incredibuild-settings' to True if you want to attempt to automatically update the settings")
return False
# if auto-update-incredibuild-settings is true, then attempt to update the values automatically
try:
ib_settings_writing = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, IB_REGISTRY_PATH, 0, winreg.KEY_SET_VALUE | winreg.KEY_READ)
except:
Logs.warn('[WARNING] Cannot access a registry entry "HKEY_LOCAL_MACHINE\\{}" for writing.'.format(IB_REGISTRY_PATH))
Logs.warn('[WARNING] Please run "{0}" as an administrator or change the value to "0" in the registry to ensure a correct operation of WAF'.format(WAF_EXECUTABLE) )
return False
# Once we get the key, attempt to update the values
is_ib_updated = True
for settings_name, set_value in valid_ib_reg_key_values:
if is_ib_updated and not _write_ib_reg_setting(ib_settings_writing, settings_name, IB_REGISTRY_PATH, set_value):
is_ib_updated = False
if not is_ib_updated:
Logs.warn('[WARNING] Unable to update registry settings for incredibuild')
return False
Logs.info('[INFO] Registry values updated for incredibuild')
return True
INTERNAL_INCREDIBUILD_LICENSE_CHECK = None
########################################################################################################
def internal_use_incredibuild(ctx, section_name, option_name, value, verification_fn):
""" If Incredibuild should be used, check for required packages """
# GUI
if not ctx.is_option_true('console_mode'):
return ctx.gui_get_attribute(section_name, option_name, value)
if not value or value != 'True':
return value
if not Utils.unversioned_sys_platform() == 'win32':
return value
_incredibuild_disclaimer(ctx)
if Logs.verbose > 1:
ctx.start_msg('Incredibuild Licence Check')
(res, warning, error) = verification_fn(ctx, option_name, value)
if not res:
if warning:
Logs.warn(warning)
if error:
if Logs.verbose > 1:
ctx.end_msg(error, color='YELLOW')
return 'False'
if Logs.verbose > 1:
ctx.end_msg('ok')
return value
########################################################################################################
def _incredibuild_disclaimer(ctx):
""" Helper function to show a disclaimer over incredibuild before asking for settings """
if getattr(ctx, 'incredibuild_disclaimer_shown', False):
return
Logs.debug('incredibuild:\nWAF is using Incredibuild for distributed Builds')
Logs.debug('incredibuild:To be able to compile with WAF, various licenses are required:')
Logs.debug('incredibuild:The "IncrediBuild for Make && Build Tools Package" is always needed')
Logs.debug('incredibuild:If some packages are missing, please ask IT')
Logs.debug('incredibuild:to assign the needed ones to your machine')
ctx.incredibuild_disclaimer_shown = True
########################################################################################################
def internal_verify_incredibuild_license(licence_name, platform_name):
""" Helper function to check if user has a incredibuild licence """
try:
result = subprocess.check_output(['xgconsole.exe', '/QUERYLICENSE'])
result = str(result).strip()
except:
error = '[ERROR] Incredibuild not found on system'
return False, "", error
if not licence_name in result:
error = '[ERROR] Incredibuild on "%s" Disabled - Missing IB licence: "%s"' % (platform_name, licence_name)
return False, "", error
return True,"", ""
def clean_arg_for_subprocess_call(arg):
# incredibuild will invoke with a command line like this:
# ['xgconsole.exe', '/command="program" "arg1" "arg2" ...', 'other incredibuild params go here']
# it is important that each arg is encapsulated in quotes when it has spaces in it
# the input in this case is a single arg from our own command line, which we assume has no quotes but may contain spaces
# the most infamous example of this might be something like this:
# arg[0] = '"d:\\ly engine\\tools\\python\\python.cmd" "d:\\ly engine\\tools\\build\waf.py"'
# arg[1] = '--bootstrap-tool-param=--3rdpartypath="d:/ly engine/3rdParty" --none'
# we want to transform this into returning:
# '"d:\\ly engine\\tools\\python\\python.cmd" "d:\\ly engine\\tools\\build\waf.py"' (unchanged)
# '"--bootstrap-tool-param=--3rdpartypath=""d:/ly engine/3rdParty"" --none"' (wrapped with quotes)
# notice we have to double escape existing quotes...
# if it starts and ends with a quote, we do nothing to it as we assume its already escaped.
if not arg:
return ''
if arg[:1] == arg[-1:] == '"':
return arg
arg = arg.replace('"', '""')
if (' ' in arg):
return '"' + arg + '"'
return arg
@conf
def invoke_waf_recursively(bld, build_metrics_supported=False, metrics_namespace=None):
"""
Check the incredibuild parameters and environment to see if we need to invoke waf through incredibuild
:param bld: The BuildContext
:return: True to short circuit the current build flow (because an incredibuild command has been invoked), False to continue the flow
"""
if not WINREG_SUPPORTED:
return False # We can't run incredibuild on systems that don't support windows registry
if not Utils.unversioned_sys_platform() == 'win32':
return False # Don't use recursive execution on non-windows hosts
if bld.is_option_true('internal_dont_check_recursive_execution'):
return False
# Skip clean_ commands
if bld.cmd.startswith('clean_'):
return False
# Skip non-build commands
if getattr(bld, 'is_project_generator', False):
return False
# Don't use IB for special single file operations
if bld.is_option_true('show_includes'):
return False
if bld.is_option_true('show_preprocessed_file'):
return False
if bld.is_option_true('show_disassembly'):
return False
if bld.options.file_filter != "":
return False
# Skip if incredibuild is disabled
if not bld.is_option_true('use_incredibuild'):
Logs.warn('[WARNING] Incredibuild disabled by build option')
return False
try:
# Get correct incredibuild installation folder to not depend on PATH
IB_settings = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, IB_REGISTRY_PATH, 0, winreg.KEY_READ)
(ib_folder, type) = winreg.QueryValueEx(IB_settings, 'Folder')
except:
Logs.warn('[WARNING] Incredibuild disabled. Cannot find incredibuild installation.')
return False
# Get the incredibuild profile file
ib_profile_xml_check = getattr(bld.options, 'incredibuild_profile', '').replace("Code/Tools/waf-1.7.13/profile.xml", "Tools/build/waf-1.7.13/profile.xml")
if os.path.exists(ib_profile_xml_check):
ib_profile_xml = ib_profile_xml_check
else:
# If the profile doesnt exist, then attempt to use the engine as the base
ib_profile_xml = os.path.join(bld.get_bintemp_folder_node().abspath(), 'profile.xml')
if not os.path.isfile(ib_profile_xml):
Logs.warn('[WARN] Unable to locate default incredibuild profile ({}). Run configure to regenerate.')
Logs.warn('[WARN] Incredibuild will be disabled.')
return False
result = subprocess.check_output([str(ib_folder) + '/xgconsole.exe', '/QUERYLICENSE'])
result = str(result).strip()
# Make & Build Tools is required
if not 'Make && Build Tools' in result:
Logs.warn('Required Make && Build Tools Package not found. Build will not be accelerated through Incredibuild')
return False
# Determine if the Dev Tool Acceleration package is available. This package is required for consoles
dev_tool_accelerated = 'IncrediBuild for Dev Tool Acceleration' in result
ib_flag_check_results = bld.check_ib_flag(bld.cmd, result, dev_tool_accelerated)
if not all(ib_flag_check_results):
return False
# Windows builds can be run without the Dev Tools Acceleration Package, but won't distribute Qt tasks.
if not dev_tool_accelerated:
Logs.warn('Dev Tool Acceleration Package not found. Qt tasks will not be distributed through Incredibuild')
# Get all specific commands, but keep msvs to execute after IB has finished
bExecuteMSVS = False
if 'msvs' in Options.commands:
bExecuteMSVS = True
Options.commands = []
cmd_line_args = []
for arg in sys.argv[1:]:
if arg == 'generate_uber_files':
continue
if arg == 'msvs':
bExecuteMSVS = True
continue
if arg == 'configure':
Logs.warn(
'[WARNING] Incredibuild disabled, running configure and build in one line is not supported with incredibuild. To build with incredibuild, run the build without the configure command on the same line')
return False
arg = clean_arg_for_subprocess_call(arg)
cmd_line_args += [arg]
if bExecuteMSVS: # Execute MSVS without IB
Options.commands += ['msvs']
command_line_options = ' '.join(cmd_line_args) # Recreate command line
# Add special option to not start IB from within IB
command_line_options += ' --internal-dont-check-recursive-execution=True'
num_jobs = bld.options.incredibuild_max_cores
# Build Command Line
Logs.debug("incredibuild: Current WAF: {} ".format(CURRENT_WAF_EXECUTABLE))
command = clean_arg_for_subprocess_call(CURRENT_WAF_EXECUTABLE) + ' --jobs=' + str(num_jobs) + ' ' + command_line_options
if build_metrics_supported:
command += ' --enable-build-metrics'
if metrics_namespace is not None:
command += ' --metrics-namespace {0}'.format(metrics_namespace)
sys.stdout.write('[WAF] Starting Incredibuild: ')
process_call = []
if dev_tool_accelerated:
process_call.append(str(ib_folder) + '/xgconsole.exe')
# If the IB profile is not blank, then attempt to use it
if len(ib_profile_xml) > 0:
ib_profile_xml_file = os.path.abspath(ib_profile_xml)
# Set the profile for incredibuild only if it exists
if os.path.exists(ib_profile_xml_file):
process_call.append('/profile={}'.format(ib_profile_xml_file))
else:
Logs.warn('[WARN] Incredibuild profile file "{}" does not exist. Using default incredibuild settings'.format(ib_profile_xml))
else:
process_call.append(str(ib_folder) + '/buildconsole.exe')
# using a profile overrides the handling of max link tasks. Unfortunately, the make&build tool doesn't support
# the profile, so we must check the registry settings to ensure that they allow parallel linking up to the
# count specified in waf. Incredibuild suggests adding an override parameter to the msbuild command to override
# this, but since we aren't using this, we warn instead
try:
# grab the limitor for number of local jobs that incredibuild will use
(ib_max_local_cpu,type) = winreg.QueryValueEx(IB_settings, 'ForceCPUCount_WhenInitiator')
# grab the limitor that incredibuild will use if a profile is not specified
(ib_max_link,type) = winreg.QueryValueEx(IB_settings, 'MaxParallelLinkTargets')
except:
Logs.warn('[WARNING] unable to query Incredibuild registry, parallel linking may be sub-optimal')
else:
ib_max_local_cpu = int(ib_max_local_cpu)
if (ib_max_local_cpu == 0):
ib_max_local_cpu = multiprocessing.cpu_count()
# executable links are limited to max_parallel_link using a semaphore. lib/dll links are limited to number
# of cores since they are generally single threaded
min_setting_needed = min(ib_max_local_cpu, int(bld.options.max_parallel_link))
ib_max_link = int(ib_max_link)
if ib_max_link < min_setting_needed:
Logs.warn('[WARNING] Incredibuild configuration \'MaxParallelLinkTargets\' limits link tasks to %d, increasing to %d will improve link throughput' % (ib_max_link, min_setting_needed))
process_call.append('/command=' + command)
process_call.append('/useidemonitor')
process_call.append('/nologo')
Logs.debug('incredibuild: process_call: ' + str(process_call))
if subprocess.call(process_call, env=os.environ.copy()):
bld.fatal("[ERROR] Build Failed")
return True
@conf
def generate_ib_profile_xml(ctx):
"""
Make sure that we have a generated profile.xml for the incredibuild commands
:param ctx: Context
"""
tab2_fmt = '\t{}'
tab4_fmt = '\t' * 2 + '{}'
profile_lines = ['',
'',
'',
tab2_fmt.format('')]
tools_list_list = ctx.generate_ib_profile_tool_elements()
for tool_element_list in tools_list_list:
for tool_element in tool_element_list:
profile_lines.append(tab4_fmt.format(tool_element))
profile_lines.append(tab2_fmt.format(''))
profile_lines.append('')
profile_content = '\n'.join(profile_lines) + '\n'
profile_xml_target = os.path.join(ctx.get_bintemp_folder_node().abspath(), 'profile.xml')
if not os.path.isfile(profile_xml_target):
write_profile = True
else:
current_content_hash = utils.calculate_string_hash(profile_content)
existing_file_hash = utils.calculate_file_hash(profile_xml_target)
write_profile = current_content_hash != existing_file_hash
if write_profile:
try:
Logs.debug('lumberyard: Updating incredibuild profile file: {}'.format(profile_xml_target))
with open(profile_xml_target, "w") as profile_file:
profile_file.write(profile_content)
except Exception as e:
Logs.warn("[WARN] Unable to write to incredibuild profile file '{}':{}".format(profile_xml_target, e))
@multi_conf
def generate_ib_profile_tool_elements(ctx):
"""
Generate a list of Tool XML Elements to inject into the incredibuild profile.xml configuration for when we invoke through IB
:param ctx: Context
:return: List of tool elements
"""
bullseye_tool_elements = [
'',
''
]
return bullseye_tool_elements