from .toolchain_profiler import ToolchainProfiler
from subprocess import PIPE
import atexit
import logging
import os
import re
import shutil
import shlex
import subprocess
import signal
import stat
import sys
import tempfile
if sys.version_info < (3, 8):
print(f'error: emscripten requires python 3.8 or above ({sys.executable} {sys.version})', file=sys.stderr)
sys.exit(1)
from . import colored_logger
DEBUG = int(os.environ.get('EMCC_DEBUG', '0'))
EMCC_LOGGING = int(os.environ.get('EMCC_LOGGING', '1'))
log_level = logging.ERROR
if DEBUG:
log_level = logging.DEBUG
elif EMCC_LOGGING:
log_level = logging.INFO
logging.basicConfig(format='%(name)s:%(levelname)s: %(message)s', level=log_level)
colored_logger.enable()
from .utils import path_from_root, exit_with_error, safe_ensure_dirs, WINDOWS, set_version_globals, memoize
from . import cache, tempfiles
from . import diagnostics
from . import config
from . import filelock
from . import utils
from .settings import settings
import contextlib
DEBUG_SAVE = DEBUG or int(os.environ.get('EMCC_DEBUG_SAVE', '0'))
PRINT_SUBPROCS = int(os.getenv('EMCC_VERBOSE', '0'))
SKIP_SUBPROCS = False
MINIMUM_NODE_VERSION = (18, 3, 0)
EXPECTED_LLVM_VERSION = 22
TEMP_DIR = None
EMSCRIPTEN_TEMP_DIR = None
logger = logging.getLogger('shared')
diagnostics.add_warning('absolute-paths', enabled=False, part_of_all=False)
diagnostics.add_warning('almost-asm')
diagnostics.add_warning('experimental')
diagnostics.add_warning('legacy-settings', enabled=False, part_of_all=False)
diagnostics.add_warning('linkflags')
diagnostics.add_warning('emcc')
diagnostics.add_warning('undefined', error=True)
diagnostics.add_warning('deprecated', shared=True)
diagnostics.add_warning('version-check')
diagnostics.add_warning('export-main')
diagnostics.add_warning('map-unrecognized-libraries')
diagnostics.add_warning('unused-command-line-argument', shared=True)
diagnostics.add_warning('pthreads-mem-growth')
diagnostics.add_warning('transpile')
diagnostics.add_warning('limited-postlink-optimizations')
diagnostics.add_warning('em-js-i64')
diagnostics.add_warning('js-compiler')
diagnostics.add_warning('compatibility')
diagnostics.add_warning('unsupported')
diagnostics.add_warning('unused-main')
diagnostics.add_warning('closure', enabled=False)
def run_process(cmd, check=True, input=None, *args, **kw):
"""Runs a subprocess returning the exit code.
By default this function will raise an exception on failure. Therefore this should only be
used if you want to handle such failures. For most subprocesses, failures are not recoverable
and should be fatal. In those cases the `check_call` wrapper should be preferred.
"""
sys.stdout.flush()
sys.stderr.flush()
kw.setdefault('text', True)
kw.setdefault('encoding', 'utf-8')
ret = subprocess.run(cmd, check=check, input=input, *args, **kw)
debug_text = '%sexecuted %s' % ('successfully ' if check else '', shlex.join(cmd))
logger.debug(debug_text)
return ret
def get_num_cores():
return int(os.environ.get('EMCC_CORES', os.cpu_count()))
def returncode_to_str(code):
assert code != 0
if code < 0:
signal_name = signal.Signals(-code).name
return f'received {signal_name} ({code})'
return f'returned {code}'
def cap_max_workers_in_pool(max_workers):
if WINDOWS:
return min(max_workers, 61)
return max_workers
def run_multiple_processes(commands,
env=None,
route_stdout_to_temp_files_suffix=None,
cwd=None):
"""Runs multiple subprocess commands.
route_stdout_to_temp_files_suffix : string
if not None, all stdouts are instead written to files, and an array
of filenames is returned.
"""
if env is None:
env = os.environ.copy()
std_outs = []
processes = {}
def get_finished_process():
while True:
for idx, proc in processes.items():
if proc.poll() is not None:
return idx
idx, proc = next(iter(processes.items()))
try:
proc.communicate(timeout=0.2)
return idx
except subprocess.TimeoutExpired:
pass
num_parallel_processes = get_num_cores()
temp_files = get_temp_files()
i = 0
num_completed = 0
while num_completed < len(commands):
if i < len(commands) and len(processes) < num_parallel_processes:
if route_stdout_to_temp_files_suffix:
stdout = temp_files.get(route_stdout_to_temp_files_suffix)
else:
stdout = None
if DEBUG:
logger.debug('Running subprocess %d/%d: %s' % (i + 1, len(commands), ' '.join(commands[i])))
print_compiler_stage(commands[i])
proc = subprocess.Popen(commands[i], stdout=stdout, stderr=None, env=env, cwd=cwd)
processes[i] = proc
if route_stdout_to_temp_files_suffix:
std_outs.append((i, stdout.name))
i += 1
else:
idx = get_finished_process()
finished_process = processes.pop(idx)
if finished_process.returncode != 0:
exit_with_error('subprocess %d/%d failed (%s)! (cmdline: %s)' % (idx + 1, len(commands), returncode_to_str(finished_process.returncode), shlex.join(commands[idx])))
num_completed += 1
if route_stdout_to_temp_files_suffix:
std_outs.sort(key=lambda x: x[0])
return [x[1] for x in std_outs]
def check_call(cmd, *args, **kw):
"""Like `run_process` above but treat failures as fatal and exit_with_error."""
print_compiler_stage(cmd)
if SKIP_SUBPROCS:
return 0
try:
return run_process(cmd, *args, **kw)
except subprocess.CalledProcessError as e:
exit_with_error("'%s' failed (%s)", shlex.join(cmd), returncode_to_str(e.returncode))
except OSError as e:
exit_with_error("'%s' failed: %s", shlex.join(cmd), str(e))
def exec_process(cmd):
print_compiler_stage(cmd)
if utils.WINDOWS:
rtn = run_process(cmd, stdin=sys.stdin, check=False).returncode
sys.exit(rtn)
else:
sys.stdout.flush()
sys.stderr.flush()
os.execvp(cmd[0], cmd)
def run_js_tool(filename, jsargs=[], node_args=[], **kw):
"""Execute a javascript tool.
This is used by emcc to run parts of the build process that are written
implemented in javascript.
"""
command = config.NODE_JS + node_args + [filename] + jsargs
return check_call(command, **kw).stdout
def get_npm_cmd(name, missing_ok=False):
if WINDOWS:
cmd = [path_from_root('node_modules/.bin', name + '.cmd')]
else:
cmd = config.NODE_JS + [path_from_root('node_modules/.bin', name)]
if not os.path.exists(cmd[-1]):
if missing_ok:
return None
else:
exit_with_error(f'{name} was not found! Please run "npm install" in Emscripten root directory to set up npm dependencies')
return cmd
@memoize
def get_clang_version():
if not os.path.exists(CLANG_CC):
exit_with_error('clang executable not found at `%s`' % CLANG_CC)
proc = check_call([CLANG_CC, '--version'], stdout=PIPE)
m = re.search(r'[Vv]ersion\s+(\d+\.\d+)', proc.stdout)
return m and m.group(1)
def check_llvm_version():
actual = get_clang_version()
if actual.startswith('%d.' % EXPECTED_LLVM_VERSION):
return True
if 'BUILDBOT_BUILDNUMBER' in os.environ:
if actual.startswith('%d.' % (EXPECTED_LLVM_VERSION + 1)):
return True
diagnostics.warning('version-check', 'LLVM version for clang executable "%s" appears incorrect (seeing "%s", expected "%s")', CLANG_CC, actual, EXPECTED_LLVM_VERSION)
return False
def get_clang_targets():
if not os.path.exists(CLANG_CC):
exit_with_error('clang executable not found at `%s`' % CLANG_CC)
try:
target_info = run_process([CLANG_CC, '-print-targets'], stdout=PIPE).stdout
except subprocess.CalledProcessError:
exit_with_error('error running `clang -print-targets`. Check your llvm installation (%s)' % CLANG_CC)
if 'Registered Targets:' not in target_info:
exit_with_error('error parsing output of `clang -print-targets`. Check your llvm installation (%s)' % CLANG_CC)
return target_info.split('Registered Targets:')[1]
def check_llvm():
targets = get_clang_targets()
if 'wasm32' not in targets:
logger.critical('LLVM has not been built with the WebAssembly backend, clang reports:')
print('===========================================================================', file=sys.stderr)
print(targets, file=sys.stderr)
print('===========================================================================', file=sys.stderr)
return False
return True
def get_node_directory():
return os.path.dirname(config.NODE_JS[0] if type(config.NODE_JS) is list else config.NODE_JS)
def env_with_node_in_path():
env = os.environ.copy()
env['PATH'] = get_node_directory() + os.pathsep + env['PATH']
return env
def _get_node_version_pair(nodejs):
actual = run_process(nodejs + ['--version'], stdout=PIPE).stdout.strip()
version = actual.replace('v', '')
version = version.split('-')[0].split('.')
version = tuple(int(v) for v in version)
return actual, version
def get_node_version(nodejs):
return _get_node_version_pair(nodejs)[1]
@memoize
def check_node_version():
try:
actual, version = _get_node_version_pair(config.NODE_JS)
except Exception as e:
diagnostics.warning('version-check', 'cannot check node version: %s', e)
return
if version < MINIMUM_NODE_VERSION:
expected = '.'.join(str(v) for v in MINIMUM_NODE_VERSION)
diagnostics.warning('version-check', f'node version appears too old (seeing "{actual}", expected "v{expected}")')
return version
def node_reference_types_flags(nodejs):
node_version = get_node_version(nodejs)
if node_version and node_version < (18, 0, 0):
return ['--experimental-wasm-reftypes']
else:
return []
def node_exception_flags(nodejs):
node_version = get_node_version(nodejs)
if node_version and node_version < (17, 0, 0):
return ['--experimental-wasm-eh']
if node_version and node_version >= (22, 0, 0) and not settings.WASM_LEGACY_EXCEPTIONS:
return ['--experimental-wasm-exnref']
return []
def node_pthread_flags(nodejs):
node_version = get_node_version(nodejs)
if node_version and node_version < (16, 0, 0):
return ['--experimental-wasm-bulk-memory', '--experimental-wasm-threads']
else:
return []
@memoize
@ToolchainProfiler.profile()
def check_node():
try:
run_process(config.NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE)
except Exception as e:
exit_with_error('the configured node executable (%s) does not seem to work, check the paths in %s (%s)', config.NODE_JS, config.EM_CONFIG, str(e))
def generate_sanity():
return f'{utils.EMSCRIPTEN_VERSION}|{config.LLVM_ROOT}\n'
@memoize
def perform_sanity_checks(quiet=False):
check_node_version()
check_llvm_version()
llvm_ok = check_llvm()
if os.environ.get('EM_IGNORE_SANITY'):
logger.info('EM_IGNORE_SANITY set, ignoring sanity checks')
return
if not quiet:
logger.info('(Emscripten: Running sanity checks)')
if not llvm_ok:
exit_with_error('failing sanity checks due to previous llvm failure')
check_node()
with ToolchainProfiler.profile_block('sanity LLVM'):
for cmd in (CLANG_CC, LLVM_AR):
if not os.path.exists(cmd) and not os.path.exists(cmd + '.exe'):
exit_with_error('cannot find %s, check the paths in %s', cmd, config.EM_CONFIG)
@ToolchainProfiler.profile()
def check_sanity(force=False, quiet=False):
"""Check that basic stuff we need (a JS engine to compile, Node.js, and Clang
and LLVM) exists.
The test runner always does this check (through |force|). emcc does this less
frequently, only when ${EM_CONFIG}_sanity does not exist or is older than
EM_CONFIG (so, we re-check sanity when the settings are changed). We also
re-check sanity and clear the cache when the version changes.
"""
if not force and os.environ.get('EMCC_SKIP_SANITY_CHECK') == '1':
return
os.environ['EMCC_SKIP_SANITY_CHECK'] = '1'
if DEBUG:
force = True
if config.FROZEN_CACHE:
if force:
perform_sanity_checks(quiet)
return
if os.environ.get('EM_IGNORE_SANITY'):
perform_sanity_checks(quiet)
return
expected = generate_sanity()
sanity_file = cache.get_path('sanity.txt')
def sanity_is_correct():
sanity_data = None
with contextlib.suppress(Exception):
sanity_data = utils.read_file(sanity_file)
if sanity_data == expected:
logger.debug(f'sanity file up-to-date: {sanity_file}')
if force:
perform_sanity_checks(quiet)
return True
return False
if sanity_is_correct():
return
with cache.lock('sanity'):
if sanity_is_correct():
return
if os.path.exists(sanity_file):
sanity_data = utils.read_file(sanity_file)
logger.info('old sanity: %s', sanity_data.strip())
logger.info('new sanity: %s', expected.strip())
logger.info('(Emscripten: config changed, clearing cache)')
cache.erase()
else:
logger.debug(f'sanity file not found: {sanity_file}')
perform_sanity_checks()
utils.write_file(sanity_file, expected)
def llvm_tool_path_with_suffix(tool, suffix):
if suffix:
tool += '-' + suffix
llvm_root = os.path.expanduser(config.LLVM_ROOT)
return os.path.join(llvm_root, exe_suffix(tool))
def llvm_tool_path(tool):
return llvm_tool_path_with_suffix(tool, config.LLVM_ADD_VERSION)
def clang_tool_path(tool):
return llvm_tool_path_with_suffix(tool, config.CLANG_ADD_VERSION)
def exe_suffix(cmd):
return cmd + '.exe' if WINDOWS else cmd
def bat_suffix(cmd):
return cmd + '.bat' if WINDOWS else cmd
def replace_suffix(filename, new_suffix):
assert new_suffix[0] == '.'
return os.path.splitext(filename)[0] + new_suffix
def replace_or_append_suffix(filename, new_suffix):
assert new_suffix[0] == '.'
return replace_suffix(filename, new_suffix) if settings.MINIMAL_RUNTIME else filename + new_suffix
@memoize
def get_emscripten_temp_dir():
"""Returns a path to EMSCRIPTEN_TEMP_DIR, creating one if it didn't exist."""
global EMSCRIPTEN_TEMP_DIR
if not EMSCRIPTEN_TEMP_DIR:
EMSCRIPTEN_TEMP_DIR = tempfile.mkdtemp(prefix='emscripten_temp_', dir=TEMP_DIR)
if not DEBUG_SAVE:
def prepare_to_clean_temp(d):
def clean_temp():
utils.delete_dir(d)
atexit.register(clean_temp)
prepare_to_clean_temp(EMSCRIPTEN_TEMP_DIR)
return EMSCRIPTEN_TEMP_DIR
def in_temp(name):
return os.path.join(get_emscripten_temp_dir(), os.path.basename(name))
def get_canonical_temp_dir(temp_dir):
return os.path.join(temp_dir, 'emscripten_temp')
def setup_temp_dirs():
global EMSCRIPTEN_TEMP_DIR, CANONICAL_TEMP_DIR, TEMP_DIR
EMSCRIPTEN_TEMP_DIR = None
TEMP_DIR = os.environ.get("EMCC_TEMP_DIR", tempfile.gettempdir())
if not os.path.isdir(TEMP_DIR):
exit_with_error(f'The temporary directory `{TEMP_DIR}` does not exist! Please make sure that the path is correct.')
CANONICAL_TEMP_DIR = get_canonical_temp_dir(TEMP_DIR)
if DEBUG:
EMSCRIPTEN_TEMP_DIR = CANONICAL_TEMP_DIR
try:
safe_ensure_dirs(EMSCRIPTEN_TEMP_DIR)
except Exception as e:
exit_with_error(str(e) + f'Could not create canonical temp dir. Check definition of TEMP_DIR in {config.EM_CONFIG}')
if 'EM_HAVE_TEMP_DIR_LOCK' not in os.environ:
filelock_name = os.path.join(EMSCRIPTEN_TEMP_DIR, 'emscripten.lock')
lock = filelock.FileLock(filelock_name)
os.environ['EM_HAVE_TEMP_DIR_LOCK'] = '1'
lock.acquire()
atexit.register(lock.release)
@memoize
def get_temp_files():
if DEBUG_SAVE:
return tempfiles.TempFiles(get_emscripten_temp_dir(), save_debug_files=True)
else:
return tempfiles.TempFiles(TEMP_DIR, save_debug_files=False)
def print_compiler_stage(cmd):
"""Emulate the '-v/-###' flags of clang/gcc by printing the sub-commands
that we run."""
def maybe_quote(arg):
if all(c.isalnum() or c in './-_' for c in arg):
return arg
else:
return f'"{arg}"'
if SKIP_SUBPROCS:
print(' ' + ' '.join([maybe_quote(a) for a in cmd]), file=sys.stderr)
sys.stderr.flush()
elif PRINT_SUBPROCS:
print(' %s %s' % (maybe_quote(cmd[0]), shlex.join(cmd[1:])), file=sys.stderr)
sys.stderr.flush()
def demangle_c_symbol_name(name):
if not is_c_symbol(name):
return '$' + name
return name[1:] if name.startswith('_') else name
def is_c_symbol(name):
return name.startswith('_')
def is_internal_global(name):
internal_start_stop_symbols = {'__start_em_asm', '__stop_em_asm',
'__start_em_js', '__stop_em_js',
'__start_em_lib_deps', '__stop_em_lib_deps',
'__em_lib_deps'}
internal_prefixes = ('__em_js__', '__em_lib_deps')
return name in internal_start_stop_symbols or any(name.startswith(p) for p in internal_prefixes)
def is_user_export(name):
if is_internal_global(name):
return False
return name not in ['__indirect_function_table', 'memory'] and not name.startswith(('dynCall_', 'orig$'))
def asmjs_mangle(name):
"""Mangle a name the way asm.js/JSBackend globals are mangled.
Prepends '_' and replaces non-alphanumerics with '_'.
Used by wasm backend for JS library consistency with asm.js.
"""
if name == '__main_argc_argv':
name = 'main'
if is_user_export(name):
return '_' + name
return name
def suffix(name):
"""Return the file extension"""
return os.path.splitext(name)[1]
def unsuffixed(name):
"""Return the filename without the extension.
If there are multiple extensions this strips only the final one.
"""
return os.path.splitext(name)[0]
def unsuffixed_basename(name):
return os.path.basename(unsuffixed(name))
def get_file_suffix(filename):
"""Parses the essential suffix of a filename, discarding Unix-style version
numbers in the name. For example for 'libz.so.1.2.8' returns '.so'"""
while filename:
filename, suffix = os.path.splitext(filename)
if not suffix[1:].isdigit():
return suffix
return ''
def make_writable(filename):
assert os.path.exists(filename)
old_mode = stat.S_IMODE(os.stat(filename).st_mode)
os.chmod(filename, old_mode | stat.S_IWUSR)
def safe_copy(src, dst):
logging.debug('copy: %s -> %s', src, dst)
src = os.path.abspath(src)
dst = os.path.abspath(dst)
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(src))
if src == dst:
return
if dst == os.devnull:
return
shutil.copy(src, dst)
make_writable(dst)
def do_replace(input_, pattern, replacement):
if pattern not in input_:
exit_with_error('expected to find pattern in input JS: %s' % pattern)
return input_.replace(pattern, replacement)
def get_llvm_target():
if settings.MEMORY64:
return 'wasm64-unknown-emscripten'
else:
return 'wasm32-unknown-emscripten'
def init():
set_version_globals()
setup_temp_dirs()
CLANG_CC = clang_tool_path('clang')
CLANG_CXX = clang_tool_path('clang++')
CLANG_SCAN_DEPS = llvm_tool_path('clang-scan-deps')
LLVM_AR = llvm_tool_path('llvm-ar')
LLVM_DWP = llvm_tool_path('llvm-dwp')
LLVM_RANLIB = llvm_tool_path('llvm-ranlib')
LLVM_NM = llvm_tool_path('llvm-nm')
LLVM_DWARFDUMP = llvm_tool_path('llvm-dwarfdump')
LLVM_OBJCOPY = llvm_tool_path('llvm-objcopy')
LLVM_STRIP = llvm_tool_path('llvm-strip')
WASM_LD = llvm_tool_path('wasm-ld')
LLVM_PROFDATA = llvm_tool_path('llvm-profdata')
LLVM_COV = llvm_tool_path('llvm-cov')
EMCC = bat_suffix(path_from_root('emcc'))
EMXX = bat_suffix(path_from_root('em++'))
EMAR = bat_suffix(path_from_root('emar'))
EMRANLIB = bat_suffix(path_from_root('emranlib'))
EM_NM = bat_suffix(path_from_root('emnm'))
FILE_PACKAGER = bat_suffix(path_from_root('tools/file_packager'))
WASM_SOURCEMAP = bat_suffix(path_from_root('tools/wasm-sourcemap'))
DYLIB_EXTENSIONS = ['.dylib', '.so']
run_via_emxx = False
init()