Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
allendowney
GitHub Repository: allendowney/cpython
Path: blob/main/Tools/wasm/wasm_build.py
12 views
1
#!/usr/bin/env python3
2
"""Build script for Python on WebAssembly platforms.
3
4
$ ./Tools/wasm/wasm_builder.py emscripten-browser build repl
5
$ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test
6
$ ./Tools/wasm/wasm_builder.py wasi build test
7
8
Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
9
"emscripten-browser", and "wasi".
10
11
Emscripten builds require a recent Emscripten SDK. The tools looks for an
12
activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
13
(Debian, Homebrew) are not supported.
14
15
WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
16
and falls back to /opt/wasi-sdk.
17
18
The 'build' Python interpreter must be rebuilt every time Python's byte code
19
changes.
20
21
./Tools/wasm/wasm_builder.py --clean build build
22
23
"""
24
import argparse
25
import enum
26
import dataclasses
27
import logging
28
import os
29
import pathlib
30
import re
31
import shlex
32
import shutil
33
import socket
34
import subprocess
35
import sys
36
import sysconfig
37
import tempfile
38
import time
39
import warnings
40
import webbrowser
41
42
# for Python 3.8
43
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
44
45
logger = logging.getLogger("wasm_build")
46
47
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
48
WASMTOOLS = SRCDIR / "Tools" / "wasm"
49
BUILDDIR = SRCDIR / "builddir"
50
CONFIGURE = SRCDIR / "configure"
51
SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
52
53
HAS_CCACHE = shutil.which("ccache") is not None
54
55
# path to WASI-SDK root
56
WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
57
58
# path to Emscripten SDK config file.
59
# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
60
EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
61
EMSDK_MIN_VERSION = (3, 1, 19)
62
EMSDK_BROKEN_VERSION = {
63
(3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
64
(3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393",
65
(3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720",
66
}
67
_MISSING = pathlib.PurePath("MISSING")
68
69
WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py"
70
71
CLEAN_SRCDIR = f"""
72
Builds require a clean source directory. Please use a clean checkout or
73
run "make clean -C '{SRCDIR}'".
74
"""
75
76
INSTALL_NATIVE = """
77
Builds require a C compiler (gcc, clang), make, pkg-config, and development
78
headers for dependencies like zlib.
79
80
Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev
81
Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel
82
"""
83
84
INSTALL_EMSDK = """
85
wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
86
https://emscripten.org/docs/getting_started/downloads.html how to install
87
Emscripten and how to activate the SDK with "emsdk_env.sh".
88
89
git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
90
cd /path/to/emsdk
91
./emsdk install latest
92
./emsdk activate latest
93
source /path/to/emsdk_env.sh
94
"""
95
96
INSTALL_WASI_SDK = """
97
wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
98
https://github.com/WebAssembly/wasi-sdk/releases and install it to
99
"/opt/wasi-sdk". Alternatively you can install the SDK in a different location
100
and point the environment variable WASI_SDK_PATH to the root directory
101
of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
102
"""
103
104
INSTALL_WASMTIME = """
105
wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
106
https://wasmtime.dev/ to install wasmtime.
107
"""
108
109
110
def parse_emconfig(
111
emconfig: pathlib.Path = EM_CONFIG,
112
) -> Tuple[pathlib.PurePath, pathlib.PurePath]:
113
"""Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS.
114
115
The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
116
environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
117
subdirectory with tools like "emconfigure".
118
"""
119
if not emconfig.exists():
120
return _MISSING, _MISSING
121
with open(emconfig, encoding="utf-8") as f:
122
code = f.read()
123
# EM_CONFIG file is a Python snippet
124
local: Dict[str, Any] = {}
125
exec(code, globals(), local)
126
emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"])
127
node_js = pathlib.Path(local["NODE_JS"])
128
return emscripten_root, node_js
129
130
131
EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig()
132
133
134
def read_python_version(configure: pathlib.Path = CONFIGURE) -> str:
135
"""Read PACKAGE_VERSION from configure script
136
137
configure and configure.ac are the canonical source for major and
138
minor version number.
139
"""
140
version_re = re.compile(r"^PACKAGE_VERSION='(\d\.\d+)'")
141
with configure.open(encoding="utf-8") as f:
142
for line in f:
143
mo = version_re.match(line)
144
if mo:
145
return mo.group(1)
146
raise ValueError(f"PACKAGE_VERSION not found in {configure}")
147
148
149
PYTHON_VERSION = read_python_version()
150
151
152
class ConditionError(ValueError):
153
def __init__(self, info: str, text: str):
154
self.info = info
155
self.text = text
156
157
def __str__(self):
158
return f"{type(self).__name__}: '{self.info}'\n{self.text}"
159
160
161
class MissingDependency(ConditionError):
162
pass
163
164
165
class DirtySourceDirectory(ConditionError):
166
pass
167
168
169
@dataclasses.dataclass
170
class Platform:
171
"""Platform-specific settings
172
173
- CONFIG_SITE override
174
- configure wrapper (e.g. emconfigure)
175
- make wrapper (e.g. emmake)
176
- additional environment variables
177
- check function to verify SDK
178
"""
179
180
name: str
181
pythonexe: str
182
config_site: Optional[pathlib.PurePath]
183
configure_wrapper: Optional[pathlib.PurePath]
184
make_wrapper: Optional[pathlib.PurePath]
185
environ: dict
186
check: Callable[[], None]
187
# Used for build_emports().
188
ports: Optional[pathlib.PurePath]
189
cc: Optional[pathlib.PurePath]
190
191
def getenv(self, profile: "BuildProfile") -> dict:
192
return self.environ.copy()
193
194
195
def _check_clean_src():
196
candidates = [
197
SRCDIR / "Programs" / "python.o",
198
SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
199
]
200
for candidate in candidates:
201
if candidate.exists():
202
raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
203
204
205
def _check_native():
206
if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]):
207
raise MissingDependency("cc", INSTALL_NATIVE)
208
if not shutil.which("make"):
209
raise MissingDependency("make", INSTALL_NATIVE)
210
if sys.platform == "linux":
211
# skip pkg-config check on macOS
212
if not shutil.which("pkg-config"):
213
raise MissingDependency("pkg-config", INSTALL_NATIVE)
214
# zlib is needed to create zip files
215
for devel in ["zlib"]:
216
try:
217
subprocess.check_call(["pkg-config", "--exists", devel])
218
except subprocess.CalledProcessError:
219
raise MissingDependency(devel, INSTALL_NATIVE) from None
220
_check_clean_src()
221
222
223
NATIVE = Platform(
224
"native",
225
# macOS has python.exe
226
pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
227
config_site=None,
228
configure_wrapper=None,
229
ports=None,
230
cc=None,
231
make_wrapper=None,
232
environ={},
233
check=_check_native,
234
)
235
236
237
def _check_emscripten():
238
if EMSCRIPTEN_ROOT is _MISSING:
239
raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
240
# sanity check
241
emconfigure = EMSCRIPTEN.configure_wrapper
242
if not emconfigure.exists():
243
raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
244
# version check
245
version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
246
if not version_txt.exists():
247
raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
248
with open(version_txt) as f:
249
version = f.read().strip().strip('"')
250
if version.endswith("-git"):
251
# git / upstream / tot-upstream installation
252
version = version[:-4]
253
version_tuple = tuple(int(v) for v in version.split("."))
254
if version_tuple < EMSDK_MIN_VERSION:
255
raise ConditionError(
256
os.fspath(version_txt),
257
f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
258
"minimum required version "
259
f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
260
)
261
broken = EMSDK_BROKEN_VERSION.get(version_tuple)
262
if broken is not None:
263
raise ConditionError(
264
os.fspath(version_txt),
265
(
266
f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known "
267
f"bugs, see {broken}."
268
),
269
)
270
if os.environ.get("PKG_CONFIG_PATH"):
271
warnings.warn(
272
"PKG_CONFIG_PATH is set and not empty. emconfigure overrides "
273
"this environment variable. Use EM_PKG_CONFIG_PATH instead."
274
)
275
_check_clean_src()
276
277
278
EMSCRIPTEN = Platform(
279
"emscripten",
280
pythonexe="python.js",
281
config_site=WASMTOOLS / "config.site-wasm32-emscripten",
282
configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
283
ports=EMSCRIPTEN_ROOT / "embuilder",
284
cc=EMSCRIPTEN_ROOT / "emcc",
285
make_wrapper=EMSCRIPTEN_ROOT / "emmake",
286
environ={
287
# workaround for https://github.com/emscripten-core/emscripten/issues/17635
288
"TZ": "UTC",
289
"EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None,
290
"PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]],
291
},
292
check=_check_emscripten,
293
)
294
295
296
def _check_wasi():
297
wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
298
if not wasm_ld.exists():
299
raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
300
wasmtime = shutil.which("wasmtime")
301
if wasmtime is None:
302
raise MissingDependency("wasmtime", INSTALL_WASMTIME)
303
_check_clean_src()
304
305
306
WASI = Platform(
307
"wasi",
308
pythonexe="python.wasm",
309
config_site=WASMTOOLS / "config.site-wasm32-wasi",
310
configure_wrapper=WASMTOOLS / "wasi-env",
311
ports=None,
312
cc=WASI_SDK_PATH / "bin" / "clang",
313
make_wrapper=None,
314
environ={
315
"WASI_SDK_PATH": WASI_SDK_PATH,
316
# workaround for https://github.com/python/cpython/issues/95952
317
"HOSTRUNNER": (
318
"wasmtime run "
319
"--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib "
320
"--mapdir /::{srcdir} --"
321
),
322
"PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]],
323
},
324
check=_check_wasi,
325
)
326
327
328
class Host(enum.Enum):
329
"""Target host triplet"""
330
331
wasm32_emscripten = "wasm32-unknown-emscripten"
332
wasm64_emscripten = "wasm64-unknown-emscripten"
333
wasm32_wasi = "wasm32-unknown-wasi"
334
wasm64_wasi = "wasm64-unknown-wasi"
335
# current platform
336
build = sysconfig.get_config_var("BUILD_GNU_TYPE")
337
338
@property
339
def platform(self) -> Platform:
340
if self.is_emscripten:
341
return EMSCRIPTEN
342
elif self.is_wasi:
343
return WASI
344
else:
345
return NATIVE
346
347
@property
348
def is_emscripten(self) -> bool:
349
cls = type(self)
350
return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
351
352
@property
353
def is_wasi(self) -> bool:
354
cls = type(self)
355
return self in {cls.wasm32_wasi, cls.wasm64_wasi}
356
357
def get_extra_paths(self) -> Iterable[pathlib.PurePath]:
358
"""Host-specific os.environ["PATH"] entries.
359
360
Emscripten's Node version 14.x works well for wasm32-emscripten.
361
wasm64-emscripten requires more recent v8 version, e.g. node 16.x.
362
Attempt to use system's node command.
363
"""
364
cls = type(self)
365
if self == cls.wasm32_emscripten:
366
return [NODE_JS.parent]
367
elif self == cls.wasm64_emscripten:
368
# TODO: look for recent node
369
return []
370
else:
371
return []
372
373
@property
374
def emport_args(self) -> List[str]:
375
"""Host-specific port args (Emscripten)."""
376
cls = type(self)
377
if self is cls.wasm64_emscripten:
378
return ["-sMEMORY64=1"]
379
elif self is cls.wasm32_emscripten:
380
return ["-sMEMORY64=0"]
381
else:
382
return []
383
384
@property
385
def embuilder_args(self) -> List[str]:
386
"""Host-specific embuilder args (Emscripten)."""
387
cls = type(self)
388
if self is cls.wasm64_emscripten:
389
return ["--wasm64"]
390
else:
391
return []
392
393
394
class EmscriptenTarget(enum.Enum):
395
"""Emscripten-specific targets (--with-emscripten-target)"""
396
397
browser = "browser"
398
browser_debug = "browser-debug"
399
node = "node"
400
node_debug = "node-debug"
401
402
@property
403
def is_browser(self):
404
cls = type(self)
405
return self in {cls.browser, cls.browser_debug}
406
407
@property
408
def emport_args(self) -> List[str]:
409
"""Target-specific port args."""
410
cls = type(self)
411
if self in {cls.browser_debug, cls.node_debug}:
412
# some libs come in debug and non-debug builds
413
return ["-O0"]
414
else:
415
return ["-O2"]
416
417
418
class SupportLevel(enum.Enum):
419
supported = "tier 3, supported"
420
working = "working, unsupported"
421
experimental = "experimental, may be broken"
422
broken = "broken / unavailable"
423
424
def __bool__(self):
425
cls = type(self)
426
return self in {cls.supported, cls.working}
427
428
429
@dataclasses.dataclass
430
class BuildProfile:
431
name: str
432
support_level: SupportLevel
433
host: Host
434
target: Union[EmscriptenTarget, None] = None
435
dynamic_linking: Union[bool, None] = None
436
pthreads: Union[bool, None] = None
437
default_testopts: str = "-j2"
438
439
@property
440
def is_browser(self) -> bool:
441
"""Is this a browser build?"""
442
return self.target is not None and self.target.is_browser
443
444
@property
445
def builddir(self) -> pathlib.Path:
446
"""Path to build directory"""
447
return BUILDDIR / self.name
448
449
@property
450
def python_cmd(self) -> pathlib.Path:
451
"""Path to python executable"""
452
return self.builddir / self.host.platform.pythonexe
453
454
@property
455
def makefile(self) -> pathlib.Path:
456
"""Path to Makefile"""
457
return self.builddir / "Makefile"
458
459
@property
460
def configure_cmd(self) -> List[str]:
461
"""Generate configure command"""
462
# use relative path, so WASI tests can find lib prefix.
463
# pathlib.Path.relative_to() does not work here.
464
configure = os.path.relpath(CONFIGURE, self.builddir)
465
cmd = [configure, "-C"]
466
platform = self.host.platform
467
if platform.configure_wrapper:
468
cmd.insert(0, os.fspath(platform.configure_wrapper))
469
470
cmd.append(f"--host={self.host.value}")
471
cmd.append(f"--build={Host.build.value}")
472
473
if self.target is not None:
474
assert self.host.is_emscripten
475
cmd.append(f"--with-emscripten-target={self.target.value}")
476
477
if self.dynamic_linking is not None:
478
assert self.host.is_emscripten
479
opt = "enable" if self.dynamic_linking else "disable"
480
cmd.append(f"--{opt}-wasm-dynamic-linking")
481
482
if self.pthreads is not None:
483
opt = "enable" if self.pthreads else "disable"
484
cmd.append(f"--{opt}-wasm-pthreads")
485
486
if self.host != Host.build:
487
cmd.append(f"--with-build-python={BUILD.python_cmd}")
488
489
if platform.config_site is not None:
490
cmd.append(f"CONFIG_SITE={platform.config_site}")
491
492
return cmd
493
494
@property
495
def make_cmd(self) -> List[str]:
496
"""Generate make command"""
497
cmd = ["make"]
498
platform = self.host.platform
499
if platform.make_wrapper:
500
cmd.insert(0, os.fspath(platform.make_wrapper))
501
return cmd
502
503
def getenv(self) -> dict:
504
"""Generate environ dict for platform"""
505
env = os.environ.copy()
506
env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
507
platenv = self.host.platform.getenv(self)
508
for key, value in platenv.items():
509
if value is None:
510
env.pop(key, None)
511
elif key == "PATH":
512
# list of path items, prefix with extra paths
513
new_path: List[pathlib.PurePath] = []
514
new_path.extend(self.host.get_extra_paths())
515
new_path.extend(value)
516
env[key] = os.pathsep.join(os.fspath(p) for p in new_path)
517
elif isinstance(value, str):
518
env[key] = value.format(
519
relbuilddir=self.builddir.relative_to(SRCDIR),
520
srcdir=SRCDIR,
521
version=PYTHON_VERSION,
522
)
523
else:
524
env[key] = value
525
return env
526
527
def _run_cmd(
528
self,
529
cmd: Iterable[str],
530
args: Iterable[str] = (),
531
cwd: Optional[pathlib.Path] = None,
532
):
533
cmd = list(cmd)
534
cmd.extend(args)
535
if cwd is None:
536
cwd = self.builddir
537
logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd)
538
return subprocess.check_call(
539
cmd,
540
cwd=os.fspath(cwd),
541
env=self.getenv(),
542
)
543
544
def _check_execute(self):
545
if self.is_browser:
546
raise ValueError(f"Cannot execute on {self.target}")
547
548
def run_build(self, *args):
549
"""Run configure (if necessary) and make"""
550
if not self.makefile.exists():
551
logger.info("Makefile not found, running configure")
552
self.run_configure(*args)
553
self.run_make("all", *args)
554
555
def run_configure(self, *args):
556
"""Run configure script to generate Makefile"""
557
os.makedirs(self.builddir, exist_ok=True)
558
return self._run_cmd(self.configure_cmd, args)
559
560
def run_make(self, *args):
561
"""Run make (defaults to build all)"""
562
return self._run_cmd(self.make_cmd, args)
563
564
def run_pythoninfo(self, *args):
565
"""Run 'make pythoninfo'"""
566
self._check_execute()
567
return self.run_make("pythoninfo", *args)
568
569
def run_test(self, target: str, testopts: Optional[str] = None):
570
"""Run buildbottests"""
571
self._check_execute()
572
if testopts is None:
573
testopts = self.default_testopts
574
return self.run_make(target, f"TESTOPTS={testopts}")
575
576
def run_py(self, *args):
577
"""Run Python with hostrunner"""
578
self._check_execute()
579
self.run_make(
580
"--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
581
)
582
583
def run_browser(self, bind="127.0.0.1", port=8000):
584
"""Run WASM webserver and open build in browser"""
585
relbuilddir = self.builddir.relative_to(SRCDIR)
586
url = f"http://{bind}:{port}/{relbuilddir}/python.html"
587
args = [
588
sys.executable,
589
os.fspath(WASM_WEBSERVER),
590
"--bind",
591
bind,
592
"--port",
593
str(port),
594
]
595
srv = subprocess.Popen(args, cwd=SRCDIR)
596
# wait for server
597
end = time.monotonic() + 3.0
598
while time.monotonic() < end and srv.returncode is None:
599
try:
600
with socket.create_connection((bind, port), timeout=0.1) as _:
601
pass
602
except OSError:
603
time.sleep(0.01)
604
else:
605
break
606
607
webbrowser.open(url)
608
609
try:
610
srv.wait()
611
except KeyboardInterrupt:
612
pass
613
614
def clean(self, all: bool = False):
615
"""Clean build directory"""
616
if all:
617
if self.builddir.exists():
618
shutil.rmtree(self.builddir)
619
elif self.makefile.exists():
620
self.run_make("clean")
621
622
def build_emports(self, force: bool = False):
623
"""Pre-build emscripten ports."""
624
platform = self.host.platform
625
if platform.ports is None or platform.cc is None:
626
raise ValueError("Need ports and CC command")
627
628
embuilder_cmd = [os.fspath(platform.ports)]
629
embuilder_cmd.extend(self.host.embuilder_args)
630
if force:
631
embuilder_cmd.append("--force")
632
633
ports_cmd = [os.fspath(platform.cc)]
634
ports_cmd.extend(self.host.emport_args)
635
if self.target:
636
ports_cmd.extend(self.target.emport_args)
637
638
if self.dynamic_linking:
639
# Trigger PIC build.
640
ports_cmd.append("-sMAIN_MODULE")
641
embuilder_cmd.append("--pic")
642
643
if self.pthreads:
644
# Trigger multi-threaded build.
645
ports_cmd.append("-sUSE_PTHREADS")
646
647
# Pre-build libbz2, libsqlite3, libz, and some system libs.
648
ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
649
# Multi-threaded sqlite3 has different suffix
650
embuilder_cmd.extend(
651
["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"]
652
)
653
654
self._run_cmd(embuilder_cmd, cwd=SRCDIR)
655
656
with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir:
657
tmppath = pathlib.Path(tmpdir)
658
main_c = tmppath / "main.c"
659
main_js = tmppath / "main.js"
660
with main_c.open("w") as f:
661
f.write("int main(void) { return 0; }\n")
662
args = [
663
os.fspath(main_c),
664
"-o",
665
os.fspath(main_js),
666
]
667
self._run_cmd(ports_cmd, args, cwd=tmppath)
668
669
670
# native build (build Python)
671
BUILD = BuildProfile(
672
"build",
673
support_level=SupportLevel.working,
674
host=Host.build,
675
)
676
677
_profiles = [
678
BUILD,
679
# wasm32-emscripten
680
BuildProfile(
681
"emscripten-browser",
682
support_level=SupportLevel.supported,
683
host=Host.wasm32_emscripten,
684
target=EmscriptenTarget.browser,
685
dynamic_linking=True,
686
),
687
BuildProfile(
688
"emscripten-browser-debug",
689
support_level=SupportLevel.working,
690
host=Host.wasm32_emscripten,
691
target=EmscriptenTarget.browser_debug,
692
dynamic_linking=True,
693
),
694
BuildProfile(
695
"emscripten-node-dl",
696
support_level=SupportLevel.supported,
697
host=Host.wasm32_emscripten,
698
target=EmscriptenTarget.node,
699
dynamic_linking=True,
700
),
701
BuildProfile(
702
"emscripten-node-dl-debug",
703
support_level=SupportLevel.working,
704
host=Host.wasm32_emscripten,
705
target=EmscriptenTarget.node_debug,
706
dynamic_linking=True,
707
),
708
BuildProfile(
709
"emscripten-node-pthreads",
710
support_level=SupportLevel.supported,
711
host=Host.wasm32_emscripten,
712
target=EmscriptenTarget.node,
713
pthreads=True,
714
),
715
BuildProfile(
716
"emscripten-node-pthreads-debug",
717
support_level=SupportLevel.working,
718
host=Host.wasm32_emscripten,
719
target=EmscriptenTarget.node_debug,
720
pthreads=True,
721
),
722
# Emscripten build with both pthreads and dynamic linking is crashing.
723
BuildProfile(
724
"emscripten-node-dl-pthreads-debug",
725
support_level=SupportLevel.broken,
726
host=Host.wasm32_emscripten,
727
target=EmscriptenTarget.node_debug,
728
dynamic_linking=True,
729
pthreads=True,
730
),
731
# wasm64-emscripten (requires Emscripten >= 3.1.21)
732
BuildProfile(
733
"wasm64-emscripten-node-debug",
734
support_level=SupportLevel.experimental,
735
host=Host.wasm64_emscripten,
736
target=EmscriptenTarget.node_debug,
737
# MEMORY64 is not compatible with dynamic linking
738
dynamic_linking=False,
739
pthreads=False,
740
),
741
# wasm32-wasi
742
BuildProfile(
743
"wasi",
744
support_level=SupportLevel.supported,
745
host=Host.wasm32_wasi,
746
),
747
# wasm32-wasi-threads
748
BuildProfile(
749
"wasi-threads",
750
support_level=SupportLevel.experimental,
751
host=Host.wasm32_wasi,
752
pthreads=True,
753
),
754
# no SDK available yet
755
# BuildProfile(
756
# "wasm64-wasi",
757
# support_level=SupportLevel.broken,
758
# host=Host.wasm64_wasi,
759
# ),
760
]
761
762
PROFILES = {p.name: p for p in _profiles}
763
764
parser = argparse.ArgumentParser(
765
"wasm_build.py",
766
description=__doc__,
767
formatter_class=argparse.RawTextHelpFormatter,
768
)
769
770
parser.add_argument(
771
"--clean",
772
"-c",
773
help="Clean build directories first",
774
action="store_true",
775
)
776
777
parser.add_argument(
778
"--verbose",
779
"-v",
780
help="Verbose logging",
781
action="store_true",
782
)
783
784
parser.add_argument(
785
"--silent",
786
help="Run configure and make in silent mode",
787
action="store_true",
788
)
789
790
parser.add_argument(
791
"--testopts",
792
help=(
793
"Additional test options for 'test' and 'hostrunnertest', e.g. "
794
"--testopts='-v test_os'."
795
),
796
default=None,
797
)
798
799
# Don't list broken and experimental variants in help
800
platforms_choices = list(p.name for p in _profiles) + ["cleanall"]
801
platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"]
802
parser.add_argument(
803
"platform",
804
metavar="PLATFORM",
805
help=f"Build platform: {', '.join(platforms_help)}",
806
choices=platforms_choices,
807
)
808
809
ops = dict(
810
build="auto build (build 'build' Python, emports, configure, compile)",
811
configure="run ./configure",
812
compile="run 'make all'",
813
pythoninfo="run 'make pythoninfo'",
814
test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)",
815
hostrunnertest="run 'make hostrunnertest TESTOPTS=...'",
816
repl="start interactive REPL / webserver + browser session",
817
clean="run 'make clean'",
818
cleanall="remove all build directories",
819
emports="build Emscripten port with embuilder (only Emscripten)",
820
)
821
ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items())
822
parser.add_argument(
823
"ops",
824
metavar="OP",
825
help=f"operation (default: build)\n\n{ops_help}",
826
choices=tuple(ops),
827
default="build",
828
nargs="*",
829
)
830
831
832
def main():
833
args = parser.parse_args()
834
logging.basicConfig(
835
level=logging.INFO if args.verbose else logging.ERROR,
836
format="%(message)s",
837
)
838
839
if args.platform == "cleanall":
840
for builder in PROFILES.values():
841
builder.clean(all=True)
842
parser.exit(0)
843
844
# additional configure and make args
845
cm_args = ("--silent",) if args.silent else ()
846
847
# nargs=* with default quirk
848
if args.ops == "build":
849
args.ops = ["build"]
850
851
builder = PROFILES[args.platform]
852
try:
853
builder.host.platform.check()
854
except ConditionError as e:
855
parser.error(str(e))
856
857
if args.clean:
858
builder.clean(all=False)
859
860
# hack for WASI
861
if builder.host.is_wasi and not SETUP_LOCAL.exists():
862
SETUP_LOCAL.touch()
863
864
# auto-build
865
if "build" in args.ops:
866
# check and create build Python
867
if builder is not BUILD:
868
logger.info("Auto-building 'build' Python.")
869
try:
870
BUILD.host.platform.check()
871
except ConditionError as e:
872
parser.error(str(e))
873
if args.clean:
874
BUILD.clean(all=False)
875
BUILD.run_build(*cm_args)
876
# build Emscripten ports with embuilder
877
if builder.host.is_emscripten and "emports" not in args.ops:
878
builder.build_emports()
879
880
for op in args.ops:
881
logger.info("\n*** %s %s", args.platform, op)
882
if op == "build":
883
builder.run_build(*cm_args)
884
elif op == "configure":
885
builder.run_configure(*cm_args)
886
elif op == "compile":
887
builder.run_make("all", *cm_args)
888
elif op == "pythoninfo":
889
builder.run_pythoninfo(*cm_args)
890
elif op == "repl":
891
if builder.is_browser:
892
builder.run_browser()
893
else:
894
builder.run_py()
895
elif op == "test":
896
builder.run_test("buildbottest", testopts=args.testopts)
897
elif op == "hostrunnertest":
898
builder.run_test("hostrunnertest", testopts=args.testopts)
899
elif op == "clean":
900
builder.clean(all=False)
901
elif op == "cleanall":
902
builder.clean(all=True)
903
elif op == "emports":
904
builder.build_emports(force=args.clean)
905
else:
906
raise ValueError(op)
907
908
print(builder.builddir)
909
parser.exit(0)
910
911
912
if __name__ == "__main__":
913
main()
914
915