Skip to content

Commit bbb2ab9

Browse files
hoodmanehoodmane
committed
gh-127111: Make web example work again
I moved the web example from `Tools/wasm` into `Tools/wasm/emscripten/web_example`. I also added a new target `build_emscripten` which is `build_wasm` but also builds the web_example. The web_example needs: 1. python.html, copied 2. python.worker.mjs copied 3. python.mjs and python.wasm output from the main linking of the Python interpreter 4. The webserver that sets COOP and COEP 5. python3.14.zip This last is created by the `wasm_assets.py` script, which required a pretty small set of changes to work fine for us. The last thing that should be done is the `python.worker.mjs` script should be made independent of the Python version: currently 3.14 is hard coded. I ran into trouble doing this, so maybe I can leave it to a followup.
1 parent ff2278e commit bbb2ab9

7 files changed

Lines changed: 83 additions & 56 deletions

File tree

‎Makefile.pre.in‎

Lines changed: 32 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@
269269
# Other subdirectories
270270
SUBDIRSTOO= Include Lib Misc
271271

272-
# assets for Emscripten browser builds
273-
WASM_ASSETS_DIR=.$(prefix)
274-
WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py
275-
276272
# Files and directories to be distributed
277273
CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in
278274
DISTFILES= README.rst ChangeLog $(CONFIGFILES)
@@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh
737733
build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
738734
python-config checksharedmods
739735

736+
.PHONY: build_emscripten
737+
build_emscripten: build_wasm web_example
738+
740739
# Check that the source is clean when building out of source.
741740
.PHONY: check-clean-src
742741
check-clean-src:
@@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
10161015
else true; \
10171016
fi
10181017

1019-
# wasm32-emscripten browser build
1020-
# wasm assets directory is relative to current build dir, e.g. "./usr/local".
1021-
# --preload-file turns a relative asset path into an absolute path.
1018+
# wasm32-emscripten browser web example
1019+
1020+
WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/
1021+
web_example/python.html: $(WEBEX_DIR)/python.html
1022+
@mkdir -p web_example
1023+
@cp $< $@
1024+
1025+
web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs
1026+
@mkdir -p web_example
1027+
@cp $< $@
10221028

1023-
.PHONY: wasm_stdlib
1024-
wasm_stdlib: $(WASM_STDLIB)
1025-
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1026-
$(srcdir)/Tools/wasm/wasm_assets.py \
1029+
web_example/server.py: $(WEBEX_DIR)/server.py
1030+
@mkdir -p web_example
1031+
@cp $< $@
1032+
1033+
WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip
1034+
$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1035+
$(WEBEX_DIR)/wasm_assets.py \
10271036
Makefile pybuilddir.txt Modules/Setup.local
1028-
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
1029-
--buildroot . --prefix $(prefix)
1037+
$(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \
1038+
--buildroot . --prefix $(prefix) -o $@
10301039

1031-
python.html: $(srcdir)/Tools/wasm/python.html python.worker.js
1032-
@cp $(srcdir)/Tools/wasm/python.html $@
1040+
web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON)
1041+
@if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \
1042+
echo "Can only build web_example when target is Emscripten" ;\
1043+
exit 1 ;\
1044+
fi
1045+
cp python.mjs web_example/python.mjs
1046+
cp python.wasm web_example/python.wasm
10331047

1034-
python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
1035-
@cp $(srcdir)/Tools/wasm/python.worker.js $@
1048+
.PHONY: web_example
1049+
web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB)
10361050

10371051
############################################################################
10381052
# Header files
@@ -3053,8 +3067,7 @@ clean-retain-profile: pycremoval
30533067
find build -name '*.py[co]' -exec rm -f {} ';' || true
30543068
-rm -f pybuilddir.txt
30553069
-rm -f _bootstrap_python
3056-
-rm -f python.html python*.js python.data python*.symbols python*.map
3057-
-rm -f $(WASM_STDLIB)
3070+
-rm -rf web_example python.mjs python.wasm python*.symbols python*.map
30583071
-rm -f Programs/_testembed Programs/_freeze_module
30593072
-rm -rf Python/deepfreeze
30603073
-rm -f Python/frozen_modules/*.h

Tools/wasm/python.html renamed to Tools/wasm/emscripten/web_example/python.html

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747

4848
async initialiseWorker() {
4949
if (!this.worker) {
50-
this.worker = new Worker(this.workerURL)
50+
this.worker = new Worker(this.workerURL, {type: "module"})
5151
this.worker.addEventListener('message', this.handleMessageFromWorker)
5252
}
5353
}
@@ -347,7 +347,7 @@
347347
programRunning(false)
348348
}
349349

350-
const pythonWorkerManager = new WorkerManager('./python.worker.js', stdio, readyCallback, finishedCallback)
350+
const pythonWorkerManager = new WorkerManager('./python.worker.mjs', stdio, readyCallback, finishedCallback)
351351
}
352352
</script>
353353
</head>

Tools/wasm/python.worker.js renamed to Tools/wasm/emscripten/web_example/python.worker.mjs

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import createEmscriptenModule from "./python.mjs";
2+
13
class StdinBuffer {
24
constructor() {
35
this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT)
@@ -59,29 +61,42 @@ const stderr = (charCode) => {
5961

6062
const stdinBuffer = new StdinBuffer()
6163

62-
var Module = {
64+
const emscriptenSettings = {
6365
noInitialRun: true,
6466
stdin: stdinBuffer.stdin,
6567
stdout: stdout,
6668
stderr: stderr,
6769
onRuntimeInitialized: () => {
6870
postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab})
71+
},
72+
async preRun(Module) {
73+
// TODO: remove fixed version number
74+
// Prevent complaints about not finding exec-prefix by making a lib-dynload directory
75+
Module.FS.mkdirTree("/lib/python3.14/lib-dynload/");
76+
Module.addRunDependency("install-stdlib");
77+
const resp = await fetch("python3.14.zip");
78+
const stdlibBuffer = await resp.arrayBuffer();
79+
Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { canOwn: true });
80+
Module.removeRunDependency("install-stdlib");
6981
}
7082
}
7183

72-
onmessage = (event) => {
84+
const modulePromise = createEmscriptenModule(emscriptenSettings);
85+
86+
87+
onmessage = async (event) => {
7388
if (event.data.type === 'run') {
89+
const Module = await modulePromise;
7490
if (event.data.files) {
7591
for (const [filename, contents] of Object.entries(event.data.files)) {
7692
Module.FS.writeFile(filename, contents)
7793
}
7894
}
79-
const ret = callMain(event.data.args)
95+
const ret = Module.callMain(event.data.args);
8096
postMessage({
8197
type: 'finished',
8298
returnCode: ret
8399
})
84100
}
85101
}
86102

87-
importScripts('python.js')

Tools/wasm/wasm_webserver.py renamed to Tools/wasm/emscripten/web_example/server.py

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,6 @@
1414

1515

1616
class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
17-
extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy()
18-
extensions_map.update(
19-
{
20-
".wasm": "application/wasm",
21-
}
22-
)
23-
2417
def end_headers(self) -> None:
2518
self.send_my_headers()
2619
super().end_headers()
@@ -42,5 +35,6 @@ def main() -> None:
4235
bind=args.bind,
4336
)
4437

38+
4539
if __name__ == "__main__":
4640
main()

Tools/wasm/wasm_assets.py renamed to Tools/wasm/emscripten/web_example/wasm_assets.py

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from typing import Dict
2020

2121
# source directory
22-
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
22+
SRCDIR = pathlib.Path(__file__).parents[4].absolute()
2323
SRCDIR_LIB = SRCDIR / "Lib"
2424

2525

@@ -28,9 +28,7 @@
2828
WASM_STDLIB_ZIP = (
2929
WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
3030
)
31-
WASM_STDLIB = (
32-
WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
33-
)
31+
WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
3432
WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
3533

3634

@@ -114,9 +112,7 @@ def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path:
114112
assert isinstance(args.builddir, pathlib.Path)
115113
data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined]
116114
if not data_name.startswith(SYSCONFIG_NAMES):
117-
raise ValueError(
118-
f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES
119-
)
115+
raise ValueError(f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES)
120116
filename = data_name + ".py"
121117
return args.builddir / filename
122118

@@ -131,7 +127,7 @@ def filterfunc(filename: str) -> bool:
131127
return pathname not in args.omit_files_absolute
132128

133129
with zipfile.PyZipFile(
134-
args.wasm_stdlib_zip,
130+
args.output,
135131
mode="w",
136132
compression=args.compression,
137133
optimize=optimize,
@@ -195,6 +191,12 @@ def path(val: str) -> pathlib.Path:
195191
default=pathlib.Path("/usr/local"),
196192
type=path,
197193
)
194+
parser.add_argument(
195+
"-o",
196+
"--output",
197+
help="output file",
198+
type=path,
199+
)
198200

199201

200202
def main() -> None:
@@ -204,7 +206,6 @@ def main() -> None:
204206
args.srcdir = SRCDIR
205207
args.srcdir_lib = SRCDIR_LIB
206208
args.wasm_root = args.buildroot / relative_prefix
207-
args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
208209
args.wasm_stdlib = args.wasm_root / WASM_STDLIB
209210
args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
210211

@@ -234,12 +235,10 @@ def main() -> None:
234235
args.wasm_dynload.mkdir(parents=True, exist_ok=True)
235236
marker = args.wasm_dynload / ".empty"
236237
marker.touch()
237-
# os.py is a marker for finding the correct lib directory.
238-
shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
239238
# The rest of stdlib that's useful in a WASM context.
240239
create_stdlib_zip(args)
241-
size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2)
242-
parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
240+
size = round(args.output.stat().st_size / 1024**2, 2)
241+
parser.exit(0, f"Created {args.output} ({size} MiB)\n")
243242

244243

245244
if __name__ == "__main__":

‎configure‎

Lines changed: 8 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎configure.ac‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1854,9 +1854,13 @@ if test "$Py_OPT" = 'true' ; then
18541854
LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition"
18551855
], [], [-Werror])
18561856
])
1857-
elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then
1858-
dnl Emscripten does not support shared extensions yet. Build
1859-
dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files.
1857+
elif test "$ac_sys_system" = "Emscripten"; then
1858+
dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files.
1859+
DEF_MAKE_ALL_RULE="build_emscripten"
1860+
REQUIRE_PGO="no"
1861+
DEF_MAKE_RULE="all"
1862+
elif test "$ac_sys_system" = "WASI"; then
1863+
dnl Build "python.wasm", "pybuilddir.txt", and "platform" files.
18601864
DEF_MAKE_ALL_RULE="build_wasm"
18611865
REQUIRE_PGO="no"
18621866
DEF_MAKE_RULE="all"
@@ -2321,14 +2325,14 @@ AS_CASE([$ac_sys_system],
23212325
AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no])
23222326
23232327
dnl Start with 20 MB and allow to grow
2324-
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"])
2328+
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"])
23252329
23262330
dnl map int64_t and uint64_t to JS bigint
23272331
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"])
23282332
23292333
dnl Include file system support
23302334
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"])
2331-
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"])
2335+
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"])
23322336
23332337
AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [
23342338
AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"])
@@ -2339,7 +2343,6 @@ AS_CASE([$ac_sys_system],
23392343
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"])
23402344
AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"])
23412345
])
2342-
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"])
23432346
dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now.
23442347
AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"])
23452348
WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map"

0 commit comments

Comments
 (0)