Repository navigation
Expand file tree
/
Copy pathci_set_matrix.py
More file actions
executable file
·246 lines (207 loc) · 8.09 KB
/
Copy pathci_set_matrix.py
File metadata and controls
executable file
·246 lines (207 loc) · 8.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
#! /usr/bin/env python3
# SPDX-FileCopyrightText: 2021 Scott Shawcroft
# SPDX-FileCopyrightText: 2021 microDev
#
# SPDX-License-Identifier: MIT
"""
This script is used in GitHub Actions to determine what docs/boards are
built based on what files were changed. The base commit varies depending
on the event that triggered run. Pull request runs will compare to the
base branch while pushes will compare to the current ref. We override this
for the adafruit/circuitpython repo so we build all docs/boards for pushes.
When making changes to the script it is useful to manually test it.
You can for instance run
```shell
tools/ci_set_matrix ports/raspberrypi/common-hal/socket/SSLSocket.c
```
and (at the time this comment was written) get a series of messages indicating
that only the single board raspberry_pi_pico_w would be built.
"""
import re
import os
import sys
import json
import yaml
import pathlib
from concurrent.futures import ThreadPoolExecutor
tools_dir = pathlib.Path(__file__).resolve().parent
top_dir = tools_dir.parent
sys.path.insert(0, str(tools_dir / "adabot"))
sys.path.insert(0, str(top_dir / "docs"))
import build_board_info
from shared_bindings_matrix import (
get_settings_from_makefile,
SUPPORTED_PORTS,
all_ports_all_boards,
)
PORT_TO_ARCH = {
"atmel-samd": "arm",
"broadcom": "aarch",
"cxd56": "arm",
"espressif": "espressif",
"litex": "riscv",
"mimxrt10xx": "arm",
"nrf": "arm",
"raspberrypi": "arm",
"stm": "arm",
}
IGNORE = [
"tools/ci_set_matrix.py",
"tools/ci_check_duplicate_usb_vid_pid.py",
]
if len(sys.argv) > 1:
print("Using files list on commandline")
changed_files = sys.argv[1:]
else:
c = os.environ["CHANGED_FILES"]
if c == "":
print("CHANGED_FILES is in environment, but value is empty")
changed_files = []
else:
print("Using files list in CHANGED_FILES")
changed_files = json.loads(os.environ["CHANGED_FILES"])
def set_output(name, value):
if "GITHUB_OUTPUT" in os.environ:
with open(os.environ["GITHUB_OUTPUT"], "at") as f:
print(f"{name}={value}", file=f)
else:
print(f"Would set GitHub actions output {name} to '{value}'")
def set_boards_to_build(build_all):
# Get boards in json format
boards_info_json = build_board_info.get_board_mapping()
all_board_ids = set()
port_to_boards = {}
board_to_port = {}
board_settings = {}
for board_id in boards_info_json:
info = boards_info_json[board_id]
if info.get("alias", False):
continue
all_board_ids.add(board_id)
port = info["port"]
if port not in port_to_boards:
port_to_boards[port] = set()
port_to_boards[port].add(board_id)
board_to_port[board_id] = port
def compute_board_settings(boards):
need = set(boards) - set(board_settings.keys())
if not need:
return
def get_settings(board):
return (
board,
get_settings_from_makefile(str(top_dir / "ports" / board_to_port[board]), board),
)
with ThreadPoolExecutor(max_workers=os.cpu_count()) as ex:
board_settings.update(ex.map(get_settings, need))
boards_to_build = all_board_ids
if not build_all:
boards_to_build = set()
board_pattern = re.compile(r"^ports/[^/]+/boards/([^/]+)/")
port_pattern = re.compile(r"^ports/([^/]+)/")
module_pattern = re.compile(
r"^(ports/[^/]+/(?:common-hal|bindings)|shared-bindings|shared-module)/([^/]+)/"
)
for p in changed_files:
# See if it is board specific
board_matches = board_pattern.search(p)
if board_matches:
board = board_matches.group(1)
boards_to_build.add(board)
continue
# See if it is port specific
port_matches = port_pattern.search(p)
port = port_matches.group(1) if port_matches else None
module_matches = module_pattern.search(p)
if port and not module_matches:
if port != "unix":
boards_to_build.update(port_to_boards[port])
continue
# Check the ignore list to see if the file isn't used on board builds.
if p in IGNORE:
continue
# Boards don't run tests or docs so ignore those as well.
if p.startswith("tests") or p.startswith("docs"):
continue
# As a (nearly) last resort, for some certain files, we compute the settings from the
# makefile for each board and determine whether to build them that way.
if p.startswith("frozen") or p.startswith("supervisor") or module_matches:
if port:
board_ids = port_to_boards[port]
else:
board_ids = all_board_ids
compute_board_settings(board_ids)
for board in board_ids:
settings = board_settings[board]
# Check frozen files to see if they are in each board.
frozen = settings.get("FROZEN_MPY_DIRS", "")
if frozen and p.startswith("frozen") and p in frozen:
boards_to_build.add(board)
continue
# Check supervisor files. This is useful for limiting workflow changes to the
# relevant boards.
supervisor = settings["SRC_SUPERVISOR"]
if p.startswith("supervisor"):
if p in supervisor:
boards_to_build.add(board)
continue
web_workflow = settings["CIRCUITPY_WEB_WORKFLOW"]
while web_workflow.startswith("$("):
web_workflow = settings[web_workflow[2:-1]]
if (
p.startswith("supervisor/shared/web_workflow/static/")
and web_workflow != "0"
):
boards_to_build.add(board)
continue
# Check module matches
if module_matches:
module = module_matches.group(2) + "/"
if module in settings["SRC_PATTERNS"]:
boards_to_build.add(board)
continue
continue
# Otherwise build it all
boards_to_build = all_board_ids
break
# Split boards by architecture.
print("Building boards:")
arch_to_boards = {"aarch": [], "arm": [], "riscv": [], "espressif": []}
for board in sorted(boards_to_build):
print(" ", board)
port = board_to_port.get(board)
# A board can appear due to its _deletion_ (rare)
# if this happens it's not in `board_to_port`.
if not port:
continue
arch = PORT_TO_ARCH[port]
arch_to_boards[arch].append(board)
# Set the step outputs for each architecture
for arch in arch_to_boards:
set_output(f"boards-{arch}", json.dumps(sorted(arch_to_boards[arch])))
def set_docs_to_build(build_all):
doc_match = build_all
if not build_all:
doc_pattern = re.compile(
r"^(?:docs|extmod/ulab|(?:(?:ports/\w+/bindings|shared-bindings)\S+\.c|conf\.py|tools/extract_pyi\.py|requirements-doc\.txt)$)|(?:-stubs|\.(?:md|MD|rst|RST))$"
)
for p in changed_files:
if doc_pattern.search(p):
doc_match = True
break
# Set the step outputs
print("Building docs:", doc_match)
set_output(f"build-doc", doc_match)
def check_changed_files():
if not changed_files:
print("Building all docs/boards")
return True
else:
print("Adding docs/boards to build based on changed files")
return False
def main():
build_all = check_changed_files()
set_docs_to_build(build_all)
set_boards_to_build(build_all)
if __name__ == "__main__":
main()