Skip to content

Commit 4b4449e

Browse files
anthonykim1anthonykim1
authored
New Unittest Execution (microsoft#19386)
* Adding execution fixes. * add map of runIDToTestItem for workspaceTestAdapter instance * update UI accordingly using new execution method * attempt to fix invalid argument * fix invalid argument * add traceback to the message * attempt to fix runID * remove import Codetypes * change runtest pattern to non optional * remove unnecessary import comments * add code for unittest execution debugger * add noop,traceError to solve mergeConflict * fix malfunctioning execution, skipped and expected failure outcome * allow cherry pick test runs for efficiency * add subtest update and directory search * leave new debugLauncher dormant and try to fix python type * ignore type for runner.run as in the past * cleanup execution.py * typescript side cleanup * move the runInstance starting up before executing * remove unnecessary debugLauncher check * get rid of TypeAlias * move uuid to header instead of inside payload * fix python test and look server unittest * test out timeout * try adding requestuuid to avoid timeout * try using rawHeaders.push * try inserting uuid inside option command * fix all server.unit.test.ts problem caused by moving uuid to header * get rid of unused variable warning * windows python test server * pass in debuglauncher to new python test server in all of server.unit.test.ts instances * make debugLauncher non optional on server.ts * fix unnecessary comments on discovery * rename requestuuid to request-uuid * rename parse_unittest_discovery_args and move to utils * attempt to fix the moduleNotFound for utils * attempt to fix moduleNotFound2 * rearrange os.path * undo rearrange os.path * test that were not found mark as skipped * fix set subtraction * remove unnecessary not found which is handled earlier by tailor run and result * fix broken discovery refresh * fix warnings regarding unused * remove unused variable * fix testData doesnt exist error * remove unused methods * comment new unittest
1 parent ae1c0ab commit 4b4449e

15 files changed

Lines changed: 501 additions & 235 deletions

File tree

‎pythonFiles/tests/unittestadapter/test_discovery.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@
1010
DEFAULT_PORT,
1111
discover_tests,
1212
parse_discovery_cli_args,
13-
parse_unittest_discovery_args,
1413
)
15-
from unittestadapter.utils import TestNodeTypeEnum
14+
from unittestadapter.utils import TestNodeTypeEnum, parse_unittest_args
1615

1716
from .helpers import TEST_DATA_PATH, is_same_tree
1817

@@ -68,7 +67,7 @@ def test_parse_unittest_args(args: List[str], expected: List[str]) -> None:
6867
"""The parse_unittest_args function should return values for the start_dir, pattern, and top_level_dir arguments
6968
when passed as command-line options, and ignore unrecognized arguments.
7069
"""
71-
actual = parse_unittest_discovery_args(args)
70+
actual = parse_unittest_args(args)
7271

7372
assert actual == expected
7473

@@ -132,7 +131,6 @@ def test_simple_discovery() -> None:
132131
actual = discover_tests(start_dir, pattern, None, uuid)
133132

134133
assert actual["status"] == "success"
135-
assert actual["uuid"] == uuid
136134
assert is_same_tree(actual.get("tests"), expected)
137135
assert "errors" not in actual
138136

@@ -148,7 +146,6 @@ def test_empty_discovery() -> None:
148146
actual = discover_tests(start_dir, pattern, None, uuid)
149147

150148
assert actual["status"] == "success"
151-
assert actual["uuid"] == uuid
152149
assert "tests" not in actual
153150
assert "errors" not in actual
154151

@@ -215,6 +212,5 @@ def test_error_discovery() -> None:
215212
actual = discover_tests(start_dir, pattern, None, uuid)
216213

217214
assert actual["status"] == "error"
218-
assert actual["uuid"] == uuid
219215
assert is_same_tree(expected, actual.get("tests"))
220216
assert len(actual.get("errors", [])) == 1

‎pythonFiles/tests/unittestadapter/test_utils.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,13 +195,15 @@ def test_build_simple_tree() -> None:
195195
"type_": TestNodeTypeEnum.test,
196196
"lineno": "13",
197197
"id_": file_path + "\\" + "TreeOne" + "\\" + "test_one",
198+
"runID": "utils_simple_tree.TreeOne.test_one",
198199
},
199200
{
200201
"name": "test_two",
201202
"path": file_path,
202203
"type_": TestNodeTypeEnum.test,
203204
"lineno": "16",
204205
"id_": file_path + "\\" + "TreeOne" + "\\" + "test_two",
206+
"runID": "utils_simple_tree.TreeOne.test_two",
205207
},
206208
],
207209
"id_": file_path + "\\" + "TreeOne",
@@ -253,13 +255,15 @@ def test_build_decorated_tree() -> None:
253255
"type_": TestNodeTypeEnum.test,
254256
"lineno": "24",
255257
"id_": file_path + "\\" + "TreeOne" + "\\" + "test_one",
258+
"runID": "utils_decorated_tree.TreeOne.test_one",
256259
},
257260
{
258261
"name": "test_two",
259262
"path": file_path,
260263
"type_": TestNodeTypeEnum.test,
261264
"lineno": "28",
262265
"id_": file_path + "\\" + "TreeOne" + "\\" + "test_two",
266+
"runID": "utils_decorated_tree.TreeOne.test_two",
263267
},
264268
],
265269
"id_": file_path + "\\" + "TreeOne",

‎pythonFiles/unittestadapter/discovery.py‎

Lines changed: 4 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from testing_tools import socket_manager
1818

1919
# If I use from utils then there will be an import error in test_discovery.py.
20-
from unittestadapter.utils import TestNode, build_test_tree
20+
from unittestadapter.utils import TestNode, build_test_tree, parse_unittest_args
2121

2222
# Add the lib path to sys.path to find the typing_extensions module.
2323
sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python"))
@@ -44,35 +44,8 @@ def parse_discovery_cli_args(args: List[str]) -> Tuple[int, Union[str, None]]:
4444
return int(parsed_args.port), parsed_args.uuid
4545

4646

47-
def parse_unittest_discovery_args(args: List[str]) -> Tuple[str, str, Union[str, None]]:
48-
"""Parse command-line arguments that should be forwarded to unittest to perform discovery.
49-
50-
Valid unittest arguments are: -v, -s, -p, -t and their long-form counterparts,
51-
however we only care about the last three.
52-
53-
The returned tuple contains the following items
54-
- start_directory: The directory where to start discovery, defaults to .
55-
- pattern: The pattern to match test files, defaults to test*.py
56-
- top_level_directory: The top-level directory of the project, defaults to None, and unittest will use start_directory behind the scenes.
57-
"""
58-
59-
arg_parser = argparse.ArgumentParser()
60-
arg_parser.add_argument("--start-directory", "-s", default=".")
61-
arg_parser.add_argument("--pattern", "-p", default="test*.py")
62-
arg_parser.add_argument("--top-level-directory", "-t", default=None)
63-
64-
parsed_args, _ = arg_parser.parse_known_args(args)
65-
66-
return (
67-
parsed_args.start_directory,
68-
parsed_args.pattern,
69-
parsed_args.top_level_directory,
70-
)
71-
72-
7347
class PayloadDict(TypedDict):
7448
cwd: str
75-
uuid: Union[str, None]
7649
status: Literal["success", "error"]
7750
tests: NotRequired[TestNode]
7851
errors: NotRequired[List[str]]
@@ -112,7 +85,7 @@ def discover_tests(
11285
}
11386
"""
11487
cwd = os.path.abspath(start_dir)
115-
payload: PayloadDict = {"cwd": cwd, "status": "success", "uuid": uuid}
88+
payload: PayloadDict = {"cwd": cwd, "status": "success"}
11689
tests = None
11790
errors: List[str] = []
11891

@@ -140,7 +113,7 @@ def discover_tests(
140113
argv = sys.argv[1:]
141114
index = argv.index("--udiscovery")
142115

143-
start_dir, pattern, top_level_dir = parse_unittest_discovery_args(argv[index + 1 :])
116+
start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :])
144117

145118
# Perform test discovery.
146119
port, uuid = parse_discovery_cli_args(argv[:index])
@@ -154,6 +127,7 @@ def discover_tests(
154127
Host: localhost:{port}
155128
Content-Length: {len(data)}
156129
Content-Type: application/json
130+
Request-uuid: {uuid}
157131
158132
{data}"""
159133
result = s.socket.sendall(request.encode("utf-8")) # type: ignore

‎pythonFiles/unittestadapter/execution.py‎

Lines changed: 33 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,14 @@
1111
from types import TracebackType
1212
from typing import Dict, List, Optional, Tuple, Type, TypeAlias, TypedDict
1313

14-
from typing_extensions import NotRequired
15-
16-
from .discovery import parse_unittest_discovery_args
17-
1814
# Add the path to pythonFiles to sys.path to find testing_tools.socket_manager.
1915
PYTHON_FILES = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
2016
sys.path.insert(0, PYTHON_FILES)
21-
22-
# from testing_tools import socket_manager
23-
2417
# Add the lib path to sys.path to find the typing_extensions module.
2518
sys.path.insert(0, os.path.join(PYTHON_FILES, "lib", "python"))
19+
from testing_tools import socket_manager
20+
from typing_extensions import NotRequired
21+
from unittestadapter.utils import parse_unittest_args
2622

2723
DEFAULT_PORT = "45454"
2824

@@ -41,15 +37,13 @@ def parse_execution_cli_args(args: List[str]) -> Tuple[int, str | None, List[str
4137
arg_parser = argparse.ArgumentParser()
4238
arg_parser.add_argument("--port", default=DEFAULT_PORT)
4339
arg_parser.add_argument("--uuid")
44-
arg_parser.add_argument("--testids")
40+
arg_parser.add_argument("--testids", nargs="+")
4541
parsed_args, _ = arg_parser.parse_known_args(args)
4642

47-
test_ids: List[str] = parsed_args.testids.split(",") if parsed_args.testids else []
48-
49-
return (int(parsed_args.port), parsed_args.uuid, test_ids)
43+
return (int(parsed_args.port), parsed_args.uuid, parsed_args.testids)
5044

5145

52-
ErrorType: TypeAlias = (
46+
ErrorType = (
5347
Tuple[Type[BaseException], BaseException, TracebackType] | Tuple[None, None, None]
5448
)
5549

@@ -66,6 +60,7 @@ class TestOutcomeEnum(str, enum.Enum):
6660

6761

6862
class UnittestTestResult(unittest.TextTestResult):
63+
6964
formatted: Dict[str, Dict[str, str | None]] = dict()
7065

7166
def startTest(self, test: unittest.TestCase):
@@ -152,7 +147,6 @@ class TestExecutionStatus(str, enum.Enum):
152147

153148
class PayloadDict(TypedDict):
154149
cwd: str
155-
uuid: str | None
156150
status: TestExecutionStatus
157151
result: NotRequired[TestResultTypeAlias]
158152
not_found: NotRequired[List[str]]
@@ -167,14 +161,14 @@ class PayloadDict(TypedDict):
167161
def run_tests(
168162
start_dir: str,
169163
test_ids: List[str],
170-
pattern: Optional[str],
164+
pattern: str,
171165
top_level_dir: Optional[str],
172166
uuid: Optional[str],
173167
) -> PayloadDict:
174168
cwd = os.path.abspath(start_dir)
175-
status = TestExecutionStatus.success
169+
status = TestExecutionStatus.error
176170
error = None
177-
payload: PayloadDict = {"cwd": cwd, "uuid": uuid, "status": status}
171+
payload: PayloadDict = {"cwd": cwd, "status": status}
178172

179173
try:
180174
# If it's a file, split path and file name.
@@ -191,20 +185,17 @@ def run_tests(
191185
"pattern": pattern,
192186
"top_level_dir": top_level_dir,
193187
}
194-
suite = loader.discover(**{k: v for k, v in args.items() if v is not None})
188+
suite = loader.discover(start_dir, pattern, top_level_dir)
195189

196190
# Run tests.
197191
runner = unittest.TextTestRunner(resultclass=UnittestTestResult)
198-
result: UnittestTestResult = runner.run(suite) # type: ignore
192+
# lets try to tailer our own suite so we can figure out running only the ones we want
193+
loader = unittest.TestLoader()
194+
tailor: unittest.TestSuite = loader.loadTestsFromNames(test_ids)
195+
result: UnittestTestResult = runner.run(tailor) # type: ignore
199196

200-
# Filter tests by id.
201-
filtered_results = {k: v for k, v in result.formatted.items() if k in test_ids}
202-
payload["result"] = filtered_results
197+
payload["result"] = result.formatted
203198

204-
# Add a payload entry with the list of test ids for tests that weren't found.
205-
not_found = set(test_ids) - set(filtered_results.keys())
206-
if not_found:
207-
payload["not_found"] = list(not_found)
208199
except Exception:
209200
status = TestExecutionStatus.error
210201
error = traceback.format_exc()
@@ -214,7 +205,7 @@ def run_tests(
214205

215206
payload["status"] = status
216207

217-
print(f"payload: \n{json.dumps(payload, indent=4)}")
208+
# print(f"payload: \n{json.dumps(payload, indent=4)}")
218209

219210
return payload
220211

@@ -224,29 +215,21 @@ def run_tests(
224215
argv = sys.argv[1:]
225216
index = argv.index("--udiscovery")
226217

227-
start_dir, pattern, top_level_dir = parse_unittest_discovery_args(argv[index + 1 :])
228-
229-
# start_path = pathlib.Path.home() / "Documents" / "Sandbox" / "unittest-subtest"
230-
# test_ids = [
231-
# "subfolder.test_two.TestClassTwo.test_two_two",
232-
# "test_one.TestClassOne.test_func_one",
233-
# "test_eight.TestClassEight.test_func_eight",
234-
# ]
235-
# uuid = "abcd"
218+
start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :])
236219

237220
# Perform test execution.
238-
port, uuid, test_ids = parse_execution_cli_args(argv[:index])
239-
run_tests(start_dir, test_ids, pattern, top_level_dir, uuid)
240-
241-
242-
# # Build the request data (it has to be a POST request or the Node side will not process it), and send it.
243-
# addr = ("localhost", port)
244-
# with socket_manager.SocketManager(addr) as s:
245-
# data = json.dumps(payload)
246-
# request = f"""POST / HTTP/1.1
247-
# Host: localhost:{port}
248-
# Content-Length: {len(data)}
249-
# Content-Type: application/json
250-
251-
# {data}"""
252-
# result = s.socket.sendall(request.encode("utf-8")) # type: ignore
221+
port, uuid, testids = parse_execution_cli_args(argv[:index])
222+
payload = run_tests(start_dir, testids, pattern, top_level_dir, uuid)
223+
224+
# Build the request data (it has to be a POST request or the Node side will not process it), and send it.
225+
addr = ("localhost", port)
226+
with socket_manager.SocketManager(addr) as s:
227+
data = json.dumps(payload)
228+
request = f"""POST / HTTP/1.1
229+
Host: localhost:{port}
230+
Content-Length: {len(data)}
231+
Content-Type: application/json
232+
Request-uuid: {uuid}
233+
234+
{data}"""
235+
result = s.socket.sendall(request.encode("utf-8")) # type: ignore

‎pythonFiles/unittestadapter/utils.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Copyright (c) Microsoft Corporation. All rights reserved.
22
# Licensed under the MIT License.
33

4+
import argparse
45
import enum
56
import inspect
67
import os
@@ -28,6 +29,7 @@ class TestData(TypedDict):
2829

2930
class TestItem(TestData):
3031
lineno: str
32+
runID: str
3133

3234

3335
class TestNode(TestData):
@@ -190,10 +192,37 @@ def build_test_tree(
190192
"lineno": lineno,
191193
"type_": TestNodeTypeEnum.test,
192194
"id_": file_path + "\\" + class_name + "\\" + function_name,
195+
"runID": test_id,
193196
} # concatenate class name and function test name
194197
current_node["children"].append(test_node)
195198

196199
if not root["children"]:
197200
root = None
198201

199202
return root, errors
203+
204+
205+
def parse_unittest_args(args: List[str]) -> Tuple[str, str, Union[str, None]]:
206+
"""Parse command-line arguments that should be forwarded to unittest to perform discovery.
207+
208+
Valid unittest arguments are: -v, -s, -p, -t and their long-form counterparts,
209+
however we only care about the last three.
210+
211+
The returned tuple contains the following items
212+
- start_directory: The directory where to start discovery, defaults to .
213+
- pattern: The pattern to match test files, defaults to test*.py
214+
- top_level_directory: The top-level directory of the project, defaults to None, and unittest will use start_directory behind the scenes.
215+
"""
216+
217+
arg_parser = argparse.ArgumentParser()
218+
arg_parser.add_argument("--start-directory", "-s", default=".")
219+
arg_parser.add_argument("--pattern", "-p", default="test*.py")
220+
arg_parser.add_argument("--top-level-directory", "-t", default=None)
221+
222+
parsed_args, _ = arg_parser.parse_known_args(args)
223+
224+
return (
225+
parsed_args.start_directory,
226+
parsed_args.pattern,
227+
parsed_args.top_level_directory,
228+
)

‎src/client/common/process/internal/scripts/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,13 @@ export function visualstudio_py_testlauncher(testArgs: string[]): string[] {
138138
return [script, ...testArgs];
139139
}
140140

141+
// execution.py
142+
// eslint-disable-next-line camelcase
143+
export function execution_py_testlauncher(testArgs: string[]): string[] {
144+
const script = path.join(SCRIPTS_DIR, 'unittestadapter', 'execution.py');
145+
return [script, ...testArgs];
146+
}
147+
141148
// tensorboard_launcher.py
142149

143150
export function tensorboardLauncher(args: string[]): string[] {

‎src/client/common/utils/localize.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,11 @@ export namespace Testing {
518518
);
519519
export const errorUnittestDiscovery = localize('Testing.errorUnittestDiscovery', 'Unittest test discovery error');
520520
export const seePythonOutput = localize('Testing.seePythonOutput', '(see Output > Python)');
521+
export const cancelUnittestExecution = localize(
522+
'Testing.cancelUnittestExecution',
523+
'Canceled unittest test execution',
524+
);
525+
export const errorUnittestExecution = localize('Testing.errorUnittestExecution', 'Unittest test execution error');
521526
}
522527

523528
export namespace OutdatedDebugger {

‎src/client/testing/common/debugLauncher.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ export class DebugLauncher implements ITestDebugLauncher {
211211
private getTestLauncherScript(testProvider: TestProvider) {
212212
switch (testProvider) {
213213
case 'unittest': {
214-
return internalScripts.visualstudio_py_testlauncher;
214+
return internalScripts.visualstudio_py_testlauncher; // old way unittest execution, debugger
215+
// return internalScripts.execution_py_testlauncher; // this is the new way to run unittest execution, debugger
215216
}
216217
case 'pytest': {
217218
return internalScripts.testlauncher;

0 commit comments

Comments
 (0)