Skip to content

Commit 591f675

Browse files
hukkinhukkin
authored
bpo-40059: Add tomllib (PEP-680) (GH-31498)
This adds a new standard library module, `tomllib`, for parsing TOML. The implementation is based on Tomli (https://github.com/hukkin/tomli). ## Steps taken (converting `tomli` to `tomllib`) - Move everything in `tomli:src/tomli` to `Lib/tomllib`. Exclude `py.typed`. - Remove `__version__ = ...` line from `Lib/tomllib/__init__.py` - Move everything in `tomli:tests` to `Lib/test/test_tomllib`. Exclude the following test data dirs recursively: - `tomli:tests/data/invalid/_external/` - `tomli:tests/data/valid/_external/` - Create `Lib/test/test_tomllib/__main__.py`: ```python import unittest from . import load_tests unittest.main() ``` - Add the following to `Lib/test/test_tomllib/__init__.py`: ```python import os from test.support import load_package_tests def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) ``` Also change `import tomli as tomllib` to `import tomllib`. - In `cpython/Lib/tomllib/_parser.py` replace `__fp` with `fp` and `__s` with `s`. Add the `/` to `load` and `loads` function signatures. - Run `make regen-stdlib-module-names` - Create `Doc/library/tomllib.rst` and reference it in `Doc/library/fileformats.rst`
1 parent 4d95fa1 commit 591f675

90 files changed

Lines changed: 1479 additions & 1 deletion

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/CODEOWNERS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ Lib/ast.py @isidentical
141141
**/*cgi* @ethanfurman
142142
**/*tarfile* @ethanfurman
143143

144+
**/*tomllib* @encukou
145+
144146
# macOS
145147
/Mac/ @python/macos-team
146148
**/*osx_support* @python/macos-team

‎Doc/library/fileformats.rst‎

Lines changed: 1 addition & 0 deletions

‎Doc/library/tomllib.rst‎

Lines changed: 117 additions & 0 deletions

‎Doc/whatsnew/3.11.rst‎

Lines changed: 2 additions & 1 deletion

‎Lib/test/test_tomllib/__init__.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# SPDX-License-Identifier: MIT
2+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
3+
# Licensed to PSF under a Contributor Agreement.
4+
5+
__all__ = ("tomllib",)
6+
7+
# By changing this one line, we can run the tests against
8+
# a different module name.
9+
import tomllib
10+
11+
import os
12+
from test.support import load_package_tests
13+
14+
def load_tests(*args):
15+
return load_package_tests(os.path.dirname(__file__), *args)

‎Lib/test/test_tomllib/__main__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import unittest
2+
3+
from . import load_tests
4+
5+
6+
unittest.main()

‎Lib/test/test_tomllib/burntsushi.py‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# SPDX-License-Identifier: MIT
2+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
3+
# Licensed to PSF under a Contributor Agreement.
4+
5+
"""Utilities for tests that are in the "burntsushi" format."""
6+
7+
import datetime
8+
from typing import Any
9+
10+
# Aliases for converting TOML compliance format [1] to BurntSushi format [2]
11+
# [1] https://github.com/toml-lang/compliance/blob/db7c3211fda30ff9ddb10292f4aeda7e2e10abc4/docs/json-encoding.md # noqa: E501
12+
# [2] https://github.com/BurntSushi/toml-test/blob/4634fdf3a6ecd6aaea5f4cdcd98b2733c2694993/README.md # noqa: E501
13+
_aliases = {
14+
"boolean": "bool",
15+
"offset datetime": "datetime",
16+
"local datetime": "datetime-local",
17+
"local date": "date-local",
18+
"local time": "time-local",
19+
}
20+
21+
22+
def convert(obj): # noqa: C901
23+
if isinstance(obj, str):
24+
return {"type": "string", "value": obj}
25+
elif isinstance(obj, bool):
26+
return {"type": "bool", "value": str(obj).lower()}
27+
elif isinstance(obj, int):
28+
return {"type": "integer", "value": str(obj)}
29+
elif isinstance(obj, float):
30+
return {"type": "float", "value": _normalize_float_str(str(obj))}
31+
elif isinstance(obj, datetime.datetime):
32+
val = _normalize_datetime_str(obj.isoformat())
33+
if obj.tzinfo:
34+
return {"type": "datetime", "value": val}
35+
return {"type": "datetime-local", "value": val}
36+
elif isinstance(obj, datetime.time):
37+
return {
38+
"type": "time-local",
39+
"value": _normalize_localtime_str(str(obj)),
40+
}
41+
elif isinstance(obj, datetime.date):
42+
return {
43+
"type": "date-local",
44+
"value": str(obj),
45+
}
46+
elif isinstance(obj, list):
47+
return [convert(i) for i in obj]
48+
elif isinstance(obj, dict):
49+
return {k: convert(v) for k, v in obj.items()}
50+
raise Exception("unsupported type")
51+
52+
53+
def normalize(obj: Any) -> Any:
54+
"""Normalize test objects.
55+
56+
This normalizes primitive values (e.g. floats), and also converts from
57+
TOML compliance format [1] to BurntSushi format [2].
58+
59+
[1] https://github.com/toml-lang/compliance/blob/db7c3211fda30ff9ddb10292f4aeda7e2e10abc4/docs/json-encoding.md # noqa: E501
60+
[2] https://github.com/BurntSushi/toml-test/blob/4634fdf3a6ecd6aaea5f4cdcd98b2733c2694993/README.md # noqa: E501
61+
"""
62+
if isinstance(obj, list):
63+
return [normalize(item) for item in obj]
64+
if isinstance(obj, dict):
65+
if "type" in obj and "value" in obj:
66+
type_ = obj["type"]
67+
norm_type = _aliases.get(type_, type_)
68+
value = obj["value"]
69+
if norm_type == "float":
70+
norm_value = _normalize_float_str(value)
71+
elif norm_type in {"datetime", "datetime-local"}:
72+
norm_value = _normalize_datetime_str(value)
73+
elif norm_type == "time-local":
74+
norm_value = _normalize_localtime_str(value)
75+
else:
76+
norm_value = value
77+
78+
if norm_type == "array":
79+
return [normalize(item) for item in value]
80+
return {"type": norm_type, "value": norm_value}
81+
return {k: normalize(v) for k, v in obj.items()}
82+
raise AssertionError("Burntsushi fixtures should be dicts/lists only")
83+
84+
85+
def _normalize_datetime_str(dt_str: str) -> str:
86+
if dt_str[-1].lower() == "z":
87+
dt_str = dt_str[:-1] + "+00:00"
88+
89+
date = dt_str[:10]
90+
rest = dt_str[11:]
91+
92+
if "+" in rest:
93+
sign = "+"
94+
elif "-" in rest:
95+
sign = "-"
96+
else:
97+
sign = ""
98+
99+
if sign:
100+
time, _, offset = rest.partition(sign)
101+
else:
102+
time = rest
103+
offset = ""
104+
105+
time = time.rstrip("0") if "." in time else time
106+
return date + "T" + time + sign + offset
107+
108+
109+
def _normalize_localtime_str(lt_str: str) -> str:
110+
return lt_str.rstrip("0") if "." in lt_str else lt_str
111+
112+
113+
def _normalize_float_str(float_str: str) -> str:
114+
as_float = float(float_str)
115+
116+
# Normalize "-0.0" and "+0.0"
117+
if as_float == 0:
118+
return "0"
119+
120+
return str(as_float)

‎Lib/test/test_tomllib/data/invalid/array-missing-comma.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
arrr = [true false]

‎Lib/test/test_tomllib/data/invalid/array-of-tables/overwrite-array-in-parent.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[[parent-table.arr]]
2+
[parent-table]
3+
not-arr = 1
4+
arr = 2

‎Lib/test/test_tomllib/data/invalid/array-of-tables/overwrite-bool-with-aot.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
a=true
2+
[[a]]

0 commit comments

Comments
 (0)