Skip to content

Commit dd9f436

Browse files
jaracoclaude
andcommitted
Normalize Unicode form when matching MANIFEST.in patterns
FileList matched MANIFEST.in patterns against on-disk names byte-for-byte with no Unicode normalization. On macOS APFS/HFS+, a file stored NFD and a pattern authored NFC denote the same file but differ byte-for-byte, so an exclude/global-exclude/recursive-exclude/prune rule could silently fail to drop a non-ASCII-named file, publishing it in the sdist despite the rule. Normalize both the pattern and the candidate path to NFC before matching, via a new unicode_utils.normalize() helper and a _NormalizedMatcher wrapper around the compiled pattern in translate_pattern. Fixes GHSA-h35f-9h28-mq5c. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 84ed591 commit dd9f436

4 files changed

Lines changed: 93 additions & 1 deletion

File tree

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
``MANIFEST.in`` matching (via ``FileList``) is now insensitive to Unicode
2+
normalization form. A pattern authored in one form (e.g. NFC, as typically
3+
saved by editors) now matches a file whose name is stored on disk in another
4+
(e.g. NFD, as produced by macOS APFS/HFS+). Previously an ``exclude``,
5+
``global-exclude``, ``recursive-exclude``, or ``prune`` rule could silently
6+
fail to drop a non-ASCII-named file from the source distribution, publishing
7+
it despite the exclusion -- see GHSA-h35f-9h28-mq5c.

‎setuptools/command/egg_info.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,27 @@
3737
PY_MAJOR = f'{sys.version_info.major}.{sys.version_info.minor}'
3838

3939

40+
class _NormalizedMatcher:
41+
"""
42+
Wrap a compiled pattern so that matching is insensitive to Unicode
43+
normalization form.
44+
45+
File names walked from disk (NFD on macOS APFS/HFS+) and patterns from
46+
``MANIFEST.in`` (typically NFC) can denote the same file while differing
47+
byte-for-byte. Normalizing both sides before matching keeps an exclusion
48+
(or inclusion) from silently failing. See GHSA-h35f-9h28-mq5c.
49+
"""
50+
51+
def __init__(self, pattern: re.Pattern) -> None:
52+
self._pattern = pattern
53+
54+
def match(self, path):
55+
return self._pattern.match(unicode_utils.normalize(path))
56+
57+
def search(self, path):
58+
return self._pattern.search(unicode_utils.normalize(path))
59+
60+
4061
def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME
4162
"""
4263
Translate a file path glob like '*.txt' in to a regular expression.
@@ -46,6 +67,11 @@ def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME
4667
"""
4768
pat = ''
4869

70+
# Normalize the pattern so it matches paths regardless of the Unicode
71+
# normalization form used on disk (GHSA-h35f-9h28-mq5c). Candidate paths
72+
# are normalized to the same form by ``_NormalizedMatcher``.
73+
glob = unicode_utils.normalize(glob)
74+
4975
# This will split on '/' within [character classes]. This is deliberate.
5076
chunks = glob.split(os.path.sep)
5177

@@ -117,7 +143,7 @@ def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME
117143
pat += sep
118144

119145
pat += r'\Z'
120-
return re.compile(pat, flags=re.MULTILINE | re.DOTALL)
146+
return _NormalizedMatcher(re.compile(pat, flags=re.MULTILINE | re.DOTALL))
121147

122148

123149
class InfoCommon:

‎setuptools/tests/test_manifest.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import shutil
1111
import sys
1212
import tempfile
13+
import unicodedata
1314

1415
import pytest
1516

@@ -157,6 +158,21 @@ def test_translated_pattern_mismatch(pattern_mismatch):
157158
assert not translate_pattern(pattern).match(target)
158159

159160

161+
def test_translate_pattern_unicode_normalization():
162+
"""
163+
Matching is insensitive to Unicode normalization form: a pattern authored
164+
in one form matches a path stored on disk in another (and vice versa), so
165+
that an exclusion cannot be bypassed by an NFC/NFD mismatch.
166+
167+
Regression test for GHSA-h35f-9h28-mq5c.
168+
"""
169+
nfc = unicodedata.normalize('NFC', 'café.txt') # 'café.txt' composed
170+
nfd = unicodedata.normalize('NFD', 'café.txt') # 'café.txt' decomposed
171+
assert nfc != nfd # the two byte forms genuinely differ
172+
assert translate_pattern(nfc).match(nfd)
173+
assert translate_pattern(nfd).match(nfc)
174+
175+
160176
class TempDirTestCase:
161177
def setup_method(self, method):
162178
self.temp_dir = tempfile.mkdtemp()
@@ -331,6 +347,35 @@ def test_graft_prune(self):
331347
files = default_files | set([ml('app/a.txt'), ml('app/b.txt'), ml('app/c.rst')])
332348
assert files == self.get_files()
333349

350+
def test_global_exclude_unicode_normalization(self):
351+
"""
352+
A ``global-exclude`` authored NFC must drop a file whose on-disk name
353+
is NFD: on macOS APFS/HFS+ the two are the same file, and even on
354+
case/normalization-exact filesystems the decomposed name can be
355+
committed and reach the build. Otherwise the file is published in the
356+
sdist despite the exclusion.
357+
358+
Regression test for GHSA-h35f-9h28-mq5c.
359+
"""
360+
nfc_name = unicodedata.normalize('NFC', 'café.txt')
361+
nfd_name = unicodedata.normalize('NFD', 'café.txt')
362+
assert nfc_name != nfd_name
363+
# write the file under its decomposed (NFD) name ...
364+
touch(os.path.join(self.temp_dir, 'app', nfd_name))
365+
# ... and exclude it with the composed (NFC) form.
366+
self.make_manifest(
367+
f"""
368+
global-include *.txt
369+
global-exclude {nfc_name}
370+
"""
371+
)
372+
leaked = {
373+
f
374+
for f in self.get_files()
375+
if unicodedata.normalize('NFC', os.path.basename(f)) == nfc_name
376+
}
377+
assert not leaked, f"excluded file leaked into manifest: {leaked}"
378+
334379

335380
class TestFileListTest(TempDirTestCase):
336381
"""

‎setuptools/unicode_utils.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,20 @@ def decompose(path):
1919
return path
2020

2121

22+
def normalize(text):
23+
"""
24+
Return *text* in a canonical Unicode form (NFC) so that names which are
25+
visually identical but encoded differently compare equal.
26+
27+
macOS APFS/HFS+ store file names in decomposed form (NFD), while patterns
28+
in ``MANIFEST.in`` are typically authored composed (NFC). The two denote
29+
the same file but differ byte-for-byte, so matching them directly lets an
30+
exclusion silently fail. Normalizing both the walked path and the pattern
31+
to a single form before matching avoids that (GHSA-h35f-9h28-mq5c).
32+
"""
33+
return unicodedata.normalize('NFC', text) if isinstance(text, str) else text
34+
35+
2236
def filesys_decode(path):
2337
"""
2438
Ensure that the given path is decoded,

0 commit comments

Comments
 (0)