Skip to content

Commit e15f12f

Browse files
committed
util.download: support Content-Type with multiple MIMEs (non-standard?)
1 parent 322ade1 commit e15f12f

1 file changed

Lines changed: 56 additions & 41 deletions

File tree

‎util.py‎

Lines changed: 56 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -980,57 +980,72 @@ def has_valid_suffix(f):
980980
return True
981981

982982
def replace_suffix(f, content_type):
983-
header_suffix = ''
984-
mime = content_type.split(';')[0]
985-
guess = guess_extension(mime, strict=False)
986-
suffix_from_mime = mime.split('/')[-1].lower()
987-
988-
# manually set suffix for some common types that isn't covered by guess_extension
989-
maps = {
990-
'audio/mp4': '.m4a',
991-
'audio/x-m4a': '.m4a',
992-
'image/webp': '.webp',
993-
}
994983

995-
if mime in maps:
996-
header_suffix = maps[mime]
997-
# if 'application/octet-stream', just use the original suffix (guess_extension gives .bin)
998-
elif mime == 'application/octet-stream':
984+
# some URLs have multiple mime types in the header, e.g.
985+
# https://prd-pspa-s3-content.s3.ap-northeast-1.amazonaws.com/public/18255769761JOGWTUQPoSQXniAYcDMKy.jpg
986+
# Content-Type: image/png; image/jpg; image/jpeg
987+
header_suffixes = []
988+
for mime in content_type.split(';'):
989+
mime = mime.strip().lower()
999990
header_suffix = ''
1000-
# only use guessed extension if it's valid (and not bin)
1001-
elif guess is not None:
1002-
header_suffix = guess
1003-
# last resort. NOTE: this does not work half the time, maybe just abandon it...
1004-
elif re.search(r'^[a-zA-Z0-9]+$', suffix_from_mime):
1005-
header_suffix = '.' + suffix_from_mime
1006-
1007-
# replace logic
1008-
# if they're the same, we don't need to do anything
1009-
if f.suffix.lower() == header_suffix:
1010-
return f
1011-
# if header_suffix is a bad one, don't do anything
1012-
if header_suffix == '' or '-' in header_suffix or '+' in header_suffix:
991+
guess = guess_extension(mime, strict=False)
992+
suffix_from_mime = mime.split('/')[-1].lower()
993+
# manually set suffix for some common types that isn't covered by guess_extension
994+
maps = {
995+
'audio/mp4': '.m4a',
996+
'audio/x-m4a': '.m4a',
997+
'image/webp': '.webp',
998+
}
999+
if mime in maps:
1000+
header_suffix = maps[mime]
1001+
# if 'application/octet-stream', just use the original suffix (guess_extension gives .bin)
1002+
elif mime == 'application/octet-stream':
1003+
header_suffix = ''
1004+
# only use guessed extension if it's valid (and not bin)
1005+
elif guess is not None:
1006+
header_suffix = guess
1007+
# last resort. NOTE: this does not work half the time, maybe just abandon it...
1008+
elif re.search(r'^[a-zA-Z0-9]+$', suffix_from_mime):
1009+
header_suffix = '.' + suffix_from_mime
1010+
1011+
# ignore invalid suffixes
1012+
if '-' in header_suffix or '+' in header_suffix:
1013+
header_suffix = ''
1014+
# early return if we have a valid suffix and it matches the file's suffix
1015+
if header_suffix:
1016+
if f.suffix.lower() == header_suffix:
1017+
return f
1018+
# don't replace equivalent extensions
1019+
ext_alias_groups = [
1020+
['.mp4', '.m4a', '.m4v', '.m4s'],
1021+
['.jpg', '.jpeg']
1022+
]
1023+
for aliases in ext_alias_groups:
1024+
if f.suffix.lower() in aliases and header_suffix in aliases:
1025+
return f
1026+
# add it to candidate suffixes otherwise
1027+
header_suffixes.append(header_suffix)
1028+
# that means all the suffixes are invalid or empty; just return the original file
1029+
if not header_suffixes:
10131030
return f
1014-
# don't replace equivalent extensions
1015-
ext_alias_groups = [
1016-
['.mp4', '.m4a', '.m4v', '.m4s'],
1017-
['.jpg', '.jpeg']
1018-
]
1019-
for aliases in ext_alias_groups:
1020-
if f.suffix.lower() in aliases and header_suffix in aliases:
1021-
return f
1022-
# likely dynamic content, use header suffix instead (silently)
1023-
if f.suffix.lower() in ['.php', '']:
1031+
# use the first valid suffix from the header since we can't determine which one is the best.
1032+
# but print a warning if there are multiple valid suffixes.
1033+
if len(header_suffixes) > 1:
1034+
print(f'[Warning] Multiple valid suffixes found in Content-Type header: {header_suffixes}. Using the first one: {header_suffixes[0]}', 1)
1035+
header_suffix = header_suffixes[0]
1036+
1037+
# likely dynamic content, just use (add) header suffix silently
1038+
if f.suffix.lower() in ['.php', '.asp', '.jsp', '.cgi', '']:
10241039
return f.with_suffix(header_suffix)
1025-
10261040
# this is to prevent that when filename has dot in it, it would cause Path obj to consider part of stem is the suffix.
10271041
# so we only replace the suffix that is "valid" (<= 3 chars etc.) but wrong.
10281042
# Not ideal, but should be good enough.
10291043
if has_valid_suffix(f):
10301044
print(f'[Warning] File suffix is different from the one in Content-Type header! {f.suffix.lower()} -> {header_suffix}', 1)
10311045
return f.with_suffix(header_suffix)
1032-
# f has a weird suffix. We assume it's part of the name stem, so we just append the header suffix (silently).
1033-
return f.with_name(f.name + header_suffix)
1046+
else:
1047+
# f has a weird suffix. We assume it's part of the name stem, so we just append the header suffix (silently).
1048+
return f.with_name(f.name + header_suffix)
10341049

10351050
def check_dupe(f, dupe=dupe, size=0):
10361051
if not f.exists():

0 commit comments

Comments
 (0)