Skip to content

Commit 8a9b841

Browse files
committed
Improve performance of json dumping
json.dump uses the Python implementation of the json encoder instead of the C implementation. While this saves a bit of memory while dumping data, the performance impact is large: python/cpython#56343 Saving a user share with 3 million files to disk previously took 1 minute 13 seconds. It now takes 14 seconds. The C implementation of the json encoder doesn't support the 'indent' argument, so stop using it.
1 parent e5f4147 commit 8a9b841

2 files changed

Lines changed: 23 additions & 20 deletions

File tree

‎pynicotine/transfers.py‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -263,21 +263,14 @@ def get_upload_list_file_name(self):
263263
def load_transfers_file(transfers_file):
264264
"""Loads a file of transfers in json format."""
265265

266-
def json_keys_to_integer(dictionary):
267-
# JSON stores file attribute types as strings, convert them back to integers
268-
try:
269-
return {int(k): v for k, v in dictionary}
270-
271-
except ValueError:
272-
return dictionary
273-
274266
transfers_file = encode_path(transfers_file)
275267

276268
if not os.path.isfile(transfers_file):
277269
return None
278270

279271
with open(transfers_file, encoding="utf-8") as handle:
280-
return json.load(handle, object_pairs_hook=json_keys_to_integer)
272+
# JSON stores file attribute types as strings, convert them back to integers with object_hook
273+
return json.load(handle, object_hook=lambda d: {int(k): v for k, v in d.items()})
281274

282275
@staticmethod
283276
def load_legacy_transfers_file(transfers_file):
@@ -2682,11 +2675,11 @@ def get_uploads(self):
26822675
for upload in reversed(self.uploads) if upload.status == "Finished"
26832676
]
26842677

2685-
def save_downloads_callback(self, filename):
2686-
json.dump(self.get_downloads(), filename, ensure_ascii=False)
2678+
def save_downloads_callback(self, file_handle):
2679+
file_handle.write(json.dumps(self.get_downloads(), check_circular=False, ensure_ascii=False))
26872680

2688-
def save_uploads_callback(self, filename):
2689-
json.dump(self.get_uploads(), filename, ensure_ascii=False)
2681+
def save_uploads_callback(self, file_handle):
2682+
file_handle.write(json.dumps(self.get_uploads(), check_circular=False, ensure_ascii=False))
26902683

26912684
def save_transfers(self):
26922685
"""Save list of transfers."""

‎pynicotine/userbrowse.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,6 @@ def load_shares_list_from_disk(self, filename):
145145

146146
filename_encoded = encode_path(filename)
147147

148-
def json_keys_to_integer(dictionary):
149-
# JSON stores file attribute types as strings, convert them back to integers
150-
return {int(k): v for k, v in dictionary}
151-
152148
try:
153149
try:
154150
# Try legacy format first
@@ -161,7 +157,8 @@ def json_keys_to_integer(dictionary):
161157
# Try new format
162158

163159
with open(filename_encoded, encoding="utf-8") as file_handle:
164-
shares_list = json.load(file_handle, object_pairs_hook=json_keys_to_integer)
160+
# JSON stores file attribute types as strings, convert them back to integers with object_hook
161+
shares_list = json.load(file_handle, object_hook=lambda d: {int(k): v for k, v in d.items()})
165162

166163
# Basic sanity check
167164
for _folder, files in shares_list:
@@ -196,8 +193,21 @@ def save_shares_list_to_disk(self, user):
196193
path = os.path.join(shares_folder, clean_file(user))
197194

198195
with open(encode_path(path), "w", encoding="utf-8") as file_handle:
199-
# Add line breaks for readability, but avoid indentation to decrease file size
200-
json.dump(list(self.user_shares[user].items()), file_handle, ensure_ascii=False, indent=0)
196+
# Dump every folder to the file individually to avoid large memory usage
197+
json_encoder = json.JSONEncoder(check_circular=False, ensure_ascii=False)
198+
is_first_item = True
199+
200+
file_handle.write("[")
201+
202+
for item in self.user_shares[user].items():
203+
if is_first_item:
204+
is_first_item = False
205+
else:
206+
file_handle.write(", ")
207+
208+
file_handle.write(json_encoder.encode(item))
209+
210+
file_handle.write("]")
201211

202212
log.add(_("Saved list of shared files for user '%(user)s' to %(dir)s"),
203213
{"user": user, "dir": shares_folder})

0 commit comments

Comments
 (0)