Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
7dd836b
Add binary I/O header for sampling profiler
pablogsal Dec 15, 2025
83603c7
Add binary writer for sampling profiler
pablogsal Dec 15, 2025
7a58811
Add binary reader for sampling profiler
pablogsal Dec 15, 2025
e3ea7a4
Build: add binary I/O files to Unix build
pablogsal Dec 15, 2025
8931b4a
Build: add binary I/O files to Windows build
pablogsal Dec 15, 2025
18287f4
Add Python bindings for binary writer and reader
pablogsal Dec 15, 2025
1f7737e
Add BinaryCollector for sampling profiler
pablogsal Dec 15, 2025
2965eff
Add BinaryReader for sampling profiler replay
pablogsal Dec 15, 2025
427f846
Add --binary and replay command to sampling profiler CLI
pablogsal Dec 15, 2025
1437629
Update collector interface for binary replay support
pablogsal Dec 15, 2025
e75513b
Add tests for binary format
pablogsal Dec 15, 2025
a520ac1
Add internal documentation for binary format
pablogsal Dec 15, 2025
3ad7a3d
Document binary format and replay command
pablogsal Dec 15, 2025
cd4f412
Fix CI
pablogsal Dec 15, 2025
596af7f
Add NEWS entry
pablogsal Dec 15, 2025
1e2400b
Regen and simplify
pablogsal Dec 15, 2025
b90e37c
Merge upstream/main into file-output
pablogsal Dec 15, 2025
788c565
Speed up general case
pablogsal Dec 15, 2025
51b3b09
Small fixes
pablogsal Dec 20, 2025
81a8f2d
Merge branch 'main' into file-output
pablogsal Dec 20, 2025
0ccea49
Address Laszlo's review
pablogsal Dec 20, 2025
dfc0d31
More fixes from review
pablogsal Dec 20, 2025
b9fc778
Fixes and more tests
pablogsal Dec 20, 2025
191458a
Fix c analizer
pablogsal Dec 22, 2025
63c6f50
Update docs
pablogsal Dec 22, 2025
046fe55
Update cli
pablogsal Dec 22, 2025
f88c01f
Fixes
pablogsal Dec 22, 2025
ed63156
Merge upstream/main
pablogsal Dec 22, 2025
c880f85
Fix big endian
pablogsal Dec 22, 2025
ad7891c
Fix big endian
pablogsal Dec 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Regen and simplify
  • Loading branch information
pablogsal committed Dec 15, 2025
commit 1e2400b53f694fb7757d3138c552d451f4268288
9 changes: 9 additions & 0 deletions Include/internal/pycore_global_objects_fini_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions Include/internal/pycore_global_strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -376,9 +376,11 @@ struct _Py_global_strings {
STRUCT_FOR_ID(co_varnames)
STRUCT_FOR_ID(code)
STRUCT_FOR_ID(col_offset)
STRUCT_FOR_ID(collector)
STRUCT_FOR_ID(command)
STRUCT_FOR_ID(comment_factory)
STRUCT_FOR_ID(compile_mode)
STRUCT_FOR_ID(compression)
STRUCT_FOR_ID(config)
STRUCT_FOR_ID(consts)
STRUCT_FOR_ID(context)
Expand Down Expand Up @@ -441,7 +443,9 @@ struct _Py_global_strings {
STRUCT_FOR_ID(event)
STRUCT_FOR_ID(eventmask)
STRUCT_FOR_ID(exc)
STRUCT_FOR_ID(exc_tb)
STRUCT_FOR_ID(exc_type)
STRUCT_FOR_ID(exc_val)
STRUCT_FOR_ID(exc_value)
STRUCT_FOR_ID(excepthook)
STRUCT_FOR_ID(exception)
Expand Down Expand Up @@ -697,6 +701,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(print_file_and_line)
STRUCT_FOR_ID(priority)
STRUCT_FOR_ID(progress)
STRUCT_FOR_ID(progress_callback)
STRUCT_FOR_ID(progress_routine)
STRUCT_FOR_ID(proto)
STRUCT_FOR_ID(protocol)
Expand Down Expand Up @@ -736,6 +741,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(reversed)
STRUCT_FOR_ID(rounding)
STRUCT_FOR_ID(salt)
STRUCT_FOR_ID(sample_interval_us)
STRUCT_FOR_ID(sched_priority)
STRUCT_FOR_ID(scheduler)
STRUCT_FOR_ID(script)
Expand Down Expand Up @@ -775,8 +781,10 @@ struct _Py_global_strings {
STRUCT_FOR_ID(spam)
STRUCT_FOR_ID(src)
STRUCT_FOR_ID(src_dir_fd)
STRUCT_FOR_ID(stack_frames)
STRUCT_FOR_ID(stacklevel)
STRUCT_FOR_ID(start)
STRUCT_FOR_ID(start_time_us)
STRUCT_FOR_ID(statement)
STRUCT_FOR_ID(stats)
STRUCT_FOR_ID(status)
Expand Down Expand Up @@ -817,6 +825,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(times)
STRUCT_FOR_ID(timespec)
STRUCT_FOR_ID(timestamp)
STRUCT_FOR_ID(timestamp_us)
STRUCT_FOR_ID(timetuple)
STRUCT_FOR_ID(timeunit)
STRUCT_FOR_ID(top)
Expand Down
9 changes: 9 additions & 0 deletions Include/internal/pycore_runtime_init_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 36 additions & 0 deletions Include/internal/pycore_unicodeobject_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 0 additions & 14 deletions InternalDocs/profiling_binary_format.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,20 +174,6 @@ frames from the previous stack are kept, and new frames are prepended.
Used when the code path changed: some frames were popped (function returns)
and new frames were pushed (different function calls).

### Compression Benefits

This delta encoding provides massive savings for typical profiling workloads:

- **CPU-bound code**: Hot loops produce many identical samples. RLE encoding
compresses 100 identical samples to just 2-3 bytes of overhead plus the
timestamp/status data.

- **I/O-bound code**: Alternating between wait and work produces similar
stacks with small variations. SUFFIX encoding captures this efficiently.

- **Call-heavy code**: Functions calling other functions share common stack
prefixes. POP_PUSH encoding only stores the changed frames.

### Thread and Interpreter Identification

Thread IDs are 64-bit values that can be large (memory addresses on some
Expand Down
2 changes: 0 additions & 2 deletions Lib/profiling/sampling/binary_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ def export(self, filename=None):

@property
def total_samples(self):
"""Total number of samples written."""
return self._writer.total_samples

def get_stats(self):
Expand All @@ -111,7 +110,6 @@ def get_stats(self):
return self._writer.get_stats()

def __enter__(self):
"""Context manager entry."""
return self

def __exit__(self, exc_type, exc_val, exc_tb):
Expand Down
3 changes: 0 additions & 3 deletions Lib/profiling/sampling/binary_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,11 @@ def __init__(self, filename):
self._reader = None

def __enter__(self):
"""Open the binary file for reading."""
import _remote_debugging
self._reader = _remote_debugging.BinaryReader(self.filename)
return self

def __exit__(self, exc_type, exc_val, exc_tb):
"""Close the binary file."""
if self._reader is not None:
self._reader.close()
self._reader = None
Expand Down Expand Up @@ -71,7 +69,6 @@ def replay_samples(self, collector, progress_callback=None):

@property
def sample_count(self):
"""Number of samples in the file."""
if self._reader is None:
raise RuntimeError("Reader not open. Use as context manager.")
return self._reader.get_info()['sample_count']
Expand Down
13 changes: 2 additions & 11 deletions Lib/profiling/sampling/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def _parse_mode(mode_string):
def _check_process_died(process):
"""Check if process died and raise an error with stderr if available."""
if process.poll() is None:
return # Process still running
return

# Process died - try to get stderr for error message
stderr_msg = ""
Expand Down Expand Up @@ -471,7 +471,6 @@ def _handle_output(collector, args, pid, mode):
sort_mode, limit, not args.no_summary, mode
)
else:
# Export to file
filename = args.outfile or _generate_output_filename(args.format, pid)
collector.export(filename)

Expand Down Expand Up @@ -726,7 +725,6 @@ def _handle_attach(args):
mode != PROFILING_MODE_WALL if mode != PROFILING_MODE_ALL else False
)

# For binary format, determine output file before creating collector
output_file = None
if args.format == "binary":
output_file = args.outfile or _generate_output_filename(args.format, args.pid)
Expand Down Expand Up @@ -806,7 +804,6 @@ def _handle_run(args):
mode != PROFILING_MODE_WALL if mode != PROFILING_MODE_ALL else False
)

# For binary format, determine output file before creating collector
output_file = None
if args.format == "binary":
output_file = args.outfile or _generate_output_filename(args.format, process.pid)
Expand Down Expand Up @@ -940,7 +937,6 @@ def _handle_replay(args):
"""Handle the 'replay' command - convert binary profile to another format."""
import os

# Check input file exists
if not os.path.exists(args.input_file):
sys.exit(f"Error: Input file not found: {args.input_file}")

Expand All @@ -956,10 +952,8 @@ def _handle_replay(args):
print(f" Sample interval: {interval} us")
print(f" Compression: {'zstd' if info.get('compression_type', 0) == 1 else 'none'}")

# Create appropriate collector
collector = _create_collector(args.format, interval, skip_idle=False)

# Replay with progress bar
def progress_callback(current, total):
if total > 0:
pct = current / total
Expand All @@ -969,20 +963,17 @@ def progress_callback(current, total):
print(f"\r [{bar}] {pct*100:5.1f}% ({current:,}/{total:,})", end="", flush=True)

count = reader.replay_samples(collector, progress_callback)
print() # Newline after progress bar
print()

# Handle output similar to other formats
if args.format == "pstats":
if args.outfile:
collector.export(args.outfile)
else:
# Print to stdout with defaults applied
sort_choice = args.sort if args.sort is not None else "nsamples"
limit = args.limit if args.limit is not None else 15
sort_mode = _sort_to_mode(sort_choice)
collector.print_stats(sort_mode, limit, not args.no_summary, PROFILING_MODE_WALL)
else:
# Export to file
filename = args.outfile or _generate_output_filename(args.format, os.getpid())
collector.export(filename)

Expand Down
Loading
Loading