Skip to content

bpo-34638: Store a weak reference to stream reader to break strong references loop #9201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
asvetlov merged 10 commits into from
Sep 12, 2018
Merged

bpo-34638: Store a weak reference to stream reader to break strong references loop #9201

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5c87303
Store a weak reference to stream readerfor breaking strong references
asvetlov Sep 11, 2018
3ac4338
Work on
asvetlov Sep 12, 2018
465d9b3
Make tests passing
asvetlov Sep 12, 2018
5447b8a
Drop debug prints
asvetlov Sep 12, 2018
521ec5d
Add NEWS entry
asvetlov Sep 12, 2018
a7b4ad1
Drop redundant assertion
asvetlov Sep 12, 2018
806b5c2
_reject_transport -> _reject_connection
asvetlov Sep 12, 2018
119bfab
Add a test for destroying a reader befor connection_made() call
asvetlov Sep 12, 2018
ced1bd2
Fix notes
asvetlov Sep 12, 2018
1670250
Fix a comment
asvetlov Sep 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Make tests passing
  • Loading branch information
@asvetlov
asvetlov committed Sep 12, 2018
commit 465d9b3e217ae2bcfb65f3baf9a34b9b8204ac8b
15 changes: 13 additions & 2 deletions Lib/asyncio/streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,9 @@ def _on_reader_gc(self, wr):
# connection_made was called
print("schedule abort")
context = {
'message': "Stream was garbage collected"
'message': ("Close transport. "
"A stream was destroyed without "
"stream.close() call")

@1st1 1st1 Sep 12, 2018 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd rephrase as:

An open stream is being garbage collected; 
call "stream.close()" explicitly.

}
if self._source_traceback:
context['source_traceback'] = self._source_traceback
Expand All @@ -231,6 +233,9 @@ def _on_reader_gc(self, wr):
self._reject_transport = True
self._stream_reader_wr = None

def _untrack_reader(self):
self._stream_reader_wr = None

@property
def _stream_reader(self):
if self._stream_reader_wr is None:
Expand All @@ -241,7 +246,9 @@ def connection_made(self, transport):
print("connection made")
if self._reject_transport:
context = {
'message': "Close transport, a stream was garbage collected"
'message': ("Close transport. "
"a stream was destroyed "
"before connection establishment")

@1st1 1st1 Sep 12, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

An open stream was garbage collected prior to 
establishing network connection; call "stream.close()" explicitly.

}
if self._source_traceback:
context['source_traceback'] = self._source_traceback
Expand Down Expand Up @@ -280,6 +287,7 @@ def connection_lost(self, exc):
self._stream_reader_wr = None
self._stream_writer = None
self._transport = None
print("connection lost end")

def data_received(self, data):
reader = self._stream_reader
Expand Down Expand Up @@ -346,6 +354,9 @@ def can_write_eof(self):
return self._transport.can_write_eof()

def close(self):
# a reader could be garbage collected / destroyed
# after connection closing

@1st1 1st1 Sep 12, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

# The reader can be safely garbage collected after 
# closing the connection.

self._protocol._untrack_reader()
return self._transport.close()

def is_closing(self):
Expand Down
5 changes: 5 additions & 0 deletions Lib/asyncio/subprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ def __repr__(self):
info.append(f'stderr={self.stderr!r}')
return '<{}>'.format(' '.join(info))

def _untrack_reader(self):
# a dummy placeholder subprocess protocol doesn't need it but
# the method is required for regular stream implementation.
pass

@1st1 1st1 Sep 12, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

`StreamWriter.close()` expects the protocol to have this method defined.


def connection_made(self, transport):
self._transport = transport

Expand Down
10 changes: 9 additions & 1 deletion Lib/test/test_asyncio/test_streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,10 +585,12 @@ def __init__(self, loop):
self.loop = loop

async def handle_client(self, client_reader, client_writer):
print("start handle_client")
data = await client_reader.readline()
client_writer.write(data)
await client_writer.drain()
client_writer.close()
print("start handle_client")

def start(self):
sock = socket.socket()
Expand Down Expand Up @@ -633,6 +635,7 @@ async def client(addr):
messages = []
self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx))

print("run 1")
# test the server variant with a coroutine as client handler
server = MyServer(self.loop)
addr = server.start()
Expand All @@ -641,13 +644,17 @@ async def client(addr):
server.stop()
self.assertEqual(msg, b"hello world!\n")

print("run 1 end")

print("run 2")
# test the server variant with a callback as client handler
server = MyServer(self.loop)
addr = server.start_callback()
msg = self.loop.run_until_complete(asyncio.Task(client(addr),
loop=self.loop))
server.stop()
self.assertEqual(msg, b"hello world!\n")
print("run 2 end")

self.assertEqual(messages, [])

Expand Down Expand Up @@ -938,7 +945,8 @@ def test_del_stream_before_sock_closing(self):
self.assertEqual(sock.fileno(), -1)

self.assertEqual(1, len(messages))
self.assertEqual('Stream was garbage collected',
self.assertEqual('Close transport. A stream was destroyed '
'without stream.close() call',
messages[0]['message'])


Expand Down