-
Notifications
You must be signed in to change notification settings - Fork 67
Expand file tree
/
Copy pathasync_chat_completion.py
More file actions
38 lines (29 loc) · 1.16 KB
/
Copy pathasync_chat_completion.py
File metadata and controls
38 lines (29 loc) · 1.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
"""Async chat completion via ``pydo.inference.aio.Client``.
Mirrors ``chat_completion_stream.py`` but uses the async client so it can
be embedded in an ``asyncio`` event loop alongside other concurrent work.
The surface is identical to the sync namespace — same constructor, same
operation groups, just ``await`` the calls and use ``async with`` so the
underlying aiohttp session closes cleanly.
Usage:
# The PAT must be created with FULL ACCESS scope to call inference APIs.
export DIGITALOCEAN_TOKEN="your-full-access-pat"
python examples/inference/async_chat_completion.py
"""
import asyncio
import os
from pydo.inference.aio import Client
async def main() -> None:
async with Client(token=os.environ["DIGITALOCEAN_TOKEN"]) as client:
resp = await client.chat.completions.create(
model="llama3.3-70b-instruct",
messages=[
{
"role": "user",
"content": "Give me a one-sentence fun fact about octopuses.",
}
],
max_tokens=128,
)
print(resp.choices[0].message.content)
if __name__ == "__main__":
asyncio.run(main())