|
| 1 | +# Copyright (c) Microsoft. All rights reserved. |
| 2 | + |
| 3 | +import asyncio |
| 4 | +import logging |
| 5 | +from datetime import datetime |
| 6 | +from random import randint |
| 7 | + |
| 8 | +from samples.concepts.realtime.utils import AudioPlayerWebRTC, AudioRecorderWebRTC, check_audio_devices |
| 9 | +from semantic_kernel import Kernel |
| 10 | +from semantic_kernel.connectors.ai import FunctionChoiceBehavior |
| 11 | +from semantic_kernel.connectors.ai.open_ai import ( |
| 12 | + ListenEvents, |
| 13 | + OpenAIRealtimeExecutionSettings, |
| 14 | + OpenAIRealtimeWebRTC, |
| 15 | + TurnDetection, |
| 16 | +) |
| 17 | +from semantic_kernel.contents import ChatHistory |
| 18 | +from semantic_kernel.contents.realtime_events import RealtimeTextEvent |
| 19 | +from semantic_kernel.functions import kernel_function |
| 20 | + |
| 21 | +logging.basicConfig(level=logging.WARNING) |
| 22 | +utils_log = logging.getLogger("samples.concepts.realtime.utils") |
| 23 | +utils_log.setLevel(logging.INFO) |
| 24 | +logger = logging.getLogger(__name__) |
| 25 | +logger.setLevel(logging.INFO) |
| 26 | + |
| 27 | +""" |
| 28 | +This simple sample demonstrates how to use the OpenAI Realtime API to create |
| 29 | +a chat bot that can listen and respond directly through audio. |
| 30 | +It requires installing: |
| 31 | +- semantic-kernel[realtime] |
| 32 | +- pyaudio |
| 33 | +- sounddevice |
| 34 | +- pydub |
| 35 | +e.g. pip install pyaudio sounddevice pydub semantic_kernel[realtime] |
| 36 | +
|
| 37 | +For more details of the exact setup, see the README.md in the realtime folder. |
| 38 | +""" |
| 39 | + |
| 40 | +# The characterics of your speaker and microphone are a big factor in a smooth conversation |
| 41 | +# so you may need to try out different devices for each. |
| 42 | +# you can also play around with the turn_detection settings to get the best results. |
| 43 | +# It has device id's set in the AudioRecorderStream and AudioPlayerAsync classes, |
| 44 | +# so you may need to adjust these for your system. |
| 45 | +# you can disable the check for available devices by commenting the line below |
| 46 | +check_audio_devices() |
| 47 | + |
| 48 | + |
| 49 | +@kernel_function |
| 50 | +def get_weather(location: str) -> str: |
| 51 | + """Get the weather for a location.""" |
| 52 | + weather_conditions = ("sunny", "hot", "cloudy", "raining", "freezing", "snowing") |
| 53 | + weather = weather_conditions[randint(0, len(weather_conditions) - 1)] # nosec |
| 54 | + logger.info(f"@ Getting weather for {location}: {weather}") |
| 55 | + return f"The weather in {location} is {weather}." |
| 56 | + |
| 57 | + |
| 58 | +@kernel_function |
| 59 | +def get_date_time() -> str: |
| 60 | + """Get the current date and time.""" |
| 61 | + logger.info("@ Getting current datetime") |
| 62 | + return f"The current date and time is {datetime.now().isoformat()}." |
| 63 | + |
| 64 | + |
| 65 | +@kernel_function |
| 66 | +def goodbye(): |
| 67 | + """When the user is done, say goodbye and then call this function.""" |
| 68 | + logger.info("@ Goodbye has been called!") |
| 69 | + raise KeyboardInterrupt |
| 70 | + |
| 71 | + |
| 72 | +async def main() -> None: |
| 73 | + print_transcript = True |
| 74 | + # create the Kernel and add a simple function for function calling. |
| 75 | + kernel = Kernel() |
| 76 | + kernel.add_functions(plugin_name="helpers", functions=[goodbye, get_weather, get_date_time]) |
| 77 | + |
| 78 | + # create the audio player and audio track |
| 79 | + # both take a device_id parameter, which is the index of the device to use, if None the default device is used |
| 80 | + audio_player = AudioPlayerWebRTC() |
| 81 | + # create the realtime client and optionally add the audio output function, this is optional |
| 82 | + # and can also be passed in the receive method |
| 83 | + realtime_client = OpenAIRealtimeWebRTC(audio_track=AudioRecorderWebRTC()) |
| 84 | + |
| 85 | + # Create the settings for the session |
| 86 | + # The realtime api, does not use a system message, but takes instructions as a parameter for a session |
| 87 | + # Another important setting is to tune the server_vad turn detection |
| 88 | + # if this is turned off (by setting turn_detection=None), you will have to send |
| 89 | + # the "input_audio_buffer.commit" and "response.create" event to the realtime api |
| 90 | + # to signal the end of the user's turn and start the response. |
| 91 | + # manual VAD is not part of this sample |
| 92 | + # for more info: https://platform.openai.com/docs/api-reference/realtime-sessions/create#realtime-sessions-create-turn_detection |
| 93 | + settings = OpenAIRealtimeExecutionSettings( |
| 94 | + instructions=""" |
| 95 | + You are a chat bot. Your name is Mosscap and |
| 96 | + you have one goal: figure out what people need. |
| 97 | + Your full name, should you need to know it, is |
| 98 | + Splendid Speckled Mosscap. You communicate |
| 99 | + effectively, but you tend to answer with long |
| 100 | + flowery prose. |
| 101 | + """, |
| 102 | + voice="alloy", |
| 103 | + turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), |
| 104 | + function_choice_behavior=FunctionChoiceBehavior.Auto(), |
| 105 | + ) |
| 106 | + # and we can add a chat history to conversation after starting it |
| 107 | + chat_history = ChatHistory() |
| 108 | + chat_history.add_user_message("Hi there, who are you?") |
| 109 | + chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") |
| 110 | + |
| 111 | + # the context manager calls the create_session method on the client and starts listening to the audio stream |
| 112 | + async with ( |
| 113 | + audio_player, |
| 114 | + realtime_client( |
| 115 | + settings=settings, |
| 116 | + chat_history=chat_history, |
| 117 | + kernel=kernel, |
| 118 | + create_response=True, |
| 119 | + ), |
| 120 | + ): |
| 121 | + async for event in realtime_client.receive(audio_output_callback=audio_player.client_callback): |
| 122 | + match event: |
| 123 | + case RealtimeTextEvent(): |
| 124 | + if print_transcript: |
| 125 | + print(event.text.text, end="") |
| 126 | + case _: |
| 127 | + # OpenAI Specific events |
| 128 | + match event.service_type: |
| 129 | + case ListenEvents.RESPONSE_CREATED: |
| 130 | + if print_transcript: |
| 131 | + print("\nMosscap (transcript): ", end="") |
| 132 | + case ListenEvents.ERROR: |
| 133 | + logger.error(event.service_event) |
| 134 | + |
| 135 | + |
| 136 | +if __name__ == "__main__": |
| 137 | + print( |
| 138 | + "Instructions: The model will start speaking immediately," |
| 139 | + "this can be turned off by removing `create_response=True` above." |
| 140 | + "The model will detect when you stop and automatically generate a response. " |
| 141 | + "Press ctrl + c to stop the program." |
| 142 | + ) |
| 143 | + asyncio.run(main()) |
0 commit comments