Skip to content

Commit fcc56cd

Browse files
johnoliverjohnoliver
authored
Java: Add OpenAI Audio to/from Text services (microsoft#5916)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
1 parent 0a7f96a commit fcc56cd

27 files changed

Lines changed: 1188 additions & 301 deletions

File tree

‎java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
package com.microsoft.semantickernel.aiservices.openai;
3+
4+
import com.azure.ai.openai.OpenAIAsyncClient;
5+
import com.microsoft.semantickernel.services.AIService;
6+
import javax.annotation.Nullable;
7+
8+
/**
9+
* Provides OpenAI service.
10+
*/
11+
public abstract class OpenAiService implements AIService {
12+
13+
private final OpenAIAsyncClient client;
14+
@Nullable
15+
private final String serviceId;
16+
private final String modelId;
17+
18+
protected OpenAiService(OpenAIAsyncClient client, @Nullable String serviceId, String modelId) {
19+
this.client = client;
20+
this.serviceId = serviceId;
21+
this.modelId = modelId;
22+
}
23+
24+
@Nullable
25+
@Override
26+
public String getModelId() {
27+
return modelId;
28+
}
29+
30+
@Override
31+
@Nullable
32+
public String getServiceId() {
33+
return serviceId;
34+
}
35+
36+
protected OpenAIAsyncClient getClient() {
37+
return client;
38+
}
39+
40+
}

‎java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
package com.microsoft.semantickernel.aiservices.openai.audio;
3+
4+
import com.azure.ai.openai.OpenAIAsyncClient;
5+
import com.azure.ai.openai.models.AudioTranscription;
6+
import com.azure.ai.openai.models.AudioTranscriptionFormat;
7+
import com.azure.ai.openai.models.AudioTranscriptionOptions;
8+
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
9+
import com.microsoft.semantickernel.exceptions.SKException;
10+
import com.microsoft.semantickernel.services.audio.AudioContent;
11+
import com.microsoft.semantickernel.services.audio.AudioToTextExecutionSettings;
12+
import com.microsoft.semantickernel.services.audio.AudioToTextService;
13+
import javax.annotation.Nullable;
14+
import reactor.core.publisher.Mono;
15+
16+
/**
17+
* Provides OpenAi implementation of audio to text service.
18+
*/
19+
public class OpenAiAudioToTextService extends OpenAiService implements AudioToTextService {
20+
21+
/**
22+
* Creates an instance of OpenAi audio to text service.
23+
*
24+
* @param client OpenAI client.
25+
* @param modelId The model ID.
26+
*/
27+
public OpenAiAudioToTextService(
28+
OpenAIAsyncClient client,
29+
String modelId) {
30+
super(client, null, modelId);
31+
}
32+
33+
@Override
34+
public Mono<String> getTextContentsAsync(
35+
AudioContent content,
36+
@Nullable AudioToTextExecutionSettings executionSettings) {
37+
38+
AudioTranscriptionOptions options = convertOptions(content, executionSettings);
39+
40+
// TODO: Should use getAudioTranscriptionTextWithResponse, and OpenAIRequestSettings.getRequestOptions()
41+
// however currently this breaks the request
42+
return getClient()
43+
.getAudioTranscription(
44+
getModelId(),
45+
options.getFilename(),
46+
options)
47+
.map(AudioTranscription::getText);
48+
}
49+
50+
private AudioTranscriptionOptions convertOptions(
51+
AudioContent content,
52+
@Nullable AudioToTextExecutionSettings executionSettings) {
53+
AudioTranscriptionOptions options = new AudioTranscriptionOptions(content.getData());
54+
55+
options.setModel(getModelId());
56+
if (executionSettings == null) {
57+
return options;
58+
}
59+
60+
if (executionSettings.getResponseFormat() != null) {
61+
options.setResponseFormat(
62+
AudioTranscriptionFormat.fromString(executionSettings.getResponseFormat()));
63+
}
64+
65+
if (executionSettings.getFilename() != null) {
66+
options.setFilename(executionSettings.getFilename());
67+
}
68+
69+
if (executionSettings.getLanguage() != null) {
70+
options.setLanguage(executionSettings.getLanguage());
71+
}
72+
73+
if (executionSettings.getPrompt() != null) {
74+
options.setPrompt(executionSettings.getPrompt());
75+
}
76+
77+
if (executionSettings.getTemperature() != null) {
78+
options.setTemperature(executionSettings.getTemperature());
79+
}
80+
return options;
81+
}
82+
83+
/**
84+
* Builder for OpenAiAudioToTextService.
85+
*
86+
* @return The builder.
87+
*/
88+
public static Builder builder() {
89+
return new Builder();
90+
}
91+
92+
/**
93+
* Represents a builder for OpenAiAudioToTextService.
94+
*/
95+
public static class Builder extends AudioToTextService.Builder {
96+
97+
/**
98+
* builds the OpenAiAudioToTextService.
99+
*/
100+
@Override
101+
public AudioToTextService build() {
102+
if (client == null) {
103+
throw new SKException("OpenAI client is required");
104+
}
105+
if (modelId == null) {
106+
throw new SKException("Model id is required");
107+
}
108+
return new OpenAiAudioToTextService(client, modelId);
109+
}
110+
}
111+
}

‎java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
package com.microsoft.semantickernel.aiservices.openai.audio;
3+
4+
import com.azure.ai.openai.OpenAIAsyncClient;
5+
import com.azure.ai.openai.models.SpeechGenerationOptions;
6+
import com.azure.ai.openai.models.SpeechGenerationResponseFormat;
7+
import com.azure.ai.openai.models.SpeechVoice;
8+
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
9+
import com.microsoft.semantickernel.exceptions.SKException;
10+
import com.microsoft.semantickernel.services.audio.AudioContent;
11+
import com.microsoft.semantickernel.services.audio.TextToAudioExecutionSettings;
12+
import com.microsoft.semantickernel.services.audio.TextToAudioService;
13+
import reactor.core.publisher.Mono;
14+
15+
/**
16+
* Provides OpenAi implementation of text to audio service.
17+
*/
18+
public class OpenAiTextToAudioService extends OpenAiService implements TextToAudioService {
19+
20+
/**
21+
* Creates an instance of OpenAi text to audio service.
22+
*
23+
* @param client OpenAI client.
24+
* @param modelId The model ID.
25+
*/
26+
public OpenAiTextToAudioService(
27+
OpenAIAsyncClient client,
28+
String modelId) {
29+
super(client, null, modelId);
30+
}
31+
32+
@Override
33+
public Mono<AudioContent> getAudioContentAsync(
34+
String text,
35+
TextToAudioExecutionSettings executionSettings) {
36+
37+
SpeechGenerationOptions options = convertOptions(text, executionSettings);
38+
39+
return getClient().generateSpeechFromText(getModelId(), options)
40+
.map(response -> new AudioContent(response.toBytes(), getModelId()));
41+
}
42+
43+
private SpeechGenerationOptions convertOptions(
44+
String text,
45+
TextToAudioExecutionSettings executionSettings) {
46+
SpeechGenerationOptions options = new SpeechGenerationOptions(
47+
text,
48+
SpeechVoice.fromString(executionSettings.getVoice()));
49+
50+
options.setModel(getModelId());
51+
52+
if (executionSettings.getResponseFormat() != null) {
53+
options.setResponseFormat(
54+
SpeechGenerationResponseFormat.fromString(executionSettings.getResponseFormat()));
55+
}
56+
57+
if (executionSettings.getSpeed() != null) {
58+
options.setSpeed(executionSettings.getSpeed());
59+
}
60+
61+
return options;
62+
}
63+
64+
/**
65+
* Creates a new builder.
66+
*
67+
* @return The builder.
68+
*/
69+
public static Builder builder() {
70+
return new Builder();
71+
}
72+
73+
/**
74+
* Represents a builder for OpenAi text to audio service.
75+
*/
76+
77+
public static class Builder extends TextToAudioService.Builder {
78+
79+
/**
80+
* Builds the OpenAi text to audio service.
81+
*
82+
* @return The OpenAi text to audio service.
83+
*/
84+
@Override
85+
public TextToAudioService build() {
86+
if (client == null) {
87+
throw new SKException("OpenAI client is required");
88+
}
89+
if (modelId == null) {
90+
throw new SKException("Model id is required");
91+
}
92+
return new OpenAiTextToAudioService(client, modelId);
93+
}
94+
}
95+
}

‎java/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.fasterxml.jackson.databind.ObjectMapper;
2525
import com.fasterxml.jackson.databind.node.ContainerNode;
2626
import com.microsoft.semantickernel.Kernel;
27+
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
2728
import com.microsoft.semantickernel.aiservices.openai.implementation.OpenAIRequestSettings;
2829
import com.microsoft.semantickernel.contextvariables.ContextVariable;
2930
import com.microsoft.semantickernel.contextvariables.ContextVariableTypes;
@@ -60,22 +61,15 @@
6061
/**
6162
* OpenAI chat completion service.
6263
*/
63-
public class OpenAIChatCompletion implements ChatCompletionService {
64+
public class OpenAIChatCompletion extends OpenAiService implements ChatCompletionService {
6465

6566
private static final Logger LOGGER = LoggerFactory.getLogger(OpenAIChatCompletion.class);
66-
private final OpenAIAsyncClient client;
67-
68-
@Nullable
69-
private final String serviceId;
70-
private final String modelId;
7167

7268
protected OpenAIChatCompletion(
7369
OpenAIAsyncClient client,
7470
String modelId,
7571
@Nullable String serviceId) {
76-
this.serviceId = serviceId;
77-
this.client = client;
78-
this.modelId = modelId;
72+
super(client, serviceId, modelId);
7973
}
8074

8175
/**
@@ -87,12 +81,6 @@ public static OpenAIChatCompletion.Builder builder() {
8781
return new OpenAIChatCompletion.Builder();
8882
}
8983

90-
@Override
91-
@Nullable
92-
public String getServiceId() {
93-
return serviceId;
94-
}
95-
9684
@Override
9785
public Mono<List<ChatMessageContent<?>>> getChatMessageContentsAsync(
9886
ChatHistory chatHistory,
@@ -161,7 +149,7 @@ private Mono<List<ChatMessageContent<?>>> internalChatMessageContentsAsync(
161149
autoInvokeAttempts)))
162150
.getOptions();
163151

164-
Mono<List<? extends ChatMessageContent>> result = client
152+
Mono<List<? extends ChatMessageContent>> result = getClient()
165153
.getChatCompletionsWithResponse(getModelId(), options,
166154
OpenAIRequestSettings.getRequestOptions())
167155
.flatMap(completionsResult -> {
@@ -216,6 +204,11 @@ private Mono<List<ChatMessageContent<?>>> internalChatMessageContentsAsync(
216204
})
217205
.flatMap(it -> it)
218206
.flatMap(msgs -> {
207+
ChatRequestMessage m = msgs.get(msgs.size() - 1);
208+
if (m instanceof ChatRequestAssistantMessage &&
209+
((ChatRequestAssistantMessage) m).getToolCalls().size() > 0) {
210+
LOGGER.error("No response");
211+
}
219212
return internalChatMessageContentsAsync(msgs, kernel, functions,
220213
invocationContext, autoInvokeAttempts - 1);
221214
})
@@ -607,31 +600,43 @@ private static ChatRequestMessage getChatRequestMessage(
607600
switch (authorRole) {
608601
case ASSISTANT:
609602
// TODO: break this out into a separate method and handle tools other than function calls
610-
ChatRequestAssistantMessage asstMessage = new ChatRequestAssistantMessage(content);
611-
List<OpenAIFunctionToolCall> toolCalls = ((OpenAIChatMessageContent<?>) message).getToolCall();
603+
ChatRequestAssistantMessage asstMessage = new ChatRequestAssistantMessage(content);
604+
List<OpenAIFunctionToolCall> toolCalls = ((OpenAIChatMessageContent<?>) message)
605+
.getToolCall();
612606
if (toolCalls != null) {
613607
asstMessage.setToolCalls(
614608
toolCalls.stream()
615609
.map(toolCall -> {
616610
KernelFunctionArguments arguments = toolCall.getArguments();
617611
String args = arguments != null && !arguments.isEmpty()
618612
? arguments.entrySet().stream()
619-
.map(entry -> String.format("\"%s\": \"%s\"", entry.getKey(), entry.getValue()))
620-
.collect(Collectors.joining("{","}",","))
613+
.map(entry -> String.format("\"%s\": \"%s\"",
614+
entry.getKey(), entry.getValue()))
615+
.collect(Collectors.joining("{", "}", ","))
621616
: "{}";
622-
FunctionCall fnCall = new FunctionCall(toolCall.getFunctionName(), args);
623-
return new ChatCompletionsFunctionToolCall(toolCall.getId(), fnCall);
617+
FunctionCall fnCall = new FunctionCall(toolCall.getFunctionName(),
618+
args);
619+
return new ChatCompletionsFunctionToolCall(toolCall.getId(),
620+
fnCall);
624621
})
625-
.collect(Collectors.toList())
626-
);
627-
return asstMessage;
622+
.collect(Collectors.toList()));
628623
}
624+
return asstMessage;
629625
case SYSTEM:
630626
return new ChatRequestSystemMessage(content);
631627
case USER:
632628
return new ChatRequestUserMessage(content);
633629
case TOOL:
634-
String id = message.getMetadata().getId();
630+
String id = null;
631+
632+
if (message.getMetadata() != null) {
633+
id = message.getMetadata().getId();
634+
}
635+
636+
if (id == null) {
637+
throw new SKException(
638+
"Require to create a tool call message, but not tool call id is available");
639+
}
635640
return new ChatRequestToolMessage(content, id);
636641
default:
637642
LOGGER.debug("Unexpected author role: " + authorRole);
@@ -641,7 +646,7 @@ private static ChatRequestMessage getChatRequestMessage(
641646
}
642647

643648
static ChatRequestMessage getChatRequestMessage(
644-
AuthorRole authorRole,
649+
AuthorRole authorRole,
645650
String content) {
646651

647652
switch (authorRole) {
@@ -660,12 +665,6 @@ static ChatRequestMessage getChatRequestMessage(
660665

661666
}
662667

663-
@Nullable
664-
@Override
665-
public String getModelId() {
666-
return modelId;
667-
}
668-
669668
/**
670669
* Builder for creating a new instance of {@link OpenAIChatCompletion}.
671670
*/

0 commit comments

Comments
 (0)