Skip to content

Commit ea864bf

Browse files
rojimarkwallace-microsoft
roji
and
markwallace-microsoft
authored
.Net: [MEVD] Support DateTime/DateTimeOffset/DateOnly/TimeOnly across providers (microsoft#13569)
See microsoft#11286 (comment) for some design notes. Closes microsoft#11286 Closes microsoft#11086 --------- Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
1 parent 889807d commit ea864bf

40 files changed

Lines changed: 626 additions & 116 deletions

File tree

‎dotnet/src/InternalUtilities/connectors/Memory/MongoDB/BsonValueFactory.cs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,15 @@ public static BsonValue Create(object? value)
2121
=> value switch
2222
{
2323
null => BsonNull.Value,
24-
Guid guid => new BsonBinaryData(guid, GuidRepresentation.Standard),
25-
object[] array => new BsonArray(Array.ConvertAll(array, Create)),
26-
Array array => new BsonArray(array),
27-
IEnumerable<object> enumerable => new BsonArray(enumerable.Select(Create)),
24+
Guid v => new BsonBinaryData(v, GuidRepresentation.Standard),
25+
DateTimeOffset v => new BsonDateTime(v.UtcDateTime),
26+
#if NET
27+
DateOnly v => new BsonDateTime(v.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc)),
28+
#endif
29+
object[] v => new BsonArray(Array.ConvertAll(v, Create)),
30+
Array v => new BsonArray(v),
31+
IEnumerable<object> v => new BsonArray(v.Select(Create)),
32+
2833
_ => BsonValue.Create(value)
2934
};
3035
}

‎dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,14 +160,24 @@ Embedding<float> e
160160
Type t when t == typeof(int?) => value.AsNullableInt32,
161161
Type t when t == typeof(long) => value.AsInt64,
162162
Type t when t == typeof(long?) => value.AsNullableInt64,
163-
Type t when t == typeof(float) => ((float)value.AsDouble),
163+
Type t when t == typeof(float) => (float)value.AsDouble,
164164
Type t when t == typeof(float?) => ((float?)value.AsNullableDouble),
165165
Type t when t == typeof(double) => value.AsDouble,
166166
Type t when t == typeof(double?) => value.AsNullableDouble,
167167
Type t when t == typeof(decimal) => value.AsDecimal,
168168
Type t when t == typeof(decimal?) => value.AsNullableDecimal,
169169
Type t when t == typeof(DateTime) => value.ToUniversalTime(),
170170
Type t when t == typeof(DateTime?) => value.ToNullableUniversalTime(),
171+
Type t when t == typeof(DateTimeOffset) => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
172+
Type t when t == typeof(DateTimeOffset?) => value.ToNullableUniversalTime() is DateTime dateTime
173+
? new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero)
174+
: null,
175+
#if NET
176+
Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.ToUniversalTime()),
177+
Type t when t == typeof(DateOnly?) => value.ToNullableUniversalTime() is DateTime dateTime
178+
? DateOnly.FromDateTime(dateTime)
179+
: null,
180+
#endif
171181

172182
_ => (object?)null
173183
};

‎dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty)
5757

5858
protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
5959
{
60-
supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types";
60+
supportedTypes = "string, int, long, double, float, bool, decimal, DateTime, DateTimeOffset,"
61+
#if NET
62+
+ " DateOnly,"
63+
#endif
64+
+ " or arrays/lists of these types";
6165

6266
if (Nullable.GetUnderlyingType(type) is Type underlyingType)
6367
{
@@ -76,7 +80,12 @@ static bool IsValid(Type type)
7680
type == typeof(float) ||
7781
type == typeof(double) ||
7882
type == typeof(decimal) ||
79-
type == typeof(DateTime);
83+
type == typeof(DateTime) ||
84+
type == typeof(DateTimeOffset) ||
85+
#if NET
86+
type == typeof(DateOnly) ||
87+
#endif
88+
false;
8089
}
8190

8291
protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)

‎dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,11 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType)
128128
// Half is also listed by the SDK, but currently not supported.
129129
Type t when t == typeof(float) => SearchFieldDataType.Double,
130130
Type t when t == typeof(double) => SearchFieldDataType.Double,
131+
Type t when t == typeof(DateTime) => SearchFieldDataType.DateTimeOffset,
131132
Type t when t == typeof(DateTimeOffset) => SearchFieldDataType.DateTimeOffset,
133+
#if NET
134+
Type t when t == typeof(DateOnly) => SearchFieldDataType.DateTimeOffset,
135+
#endif
132136

133137
Type t when t == typeof(string[]) => SearchFieldDataType.Collection(SearchFieldDataType.String),
134138
Type t when t == typeof(List<string>) => SearchFieldDataType.Collection(SearchFieldDataType.String),
@@ -142,8 +146,14 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType)
142146
Type t when t == typeof(List<float>) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
143147
Type t when t == typeof(double[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
144148
Type t when t == typeof(List<double>) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
149+
Type t when t == typeof(DateTime[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
150+
Type t when t == typeof(List<DateTime>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
145151
Type t when t == typeof(DateTimeOffset[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
146152
Type t when t == typeof(List<DateTimeOffset>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
153+
#if NET
154+
Type t when t == typeof(DateOnly[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
155+
Type t when t == typeof(List<DateOnly>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
156+
#endif
147157

148158
_ => throw new NotSupportedException($"Data type '{propertyType}' for {nameof(VectorStoreDataProperty)} is not supported by the Azure AI Search VectorStore.")
149159
};

‎dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public JsonObject MapFromDataToStorageModel(Dictionary<string, object?> dataMode
4141
if (dataModel.TryGetValue(dataProperty.ModelName, out var dataValue))
4242
{
4343
jsonObject[dataProperty.StorageName] = dataValue is not null ?
44-
JsonSerializer.SerializeToNode(dataValue, dataProperty.Type, jsonSerializerOptions) :
44+
JsonSerializer.SerializeToNode(ConvertToStorageValue(dataValue), GetStorageType(dataProperty.Type), jsonSerializerOptions) :
4545
null;
4646
}
4747
}
@@ -183,6 +183,10 @@ public JsonObject MapFromDataToStorageModel(Dictionary<string, object?> dataMode
183183
Type t when t == typeof(DateTime?) => value.GetValue<DateTime?>(),
184184
Type t when t == typeof(DateTimeOffset) => value.GetValue<DateTimeOffset>(),
185185
Type t when t == typeof(DateTimeOffset?) => value.GetValue<DateTimeOffset?>(),
186+
#if NET
187+
Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.GetValue<DateTimeOffset>().DateTime),
188+
Type t when t == typeof(DateOnly?) => (DateOnly?)DateOnly.FromDateTime(value.GetValue<DateTimeOffset>().DateTime),
189+
#endif
186190

187191
_ => (object?)null
188192
};
@@ -242,4 +246,38 @@ public JsonObject MapFromDataToStorageModel(Dictionary<string, object?> dataMode
242246

243247
throw new UnreachableException($"Unsupported property type '{propertyType.Name}'.");
244248
}
249+
250+
/// <summary>
251+
/// Converts a value to its storage representation for Azure AI Search.
252+
/// Azure AI Search only supports <see cref="DateTimeOffset"/> for date/time types, and always internally
253+
/// converts to UTC for storage.
254+
/// </summary>
255+
/// <summary>
256+
/// Gets the storage type for a given property type. Azure AI Search stores DateTime and DateOnly as DateTimeOffset.
257+
/// </summary>
258+
private static Type GetStorageType(Type propertyType)
259+
{
260+
var underlying = Nullable.GetUnderlyingType(propertyType) ?? propertyType;
261+
262+
if (underlying == typeof(DateTime)
263+
#if NET
264+
|| underlying == typeof(DateOnly)
265+
#endif
266+
)
267+
{
268+
return propertyType == underlying ? typeof(DateTimeOffset) : typeof(DateTimeOffset?);
269+
}
270+
271+
return propertyType;
272+
}
273+
274+
private static object ConvertToStorageValue(object value)
275+
=> value switch
276+
{
277+
DateTime dateTime => new DateTimeOffset(dateTime, TimeSpan.Zero),
278+
#if NET
279+
DateOnly dateOnly => new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero),
280+
#endif
281+
_ => value
282+
};
245283
}

‎dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,17 @@ private void GenerateLiteral(object? value)
119119
this._filter.Append('\'').Append(g.ToString()).Append('\'');
120120
return;
121121

122+
case DateTime d:
123+
this._filter.Append(new DateTimeOffset(d, TimeSpan.Zero).ToString("o"));
124+
return;
122125
case DateTimeOffset d:
123126
this._filter.Append(d.ToString("o"));
124127
return;
128+
#if NET
129+
case DateOnly d:
130+
this._filter.Append(new DateTimeOffset(d.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero).ToString("o"));
131+
return;
132+
#endif
125133

126134
case Array:
127135
throw new NotImplementedException();

‎dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,29 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex,
2020

2121
var jsonObject = JsonSerializer.SerializeToNode(dataModel, jsonSerializerOptions)!.AsObject();
2222

23+
// Azure AI Search only supports Edm.DateTimeOffset for date/time types.
24+
// DateTime properties may be serialized by STJ without timezone info (when Kind=Unspecified),
25+
// which Azure AI Search rejects. Convert them to DateTimeOffset (UTC).
26+
// DateOnly properties are serialized as "yyyy-MM-dd", which also isn't accepted.
27+
// Convert them to full DateTimeOffset representation (midnight UTC).
28+
foreach (var dataProperty in model.DataProperties)
29+
{
30+
var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type;
31+
32+
if (propertyType == typeof(DateTime) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null)
33+
{
34+
var dateTime = dtNode.Deserialize<DateTime>(jsonSerializerOptions);
35+
jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(DateTime.SpecifyKind(dateTime, DateTimeKind.Utc), TimeSpan.Zero));
36+
}
37+
#if NET
38+
else if (propertyType == typeof(DateOnly) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null)
39+
{
40+
var dateOnly = dateNode.Deserialize<DateOnly>(jsonSerializerOptions);
41+
jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero));
42+
}
43+
#endif
44+
}
45+
2346
// Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above.
2447
// Also, for Embedding<T> properties we also need to overwrite with a simple array (since Embedding<T> gets serialized as a complex object).
2548
for (var i = 0; i < model.VectorProperties.Count; i++)
@@ -64,6 +87,27 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex,
6487

6588
public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors)
6689
{
90+
// Azure AI Search stores DateTime properties as Edm.DateTimeOffset.
91+
// Convert them back to DateTime so STJ can deserialize them into the correct type.
92+
// DateOnly properties also need to be converted back from DateTimeOffset.
93+
foreach (var dataProperty in model.DataProperties)
94+
{
95+
var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type;
96+
97+
if (propertyType == typeof(DateTime) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null)
98+
{
99+
var dateTimeOffset = dtNode.Deserialize<DateTimeOffset>(jsonSerializerOptions);
100+
storageModel[dataProperty.StorageName] = JsonValue.Create(dateTimeOffset.UtcDateTime);
101+
}
102+
#if NET
103+
else if (propertyType == typeof(DateOnly) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null)
104+
{
105+
var dateTimeOffset = dateNode.Deserialize<DateTimeOffset>(jsonSerializerOptions);
106+
storageModel[dataProperty.StorageName] = JsonValue.Create(DateOnly.FromDateTime(dateTimeOffset.DateTime));
107+
}
108+
#endif
109+
}
110+
67111
if (includeVectors)
68112
{
69113
foreach (var vectorProperty in model.VectorProperties)

‎dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,11 @@ protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)
3939

4040
internal static bool IsDataPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes)
4141
{
42-
supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types";
42+
supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset,"
43+
#if NET
44+
+ " DateOnly,"
45+
#endif
46+
+ " or arrays/lists of these types";
4347

4448
if (Nullable.GetUnderlyingType(type) is Type underlyingType)
4549
{
@@ -57,6 +61,10 @@ static bool IsValid(Type type)
5761
type == typeof(double) ||
5862
type == typeof(float) ||
5963
type == typeof(bool) ||
64+
type == typeof(DateTime) ||
65+
#if NET
66+
type == typeof(DateOnly) ||
67+
#endif
6068
type == typeof(DateTimeOffset);
6169
}
6270

‎dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,22 @@ private void TranslateConstant(object? value)
136136
.Append("Z\"");
137137
return;
138138

139+
case DateTime v:
140+
this._sql
141+
.Append('"')
142+
.Append(v.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture))
143+
.Append('"');
144+
return;
145+
146+
#if NET
147+
case DateOnly v:
148+
this._sql
149+
.Append('"')
150+
.Append(v.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture))
151+
.Append('"');
152+
return;
153+
#endif
154+
139155
case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)):
140156
this._sql.Append('[');
141157

‎dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlModelBuilder.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty)
3535

3636
protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
3737
{
38-
supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types";
38+
supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset,"
39+
#if NET
40+
+ " DateOnly,"
41+
#endif
42+
+ " or arrays/lists of these types";
3943

4044
if (Nullable.GetUnderlyingType(type) is Type underlyingType)
4145
{
@@ -53,6 +57,10 @@ static bool IsValid(Type type)
5357
type == typeof(long) ||
5458
type == typeof(float) ||
5559
type == typeof(double) ||
60+
type == typeof(DateTime) ||
61+
#if NET
62+
type == typeof(DateOnly) ||
63+
#endif
5664
type == typeof(DateTimeOffset);
5765
}
5866

0 commit comments

Comments
 (0)