Skip to content

Commit b7c4d1e

Browse files
lemirelemire
authored
Adding test for issue 1729. (simdjson#1730)
* Adding test for issue 1729. * Adding comment. * Trying to move to 11.7. * Tweaking. * More tweaking. * Adding additional test. * Missing "<<". * Minor update. * Removing legacy systems.
1 parent 4a2f10e commit b7c4d1e

2 files changed

Lines changed: 183 additions & 43 deletions

File tree

‎.drone.yml‎

Lines changed: 0 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,4 @@
11
kind: pipeline
2-
name: i386-gcc # we do not support 32-bit systems, but we run tests
3-
platform: { os: linux, arch: amd64 }
4-
steps:
5-
- name: Build and Test
6-
image: i386/ubuntu
7-
environment:
8-
CC: gcc
9-
CXX: g++
10-
BUILD_FLAGS: -- -j
11-
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
12-
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
13-
commands:
14-
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
15-
- apt-get install -y g++ cmake gcc git
16-
- mkdir build
17-
- cd build
18-
- cmake $CMAKE_FLAGS ..
19-
- cmake --build . $BUILD_FLAGS
20-
- ctest $CTEST_FLAGS
21-
---
22-
kind: pipeline
23-
name: i386-clang # we do not support 32-bit systems, but we run tests
24-
platform: { os: linux, arch: amd64 }
25-
steps:
26-
- name: Build and Test
27-
image: i386/ubuntu
28-
environment:
29-
CC: clang-6.0
30-
CXX: clang++-6.0
31-
BUILD_FLAGS: -- -j
32-
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
33-
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
34-
commands:
35-
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
36-
- apt-get install -y clang++-6.0 cmake git
37-
- mkdir build
38-
- cd build
39-
- cmake $CMAKE_FLAGS ..
40-
- cmake --build . $BUILD_FLAGS
41-
- ctest $CTEST_FLAGS
42-
---
43-
kind: pipeline
442
name: gcc9
453
platform: { os: linux, arch: amd64 }
464
steps:

‎tests/ondemand/ondemand_document_stream_tests.cpp‎

Lines changed: 183 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,190 @@ namespace document_stream_tests {
99
bool process_doc(T &docref) {
1010
int64_t val;
1111
ASSERT_SUCCESS(docref.at_pointer("/4").get(val));
12-
//ASSERT_SUCCESS(err);
1312
ASSERT_EQUAL(val, 5);
1413
return true;
1514
}
15+
bool truncated_utf8() {
16+
TEST_START();
17+
// truncated UTF-8
18+
auto json = "\"document1\" \xC3"_padded;
19+
20+
ondemand::parser parser;
21+
ondemand::document_stream stream;
22+
auto oderror = parser.iterate_many(json).get(stream);
23+
if (oderror) {
24+
std::cerr << "ondemand iterate_many error: " << oderror << std::endl;
25+
return false;
26+
}
27+
int document_index = 0;
28+
auto i = stream.begin();
29+
for (; i != stream.end(); ++i) {
30+
ondemand::document_reference doc;
31+
auto err = (*i).get(doc);
32+
document_index++;
33+
// With the fallback routine, we might detect a 'document' since
34+
// it does not do UTF-8 validation in stage one, but it will do
35+
// so when we access the document.
36+
if(err == SUCCESS) {
37+
ondemand::json_type t;
38+
err = doc.type().get(t);
39+
}
40+
if((err == SUCCESS) && (document_index != 1)) {
41+
std::cerr << "Only the first document should be valid." << std::endl;
42+
return false;
43+
}
44+
if((err != SUCCESS) && (document_index != 2)) {
45+
std::cerr << "ondemand iterate_many error: " << err << std::endl;
46+
return false;
47+
}
48+
}
49+
if(document_index < 1) {
50+
std::cerr << "ondemand should have accessed at least one document" << std::endl;
51+
return false;
52+
}
53+
TEST_SUCCEED();
54+
}
55+
bool issue1729() {
56+
// This example is not a good application of iterate_many since there is
57+
// a single JSON document.
58+
TEST_START();
59+
auto json = R"({
60+
"t": 5649,
61+
"ng": 5,
62+
"lu": 4086,
63+
"g": [{
64+
"t": "Temps",
65+
"xvy": 0,
66+
"pd": 60,
67+
"sz": 3,
68+
"l": [
69+
"Low Temp",
70+
"High Temp",
71+
"Smooth Avg"
72+
],
73+
"c": [
74+
"green",
75+
"orange",
76+
"cyan"
77+
],
78+
"d": [
79+
80.48750305,
80+
80.82499694,
81+
80.65625
82+
]
83+
},
84+
{
85+
"t": "Pump Status",
86+
"xvy": 0,
87+
"pd": 60,
88+
"sz": 3,
89+
"l": [
90+
"Pump",
91+
"Thermal",
92+
"Light"
93+
],
94+
"c": [
95+
"green",
96+
"orange",
97+
"cyan"
98+
],
99+
"d": [
100+
0,
101+
0,
102+
0
103+
]
104+
},
105+
{
106+
"t": "Lux",
107+
"xvy": 0,
108+
"pd": 60,
109+
"sz": 4,
110+
"l": [
111+
"Value",
112+
"Smooth",
113+
"Low",
114+
"High"
115+
],
116+
"c": [
117+
"green",
118+
"orange",
119+
"cyan",
120+
"yellow"
121+
],
122+
"d": [
123+
2274.62939453,
124+
2277.45947265,
125+
4050,
126+
4500
127+
]
128+
},
129+
{
130+
"t": "Temp Diff",
131+
"xvy": 0,
132+
"pd": 60,
133+
"sz": 4,
134+
"l": [
135+
"dFarenheit",
136+
"sum",
137+
"Low",
138+
"High"
139+
],
140+
"c": [
141+
"green",
142+
"orange",
143+
"cyan",
144+
"yellow"
145+
],
146+
"d": [
147+
0,
148+
0,
149+
0.5,
150+
10
151+
]
152+
},
153+
{
154+
"t": "Power",
155+
"xvy": 0,
156+
"pd": 60,
157+
"sz": 3,
158+
"l": [
159+
"watts (est. ligth) ",
160+
"watts (1.9~gpm) ",
161+
"watts (1gpm) "
162+
],
163+
"c": [
164+
"green",
165+
"orange",
166+
"cyan"
167+
],
168+
"d": [
169+
181.78063964,
170+
114.88922882,
171+
59.35943603
172+
]
173+
}
174+
]
175+
})"_padded;
176+
177+
ondemand::parser parser;
178+
ondemand::document_stream stream;
179+
auto oderror = parser.iterate_many(json).get(stream);
180+
if (oderror) {
181+
std::cerr << "ondemand iterate_many error: " << oderror << std::endl;
182+
return false;
183+
}
184+
auto i = stream.begin();
185+
for (; i != stream.end(); ++i) {
186+
ondemand::document_reference doc;
187+
auto err = (*i).get(doc);
188+
if(err != SUCCESS) {
189+
std::cerr << "ondemand iterate_many error: " << err << std::endl;
190+
return false;
191+
}
192+
}
193+
TEST_SUCCEED();
194+
}
195+
16196

17197
bool issue1683() {
18198
TEST_START();
@@ -536,6 +716,8 @@ namespace document_stream_tests {
536716

537717
bool run() {
538718
return
719+
truncated_utf8() &&
720+
issue1729() &&
539721
fuzzaccess() &&
540722
issue1683() &&
541723
issue1668() &&

0 commit comments

Comments
 (0)