Skip to content

Commit a6e4933

Browse files
author
Daniel Lemire
committed
Exposing the string minifier.
1 parent b4837f2 commit a6e4933

7 files changed

Lines changed: 128 additions & 5 deletions

File tree

‎benchmark/minifiercompetition.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,7 @@ int main(int argc, char *argv[]) {
126126

127127
char *mini_buffer = simdjson::internal::allocate_padded_buffer(p.size() + 1);
128128
size_t minisize;
129-
auto minierror = simdjson::active_implementation->minify((const uint8_t *)p.data(), p.size(),
130-
(uint8_t *)mini_buffer, minisize);
129+
auto minierror = minify_string(p.data(), p.size(),mini_buffer, minisize);
131130
if (!minierror) { std::cerr << minierror << std::endl; exit(1); }
132131
mini_buffer[minisize] = '\0';
133132

‎doc/basics.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,24 @@ And another one:
168168
cout << "number: " << v << endl;
169169
```
170170

171+
Minifying JSON strings without parsing
172+
----------------------
173+
174+
In some cases, you may have valid JSON strings that you do not wish to parse but that you wish to minify. That is, you wish to remove all unnecessary spaces. We have a fast function for this purpose (`minify_string`). This function does not validate your content, and it does not parse it. Instead, it assumes that your string is valid UTF-8. It is much faster than parsing the string and re-serializing it in minified form. Usage is relatively simple. You must pass an input pointer with a length parameter, as well as an output pointer and an output length parameter (by reference). The output length parameter is not read, but written to. The output pointer should point to a valid memory region that is slightly overallocated (by `simdjson::SIMDJSON_PADDING`) compared to the original string length. The input pointer and input length are read, but not written to.
175+
176+
```C++
177+
// Starts with a valid JSON document as a string.
178+
// It does not have to be null-terminated.
179+
const char * some_string = "[ 1, 2, 3, 4] ";
180+
size_t length = strlen(some_string);
181+
// Create a buffer to receive the minified string. Make sure that there is enough room,
182+
// including some padding (simdjson::SIMDJSON_PADDING).
183+
std::unique_ptr<char> buffer{new(std::nothrow) char[length + simdjson::SIMDJSON_PADDING]};
184+
size_t new_length{}; // It will receive the minified length.
185+
auto error = simdjson::minify_string(some_string, length, buffer.get(), new_length);
186+
// The buffer variable now has "[1,2,3,4]" and new_length has value 9.
187+
```
188+
171189
172190
C++17 Support
173191
-------------

‎include/simdjson/implementation.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,11 @@ class implementation {
7272
/**
7373
* @private For internal implementation use
7474
*
75-
* Run a full document parse (ensure_capacity, stage1 and stage2).
75+
* Minify the input string assuming that it represents a JSON string, does not parse or validate.
7676
*
7777
* Overridden by each implementation.
7878
*
79-
* @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes.
79+
* @param buf the json document to minify.
8080
* @param len the length of the json document.
8181
* @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes.
8282
* @param dst_len the number of bytes written. Output only.

‎include/simdjson/minify.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,24 @@
99

1010
namespace simdjson {
1111

12+
13+
14+
/**
15+
*
16+
* Minify the input string assuming that it represents a JSON string, does not parse or validate.
17+
* This function is much faster than parsing a JSON string and then writing a minified version of it.
18+
* However, it does not validate the input.
19+
*
20+
*
21+
* @param buf the json document to minify.
22+
* @param len the length of the json document.
23+
* @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes.
24+
* @param dst_len the number of bytes written. Output only.
25+
* @return the error code, or SUCCESS if there was no error.
26+
*/
27+
WARN_UNUSED error_code minify_string(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept;
28+
29+
1230
/**
1331
* Minifies a JSON element or document, printing the smallest possible valid JSON.
1432
*

‎src/implementation.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,4 +134,9 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes
134134
SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{};
135135
SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton};
136136

137+
WARN_UNUSED error_code minify_string(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept {
138+
return active_implementation->minify((const uint8_t *)buf, len, (uint8_t *)dst, dst_len);
139+
}
140+
141+
137142
} // namespace simdjson

‎tests/basictests.cpp‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1712,6 +1712,62 @@ namespace type_tests {
17121712

17131713
}
17141714

1715+
1716+
1717+
namespace minify_string_tests {
1718+
1719+
bool check_minification(const char * input, size_t length, const char * expected, size_t expected_length) {
1720+
std::unique_ptr<char> buffer{new(std::nothrow) char[length + simdjson::SIMDJSON_PADDING]};
1721+
if(buffer.get() == nullptr) {
1722+
std::cerr << "cannot alloc " << std::endl;
1723+
return false;
1724+
}
1725+
size_t newlength{};
1726+
auto error = simdjson::minify_string(input, length, buffer.get(), newlength);
1727+
if(error != simdjson::SUCCESS) {
1728+
std::cerr << "error " << error << std::endl;
1729+
return false;
1730+
}
1731+
// memcmp
1732+
if(newlength != expected_length) {
1733+
std::cerr << "lengths do not match " << std::endl;
1734+
return false;
1735+
}
1736+
for(size_t i = 0; i < newlength; i++) {
1737+
if(buffer.get()[i] != expected[i]) {
1738+
std::cerr << "Inputs do not match (but same length) " << std::endl;
1739+
return false;
1740+
}
1741+
}
1742+
return true;
1743+
}
1744+
1745+
bool test_minify_string() {
1746+
std::cout << "Running " << __func__ << std::endl;
1747+
const std::string test = R"({ "foo" : 1, "bar" : [ 1, 2, 3 ], "baz": { "a": 1, "b": 2, "c": 3 } })";
1748+
const std::string minified(R"({"foo":1,"bar":[1,2,3],"baz":{"a":1,"b":2,"c":3}})");
1749+
return check_minification(test.c_str(), test.size(), minified.c_str(), minified.size());
1750+
}
1751+
bool test_minify_string_array() {
1752+
std::cout << "Running " << __func__ << std::endl;
1753+
std::string test("[ 1, 2, 3]");
1754+
std::string minified("[1,2,3]");
1755+
return check_minification(test.c_str(), test.size(), minified.c_str(), minified.size());
1756+
}
1757+
bool test_minify_string_object() {
1758+
std::cout << "Running " << __func__ << std::endl;
1759+
std::string test(R"({ "foo " : 1, "b ar" : [ 1, 2, 3 ], "baz": { "a": 1, "b": 2, "c": 3 } })");
1760+
std::string minified(R"({"foo ":1,"b ar":[1,2,3],"baz":{"a":1,"b":2,"c":3}})");
1761+
return check_minification(test.c_str(), test.size(), minified.c_str(), minified.size());
1762+
}
1763+
bool run() {
1764+
return test_minify_string() &&
1765+
test_minify_string_array() &&
1766+
test_minify_string_object();
1767+
}
1768+
}
1769+
1770+
17151771
namespace format_tests {
17161772
using namespace simdjson;
17171773
using namespace simdjson::dom;
@@ -1968,7 +2024,8 @@ int main(int argc, char *argv[]) {
19682024
printf("unsupported CPU\n");
19692025
}
19702026
std::cout << "Running basic tests." << std::endl;
1971-
if (parse_api_tests::run() &&
2027+
if (minify_string_tests::run() &&
2028+
parse_api_tests::run() &&
19722029
dom_api_tests::run() &&
19732030
type_tests::run() &&
19742031
format_tests::run() &&

‎tests/readme_examples.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,10 +239,36 @@ void performance_3() {
239239
SIMDJSON_POP_DISABLE_WARNINGS
240240
#endif
241241

242+
void minify_string() {
243+
const char * some_string = "[ 1, 2, 3, 4] ";
244+
size_t length = strlen(some_string);
245+
std::unique_ptr<char> buffer{new(std::nothrow) char[length + simdjson::SIMDJSON_PADDING]};
246+
size_t new_length{};
247+
auto error = simdjson::minify_string(some_string, length, buffer.get(), new_length);
248+
if(error != simdjson::SUCCESS) {
249+
std::cerr << "error " << error << std::endl;
250+
abort();
251+
} else {
252+
const char * expected_string = "[1,2,3,4]";
253+
size_t expected_length = strlen(expected_string);
254+
if(expected_length != new_length) {
255+
std::cerr << "mismatched length (error) " << std::endl;
256+
abort();
257+
}
258+
for(size_t i = 0; i < new_length; i++) {
259+
if(expected_string[i] != buffer.get()[i]) {
260+
std::cerr << "mismatched content (error) " << std::endl;
261+
abort();
262+
}
263+
}
264+
}
265+
}
266+
242267
int main() {
243268
basics_dom_1();
244269
basics_dom_2();
245270
basics_dom_3();
246271
basics_dom_4();
272+
minify_string();
247273
return 0;
248274
}

0 commit comments

Comments
 (0)