Skip to content

Commit bf4089b

Browse files
lemirelemire
committed
Removing custom types (more standard code).
1 parent bb83ad4 commit bf4089b

20 files changed

Lines changed: 284 additions & 309 deletions

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ parsehisto: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
145145
cppcheck:
146146
cppcheck --enable=all src/*.cpp benchmarks/*.cpp tests/*.cpp -Iinclude -I. -Ibenchmark/linux
147147

148+
everything: $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)
148149

149150
clean:
150151
rm -f $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)

‎benchmark/minifiercompetition.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ int main(int argc, char *argv[]) {
140140
printf("failed to allocate memory\n");
141141
return EXIT_FAILURE;
142142
}
143-
BEST_TIME("simdjson orig", json_parse((const u8*)buffer, p.size(), pj), true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
143+
BEST_TIME("simdjson orig", json_parse((const uint8_t*)buffer, p.size(), pj), true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
144144

145145
ParsedJson pj2;
146146
bool isallocok2 = pj2.allocateCapacity(p.size(), 1024);
@@ -149,7 +149,7 @@ int main(int argc, char *argv[]) {
149149
return EXIT_FAILURE;
150150
}
151151

152-
BEST_TIME("simdjson despaced", json_parse((const u8*)buffer, minisize, pj2), true, memcpy(buffer, minibuffer, p.size()), repeat, volume, !justdata);
152+
BEST_TIME("simdjson despaced", json_parse((const uint8_t*)buffer, minisize, pj2), true, memcpy(buffer, minibuffer, p.size()), repeat, volume, !justdata);
153153
free((void*)p.data());
154154
free(buffer);
155155
free(ast_buffer);

‎benchmark/parse.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,9 @@ int main(int argc, char *argv[]) {
8585
cout << "[verbose] loaded " << filename << " (" << p.size() << " bytes)"
8686
<< endl;
8787
#if defined(DEBUG)
88-
const u32 iterations = 1;
88+
const uint32_t iterations = 1;
8989
#else
90-
const u32 iterations =
90+
const uint32_t iterations =
9191
forceoneiteration ? 1 : (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
9292
#endif
9393
vector<double> res;
@@ -108,7 +108,7 @@ int main(int argc, char *argv[]) {
108108
evts.push_back(PERF_COUNT_HW_CACHE_REFERENCES);
109109
evts.push_back(PERF_COUNT_HW_CACHE_MISSES);
110110
LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
111-
vector<u64> results;
111+
vector<uint64_t> results;
112112
results.resize(evts.size());
113113
unsigned long cy0 = 0, cy1 = 0, cy2 = 0, cy3 = 0;
114114
unsigned long cl0 = 0, cl1 = 0, cl2 = 0, cl3 = 0;
@@ -118,7 +118,7 @@ int main(int argc, char *argv[]) {
118118
#endif
119119
bool isok = true;
120120

121-
for (u32 i = 0; i < iterations; i++) {
121+
for (uint32_t i = 0; i < iterations; i++) {
122122
if (verbose)
123123
cout << "[verbose] iteration # " << i << endl;
124124
#ifndef SQUASH_COUNTERS

‎benchmark/parseandstatcompetition.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,14 @@ stat_t simdjson_computestats(const std::string_view &p) {
5959
answer.true_count = 0;
6060
answer.false_count = 0;
6161
size_t tapeidx = 0;
62-
u64 tape_val = pj.tape[tapeidx++];
63-
u8 type = (tape_val >> 56);
62+
uint64_t tape_val = pj.tape[tapeidx++];
63+
uint8_t type = (tape_val >> 56);
6464
size_t howmany = 0;
6565
assert(type == 'r');
6666
howmany = tape_val & JSONVALUEMASK;
6767
for (; tapeidx < howmany; tapeidx++) {
6868
tape_val = pj.tape[tapeidx];
69-
// u64 payload = tape_val & JSONVALUEMASK;
69+
// uint64_t payload = tape_val & JSONVALUEMASK;
7070
type = (tape_val >> 56);
7171
switch (type) {
7272
case 'l': // we have a long int

‎benchmark/statisticalmodel.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,15 @@
99

1010
using namespace std;
1111

12-
size_t count_nonasciibytes(const u8 *input, size_t length) {
12+
size_t count_nonasciibytes(const uint8_t *input, size_t length) {
1313
size_t count = 0;
1414
for (size_t i = 0; i < length; i++) {
1515
count += input[i] >> 7;
1616
}
1717
return count;
1818
}
1919

20-
size_t count_backslash(const u8 *input, size_t length) {
20+
size_t count_backslash(const uint8_t *input, size_t length) {
2121
size_t count = 0;
2222
for (size_t i = 0; i < length; i++) {
2323
count += (input[i] == '\\') ? 1 : 0;
@@ -50,9 +50,9 @@ stat_t simdjson_computestats(const std::string_view &p) {
5050
if (!answer.valid) {
5151
return answer;
5252
}
53-
answer.backslash_count = count_backslash((const u8 *)p.data(), p.size());
53+
answer.backslash_count = count_backslash((const uint8_t *)p.data(), p.size());
5454
answer.nonasciibyte_count =
55-
count_nonasciibytes((const u8 *)p.data(), p.size());
55+
count_nonasciibytes((const uint8_t *)p.data(), p.size());
5656
answer.byte_count = p.size();
5757
answer.integer_count = 0;
5858
answer.float_count = 0;
@@ -64,14 +64,14 @@ stat_t simdjson_computestats(const std::string_view &p) {
6464
answer.string_count = 0;
6565
answer.structural_indexes_count = pj.n_structural_indexes;
6666
size_t tapeidx = 0;
67-
u64 tape_val = pj.tape[tapeidx++];
68-
u8 type = (tape_val >> 56);
67+
uint64_t tape_val = pj.tape[tapeidx++];
68+
uint8_t type = (tape_val >> 56);
6969
size_t howmany = 0;
7070
assert(type == 'r');
7171
howmany = tape_val & JSONVALUEMASK;
7272
for (; tapeidx < howmany; tapeidx++) {
7373
tape_val = pj.tape[tapeidx];
74-
// u64 payload = tape_val & JSONVALUEMASK;
74+
// uint64_t payload = tape_val & JSONVALUEMASK;
7575
type = (tape_val >> 56);
7676
switch (type) {
7777
case 'l': // we have a long int
@@ -173,7 +173,7 @@ int main(int argc, char *argv[]) {
173173
LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
174174
unsigned long cy1 = 0, cy2 = 0, cy3 = 0;
175175
unsigned long cl1 = 0, cl2 = 0, cl3 = 0;
176-
vector<u64> results;
176+
vector<uint64_t> results;
177177
results.resize(evts.size());
178178
for (u32 i = 0; i < iterations; i++) {
179179
unified.start();

‎include/simdjson/common_defs.h‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,7 @@
66
#define SIMDJSON_PADDING sizeof(__m256i)
77

88

9-
typedef unsigned char u8;
10-
typedef unsigned short u16;
11-
typedef unsigned int u32;
12-
typedef unsigned long long u64;
13-
typedef signed char s8;
14-
typedef signed short s16;
15-
typedef signed int s32;
16-
typedef signed long long s64;
9+
1710

1811
#ifdef _MSC_VER
1912
/* Microsoft C/C++-compatible compiler */
@@ -22,8 +15,6 @@ typedef signed long long s64;
2215
#include <x86intrin.h>
2316
#endif
2417

25-
typedef __m128i m128;
26-
typedef __m256i m256;
2718

2819
// Align to N-byte boundary
2920
#define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
@@ -44,20 +35,3 @@ typedef __m256i m256;
4435
#define unlikely(x) __builtin_expect(!!(x), 0)
4536
#endif
4637

47-
/*static inline u32 ctz64(u64 x) {
48-
assert(x); // behaviour not defined for x == 0
49-
#if defined(_WIN64)
50-
unsigned long r;
51-
_BitScanForward64(&r, x);
52-
return r;
53-
#elif defined(_WIN32)
54-
unsigned long r;
55-
if (_BitScanForward(&r, (u32)x)) {
56-
return (u32)r;
57-
}
58-
_BitScanForward(&r, x >> 32);
59-
return (u32)(r + 32);
60-
#else
61-
return (u32)__builtin_ctzll(x);
62-
#endif
63-
}*/

‎include/simdjson/jsoncharutils.h‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
// these are the chars that can follow a true/false/null or number atom
1212
// and nothing else
13-
const u32 structural_or_whitespace_negated[256] = {
13+
const uint32_t structural_or_whitespace_negated[256] = {
1414
0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,
1515
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1616
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
@@ -29,11 +29,11 @@ const u32 structural_or_whitespace_negated[256] = {
2929

3030
// return non-zero if not a structural or whitespace char
3131
// zero otherwise
32-
really_inline u32 is_not_structural_or_whitespace(u8 c) {
32+
really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) {
3333
return structural_or_whitespace_negated[c];
3434
}
3535

36-
const u32 structural_or_whitespace[256] = {
36+
const uint32_t structural_or_whitespace[256] = {
3737
1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3838
0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
3939
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -46,7 +46,7 @@ const u32 structural_or_whitespace[256] = {
4646
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4747
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
4848

49-
really_inline u32 is_structural_or_whitespace(u8 c) {
49+
really_inline uint32_t is_structural_or_whitespace(uint8_t c) {
5050
return structural_or_whitespace[c];
5151
}
5252

@@ -67,22 +67,22 @@ const char digittoval[256] = {
6767
-1, -1, -1, -1, -1, -1, -1, -1, -1};
6868

6969
// return true if we have a valid hex between 0000 and FFFF
70-
/*inline bool hex_to_u32(const u8 *src, u32 *res) {
71-
u8 v1 = src[0];
72-
u8 v2 = src[1];
73-
u8 v3 = src[2];
74-
u8 v4 = src[3];
70+
/*inline bool hex_to_u32(const uint8_t *src, uint32_t *res) {
71+
uint8_t v1 = src[0];
72+
uint8_t v2 = src[1];
73+
uint8_t v3 = src[2];
74+
uint8_t v4 = src[3];
7575
*res = digittoval[v1] << 12 | digittoval[v2] << 8 | digittoval[v3] << 4 |
7676
digittoval[v4];
7777
return (int32_t)(*res) >= 0;
7878
}*/
7979

8080
// returns a value with the highest bit set if it is not valid
81-
uint32_t hex_to_u32_nocheck(const u8 *src) {
82-
u8 v1 = src[0];
83-
u8 v2 = src[1];
84-
u8 v3 = src[2];
85-
u8 v4 = src[3];
81+
uint32_t hex_to_u32_nocheck(const uint8_t *src) {
82+
uint8_t v1 = src[0];
83+
uint8_t v2 = src[1];
84+
uint8_t v3 = src[2];
85+
uint8_t v4 = src[3];
8686
return digittoval[v1] << 12 | digittoval[v2] << 8 | digittoval[v3] << 4 |
8787
digittoval[v4];
8888
}
@@ -99,7 +99,7 @@ uint32_t hex_to_u32_nocheck(const u8 *src) {
9999
//
100100
// Note: we assume that surrogates are treated separately
101101
//
102-
inline size_t codepoint_to_utf8(uint32_t cp, u8 *c) {
102+
inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
103103
if (cp <= 0x7F) {
104104
c[0] = cp;
105105
return 1; // ascii

‎include/simdjson/jsonparser.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
// The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
2020
// all bytes at and after buf + len are ignored (can be garbage).
2121
WARN_UNUSED
22-
bool json_parse(const u8 *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);
22+
bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);
2323

2424
// Parse a document found in buf, need to preallocate ParsedJson.
2525
// Return false in case of a failure. You can also check validity
@@ -31,7 +31,7 @@ bool json_parse(const u8 *buf, size_t len, ParsedJson &pj, bool reallocifneeded
3131
// all bytes at and after buf + len are ignored (can be garbage).
3232
WARN_UNUSED
3333
static inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
34-
return json_parse((const u8 *) buf, len, pj, reallocifneeded);
34+
return json_parse((const uint8_t *) buf, len, pj, reallocifneeded);
3535
}
3636

3737
// Parse a document found in buf, need to preallocate ParsedJson.
@@ -56,7 +56,7 @@ static inline bool json_parse(const std::string_view &s, ParsedJson &pj, bool re
5656
// the input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
5757
// all bytes at and after buf + len are ignored (can be garbage).
5858
WARN_UNUSED
59-
ParsedJson build_parsed_json(const u8 *buf, size_t len, bool reallocifneeded = true);
59+
ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneeded = true);
6060

6161
WARN_UNUSED
6262
// Build a ParsedJson object. You can check validity
@@ -66,7 +66,7 @@ WARN_UNUSED
6666
// The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
6767
// all bytes at and after buf + len are ignored (can be garbage).
6868
static inline ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded = true) {
69-
return build_parsed_json((const u8 *) buf, len, reallocifneeded);
69+
return build_parsed_json((const uint8_t *) buf, len, reallocifneeded);
7070
}
7171

7272
// convenience function

‎include/simdjson/numberparsing.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,8 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
159159
// Note: a redesign could avoid this function entirely.
160160
//
161161
static never_inline bool
162-
parse_float(const u8 *const buf,
163-
ParsedJson &pj, const u32 offset,
162+
parse_float(const uint8_t *const buf,
163+
ParsedJson &pj, const uint32_t offset,
164164
bool found_minus) {
165165
const char *p = (const char *)(buf + offset);
166166
bool negative = false;
@@ -261,9 +261,9 @@ parse_float(const u8 *const buf,
261261
//
262262
// This function will almost never be called!!!
263263
//
264-
static never_inline bool parse_large_integer(const u8 *const buf,
264+
static never_inline bool parse_large_integer(const uint8_t *const buf,
265265
ParsedJson &pj,
266-
const u32 offset,
266+
const uint32_t offset,
267267
bool found_minus) {
268268
const char *p = (const char *)(buf + offset);
269269

@@ -336,9 +336,9 @@ static never_inline bool parse_large_integer(const u8 *const buf,
336336

337337
// parse the number at buf + offset
338338
// define JSON_TEST_NUMBERS for unit testing
339-
static really_inline bool parse_number(const u8 *const buf,
339+
static really_inline bool parse_number(const uint8_t *const buf,
340340
ParsedJson &pj,
341-
const u32 offset,
341+
const uint32_t offset,
342342
bool found_minus) {
343343
#ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes useful to skip parsing
344344
pj.write_tape_s64(0); // always write zero

0 commit comments

Comments
 (0)