Skip to content

Commit 3f993c3

Browse files
committed
Use an enum for decoding_state. It makes the code a little more
understandable.
1 parent 16c7075 commit 3f993c3

2 files changed

Lines changed: 18 additions & 12 deletions

File tree

‎Parser/tokenizer.c‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ tok_new(void)
139139
tok->alterror = 1;
140140
tok->alttabsize = 1;
141141
tok->altindstack[0] = 0;
142-
tok->decoding_state = 0;
142+
tok->decoding_state = STATE_INIT;
143143
tok->decoding_erred = 0;
144144
tok->read_coding_spec = 0;
145145
tok->encoding = NULL;
@@ -280,15 +280,15 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
280280
if (cs != NULL) {
281281
tok->read_coding_spec = 1;
282282
if (tok->encoding == NULL) {
283-
assert(tok->decoding_state == 1); /* raw */
283+
assert(tok->decoding_state == STATE_RAW);
284284
if (strcmp(cs, "utf-8") == 0 ||
285285
strcmp(cs, "iso-8859-1") == 0) {
286286
tok->encoding = cs;
287287
} else {
288288
r = set_readline(tok, cs);
289289
if (r) {
290290
tok->encoding = cs;
291-
tok->decoding_state = -1;
291+
tok->decoding_state = STATE_NORMAL;
292292
}
293293
else
294294
PyMem_FREE(cs);
@@ -318,7 +318,7 @@ check_bom(int get_char(struct tok_state *),
318318
struct tok_state *tok)
319319
{
320320
int ch = get_char(tok);
321-
tok->decoding_state = 1;
321+
tok->decoding_state = STATE_RAW;
322322
if (ch == EOF) {
323323
return 1;
324324
} else if (ch == 0xEF) {
@@ -330,11 +330,11 @@ check_bom(int get_char(struct tok_state *),
330330
} else if (ch == 0xFE) {
331331
ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;
332332
if (!set_readline(tok, "utf-16-be")) return 0;
333-
tok->decoding_state = -1;
333+
tok->decoding_state = STATE_NORMAL;
334334
} else if (ch == 0xFF) {
335335
ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;
336336
if (!set_readline(tok, "utf-16-le")) return 0;
337-
tok->decoding_state = -1;
337+
tok->decoding_state = STATE_NORMAL;
338338
#endif
339339
} else {
340340
unget_char(ch, tok);
@@ -494,12 +494,12 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
494494
char *line = NULL;
495495
int badchar = 0;
496496
for (;;) {
497-
if (tok->decoding_state < 0) {
497+
if (tok->decoding_state == STATE_NORMAL) {
498498
/* We already have a codec associated with
499499
this input. */
500500
line = fp_readl(s, size, tok);
501501
break;
502-
} else if (tok->decoding_state > 0) {
502+
} else if (tok->decoding_state == STATE_RAW) {
503503
/* We want a 'raw' read. */
504504
line = Py_UniversalNewlineFgets(s, size,
505505
tok->fp, NULL);
@@ -510,7 +510,7 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
510510
reader functions from now on. */
511511
if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
512512
return error_ret(tok);
513-
assert(tok->decoding_state != 0);
513+
assert(tok->decoding_state != STATE_INIT);
514514
}
515515
}
516516
if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
@@ -550,7 +550,7 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
550550
static int
551551
decoding_feof(struct tok_state *tok)
552552
{
553-
if (tok->decoding_state >= 0) {
553+
if (tok->decoding_state != STATE_NORMAL) {
554554
return feof(tok->fp);
555555
} else {
556556
PyObject* buf = tok->decoding_buffer;
@@ -700,7 +700,7 @@ PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
700700
return NULL;
701701
}
702702
strcpy(tok->encoding, enc);
703-
tok->decoding_state = -1;
703+
tok->decoding_state = STATE_NORMAL;
704704
}
705705
return tok;
706706
}

‎Parser/tokenizer.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@ extern "C" {
1212

1313
#define MAXINDENT 100 /* Max indentation level */
1414

15+
enum decoding_state {
16+
STATE_INIT,
17+
STATE_RAW,
18+
STATE_NORMAL, /* have a codec associated with input */
19+
};
20+
1521
/* Tokenizer state */
1622
struct tok_state {
1723
/* Input state; buf <= cur <= inp <= end */
@@ -40,7 +46,7 @@ struct tok_state {
4046
int alttabsize; /* Alternate tab spacing */
4147
int altindstack[MAXINDENT]; /* Stack of alternate indents */
4248
/* Stuff for PEP 0263 */
43-
int decoding_state; /* -1:decoding, 0:init, 1:raw */
49+
enum decoding_state decoding_state;
4450
int decoding_erred; /* whether erred in decoding */
4551
int read_coding_spec; /* whether 'coding:...' has been read */
4652
char *encoding;

0 commit comments

Comments
 (0)