diff options
Diffstat (limited to 'py/lexer.c')
| -rw-r--r-- | py/lexer.c | 35 |
1 files changed, 19 insertions, 16 deletions
diff --git a/py/lexer.c b/py/lexer.c index 89ecc386e..76abedd45 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -490,22 +490,25 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) { } } if (c != MP_LEXER_EOF) { - #if MICROPY_PY_BUILTINS_STR_UNICODE - if (c < 0x110000 && !is_bytes) { - vstr_add_char(&lex->vstr, c); - } else if (c < 0x100 && is_bytes) { - vstr_add_byte(&lex->vstr, c); - } - #else - // without unicode everything is just added as an 8-bit byte - if (c < 0x100) { - vstr_add_byte(&lex->vstr, c); - } - #endif - else { - // unicode character out of range - // this raises a generic SyntaxError; could provide more info - lex->tok_kind = MP_TOKEN_INVALID; + if (MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) { + if (c < 0x110000 && !is_bytes) { + vstr_add_char(&lex->vstr, c); + } else if (c < 0x100 && is_bytes) { + vstr_add_byte(&lex->vstr, c); + } else { + // unicode character out of range + // this raises a generic SyntaxError; could provide more info + lex->tok_kind = MP_TOKEN_INVALID; + } + } else { + // without unicode everything is just added as an 8-bit byte + if (c < 0x100) { + vstr_add_byte(&lex->vstr, c); + } else { + // 8-bit character out of range + // this raises a generic SyntaxError; could provide more info + lex->tok_kind = MP_TOKEN_INVALID; + } } } } else { |