py: Implement partial PEP-498 (f-string) support.

This implements (most of) the PEP-498 spec for f-strings and is based on
https://github.com/micropython/micropython/pull/4998 by @klardotsh.

It is implemented in the lexer as a syntax translation to `str.format`:
  f"{a}" --> "{}".format(a)

It also supports:
  f"{a=}" --> "a={}".format(a)

This is done by extracting the arguments into a temporary vstr buffer,
then after the string has been tokenized, the lexer input queue is saved
and the contents of the temporary vstr buffer are injected into the lexer
instead.

There are four main limitations:
- raw f-strings (`fr` or `rf` prefixes) are not supported and will raise
  `SyntaxError: raw f-strings are not supported`.

- literal concatenation of f-strings with adjacent strings will fail
    "{}" f"{a}" --> "{}{}".format(a)    (str.format will incorrectly use
                                         the braces from the non-f-string)
    f"{a}" f"{a}" --> "{}".format(a) "{}".format(a) (cannot concatenate)

- PEP-498 requires the full parser to understand the interpolated
  argument, however because this entirely runs in the lexer it cannot
  resolve nested braces in expressions like
    f"{'}'}"

- The !r, !s, and !a conversions are not supported.

Includes tests and cpydiffs.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

1 files changed, 11 insertions, 0 deletions

diff --git a/py/lexer.h b/py/lexer.h
index 91767a44b..e16b9a8ce 100644
--- a/py/lexer.h
+++ b/py/lexer.h
@@ -44,6 +44,10 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_INVALID,
     MP_TOKEN_DEDENT_MISMATCH,
     MP_TOKEN_LONELY_STRING_OPEN,
+    #if MICROPY_PY_FSTRINGS
+    MP_TOKEN_MALFORMED_FSTRING,
+    MP_TOKEN_FSTRING_RAW,
+    #endif
 
     MP_TOKEN_NEWLINE,
     MP_TOKEN_INDENT,
@@ -158,6 +162,9 @@ typedef struct _mp_lexer_t {
     mp_reader_t reader;         // stream source
 
     unichar chr0, chr1, chr2;   // current cached characters from source
+    #if MICROPY_PY_FSTRINGS
+    unichar chr0_saved, chr1_saved, chr2_saved; // current cached characters from alt source
+    #endif
 
     size_t line;                // current source line
     size_t column;              // current source column
@@ -173,6 +180,10 @@ typedef struct _mp_lexer_t {
     size_t tok_column;          // token source column
     mp_token_kind_t tok_kind;   // token kind
     vstr_t vstr;                // token data
+    #if MICROPY_PY_FSTRINGS
+    vstr_t fstring_args;        // extracted arguments to pass to .format()
+    size_t fstring_args_idx;    // how many bytes of fstring_args have been read
+    #endif
 } mp_lexer_t;
 
 mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader);