py: Implement partial PEP-498 (f-string) support.

This implements (most of) the PEP-498 spec for f-strings and is based on
https://github.com/micropython/micropython/pull/4998 by @klardotsh.

It is implemented in the lexer as a syntax translation to `str.format`:
  f"{a}" --> "{}".format(a)

It also supports:
  f"{a=}" --> "a={}".format(a)

This is done by extracting the arguments into a temporary vstr buffer,
then after the string has been tokenized, the lexer input queue is saved
and the contents of the temporary vstr buffer are injected into the lexer
instead.

There are four main limitations:
- raw f-strings (`fr` or `rf` prefixes) are not supported and will raise
  `SyntaxError: raw f-strings are not supported`.

- literal concatenation of f-strings with adjacent strings will fail
    "{}" f"{a}" --> "{}{}".format(a)    (str.format will incorrectly use
                                         the braces from the non-f-string)
    f"{a}" f"{a}" --> "{}".format(a) "{}".format(a) (cannot concatenate)

- PEP-498 requires the full parser to understand the interpolated
  argument, however because this entirely runs in the lexer it cannot
  resolve nested braces in expressions like
    f"{'}'}"

- The !r, !s, and !a conversions are not supported.

Includes tests and cpydiffs.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

10 files changed, 138 insertions, 5 deletions

diff --git a/tests/basics/string_fstring.py b/tests/basics/string_fstring.py
new file mode 100644
index 000000000..efb7e5a8e
--- /dev/null
+++ b/tests/basics/string_fstring.py
@@ -0,0 +1,57 @@
+def f():
+    return 4
+def g(_):
+    return 5
+def h():
+    return 6
+
+print(f'no interpolation')
+print(f"no interpolation")
+print(f"""no interpolation""")
+
+x, y = 1, 2
+print(f'{x}')
+print(f'{x:08x}')
+print(f'{x=}')
+print(f'{x=:08x}')
+print(f'a {x} b {y} c')
+print(f'a {x:08x} b {y} c')
+print(f'a {x=} b {y} c')
+print(f'a {x=:08x} b {y} c')
+
+print(f'a {"hello"} b')
+print(f'a {f() + g("foo") + h()} b')
+print(f'a {f() + g("foo") + h()=} b')
+print(f'a {f() + g("foo") + h()=:08x} b')
+
+def foo(a, b):
+    return f'{x}{y}{a}{b}'
+print(foo(7, 8))
+
+# PEP-0498 specifies that '\\' and '#' must be disallowed explicitly, whereas
+# MicroPython relies on the syntax error as a result of the substitution.
+
+print(f"\\")
+print(f'#')
+try:
+    eval("f'{\}'")
+except SyntaxError:
+    print('SyntaxError')
+try:
+    eval("f'{#}'")
+except SyntaxError:
+    print('SyntaxError')
+
+
+# PEP-0498 specifies that handling of double braces '{{' or '}}' should
+# behave like str.format.
+print(f'{{}}')
+print(f'{{{4*10}}}', '{40}')
+
+# A single closing brace, unlike str.format should raise a syntax error.
+# MicroPython instead raises ValueError at runtime from the substitution.
+try:
+    eval("f'{{}'")
+except (ValueError, SyntaxError):
+    # MicroPython incorrectly raises ValueError here.
+    print('SyntaxError')
diff --git a/tests/cmdline/cmd_parsetree.py b/tests/cmdline/cmd_parsetree.py
index 50da36954..483ea8937 100644
--- a/tests/cmdline/cmd_parsetree.py
+++ b/tests/cmdline/cmd_parsetree.py
@@ -10,3 +10,4 @@ d = b"bytes"
 e = b"a very long bytes that will not be interned"
 f = 123456789012345678901234567890
 g = 123
+h = f"fstring: '{b}'"
diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp
index e64f4f782..cc8ba82c0 100644
--- a/tests/cmdline/cmd_parsetree.py.exp
+++ b/tests/cmdline/cmd_parsetree.py.exp
@@ -1,6 +1,6 @@
 ----------------
-[   4] \(rule\|file_input_2\)(1) (n=9)
-         tok(4)
+[   4] \(rule\|file_input_2\)(1) (n=10)
+         tok(6)
 [   4]   \(rule\|for_stmt\)(22) (n=4)
            id(i)
 [   4]     \(rule\|atom_paren\)(45) (n=1)
@@ -9,7 +9,7 @@
            NULL
 [   6]   \(rule\|expr_stmt\)(5) (n=2)
            id(a)
-           tok(14)
+           tok(16)
 [   7]   \(rule\|expr_stmt\)(5) (n=2)
            id(b)
            str(str)
@@ -28,6 +28,16 @@
 [  12]   \(rule\|expr_stmt\)(5) (n=2)
            id(g)
            int(123)
+[  13]   \(rule\|expr_stmt\)(5) (n=2)
+           id(h)
+[  13]     \(rule\|atom_expr_normal\)(44) (n=2)
+[  13]       literal const(\.\+)
+[  13]       \(rule\|atom_expr_trailers\)(142) (n=2)
+[  13]         \(rule\|trailer_period\)(50) (n=1)
+                 id(format)
+[  13]         \(rule\|trailer_paren\)(48) (n=1)
+[  13]           \(rule\|arglist\)(164) (n=1)
+                   id(b)
 ----------------
 File cmdline/cmd_parsetree.py, code block '<module>' (descriptor: \.\+, bytecode @\.\+ bytes)
 Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
@@ -46,6 +56,7 @@ arg names:
   bc=32 line=10
   bc=37 line=11
   bc=42 line=12
+  bc=48 line=13
 00 BUILD_TUPLE 0
 02 GET_ITER_STACK
 03 FOR_ITER 12
@@ -65,8 +76,13 @@ arg names:
 39 STORE_NAME f
 42 LOAD_CONST_SMALL_INT 123
 45 STORE_NAME g
-48 LOAD_CONST_NONE
-49 RETURN_VALUE
+48 LOAD_CONST_OBJ \.\+
+50 LOAD_METHOD format
+53 LOAD_NAME b (cache=0)
+57 CALL_METHOD n=1 nkw=0
+59 STORE_NAME h
+62 LOAD_CONST_NONE
+63 RETURN_VALUE
 mem: total=\\d\+, current=\\d\+, peak=\\d\+
 stack: \\d\+ out of \\d\+
 GC: total: \\d\+, used: \\d\+, free: \\d\+
diff --git a/tests/cpydiff/core_fstring_concat.py b/tests/cpydiff/core_fstring_concat.py
new file mode 100644
index 000000000..fd83527b5
--- /dev/null
+++ b/tests/cpydiff/core_fstring_concat.py
@@ -0,0 +1,12 @@
+"""
+categories: Core
+description: f-strings don't support concatenation with adjacent literals if the adjacent literals contain braces
+cause: MicroPython is optimised for code space.
+workaround: Use the + operator between literal strings when either is an f-string
+"""
+
+x = 1
+print("aa" f"{x}")
+print(f"{x}" "ab")
+print("a{}a" f"{x}")
+print(f"{x}" "a{}b")
diff --git a/tests/cpydiff/core_fstring_parser.py b/tests/cpydiff/core_fstring_parser.py
new file mode 100644
index 000000000..6917f3cfa
--- /dev/null
+++ b/tests/cpydiff/core_fstring_parser.py
@@ -0,0 +1,9 @@
+"""
+categories: Core
+description: f-strings cannot support expressions that require parsing to resolve nested braces
+cause: MicroPython is optimised for code space.
+workaround: Only use simple expressions inside f-strings
+"""
+
+f'{"hello {} world"}'
+f"{repr({})}"
diff --git a/tests/cpydiff/core_fstring_raw.py b/tests/cpydiff/core_fstring_raw.py
new file mode 100644
index 000000000..84e265f70
--- /dev/null
+++ b/tests/cpydiff/core_fstring_raw.py
@@ -0,0 +1,8 @@
+"""
+categories: Core
+description: Raw f-strings are not supported
+cause: MicroPython is optimised for code space.
+workaround: Unknown
+"""
+
+rf"hello"
diff --git a/tests/cpydiff/core_fstring_repr.py b/tests/cpydiff/core_fstring_repr.py
new file mode 100644
index 000000000..fcadcbf1b
--- /dev/null
+++ b/tests/cpydiff/core_fstring_repr.py
@@ -0,0 +1,18 @@
+"""
+categories: Core
+description: f-strings don't support the !r, !s, and !a conversions
+cause: MicroPython is optimised for code space.
+workaround: Use repr(), str(), and ascii() explictly.
+"""
+
+
+class X:
+    def __repr__(self):
+        return "repr"
+
+    def __str__(self):
+        return "str"
+
+
+print(f"{X()!r}")
+print(f"{X()!s}")
diff --git a/tests/feature_check/fstring.py b/tests/feature_check/fstring.py
new file mode 100644
index 000000000..14792bce0
--- /dev/null
+++ b/tests/feature_check/fstring.py
@@ -0,0 +1,3 @@
+# check whether f-strings (PEP-498) are supported
+a = 1
+print(f"a={a}")
diff --git a/tests/feature_check/fstring.py.exp b/tests/feature_check/fstring.py.exp
new file mode 100644
index 000000000..73cdb8bcc
--- /dev/null
+++ b/tests/feature_check/fstring.py.exp
@@ -0,0 +1 @@
+a=1
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 619df5ed3..3e97a7c87 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -290,6 +290,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
     skip_const = False
     skip_revops = False
     skip_io_module = False
+    skip_fstring = False
     skip_endian = False
     has_complex = True
     has_coverage = False
@@ -348,6 +349,11 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         if output != b"uio\n":
             skip_io_module = True
 
+        # Check if fstring feature is enabled, and skip such tests if it doesn't
+        output = run_feature_check(pyb, args, base_path, "fstring.py")
+        if output != b"a=1\n":
+            skip_fstring = True
+
         # Check if emacs repl is supported, and skip such tests if it's not
         t = run_feature_check(pyb, args, base_path, "repl_emacs_check.py")
         if "True" not in str(t, "ascii"):
@@ -543,6 +549,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         is_async = test_name.startswith(("async_", "uasyncio_"))
         is_const = test_name.startswith("const")
         is_io_module = test_name.startswith("io_")
+        is_fstring = test_name.startswith("string_fstring")
 
         skip_it = test_file in skip_tests
         skip_it |= skip_native and is_native
@@ -555,6 +562,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         skip_it |= skip_const and is_const
         skip_it |= skip_revops and "reverse_op" in test_name
         skip_it |= skip_io_module and is_io_module
+        skip_it |= skip_fstring and is_fstring
 
         if args.list_tests:
             if not skip_it: