extmod/vfs: Add finaliser to ilistdir to close directory handle.

When iterating over filesystem/folders with os.iterdir(), an open file
(directory) handle is used internally.  Currently this file handle is only
closed once the iterator is completely drained, eg. once all entries have
been looped over / converted into list etc.

If a program opens an iterdir but does not loop over it, or starts to loop
over the iterator but breaks out of the loop, then the handle never gets
closed.  In this state, when the iter object is cleaned up by the garbage
collector this open handle can cause corruption of the filesystem.

Fixes issues #6568 and #8506.

6 files changed, 310 insertions, 0 deletions

diff --git a/tests/extmod/vfs_fat_ilistdir_del.py b/tests/extmod/vfs_fat_ilistdir_del.py
new file mode 100644
index 000000000..a833e9ac1
--- /dev/null
+++ b/tests/extmod/vfs_fat_ilistdir_del.py
@@ -0,0 +1,75 @@
+# Test ilistdir __del__ for VfsFat using a RAM device.
+import gc
+
+try:
+    import uos
+
+    uos.VfsFat
+except (ImportError, AttributeError):
+    print("SKIP")
+    raise SystemExit
+
+
+class RAMBlockDevice:
+    ERASE_BLOCK_SIZE = 4096
+
+    def __init__(self, blocks):
+        self.data = bytearray(blocks * self.ERASE_BLOCK_SIZE)
+
+    def readblocks(self, block, buf, off=0):
+        addr = block * self.ERASE_BLOCK_SIZE + off
+        for i in range(len(buf)):
+            buf[i] = self.data[addr + i]
+
+    def writeblocks(self, block, buf, off=0):
+        addr = block * self.ERASE_BLOCK_SIZE + off
+        for i in range(len(buf)):
+            self.data[addr + i] = buf[i]
+
+    def ioctl(self, op, arg):
+        if op == 4:  # block count
+            return len(self.data) // self.ERASE_BLOCK_SIZE
+        if op == 5:  # block size
+            return self.ERASE_BLOCK_SIZE
+        if op == 6:  # erase block
+            return 0
+
+
+def test(bdev, vfs_class):
+    vfs_class.mkfs(bdev)
+    vfs = vfs_class(bdev)
+    vfs.mkdir("/test_d1")
+    vfs.mkdir("/test_d2")
+    vfs.mkdir("/test_d3")
+
+    for i in range(10):
+        print(i)
+
+        # We want to partially iterate the ilistdir iterator to leave it in an
+        # open state, which will then test the finaliser when it's garbage collected.
+        idir = vfs.ilistdir("/")
+        print(any(idir))
+
+        # Alternate way of partially iterating the ilistdir object, modifying the
+        # filesystem while it's open.
+        for dname, *_ in vfs.ilistdir("/"):
+            vfs.rmdir(dname)
+            break
+        vfs.mkdir(dname)
+
+        # Also create a fully drained iterator and ensure trying to re-use it
+        # throws the correct exception.
+        idir_emptied = vfs.ilistdir("/")
+        l = list(idir_emptied)
+        print(len(l))
+        try:
+            next(idir_emptied)
+        except StopIteration:
+            pass
+
+        gc.collect()
+        vfs.open("/test", "w").close()
+
+
+bdev = RAMBlockDevice(30)
+test(bdev, uos.VfsFat)
diff --git a/tests/extmod/vfs_fat_ilistdir_del.py.exp b/tests/extmod/vfs_fat_ilistdir_del.py.exp
new file mode 100644
index 000000000..0ab2b019f
--- /dev/null
+++ b/tests/extmod/vfs_fat_ilistdir_del.py.exp
@@ -0,0 +1,30 @@
+0
+True
+3
+1
+True
+4
+2
+True
+4
+3
+True
+4
+4
+True
+4
+5
+True
+4
+6
+True
+4
+7
+True
+4
+8
+True
+4
+9
+True
+4
diff --git a/tests/extmod/vfs_lfs_ilistdir_del.py b/tests/extmod/vfs_lfs_ilistdir_del.py
new file mode 100644
index 000000000..073576986
--- /dev/null
+++ b/tests/extmod/vfs_lfs_ilistdir_del.py
@@ -0,0 +1,75 @@
+# Test ilistdir __del__ for VfsLittle using a RAM device.
+import gc
+
+try:
+    import uos
+
+    uos.VfsLfs2
+except (ImportError, AttributeError):
+    print("SKIP")
+    raise SystemExit
+
+
+class RAMBlockDevice:
+    ERASE_BLOCK_SIZE = 1024
+
+    def __init__(self, blocks):
+        self.data = bytearray(blocks * self.ERASE_BLOCK_SIZE)
+
+    def readblocks(self, block, buf, off):
+        addr = block * self.ERASE_BLOCK_SIZE + off
+        for i in range(len(buf)):
+            buf[i] = self.data[addr + i]
+
+    def writeblocks(self, block, buf, off):
+        addr = block * self.ERASE_BLOCK_SIZE + off
+        for i in range(len(buf)):
+            self.data[addr + i] = buf[i]
+
+    def ioctl(self, op, arg):
+        if op == 4:  # block count
+            return len(self.data) // self.ERASE_BLOCK_SIZE
+        if op == 5:  # block size
+            return self.ERASE_BLOCK_SIZE
+        if op == 6:  # erase block
+            return 0
+
+
+def test(bdev, vfs_class):
+    vfs_class.mkfs(bdev)
+    vfs = vfs_class(bdev)
+    vfs.mkdir("/test_d1")
+    vfs.mkdir("/test_d2")
+    vfs.mkdir("/test_d3")
+
+    for i in range(10):
+        print(i)
+
+        # We want to partially iterate the ilistdir iterator to leave it in an
+        # open state, which will then test the finaliser when it's garbage collected.
+        idir = vfs.ilistdir("/")
+        print(any(idir))
+
+        # Alternate way of partially iterating the ilistdir object, modifying the
+        # filesystem while it's open.
+        for dname, *_ in vfs.ilistdir("/"):
+            vfs.rmdir(dname)
+            break
+        vfs.mkdir(dname)
+
+        # Also create a fully drained iterator and ensure trying to re-use it
+        # throws the correct exception.
+        idir_emptied = vfs.ilistdir("/")
+        l = list(idir_emptied)
+        print(len(l))
+        try:
+            next(idir_emptied)
+        except StopIteration:
+            pass
+
+        gc.collect()
+        vfs.open("/test", "w").close()
+
+
+bdev = RAMBlockDevice(30)
+test(bdev, uos.VfsLfs2)
diff --git a/tests/extmod/vfs_lfs_ilistdir_del.py.exp b/tests/extmod/vfs_lfs_ilistdir_del.py.exp
new file mode 100644
index 000000000..0ab2b019f
--- /dev/null
+++ b/tests/extmod/vfs_lfs_ilistdir_del.py.exp
@@ -0,0 +1,30 @@
+0
+True
+3
+1
+True
+4
+2
+True
+4
+3
+True
+4
+4
+True
+4
+5
+True
+4
+6
+True
+4
+7
+True
+4
+8
+True
+4
+9
+True
+4
diff --git a/tests/extmod/vfs_posix_ilistdir_del.py b/tests/extmod/vfs_posix_ilistdir_del.py
new file mode 100644
index 000000000..edb50dfd6
--- /dev/null
+++ b/tests/extmod/vfs_posix_ilistdir_del.py
@@ -0,0 +1,70 @@
+# Test ilistdir __del__ for VfsPosix.
+import gc
+
+try:
+    import os
+
+    os.VfsPosix
+except (ImportError, AttributeError):
+    print("SKIP")
+    raise SystemExit
+
+
+def test(testdir):
+    vfs = os.VfsPosix(testdir)
+    vfs.mkdir("/test_d1")
+    vfs.mkdir("/test_d2")
+    vfs.mkdir("/test_d3")
+
+    for i in range(10):
+        print(i)
+
+        # We want to partially iterate the ilistdir iterator to leave it in an
+        # open state, which will then test the finaliser when it's garbage collected.
+        idir = vfs.ilistdir("/")
+        print(any(idir))
+
+        # Alternate way of partially iterating the ilistdir object, modifying the
+        # filesystem while it's open.
+        for dname, *_ in vfs.ilistdir("/"):
+            vfs.rmdir(dname)
+            break
+        vfs.mkdir(dname)
+
+        # Also create a fully drained iterator and ensure trying to re-use it
+        # throws the correct exception.
+        idir_emptied = vfs.ilistdir("/")
+        l = list(idir_emptied)
+        print(len(l))
+        try:
+            next(idir_emptied)
+        except StopIteration:
+            pass
+
+        gc.collect()
+
+        # Create and delete a file, try to flush out any filesystem
+        # corruption that may be caused over the loops.
+        vfs.open("/test", "w").close()
+        vfs.remove("/test")
+
+
+# We need an empty directory for testing.
+# Skip the test if it already exists.
+temp_dir = "vfs_posix_ilistdir_del_test_dir"
+try:
+    os.stat(temp_dir)
+    print("SKIP")
+    raise SystemExit
+except OSError:
+    pass
+
+os.mkdir(temp_dir)
+
+test(temp_dir)
+
+# Remove tempdir.
+for td in os.listdir(temp_dir):
+    os.rmdir("/".join((temp_dir, td)))
+
+os.rmdir(temp_dir)
diff --git a/tests/extmod/vfs_posix_ilistdir_del.py.exp b/tests/extmod/vfs_posix_ilistdir_del.py.exp
new file mode 100644
index 000000000..c30ba4132
--- /dev/null
+++ b/tests/extmod/vfs_posix_ilistdir_del.py.exp
@@ -0,0 +1,30 @@
+0
+True
+3
+1
+True
+3
+2
+True
+3
+3
+True
+3
+4
+True
+3
+5
+True
+3
+6
+True
+3
+7
+True
+3
+8
+True
+3
+9
+True
+3