dbfs: proper cache invalidation for databricks filesystem

Author: isidentical

fsspec/implementations/dbfs.py

@@ -118,6 +118,7 @@ def makedirs(self, path, exist_ok=True):
                 raise FileExistsError(e.message)
 
             raise e
+        self.invalidate_cache(self._parent(path))
 
     def mkdir(self, path, create_parents=True, **kwargs):
         """
@@ -163,6 +164,7 @@ def rm(self, path, recursive=False):
                 raise OSError(e.message)
 
             raise e
+        self.invalidate_cache(self._parent(path))
 
     def mv(self, source_path, destination_path, recursive=False, maxdepth=None):
         """
@@ -205,6 +207,8 @@ def mv(self, source_path, destination_path, recursive=False, maxdepth=None):
                 raise FileExistsError(e.message)
 
             raise e
+        self.invalidate_cache(self._parent(source_path))
+        self.invalidate_cache(self._parent(destination_path))
 
     def _open(self, path, mode="rb", block_size="default", **kwargs):
         """
@@ -364,6 +368,13 @@ def _get_data(self, path, start, end):
 
             raise e
 
+    def invalidate_cache(self, path=None):
+        if path is None:
+            self.dircache.clear()
+        else:
+            self.dircache.pop(path, None)
+        super().invalidate_cache(path)
+
 
 class DatabricksFile(AbstractBufferedFile):
     """