Skip to content

Feedback after release #662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions fsspec/implementations/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,17 +58,7 @@ def glob(self, path, **kwargs):
return super().glob(path, **kwargs)

def info(self, path, **kwargs):
if isinstance(path, str):
path = self._strip_protocol(path)
out = os.stat(path, follow_symlinks=False)
link = os.path.islink(path)
if os.path.isdir(path):
t = "directory"
elif os.path.isfile(path):
t = "file"
else:
t = "other"
else:
if hasattr(path, "stat"):
# scandir DirEntry
out = path.stat(follow_symlinks=False)
link = path.is_symlink()
Expand All @@ -79,6 +69,17 @@ def info(self, path, **kwargs):
else:
t = "other"
path = self._strip_protocol(path.path)
else:
# str or path-like
path = self._strip_protocol(path)
out = os.stat(path, follow_symlinks=False)
link = os.path.islink(path)
if os.path.isdir(path):
t = "directory"
elif os.path.isfile(path):
t = "file"
else:
t = "other"
result = {
"name": path,
"size": out.st_size,
Expand Down Expand Up @@ -123,6 +124,9 @@ def mv_file(self, path1, path2, **kwargs):
def rm(self, path, recursive=False, maxdepth=None):
path = self._strip_protocol(path).rstrip("/")
if recursive and self.isdir(path):

if os.path.abspath(path) == os.getcwd():
raise ValueError("Cannot delete current working directory")
shutil.rmtree(path)
else:
os.remove(path)
Expand Down Expand Up @@ -163,7 +167,7 @@ def _strip_protocol(cls, path):
path = stringify_path(path)
if path.startswith("file://"):
path = path[7:]
return make_path_posix(path)
return make_path_posix(path).rstrip("/")

def _isfilestore(self):
# Inheriting from DaskFileSystem makes this False (S3, etc. were)
Expand Down
12 changes: 12 additions & 0 deletions fsspec/implementations/tests/test_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,7 @@ def test_strip_protocol_expanduser():
assert path != stripped
assert "file://" not in stripped
assert stripped.startswith(os.path.expanduser("~").replace("\\", "/"))
assert not LocalFileSystem._strip_protocol("./").endswith("/")


def test_iterable(tmpdir):
Expand Down Expand Up @@ -665,6 +666,17 @@ def test_transaction(tmpdir):
assert content == read_content


def test_delete_cwd(tmpdir):
cwd = os.getcwd()
fs = LocalFileSystem()
try:
os.chdir(tmpdir)
with pytest.raises(ValueError):
fs.rm(".", recursive=True)
finally:
os.chdir(cwd)


@pytest.mark.parametrize(
"opener, ext", [(bz2.open, ".bz2"), (gzip.open, ".gz"), (open, "")]
)
Expand Down
30 changes: 14 additions & 16 deletions fsspec/registry.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import importlib
from distutils.version import LooseVersion

__all__ = ["registry", "get_filesystem_class", "default"]

Expand Down Expand Up @@ -187,8 +186,6 @@ def register_implementation(name, cls, clobber=True, errtxt=None):
"reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"},
}

minversions = {"s3fs": LooseVersion("0.3.0"), "gcsfs": LooseVersion("0.3.0")}


def get_filesystem_class(protocol):
"""Fetch named protocol implementation from the registry
Expand Down Expand Up @@ -221,19 +218,20 @@ def get_filesystem_class(protocol):


def _import_class(cls, minv=None):
mod, name = cls.rsplit(".", 1)
minv = minv or minversions
minversion = minv.get(mod, None)

mod = importlib.import_module(mod)
if minversion:
version = getattr(mod, "__version__", None)
if version and LooseVersion(version) < minversion:
raise RuntimeError(
"'{}={}' is installed, but version '{}' or "
"higher is required".format(mod.__name__, version, minversion)
)
return getattr(mod, name)
"""Take a string FQP and return the imported class or identifier

clas is of the form "package.module.klass" or "package.module:subobject.klass"
"""
if ":" in cls:
mod, name = cls.rsplit(":", 1)
mod = importlib.import_module(mod)
for part in name.split("."):
mod = getattr(mod, part)
return mod
else:
mod, name = cls.rsplit(".", 1)
mod = importlib.import_module(mod)
return getattr(mod, name)


def filesystem(protocol, **storage_options):
Expand Down
16 changes: 0 additions & 16 deletions fsspec/tests/test_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,6 @@ def clean_imports():
sys.modules["fsspec"] = real_module


@pytest.mark.parametrize(
"protocol,module,minversion,oldversion",
[("s3", "s3fs", "0.3.0", "0.1.0"), ("gs", "gcsfs", "0.3.0", "0.1.0")],
)
def test_minversion_s3fs(protocol, module, minversion, oldversion, monkeypatch):
_registry.clear()
mod = pytest.importorskip(module, minversion)

assert get_filesystem_class("s3") is not None
_registry.clear()

monkeypatch.setattr(mod, "__version__", oldversion)
with pytest.raises(RuntimeError, match=minversion):
get_filesystem_class(protocol)


def test_registry_readonly():
get_filesystem_class("file")
assert "file" in registry
Expand Down