Skip to content

RF: Update and factor mount table parsing #2444

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Feb 16, 2018
Merged

RF: Update and factor mount table parsing #2444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e511566
fstype parsing for cifs can fail on Mac OS
afni-dglen Feb 7, 2018
547e2de
fstype parsing for cifs can fail on Mac OS
afni-dglen Feb 7, 2018
a0d48fd
more proper integration
afni-dglen Feb 7, 2018
9e763ba
tab/space fix
afni-dglen Feb 7, 2018
832fecf
RF: Update and factor mount table parsing code
effigies Feb 16, 2018
0f4b40b
TEST: Add tests for mount table parsing
effigies Feb 16, 2018
9476963
Merge remote-tracking branch 'upstream/master' into fix/filesystem_table
effigies Feb 16, 2018
e8b0341
STY: flake8
effigies Feb 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 31 additions & 16 deletions nipype/utils/filemanip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,34 +275,49 @@ def hash_timestamp(afile):
return md5hex


def _generate_cifs_table():
"""Construct a reverse-length-ordered list of mount points that
fall under a CIFS mount.

This precomputation allows efficient checking for whether a given path
would be on a CIFS filesystem.
def _parse_mount_table(exit_code, output):
"""Parses the output of ``mount`` to produce (path, fs_type) pairs

On systems without a ``mount`` command, or with no CIFS mounts, returns an
empty list.
Separated from _generate_cifs_table to enable testing logic with real
outputs
"""
exit_code, output = sp.getstatusoutput("mount")
# Not POSIX
if exit_code != 0:
return []

# Linux mount example: sysfs on /sys type sysfs (rw,nosuid,nodev,noexec)
# <PATH>^^^^ ^^^^^<FSTYPE>
# OSX mount example: /dev/disk2 on / (hfs, local, journaled)
# <PATH>^ ^^^<FSTYPE>
pattern = re.compile(r'.*? on (/.*?) (?:type |\()([^\s,]+)(?:, |\)| )')

# (path, fstype) tuples, sorted by path length (longest first)
mount_info = sorted(
(line.split()[2:5:2] for line in output.splitlines()),
key=lambda x: len(x[0]),
reverse=True)
cifs_paths = [path for path, fstype in mount_info if fstype == 'cifs']
mount_info = sorted((pattern.match(l).groups()
for l in output.splitlines()),
key=lambda x: len(x[0]), reverse=True)
cifs_paths = [path for path, fstype in mount_info
if fstype.lower() == 'cifs']

return [
mount for mount in mount_info
if any(mount[0].startswith(path) for path in cifs_paths)
]


def _generate_cifs_table():
"""Construct a reverse-length-ordered list of mount points that
fall under a CIFS mount.

This precomputation allows efficient checking for whether a given path
would be on a CIFS filesystem.

On systems without a ``mount`` command, or with no CIFS mounts, returns an
empty list.
"""
exit_code, output = sp.getstatusoutput("mount")
return _parse_mount_table(exit_code, output)


_cifs_table = _generate_cifs_table()


Expand Down Expand Up @@ -763,8 +778,8 @@ def emptydirs(path, noexist_ok=False):
elcont = os.listdir(path)
if ex.errno == errno.ENOTEMPTY and not elcont:
fmlogger.warning(
'An exception was raised trying to remove old %s, but the path '
'seems empty. Is it an NFS mount?. Passing the exception.',
'An exception was raised trying to remove old %s, but the path'
' seems empty. Is it an NFS mount?. Passing the exception.',
path)
elif ex.errno == errno.ENOTEMPTY and elcont:
fmlogger.debug('Folder %s contents (%d items).', path, len(elcont))
Expand Down
135 changes: 130 additions & 5 deletions nipype/utils/tests/test_filemanip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,9 @@
from ...testing import TempFATFS
from ...utils.filemanip import (
save_json, load_json, fname_presuffix, fnames_presuffix, hash_rename,
check_forhash, _cifs_table, on_cifs, copyfile, copyfiles, filename_to_list,
list_to_filename, check_depends, split_filename, get_related_files)

import numpy as np
check_forhash, _parse_mount_table, _cifs_table, on_cifs, copyfile,
copyfiles, filename_to_list, list_to_filename, check_depends,
split_filename, get_related_files)


def _ignore_atime(stat):
Expand Down Expand Up @@ -123,7 +122,7 @@ def test_copyfiles(_temp_analyze_files, _temp_analyze_files_prime):
pth, fname = os.path.split(orig_img2)
new_img2 = os.path.join(pth, 'secondfile.img')
new_hdr2 = os.path.join(pth, 'secondfile.hdr')
newfiles = copyfiles([orig_img1, orig_img2], [new_img1, new_img2])
copyfiles([orig_img1, orig_img2], [new_img1, new_img2])
assert os.path.exists(new_img1)
assert os.path.exists(new_hdr1)
assert os.path.exists(new_img2)
Expand Down Expand Up @@ -335,6 +334,132 @@ def test_related_files(file, length, expected_files):
assert ef in related_files


MOUNT_OUTPUTS = (
# Linux, no CIFS
(r'''sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=8121732k,nr_inodes=2030433,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=1628440k,mode=755)
/dev/nvme0n1p2 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
efivarfs on /sys/firmware/efi/efivars type efivarfs (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
/dev/nvme0n1p1 on /boot/efi type vfat (rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro)
/dev/nvme0n1p2 on /var/lib/docker/aufs type ext4 (rw,relatime,errors=remount-ro,data=ordered)
gvfsd-fuse on /run/user/1002/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1002,group_id=1002)
''', 0, []),
# OS X, no CIFS
(r'''/dev/disk2 on / (hfs, local, journaled)
devfs on /dev (devfs, local, nobrowse)
map -hosts on /net (autofs, nosuid, automounted, nobrowse)
map auto_home on /home (autofs, automounted, nobrowse)
map -fstab on /Network/Servers (autofs, automounted, nobrowse)
/dev/disk3s2 on /Volumes/MyBookData (hfs, local, nodev, nosuid, journaled)
afni:/elrond0 on /Volumes/afni (nfs)
afni:/var/www/INCOMING on /Volumes/INCOMING (nfs)
afni:/fraid on /Volumes/afni (nfs, asynchronous)
boromir:/raid.bot on /Volumes/raid.bot (nfs)
elros:/volume2/AFNI_SHARE on /Volumes/AFNI_SHARE (nfs)
map -static on /Volumes/safni (autofs, automounted, nobrowse)
map -static on /Volumes/raid.top (autofs, automounted, nobrowse)
/dev/disk1s3 on /Volumes/Boot OS X (hfs, local, journaled, nobrowse)
''', 0, []),
# Non-zero exit code
('', 1, []),
# Variant of Linux example with CIFS added manually
(r'''sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=8121732k,nr_inodes=2030433,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=1628440k,mode=755)
/dev/nvme0n1p2 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
efivarfs on /sys/firmware/efi/efivars type efivarfs (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
/dev/nvme0n1p1 on /boot/efi type vfat (rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro)
/dev/nvme0n1p2 on /var/lib/docker/aufs type ext4 (rw,relatime,errors=remount-ro,data=ordered)
gvfsd-fuse on /run/user/1002/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1002,group_id=1002)
''', 0, []),
# Variant of OS X example with CIFS added manually
(r'''/dev/disk2 on / (hfs, local, journaled)
devfs on /dev (devfs, local, nobrowse)
afni:/elrond0 on /Volumes/afni (cifs)
afni:/var/www/INCOMING on /Volumes/INCOMING (nfs)
afni:/fraid on /Volumes/afni/fraid (nfs, asynchronous)
boromir:/raid.bot on /Volumes/raid.bot (nfs)
elros:/volume2/AFNI_SHARE on /Volumes/AFNI_SHARE (nfs)
''', 0, [('/Volumes/afni/fraid', 'nfs'), ('/Volumes/afni', 'cifs')]),
# From Windows: docker run --rm -it -v C:\:/data busybox mount
(r'''overlay on / type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/26UTYITLF24YE7KEGTMHUNHPPG:/var/lib/docker/overlay2/l/SWGNP3T2EEB4CNBJFN3SDZLXHP,upperdir=/var/lib/docker/overlay2/a4c54ab1aa031bb5a14a424abd655510521e183ee4fa4158672e8376c89df394/diff,workdir=/var/lib/docker/overlay2/a4c54ab1aa031bb5a14a424abd655510521e183ee4fa4158672e8376c89df394/work)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev type tmpfs (rw,nosuid,size=65536k,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666)
sysfs on /sys type sysfs (ro,nosuid,nodev,noexec,relatime)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,relatime,mode=755)
cpuset on /sys/fs/cgroup/cpuset type cgroup (ro,nosuid,nodev,noexec,relatime,cpuset)
cpu on /sys/fs/cgroup/cpu type cgroup (ro,nosuid,nodev,noexec,relatime,cpu)
cpuacct on /sys/fs/cgroup/cpuacct type cgroup (ro,nosuid,nodev,noexec,relatime,cpuacct)
blkio on /sys/fs/cgroup/blkio type cgroup (ro,nosuid,nodev,noexec,relatime,blkio)
memory on /sys/fs/cgroup/memory type cgroup (ro,nosuid,nodev,noexec,relatime,memory)
devices on /sys/fs/cgroup/devices type cgroup (ro,nosuid,nodev,noexec,relatime,devices)
freezer on /sys/fs/cgroup/freezer type cgroup (ro,nosuid,nodev,noexec,relatime,freezer)
net_cls on /sys/fs/cgroup/net_cls type cgroup (ro,nosuid,nodev,noexec,relatime,net_cls)
perf_event on /sys/fs/cgroup/perf_event type cgroup (ro,nosuid,nodev,noexec,relatime,perf_event)
net_prio on /sys/fs/cgroup/net_prio type cgroup (ro,nosuid,nodev,noexec,relatime,net_prio)
hugetlb on /sys/fs/cgroup/hugetlb type cgroup (ro,nosuid,nodev,noexec,relatime,hugetlb)
pids on /sys/fs/cgroup/pids type cgroup (ro,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/systemd type cgroup (ro,nosuid,nodev,noexec,relatime,name=systemd)
mqueue on /dev/mqueue type mqueue (rw,nosuid,nodev,noexec,relatime)
//10.0.75.1/C on /data type cifs (rw,relatime,vers=3.02,sec=ntlmsspi,cache=strict,username=filo,domain=MSI,uid=0,noforceuid,gid=0,noforcegid,addr=10.0.75.1,file_mode=0755,dir_mode=0755,iocharset=utf8,nounix,serverino,mapposix,nobrl,mfsymlinks,noperm,rsize=1048576,wsize=1048576,echo_interval=60,actimeo=1)
/dev/sda1 on /etc/resolv.conf type ext4 (rw,relatime,data=ordered)
/dev/sda1 on /etc/hostname type ext4 (rw,relatime,data=ordered)
/dev/sda1 on /etc/hosts type ext4 (rw,relatime,data=ordered)
shm on /dev/shm type tmpfs (rw,nosuid,nodev,noexec,relatime,size=65536k)
devpts on /dev/console type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666)
proc on /proc/bus type proc (ro,relatime)
proc on /proc/fs type proc (ro,relatime)
proc on /proc/irq type proc (ro,relatime)
proc on /proc/sys type proc (ro,relatime)
proc on /proc/sysrq-trigger type proc (ro,relatime)
tmpfs on /proc/kcore type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/timer_list type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/sched_debug type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/scsi type tmpfs (ro,relatime)
tmpfs on /sys/firmware type tmpfs (ro,relatime)
''', 0, [('/data', 'cifs')])
)


@pytest.mark.parametrize("output, exit_code, expected", MOUNT_OUTPUTS)
def test_parse_mount_table(output, exit_code, expected):
assert _parse_mount_table(exit_code, output) == expected


def test_cifs_check():
assert isinstance(_cifs_table, list)
assert isinstance(on_cifs('/'), bool)
Expand Down