Skip to content

Replace chars when standard out can't be decoded #2238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 20, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 11 additions & 14 deletions nipype/interfaces/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
from ..utils.provenance import write_provenance
from ..utils.misc import is_container, trim, str2bool
from ..utils.filemanip import (md5, hash_infile, FileNotFoundError, hash_timestamp,
split_filename, to_str)
split_filename, to_str, read_stream)
from .traits_extension import (
traits, Undefined, TraitDictObject, TraitListObject, TraitError, isdefined,
File, Directory, DictStrStr, has_metadata, ImageFile)
Expand Down Expand Up @@ -1268,9 +1268,7 @@ def __init__(self, name, impl):
self._buf = ''
self._rows = []
self._lastidx = 0
self.default_encoding = locale.getdefaultlocale()[1]
if self.default_encoding is None:
self.default_encoding = 'UTF-8'
self.default_encoding = locale.getdefaultlocale()[1] or 'UTF-8'

def fileno(self):
"Pass-through for file descriptor."
Expand Down Expand Up @@ -1349,10 +1347,6 @@ def run_command(runtime, output=None, timeout=0.01):
cmdline = runtime.cmdline
env = _canonicalize_env(runtime.environ)

default_encoding = locale.getdefaultlocale()[1]
if default_encoding is None:
default_encoding = 'UTF-8'

errfile = None
outfile = None
stdout = sp.PIPE
Expand Down Expand Up @@ -1420,19 +1414,22 @@ def _process(drain=0):

if output == 'allatonce':
stdout, stderr = proc.communicate()
result['stdout'] = stdout.decode(default_encoding).split('\n')
result['stderr'] = stderr.decode(default_encoding).split('\n')
result['stdout'] = read_stream(stdout, logger=iflogger)
result['stderr'] = read_stream(stderr, logger=iflogger)

elif output.startswith('file'):
proc.wait()
if outfile is not None:
stdout.flush()
result['stdout'] = [line.decode(default_encoding).strip()
for line in open(outfile, 'rb').readlines()]
with open(outfile, 'rb') as ofh:
stdoutstr = ofh.read()
result['stdout'] = read_stream(stdoutstr, logger=iflogger)

if errfile is not None:
stderr.flush()
result['stderr'] = [line.decode(default_encoding).strip()
for line in open(errfile, 'rb').readlines()]
with open(errfile, 'rb') as efh:
stderrstr = efh.read()
result['stderr'] = read_stream(stderrstr, logger=iflogger)

if output == 'file':
result['merged'] = result['stdout']
Expand Down
30 changes: 26 additions & 4 deletions nipype/utils/filemanip.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,13 @@

"""
from __future__ import print_function, division, unicode_literals, absolute_import
from builtins import str, bytes, open

from future import standard_library
standard_library.install_aliases()

import sys
import pickle
import subprocess
import gzip
import hashlib
import locale
from hashlib import md5
import os
import re
Expand All @@ -23,10 +20,15 @@
import simplejson as json
import numpy as np

from builtins import str, bytes, open

from .. import logging, config
from .misc import is_container
from ..interfaces.traits_extension import isdefined

from future import standard_library
standard_library.install_aliases()

fmlogger = logging.getLogger('utils')


Expand Down Expand Up @@ -596,6 +598,26 @@ def crash2txt(filename, record):
fp.write(''.join(record['traceback']))


def read_stream(stream, logger=None, encoding=None):
"""
Robustly reads a stream, sending a warning to a logger
if some decoding error was raised.

>>> read_stream(bytearray([65, 0xc7, 65, 10, 66])) # doctest: +ELLIPSIS +ALLOW_UNICODE
['A...A', 'B']


"""
default_encoding = encoding or locale.getdefaultlocale()[1] or 'UTF-8'
logger = logger or fmlogger
try:
out = stream.decode(default_encoding)
except UnicodeDecodeError as err:
out = stream.decode(default_encoding, errors='replace')
logger.warning('Error decoding string: %s', err)
return out.splitlines()


def savepkl(filename, record):
if filename.endswith('pklz'):
pkl_file = gzip.open(filename, 'wb')
Expand Down