Replace chars when standard out can't be decoded

nipype/interfaces/base.py

@@ -38,7 +38,7 @@
 from ..utils.provenance import write_provenance
 from ..utils.misc import is_container, trim, str2bool
 from ..utils.filemanip import (md5, hash_infile, FileNotFoundError, hash_timestamp,
-                               split_filename, to_str)
+                               split_filename, to_str, read_stream)
 from .traits_extension import (
     traits, Undefined, TraitDictObject, TraitListObject, TraitError, isdefined,
     File, Directory, DictStrStr, has_metadata, ImageFile)
@@ -1268,9 +1268,7 @@ def __init__(self, name, impl):
         self._buf = ''
         self._rows = []
         self._lastidx = 0
-        self.default_encoding = locale.getdefaultlocale()[1]
-        if self.default_encoding is None:
-            self.default_encoding = 'UTF-8'
+        self.default_encoding = locale.getdefaultlocale()[1] or 'UTF-8'
 
     def fileno(self):
         "Pass-through for file descriptor."
@@ -1349,10 +1347,6 @@ def run_command(runtime, output=None, timeout=0.01):
     cmdline = runtime.cmdline
     env = _canonicalize_env(runtime.environ)
 
-    default_encoding = locale.getdefaultlocale()[1]
-    if default_encoding is None:
-        default_encoding = 'UTF-8'
-
     errfile = None
     outfile = None
     stdout = sp.PIPE
@@ -1420,19 +1414,22 @@ def _process(drain=0):
 
     if output == 'allatonce':
         stdout, stderr = proc.communicate()
-        result['stdout'] = stdout.decode(default_encoding).split('\n')
-        result['stderr'] = stderr.decode(default_encoding).split('\n')
+        result['stdout'] = read_stream(stdout, logger=iflogger)
+        result['stderr'] = read_stream(stderr, logger=iflogger)
 
     elif output.startswith('file'):
         proc.wait()
         if outfile is not None:
             stdout.flush()
-            result['stdout'] = [line.decode(default_encoding).strip()
-                                for line in open(outfile, 'rb').readlines()]
+            with open(outfile, 'rb') as ofh:
+                stdoutstr = ofh.read()
+            result['stdout'] = read_stream(stdoutstr, logger=iflogger)
+
         if errfile is not None:
             stderr.flush()
-            result['stderr'] = [line.decode(default_encoding).strip()
-                                for line in open(errfile, 'rb').readlines()]
+            with open(errfile, 'rb') as efh:
+                stderrstr = efh.read()
+            result['stderr'] = read_stream(stderrstr, logger=iflogger)
 
         if output == 'file':
             result['merged'] = result['stdout']

nipype/utils/filemanip.py

@@ -5,16 +5,13 @@
 
 """
 from __future__ import print_function, division, unicode_literals, absolute_import
-from builtins import str, bytes, open
-
-from future import standard_library
-standard_library.install_aliases()
 
 import sys
 import pickle
 import subprocess
 import gzip
 import hashlib
+import locale
 from hashlib import md5
 import os
 import re
@@ -23,10 +20,15 @@
 import simplejson as json
 import numpy as np
 
+from builtins import str, bytes, open
+
 from .. import logging, config
 from .misc import is_container
 from ..interfaces.traits_extension import isdefined
 
+from future import standard_library
+standard_library.install_aliases()
+
 fmlogger = logging.getLogger('utils')
 
 
@@ -596,6 +598,26 @@ def crash2txt(filename, record):
         fp.write(''.join(record['traceback']))
 
 
+def read_stream(stream, logger=None, encoding=None):
+    """
+    Robustly reads a stream, sending a warning to a logger
+    if some decoding error was raised.
+
+    >>> read_stream(bytearray([65, 0xc7, 65, 10, 66]))  # doctest: +ELLIPSIS +ALLOW_UNICODE
+    ['A...A', 'B']
+
+
+    """
+    default_encoding = encoding or locale.getdefaultlocale()[1] or 'UTF-8'
+    logger = logger or fmlogger
+    try:
+        out = stream.decode(default_encoding)
+    except UnicodeDecodeError as err:
+        out = stream.decode(default_encoding, errors='replace')
+        logger.warning('Error decoding string: %s', err)
+    return out.splitlines()
+
+
 def savepkl(filename, record):
     if filename.endswith('pklz'):
         pkl_file = gzip.open(filename, 'wb')