Skip to content

Commit e4e95f3

Browse files
authored
Merge pull request #2238 from nipy/oesteban-patch-3
Replace chars when standard out can't be decoded
2 parents 3c3d2e9 + 70ab1aa commit e4e95f3

File tree

2 files changed

+37
-18
lines changed

2 files changed

+37
-18
lines changed

nipype/interfaces/base.py

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
from ..utils.provenance import write_provenance
3939
from ..utils.misc import is_container, trim, str2bool
4040
from ..utils.filemanip import (md5, hash_infile, FileNotFoundError, hash_timestamp,
41-
split_filename, to_str)
41+
split_filename, to_str, read_stream)
4242
from .traits_extension import (
4343
traits, Undefined, TraitDictObject, TraitListObject, TraitError, isdefined,
4444
File, Directory, DictStrStr, has_metadata, ImageFile)
@@ -1268,9 +1268,7 @@ def __init__(self, name, impl):
12681268
self._buf = ''
12691269
self._rows = []
12701270
self._lastidx = 0
1271-
self.default_encoding = locale.getdefaultlocale()[1]
1272-
if self.default_encoding is None:
1273-
self.default_encoding = 'UTF-8'
1271+
self.default_encoding = locale.getdefaultlocale()[1] or 'UTF-8'
12741272

12751273
def fileno(self):
12761274
"Pass-through for file descriptor."
@@ -1349,10 +1347,6 @@ def run_command(runtime, output=None, timeout=0.01):
13491347
cmdline = runtime.cmdline
13501348
env = _canonicalize_env(runtime.environ)
13511349

1352-
default_encoding = locale.getdefaultlocale()[1]
1353-
if default_encoding is None:
1354-
default_encoding = 'UTF-8'
1355-
13561350
errfile = None
13571351
outfile = None
13581352
stdout = sp.PIPE
@@ -1420,19 +1414,22 @@ def _process(drain=0):
14201414

14211415
if output == 'allatonce':
14221416
stdout, stderr = proc.communicate()
1423-
result['stdout'] = stdout.decode(default_encoding).split('\n')
1424-
result['stderr'] = stderr.decode(default_encoding).split('\n')
1417+
result['stdout'] = read_stream(stdout, logger=iflogger)
1418+
result['stderr'] = read_stream(stderr, logger=iflogger)
14251419

14261420
elif output.startswith('file'):
14271421
proc.wait()
14281422
if outfile is not None:
14291423
stdout.flush()
1430-
result['stdout'] = [line.decode(default_encoding).strip()
1431-
for line in open(outfile, 'rb').readlines()]
1424+
with open(outfile, 'rb') as ofh:
1425+
stdoutstr = ofh.read()
1426+
result['stdout'] = read_stream(stdoutstr, logger=iflogger)
1427+
14321428
if errfile is not None:
14331429
stderr.flush()
1434-
result['stderr'] = [line.decode(default_encoding).strip()
1435-
for line in open(errfile, 'rb').readlines()]
1430+
with open(errfile, 'rb') as efh:
1431+
stderrstr = efh.read()
1432+
result['stderr'] = read_stream(stderrstr, logger=iflogger)
14361433

14371434
if output == 'file':
14381435
result['merged'] = result['stdout']

nipype/utils/filemanip.py

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,13 @@
55
66
"""
77
from __future__ import print_function, division, unicode_literals, absolute_import
8-
from builtins import str, bytes, open
9-
10-
from future import standard_library
11-
standard_library.install_aliases()
128

139
import sys
1410
import pickle
1511
import subprocess
1612
import gzip
1713
import hashlib
14+
import locale
1815
from hashlib import md5
1916
import os
2017
import re
@@ -23,10 +20,15 @@
2320
import simplejson as json
2421
import numpy as np
2522

23+
from builtins import str, bytes, open
24+
2625
from .. import logging, config
2726
from .misc import is_container
2827
from ..interfaces.traits_extension import isdefined
2928

29+
from future import standard_library
30+
standard_library.install_aliases()
31+
3032
fmlogger = logging.getLogger('utils')
3133

3234

@@ -596,6 +598,26 @@ def crash2txt(filename, record):
596598
fp.write(''.join(record['traceback']))
597599

598600

601+
def read_stream(stream, logger=None, encoding=None):
602+
"""
603+
Robustly reads a stream, sending a warning to a logger
604+
if some decoding error was raised.
605+
606+
>>> read_stream(bytearray([65, 0xc7, 65, 10, 66])) # doctest: +ELLIPSIS +ALLOW_UNICODE
607+
['A...A', 'B']
608+
609+
610+
"""
611+
default_encoding = encoding or locale.getdefaultlocale()[1] or 'UTF-8'
612+
logger = logger or fmlogger
613+
try:
614+
out = stream.decode(default_encoding)
615+
except UnicodeDecodeError as err:
616+
out = stream.decode(default_encoding, errors='replace')
617+
logger.warning('Error decoding string: %s', err)
618+
return out.splitlines()
619+
620+
599621
def savepkl(filename, record):
600622
if filename.endswith('pklz'):
601623
pkl_file = gzip.open(filename, 'wb')

0 commit comments

Comments
 (0)