Merge remote-tracking branch 'upstream/master' into remove-sparse

Author: TomAugspurger

doc/source/development/contributing.rst

@@ -208,7 +208,7 @@ We'll now kick off a three-step process:
 
    # Build and install pandas
    python setup.py build_ext --inplace -j 4
-   python -m pip install -e --no-build-isolation .
+   python -m pip install -e . --no-build-isolation
 
 At this point you should be able to import pandas from your locally built version::
 
@@ -252,7 +252,7 @@ You'll need to have at least python3.5 installed on your system.
 
    # Build and install pandas
    python setup.py build_ext --inplace -j 4
-   python -m pip install -e --no-build-isolation .
+   python -m pip install -e . --no-build-isolation
 
 Creating a branch
 -----------------

doc/source/user_guide/options.rst

@@ -353,7 +353,7 @@ display.max_colwidth                    50           The maximum width in charac
                                                      a column in the repr of a pandas
                                                      data structure. When the column overflows,
                                                      a "..." placeholder is embedded in
-                                                     the output.
+                                                     the output. 'None' value means unlimited.
 display.max_info_columns                100          max_info_columns is used in DataFrame.info
                                                      method to decide if per column information
                                                      will be printed.

doc/source/whatsnew/v1.0.0.rst

@@ -21,6 +21,8 @@ including other versions of pandas.
 Enhancements
 ~~~~~~~~~~~~
 
+- :meth:`DataFrame.to_string` added the ``max_colwidth`` parameter to control when wide columns are truncated (:issue:`9784`)
+-
 
 .. _whatsnew_1000.enhancements.other:
 
@@ -200,6 +202,7 @@ I/O
 - Bug in :meth:`DataFrame.to_json` where using a Tuple as a column or index value and using ``orient="columns"`` or ``orient="index"`` would produce invalid JSON (:issue:`20500`)
 - Improve infinity parsing. :meth:`read_csv` now interprets ``Infinity``, ``+Infinity``, ``-Infinity`` as floating point values (:issue:`10065`)
 - Bug in :meth:`DataFrame.to_csv` where values were truncated when the length of ``na_rep`` was shorter than the text input data. (:issue:`25099`)
+- Bug in :func:`DataFrame.to_string` where values were truncated using display options instead of outputting the full content (:issue:`9784`)
 
 Plotting
 ^^^^^^^^

pandas/core/arrays/sparse.py

@@ -244,10 +244,10 @@ def construct_from_string(cls, string):
         if string.startswith("Sparse"):
             try:
                 sub_type, has_fill_value = cls._parse_subtype(string)
-                result = SparseDtype(sub_type)
-            except Exception:
+            except ValueError:
                 raise TypeError(msg)
             else:
+                result = SparseDtype(sub_type)
                 msg = (
                     "Could not construct SparseDtype from '{}'.\n\nIt "
                     "looks like the fill_value in the string is not "

pandas/core/base.py

@@ -1289,17 +1289,17 @@ def value_counts(
 
         Parameters
         ----------
-        normalize : boolean, default False
+        normalize : bool, default False
             If True then the object returned will contain the relative
             frequencies of the unique values.
-        sort : boolean, default True
+        sort : bool, default True
             Sort by frequencies.
-        ascending : boolean, default False
+        ascending : bool, default False
             Sort in ascending order.
-        bins : integer, optional
+        bins : int, optional
             Rather than count values, group them into half-open bins,
             a convenience for ``pd.cut``, only works with numeric data.
-        dropna : boolean, default True
+        dropna : bool, default True
             Don't include counts of NaN.
 
         Returns
@@ -1496,7 +1496,7 @@ def memory_usage(self, deep=False):
         size_hint="",
         sort=textwrap.dedent(
             """\
-            sort : boolean, default False
+            sort : bool, default False
                 Sort `uniques` and shuffle `labels` to maintain the
                 relationship.
             """

pandas/core/common.py

@@ -445,15 +445,15 @@ def pipe(obj, func, *args, **kwargs):
 
     Parameters
     ----------
-    func : callable or tuple of (callable, string)
+    func : callable or tuple of (callable, str)
         Function to apply to this object or, alternatively, a
         ``(callable, data_keyword)`` tuple where ``data_keyword`` is a
         string indicating the keyword of `callable`` that expects the
         object.
-    args : iterable, optional
-        positional arguments passed into ``func``.
-    kwargs : dict, optional
-        a dictionary of keyword arguments passed into ``func``.
+    *args : iterable, optional
+        Positional arguments passed into ``func``.
+    **kwargs : dict, optional
+        A dictionary of keyword arguments passed into ``func``.
 
     Returns
     -------

pandas/core/config_init.py

@@ -148,10 +148,10 @@ def use_numexpr_cb(key):
 """
 
 max_colwidth_doc = """
-: int
+: int or None
     The maximum width in characters of a column in the repr of
     a pandas data structure. When the column overflows, a "..."
-    placeholder is embedded in the output.
+    placeholder is embedded in the output. A 'None' value means unlimited.
 """
 
 colheader_justify_doc = """
@@ -340,7 +340,9 @@ def is_terminal():
         validator=is_instance_factory([type(None), int]),
     )
     cf.register_option("max_categories", 8, pc_max_categories_doc, validator=is_int)
-    cf.register_option("max_colwidth", 50, max_colwidth_doc, validator=is_int)
+    cf.register_option(
+        "max_colwidth", 50, max_colwidth_doc, validator=is_nonnegative_int
+    )
     if is_terminal():
         max_cols = 0  # automatically determine optimal number of columns
     else:

pandas/core/dtypes/common.py

@@ -2034,10 +2034,8 @@ def pandas_dtype(dtype):
     # raise a consistent TypeError if failed
     try:
         npdtype = np.dtype(dtype)
-    except Exception:
-        # we don't want to force a repr of the non-string
-        if not isinstance(dtype, str):
-            raise TypeError("data type not understood")
+    except SyntaxError:
+        # np.dtype uses `eval` which can raise SyntaxError
         raise TypeError("data type '{}' not understood".format(dtype))
 
     # Any invalid dtype (such as pd.Timestamp) should raise an error.

pandas/core/frame.py

@@ -641,6 +641,7 @@ def __repr__(self):
         max_rows = get_option("display.max_rows")
         min_rows = get_option("display.min_rows")
         max_cols = get_option("display.max_columns")
+        max_colwidth = get_option("display.max_colwidth")
         show_dimensions = get_option("display.show_dimensions")
         if get_option("display.expand_frame_repr"):
             width, _ = console.get_console_size()
@@ -652,6 +653,7 @@ def __repr__(self):
             min_rows=min_rows,
             max_cols=max_cols,
             line_width=width,
+            max_colwidth=max_colwidth,
             show_dimensions=show_dimensions,
         )
 
@@ -730,12 +732,17 @@ def to_string(
         show_dimensions=False,
         decimal=".",
         line_width=None,
+        max_colwidth=None,
     ):
         """
         Render a DataFrame to a console-friendly tabular output.
         %(shared_params)s
         line_width : int, optional
             Width to wrap a line in characters.
+        max_colwidth : int, optional
+            Max width to truncate each column in characters. By default, no limit.
+
+            .. versionadded:: 1.0.0
         %(returns)s
         See Also
         --------
@@ -752,26 +759,29 @@ def to_string(
         2     3     6
         """
 
-        formatter = fmt.DataFrameFormatter(
-            self,
-            columns=columns,
-            col_space=col_space,
-            na_rep=na_rep,
-            formatters=formatters,
-            float_format=float_format,
-            sparsify=sparsify,
-            justify=justify,
-            index_names=index_names,
-            header=header,
-            index=index,
-            min_rows=min_rows,
-            max_rows=max_rows,
-            max_cols=max_cols,
-            show_dimensions=show_dimensions,
-            decimal=decimal,
-            line_width=line_width,
-        )
-        return formatter.to_string(buf=buf)
+        from pandas import option_context
+
+        with option_context("display.max_colwidth", max_colwidth):
+            formatter = fmt.DataFrameFormatter(
+                self,
+                columns=columns,
+                col_space=col_space,
+                na_rep=na_rep,
+                formatters=formatters,
+                float_format=float_format,
+                sparsify=sparsify,
+                justify=justify,
+                index_names=index_names,
+                header=header,
+                index=index,
+                min_rows=min_rows,
+                max_rows=max_rows,
+                max_cols=max_cols,
+                show_dimensions=show_dimensions,
+                decimal=decimal,
+                line_width=line_width,
+            )
+            return formatter.to_string(buf=buf)
 
     # ----------------------------------------------------------------------
 
@@ -2098,8 +2108,12 @@ def to_parquet(
             Name of the compression to use. Use ``None`` for no compression.
         index : bool, default None
             If ``True``, include the dataframe's index(es) in the file output.
-            If ``False``, they will not be written to the file. If ``None``,
-            the behavior depends on the chosen engine.
+            If ``False``, they will not be written to the file.
+            If ``None``, similar to ``True`` the dataframe's index(es)
+            will be saved. However, instead of being saved as values,
+            the RangeIndex will be stored as a range in the metadata so it
+            doesn't require much space and is faster. Other indexes will
+            be included as columns in the file output.
 
             .. versionadded:: 0.24.0
 

pandas/core/generic.py

@@ -2310,7 +2310,7 @@ def to_json(
             floating point values.
         force_ascii : bool, default True
             Force encoded string to be ASCII.
-        date_unit : string, default 'ms' (milliseconds)
+        date_unit : str, default 'ms' (milliseconds)
             The time unit to encode to, governs timestamp and ISO8601
             precision.  One of 's', 'ms', 'us', 'ns' for second, millisecond,
             microsecond, and nanosecond respectively.
@@ -2530,8 +2530,9 @@ def to_msgpack(self, path_or_buf=None, encoding="utf-8", **kwargs):
 
         Parameters
         ----------
-        path : string File path, buffer-like, or None
-            if None, return generated bytes
+        path : str, buffer-like, or None
+            Destination for the serialized object.
+            If None, return generated bytes
         append : bool whether to append to an existing msgpack
             (default is False)
         compress : type of compressor (zlib or blosc), default to None (no
@@ -4618,8 +4619,9 @@ def filter(self, items=None, like=None, regex=None, axis=None):
             Keep labels from axis for which "like in label == True".
         regex : str (regular expression)
             Keep labels from axis for which re.search(regex, label) == True.
-        axis : int or string axis name
-            The axis to filter on.  By default this is the info axis,
+        axis : {0 or ‘index’, 1 or ‘columns’, None}, default None
+            The axis to filter on, expressed either as an index (int)
+            or axis name (str). By default this is the info axis,
             'index' for Series, 'columns' for DataFrame.
 
         Returns
@@ -4852,7 +4854,7 @@ def sample(
         random_state : int or numpy.random.RandomState, optional
             Seed for the random number generator (if int), or numpy RandomState
             object.
-        axis : int or string, optional
+        axis : {0 or ‘index’, 1 or ‘columns’, None}, default None
             Axis to sample. Accepts axis number or name. Default is stat axis
             for given data type (0 for Series and DataFrames).
 
@@ -8445,7 +8447,7 @@ def first(self, offset):
 
         Parameters
         ----------
-        offset : string, DateOffset, dateutil.relativedelta
+        offset : str, DateOffset, dateutil.relativedelta
 
         Returns
         -------
@@ -8508,7 +8510,7 @@ def last(self, offset):
 
         Parameters
         ----------
-        offset : string, DateOffset, dateutil.relativedelta
+        offset : str, DateOffset, dateutil.relativedelta
 
         Returns
         -------
@@ -9116,7 +9118,7 @@ def _where(
 
         Parameters
         ----------
-        cond : boolean %(klass)s, array-like, or callable
+        cond : bool %(klass)s, array-like, or callable
             Where `cond` is %(cond)s, keep the original value. Where
             %(cond_rev)s, replace with corresponding value from `other`.
             If `cond` is callable, it is computed on the %(klass)s and
@@ -9423,9 +9425,10 @@ def tshift(self, periods=1, freq=None, axis=0):
         ----------
         periods : int
             Number of periods to move, can be positive or negative
-        freq : DateOffset, timedelta, or time rule string, default None
-            Increment to use from the tseries module or time rule (e.g. 'EOM')
-        axis : int or basestring
+        freq : DateOffset, timedelta, or str, default None
+            Increment to use from the tseries module
+            or time rule expressed as a string (e.g. 'EOM')
+        axis : {0 or ‘index’, 1 or ‘columns’, None}, default 0
             Corresponds to the axis that contains the Index
 
         Returns
@@ -9483,9 +9486,9 @@ def truncate(self, before=None, after=None, axis=None, copy=True):
 
         Parameters
         ----------
-        before : date, string, int
+        before : date, str, int
             Truncate all rows before this index value.
-        after : date, string, int
+        after : date, str, int
             Truncate all rows after this index value.
         axis : {0 or 'index', 1 or 'columns'}, optional
             Axis to truncate. Truncates the index (rows) by default.
@@ -10294,7 +10297,7 @@ def _check_percentile(self, q):
             How to handle NAs before computing percent changes.
         limit : int, default None
             The number of consecutive NAs to fill before stopping.
-        freq : DateOffset, timedelta, or offset alias string, optional
+        freq : DateOffset, timedelta, or str, optional
             Increment to use from time series API (e.g. 'M' or BDay()).
         **kwargs
             Additional keyword arguments are passed into

pandas/io/clipboards.py

@@ -131,7 +131,7 @@ def to_clipboard(obj, excel=True, sep=None, **kwargs):  # pragma: no cover
 
     if isinstance(obj, ABCDataFrame):
         # str(df) has various unhelpful defaults, like truncation
-        with option_context("display.max_colwidth", 999999):
+        with option_context("display.max_colwidth", None):
             objstr = obj.to_string(**kwargs)
     else:
         objstr = str(obj)

pandas/io/formats/html.py

@@ -377,7 +377,7 @@ def _write_header(self, indent: int) -> None:
         self.write("</thead>", indent)
 
     def _get_formatted_values(self) -> Dict[int, List[str]]:
-        with option_context("display.max_colwidth", 999999):
+        with option_context("display.max_colwidth", None):
             fmt_values = {i: self.fmt._format_col(i) for i in range(self.ncols)}
         return fmt_values
 

pandas/io/parquet.py

@@ -227,8 +227,12 @@ def to_parquet(
         Name of the compression to use. Use ``None`` for no compression.
     index : bool, default None
         If ``True``, include the dataframe's index(es) in the file output. If
-        ``False``, they will not be written to the file. If ``None``, the
-        engine's default behavior will be used.
+        ``False``, they will not be written to the file.
+        If ``None``, similar to ``True`` the dataframe's index(es)
+        will be saved. However, instead of being saved as values,
+        the RangeIndex will be stored as a range in the metadata so it
+        doesn't require much space and is faster. Other indexes will
+        be included as columns in the file output.
 
         .. versionadded:: 0.24.0
 

pandas/io/pytables.py

@@ -1782,7 +1782,7 @@ def convert(self, values, nan_rep, encoding, errors, start=None, stop=None):
         # making an Index instance could throw a number of different errors
         try:
             self.values = Index(values, **kwargs)
-        except Exception:  # noqa: E722
+        except Exception:
 
             # if the output freq is different that what we recorded,
             # it should be None (see also 'doc example part 2')