Skip to content

ENH: Add lazy copy for sort_values #50643

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jan 13, 2023
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,7 @@ Indexing
- Bug in :meth:`DataFrame.__setitem__` raising when indexer is a :class:`DataFrame` with ``boolean`` dtype (:issue:`47125`)
- Bug in :meth:`DataFrame.reindex` filling with wrong values when indexing columns and index for ``uint`` dtypes (:issue:`48184`)
- Bug in :meth:`DataFrame.loc` when setting :class:`DataFrame` with different dtypes coercing values to single dtype (:issue:`50467`)
- Bug in :meth:`DataFrame.sort_values` incorrectly returning object when ``by`` is empty list and ``inplace=True`` (:issue:`1`)
- Bug in :meth:`DataFrame.loc` coercing dtypes when setting values with a list indexer (:issue:`49159`)
- Bug in :meth:`Series.loc` raising error for out of bounds end of slice indexer (:issue:`50161`)
- Bug in :meth:`DataFrame.loc` raising ``ValueError`` with ``bool`` indexer and :class:`MultiIndex` (:issue:`47687`)
Expand Down
12 changes: 11 additions & 1 deletion pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
from pandas._libs.hashtable import duplicated
from pandas._libs.lib import (
NoDefault,
array_equal_fast,
no_default,
)
from pandas._typing import (
Expand Down Expand Up @@ -6688,7 +6689,16 @@ def sort_values(
k, kind=kind, ascending=ascending, na_position=na_position, key=key
)
else:
return self.copy()
if inplace:
return self._update_inplace(self)
else:
return self.copy(deep=None)

if array_equal_fast(indexer, np.arange(0, len(indexer), dtype=indexer.dtype)):
if inplace:
return self._update_inplace(self)
else:
return self.copy(deep=None)

new_data = self._mgr.take(
indexer, axis=self._get_block_manager_axis(axis), verify=False
Expand Down
12 changes: 11 additions & 1 deletion pandas/core/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@
properties,
reshape,
)
from pandas._libs.lib import no_default
from pandas._libs.lib import (
array_equal_fast,
no_default,
)
from pandas._typing import (
AggFuncType,
AlignJoin,
Expand Down Expand Up @@ -3548,6 +3551,13 @@ def sort_values(
values_to_sort = ensure_key_mapped(self, key)._values if key else self._values
sorted_index = nargsort(values_to_sort, kind, bool(ascending), na_position)

if array_equal_fast(
sorted_index, np.arange(0, len(sorted_index), dtype=sorted_index.dtype)
):
if inplace:
return self._update_inplace(self)
return self.copy(deep=None)

result = self._constructor(
self._values[sorted_index], index=self.index[sorted_index]
)
Expand Down
43 changes: 43 additions & 0 deletions pandas/tests/copy_view/test_methods.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import numpy as np
import pytest

import pandas as pd
from pandas import (
DataFrame,
Index,
Expand Down Expand Up @@ -575,6 +576,48 @@ def test_sort_index(using_copy_on_write):
tm.assert_series_equal(ser, ser_orig)


@pytest.mark.parametrize(
"obj, kwargs", [(Series([1, 2, 3]), {}), (DataFrame({"a": [1, 2, 3]}), {"by": "a"})]
)
def test_sort_values(using_copy_on_write, obj, kwargs):
obj_orig = obj.copy()
obj2 = obj.sort_values(**kwargs)

if using_copy_on_write:
assert np.shares_memory(obj.values, obj2.values)
else:
assert not np.shares_memory(obj.values, obj2.values)

# mutating df triggers a copy-on-write for the column / block
obj2.iloc[0] = 0
assert not np.shares_memory(obj2.values, obj.values)
tm.assert_equal(obj, obj_orig)


@pytest.mark.parametrize(
"obj, kwargs", [(Series([1, 2, 3]), {}), (DataFrame({"a": [1, 2, 3]}), {"by": "a"})]
)
def test_sort_values_inplace(using_copy_on_write, obj, kwargs, using_array_manager):
obj_orig = obj.copy()
view = obj[:]
obj.sort_values(inplace=True, **kwargs)

assert np.shares_memory(obj.values, view.values)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, this is failing with ArrayManager (failing CI)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.values on a DataFrame is always a copy for ArrayManager, so the generic obj.values doesn't work here:

Suggested change
assert np.shares_memory(obj.values, view.values)
if isinstance(obj, DataFrame):
assert np.shares_memory(get_array(obj, "a"), get_array(view, "a"))
else:
assert np.shares_memory(obj.values, view.values)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You will have to use the same pattern below, so can probably simplify this if we give the Series a name so can use get_array in all cases.

Copy link
Member

@jorisvandenbossche jorisvandenbossche Jan 13, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While testing this, I just pushed the changes I was doing locally, hopefully you weren't yet working on it.


# mutating obj triggers a copy-on-write for the column / block
obj.iloc[0] = 0
if (
using_copy_on_write
or using_array_manager
and isinstance(obj, DataFrame)
and pd.options.mode.copy_on_write
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ArrayManager never uses Copy-on-Write (we should maybe error when setting those two options at the same time), so this check (and the or using_array_manager two lines above) shouldn't be needed. I suppose the reason it seemingly did give a copy is because of the .values giving a copy

):
assert not np.shares_memory(view.values, obj.values)
tm.assert_equal(view, obj_orig)
else:
assert np.shares_memory(view.values, obj.values)


def test_reorder_levels(using_copy_on_write):
index = MultiIndex.from_tuples(
[(1, 1), (1, 2), (2, 1), (2, 2)], names=["one", "two"]
Expand Down
8 changes: 8 additions & 0 deletions pandas/tests/frame/methods/test_sort_values.py
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,14 @@ def test_sort_values_reshaping(self):

tm.assert_frame_equal(df, expected)

def test_sort_values_no_by_inplace(self):
# GH#
df = DataFrame({"a": [1, 2, 3]})
expected = df.copy()
result = df.sort_values(by=[], inplace=True)
tm.assert_frame_equal(df, expected)
assert result is None


class TestDataFrameSortKey: # test key sorting (issue 27237)
def test_sort_values_inplace_key(self, sort_by_key):
Expand Down