Skip to content

Commit 1829a61

Browse files
arw2019arw2019
authored
BUG in Series.interpolate: limit_area/limit_direction kwargs with method="pad"/"bfill" have no effect (#38106)
1 parent 43928e6 commit 1829a61

File tree

4 files changed

+161
-5
lines changed

4 files changed

+161
-5
lines changed

‎doc/source/whatsnew/v1.2.0.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -670,6 +670,7 @@ Missing
670670

671671
- Bug in :meth:`.SeriesGroupBy.transform` now correctly handles missing values for ``dropna=False`` (:issue:`35014`)
672672
- Bug in :meth:`Series.nunique` with ``dropna=True`` was returning incorrect results when both ``NA`` and ``None`` missing values were present (:issue:`37566`)
673+
- Bug in :meth:`Series.interpolate` where kwarg ``limit_area`` and ``limit_direction`` had no effect when using methods ``pad`` and ``backfill`` (:issue:`31048`)
673674
-
674675

675676
MultiIndex

‎pandas/core/internals/blocks.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1261,6 +1261,7 @@ def interpolate(
12611261
axis=axis,
12621262
inplace=inplace,
12631263
limit=limit,
1264+
limit_area=limit_area,
12641265
downcast=downcast,
12651266
)
12661267
# validate the interp method
@@ -1287,6 +1288,7 @@ def _interpolate_with_fill(
12871288
axis: int = 0,
12881289
inplace: bool = False,
12891290
limit: Optional[int] = None,
1291+
limit_area: Optional[str] = None,
12901292
downcast: Optional[str] = None,
12911293
) -> List["Block"]:
12921294
""" fillna but using the interpolate machinery """
@@ -1301,6 +1303,7 @@ def _interpolate_with_fill(
13011303
method=method,
13021304
axis=axis,
13031305
limit=limit,
1306+
limit_area=limit_area,
13041307
)
13051308

13061309
blocks = [self.make_block_same_class(values, ndim=self.ndim)]

‎pandas/core/missing.py

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
"""
22
Routines for filling missing data.
33
"""
4-
4+
from functools import partial
55
from typing import Any, List, Optional, Set, Union
66

77
import numpy as np
88

99
from pandas._libs import algos, lib
10-
from pandas._typing import ArrayLike, DtypeObj
10+
from pandas._typing import ArrayLike, Axis, DtypeObj
1111
from pandas.compat._optional import import_optional_dependency
1212

1313
from pandas.core.dtypes.cast import infer_dtype_from_array
@@ -528,16 +528,92 @@ def _cubicspline_interpolate(xi, yi, x, axis=0, bc_type="not-a-knot", extrapolat
528528
return P(x)
529529

530530

531+
def _interpolate_with_limit_area(
532+
values: ArrayLike, method: str, limit: Optional[int], limit_area: Optional[str]
533+
) -> ArrayLike:
534+
"""
535+
Apply interpolation and limit_area logic to values along a to-be-specified axis.
536+
537+
Parameters
538+
----------
539+
values: array-like
540+
Input array.
541+
method: str
542+
Interpolation method. Could be "bfill" or "pad"
543+
limit: int, optional
544+
Index limit on interpolation.
545+
limit_area: str
546+
Limit area for interpolation. Can be "inside" or "outside"
547+
548+
Returns
549+
-------
550+
values: array-like
551+
Interpolated array.
552+
"""
553+
554+
invalid = isna(values)
555+
556+
if not invalid.all():
557+
first = find_valid_index(values, "first")
558+
last = find_valid_index(values, "last")
559+
560+
values = interpolate_2d(
561+
values,
562+
method=method,
563+
limit=limit,
564+
)
565+
566+
if limit_area == "inside":
567+
invalid[first : last + 1] = False
568+
elif limit_area == "outside":
569+
invalid[:first] = invalid[last + 1 :] = False
570+
571+
values[invalid] = np.nan
572+
573+
return values
574+
575+
531576
def interpolate_2d(
532577
values,
533-
method="pad",
534-
axis=0,
535-
limit=None,
578+
method: str = "pad",
579+
axis: Axis = 0,
580+
limit: Optional[int] = None,
581+
limit_area: Optional[str] = None,
536582
):
537583
"""
538584
Perform an actual interpolation of values, values will be make 2-d if
539585
needed fills inplace, returns the result.
586+
587+
Parameters
588+
----------
589+
values: array-like
590+
Input array.
591+
method: str, default "pad"
592+
Interpolation method. Could be "bfill" or "pad"
593+
axis: 0 or 1
594+
Interpolation axis
595+
limit: int, optional
596+
Index limit on interpolation.
597+
limit_area: str, optional
598+
Limit area for interpolation. Can be "inside" or "outside"
599+
600+
Returns
601+
-------
602+
values: array-like
603+
Interpolated array.
540604
"""
605+
if limit_area is not None:
606+
return np.apply_along_axis(
607+
partial(
608+
_interpolate_with_limit_area,
609+
method=method,
610+
limit=limit,
611+
limit_area=limit_area,
612+
),
613+
axis,
614+
values,
615+
)
616+
541617
orig_values = values
542618

543619
transf = (lambda x: x) if axis == 0 else (lambda x: x.T)

‎pandas/tests/series/methods/test_interpolate.py

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,82 @@ def test_interp_limit_direction_raises(self, method, limit_direction, expected):
458458
with pytest.raises(ValueError, match=msg):
459459
s.interpolate(method=method, limit_direction=limit_direction)
460460

461+
@pytest.mark.parametrize(
462+
"data, expected_data, kwargs",
463+
(
464+
(
465+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
466+
[np.nan, np.nan, 3.0, 3.0, 3.0, 3.0, 7.0, np.nan, np.nan],
467+
{"method": "pad", "limit_area": "inside"},
468+
),
469+
(
470+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
471+
[np.nan, np.nan, 3.0, 3.0, np.nan, np.nan, 7.0, np.nan, np.nan],
472+
{"method": "pad", "limit_area": "inside", "limit": 1},
473+
),
474+
(
475+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
476+
[np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, 7.0],
477+
{"method": "pad", "limit_area": "outside"},
478+
),
479+
(
480+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
481+
[np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, np.nan],
482+
{"method": "pad", "limit_area": "outside", "limit": 1},
483+
),
484+
(
485+
[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
486+
[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
487+
{"method": "pad", "limit_area": "outside", "limit": 1},
488+
),
489+
(
490+
range(5),
491+
range(5),
492+
{"method": "pad", "limit_area": "outside", "limit": 1},
493+
),
494+
),
495+
)
496+
def test_interp_limit_area_with_pad(self, data, expected_data, kwargs):
497+
# GH26796
498+
499+
s = Series(data)
500+
expected = Series(expected_data)
501+
result = s.interpolate(**kwargs)
502+
tm.assert_series_equal(result, expected)
503+
504+
@pytest.mark.parametrize(
505+
"data, expected_data, kwargs",
506+
(
507+
(
508+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
509+
[np.nan, np.nan, 3.0, 7.0, 7.0, 7.0, 7.0, np.nan, np.nan],
510+
{"method": "bfill", "limit_area": "inside"},
511+
),
512+
(
513+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
514+
[np.nan, np.nan, 3.0, np.nan, np.nan, 7.0, 7.0, np.nan, np.nan],
515+
{"method": "bfill", "limit_area": "inside", "limit": 1},
516+
),
517+
(
518+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
519+
[3.0, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan],
520+
{"method": "bfill", "limit_area": "outside"},
521+
),
522+
(
523+
[np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan],
524+
[np.nan, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan],
525+
{"method": "bfill", "limit_area": "outside", "limit": 1},
526+
),
527+
),
528+
)
529+
def test_interp_limit_area_with_backfill(self, data, expected_data, kwargs):
530+
# GH26796
531+
532+
s = Series(data)
533+
expected = Series(expected_data)
534+
result = s.interpolate(**kwargs)
535+
tm.assert_series_equal(result, expected)
536+
461537
def test_interp_limit_direction(self):
462538
# These tests are for issue #9218 -- fill NaNs in both directions.
463539
s = Series([1, 3, np.nan, np.nan, np.nan, 11])

0 commit comments

Comments
 (0)