BUG: another ValueError: buffer source array is read-only

[dmitra79]

• I have checked that this issue has not already been reported.

• [X ] I have confirmed this bug exists on the latest version of pandas.

• (optional) I have confirmed this bug exists on the master branch of pandas.

---

This seems related to: #37174
however it shows up under different conditions - appears to be triggered by comparison involving NaN.

Code Sample, a copy-pastable example

print(temp)
temp[temp>180]=temp[temp>180]-360

Output:

2016-12-27 02:02:09.000    182.220
2016-12-27 02:02:09.017    182.226
2016-12-27 02:02:09.033    182.227
2016-12-27 02:02:09.050    182.222
2016-12-27 02:02:09.066        NaN
dtype: float64
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/series.py in __setitem__(self, key, value)
    999         try:
-> 1000             self._set_with_engine(key, value)
   1001         except (KeyError, ValueError):

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/series.py in _set_with_engine(self, key, value)
   1032         # fails with AttributeError for IntervalIndex
-> 1033         loc = self.index._engine.get_loc(key)
   1034         validate_numeric_casting(self.dtype, value)

pandas/_libs/index.pyx in pandas._libs.index.DatetimeEngine.get_loc()

pandas/_libs/index.pyx in pandas._libs.index.DatetimeEngine.get_loc()

TypeError: '2016-12-27 02:02:09.000     True
2016-12-27 02:02:09.017     True
2016-12-27 02:02:09.033     True
2016-12-27 02:02:09.050     True
2016-12-27 02:02:09.066    False
dtype: bool' is an invalid key

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-33-85e39ae5f3f1> in <module>
      3 temp=temp.head(5)
      4 print(temp.head())
----> 5 temp[temp>180]=temp[temp>180]-360

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/series.py in __setitem__(self, key, value)
   1018                 key = np.asarray(key, dtype=bool)
   1019                 try:
-> 1020                     self._where(~key, value, inplace=True)
   1021                 except InvalidIndexError:
   1022                     self.iloc[key] = value

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/generic.py in _where(self, cond, other, inplace, axis, level, errors, try_cast)
   8778             if other.ndim <= self.ndim:
   8779 
-> 8780                 _, other = self.align(
   8781                     other, join="left", axis=axis, level=level, fill_value=np.nan
   8782                 )

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/series.py in align(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis, broadcast_axis)
   4272         broadcast_axis=None,
   4273     ):
-> 4274         return super().align(
   4275             other,
   4276             join=join,

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/generic.py in align(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis, broadcast_axis)
   8557             )
   8558         elif isinstance(other, ABCSeries):
-> 8559             return self._align_series(
   8560                 other,
   8561                 join=join,

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/generic.py in _align_series(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis)
   8660                 join_index, lidx, ridx = None, None, None
   8661             else:
-> 8662                 join_index, lidx, ridx = self.index.join(
   8663                     other.index, how=join, level=level, return_indexers=True
   8664                 )

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/indexes/datetimelike.py in join(self, other, how, level, return_indexers, sort)
    884 
    885         this, other = self._maybe_utc_convert(other)
--> 886         return Index.join(
    887             this,
    888             other,

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/indexes/base.py in join(self, other, how, level, return_indexers, sort)
   3500         elif self.is_monotonic and other.is_monotonic:
   3501             try:
-> 3502                 return self._join_monotonic(
   3503                     other, how=how, return_indexers=return_indexers
   3504                 )

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/indexes/base.py in _join_monotonic(self, other, how, return_indexers)
   3800                 join_index = self
   3801                 lidx = None
-> 3802                 ridx = self._left_indexer_unique(sv, ov)
   3803             elif how == "right":
   3804                 join_index = other

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/core/indexes/datetimelike.py in wrapper(left, right)
     62             right = right.view("i8")
     63 
---> 64         results = joinf(left, right)
     65         if with_indexers:
     66             # dtype should be timedelta64[ns] for TimedeltaIndex

pandas/_libs/join.pyx in pandas._libs.join.left_join_indexer_unique()

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/_libs/join.cpython-38-x86_64-linux-gnu.so in View.MemoryView.memoryview_cwrapper()

~/anaconda3/envs/mindsynchro/lib/python3.8/site-packages/pandas/_libs/join.cpython-38-x86_64-linux-gnu.so in View.MemoryView.memoryview.__cinit__()

ValueError: buffer source array is read-only

Problem description

See error above

Expected Output

I would have expected:

• the 'NA' value to be ignored in comparison without causing an error
• If an error is due to NA, I expect to see appropriate message, instead of 'read-only' being thrown

Output of pd.show_versions()

INSTALLED VERSIONS
commit : db08276
python : 3.8.6.final.0
python-bits : 64
OS : Linux
OS-release : 4.15.0-118-generic
Version : #119-Ubuntu SMP Tue Sep 8 12:30:01 UTC 2020
machine : x86_64
processor : x86_64
byteorder : little
LC_ALL : None
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8

pandas : 1.1.3
numpy : 1.19.2
pytz : 2020.1
dateutil : 2.8.1
pip : 20.2.3
setuptools : 49.6.0.post20201009
Cython : 0.29.21
lxml.etree : 4.5.2
jinja2 : 2.11.2
IPython : 7.18.1
fsspec : 0.8.4
fastparquet : 0.4.1
matplotlib : 3.3.2
pyarrow : 0.17.1
scipy : 1.5.2
xarray : 0.16.1
numba : 0.51.2

[dmitra79]

@dsaxton Does #37181 cover this?

[dsaxton]

@dsaxton Does #37181 cover this?

It seems like a different issue to me. Can you create a copy / pasteable reproducible example so others can investigate?

[dmitra79]

@dsaxton Here is an example:

di=np.array(['2016-12-27 02:02:09', '2016-12-27 02:02:09.017000',
               '2016-12-27 02:02:09.033000', '2016-12-27 02:02:09.050000',
               '2016-12-27 02:02:09.066000'],dtype='datetime64[ns]')
di.setflags(write=False)
v1=[95.746, 95.735, 95.722, 95.709,    np.nan]
v2=[-86.474, -86.491, -86.505, -86.513,   np.nan]
temp=pd.DataFrame({'v1':v1,'v2':v2}, index=di)#res_df.index[0:5])
temp=temp.astype('float64')
temp2=temp.v1-temp.v2
temp2[temp2>180]=temp2[temp2>180]-360

[zeromh]

I was have the same error when using the rank method. Should I file as a new Issue, or is this all related?

arr = np.arange(10)
arr.setflags(write=False)
data = pd.Series(arr)
data.rank()

Output:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-55-b04e42a69641> in <module>
      2 arr.setflags(write=False)
      3 data = pd.Series(arr)
----> 4 data.rank()

~/anaconda/envs/xfactor/lib/python3.8/site-packages/pandas/core/generic.py in rank(self, axis, method, numeric_only, na_option, ascending, pct)
   8331         if numeric_only is None:
   8332             try:
-> 8333                 return ranker(self)
   8334             except TypeError:
   8335                 numeric_only = True

~/anaconda/envs/xfactor/lib/python3.8/site-packages/pandas/core/generic.py in ranker(data)
   8316 
   8317         def ranker(data):
-> 8318             ranks = algos.rank(
   8319                 data.values,
   8320                 axis=axis,

~/anaconda/envs/xfactor/lib/python3.8/site-packages/pandas/core/algorithms.py in rank(values, axis, method, na_option, ascending, pct)
    929     if values.ndim == 1:
    930         values = _get_values_for_rank(values)
--> 931         ranks = algos.rank_1d(
    932             values,
    933             ties_method=method,

pandas/_libs/algos.pyx in pandas._libs.algos.rank_1d()

~/anaconda/envs/xfactor/lib/python3.8/site-packages/pandas/_libs/algos.cpython-38-darwin.so in View.MemoryView.memoryview_cwrapper()

~/anaconda/envs/xfactor/lib/python3.8/site-packages/pandas/_libs/algos.cpython-38-darwin.so in View.MemoryView.memoryview.__cinit__()

ValueError: buffer source array is read-only

[dsaxton]

I was have the same error when using the rank method. Should I file as a new Issue, or is this all related?

I think it's related but the fix won't be identical, so probably warrants a separate issue. From some comments it looks like this might require at least Cython 3.

[dmitra79]

Any movement on this? It's still marked as 'Needs Triage'?

[jreback]

rank is already patched for master and 1.1.4

@dmitra79 can u try on master

[dmitra79]

@jreback

I just tried this on master (created clean environment, downloaded and buolt master) and got the same error:

>>> import pandas as pd
>>> import numpy as np
>>> di=np.array(['2016-12-27 02:02:09', '2016-12-27 02:02:09.017000',
...                '2016-12-27 02:02:09.033000', '2016-12-27 02:02:09.050000',
...                '2016-12-27 02:02:09.066000'],dtype='datetime64[ns]')
>>> di.setflags(write=False)
>>> v1=[95.746, 95.735, 95.722, 95.709,    np.nan]
>>> v2=[-86.474, -86.491, -86.505, -86.513,   np.nan]
>>> temp=pd.DataFrame({'v1':v1,'v2':v2}, index=di)#res_df.index[0:5])
>>> temp=temp.astype('float64')
>>> temp2=temp.v1-temp.v2
>>> temp2[temp2>180]=temp2[temp2>180]-360
Traceback (most recent call last):
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/series.py", line 1011, in __setitem__
    self._set_with_engine(key, value)
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/series.py", line 1044, in _set_with_engine
    loc = self.index._engine.get_loc(key)
  File "pandas/_libs/index.pyx", line 413, in pandas._libs.index.DatetimeEngine.get_loc
  File "pandas/_libs/index.pyx", line 420, in pandas._libs.index.DatetimeEngine.get_loc
TypeError: '2016-12-27 02:02:09.000     True
2016-12-27 02:02:09.017     True
2016-12-27 02:02:09.033     True
2016-12-27 02:02:09.050     True
2016-12-27 02:02:09.066    False
dtype: bool' is an invalid key

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/series.py", line 1031, in __setitem__
    self._where(~key, value, inplace=True)
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/generic.py", line 8931, in _where
    _, other = self.align(
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/series.py", line 4255, in align
    return super().align(
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/generic.py", line 8702, in align
    return self._align_series(
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/generic.py", line 8809, in _align_series
    join_index, lidx, ridx = self.index.join(
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/indexes/datetimelike.py", line 911, in join
    return Index.join(
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/indexes/base.py", line 3631, in join
    return self._join_monotonic(
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/indexes/base.py", line 3936, in _join_monotonic
    ridx = self._left_indexer_unique(sv, ov)
  File "/home/z0022z7b/anaconda3/envs/test_env/lib/python3.8/site-packages/pandas-1.2.0.dev0+1029.gc583144d4-py3.8-linux-x86_64.egg/pandas/core/indexes/datetimelike.py", line 64, in wrapper
    results = joinf(left, right)
  File "pandas/_libs/join.pyx", line 271, in pandas._libs.join.left_join_indexer_unique
  File "stringsource", line 658, in View.MemoryView.memoryview_cwrapper
  File "stringsource", line 349, in View.MemoryView.memoryview.__cinit__
ValueError: buffer source array is read-only