SAS7BDAT parser: Fast byteswap

doc/source/whatsnew/v1.6.0.rst

@@ -149,7 +149,7 @@ Performance improvements
 - Performance improvement in :meth:`DataFrame.join` when joining on a subset of a :class:`MultiIndex` (:issue:`48611`)
 - Performance improvement for :meth:`MultiIndex.intersection` (:issue:`48604`)
 - Performance improvement in ``var`` for nullable dtypes (:issue:`48379`).
-- Performance improvement to :func:`read_sas` with ``blank_missing=True`` (:issue:`48502`)
+- Performance improvements to :func:`read_sas` (:issue:`47403`, :issue:`47405`, :issue:`47656`, :issue:`48502`)
 - Memory improvement in :meth:`RangeIndex.sort_values` (:issue:`48801`)
 
 .. ---------------------------------------------------------------------------

pandas/io/sas/_byteswap.pyi

@@ -0,0 +1,5 @@
+def read_float_with_byteswap(data: bytes, offset: int, byteswap: bool) -> float: ...
+def read_double_with_byteswap(data: bytes, offset: int, byteswap: bool) -> float: ...
+def read_uint16_with_byteswap(data: bytes, offset: int, byteswap: bool) -> int: ...
+def read_uint32_with_byteswap(data: bytes, offset: int, byteswap: bool) -> int: ...
+def read_uint64_with_byteswap(data: bytes, offset: int, byteswap: bool) -> int: ...

pandas/io/sas/byteswap.pyx

@@ -0,0 +1,92 @@
+"""
+The following are faster versions of struct.unpack that avoid the overhead of Python function calls.
+
+In the SAS7BDAT parser, they may be called up to (n_rows * n_cols) times.
+"""
+from cython cimport Py_ssize_t
+from libc.stdint cimport (
+    uint16_t,
+    uint32_t,
+    uint64_t,
+)
+
+
+def read_float_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
+    assert offset + 4 < len(data)
+    cdef:
+        const char *data_ptr = data
+        float res = (<float*>(data_ptr + offset))[0]
+    if byteswap:
+        res = _byteswap_float(res)
+    return res
+
+
+def read_double_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
+    assert offset + 8 < len(data)
+    cdef:
+        const char *data_ptr = data
+        double res = (<double*>(data_ptr + offset))[0]
+    if byteswap:
+        res = _byteswap_double(res)
+    return res
+
+
+def read_uint16_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
+    assert offset + 2 < len(data)
+    cdef:
+        const char *data_ptr = data
+        uint16_t res = (<uint16_t *>(data_ptr + offset))[0]
+    if byteswap:
+        res = _byteswap2(res)
+    return res
+
+
+def read_uint32_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
+    assert offset + 4 < len(data)
+    cdef:
+        const char *data_ptr = data
+        uint32_t res = (<uint32_t *>(data_ptr + offset))[0]
+    if byteswap:
+        res = _byteswap4(res)
+    return res
+
+
+def read_uint64_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
+    assert offset + 8 < len(data)
+    cdef:
+        const char *data_ptr = data
+        uint64_t res = (<uint64_t *>(data_ptr + offset))[0]
+    if byteswap:
+        res = _byteswap8(res)
+    return res
+
+
+# Byteswapping
+
+cdef extern from *:
+    """
+    #ifdef _MSC_VER
+        #define _byteswap2 _byteswap_ushort
+        #define _byteswap4 _byteswap_ulong
+        #define _byteswap8 _byteswap_uint64
+    #else
+        #define _byteswap2 __builtin_bswap16
+        #define _byteswap4 __builtin_bswap32
+        #define _byteswap8 __builtin_bswap64
+    #endif
+    """
+    uint16_t _byteswap2(uint16_t)
+    uint32_t _byteswap4(uint32_t)
+    uint64_t _byteswap8(uint64_t)
+
+
+cdef inline float _byteswap_float(float num):
+    cdef uint32_t *intptr = <uint32_t *>&num
+    intptr[0] = _byteswap4(intptr[0])
+    return num
+
+
+cdef inline double _byteswap_double(double num):
+    cdef uint64_t *intptr = <uint64_t *>&num
+    intptr[0] = _byteswap8(intptr[0])
+    return num

pandas/io/sas/sas7bdat.py

@@ -20,7 +20,7 @@
     datetime,
     timedelta,
 )
-import struct
+import sys
 from typing import cast
 
 import numpy as np
@@ -42,6 +42,13 @@
 )
 
 from pandas.io.common import get_handle
+from pandas.io.sas._byteswap import (
+    read_double_with_byteswap,
+    read_float_with_byteswap,
+    read_uint16_with_byteswap,
+    read_uint32_with_byteswap,
+    read_uint64_with_byteswap,
+)
 from pandas.io.sas._sas import Parser
 import pandas.io.sas.sas_constants as const
 from pandas.io.sas.sasreader import ReaderBase
@@ -261,8 +268,10 @@ def _get_properties(self) -> None:
         buf = self._read_bytes(const.endianness_offset, const.endianness_length)
         if buf == b"\x01":
             self.byte_order = "<"
+            self.need_byteswap = sys.byteorder == "big"
         else:
             self.byte_order = ">"
+            self.need_byteswap = sys.byteorder == "little"
 
         # Get encoding information
         buf = self._read_bytes(const.encoding_offset, const.encoding_length)[0]
@@ -349,37 +358,46 @@ def __next__(self) -> DataFrame:
 
     # Read a single float of the given width (4 or 8).
     def _read_float(self, offset: int, width: int):
-        if width not in (4, 8):
+        assert self._cached_page is not None
+        if width == 4:
+            return read_float_with_byteswap(
+                self._cached_page, offset, self.need_byteswap
+            )
+        elif width == 8:
+            return read_double_with_byteswap(
+                self._cached_page, offset, self.need_byteswap
+            )
+        else:
             self.close()
             raise ValueError("invalid float width")
-        buf = self._read_bytes(offset, width)
-        fd = "f" if width == 4 else "d"
-        return struct.unpack(self.byte_order + fd, buf)[0]
 
     # Read a single signed integer of the given width (1, 2, 4 or 8).
     def _read_int(self, offset: int, width: int) -> int:
-        if width not in (1, 2, 4, 8):
+        assert self._cached_page is not None
+        if width == 1:
+            return self._read_bytes(offset, 1)[0]
+        elif width == 2:
+            return read_uint16_with_byteswap(
+                self._cached_page, offset, self.need_byteswap
+            )
+        elif width == 4:
+            return read_uint32_with_byteswap(
+                self._cached_page, offset, self.need_byteswap
+            )
+        elif width == 8:
+            return read_uint64_with_byteswap(
+                self._cached_page, offset, self.need_byteswap
+            )
+        else:
             self.close()
             raise ValueError("invalid int width")
-        buf = self._read_bytes(offset, width)
-        it = {1: "b", 2: "h", 4: "l", 8: "q"}[width]
-        iv = struct.unpack(self.byte_order + it, buf)[0]
-        return iv
 
     def _read_bytes(self, offset: int, length: int):
-        if self._cached_page is None:
-            self._path_or_buf.seek(offset)
-            buf = self._path_or_buf.read(length)
-            if len(buf) < length:
-                self.close()
-                msg = f"Unable to read {length:d} bytes from file position {offset:d}."
-                raise ValueError(msg)
-            return buf
-        else:
-            if offset + length > len(self._cached_page):
-                self.close()
-                raise ValueError("The cached page is too small.")
-            return self._cached_page[offset : offset + length]
+        assert self._cached_page is not None
+        if offset + length > len(self._cached_page):
+            self.close()
+            raise ValueError("The cached page is too small.")
+        return self._cached_page[offset : offset + length]
 
     def _read_and_convert_header_text(self, offset: int, length: int) -> str | bytes:
         return self._convert_header_text(

pandas/tests/io/sas/test_byteswap.py

@@ -0,0 +1,54 @@
+from hypothesis import (
+    assume,
+    example,
+    given,
+    strategies as st,
+)
+import numpy as np
+import pytest
+
+import pandas._testing as tm
+
+from pandas.io.sas._byteswap import (
+    read_double_with_byteswap,
+    read_float_with_byteswap,
+    read_uint16_with_byteswap,
+    read_uint32_with_byteswap,
+    read_uint64_with_byteswap,
+)
+
+
+@given(read_offset=st.integers(0, 11), number=st.integers(min_value=0))
+@example(number=2**16, read_offset=0)
+@example(number=2**32, read_offset=0)
+@example(number=2**64, read_offset=0)
+@pytest.mark.parametrize("int_type", [np.uint16, np.uint32, np.uint64])
+@pytest.mark.parametrize("should_byteswap", [True, False])
+def test_int_byteswap(read_offset, number, int_type, should_byteswap):
+    assume(number < 2 ** (8 * int_type(0).itemsize))
+    _test(number, int_type, read_offset, should_byteswap)
+
+
+@given(read_offset=st.integers(0, 11), number=st.floats())
+@pytest.mark.parametrize("float_type", [np.float32, np.float64])
+@pytest.mark.parametrize("should_byteswap", [True, False])
+def test_float_byteswap(read_offset, number, float_type, should_byteswap):
+    _test(number, float_type, read_offset, should_byteswap)
+
+
+def _test(number, number_type, read_offset, should_byteswap):
+    number = number_type(number)
+    data = np.random.default_rng().integers(0, 256, size=20, dtype="uint8")
+    data[read_offset : read_offset + number.itemsize] = number[None].view("uint8")
+    swap_func = {
+        np.float32: read_float_with_byteswap,
+        np.float64: read_double_with_byteswap,
+        np.uint16: read_uint16_with_byteswap,
+        np.uint32: read_uint32_with_byteswap,
+        np.uint64: read_uint64_with_byteswap,
+    }[type(number)]
+    output_number = number_type(swap_func(bytes(data), read_offset, should_byteswap))
+    if should_byteswap:
+        tm.assert_equal(output_number, number.byteswap())
+    else:
+        tm.assert_equal(output_number, number)

setup.py

@@ -226,6 +226,7 @@ class CheckSDist(sdist_class):
         "pandas/_libs/window/indexers.pyx",
         "pandas/_libs/writers.pyx",
         "pandas/io/sas/sas.pyx",
+        "pandas/io/sas/byteswap.pyx",
     ]
 
     _cpp_pyxfiles = [
@@ -571,6 +572,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
     "_libs.window.indexers": {"pyxfile": "_libs/window/indexers"},
     "_libs.writers": {"pyxfile": "_libs/writers"},
     "io.sas._sas": {"pyxfile": "io/sas/sas"},
+    "io.sas._byteswap": {"pyxfile": "io/sas/byteswap"},
 }
 
 extensions = []