[libc++] Optimize {std,ranges}::distance for segmented iterators #133612
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
winner245
force-pushed
the
segmented-distance
branch
8 times, most recently
from
fd06544 to
25effb0
Compare
winner245
force-pushed
the
segmented-distance
branch
from
25effb0 to
5795a5a
Compare
llvmbot
added
the
libc++
|
@llvm/pr-subscribers-libcxx Author: Peng Liu (winner245) ChangesThis patch enhances the performance of The table below summarizes the peak performance improvements observed across different segment sizes, with the total element count BenchmarksSegment size = 1024Segment size = 256Full diff: https://github.com/llvm/llvm-project/pull/133612.diff 4 Files Affected:
diff --git a/libcxx/include/__iterator/distance.h b/libcxx/include/__iterator/distance.h
index 1732aa527f64a..8d14299191e30 100644
--- a/libcxx/include/__iterator/distance.h
+++ b/libcxx/include/__iterator/distance.h
@@ -10,41 +10,82 @@
#ifndef _LIBCPP___ITERATOR_DISTANCE_H
#define _LIBCPP___ITERATOR_DISTANCE_H
+#include <__algorithm/for_each_segment.h>
#include <__config>
#include <__iterator/concepts.h>
#include <__iterator/incrementable_traits.h>
#include <__iterator/iterator_traits.h>
+#include <__iterator/segmented_iterator.h>
#include <__ranges/access.h>
#include <__ranges/concepts.h>
#include <__ranges/size.h>
#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
#include <__type_traits/remove_cvref.h>
+#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
_LIBCPP_BEGIN_NAMESPACE_STD
-template <class _InputIter>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
-__distance(_InputIter __first, _InputIter __last, input_iterator_tag) {
- typename iterator_traits<_InputIter>::difference_type __r(0);
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter>
+using __iter_distance_t _LIBCPP_NODEBUG = std::iter_difference_t<_Iter>;
+#else
+template <class _Iter>
+using __iter_distance_t _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::difference_type;
+#endif
+
+template <class _InputIter, class _Sent>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_InputIter>
+__distance(_InputIter __first, _Sent __last) {
+ __iter_distance_t<_InputIter> __r(0);
for (; __first != __last; ++__first)
++__r;
return __r;
}
-template <class _RandIter>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_RandIter>::difference_type
-__distance(_RandIter __first, _RandIter __last, random_access_iterator_tag) {
+template <class _RandIter, __enable_if_t<__has_random_access_iterator_category<_RandIter>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_RandIter>
+__distance(_RandIter __first, _RandIter __last) {
return __last - __first;
}
+template <class _SegmentedIter, class _Difference>
+struct __segment_distance {
+ using _Traits _LIBCPP_NODEBUG = __segmented_iterator_traits<_SegmentedIter>;
+
+ _Difference& __r_;
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __segment_distance(_Difference& __r) : __r_(__r) {}
+
+ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+ operator()(typename _Traits::__local_iterator __lfirst, typename _Traits::__local_iterator __llast) {
+ __r_ += std::__distance(__lfirst, __llast);
+ }
+};
+
+template <class _SegmentedIter,
+ __enable_if_t<!__has_random_access_iterator_category<_SegmentedIter>::value &&
+ __is_segmented_iterator<_SegmentedIter>::value,
+ int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_SegmentedIter>
+__distance(_SegmentedIter __first, _SegmentedIter __last) {
+ using __difference_type = __iter_distance_t<_SegmentedIter>;
+ __difference_type __r(0);
+ std::__for_each_segment(__first, __last, std::__segment_distance<_SegmentedIter, __difference_type>(__r));
+ return __r;
+}
+
template <class _InputIter>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_InputIter>
distance(_InputIter __first, _InputIter __last) {
- return std::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category());
+ return std::__distance(__first, __last);
}
#if _LIBCPP_STD_VER >= 20
@@ -56,12 +97,7 @@ struct __distance {
template <class _Ip, sentinel_for<_Ip> _Sp>
requires(!sized_sentinel_for<_Sp, _Ip>)
_LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip __first, _Sp __last) const {
- iter_difference_t<_Ip> __n = 0;
- while (__first != __last) {
- ++__first;
- ++__n;
- }
- return __n;
+ return std::__distance(std::move(__first), std::move(__last));
}
template <class _Ip, sized_sentinel_for<decay_t<_Ip>> _Sp>
@@ -92,4 +128,6 @@ inline constexpr auto distance = __distance{};
_LIBCPP_END_NAMESPACE_STD
+_LIBCPP_POP_MACROS
+
#endif // _LIBCPP___ITERATOR_DISTANCE_H
diff --git a/libcxx/test/benchmarks/iterators/distance.bench.cpp b/libcxx/test/benchmarks/iterators/distance.bench.cpp
new file mode 100644
index 0000000000000..db9e6a9201658
--- /dev/null
+++ b/libcxx/test/benchmarks/iterators/distance.bench.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <cstddef>
+#include <deque>
+#include <iterator>
+#include <ranges>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+
+int main(int argc, char** argv) {
+ auto std_distance = [](auto first, auto last) { return std::distance(first, last); };
+
+ // {std,ranges}::distance
+ {
+ auto bm = []<class Container>(std::string name, auto distance, std::size_t seg_size) {
+ benchmark::RegisterBenchmark(
+ name,
+ [distance, seg_size](auto& st) {
+ std::size_t const size = st.range(0);
+ std::size_t const segments = (size + seg_size - 1) / seg_size;
+ Container c(segments);
+ for (std::size_t i = 0, n = size; i < segments; ++i, n -= seg_size) {
+ c[i].resize(std::min(seg_size, n));
+ }
+
+ auto view = c | std::views::join;
+ auto first = view.begin();
+ auto last = view.end();
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::DoNotOptimize(c);
+ auto result = distance(first, last);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(4096)
+ ->Arg(8192)
+ ->Arg(1 << 14)
+ ->Arg(1 << 16)
+ ->Arg(1 << 18)
+ ->Arg(1 << 20);
+ };
+ bm.operator()<std::vector<std::vector<int>>>("std::distance(join_view(vector<vector<int>>))", std_distance, 256);
+ bm.operator()<std::deque<std::deque<int>>>("std::distance(join_view(deque<deque<int>>))", std_distance, 256);
+ bm.operator()<std::vector<std::vector<int>>>(
+ "rng::distance(join_view(vector<vector<int>>)", std::ranges::distance, 256);
+ bm.operator()<std::deque<std::deque<int>>>(
+ "rng::distance(join_view(deque<deque<int>>)", std::ranges::distance, 256);
+
+ // bm.operator()<std::vector<std::vector<int>>>("std::distance(join_view(vector<vector<int>>))", std_distance, 1024);
+ // bm.operator()<std::deque<std::deque<int>>>("std::distance(join_view(deque<deque<int>>))", std_distance, 1024);
+ // bm.operator()<std::vector<std::vector<int>>>("rng::distance(join_view(vector<vector<int>>)", std::ranges::distance, 1024);
+ // bm.operator()<std::deque<std::deque<int>>>("rng::distance(join_view(deque<deque<int>>)", std::ranges::distance, 1024);
+ }
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
diff --git a/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp
index 13caefff92365..bcaeee14cad6d 100644
--- a/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/iterator.operations/distance.pass.cpp
@@ -16,38 +16,73 @@
// Iter::difference_type
// distance(Iter first, Iter last); // constexpr in C++17
-#include <iterator>
+#include <array>
#include <cassert>
+#include <deque>
+#include <iterator>
+#include <vector>
#include <type_traits>
#include "test_macros.h"
#include "test_iterators.h"
template <class It>
-TEST_CONSTEXPR_CXX17
-void check_distance(It first, It last, typename std::iterator_traits<It>::difference_type dist)
-{
- typedef typename std::iterator_traits<It>::difference_type Difference;
- static_assert(std::is_same<decltype(std::distance(first, last)), Difference>::value, "");
- assert(std::distance(first, last) == dist);
+TEST_CONSTEXPR_CXX17 void check_distance(It first, It last, typename std::iterator_traits<It>::difference_type dist) {
+ typedef typename std::iterator_traits<It>::difference_type Difference;
+ static_assert(std::is_same<decltype(std::distance(first, last)), Difference>::value, "");
+ assert(std::distance(first, last) == dist);
}
-TEST_CONSTEXPR_CXX17 bool tests()
-{
- const char* s = "1234567890";
- check_distance(cpp17_input_iterator<const char*>(s), cpp17_input_iterator<const char*>(s+10), 10);
- check_distance(forward_iterator<const char*>(s), forward_iterator<const char*>(s+10), 10);
- check_distance(bidirectional_iterator<const char*>(s), bidirectional_iterator<const char*>(s+10), 10);
- check_distance(random_access_iterator<const char*>(s), random_access_iterator<const char*>(s+10), 10);
- check_distance(s, s+10, 10);
- return true;
+#if TEST_STD_VER >= 20
+void test_deque() {
+ using Container = std::deque<std::deque<double>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::distance(view.begin(), view.end()) == n);
+}
+#endif
+
+TEST_CONSTEXPR_CXX17 bool tests() {
+ const char* s = "1234567890";
+ check_distance(cpp17_input_iterator<const char*>(s), cpp17_input_iterator<const char*>(s + 10), 10);
+ check_distance(forward_iterator<const char*>(s), forward_iterator<const char*>(s + 10), 10);
+ check_distance(bidirectional_iterator<const char*>(s), bidirectional_iterator<const char*>(s + 10), 10);
+ check_distance(random_access_iterator<const char*>(s), random_access_iterator<const char*>(s + 10), 10);
+ check_distance(s, s + 10, 10);
+
+#if TEST_STD_VER >= 20
+ {
+ using Container = std::vector<std::vector<int>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::distance(view.begin(), view.end()) == n);
+ }
+ {
+ using Container = std::array<std::array<char, 3>, 10>;
+ Container c;
+ auto view = c | std::views::join;
+ assert(std::distance(view.begin(), view.end()) == 30);
+ }
+ if (!TEST_IS_CONSTANT_EVALUATED)
+ test_deque();
+#endif
+ return true;
}
-int main(int, char**)
-{
- tests();
+int main(int, char**) {
+ tests();
#if TEST_STD_VER >= 17
- static_assert(tests(), "");
+ static_assert(tests(), "");
#endif
- return 0;
+ return 0;
}
diff --git a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp
index b4199b73ad76a..a5fcf337387c2 100644
--- a/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/range.iter.ops/range.iter.ops.distance/iterator_sentinel.pass.cpp
@@ -15,18 +15,21 @@
// template<class I, sized_sentinel_for<decay_t<I>> S>
// constexpr iter_difference_t<I> ranges::distance(I&& first, S last); // TODO: update when LWG3664 is resolved
-#include <iterator>
+#include <array>
#include <cassert>
+#include <deque>
+#include <iterator>
+#include <vector>
#include "test_iterators.h"
#include "test_macros.h"
-template<class It, class Sent>
+template <class It, class Sent>
constexpr void test_unsized() {
static_assert(std::sentinel_for<Sent, It> && !std::sized_sentinel_for<Sent, It>);
- int a[3] = {1,2,3};
+ int a[3] = {1, 2, 3};
{
- It first = It(a);
+ It first = It(a);
auto last = Sent(It(a));
assert(std::ranges::distance(first, last) == 0);
assert(std::ranges::distance(It(a), last) == 0);
@@ -36,7 +39,7 @@ constexpr void test_unsized() {
}
{
auto check = [&a]<class ItQual, class SentQual> {
- It first = It(a);
+ It first = It(a);
Sent last = Sent(It(a + 3));
assert(std::ranges::distance(static_cast<ItQual>(first), static_cast<SentQual>(last)) == 3);
};
@@ -61,13 +64,13 @@ constexpr void test_unsized() {
}
}
-template<class It, class Sent>
+template <class It, class Sent>
constexpr void test_sized() {
static_assert(std::sized_sentinel_for<Sent, It>);
- int a[] = {1,2,3};
+ int a[] = {1, 2, 3};
{
auto check = [&a]<class ItQual, class SentQual> {
- It first = It(a + 3);
+ It first = It(a + 3);
Sent last = Sent(It(a));
assert(std::ranges::distance(static_cast<ItQual>(first), static_cast<SentQual>(last)) == -3);
};
@@ -91,7 +94,7 @@ constexpr void test_sized() {
check.template operator()<const It&&, const Sent&&>();
}
{
- It first = It(a);
+ It first = It(a);
auto last = Sent(It(a));
assert(std::ranges::distance(first, last) == 0);
assert(std::ranges::distance(It(a), last) == 0);
@@ -100,7 +103,7 @@ constexpr void test_sized() {
ASSERT_SAME_TYPE(decltype(std::ranges::distance(It(a), Sent(It(a)))), std::iter_difference_t<It>);
}
{
- It first = It(a);
+ It first = It(a);
auto last = Sent(It(a + 3));
assert(std::ranges::distance(first, last) == 3);
assert(std::ranges::distance(It(a), last) == 3);
@@ -110,13 +113,17 @@ constexpr void test_sized() {
}
struct StrideCounter {
- int *it_;
- int *inc_;
- using value_type = int;
+ int* it_;
+ int* inc_;
+ using value_type = int;
using difference_type = int;
explicit StrideCounter();
- constexpr explicit StrideCounter(int *it, int *inc) : it_(it), inc_(inc) {}
- constexpr auto& operator++() { ++it_; *inc_ += 1; return *this; }
+ constexpr explicit StrideCounter(int* it, int* inc) : it_(it), inc_(inc) {}
+ constexpr auto& operator++() {
+ ++it_;
+ *inc_ += 1;
+ return *this;
+ }
StrideCounter operator++(int);
int& operator*() const;
bool operator==(StrideCounter) const;
@@ -125,11 +132,11 @@ static_assert(std::forward_iterator<StrideCounter>);
static_assert(!std::sized_sentinel_for<StrideCounter, StrideCounter>);
struct SizedStrideCounter {
- int *it_;
- int *minus_;
+ int* it_;
+ int* minus_;
using value_type = int;
explicit SizedStrideCounter();
- constexpr explicit SizedStrideCounter(int *it, int *minus) : it_(it), minus_(minus) {}
+ constexpr explicit SizedStrideCounter(int* it, int* minus) : it_(it), minus_(minus) {}
SizedStrideCounter& operator++();
SizedStrideCounter operator++(int);
int& operator*() const;
@@ -147,22 +154,34 @@ constexpr void test_stride_counting() {
int a[] = {1, 2, 3};
int inc = 0;
StrideCounter first(a, &inc);
- StrideCounter last(a+3, nullptr);
+ StrideCounter last(a + 3, nullptr);
std::same_as<int> auto result = std::ranges::distance(first, last);
assert(result == 3);
assert(inc == 3);
}
{
- int a[] = {1, 2, 3};
+ int a[] = {1, 2, 3};
int minus = 0;
SizedStrideCounter first(a, &minus);
- SizedStrideCounter last(a+3, nullptr);
+ SizedStrideCounter last(a + 3, nullptr);
std::same_as<int> auto result = std::ranges::distance(first, last);
assert(result == 3);
assert(minus == 1);
}
}
+void test_deque() {
+ using Container = std::deque<std::deque<double>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::ranges::distance(view.begin(), view.end()) == n);
+}
+
constexpr bool test() {
{
int a[] = {1, 2, 3};
@@ -197,7 +216,7 @@ constexpr bool test() {
test_sized<contiguous_iterator<int*>, contiguous_iterator<int*>>();
{
- using It = cpp20_input_iterator<int*>; // non-copyable, thus not a sentinel for itself
+ using It = cpp20_input_iterator<int*>; // non-copyable, thus not a sentinel for itself
static_assert(!std::is_copy_constructible_v<It>);
static_assert(!std::sentinel_for<It, It>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, It&>);
@@ -206,10 +225,10 @@ constexpr bool test() {
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&&, It&&>);
}
{
- using It = cpp20_input_iterator<int*>; // non-copyable
- using Sent = sentinel_wrapper<It>; // not a sized sentinel
+ using It = cpp20_input_iterator<int*>; // non-copyable
+ using Sent = sentinel_wrapper<It>; // not a sized sentinel
static_assert(std::sentinel_for<Sent, It> && !std::sized_sentinel_for<Sent, It>);
- int a[] = {1,2,3};
+ int a[] = {1, 2, 3};
Sent last = Sent(It(a + 3));
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, Sent&>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, Sent&&>);
@@ -217,7 +236,7 @@ constexpr bool test() {
assert(std::ranges::distance(It(a), Sent(It(a + 3))) == 3);
}
{
- using It = cpp17_input_iterator<int*>; // not a sentinel for itself
+ using It = cpp17_input_iterator<int*>; // not a sentinel for itself
static_assert(!std::sentinel_for<It, It>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, It&>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), It&, It&&>);
@@ -231,6 +250,26 @@ constexpr bool test() {
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), int, int*>);
static_assert(!std::is_invocable_v<decltype(std::ranges::distance), int*, char*>);
+ {
+ using Container = std::vector<std::vector<int>>;
+ Container c;
+ auto view = c | std::views::join;
+ Container::difference_type n = 0;
+ for (std::size_t i = 0; i < 10; ++i) {
+ n += i;
+ c.push_back(Container::value_type(i));
+ }
+ assert(std::ranges::distance(view.begin(), view.end()) == n);
+ }
+ {
+ using Container = std::array<std::array<char, 3>, 10>;
+ Container c;
+ auto view = c | std::views::join;
+ assert(std::ranges::distance(view.begin(), view.end()) == 30);
+ }
+ if (!TEST_IS_CONSTANT_EVALUATED)
+ test_deque();
+
return true;
}
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This patch enhances the performance of
std::distanceandstd::ranges::distancefor non-random-access segmented iterators. The original implementation operates in linear time,O(n), wherenis the total number of elements. The optimized version reduces this to approximatelyO(n / segment_size)by leveraging segmented structure, wheresegment_sizeis the average size of each segment.The table below summarizes the peak performance improvements observed across different segment sizes, with the total element count
nranging up to1 << 20(1,048,576 elements), based on benchmark results.Benchmarks
Segment size = 1024
Segment size = 256