Skip to content

Commit 5b43413

Browse files
den818den818
committed
fix
1 parent 6a74560 commit 5b43413

File tree

4 files changed

+161
-12
lines changed

4 files changed

+161
-12
lines changed

‎clickhouse/client.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -567,7 +567,6 @@ bool Client::Impl::ReadBlock(InputStream& input, Block* block) {
567567
col->LoadSerializationKind(&input);
568568
}
569569

570-
571570
if (num_rows && !col->Load(&input, num_rows)) {
572571
throw ProtocolError("can't load column '" + name + "' of type " + type);
573572
}
@@ -782,12 +781,15 @@ void Client::Impl::WriteBlock(const Block& block, OutputStream& output) {
782781
WireFormat::WriteString(output, bi.Name());
783782
WireFormat::WriteString(output, bi.Type()->GetName());
784783

784+
bool has_custom = bi.Column()->HasCustomSerialization();
785785
if (server_info_.revision >= DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION) {
786-
bool has_custom = bi.Column()->HasCustomSerialization();
787786
WireFormat::WriteFixed(output, static_cast<uint8_t>(has_custom));
788787
if (has_custom) {
789788
bi.Column()->SaveSerializationKind(&output);
790789
}
790+
} else {
791+
// Current implementation works only for server version >= v22.1.2.2-stable
792+
throw UnimplementedError(std::string("Can't send column with custom serialisation to a server, server version is too old"));
791793
}
792794

793795
// Empty columns are not serialized and occupy exactly 0 bytes.

‎clickhouse/columns/string.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ void ColumnFixedString::SetSerializationKind(Serialization::Kind kind) {
126126
serialization_ = Serialization::MakeDefault(this);
127127
break;
128128
case Serialization::Kind::SPARSE:
129-
serialization_ = Serialization::MakeSparse(this, std::string('\0', FixedSize()));
129+
serialization_ = Serialization::MakeSparse(this, std::string(FixedSize(), '\0'));
130130
break;
131131
default:
132132
throw UnimplementedError("Serialization kind:" + std::to_string(static_cast<int>(kind))

‎clickhouse/columns/tuple.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ void ColumnTuple::Swap(Column& other) {
9898
}
9999

100100
bool ColumnTuple::LoadSerializationKind(InputStream* input) {
101+
if (!Column::LoadSerializationKind(input)) {
102+
return false;
103+
}
101104
for (auto & column : columns_) {
102105
if (!(column->LoadSerializationKind(input))) {
103106
return false;
@@ -107,6 +110,7 @@ bool ColumnTuple::LoadSerializationKind(InputStream* input) {
107110
}
108111

109112
void ColumnTuple::SaveSerializationKind(OutputStream* output) {
113+
Column::SaveSerializationKind(output);
110114
for (auto & column : columns_) {
111115
column->SaveSerializationKind(output);
112116
}

‎ut/sparse_ut.cpp

Lines changed: 152 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ namespace
1010
using namespace clickhouse;
1111
}
1212

13+
#define DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION 54454
14+
1315
static const auto localHostEndpoint = ClientOptions()
1416
.SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost"))
1517
.SetPort( getEnvOrDefault<size_t>("CLICKHOUSE_PORT", "9000"))
@@ -30,19 +32,31 @@ class SparseCase : public testing::Test {
3032
}
3133

3234
template <typename T>
33-
std::shared_ptr<T> createTableWithOneColumn()
35+
bool createTable(std::shared_ptr<T> col)
3436
{
35-
auto col = std::make_shared<T>();
3637
const auto type_name = col->GetType().GetName();
3738

3839
client_->Execute("DROP TABLE IF EXISTS " + table_name + ";");
39-
client_->Execute("CREATE TABLE IF NOT EXISTS " + table_name + "( id UInt64," + column_name + " " + type_name + " )"
40-
" ENGINE = MergeTree ORDER BY id "
41-
" SETTINGS index_granularity = 32, "
42-
" ratio_of_defaults_for_sparse_serialization = 0.1;");
40+
try {
41+
client_->Execute("CREATE TABLE IF NOT EXISTS " + table_name + "( id UInt64," + column_name + " " + type_name + " )"
42+
" ENGINE = MergeTree ORDER BY id "
43+
" SETTINGS index_granularity = 32, "
44+
" ratio_of_defaults_for_sparse_serialization = 0.1;");
45+
} catch (const std::exception & e) {
46+
std::cerr << "Got error while create table: " << e.what() << std::endl;
47+
// DB::Exception: clickhouse_cpp_cicd: Cannot execute query in readonly mode
48+
if (std::string(e.what()).find("Cannot execute query in readonly mode") != std::string::npos) {
49+
return false;
50+
}
51+
//DB::Exception: clickhouse_cpp_cicd: Not enough privileges. To execute this query it's necessary to have grant CREATE TABLE ON default.test_clickhouse_cpp_test_ut_sparse_table
52+
if (std::string(e.what()).find("Not enough privileges") != std::string::npos) {
53+
return false;
54+
}
55+
throw;
56+
}
4357

4458
client_->Execute("SYSTEM STOP MERGES test_clickhouse_sparse_table;");
45-
return col;
59+
return true;
4660
}
4761

4862
std::string getOneColumnSelectQuery() const
@@ -77,9 +91,15 @@ class SparseCase : public testing::Test {
7791

7892
TEST_F(SparseCase, UInt64_Load) {
7993

80-
createTableWithOneColumn<ColumnUInt64>();
94+
if (client_->GetServerInfo().revision < DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION) {
95+
GTEST_SKIP();
96+
}
97+
98+
if (!createTable(std::make_shared<ColumnUInt64>())) {
99+
GTEST_SKIP();
100+
}
81101

82-
client_->Execute("INSERT INTO " + table_name +" "
102+
client_->Execute("INSERT INTO " + table_name +
83103
" SELECT number, "
84104
" if (number % 10 = 0, number, 0)"
85105
" FROM numbers(1000);");
@@ -95,6 +115,7 @@ TEST_F(SparseCase, UInt64_Load) {
95115
ASSERT_EQ(1U, block.GetColumnCount());
96116
ASSERT_TRUE(block[0]->GetType().IsEqual(Type::CreateSimple<uint64_t>()));
97117
auto cl = block[0]->AsStrict<ColumnUInt64>();
118+
ASSERT_EQ(cl->GetSerialization()->GetKind(), Serialization::Kind::SPARSE);
98119
for (size_t i = 0; i < 1000; ++i) {
99120
if (i % 10 == 0) {
100121
EXPECT_EQ(cl->At(i), i);
@@ -103,6 +124,128 @@ TEST_F(SparseCase, UInt64_Load) {
103124
}
104125
}
105126
});
127+
}
128+
129+
TEST_F(SparseCase, String_Load) {
130+
131+
if (client_->GetServerInfo().revision < DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION) {
132+
GTEST_SKIP();
133+
}
134+
135+
if (!createTable(std::make_shared<ColumnString>())) {
136+
GTEST_SKIP();
137+
}
138+
139+
client_->Execute("INSERT INTO " + table_name +
140+
" SELECT number, "
141+
" if (number % 10 = 0, toString(number), '')"
142+
" FROM numbers(1000);");
143+
144+
checkSerializationKind();
145+
146+
client_->Select(getOneColumnSelectQuery(),
147+
[&](const Block& block) {
148+
// std::cerr << PrettyPrintBlock{block} << std::endl;
149+
if (block.GetRowCount() == 0)
150+
return;
151+
EXPECT_EQ(1000u, block.GetRowCount());
152+
ASSERT_EQ(1U, block.GetColumnCount());
153+
ASSERT_TRUE(block[0]->GetType().IsEqual(Type::CreateString()));
154+
auto cl = block[0]->AsStrict<ColumnString>();
155+
ASSERT_EQ(cl->GetSerialization()->GetKind(), Serialization::Kind::SPARSE);
156+
for (size_t i = 0; i < 1000; ++i) {
157+
if (i % 10 == 0) {
158+
EXPECT_EQ(cl->At(i), std::to_string(i));
159+
} else {
160+
EXPECT_EQ(cl->At(i), "");
161+
}
162+
}
163+
});
164+
}
165+
166+
TEST_F(SparseCase, FixedString_Load) {
167+
const size_t FixedStringSize = 10;
168+
169+
if (client_->GetServerInfo().revision < DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION) {
170+
GTEST_SKIP();
171+
}
172+
173+
if (!createTable(std::make_shared<ColumnFixedString>(FixedStringSize))) {
174+
GTEST_SKIP();
175+
}
176+
177+
client_->Execute("INSERT INTO " + table_name +
178+
" SELECT number, "
179+
" if (number % 10 = 0, toString(number), '')"
180+
" FROM numbers(1000);");
181+
182+
checkSerializationKind();
183+
184+
auto rpad = [](const std::string& val) {
185+
std::string res = val;
186+
res.append(std::string(FixedStringSize - res.size(), '\0'));
187+
return res;
188+
};
189+
190+
client_->Select(getOneColumnSelectQuery(),
191+
[&](const Block& block) {
192+
//std::cerr << PrettyPrintBlock{block} << std::endl;
193+
if (block.GetRowCount() == 0)
194+
return;
195+
EXPECT_EQ(1000u, block.GetRowCount());
196+
ASSERT_EQ(1U, block.GetColumnCount());
197+
ASSERT_TRUE(block[0]->GetType().IsEqual(Type::CreateString(FixedStringSize)));
198+
auto cl = block[0]->AsStrict<ColumnFixedString>();
199+
EXPECT_EQ(cl->GetSerialization()->GetKind(), Serialization::Kind::SPARSE);
200+
for (size_t i = 0; i < 1000; ++i) {
201+
if (i % 10 == 0) {
202+
EXPECT_EQ(cl->At(i), rpad(std::to_string(i)));
203+
} else {
204+
EXPECT_EQ(cl->At(i), rpad(""));
205+
}
206+
}
207+
});
208+
}
106209

107210

211+
TEST_F(SparseCase, Tuple_Load) {
212+
213+
if (client_->GetServerInfo().revision < DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION) {
214+
GTEST_SKIP();
215+
}
216+
217+
auto tuple = std::make_shared<ColumnTuple>(std::vector<ColumnRef>({
218+
std::make_shared<ColumnUInt64>(),
219+
std::make_shared<ColumnString>()
220+
}));
221+
222+
if (!createTable(tuple)) {
223+
GTEST_SKIP();
224+
}
225+
226+
client_->Execute("INSERT INTO " + table_name +
227+
" SELECT number, "
228+
" (if (number % 10 = 0, number, 0), repeat('a', number % 10 + 1))"
229+
" FROM numbers(1000);");
230+
231+
client_->Select(getOneColumnSelectQuery(),
232+
[&](const Block& block) {
233+
//std::cerr << PrettyPrintBlock{block} << std::endl;
234+
if (block.GetRowCount() == 0)
235+
return;
236+
EXPECT_EQ(1000u, block.GetRowCount());
237+
ASSERT_EQ(1U, block.GetColumnCount());
238+
auto cl = block[0]->AsStrict<ColumnTuple>();
239+
ASSERT_EQ(2U, cl->TupleSize());
240+
EXPECT_EQ((*cl)[0]->GetSerialization()->GetKind(), Serialization::Kind::SPARSE);
241+
EXPECT_EQ((*cl)[1]->GetSerialization()->GetKind(), Serialization::Kind::DEFAULT);
242+
auto cl_int = (*cl)[0]->AsStrict<ColumnUInt64>();
243+
for (size_t i = 0; i < 1000; ++i) {
244+
if (i % 10 == 0) {
245+
EXPECT_EQ(cl_int->At(i), i);
246+
} else {
247+
EXPECT_EQ(cl_int->At(i), 0u);
248+
}
249+
}
250+
});
108251
}

0 commit comments

Comments
 (0)