Description
Hi,
TL;DR: I am getting bad_alloc error when creating clickhouse connections on a node that used to run clickhouse fine.
I have clickhouse-cpp and clickhouse-server set up on a centOS node with the following OS:
uname -a Linux fpga01.cluster 3.10.0-1127.19.1.el7.x86_64 #1 SMP Tue Aug 25 17:23:54 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
I have been able to run my client application to connect to the clickhouse. However, after restarting the application I am getting the following error:
[clickhouse_create_tables] host: localhost, user : myuser, password: thepassword, dbname: mydb, pool_size: 8 [ClickHouseConnectionPool] creating clickhosue connection for pool : 0 out of 8 terminate called after throwing an instance of 'std::bad_alloc' what(): std::bad_alloc Aborted
I believe there is something wrong with clickhouse-cpp here. I can run the application on a new node with similar specs. I tried restarting the clickhouse-server and the issue still persists.
When I check the node memory I see enough free memory available.
free -h total used free shared buff/cache available Mem: 125G 11G 101G 924M 12G 112G Swap: 3.7G 0B 3.7G
Here is the code snippet that is creating the database connections, I do not see connection 1 establishing :
class ClickHouseConnectionPool {
public:
ClickHouseConnectionPool(const std::string& host, const std::string& user, const std::string& password, const std::string& dbname, int pool_size = inam_db_poll_size)
: host_(host), user_(user), password_(password), dbname_(dbname),
pool_size_(pool_size), num_connections_(0), pool_(),
mutex_(), cv_(), available_connections_()
{
clickhouse::ClientOptions options;
options.SetHost(host_)
.SetUser(user_)
.SetPassword(password_)
.SetDefaultDatabase(dbname_)
.SetConnectionRecvTimeout(std::chrono::milliseconds(1000*inam_db_read_timeout))
.SetConnectionSendTimeout(std::chrono::milliseconds(1000*inam_db_write_timeout))
.SetConnectionConnectTimeout(std::chrono::seconds(inam_db_connect_timeout))
.TcpKeepAlive(true)
.SetTcpKeepAliveIdle(std::chrono::seconds( 3 * inam_db_write_timeout))
.SetTcpKeepAliveInterval(std::chrono::seconds(inam_db_write_timeout))
.SetTcpKeepAliveCount(30);
for (int i = 0; i < pool_size_; i++) {
PRINT_DEBUG(DEBUG_DB_verbose > 2, "creating clickhosue connection for pool : %d out of %d\n", i , pool_size);
add_connection(options);
}
}
~ClickHouseConnectionPool() {
for (auto& conn : pool_) {
delete conn;
}
}
ClickHouseConnectionPool(const ClickHouseConnectionPool&) = delete;
ClickHouseConnectionPool& operator=(const ClickHouseConnectionPool&) = delete;
ClickHouseConnectionPool(ClickHouseConnectionPool&&) = delete;
ClickHouseConnectionPool& operator=(ClickHouseConnectionPool&&) = delete;
clickhouse::Client* acquire_connection() {
std::unique_lock<std::mutex> lock(mutex_);
while (available_connections_.empty()) {
cv_.wait(lock);
}
PRINT_DEBUG(DEBUG_DB_verbose > 3, "acquiring clickhosue connection from pool, remaining connection are %d out of %d\n",num_connections_--, pool_size_);
auto* conn = available_connections_.back();
available_connections_.pop_back();
num_connections_--;
return conn;
}
void release_connection(clickhouse::Client* conn) {
std::unique_lock<std::mutex> lock(mutex_);
available_connections_.push_back(conn);
PRINT_DEBUG(DEBUG_DB_verbose > 3, "release clickhosue connection for pool\n");
num_connections_++;
cv_.notify_one();
}
int num_used_connections() const {
std::unique_lock<std::mutex> lock(mutex_);
return num_connections_ - available_connections_.size();
}
int num_available_connections() const {
std::unique_lock<std::mutex> lock(mutex_);
return available_connections_.size();
}
private:
void add_connection(const clickhouse::ClientOptions& options) {
auto* conn = new clickhouse::Client(options);
pool_.push_back(conn);
available_connections_.push_back(conn);
PRINT_DEBUG(DEBUG_DB_verbose > 3, "created clickhosue connection for pool : %d\n",num_connections_);
num_connections_++;
}
const std::string host_;
const std::string user_;
const std::string password_;
const std::string dbname_;
const int pool_size_;
int num_connections_;
std::vector<clickhouse::Client*> pool_;
mutable std::mutex mutex_;
//std::mutex mutex_;
std::condition_variable cv_;
std::vector<clickhouse::Client*> available_connections_;
};
### Tasks