Merge branch 'main' into raft

Author: mergify[bot]

Cargo.lock

@@ -8,6 +8,7 @@ members = [
     "common/clickhouse-srv",
     "common/containers",
     "common/dal",
+    "common/dal2",
     "common/datablocks",
     "common/datavalues",
     "common/exception",

Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "common-dal2"
+version = "0.1.0"
+authors = ["Databend Authors <[email protected]>"]
+license = "Apache-2.0"
+publish = false
+edition = "2021"
+
+[lib]
+doctest = false
+test = false
+
+[dependencies]
+common-exception = { path = "../exception" }
+
+async-compat = "0.2.1"
+async-trait = "0.1.52"
+bytes = "1.1.0"
+futures = "0.3.19"
+reqwest = "0.11.7"
+rusoto_core = "0.47.0"
+rusoto_s3 = "0.47.0"
+rusoto_credential = "0.47.0"
+tokio = {version="1.15.0",features=["fs"]}

common/dal2/Cargo.toml

@@ -0,0 +1,72 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use common_exception::Result;
+
+use crate::ops::Delete;
+use crate::ops::Object;
+use crate::ops::Read;
+use crate::ops::ReadBuilder;
+use crate::ops::Stat;
+use crate::ops::Write;
+use crate::ops::WriteBuilder;
+
+pub struct DataAccessor<'d, S> {
+    s: Arc<S>,
+    phantom: PhantomData<&'d ()>,
+}
+
+impl<'d, S> DataAccessor<'d, S> {
+    pub fn new(s: S) -> DataAccessor<'d, S> {
+        DataAccessor {
+            s: Arc::new(s),
+            phantom: PhantomData::default(),
+        }
+    }
+}
+
+impl<'d, S> DataAccessor<'d, S>
+where S: Read<S>
+{
+    pub fn read(&self, path: &'d str) -> ReadBuilder<S> {
+        ReadBuilder::new(self.s.clone(), path)
+    }
+}
+
+impl<'d, S> DataAccessor<'d, S>
+where S: Write<S>
+{
+    pub fn write(&self, path: &'d str, size: u64) -> WriteBuilder<S> {
+        WriteBuilder::new(self.s.clone(), path, size)
+    }
+}
+
+impl<'d, S> DataAccessor<'d, S>
+where S: Stat<S>
+{
+    pub async fn stat(&self, path: &'d str) -> Result<Object> {
+        self.s.stat(path).await
+    }
+}
+
+impl<'d, S> DataAccessor<'d, S>
+where S: Delete<S>
+{
+    pub async fn delete(&self, path: &'d str) -> Result<()> {
+        self.s.delete(path).await
+    }
+}

common/dal2/src/dal.rs

@@ -0,0 +1,19 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod dal;
+mod ops;
+
+pub mod services;
+pub use dal::DataAccessor;

common/dal2/src/lib.rs

@@ -0,0 +1,24 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+use common_exception::Result;
+
+#[async_trait]
+pub trait Delete<S: Send + Sync>: Send + Sync {
+    async fn delete(&self, path: &str) -> Result<()> {
+        let _ = path;
+        unimplemented!()
+    }
+}

common/dal2/src/ops/delete.rs

@@ -0,0 +1,39 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::pin::Pin;
+use std::task::Poll;
+
+pub type Reader = Box<dyn futures::io::AsyncRead + Unpin + Send>;
+
+pub struct CallbackReader {
+    inner: Reader,
+    f: Box<dyn Fn(usize)>,
+}
+
+impl futures::AsyncRead for CallbackReader {
+    fn poll_read(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<std::io::Result<usize>> {
+        let r = Pin::new(&mut self.inner).poll_read(cx, buf);
+
+        if let Poll::Ready(Ok(len)) = r {
+            (self.f)(len);
+        };
+
+        r
+    }
+}

common/dal2/src/ops/io.rs

@@ -0,0 +1,29 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod delete;
+mod io;
+mod object;
+mod read;
+mod stat;
+mod write;
+
+pub use delete::Delete;
+pub use io::Reader;
+pub use object::Object;
+pub use read::Read;
+pub use read::ReadBuilder;
+pub use stat::Stat;
+pub use write::Write;
+pub use write::WriteBuilder;

common/dal2/src/ops/mod.rs

@@ -0,0 +1,18 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub struct Object {
+    pub path: String,
+    pub size: u64,
+}

common/dal2/src/ops/object.rs

@@ -0,0 +1,65 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_exception::Result;
+
+use super::io::Reader;
+
+#[async_trait]
+pub trait Read<S: Send + Sync>: Send + Sync {
+    async fn read(&self, args: &ReadBuilder<S>) -> Result<Reader> {
+        let _ = args;
+        unimplemented!()
+    }
+}
+
+pub struct ReadBuilder<'p, S> {
+    s: Arc<S>,
+
+    pub path: &'p str,
+    pub offset: Option<u64>,
+    pub size: Option<u64>,
+}
+
+impl<'p, S> ReadBuilder<'p, S> {
+    pub fn new(s: Arc<S>, path: &'p str) -> Self {
+        Self {
+            s,
+            path,
+            offset: None,
+            size: None,
+        }
+    }
+
+    pub fn offset(&mut self, offset: u64) -> &mut Self {
+        self.offset = Some(offset);
+
+        self
+    }
+
+    pub fn size(&mut self, size: u64) -> &mut Self {
+        self.size = Some(size);
+
+        self
+    }
+}
+
+impl<'p, S: Read<S>> ReadBuilder<'p, S> {
+    pub async fn run(&mut self) -> Result<Reader> {
+        self.s.read(self).await
+    }
+}

common/dal2/src/ops/read.rs

@@ -0,0 +1,26 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+use common_exception::Result;
+
+use crate::ops::Object;
+
+#[async_trait]
+pub trait Stat<S: Send + Sync>: Send + Sync {
+    async fn stat(&self, path: &str) -> Result<Object> {
+        let _ = path;
+        unimplemented!()
+    }
+}