From 62976bca2fbfb4e1d032128ad6ff31c5e490b48a Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Mon, 24 Feb 2025 21:23:06 +0800 Subject: [PATCH 01/17] fix(interactive): disable repair wheel in coordinator dockerfile (#4517) Fixes #4454 --------- Signed-off-by: siyuan0322 fix aocc Committed-by: xiaolei.zl@alibaba-inc.com from Dev container fix installation Committed-by: xiaolei.zl@alibaba-inc.com from Dev container fix linking,todo: verify correctness Committed-by: xiaolei.zl from Dev container impl kafka writer and parser Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container minor fix Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container fix Committed-by: xiaolei.zl from Dev container Committed-by: xiaolei.zl from Dev container --- .github/workflows/interactive.yml | 17 ++ .gitmodules | 3 + .../interactive/development/dev_and_test.md | 37 +++++ flex/CMakeLists.txt | 33 ++++ flex/engines/graph_db/CMakeLists.txt | 9 + .../graph_db/database/wal/kafka_wal_parser.cc | 154 ++++++++++++++++++ .../graph_db/database/wal/kafka_wal_parser.h | 67 ++++++++ .../graph_db/database/wal/kafka_wal_writer.cc | 79 +++++++++ .../graph_db/database/wal/kafka_wal_writer.h | 79 +++++++++ flex/tests/hqps/kafka_test.cc | 87 ++++++++++ flex/third_party/cppkafka | 1 + flex/third_party/cppkafka_without_boost.patch | 55 +++++++ .../graphscope/gsctl/scripts/install_deps.sh | 6 +- 13 files changed, 624 insertions(+), 3 deletions(-) create mode 100644 flex/engines/graph_db/database/wal/kafka_wal_parser.cc create mode 100644 flex/engines/graph_db/database/wal/kafka_wal_parser.h create mode 100644 flex/engines/graph_db/database/wal/kafka_wal_writer.cc create mode 100644 flex/engines/graph_db/database/wal/kafka_wal_writer.h create mode 100644 flex/tests/hqps/kafka_test.cc create mode 160000 flex/third_party/cppkafka create mode 100644 flex/third_party/cppkafka_without_boost.patch diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index dcd0526ee0c7..f95c8f12ac92 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -377,6 +377,23 @@ jobs: bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \ ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml + - name: Run kafka wal test + env: + GS_TEST_DIR: ${{ github.workspace }}/gstest + run: | + wget https://dlcdn.apache.org/kafka/3.9.0/kafka_2.13-3.9.0.tgz + tar -zxf kafka_2.13-3.9.0.tgz + cd kafka_2.13-3.9.0 + KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/kraft/reconfig-server.properties + bin/kafka-server-start.sh config/kraft/reconfig-server.properties & + + bin/kafka-topics.sh --create --topic kafka-test --bootstrap-server localhost:9092 + + ./tests/hqps/kafka_test localhost:902 kafka-test + + pkill -f kafka-server-start.sh + - name: Run Gremlin test on modern graph env: GS_TEST_DIR: ${{ github.workspace }}/gstest diff --git a/.gitmodules b/.gitmodules index 1203c4c4d451..48cda2149aa8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "flex/third_party/parallel-hashmap"] path = flex/third_party/parallel-hashmap url = https://github.com/greg7mdp/parallel-hashmap.git +[submodule "flex/third_party/cppkafka"] + path = flex/third_party/cppkafka + url = https://github.com/mfontanini/cppkafka.git diff --git a/docs/flex/interactive/development/dev_and_test.md b/docs/flex/interactive/development/dev_and_test.md index 9e35bdf285ef..20f74b179ea1 100644 --- a/docs/flex/interactive/development/dev_and_test.md +++ b/docs/flex/interactive/development/dev_and_test.md @@ -262,3 +262,40 @@ In Interactive's execution engine, transactions such as `ReadTransaction`, `Upda 2. If a transaction returns `false` during the `commit()` process, the error occurred prior to applying the WAL to the graph data. This type of failure could arise during the construction of the WAL or during its writing phase. 3. It is important to note that errors can still occur when replaying the WAL to the graph database. Replaying might fail due to limitations in resources or due to unforeseen bugs. **However,** any errors encountered during this stage will be handled via exceptions or may result in process failure. Currently, there is no established mechanism to handle such failures. Future improvements should focus on implementing failover strategies, potentially allowing the GraphDB to continue replaying the WAL until it succeeds. + +## Persisting WAL to kafka + +Kafka-based WAL storages is also provided. Follows [kafka-quick-start](https://kafka.apache.org/quickstart). + +### Install kafka + +```bash +wget https://dlcdn.apache.org/kafka/3.9.0/kafka_2.13-3.9.0.tgz +tar -zxf kafka_2.13-3.9.0.tgz +cd kafka_2.13-3.9.0 +``` + +### kafka with kraft + +```bash +KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" +bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/kraft/reconfig-server.properties +bin/kafka-server-start.sh config/kraft/reconfig-server.properties +``` + +### Create topic + +```bash +bin/kafka-topics.sh --create --topic kafka-test --bootstrap-server localhost:9092 +# describe the topic +bin/kafka-topics.sh --describe --topic kafka-test --bootstrap-server localhost:9092 +``` + +### Test KafkaWalWriter and KafkaWalParser + +```bash +cd flex && mkdir build +cd build && cmake .. -DBUILD_TEST=ON && make -j +./tests/hqps/kafka_test localhost:902 kafka-test +# run the kafka tst +``` diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index 2c27898a9e8e..7143b334a113 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -17,6 +17,7 @@ option(USE_PTHASH "Whether to use pthash" OFF) option(OPTIMIZE_FOR_HOST "Whether to optimize on host" ON) # Whether to build optimized code on host option(USE_STATIC_ARROW "Whether to use static arrow" OFF) # Whether to link arrow statically, default is OFF option(BUILD_WITH_OTEL "Whether to build with opentelemetry-cpp" OFF) # Whether to build with opentelemetry-cpp, default is OFF +option(BUILD_KAFKA_WAL_WRITER_PARSER "Whether to build kafka wal writer and wal parser" ON) # Whether to build kafka wal writer and wal parser, default is ON #print options message(STATUS "Build test: ${BUILD_TEST}") @@ -165,6 +166,38 @@ endif () find_package(Protobuf REQUIRED) include_directories(${Protobuf_INCLUDE_DIRS}) +#find CppKafka------------------------------------------------------------------- +if (BUILD_KAFKA_WAL_WRITER_PARSER) + find_package(CppKafka) + if (NOT CppKafka_FOUND) + message(STATUS "cppkafka not found, try to build with third_party/cppkafka") + set(CPPKAFKA_BOOST_STATIC_LIBS OFF) + add_subdirectory(third_party/cppkafka) + if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka/CMakeLists.txt) + message(FATAL_ERROR "cppkafka not found, please run git submodule update --init --recursive") + endif () + # Apply third_party/cppkafka_without_boost.patch + # to remove the dependency of boost in cppkafka + # run git apply ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka_without_boost.patch + add_custom_target(apply_patch + COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka_without_boost.patch || true + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka + COMMENT "Applying patch to cppkafka" + VERBATIM) + + include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka/include) + set (CppKafka_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka/build) + list (APPEND CMAKE_PREFIX_PATH ${CppKafka_INSTALL_DIR}) + set(CppKafka_LIBRARIES cppkafka) + else() + include_directories(SYSTEM ${CppKafka_INCLUDE_DIRS}) + message(STATUS "cpp kafka include dir: ${CppKafka_INCLUDE_DIRS}") + set(CppKafka_LIBRARIES CppKafka::cppkafka) + endif () + add_definitions(-DBUILD_KAFKA_WAL_WRITER_PARSER) + message(STATUS "cppkafka found") +endif() + if (BUILD_WITH_OTEL) find_package(opentelemetry-cpp CONFIG) if (OPENTELEMETRY_CPP_FOUND) diff --git a/flex/engines/graph_db/CMakeLists.txt b/flex/engines/graph_db/CMakeLists.txt index fb20dbf56750..1e2f0ba37b52 100644 --- a/flex/engines/graph_db/CMakeLists.txt +++ b/flex/engines/graph_db/CMakeLists.txt @@ -4,11 +4,20 @@ file(GLOB_RECURSE GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/app/*.cc" "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/*.cc" "${CMAKE_CURRENT_SOURCE_DIR}/app/builtin/*.cc") +if (BUILD_KAFKA_WAL_WRITER_PARSER) + list(APPEND GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/kafka_wal_writer.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/kafka_wal_parser.cc") +endif() + add_library(flex_graph_db SHARED ${GRAPH_DB_SRC_FILES}) target_include_directories(flex_graph_db PUBLIC $) target_link_libraries(flex_graph_db flex_rt_mutable_graph ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(flex_graph_db runtime_execute) +if (BUILD_KAFKA_WAL_WRITER_PARSER) + target_link_libraries(flex_graph_db ${CppKafka_LIBRARIES}) +endif() +add_dependencies(flex_graph_db apply_patch) install_flex_target(flex_graph_db) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/database/graph_db.h diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc new file mode 100644 index 000000000000..2fdc9263964b --- /dev/null +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc @@ -0,0 +1,154 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/database/wal/kafka_wal_parser.h" +#include +#include +#include +#include +#include "flex/engines/graph_db/database/wal/wal.h" + +namespace gs { + +std::vector get_all_topic_partitions( + const cppkafka::Configuration& config, const std::string& topic_name) { + std::vector partitions; + cppkafka::Consumer consumer(config); // tmp consumer + auto meta_vector = consumer.get_metadata().get_topics({topic_name}); + if (meta_vector.empty()) { + LOG(WARNING) << "Failed to get metadata for topic " << topic_name + << ", maybe the topic does not exist"; + return {}; + } + auto metadata = meta_vector.front().get_partitions(); + for (const auto& partition : metadata) { + partitions.push_back(cppkafka::TopicPartition( + topic_name, partition.get_id(), 0)); // from the beginning + } + return partitions; +} + +std::unique_ptr KafkaWalParser::Make() { + const char* broker_list = std::getenv("KAFKA_BROKER_LIST"); + if (broker_list == nullptr) { + LOG(FATAL) << "KAFKA_BROKER_LIST is not set"; + } + const char* group_id = std::getenv("KAFKA_GROUP_ID"); + std::string group_id_str = group_id ? group_id : "interactive_consumer"; + cppkafka::Configuration config = {{"metadata.broker.list", broker_list}, + {"group.id", group_id_str}}; + return std::unique_ptr(new KafkaWalParser(config)); +} + +KafkaWalParser::KafkaWalParser(const cppkafka::Configuration& config) + : consumer_(nullptr), + insert_wal_list_(NULL), + insert_wal_list_size_(0), + last_ts_(0), + config_(config) { + consumer_ = std::make_unique(config); +} + +void KafkaWalParser::open(const std::string& topic_name) { + auto topic_partitions = get_all_topic_partitions(config_, topic_name); + open(topic_partitions); +} + +void KafkaWalParser::open( + const std::vector& topic_partitions) { + consumer_->assign(topic_partitions); + insert_wal_list_ = static_cast( + mmap(NULL, IWalWriter::MAX_WALS_NUM * sizeof(WalContentUnit), + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0)); + insert_wal_list_size_ = IWalWriter::MAX_WALS_NUM; + + while (true) { + auto msgs = consumer_->poll_batch(MAX_BATCH_SIZE); + if (msgs.empty() || msgs.empty()) { + LOG(INFO) << "No message are polled, the topic has been all consumed."; + break; + } + for (auto& msg : msgs) { + if (msg) { + if (msg.get_error()) { + if (!msg.is_eof()) { + LOG(INFO) << "[+] Received error notification: " << msg.get_error(); + } + } else { + message_vector_.emplace_back(msg.get_payload()); + } + } + } + } + + for (auto& wal : message_vector_) { + LOG(INFO) << "Got wal:" << wal.size(); + const char* payload = wal.data(); + const WalHeader* header = reinterpret_cast(payload); + uint32_t cur_ts = header->timestamp; + if (cur_ts == 0) { + LOG(WARNING) << "Invalid timestamp 0, skip"; + continue; + } + int length = header->length; + if (header->type) { + UpdateWalUnit unit; + unit.timestamp = cur_ts; + unit.ptr = const_cast(payload + sizeof(WalHeader)); + unit.size = length; + update_wal_list_.push_back(unit); + } else { + if (insert_wal_list_[cur_ts].ptr) { + LOG(WARNING) << "Duplicated timestamp " << cur_ts << ", skip"; + } + insert_wal_list_[cur_ts].ptr = + const_cast(payload + sizeof(WalHeader)); + insert_wal_list_[cur_ts].size = length; + } + last_ts_ = std::max(cur_ts, last_ts_); + } + + LOG(INFO) << "last_ts: " << last_ts_; + if (!update_wal_list_.empty()) { + std::sort(update_wal_list_.begin(), update_wal_list_.end(), + [](const UpdateWalUnit& lhs, const UpdateWalUnit& rhs) { + return lhs.timestamp < rhs.timestamp; + }); + } +} + +void KafkaWalParser::close() { + if (consumer_) { + consumer_.reset(); + } + if (insert_wal_list_ != NULL) { + munmap(insert_wal_list_, insert_wal_list_size_ * sizeof(WalContentUnit)); + } +} + +uint32_t KafkaWalParser::last_ts() const { return last_ts_; } + +const WalContentUnit& KafkaWalParser::get_insert_wal(uint32_t ts) const { + if (insert_wal_list_[ts].ptr == NULL) { + LOG(WARNING) << "No WAL for timestamp " << ts; + } + return insert_wal_list_[ts]; +} +const std::vector& KafkaWalParser::get_update_wals() const { + return update_wal_list_; +} + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.h b/flex/engines/graph_db/database/wal/kafka_wal_parser.h new file mode 100644 index 000000000000..7748ccc2f2e9 --- /dev/null +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.h @@ -0,0 +1,67 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_PARSER_H_ +#define ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_PARSER_H_ + +#include +#include "flex/engines/graph_db/database/wal/wal.h" + +#include "cppkafka/cppkafka.h" + +namespace gs { + +/* + * Get all partitions of the given topic. + */ +std::vector get_all_topic_partitions( + const cppkafka::Configuration& config, const std::string& topic_name); + +class KafkaWalParser : public IWalParser { + public: + static constexpr const std::chrono::milliseconds POLL_TIMEOUT = + std::chrono::milliseconds(100); + static constexpr const size_t MAX_BATCH_SIZE = 1000; + + static std::unique_ptr Make(); + + // always track all partitions and from begining + KafkaWalParser(const cppkafka::Configuration& config); + ~KafkaWalParser() { close(); } + + void open(const std::string& topic_name) override; + void close() override; + + uint32_t last_ts() const override; + const WalContentUnit& get_insert_wal(uint32_t ts) const override; + const std::vector& get_update_wals() const override; + + //////Kafka specific methods + void open(const std::vector& partitions); + + private: + std::unique_ptr consumer_; + WalContentUnit* insert_wal_list_; + size_t insert_wal_list_size_; + uint32_t last_ts_; + + std::vector update_wal_list_; + std::vector message_vector_; // used to hold the polled messages + cppkafka::Configuration config_; +}; + +} // namespace gs + +#endif // ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_PARSER_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/database/wal/kafka_wal_writer.cc b/flex/engines/graph_db/database/wal/kafka_wal_writer.cc new file mode 100644 index 000000000000..c94ec09fa42a --- /dev/null +++ b/flex/engines/graph_db/database/wal/kafka_wal_writer.cc @@ -0,0 +1,79 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/database/wal/kafka_wal_writer.h" +#include "flex/engines/graph_db/database/wal/wal.h" + +#include +#include + +namespace gs { + +std::unique_ptr KafkaWalWriter::Make() { + // Get env KAFKA_BROKER_LIST + const char* broker_list = std::getenv("KAFKA_BROKER_LIST"); + if (broker_list == nullptr) { + LOG(FATAL) << "KAFKA_BROKER_LIST is not set"; + } + return std::unique_ptr(new KafkaWalWriter(broker_list)); +} + +void KafkaWalWriter::open(const std::string& topic, int thread_id) { + if (thread_id_ != -1 || producer_) { + LOG(FATAL) << "KafkaWalWriter has been opened"; + } + thread_id_ = thread_id; + if (!kafka_brokers_.empty()) { + if (topic.empty()) { + LOG(FATAL) << "Kafka topic is empty"; + } + kafka_topic_ = topic; + cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers_}}; + producer_ = + std::make_shared>(config); + producer_->set_max_number_retries(DEFAULT_MAX_RETRIES); + builder_.topic(kafka_topic_).partition(thread_id_); + } else { + LOG(FATAL) << "Kafka brokers is empty"; + } +} + +void KafkaWalWriter::close() { + if (producer_) { + producer_->flush(); + producer_.reset(); + thread_id_ = -1; + kafka_topic_.clear(); + } +} + +bool KafkaWalWriter::append(const char* data, size_t length) { + try { + producer_->sync_produce(builder_.payload({data, length})); + producer_->flush(true); + } catch (const cppkafka::HandleException& e) { + LOG(ERROR) << "Failed to send to kafka: " << e.what(); + return false; + } + VLOG(10) << "Finished sending to kafka with message size: " << length + << ", partition: " << thread_id_; + return true; +} + +const bool KafkaWalWriter::registered_ = WalWriterFactory::RegisterWalWriter( + "kafka", static_cast( + &KafkaWalWriter::Make)); + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/database/wal/kafka_wal_writer.h b/flex/engines/graph_db/database/wal/kafka_wal_writer.h new file mode 100644 index 000000000000..e031e62ff488 --- /dev/null +++ b/flex/engines/graph_db/database/wal/kafka_wal_writer.h @@ -0,0 +1,79 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_WRITER_H_ +#define ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_WRITER_H_ + +#include +#include +#include "flex/engines/graph_db/database/wal/wal.h" + +#include "cppkafka/cppkafka.h" + +namespace gs { + +/** + * KafkaWalWriter is a concrete implementation of IWalWriter that writes WAL + * entries to a Kafka topic. + * It uses cppkafka to produce messages to Kafka. + * The Kafka brokers are specified by the environment variable + * KAFKA_BROKER_LIST. The Kafka topic is specified by the open() method, the + * thread_id is used as the partition number. + * + * After restarting the GraphDB service, the KafkaWalWriter will continue to + * write to the same Kafka topic and partition. Consumers should be able to + * select the WAL entries by the timestamp. + */ +class KafkaWalWriter : public IWalWriter { + public: + static constexpr const int DEFAULT_MAX_RETRIES = 10; + + static std::unique_ptr Make(); + + KafkaWalWriter(const std::string& kafka_brokers) + : thread_id_(-1), + kafka_brokers_(kafka_brokers), + kafka_topic_(""), + producer_(nullptr), + builder_("") {} // brokers could be a list of brokers + + ~KafkaWalWriter() { close(); } + + void open(const std::string& kafka_topic, int thread_id) override; + void close() override; + bool append(const char* data, size_t length) override; + std::string type() const override { return "kafka"; } + + //////Kafka specific methods + + inline std::tuple getCurrentOffset() { + return producer_->get_producer().query_offsets( + cppkafka::TopicPartition(kafka_topic_, thread_id_)); + } + + private: + int32_t thread_id_; + std::string kafka_brokers_; + std::string kafka_topic_; + std::shared_ptr> producer_; + cppkafka::MessageBuilder builder_; + int64_t cur_offset_; + + static const bool registered_; +}; + +} // namespace gs + +#endif // ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_WRITER_H_ \ No newline at end of file diff --git a/flex/tests/hqps/kafka_test.cc b/flex/tests/hqps/kafka_test.cc new file mode 100644 index 000000000000..56c47e687a79 --- /dev/null +++ b/flex/tests/hqps/kafka_test.cc @@ -0,0 +1,87 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include "grape/serialization/in_archive.h" +#include "grape/serialization/out_archive.h" + +#include + +int main(int argc, char** argv) { + if (argc < 3) { + std::cerr << "usage: kafka_test " + << std::endl; + return -1; + } + + std::string kafka_brokers = argv[1]; + std::string kafka_topic = argv[2]; + LOG(INFO) << "Kafka brokers: " << kafka_brokers; + + // Write messages to the specified kafka topic, and read them back. + gs::KafkaWalWriter writer(kafka_brokers); + cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers}, + {"group.id", "test"}, + {"enable.auto.commit", false}}; + gs::KafkaWalParser parser(config); + writer.open(kafka_topic, 0); + + // Let user enter number of messages to write + LOG(INFO) << "Enter number of messages to write: "; + int num_messages = 3; + for (int i = 0; i < num_messages; ++i) { + std::string message = "message " + std::to_string(i); + grape::InArchive in_archive; + in_archive.Resize(sizeof(gs::WalHeader)); + in_archive << message; + auto header = reinterpret_cast(in_archive.GetBuffer()); + header->timestamp = i + 1; + header->type = 0; + header->length = in_archive.GetSize() - sizeof(gs::WalHeader); + LOG(INFO) << "Writing wal: " << header->timestamp << ", " << header->length; + writer.append(in_archive.GetBuffer(), in_archive.GetSize()); + } + auto offset = writer.getCurrentOffset(); + LOG(INFO) << "Current offset: " << std::get<0>(offset) << ", " + << std::get<1>(offset); + writer.close(); + LOG(INFO) << "Messages have been written to Kafka topic: " << argv[2] + << std::endl; + + // Digest the messages + std::vector topic_partitions = { + {kafka_topic, 0, 0}}; + + // parser.open(topic_partitions); + parser.open(kafka_topic); + auto last_ts = parser.last_ts(); + for (int i = 0; i < num_messages; ++i) { + const gs::WalContentUnit& wal = parser.get_insert_wal(last_ts + i); + if (wal.ptr) { + grape::OutArchive out_archive; + out_archive.SetSlice(wal.ptr, wal.size); + std::string message; + out_archive >> message; + LOG(INFO) << "Read message: " << message; + } else { + LOG(ERROR) << "No message for timestamp " << last_ts + i; + } + } + + return 0; +} \ No newline at end of file diff --git a/flex/third_party/cppkafka b/flex/third_party/cppkafka new file mode 160000 index 000000000000..9c5ea0e33248 --- /dev/null +++ b/flex/third_party/cppkafka @@ -0,0 +1 @@ +Subproject commit 9c5ea0e332486961e612deacc6e3f0c1874c688d diff --git a/flex/third_party/cppkafka_without_boost.patch b/flex/third_party/cppkafka_without_boost.patch new file mode 100644 index 000000000000..6a6e8e74d6c7 --- /dev/null +++ b/flex/third_party/cppkafka_without_boost.patch @@ -0,0 +1,55 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 6c667df..8ebae74 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -104,24 +104,6 @@ if (NOT CPPKAFKA_PKGCONFIG_DIR) + set(CPPKAFKA_PKGCONFIG_DIR share/pkgconfig) + endif() + +-# Look for Boost (just need boost.optional headers here) +-find_package(Boost REQUIRED ${FIND_PACKAGE_QUIET}) +- +-if (Boost_FOUND) +- find_package(Boost COMPONENTS program_options ${FIND_PACKAGE_QUIET}) +- set(Boost_USE_STATIC_LIBS ${CPPKAFKA_BOOST_STATIC_LIBS}) +- set(Boost_USE_MULTITHREADED ${CPPKAFKA_BOOST_USE_MULTITHREADED}) +- include_directories(${Boost_INCLUDE_DIRS}) +- link_directories(${Boost_LIBRARY_DIRS}) +- if (CPPKAFKA_CMAKE_VERBOSE) +- message(STATUS "Boost include dir: ${Boost_INCLUDE_DIRS}") +- message(STATUS "Boost library dir: ${Boost_LIBRARY_DIRS}") +- message(STATUS "Boost use static libs: ${Boost_USE_STATIC_LIBS}") +- message(STATUS "Boost is multi-threaded: ${CPPKAFKA_BOOST_USE_MULTITHREADED}") +- message(STATUS "Boost libraries: ${Boost_LIBRARIES}") +- endif() +-endif() +- + # Try to find the RdKafka configuration file if present. + # This will search default system locations as well as RdKafka_ROOT and RdKafka_Dir paths if specified. + find_package(RdKafka ${FIND_PACKAGE_QUIET} CONFIG) +diff --git a/cmake/config.cmake.in b/cmake/config.cmake.in +index 32f4398..749790c 100644 +--- a/cmake/config.cmake.in ++++ b/cmake/config.cmake.in +@@ -8,7 +8,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_LIST_DIR}") + set(RDKAFKA_MIN_VERSION_HEX "@RDKAFKA_MIN_VERSION_HEX@") + + # Find boost optional +-find_dependency(Boost REQUIRED) ++# find_dependency(Boost REQUIRED) + + # Try to find the RdKafka configuration file if present. + # This will search default system locations as well as RdKafka_ROOT and RdKafka_DIR paths if specified. +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 214ccd4..7d73ad6 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -51,7 +51,7 @@ set_target_properties(${TARGET_NAME} PROPERTIES + VERSION ${CPPKAFKA_VERSION} + SOVERSION ${CPPKAFKA_VERSION}) + # In CMake >= 3.15 Boost::boost == Boost::headers +-target_link_libraries(${TARGET_NAME} PUBLIC RdKafka::rdkafka Boost::boost) ++target_link_libraries(${TARGET_NAME} PUBLIC RdKafka::rdkafka) + if (WIN32) + # On windows ntohs and related are in ws2_32 + target_link_libraries(${TARGET_NAME} PUBLIC ws2_32.lib) diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index 397e3b76b6a0..33ac3da71e73 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -866,9 +866,9 @@ install_analytical_java_dependencies() { fi } -INTERACTIVE_MACOS=("apache-arrow" "rapidjson" "boost" "glog" "gflags") -INTERACTIVE_UBUNTU=("rapidjson-dev" "libgoogle-glog-dev" "libgflags-dev") -INTERACTIVE_CENTOS=("rapidjson-devel") +INTERACTIVE_MACOS=("apache-arrow" "rapidjson" "boost" "glog" "gflags" "librdkafka") +INTERACTIVE_UBUNTU=("rapidjson-dev" "libgoogle-glog-dev" "libgflags-dev" "librdkafka-dev") +INTERACTIVE_CENTOS=("rapidjson-devel" "librdkafka-devel") install_interactive_dependencies() { # dependencies package From 56d4fc58b7ed896a1697570cfb8d7f4040ef1def Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Tue, 25 Mar 2025 17:42:13 +0800 Subject: [PATCH 02/17] fix cmakelist.txt --- .gitmodules | 3 - flex/CMakeLists.txt | 47 +++---- flex/engines/graph_db/CMakeLists.txt | 7 +- .../graph_db/app/kafka_wal_ingester_app.cc | 121 ++++++++++++++++++ .../graph_db/app/kafka_wal_ingester_app.h | 45 +++++++ flex/engines/graph_db/database/graph_db.cc | 36 ++++++ flex/engines/graph_db/database/graph_db.h | 15 +++ .../graph_db/database/insert_transaction.cc | 2 + .../graph_db/database/insert_transaction.h | 2 + .../graph_db/database/update_transaction.cc | 2 + .../graph_db/database/update_transaction.h | 2 + flex/tests/hqps/kafka_test.cc | 9 +- flex/third_party/cppkafka | 1 - flex/third_party/cppkafka_without_boost.patch | 55 -------- .../graphscope/gsctl/scripts/install_deps.sh | 2 + 15 files changed, 251 insertions(+), 98 deletions(-) create mode 100644 flex/engines/graph_db/app/kafka_wal_ingester_app.cc create mode 100644 flex/engines/graph_db/app/kafka_wal_ingester_app.h delete mode 160000 flex/third_party/cppkafka delete mode 100644 flex/third_party/cppkafka_without_boost.patch diff --git a/.gitmodules b/.gitmodules index 48cda2149aa8..1203c4c4d451 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,6 +13,3 @@ [submodule "flex/third_party/parallel-hashmap"] path = flex/third_party/parallel-hashmap url = https://github.com/greg7mdp/parallel-hashmap.git -[submodule "flex/third_party/cppkafka"] - path = flex/third_party/cppkafka - url = https://github.com/mfontanini/cppkafka.git diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index 7143b334a113..3c654fe10282 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -122,6 +122,21 @@ if (NOT yaml-cpp_FOUND) message(FATAL_ERROR "yaml-cpp not found, please install the yaml-cpp library") endif () +#find CppKafka------------------------------------------------------------------- +if (BUILD_KAFKA_WAL_WRITER_PARSER) + find_package(CppKafka) + if (NOT CppKafka_FOUND) + message(FATAL_ERROR "cppkafka not found, please install cppkafka library") + else() + include_directories(SYSTEM ${CppKafka_INCLUDE_DIRS}) + message(STATUS "cpp kafka include dir: ${CppKafka_INCLUDE_DIRS}") + set(CppKafka_LIBRARIES CppKafka::cppkafka) + add_definitions(-DBUILD_KAFKA_WAL_WRITER_PARSER) + message(STATUS "cppkafka found") + endif () +endif() + + #find boost---------------------------------------------------------------------- find_package(Boost REQUIRED COMPONENTS system filesystem # required by folly @@ -166,38 +181,6 @@ endif () find_package(Protobuf REQUIRED) include_directories(${Protobuf_INCLUDE_DIRS}) -#find CppKafka------------------------------------------------------------------- -if (BUILD_KAFKA_WAL_WRITER_PARSER) - find_package(CppKafka) - if (NOT CppKafka_FOUND) - message(STATUS "cppkafka not found, try to build with third_party/cppkafka") - set(CPPKAFKA_BOOST_STATIC_LIBS OFF) - add_subdirectory(third_party/cppkafka) - if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka/CMakeLists.txt) - message(FATAL_ERROR "cppkafka not found, please run git submodule update --init --recursive") - endif () - # Apply third_party/cppkafka_without_boost.patch - # to remove the dependency of boost in cppkafka - # run git apply ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka_without_boost.patch - add_custom_target(apply_patch - COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka_without_boost.patch || true - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka - COMMENT "Applying patch to cppkafka" - VERBATIM) - - include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka/include) - set (CppKafka_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cppkafka/build) - list (APPEND CMAKE_PREFIX_PATH ${CppKafka_INSTALL_DIR}) - set(CppKafka_LIBRARIES cppkafka) - else() - include_directories(SYSTEM ${CppKafka_INCLUDE_DIRS}) - message(STATUS "cpp kafka include dir: ${CppKafka_INCLUDE_DIRS}") - set(CppKafka_LIBRARIES CppKafka::cppkafka) - endif () - add_definitions(-DBUILD_KAFKA_WAL_WRITER_PARSER) - message(STATUS "cppkafka found") -endif() - if (BUILD_WITH_OTEL) find_package(opentelemetry-cpp CONFIG) if (OPENTELEMETRY_CPP_FOUND) diff --git a/flex/engines/graph_db/CMakeLists.txt b/flex/engines/graph_db/CMakeLists.txt index 1e2f0ba37b52..740d3466c9c0 100644 --- a/flex/engines/graph_db/CMakeLists.txt +++ b/flex/engines/graph_db/CMakeLists.txt @@ -4,9 +4,9 @@ file(GLOB_RECURSE GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/app/*.cc" "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/*.cc" "${CMAKE_CURRENT_SOURCE_DIR}/app/builtin/*.cc") -if (BUILD_KAFKA_WAL_WRITER_PARSER) - list(APPEND GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/kafka_wal_writer.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/kafka_wal_parser.cc") +if (NOT BUILD_KAFKA_WAL_WRITER_PARSER) + list(REMOVE_ITEM GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/kafka_wal_writer.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/database/wal/kafka_wal_parser.cc") endif() add_library(flex_graph_db SHARED ${GRAPH_DB_SRC_FILES}) @@ -17,7 +17,6 @@ target_link_libraries(flex_graph_db runtime_execute) if (BUILD_KAFKA_WAL_WRITER_PARSER) target_link_libraries(flex_graph_db ${CppKafka_LIBRARIES}) endif() -add_dependencies(flex_graph_db apply_patch) install_flex_target(flex_graph_db) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/database/graph_db.h diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc new file mode 100644 index 000000000000..d3128e064160 --- /dev/null +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -0,0 +1,121 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/app/kafka_wal_ingester_app.h" +#include "cppkafka/cppkafka.h" +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/database/wal/kafka_wal_parser.h" + +namespace gs { +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER + +class KafkaWalConsumer { + public: + struct CustomComparator { + inline bool operator()(const std::string& lhs, const std::string& rhs) { + const WalHeader* header1 = reinterpret_cast(lhs.data()); + const WalHeader* header2 = reinterpret_cast(rhs.data()); + return header1->timestamp > header2->timestamp; + } + }; + static constexpr const std::chrono::milliseconds POLL_TIMEOUT = + std::chrono::milliseconds(100); + + // always track all partitions and from begining + KafkaWalConsumer(cppkafka::Configuration config, + const std::string& topic_name, int32_t thread_num) { + auto topic_partitions = get_all_topic_partitions(config, topic_name); + consumers_.reserve(topic_partitions.size()); + for (size_t i = 0; i < topic_partitions.size(); ++i) { + consumers_.emplace_back(std::make_unique(config)); + consumers_.back()->assign({topic_partitions[i]}); + } + } + + std::string poll() { + for (auto& consumer : consumers_) { + auto msg = consumer->poll(); + if (msg) { + if (msg.get_error()) { + if (!msg.is_eof()) { + LOG(INFO) << "[+] Received error notification: " << msg.get_error(); + } + } else { + std::string payload = msg.get_payload(); + message_queue_.push(payload); + consumer->commit(msg); + } + } + } + if (message_queue_.empty()) { + return ""; + } + std::string payload = message_queue_.top(); + message_queue_.pop(); + return payload; + } + + private: + std::vector> consumers_; + std::priority_queue, CustomComparator> + message_queue_; +}; + +bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, + Encoder& output) { + // TODO: get value from config + std::string kafka_brokers = std::string(input.get_string()); + std::string brokers = std::string(input.get_string()); + std::string topic_name = std::string(input.get_string()); + std::string group_id = std::string(input.get_string()); + std::string engine_endpoint = std::string(input.get_string()); + LOG(INFO) << "Kafka brokers: " << kafka_brokers; + LOG(INFO) << "engine endpoint: " << engine_endpoint; + + cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers}, + {"group.id", group_id}, + // Disable auto commit + {"enable.auto.commit", false}}; + gs::KafkaWalConsumer consumer(config, topic_name, 1); + // TODO: how to make it stop + while (!graph.db().kafka_wal_ingester_state()) { + auto res = consumer.poll(); + if (res.empty()) { + std::this_thread::sleep_for(gs::KafkaWalConsumer::POLL_TIMEOUT); + continue; + } + + auto header = reinterpret_cast(res.data()); + if (header->type == 0) { + auto txn = graph.GetInsertTransaction(); + txn.IngestWal(graph.graph(), txn.timestamp(), + const_cast(res.data()) + sizeof(WalHeader), + header->length, txn.allocator()); + } else if (header->type == 1) { + auto txn = graph.GetUpdateTransaction(); + txn.IngestWal(graph.graph(), graph.db().work_dir(), txn.timestamp(), + const_cast(res.data()) + sizeof(WalHeader), + header->length, txn.allocator()); + } else { + LOG(ERROR) << "Unknown wal type: " << header->type; + } + } + return true; +} +AppWrapper KafkaWalIngesterAppFactory::CreateApp(const GraphDB& db) { + return AppWrapper(new KafkaWalIngesterApp(), NULL); +} +#endif +} // namespace gs diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.h b/flex/engines/graph_db/app/kafka_wal_ingester_app.h new file mode 100644 index 000000000000..bb4d33b379d9 --- /dev/null +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.h @@ -0,0 +1,45 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_KAFKA_WAL_INGESTER_APP_H_ +#define ENGINES_KAFKA_WAL_INGESTER_APP_H_ + +#include "flex/engines/graph_db/app/app_base.h" +#include "flex/engines/graph_db/database/graph_db_session.h" + +namespace gs { +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +// Ingest wal from kafka +class KafkaWalIngesterApp : public WriteAppBase { + public: + KafkaWalIngesterApp() {} + + AppType type() const override { return AppType::kBuiltIn; } + + bool Query(GraphDBSession& graph, Decoder& input, Encoder& output) override; +}; + +class KafkaWalIngesterAppFactory : public AppFactoryBase { + public: + KafkaWalIngesterAppFactory() = default; + ~KafkaWalIngesterAppFactory() = default; + + AppWrapper CreateApp(const GraphDB& db) override; +}; +#endif + +} // namespace gs + +#endif // ENGINES_KAFKA_WAL_INGESTER_APP_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index 9d39abd5916f..f50bbbabdbe5 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -28,6 +28,10 @@ #include "flex/engines/graph_db/runtime/utils/cypher_runner_impl.h" #include "flex/utils/yaml_utils.h" +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +#include "flex/engines/graph_db/app/kafka_wal_ingester_app.h" +#endif + #include "flex/third_party/httplib.h" namespace gs { @@ -274,6 +278,38 @@ void GraphDB::Close() { std::fill(app_factories_.begin(), app_factories_.end(), nullptr); } +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +bool GraphDB::kafka_wal_ingester_state() const { + return kafka_wal_ingester_thread_running_.load(); +} + +void GraphDB::start_kafka_wal_ingester(const std::string& kafka_brokers, + const std::string& brokers, + const std::string& topic_name, + const std::string& group_id, + const std::string& engine_endpoint) { + if (kafka_wal_ingester_thread_running_) { + kafka_wal_ingester_thread_running_ = false; + kafka_wal_ingester_thread_.join(); + } + std::vector buffer; + gs::Encoder encoder(buffer); + encoder.put_string(kafka_brokers); + encoder.put_string(brokers); + encoder.put_string(topic_name); + encoder.put_string(group_id); + encoder.put_string(engine_endpoint); + gs::Decoder decoder(buffer.data(), buffer.size()); + KafkaWalIngesterApp().Query(GetSession(0), decoder, encoder); +} + +void GraphDB::stop_kafka_wal_ingester() { + kafka_wal_ingester_thread_running_ = false; + kafka_wal_ingester_thread_.join(); +} + +#endif + ReadTransaction GraphDB::GetReadTransaction(int thread_id) { return contexts_[thread_id].session.GetReadTransaction(); } diff --git a/flex/engines/graph_db/database/graph_db.h b/flex/engines/graph_db/database/graph_db.h index 2481d403c4d8..e337eade6886 100644 --- a/flex/engines/graph_db/database/graph_db.h +++ b/flex/engines/graph_db/database/graph_db.h @@ -169,6 +169,18 @@ class GraphDB { inline const GraphDBConfig& config() const { return config_; } +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER + bool kafka_wal_ingester_state() const; + + void start_kafka_wal_ingester(const std::string& kafka_brokers, + const std::string& brokers, + const std::string& topic_name, + const std::string& group_id, + const std::string& engine_endpoint); + + void stop_kafka_wal_ingester(); +#endif + private: bool registerApp(const std::string& path, uint8_t index = 0); @@ -204,6 +216,9 @@ class GraphDB { std::thread monitor_thread_; bool monitor_thread_running_; + std::thread kafka_wal_ingester_thread_; + std::atomic kafka_wal_ingester_thread_running_; + timestamp_t last_compaction_ts_; bool compact_thread_running_ = false; std::thread compact_thread_; diff --git a/flex/engines/graph_db/database/insert_transaction.cc b/flex/engines/graph_db/database/insert_transaction.cc index a6cc61a942dd..7a9ac09f3e57 100644 --- a/flex/engines/graph_db/database/insert_transaction.cc +++ b/flex/engines/graph_db/database/insert_transaction.cc @@ -226,6 +226,8 @@ const Schema& InsertTransaction::schema() const { return graph_.schema(); } const GraphDBSession& InsertTransaction::GetSession() const { return session_; } +Allocator& InsertTransaction::allocator() const { return alloc_; } + #define likely(x) __builtin_expect(!!(x), 1) bool InsertTransaction::get_vertex_with_retries(MutablePropertyFragment& graph, diff --git a/flex/engines/graph_db/database/insert_transaction.h b/flex/engines/graph_db/database/insert_transaction.h index c954ccd6a0ff..7a5fcd9357ef 100644 --- a/flex/engines/graph_db/database/insert_transaction.h +++ b/flex/engines/graph_db/database/insert_transaction.h @@ -61,6 +61,8 @@ class InsertTransaction { const GraphDBSession& GetSession() const; + Allocator& allocator() const; + private: void clear(); diff --git a/flex/engines/graph_db/database/update_transaction.cc b/flex/engines/graph_db/database/update_transaction.cc index 7d8a53eefdb9..06b5effa9af1 100644 --- a/flex/engines/graph_db/database/update_transaction.cc +++ b/flex/engines/graph_db/database/update_transaction.cc @@ -102,6 +102,8 @@ UpdateTransaction::~UpdateTransaction() { release(); } timestamp_t UpdateTransaction::timestamp() const { return timestamp_; } +Allocator& UpdateTransaction::allocator() { return alloc_; } + bool UpdateTransaction::Commit() { if (timestamp_ == std::numeric_limits::max()) { return true; diff --git a/flex/engines/graph_db/database/update_transaction.h b/flex/engines/graph_db/database/update_transaction.h index 12988e8f1135..beff87579b66 100644 --- a/flex/engines/graph_db/database/update_transaction.h +++ b/flex/engines/graph_db/database/update_transaction.h @@ -153,6 +153,8 @@ class UpdateTransaction { const GraphDBSession& GetSession() const; + Allocator& allocator(); + private: friend class GraphDBSession; bool batch_commit(UpdateBatch& batch); diff --git a/flex/tests/hqps/kafka_test.cc b/flex/tests/hqps/kafka_test.cc index 56c47e687a79..087cb2657339 100644 --- a/flex/tests/hqps/kafka_test.cc +++ b/flex/tests/hqps/kafka_test.cc @@ -15,11 +15,12 @@ #include -#include -#include #include "grape/serialization/in_archive.h" #include "grape/serialization/out_archive.h" - +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +#include +#include +#endif #include int main(int argc, char** argv) { @@ -28,6 +29,7 @@ int main(int argc, char** argv) { << std::endl; return -1; } +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER std::string kafka_brokers = argv[1]; std::string kafka_topic = argv[2]; @@ -82,6 +84,7 @@ int main(int argc, char** argv) { LOG(ERROR) << "No message for timestamp " << last_ts + i; } } +#endif return 0; } \ No newline at end of file diff --git a/flex/third_party/cppkafka b/flex/third_party/cppkafka deleted file mode 160000 index 9c5ea0e33248..000000000000 --- a/flex/third_party/cppkafka +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9c5ea0e332486961e612deacc6e3f0c1874c688d diff --git a/flex/third_party/cppkafka_without_boost.patch b/flex/third_party/cppkafka_without_boost.patch deleted file mode 100644 index 6a6e8e74d6c7..000000000000 --- a/flex/third_party/cppkafka_without_boost.patch +++ /dev/null @@ -1,55 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 6c667df..8ebae74 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -104,24 +104,6 @@ if (NOT CPPKAFKA_PKGCONFIG_DIR) - set(CPPKAFKA_PKGCONFIG_DIR share/pkgconfig) - endif() - --# Look for Boost (just need boost.optional headers here) --find_package(Boost REQUIRED ${FIND_PACKAGE_QUIET}) -- --if (Boost_FOUND) -- find_package(Boost COMPONENTS program_options ${FIND_PACKAGE_QUIET}) -- set(Boost_USE_STATIC_LIBS ${CPPKAFKA_BOOST_STATIC_LIBS}) -- set(Boost_USE_MULTITHREADED ${CPPKAFKA_BOOST_USE_MULTITHREADED}) -- include_directories(${Boost_INCLUDE_DIRS}) -- link_directories(${Boost_LIBRARY_DIRS}) -- if (CPPKAFKA_CMAKE_VERBOSE) -- message(STATUS "Boost include dir: ${Boost_INCLUDE_DIRS}") -- message(STATUS "Boost library dir: ${Boost_LIBRARY_DIRS}") -- message(STATUS "Boost use static libs: ${Boost_USE_STATIC_LIBS}") -- message(STATUS "Boost is multi-threaded: ${CPPKAFKA_BOOST_USE_MULTITHREADED}") -- message(STATUS "Boost libraries: ${Boost_LIBRARIES}") -- endif() --endif() -- - # Try to find the RdKafka configuration file if present. - # This will search default system locations as well as RdKafka_ROOT and RdKafka_Dir paths if specified. - find_package(RdKafka ${FIND_PACKAGE_QUIET} CONFIG) -diff --git a/cmake/config.cmake.in b/cmake/config.cmake.in -index 32f4398..749790c 100644 ---- a/cmake/config.cmake.in -+++ b/cmake/config.cmake.in -@@ -8,7 +8,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_LIST_DIR}") - set(RDKAFKA_MIN_VERSION_HEX "@RDKAFKA_MIN_VERSION_HEX@") - - # Find boost optional --find_dependency(Boost REQUIRED) -+# find_dependency(Boost REQUIRED) - - # Try to find the RdKafka configuration file if present. - # This will search default system locations as well as RdKafka_ROOT and RdKafka_DIR paths if specified. -diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index 214ccd4..7d73ad6 100644 ---- a/src/CMakeLists.txt -+++ b/src/CMakeLists.txt -@@ -51,7 +51,7 @@ set_target_properties(${TARGET_NAME} PROPERTIES - VERSION ${CPPKAFKA_VERSION} - SOVERSION ${CPPKAFKA_VERSION}) - # In CMake >= 3.15 Boost::boost == Boost::headers --target_link_libraries(${TARGET_NAME} PUBLIC RdKafka::rdkafka Boost::boost) -+target_link_libraries(${TARGET_NAME} PUBLIC RdKafka::rdkafka) - if (WIN32) - # On windows ntohs and related are in ws2_32 - target_link_libraries(${TARGET_NAME} PUBLIC ws2_32.lib) diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index 33ac3da71e73..a8169c532b32 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -881,6 +881,7 @@ install_interactive_dependencies() { # hiactor is only supported on ubuntu install_hiactor install_mimalloc + install_cppkafka ${SUDO} sh -c 'echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf' ${SUDO} sysctl -p /etc/sysctl.conf else @@ -888,6 +889,7 @@ install_interactive_dependencies() { install_arrow install_boost install_mimalloc + install_cppkafka fi # libgrape-lite install_libgrape_lite "v0.3.2" From 6f323492e3a4070424b142d1bc204e8729dee74b Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Wed, 26 Mar 2025 15:11:22 +0800 Subject: [PATCH 03/17] fix ci --- .github/workflows/interactive.yml | 11 ++- .../graph_db/app/kafka_wal_ingester_app.cc | 12 +-- flex/engines/graph_db/database/graph_db.cc | 34 ++++---- flex/engines/graph_db/database/graph_db.h | 10 +-- .../graph_db/database/wal/kafka_wal_parser.cc | 31 +++----- .../graph_db/database/wal/kafka_wal_parser.h | 10 +-- flex/tests/hqps/kafka_wal_ingester_test.cc | 77 +++++++++++++++++++ 7 files changed, 130 insertions(+), 55 deletions(-) create mode 100644 flex/tests/hqps/kafka_wal_ingester_test.cc diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index db3023aadf53..d5600606f96f 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -371,6 +371,7 @@ jobs: env: GS_TEST_DIR: ${{ github.workspace }}/gstest run: | + cd ${GITHUB_WORKSPACE}/flex/build wget https://dlcdn.apache.org/kafka/3.9.0/kafka_2.13-3.9.0.tgz tar -zxf kafka_2.13-3.9.0.tgz cd kafka_2.13-3.9.0 @@ -379,10 +380,12 @@ jobs: bin/kafka-server-start.sh config/kraft/reconfig-server.properties & bin/kafka-topics.sh --create --topic kafka-test --bootstrap-server localhost:9092 - - ./tests/hqps/kafka_test localhost:902 kafka-test - - pkill -f kafka-server-start.sh + ../tests/hqps/kafka_test localhost:9092 kafka-test + bin/kafka-topics.sh --delete --topic kafka-test --bootstrap-server localhost:9092 + bin/kafka-topics.sh --create --topic kafka-test --bootstrap-server localhost:9092 + ../tests/hqps/kafka_wal_ingester_test .. localhost:9092 kafka-test + bin/kafka-topics.sh --delete --topic kafka-test --bootstrap-server localhost:9092 + pkill -f kafka - name: Run Gremlin test on modern graph env: diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc index d3128e064160..3799329307e3 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -77,20 +77,18 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, Encoder& output) { // TODO: get value from config std::string kafka_brokers = std::string(input.get_string()); - std::string brokers = std::string(input.get_string()); - std::string topic_name = std::string(input.get_string()); std::string group_id = std::string(input.get_string()); - std::string engine_endpoint = std::string(input.get_string()); + bool enable_auto_commit = input.get_byte(); + std::string topic_name = std::string(input.get_string()); LOG(INFO) << "Kafka brokers: " << kafka_brokers; - LOG(INFO) << "engine endpoint: " << engine_endpoint; cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers}, {"group.id", group_id}, // Disable auto commit - {"enable.auto.commit", false}}; + {"enable.auto.commit", enable_auto_commit}}; gs::KafkaWalConsumer consumer(config, topic_name, 1); // TODO: how to make it stop - while (!graph.db().kafka_wal_ingester_state()) { + while (graph.db().kafka_wal_ingester_state()) { auto res = consumer.poll(); if (res.empty()) { std::this_thread::sleep_for(gs::KafkaWalConsumer::POLL_TIMEOUT); @@ -103,11 +101,13 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, txn.IngestWal(graph.graph(), txn.timestamp(), const_cast(res.data()) + sizeof(WalHeader), header->length, txn.allocator()); + txn.Commit(); } else if (header->type == 1) { auto txn = graph.GetUpdateTransaction(); txn.IngestWal(graph.graph(), graph.db().work_dir(), txn.timestamp(), const_cast(res.data()) + sizeof(WalHeader), header->length, txn.allocator()); + txn.Commit(); } else { LOG(ERROR) << "Unknown wal type: " << header->type; } diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index f2f5511c2d8e..a3a5ff20b68f 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -56,7 +56,8 @@ struct SessionLocalContext { char _padding2[4096 - sizeof(GraphDBSession) % 4096]; }; -GraphDB::GraphDB() = default; +GraphDB::GraphDB() + : monitor_thread_running_(false), compact_thread_running_(false) {} GraphDB::~GraphDB() { if (compact_thread_running_) { compact_thread_running_ = false; @@ -283,24 +284,25 @@ bool GraphDB::kafka_wal_ingester_state() const { return kafka_wal_ingester_thread_running_.load(); } -void GraphDB::start_kafka_wal_ingester(const std::string& kafka_brokers, - const std::string& brokers, - const std::string& topic_name, - const std::string& group_id, - const std::string& engine_endpoint) { +void GraphDB::start_kafka_wal_ingester(const cppkafka::Configuration& config, + const std::string& topic_name) { if (kafka_wal_ingester_thread_running_) { kafka_wal_ingester_thread_running_ = false; - kafka_wal_ingester_thread_.join(); + if (kafka_wal_ingester_thread_.joinable()) { + kafka_wal_ingester_thread_.join(); + } } - std::vector buffer; - gs::Encoder encoder(buffer); - encoder.put_string(kafka_brokers); - encoder.put_string(brokers); - encoder.put_string(topic_name); - encoder.put_string(group_id); - encoder.put_string(engine_endpoint); - gs::Decoder decoder(buffer.data(), buffer.size()); - KafkaWalIngesterApp().Query(GetSession(0), decoder, encoder); + kafka_wal_ingester_thread_running_ = true; + kafka_wal_ingester_thread_ = std::thread([&]() { + std::vector buffer; + gs::Encoder encoder(buffer); + encoder.put_string(config.get("metadata.broker.list")); + encoder.put_string(config.get("group.id")); + encoder.put_byte(config.get("enable.auto.commit") == "true"); + encoder.put_string(topic_name); + gs::Decoder decoder(buffer.data(), buffer.size()); + KafkaWalIngesterApp().Query(GetSession(0), decoder, encoder); + }); } void GraphDB::stop_kafka_wal_ingester() { diff --git a/flex/engines/graph_db/database/graph_db.h b/flex/engines/graph_db/database/graph_db.h index e337eade6886..9233b5a4c137 100644 --- a/flex/engines/graph_db/database/graph_db.h +++ b/flex/engines/graph_db/database/graph_db.h @@ -33,6 +33,9 @@ #include "flex/storages/rt_mutable_graph/loader/loader_factory.h" #include "flex/storages/rt_mutable_graph/loading_config.h" #include "flex/storages/rt_mutable_graph/mutable_property_fragment.h" +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +#include "cppkafka/cppkafka.h" +#endif namespace gs { @@ -172,11 +175,8 @@ class GraphDB { #ifdef BUILD_KAFKA_WAL_WRITER_PARSER bool kafka_wal_ingester_state() const; - void start_kafka_wal_ingester(const std::string& kafka_brokers, - const std::string& brokers, - const std::string& topic_name, - const std::string& group_id, - const std::string& engine_endpoint); + void start_kafka_wal_ingester(const cppkafka::Configuration& config, + const std::string& topic_name); void stop_kafka_wal_ingester(); #endif diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc index 2fdc9263964b..e3868e529dc9 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc @@ -14,10 +14,6 @@ */ #include "flex/engines/graph_db/database/wal/kafka_wal_parser.h" -#include -#include -#include -#include #include "flex/engines/graph_db/database/wal/wal.h" namespace gs { @@ -40,7 +36,7 @@ std::vector get_all_topic_partitions( return partitions; } -std::unique_ptr KafkaWalParser::Make() { +std::unique_ptr KafkaWalParser::Make(const std::string&) { const char* broker_list = std::getenv("KAFKA_BROKER_LIST"); if (broker_list == nullptr) { LOG(FATAL) << "KAFKA_BROKER_LIST is not set"; @@ -53,11 +49,7 @@ std::unique_ptr KafkaWalParser::Make() { } KafkaWalParser::KafkaWalParser(const cppkafka::Configuration& config) - : consumer_(nullptr), - insert_wal_list_(NULL), - insert_wal_list_size_(0), - last_ts_(0), - config_(config) { + : consumer_(nullptr), last_ts_(0), config_(config) { consumer_ = std::make_unique(config); } @@ -69,11 +61,7 @@ void KafkaWalParser::open(const std::string& topic_name) { void KafkaWalParser::open( const std::vector& topic_partitions) { consumer_->assign(topic_partitions); - insert_wal_list_ = static_cast( - mmap(NULL, IWalWriter::MAX_WALS_NUM * sizeof(WalContentUnit), - PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, - -1, 0)); - insert_wal_list_size_ = IWalWriter::MAX_WALS_NUM; + insert_wal_list_.resize(4096); while (true) { auto msgs = consumer_->poll_batch(MAX_BATCH_SIZE); @@ -95,7 +83,7 @@ void KafkaWalParser::open( } for (auto& wal : message_vector_) { - LOG(INFO) << "Got wal:" << wal.size(); + VLOG(1) << "Got wal:" << wal.size(); const char* payload = wal.data(); const WalHeader* header = reinterpret_cast(payload); uint32_t cur_ts = header->timestamp; @@ -111,6 +99,9 @@ void KafkaWalParser::open( unit.size = length; update_wal_list_.push_back(unit); } else { + if (cur_ts >= insert_wal_list_.size()) { + insert_wal_list_.resize(cur_ts + 1); + } if (insert_wal_list_[cur_ts].ptr) { LOG(WARNING) << "Duplicated timestamp " << cur_ts << ", skip"; } @@ -134,9 +125,7 @@ void KafkaWalParser::close() { if (consumer_) { consumer_.reset(); } - if (insert_wal_list_ != NULL) { - munmap(insert_wal_list_, insert_wal_list_size_ * sizeof(WalContentUnit)); - } + insert_wal_list_.clear(); } uint32_t KafkaWalParser::last_ts() const { return last_ts_; } @@ -151,4 +140,8 @@ const std::vector& KafkaWalParser::get_update_wals() const { return update_wal_list_; } +const bool KafkaWalParser::registered_ = WalParserFactory::RegisterWalParser( + "kafaka", static_cast( + &KafkaWalParser::Make)); + } // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.h b/flex/engines/graph_db/database/wal/kafka_wal_parser.h index 7748ccc2f2e9..8710eea0ebd0 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.h +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.h @@ -17,9 +17,8 @@ #define ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_PARSER_H_ #include -#include "flex/engines/graph_db/database/wal/wal.h" - #include "cppkafka/cppkafka.h" +#include "flex/engines/graph_db/database/wal/wal.h" namespace gs { @@ -35,7 +34,7 @@ class KafkaWalParser : public IWalParser { std::chrono::milliseconds(100); static constexpr const size_t MAX_BATCH_SIZE = 1000; - static std::unique_ptr Make(); + static std::unique_ptr Make(const std::string&); // always track all partitions and from begining KafkaWalParser(const cppkafka::Configuration& config); @@ -53,13 +52,14 @@ class KafkaWalParser : public IWalParser { private: std::unique_ptr consumer_; - WalContentUnit* insert_wal_list_; - size_t insert_wal_list_size_; + std::vector insert_wal_list_; uint32_t last_ts_; std::vector update_wal_list_; std::vector message_vector_; // used to hold the polled messages cppkafka::Configuration config_; + + static const bool registered_; }; } // namespace gs diff --git a/flex/tests/hqps/kafka_wal_ingester_test.cc b/flex/tests/hqps/kafka_wal_ingester_test.cc new file mode 100644 index 000000000000..79aadabace64 --- /dev/null +++ b/flex/tests/hqps/kafka_wal_ingester_test.cc @@ -0,0 +1,77 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +#include +#include +#endif +#include "grape/serialization/in_archive.h" +#include "grape/serialization/out_archive.h" + +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/database/graph_db_session.h" +#include "flex/storages/rt_mutable_graph/schema.h" + +int main(int argc, char** argv) { +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER + gs::Schema schema; + schema.add_vertex_label( + "PERSON", + { + gs::PropertyType::kInt64, // version + }, + {"weight"}, + {std::tuple( + gs::PropertyType::kInt64, "id", 0)}, + {gs::StorageStrategy::kMem, gs::StorageStrategy::kMem}, 4096); + gs::GraphDB db; + std::string work_dir = argv[1]; + std::string kafka_brokers = argv[2]; + std::string kafka_topic = argv[3]; + db.Open(schema, work_dir, 1); + + gs::KafkaWalWriter writer(kafka_brokers); + writer.open(kafka_topic, 0); + grape::InArchive in_archive; + in_archive.Resize(sizeof(gs::WalHeader)); + gs::label_t label = db.schema().get_vertex_label_id("PERSON"); + in_archive << static_cast(0) << label; + int64_t id = 998244353; + in_archive << id; + int64_t weight = 100; + in_archive << weight; + auto header = reinterpret_cast(in_archive.GetBuffer()); + header->timestamp = 1; + header->type = 0; + header->length = in_archive.GetSize() - sizeof(gs::WalHeader); + writer.append(in_archive.GetBuffer(), in_archive.GetSize()); + writer.close(); + cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers}, + {"group.id", "test"}, + {"enable.auto.commit", false}}; + db.start_kafka_wal_ingester(config, kafka_topic); + std::this_thread::sleep_for(std::chrono::seconds(1)); + db.stop_kafka_wal_ingester(); + auto txn = db.GetReadTransaction(0); + gs::vid_t lid; + CHECK(txn.GetVertexNum(label) == 1); + CHECK(txn.GetVertexIndex(label, id, lid)); + auto iter = txn.GetVertexIterator(label); + CHECK(iter.GetField(0).AsInt64() == 100); + std::cout << "Vertex id: " << lid << std::endl; + db.Close(); +#endif + return 0; +} \ No newline at end of file From 3b11fe111eff915ac0a50aa2eab391907b3fdf18 Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Wed, 26 Mar 2025 17:18:18 +0800 Subject: [PATCH 04/17] get parameters from uri --- .../graph_db/app/kafka_wal_ingester_app.cc | 22 ++++---- flex/engines/graph_db/database/graph_db.cc | 12 +++- .../graph_db/database/wal/kafka_wal_parser.cc | 56 ++++++++++++++++--- .../graph_db/database/wal/kafka_wal_writer.cc | 34 ++++++++--- .../graph_db/database/wal/kafka_wal_writer.h | 4 +- flex/tests/hqps/kafka_test.cc | 5 +- flex/tests/hqps/kafka_wal_ingester_test.cc | 6 +- 7 files changed, 103 insertions(+), 36 deletions(-) diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc index 3799329307e3..2c42bdde598a 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -75,17 +75,19 @@ class KafkaWalConsumer { bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, Encoder& output) { - // TODO: get value from config - std::string kafka_brokers = std::string(input.get_string()); - std::string group_id = std::string(input.get_string()); - bool enable_auto_commit = input.get_byte(); - std::string topic_name = std::string(input.get_string()); - LOG(INFO) << "Kafka brokers: " << kafka_brokers; + cppkafka::Configuration config; + std::string topic_name; + while (!input.empty()) { + auto key = input.get_string(); + auto value = input.get_string(); + if (key == "topic_name") { + topic_name = value; + } else { + config.set(std::string(key), std::string(value)); + } + } + LOG(INFO) << "Kafka brokers: " << config.get("metadata.broker.list"); - cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers}, - {"group.id", group_id}, - // Disable auto commit - {"enable.auto.commit", enable_auto_commit}}; gs::KafkaWalConsumer consumer(config, topic_name, 1); // TODO: how to make it stop while (graph.db().kafka_wal_ingester_state()) { diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index a3a5ff20b68f..0e889a5c9082 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -293,13 +293,19 @@ void GraphDB::start_kafka_wal_ingester(const cppkafka::Configuration& config, } } kafka_wal_ingester_thread_running_ = true; + kafka_wal_ingester_thread_ = std::thread([&]() { std::vector buffer; gs::Encoder encoder(buffer); - encoder.put_string(config.get("metadata.broker.list")); - encoder.put_string(config.get("group.id")); - encoder.put_byte(config.get("enable.auto.commit") == "true"); + encoder.put_string("topic_name"); encoder.put_string(topic_name); + const auto& mp = config.get_all(); + encoder.put_string("metadata.broker.list"); + encoder.put_string(mp.at("metadata.broker.list")); + encoder.put_string("group.id"); + encoder.put_string(mp.at("group.id")); + encoder.put_string("enable.auto.commit"); + encoder.put_string(mp.at("enable.auto.commit")); gs::Decoder decoder(buffer.data(), buffer.size()); KafkaWalIngesterApp().Query(GetSession(0), decoder, encoder); }); diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc index e3868e529dc9..8744bcdce9cb 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc @@ -36,16 +36,54 @@ std::vector get_all_topic_partitions( return partitions; } -std::unique_ptr KafkaWalParser::Make(const std::string&) { - const char* broker_list = std::getenv("KAFKA_BROKER_LIST"); - if (broker_list == nullptr) { - LOG(FATAL) << "KAFKA_BROKER_LIST is not set"; +std::unique_ptr KafkaWalParser::Make(const std::string& uri) { + // uri should be like + // "kafka://localhost:9092,localhost:9093/my_topic?group.id=my_consumer_group& + // auto.offset.reset=earliest&enable.auto.commit=false"; + const std::string prefix = "kafka://"; + if (uri.find(prefix) != 0) { + LOG(FATAL) << "Invalid uri: " << uri; } - const char* group_id = std::getenv("KAFKA_GROUP_ID"); - std::string group_id_str = group_id ? group_id : "interactive_consumer"; - cppkafka::Configuration config = {{"metadata.broker.list", broker_list}, - {"group.id", group_id_str}}; - return std::unique_ptr(new KafkaWalParser(config)); + + std::string hosts_part = uri.substr(prefix.length()); + size_t query_pos = hosts_part.find('/'); + std::string hosts; + std::string query; + cppkafka::Configuration config; + if (query_pos != std::string::npos) { + hosts = hosts_part.substr(0, query_pos); + query = hosts_part.substr(query_pos + 1); + } else { + LOG(FATAL) << "Invalid uri: " << uri; + } + size_t top_pos = query.find('?'); + std::string topic_name; + if (top_pos != std::string::npos) { + topic_name = query.substr(0, top_pos); + query = query.substr(top_pos + 1); + } else { + LOG(FATAL) << "Invalid uri: " << uri; + } + std::istringstream query_stream(query); + std::string pair; + while (std::getline(query_stream, pair, '&')) { + size_t eq_pos = pair.find('='); + if (eq_pos != std::string::npos) { + std::string key = pair.substr(0, eq_pos); + std::string value = pair.substr(eq_pos + 1); + if (key == "group.id") { + config.set("group.id", value); + } else if (key == "auto.offset.reset") { + config.set("auto.offset.reset", value); + } else if (key == "enable.auto.commit") { + config.set("enable.auto.commit", value); + } + } + } + + auto parser = std::unique_ptr(new KafkaWalParser(config)); + parser->open(topic_name); + return parser; } KafkaWalParser::KafkaWalParser(const cppkafka::Configuration& config) diff --git a/flex/engines/graph_db/database/wal/kafka_wal_writer.cc b/flex/engines/graph_db/database/wal/kafka_wal_writer.cc index c94ec09fa42a..cd07ab2558f1 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_writer.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_writer.cc @@ -22,15 +22,35 @@ namespace gs { std::unique_ptr KafkaWalWriter::Make() { - // Get env KAFKA_BROKER_LIST - const char* broker_list = std::getenv("KAFKA_BROKER_LIST"); - if (broker_list == nullptr) { - LOG(FATAL) << "KAFKA_BROKER_LIST is not set"; - } - return std::unique_ptr(new KafkaWalWriter(broker_list)); + return std::unique_ptr(new KafkaWalWriter()); } -void KafkaWalWriter::open(const std::string& topic, int thread_id) { +void KafkaWalWriter::open(const std::string& uri, int thread_id) { + const std::string prefix = "kafka://"; + if (uri.find(prefix) != 0) { + LOG(FATAL) << "Invalid uri: " << uri; + } + + std::string hosts_part = uri.substr(prefix.length()); + size_t query_pos = hosts_part.find('/'); + std::string hosts; + std::string query; + cppkafka::Configuration config; + if (query_pos != std::string::npos) { + hosts = hosts_part.substr(0, query_pos); + query = hosts_part.substr(query_pos + 1); + } else { + LOG(FATAL) << "Invalid uri: " << uri; + } + kafka_brokers_ = hosts; + size_t top_pos = query.find('?'); + std::string topic; + if (top_pos != std::string::npos) { + topic = query.substr(0, top_pos); + } else { + topic = query; + } + if (thread_id_ != -1 || producer_) { LOG(FATAL) << "KafkaWalWriter has been opened"; } diff --git a/flex/engines/graph_db/database/wal/kafka_wal_writer.h b/flex/engines/graph_db/database/wal/kafka_wal_writer.h index e031e62ff488..1cd2ff6ad9d0 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_writer.h +++ b/flex/engines/graph_db/database/wal/kafka_wal_writer.h @@ -42,9 +42,9 @@ class KafkaWalWriter : public IWalWriter { static std::unique_ptr Make(); - KafkaWalWriter(const std::string& kafka_brokers) + KafkaWalWriter() : thread_id_(-1), - kafka_brokers_(kafka_brokers), + kafka_brokers_(""), kafka_topic_(""), producer_(nullptr), builder_("") {} // brokers could be a list of brokers diff --git a/flex/tests/hqps/kafka_test.cc b/flex/tests/hqps/kafka_test.cc index 087cb2657339..168124f7e2d1 100644 --- a/flex/tests/hqps/kafka_test.cc +++ b/flex/tests/hqps/kafka_test.cc @@ -36,12 +36,13 @@ int main(int argc, char** argv) { LOG(INFO) << "Kafka brokers: " << kafka_brokers; // Write messages to the specified kafka topic, and read them back. - gs::KafkaWalWriter writer(kafka_brokers); + gs::KafkaWalWriter writer; cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers}, {"group.id", "test"}, {"enable.auto.commit", false}}; gs::KafkaWalParser parser(config); - writer.open(kafka_topic, 0); + std::string uri = "kafka://" + kafka_brokers + "/" + kafka_topic; + writer.open(uri, 0); // Let user enter number of messages to write LOG(INFO) << "Enter number of messages to write: "; diff --git a/flex/tests/hqps/kafka_wal_ingester_test.cc b/flex/tests/hqps/kafka_wal_ingester_test.cc index 79aadabace64..65b22d6cab8d 100644 --- a/flex/tests/hqps/kafka_wal_ingester_test.cc +++ b/flex/tests/hqps/kafka_wal_ingester_test.cc @@ -41,9 +41,9 @@ int main(int argc, char** argv) { std::string kafka_brokers = argv[2]; std::string kafka_topic = argv[3]; db.Open(schema, work_dir, 1); - - gs::KafkaWalWriter writer(kafka_brokers); - writer.open(kafka_topic, 0); + std::string uri = "kafka://" + kafka_brokers + "/" + kafka_topic; + gs::KafkaWalWriter writer; + writer.open(uri, 0); grape::InArchive in_archive; in_archive.Resize(sizeof(gs::WalHeader)); gs::label_t label = db.schema().get_vertex_label_id("PERSON"); From 46a5f51cd2087b1a5ba8237590871884bccdae15 Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Wed, 26 Mar 2025 19:10:40 +0800 Subject: [PATCH 05/17] refactor ci --- .../graph_db/app/kafka_wal_ingester_app.cc | 1 - flex/engines/graph_db/database/graph_db.cc | 19 +++-- .../graph_db/database/wal/kafka_wal_parser.cc | 10 ++- flex/tests/hqps/kafka_wal_ingester_test.cc | 77 +++++++++++-------- 4 files changed, 65 insertions(+), 42 deletions(-) diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc index 2c42bdde598a..6f78545d56a9 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -96,7 +96,6 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, std::this_thread::sleep_for(gs::KafkaWalConsumer::POLL_TIMEOUT); continue; } - auto header = reinterpret_cast(res.data()); if (header->type == 0) { auto txn = graph.GetInsertTransaction(); diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index 0e889a5c9082..c3c3559eb48c 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -299,13 +299,18 @@ void GraphDB::start_kafka_wal_ingester(const cppkafka::Configuration& config, gs::Encoder encoder(buffer); encoder.put_string("topic_name"); encoder.put_string(topic_name); - const auto& mp = config.get_all(); - encoder.put_string("metadata.broker.list"); - encoder.put_string(mp.at("metadata.broker.list")); - encoder.put_string("group.id"); - encoder.put_string(mp.at("group.id")); - encoder.put_string("enable.auto.commit"); - encoder.put_string(mp.at("enable.auto.commit")); + if (config.has_property("metadata.broker.list")) { + encoder.put_string("metadata.broker.list"); + encoder.put_string(config.get("metadata.broker.list")); + } + if (config.has_property("group.id")) { + encoder.put_string("group.id"); + encoder.put_string(config.get("group.id")); + } + if (config.has_property("enable.auto.commit")) { + encoder.put_string("enable.auto.commit"); + encoder.put_string(config.get("enable.auto.commit")); + } gs::Decoder decoder(buffer.data(), buffer.size()); KafkaWalIngesterApp().Query(GetSession(0), decoder, encoder); }); diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc index 8744bcdce9cb..96cc44a64dd8 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc @@ -22,6 +22,10 @@ std::vector get_all_topic_partitions( const cppkafka::Configuration& config, const std::string& topic_name) { std::vector partitions; cppkafka::Consumer consumer(config); // tmp consumer + LOG(INFO) << config.get("metadata.broker.list"); + LOG(INFO) << config.get("group.id"); + + LOG(INFO) << "Get metadata for topic " << topic_name; auto meta_vector = consumer.get_metadata().get_topics({topic_name}); if (meta_vector.empty()) { LOG(WARNING) << "Failed to get metadata for topic " << topic_name @@ -50,12 +54,14 @@ std::unique_ptr KafkaWalParser::Make(const std::string& uri) { std::string hosts; std::string query; cppkafka::Configuration config; + if (query_pos != std::string::npos) { hosts = hosts_part.substr(0, query_pos); query = hosts_part.substr(query_pos + 1); } else { LOG(FATAL) << "Invalid uri: " << uri; } + config.set("metadata.broker.list", hosts); size_t top_pos = query.find('?'); std::string topic_name; if (top_pos != std::string::npos) { @@ -179,7 +185,7 @@ const std::vector& KafkaWalParser::get_update_wals() const { } const bool KafkaWalParser::registered_ = WalParserFactory::RegisterWalParser( - "kafaka", static_cast( - &KafkaWalParser::Make)); + "kafka", static_cast( + &KafkaWalParser::Make)); } // namespace gs \ No newline at end of file diff --git a/flex/tests/hqps/kafka_wal_ingester_test.cc b/flex/tests/hqps/kafka_wal_ingester_test.cc index 65b22d6cab8d..15c9d8d7f7cf 100644 --- a/flex/tests/hqps/kafka_wal_ingester_test.cc +++ b/flex/tests/hqps/kafka_wal_ingester_test.cc @@ -40,38 +40,51 @@ int main(int argc, char** argv) { std::string work_dir = argv[1]; std::string kafka_brokers = argv[2]; std::string kafka_topic = argv[3]; - db.Open(schema, work_dir, 1); - std::string uri = "kafka://" + kafka_brokers + "/" + kafka_topic; - gs::KafkaWalWriter writer; - writer.open(uri, 0); - grape::InArchive in_archive; - in_archive.Resize(sizeof(gs::WalHeader)); - gs::label_t label = db.schema().get_vertex_label_id("PERSON"); - in_archive << static_cast(0) << label; - int64_t id = 998244353; - in_archive << id; - int64_t weight = 100; - in_archive << weight; - auto header = reinterpret_cast(in_archive.GetBuffer()); - header->timestamp = 1; - header->type = 0; - header->length = in_archive.GetSize() - sizeof(gs::WalHeader); - writer.append(in_archive.GetBuffer(), in_archive.GetSize()); - writer.close(); - cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers}, - {"group.id", "test"}, - {"enable.auto.commit", false}}; - db.start_kafka_wal_ingester(config, kafka_topic); - std::this_thread::sleep_for(std::chrono::seconds(1)); - db.stop_kafka_wal_ingester(); - auto txn = db.GetReadTransaction(0); - gs::vid_t lid; - CHECK(txn.GetVertexNum(label) == 1); - CHECK(txn.GetVertexIndex(label, id, lid)); - auto iter = txn.GetVertexIterator(label); - CHECK(iter.GetField(0).AsInt64() == 100); - std::cout << "Vertex id: " << lid << std::endl; - db.Close(); + gs::GraphDBConfig config(schema, work_dir, "", 1); + std::string uri = "kafka://" + kafka_brokers + "/" + kafka_topic + + "?group.id=test&enable.auto.commit=false"; + config.wal_uri = uri; + db.Open(config); + { + auto txn = db.GetInsertTransaction(0); + gs::label_t label = db.schema().get_vertex_label_id("PERSON"); + int64_t id = 998244353; + int64_t weight = 100; + txn.AddVertex(label, id, {gs::Any(weight)}); + txn.Commit(); + } + + gs::GraphDB db2; + db2.Open(config); + + CHECK(db2.GetReadTransaction(0).GetVertexNum(0) == 1) + << "Vertex num: " << db2.GetReadTransaction(0).GetVertexNum(0); + cppkafka::Configuration config1 = {{"metadata.broker.list", kafka_brokers}, + {"group.id", "test"}, + {"enable.auto.commit", false}}; + db2.start_kafka_wal_ingester(config1, kafka_topic); + + { + auto txn = db.GetInsertTransaction(0); + gs::label_t label = db.schema().get_vertex_label_id("PERSON"); + int64_t id = 998244354; + int64_t weight = 200; + txn.AddVertex(label, id, {gs::Any(weight)}); + txn.Commit(); + } + LOG(INFO) << db.GetReadTransaction(0).GetVertexNum(0); + + std::this_thread::sleep_for(std::chrono::seconds(3)); + { + auto txn = db2.GetReadTransaction(0); + CHECK(txn.GetVertexNum(0) == 3) << "Vertex num: " << txn.GetVertexNum(0); + gs::vid_t lid; + CHECK(txn.GetVertexIndex(0, 998244353L, lid)); + LOG(INFO) << "Vertex id: " << lid; + CHECK(txn.GetVertexIndex(0, 998244354L, lid)); + LOG(INFO) << "Vertex id: " << lid; + db2.stop_kafka_wal_ingester(); + } #endif return 0; } \ No newline at end of file From 318236076451e45cbdf52fd0f47a4994819e0980 Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Wed, 26 Mar 2025 19:58:19 +0800 Subject: [PATCH 06/17] fix offset --- .../graph_db/app/kafka_wal_ingester_app.cc | 2 +- flex/engines/graph_db/database/graph_db.cc | 4 ++++ .../graph_db/database/wal/kafka_wal_parser.cc | 21 +++++++++++-------- .../graph_db/database/wal/kafka_wal_parser.h | 3 ++- flex/tests/hqps/kafka_wal_ingester_test.cc | 9 ++++---- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc index 6f78545d56a9..8588000e31d3 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -36,7 +36,7 @@ class KafkaWalConsumer { // always track all partitions and from begining KafkaWalConsumer(cppkafka::Configuration config, const std::string& topic_name, int32_t thread_num) { - auto topic_partitions = get_all_topic_partitions(config, topic_name); + auto topic_partitions = get_all_topic_partitions(config, topic_name, false); consumers_.reserve(topic_partitions.size()); for (size_t i = 0; i < topic_partitions.size(); ++i) { consumers_.emplace_back(std::make_unique(config)); diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index c3c3559eb48c..bbe08e64b202 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -311,6 +311,10 @@ void GraphDB::start_kafka_wal_ingester(const cppkafka::Configuration& config, encoder.put_string("enable.auto.commit"); encoder.put_string(config.get("enable.auto.commit")); } + if (config.has_property("auto.offset.reset")) { + encoder.put_string("auto.offset.reset"); + encoder.put_string(config.get("auto.offset.reset")); + } gs::Decoder decoder(buffer.data(), buffer.size()); KafkaWalIngesterApp().Query(GetSession(0), decoder, encoder); }); diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc index 96cc44a64dd8..89f02de98b7e 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc @@ -19,7 +19,8 @@ namespace gs { std::vector get_all_topic_partitions( - const cppkafka::Configuration& config, const std::string& topic_name) { + const cppkafka::Configuration& config, const std::string& topic_name, + bool from_beginning) { std::vector partitions; cppkafka::Consumer consumer(config); // tmp consumer LOG(INFO) << config.get("metadata.broker.list"); @@ -34,16 +35,20 @@ std::vector get_all_topic_partitions( } auto metadata = meta_vector.front().get_partitions(); for (const auto& partition : metadata) { - partitions.push_back(cppkafka::TopicPartition( - topic_name, partition.get_id(), 0)); // from the beginning + if (from_beginning) { + partitions.push_back( + cppkafka::TopicPartition(topic_name, partition.get_id(), 0)); + } else { + partitions.push_back(cppkafka::TopicPartition( + topic_name, partition.get_id())); // from the beginning + } } return partitions; } std::unique_ptr KafkaWalParser::Make(const std::string& uri) { // uri should be like - // "kafka://localhost:9092,localhost:9093/my_topic?group.id=my_consumer_group& - // auto.offset.reset=earliest&enable.auto.commit=false"; + // "kafka://localhost:9092,localhost:9093/my_topic?group.id=my_consumer_group" const std::string prefix = "kafka://"; if (uri.find(prefix) != 0) { LOG(FATAL) << "Invalid uri: " << uri; @@ -79,13 +84,10 @@ std::unique_ptr KafkaWalParser::Make(const std::string& uri) { std::string value = pair.substr(eq_pos + 1); if (key == "group.id") { config.set("group.id", value); - } else if (key == "auto.offset.reset") { - config.set("auto.offset.reset", value); - } else if (key == "enable.auto.commit") { - config.set("enable.auto.commit", value); } } } + config.set("enable.auto.commit", false); auto parser = std::unique_ptr(new KafkaWalParser(config)); parser->open(topic_name); @@ -124,6 +126,7 @@ void KafkaWalParser::open( } } } + consumer_->commit(); } for (auto& wal : message_vector_) { diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.h b/flex/engines/graph_db/database/wal/kafka_wal_parser.h index 8710eea0ebd0..75e02b5389e6 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.h +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.h @@ -26,7 +26,8 @@ namespace gs { * Get all partitions of the given topic. */ std::vector get_all_topic_partitions( - const cppkafka::Configuration& config, const std::string& topic_name); + const cppkafka::Configuration& config, const std::string& topic_name, + bool from_beginning = true); class KafkaWalParser : public IWalParser { public: diff --git a/flex/tests/hqps/kafka_wal_ingester_test.cc b/flex/tests/hqps/kafka_wal_ingester_test.cc index 15c9d8d7f7cf..f6ff1d99f4c8 100644 --- a/flex/tests/hqps/kafka_wal_ingester_test.cc +++ b/flex/tests/hqps/kafka_wal_ingester_test.cc @@ -41,8 +41,8 @@ int main(int argc, char** argv) { std::string kafka_brokers = argv[2]; std::string kafka_topic = argv[3]; gs::GraphDBConfig config(schema, work_dir, "", 1); - std::string uri = "kafka://" + kafka_brokers + "/" + kafka_topic + - "?group.id=test&enable.auto.commit=false"; + std::string uri = + "kafka://" + kafka_brokers + "/" + kafka_topic + "?group.id=test"; config.wal_uri = uri; db.Open(config); { @@ -61,7 +61,8 @@ int main(int argc, char** argv) { << "Vertex num: " << db2.GetReadTransaction(0).GetVertexNum(0); cppkafka::Configuration config1 = {{"metadata.broker.list", kafka_brokers}, {"group.id", "test"}, - {"enable.auto.commit", false}}; + {"enable.auto.commit", false}, + {"auto.offset.reset", "earliest"}}; db2.start_kafka_wal_ingester(config1, kafka_topic); { @@ -77,7 +78,7 @@ int main(int argc, char** argv) { std::this_thread::sleep_for(std::chrono::seconds(3)); { auto txn = db2.GetReadTransaction(0); - CHECK(txn.GetVertexNum(0) == 3) << "Vertex num: " << txn.GetVertexNum(0); + CHECK(txn.GetVertexNum(0) == 2) << "Vertex num: " << txn.GetVertexNum(0); gs::vid_t lid; CHECK(txn.GetVertexIndex(0, 998244353L, lid)); LOG(INFO) << "Vertex id: " << lid; From 81cb82331ff0dd101e4674501ab75f83a197fbed Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Wed, 26 Mar 2025 20:17:05 +0800 Subject: [PATCH 07/17] insert more vertices --- .github/workflows/interactive.yml | 2 +- flex/tests/hqps/kafka_wal_ingester_test.cc | 36 ++++++++++++---------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index d5600606f96f..71b62da5b67b 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -385,7 +385,7 @@ jobs: bin/kafka-topics.sh --create --topic kafka-test --bootstrap-server localhost:9092 ../tests/hqps/kafka_wal_ingester_test .. localhost:9092 kafka-test bin/kafka-topics.sh --delete --topic kafka-test --bootstrap-server localhost:9092 - pkill -f kafka + ps aux | grep kafka | grep -v grep | awk '{print $2}' | xargs kill -9 - name: Run Gremlin test on modern graph env: diff --git a/flex/tests/hqps/kafka_wal_ingester_test.cc b/flex/tests/hqps/kafka_wal_ingester_test.cc index f6ff1d99f4c8..fd17bca9b1d3 100644 --- a/flex/tests/hqps/kafka_wal_ingester_test.cc +++ b/flex/tests/hqps/kafka_wal_ingester_test.cc @@ -46,18 +46,20 @@ int main(int argc, char** argv) { config.wal_uri = uri; db.Open(config); { - auto txn = db.GetInsertTransaction(0); - gs::label_t label = db.schema().get_vertex_label_id("PERSON"); - int64_t id = 998244353; - int64_t weight = 100; - txn.AddVertex(label, id, {gs::Any(weight)}); - txn.Commit(); + for (int i = 0; i < 100; ++i) { + auto txn = db.GetInsertTransaction(0); + gs::label_t label = db.schema().get_vertex_label_id("PERSON"); + int64_t id = i; + int64_t weight = i * 2 + 1; + txn.AddVertex(label, id, {gs::Any(weight)}); + txn.Commit(); + } } gs::GraphDB db2; db2.Open(config); - CHECK(db2.GetReadTransaction(0).GetVertexNum(0) == 1) + CHECK(db2.GetReadTransaction(0).GetVertexNum(0) == 100) << "Vertex num: " << db2.GetReadTransaction(0).GetVertexNum(0); cppkafka::Configuration config1 = {{"metadata.broker.list", kafka_brokers}, {"group.id", "test"}, @@ -66,23 +68,25 @@ int main(int argc, char** argv) { db2.start_kafka_wal_ingester(config1, kafka_topic); { - auto txn = db.GetInsertTransaction(0); - gs::label_t label = db.schema().get_vertex_label_id("PERSON"); - int64_t id = 998244354; - int64_t weight = 200; - txn.AddVertex(label, id, {gs::Any(weight)}); - txn.Commit(); + for (int i = 100; i < 200; ++i) { + auto txn = db.GetInsertTransaction(0); + gs::label_t label = db.schema().get_vertex_label_id("PERSON"); + int64_t id = i; + int64_t weight = 200; + txn.AddVertex(label, id, {gs::Any(weight)}); + txn.Commit(); + } } LOG(INFO) << db.GetReadTransaction(0).GetVertexNum(0); std::this_thread::sleep_for(std::chrono::seconds(3)); { auto txn = db2.GetReadTransaction(0); - CHECK(txn.GetVertexNum(0) == 2) << "Vertex num: " << txn.GetVertexNum(0); + CHECK(txn.GetVertexNum(0) == 200) << "Vertex num: " << txn.GetVertexNum(0); gs::vid_t lid; - CHECK(txn.GetVertexIndex(0, 998244353L, lid)); + CHECK(txn.GetVertexIndex(0, 90L, lid)); LOG(INFO) << "Vertex id: " << lid; - CHECK(txn.GetVertexIndex(0, 998244354L, lid)); + CHECK(txn.GetVertexIndex(0, 188L, lid)); LOG(INFO) << "Vertex id: " << lid; db2.stop_kafka_wal_ingester(); } From 0d7bd50e3c04bc720fd4a26de54f22754ed80d85 Mon Sep 17 00:00:00 2001 From: zhanglei1949 Date: Thu, 27 Mar 2025 09:23:20 +0800 Subject: [PATCH 08/17] fixing ci --- flex/engines/graph_db/database/wal/kafka_wal_writer.h | 1 - k8s/dockerfiles/flex-interactive.Dockerfile | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/flex/engines/graph_db/database/wal/kafka_wal_writer.h b/flex/engines/graph_db/database/wal/kafka_wal_writer.h index 1cd2ff6ad9d0..83e98afe466b 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_writer.h +++ b/flex/engines/graph_db/database/wal/kafka_wal_writer.h @@ -69,7 +69,6 @@ class KafkaWalWriter : public IWalWriter { std::string kafka_topic_; std::shared_ptr> producer_; cppkafka::MessageBuilder builder_; - int64_t cur_offset_; static const bool registered_; }; diff --git a/k8s/dockerfiles/flex-interactive.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile index 72447d90fd69..547a135bc830 100644 --- a/k8s/dockerfiles/flex-interactive.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -103,6 +103,7 @@ RUN find /opt/flex/lib/ -name "*.a" -type f -delete # include COPY --from=builder /opt/flex/include/ /opt/graphscope/include/ /opt/vineyard/include/ /opt/flex/include/ COPY --from=builder /opt/graphscope/lib/libgrape-lite.so /opt/flex/lib/ +COPY --from=builder /opt/graphscope/lib/libcppkafka.so* /opt/flex/lib/ # copy the builtin graph, modern_graph RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/ From be2b5813338378634dcb030826d7cdcf2951fdef Mon Sep 17 00:00:00 2001 From: zhanglei1949 Date: Thu, 27 Mar 2025 10:33:40 +0800 Subject: [PATCH 09/17] fixing ci --- .github/workflows/interactive.yml | 1 + flex/CMakeLists.txt | 2 +- k8s/dockerfiles/flex-interactive.Dockerfile | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 71b62da5b67b..c03a3de9294c 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -372,6 +372,7 @@ jobs: GS_TEST_DIR: ${{ github.workspace }}/gstest run: | cd ${GITHUB_WORKSPACE}/flex/build + cmake .. -DBUILD_KAFKA_WAL_WRITER_PARSER=ON && make -j$(nproc) wget https://dlcdn.apache.org/kafka/3.9.0/kafka_2.13-3.9.0.tgz tar -zxf kafka_2.13-3.9.0.tgz cd kafka_2.13-3.9.0 diff --git a/flex/CMakeLists.txt b/flex/CMakeLists.txt index d11169754da1..f546c67f3215 100644 --- a/flex/CMakeLists.txt +++ b/flex/CMakeLists.txt @@ -18,7 +18,7 @@ option(OPTIMIZE_FOR_HOST "Whether to optimize on host" ON) # Whether to build op option(USE_STATIC_ARROW "Whether to use static arrow" OFF) # Whether to link arrow statically, default is OFF option(BUILD_WITH_OTEL "Whether to build with opentelemetry-cpp" OFF) # Whether to build with opentelemetry-cpp, default is OFF -option(BUILD_KAFKA_WAL_WRITER_PARSER "Whether to build kafka wal writer and wal parser" ON) # Whether to build kafka wal writer and wal parser, default is ON +option(BUILD_KAFKA_WAL_WRITER_PARSER "Whether to build kafka wal writer and wal parser" OFF) # Whether to build kafka wal writer and wal parser, default is OFF option(BUILD_WITH_OSS "Whether to build with oss support" OFF) # Whether to build with oss support, default is OFF #print options diff --git a/k8s/dockerfiles/flex-interactive.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile index 547a135bc830..b86151cc09ea 100644 --- a/k8s/dockerfiles/flex-interactive.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -27,7 +27,9 @@ COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope # install flex RUN . ${HOME}/.cargo/env && cd ${HOME}/GraphScope/flex && \ - git submodule update --init && mkdir build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=/opt/flex -DBUILD_DOC=OFF -DBUILD_TEST=OFF -DOPTIMIZE_FOR_HOST=${OPTIMIZE_FOR_HOST} -DUSE_STATIC_ARROW=ON && make -j ${PARALLEL} && make install && \ + git submodule update --init && mkdir build && cd build && \ + cmake .. -DCMAKE_INSTALL_PREFIX=/opt/flex -DBUILD_DOC=OFF -DBUILD_TEST=OFF \ + -DOPTIMIZE_FOR_HOST=${OPTIMIZE_FOR_HOST} -DUSE_STATIC_ARROW=ON -DBUILD_KAFKA_WAL_WRITER_PARSER=ON && make -j ${PARALLEL} && make install && \ cd ~/GraphScope/interactive_engine/ && mvn clean package -Pexperimental -DskipTests && \ cd ~/GraphScope/interactive_engine/compiler && cp target/compiler-0.0.1-SNAPSHOT.jar /opt/flex/lib/ && \ cp target/libs/*.jar /opt/flex/lib/ && \ @@ -147,6 +149,7 @@ COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libprotobuf* /usr/lib/$PLATFORM /usr/lib/$PLATFORM-linux-gnu/libglog*.so* \ /usr/lib/$PLATFORM-linux-gnu/libgflags*.so* \ /usr/lib/$PLATFORM-linux-gnu/libicudata.so* \ + /usr/lib/$PLATFORM-linux-gnu/librdkafka* \ /usr/lib/$PLATFORM-linux-gnu/ RUN sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/libLLVM*.so* && sudo rm -rf /opt/flex/lib/libseastar.a && \ From 222a49e8726bac67617ff54707fe7bfa400f4ae3 Mon Sep 17 00:00:00 2001 From: zhanglei1949 Date: Thu, 27 Mar 2025 11:27:34 +0800 Subject: [PATCH 10/17] submodule --- .github/workflows/interactive.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index c03a3de9294c..5dde2d417aee 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -371,6 +371,7 @@ jobs: env: GS_TEST_DIR: ${{ github.workspace }}/gstest run: | + git submodule update --init cd ${GITHUB_WORKSPACE}/flex/build cmake .. -DBUILD_KAFKA_WAL_WRITER_PARSER=ON && make -j$(nproc) wget https://dlcdn.apache.org/kafka/3.9.0/kafka_2.13-3.9.0.tgz From f32e3eab9d7f5df7fefea5e1611b62ed491aaf8e Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Thu, 27 Mar 2025 14:48:22 +0800 Subject: [PATCH 11/17] append wal when abort --- .../graph_db/app/kafka_wal_ingester_app.cc | 11 +++++------ .../engines/graph_db/database/graph_db_session.cc | 2 ++ flex/engines/graph_db/database/graph_db_session.h | 2 ++ .../graph_db/database/insert_transaction.cc | 7 +++++++ .../database/single_edge_insert_transaction.cc | 7 +++++++ .../database/single_vertex_insert_transaction.cc | 7 +++++++ .../graph_db/database/update_transaction.cc | 15 +++++++++++++-- 7 files changed, 43 insertions(+), 8 deletions(-) diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc index 8588000e31d3..12fcd7321353 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -89,7 +89,7 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, LOG(INFO) << "Kafka brokers: " << config.get("metadata.broker.list"); gs::KafkaWalConsumer consumer(config, topic_name, 1); - // TODO: how to make it stop + while (graph.db().kafka_wal_ingester_state()) { auto res = consumer.poll(); if (res.empty()) { @@ -98,11 +98,10 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, } auto header = reinterpret_cast(res.data()); if (header->type == 0) { - auto txn = graph.GetInsertTransaction(); - txn.IngestWal(graph.graph(), txn.timestamp(), - const_cast(res.data()) + sizeof(WalHeader), - header->length, txn.allocator()); - txn.Commit(); + gs::InsertTransaction::IngestWal( + graph.graph(), txn.timestamp(), + const_cast(res.data()) + sizeof(WalHeader), header->length, + txn.allocator()); } else if (header->type == 1) { auto txn = graph.GetUpdateTransaction(); txn.IngestWal(graph.graph(), graph.db().work_dir(), txn.timestamp(), diff --git a/flex/engines/graph_db/database/graph_db_session.cc b/flex/engines/graph_db/database/graph_db_session.cc index 03d34b4d43b4..b3f28c27ffee 100644 --- a/flex/engines/graph_db/database/graph_db_session.cc +++ b/flex/engines/graph_db/database/graph_db_session.cc @@ -248,6 +248,8 @@ AppBase* GraphDBSession::GetApp(int type) { return app; } +Allocator& GraphDBSession::allocator() { return alloc_; } + #undef likely // likely Result> diff --git a/flex/engines/graph_db/database/graph_db_session.h b/flex/engines/graph_db/database/graph_db_session.h index 84aeed45623a..1902004a8b55 100644 --- a/flex/engines/graph_db/database/graph_db_session.h +++ b/flex/engines/graph_db/database/graph_db_session.h @@ -108,6 +108,8 @@ class GraphDBSession { AppBase* GetApp(const std::string& name); + Allocator& allocator(); + private: Result> parse_query_type_from_cypher_json(const std::string_view& input); diff --git a/flex/engines/graph_db/database/insert_transaction.cc b/flex/engines/graph_db/database/insert_transaction.cc index 7a9ac09f3e57..eb0044544b02 100644 --- a/flex/engines/graph_db/database/insert_transaction.cc +++ b/flex/engines/graph_db/database/insert_transaction.cc @@ -173,6 +173,13 @@ bool InsertTransaction::Commit() { void InsertTransaction::Abort() { if (timestamp_ != std::numeric_limits::max()) { + auto header = reinterpret_cast(arc_.GetBuffer()); + header->length = 0; + header->timestamp = timestamp_; + header->type = 0; + if (!logger_.append(arc_.GetBuffer(), arc_.GetSize())) { + LOG(ERROR) << "Failed to append wal log"; + } LOG(ERROR) << "aborting " << timestamp_ << "-th transaction (insert)"; vm_.release_insert_timestamp(timestamp_); clear(); diff --git a/flex/engines/graph_db/database/single_edge_insert_transaction.cc b/flex/engines/graph_db/database/single_edge_insert_transaction.cc index f9c00dca0772..f8f4b6a904e3 100644 --- a/flex/engines/graph_db/database/single_edge_insert_transaction.cc +++ b/flex/engines/graph_db/database/single_edge_insert_transaction.cc @@ -98,6 +98,13 @@ bool SingleEdgeInsertTransaction::AddEdge(label_t src_label, const Any& src, void SingleEdgeInsertTransaction::Abort() { if (timestamp_ != std::numeric_limits::max()) { + auto header = reinterpret_cast(arc_.GetBuffer()); + header->length = 0; + header->timestamp = timestamp_; + header->type = 0; + if (!logger_.append(arc_.GetBuffer(), arc_.GetSize())) { + LOG(ERROR) << "Failed to append wal log"; + } LOG(ERROR) << "aborting " << timestamp_ << "-th transaction (single edge insert)"; vm_.release_insert_timestamp(timestamp_); diff --git a/flex/engines/graph_db/database/single_vertex_insert_transaction.cc b/flex/engines/graph_db/database/single_vertex_insert_transaction.cc index 188d1bfe72b4..c0c201c68430 100644 --- a/flex/engines/graph_db/database/single_vertex_insert_transaction.cc +++ b/flex/engines/graph_db/database/single_vertex_insert_transaction.cc @@ -178,6 +178,13 @@ bool SingleVertexInsertTransaction::Commit() { void SingleVertexInsertTransaction::Abort() { if (timestamp_ != std::numeric_limits::max()) { + auto header = reinterpret_cast(arc_.GetBuffer()); + header->length = 0; + header->timestamp = timestamp_; + header->type = 0; + if (!logger_.append(arc_.GetBuffer(), arc_.GetSize())) { + LOG(ERROR) << "Failed to append wal log"; + } LOG(ERROR) << "aborting " << timestamp_ << "-th transaction (single vertex insert)"; vm_.release_insert_timestamp(timestamp_); diff --git a/flex/engines/graph_db/database/update_transaction.cc b/flex/engines/graph_db/database/update_transaction.cc index 06b5effa9af1..08980186d7f2 100644 --- a/flex/engines/graph_db/database/update_transaction.cc +++ b/flex/engines/graph_db/database/update_transaction.cc @@ -98,7 +98,7 @@ UpdateTransaction::UpdateTransaction(const GraphDBSession& session, updated_edge_data_.resize(csr_num); } -UpdateTransaction::~UpdateTransaction() { release(); } +UpdateTransaction::~UpdateTransaction() { Abort(); } timestamp_t UpdateTransaction::timestamp() const { return timestamp_; } @@ -129,7 +129,18 @@ bool UpdateTransaction::Commit() { return true; } -void UpdateTransaction::Abort() { release(); } +void UpdateTransaction::Abort() { + if (timestamp_ != std::numeric_limits::max()) { + auto header = reinterpret_cast(arc_.GetBuffer()); + header->length = 0; + header->timestamp = timestamp_; + header->type = 0; + if (!logger_.append(arc_.GetBuffer(), arc_.GetSize())) { + LOG(ERROR) << "Failed to append wal log"; + } + release(); + } +} bool UpdateTransaction::AddVertex(label_t label, const Any& oid, const std::vector& props) { From 90b5892938fb9355f151315c5456677bf1e38891 Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Thu, 27 Mar 2025 20:37:29 +0800 Subject: [PATCH 12/17] refactor ingest wal --- .../graph_db/app/kafka_wal_ingester_app.cc | 184 +++++++++++++----- flex/engines/graph_db/database/graph_db.cc | 6 +- flex/engines/graph_db/database/graph_db.h | 4 + .../graph_db/database/graph_db_session.cc | 2 + .../graph_db/database/graph_db_session.h | 2 + .../graph_db/database/insert_transaction.cc | 2 - .../graph_db/database/insert_transaction.h | 2 - .../graph_db/database/update_transaction.cc | 2 - .../graph_db/database/update_transaction.h | 2 - .../graph_db/database/version_manager.cc | 7 + .../graph_db/database/version_manager.h | 2 + .../graph_db/database/wal/kafka_wal_parser.cc | 62 +++--- flex/engines/graph_db/database/wal/wal.cc | 2 +- flex/tests/hqps/kafka_wal_ingester_test.cc | 45 ++++- 14 files changed, 225 insertions(+), 99 deletions(-) diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc index 12fcd7321353..ff7d96dbe9f5 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -21,21 +21,120 @@ namespace gs { #ifdef BUILD_KAFKA_WAL_WRITER_PARSER +struct WalIngester { + GraphDBSession& session_; + timestamp_t begin_; + timestamp_t end_; + timestamp_t ingested_plus_one_; + std::vector data_; + // 0: not exist, 1: exist, 2: ingested + std::vector states_; + + void resize() { + size_t n = data_.size(); + std::vector new_data(n + 4096); + std::vector new_states(n + 4096, 0); + size_t idx = (ingested_plus_one_ - begin_) % n; + for (size_t i = 0; i < n; ++i) { + new_data[i] = data_[idx]; + new_states[i] = states_[idx]; + if (states_[idx]) { + end_ = ingested_plus_one_ + i + 1; + } + ++idx; + idx %= n; + } + data_ = std::move(new_data); + states_ = std::move(new_states); + begin_ = ingested_plus_one_; + } + WalIngester(GraphDBSession& session, timestamp_t cur) + : session_(session), + begin_(cur), + end_(cur), + ingested_plus_one_(cur), + data_(4096), + states_(4096, 0) {} + + bool empty() const { return ingested_plus_one_ == end_; } + + void ingest_impl(const std::string& data) { + auto header = reinterpret_cast(data.data()); + if (header->type == 0) { + InsertTransaction::IngestWal( + session_.graph(), header->timestamp, + const_cast(data.data()) + sizeof(WalHeader), header->length, + session_.allocator()); + } else { + auto txn = session_.GetUpdateTransaction(); + auto header = reinterpret_cast(data.data()); + txn.IngestWal(session_.graph(), session_.db().work_dir(), + header->timestamp, + const_cast(data.data()) + sizeof(WalHeader), + header->length, session_.allocator()); + txn.Commit(); + } + } + + void ingest() { + size_t idx = (ingested_plus_one_ - begin_) % data_.size(); + bool flag = false; + while (states_[idx] == 2 || states_[idx] == 1) { + if (states_[idx] == 1) { + ingest_impl(data_[idx]); + } + states_[idx] = 0; + ++ingested_plus_one_; + ++idx; + idx %= data_.size(); + flag = true; + } + if (flag) { + session_.commit(ingested_plus_one_); + } + } + void push(const std::string& data) { + auto header = reinterpret_cast(data.data()); + if (header->timestamp < begin_) { + LOG(ERROR) << "Invalid timestamp: " << header->timestamp; + return; + } + size_t index; + size_t n = data_.size(); + if (header->timestamp < end_) { + index = (header->timestamp - begin_) % n; + } else if (header->timestamp - begin_ < n) { + index = header->timestamp - begin_; + end_ = header->timestamp + 1; + } else { + ingest(); + while (header->timestamp - ingested_plus_one_ + 1 > states_.size()) { + resize(); + } + index = (header->timestamp - begin_) % states_.size(); + end_ = header->timestamp + 1; + } + if (header->length == 0) { + states_[index] = 2; + } else if (header->type == 0) { + ingest_impl(data); + states_[index] = 2; + } else { + states_[index] = 1; + data_[index] = data; + } + } +}; + class KafkaWalConsumer { public: - struct CustomComparator { - inline bool operator()(const std::string& lhs, const std::string& rhs) { - const WalHeader* header1 = reinterpret_cast(lhs.data()); - const WalHeader* header2 = reinterpret_cast(rhs.data()); - return header1->timestamp > header2->timestamp; - } - }; static constexpr const std::chrono::milliseconds POLL_TIMEOUT = std::chrono::milliseconds(100); // always track all partitions and from begining - KafkaWalConsumer(cppkafka::Configuration config, - const std::string& topic_name, int32_t thread_num) { + KafkaWalConsumer(WalIngester& ingester, cppkafka::Configuration config, + const std::string& topic_name, int32_t thread_num) + : ingester_(ingester) { auto topic_partitions = get_all_topic_partitions(config, topic_name, false); consumers_.reserve(topic_partitions.size()); for (size_t i = 0; i < topic_partitions.size(); ++i) { @@ -44,35 +143,33 @@ class KafkaWalConsumer { } } - std::string poll() { + void poll() { for (auto& consumer : consumers_) { - auto msg = consumer->poll(); - if (msg) { - if (msg.get_error()) { - if (!msg.is_eof()) { - LOG(INFO) << "[+] Received error notification: " << msg.get_error(); + auto msgs = consumer->poll_batch(1024); + for (const auto& msg : msgs) { + if (msg) { + if (msg.get_error()) { + if (!msg.is_eof()) { + LOG(INFO) << "[+] Received error notification: " + << msg.get_error(); + } + } else { + std::string payload = msg.get_payload(); + ingester_.push(payload); + consumer->commit(msg); } - } else { - std::string payload = msg.get_payload(); - message_queue_.push(payload); - consumer->commit(msg); } } } - if (message_queue_.empty()) { - return ""; - } - std::string payload = message_queue_.top(); - message_queue_.pop(); - return payload; } private: std::vector> consumers_; - std::priority_queue, CustomComparator> - message_queue_; + WalIngester& ingester_; }; +void Ingest(const std::string& data, GraphDBSession& session) {} + bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, Encoder& output) { cppkafka::Configuration config; @@ -87,30 +184,17 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, } } LOG(INFO) << "Kafka brokers: " << config.get("metadata.broker.list"); - - gs::KafkaWalConsumer consumer(config, topic_name, 1); - + timestamp_t cur_ts = graph.db().get_last_ingested_wal_ts() + 1; + gs::WalIngester ingester(graph, cur_ts); + gs::KafkaWalConsumer consumer(ingester, config, topic_name, 1); while (graph.db().kafka_wal_ingester_state()) { - auto res = consumer.poll(); - if (res.empty()) { - std::this_thread::sleep_for(gs::KafkaWalConsumer::POLL_TIMEOUT); - continue; - } - auto header = reinterpret_cast(res.data()); - if (header->type == 0) { - gs::InsertTransaction::IngestWal( - graph.graph(), txn.timestamp(), - const_cast(res.data()) + sizeof(WalHeader), header->length, - txn.allocator()); - } else if (header->type == 1) { - auto txn = graph.GetUpdateTransaction(); - txn.IngestWal(graph.graph(), graph.db().work_dir(), txn.timestamp(), - const_cast(res.data()) + sizeof(WalHeader), - header->length, txn.allocator()); - txn.Commit(); - } else { - LOG(ERROR) << "Unknown wal type: " << header->type; - } + consumer.poll(); + ingester.ingest(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + while (!ingester.empty()) { + consumer.poll(); + ingester.ingest(); } return true; } diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index bbe08e64b202..b0b687eec8ca 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -57,7 +57,10 @@ struct SessionLocalContext { }; GraphDB::GraphDB() - : monitor_thread_running_(false), compact_thread_running_(false) {} + : monitor_thread_running_(false), + kafka_wal_ingester_thread_running_(false), + last_ingested_wal_ts_(0), + compact_thread_running_(false) {} GraphDB::~GraphDB() { if (compact_thread_running_) { compact_thread_running_ = false; @@ -539,6 +542,7 @@ void GraphDB::openWalAndCreateContexts(const GraphDBConfig& config, } auto wal_parser = WalParserFactory::CreateWalParser(wal_uri); ingestWals(*wal_parser, data_dir, thread_num_); + last_ingested_wal_ts_ = wal_parser->last_ts(); for (int i = 0; i < thread_num_; ++i) { contexts_[i].logger->open(wal_uri, i); diff --git a/flex/engines/graph_db/database/graph_db.h b/flex/engines/graph_db/database/graph_db.h index 9233b5a4c137..b324f27caa88 100644 --- a/flex/engines/graph_db/database/graph_db.h +++ b/flex/engines/graph_db/database/graph_db.h @@ -181,6 +181,9 @@ class GraphDB { void stop_kafka_wal_ingester(); #endif + uint64_t get_last_ingested_wal_ts() const { return last_ingested_wal_ts_; } + void set_last_ingested_wal_ts(uint64_t ts) { last_ingested_wal_ts_ = ts; } + private: bool registerApp(const std::string& path, uint8_t index = 0); @@ -218,6 +221,7 @@ class GraphDB { std::thread kafka_wal_ingester_thread_; std::atomic kafka_wal_ingester_thread_running_; + uint64_t last_ingested_wal_ts_; timestamp_t last_compaction_ts_; bool compact_thread_running_ = false; diff --git a/flex/engines/graph_db/database/graph_db_session.cc b/flex/engines/graph_db/database/graph_db_session.cc index b3f28c27ffee..b2b70a9d1187 100644 --- a/flex/engines/graph_db/database/graph_db_session.cc +++ b/flex/engines/graph_db/database/graph_db_session.cc @@ -67,6 +67,8 @@ const MutablePropertyFragment& GraphDBSession::graph() const { const GraphDB& GraphDBSession::db() const { return db_; } +void GraphDBSession::commit(timestamp_t ts) { db_.version_manager_.commit(ts); } + MutablePropertyFragment& GraphDBSession::graph() { return db_.graph(); } const Schema& GraphDBSession::schema() const { return db_.schema(); } diff --git a/flex/engines/graph_db/database/graph_db_session.h b/flex/engines/graph_db/database/graph_db_session.h index 1902004a8b55..8fce571c02cf 100644 --- a/flex/engines/graph_db/database/graph_db_session.h +++ b/flex/engines/graph_db/database/graph_db_session.h @@ -110,6 +110,8 @@ class GraphDBSession { Allocator& allocator(); + void commit(timestamp_t ts); + private: Result> parse_query_type_from_cypher_json(const std::string_view& input); diff --git a/flex/engines/graph_db/database/insert_transaction.cc b/flex/engines/graph_db/database/insert_transaction.cc index eb0044544b02..dea4afe769f9 100644 --- a/flex/engines/graph_db/database/insert_transaction.cc +++ b/flex/engines/graph_db/database/insert_transaction.cc @@ -233,8 +233,6 @@ const Schema& InsertTransaction::schema() const { return graph_.schema(); } const GraphDBSession& InsertTransaction::GetSession() const { return session_; } -Allocator& InsertTransaction::allocator() const { return alloc_; } - #define likely(x) __builtin_expect(!!(x), 1) bool InsertTransaction::get_vertex_with_retries(MutablePropertyFragment& graph, diff --git a/flex/engines/graph_db/database/insert_transaction.h b/flex/engines/graph_db/database/insert_transaction.h index 7a5fcd9357ef..c954ccd6a0ff 100644 --- a/flex/engines/graph_db/database/insert_transaction.h +++ b/flex/engines/graph_db/database/insert_transaction.h @@ -61,8 +61,6 @@ class InsertTransaction { const GraphDBSession& GetSession() const; - Allocator& allocator() const; - private: void clear(); diff --git a/flex/engines/graph_db/database/update_transaction.cc b/flex/engines/graph_db/database/update_transaction.cc index 08980186d7f2..3ad80ec6043a 100644 --- a/flex/engines/graph_db/database/update_transaction.cc +++ b/flex/engines/graph_db/database/update_transaction.cc @@ -102,8 +102,6 @@ UpdateTransaction::~UpdateTransaction() { Abort(); } timestamp_t UpdateTransaction::timestamp() const { return timestamp_; } -Allocator& UpdateTransaction::allocator() { return alloc_; } - bool UpdateTransaction::Commit() { if (timestamp_ == std::numeric_limits::max()) { return true; diff --git a/flex/engines/graph_db/database/update_transaction.h b/flex/engines/graph_db/database/update_transaction.h index beff87579b66..12988e8f1135 100644 --- a/flex/engines/graph_db/database/update_transaction.h +++ b/flex/engines/graph_db/database/update_transaction.h @@ -153,8 +153,6 @@ class UpdateTransaction { const GraphDBSession& GetSession() const; - Allocator& allocator(); - private: friend class GraphDBSession; bool batch_commit(UpdateBatch& batch); diff --git a/flex/engines/graph_db/database/version_manager.cc b/flex/engines/graph_db/database/version_manager.cc index 9ff538b67505..38f261dba05d 100644 --- a/flex/engines/graph_db/database/version_manager.cc +++ b/flex/engines/graph_db/database/version_manager.cc @@ -98,6 +98,13 @@ void VersionManager::release_insert_timestamp(uint32_t ts) { pending_reqs_.fetch_sub(1); } +void VersionManager::commit(uint32_t ts) { + lock_.lock(); + read_ts_.store(ts); + write_ts_.store(ts + 1); + lock_.unlock(); +} + uint32_t VersionManager::acquire_update_timestamp() { int expected_update_reqs = 0; while ( diff --git a/flex/engines/graph_db/database/version_manager.h b/flex/engines/graph_db/database/version_manager.h index 2a7d4f3fd85a..41112087da1f 100644 --- a/flex/engines/graph_db/database/version_manager.h +++ b/flex/engines/graph_db/database/version_manager.h @@ -51,6 +51,8 @@ class VersionManager { void release_update_timestamp(uint32_t ts); bool revert_update_timestamp(uint32_t ts); + void commit(uint32_t ts); + private: std::atomic write_ts_{1}; std::atomic read_ts_{0}; diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc index 89f02de98b7e..268f8f37e665 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc @@ -108,10 +108,10 @@ void KafkaWalParser::open( const std::vector& topic_partitions) { consumer_->assign(topic_partitions); insert_wal_list_.resize(4096); - + uint32_t cnt = 0; while (true) { auto msgs = consumer_->poll_batch(MAX_BATCH_SIZE); - if (msgs.empty() || msgs.empty()) { + if (msgs.empty() && cnt == last_ts_) { LOG(INFO) << "No message are polled, the topic has been all consumed."; break; } @@ -123,42 +123,40 @@ void KafkaWalParser::open( } } else { message_vector_.emplace_back(msg.get_payload()); + const std::string& wal = message_vector_.back(); + auto header = reinterpret_cast(wal.data()); + if (header->timestamp == 0) { + LOG(WARNING) << "Invalid timestamp 0, skip"; + continue; + } + if (header->type) { + UpdateWalUnit unit; + unit.timestamp = header->timestamp; + unit.ptr = const_cast(wal.data() + sizeof(WalHeader)); + unit.size = header->length; + update_wal_list_.push_back(unit); + cnt++; + } else { + if (header->timestamp >= insert_wal_list_.size()) { + insert_wal_list_.resize(header->timestamp + 1); + } + if (insert_wal_list_[header->timestamp].ptr) { + LOG(WARNING) << "Duplicated timestamp " << header->timestamp + << ", skip"; + continue; + } + cnt++; + insert_wal_list_[header->timestamp].ptr = + const_cast(wal.data() + sizeof(WalHeader)); + insert_wal_list_[header->timestamp].size = header->length; + } + last_ts_ = std::max(header->timestamp, last_ts_); } } } consumer_->commit(); } - for (auto& wal : message_vector_) { - VLOG(1) << "Got wal:" << wal.size(); - const char* payload = wal.data(); - const WalHeader* header = reinterpret_cast(payload); - uint32_t cur_ts = header->timestamp; - if (cur_ts == 0) { - LOG(WARNING) << "Invalid timestamp 0, skip"; - continue; - } - int length = header->length; - if (header->type) { - UpdateWalUnit unit; - unit.timestamp = cur_ts; - unit.ptr = const_cast(payload + sizeof(WalHeader)); - unit.size = length; - update_wal_list_.push_back(unit); - } else { - if (cur_ts >= insert_wal_list_.size()) { - insert_wal_list_.resize(cur_ts + 1); - } - if (insert_wal_list_[cur_ts].ptr) { - LOG(WARNING) << "Duplicated timestamp " << cur_ts << ", skip"; - } - insert_wal_list_[cur_ts].ptr = - const_cast(payload + sizeof(WalHeader)); - insert_wal_list_[cur_ts].size = length; - } - last_ts_ = std::max(cur_ts, last_ts_); - } - LOG(INFO) << "last_ts: " << last_ts_; if (!update_wal_list_.empty()) { std::sort(update_wal_list_.begin(), update_wal_list_.end(), diff --git a/flex/engines/graph_db/database/wal/wal.cc b/flex/engines/graph_db/database/wal/wal.cc index 14d9c147651d..3053c3191185 100644 --- a/flex/engines/graph_db/database/wal/wal.cc +++ b/flex/engines/graph_db/database/wal/wal.cc @@ -101,7 +101,7 @@ std::unique_ptr WalParserFactory::CreateWalParser( bool WalParserFactory::RegisterWalParser( const std::string& wal_writer_type, WalParserFactory::wal_parser_initializer_t initializer) { - LOG(INFO) << "Registering wal writer of type: " << wal_writer_type; + LOG(INFO) << "Registering wal parser of type: " << wal_writer_type; auto& known_parsers_ = getKnownWalParsers(); known_parsers_.emplace(wal_writer_type, initializer); return true; diff --git a/flex/tests/hqps/kafka_wal_ingester_test.cc b/flex/tests/hqps/kafka_wal_ingester_test.cc index fd17bca9b1d3..92f477258e73 100644 --- a/flex/tests/hqps/kafka_wal_ingester_test.cc +++ b/flex/tests/hqps/kafka_wal_ingester_test.cc @@ -13,6 +13,7 @@ * limitations under the License. */ #include +#include #ifdef BUILD_KAFKA_WAL_WRITER_PARSER #include #include @@ -68,13 +69,31 @@ int main(int argc, char** argv) { db2.start_kafka_wal_ingester(config1, kafka_topic); { + std::vector threads; for (int i = 100; i < 200; ++i) { - auto txn = db.GetInsertTransaction(0); - gs::label_t label = db.schema().get_vertex_label_id("PERSON"); - int64_t id = i; - int64_t weight = 200; - txn.AddVertex(label, id, {gs::Any(weight)}); - txn.Commit(); + threads.emplace_back([&db, i] { + auto txn = db.GetInsertTransaction(0); + gs::label_t label = db.schema().get_vertex_label_id("PERSON"); + int64_t id = i; + int64_t weight = 200; + txn.AddVertex(label, id, {gs::Any(weight)}); + std::random_device rd; + std::mt19937 gen(rd()); + std::this_thread::sleep_for(std::chrono::milliseconds(gen() % 1000)); + if (i % 20 == 0) { + txn.Abort(); + } else { + txn.Commit(); + } + { + auto txn = db.GetUpdateTransaction(0); + txn.AddVertex(label, id - 100, {gs::Any(weight - 1)}); + txn.Commit(); + } + }); + } + for (auto& thrd : threads) { + thrd.join(); } } LOG(INFO) << db.GetReadTransaction(0).GetVertexNum(0); @@ -82,12 +101,24 @@ int main(int argc, char** argv) { std::this_thread::sleep_for(std::chrono::seconds(3)); { auto txn = db2.GetReadTransaction(0); - CHECK(txn.GetVertexNum(0) == 200) << "Vertex num: " << txn.GetVertexNum(0); + CHECK(txn.GetVertexNum(0) == 195) << "Vertex num: " << txn.GetVertexNum(0); gs::vid_t lid; CHECK(txn.GetVertexIndex(0, 90L, lid)); LOG(INFO) << "Vertex id: " << lid; CHECK(txn.GetVertexIndex(0, 188L, lid)); LOG(INFO) << "Vertex id: " << lid; + auto iter = txn.GetVertexIterator(0); + int cnt = 0; + while (iter.IsValid()) { + if (cnt < 100) { + CHECK(iter.GetField(0).AsInt64() == 199); + } else { + CHECK(iter.GetField(0).AsInt64() == 200); + } + cnt++; + iter.Next(); + } + db2.stop_kafka_wal_ingester(); } #endif From cc212e75f376f6071521acac5a5ade316415aff7 Mon Sep 17 00:00:00 2001 From: Zhang Lei Date: Fri, 28 Mar 2025 14:12:46 +0800 Subject: [PATCH 13/17] Update docs/flex/interactive/development/dev_and_test.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/flex/interactive/development/dev_and_test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/flex/interactive/development/dev_and_test.md b/docs/flex/interactive/development/dev_and_test.md index a47895895ba4..a895bb70b1d1 100644 --- a/docs/flex/interactive/development/dev_and_test.md +++ b/docs/flex/interactive/development/dev_and_test.md @@ -297,5 +297,5 @@ bin/kafka-topics.sh --describe --topic kafka-test --bootstrap-server localhost:9 cd flex && mkdir build cd build && cmake .. -DBUILD_TEST=ON && make -j ./tests/hqps/kafka_test localhost:902 kafka-test -# run the kafka tst +# run the kafka test ``` From 2dfd95892f1fc35fc04592d39b2a37889914de91 Mon Sep 17 00:00:00 2001 From: Zhang Lei Date: Fri, 28 Mar 2025 14:13:06 +0800 Subject: [PATCH 14/17] Update docs/flex/interactive/development/dev_and_test.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/flex/interactive/development/dev_and_test.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/flex/interactive/development/dev_and_test.md b/docs/flex/interactive/development/dev_and_test.md index a895bb70b1d1..52707165157f 100644 --- a/docs/flex/interactive/development/dev_and_test.md +++ b/docs/flex/interactive/development/dev_and_test.md @@ -265,8 +265,7 @@ In Interactive's execution engine, transactions such as `ReadTransaction`, `Upda ## Persisting WAL to kafka -Kafka-based WAL storages is also provided. Follows [kafka-quick-start](https://kafka.apache.org/quickstart). - +Kafka-based WAL storage is also provided. Follows [kafka-quick-start](https://kafka.apache.org/quickstart). ### Install kafka ```bash From a3e33388b88cfd9a358c6091525a465d78ba88de Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Mon, 31 Mar 2025 11:37:51 +0800 Subject: [PATCH 15/17] fix --- .../graph_db/app/kafka_wal_ingester_app.cc | 37 ++++--- .../graph_db/app/kafka_wal_ingester_app.h | 5 +- flex/engines/graph_db/database/graph_db.cc | 53 ---------- flex/engines/graph_db/database/graph_db.h | 14 --- .../graph_db/database/wal/kafka_wal_parser.cc | 78 +++------------ .../graph_db/database/wal/kafka_wal_parser.h | 7 -- .../graph_db/database/wal/kafka_wal_utils.cc | 97 +++++++++++++++++++ .../graph_db/database/wal/kafka_wal_utils.h | 38 ++++++++ .../graph_db/database/wal/kafka_wal_writer.cc | 38 +++----- flex/engines/http_server/graph_db_service.cc | 17 ++++ flex/engines/http_server/graph_db_service.h | 12 +++ .../engines/http_server/kafka_wal_ingester.cc | 52 ++++++++++ flex/engines/http_server/kafka_wal_ingester.h | 34 +++++++ flex/tests/hqps/CMakeLists.txt | 2 +- flex/tests/hqps/kafka_wal_ingester_test.cc | 10 +- 15 files changed, 314 insertions(+), 180 deletions(-) create mode 100644 flex/engines/graph_db/database/wal/kafka_wal_utils.cc create mode 100644 flex/engines/graph_db/database/wal/kafka_wal_utils.h create mode 100644 flex/engines/http_server/kafka_wal_ingester.cc create mode 100644 flex/engines/http_server/kafka_wal_ingester.h diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc index ff7d96dbe9f5..8496aa63187e 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.cc +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.cc @@ -16,12 +16,13 @@ #include "flex/engines/graph_db/app/kafka_wal_ingester_app.h" #include "cppkafka/cppkafka.h" #include "flex/engines/graph_db/database/graph_db.h" -#include "flex/engines/graph_db/database/wal/kafka_wal_parser.h" +#include "flex/engines/graph_db/database/wal/kafka_wal_utils.h" namespace gs { #ifdef BUILD_KAFKA_WAL_WRITER_PARSER struct WalIngester { + constexpr static size_t BUFFSIZ = 4096; GraphDBSession& session_; timestamp_t begin_; timestamp_t end_; @@ -31,30 +32,32 @@ struct WalIngester { std::vector states_; void resize() { - size_t n = data_.size(); - std::vector new_data(n + 4096); - std::vector new_states(n + 4096, 0); - size_t idx = (ingested_plus_one_ - begin_) % n; - for (size_t i = 0; i < n; ++i) { + size_t origin_len = data_.size(); + std::vector new_data(origin_len + BUFFSIZ); + std::vector new_states(origin_len + BUFFSIZ, 0); + size_t idx = (ingested_plus_one_ - begin_) % origin_len; + for (size_t i = 0; i < origin_len; ++i) { new_data[i] = data_[idx]; new_states[i] = states_[idx]; if (states_[idx]) { end_ = ingested_plus_one_ + i + 1; } ++idx; - idx %= n; + idx %= origin_len; } data_ = std::move(new_data); states_ = std::move(new_states); begin_ = ingested_plus_one_; } + + timestamp_t last_ingested() const { return ingested_plus_one_ - 1; } WalIngester(GraphDBSession& session, timestamp_t cur) : session_(session), begin_(cur), end_(cur), ingested_plus_one_(cur), - data_(4096), - states_(4096, 0) {} + data_(BUFFSIZ), + states_(BUFFSIZ, 0) {} bool empty() const { return ingested_plus_one_ == end_; } @@ -77,7 +80,8 @@ struct WalIngester { } void ingest() { - size_t idx = (ingested_plus_one_ - begin_) % data_.size(); + size_t len = data_.size(); + size_t idx = (ingested_plus_one_ - begin_) % len; bool flag = false; while (states_[idx] == 2 || states_[idx] == 1) { if (states_[idx] == 1) { @@ -86,7 +90,7 @@ struct WalIngester { states_[idx] = 0; ++ingested_plus_one_; ++idx; - idx %= data_.size(); + idx %= len; flag = true; } if (flag) { @@ -180,6 +184,7 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, if (key == "topic_name") { topic_name = value; } else { + LOG(INFO) << "Kafka config: " << key << " = " << value; config.set(std::string(key), std::string(value)); } } @@ -187,7 +192,7 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, timestamp_t cur_ts = graph.db().get_last_ingested_wal_ts() + 1; gs::WalIngester ingester(graph, cur_ts); gs::KafkaWalConsumer consumer(ingester, config, topic_name, 1); - while (graph.db().kafka_wal_ingester_state()) { + while (!force_stop_.load()) { consumer.poll(); ingester.ingest(); std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -196,8 +201,16 @@ bool KafkaWalIngesterApp::Query(GraphDBSession& graph, Decoder& input, consumer.poll(); ingester.ingest(); } + int64_t ts = ingester.last_ingested(); + output.put_long(ts); + return true; +} + +bool KafkaWalIngesterApp::terminal() { + force_stop_.store(true); return true; } + AppWrapper KafkaWalIngesterAppFactory::CreateApp(const GraphDB& db) { return AppWrapper(new KafkaWalIngesterApp(), NULL); } diff --git a/flex/engines/graph_db/app/kafka_wal_ingester_app.h b/flex/engines/graph_db/app/kafka_wal_ingester_app.h index bb4d33b379d9..1ac63bdd5cdc 100644 --- a/flex/engines/graph_db/app/kafka_wal_ingester_app.h +++ b/flex/engines/graph_db/app/kafka_wal_ingester_app.h @@ -24,11 +24,14 @@ namespace gs { // Ingest wal from kafka class KafkaWalIngesterApp : public WriteAppBase { public: - KafkaWalIngesterApp() {} + KafkaWalIngesterApp() : force_stop_(false) {} AppType type() const override { return AppType::kBuiltIn; } bool Query(GraphDBSession& graph, Decoder& input, Encoder& output) override; + + bool terminal(); + std::atomic force_stop_{false}; }; class KafkaWalIngesterAppFactory : public AppFactoryBase { diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index b0b687eec8ca..4a80183fb82b 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -28,10 +28,6 @@ #include "flex/engines/graph_db/runtime/utils/cypher_runner_impl.h" #include "flex/utils/yaml_utils.h" -#ifdef BUILD_KAFKA_WAL_WRITER_PARSER -#include "flex/engines/graph_db/app/kafka_wal_ingester_app.h" -#endif - #include "flex/third_party/httplib.h" namespace gs { @@ -58,7 +54,6 @@ struct SessionLocalContext { GraphDB::GraphDB() : monitor_thread_running_(false), - kafka_wal_ingester_thread_running_(false), last_ingested_wal_ts_(0), compact_thread_running_(false) {} GraphDB::~GraphDB() { @@ -282,54 +277,6 @@ void GraphDB::Close() { std::fill(app_factories_.begin(), app_factories_.end(), nullptr); } -#ifdef BUILD_KAFKA_WAL_WRITER_PARSER -bool GraphDB::kafka_wal_ingester_state() const { - return kafka_wal_ingester_thread_running_.load(); -} - -void GraphDB::start_kafka_wal_ingester(const cppkafka::Configuration& config, - const std::string& topic_name) { - if (kafka_wal_ingester_thread_running_) { - kafka_wal_ingester_thread_running_ = false; - if (kafka_wal_ingester_thread_.joinable()) { - kafka_wal_ingester_thread_.join(); - } - } - kafka_wal_ingester_thread_running_ = true; - - kafka_wal_ingester_thread_ = std::thread([&]() { - std::vector buffer; - gs::Encoder encoder(buffer); - encoder.put_string("topic_name"); - encoder.put_string(topic_name); - if (config.has_property("metadata.broker.list")) { - encoder.put_string("metadata.broker.list"); - encoder.put_string(config.get("metadata.broker.list")); - } - if (config.has_property("group.id")) { - encoder.put_string("group.id"); - encoder.put_string(config.get("group.id")); - } - if (config.has_property("enable.auto.commit")) { - encoder.put_string("enable.auto.commit"); - encoder.put_string(config.get("enable.auto.commit")); - } - if (config.has_property("auto.offset.reset")) { - encoder.put_string("auto.offset.reset"); - encoder.put_string(config.get("auto.offset.reset")); - } - gs::Decoder decoder(buffer.data(), buffer.size()); - KafkaWalIngesterApp().Query(GetSession(0), decoder, encoder); - }); -} - -void GraphDB::stop_kafka_wal_ingester() { - kafka_wal_ingester_thread_running_ = false; - kafka_wal_ingester_thread_.join(); -} - -#endif - ReadTransaction GraphDB::GetReadTransaction(int thread_id) { return contexts_[thread_id].session.GetReadTransaction(); } diff --git a/flex/engines/graph_db/database/graph_db.h b/flex/engines/graph_db/database/graph_db.h index b324f27caa88..53844556eb89 100644 --- a/flex/engines/graph_db/database/graph_db.h +++ b/flex/engines/graph_db/database/graph_db.h @@ -33,9 +33,6 @@ #include "flex/storages/rt_mutable_graph/loader/loader_factory.h" #include "flex/storages/rt_mutable_graph/loading_config.h" #include "flex/storages/rt_mutable_graph/mutable_property_fragment.h" -#ifdef BUILD_KAFKA_WAL_WRITER_PARSER -#include "cppkafka/cppkafka.h" -#endif namespace gs { @@ -172,15 +169,6 @@ class GraphDB { inline const GraphDBConfig& config() const { return config_; } -#ifdef BUILD_KAFKA_WAL_WRITER_PARSER - bool kafka_wal_ingester_state() const; - - void start_kafka_wal_ingester(const cppkafka::Configuration& config, - const std::string& topic_name); - - void stop_kafka_wal_ingester(); -#endif - uint64_t get_last_ingested_wal_ts() const { return last_ingested_wal_ts_; } void set_last_ingested_wal_ts(uint64_t ts) { last_ingested_wal_ts_ = ts; } @@ -219,8 +207,6 @@ class GraphDB { std::thread monitor_thread_; bool monitor_thread_running_; - std::thread kafka_wal_ingester_thread_; - std::atomic kafka_wal_ingester_thread_running_; uint64_t last_ingested_wal_ts_; timestamp_t last_compaction_ts_; diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc index 268f8f37e665..694989fada79 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.cc @@ -14,81 +14,31 @@ */ #include "flex/engines/graph_db/database/wal/kafka_wal_parser.h" +#include "flex/engines/graph_db/database/wal/kafka_wal_utils.h" #include "flex/engines/graph_db/database/wal/wal.h" namespace gs { -std::vector get_all_topic_partitions( - const cppkafka::Configuration& config, const std::string& topic_name, - bool from_beginning) { - std::vector partitions; - cppkafka::Consumer consumer(config); // tmp consumer - LOG(INFO) << config.get("metadata.broker.list"); - LOG(INFO) << config.get("group.id"); - - LOG(INFO) << "Get metadata for topic " << topic_name; - auto meta_vector = consumer.get_metadata().get_topics({topic_name}); - if (meta_vector.empty()) { - LOG(WARNING) << "Failed to get metadata for topic " << topic_name - << ", maybe the topic does not exist"; - return {}; - } - auto metadata = meta_vector.front().get_partitions(); - for (const auto& partition : metadata) { - if (from_beginning) { - partitions.push_back( - cppkafka::TopicPartition(topic_name, partition.get_id(), 0)); - } else { - partitions.push_back(cppkafka::TopicPartition( - topic_name, partition.get_id())); // from the beginning - } - } - return partitions; -} - std::unique_ptr KafkaWalParser::Make(const std::string& uri) { // uri should be like // "kafka://localhost:9092,localhost:9093/my_topic?group.id=my_consumer_group" - const std::string prefix = "kafka://"; - if (uri.find(prefix) != 0) { - LOG(FATAL) << "Invalid uri: " << uri; - } - - std::string hosts_part = uri.substr(prefix.length()); - size_t query_pos = hosts_part.find('/'); - std::string hosts; - std::string query; - cppkafka::Configuration config; - - if (query_pos != std::string::npos) { - hosts = hosts_part.substr(0, query_pos); - query = hosts_part.substr(query_pos + 1); - } else { - LOG(FATAL) << "Invalid uri: " << uri; + auto res = parse_uri(uri); + if (!res) { + LOG(FATAL) << "Failed to parse uri: " << uri; + return nullptr; } - config.set("metadata.broker.list", hosts); - size_t top_pos = query.find('?'); + gs::Decoder decoder(res.value().data(), res.value().size()); std::string topic_name; - if (top_pos != std::string::npos) { - topic_name = query.substr(0, top_pos); - query = query.substr(top_pos + 1); - } else { - LOG(FATAL) << "Invalid uri: " << uri; - } - std::istringstream query_stream(query); - std::string pair; - while (std::getline(query_stream, pair, '&')) { - size_t eq_pos = pair.find('='); - if (eq_pos != std::string::npos) { - std::string key = pair.substr(0, eq_pos); - std::string value = pair.substr(eq_pos + 1); - if (key == "group.id") { - config.set("group.id", value); - } + cppkafka::Configuration config; + while (!decoder.empty()) { + auto key = decoder.get_string(); + auto value = decoder.get_string(); + if (key == "topic_name") { + topic_name = value; + } else { + config.set(std::string(key), std::string(value)); } } - config.set("enable.auto.commit", false); - auto parser = std::unique_ptr(new KafkaWalParser(config)); parser->open(topic_name); return parser; diff --git a/flex/engines/graph_db/database/wal/kafka_wal_parser.h b/flex/engines/graph_db/database/wal/kafka_wal_parser.h index 75e02b5389e6..78eeaac401e6 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_parser.h +++ b/flex/engines/graph_db/database/wal/kafka_wal_parser.h @@ -22,13 +22,6 @@ namespace gs { -/* - * Get all partitions of the given topic. - */ -std::vector get_all_topic_partitions( - const cppkafka::Configuration& config, const std::string& topic_name, - bool from_beginning = true); - class KafkaWalParser : public IWalParser { public: static constexpr const std::chrono::milliseconds POLL_TIMEOUT = diff --git a/flex/engines/graph_db/database/wal/kafka_wal_utils.cc b/flex/engines/graph_db/database/wal/kafka_wal_utils.cc new file mode 100644 index 000000000000..59e362139dd7 --- /dev/null +++ b/flex/engines/graph_db/database/wal/kafka_wal_utils.cc @@ -0,0 +1,97 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "flex/engines/graph_db/database/wal/kafka_wal_utils.h" +#include "glog/logging.h" + +namespace gs { +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +std::vector get_all_topic_partitions( + const cppkafka::Configuration& config, const std::string& topic_name, + bool from_beginning) { + std::vector partitions; + cppkafka::Consumer consumer(config); // tmp consumer + LOG(INFO) << config.get("metadata.broker.list"); + LOG(INFO) << config.get("group.id"); + + LOG(INFO) << "Get metadata for topic " << topic_name; + auto meta_vector = consumer.get_metadata().get_topics({topic_name}); + if (meta_vector.empty()) { + LOG(WARNING) << "Failed to get metadata for topic " << topic_name + << ", maybe the topic does not exist"; + return {}; + } + auto metadata = meta_vector.front().get_partitions(); + for (const auto& partition : metadata) { + if (from_beginning) { + partitions.push_back( + cppkafka::TopicPartition(topic_name, partition.get_id(), 0)); + } else { + partitions.push_back(cppkafka::TopicPartition( + topic_name, partition.get_id())); // from the beginning + } + } + return partitions; +} + +std::optional> parse_uri(const std::string& wal_uri) { + std::vector buf; + gs::Encoder encoder(buf); + + const std::string prefix = "kafka://"; + if (wal_uri.find(prefix) != 0) { + LOG(ERROR) << "Invalid uri: " << wal_uri; + return std::nullopt; + } + + std::string hosts_part = wal_uri.substr(prefix.length()); + size_t query_pos = hosts_part.find('/'); + std::string hosts; + std::string query; + hosts = hosts_part.substr(0, query_pos); + query = hosts_part.substr(query_pos + 1); + + std::string kafka_brokers = hosts; + encoder.put_string(std::string("metadata.broker.list")); + encoder.put_string(kafka_brokers); + size_t top_pos = query.find('?'); + std::string topic; + if (top_pos != std::string::npos) { + topic = query.substr(0, top_pos); + query = query.substr(top_pos + 1); + } else { + LOG(ERROR) << "Invalid uri: " << wal_uri; + return std::nullopt; + } + encoder.put_string(std::string("topic_name")); + encoder.put_string(topic); + std::istringstream query_stream(query); + std::string pair; + while (std::getline(query_stream, pair, '&')) { + size_t eq_pos = pair.find('='); + if (eq_pos != std::string::npos) { + std::string key = pair.substr(0, eq_pos); + std::string value = pair.substr(eq_pos + 1); + encoder.put_string(key); + encoder.put_string(value); + } + } + encoder.put_string(std::string("enable.auto.commit")); + encoder.put_string(std::string("false")); + + return buf; +} +#endif + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/database/wal/kafka_wal_utils.h b/flex/engines/graph_db/database/wal/kafka_wal_utils.h new file mode 100644 index 000000000000..2354ed235ac9 --- /dev/null +++ b/flex/engines/graph_db/database/wal/kafka_wal_utils.h @@ -0,0 +1,38 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLEX_ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_UTILS_H_ +#define FLEX_ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_UTILS_H_ +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +#include +#include +#include +#include "cppkafka/cppkafka.h" +#include "flex/utils/app_utils.h" + +namespace gs { +/* + * Get all partitions of the given topic. + */ +std::vector get_all_topic_partitions( + const cppkafka::Configuration& config, const std::string& topic_name, + bool from_beginning = true); + +std::optional> parse_uri(const std::string& wal_uri); + +} // namespace gs +#endif + +#endif // FLEX_ENGINES_GRAPH_DB_DATABASE_WAL_KAFKA_WAL_UTILS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/database/wal/kafka_wal_writer.cc b/flex/engines/graph_db/database/wal/kafka_wal_writer.cc index cd07ab2558f1..340befc4ed31 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_writer.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_writer.cc @@ -14,6 +14,7 @@ */ #include "flex/engines/graph_db/database/wal/kafka_wal_writer.h" +#include "flex/engines/graph_db/database/wal/kafka_wal_utils.h" #include "flex/engines/graph_db/database/wal/wal.h" #include @@ -26,29 +27,19 @@ std::unique_ptr KafkaWalWriter::Make() { } void KafkaWalWriter::open(const std::string& uri, int thread_id) { - const std::string prefix = "kafka://"; - if (uri.find(prefix) != 0) { - LOG(FATAL) << "Invalid uri: " << uri; + auto res = parse_uri(uri); + if (!res) { + LOG(FATAL) << "Failed to parse uri: " << uri; } - - std::string hosts_part = uri.substr(prefix.length()); - size_t query_pos = hosts_part.find('/'); - std::string hosts; - std::string query; - cppkafka::Configuration config; - if (query_pos != std::string::npos) { - hosts = hosts_part.substr(0, query_pos); - query = hosts_part.substr(query_pos + 1); - } else { - LOG(FATAL) << "Invalid uri: " << uri; - } - kafka_brokers_ = hosts; - size_t top_pos = query.find('?'); - std::string topic; - if (top_pos != std::string::npos) { - topic = query.substr(0, top_pos); - } else { - topic = query; + gs::Decoder decoder(res.value().data(), res.value().size()); + while (!decoder.empty()) { + auto key = decoder.get_string(); + auto value = decoder.get_string(); + if (key == "metadata.broker.list") { + kafka_brokers_ = value; + } else if (key == "topic_name") { + kafka_topic_ = value; + } } if (thread_id_ != -1 || producer_) { @@ -56,10 +47,9 @@ void KafkaWalWriter::open(const std::string& uri, int thread_id) { } thread_id_ = thread_id; if (!kafka_brokers_.empty()) { - if (topic.empty()) { + if (kafka_topic_.empty()) { LOG(FATAL) << "Kafka topic is empty"; } - kafka_topic_ = topic; cppkafka::Configuration config = {{"metadata.broker.list", kafka_brokers_}}; producer_ = std::make_shared>(config); diff --git a/flex/engines/http_server/graph_db_service.cc b/flex/engines/http_server/graph_db_service.cc index eb27be41b1e4..025be75d4e35 100644 --- a/flex/engines/http_server/graph_db_service.cc +++ b/flex/engines/http_server/graph_db_service.cc @@ -425,6 +425,23 @@ bool GraphDBService::stop_compiler_subprocess() { return true; } +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +bool GraphDBService::start_kafka_wal_ingester() { + kafka_wal_ingester_ = std::make_unique(); + kafka_wal_ingester_->open(gs::GraphDB::get(), service_config_.wal_uri); + return true; +} + +bool GraphDBService::stop_kafka_wal_ingester() { + if (kafka_wal_ingester_) { + kafka_wal_ingester_->close(); + kafka_wal_ingester_.reset(); + return true; + } + return false; +} +#endif + std::string GraphDBService::find_interactive_class_path() { std::string interactive_home = DEFAULT_INTERACTIVE_HOME; if (std::getenv("INTERACTIVE_HOME")) { diff --git a/flex/engines/http_server/graph_db_service.h b/flex/engines/http_server/graph_db_service.h index 0689cf1003ba..56cefcc1f6ff 100644 --- a/flex/engines/http_server/graph_db_service.h +++ b/flex/engines/http_server/graph_db_service.h @@ -28,6 +28,9 @@ #include "flex/storages/metadata/metadata_store_factory.h" #include "flex/utils/result.h" #include "flex/utils/service_utils.h" +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +#include "flex/engines/http_server/kafka_wal_ingester.h" +#endif #include #include @@ -175,6 +178,12 @@ class GraphDBService { bool check_compiler_ready() const; +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER + bool start_kafka_wal_ingester(); + + bool stop_kafka_wal_ingester(); +#endif + private: GraphDBService() = default; @@ -197,6 +206,9 @@ class GraphDBService { boost::process::child compiler_process_; // handler for metadata store std::shared_ptr metadata_store_; +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER + std::unique_ptr kafka_wal_ingester_; +#endif }; } // namespace server diff --git a/flex/engines/http_server/kafka_wal_ingester.cc b/flex/engines/http_server/kafka_wal_ingester.cc new file mode 100644 index 000000000000..cf6ab53b5f2b --- /dev/null +++ b/flex/engines/http_server/kafka_wal_ingester.cc @@ -0,0 +1,52 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "flex/engines/http_server/kafka_wal_ingester.h" +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/database/graph_db_session.h" +#include "flex/engines/graph_db/database/wal/kafka_wal_utils.h" + +namespace server { +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER + +bool KafkaWalIngester::open(gs::GraphDB& db, const std::string& wal_uri) { + ingester_ = std::make_unique(); + ingester_thread_ = std::thread([&]() { + auto res = gs::parse_uri(wal_uri); + if (!res) { + LOG(ERROR) << "Failed to parse uri: " << wal_uri; + return; + } + gs::Decoder decoder(res.value().data(), res.value().size()); + std::vector buf; + gs::Encoder encoder(buf); + ingester_->Query(db.GetSession(0), decoder, encoder); + gs::Decoder output(buf.data(), buf.size()); + db.set_last_ingested_wal_ts(output.get_long()); + }); + return true; +} + +bool KafkaWalIngester::close() { + if (ingester_) { + ingester_->terminal(); + } + if (ingester_thread_.joinable()) { + ingester_thread_.join(); + } + return true; +} +#endif + +} // namespace server \ No newline at end of file diff --git a/flex/engines/http_server/kafka_wal_ingester.h b/flex/engines/http_server/kafka_wal_ingester.h new file mode 100644 index 000000000000..16f02b0460c8 --- /dev/null +++ b/flex/engines/http_server/kafka_wal_ingester.h @@ -0,0 +1,34 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#ifdef BUILD_KAFKA_WAL_WRITER_PARSER +#include "flex/engines/graph_db/app/kafka_wal_ingester_app.h" +namespace server { +class KafkaWalIngester { + public: + KafkaWalIngester() : force_stop_(false), ingester_(nullptr) {} + + bool open(gs::GraphDB&, const std::string& uri); + + bool close(); + std::atomic force_stop_{false}; + std::unique_ptr ingester_; + std::thread ingester_thread_; +}; +} // namespace server +#endif \ No newline at end of file diff --git a/flex/tests/hqps/CMakeLists.txt b/flex/tests/hqps/CMakeLists.txt index 2aa5ac05bf61..857f9dce7e06 100644 --- a/flex/tests/hqps/CMakeLists.txt +++ b/flex/tests/hqps/CMakeLists.txt @@ -10,7 +10,7 @@ foreach(f ${GS_TEST_FILES}) set(T_NAME ${CMAKE_MATCH_1}) message(STATUS "Found graphscope test - " ${T_NAME}) add_executable(${T_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/${T_NAME}.cc) - target_link_libraries(${T_NAME} flex_plan_proto flex_graph_db flex_utils) + target_link_libraries(${T_NAME} flex_plan_proto flex_graph_db flex_utils flex_server) if (BUILD_WITH_OSS) target_link_libraries(${T_NAME} cpp-sdk) endif() diff --git a/flex/tests/hqps/kafka_wal_ingester_test.cc b/flex/tests/hqps/kafka_wal_ingester_test.cc index 92f477258e73..2b32b8d0fb95 100644 --- a/flex/tests/hqps/kafka_wal_ingester_test.cc +++ b/flex/tests/hqps/kafka_wal_ingester_test.cc @@ -17,6 +17,7 @@ #ifdef BUILD_KAFKA_WAL_WRITER_PARSER #include #include +#include "flex/engines/http_server/kafka_wal_ingester.h" #endif #include "grape/serialization/in_archive.h" #include "grape/serialization/out_archive.h" @@ -66,8 +67,8 @@ int main(int argc, char** argv) { {"group.id", "test"}, {"enable.auto.commit", false}, {"auto.offset.reset", "earliest"}}; - db2.start_kafka_wal_ingester(config1, kafka_topic); - + server::KafkaWalIngester ingester; + ingester.open(db2, uri); { std::vector threads; for (int i = 100; i < 200; ++i) { @@ -98,7 +99,7 @@ int main(int argc, char** argv) { } LOG(INFO) << db.GetReadTransaction(0).GetVertexNum(0); - std::this_thread::sleep_for(std::chrono::seconds(3)); + std::this_thread::sleep_for(std::chrono::seconds(4)); { auto txn = db2.GetReadTransaction(0); CHECK(txn.GetVertexNum(0) == 195) << "Vertex num: " << txn.GetVertexNum(0); @@ -118,8 +119,9 @@ int main(int argc, char** argv) { cnt++; iter.Next(); } + ingester.close(); - db2.stop_kafka_wal_ingester(); + // db2.stop_kafka_wal_ingester(); } #endif return 0; From c5b49318de43ac2f79e59f6659d8cb04ebdfe2f5 Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Mon, 31 Mar 2025 13:01:05 +0800 Subject: [PATCH 16/17] fix ci --- flex/engines/graph_db/database/wal/kafka_wal_utils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flex/engines/graph_db/database/wal/kafka_wal_utils.cc b/flex/engines/graph_db/database/wal/kafka_wal_utils.cc index 59e362139dd7..90d2430bd41d 100644 --- a/flex/engines/graph_db/database/wal/kafka_wal_utils.cc +++ b/flex/engines/graph_db/database/wal/kafka_wal_utils.cc @@ -71,8 +71,8 @@ std::optional> parse_uri(const std::string& wal_uri) { topic = query.substr(0, top_pos); query = query.substr(top_pos + 1); } else { - LOG(ERROR) << "Invalid uri: " << wal_uri; - return std::nullopt; + topic = query; + query = ""; } encoder.put_string(std::string("topic_name")); encoder.put_string(topic); From 2f15484d306fe8c2790dcfca28133858766a9b62 Mon Sep 17 00:00:00 2001 From: liulx20 <519459125@qq.com> Date: Mon, 31 Mar 2025 15:27:54 +0800 Subject: [PATCH 17/17] fix --- flex/bin/rt_server.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/flex/bin/rt_server.cc b/flex/bin/rt_server.cc index c6bbbbc8cf00..91684c2e11c4 100644 --- a/flex/bin/rt_server.cc +++ b/flex/bin/rt_server.cc @@ -108,6 +108,7 @@ int main(int argc, char** argv) { service_config.query_port = http_port; service_config.start_admin_service = false; service_config.start_compiler = false; + service_config.wal_uri = config.wal_uri; service_config.set_sharding_mode(vm["sharding-mode"].as()); server::GraphDBService::get().init(service_config);