Initial commit

cmake/AddCrate.cmake

@@ -0,0 +1,33 @@
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(CARGO_CMD cargo build)
+    set(TARGET_DIR "debug")
+else ()
+    set(CARGO_CMD cargo build --release)
+    set(TARGET_DIR "release")
+endif ()
+
+add_custom_target(
+    cargo_build_all ALL
+    COMMAND ${CARGO_CMD}
+    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
+)
+
+macro(add_crate NAME)
+    set(LIBRARY "${PROJECT_SOURCE_DIR}/target/${TARGET_DIR}/lib${NAME}.a")
+    set(SOURCES
+        "${PROJECT_SOURCE_DIR}/target/cxxbridge/${NAME}/src/cxx.rs.h"
+        "${PROJECT_SOURCE_DIR}/target/cxxbridge/${NAME}/src/cxx.rs.cc"
+    )
+
+    add_custom_command(
+        OUTPUT ${SOURCES} ${LIBRARY}
+        COMMAND ${CARGO_CMD}
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${NAME}"
+    )
+
+    add_library(${NAME} STATIC ${SOURCES} ${LIBRARY})
+    target_link_libraries(${NAME} pthread dl ${LIBRARY})
+    target_include_directories(${NAME} PUBLIC "${PROJECT_SOURCE_DIR}/target/cxxbridge")
+    target_compile_options(${NAME} PUBLIC -Wno-dollar-in-identifier-extension)
+    add_dependencies(${NAME} cargo_build_all)
+endmacro()

cmake/ApacheArrow.cmake

@@ -0,0 +1,54 @@
+add_library(apache_arrow_static INTERFACE)
+add_library(arrow_static STATIC IMPORTED)
+add_library(parquet_static STATIC IMPORTED)
+add_library(arrow_dependencies STATIC IMPORTED)
+
+set(PREFIX "${CMAKE_CURRENT_BINARY_DIR}")
+set(ARROW_RELEASE_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/src/apache-arrow-cpp/cpp/build/release")
+
+# https://cmake.org/cmake/help/latest/policy/CMP0097.html
+# Starting with CMake 3.16, explicitly setting GIT_SUBMODULES to an empty string
+# means no submodules will be initialized or updated.
+cmake_policy(SET CMP0097 NEW)
+
+include(ExternalProject)
+ExternalProject_Add(
+    apache-arrow-cpp
+    PREFIX ${PREFIX}
+    GIT_REPOSITORY https://github.com/apache/arrow.git
+    GIT_TAG b7d2f7ffca66c868bd2fce5b3749c6caa002a7f0
+    GIT_SHALLOW ON
+    GIT_PROGRESS ON
+    GIT_SUBMODULES ""
+    SOURCE_SUBDIR "cpp"
+    BUILD_IN_SOURCE ON
+    INSTALL_DIR ${PREFIX}
+    CONFIGURE_COMMAND bash -x -c "\
+    ( cd thirdparty && [[ -f export.sh ]] || ./download_dependencies.sh | tee export.sh ) && \
+    source thirdparty/export.sh && cmake -S . -B . \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DARROW_USE_CCACHE=OFF \
+        -DARROW_USE_SCCACHE=OFF \
+        -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+        -DARROW_BUILD_STATIC=ON \
+        -DARROW_JEMALLOC=ON \
+        -DARROW_SIMD_LEVEL=DEFAULT \
+        -DARROW_BUILD_EXAMPLES=OFF \
+        -DARROW_PARQUET=ON -DARROW_CSV=ON \
+        -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_ZLIB=ON"
+    BUILD_COMMAND bash -x -c "source thirdparty/export.sh && cmake --build . -j"
+    INSTALL_COMMAND cmake --install . --prefix "${PREFIX}"
+    BUILD_BYPRODUCTS
+        "${ARROW_RELEASE_BUILD_DIR}/libarrow.a"
+        "${ARROW_RELEASE_BUILD_DIR}/libparquet.a"
+        "${ARROW_RELEASE_BUILD_DIR}/libarrow_bundled_dependencies.a"
+  )
+
+add_dependencies(arrow_static apache-arrow-cpp)
+add_dependencies(parquet_static apache-arrow-cpp)
+add_dependencies(arrow_dependencies apache-arrow-cpp)
+set_target_properties(arrow_static PROPERTIES IMPORTED_LOCATION "${ARROW_RELEASE_BUILD_DIR}/libarrow.a")
+set_target_properties(parquet_static PROPERTIES IMPORTED_LOCATION "${ARROW_RELEASE_BUILD_DIR}/libparquet.a")
+set_target_properties(arrow_dependencies PROPERTIES IMPORTED_LOCATION "${ARROW_RELEASE_BUILD_DIR}/libarrow_bundled_dependencies.a")
+target_include_directories(apache_arrow_static SYSTEM INTERFACE "${PREFIX}/include")
+target_link_libraries(apache_arrow_static INTERFACE parquet_static arrow_static arrow_dependencies)

cmake/CLangFormat.cmake

@@ -0,0 +1,24 @@
+set(CLANG_FORMAT "/usr/bin/clang-format-14")
+if(EXISTS ${CLANG_FORMAT})
+    message(STATUS "Found clang-format at ${CLANG_FORMAT}")
+
+    set(SOURCE_DIRS
+        ${CMAKE_SOURCE_DIR}/src
+        ${CMAKE_SOURCE_DIR}/tests
+        ${CMAKE_SOURCE_DIR}/benchmarks
+    )
+
+    # For now, it just hard codes the source files list to globs. That works
+    # fine until we have another directory in `src/`. We should ideally gather
+    # this from SOURCE_FILES list. But, should filter the thirs_party sources.
+    # Taking a quick route for now. We should deal with it sometime down the line.
+    add_custom_target(format
+            COMMENT "Running clang-format"
+            COMMAND find ${SOURCE_DIRS} -name '*.cc' -o -name '*.cpp' -o -name '*.h' | grep -v "_generated.h" | xargs ${CLANG_FORMAT} -i)
+
+    add_custom_target(check-format
+            COMMENT "Running clang-format"
+            COMMAND find ${SOURCE_DIRS} -name '*.cc' -o -name '*.cpp' -o -name '*.h' | grep -v "_generated.h" | xargs ${CLANG_FORMAT} --Werror --dry-run)
+else()
+    message(FATAL_ERROR "clang-format-14 not found")
+endif()

cmake/CLangTidy.cmake

@@ -0,0 +1,52 @@
+# clang-tidy generates too many warnings, so just disable it by default.
+option(ENABLE_CLANG_TIDY "Run clang-tidy during build" OFF)
+
+find_program(CLANG_TIDY NAMES clang-tidy-14)
+if(CLANG_TIDY)
+    if(CMake_SOURCE_DIR STREQUAL CMake_BINARY_DIR)
+        message(FATAL_ERROR "CMake_RUN_CLANG_TIDY requires an out-of-source build!")
+    endif()
+
+    if(NOT CMAKE_EXPORT_COMPILE_COMMANDS)
+        message(WARNING "CMAKE_EXPORT_COMPILE_COMMANDS=OFF, clang-tidy may not works!!!")
+    endif()
+
+    set(HEADER_FILTER "${CMAKE_SOURCE_DIR}/\\(src\\|tests\\|benchmarks\\|demos\\)")
+
+    if(ENABLE_CLANG_TIDY)
+        set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY} --header-filter ${HEADER_FILTER})
+
+        # Create a preprocessor definition that depends on .clang-tidy content so
+        # the compile command will change when .clang-tidy changes.  This ensures
+        # that a subsequent build re-runs clang-tidy on all sources even if they
+        # do not otherwise need to be recompiled.  Nothing actually uses this
+        # definition.  We add it to targets on which we run clang-tidy just to
+        # get the build dependency on the .clang-tidy file.
+        file(SHA1 ${CMAKE_CURRENT_SOURCE_DIR}/.clang-tidy clang_tidy_sha1)
+        set(CLANG_TIDY_DEFINITIONS "CLANG_TIDY_SHA1=${clang_tidy_sha1}")
+        unset(clang_tidy_sha1)
+
+        configure_file(.clang-tidy .clang-tidy COPYONLY)
+    endif()
+
+    set(SOURCE_DIRS
+        ${CMAKE_SOURCE_DIR}/src
+        ${CMAKE_SOURCE_DIR}/tests
+        ${CMAKE_SOURCE_DIR}/demos
+        ${CMAKE_SOURCE_DIR}/benchmarks
+    )
+
+    # For now, it just hard codes the source files list to globs. That works
+    # fine until we have another directory in `src/`. We should ideally gather
+    # this from SOURCE_FILES list. But, should filter the thirs_party sources.
+    # Taking a quick route for now. We should deal with it sometime down the line.
+    add_custom_target(clang-tidy
+        COMMENT "Running clang-tidy"
+        COMMAND run-clang-tidy-14 -header-filter ${HEADER_FILTER} `find ${SOURCE_DIRS} -name "*.cc" -o -name "*.cpp" -not -name "*.actor.cpp" ` -quiet)
+
+    add_custom_target(clang-tidy-fix
+        COMMENT "Running clang-tidy -fix"
+        COMMAND run-clang-tidy-14 -header-filter ${HEADER_FILTER} `find ${SOURCE_DIRS} -name "*.cc" -o -name "*.cpp" -not -name "*.actor.cpp" ` -fix -quiet)
+else()
+    message(WARNING "clang-tidy-14 not found!!!")
+endif()

cmake/CTestCustom.cmake

@@ -0,0 +1,12 @@
+file (STRINGS "@CMAKE_BINARY_DIR@/CTestTestfile.cmake" LINES)
+
+# overwrite the file....
+file(WRITE "@CMAKE_BINARY_DIR@/CTestTestfile.cmake" "")
+
+# loop through the lines,
+foreach(LINE IN LISTS LINES)
+  # remove unwanted parts
+  string(REGEX REPLACE ".*third_party.*" "" STRIPPED "${LINE}")
+  # and write the (changed) line ...
+  file(APPEND "@CMAKE_BINARY_DIR@/CTestTestfile.cmake" "${STRIPPED}\n")
+endforeach()

cmake/CodeCoverage.cmake

@@ -0,0 +1,27 @@
+option(ENABLE_CODE_COVERAGE "Enable code coverage" OFF)
+if(ENABLE_CODE_COVERAGE)
+    if(CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" )
+        message(STATUS "Enable code coverage with debug mode.")
+    else()
+        message(WARNING "Code coverage with no debug mode!!!")
+    endif()
+
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+        set(COVERAGE_COMPILER_FLAGS "-g -fprofile-arcs -ftest-coverage")
+        link_libraries(gcov)
+    elseif(CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang")
+        set(COVERAGE_COMPILER_FLAGS "-g -fprofile-instr-generate -fcoverage-mapping")
+    else()
+        message(FATAL_ERROR "Unknown compiler iid ${CMAKE_CXX_COMPILER_ID}")
+    endif()
+
+    include(CheckCXXCompilerFlag)
+    check_cxx_compiler_flag(-fprofile-abs-path HAVE_fprofile_abs_path)
+    if(HAVE_fprofile_abs_path)
+        set(COVERAGE_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS} -fprofile-abs-path")
+    endif()
+
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COVERAGE_COMPILER_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_COMPILER_FLAGS}")
+    message(STATUS "Appending code coverage compiler flags: ${COVERAGE_COMPILER_FLAGS}")
+endif()

cmake/CompileFlags.cmake

@@ -0,0 +1,21 @@
+macro(store_compile_flags)
+set(ORIGINAL_C_FLAGS "${CMAKE_C_FLAGS}")
+set(ORIGINAL_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
+set(ORIGINAL_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+set(ORIGINAL_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
+set(ORIGINAL_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+set(ORIGINAL_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
+set(ORIGINAL_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+set(ORIGINAL_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+endmacro()
+
+macro(restore_compile_flags)
+set(CMAKE_C_FLAGS "${ORIGINAL_C_FLAGS}")
+set(CMAKE_C_FLAGS_DEBUG "${ORIGINAL_C_FLAGS_DEBUG}")
+set(CMAKE_C_FLAGS_RELEASE "${ORIGINAL_C_FLAGS_RELEASE}")
+set(CMAKE_C_FLAGS_RELWITHDEBINFO "${ORIGINAL_C_FLAGS_RELWITHDEBINFO}")
+set(CMAKE_CXX_FLAGS "${ORIGINAL_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS_DEBUG "${ORIGINAL_CXX_FLAGS_DEBUG}")
+set(CMAKE_CXX_FLAGS_RELEASE "${ORIGINAL_CXX_FLAGS_RELEASE}")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${ORIGINAL_CXX_FLAGS_RELWITHDEBINFO}")
+endmacro()

cmake/DumpConfig.cmake

@@ -0,0 +1,15 @@
+if (ENABLE_FUSE_APPLICATION)
+add_custom_target(dump-config
+            COMMENT "Running dump config"
+            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'admin_cli' -o -name 'hf3fs_fuse_main' | xargs -I {} bash -c '{} --dump_default_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`.toml'
+            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'hf3fs_fuse_main' | xargs -I {} bash -c '{} --dump_default_app_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_app.toml'
+            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'hf3fs_fuse_main' | xargs -I {} bash -c '{} --dump_default_launcher_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_launcher.toml')
+add_dependencies(dump-config meta_main mgmtd_main storage_main admin_cli hf3fs_fuse_main)
+else()
+add_custom_target(dump-config
+            COMMENT "Running dump config"
+            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'admin_cli' | xargs -I {} bash -c '{} --dump_default_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`.toml'
+            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' | xargs -I {} bash -c '{} --dump_default_app_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_app.toml'
+            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' | xargs -I {} bash -c '{} --dump_default_launcher_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_launcher.toml')
+add_dependencies(dump-config meta_main mgmtd_main storage_main admin_cli)
+endif()

cmake/GitVersion.cmake

@@ -0,0 +1,127 @@
+find_package(Git REQUIRED)
+
+if (NOT DEFINED PRE_CONFIGURE_DIR)
+    set(PRE_CONFIGURE_DIR ${PROJECT_SOURCE_DIR}/src/common/utils)
+endif ()
+
+if (NOT DEFINED POST_BUILD_DIR)
+    set(POST_BUILD_DIR ${PROJECT_BINARY_DIR})
+endif ()
+
+set(PRE_CONFIGURE_FILE ${PRE_CONFIGURE_DIR}/VersionInfo.cc.in)
+set(POST_CONFIGURE_FILE ${POST_BUILD_DIR}/src/common/utils/VersionInfo.cc)
+
+function(CheckGitWrite git_hash)
+    file(WRITE ${POST_BUILD_DIR}/git-state.txt ${git_hash})
+endfunction()
+
+function(CheckGitRead git_hash)
+    if (EXISTS ${POST_BUILD_DIR}/git-state.txt)
+        file(STRINGS ${POST_BUILD_DIR}/git-state.txt CONTENT)
+        LIST(GET CONTENT 0 var)
+        set(${git_hash} ${var} PARENT_SCOPE)
+    endif ()
+endfunction()
+
+function(CheckGitVersion)
+    # Get the latest abbreviated commit hash of the working branch
+    execute_process(
+            COMMAND ${GIT_EXECUTABLE} rev-parse --short=8 HEAD
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            OUTPUT_VARIABLE BUILD_COMMIT_HASH_SHORT
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    execute_process(
+            COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            OUTPUT_VARIABLE BUILD_COMMIT_HASH_FULL
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    execute_process(
+            COMMAND ${GIT_EXECUTABLE} log -1 --format=%at --date=local
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            OUTPUT_VARIABLE BUILD_TIMESTAMP
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    execute_process(
+            COMMAND date -d @${BUILD_TIMESTAMP} +%Y%m%d
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            OUTPUT_VARIABLE BUILD_DATE
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    execute_process(
+            COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            OUTPUT_VARIABLE BUILD_TAG
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    execute_process(
+            COMMAND bash -c "${GIT_EXECUTABLE} describe --tags --long | sed -E 's/(.*)-([0-9]+)-(\\w+)/\\2/g'"
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            OUTPUT_VARIABLE BUILD_TAG_SEQ_NUM
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    if(NOT DEFINED BUILD_TAG OR BUILD_TAG STREQUAL "")
+        set(BUILD_TAG "250228")
+    endif()
+    if(NOT DEFINED BUILD_TAG_SEQ_NUM OR BUILD_TAG_SEQ_NUM STREQUAL "")
+        set(BUILD_TAG_SEQ_NUM "1")
+    endif()
+    message(STATUS "Git Commit hash: ${BUILD_COMMIT_HASH_SHORT} ${BUILD_COMMIT_HASH_FULL}")
+    message(STATUS "Git Commit Date & Timestamp: ${BUILD_DATE} ${BUILD_TIMESTAMP}")
+    message(STATUS "Git Commit Tag & Seq Num: ${BUILD_TAG} ${BUILD_TAG_SEQ_NUM}")
+
+    set(BUILD_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}")
+    set(BUILD_VERSION_MINOR "${PROJECT_VERSION_MINOR}")
+    set(BUILD_VERSION_PATCH "${PROJECT_VERSION_PATCH}")
+    set(BUILD_VERSION "${PROJECT_VERSION}")
+
+    CheckGitRead(GIT_HASH_CACHE)
+
+    if (NOT DEFINED GIT_HASH_CACHE)
+        set(GIT_HASH_CACHE "INVALID")
+    endif ()
+
+    if (NOT DEFINED BUILD_ON_RELEASE_BRANCH)
+      set(BUILD_ON_RELEASE_BRANCH "false")
+    endif ()
+
+    if (NOT DEFINED BUILD_PIPELINE_ID)
+        set(BUILD_PIPELINE_ID "999999")
+    endif()
+
+    # Only update the git_version.cpp if the hash has changed. This will
+    # prevent us from rebuilding the project more than we need to.
+    if (NOT ${BUILD_COMMIT_HASH_FULL} STREQUAL ${GIT_HASH_CACHE} OR NOT EXISTS ${POST_CONFIGURE_FILE})
+        # Set che GIT_HASH_CACHE variable the next build won't have
+        # to regenerate the source file.
+        CheckGitWrite("${BUILD_COMMIT_HASH_FULL}")
+        configure_file(${PRE_CONFIGURE_FILE} ${POST_CONFIGURE_FILE} @ONLY)
+    endif ()
+endfunction()
+
+function(CheckGitSetup project_src_dir)
+    add_custom_target(AlwaysCheckGit COMMAND ${CMAKE_COMMAND}
+        -DRUN_CHECK_GIT_VERSION=1
+        -DPRE_CONFIGURE_DIR=${PRE_CONFIGURE_DIR}
+        -DPOST_BUILD_DIR=${POST_BUILD_DIR}
+        -DGIT_HASH_CACHE=${GIT_HASH_CACHE}
+        -DPROJECT_VERSION_MAJOR=${PROJECT_VERSION_MAJOR}
+        -DPROJECT_VERSION_MINOR=${PROJECT_VERSION_MINOR}
+        -DPROJECT_VERSION_PATCH=${PROJECT_VERSION_PATCH}
+        -DPROJECT_VERSION=${PROJECT_VERSION}
+        -DPROJECT_SOURCE_DIR=${project_src_dir}
+        -P ${PROJECT_SOURCE_DIR}/cmake/GitVersion.cmake
+        DEPENDS ${PRE_CONFIGURE_FILE}
+        BYPRODUCTS ${POST_CONFIGURE_FILE}
+    )
+
+    add_library(version-info STATIC ${POST_CONFIGURE_FILE})
+    target_include_directories(version-info PUBLIC ${PROJECT_SOURCE_DIR}/src)
+    add_dependencies(version-info AlwaysCheckGit)
+    CheckGitVersion()
+endfunction()
+
+if (RUN_CHECK_GIT_VERSION)
+    CheckGitVersion()
+endif ()

cmake/Jemalloc.cmake

@@ -0,0 +1,20 @@
+add_library(jemalloc INTERFACE)
+add_library(hf3fs_jemalloc_shared SHARED IMPORTED)
+
+include(ExternalProject)
+set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/third_party/jemalloc")
+
+ExternalProject_add(Hf3fsJemalloc_project
+  SOURCE_DIR "${PROJECT_SOURCE_DIR}/third_party/jemalloc"
+  BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
+  "${JEMALLOC_DIR}/lib/libjemalloc.so.2"
+  CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=${JEMALLOC_DIR} --disable-cxx --enable-prof --disable-initial-exec-tls
+  BUILD_IN_SOURCE ON
+  BUILD_COMMAND make -j 6
+  INSTALL_DIR "${JEMALLOC_DIR}"
+  INSTALL_COMMAND make install)
+
+add_dependencies(hf3fs_jemalloc_shared Hf3fsJemalloc_project)
+set_target_properties(hf3fs_jemalloc_shared PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.so.2")
+target_include_directories(hf3fs_jemalloc_shared INTERFACE "${JEMALLOC_DIR}/include")
+target_link_libraries(jemalloc INTERFACE hf3fs_jemalloc_shared)

cmake/Sanitizers.cmake

@@ -0,0 +1,242 @@
+#
+# Copyright (C) 2018-2022 by George Cave - gcave@stablecoder.ca
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+include(CheckCXXSourceCompiles)
+
+set(SANITIZER "" CACHE STRING "Sanitizer: ASAN, MSAN, UBSAN, TSAN")
+if(SANITIZER STREQUAL "ASAN")
+    message(STATUS "Build with Address Sanitizer.")
+    set(USE_SANITIZER "Address")
+    set(USE_ASAN ON)
+elseif(SANITIZER STREQUAL "MSAN")
+    message(STATUS "Build with Memory Sanitizer.")
+    set(USE_SANITIZER "Memory")
+    set(USE_MSAN ON)
+elseif(SANITIZER STREQUAL "UBSAN")
+    message(STATUS "Build with Undefined behavior Sanitizer.")
+    message(WARNING "With clang++ v14.0.6 undefined behavior sanitizer doesn't work with coroutine!!!")
+    set(USE_SANITIZER "Undefined")
+    set(USE_UBSAN ON)
+elseif(SANITIZER STREQUAL "TSAN")
+    message(STATUS "Build with Thread Sanitizer.")
+    set(USE_SANITIZER "Thread")
+    set(USE_TSAN ON)
+else()
+    message(STATUS "Sanitizer not enabled. ${SANITIZER}")
+    set(USE_SANITIZER "")
+endif()
+
+# set(USE_SANITIZER "" CACHE STRING
+#     "Compile with a sanitizer. Options are: Address, Memory, MemoryWithOrigins, Undefined, Thread, Leak, 'Address;Undefined', CFI"
+# )
+
+function(append value)
+    foreach(variable ${ARGN})
+        set(${variable}
+                "${${variable}} ${value}"
+                PARENT_SCOPE)
+    endforeach(variable)
+endfunction()
+
+function(append_quoteless value)
+    foreach(variable ${ARGN})
+        set(${variable}
+                ${${variable}} ${value}
+                PARENT_SCOPE)
+    endforeach(variable)
+endfunction()
+
+function(test_san_flags return_var flags)
+    set(QUIET_BACKUP ${CMAKE_REQUIRED_QUIET})
+    set(CMAKE_REQUIRED_QUIET TRUE)
+    unset(${return_var} CACHE)
+    set(FLAGS_BACKUP ${CMAKE_REQUIRED_FLAGS})
+    set(CMAKE_REQUIRED_FLAGS "${flags}")
+    check_cxx_source_compiles("int main() { return 0; }" ${return_var})
+    set(CMAKE_REQUIRED_FLAGS "${FLAGS_BACKUP}")
+    set(CMAKE_REQUIRED_QUIET "${QUIET_BACKUP}")
+endfunction()
+
+if(USE_SANITIZER)
+    append("-fno-omit-frame-pointer" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+
+    unset(SANITIZER_SELECTED_FLAGS)
+
+    if(UNIX)
+        if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+            append("-O1" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+        endif()
+
+        if(USE_SANITIZER MATCHES "([Aa]ddress)")
+            # Optional: -fno-optimize-sibling-calls -fsanitize-address-use-after-scope
+            message(STATUS "Testing with Address sanitizer")
+            set(SANITIZER_ADDR_FLAG "-fsanitize=address")
+            test_san_flags(SANITIZER_ADDR_AVAILABLE ${SANITIZER_ADDR_FLAG})
+            if(SANITIZER_ADDR_AVAILABLE)
+                message(STATUS "  Building with Address sanitizer")
+                append("${SANITIZER_ADDR_FLAG}" SANITIZER_SELECTED_FLAGS)
+
+                if(AFL)
+                    append_quoteless(AFL_USE_ASAN=1 CMAKE_C_COMPILER_LAUNCHER
+                                                     CMAKE_CXX_COMPILER_LAUNCHER)
+                endif()
+            else()
+                message(
+                    FATAL_ERROR
+                        "Address sanitizer not available for ${CMAKE_CXX_COMPILER}")
+            endif()
+        endif()
+
+        if(USE_SANITIZER MATCHES "([Mm]emory([Ww]ith[Oo]rigins)?)")
+            # Optional: -fno-optimize-sibling-calls -fsanitize-memory-track-origins=2
+            set(SANITIZER_MEM_FLAG "-fsanitize=memory")
+            if(USE_SANITIZER MATCHES "([Mm]emory[Ww]ith[Oo]rigins)")
+                message(STATUS "Testing with MemoryWithOrigins sanitizer")
+                append("-fsanitize-memory-track-origins" SANITIZER_MEM_FLAG)
+            else()
+                message(STATUS "Testing with Memory sanitizer")
+            endif()
+            test_san_flags(SANITIZER_MEM_AVAILABLE ${SANITIZER_MEM_FLAG})
+            if(SANITIZER_MEM_AVAILABLE)
+                if(USE_SANITIZER MATCHES "([Mm]emory[Ww]ith[Oo]rigins)")
+                    message(STATUS "  Building with MemoryWithOrigins sanitizer")
+                else()
+                    message(STATUS "  Building with Memory sanitizer")
+                endif()
+                append("${SANITIZER_MEM_FLAG}" SANITIZER_SELECTED_FLAGS)
+
+                if(AFL)
+                    append_quoteless(AFL_USE_MSAN=1 CMAKE_C_COMPILER_LAUNCHER
+                                                     CMAKE_CXX_COMPILER_LAUNCHER)
+                endif()
+            else()
+                message(
+                    FATAL_ERROR
+                        "Memory [With Origins] sanitizer not available for ${CMAKE_CXX_COMPILER}"
+                )
+            endif()
+        endif()
+
+        if(USE_SANITIZER MATCHES "([Uu]ndefined)")
+            message(STATUS "Testing with Undefined Behaviour sanitizer")
+            set(SANITIZER_UB_FLAG "-fsanitize=undefined")
+            if(EXISTS "${BLACKLIST_FILE}")
+                append("-fsanitize-blacklist=${BLACKLIST_FILE}" SANITIZER_UB_FLAG)
+            endif()
+            test_san_flags(SANITIZER_UB_AVAILABLE ${SANITIZER_UB_FLAG})
+            if(SANITIZER_UB_AVAILABLE)
+                message(STATUS "  Building with Undefined Behaviour sanitizer")
+                append("${SANITIZER_UB_FLAG}" SANITIZER_SELECTED_FLAGS)
+
+                if(AFL)
+                    append_quoteless(AFL_USE_UBSAN=1 CMAKE_C_COMPILER_LAUNCHER
+                                                     CMAKE_CXX_COMPILER_LAUNCHER)
+                endif()
+            else()
+                message(
+                    FATAL_ERROR
+                        "Undefined Behaviour sanitizer not available for ${CMAKE_CXX_COMPILER}"
+                )
+            endif()
+        endif()
+
+        if(USE_SANITIZER MATCHES "([Tt]hread)")
+            message(STATUS "Testing with Thread sanitizer")
+            set(SANITIZER_THREAD_FLAG "-fsanitize=thread -fsanitize-ignorelist=${CMAKE_SOURCE_DIR}/tsan_ignore.txt")
+            test_san_flags(SANITIZER_THREAD_AVAILABLE ${SANITIZER_THREAD_FLAG})
+            if(SANITIZER_THREAD_AVAILABLE)
+                message(STATUS "  Building with Thread sanitizer")
+                append("${SANITIZER_THREAD_FLAG}" SANITIZER_SELECTED_FLAGS)
+
+                if(AFL)
+                    append_quoteless(AFL_USE_TSAN=1 CMAKE_C_COMPILER_LAUNCHER
+                                                     CMAKE_CXX_COMPILER_LAUNCHER)
+                endif()
+            else()
+                message(
+                    FATAL_ERROR "Thread sanitizer not available for ${CMAKE_CXX_COMPILER}"
+                )
+            endif()
+        endif()
+
+        if(USE_SANITIZER MATCHES "([Ll]eak)")
+            message(STATUS "Testing with Leak sanitizer")
+            set(SANITIZER_LEAK_FLAG "-fsanitize=leak")
+            test_san_flags(SANITIZER_LEAK_AVAILABLE ${SANITIZER_LEAK_FLAG})
+            if(SANITIZER_LEAK_AVAILABLE)
+                message(STATUS "  Building with Leak sanitizer")
+                append("${SANITIZER_LEAK_FLAG}" SANITIZER_SELECTED_FLAGS)
+
+                if(AFL)
+                    append_quoteless(AFL_USE_LSAN=1 CMAKE_C_COMPILER_LAUNCHER
+                                                     CMAKE_CXX_COMPILER_LAUNCHER)
+                endif()
+            else()
+                message(
+                    FATAL_ERROR "Thread sanitizer not available for ${CMAKE_CXX_COMPILER}"
+                )
+            endif()
+        endif()
+
+        if(USE_SANITIZER MATCHES "([Cc][Ff][Ii])")
+            message(STATUS "Testing with Control Flow Integrity(CFI) sanitizer")
+            set(SANITIZER_CFI_FLAG "-fsanitize=cfi")
+            test_san_flags(SANITIZER_CFI_AVAILABLE ${SANITIZER_CFI_FLAG})
+            if(SANITIZER_CFI_AVAILABLE)
+                message(STATUS "  Building with Control Flow Integrity(CFI) sanitizer")
+                append("${SANITIZER_LEAK_FLAG}" SANITIZER_SELECTED_FLAGS)
+
+                if(AFL)
+                    append_quoteless(AFL_USE_CFISAN=1 CMAKE_C_COMPILER_LAUNCHER
+                                                     CMAKE_CXX_COMPILER_LAUNCHER)
+                endif()
+            else()
+                message(
+                    FATAL_ERROR
+                        "Control Flow Integrity(CFI) sanitizer not available for ${CMAKE_CXX_COMPILER}"
+                )
+            endif()
+        endif()
+
+        message(STATUS "Sanitizer flags: ${SANITIZER_SELECTED_FLAGS}")
+        test_san_flags(SANITIZER_SELECTED_COMPATIBLE ${SANITIZER_SELECTED_FLAGS})
+        if(SANITIZER_SELECTED_COMPATIBLE)
+            message(STATUS " Building with ${SANITIZER_SELECTED_FLAGS}")
+            append("${SANITIZER_SELECTED_FLAGS}" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+        else()
+            message(
+                FATAL_ERROR
+                    " Sanitizer flags ${SANITIZER_SELECTED_FLAGS} are not compatible.")
+        endif()
+    elseif(MSVC)
+        if(USE_SANITIZER MATCHES "([Aa]ddress)")
+            message(STATUS "Building with Address sanitizer")
+            append("-fsanitize=address" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+
+            if(AFL)
+                append_quoteless(AFL_USE_ASAN=1 CMAKE_C_COMPILER_LAUNCHER
+                                                 CMAKE_CXX_COMPILER_LAUNCHER)
+            endif()
+        else()
+            message(
+                FATAL_ERROR
+                    "This sanitizer not yet supported in the MSVC environment: ${USE_SANITIZER}"
+            )
+        endif()
+    else()
+        message(FATAL_ERROR "USE_SANITIZER is not supported on this platform.")
+    endif()
+
+endif()

cmake/Target.cmake

@@ -0,0 +1,109 @@
+# Check if IPO is supported
+include(CheckIPOSupported)
+check_ipo_supported(RESULT HAVE_IPO)
+
+# Enable IPO in non-debug build
+macro(target_enable_ipo NAME)
+    if(NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" AND HAVE_IPO)
+        set_property(TARGET ${NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
+        message (STATUS "Enabled IPO for target: ${NAME}")
+    endif()
+endmacro()
+
+macro(target_add_lib NAME)
+    file(GLOB_RECURSE FILES CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.h")
+    add_library(${NAME} STATIC ${FILES} ${FBS_FILES})
+    target_include_directories(${NAME}
+        PUBLIC
+            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
+            ${PROJECT_SOURCE_DIR}
+            ${PROJECT_BINARY_DIR}/src
+            ${PROJECT_BINARY_DIR}
+    )
+    target_link_libraries(${NAME} ${ARGN} "")
+endmacro()
+
+macro(target_add_shared_lib NAME)
+    file(GLOB_RECURSE FILES CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.h")
+    add_library(${NAME} SHARED ${FILES} ${FBS_FILES})
+    target_include_directories(${NAME}
+        PUBLIC
+            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
+            ${PROJECT_SOURCE_DIR}
+            ${PROJECT_BINARY_DIR}/src
+            ${PROJECT_BINARY_DIR}
+    )
+    target_link_libraries(${NAME} ${ARGN} "")
+    target_enable_ipo(${NAME})
+endmacro()
+
+macro(target_add_bin NAME MAIN_FILE)
+  add_executable(${NAME} ${MAIN_FILE})
+    target_link_libraries(${NAME} ${ARGN} "")
+    target_include_directories(${NAME}
+        PUBLIC
+            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
+            ${PROJECT_SOURCE_DIR}
+            ${PROJECT_SOURCE_DIR}/src/lib/api
+            ${PROJECT_BINARY_DIR}/src
+            ${PROJECT_BINARY_DIR}
+    )
+    set_target_properties(${NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+    target_enable_ipo(${NAME})
+endmacro()
+
+macro(target_add_test NAME)
+    file(GLOB_RECURSE FILES CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
+    add_executable(${NAME} ${FILES})
+    target_link_libraries(${NAME} gmock test_main ${ARGN} "")
+    target_include_directories(${NAME}
+        PUBLIC
+            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
+            ${PROJECT_SOURCE_DIR}
+            ${PROJECT_BINARY_DIR}/src
+            ${PROJECT_BINARY_DIR}
+    )
+    add_test(NAME ${NAME} COMMAND ${NAME})
+    set_target_properties(${NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests")
+endmacro()
+
+macro(target_add_fbs NAME PATH)
+    cmake_parse_arguments(FBS "SERVICE" "" "DEPS" ${ARGN})
+
+    message("Fbs " ${NAME} " FBS_SERVICE " ${FBS_SERVICE} " ARGN " ${ARGN})
+  
+    set(FLATBUFFERS_FLATC_SCHEMA_EXTRA_ARGS
+        --scoped-enums
+        --gen-object-api
+        --gen-mutable
+        --gen-compare
+        --cpp-std=c++17
+        --python
+        --hf3fs
+        --keep-prefix
+    )
+
+    get_filename_component(NAME_WE ${PATH} NAME_WE)
+    get_filename_component(DIR ${PATH} DIRECTORY)
+
+    build_flatbuffers(${PATH}
+        "${CMAKE_CURRENT_SOURCE_DIR}/${DIR};${CMAKE_SOURCE_DIR}/src"
+        "${NAME_WE}-generated"
+        ""
+        "${CMAKE_CURRENT_BINARY_DIR}/${DIR}"
+        ""
+        ""
+        )
+
+      add_library(${NAME} INTERFACE)
+      target_link_libraries(${NAME} INTERFACE common)
+      target_include_directories(${NAME}
+          INTERFACE
+              $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
+              ${PROJECT_SOURCE_DIR}
+              ${PROJECT_BINARY_DIR}/src
+              ${PROJECT_BINARY_DIR}
+      )
+
+    add_dependencies(${NAME} "${NAME_WE}-generated" ${FBS_DEPS})
+endmacro()