DeepSeek/3FS
1
0
Fork 0
mirror of https://github.com/deepseek-ai/3FS synced 2025-06-26 18:16:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
dev
2025-02-27 21:53:53 +08:00
commit 815e55e4c0
1291 changed files with 185445 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

338
configs/admin_cli.toml Normal file
View File

@@ -0,0 +1,338 @@
break_multi_line_command_on_failure = false
cluster_id = ''
log = 'DBG:normal; normal=file:path=/var/log/3fs/cli.log,async=true,sync_level=ERR'
num_timeout_ms = 1000
profile = false
verbose = false
[client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[client.io_worker.transport_pool]
max_connections = 1
[client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[client.rdma_control]
max_concurrent_transmission = 64
[client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[fdb]
casual_read_risky = false
clusterFile = ''
default_backoff = 0
enableMultipleClient = false
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'
[ib_devices]
allow_no_usable_devices = false
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
default_pkey_index = 0
default_roce_pkey_index = 0
default_traffic_class = 0
device_filter = []
fork_safe = true
prefer_ibdevice = true
skip_inactive_ports = true
skip_unusable_device = true
subnets = []
[meta_client]
check_server_interval = '5s'
dynamic_stripe = false
max_concurrent_requests = 128
network_type = 'RDMA'
remove_chunks_batch_size = 32
remove_chunks_max_iters = 1024
selection_mode = 'RandomFollow'
[meta_client.background_closer]
prune_session_batch_count = 128
prune_session_batch_interval = '10s'
retry_first_wait = '100ms'
retry_max_wait = '10s'
task_scan = '50ms'
[meta_client.background_closer.coroutine_pool]
coroutines_num = 8
enable_work_stealing = false
queue_size = 128
[meta_client.retry_default]
max_failures_before_failover = 1
retry_fast = '1s'
retry_init_wait = '500ms'
retry_max_wait = '5s'
retry_send = 1
retry_total_time = '1min'
rpc_timeout = '5s'
[mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '1s'
enable_auto_extend_client_session = false
enable_auto_heartbeat = false
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100
[monitor]
collect_period = '1s'
num_collectors = 1
reporters = []
[storage_client]
check_overlapping_read_buffers = true
check_overlapping_write_buffers = false
chunk_checksum_type = 'CRC32C'
create_net_client_for_updates = false
implementation_type = 'RPC'
max_inline_read_bytes = '0'
max_inline_write_bytes = '0'
max_read_io_bytes = '0'
[storage_client.net_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[storage_client.net_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[storage_client.net_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[storage_client.net_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[storage_client.net_client.io_worker.transport_pool]
max_connections = 1
[storage_client.net_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[storage_client.net_client.rdma_control]
max_concurrent_transmission = 64
[storage_client.net_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[storage_client.net_client_for_updates]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[storage_client.net_client_for_updates.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[storage_client.net_client_for_updates.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[storage_client.net_client_for_updates.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[storage_client.net_client_for_updates.io_worker.transport_pool]
max_connections = 1
[storage_client.net_client_for_updates.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[storage_client.net_client_for_updates.rdma_control]
max_concurrent_transmission = 64
[storage_client.net_client_for_updates.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[storage_client.retry]
init_wait_time = '10s'
max_failures_before_failover = 1
max_retry_time = '1min'
max_wait_time = '30s'
[storage_client.traffic_control.query]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage_client.traffic_control.read]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage_client.traffic_control.remove]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage_client.traffic_control.truncate]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage_client.traffic_control.write]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[user_info]
gid = -1
gids = []
token = ''
uid = -1

557
configs/hf3fs_client_agent.toml Normal file
View File

@@ -0,0 +1,557 @@
[[common.log.categories]]
categories = [ '.' ]
handlers = [ 'normal', 'err', 'fatal' ]
inherit = true
level = 'INFO'
propagate = 'NONE'
[[common.log.handlers]]
async = true
file_path = ''
max_file_size = '10MB'
max_files = 100
name = 'normal'
rotate = true
rotate_on_open = false
start_level = 'NONE'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = ''
max_file_size = '10MB'
max_files = 100
name = 'err'
rotate = true
rotate_on_open = false
start_level = 'ERR'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = ''
max_file_size = '10MB'
max_files = 100
name = 'fatal'
rotate = true
rotate_on_open = false
start_level = 'FATAL'
stream_type = 'STDERR'
writer_type = 'STREAM'
[common.memory]
prof_active = false
prof_prefix = ''
[common.monitor]
reporters = []
[server.agent]
allow_read_holes = true
auth_timeout = '5min'
background_ibreg = true
list_entry_limit = 100
max_concurrent_iovallocs = 0
mock_storage_dir = ''
mount_name = ''
read_only_mode = false
truncate_if_write_after_eof = false
[server.agent.inode_cache]
capacity = 4194304
entry_lifetime = '5min'
[server.agent.limit_per_process]
fd = 1048576
shm = 1048576
[server.agent.periodic_sync]
interval = '10min'
on = false
[server.agent.proc_watch]
interval = '1min'
on = true
[server.agent.storage_io.read]
enableChecksum = false
[server.agent.storage_io.read.debug]
bypass_disk_io = false
bypass_rdma_xmit = false
inject_random_client_error = false
inject_random_server_error = false
max_num_of_injection_points = 100
[server.agent.storage_io.read.retry]
init_wait_time = '0ns'
max_retry_time = '0ns'
max_wait_time = '0ns'
retry_permanent_error = false
[server.agent.storage_io.read.targetSelection]
mode = 'Default'
targetIndex = 0
trafficZone = ''
[server.agent.storage_io.write]
enableChecksum = true
[server.agent.storage_io.write.debug]
bypass_disk_io = false
bypass_rdma_xmit = false
inject_random_client_error = false
inject_random_server_error = false
max_num_of_injection_points = 100
[server.agent.storage_io.write.retry]
init_wait_time = '0ns'
max_retry_time = '0ns'
max_wait_time = '0ns'
retry_permanent_error = false
[server.agent.storage_io.write.targetSelection]
mode = 'Default'
targetIndex = 0
trafficZone = ''
[server.agent.storage_ops]
enable_read = true
enable_write = true
[server.background_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.background_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.background_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.background_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.background_client.io_worker.transport_pool]
max_connections = 1
[server.background_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.background_client.rdma_control]
max_concurrent_transmission = 64
[server.background_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.independent_thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 64
num_proc_threads = 64
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'LOCAL'
services = [ 'ClientAgentSerde' ]
use_independent_thread_pool = false
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
filter_list = []
listen_port = 0
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'TCP'
services = [ 'Core' ]
use_independent_thread_pool = true
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
filter_list = []
listen_port = 9000
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.meta]
check_server_interval = '5s'
dynamic_stripe = false
max_concurrent_requests = 128
network_type = 'RDMA'
remove_chunks_batch_size = 32
remove_chunks_max_iters = 1024
selection_mode = 'RandomFollow'
[server.meta.background_closer]
prune_session_batch_count = 128
prune_session_batch_interval = '10s'
retry_first_wait = '100ms'
retry_max_wait = '10s'
task_scan = '50ms'
[server.meta.background_closer.coroutine_pool]
coroutines_num = 8
enable_work_stealing = false
queue_size = 128
[server.meta.retry_default]
max_failures_before_failover = 1
retry_init_wait = '500ms'
retry_max_wait = '5s'
retry_send = 1
retry_total_time = '1min'
rpc_timeout = '2s'
[server.meta.retry_truncate]
max_failures_before_failover = 1
retry_init_wait = '2s'
retry_max_wait = '5s'
retry_send = 1
retry_total_time = '1min'
rpc_timeout = '15s'
[server.mgmtd]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = true
enable_auto_heartbeat = false
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100
[server.storage]
check_overlapping_read_buffers = true
check_overlapping_write_buffers = false
chunk_checksum_type = 'CRC32C'
create_net_client_for_updates = false
implementation_type = 'RPC'
max_inline_read_bytes = '0'
max_inline_write_bytes = '0'
max_read_io_bytes = '0'
[server.storage.net_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.storage.net_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.storage.net_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.storage.net_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.storage.net_client.io_worker.transport_pool]
max_connections = 1
[server.storage.net_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.storage.net_client.rdma_control]
max_concurrent_transmission = 64
[server.storage.net_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.storage.net_client_for_updates]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.storage.net_client_for_updates.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.storage.net_client_for_updates.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.storage.net_client_for_updates.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.storage.net_client_for_updates.io_worker.transport_pool]
max_connections = 1
[server.storage.net_client_for_updates.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.storage.net_client_for_updates.rdma_control]
max_concurrent_transmission = 64
[server.storage.net_client_for_updates.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.storage.retry]
init_wait_time = '10s'
max_failures_before_failover = 1
max_retry_time = '1min'
max_wait_time = '30s'
[server.storage.traffic_control.query]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage.traffic_control.read]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage.traffic_control.remove]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage.traffic_control.truncate]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage.traffic_control.write]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true

1
configs/hf3fs_client_agent_app.toml Normal file
View File

@@ -0,0 +1 @@

88
configs/hf3fs_client_agent_launcher.toml Normal file
View File

@@ -0,0 +1,88 @@
allow_dev_version = true
cluster_id = ''
[client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[client.io_worker.transport_pool]
max_connections = 1
[client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[client.rdma_control]
max_concurrent_transmission = 64
[client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[ib_devices]
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
default_pkey_index = 0
default_traffic_class = 0
device_filter = []
fork_safe = true
skip_inactive_ports = true
subnets = []
[mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = true
enable_auto_heartbeat = false
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100

450
configs/hf3fs_fuse_main.toml Normal file
View File

@@ -0,0 +1,450 @@
attr_timeout = 30.0
batch_io_coros = 128
check_rmrf = true
chunk_size_limit = '0'
dryrun_bench_mode = false
enable_interrupt = false
enable_priority = false
enable_read_cache = true
enable_writeback_cache = false
entry_timeout = 30.0
fdatasync_update_length = false
flush_on_stat = true
fsync_length_hint = false
io_job_deq_timeout = '1ms'
io_jobq_size = 1024
iov_limit = '1MB'
max_background = 32
max_idle_threads = 10
max_jobs_per_ioring = 32
max_readahead = '16MB'
max_threads = 256
max_uid = '1M'
memset_before_read = false
negative_timeout = 5.0
notify_inval_threads = 32
rdma_buf_pool_size = 1024
readonly = false
submit_wait_jitter = '1ms'
symlink_timeout = 5.0
sync_on_stat = true
time_granularity = '1s'
[client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[client.io_worker.transport_pool]
max_connections = 1
[client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[client.rdma_control]
max_concurrent_transmission = 64
[client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[[common.log.categories]]
categories = [ '.' ]
handlers = [ 'normal', 'err', 'fatal' ]
inherit = true
level = 'INFO'
propagate = 'NONE'
[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/hf3fs_fuse_main.log'
max_file_size = '100MB'
max_files = 10
name = 'normal'
rotate = true
rotate_on_open = false
start_level = 'NONE'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/hf3fs_fuse_main-err.log'
max_file_size = '100MB'
max_files = 10
name = 'err'
rotate = true
rotate_on_open = false
start_level = 'ERR'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/hf3fs_fuse_main-fatal.log'
max_file_size = '100MB'
max_files = 10
name = 'fatal'
rotate = true
rotate_on_open = false
start_level = 'FATAL'
stream_type = 'STDERR'
writer_type = 'STREAM'
[common.memory]
prof_active = false
prof_prefix = ''
[common.monitor]
collect_period = '1s'
num_collectors = 1
[[common.monitor.reporters]]
type = 'monitor_collector'
[common.monitor.reporters.monitor_collector]
remote_ip = ''
[io_bufs]
max_buf_size = '1MB'
max_readahead = '256KB'
write_buf_size = '1MB'
[io_jobq_sizes]
hi = 32
lo = 4096
[io_worker_coros]
hi = 8
lo = 8
[meta]
check_server_interval = '5s'
dynamic_stripe = true
max_concurrent_requests = 128
network_type = 'RDMA'
remove_chunks_batch_size = 32
remove_chunks_max_iters = 1024
selection_mode = 'RandomFollow'
[meta.background_closer]
prune_session_batch_count = 128
prune_session_batch_interval = '10s'
retry_first_wait = '100ms'
retry_max_wait = '10s'
task_scan = '50ms'
[meta.background_closer.coroutine_pool]
coroutines_num = 8
enable_work_stealing = false
queue_size = 128
[meta.retry_default]
max_failures_before_failover = 1
retry_fast = '1s'
retry_init_wait = '500ms'
retry_max_wait = '5s'
retry_send = 1
retry_total_time = '1min'
rpc_timeout = '5s'
[mgmtd]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = true
enable_auto_heartbeat = false
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100
[periodic_sync]
enable = true
flush_write_buf = true
interval = '30s'
limit = 1000
[periodic_sync.worker]
coroutines_num = 4
enable_work_stealing = false
queue_size = 1024
[storage]
check_overlapping_read_buffers = true
check_overlapping_write_buffers = false
chunk_checksum_type = 'CRC32C'
create_net_client_for_updates = false
implementation_type = 'RPC'
max_inline_read_bytes = '0'
max_inline_write_bytes = '0'
max_read_io_bytes = '0'
[storage.net_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[storage.net_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[storage.net_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[storage.net_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[storage.net_client.io_worker.transport_pool]
max_connections = 1
[storage.net_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[storage.net_client.rdma_control]
max_concurrent_transmission = 64
[storage.net_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[storage.net_client_for_updates]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[storage.net_client_for_updates.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[storage.net_client_for_updates.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[storage.net_client_for_updates.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[storage.net_client_for_updates.io_worker.transport_pool]
max_connections = 1
[storage.net_client_for_updates.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[storage.net_client_for_updates.rdma_control]
max_concurrent_transmission = 64
[storage.net_client_for_updates.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[storage.retry]
init_wait_time = '10s'
max_failures_before_failover = 1
max_retry_time = '1min'
max_wait_time = '30s'
[storage.traffic_control.query]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage.traffic_control.read]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage.traffic_control.remove]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage.traffic_control.truncate]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage.traffic_control.write]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[storage_io.read]
allowReadUncommitted = false
enableChecksum = false
[storage_io.read.debug]
bypass_disk_io = false
bypass_rdma_xmit = false
inject_random_client_error = false
inject_random_server_error = false
max_num_of_injection_points = 100
[storage_io.read.retry]
init_wait_time = '0ns'
max_retry_time = '0ns'
max_wait_time = '0ns'
retry_permanent_error = false
[storage_io.read.targetSelection]
mode = 'Default'
targetIndex = 0
trafficZone = ''
[storage_io.write]
enableChecksum = true
[storage_io.write.debug]
bypass_disk_io = false
bypass_rdma_xmit = false
inject_random_client_error = false
inject_random_server_error = false
max_num_of_injection_points = 100
[storage_io.write.retry]
init_wait_time = '0ns'
max_retry_time = '0ns'
max_wait_time = '0ns'
retry_permanent_error = false
[storage_io.write.targetSelection]
mode = 'Default'
targetIndex = 0
trafficZone = ''

1
configs/hf3fs_fuse_main_app.toml Normal file
View File

@@ -0,0 +1 @@

95
configs/hf3fs_fuse_main_launcher.toml Normal file
View File

@@ -0,0 +1,95 @@
allow_other = true
cluster_id = ''
mountpoint = ''
token_file = ''
[client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[client.io_worker.transport_pool]
max_connections = 1
[client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[client.rdma_control]
max_concurrent_transmission = 64
[client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[ib_devices]
allow_no_usable_devices = false
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
default_pkey_index = 0
default_roce_pkey_index = 0
default_traffic_class = 0
device_filter = []
fork_safe = true
prefer_ibdevice = true
skip_inactive_ports = true
skip_unusable_device = true
subnets = []
[mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = true
enable_auto_heartbeat = false
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100

633
configs/meta_main.toml Normal file
View File

@@ -0,0 +1,633 @@
[[common.log.categories]]
categories = [ '.' ]
handlers = [ 'normal', 'err', 'fatal' ]
inherit = true
level = 'INFO'
propagate = 'NONE'
[[common.log.categories]]
categories = [ 'eventlog' ]
handlers = [ 'event' ]
inherit = false
level = 'INFO'
propagate = 'ERR'
[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/hf3fs_meta_main.log'
max_file_size = '100MB'
max_files = 10
name = 'normal'
rotate = true
rotate_on_open = false
start_level = 'NONE'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/hf3fs_meta_main-err.log'
max_file_size = '100MB'
max_files = 10
name = 'err'
rotate = true
rotate_on_open = false
start_level = 'ERR'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/hf3fs_meta_main-fatal.log'
max_file_size = '100MB'
max_files = 10
name = 'fatal'
rotate = true
rotate_on_open = false
start_level = 'FATAL'
stream_type = 'STDERR'
writer_type = 'STREAM'
[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/hf3fs_meta_main-event.log'
max_file_size = '100MB'
max_files = 10
name = 'event'
rotate = true
rotate_on_open = false
start_level = 'INFO'
stream_type = 'STDERR'
writer_type = 'EVENT'
[common.memory]
prof_active = false
prof_prefix = ''
[common.monitor]
collect_period = '1s'
num_collectors = 1
[[common.monitor.reporters]]
type = 'monitor_collector'
[common.monitor.reporters.monitor_collector]
remote_ip = ''
[server]
use_memkv = false
[server.background_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.background_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.background_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.background_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.background_client.io_worker.transport_pool]
max_connections = 1
[server.background_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.background_client.rdma_control]
max_concurrent_transmission = 64
[server.background_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.independent_thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'RDMA'
services = [ 'MetaSerde' ]
use_independent_thread_pool = false
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 8001
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'TCP'
services = [ 'Core' ]
use_independent_thread_pool = true
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 9001
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.fdb]
casual_read_risky = false
clusterFile = ''
default_backoff = 0
enableMultipleClient = false
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'
[server.kv_engine]
use_memkv = false
[server.kv_engine.fdb]
casual_read_risky = false
clusterFile = ''
default_backoff = 0
enableMultipleClient = false
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'
[server.meta]
acl_cache_time = '15s'
allow_directly_move_to_trash = false
allow_owner_change_immutable = false
allow_stat_deleted_inodes = true
authenticate = false
batch_stat_by_path_concurrent = 4
batch_stat_concurrent = 8
check_file_hole = false
dynamic_stripe = false
dynamic_stripe_growth = 2
dynamic_stripe_initial = 16
enable_new_chunk_engine = false
grv_cache = false
idempotent_record_clean = '1min'
idempotent_record_expire = '30min'
idempotent_remove = true
idempotent_rename = false
iflags_chain_allocation = false
iflags_chunk_engine = true
ignore_length_hint = false
inodeId_abort_on_duplicate = false
inodeId_check_unique = true
list_default_limit = 128
max_batch_operations = 4096
max_directory_depth = 64
max_remove_chunks_per_request = 32
max_symlink_count = 10
max_symlink_depth = 4
operation_timeout = '5s'
otrunc_replace_file = true
otrunc_replace_file_threshold = '1GB'
readonly = false
recursive_remove_check_owner = true
recursive_remove_perm_check = 1024
statfs_cache_time = '1min'
statfs_space_imbalance_threshold = 5
statfs_update_interval = '5s'
sync_on_prune_session = false
time_granularity = '1s'
[server.meta.background_hole_checker]
coroutines_num = 16
enable_work_stealing = false
queue_size = 4096
[server.meta.distributor]
timeout = '30s'
update_interval = '1s'
[server.meta.event_trace_log]
dump_interval = '30s'
enabled = true
max_num_writers = 1
max_row_group_length = 100000
trace_file_dir = '.'
[server.meta.forward]
addr_type = 'RDMA'
debug = true
timeout = '10s'
[server.meta.gc]
check_session = true
distributed_gc = true
enable = true
gc_delay_free_space_threshold = 5
gc_directory_concurrent = 4
gc_directory_delay = '0ns'
gc_directory_entry_batch = 32
gc_directory_entry_concurrent = 4
gc_file_concurrent = 32
gc_file_delay = '5min'
large_file_chunks = 128
recursive_perm_check = true
remove_chunks_batch_size = 32
retry_delay = '10min'
scan_batch = 4096
scan_interval = '200ms'
small_file_chunks = 32
txn_low_priority = false
[server.meta.gc.retry_remove_chunks]
init_wait_time = '10s'
max_retry_time = '30s'
max_wait_time = '10s'
retry_permanent_error = false
[server.meta.gc.workers]
coroutines_num = 8
enable_work_stealing = false
queue_size = 1024
[server.meta.retry_remove_chunks]
init_wait_time = '10s'
max_retry_time = '30s'
max_wait_time = '10s'
retry_permanent_error = false
[server.meta.retry_transaction]
max_backoff = '1s'
max_retry_count = 10
[server.meta.session_manager]
enable = true
scan_batch = 1024
scan_interval = '5min'
session_timeout = '5min'
sync_on_prune_session = false
[server.meta.session_manager.close_workers]
coroutines_num = 32
enable_work_stealing = false
queue_size = 1024
[server.meta.session_manager.scan_workers]
coroutines_num = 8
enable_work_stealing = false
queue_size = 128
[server.meta.user_cache]
buckets = 127
exist_ttl = '5min'
inexist_ttl = '10s'
[server.mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = false
enable_auto_heartbeat = true
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100
[server.storage_client]
check_overlapping_read_buffers = true
check_overlapping_write_buffers = false
chunk_checksum_type = 'CRC32C'
create_net_client_for_updates = false
implementation_type = 'RPC'
max_inline_read_bytes = '0'
max_inline_write_bytes = '0'
max_read_io_bytes = '0'
[server.storage_client.net_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.storage_client.net_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.storage_client.net_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.storage_client.net_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.storage_client.net_client.io_worker.transport_pool]
max_connections = 1
[server.storage_client.net_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.storage_client.net_client.rdma_control]
max_concurrent_transmission = 64
[server.storage_client.net_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.storage_client.net_client_for_updates]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.storage_client.net_client_for_updates.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.storage_client.net_client_for_updates.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.storage_client.net_client_for_updates.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.storage_client.net_client_for_updates.io_worker.transport_pool]
max_connections = 1
[server.storage_client.net_client_for_updates.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.storage_client.net_client_for_updates.rdma_control]
max_concurrent_transmission = 64
[server.storage_client.net_client_for_updates.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.storage_client.retry]
init_wait_time = '2s'
max_failures_before_failover = 1
max_retry_time = '5s'
max_wait_time = '5s'
[server.storage_client.traffic_control.query]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.read]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.remove]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.truncate]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.write]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true

2
configs/meta_main_app.toml Normal file
View File

@@ -0,0 +1,2 @@
allow_empty_node_id = true
node_id = 0

93
configs/meta_main_launcher.toml Normal file
View File

@@ -0,0 +1,93 @@
allow_dev_version = true
cluster_id = ''
[client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[client.io_worker.transport_pool]
max_connections = 1
[client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[client.rdma_control]
max_concurrent_transmission = 64
[client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[ib_devices]
allow_no_usable_devices = false
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
default_pkey_index = 0
default_roce_pkey_index = 0
default_traffic_class = 0
device_filter = []
fork_safe = true
prefer_ibdevice = true
skip_inactive_ports = true
skip_unusable_device = true
subnets = []
[mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = false
enable_auto_heartbeat = true
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100

232
configs/mgmtd_main.toml Normal file
View File

@@ -0,0 +1,232 @@
[[common.log.categories]]
categories = [ '.' ]
handlers = [ 'normal', 'err', 'fatal' ]
inherit = true
level = 'INFO'
propagate = 'NONE'
[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/mgmtd_main.log'
max_file_size = '100MB'
max_files = 10
name = 'normal'
rotate = true
rotate_on_open = false
start_level = 'NONE'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/mgmtd_main-err.log'
max_file_size = '100MB'
max_files = 10
name = 'err'
rotate = true
rotate_on_open = false
start_level = 'ERR'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/mgmtd_main-fatal.log'
max_file_size = '100MB'
max_files = 10
name = 'fatal'
rotate = true
rotate_on_open = false
start_level = 'FATAL'
stream_type = 'STDERR'
writer_type = 'STREAM'
[common.memory]
prof_active = false
prof_prefix = ''
[common.monitor]
collect_period = '1s'
num_collectors = 1
[[common.monitor.reporters]]
type = 'monitor_collector'
[common.monitor.reporters.monitor_collector]
remote_ip = ""
[server.base.independent_thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'RDMA'
services = [ 'Mgmtd' ]
use_independent_thread_pool = false
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 8000
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'TCP'
services = [ 'Core' ]
use_independent_thread_pool = true
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 9000
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.service]
allow_heartbeat_from_unregistered = true
authenticate = false
bootstrapping_length = '2min'
bump_routing_info_version_interval = '5s'
check_status_interval = '10s'
client_session_timeout = '20min'
enable_routinginfo_cache = true
extend_lease_check_release_version = true
extend_lease_interval = '10s'
heartbeat_fail_interval = '1min'
heartbeat_ignore_stale_targets = true
heartbeat_ignore_unknown_targets = false
heartbeat_timestamp_valid_window = '30s'
lease_length = '1min'
new_chain_bootstrap_interval = '2min'
only_accept_client_uuid = false
retry_times_on_txn_errors = -1
send_heartbeat = true
send_heartbeat_interval = '10s'
suspicious_lease_interval = '20s'
target_info_load_interval = '1s'
target_info_persist_batch = 1000
target_info_persist_interval = '1s'
try_adjust_target_order_as_preferred = false
update_chains_interval = '1s'
update_metrics_interval = '1s'
validate_lease_on_write = true
[server.service.retry_transaction]
max_backoff = '1s'
max_retry_count = 10
[server.service.user_cache]
buckets = 127
exist_ttl = '5min'
inexist_ttl = '10s'

2
configs/mgmtd_main_app.toml Normal file
View File

@@ -0,0 +1,2 @@
allow_empty_node_id = true
node_id = 0

44
configs/mgmtd_main_launcher.toml Normal file
View File

@@ -0,0 +1,44 @@
allow_dev_version = true
cluster_id = ''
use_memkv = false
[fdb]
casual_read_risky = false
clusterFile = ''
default_backoff = 0
enableMultipleClient = false
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'
[ib_devices]
allow_no_usable_devices = false
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
default_pkey_index = 0
default_roce_pkey_index = 0
default_traffic_class = 0
device_filter = []
fork_safe = true
prefer_ibdevice = true
skip_inactive_ports = true
skip_unusable_device = true
subnets = []
[kv_engine]
use_memkv = false
[kv_engine.fdb]
casual_read_risky = false
clusterFile = ''
default_backoff = 0
enableMultipleClient = false
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'

141
configs/monitor_collector_main.toml Normal file
View File

@@ -0,0 +1,141 @@
[common]
cluster_id = ''
[common.ib_devices]
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
device_filter = []
subnets = []
[[common.log.categories]]
categories = [ '.' ]
handlers = [ 'normal', 'err', 'fatal' ]
inherit = true
level = 'INFO'
propagate = 'NONE'
[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/monitor_collector_main.log'
max_file_size = '100MB'
max_files = 10
name = 'normal'
rotate = true
rotate_on_open = false
start_level = 'NONE'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/monitor_collector_main-err.log'
max_file_size = '100MB'
max_files = 10
name = 'err'
rotate = true
rotate_on_open = false
start_level = 'ERR'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/monitor_collector_main-fatal.log'
max_file_size = '100MB'
max_files = 10
name = 'fatal'
rotate = true
rotate_on_open = false
start_level = 'FATAL'
stream_type = 'STDERR'
writer_type = 'STREAM'
[server.base.independent_thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[[server.base.groups]]
#default_timeout = '1s'
#drop_connections_interval = '1h'
network_type = 'TCP'
services = [ 'MonitorCollector' ]
use_independent_thread_pool = false
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drop_connections = 0
event_ack_batch = 128
#gid_index = 0
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
pkey_index = 0
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
traffic_class = 0
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
filter_list = []
listen_port = 10000
listen_queue_depth = 4096
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
[server.monitor_collector]
batch_commit_size = 4096
conn_threads = 32
queue_capacity = 204800
[server.monitor_collector.reporter]
type = 'clickhouse'
[server.monitor_collector.reporter.clickhouse]
db = ''
host = ''
passwd = ''
port = ''
user = ''

509
configs/storage_main.toml Normal file
View File

@@ -0,0 +1,509 @@
[[common.log.categories]]
categories = [ '.' ]
handlers = [ 'normal', 'err', 'fatal' ]
inherit = true
level = 'INFO'
propagate = 'NONE'
[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/storage_main.log'
max_file_size = '100MB'
max_files = 10
name = 'normal'
rotate = true
rotate_on_open = false
start_level = 'NONE'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/storage_main-err.log'
max_file_size = '100MB'
max_files = 10
name = 'err'
rotate = true
rotate_on_open = false
start_level = 'ERR'
stream_type = 'STDERR'
writer_type = 'FILE'
[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/storage_main-fatal.log'
max_file_size = '100MB'
max_files = 10
name = 'fatal'
rotate = true
rotate_on_open = false
start_level = 'FATAL'
stream_type = 'STDERR'
writer_type = 'STREAM'
[common.memory]
prof_active = false
prof_prefix = ''
[common.monitor]
collect_period = '1s'
num_collectors = 1
[[common.monitor.reporters]]
type = 'monitor_collector'
[common.monitor.reporters.monitor_collector]
remote_ip = ""
[server]
speed_up_quit = true
use_coroutines_pool_read = true
use_coroutines_pool_update = true
[server.aio_read_worker]
enable_io_uring = true
inflight_control_offset = 128
ioengine = 'libaio'
max_events = 512
min_complete = 128
num_threads = 32
queue_size = 4096
wait_all_inflight = false
[server.allocate_worker]
max_remain_groups = 8
max_remain_ultra_groups = 4
max_reserved_chunks = '1GB'
min_remain_groups = 4
min_remain_ultra_groups = 0
[server.base.independent_thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 32
num_proc_threads = 32
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'RDMA'
services = [ 'StorageSerde' ]
use_independent_thread_pool = false
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 8000
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'TCP'
services = [ 'Core' ]
use_independent_thread_pool = true
[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 9000
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.buffer_pool]
big_rdmabuf_count = 64
big_rdmabuf_size = '64MB'
rdmabuf_count = 1024
rdmabuf_size = '4MB'
[server.check_worker]
disk_low_space_threshold = 0.95999999999999996
disk_reject_create_chunk_threshold = 0.97999999999999998
emergency_recycling_ratio = 0.94999999999999996
update_target_size_interval = '10s'
[server.client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.client.io_worker.transport_pool]
max_connections = 1
[server.client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.client.rdma_control]
max_concurrent_transmission = 64
[server.client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.coroutines_pool_default]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.coroutines_pool_read]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.coroutines_pool_sync]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.coroutines_pool_update]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.dump_worker]
dump_interval = '1day'
dump_root_path = ''
high_cpu_usage_threshold = 100
[server.forward_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[server.forward_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.forward_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.forward_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.forward_client.io_worker.transport_pool]
max_connections = 1
[server.forward_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[server.forward_client.rdma_control]
max_concurrent_transmission = 64
[server.forward_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.mgmtd]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = false
enable_auto_heartbeat = true
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100
[server.reliable_forwarding]
max_inline_forward_bytes = '0'
retry_first_wait = '100ms'
retry_max_wait = '1s'
retry_total_time = '1min'
[server.reliable_update]
clean_up_expired_clients = false
expired_clients_timeout = '1h'
[server.storage]
apply_transmission_before_getting_semaphore = true
batch_read_ignore_chain_version = false
batch_read_job_split_size = 1024
max_concurrent_rdma_reads = 256
max_concurrent_rdma_writes = 256
max_num_results_per_query = 100
post_buffer_per_bytes = '64KB'
rdma_transmission_req_timeout = '0ns'
read_only = false
[server.storage.event_trace_log]
dump_interval = '30s'
enabled = true
max_num_writers = 1
max_row_group_length = 100000
trace_file_dir = '.'
[server.storage.write_worker]
bg_num_threads = 8
num_threads = 32
queue_size = 4096
[server.sync_meta_kv_worker]
sync_meta_kv_interval = '1min'
[server.sync_worker]
batch_size = 16
full_sync_chains = []
full_sync_level = 'NONE'
num_channels = 1024
num_threads = 16
sync_start_timeout = '10s'
[server.sync_worker.batch_concurrency_limiter]
max_concurrency = 64
[server.sync_worker.pool]
coroutines_num = 64
enable_work_stealing = false
queue_size = 1024
[server.targets]
allow_disk_without_uuid = false
collect_all_fds = true
create_engine_path = true
space_info_cache_timeout = '5s'
target_num_per_path = 0
target_paths = []
[server.targets.storage_target]
force_persist = true
kv_path = ''
migrate_kv_store = false
mutex_num = 257
point_query_strategy = 'NONE'
[server.targets.storage_target.file_store]
preopen_chunk_size_list = []
[server.targets.storage_target.kv_store]
create_if_missing = false
integrate_leveldb_log = false
leveldb_block_cache_size = '8GB'
leveldb_iterator_fill_cache = true
leveldb_shared_block_cache = true
leveldb_sst_file_size = '16MB'
leveldb_write_buffer_size = '16MB'
rocksdb_avoid_flush_during_recovery = false
rocksdb_avoid_flush_during_shutdown = false
rocksdb_avoid_unnecessary_blocking_io = false
rocksdb_block_cache_size = '8GB'
rocksdb_block_size = '4KB'
rocksdb_bloom_filter_bits_per_key = 10
rocksdb_compression = 'kNoCompression'
rocksdb_enable_bloom_filter = true
rocksdb_enable_pipelined_write = false
rocksdb_enable_prefix_transform = true
rocksdb_keep_log_file_num = 10
rocksdb_level0_file_num_compaction_trigger = 4
rocksdb_lowest_used_cache_tier = 'kNonVolatileBlockTier'
rocksdb_max_manifest_file_size = '64MB'
rocksdb_num_levels = 7
rocksdb_prepopulate_block_cache = 'kDisable'
rocksdb_readahead_size = '2MB'
rocksdb_shared_block_cache = true
rocksdb_stats_dump_period = '2min'
rocksdb_target_file_size_base = '64MB'
rocksdb_target_file_size_multiplier = 1
rocksdb_threads_num = 8
rocksdb_unordered_write = false
rocksdb_wal_recovery_mode = 'kTolerateCorruptedTailRecords'
rocksdb_write_buffer_size = '16MB'
sync_when_write = true
type = 'LevelDB'
[server.targets.storage_target.meta_store]
allocate_size = '256MB'
punch_hole_batch_size = 16
recycle_batch_size = 256
removed_chunk_expiration_time = '3day'
removed_chunk_force_recycled_time = '1h'

2
configs/storage_main_app.toml Normal file
View File

@@ -0,0 +1,2 @@
allow_empty_node_id = true
node_id = 0

93
configs/storage_main_launcher.toml Normal file
View File

@@ -0,0 +1,93 @@
allow_dev_version = true
cluster_id = ''
[client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false
[client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[client.io_worker.transport_pool]
max_connections = 1
[client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'
[client.rdma_control]
max_concurrent_transmission = 64
[client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[ib_devices]
allow_no_usable_devices = false
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
default_pkey_index = 0
default_roce_pkey_index = 0
default_traffic_class = 0
device_filter = []
fork_safe = true
prefer_ibdevice = true
skip_inactive_ports = true
skip_unusable_device = true
subnets = []
[mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = false
enable_auto_heartbeat = true
enable_auto_refresh = true
mgmtd_server_addresses = []
work_queue_size = 100