kafka参数配置

kafka参数配置

server.properties

############################# Server Basics #############################

# The id of the broker. This must be set to a unique integer for each broker.
broker.id=0341

############################# Socket Server Settings #############################

# The port the socket server listens on
port=9092

# Hostname the broker will bind to. If not set, the server will bind to all interfaces
host.name=ip
# 空连接的超时时间
connections.max.idle.ms=600000

# Hostname the broker will advertise to producers and consumers. If not set, it uses the
# value for "host.name" if configured.  Otherwise, it will use the value returned from
# java.net.InetAddress.getCanonicalHostName().
#advertised.host.name=<hostname routable by clients>

# The port to publish to ZooKeeper for clients to use. If this is not set,
# it will publish the same port that the broker binds to.
#advertised.port=<port accessible by clients>

# The number of threads handling network requests
num.network.threads=4
 
# The number of threads doing disk I/O
num.io.threads=16

#socket 连接缓存大小
# The send buffer (SO_SNDBUF) used by the socket server
socket.send.buffer.bytes=1048576
# The receive buffer (SO_RCVBUF) used by the socket server
socket.receive.buffer.bytes=1048576

#请求最大大小
# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600

#在queue停止读取新请求之前队列允许的最大请求数量
queued.max.requests=1000
#清除consumer历史获取request记录
fetch.purgatory.purge.interval.requests=1000
#清除producer历史生成request记录
producer.purgatory.purge.interval.requests=1000

############################# Replica Info ##########################
#复制线程数
num.replica.fetchers=2
#flower最大时间没有发出获取副本的相应，leader认为其挂了，将其从同步副本中删除
replica.lag.time.max.ms=10000
#flower最多条记录，同上
replica.lag.max.messages=4000
#复制数据到leader的请求超时时间
replica.socket.timeout.ms=30000
#复制数据的socket buffer大小
replica.socket.receive.buffer.bytes=65536
#备份时每次fetch的最大值
replica.fetch.max.bytes=1048576
#leader获得复制数据请求相应的最小byte
replica.fetch.min.bytes=1
#复制数据响应leader的最大时间
replica.fetch.wait.max.ms=500
#每个replica检查是否将最高水位进行固化的频率
replica.high.watermark.checkpoint.interval.ms=5000

############################# Log Basics #############################

# A comma seperated list of directories under which to store log files
log.dirs=/data/kafka_2.10-0.8.2.1/data,/data/kafka_2.10-0.8.2.1/data2,/data/kafka_2.10-0.8.2.1/data3

############################# Log Flush Policy #############################

log.flush.interval.ms=10000
log.flush.interval.messages=20000
log.flush.scheduler.interval.ms=2000
#log.flush.offset.checkpoint.interval.ms=60000

############################## Log Retention Policy ##############################

#log.retention.minutes=
#log.retention.bytes=1073741824
#默认topic保存时间
log.retention.hours=24
#log删除检查时间间隔
log.retention.check.interval.ms=300000
log.cleanup.policy=delete
#log.delete.delay.ms=60000

#每个log segment允许包含offset的最大大小
log.index.size.max.bytes=10485760
#每个fetch留给offset的大小，不用理会
log.index.interval.bytes=4096
#每个log segment的最大尺寸,和时长
log.segment.bytes=1073741824
log.roll.hours=168
#log.roll.jitter.{ms,hours}
log.segment.delete.delay.ms=60000

log.cleaner.enable=false
#log.cleaner.threads=1
#log.cleaner.io.max.bytes.per.second=None
#log.cleaner.dedupe.buffer.size=500*1024*1024
#log.cleaner.io.buffer.size=512*1024
#log.cleaner.io.buffer.load.factor=0.9
#log.cleaner.backoff.ms=15000
#log.cleaner.min.cleanable.ratio=0.5
#log.cleaner.delete.retention.ms=2 day

#################################### Zookeeper ###################################

zookeeper.connect=host34:2181,host36:2181,host38:2181/kafka-dirktest
zookeeper.session.timeout.ms=6000
zookeeper.connection.timeout.ms=6000
zookeeper.sync.time.ms=2000

################################## Other Settings ################################
#是否让producer建立topic
auto.create.topics.enable=true

default.replication.factor=2
num.partitions=4

#最小同步replica数
min.insync.replicas=2

message.max.bytes=1000000
#记录offset和file对应索引的元数据文件大小
offset.metadata.max.bytes=4096

controller.socket.timeout.ms=30000
#controller.message.queue.size=10

#关闭broker前尝试将leader partition迁移
controlled.shutdown.enable=true
controlled.shutdown.max.retries=3
controlled.shutdown.retry.backoff.ms=5000

auto.leader.rebalance.enable=false
#leader.imbalance.per.broker.percentage=10
#leader.imbalance.check.interval.seconds=300

#后台进程数
background.threads=10

num.recovery.threads.per.data.dir=1
#将不在isr的replica作为leader
unclean.leader.election.enable=false
#是否可以删除topic
delete.topic.enable=true


#offsets.topic.num.partitions
#offsets.topic.retention.minutes
#offsets.retention.check.interval.ms
#offsets.topic.replication.factor
#offsets.topic.segment.bytes
#offsets.load.buffer.size
#offsets.commit.required.acks
#offsets.commit.timeout.ms