Configuration Reference

Master Configuration

# Identity
celeborn.master.host=<hostname>
celeborn.master.port=9097
celeborn.master.http.port=9098
# Slot assignment (roundrobin or loadaware)
celeborn.master.slot.assign.policy=loadaware
# Worker health management
celeborn.master.heartbeat.timeout=120s
celeborn.master.worker.timeout=600s

Worker Configuration

# Storage — multiple disks with disk type annotation
celeborn.worker.storage.dirs=/mnt/nvme1:disktype=SSD,/mnt/nvme2:disktype=SSD,/mnt/hdd1:disktype=HDD
celeborn.storage.availableTypes=SSD,HDD
# Memory (set in celeborn-env.sh)
CELEBORN_WORKER_MEMORY=2g
CELEBORN_WORKER_OFFHEAP_MEMORY=4g
# Flush buffer sizes
celeborn.worker.flusher.buffer.size=256k
celeborn.worker.flusher.threads=4
# Threading (increase for HDFS/S3 workloads)
celeborn.worker.sortPartition.threads=64
celeborn.worker.commitFiles.threads=128
celeborn.worker.commitFiles.timeout=240s

Client Configuration (Spark)

# Enable Celeborn shuffle manager
spark.shuffle.manager=org.apache.spark.shuffle.celeborn.SparkShuffleManager
spark.celeborn.master.endpoints=master1:9097,master2:9097,master3:9097
# Disable YARN ESS (required when using Celeborn)
spark.shuffle.service.enabled=false
# Serialization (required for Celeborn)
spark.serializer=org.apache.spark.serializer.KryoSerializer
# Replication (recommended for production)
spark.celeborn.client.push.replicate.enabled=true
# Writer mode: hash (moderate partitions) or sort (high partitions)
spark.celeborn.client.spark.shuffle.writer=hash
# Dynamic allocation (Spark 3.5+)
spark.dynamicAllocation.enabled=true
spark.dynamicAllocation.shuffleTracking.enabled=false
spark.shuffle.sort.io.plugin.class=org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO