Configuration Reference

Master Configuration

Bash

# Identity

celeborn.master.host=<hostname>

celeborn.master.port=9097

celeborn.master.http.port=9098

# Slot assignment (roundrobin or loadaware)

celeborn.master.slot.assign.policy=loadaware

# Worker health management

celeborn.master.heartbeat.timeout=120s

celeborn.master.worker.timeout=600s

Bash

# Storage — multiple disks with disk type annotation

celeborn.worker.storage.dirs=/mnt/nvme1:disktype=SSD,/mnt/nvme2:disktype=SSD,/mnt/hdd1:disktype=HDD

celeborn.storage.availableTypes=SSD,HDD

# Memory (set in celeborn-env.sh)

CELEBORN_WORKER_MEMORY=2g

CELEBORN_WORKER_OFFHEAP_MEMORY=4g

# Flush buffer sizes

celeborn.worker.flusher.buffer.size=256k

celeborn.worker.flusher.threads=4

# Threading (increase for HDFS/S3 workloads)

celeborn.worker.sortPartition.threads=64

celeborn.worker.commitFiles.threads=128

celeborn.worker.commitFiles.timeout=240s

Bash

# Enable Celeborn shuffle manager

spark.shuffle.manager=org.apache.spark.shuffle.celeborn.SparkShuffleManager

spark.celeborn.master.endpoints=master1:9097,master2:9097,master3:9097

# Disable YARN ESS (required when using Celeborn)

spark.shuffle.service.enabled=false

# Serialization (required for Celeborn)

spark.serializer=org.apache.spark.serializer.KryoSerializer

# Replication (recommended for production)

spark.celeborn.client.push.replicate.enabled=true

# Writer mode: hash (moderate partitions) or sort (high partitions)

spark.celeborn.client.spark.shuffle.writer=hash

# Dynamic allocation (Spark 3.5+)

spark.dynamicAllocation.enabled=true

spark.dynamicAllocation.shuffleTracking.enabled=false

spark.shuffle.sort.io.plugin.class=org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO