Performance Tuning

Memory

# celeborn-env.sh
 
export CELEBORN_MASTER_MEMORY=4g
 
export CELEBORN_WORKER_MEMORY=2g
 
export CELEBORN_WORKER_OFFHEAP_MEMORY=8g
 
 
 
# JVM GC tuning
 
export CELEBORN_MASTER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200"
 
export CELEBORN_WORKER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:MaxDirectMemorySize=8g"

Buffer Tuning

celeborn.worker.flusher.buffer.size=256k # Local disk
 
celeborn.worker.flusher.hdfs.buffer.size=4m # HDFS (must be >= 4 MB)
 
celeborn.worker.flusher.s3.buffer.size=6m # S3

Threading

# Partition sorting and commit (increase for HDFS/S3)
 
celeborn.worker.sortPartition.threads=64
celeborn.worker.commitFiles.threads=128
 
 
 
# Network and RPC
 
celeborn.data.io.threads=32
celeborn.rpc.dispatcher.numThreads=32

Network

# High throughput: increase connections per peer
 
celeborn.data.io.numConnectionsPerPeer=16
celeborn.client.push.maxReqsInFlight=128
 
 
 
# Timeouts (increase for large shuffles or slow storage)
 
celeborn.rpc.askTimeout=240s
celeborn.worker.commitFiles.timeout=240s

Spark Client Tuning

# Hash writer: better for moderate partition counts
 
spark.celeborn.client.spark.shuffle.writer=hash
spark.celeborn.client.push.buffer.max.size=64k
 
# Sort writer: better for very high partition counts (> 100k)
spark.celeborn.client.spark.shuffle.writer=sort
 
# Disable local shuffle reader for consistent performance
spark.sql.adaptive.localShuffleReader.enabled=false