Performance Tuning

Memory

# celeborn-env.sh
export CELEBORN_MASTER_MEMORY=4g
export CELEBORN_WORKER_MEMORY=2g
export CELEBORN_WORKER_OFFHEAP_MEMORY=8g
# JVM GC tuning
export CELEBORN_MASTER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200"
export CELEBORN_WORKER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:MaxDirectMemorySize=8g"

Buffer Tuning

celeborn.worker.flusher.buffer.size=256k # Local disk
celeborn.worker.flusher.hdfs.buffer.size=4m # HDFS (must be >= 4 MB)
celeborn.worker.flusher.s3.buffer.size=6m # S3

Threading

# Partition sorting and commit (increase for HDFS/S3)
celeborn.worker.sortPartition.threads=64
celeborn.worker.commitFiles.threads=128
# Network and RPC
celeborn.data.io.threads=32
celeborn.rpc.dispatcher.numThreads=32

Network

# High throughput: increase connections per peer
celeborn.data.io.numConnectionsPerPeer=16
celeborn.client.push.maxReqsInFlight=128
# Timeouts (increase for large shuffles or slow storage)
celeborn.rpc.askTimeout=240s
celeborn.worker.commitFiles.timeout=240s

Spark Client Tuning

# Hash writer: better for moderate partition counts
spark.celeborn.client.spark.shuffle.writer=hash
spark.celeborn.client.push.buffer.max.size=64k
# Sort writer: better for very high partition counts (> 100k)
spark.celeborn.client.spark.shuffle.writer=sort
# Disable local shuffle reader for consistent performance
spark.sql.adaptive.localShuffleReader.enabled=false