Custom Functions

SQL UDFs

-- Register a SQL UDF
CREATE FUNCTION square AS 'org.example.Square';
SELECT square(column_name) FROM table_name;

# Register via Python
class Square:
def evaluate(self, row):
return row[0] * row[0]
spark.udaf.register("square", Square())
spark.sql("SELECT square(value) FROM mytable").show()

Python UDFs (Standard)

from pyspark.sql.functions import udf
from pyspark.sql.types import DoubleType
@udf(returnType=DoubleType())
def square(x):
return float(x * x)
data = [("Alice", 3), ("Bob", 4), ("Charlie", 5)]
df = spark.createDataFrame(data, ["name", "value"])
df.withColumn("squared", square("value")).show()