PySpark Fingertip Commands Cheat Sheet

df = spark.read.csv("file.csv", header=True)
df = spark.read.parquet("file.parquet", header=True)

df.withColumn("col1", col("col1").cast("double"))

df.printSchema()

df.select("col").distinct().count()

# Filter entries of age, only keep those records of
 which the values are >24 

  df.filter(df["age"]>24).show()

#Syntax
df.withColumnRenamed("old_name","new_name")

#Example
df = df.withColumnRenamed('CallNumber', 'PhoneNumber')

# Return first n rows
  df.head() 
# Return first row
  df.first() 
# Return the first n rows
  df.take(2) 
# Print the schema of df
  df.printSchema() 
# Print the (logical and physical) plans
 df.explain()
#Get All column names from DataFrame
  df.columns

# Get_row count
  rows = empDF.count()

# Get_columns count
  cols = len(empDF.columns)

df.select("col1","col2").show()

 # Select All columns 
df.select("*").show()

# Fill all null values with 0
  df.fillna(0)

#Fill specific columns with specified values
  df.fillna({col1: 0, "col2":"missing"})

#syntax:
  joined_df = df1.join(df2, on="key_column", how="inner")

#example
  joined_df = empDF.join(deptDF,empDF.emp_dept_id ==  deptDF.dept_id,"inner") \
     .show(truncate=False)

df.withColumn("new_col", col("col1") + col("col2"))

df.drop("col1")

#syntax
df.groupBy("col1").agg({"col2":"mean"})

#examples
df.groupBy("department") \
    .agg(sum("salary").alias("sum_salary"), \
         avg("salary").alias("avg_salary"), \
         sum("bonus").alias("sum_bonus"), \
         max("bonus").alias("max_bonus") \
     ) \
    .show(truncate=False)

spark.stop()