Do you already know Python and work with Pandas? Do you work with Big Data? Then PySpark should be your friend! PySpark is a Python API for Spark which is a general-purpose distributed data processing engine. It does computations in a distributed manner which enables the ability to analyse a large amount of data in a short time.
|
#SPARK titanic_sp = spark.table("titanic_train")
|
#PANDAS titanic_pd = titanic_sp.select("*").toPandas()
|
#FromPdBackToSPARK pysparkDF2 = spark.createDataFrame(pandasDF)
|
|
titanic_sp.show()
|
display(titanic_pd)
|
|
titanic_sp.printSchema()
|
titanic_pd.info()
|
|
column_renamed=titanic_sp.withColumnRenamed("Name","PassengerName").columns
|
titanic_pd.rename(columns={'Name': 'PassengerName'}).columns
|
|
#SPARK print((titanic_sp.count(), len(titanic_sp.columns)))
|
#PANDAS titanic_pd.shape
|
|
flight_data = flight_data.drop(columns_to_drop, axis = 1)
|
flight_data = flight_data.drop(*columns_to_drop)
|
|
titanic_sp.select('Survived').distinct().show()
|
titanic_pd['Survived'].unique()
|
|
titanic_sp.columns
|
titanic_pd.columns
|
|
titanic_sp.dtypes
|
titanic_pd.dtypes
|
|
flight_data = flight_data.withColumn('dt_departure', f.to_timestamp(flight_data.departure_datetime, 'yyyy-MM-dd HHmm'))
|
flight_data['dt_departure'] = pd.to_datetime(flight_data['dt_departure_datetime'], format='%Y-%m-%d %H%M')
|
|
df.describe()
|
df.describe().show()
|
|
df.groupBy("Company").agg({'Sales':'sum'}).show()
|
pysparkDF.groupBy("gender") \ .agg(mean("age"),mean("salary"),max("salary")) \ .show()
|
|
df[df['species'].isin(['Chinstrap', 'Gentoo'])].show(5)
|
df[df['species'].isin(['Chinstrap', 'Gentoo'])].head()
|
|
df[df['species'].rlike('G.')].show(5)
|
df[df['species'].str.match('G.')].head()
|
|
df[df['flipper'].between(225,229)].show(5)
|
df[df['flipper'].between(225,229)].head()
|
|
df[df['mass'].isNull()].show(5)
|
df[df['mass'].isnull()].head()
|
|
df[(df['mass']<3400) & (df['sex']=='Male')].head()
|
df[(df['mass']<3400) & (df['sex']=='Male')].show(5)
|
|
df[~df['flipper'].between(225,229)].show(5)
|
df[~df['flipper'].between(225,229)].head()
|
|
flight_data['departure_delay_status'] = np.where(flight_data['departure_delay'] > 90, 'Heavy', 'Moderate')
|
flight_data = flight_data.withColumn('departure_delay_status', f.when(flight_data.departure_delay > 90, 'Heavy').otherwise('Moderate')
|
No comments yet.
Cheatography n. The art of mapping complex subjects into clear, instantly navigable reference guides.
