pandas remove duplicates

# To drop all duplicate rows:
df = df.drop_duplicates()

# To remove all rows which have a duplicate, i.e. if there are 
# two copies of a row, keep neither as opposed to one:
df = df.drop_duplicates(keep = False)

# To drop all but the first occurence of a row with each
# distinct age:
df = df.drop_duplicates(subset = 'Age', keep = 'first')

# To drop all but the last occurence of a row with each
# distinct age-height combination:
df = drop_duplicates(subset = ['Age','Height'], keep = 'last')

        

import pandas as pd 
  
# making data frame from csv file 
data = pd.read_csv("employees.csv") 
  
# sorting by first name 
data.sort_values("First Name", inplace = True) 
  
# dropping ALL duplicte values 
data.drop_duplicates(subset ="First Name",keep = False, inplace = True) 
  
# displaying data 
print(data)

df = df.loc[:,~df.columns.duplicated()]

df = df.drop_duplicates()
p

# Return a new DataFrame with duplicate rows removed

from pyspark.sql import Row
df = sc.parallelize([
  Row(name='Alice', age=5, height=80),
  Row(name='Alice', age=5, height=80),
  Row(name='Alice', age=10, height=80)]).toDF()
df.dropDuplicates().show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# |  5|    80|Alice|
# | 10|    80|Alice|
# +---+------+-----+

df.dropDuplicates(['name', 'height']).show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# |  5|    80|Alice|
# +---+------+-----+