Pandas for Machine Learning & Deep Learning

## Numpy

import pandas as pd



# Create a Data Frame

data = {'Category':['A', 'B', 'C', 'A','B','C','C','A'],

        'Value':[10, 20, 40, 45, 50, 90,100, 200]}



df = pd.DataFrame(data)





# Grouping by 'category' and calculating the sum and mean

grouped = df.groupby('Category').agg({'Value': ['sum', 'mean']})



print("Grouped Data Frame with Aggregation: \n\n", grouped)

import pandas as pd



# Create dataframe

data = {

    'Category':['A','A','B','B'],

    'Values':[10, 20, 30,40]

}

df = pd.DataFrame(data)



# Grouping by 'category' and calculating the sum and mean

grouped_df = df.groupby('Category').agg({'Values': ['sum', 'mean','max']})



print("Grouped Data Frame with with multiple functions \n\n", grouped_df)

import pandas as pd



# Create Multi Index Dataframe

arrays = [['A', 'A', 'B', 'B'], [2020, 2021, 2022, 2023]]

index = pd.MultiIndex.from_arrays(arrays, names=('Category', 'Year'))

data = [100, 150, 200, 250]

df = pd.DataFrame(data, index=index, columns=['Sales'])



# Accessing data in a mutiindex Dataframe

sales_in_2020 = df.xs(2020, level='Year')



print("Sales Data in 2020: ", sales_in_2020)

import pandas as pd



# Create Dataframe

data = {'Name': ['Alice', 'Bob', 'Charlie'], 'Score':[85,90,95]}

df = pd.DataFrame(data)



# Applying a lambda function to modify scores

df['Adjusted Score'] = df['Score'].apply(lambda x: x+5 if x<90 else x)



print("Dataframe with Adjusted Scores :\n",df)

import pandas as pd



# Create two dataframe

df1 = pd.DataFrame({'ID':[1,2,3], 'Name':['Alice', 'Bob', 'Charlie']})

df2 = pd.DataFrame({'Emp_ID':[2,3,4], 'Department':['HR','Finance','IT']})



#Merging with different keys

merged_df = pd.merge(df1, df2, left_on='ID', right_on='Emp_ID', how='outer')



print("Merged Dataframe: \n", merged_df)

import pandas as pd

import numpy as np



# Creating dataframe with missing values

data = {'Name':['Alice', 'Bob', np.nan, 'David'], 'Age':[13,np.nan,32,20]}

df = pd.DataFrame(data)



# Filling Missing Values using a custom function

df['Name'].fillna('Unknown', inplace=True)

df['Age'].fillna(df['Age'].median(), inplace=True)



print("Dataframe after handling Missing values: \n", df)

import pandas as pd

import numpy as np



# Creating dataframe with missing values

data = {'Name':['Alice', 'Bob', np.nan, 'David'], 'Age':[13,np.nan,32,20]}

df = pd.DataFrame(data)



# Filling Missing Values using ffill & bfill

df_ffill = df.fillna(method = 'ffill')

df_bfill = df.fillna(method = 'bfill')



print("Dataframe with forward fill: \n", df_ffill)

print("Dataframe with backwark fill: \n", df_bfill)

import pandas as pd



# Create dataframe

data = {'Date':['2023-01-01', '2023-01-02','2023-01-01','2023-01-02'],

        'City':['NY', 'NY','LA', 'LA'],

        'Sales':[200,250,300,400]

        }



df = pd.DataFrame(data)



# Pivoting the Dataframe

pivot_df = df.pivot(index='Date', columns='City', values='Sales')



print('Pivoted Dataframe : \n', pivot_df)

import pandas as pd



# Create dataframe

data = {'Date':['2023-01-01', '2023-01-02','2023-01-01','2023-01-02'],

        'City':['NY', 'NY','LA', 'LA'],

        'Type': ['Online', 'Store', 'Online','Store'],

        'Sales':[200,250,300,400]

        }



df = pd.DataFrame(data)



# Pivoting to create a Dataframe with multi-index columns

pivot_df = df.pivot(index='Date', columns=['City','Type'], values='Sales')



print('Pivoted Dataframe with multi-index columns: \n', pivot_df)

import pandas as pd



# Create dataframe

data = {'ID':[1,2], 'Math':[90,79], 'Science':[85,88]}

df = pd.DataFrame(data)



# Melting the dataframe

melted_dataframe = pd.melt(df,id_vars=['ID'], value_vars=['Math', 'Science'], var_name='Subject', value_name='Score')



print("Melted Dataframe : \n", melted_dataframe)

import pandas as pd



# Create time Series Dataframe

date_range = pd.date_range(start='2023-01-01', periods=10, freq='D')

data = {'Sales':[100, 200, 150, 300, 250, 400, 300, 350, 300, 400]}

df = pd.DataFrame(data, index=date_range)



# Resampling to find weekly sales

weekly_sales = df.resample('W').sum()



print("Weekly Sales : \n", weekly_sales)

import pandas as pd



# Create time Series Dataframe

date_range = pd.date_range(start='2023-01-01', periods=10, freq='D')

data = {'Sales':[100, 200, 150, 300, 250, 400, 300, 350, 300, 400]}

df = pd.DataFrame(data, index=date_range)



# Resampling to find weekly sales

resampled_df = df.resample('W').agg({'Sales':['sum', 'mean', 'max']})



print("Resampled Dataframe with different aggregation : \n", resampled_df)

import pandas as pd



# Create dataframe

data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'], 'Score':[85, 80,40,70], 'Passed':[True, False, True, False]}

df = pd.DataFrame(data)



# Filtering with multiple conditions

filtered_df = df[(df['Score'] > 80) & (df['Passed'] == True)]



print("FIltered Dataframe : \n", filtered_df)

import pandas as pd



# Create dataframe

data = {'Name':['Alice', 'Bob', 'Charlie', 'David'], 'Score': [85, 70, 95,60]}

df = pd.DataFrame(data)



# Creating a new column with categorical data

df['Performance'] = pd.cut(df['Score'], bins=[0,70,90,100], labels=['Poor', 'Average', 'Excellent'])



print('Dataframe with categorical performance : \n', df)

import pandas as pd



# Create dataframe

data = {'Score': [85, 70, 95,60, 100, 90]}

df = pd.DataFrame(data)



# Binning the scores into categories

df['Grade'] = pd.cut(df['Score'], bins=[0,59,69,79,89,100], labels=['F', 'D', 'C', 'B','A'])



print('Dataframe with binned scores : \n', df)

import pandas as pd



# Create a time series dataframe

data_range = pd.date_range(start='2023-01-01', periods=10, freq='D')

data = {'Sales': [100, 200, 150, 300, 250, 400, 300, 350, 300, 400]}

df = pd.DataFrame(data, index= data_range)



# Calculating a rolling mean with a window of 3 days

df['Rolling Mean'] = df['Sales'].rolling(window=3).mean()



print("Dataframe with rolling mean : \n", df)

import pandas as pd



# Create Dataframe

data = {'Date': pd.date_range(start='2023-01-01', periods=5, freq='D'),

        'Temperature': [30, 32, 35,33, 31]}



df = pd.DataFrame(data)



# Shifting data by one to create a lag

df['Prev Day Temp'] = df['Temperature'].shift(1)



print('Dataframe with shifted data: \n', df)

import pandas as pd



# Create dataframe

data = {'Sales': [100, 200, 150, 300, 250]}

df = pd.DataFrame(data)



# Calculating cumulative sum and product

df['cumulative sum'] = df['Sales'].cumsum()

df['cumulative product'] = df['Sales'].cumprod()



print('DataFrame with cumulative operations:\n', df)

import pandas as pd



# Create two dataframe

df1 = pd.DataFrame({'ID':[1,2,3], 'Year':[2020,2021,2023], 'Name':['Alice','Bob','Charlie'] })

df2 = pd.DataFrame({'ID':[2,3,1], 'Year':[2023,2023,2020], 'Score':[88,92,75]})



# Merging on multiple columns

merged_df = pd.merge(df1,df2, on=['ID','Year'], how='inner')



print('Merged Dataframe on multiple columns : \n', merged_df)

import pandas as pd



# Create dataframe

data = {'Value': [10, 12, 14, 100, 15, 13, 12]}

df = pd.DataFrame(data)



# Indentifying outliers using the IQR method

Q1 = df['Value'].quantile(0.25)

Q3 = df['Value'].quantile(0.75)

IQR = Q3 - Q1

outliers = df[(df['Value'] < (Q1 - 1.5 * IQR)) | (df['Value'] > (Q3 + 1.5 * IQR))]



print('Outliers: \n', outliers)

import pandas as pd



# Create dataframe

data = {'City':['NY', 'LA', 'NY','SF','LA'], 'Year':[2020,2020,2021,2021,2020], 'Sales':[100, 150, 200, 250,300]}

df = pd.DataFrame(data)



# Create a pivot table with multiple aggregation

pivot_table = pd.pivot_table(df, values='Sales', index='City', columns='Year', aggfunc=['sum', 'mean'])



print('Pivot table with multiple aggregation: \n', pivot_table)

import pandas as pd



# Create dataframe

data = {'City':['NY', 'LA', 'NY','SF','LA'], 'Year':[2020,2020,2021,2021,2020], 'Sales':[100, 150, 200, 250,300]}

df = pd.DataFrame(data)



# Create a pivot table with multiple indexes

pivot_table = pd.pivot_table(df, values='Sales', index=['City', 'Year'], aggfunc='sum')



print('Pivot table with multiple indexes: \n', pivot_table)

import pandas as pd



# Create dataframe

data = {'Name': ['Alice', 'Bob', 'Charlie'], 'Department':['HR', 'IT', 'Finance']}

df = pd.DataFrame(data)



# Mapping department to codes

department_map = {'HR':1, 'IT':2, 'Finance':3}

df['Dept Code'] = df['Department'].map(department_map)



print("Dataframe with mapped values: \n", df)

iimport pandas as pd



# Create a dataframe with duplicate rows

data = {'ID':[1,2,2,3,4,4], 'Value': [10, 20,20,30,40,40]}

df = pd.DataFrame(data)



# Detecting duplicates

duplicates = df[df.duplicated()]



print('Duplicate Row : \n', duplicates)

import pandas as pd



# Create a dataframe with lists in a column

data = {'ID':[1,2], 'Hobbies':[['Reading', 'Swimming'], ['Running', 'Cycling'] ]}

df = pd.DataFrame(data)



# Explodeing the 'Hobbies' Column

exploded_df = df.explode('Hobbies')



print("Dataframe after exploring lists: \n", exploded_df)

import pandas as pd



# Create dataframe

data = {'Name' : ['Alice', 'Bob', 'Charlie', 'David'], 'Score': [85,90,78,92]}

df = pd.DataFrame(data)



# Ranking scores in descending order

df['Rank'] = df['Score'].rank(ascending=False)



print('Dataframe with ranked scores : \n', df)

import pandas as pd



# Create dataframe

data = {'A': [1,2,3], 'B':[4,5,6]}

df = pd.DataFrame(data)



# Applying an operation to each element in the Dataframe

df_transformed = df.applymap(lambda x:x**2)



print("Dataframe with squared values : \n", df_transformed)

import pandas as pd



# Create dataframe

data = {'A': [1,2,3], 'B':[4,5,6]}

df = pd.DataFrame(data)



# Applying a function to rows and columns using the axis parameter

row_sum = df.apply(lambda x: x.sum() , axis=1)

col_sum = df.apply(lambda x: x.sum() , axis=0)



print("Row wise sum : \n", row_sum)

print("Column wise sum : \n", col_sum)

import pandas as pd



# Create dataframe

data = {'City': ['NY', 'LA', 'SF', 'NY']}

df = pd.DataFrame(data)



# Creating dummy varibales

dummies = pd.get_dummies(df['City'], prefix='City')



print('Dummy varibales dataframe: \n', dummies)

import pandas as pd



# Create dataframe

data = {'Name' : ['Alice', 'Bob', 'Charlie', 'David'], 'Score': [85,90,78,92]}

df = pd.DataFrame(data)



# Use query for filtering

filtered_df = df.query('Score > 80')



print('Filtered dataframe using query : \n', filtered_df)

import pandas as pd



# Create dataframe

data = {'Name' : ['Alice', 'Bob', 'Charlie', 'David'], 'Score': [85,90,78,92]}

df = pd.DataFrame(data)



# Adding a new column using assign

df = df.assign(Grade= lambda x: ['A' if Score >= 90 else 'B' for Score in x['Score']])



print("Dataframe with assigned grade column : \n", df)

import pandas as pd



# Create dataframe

data = {

    'Category':['A','A','B','B'],

    'Value':[10, 20, 30,40]

}



df = pd.DataFrame(data)



# Applying a custom function with groupby

grouped_df = df.groupby('Category') ['Value'].apply(lambda x: x.max() - x.min())



print("Custom Aggregation result : \n", grouped_df)

import pandas as pd



# Create dataframe

data = {

    'Team':['A','A','B','B'],

    'Points':[10, 20, 30,40]

}

df = pd.DataFrame(data)



# Applying a function to grouped data

grouped_df = df.groupby('Team') ['Points'].transform(lambda x: x/x.max() )



print("Dataframe with applied functions to grouped data : \n", grouped_df)

import pandas as pd



# Create dataframe

data = {

    'A': range(1000),

    'B':range(1000, 2000)

}

df = pd.DataFrame(data)



# Displaying dataframe info and memory usage

df_info = df.info(memory_usage='deep')



print('Dataframe info and Memory usage : \n', df_info)

import pandas as pd



# Create dataframe

data = {'Name' : ['Alice', 'Bob', 'Charlie', 'David'], 'City': ['NY','LA','SF','NY']}

df = pd.DataFrame(data)



# filter with isin()

filtered_df = df[df['City'].isin(['NY','SF'])]



print("Filtered Dataframe with isin method: \n", filtered_df)

import pandas as pd



# Create dataframes

df1 = pd.DataFrame({

    'A':[1,2],

    'B':[3,4]

})



df2 = pd.DataFrame({

    'A': [5,6],

    'B': [7,8]

})



# Concatening Dataframes

concat_df = pd.concat([df1, df2], ignore_index=True)



print("COncatenated DataFrame : \n", concat_df)

import pandas as pd



# Create dataframe

data = {'Name' : ['Alice', 'Bob', 'Charlie', 'David'], 'Score': [85,90,78,92], 'Age': [25,37,35,21]}

df = pd.DataFrame(data)



# Sorting by multiple columns

sorted_df = df.sort_values(by=['Score', 'Age'], ascending=[False, True])



print("Sorted Dataframe by multiple columns: \n", sorted_df)

import pandas as pd



# Create dataframe

data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'], 'Score': [85, 90, 78, 92]}

df = pd.DataFrame(data)



# Applying styles to highlight scores above 80

styled_df = df.style.map(

    lambda x: 'background-color: yellow' if isinstance(x, int) and x > 80 else ''

)



# Save styled DataFrame to HTML

styled_df.to_html("styled_dataframe.html")



# Displaying the styled Dataframe

print("Styled dataframe with conditional formatting has been saved to 'styled_dataframe.html'.")

import pandas as pd



# Create a dataframe with nested lists

data = {

    'ID' : [1,2],

    'Hobbies':[['Reading', 'Swimming', 'Gaming'], ['Hiking', 'Drawing']]

}

df = pd.DataFrame(data)



# Exploding the 'Hobbies' column

exploded_df = df.explode('Hobbies')



print("Dataframe after exploding nested json columns:\n", exploded_df)

import pandas as pd



# Create dataframe

data = {'Name': ['Alice', 'Bob', 'Charlie', 'David'], 'Score': [85, 90, 78, 92]}

df = pd.DataFrame(data)



# Custom function for modifying the dataframe

def add_grade_column(df):

    df['Grade'] = df['Score'].apply(lambda x: 'A' if x >=90 else 'B')

    return df



# Using pipe() for method chaining

df = df.pipe(add_grade_column)



print("Dataframe after using pipe : \n", df)

import pandas as pd



# Create Multi Index Dataframe

arrays = [['A', 'A', 'B', 'B'], [2020, 2021, 2022, 2023]]

index = pd.MultiIndex.from_arrays(arrays, names=('Category', 'Year'))

data = [100, 150, 200, 250]

df = pd.DataFrame(data, index=index, columns=['Sales'])



# Slicing the  mutiindex Dataframe

sliced_df = df.loc['A']



print("Sliced  mutiindex Dataframe \n", sliced_df)

import pandas as pd



# Create a dataframe

data = {

    'Math': [90, 80],

    'Science':[85,88]

}

df = pd.DataFrame(data)



# Adding prefix to column names

df_prefixed = df.add_prefix('Grade_')



# Adding suffix to column names

df_suffixed = df.add_suffix('_Score')



print("Dataframe with Prefix :\n", df_prefixed)

print('\n')

print("Dataframe with Suffix :\n", df_suffixed)