18 KiB
Pandas¶
"high-performance, easy-to-use data structures and data analysis tools" https://pandas.pydata.org/
Pandas is typically imported as pd.
import pandas as pd
# We will start with the Iris dataset from a previous lecture
iris_dataset = {'sepal length (cm)': [5.1, 4.9, 4.7, 4.6, 5.0, 5.4, 4.6, 5.0, 4.4, 4.9, 5.4, 4.8, 4.8, 4.3, 5.8, 5.7, 5.4, 5.1, 5.7, 5.1, 5.4, 5.1, 4.6, 5.1, 4.8, 5.0, 5.0, 5.2, 5.2, 4.7, 4.8, 5.4, 5.2, 5.5, 4.9, 5.0, 5.5, 4.9, 4.4, 5.1, 5.0, 4.5, 4.4, 5.0, 5.1, 4.8, 5.1, 4.6, 5.3, 5.0, 7.0, 6.4, 6.9, 5.5, 6.5, 5.7, 6.3, 4.9, 6.6, 5.2, 5.0, 5.9, 6.0, 6.1, 5.6, 6.7, 5.6, 5.8, 6.2, 5.6, 5.9, 6.1, 6.3, 6.1, 6.4, 6.6, 6.8, 6.7, 6.0, 5.7, 5.5, 5.5, 5.8, 6.0, 5.4, 6.0, 6.7, 6.3, 5.6, 5.5, 5.5, 6.1, 5.8, 5.0, 5.6, 5.7, 5.7, 6.2, 5.1, 5.7, 6.3, 5.8, 7.1, 6.3, 6.5, 7.6, 4.9, 7.3, 6.7, 7.2, 6.5, 6.4, 6.8, 5.7, 5.8, 6.4, 6.5, 7.7, 7.7, 6.0, 6.9, 5.6, 7.7, 6.3, 6.7, 7.2, 6.2, 6.1, 6.4, 7.2, 7.4, 7.9, 6.4, 6.3, 6.1, 7.7, 6.3, 6.4, 6.0, 6.9, 6.7, 6.9, 5.8, 6.8, 6.7, 6.7, 6.3, 6.5, 6.2, 5.9], 'sepal width (cm)': [3.5, 3.0, 3.2, 3.1, 3.6, 3.9, 3.4, 3.4, 2.9, 3.1, 3.7, 3.4, 3.0, 3.0, 4.0, 4.4, 3.9, 3.5, 3.8, 3.8, 3.4, 3.7, 3.6, 3.3, 3.4, 3.0, 3.4, 3.5, 3.4, 3.2, 3.1, 3.4, 4.1, 4.2, 3.1, 3.2, 3.5, 3.6, 3.0, 3.4, 3.5, 2.3, 3.2, 3.5, 3.8, 3.0, 3.8, 3.2, 3.7, 3.3, 3.2, 3.2, 3.1, 2.3, 2.8, 2.8, 3.3, 2.4, 2.9, 2.7, 2.0, 3.0, 2.2, 2.9, 2.9, 3.1, 3.0, 2.7, 2.2, 2.5, 3.2, 2.8, 2.5, 2.8, 2.9, 3.0, 2.8, 3.0, 2.9, 2.6, 2.4, 2.4, 2.7, 2.7, 3.0, 3.4, 3.1, 2.3, 3.0, 2.5, 2.6, 3.0, 2.6, 2.3, 2.7, 3.0, 2.9, 2.9, 2.5, 2.8, 3.3, 2.7, 3.0, 2.9, 3.0, 3.0, 2.5, 2.9, 2.5, 3.6, 3.2, 2.7, 3.0, 2.5, 2.8, 3.2, 3.0, 3.8, 2.6, 2.2, 3.2, 2.8, 2.8, 2.7, 3.3, 3.2, 2.8, 3.0, 2.8, 3.0, 2.8, 3.8, 2.8, 2.8, 2.6, 3.0, 3.4, 3.1, 3.0, 3.1, 3.1, 3.1, 2.7, 3.2, 3.3, 3.0, 2.5, 3.0, 3.4, 3.0], 'petal length (cm)': [1.4, 1.4, 1.3, 1.5, 1.4, 1.7, 1.4, 1.5, 1.4, 1.5, 1.5, 1.6, 1.4, 1.1, 1.2, 1.5, 1.3, 1.4, 1.7, 1.5, 1.7, 1.5, 1.0, 1.7, 1.9, 1.6, 1.6, 1.5, 1.4, 1.6, 1.6, 1.5, 1.5, 1.4, 1.5, 1.2, 1.3, 1.4, 1.3, 1.5, 1.3, 1.3, 1.3, 1.6, 1.9, 1.4, 1.6, 1.4, 1.5, 1.4, 4.7, 4.5, 4.9, 4.0, 4.6, 4.5, 4.7, 3.3, 4.6, 3.9, 3.5, 4.2, 4.0, 4.7, 3.6, 4.4, 4.5, 4.1, 4.5, 3.9, 4.8, 4.0, 4.9, 4.7, 4.3, 4.4, 4.8, 5.0, 4.5, 3.5, 3.8, 3.7, 3.9, 5.1, 4.5, 4.5, 4.7, 4.4, 4.1, 4.0, 4.4, 4.6, 4.0, 3.3, 4.2, 4.2, 4.2, 4.3, 3.0, 4.1, 6.0, 5.1, 5.9, 5.6, 5.8, 6.6, 4.5, 6.3, 5.8, 6.1, 5.1, 5.3, 5.5, 5.0, 5.1, 5.3, 5.5, 6.7, 6.9, 5.0, 5.7, 4.9, 6.7, 4.9, 5.7, 6.0, 4.8, 4.9, 5.6, 5.8, 6.1, 6.4, 5.6, 5.1, 5.6, 6.1, 5.6, 5.5, 4.8, 5.4, 5.6, 5.1, 5.1, 5.9, 5.7, 5.2, 5.0, 5.2, 5.4, 5.1], 'petal width (cm)': [0.2, 0.2, 0.2, 0.2, 0.2, 0.4, 0.3, 0.2, 0.2, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.4, 0.4, 0.3, 0.3, 0.3, 0.2, 0.4, 0.2, 0.5, 0.2, 0.2, 0.4, 0.2, 0.2, 0.2, 0.2, 0.4, 0.1, 0.2, 0.2, 0.2, 0.2, 0.1, 0.2, 0.2, 0.3, 0.3, 0.2, 0.6, 0.4, 0.3, 0.2, 0.2, 0.2, 0.2, 1.4, 1.5, 1.5, 1.3, 1.5, 1.3, 1.6, 1.0, 1.3, 1.4, 1.0, 1.5, 1.0, 1.4, 1.3, 1.4, 1.5, 1.0, 1.5, 1.1, 1.8, 1.3, 1.5, 1.2, 1.3, 1.4, 1.4, 1.7, 1.5, 1.0, 1.1, 1.0, 1.2, 1.6, 1.5, 1.6, 1.5, 1.3, 1.3, 1.3, 1.2, 1.4, 1.2, 1.0, 1.3, 1.2, 1.3, 1.3, 1.1, 1.3, 2.5, 1.9, 2.1, 1.8, 2.2, 2.1, 1.7, 1.8, 1.8, 2.5, 2.0, 1.9, 2.1, 2.0, 2.4, 2.3, 1.8, 2.2, 2.3, 1.5, 2.3, 2.0, 2.0, 1.8, 2.1, 1.8, 1.8, 1.8, 2.1, 1.6, 1.9, 2.0, 2.2, 1.5, 1.4, 2.3, 2.4, 1.8, 1.8, 2.1, 2.4, 2.3, 1.9, 2.3, 2.5, 2.3, 1.9, 2.0, 2.3, 1.8], 'species': ['setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'setosa', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'versicolor', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica', 'virginica']}
pandas DataFrame¶
The primary interest in pandas is the DataFrame. A DataFrame is a type, conceptually related to a numpy array, for containing large amounts of data and operating efficiently on it. With DataFrames, however, there is typically more structure. A DataFrame is always two dimensional, with every element in a column having the same data type. There are multiple columns, each with a name and potentially different datatypes. The easiest way to think about a DataFrame is like a well-organized spreadsheet. Indeed, DataFrames are great for doing the kind of calculations you might do in spreadsheets.
Creation¶
One way to create a pandas DataFrame is by using its constructor, DataFrame(). If provided one argument, a dictionary, it will create a new DataFrame instance with a column from each item in the dict. The dict key becomes the column name and the dict value (a Python sequence) becomes are the column data values. Pandas will infer the datatype for the column. It is required that the length of all sequences in the dict are identical so that each column in the DataFrame has the same length.
As an example, let's load our Iris dataset.
df = pd.DataFrame(iris_dataset)
df
The head() and tail() methods both return dataframes which are a subset of the original dataframe, with the top and bottom rows, respectively:
df.head()
df.tail()
Note that jupyter and pandas work nicely together to give the nicely formatted output you see above. Here is plain print():
print(df.tail())
What's happening behind the scenes is that Pandas knows how to use the diplay() function from IPython.
from IPython.display import display
display(df.head())
The DataFrame.groupby() method¶
One of the most useful aspects of dataframes is the groupby() method, which returns an iterator that steps through the original dataframe by returning subsets (groups) which all have been selected based on a common value. An example will make this more clear.
Here we will step through our original dataframe grouping by species. The iterator from groupby() returns, on each iteration, a tuple of (group_value, group_data_frame). Let's look at this in action:
for species, gdf in df.groupby('species'):
print(f"species: Iris {species}")
display(gdf.head())
Let's take a closer look at this iteration aspect:
my_iter = df.groupby('species')
print(type(my_iter))
my_iter = df.groupby('species')
for x in my_iter:
# species = x[0]
# gdf = x[1]
species, gdf = x
# (species, gdf) = x
print(species, len(gdf))
for species, gdf in df.groupby('species'):
print(species, len(gdf))
for species, gdf in df.groupby('species'):
print(f"=============== {species} ============")
display(gdf)
df.describe()
for species, gdf in df.groupby('species'):
print(f"=============== {species} ============")
display(gdf.describe())
More about Pandas DataFrames¶
Let's get started by making a sample dataframe with fake data:
sample_df = pd.DataFrame({'number':[1,2,3,6,2,3,2,2,1,2], 'color':['blue','blue','red','red','red','blue','blue','red','green','yellow']})
display(sample_df)
Getting a Series from a DataFrame in Pandas¶
You can get a Series (a Pandas 1D array) from a dataframe column by indexing with the column name:
colors = sample_df['color']
display(colors)
Above we got a Series from a DataFrame column with dictionary-like get item using square brackets and a string with the name of the column. In addition to this approach, Pandas also has an ergonomic feature where columns with names that are valid Python can be used with a dot (.) as if they were variables (also called "attributes") of the DataFrame instance.
# This does the same thing as above because "color" is a valid Python attribute name.
colors = sample_df.color
display(colors)
type(colors)
A Series has many useful methods, such as .unique() and .mean()
colors.unique()
sample_df['number'].mean()
df = pd.read_csv('iris.csv')
df
df.head()
df['species'].unique()
Conditions and selecting part of the data from a DataFrame.¶
Let's consider an equality condition. Let's check every row to test if the 'color' column is equal to 'blue':
sample_df
sample_df['color']
sample_df['color']=='blue'
As usual, we can assign this result to a variable:
condition = sample_df['color']=='blue'
We can see the type of this result is another Series
type(condition)
One very useful thing in Pandas is to create a new DataFrame based on a condition from an old one. Let's make a new DataFrame from only the rows with a blue color:
condition = sample_df['color']=='blue'
blue_sample_df = sample_df[ condition ]
display(blue_sample_df)
This could of course be written so:
blue_sample_df = sample_df[ sample_df['color']=='blue' ]
display(blue_sample_df)
A key concept in Pandas is iterating over a dataframe, grouping by values in one (or more) columns¶
This allows doing a lot of powerful datascience work which requires nothing more than storing your data in a well-organized format. This of course has other advantages as well. Let's have a look at our groupby() example again:
df = pd.read_csv('iris.csv')
for species, gdf in df.groupby('species'):
print(f"=============== {species} ============")
display(gdf.describe())
import seaborn as sns
sns.stripplot(x="species", y="sepal_width", data=df);
sns.stripplot(x="species", y="sepal_width", data=pd.read_csv('iris.csv'));
Let's take a minute to look at the seaborn gallery.
And while we are at it, we should not forget the matplotlib gallery.