# Set up packages for lecture. Don't worry about understanding this code,
# but make sure to run it if you're following along.
import numpy as np
import babypandas as bpd
import pandas as pd
from matplotlib_inline.backend_inline import set_matplotlib_formats
import matplotlib.pyplot as plt
set_matplotlib_formats("svg")
plt.style.use('ggplot')

plt.rcParams['figure.figsize'] = (10, 5)

np.set_printoptions(threshold=20, precision=2, suppress=True)
pd.set_option("display.max_rows", 7)
pd.set_option("display.max_columns", 8)
pd.set_option("display.precision", 2)

# Animations
from IPython.display import display
import ipywidgets as widgets

import warnings
warnings.filterwarnings('ignore')

# Demonstration code
def r_scatter(r):
    "Generate a scatter plot with a correlation approximately r"
    x = np.random.normal(0, 1, 1000)
    z = np.random.normal(0, 1, 1000)
    y = r * x + (np.sqrt(1 - r ** 2)) * z
    plt.scatter(x, y)
    plt.xlim(-4, 4)
    plt.ylim(-4, 4)
    
def show_scatter_grid():
    plt.subplots(1, 4, figsize=(10, 2))
    for i, r in enumerate([-1, -2/3, -1/3, 0]):
        plt.subplot(1, 4, i+1)
        r_scatter(r)
        plt.title(f'r = {np.round(r, 2)}')
    plt.show()
    plt.subplots(1, 4, figsize=(10, 2))
    for i, r in enumerate([1, 2/3, 1/3]):
        plt.subplot(1, 4, i+1)
        r_scatter(r)
        plt.title(f'r = {np.round(r, 2)}')
    plt.subplot(1, 4, 4)
    plt.axis('off')
    plt.show()

Lecture 24 – Correlation

DSC 10, Spring 2023

Announcements

• Lab 7 is due on Saturday 6/3 at 11:59PM.
• The Final Project is due on Tuesday 6/6 at 11:59PM.
  • Issues saving your Final Project notebook? Watch this video!

Agenda

• Association.
• Correlation.
• Regression.

Remember to review the end of Lecture 23 for a high-level summary of the second half of the class so far.

Association

Prediction

• Suppose we have a dataset with at least two numerical variables.

• We're interested in predicting one variable based on another:
  • Given my education level, what is my income?
  • Given my height, how tall will my kid be as an adult?
  • Given my age, how many countries have I visited?

• To do this effectively, we need to first observe a pattern between the two numerical variables.

• To see if a pattern exists, we'll need to draw a scatter plot.

Association

• An association is any relationship or link 🔗 between two variables in a scatter plot. Associations can be linear or non-linear.

• If two variables have a positive association ↗️, then as one variable increases, the other tends to increase.
• If two variables have a negative association ↘️, then as one variable increases, the other tends to decrease.

• If two variables are associated, then we can predict the value of one variable based on the value of the other.

Example: Hybrid cars 🚗

hybrid = bpd.read_csv('data/hybrid.csv')
hybrid

	vehicle	year	price	acceleration	mpg	class
0	Prius (1st Gen)	1997	24509.74	7.46	41.26	Compact
1	Tino	2000	35354.97	8.20	54.10	Compact
2	Prius (2nd Gen)	2000	26832.25	7.97	45.23	Compact
...	...	...	...	...	...	...
150	C-Max Energi Plug-in	2013	32950.00	11.76	43.00	Midsize
151	Fusion Energi Plug-in	2013	38700.00	11.76	43.00	Midsize
152	Chevrolet Volt	2013	39145.00	11.11	37.00	Compact

153 rows × 6 columns

'price' vs. 'acceleration'

Is there an association between these two variables? If so, what kind?

hybrid.plot(kind='scatter', x='acceleration', y='price');

'price' vs. 'mpg'

Is there an association between these two variables? If so, what kind?

hybrid.plot(kind='scatter', x='mpg', y='price');

Observations:

• There is a negative association – cars with better fuel economy tended to be cheaper.
  • Why do we think that is? 🤔
• The association looks more curved than linear.
  • It may roughly follow $y \approx \frac{1}{x}$.

Linear changes in units

• A linear change in units doesn't change the shape of the plot, it only changes the scale of the plot.
  • Linear change means adding or subtracting a constant, and multiplying or dividing by a constant.

• In other words, instead of plotting price in dollars and fuel economy in miles per gallon, we can plot price in Yen (🇯🇵) and fuel economy in kilometers per liter and the plot would look the same, just with different axes:

hybrid.assign(
    km_per_liter=hybrid.get('mpg') * 0.425144,
    yen=hybrid.get('price') * 139.77 
).plot(kind='scatter', x='km_per_liter', y='yen');

Converting columns to standard units

• Recall: Suppose $x$ is a numerical variable, and $x_i$ is one value of that variable. To convert $x_i$ to standard units, $$x_{i \: \text{(su)}} = \frac{x_i - \text{mean of $x$}}{\text{SD of $x$}}$$

• Converting columns to standard units makes different scatter plots comparable, by putting the $x$ and $y$ axes on the same scale.
  • Both axes measure the number of standard deviations above the mean.

• Converting columns to standard units doesn't change shape of the scatter plot, because the conversion is linear.

def standard_units(any_numbers):
    "Convert any array of numbers to standard units."
    any_numbers = np.array(any_numbers)
    return (any_numbers - any_numbers.mean()) / np.std(any_numbers)

def standardize(df):
    """Return a DataFrame in which all columns of df are converted to standard units."""
    df_su = bpd.DataFrame()
    for column in df.columns:
        df_su = df_su.assign(**{column + ' (su)': standard_units(df.get(column))})
    return df_su

Standard units for hybrid cars

For a given pair of variables:

• Which cars are average in both variables?
• Which cars are well above or well below average in both variables?

hybrid_su = standardize(hybrid.get(['price', 'acceleration', 'mpg'])).assign(vehicle=hybrid.get('vehicle'))
hybrid_su

	price (su)	acceleration (su)	mpg (su)	vehicle
0	-6.94e-01	-1.54	0.59	Prius (1st Gen)
1	-1.86e-01	-1.28	1.76	Tino
2	-5.85e-01	-1.36	0.95	Prius (2nd Gen)
...	...	...	...	...
150	-2.98e-01	-0.07	0.75	C-Max Energi Plug-in
151	-2.90e-02	-0.07	0.75	Fusion Energi Plug-in
152	-8.17e-03	-0.29	0.20	Chevrolet Volt

153 rows × 4 columns

'price' vs. 'acceleration'

hybrid_su.plot(kind='scatter', x='acceleration (su)', y='price (su)');

Which cars have 'acceleration's and 'price's that are more than 2 SDs above average?

hybrid_su[(hybrid_su.get('acceleration (su)') > 2) &
          (hybrid_su.get('price (su)') > 2)]

	price (su)	acceleration (su)	mpg (su)	vehicle
47	2.71	2.05	-1.46	ActiveHybrid X6
60	3.04	2.88	-1.16	ActiveHybrid 7
95	2.96	2.12	-1.35	ActiveHybrid 7i
146	2.11	2.12	-0.90	ActiveHybrid 7L
147	2.66	2.24	-0.90	Panamera S

'price' vs. 'mpg'

hybrid_su.plot(kind='scatter', x='mpg (su)', y='price (su)');

Which cars have close to average 'mpg's and close to average 'price's?

hybrid_su[(hybrid_su.get('mpg (su)') <= 0.3) &
          (hybrid_su.get('mpg (su)') >= -0.3) &
          (hybrid_su.get('price (su)') <= 0.3) &
          (hybrid_su.get('price (su)') >= -0.3)]

	price (su)	acceleration (su)	mpg (su)	vehicle
10	-1.24e-01	-0.56	-0.26	Escape
22	-2.13e-01	-1.02	-0.17	Mercury Mariner
57	-8.47e-02	0.72	-0.11	Audi Q5
...	...	...	...	...
70	-2.14e-01	-0.07	0.02	HS 250h
102	-2.69e-03	-0.29	0.20	Chevrolet Volt
152	-8.17e-03	-0.29	0.20	Chevrolet Volt

8 rows × 4 columns

Observation on associations in standard units

• If two variables are positively associated ↗️,
  • when one variable is positive, the other tends to be positive, and
  • when one variable is negative, the other also tends to be negative.

• If two variables are negatively associated ↘️,
  • when one variable is positive, the other tends to be negative, and vice versa.

• If two variables aren't associated, there should be no such pattern.

Correlation

Definition: Correlation coefficient

The correlation coefficient $r$ of two variables $x$ and $y$ is defined as the

• average value of the
• product of $x$ and $y$
• when both are measured in standard units.

If x and y are two Series or arrays,

r = (x_su * y_su).mean()

where x_su and y_su are x and y converted to standard units.

def calculate_r(df, x, y):
    x_su = df.get(x + ' (su)')
    y_su = df.get(y + ' (su)')
    return (x_su * y_su).mean()

Let's calculate $r$ for 'acceleration' and 'price'.

hybrid_su

	price (su)	acceleration (su)	mpg (su)	vehicle
0	-6.94e-01	-1.54	0.59	Prius (1st Gen)
1	-1.86e-01	-1.28	1.76	Tino
2	-5.85e-01	-1.36	0.95	Prius (2nd Gen)
...	...	...	...	...
150	-2.98e-01	-0.07	0.75	C-Max Energi Plug-in
151	-2.90e-02	-0.07	0.75	Fusion Energi Plug-in
152	-8.17e-03	-0.29	0.20	Chevrolet Volt

153 rows × 4 columns

r_acc_price = calculate_r(hybrid_su, 'acceleration', 'price')
r_acc_price

0.6955778996913982

hybrid_su.plot(kind='scatter', x='acceleration (su)', y='price (su)')
plt.axvline(0, color='black');
plt.axhline(0, color='black');

Note that the correlation is positive, and most data points fall in the lower left and upper right quadrants!

Let's now calculate $r$ for 'mpg' and 'price'.

hybrid_su

	price (su)	acceleration (su)	mpg (su)	vehicle
0	-6.94e-01	-1.54	0.59	Prius (1st Gen)
1	-1.86e-01	-1.28	1.76	Tino
2	-5.85e-01	-1.36	0.95	Prius (2nd Gen)
...	...	...	...	...
150	-2.98e-01	-0.07	0.75	C-Max Energi Plug-in
151	-2.90e-02	-0.07	0.75	Fusion Energi Plug-in
152	-8.17e-03	-0.29	0.20	Chevrolet Volt

153 rows × 4 columns

r_mpg_price = calculate_r(hybrid_su, 'mpg', 'price')
r_mpg_price

-0.5318263633683789

hybrid_su.plot(kind='scatter', x='mpg (su)', y='price (su)');
plt.axvline(0, color='black');
plt.axhline(0, color='black');

Note that the correlation is negative, and most data points fall in the upper left and lower right quadrants!

The correlation coefficient, $r$

• $r$ measures how clustered points are around a straight line – it measures linear association.
  • If two variables are correlated, it means they are linearly associated.

• $r$ is always between $-1$ and $1$.
  • If $r = 1$, the scatter plot is a line of slope 1.
  • If $r = -1$, the scatter plot is a line of slope -1.
  • If $r = 0$, there is no linear association (uncorrelated).

show_scatter_grid()