# Set up packages for lecture. Don't worry about understanding this code, but
# make sure to run it if you're following along.
import numpy as np
import babypandas as bpd
import pandas as pd

import matplotlib.pyplot as plt
from matplotlib_inline.backend_inline import set_matplotlib_formats
set_matplotlib_formats("svg")
plt.style.use('ggplot')
plt.rcParams['figure.figsize'] = (10, 5)

np.set_printoptions(threshold=20, precision=2, suppress=True)
pd.set_option("display.max_rows", 7)
pd.set_option("display.max_columns", 8)
pd.set_option("display.precision", 2)

from IPython.display import display, IFrame, YouTubeVideo

def show_grouping_animation():
    src = "https://docs.google.com/presentation/d/e/2PACX-1vTgVlFngQcLMYHP-z1vq5lVXjsBgcHebc-3TX7SW6L_gjX6TD1gsflvVDQUpWiDdeEPqJASenUIfBVd/embed?start=false&loop=false&delayms=60000"
    width = 960
    height = 509
    display(IFrame(src, width, height))
    
import warnings
warnings.simplefilter('ignore')

Lecture 6 – Data Visualization 📈

DSC 10, Spring 2023

Announcements

• Lab 1 is due tomorrow at 11:59PM.
• Homework 1 is due on Tuesday at 11:59PM.
• Come to office hours for help! The schedule is here.
• Make sure to try the "challenge problems" at the end of Wednesday's lecture and watch this walkthrough video for the answers.

Don't forget about these resources!

• DSC 10 Reference Sheet 📌.
• babypandas notes.
• babypandas documentation.
• The Resources tab of the course website.

Agenda

• Recap: GroupBy.
• Why visualize?
• Terminology.
• Scatter plots.
• Line plots.
• Bar charts.

Aside: Keyboard shortcuts

There are several keyboard shortcuts built into Jupyter Notebooks designed to help you save time. To see them, either click the keyboard button in the toolbar above or hit the H key on your keyboard (as long as you're not actively editing a cell).

Particularly useful shortcuts:

Action	Keyboard shortcut
Run cell + jump to next cell	SHIFT + ENTER
Save the notebook	CTRL/CMD + S
Create new cell above/below	A/B
Delete cell	DD

Recap: GroupBy

show_grouping_animation()

Run the cell below to load in the requests DataFrame from last class.

requests = bpd.read_csv('data/get-it-done-requests.csv')
requests = requests.assign(total=requests.get('closed') + requests.get('open'))
requests

	neighborhood	service	closed	open	total
0	Balboa Park	Dead Animal	11	0	11
1	Balboa Park	Development Services - Code Enforcement	2	0	2
2	Balboa Park	Encampment	215	20	235
...	...	...	...	...	...
1418	Via De La Valle	Pothole	11	7	18
1419	Via De La Valle	Sidewalk Repair Issue	0	1	1
1420	Via De La Valle	Street Sweeping	1	0	1

1421 rows × 5 columns

Which neighborhood had the most requests?

requests

	neighborhood	service	closed	open	total
0	Balboa Park	Dead Animal	11	0	11
1	Balboa Park	Development Services - Code Enforcement	2	0	2
2	Balboa Park	Encampment	215	20	235
...	...	...	...	...	...
1418	Via De La Valle	Pothole	11	7	18
1419	Via De La Valle	Sidewalk Repair Issue	0	1	1
1420	Via De La Valle	Street Sweeping	1	0	1

1421 rows × 5 columns

requests.groupby('neighborhood').sum()

	closed	open	total
neighborhood
Balboa Park	1173	261	1434
Barrio Logan	1185	201	1386
Black Mountain Ranch	151	44	195
...	...	...	...
University	1614	620	2234
Uptown	5654	1691	7345
Via De La Valle	13	12	25

57 rows × 3 columns

# Note the use of .index – remember, the index isn't a column!
(
    requests
    .groupby('neighborhood')
    .sum()
    .sort_values(by='total', ascending=False)
    .index[0]
)

'Downtown'

Example: Number of different services

How do we find the number of different services requested in each neighborhood?

As always when using groupby, there are two steps:

1. Choose a column to group by.
   • Here, 'neighborhood' seems like a good choice.

1. Choose an aggregation method.
   • Common aggregation methods include .count(), .sum(), .mean(), .median(), .max(), and .min().

# How many different requests are there for the neighborhood 'University'?
requests[requests.get('neighborhood') == 'University']

	neighborhood	service	closed	open	total
1354	University	Dead Animal	25	0	25
1355	University	Development Services - Code Enforcement	7	2	9
1356	University	Encampment	55	27	82
...	...	...	...	...	...
1381	University	Tree Maintenance	47	8	55
1382	University	Waste on Private Property	5	1	6
1383	University	Weed Cleanup	1	4	5

30 rows × 5 columns

# How do we find this result for every neighborhood?

Observation #4

The column names of the output of .groupby don't make sense when using the .count() aggregation method.

num_diff_services = requests.groupby('neighborhood').count()
num_diff_services

	service	closed	open	total
neighborhood
Balboa Park	28	28	28	28
Barrio Logan	28	28	28	28
Black Mountain Ranch	24	24	24	24
...	...	...	...	...
University	30	30	30	30
Uptown	31	31	31	31
Via De La Valle	6	6	6	6

57 rows × 4 columns

Consider dropping unneeded columns and renaming columns as follows:

1. Use .assign to create a new column containing the same values as the old column(s).
2. Use .drop(columns=list_of_column_labels) to drop the old column(s). Alternatively, use .get(list_of_column_labels) to keep only the columns in the given list. The columns will appear in the order you specify, so this is also useful for reordering columns!

num_diff_services = num_diff_services.assign(
                    count_of_services=num_diff_services.get('open')
                    ).drop(columns=['service', 'closed', 'open', 'total'])
num_diff_services

	count_of_services
neighborhood
Balboa Park	28
Barrio Logan	28
Black Mountain Ranch	24
...	...
University	30
Uptown	31
Via De La Valle	6

57 rows × 1 columns

Why visualize?

Run these cells to load the Little Women data from Lecture 1.

chapters = open('data/lw.txt').read().split('CHAPTER ')[1:]

# Counts of names in the chapters of Little Women.

counts = bpd.DataFrame().assign(
    Amy=np.char.count(chapters, 'Amy'),
    Beth=np.char.count(chapters, 'Beth'),
    Jo=np.char.count(chapters, 'Jo'),
    Meg=np.char.count(chapters, 'Meg'),
    Laurie=np.char.count(chapters, 'Laurie'),
)

# Cumulative number of times each name appears.

lw_counts = bpd.DataFrame().assign(
    Amy=np.cumsum(counts.get('Amy')),
    Beth=np.cumsum(counts.get('Beth')),
    Jo=np.cumsum(counts.get('Jo')),
    Meg=np.cumsum(counts.get('Meg')),
    Laurie=np.cumsum(counts.get('Laurie')),
    Chapter=np.arange(1, 48, 1)
)

lw_counts

	Amy	Beth	Jo	Meg	Laurie	Chapter
0	23	26	44	26	0	1
1	36	38	65	46	0	2
2	38	40	127	82	16	3
...	...	...	...	...	...	...
44	633	461	1450	675	581	45
45	635	462	1506	679	583	46
46	645	465	1543	685	596	47

47 rows × 6 columns

Little Women

In Lecture 1, we were able to answer questions about the plot of Little Women without having to read the novel and without having to understand Python code. Some of those questions included:

• Who is the main character?
• Which pair of characters gets married in Chapter 35?

We answered these questions from a data visualization alone!

lw_counts.plot(x='Chapter');

Napoleon's March

"Probably the best statistical graphic ever drawn, this map by Charles Joseph Minard portrays the losses suffered by Napoleon's army in the Russian campaign of 1812." (source)

Why visualize?

• Computers are better than humans at crunching numbers, but humans are better at identifying visual patterns.

• Visualizations allow us to understand lots of data quickly – they make it easier to spot trends and communicate our results with others.

• There are many types of visualizations; in this class, we'll look at scatter plots, line plots, bar charts, and histograms, but there are many others.
  • The right choice depends on the type of data.

Terminology

Individuals and variables

• Individual (row): Person/place/thing for which data is recorded. Also called an observation.

• Variable (column): Something that is recorded for each individual. Also called a feature.

Types of variables

There are two main types of variables:

• Numerical: It makes sense to do arithmetic with the values.
• Categorical: Values fall into categories, that may or may not have some order to them.

Note that here, "variable" does not mean a variable in Python, but rather it means a column in a DataFrame.

Examples of numerical variables

• Salaries of NBA players 🏀.
  • Individual: An NBA player.
  • Variable: Their salary.

• Movie gross earnings 💰.
  • Individual: A movie.
  • Variable: Its gross earnings.

• Booster doses administered per day 💉.
  • Individual: Date.
  • Variable: Number of booster doses administered on that date.

Examples of categorical variables

• Movie genres 🎬.
  • Individual: A movie.
  • Variable: Its genre.

• Zip codes 🏠.
  • Individual: US resident.
  • Variable: Zip code.
    • Even though they look like numbers, zip codes are categorical (arithmetic doesn't make sense).

• Level of prior programming experience for students in DSC 10 🧑‍🎓.
  • Individual: Student in DSC 10.
  • Variable: Their level of prior programming experience, e.g. none, low, medium, or high.
    • There is an order to these categories!

Concept Check ✅ – Answer at cc.dsc10.com

Which of these is not a numerical variable?

A. Fuel economy in miles per gallon.

B. Number of quarters at UCSD.

C. College at UCSD (Sixth, Seventh, etc).

D. Bank account number.

E. More than one of these are not numerical variables.

Types of visualizations

The type of visualization we create depends on the kinds of variables we're visualizing.

• Scatter plot: Numerical vs. numerical.
• Line plot: Sequential numerical (time) vs. numerical.
• Bar chart: Categorical vs. numerical.
• Histogram: Numerical.
  • Will cover next time.

We may interchange the words "plot", "chart", and "graph"; they all mean the same thing.

Scatter plots

Dataset of 50 top-grossing actors

Column	Contents

'Actor'|Name of actor 'Total Gross'| Total gross domestic box office receipt, in millions of dollars, of all of the actor’s movies 'Number of Movies'| The number of movies the actor has been in 'Average per Movie'| Total gross divided by number of movies '#1 Movie'| The highest grossing movie the actor has been in 'Gross'| Gross domestic box office receipt, in millions of dollars, of the actor’s #1 Movie

actors = bpd.read_csv('data/actors.csv').set_index('Actor')
actors

	Total Gross	Number of Movies	Average per Movie	#1 Movie	Gross
Actor
Harrison Ford	4871.7	41	118.8	Star Wars: The Force Awakens	936.7
Samuel L. Jackson	4772.8	69	69.2	The Avengers	623.4
Morgan Freeman	4468.3	61	73.3	The Dark Knight	534.9
...	...	...	...	...	...
Sandra Bullock	2462.6	35	70.4	Minions	336.0
Chris Evans	2457.8	23	106.9	The Avengers	623.4
Anne Hathaway	2416.5	25	96.7	The Dark Knight Rises	448.1

50 rows × 5 columns

Scatter plots

What is the relationship between 'Number of Movies' and 'Total Gross'?

actors.plot(kind='scatter', x='Number of Movies', y='Total Gross');

Scatter plots

• Scatter plots visualize the relationship between two numerical variables.
• To create one from a DataFrame df, use

  df.plot(
    kind='scatter', 
    x=x_column_for_horizontal, 
    y=y_column_for_vertical
  )

• The resulting scatter plot has one point per row of df.
• If you put a semicolon after a call to .plot, it will hide the weird text output that displays.

Scatter plots

What is the relationship between 'Number of Movies' and 'Average per Movie'?

actors.plot(kind='scatter', x='Number of Movies', y='Average per Movie');

Note that in the above plot, there's a negative association and an outlier.

Who was in 60 or more movies?

actors[actors.get('Number of Movies') >= 60]

	Total Gross	Number of Movies	Average per Movie	#1 Movie	Gross
Actor
Samuel L. Jackson	4772.8	69	69.2	The Avengers	623.4
Morgan Freeman	4468.3	61	73.3	The Dark Knight	534.9
Bruce Willis	3189.4	60	53.2	Sixth Sense	293.5
Robert DeNiro	3081.3	79	39.0	Meet the Fockers	279.3
Liam Neeson	2942.7	63	46.7	The Phantom Menace	474.5

Who is the outlier?

Whoever they are, they made very few, high grossing movies.

actors[actors.get('Number of Movies') < 10]

	Total Gross	Number of Movies	Average per Movie	#1 Movie	Gross
Actor
Anthony Daniels	3162.9	7	451.8	Star Wars: The Force Awakens	936.7

Line plots 📉

Dataset aggregating movies by year

Column	Content

'Year'| Year 'Total Gross in Billions'| Total domestic box office gross, in billions of dollars, of all movies released 'Number of Movies'| Number of movies released '#1 Movie'| Highest grossing movie

movies_by_year = bpd.read_csv('data/movies_by_year.csv').set_index('Year')
movies_by_year

	Total Gross in Billions	Number of Movies	#1 Movie
Year
2022	5.64	380	Top Gun: Maverick
2021	4.48	439	Spider-Man: No Way Home
2020	2.11	456	Bad Boys for Life
...	...	...	...
1979	1.23	40	Superman
1978	0.83	13	Grease
1977	0.44	9	Star Wars: Episode IV - A New Hope

46 rows × 3 columns

Line plots

How has the number of movies changed over time? 🤔

movies_by_year.plot(kind='line', y='Number of Movies');

Line plots

• Line plots show trends in numerical variables over time.
• To create one from a DataFrame df, use

  df.plot(
    kind='line', 
    x=x_column_for_horizontal, 
    y=y_column_for_vertical
  )

Plotting tip

• If you want the x-axis to be the index, omit the x= argument!
• Doesn't work for scatter plots, but works for most other plot types.

movies_by_year.plot(kind='line', y='Number of Movies');

Zooming in

We can create a line plot of just 2000 onwards by querying movies_by_year before calling .plot.

movies_by_year[movies_by_year.index >= 2000].plot(kind='line', y='Number of Movies');

What do you think explains the declines around 2008 and 2020?

How did this affect total gross?

movies_by_year[movies_by_year.index >= 2000].plot(kind='line', y='Total Gross in Billions');

What was the top grossing movie of 2018?

...

Ellipsis

Extra video on line plots

If you're curious how line plots work under the hood, watch this video we made a few quarters ago.

YouTubeVideo('glzZ04D1kDg')

Bar charts 📊

Dataset of the top 200 songs in the US on Spotify as of Thursday (4/13/2023)

Downloaded from here – check it out!

charts = (bpd.read_csv('data/regional-us-daily-2023-04-13.csv')
          .set_index('rank')
          .get(['track_name', 'artist_names', 'streams', 'uri'])
         )
charts

	track_name	artist_names	streams	uri
rank
1	Last Night	Morgan Wallen	1801636	spotify:track:7K3BhSpAxZBznislvUMVtn
2	Search & Rescue	Drake	1515162	spotify:track:7aRCf5cLOFN1U7kvtChY1G
3	Kill Bill	SZA	1412326	spotify:track:1Qrg8KqiBpW07V7PNxwwwL
...	...	...	...	...
198	Redbone	Childish Gambino	291222	spotify:track:0wXuerDYiBnERgIpbb3JBR
199	You're On Your Own, Kid	Taylor Swift	290995	spotify:track:4D7BCuvgdJlYvlX5WlN54t
200	Fall In Love	Bailey Zimmerman	290535	spotify:track:5gVCfYmQRPy1QJifP8f5gg

200 rows × 4 columns

Bar charts

How many streams do the top 10 songs have?

charts

	track_name	artist_names	streams	uri
rank
1	Last Night	Morgan Wallen	1801636	spotify:track:7K3BhSpAxZBznislvUMVtn
2	Search & Rescue	Drake	1515162	spotify:track:7aRCf5cLOFN1U7kvtChY1G
3	Kill Bill	SZA	1412326	spotify:track:1Qrg8KqiBpW07V7PNxwwwL
...	...	...	...	...
198	Redbone	Childish Gambino	291222	spotify:track:0wXuerDYiBnERgIpbb3JBR
199	You're On Your Own, Kid	Taylor Swift	290995	spotify:track:4D7BCuvgdJlYvlX5WlN54t
200	Fall In Love	Bailey Zimmerman	290535	spotify:track:5gVCfYmQRPy1QJifP8f5gg

200 rows × 4 columns

charts.take(np.arange(10))

	track_name	artist_names	streams	uri
rank
1	Last Night	Morgan Wallen	1801636	spotify:track:7K3BhSpAxZBznislvUMVtn
2	Search & Rescue	Drake	1515162	spotify:track:7aRCf5cLOFN1U7kvtChY1G
3	Kill Bill	SZA	1412326	spotify:track:1Qrg8KqiBpW07V7PNxwwwL
...	...	...	...	...
8	La Bebe - Remix	Yng Lvcas, Peso Pluma	906832	spotify:track:2UW7JaomAMuX9pZrjVpHAU
9	You Proof	Morgan Wallen	833741	spotify:track:5W4kiM2cUYBJXKRudNyxjW
10	Cupid - Twin Ver.	FIFTY FIFTY	787840	spotify:track:7FbrGaHYVDmfr7KoLIZnQ7

10 rows × 4 columns

charts.take(np.arange(10)).plot(kind='barh', x='track_name', y='streams');

Bar charts

• Bar charts visualize the relationship between a categorical variable and a numerical variable.
• In a bar chart...
  • The thickness and spacing of bars is arbitrary.
  • The order of the categorical labels doesn't matter.
• To create one from a DataFrame df, use

  df.plot(
    kind='barh', 
    x=categorical_column_name, 
    y=numerical_column_name
  )

• The "h" in 'barh' stands for "horizontal".
  • It's easier to read labels this way.
• In the previous chart, we set y='Streams' even though streams are measured by x-axis length.

# The bars appear in the opposite order relative to the DataFrame.
(charts
 .take(np.arange(10))
 .sort_values(by='streams')
 .plot(kind='barh', x='track_name', y='streams')
);

# Change "barh" to "bar" to get a vertical bar chart. These are a little harder to read.
(charts
 .take(np.arange(10))
 .sort_values(by='streams')
 .plot(kind='bar', x='track_name', y='streams')
);

Aside: How many streams did The Weeknd's songs on the chart receive?

(charts
 [charts.get('artist_names') == 'The Weeknd']
 .sort_values('streams')
 .plot(kind='barh', x='track_name', y='streams')
);

It seems like we're missing some popular songs...

How do we include songs with other artists, as well?

Answer: Using .str.contains.

weeknd = charts[charts.get('artist_names').str.contains('The Weeknd')]
weeknd

	track_name	artist_names	streams	uri
rank
12	Creepin' (with The Weeknd & 21 Savage)	Metro Boomin, The Weeknd, 21 Savage	783095	spotify:track:2dHHgzDwk4BJdRwy9uXhTO
26	Die For You	The Weeknd	658995	spotify:track:2LBqCSwhJGcFQeTHMVGwy3
41	Die For You (with Ariana Grande) - Remix	The Weeknd, Ariana Grande	572829	spotify:track:4W4fNrZYkobj539TOWsLO2
97	Starboy	The Weeknd, Daft Punk	387522	spotify:track:7MXVkk9YMctZqd1Srtv4MB
110	Stargirl Interlude	The Weeknd, Lana Del Rey	373574	spotify:track:5gDWsRxpJ2lZAffh5p7K0w
171	The Hills	The Weeknd	312251	spotify:track:7fBv7CLKzipRk6EC6TWHOB
180	Blinding Lights	The Weeknd	305384	spotify:track:0VjIjW4GlUZAMYd2vXMi3b

weeknd.sort_values('streams').plot(kind='barh', x='track_name', y='streams');

Fun demo 🎵

# Run this cell, don't worry about what it does.
def show_spotify(uri):
    code = uri[uri.rfind(':')+1:]
    src = f"https://open.spotify.com/embed/track/{code}"
    width = 400
    height = 75
    display(IFrame(src, width, height))

Let's find the URI of a song we care about.

charts

	track_name	artist_names	streams	uri
rank
1	Last Night