Weather Visualization for Portland, OR: 1940 - 2020¶
- Notebook Author: Trenton McKinney
- Just for fun
In [1]:
import pandas as pd
import matplotlib.pyplot as plt
from pathlib import Path
import seaborn as sns
import numpy as np
import calendar
from math import floor
from datetime import date
In [2]:
pd.set_option('display.max_columns', 700)
pd.set_option('display.max_rows', 100)
pd.set_option('display.min_rows', 10)
pd.set_option('display.expand_frame_repr', True)
In [3]:
plt.rcParams['figure.figsize'] = (16.0, 10.0)
# plt.style.use('ggplot')
# sns.set_style("white")
Load and Clean Data¶
- The data comes from NOAA Nation Weather Service Forecast Office: Portland, OR
- Portland_dailyclimatedata.csv only comes with data through 2019
- Portland_dailyclimatedata1940-2019.xlsx has data through 2020
- The 2020 data is on a worksheet
2020 - Data prior to 2020 in on worksheet
Portland_dailyclimatedata1940-2
- The 2020 data is on a worksheet
Load a local copy¶
In [4]:
p = Path().cwd().parents[0] / 'data/Portland_dailyclimatedata1940-2019.xlsx'
In [5]:
# Data through the end of 2019
pdx_19 = pd.read_excel(p, sheet_name='Portland_dailyclimatedata1940-2', skiprows=list(range(6)))
# Data for 2020
pdx_20 = pd.read_excel(p, sheet_name='2020')
View initial dataframe¶
In [6]:
pdx_19.head()
Out[6]:
In [7]:
pdx_20.head()
Out[7]:
Transform and clean data¶
In [8]:
# Drop unused column
pdx_19.drop(columns=['AVG or Total'], inplace=True)
pdx_20.drop(columns=['Departure', 'HDD', 'CDD', 'New_Snow', 'Snow_Depth'], inplace=True)
In [9]:
# add prefix to days for use with wide_to_long
pdx_19.columns = list(pdx_19.columns[:3]) + [f'v_{day}' for day in pdx_19.columns[3:]]
In [10]:
# Select TX (max temp) and TN (min temp)
pdx_19.rename(columns={'Unnamed: 2': 'TYPE'}, inplace=True)
pdx_19 = pdx_19[pdx_19.TYPE.isin(['TX', 'TN', 'PR'])]
In [11]:
# change name to facilitate wide_to_long
pdx_20.rename(columns={'Date': 'date', 'MAX': 'v1', 'MIN': 'v2', 'AVG': 'v3', 'Precip': 'v4'}, inplace=True)
In [12]:
pdx_20.head()
Out[12]:
In [13]:
pdx_19.head()
Out[13]:
Convert to Tidy format¶
1940 - 2019 Data¶
In [14]:
# reshape the data to tidy
pdx = pd.wide_to_long(pdx_19, stubnames='v', sep='_', i=['YR', 'MO', 'TYPE'], j='day').reset_index()
In [15]:
# Give a more descriptive name
pdx.TYPE = pdx.TYPE.map({'TX': 'MAX', 'TN': 'MIN', 'PR': 'PRE'})
In [16]:
# rename so they can be used to create a date column
pdx.rename(columns={'YR': 'year', 'MO': 'month'}, inplace=True)
In [17]:
# - denotes days that don't exist for a given month; drop those
pdx = pdx[pdx.v != '-'].copy()
In [18]:
# create date column
pdx['date'] = pd.to_datetime(pdx[['year', 'month', 'day']])
In [19]:
pdx.head()
Out[19]:
2020 Data¶
In [20]:
pdx_20_l = pd.wide_to_long(pdx_20, stubnames=['v'], i=['date'], j='TYPE').reset_index()
In [21]:
pdx_20_l.TYPE = pdx_20_l.TYPE.map({1: 'MAX', 2: 'MIN', 3: 'AVG', 4: 'PRE'})
In [22]:
pdx_20_l['year'] = pdx_20_l.date.dt.year
pdx_20_l['month'] = pdx_20_l.date.dt.month
pdx_20_l['day'] = pdx_20_l.date.dt.day
In [23]:
pdx_20_l
Out[23]:
Join all date ranges¶
In [24]:
pdx = pd.concat([pdx, pdx_20_l]).reset_index(drop=True)
In [25]:
# Set dtypes to float
pdx.v.replace({'M': np.nan, 'T': np.nan}, inplace=True)
pdx.v = pdx.v.astype('float')
In [26]:
# drop na
pdx.dropna(inplace=True)
In [27]:
# add decade
pdx['dec'] = pdx.year.apply(lambda x: f"{floor(x/10)*10}'s")
create temp only dataframe¶
In [28]:
temp = pdx[~(pdx.TYPE == 'PRE')].copy().reset_index(drop=True)
In [29]:
temp.TYPE.unique()
Out[29]:
In [30]:
# add bins
temp['range'] = pd.cut(temp.v, bins=[0, 64, 74, 84, 94, 200], labels=['< 65', '65 - 74', '75 - 84', '85 - 94', '>= 95'])
In [31]:
display(temp.head())
display(temp.tail())
create precipitation only dataframe¶
In [32]:
# convert from str to float
precip = pdx[(pdx.TYPE == 'PRE')].copy().reset_index(drop=True)
In [33]:
precip.TYPE.unique()
Out[33]:
In [34]:
display(precip.head())
display(precip.tail())
Create max temperature dataframe¶
In [35]:
pdx_max = temp[temp.TYPE == 'MAX'].reset_index(drop=True).reset_index(drop=True)
pdx_max
Out[35]:
Create January to May Max Temperature¶
In [36]:
pdx_max_jan_may = pdx_max[(pdx_max.date.dt.month >= 1) & (pdx_max.date.dt.month < 6)]
Groupby year and range¶
In [37]:
pdx_max_g = pdx_max_jan_may.groupby([pdx_max_jan_may.date.dt.year, 'range'])['v'].agg('count').reset_index(level=0)
In [38]:
display(pdx_max_g.head())
display(pdx_max_g.tail())
Plot MAX Temperature data¶
In [39]:
years = list(range(1950, 2031, 10))
with sns.axes_style("darkgrid"):
for year in years:
plt.figure()
data = pdx_max_g[(pdx_max_g.date >= year - 10) & (pdx_max_g.date < year)]
ax = sns.barplot(x=data.index, y=data.v, hue=data.date)
plt.annotate('Prepared By: Trenton McKinney', xy=(3, 140.1), xytext=(3, 140.1), fontsize=8)
for p in ax.patches:
if p.get_height() > 0:
ax.annotate(format(p.get_height(), '.0f'),
(p.get_x() + p.get_width() / 2., p.get_height()),
ha = 'center', va = 'center', fontsize=8,
xytext = (0, 10), textcoords = 'offset points')
plt.ylim(0, 150)
plt.ylabel('Days')
plt.xlabel('High Temperatures °F')
plt.title(f"Portland, OR\nJan - May High Temperature Days: {year-10}'s")
Create June Max Temperature¶
In [40]:
pdx_max_june = pdx_max[(pdx_max.date.dt.month == 6)]
pdx_max_june = pdx_max_june.groupby([pdx_max_june.date.dt.year, 'range'])['v'].agg('count').reset_index(level=0)
pdx_max_june
Out[40]:
Plot June¶
In [41]:
years = list(range(1950, 2031, 10))
with sns.axes_style("darkgrid"):
for year in years:
plt.figure()
data = pdx_max_june[(pdx_max_june.date >= year - 10) & (pdx_max_june.date < year)]
ax = sns.barplot(x=data.index, y=data.v, hue=data.date)
plt.annotate('Prepared By: Trenton McKinney', xy=(0, 24.3), xytext=(0, 24.3), fontsize=8)
for p in ax.patches:
if p.get_height() > 0:
ax.annotate(format(p.get_height(), '.0f'),
(p.get_x() + p.get_width() / 2., p.get_height()),
ha='center', va='center', fontsize=9,
xytext=(0, 10), textcoords='offset points')
plt.ylim(0, 25)
plt.ylabel('Days')
plt.xlabel('High Temperatures °F')
plt.title(f"Portland, OR\nJune High Temperature Days: {year-10}'s")
Resample max monthly mean¶
In [42]:
pdx_m_mean = pdx_max.set_index('date').resample('m').mean()
In [43]:
years = list(range(1950, 2031, 10))
with sns.axes_style("darkgrid"):
for year in years:
data = pdx_m_mean[(pdx_m_mean.index.year >= year - 10) & (pdx_m_mean.index.year < year)]
ax = sns.lineplot(x=data.index, y=data.v)
plt.annotate('Prepared By: Trenton McKinney', xy=(730120, 32), xytext=(730120, 32), fontsize=10)
plt.xlabel('Year')
plt.ylabel('Monthly Mean Max Temperature °F')
plt.title(f"Portland, OR\nResampled Monthly Mean High Temperature")
Mean: Resample max yearly¶
In [44]:
pdx_y_mean = pdx_max[pdx_max.year > 1940].set_index('date').resample('y').mean()
# filtered out 1940 because it's a partial year (only 2 months)
In [45]:
years = list(range(1950, 2021, 10))
with sns.axes_style("darkgrid"):
for year in years:
data = pdx_y_mean[(pdx_y_mean.index.year >= year - 10) & (pdx_y_mean.index.year < year)]
ax = sns.lineplot(x=data.index, y=data.v)
plt.annotate('Prepared By: Trenton McKinney', xy=(730120, 58.7), xytext=(730120, 58.7), fontsize=9)
plt.xlabel('Year')
plt.ylabel('Yearly Mean Max Temperature °F')
plt.title(f"Portland, OR\nResampled Yearly Mean High Temperature")
Mean: Groupby year & month¶
In [46]:
mym = pdx_max.groupby([pdx_max.date.dt.year, pdx_max.date.dt.month])['v'].mean().unstack(level=0)
mym.iloc[:, :5]
Out[46]:
In [47]:
ix = list(range(0, len(mym.columns), 10))
with sns.axes_style("darkgrid"):
for i in ix:
plt.figure()
data = mym.iloc[:, i:i+10]
sns.lineplot(data=data, markers=None, dashes=False)
plt.annotate('Prepared By: Trenton McKinney', xy=(10, 31), xytext=(10, 31), fontsize=10)
plt.xticks(np.arange(1, 13), calendar.month_name[1:13])
plt.ylim(30, 100)
plt.xlabel('Month')
plt.ylabel('Mean Max Temperature °F')
plt.title(f"Portland, OR\nMonthly Mean per Year High Temperature")
plt.show()
Mean: Groupby decade & month¶
In [48]:
mymd = pdx_max.groupby([pdx_max.dec, pdx_max.date.dt.month])['v'].mean().unstack(level=0)
mymd
Out[48]:
In [49]:
with sns.axes_style("darkgrid"):
plt.figure()
sns.lineplot(data=mymd, markers=None, dashes=False)
plt.annotate('Prepared By: Trenton McKinney', xy=(10, 40.5), xytext=(10, 40.5), fontsize=10)
plt.xticks(np.arange(1, 13), calendar.month_name[1:13])
plt.ylim(40, 85)
plt.xlabel('Month')
plt.ylabel('Mean Max Temperature °F')
plt.title(f"Portland, OR\nMonthly Mean Per Decade High Temperature")
plt.show()
Precipitation¶
2020¶
In [50]:
# PRE: precipitation data only
pdx_20_lp = precip[precip.year == 2020]
pdx_20_lp.head()
Out[50]:
In [51]:
plt.figure(figsize=(25, 10))
with sns.axes_style("darkgrid"):
sns.barplot(pdx_20_lp.date, pdx_20_lp.v)
plt.annotate('Prepared By: Trenton McKinney', xy=(130, 1.03), xytext=(130, 1.03), fontsize=8)
ticks, labels = plt.xticks(rotation=90)
labels = [label.get_text()[:10] for label in labels]
plt.xticks(ticks=ticks, labels=labels)
plt.ylabel('Precipitation (inches)')
plt.xlabel('Date')
plt.title(f"Portland, OR\n 2020: Jan - June 16\nPrecipitation Days")
In [52]:
with sns.axes_style("darkgrid"):
sns.boxplot(pdx_20_lp.date.dt.month, pdx_20_lp.v)
plt.xticks(np.arange(0, 6), calendar.month_name[1:7])
# plt.annotate('Prepared By: Trenton McKinney', xy=(3, 0.9), xytext=(3, 0.9), fontsize=8)
plt.ylabel('Precipitation (inches)')
plt.xlabel('Month')
plt.title(f"Portland, OR\n 2020: Jan - June 16\nPrecipitation")
1940 - 2020¶
In [53]:
pdx_precip = precip
Mean Monthly Precipitation per Decade: 1940 - 2020¶
In [54]:
pmymd = pdx_precip.groupby([pdx_precip.dec, pdx_precip.month])['v'].mean().unstack(level=0)
pmymd
Out[54]:
In [55]:
with sns.axes_style("darkgrid"):
plt.figure()
sns.lineplot(data=pmymd, markers=None, dashes=False)
# plt.annotate('Prepared By: Trenton McKinney', xy=(10, 0.06), xytext=(10, 0.06), fontsize=10)
plt.xticks(np.arange(1, 13), calendar.month_name[1:13])
plt.xlabel('Month')
plt.ylabel('Mean Precipitation (inches)')
plt.title(f"Portland, OR\nMonthly Mean Amount of Precipitation by Decade")
plt.show()
In [56]:
pmymds = pmymd.stack().reset_index()
with sns.axes_style("darkgrid"):
plt.figure(figsize=(20, 10))
ax = sns.barplot('month', 0, data=pmymds, hue='dec')
for p in ax.patches:
if p.get_height() > 0:
ax.annotate(format(p.get_height(), '.2f'),
(p.get_x() + p.get_width() / 2., p.get_height()),
ha='center', va='center', fontsize=9, rotation=90,
xytext=(0, 10), textcoords='offset points')
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
# plt.annotate('Prepared By: Trenton McKinney', xy=(9, 0.351), xytext=(9, 0.351), fontsize=10)
plt.xticks(np.arange(0, 12), calendar.month_name[1:13])
plt.xlabel('Month')
plt.ylabel('Mean Precipitation (inches)')
plt.title(f"Portland, OR\nMonthly Mean Amount of Precipitation by Decade")
plt.show()
- Note the 2020's are only made up of 5.5 months of data
- Other decades are the mean over 10 years.
Total Precipitation Days: 1940 - 2020¶
In [57]:
precip_days_year = pdx_precip[pdx_precip.v > 0].groupby('year', as_index=False)['year'].agg({'count': 'count'})
with sns.axes_style("darkgrid"):
ax = sns.barplot(x='year', y='count', data=precip_days_year)
plt.xticks(rotation=90)
# plt.annotate('Prepared By: Trenton McKinney', xy=(3, 140.1), xytext=(3, 140.1), fontsize=8)
for p in ax.patches:
if p.get_height() > 0:
ax.annotate(format(p.get_height(), '.0f'),
(p.get_x() + p.get_width() / 2., p.get_height()),
ha = 'center', va = 'center', fontsize=9, rotation=90,
xytext = (0, 10), textcoords = 'offset points')
plt.ylabel('Days of Precipitation')
plt.xlabel('Year')
plt.title(f"Portland, OR\nPrecipitation Days Per Year")
plt.show()
- 1940 and 2020 do not have complete data for the year
Mean Monthly Precipitation: 1940 - 2020¶
In [58]:
precip_days_month = pdx_precip.groupby('month', as_index=False)['v'].agg({'mean': 'mean'})
ax = sns.barplot(x='month', y='mean', data=precip_days_month)
plt.xticks(rotation=90)
# plt.annotate('Prepared By: Trenton McKinney', xy=(3, 140.1), xytext=(3, 140.1), fontsize=8)
plt.xticks(np.arange(0, 12), calendar.month_name[1:13])
for p in ax.patches:
if p.get_height() > 0:
ax.annotate(format(p.get_height(), '.2f'),
(p.get_x() + p.get_width() / 2., p.get_height()),
ha = 'center', va = 'center', fontsize=8,
xytext = (0, 10), textcoords = 'offset points')
plt.ylabel('Mean Precipitation (inches)')
plt.xlabel('Month')
plt.title(f"Portland, OR\nMean Precipitation Per Month: 1940 - 2020")
plt.show()
Total Jan - May Precipitation: 2018 - 2020¶
In [59]:
gb_18_20 = pdx_precip[(pdx_precip.year.isin([2018, 2019, 2020])) & (pdx_precip.month < 7)].groupby(['year', 'month']).agg({'v': sum}).reset_index()
In [60]:
ax = sns.barplot(x=gb_18_20.month, y=gb_18_20.v, hue=gb_18_20.year)
plt.xticks(np.arange(0, 6), calendar.month_name[1:7])
for p in ax.patches:
if p.get_height() > 0:
ax.annotate(format(p.get_height(), '.2f'),
(p.get_x() + p.get_width() / 2., p.get_height()),
ha = 'center', va = 'center', fontsize=8,
xytext = (0, 10), textcoords = 'offset points')
plt.ylabel('Total Precipitation (inches)')
plt.xlabel('Month')
plt.title(f"Portland, OR\nTotal Monthly Precipitation\n(Jan - June): 2018 - 2020")
plt.show()
Monthly Total Precipitation¶
In [61]:
years = list(range(1950, 2031, 10))
with sns.axes_style("darkgrid"):
for year in years:
plt.figure()
data = pdx_precip[(pdx_precip.year >= year - 10) & (pdx_precip.year < year)]
data = data.groupby(['year', 'month']).agg({'v': sum}).reset_index()
ax = sns.barplot(x=data.month, y=data.v, hue=data.year)
# plt.annotate('Prepared By: Trenton McKinney', xy=(3, 140.1), xytext=(3, 140.1), fontsize=8)
for p in ax.patches:
if p.get_height() > 0:
ax.annotate(format(p.get_height(), '.2f'),
(p.get_x() + p.get_width() / 2., p.get_height() + 0.2),
ha = 'center', va = 'center', fontsize=9, rotation=90,
xytext = (0, 10), textcoords = 'offset points')
plt.xticks(np.arange(0, 12), calendar.month_name[1:13])
plt.ylim(0, 16)
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.ylabel('Total Rain (inches)')
plt.xlabel('Month')
plt.title(f"Portland, OR\nTotal Rain Per Month: {year-10}'s")
2011 - 2020¶
In [62]:
pdx_precip_11_00 = pdx_precip[(pdx_precip.year > 2010)]
pdx_precip_11_00
Out[62]:
In [63]:
with sns.axes_style("darkgrid"):
sns.boxplot(pdx_precip_11_00.date.dt.year, pdx_precip_11_00.v, color='lightgray')
sns.swarmplot(pdx_precip_11_00.date.dt.year, pdx_precip_11_00.v, size=2.5)
plt.annotate('Prepared By: Trenton McKinney', xy=(7, 2.51), xytext=(7, 2.51), fontsize=8)
plt.ylabel('Precipitation (inches)')
plt.xlabel('Year')
plt.title(f"Portland, OR\n 2011 - 2020 Amount of Precipitation")
