Week3_Toronto_Neighborhoods_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Segmenting and Clustering Neighborhoods in Toronto"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set up dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Libraries imported.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # library to handle data in a vectorized manner\n",
    "\n",
    "import pandas as pd # library for data analysis\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "\n",
    "import json # library to handle JSON files\n",
    "\n",
    "#!conda install -c conda-forge geopy --yes \n",
    "from geopy.geocoders import Nominatim # convert an address into latitude and longitude values\n",
    "\n",
    "import requests # library to handle requests\n",
    "from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe\n",
    "\n",
    "# Matplotlib and associated plotting modules\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.colors as colors\n",
    "\n",
    "# import k-means from clustering stage\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "#!conda install -c conda-forge folium=0.5.0 --yes\n",
    "import folium # map rendering library\n",
    "\n",
    "import requests # library to fetch URLs\n",
    "\n",
    "from bs4 import BeautifulSoup # library to scrape data from a webpage\n",
    "\n",
    "print('Libraries imported.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape Postal Codes from Wikipedia\n",
    "Scrape the postal codes table from this Wikipedia page:\n",
    "https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\n",
    "\n",
    "<i>\"This is a list of postal codes in Canada where the first letter is M. Postal codes beginning with M are located within the city of Toronto in the province of Ontario. Only the first three characters are listed, corresponding to the Forward Sortation Area.\"</i>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data downloaded from: https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\n"
     ]
    }
   ],
   "source": [
    "url = 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\n",
    "\n",
    "response = requests.get(url)\n",
    "if response.status_code != 200:\n",
    "    print('Failed to get data:', response.status_code)\n",
    "else:\n",
    "    print(f'Data downloaded from: {url}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td>Not assigned</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M2A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td>Not assigned</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Harbourfront</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Postcode           Borough     Neighbourhood\n",
       "0      M1A      Not assigned      Not assigned\n",
       "1      M2A      Not assigned      Not assigned\n",
       "2      M3A        North York         Parkwoods\n",
       "3      M4A        North York  Victoria Village\n",
       "4      M5A  Downtown Toronto      Harbourfront"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup = BeautifulSoup(response.text, 'html.parser')\n",
    "table = soup.find(\"table\", {\"class\":\"wikitable sortable\"})\n",
    "df = pd.read_html(str(table),header=0)[0]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean up the postal code dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed 77 rows with unassigned boroughs.\n",
      "As expected, there is only one borough in each postcode. Yay!\n",
      "1 rows with unassigned neighbourhoods (now assigned to borough).\n",
      "Collapsed 108 duplicated postcodes into single rows.\n",
      "\n",
      "\n",
      "  Postcode      Borough                           Neighbourhood\n",
      "0      M1B  Scarborough                          Rouge, Malvern\n",
      "1      M1C  Scarborough  Highland Creek, Rouge Hill, Port Union\n",
      "2      M1E  Scarborough       Guildwood, Morningside, West Hill\n",
      "3      M1G  Scarborough                                  Woburn\n",
      "4      M1H  Scarborough                               Cedarbrae\n"
     ]
    }
   ],
   "source": [
    "# Remove unassigned boroughs\n",
    "n = df.shape[0]\n",
    "df = df.loc[df['Borough']!='Not assigned',:]\n",
    "print(f'Removed {n-df.shape[0]:,} rows with unassigned boroughs.')\n",
    "del n\n",
    "\n",
    "# Check if a postcode can have more than one borough:\n",
    "if np.max(df.groupby('Postcode')['Borough'].nunique())==1:\n",
    "    print('As expected, there is only one borough in each postcode. Yay!')\n",
    "else:\n",
    "    print('Watch out! There is at least one postcode with multiple distinct boroughs...')\n",
    "\n",
    "# Give unassigned neighborhoods the name of their borough\n",
    "n = sum(df['Neighbourhood']=='Not assigned')\n",
    "print(f'{n:,} rows with unassigned neighbourhoods (now assigned to borough).')\n",
    "del n\n",
    "\n",
    "df.loc[df['Neighbourhood']=='Not assigned','Neighbourhood']=df['Borough']\n",
    "\n",
    "# Combine multiple rows for a given postcode into one comma-separated list of neighbourhoods\n",
    "n = df.shape[0]\n",
    "df=df.groupby(['Postcode','Borough'])['Neighbourhood'].apply(', '.join).reset_index()\n",
    "print(f'Collapsed {n-df.shape[0]:,} duplicated postcodes into single rows.')\n",
    "del n\n",
    "print('\\n')\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cleaned dataframe has 103 total rows, making up the following buroughs:\n",
      "North York          24\n",
      "Downtown Toronto    18\n",
      "Scarborough         17\n",
      "Etobicoke           12\n",
      "Central Toronto      9\n",
      "West Toronto         6\n",
      "East York            5\n",
      "York                 5\n",
      "East Toronto         5\n",
      "Queen's Park         1\n",
      "Mississauga          1\n",
      "Name: Borough, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(f'The cleaned dataframe has {df.shape[0]:,} total rows, making up the following buroughs:')\n",
    "print(df['Borough'].value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>In the last cell of the notebook, use the .shape method to print the number of rows in the dataframe:</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103\n"
     ]
    }
   ],
   "source": [
    "print(df.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Latitude and Longitude Coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data downloaded from: https://cocl.us/Geospatial_data\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postal Code</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1B</td>\n",
       "      <td>43.806686</td>\n",
       "      <td>-79.194353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M1C</td>\n",
       "      <td>43.784535</td>\n",
       "      <td>-79.160497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M1E</td>\n",
       "      <td>43.763573</td>\n",
       "      <td>-79.188711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M1G</td>\n",
       "      <td>43.770992</td>\n",
       "      <td>-79.216917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M1H</td>\n",
       "      <td>43.773136</td>\n",
       "      <td>-79.239476</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Postal Code   Latitude  Longitude\n",
       "0         M1B  43.806686 -79.194353\n",
       "1         M1C  43.784535 -79.160497\n",
       "2         M1E  43.763573 -79.188711\n",
       "3         M1G  43.770992 -79.216917\n",
       "4         M1H  43.773136 -79.239476"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download csv from course website\n",
    "url = 'https://cocl.us/Geospatial_data'\n",
    "lat_lng_coords = pd.read_csv('https://cocl.us/Geospatial_data')\n",
    "print(f'Data downloaded from: {url}')\n",
    "\n",
    "lat_lng_coords.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merge results:\n",
      "both          103\n",
      "right_only      0\n",
      "left_only       0\n",
      "Name: _merge, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Combine the two dataframes to create the specified dataframe for this assignment\n",
    "df = pd.merge(\n",
    "    left=df, right=lat_lng_coords,\n",
    "    left_on='Postcode',right_on='Postal Code',\n",
    "    how='left', validate='1:1', indicator=True)\n",
    "\n",
    "# Confirm that all of the postal codes matched to a latitude/longitude coordinate\n",
    "print('Merge results:')\n",
    "print(df['_merge'].value_counts())\n",
    "\n",
    "# Adjust the column names to match the assignment\n",
    "df.drop(['Postal Code','_merge'],axis=1,inplace=True)\n",
    "df.rename(columns={'Postcode':'PostalCode','Neighbourhood':'Neighborhood'},inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Display the first 15 lines of the dataframe.</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1B</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Rouge, Malvern</td>\n",
       "      <td>43.806686</td>\n",
       "      <td>-79.194353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M1C</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Highland Creek, Rouge Hill, Port Union</td>\n",
       "      <td>43.784535</td>\n",
       "      <td>-79.160497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M1E</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Guildwood, Morningside, West Hill</td>\n",
       "      <td>43.763573</td>\n",
       "      <td>-79.188711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M1G</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Woburn</td>\n",
       "      <td>43.770992</td>\n",
       "      <td>-79.216917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M1H</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cedarbrae</td>\n",
       "      <td>43.773136</td>\n",
       "      <td>-79.239476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>M1J</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough Village</td>\n",
       "      <td>43.744734</td>\n",
       "      <td>-79.239476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>M1K</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>East Birchmount Park, Ionview, Kennedy Park</td>\n",
       "      <td>43.727929</td>\n",
       "      <td>-79.262029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>M1L</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Clairlea, Golden Mile, Oakridge</td>\n",
       "      <td>43.711112</td>\n",
       "      <td>-79.284577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>M1M</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cliffcrest, Cliffside, Scarborough Village West</td>\n",
       "      <td>43.716316</td>\n",
       "      <td>-79.239476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>M1N</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Birch Cliff, Cliffside West</td>\n",
       "      <td>43.692657</td>\n",
       "      <td>-79.264848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>M1P</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Dorset Park, Scarborough Town Centre, Wexford ...</td>\n",
       "      <td>43.757410</td>\n",
       "      <td>-79.273304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>M1R</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Maryvale, Wexford</td>\n",
       "      <td>43.750072</td>\n",
       "      <td>-79.295849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>M1S</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Agincourt</td>\n",
       "      <td>43.794200</td>\n",
       "      <td>-79.262029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>M1T</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Clarks Corners, Sullivan, Tam O'Shanter</td>\n",
       "      <td>43.781638</td>\n",
       "      <td>-79.304302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>M1V</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Agincourt North, L'Amoreaux East, Milliken, St...</td>\n",
       "      <td>43.815252</td>\n",
       "      <td>-79.284577</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Postcode      Borough                                      Neighbourhood  \\\n",
       "0       M1B  Scarborough                                     Rouge, Malvern   \n",
       "1       M1C  Scarborough             Highland Creek, Rouge Hill, Port Union   \n",
       "2       M1E  Scarborough                  Guildwood, Morningside, West Hill   \n",
       "3       M1G  Scarborough                                             Woburn   \n",
       "4       M1H  Scarborough                                          Cedarbrae   \n",
       "5       M1J  Scarborough                                Scarborough Village   \n",
       "6       M1K  Scarborough        East Birchmount Park, Ionview, Kennedy Park   \n",
       "7       M1L  Scarborough                    Clairlea, Golden Mile, Oakridge   \n",
       "8       M1M  Scarborough    Cliffcrest, Cliffside, Scarborough Village West   \n",
       "9       M1N  Scarborough                        Birch Cliff, Cliffside West   \n",
       "10      M1P  Scarborough  Dorset Park, Scarborough Town Centre, Wexford ...   \n",
       "11      M1R  Scarborough                                  Maryvale, Wexford   \n",
       "12      M1S  Scarborough                                          Agincourt   \n",
       "13      M1T  Scarborough            Clarks Corners, Sullivan, Tam O'Shanter   \n",
       "14      M1V  Scarborough  Agincourt North, L'Amoreaux East, Milliken, St...   \n",
       "\n",
       "     Latitude  Longitude  \n",
       "0   43.806686 -79.194353  \n",
       "1   43.784535 -79.160497  \n",
       "2   43.763573 -79.188711  \n",
       "3   43.770992 -79.216917  \n",
       "4   43.773136 -79.239476  \n",
       "5   43.744734 -79.239476  \n",
       "6   43.727929 -79.262029  \n",
       "7   43.711112 -79.284577  \n",
       "8   43.716316 -79.239476  \n",
       "9   43.692657 -79.264848  \n",
       "10  43.757410 -79.273304  \n",
       "11  43.750072 -79.295849  \n",
       "12  43.794200 -79.262029  \n",
       "13  43.781638 -79.304302  \n",
       "14  43.815252 -79.284577  "
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(15)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}