01-dataset-sample-rows.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def glimpse(df, maxvals=10, maxlen=110):\n",
    "    print('Shape: ', df.shape)\n",
    "    \n",
    "    def pad(y):\n",
    "        max_len = max([len(x) for x in y])\n",
    "        return [x.ljust(max_len) for x in y]\n",
    "    \n",
    "    # Column Name\n",
    "    toprnt = pad(df.columns.tolist())\n",
    "    \n",
    "    # Column Type\n",
    "    toprnt = pad([toprnt[i] + ' ' + str(df.iloc[:,i].dtype) for i in range(df.shape[1])])\n",
    "    \n",
    "    # Num NAs\n",
    "    num_nas = [df.iloc[:,i].isnull().sum() for i in range(df.shape[1])]\n",
    "    num_nas_ratio = [int(round(x*100/df.shape[0])) for x in num_nas]\n",
    "    num_nas_str = [str(x) + ' (' + str(y) + '%)' for x,y in zip(num_nas, num_nas_ratio)]\n",
    "    max_len = max([len(x) for x in num_nas_str])\n",
    "    num_nas_str = [x.rjust(max_len) for x in num_nas_str]\n",
    "    toprnt = [x + ' ' + y + ' NAs' for x,y in zip(toprnt, num_nas_str)]\n",
    "    \n",
    "    # Separator\n",
    "    toprnt = [x + ' : ' for x in toprnt]\n",
    "    \n",
    "    # Values\n",
    "    toprnt = [toprnt[i] + ', '.join([str(y) for y in df.iloc[:min([maxvals,df.shape[0]]), i]]) for i in range(df.shape[1])]\n",
    "    \n",
    "    # Trim to maxlen\n",
    "    toprnt = [x[:min(maxlen, len(x))] for x in toprnt]\n",
    "    \n",
    "    for x in toprnt:\n",
    "        print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape:  (3777547, 4)\n",
      "session_id       object 0 (0%) NAs : 0_00001e52-55c0-470c-8d32-85c691c2019a, 0_00001e52-55c0-470c-8d32-85c691c\n",
      "track_id_clean   object 0 (0%) NAs : t_2e6b3753-5b16-4aa2-97f8-c77a81377139, t_fb588cbd-afee-4155-86a4-f871594\n",
      "session_position int64  0 (0%) NAs : 9, 10, 11, 12, 13, 14, 15, 16, 11, 12\n",
      "session_length   int64  0 (0%) NAs : 16, 16, 16, 16, 16, 16, 16, 16, 20, 20\n"
     ]
    }
   ],
   "source": [
    "glimpse(pd.read_csv('./data/test_set/log_input_20180715_000000000000.csv.gz'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape:  (3673820, 21)\n",
      "session_id                      object 0 (0%) NAs : 0_00001e52-55c0-470c-8d32-85c691c2019a, 0_00001e52-55c0-47\n",
      "session_position                int64  0 (0%) NAs : 1, 2, 3, 4, 5, 6, 7, 8, 1, 2\n",
      "session_length                  int64  0 (0%) NAs : 16, 16, 16, 16, 16, 16, 16, 16, 20, 20\n",
      "track_id_clean                  object 0 (0%) NAs : t_a0b2a533-caad-4a72-af14-e647bd7e9c1f, t_a998907c-e11c-4a\n",
      "skip_1                          bool   0 (0%) NAs : False, False, False, False, True, False, False, False, Tru\n",
      "skip_2                          bool   0 (0%) NAs : True, False, False, True, True, True, False, True, True, F\n",
      "skip_3                          bool   0 (0%) NAs : True, True, False, True, True, True, True, True, True, Fal\n",
      "not_skipped                     bool   0 (0%) NAs : False, False, True, False, False, False, False, False, Fal\n",
      "context_switch                  int64  0 (0%) NAs : 0, 1, 0, 0, 0, 0, 0, 0, 0, 0\n",
      "no_pause_before_play            int64  0 (0%) NAs : 0, 1, 0, 0, 1, 0, 0, 1, 0, 1\n",
      "short_pause_before_play         int64  0 (0%) NAs : 0, 0, 0, 1, 0, 1, 0, 0, 0, 0\n",
      "long_pause_before_play          int64  0 (0%) NAs : 0, 0, 1, 1, 0, 1, 1, 0, 0, 0\n",
      "hist_user_behavior_n_seekfwd    int64  0 (0%) NAs : 1, 0, 0, 0, 0, 0, 0, 0, 0, 0\n",
      "hist_user_behavior_n_seekback   int64  0 (0%) NAs : 0, 0, 1, 0, 0, 0, 0, 0, 0, 0\n",
      "hist_user_behavior_is_shuffle   bool   0 (0%) NAs : False, False, False, False, False, False, False, False, Tr\n",
      "hour_of_day                     int64  0 (0%) NAs : 21, 21, 21, 21, 21, 21, 21, 21, 14, 14\n",
      "date                            object 0 (0%) NAs : 2018-07-14, 2018-07-14, 2018-07-14, 2018-07-14, 2018-07-14\n",
      "premium                         bool   0 (0%) NAs : False, False, False, False, False, False, False, False, Tr\n",
      "context_type                    object 0 (0%) NAs : catalog, user_collection, user_collection, user_collection\n",
      "hist_user_behavior_reason_start object 0 (0%) NAs : appload, clickrow, clickrow, trackdone, fwdbtn, fwdbtn, cl\n",
      "hist_user_behavior_reason_end   object 0 (0%) NAs : endplay, endplay, trackdone, fwdbtn, fwdbtn, endplay, fwdb\n"
     ]
    }
   ],
   "source": [
    "glimpse(pd.read_csv('./data/test_set/log_prehistory_20180715_000000000000.csv.gz'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Track features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- track_id: unique identifier\n",
    "- duration: length of track in seconds\n",
    "- release_year: estimate of year the track was released\n",
    "- us_populairty_estimate: estimate of the US popularity percentile of the track as of Oct 12, 2018\n",
    "- acousticness: A confidence measure from 0.0 to 1.0 of whether the track is acoustic. 1.0 represents high confidence the track is acoustic.\n",
    "- beat_strength: ??\n",
    "- bounciness: ??\n",
    "- dyn_range_mean: ??\n",
    "- energy: Energy is a measure from 0.0 to 1.0 and represents a perceptual measure of intensity and activity. Typically, energetic tracks feel fast, loud, and noisy. For example, death metal has high energy, while a Bach prelude scores low on the scale. Perceptual features contributing to this attribute include dynamic range, perceived loudness, timbre, onset rate, and general entropy.\n",
    "- flatness: ??\n",
    "- instrumentalness: Predicts whether a track contains no vocals. \"Ooh\" and \"aah\" sounds are treated as instumental in this context. Rap or spoken word tracks are clearly \"vocal\". The closer the instrumentalness value is to 1.0, the greater likelihood the track contains no vocal content. Values above 0.5 are intended to represent instrumental tracks, but confidence is higher as the value approaches 1.0.\n",
    "- key: The estimated overall key of the track. Integers map to pitches using standard Pitch Class notation (https://en.wikipedia.org/wiki/Pitch_class). E.g. 0 = C, 1 = C♯/D♭, 2 = D, and so on.\n",
    "- liveness: Detects the presence of an audience in the recording. Higher liveness values represent an increased probability that the track was performed live. A value above 0.8 provides strong likelihood that the track is live.\n",
    "- loudness: The overall loudness of a track in decibels (dB). Loudness values are averaged across the entire track and are useful for comparing relative loudness of tracks. Loudness is the quality of a sound that is the primary psychological correlate of physical strength (amplitude). Values typical range between -60 and 0 db. \n",
    "- mechanism: ??\n",
    "- mode: Model indicates the modality (major or minor) of a track, the type of scale from which its melodic content is derived. \n",
    "- organism: ??\n",
    "- speechiness: Speechiness detects the presence of spoken words in a track. The more exclusively speech-like the recording (e.g. talk show, audio book, poetry), the closer to 1.0 the attribute value. Values above 0.66 describe tracks that are probably made entirely of spoken words. Values between 0.33 and 0.66 describe tracks that may contain both music and speech, either in sections or layered, including such cases as rap music. Values below 0.33 most likely represent music and other non-speech-like tracks. \n",
    "- tempo: The overall estimated tempo of a track in beats per minute (BPM). In musical terminology, tempo is the speed or pace of a given piece and derives directly from the average beat duration. \n",
    "- time_signature: An estimated overall time signature of a track. The time signature (meter) is a notational convention to specify how many beats are in each bar (or measure).\n",
    "- valence: A measure from 0.0 to 1.0 describing the musical positiveness conveyed by a track. Tracks with high valence sound more positive (e.g. happy, cheerful, euphoric), while tracks with low valence sound more negative (e.g. sad, depressed, angry). \n",
    "- See https://developer.spotify.com/documentation/web-api/reference/tracks/get-audio-features/\n",
    "- acoustic_vector_*: See http://benanne.github.io/2014/08/05/spotify-cnns.html and http://papers.nips.cc/paper/5004-deep-content-based-\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "track_features = pd.read_csv('./data/track_features/part-00065-6667ece7-5ba8-45f2-860c-f7cafe499e31-c000.csv.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>track_id</th>\n",
       "      <td>t_52d899dd-cdde-458d-b37c-05d145da6b49</td>\n",
       "      <td>t_52d8a762-dea1-4345-830c-bad7d19a29b8</td>\n",
       "      <td>t_52d8a89c-55cf-4a73-9719-66d2e3d9892c</td>\n",
       "      <td>t_52d8a901-c97f-4478-86d2-64847b5cecf4</td>\n",
       "      <td>t_52d8ad56-d7a9-4493-89e3-fe8e0af2a02b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>duration</th>\n",
       "      <td>196.64</td>\n",
       "      <td>179.294</td>\n",
       "      <td>93.4661</td>\n",
       "      <td>132.692</td>\n",
       "      <td>134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>release_year</th>\n",
       "      <td>1986</td>\n",
       "      <td>2014</td>\n",
       "      <td>2015</td>\n",
       "      <td>2010</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>us_popularity_estimate</th>\n",
       "      <td>94.4268</td>\n",
       "      <td>98.8405</td>\n",
       "      <td>94.5854</td>\n",
       "      <td>92.3149</td>\n",
       "      <td>98.8441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acousticness</th>\n",
       "      <td>0.0411283</td>\n",
       "      <td>0.00346559</td>\n",
       "      <td>0.977611</td>\n",
       "      <td>0.166812</td>\n",
       "      <td>0.283672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beat_strength</th>\n",
       "      <td>0.708103</td>\n",
       "      <td>0.310303</td>\n",
       "      <td>0.381506</td>\n",
       "      <td>0.326771</td>\n",
       "      <td>0.471132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bounciness</th>\n",
       "      <td>0.761992</td>\n",
       "      <td>0.424663</td>\n",
       "      <td>0.544087</td>\n",
       "      <td>0.256913</td>\n",
       "      <td>0.595489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>danceability</th>\n",
       "      <td>0.817674</td>\n",
       "      <td>0.470244</td>\n",
       "      <td>0.48831</td>\n",
       "      <td>0.505234</td>\n",
       "      <td>0.503046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dyn_range_mean</th>\n",
       "      <td>11.3771</td>\n",
       "      <td>7.73711</td>\n",
       "      <td>9.33401</td>\n",
       "      <td>4.61247</td>\n",
       "      <td>9.69354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>energy</th>\n",
       "      <td>0.683146</td>\n",
       "      <td>0.775533</td>\n",
       "      <td>0.0277748</td>\n",
       "      <td>0.944617</td>\n",
       "      <td>0.811545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flatness</th>\n",
       "      <td>0.997615</td>\n",
       "      <td>1.00845</td>\n",
       "      <td>1.06849</td>\n",
       "      <td>0.908481</td>\n",
       "      <td>0.961257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instrumentalness</th>\n",
       "      <td>0.00530026</td>\n",
       "      <td>0.030034</td>\n",
       "      <td>0.917592</td>\n",
       "      <td>0.942238</td>\n",
       "      <td>1.4554e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key</th>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>liveness</th>\n",
       "      <td>0.0787028</td>\n",
       "      <td>0.674304</td>\n",
       "      <td>0.115119</td>\n",
       "      <td>0.0773328</td>\n",
       "      <td>0.292384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>loudness</th>\n",
       "      <td>-10.953</td>\n",
       "      <td>-5.657</td>\n",
       "      <td>-19.689</td>\n",
       "      <td>-8.961</td>\n",
       "      <td>-5.272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mechanism</th>\n",
       "      <td>0.905556</td>\n",
       "      <td>0.767606</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.857759</td>\n",
       "      <td>0.287582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mode</th>\n",
       "      <td>minor</td>\n",
       "      <td>major</td>\n",
       "      <td>major</td>\n",
       "      <td>minor</td>\n",
       "      <td>major</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organism</th>\n",
       "      <td>0.0728399</td>\n",
       "      <td>0.164346</td>\n",
       "      <td>0.893231</td>\n",
       "      <td>0.155014</td>\n",
       "      <td>0.542222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>speechiness</th>\n",
       "      <td>0.0350051</td>\n",
       "      <td>0.0486611</td>\n",
       "      <td>0.0636742</td>\n",
       "      <td>0.0549319</td>\n",
       "      <td>0.448734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tempo</th>\n",
       "      <td>128.346</td>\n",
       "      <td>169.67</td>\n",
       "      <td>120.052</td>\n",
       "      <td>129.973</td>\n",
       "      <td>160.117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time_signature</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>valence</th>\n",
       "      <td>0.980463</td>\n",
       "      <td>0.264184</td>\n",
       "      <td>0.234709</td>\n",
       "      <td>0.176885</td>\n",
       "      <td>0.735359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_0</th>\n",
       "      <td>-0.0152818</td>\n",
       "      <td>-0.0974435</td>\n",
       "      <td>0.830604</td>\n",
       "      <td>0.283323</td>\n",
       "      <td>-0.964598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_1</th>\n",
       "      <td>-0.0389102</td>\n",
       "      <td>-0.0319587</td>\n",
       "      <td>-0.715818</td>\n",
       "      <td>-0.137359</td>\n",
       "      <td>0.255574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_2</th>\n",
       "      <td>0.231298</td>\n",
       "      <td>0.0469461</td>\n",
       "      <td>-0.26032</td>\n",
       "      <td>-0.172633</td>\n",
       "      <td>0.168999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_3</th>\n",
       "      <td>0.225196</td>\n",
       "      <td>-0.0980514</td>\n",
       "      <td>0.272471</td>\n",
       "      <td>0.101912</td>\n",
       "      <td>0.245326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_4</th>\n",
       "      <td>0.402138</td>\n",
       "      <td>-0.533764</td>\n",
       "      <td>-0.400693</td>\n",
       "      <td>-0.26147</td>\n",
       "      <td>-0.326109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_5</th>\n",
       "      <td>-0.0493544</td>\n",
       "      <td>0.138756</td>\n",
       "      <td>-0.171517</td>\n",
       "      <td>-0.110278</td>\n",
       "      <td>-0.117049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_6</th>\n",
       "      <td>0.342457</td>\n",
       "      <td>-0.117714</td>\n",
       "      <td>0.167033</td>\n",
       "      <td>0.207227</td>\n",
       "      <td>-0.57656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic_vector_7</th>\n",
       "      <td>-0.376513</td>\n",
       "      <td>0.672062</td>\n",
       "      <td>0.0382663</td>\n",
       "      <td>0.246649</td>\n",
       "      <td>0.123986</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                             0  \\\n",
       "track_id                t_52d899dd-cdde-458d-b37c-05d145da6b49   \n",
       "duration                                                196.64   \n",
       "release_year                                              1986   \n",
       "us_popularity_estimate                                 94.4268   \n",
       "acousticness                                         0.0411283   \n",
       "beat_strength                                         0.708103   \n",
       "bounciness                                            0.761992   \n",
       "danceability                                          0.817674   \n",
       "dyn_range_mean                                         11.3771   \n",
       "energy                                                0.683146   \n",
       "flatness                                              0.997615   \n",
       "instrumentalness                                    0.00530026   \n",
       "key                                                         11   \n",
       "liveness                                             0.0787028   \n",
       "loudness                                               -10.953   \n",
       "mechanism                                             0.905556   \n",
       "mode                                                     minor   \n",
       "organism                                             0.0728399   \n",
       "speechiness                                          0.0350051   \n",
       "tempo                                                  128.346   \n",
       "time_signature                                               4   \n",
       "valence                                               0.980463   \n",
       "acoustic_vector_0                                   -0.0152818   \n",
       "acoustic_vector_1                                   -0.0389102   \n",
       "acoustic_vector_2                                     0.231298   \n",
       "acoustic_vector_3                                     0.225196   \n",
       "acoustic_vector_4                                     0.402138   \n",
       "acoustic_vector_5                                   -0.0493544   \n",
       "acoustic_vector_6                                     0.342457   \n",
       "acoustic_vector_7                                    -0.376513   \n",
       "\n",
       "                                                             1  \\\n",
       "track_id                t_52d8a762-dea1-4345-830c-bad7d19a29b8   \n",
       "duration                                               179.294   \n",
       "release_year                                              2014   \n",
       "us_popularity_estimate                                 98.8405   \n",
       "acousticness                                        0.00346559   \n",
       "beat_strength                                         0.310303   \n",
       "bounciness                                            0.424663   \n",
       "danceability                                          0.470244   \n",
       "dyn_range_mean                                         7.73711   \n",
       "energy                                                0.775533   \n",
       "flatness                                               1.00845   \n",
       "instrumentalness                                      0.030034   \n",
       "key                                                          9   \n",
       "liveness                                              0.674304   \n",
       "loudness                                                -5.657   \n",
       "mechanism                                             0.767606   \n",
       "mode                                                     major   \n",
       "organism                                              0.164346   \n",
       "speechiness                                          0.0486611   \n",
       "tempo                                                   169.67   \n",
       "time_signature                                               4   \n",
       "valence                                               0.264184   \n",
       "acoustic_vector_0                                   -0.0974435   \n",
       "acoustic_vector_1                                   -0.0319587   \n",
       "acoustic_vector_2                                    0.0469461   \n",
       "acoustic_vector_3                                   -0.0980514   \n",
       "acoustic_vector_4                                    -0.533764   \n",
       "acoustic_vector_5                                     0.138756   \n",
       "acoustic_vector_6                                    -0.117714   \n",
       "acoustic_vector_7                                     0.672062   \n",
       "\n",
       "                                                             2  \\\n",
       "track_id                t_52d8a89c-55cf-4a73-9719-66d2e3d9892c   \n",
       "duration                                               93.4661   \n",
       "release_year                                              2015   \n",
       "us_popularity_estimate                                 94.5854   \n",
       "acousticness                                          0.977611   \n",
       "beat_strength                                         0.381506   \n",
       "bounciness                                            0.544087   \n",
       "danceability                                           0.48831   \n",
       "dyn_range_mean                                         9.33401   \n",
       "energy                                               0.0277748   \n",
       "flatness                                               1.06849   \n",
       "instrumentalness                                      0.917592   \n",
       "key                                                          0   \n",
       "liveness                                              0.115119   \n",
       "loudness                                               -19.689   \n",
       "mechanism                                                  0.2   \n",
       "mode                                                     major   \n",
       "organism                                              0.893231   \n",
       "speechiness                                          0.0636742   \n",
       "tempo                                                  120.052   \n",
       "time_signature                                               3   \n",
       "valence                                               0.234709   \n",
       "acoustic_vector_0                                     0.830604   \n",
       "acoustic_vector_1                                    -0.715818   \n",
       "acoustic_vector_2                                     -0.26032   \n",
       "acoustic_vector_3                                     0.272471   \n",
       "acoustic_vector_4                                    -0.400693   \n",
       "acoustic_vector_5                                    -0.171517   \n",
       "acoustic_vector_6                                     0.167033   \n",
       "acoustic_vector_7                                    0.0382663   \n",
       "\n",
       "                                                             3  \\\n",
       "track_id                t_52d8a901-c97f-4478-86d2-64847b5cecf4   \n",
       "duration                                               132.692   \n",
       "release_year                                              2010   \n",
       "us_popularity_estimate                                 92.3149   \n",
       "acousticness                                          0.166812   \n",
       "beat_strength                                         0.326771   \n",
       "bounciness                                            0.256913   \n",
       "danceability                                          0.505234   \n",
       "dyn_range_mean                                         4.61247   \n",
       "energy                                                0.944617   \n",
       "flatness                                              0.908481   \n",
       "instrumentalness                                      0.942238   \n",
       "key                                                          9   \n",
       "liveness                                             0.0773328   \n",
       "loudness                                                -8.961   \n",
       "mechanism                                             0.857759   \n",
       "mode                                                     minor   \n",
       "organism                                              0.155014   \n",
       "speechiness                                          0.0549319   \n",
       "tempo                                                  129.973   \n",
       "time_signature                                               4   \n",
       "valence                                               0.176885   \n",
       "acoustic_vector_0                                     0.283323   \n",
       "acoustic_vector_1                                    -0.137359   \n",
       "acoustic_vector_2                                    -0.172633   \n",
       "acoustic_vector_3                                     0.101912   \n",
       "acoustic_vector_4                                     -0.26147   \n",
       "acoustic_vector_5                                    -0.110278   \n",
       "acoustic_vector_6                                     0.207227   \n",
       "acoustic_vector_7                                     0.246649   \n",
       "\n",
       "                                                             4  \n",
       "track_id                t_52d8ad56-d7a9-4493-89e3-fe8e0af2a02b  \n",
       "duration                                                   134  \n",
       "release_year                                              2018  \n",
       "us_popularity_estimate                                 98.8441  \n",
       "acousticness                                          0.283672  \n",
       "beat_strength                                         0.471132  \n",
       "bounciness                                            0.595489  \n",
       "danceability                                          0.503046  \n",
       "dyn_range_mean                                         9.69354  \n",
       "energy                                                0.811545  \n",
       "flatness                                              0.961257  \n",
       "instrumentalness                                    1.4554e-14  \n",
       "key                                                          8  \n",
       "liveness                                              0.292384  \n",
       "loudness                                                -5.272  \n",
       "mechanism                                             0.287582  \n",
       "mode                                                     major  \n",
       "organism                                              0.542222  \n",
       "speechiness                                           0.448734  \n",
       "tempo                                                  160.117  \n",
       "time_signature                                               4  \n",
       "valence                                               0.735359  \n",
       "acoustic_vector_0                                    -0.964598  \n",
       "acoustic_vector_1                                     0.255574  \n",
       "acoustic_vector_2                                     0.168999  \n",
       "acoustic_vector_3                                     0.245326  \n",
       "acoustic_vector_4                                    -0.326109  \n",
       "acoustic_vector_5                                    -0.117049  \n",
       "acoustic_vector_6                                     -0.57656  \n",
       "acoustic_vector_7                                     0.123986  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "track_features.head().T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training set - session logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- session_id: unique identifier for the session that this row is a part of\n",
    "- session_position: {1-20} - position of row within the session\n",
    "- session_length: {10-20} - number of rows in session\n",
    "- track_id_clean: unique identifier for the track played. This is linked with track_id in the track features and metadata table.\n",
    "- skip_1: Boolean indicating if the track was only played very briefly\n",
    "- skip_2: Boolean indicating if the track was only played briefly\n",
    "- skip_3: Boolean indicating if most of track was played\n",
    "- not_skipped: Boolean indicating that the track was played in its entirety\n",
    "- context_switch: Boolean indicating if the user changed context between previous row and the current row. This could for example occur if the user switched from one playlist to another.\n",
    "- no_pause_before_play: Boolean indicating if there was no pause between playback of the previous track and this track.\n",
    "- short_pause_before_play: Boolean indicating if there was a short pause between playback of the previous track and this track\n",
    "- long_pause_before_play: Boolean indicating if there was a long pause between playback of the previous track and this track\n",
    "- hist_user_behavior_n_seekfwd: Number of times the user did a seek forward within track\n",
    "- hist_user_behavior_n_seekback: Number of times the user did a seek back within track\n",
    "- hist_user_behavior_is_shuffle: Boolean indicating if the user encountered this track while shuffle mode was activated\n",
    "- hour_of_day: {0-23} - The hour of the day\n",
    "- date\n",
    "- premium: Boolean indicating if the user was on premium or not. This has potential implications for skipping behavior.\n",
    "- context_type: E.g. editorial playlist - what type of context the playback occurred within\n",
    "- hist_user_behavior_reason_start: E.g. fwdbtn - the user action which led to the current track being played\n",
    "- hist_user_behavior_reason_end: E.g. trackdone - the user action which led to the current track playback ending\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set = pd.read_csv('./data/training_set/log_0_20180822_000000000000.csv.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>session_id</th>\n",
       "      <td>38_00006771-7eb1-4024-b52f-014b3b7b0488</td>\n",
       "      <td>38_00006771-7eb1-4024-b52f-014b3b7b0488</td>\n",
       "      <td>38_00006771-7eb1-4024-b52f-014b3b7b0488</td>\n",
       "      <td>38_00006771-7eb1-4024-b52f-014b3b7b0488</td>\n",
       "      <td>38_00006771-7eb1-4024-b52f-014b3b7b0488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>session_position</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>session_length</th>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>track_id_clean</th>\n",
       "      <td>t_2b723bb2-0a80-450c-b5e9-20d055175256</td>\n",
       "      <td>t_889887f2-02b4-4cb7-8207-c7df6b115854</td>\n",
       "      <td>t_f36f81cd-c827-43c2-82a0-78bdee27d503</td>\n",
       "      <td>t_830a9de3-ddba-44b8-84be-a5472e925bc2</td>\n",
       "      <td>t_a1817b44-24c4-4289-bde4-046a1315f28d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>skip_1</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>skip_2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>skip_3</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not_skipped</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>context_switch</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_pause_before_play</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>short_pause_before_play</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>long_pause_before_play</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_user_behavior_n_seekfwd</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_user_behavior_n_seekback</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_user_behavior_is_shuffle</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hour_of_day</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <td>2018-08-22</td>\n",
       "      <td>2018-08-22</td>\n",
       "      <td>2018-08-22</td>\n",
       "      <td>2018-08-22</td>\n",
       "      <td>2018-08-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>premium</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>context_type</th>\n",
       "      <td>catalog</td>\n",
       "      <td>catalog</td>\n",
       "      <td>catalog</td>\n",
       "      <td>catalog</td>\n",
       "      <td>user_collection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_user_behavior_reason_start</th>\n",
       "      <td>trackdone</td>\n",
       "      <td>clickrow</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>clickrow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_user_behavior_reason_end</th>\n",
       "      <td>remote</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>endplay</td>\n",
       "      <td>endplay</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                       0  \\\n",
       "session_id                       38_00006771-7eb1-4024-b52f-014b3b7b0488   \n",
       "session_position                                                       1   \n",
       "session_length                                                        17   \n",
       "track_id_clean                    t_2b723bb2-0a80-450c-b5e9-20d055175256   \n",
       "skip_1                                                             False   \n",
       "skip_2                                                             False   \n",
       "skip_3                                                              True   \n",
       "not_skipped                                                        False   \n",
       "context_switch                                                         0   \n",
       "no_pause_before_play                                                   0   \n",
       "short_pause_before_play                                                0   \n",
       "long_pause_before_play                                                 0   \n",
       "hist_user_behavior_n_seekfwd                                           0   \n",
       "hist_user_behavior_n_seekback                                          0   \n",
       "hist_user_behavior_is_shuffle                                      False   \n",
       "hour_of_day                                                           11   \n",
       "date                                                          2018-08-22   \n",
       "premium                                                             True   \n",
       "context_type                                                     catalog   \n",
       "hist_user_behavior_reason_start                                trackdone   \n",
       "hist_user_behavior_reason_end                                     remote   \n",
       "\n",
       "                                                                       1  \\\n",
       "session_id                       38_00006771-7eb1-4024-b52f-014b3b7b0488   \n",
       "session_position                                                       2   \n",
       "session_length                                                        17   \n",
       "track_id_clean                    t_889887f2-02b4-4cb7-8207-c7df6b115854   \n",
       "skip_1                                                             False   \n",
       "skip_2                                                             False   \n",
       "skip_3                                                             False   \n",
       "not_skipped                                                         True   \n",
       "context_switch                                                         1   \n",
       "no_pause_before_play                                                   1   \n",
       "short_pause_before_play                                                0   \n",
       "long_pause_before_play                                                 0   \n",
       "hist_user_behavior_n_seekfwd                                           0   \n",
       "hist_user_behavior_n_seekback                                          0   \n",
       "hist_user_behavior_is_shuffle                                      False   \n",
       "hour_of_day                                                           11   \n",
       "date                                                          2018-08-22   \n",
       "premium                                                             True   \n",
       "context_type                                                     catalog   \n",
       "hist_user_behavior_reason_start                                 clickrow   \n",
       "hist_user_behavior_reason_end                                  trackdone   \n",
       "\n",
       "                                                                       2  \\\n",
       "session_id                       38_00006771-7eb1-4024-b52f-014b3b7b0488   \n",
       "session_position                                                       3   \n",
       "session_length                                                        17   \n",
       "track_id_clean                    t_f36f81cd-c827-43c2-82a0-78bdee27d503   \n",
       "skip_1                                                             False   \n",
       "skip_2                                                             False   \n",
       "skip_3                                                             False   \n",
       "not_skipped                                                         True   \n",
       "context_switch                                                         0   \n",
       "no_pause_before_play                                                   1   \n",
       "short_pause_before_play                                                0   \n",
       "long_pause_before_play                                                 0   \n",
       "hist_user_behavior_n_seekfwd                                           0   \n",
       "hist_user_behavior_n_seekback                                          0   \n",
       "hist_user_behavior_is_shuffle                                      False   \n",
       "hour_of_day                                                           11   \n",
       "date                                                          2018-08-22   \n",
       "premium                                                             True   \n",
       "context_type                                                     catalog   \n",
       "hist_user_behavior_reason_start                                trackdone   \n",
       "hist_user_behavior_reason_end                                  trackdone   \n",
       "\n",
       "                                                                       3  \\\n",
       "session_id                       38_00006771-7eb1-4024-b52f-014b3b7b0488   \n",
       "session_position                                                       4   \n",
       "session_length                                                        17   \n",
       "track_id_clean                    t_830a9de3-ddba-44b8-84be-a5472e925bc2   \n",
       "skip_1                                                             False   \n",
       "skip_2                                                             False   \n",
       "skip_3                                                              True   \n",
       "not_skipped                                                        False   \n",
       "context_switch                                                         0   \n",
       "no_pause_before_play                                                   0   \n",
       "short_pause_before_play                                                1   \n",
       "long_pause_before_play                                                 1   \n",
       "hist_user_behavior_n_seekfwd                                           0   \n",
       "hist_user_behavior_n_seekback                                          0   \n",
       "hist_user_behavior_is_shuffle                                      False   \n",
       "hour_of_day                                                           11   \n",
       "date                                                          2018-08-22   \n",
       "premium                                                             True   \n",
       "context_type                                                     catalog   \n",
       "hist_user_behavior_reason_start                                trackdone   \n",
       "hist_user_behavior_reason_end                                    endplay   \n",
       "\n",
       "                                                                       4  \n",
       "session_id                       38_00006771-7eb1-4024-b52f-014b3b7b0488  \n",
       "session_position                                                       5  \n",
       "session_length                                                        17  \n",
       "track_id_clean                    t_a1817b44-24c4-4289-bde4-046a1315f28d  \n",
       "skip_1                                                             False  \n",
       "skip_2                                                             False  \n",
       "skip_3                                                              True  \n",
       "not_skipped                                                        False  \n",
       "context_switch                                                         1  \n",
       "no_pause_before_play                                                   1  \n",
       "short_pause_before_play                                                0  \n",
       "long_pause_before_play                                                 0  \n",
       "hist_user_behavior_n_seekfwd                                           0  \n",
       "hist_user_behavior_n_seekback                                          0  \n",
       "hist_user_behavior_is_shuffle                                      False  \n",
       "hour_of_day                                                           11  \n",
       "date                                                          2018-08-22  \n",
       "premium                                                             True  \n",
       "context_type                                             user_collection  \n",
       "hist_user_behavior_reason_start                                 clickrow  \n",
       "hist_user_behavior_reason_end                                    endplay  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_set.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}