Merge pull request #44 from quandl/AP-1699/datatables

Ap 1699/datatables

.travis.yml

@@ -1,4 +1,8 @@
 language: r
+r:
+  - oldrel
+  - release
+cache: packages
 # Be strict when checking our package
 warnings_are_errors: true
 sudo: required

ChangeLog

@@ -1,3 +1,9 @@
+##############################
+####Version 2.7.0 to 2.8.0####
+##############################
+
+- Add functionality for retrieving datatable data
+
 ##############################
 ####Version 2.6.1 to 2.7.0####
 ##############################

DESCRIPTION

@@ -1,13 +1,13 @@
 Package: Quandl
 Title: API Wrapper for Quandl.com
-Version: 2.7.0
-Date: 2015-08-11
+Version: 2.8.0
+Date: 2016-04-05
 Authors@R: c(person(
-    "Raymond McTaggart", role=c("aut", "cre"), email="raymond@quandl.com"),
+    "Raymond McTaggart", role=c("aut", "cre"), email="ray@quandl.com"),
     person("Gergely Daroczi", role="aut"),
     person("Clement Leung", role="aut", email="clement@quandl.com"),
     person("Quandl Inc.", role="cph"))
-Maintainer: Raymond McTaggart <raymond@quandl.com>
+Maintainer: Raymond McTaggart <ray@quandl.com>
 Description: Functions for interacting directly with the Quandl API to offer data in a number of formats usable in R, downloading a zip with all data from a Quandl database, and the ability to search. This R package uses the Quandl API. For more information go to https://www.quandl.com/docs/api. For more help on the package itself go to https://www.quandl.com/help/r.
 Imports:
     httr (>= 0.6.1),

LICENSE

@@ -1,2 +1,2 @@
-YEAR: 2015
+YEAR: 2016
 COPYRIGHT HOLDER: Quandl Inc.

NAMESPACE

@@ -6,6 +6,7 @@ export(Quandl.auth)
 export(Quandl.database.bulk_download_to_file)
 export(Quandl.database.bulk_download_url)
 export(Quandl.dataset.get)
+export(Quandl.datatable)
 export(Quandl.search)
 export(metaData)
 export(quandl.api)

R/Quandlapi.R

@@ -40,6 +40,9 @@ quandl.api.download_file <- function(path, filename, ...) {
 
 quandl.api.build_request <- function(path, ...) {
   params <- list(...)
+  # ensure vectors get converted into v3 api supported query params
+  # e.g., qopts.columns=c('ticker', 'rev') -> list('qopts.columns[]'=ticker,'qopts.columns[]'=rev)
+  params <- quandl.api.build_query_params(params)
   # ensure Dates convert to characters or else curl will convert the Dates to timestamp
   params <- quandl.api.convert_dates_to_character(params)
 
@@ -80,3 +83,34 @@ quandl.api.convert_dates_to_character <- function(params) {
   }
   lapply(params, convert_date_to_character)
 }
+
+quandl.api.build_query_params <- function(params) {
+  if (length(params) <= 0) {
+    return(params)
+  }
+  mod_params <- list()
+  for(i in 1:length(params)) {
+    # keep the params the same if not a vector
+    converted_params <- params[i]
+
+    # check val to see if vector
+    # if so, convert
+    if (length(params[[i]]) > 1) {
+      converted_params <- quandl.api.convert_vector_params(names(params[i]), params[[i]])
+    }
+    mod_params <- c(mod_params, converted_params)
+  }
+  return(mod_params)
+}
+
+quandl.api.convert_vector_params <- function(name, vector_values) {
+  mod_query_name <- paste0(name, '[]')
+  mod_query_list <- list()
+
+  for(val in vector_values) {
+    l <- list()
+    l[[mod_query_name]] <- val
+    mod_query_list <- c(mod_query_list, l)
+  }
+  return(mod_query_list)
+}

R/Quandldatabase.R

@@ -51,13 +51,7 @@ Quandl.database.bulk_download_url <- function(database_code, ...) {
 #' }
 #' @export
 Quandl.database.bulk_download_to_file <- function(database_code, filename, ...) {
-  if (dir.exists(filename)) {
-    stop("Please add a filename to your directory path, e.g., ", filename, '/', database_code, '.zip', call. = FALSE)
-  }
   dirname <- dirname(filename)
-  if (!dir.exists(dirname)) {
-    stop(dirname, " directory does not exist!", call. = FALSE)
-  }
   quandl.api.download_file(Quandl.database.download_url_path(database_code), filename = filename, ...)
 }
 

R/Quandldatatable.R

@@ -0,0 +1,73 @@
+#' Retrieves Data from the Quandl Datatable endpoint
+#'
+#' @details Set your \code{api_key} with \code{Quandl.api_key} function. For instructions on finding your api key go to \url{https://www.quandl.com/account/api}
+#'
+#' @param code Datatable code on Quandl specified as a string.
+#' @param paginate When set to TRUE, fetches up to 1,000,000 rows of data
+#' @param ... Additional named values that are interpreted as Quandl API parameters.
+#' @return Returns a data.frame.
+#' @seealso \code{\link{Quandl.api_key}}
+#' @examples \dontrun{
+#' Quandl.datatable('ZACKS/FC', paginate=TRUE)
+#' }
+#' @export
+Quandl.datatable <- function(code, paginate=FALSE, ...) {
+  path <- paste0("datatables/", code)
+  params <- list(...)
+
+  # make request for first page of data
+  json <- do.call(quandl.api, c(path=path, params))
+  datatable <- json$datatable
+  data <- datatable$data
+  # contains a list of names and corresponding types
+  columns <- datatable$columns
+  next_cursor_id <- json$meta$next_cursor_id
+  df <- as.data.frame(data, stringsAsFactors=FALSE)
+
+  # continue to make requests for data if paginate=TRUE and there is data
+  while (isTRUE(paginate) && !is.null(next_cursor_id)) {
+    params['qopts.cursor_id'] <- next_cursor_id
+    json <- do.call(quandl.api, c(path=path, params))
+    df_page <- as.data.frame(json$datatable$data, stringsAsFactors=FALSE)
+    df <- rbind(df, df_page)
+    next_cursor_id <- json$meta$next_cursor_id
+
+    # only fetch a maximum of 1,000,000 rows
+    if (nrow(df) >= quandl.datatable.max_rows() && !is.null(next_cursor_id)) {
+      warning(paste("This call returns a larger amount of data than Quandl.datatable() allows.",
+                    "Please view our documentation on developer methods to request more data.",
+                    "https://github.com/quandl/quandl-r/blob/master/README.md#datatables"), call. = FALSE)
+      break
+    }
+  }
+
+  if (!isTRUE(paginate) && !is.null(next_cursor_id)) {
+    warning(paste("This call returns more data. To request more pages, please set paginate=TRUE",
+                  "in your Quandl.datatable() call. For more information see our documentation:",
+                  "https://github.com/quandl/quandl-r/blob/master/README.md#datatables"), call. = FALSE)
+  }
+
+  names(df) <- columns[,1]
+  df <- quandl.datatable.convert_df_columns(df, columns[,2])
+
+  return(df)
+}
+
+quandl.datatable.convert_df_columns <- function(df, column_types) {
+  if (length(column_types) <= 0) {
+    return(df)
+  }
+  column_types <- tolower(column_types)
+  for(i in 1:length(column_types)) {
+    if (grepl("^float|^bigdecimal|^integer", column_types[i])) {
+      df[,i] <- as.numeric(df[,i])
+    } else if (grepl("^date", column_types[i])) {
+      df[,i] <- as.Date(df[,i])
+    }
+  }
+  return(df)
+}
+
+quandl.datatable.max_rows <- function() {
+  return(1000000)
+}

README.md

@@ -1,15 +1,15 @@
-Quandl R Package
+Quandl R Package [![Build Status](https://travis-ci.org/quandl/quandl-r.svg?branch=master)](https://travis-ci.org/quandl/quandl-r)
 =========
 
-This is Quandl's R package. The Quandl R package uses the [Quandl API](https://www.quandl.com/docs/api). The official Quandl R package manual can be found [here](https://cran.r-project.org/web/packages/Quandl/index.html).
+This is Quandl's R package. The Quandl R package uses the [Quandl API](https://www.quandl.com/docs/api). The official Quandl R package manual can be found [here](https://cran.r-project.org/package=Quandl).
 
-License: MIT
+License provided by MIT.
 
 For more information please contact raymond@quandl.com
 
 # Installation
 
-Using the [devtools](https://cran.r-project.org/web/packages/devtools/index.html) package:
+To install the [devtools](https://cran.r-project.org/package=devtools) package:
 
     install.packages("devtools")
     library(devtools)
@@ -26,14 +26,12 @@ Note that the version on CRAN might not reflect the most recent changes made to
 
 # Authentication
 
-To extend your access to the Quandl API, use your [api key](https://www.quandl.com/docs/api#api-keys). To do this sign into your account (or create one) and go to your [account api key page](https://www.quandl.com/account/api). Then input your api key (with quotes):
+To make full use of the package we recommend you set your [api key](https://www.quandl.com/docs/api#api-keys). To do this create or sign into your account and go to your [account api key page](https://www.quandl.com/account/api). Then input your API key (with quotes):
 
 ```r
 Quandl.api_key("tEsTkEy123456789")
 ```
 
-This will then extend your usage.
-
 # Usage
 
 The Quandl package functions use the Quandl API. Optional Quandl API query parameters can be passed into each function. For more information on supported query parameters, please see the [Quandl API documentation page](https://www.quandl.com/docs/api). Once you find the data you would like to load into R on Quandl, copy the Quandl code from the description box and paste it into the function.
@@ -42,17 +40,11 @@ The Quandl package functions use the Quandl API. Optional Quandl API query param
 data <- Quandl("NSE/OIL")
 ```
 
-To reset the package to not use an api key:
-
-```r
-Quandl.api_key(NULL)
-```
-
 ## Graphing Data Example
-To create a graph of the Nasdaq, with a monthly frequency
+To create a graph of Google's performance month-over-month:
 
 ```r
-plot(stl(Quandl("GOOG/NASDAQ_GOOG",type="ts",collapse="monthly")[,1],s.window="per"))
+plot(stl(Quandl("WIKI/GOOG",type="ts",collapse="monthly")[,11],s.window="per"))
 ```
 
 Note: `collapse` is a Quandl API query parameter. Click [here](https://www.quandl.com/docs/api#retrieve-data-and-metadata) for a full list of query parameter options.  
@@ -62,9 +54,9 @@ Note: `collapse` is a Quandl API query parameter. Click [here](https://www.quand
 The supported return types for the `Quandl(code)` function are:
 * raw (which returns a data.frame)
 * [ts](https://stat.ethz.ch/R-manual/R-devel/library/stats/html/ts.html)
-* [zoo](https://cran.r-project.org/web/packages/zoo/index.html)
-* [xts](https://cran.r-project.org/web/packages/xts/index.html)
-* [timeSeries](https://cran.r-project.org/web/packages/timeSeries/index.html)
+* [zoo](https://cran.r-project.org/package=zoo)
+* [xts](https://cran.r-project.org/package=xts)
+* [timeSeries](https://cran.r-project.org/package=timeSeries)
 
 To request a specific type, assign the `type` argument the return type:
 
@@ -105,7 +97,7 @@ data <- Quandl('NSE/OIL', collapse = "quarterly", type = "zoo", limit = 3, force
 ```
 
 ## Merged Dataset Data
-To get a merged representation of multiple Quandl code data, specify a vector of Quandl codes:
+If you want to get multiple codes at once, delimit the codes with ',', and put them into an array. This will return a multiset.
 
 ```r
 merged_data <- Quandl(c('GOOG/NASDAQ_AAPL', 'GOOG/NASDAQ_MSFT'))
@@ -119,15 +111,37 @@ merged_data <- Quandl(c('GOOG/NASDAQ_AAPL.1', 'GOOG/NASDAQ_MSFT.2'))
 
 ## Downloading Entire Database
 
-An entire Database's data can be downloaded. For example, to download database `ZEA`:
+An entire database's data can be downloaded. For example, to download the database `ZEA`:
 
 ```r
 Quandl.database.bulk_download_to_file("ZEA", "./ZEA.zip")
 ```
 
-Please set your [api key](#authentication) to download [premium databases](https://www.quandl.com/search?type=premium) you are subscribed to.
+Note you must set your [api key](#authentication) to download [premium databases](https://www.quandl.com/search?type=premium) to which you are subscribed.
 
 For a full list of optional query parameters for downloading an entire database, click [here](https://www.quandl.com/docs/api#entire-database).
+
+## Datatables
+
+To retrieve Datatable data, provide a Datatable code to the Quandl datatables function:
+
+```r
+data = Quandl.datatable('ZACKS/FC')
+```
+
+The output format is `data.frame`. Given the volume of data stored in datatables, this call will retrieve the first page of the ZACKS/FC datatable. You may turn on pagination to return more data by using:
+
+```r
+data = Quandl.datatable('ZACKS/FC', paginate=TRUE)
+```
+
+This will retrieve multiple pages of data and merge them together as if they were one large page. In some cases, however, you will still exceed the request limit. In this case we recommend you filter your data using the available query parameters, as in the following example:
+
+```r
+Quandl.datatable('ZACKS/FC', ticker=c('AAPL', 'MSFT'), per_end_date.gt='2015-01-01', qopts.columns=c('ticker', 'per_end_date', 'tot_revnu'))
+```
+
+In this query we are asking for more pages of data, ticker values of either AAPL or MSFT and a per_end_date that is greater than or equal to 2015-01-01. We are also filtering the returned columns on ticker, per_end_date and tot_revnu rather than all available columns.
 
 ## Search
 Searching Quandl from within the R console is now supported. The search function is:

man/Quandl.datatable.Rd

@@ -0,0 +1,33 @@
+% Generated by roxygen2 (4.1.1): do not edit by hand
+% Please edit documentation in R/Quandldatatable.R
+\name{Quandl.datatable}
+\alias{Quandl.datatable}
+\title{Retrieves Data from the Quandl Datatable endpoint}
+\usage{
+Quandl.datatable(code, paginate = FALSE, ...)
+}
+\arguments{
+\item{code}{Datatable code on Quandl specified as a string.}
+
+\item{paginate}{When set to TRUE, fetches up to 1,000,000 rows of data}
+
+\item{...}{Additional named values that are interpreted as Quandl API parameters.}
+}
+\value{
+Returns a data.frame.
+}
+\description{
+Retrieves Data from the Quandl Datatable endpoint
+}
+\details{
+Set your \code{api_key} with \code{Quandl.api_key} function. For instructions on finding your api key go to \url{https://www.quandl.com/account/api}
+}
+\examples{
+\dontrun{
+Quandl.datatable('ZACKS/FC', paginate=TRUE)
+}
+}
+\seealso{
+\code{\link{Quandl.api_key}}
+}
+

tests/testthat/test-api.r

@@ -38,11 +38,22 @@ test_that('request headers and query params are constructed', {
   expected_headers <- list(
       Accept = 'application/json, application/vnd.quandl+json;version=2015-04-09',
       `Request-Source` = 'R',
-      `Request-Source-Version` = '2.7.0',
+      `Request-Source-Version` = '2.8.0',
       `X-Api-Token` = 'test_key'
     )
   expected_url <- "https://www.quandl.com/api/v3/datasets"
   expect_equal(results, list(request_url = expected_url, headers = expected_headers, params = expected_params))
 })
 
+context('quandl.api.build_query_params')
+test_that('query params with array values are converted properly', {
+  params <- list()
+  params$param1 <- 'foo'
+  params$param2 <- c('hello', 'world', 'bar')
+  params$param3 <- 'cool'
+
+  expected_params <- list(param1='foo', 'param2[]'='hello', 'param2[]'='world', 'param2[]'='bar', param3='cool')
+  expect_equal(quandl.api.build_query_params(params), expected_params)
+})
+
 reset_config()

tests/testthat/test-database.r

@@ -9,31 +9,6 @@ test_that("download database url is constructed correctly", {
 })
 
 with_mock(
-  # this is not a folder
-  dir.exists = function(paths) {
-    FALSE
-  },
-  test_that("bulk download errors when invalid filepath given", {
-    expect_error(Quandl.database.bulk_download_to_file("NSE", "path/does/not/exist"), "path/does/not directory does not exist!", fixed = TRUE)
-  })
-)
-
-with_mock(
-  dir.exists = function(paths) {
-    TRUE
-  },
-  test_that("bulk download errors when invalid filepath given", {
-    expect_error(Quandl.database.bulk_download_to_file("NSE", "path/is/folder"), "Please add a filename to your directory path, e.g., path/is/folder/NSE.zip", fixed = TRUE)
-  })
-)
-
-with_mock(
-  dir.exists = function(paths) {
-    if (paths == "folder/exists/NSE.zip") {
-      return(FALSE)
-    }
-    return(TRUE)
-  },
   `Quandl:::quandl.api.download_file` = function(path, filename, ...) {
     test_that("correct arguments are passed to api layer", {
       params <- list(...)
@@ -45,4 +20,4 @@ with_mock(
   Quandl.database.bulk_download_to_file("NSE", "folder/exists/NSE.zip", download_type = "partial")
 )
 
-reset_config()
+reset_config()