Update R docs to pass checks

r/Makefile

@@ -34,7 +34,7 @@ build: doc
 	R CMD build .
 
 check: build
-	-export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE && export ARROW_R_DEV=$(ARROW_R_DEV) && R CMD check --as-cran --run-donttest arrow_$(VERSION).tar.gz
+	-export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE && export ARROW_R_DEV=$(ARROW_R_DEV) && export _R_CHECK_TESTS_NLINES_=0 && R CMD check --as-cran --run-donttest arrow_$(VERSION).tar.gz
 	rm -rf arrow.Rcheck/
 
 clean:

r/NAMESPACE

@@ -137,6 +137,7 @@ export(cast_options)
 export(chunked_array)
 export(codec_is_available)
 export(contains)
+export(data_source)
 export(date32)
 export(date64)
 export(decimal)

r/R/dataset.R

@@ -17,11 +17,12 @@
 
 #' Open a multi-file dataset
 #'
-#' @param sources String path to a directory containing the data files
+#' @param sources Either (1) a string path to a directory containing data files,
+#' or (2) a list of `DataSourceDiscovery` objects as created by [data_source()].
 #' @param schema [Schema] for the dataset. If `NULL` (the default), the schema
-#' will be inferred from the files
-#' @param partition One of
-#'   * A `Schema`, in which case the file paths relative to `path` will be
+#' will be inferred from the data sources.
+#' @param partition When `sources` is a file path, one of
+#'   * A `Schema`, in which case the file paths relative to `sources` will be
 #'    parsed, and path segments will be matched with the schema fields. For
 #'    example, `schema(year = int16(), month = int8())` would create partitions
 #'    for file paths like "2019/01/file.parquet", "2019/02/file.parquet", etc.
@@ -32,7 +33,8 @@
 #'    by [hive_partition()] which parses explicit or autodetected fields from
 #'    Hive-style path segments
 #'   * `NULL` for no partitioning
-#' @param ... additional arguments passed to `DataSourceDiscovery$create()`
+#' @param ... additional arguments passed to `data_source()`, when `sources` is
+#' a file path, otherwise ignored.
 #' @return A [Dataset] R6 object. Use `dplyr` methods on it to query the data,
 #' or call `$NewScan()` to construct a query directly.
 #' @export
@@ -40,7 +42,7 @@
 #' @include arrow-package.R
 open_dataset <- function(sources, schema = NULL, partition = hive_partition(), ...) {
   if (is.character(sources)) {
-    sources <- list(data_source(sources, partition, ...))
+    sources <- list(data_source(sources, partition = partition, ...))
   }
   assert_is_list_of(sources, "DataSourceDiscovery")
   if (is.null(schema)) {
@@ -51,21 +53,6 @@ open_dataset <- function(sources, schema = NULL, partition = hive_partition(), .
   Dataset$create(map(sources, ~.$Finish(schema)), schema)
 }
 
-data_source <- function(path, partition = hive_partition(), ...) {
-  if (!is.null(partition)) {
-    if (inherits(partition, "Schema")) {
-      partition <- SchemaPartitionScheme$create(partition)
-    } else if (is.character(partition)) {
-      # These are the column/field names, and we should autodetect their types
-      partition <- SchemaPartitionSchemeDiscovery$create(partition)
-    }
-    assert_is(partition, c("PartitionScheme", "PartitionSchemeDiscovery"))
-  }
-  # Note that we technically return a DataSourceDiscovery, not DataSource
-  # This is because we may need to set a common schema across multiple Sources
-  DataSourceDiscovery$create(path, partition_scheme = partition, ...)
-}
-
 #' Multi-file datasets
 #'
 #' @description
@@ -117,17 +104,12 @@ names.Dataset <- function(x) names(x$schema)
 #' `FileSystemDataSourceDiscovery` is a subclass of `DataSourceDiscovery` for
 #' discovering files in the local file system, the only currently supported
 #' file system.
+#'
+#' In general, you'll deal with `DataSourceDiscovery` rather than `DataSource`
+#' itself.
 #' @section Factory:
-#' The `DataSourceDiscovery$create()` factory method instantiates a
-#' `DataSourceDiscovery` and takes the following arguments:
-#' * `path`: A string file path containing data files
-#' * `filesystem`: Currently only "local" is supported
-#' * `format`: Currently only "parquet" is supported
-#' * `allow_non_existent`: logical: is `path` allowed to not exist? Default
-#' `FALSE`. See [FileSelector].
-#' * `recursive`: logical: should files be discovered in subdirectories of
-#' * `path`? Default `TRUE`.
-#' * `...` Additional arguments passed to the [FileSystem] `$create()` method
+#' For the `DataSourceDiscovery$create()` factory method, see [data_source()],
+#' an alias for it.
 #'
 #' `FileSystemDataSourceDiscovery$create()` is a lower-level factory method and
 #' takes the following arguments:
@@ -135,7 +117,9 @@ names.Dataset <- function(x) names(x$schema)
 #' * `selector`: A [FileSelector]
 #' * `format`: Currently only "parquet" is supported
 #' @section Methods:
-#' `DataSource` has no defined methods. It is just passed to `Dataset$create()`.
+#' `DataSource` has one defined method:
+#'
+#' - `$schema`: Active binding, returns the [Schema] of the DataSource
 #'
 #' `DataSourceDiscovery` and its subclasses have the following methods:
 #'
@@ -150,7 +134,6 @@ DataSource <- R6Class("DataSource", inherit = Object,
     #' @description
     #' Return the DataSource's `Schema`
     schema = function() {
-      print("hello")
       shared_ptr(Schema, dataset___DataSource__schema(self))
     }
   )
@@ -175,9 +158,9 @@ DataSourceDiscovery <- R6Class("DataSourceDiscovery", inherit = Object,
 DataSourceDiscovery$create <- function(path,
                                        filesystem = c("auto", "local"),
                                        format = c("parquet", "arrow", "ipc"),
+                                       partition = NULL,
                                        allow_non_existent = FALSE,
                                        recursive = TRUE,
-                                       partition_scheme = NULL,
                                        ...) {
   if (!inherits(filesystem, "FileSystem")) {
     filesystem <- match.arg(filesystem)
@@ -195,11 +178,51 @@ DataSourceDiscovery$create <- function(path,
     allow_non_existent = allow_non_existent,
     recursive = recursive
   )
+
   format <- FileFormat$create(match.arg(format))
-  # This may also require different initializers
-  FileSystemDataSourceDiscovery$create(filesystem, selector, format, partition_scheme)
+
+  if (!is.null(partition)) {
+    if (inherits(partition, "Schema")) {
+      partition <- SchemaPartitionScheme$create(partition)
+    } else if (is.character(partition)) {
+      # These are the column/field names, and we should autodetect their types
+      partition <- SchemaPartitionSchemeDiscovery$create(partition)
+    }
+  }
+  FileSystemDataSourceDiscovery$create(filesystem, selector, format, partition)
 }
 
+#' Create a DataSource for a Dataset
+#'
+#' @param path A string file path containing data files
+#' @param filesystem A string identifier for the filesystem corresponding to
+#' `path`. Currently only "local" is supported.
+#' @param format A string identifier of the format of the files in `path`.
+#' Currently supported options are "parquet", "arrow", and "ipc" (an alias for
+#' the Arrow file format)
+#' @param partition One of
+#'   * A `Schema`, in which case the file paths relative to `sources` will be
+#'    parsed, and path segments will be matched with the schema fields. For
+#'    example, `schema(year = int16(), month = int8())` would create partitions
+#'    for file paths like "2019/01/file.parquet", "2019/02/file.parquet", etc.
+#'   * A character vector that defines the field names corresponding to those
+#'    path segments (that is, you're providing the names that would correspond
+#'    to a `Schema` but the types will be autodetected)
+#'   * A `HivePartitionScheme` or `HivePartitionSchemeDiscovery`, as returned
+#'    by [hive_partition()] which parses explicit or autodetected fields from
+#'    Hive-style path segments
+#'   * `NULL` for no partitioning
+#' @param allow_non_existent logical: is `path` allowed to not exist? Default
+#' `FALSE`. See [FileSelector].
+#' @param recursive logical: should files be discovered in subdirectories of
+#' `path`? Default `TRUE`.
+#' @param ... Additional arguments passed to the [FileSystem] `$create()` method
+#' @return A `DataSourceDiscovery` object. Pass this to [open_dataset()],
+#' in a list potentially with other `DataSourceDiscovery` objects, to create
+#' a `Dataset`.
+#' @export
+data_source <- DataSourceDiscovery$create
+
 #' @usage NULL
 #' @format NULL
 #' @rdname DataSource

r/README.md

@@ -37,15 +37,11 @@ Conda users on Linux and macOS can install `arrow` from conda-forge with
     conda install -c conda-forge r-arrow
 
 On macOS and Windows, installing a binary package from CRAN will handle
-Arrow’s C++ dependencies for you. On Linux, unless you use `conda`
-you’ll need to first install the C++ library. See the [Arrow project
-installation page](https://arrow.apache.org/install/) to find
-pre-compiled binary packages for some common Linux distributions,
-including Debian, Ubuntu, and CentOS. You’ll need to install
-`libparquet-dev` on Debian and Ubuntu, or `parquet-devel` on CentOS.
-This will also automatically install the Arrow C++ library as a
-dependency. Other Linux distributions must install the C++ library from
-source.
+Arrow’s C++ dependencies for you. On Linux, unless you use `conda`, the
+R package will have to compile its bindings from source and it will need
+to find or download the C++ dependencies. As of the 0.16.0 release, this
+dependency resolution is automatic on most common Linux distributions.
+See `vignette("install", package = "arrow")` for details.
 
 If you install the `arrow` package from source and the C++ library is
 not found, the R package functions will notify you that Arrow is not
@@ -122,16 +118,13 @@ Binary R packages for macOS and Windows are built daily and hosted at
 <https://dl.bintray.com/ursalabs/arrow-r/>. To install from there:
 
 ``` r
-install.packages("arrow", repos="https://dl.bintray.com/ursalabs/arrow-r")
+install.packages("arrow", repos = "https://dl.bintray.com/ursalabs/arrow-r")
 ```
 
 These daily package builds are not official Apache releases and are not
 recommended for production use. They may be useful for testing bug fixes
 and new features under active development.
 
-Linux users will need to build the Arrow C++ library from source, then
-install the R package from source, as described in the next section.
-
 ## Developing
 
 Windows and macOS users who wish to contribute to the R package and
@@ -202,7 +195,8 @@ Arrow C++ was put in `make install`, e.g. `export
 R_LD_LIBRARY_PATH=/usr/local/lib`, and retry installing the R package.
 
 For any other build/configuration challenges, see the [C++ developer
-guide](https://arrow.apache.org/docs/developers/cpp.html#building).
+guide](https://arrow.apache.org/docs/developers/cpp.html#building) and
+`vignette("install", package = "arrow")`.
 
 ### Editing Rcpp code
 
@@ -255,6 +249,6 @@ from the command line (`make test`, `make doc`, `make clean`, etc.)
 ### Full package validation
 
 ``` shell
-R CMD build --keep-empty-dirs .
-R CMD check arrow_*.tar.gz --as-cran --no-manual
+R CMD build .
+R CMD check arrow_*.tar.gz --as-cran
 ```

r/man/DataSource.Rd

@@ -13,21 +13,16 @@ scheme. \code{DataSourceDiscovery} is used to create a \code{DataSource}, inspec
 \code{FileSystemDataSourceDiscovery} is a subclass of \code{DataSourceDiscovery} for
 discovering files in the local file system, the only currently supported
 file system.
+
+In general, you'll deal with \code{DataSourceDiscovery} rather than \code{DataSource}
+itself.
+
+Return the DataSource's \code{Schema}
 }
 \section{Factory}{
 
-The \code{DataSourceDiscovery$create()} factory method instantiates a
-\code{DataSourceDiscovery} and takes the following arguments:
-\itemize{
-\item \code{path}: A string file path containing data files
-\item \code{filesystem}: Currently only "local" is supported
-\item \code{format}: Currently only "parquet" is supported
-\item \code{allow_non_existent}: logical: is \code{path} allowed to not exist? Default
-\code{FALSE}. See \link{FileSelector}.
-\item \code{recursive}: logical: should files be discovered in subdirectories of
-\item \code{path}? Default \code{TRUE}.
-\item \code{...} Additional arguments passed to the \link{FileSystem} \verb{$create()} method
-}
+For the \code{DataSourceDiscovery$create()} factory method, see \code{\link[=data_source]{data_source()}},
+an alias for it.
 
 \code{FileSystemDataSourceDiscovery$create()} is a lower-level factory method and
 takes the following arguments:
@@ -40,7 +35,10 @@ takes the following arguments:
 
 \section{Methods}{
 
-\code{DataSource} has no defined methods. It is just passed to \code{Dataset$create()}.
+\code{DataSource} has one defined method:
+\itemize{
+\item \verb{$schema}: Active binding, returns the \link{Schema} of the DataSource
+}
 
 \code{DataSourceDiscovery} and its subclasses have the following methods:
 \itemize{

r/man/data_source.Rd

@@ -0,0 +1,57 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/dataset.R
+\name{data_source}
+\alias{data_source}
+\title{Create a DataSource for a Dataset}
+\usage{
+data_source(
+  path,
+  filesystem = c("auto", "local"),
+  format = c("parquet", "arrow", "ipc"),
+  partition = NULL,
+  allow_non_existent = FALSE,
+  recursive = TRUE,
+  ...
+)
+}
+\arguments{
+\item{path}{A string file path containing data files}
+
+\item{filesystem}{A string identifier for the filesystem corresponding to
+\code{path}. Currently only "local" is supported.}
+
+\item{format}{A string identifier of the format of the files in \code{path}.
+Currently supported options are "parquet", "arrow", and "ipc" (an alias for
+the Arrow file format)}
+
+\item{partition}{One of
+\itemize{
+\item A \code{Schema}, in which case the file paths relative to \code{sources} will be
+parsed, and path segments will be matched with the schema fields. For
+example, \code{schema(year = int16(), month = int8())} would create partitions
+for file paths like "2019/01/file.parquet", "2019/02/file.parquet", etc.
+\item A character vector that defines the field names corresponding to those
+path segments (that is, you're providing the names that would correspond
+to a \code{Schema} but the types will be autodetected)
+\item A \code{HivePartitionScheme} or \code{HivePartitionSchemeDiscovery}, as returned
+by \code{\link[=hive_partition]{hive_partition()}} which parses explicit or autodetected fields from
+Hive-style path segments
+\item \code{NULL} for no partitioning
+}}
+
+\item{allow_non_existent}{logical: is \code{path} allowed to not exist? Default
+\code{FALSE}. See \link{FileSelector}.}
+
+\item{recursive}{logical: should files be discovered in subdirectories of
+\code{path}? Default \code{TRUE}.}
+
+\item{...}{Additional arguments passed to the \link{FileSystem} \verb{$create()} method}
+}
+\value{
+A \code{DataSourceDiscovery} object. Pass this to \code{\link[=open_dataset]{open_dataset()}},
+in a list potentially with other \code{DataSourceDiscovery} objects, to create
+a \code{Dataset}.
+}
+\description{
+Create a DataSource for a Dataset
+}

r/man/open_dataset.Rd

@@ -4,17 +4,18 @@
 \alias{open_dataset}
 \title{Open a multi-file dataset}
 \usage{
-open_dataset(path, schema = NULL, partition = hive_partition(), ...)
+open_dataset(sources, schema = NULL, partition = hive_partition(), ...)
 }
 \arguments{
-\item{path}{String path to a directory containing the data files}
+\item{sources}{Either (1) a string path to a directory containing data files,
+or (2) a list of \code{DataSourceDiscovery} objects as created by \code{\link[=data_source]{data_source()}}.}
 
 \item{schema}{\link{Schema} for the dataset. If \code{NULL} (the default), the schema
-will be inferred from the files}
+will be inferred from the data sources.}
 
-\item{partition}{One of
+\item{partition}{When \code{sources} is a file path, one of
 \itemize{
-\item A \code{Schema}, in which case the file paths relative to \code{path} will be
+\item A \code{Schema}, in which case the file paths relative to \code{sources} will be
 parsed, and path segments will be matched with the schema fields. For
 example, \code{schema(year = int16(), month = int8())} would create partitions
 for file paths like "2019/01/file.parquet", "2019/02/file.parquet", etc.
@@ -27,7 +28,8 @@ Hive-style path segments
 \item \code{NULL} for no partitioning
 }}
 
-\item{...}{additional arguments passed to \code{DataSourceDiscovery$create()}}
+\item{...}{additional arguments passed to \code{data_source()}, when \code{sources} is
+a file path, otherwise ignored.}
 }
 \value{
 A \link{Dataset} R6 object. Use \code{dplyr} methods on it to query the data,

r/tests/testthat/test-dataset.R

@@ -271,7 +271,6 @@ test_that("Assembling a Dataset manually and getting a Table", {
   datasource <- dsd$Finish(schm)
   expect_is(datasource, "DataSource")
   expect_is(datasource$schema, "Schema")
-  print(str(datasource$schema))
   expect_equal(names(schm), names(datasource$schema))
 
   ds <- Dataset$create(list(datasource), schm)