mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 08:35:46 +08:00
GH-38217: [R] Map format = "text" to format = "csv" in write_dataset() (#50367)
### Rationale for this change `write_dataset()` errors on `format="text"` even though `open_dataset()` allows it. ### What changes are included in this PR? Map "text" to "csv" so `write_dataset()` mirrors `open_dataset()` ### Are these changes tested? Yes ### Are there any user-facing changes? No * GitHub Issue: #38217 Authored-by: Nic Crane <thisisnic@gmail.com> Signed-off-by: Nic Crane <thisisnic@gmail.com>
This commit is contained in:
1 parent
9a18992678
commit
0b250b73d4
4 files changed
+57
-27
No files matched your search
@@ -78,7 +78,13 @@ FileFormat <- R6Class(
|
||||
|
||||
FileFormat$create <- function(format, schema = NULL, partitioning = NULL, ...) {
|
||||
opt_names <- names(list(...))
|
||||
if (format %in% c("csv", "text", "txt") || any(opt_names %in% c("delim", "delimiter"))) {
|
||||
if (format == "txt") {
|
||||
.Deprecated(
|
||||
msg = '`format = "txt"` is deprecated; use `format = "csv"` instead.'
|
||||
)
|
||||
format <- "csv"
|
||||
}
|
||||
if (format %in% c("csv", "text") || any(opt_names %in% c("delim", "delimiter"))) {
|
||||
CsvFileFormat$create(schema = schema, partitioning = partitioning, ...)
|
||||
} else if (format == "tsv") {
|
||||
# This delimiter argument is ignored.
|
||||
@@ -691,7 +697,7 @@ FileWriteOptions <- R6Class(
|
||||
"codec",
|
||||
"null_fallback"
|
||||
)
|
||||
} else if (format %in% c("csv", "tsv", "txt", "text")) {
|
||||
} else if (format %in% c("csv", "tsv", "text")) {
|
||||
supported_args <- c(
|
||||
names(formals(csv_write_options)),
|
||||
names(formals(readr_to_csv_write_options))
|
||||
@@ -747,7 +753,7 @@ FileWriteOptions <- R6Class(
|
||||
get_ipc_metadata_version(args$metadata_version)
|
||||
)
|
||||
}
|
||||
} else if (self$type %in% c("csv", "tsv", "txt", "text")) {
|
||||
} else if (self$type %in% c("csv", "tsv", "text")) {
|
||||
arrow_opts <- names(formals(csv_write_options))
|
||||
readr_opts <- names(formals(readr_to_csv_write_options))
|
||||
readr_only_opts <- setdiff(readr_opts, arrow_opts)
|
||||
|
||||
+9
-4
@@ -148,9 +148,17 @@ write_dataset <- function(
|
||||
msg = '`format = "feather"` is deprecated; use `format = "ipc"` instead.'
|
||||
)
|
||||
}
|
||||
if (format == "txt") {
|
||||
.Deprecated(
|
||||
msg = '`format = "txt"` is deprecated; use `format = "csv"` instead.'
|
||||
)
|
||||
}
|
||||
if (format %in% c("feather", "ipc")) {
|
||||
format <- "arrow"
|
||||
}
|
||||
if (format %in% c("text", "txt")) {
|
||||
format <- "csv"
|
||||
}
|
||||
if (inherits(dataset, "arrow_dplyr_query")) {
|
||||
# partitioning vars need to be in the `select` schema
|
||||
dataset <- ensure_group_vars(dataset)
|
||||
@@ -194,9 +202,6 @@ write_dataset <- function(
|
||||
path_and_fs <- get_path_and_filesystem(path)
|
||||
|
||||
dots <- list(...)
|
||||
if (format %in% c("txt", "text") && !any(c("delimiter", "delim") %in% names(dots))) {
|
||||
stop("A delimiter must be given for a txt format.")
|
||||
}
|
||||
if (format == "tsv" && any(c("delimiter", "delim") %in% names(dots))) {
|
||||
stop("Can't set a delimiter for the tsv format.")
|
||||
}
|
||||
@@ -316,7 +321,7 @@ write_delim_dataset <- function(
|
||||
write_dataset(
|
||||
dataset = dataset,
|
||||
path = path,
|
||||
format = "txt",
|
||||
format = "text",
|
||||
partitioning = partitioning,
|
||||
basename_template = basename_template,
|
||||
hive_style = hive_style,
|
||||
|
||||
@@ -736,3 +736,11 @@ test_that("more informative error when column inferred as null due to sparse dat
|
||||
"column type being inferred as"
|
||||
)
|
||||
})
|
||||
|
||||
test_that("FileFormat$create(format = 'txt') is deprecated and maps to csv", {
|
||||
expect_warning(
|
||||
fmt <- FileFormat$create("txt"),
|
||||
"deprecated"
|
||||
)
|
||||
expect_r6_class(fmt, "CsvFileFormat")
|
||||
})
|
||||
@@ -906,26 +906,6 @@ test_that("max_rows_per_group is adjusted if at odds with max_rows_per_file in w
|
||||
)
|
||||
})
|
||||
|
||||
test_that("Writing a flat file dataset without a delimiter throws an error.", {
|
||||
df <- tibble(
|
||||
int = 1:10,
|
||||
dbl = as.numeric(1:10),
|
||||
lgl = rep(c(TRUE, FALSE, NA, TRUE, FALSE), 2),
|
||||
chr = letters[1:10],
|
||||
)
|
||||
|
||||
dst_dir <- make_temp_dir()
|
||||
expect_error(
|
||||
write_dataset(df, dst_dir, format = "txt"),
|
||||
"A delimiter must be given for a txt format."
|
||||
)
|
||||
|
||||
expect_error(
|
||||
write_dataset(df, dst_dir, format = "text"),
|
||||
"A delimiter must be given for a txt format."
|
||||
)
|
||||
})
|
||||
|
||||
test_that("Dataset can write flat files using readr::write_csv() options.", {
|
||||
df <- tibble(
|
||||
int = 1:10,
|
||||
@@ -1054,3 +1034,34 @@ test_that("Row order is preserved when writing large parquet dataset", {
|
||||
# But ordered is exactly equal.
|
||||
expect_equal(ordered_ds$x, df$x)
|
||||
})
|
||||
|
||||
test_that("write_dataset maps format 'text' to 'csv' (GH-38217)", {
|
||||
df <- tibble(int = 1:10, dbl = as.numeric(1:10), chr = letters[1:10])
|
||||
|
||||
dst_dir <- make_temp_dir()
|
||||
write_dataset(df, dst_dir, format = "text")
|
||||
|
||||
expect_identical(dir(dst_dir), "part-0.csv")
|
||||
expect_equal(open_dataset(dst_dir, format = "csv") |> collect(), df)
|
||||
})
|
||||
|
||||
test_that("write_dataset format = 'txt' is deprecated and maps to 'csv'", {
|
||||
df <- tibble(int = 1:10, dbl = as.numeric(1:10), chr = letters[1:10])
|
||||
|
||||
dst_dir <- make_temp_dir()
|
||||
expect_warning(
|
||||
write_dataset(df, dst_dir, format = "txt"),
|
||||
"deprecated"
|
||||
)
|
||||
|
||||
expect_identical(dir(dst_dir), "part-0.csv")
|
||||
expect_equal(open_dataset(dst_dir, format = "csv") |> collect(), df)
|
||||
|
||||
# A delimiter passed alongside "txt" still reaches the writer
|
||||
dst_dir2 <- make_temp_dir()
|
||||
expect_warning(
|
||||
write_dataset(df, dst_dir2, format = "txt", delimiter = ";"),
|
||||
"deprecated"
|
||||
)
|
||||
expect_equal(open_dataset(dst_dir2, format = "csv", delimiter = ";") |> collect(), df)
|
||||
})
|
||||
Reference in new issue
Block a user