pkgname <- "arrow"
source(file.path(R.home("share"), "R", "examples-header.R"))
options(warn = 1)
library('arrow')

base::assign(".oldSearch", base::search(), pos = 'CheckExEnv')
base::assign(".old_wd", base::getwd(), pos = 'CheckExEnv')
cleanEx()
nameEx("Buffer-class")
### * Buffer-class

flush(stderr()); flush(stdout())

### Name: Buffer
### Title: Buffer class
### Aliases: Buffer

### ** Examples

my_buffer <- buffer(c(1, 2, 3, 4))
my_buffer$is_mutable
my_buffer$ZeroPadding()
my_buffer$size
my_buffer$capacity



cleanEx()
nameEx("ChunkedArray-class")
### * ChunkedArray-class

flush(stderr()); flush(stdout())

### Name: ChunkedArray
### Title: ChunkedArray class
### Aliases: ChunkedArray

### ** Examples

# Pass items into chunked_array as separate objects to create chunks
class_scores <- chunked_array(c(87, 88, 89), c(94, 93, 92), c(71, 72, 73))
class_scores$num_chunks

# When taking a Slice from a chunked_array, chunks are preserved
class_scores$Slice(2, length = 5)

# You can combine Take and SortIndices to return a ChunkedArray with 1 chunk
# containing all values, ordered.
class_scores$Take(class_scores$SortIndices(descending = TRUE))

# If you pass a list into chunked_array, you get a list of length 1
list_scores <- chunked_array(list(c(9.9, 9.6, 9.5), c(8.2, 8.3, 8.4), c(10.0, 9.9, 9.8)))
list_scores$num_chunks

# When constructing a ChunkedArray, the first chunk is used to infer type.
doubles <- chunked_array(c(1, 2, 3), c(5L, 6L, 7L))
doubles$type

# Concatenating chunked arrays returns a new chunked array containing all chunks
a <- chunked_array(c(1, 2), 3)
b <- chunked_array(c(4, 5), 6)
c(a, b)



cleanEx()
nameEx("CsvFileFormat")
### * CsvFileFormat

flush(stderr()); flush(stdout())

### Name: CsvFileFormat
### Title: CSV dataset file format
### Aliases: CsvFileFormat

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
# Set up directory for examples
tf <- tempfile()
dir.create(tf)
on.exit(unlink(tf))
df <- data.frame(x = c("1", "2", "NULL"))
write.table(df, file.path(tf, "file1.txt"), sep = ",", row.names = FALSE)

# Create CsvFileFormat object with Arrow-style null_values option
format <- CsvFileFormat$create(convert_options = list(null_values = c("", "NA", "NULL")))
open_dataset(tf, format = format)

# Use readr-style options
format <- CsvFileFormat$create(na = c("", "NA", "NULL"))
open_dataset(tf, format = format)
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("Field")
### * Field

flush(stderr()); flush(stdout())

### Name: field
### Title: Create a Field
### Aliases: field

### ** Examples

field("x", int32())



cleanEx()
nameEx("FileFormat")
### * FileFormat

flush(stderr()); flush(stdout())

### Name: FileFormat
### Title: Dataset file formats
### Aliases: FileFormat ParquetFileFormat IpcFileFormat

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
## Semi-colon delimited files
# Set up directory for examples
tf <- tempfile()
dir.create(tf)
on.exit(unlink(tf))
write.table(mtcars, file.path(tf, "file1.txt"), sep = ";", row.names = FALSE)

# Create FileFormat object
format <- FileFormat$create(format = "text", delimiter = ";")

open_dataset(tf, format = format)
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("JsonFileFormat")
### * JsonFileFormat

flush(stderr()); flush(stdout())

### Name: JsonFileFormat
### Title: JSON dataset file format
### Aliases: JsonFileFormat

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("ParquetFileReader")
### * ParquetFileReader

flush(stderr()); flush(stdout())

### Name: ParquetFileReader
### Title: ParquetFileReader class
### Aliases: ParquetFileReader

### ** Examples

## Don't show: 
if (arrow_with_parquet()) withAutoprint({ # examplesIf
## End(Don't show)
f <- system.file("v0.7.1.parquet", package = "arrow")
pq <- ParquetFileReader$create(f)
pq$GetSchema()
if (codec_is_available("snappy")) {
  # This file has compressed data columns
  tab <- pq$ReadTable()
  tab$schema
}
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("RecordBatchReader")
### * RecordBatchReader

flush(stderr()); flush(stdout())

### Name: RecordBatchReader
### Title: RecordBatchReader classes
### Aliases: RecordBatchReader RecordBatchStreamReader
###   RecordBatchFileReader

### ** Examples

tf <- tempfile()
on.exit(unlink(tf))

batch <- record_batch(chickwts)

# This opens a connection to the file in Arrow
file_obj <- FileOutputStream$create(tf)
# Pass that to a RecordBatchWriter to write data conforming to a schema
writer <- RecordBatchFileWriter$create(file_obj, batch$schema)
writer$write(batch)
# You may write additional batches to the stream, provided that they have
# the same schema.
# Call "close" on the writer to indicate end-of-file/stream
writer$close()
# Then, close the connection--closing the IPC message does not close the file
file_obj$close()

# Now, we have a file we can read from. Same pattern: open file connection,
# then pass it to a RecordBatchReader
read_file_obj <- ReadableFile$create(tf)
reader <- RecordBatchFileReader$create(read_file_obj)
# RecordBatchFileReader knows how many batches it has (StreamReader does not)
reader$num_record_batches
# We could consume the Reader by calling $read_next_batch() until all are,
# consumed, or we can call $read_table() to pull them all into a Table
tab <- reader$read_table()
# Call as.data.frame to turn that Table into an R data.frame
df <- as.data.frame(tab)
# This should be the same data we sent
all.equal(df, chickwts, check.attributes = FALSE)
# Unlike the Writers, we don't have to close RecordBatchReaders,
# but we do still need to close the file connection
read_file_obj$close()



cleanEx()
nameEx("RecordBatchWriter")
### * RecordBatchWriter

flush(stderr()); flush(stdout())

### Name: RecordBatchWriter
### Title: RecordBatchWriter classes
### Aliases: RecordBatchWriter RecordBatchStreamWriter
###   RecordBatchFileWriter

### ** Examples

tf <- tempfile()
on.exit(unlink(tf))

batch <- record_batch(chickwts)

# This opens a connection to the file in Arrow
file_obj <- FileOutputStream$create(tf)
# Pass that to a RecordBatchWriter to write data conforming to a schema
writer <- RecordBatchFileWriter$create(file_obj, batch$schema)
writer$write(batch)
# You may write additional batches to the stream, provided that they have
# the same schema.
# Call "close" on the writer to indicate end-of-file/stream
writer$close()
# Then, close the connection--closing the IPC message does not close the file
file_obj$close()

# Now, we have a file we can read from. Same pattern: open file connection,
# then pass it to a RecordBatchReader
read_file_obj <- ReadableFile$create(tf)
reader <- RecordBatchFileReader$create(read_file_obj)
# RecordBatchFileReader knows how many batches it has (StreamReader does not)
reader$num_record_batches
# We could consume the Reader by calling $read_next_batch() until all are,
# consumed, or we can call $read_table() to pull them all into a Table
tab <- reader$read_table()
# Call as.data.frame to turn that Table into an R data.frame
df <- as.data.frame(tab)
# This should be the same data we sent
all.equal(df, chickwts, check.attributes = FALSE)
# Unlike the Writers, we don't have to close RecordBatchReaders,
# but we do still need to close the file connection
read_file_obj$close()



cleanEx()
nameEx("Scalar-class")
### * Scalar-class

flush(stderr()); flush(stdout())

### Name: Scalar
### Title: Arrow scalars
### Aliases: Scalar

### ** Examples

Scalar$create(pi)
Scalar$create(404)
# If you pass a vector into Scalar$create, you get a list containing your items
Scalar$create(c(1, 2, 3))

# Comparisons
my_scalar <- Scalar$create(99)
my_scalar$ApproxEquals(Scalar$create(99.00001)) # FALSE
my_scalar$ApproxEquals(Scalar$create(99.000009)) # TRUE
my_scalar$Equals(Scalar$create(99.000009)) # FALSE
my_scalar$Equals(Scalar$create(99L)) # FALSE (types don't match)

my_scalar$ToString()



cleanEx()
nameEx("Scanner")
### * Scanner

flush(stderr()); flush(stdout())

### Name: Scanner
### Title: Scan the contents of a dataset
### Aliases: Scanner ScannerBuilder

### ** Examples

## Don't show: 
if (arrow_with_dataset() & arrow_with_parquet()) withAutoprint({ # examplesIf
## End(Don't show)
# Set up directory for examples
tf <- tempfile()
dir.create(tf)
on.exit(unlink(tf))

write_dataset(mtcars, tf, partitioning="cyl")

ds <- open_dataset(tf)

scan_builder <- ds$NewScan()
scan_builder$Filter(Expression$field_ref("hp") > 100)
scan_builder$Project(list(hp_times_ten = 10 * Expression$field_ref("hp")))

# Once configured, call $Finish()
scanner <- scan_builder$Finish()

# Can get results as a table
as.data.frame(scanner$ToTable())

# Or as a RecordBatchReader
scanner$ToRecordBatchReader()
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("add_filename")
### * add_filename

flush(stderr()); flush(stdout())

### Name: add_filename
### Title: Add the data filename as a column
### Aliases: add_filename
### Keywords: internal

### ** Examples

## Not run: 
##D open_dataset("nyc-taxi") |> mutate(
##D   file =
##D     add_filename()
##D )
##D 
##D # To use a verb like mutate() with add_filename() we need to first call
##D # compute()
##D open_dataset("nyc-taxi") |>
##D   mutate(file = add_filename()) |>
##D   compute() |>
##D   mutate(filename_length = nchar(file))
## End(Not run)




cleanEx()
nameEx("array-class")
### * array-class

flush(stderr()); flush(stdout())

### Name: Array
### Title: Array Classes
### Aliases: Array DictionaryArray StructArray ListArray LargeListArray
###   FixedSizeListArray MapArray

### ** Examples

my_array <- Array$create(1:10)
my_array$type
my_array$cast(int8())

# Check if value is null; zero-indexed
na_array <- Array$create(c(1:5, NA))
na_array$IsNull(0)
na_array$IsNull(5)
na_array$IsValid(5)
na_array$null_count

# zero-copy slicing; the offset of the new Array will be the same as the index passed to $Slice
new_array <- na_array$Slice(5)
new_array$offset

# Compare 2 arrays
na_array2 <- na_array
na_array2 == na_array # element-wise comparison
na_array2$Equals(na_array) # overall comparison



cleanEx()
nameEx("arrow_array")
### * arrow_array

flush(stderr()); flush(stdout())

### Name: arrow_array
### Title: Create an Arrow Array
### Aliases: arrow_array

### ** Examples

my_array <- arrow_array(1:10)

# Compare 2 arrays
na_array <- arrow_array(c(1:5, NA))
na_array2 <- na_array
na_array2 == na_array # element-wise comparison



cleanEx()
nameEx("as_arrow_array")
### * as_arrow_array

flush(stderr()); flush(stdout())

### Name: as_arrow_array
### Title: Convert an object to an Arrow Array
### Aliases: as_arrow_array as_arrow_array.Array as_arrow_array.Scalar
###   as_arrow_array.ChunkedArray

### ** Examples

as_arrow_array(1:5)




cleanEx()
nameEx("as_arrow_table")
### * as_arrow_table

flush(stderr()); flush(stdout())

### Name: as_arrow_table
### Title: Convert an object to an Arrow Table
### Aliases: as_arrow_table as_arrow_table.default as_arrow_table.Table
###   as_arrow_table.RecordBatch as_arrow_table.data.frame
###   as_arrow_table.RecordBatchReader as_arrow_table.Dataset
###   as_arrow_table.arrow_dplyr_query as_arrow_table.Schema

### ** Examples

# use as_arrow_table() for a single object
as_arrow_table(data.frame(col1 = 1, col2 = "two"))

# use arrow_table() to create from columns
arrow_table(col1 = 1, col2 = "two")




cleanEx()
nameEx("as_chunked_array")
### * as_chunked_array

flush(stderr()); flush(stdout())

### Name: as_chunked_array
### Title: Convert an object to an Arrow ChunkedArray
### Aliases: as_chunked_array as_chunked_array.ChunkedArray
###   as_chunked_array.Array

### ** Examples

as_chunked_array(1:5)




cleanEx()
nameEx("as_data_type")
### * as_data_type

flush(stderr()); flush(stdout())

### Name: as_data_type
### Title: Convert an object to an Arrow DataType
### Aliases: as_data_type as_data_type.DataType as_data_type.Field
###   as_data_type.Schema

### ** Examples

as_data_type(int32())




cleanEx()
nameEx("as_record_batch")
### * as_record_batch

flush(stderr()); flush(stdout())

### Name: as_record_batch
### Title: Convert an object to an Arrow RecordBatch
### Aliases: as_record_batch as_record_batch.RecordBatch
###   as_record_batch.Table as_record_batch.arrow_dplyr_query
###   as_record_batch.data.frame

### ** Examples

# use as_record_batch() for a single object
as_record_batch(data.frame(col1 = 1, col2 = "two"))

# use record_batch() to create from columns
record_batch(col1 = 1, col2 = "two")




cleanEx()
nameEx("as_record_batch_reader")
### * as_record_batch_reader

flush(stderr()); flush(stdout())

### Name: as_record_batch_reader
### Title: Convert an object to an Arrow RecordBatchReader
### Aliases: as_record_batch_reader
###   as_record_batch_reader.RecordBatchReader as_record_batch_reader.Table
###   as_record_batch_reader.RecordBatch as_record_batch_reader.data.frame
###   as_record_batch_reader.Dataset as_record_batch_reader.function
###   as_record_batch_reader.arrow_dplyr_query
###   as_record_batch_reader.Scanner

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
reader <- as_record_batch_reader(data.frame(col1 = 1, col2 = "two"))
reader$read_next_batch()
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("as_schema")
### * as_schema

flush(stderr()); flush(stdout())

### Name: as_schema
### Title: Convert an object to an Arrow Schema
### Aliases: as_schema as_schema.Schema as_schema.StructType

### ** Examples

as_schema(schema(col1 = int32()))




cleanEx()
nameEx("call_function")
### * call_function

flush(stderr()); flush(stdout())

### Name: call_function
### Title: Call an Arrow compute function
### Aliases: call_function

### ** Examples

a <- Array$create(c(1L, 2L, 3L, NA, 5L))
s <- Scalar$create(4L)
call_function("coalesce", a, s)

a <- Array$create(rnorm(10000))
call_function("quantile", a, options = list(q = seq(0, 1, 0.25)))



cleanEx()
nameEx("cast")
### * cast

flush(stderr()); flush(stdout())

### Name: cast
### Title: Change the type of an array or column
### Aliases: cast
### Keywords: internal

### ** Examples

## Not run: 
##D mtcars |>
##D   arrow_table() |>
##D   mutate(cyl = cast(cyl, string()))
## End(Not run)



cleanEx()
nameEx("chunked_array")
### * chunked_array

flush(stderr()); flush(stdout())

### Name: chunked_array
### Title: Create a Chunked Array
### Aliases: chunked_array

### ** Examples

# Pass items into chunked_array as separate objects to create chunks
class_scores <- chunked_array(c(87, 88, 89), c(94, 93, 92), c(71, 72, 73))

# If you pass a list into chunked_array, you get a list of length 1
list_scores <- chunked_array(list(c(9.9, 9.6, 9.5), c(8.2, 8.3, 8.4), c(10.0, 9.9, 9.8)))

# When constructing a ChunkedArray, the first chunk is used to infer type.
infer_type(chunked_array(c(1, 2, 3), c(5L, 6L, 7L)))

# Concatenating chunked arrays returns a new chunked array containing all chunks
a <- chunked_array(c(1, 2), 3)
b <- chunked_array(c(4, 5), 6)
c(a, b)



cleanEx()
nameEx("codec_is_available")
### * codec_is_available

flush(stderr()); flush(stdout())

### Name: codec_is_available
### Title: Check whether a compression codec is available
### Aliases: codec_is_available

### ** Examples

codec_is_available("gzip")



cleanEx()
nameEx("concat_arrays")
### * concat_arrays

flush(stderr()); flush(stdout())

### Name: concat_arrays
### Title: Concatenate zero or more Arrays
### Aliases: concat_arrays c.Array

### ** Examples

concat_arrays(Array$create(1:3), Array$create(4:5))



cleanEx()
nameEx("concat_tables")
### * concat_tables

flush(stderr()); flush(stdout())

### Name: concat_tables
### Title: Concatenate one or more Tables
### Aliases: concat_tables

### ** Examples

tbl <- arrow_table(name = rownames(mtcars), mtcars)
prius <- arrow_table(name = "Prius", mpg = 58, cyl = 4, disp = 1.8)
combined <- concat_tables(tbl, prius)
tail(combined)$to_data_frame()



cleanEx()
nameEx("copy_files")
### * copy_files

flush(stderr()); flush(stdout())

### Name: copy_files
### Title: Copy files between FileSystems
### Aliases: copy_files

### ** Examples

## Don't show: 
if (FALSE) withAutoprint({ # examplesIf
## End(Don't show)
# Copy an S3 bucket's files to a local directory:
copy_files("s3://your-bucket-name", "local-directory")
# Using a FileSystem object
copy_files(s3_bucket("your-bucket-name"), "local-directory")
# Or go the other way, from local to S3
copy_files("local-directory", s3_bucket("your-bucket-name"))
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("create_package_with_all_dependencies")
### * create_package_with_all_dependencies

flush(stderr()); flush(stdout())

### Name: create_package_with_all_dependencies
### Title: Create a source bundle that includes all thirdparty dependencies
### Aliases: create_package_with_all_dependencies

### ** Examples

## Not run: 
##D new_pkg <- create_package_with_all_dependencies()
##D # Note: this works when run in the same R session, but it's meant to be
##D # copied to a different computer.
##D install.packages(new_pkg, dependencies = c("Depends", "Imports", "LinkingTo"))
## End(Not run)



cleanEx()
nameEx("csv_convert_options")
### * csv_convert_options

flush(stderr()); flush(stdout())

### Name: csv_convert_options
### Title: CSV Convert Options
### Aliases: csv_convert_options

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
tf <- tempfile()
on.exit(unlink(tf))
writeLines("x\n1\nNULL\n2\nNA", tf)
read_csv_arrow(tf, convert_options = csv_convert_options(null_values = c("", "NA", "NULL")))
open_csv_dataset(tf, convert_options = csv_convert_options(null_values = c("", "NA", "NULL")))
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("csv_parse_options")
### * csv_parse_options

flush(stderr()); flush(stdout())

### Name: csv_parse_options
### Title: CSV Parsing Options
### Aliases: csv_parse_options

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
tf <- tempfile()
on.exit(unlink(tf))
writeLines("x\n1\n\n2", tf)
read_csv_arrow(tf, parse_options = csv_parse_options(ignore_empty_lines = FALSE))
open_csv_dataset(tf, parse_options = csv_parse_options(ignore_empty_lines = FALSE))
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("csv_read_options")
### * csv_read_options

flush(stderr()); flush(stdout())

### Name: csv_read_options
### Title: CSV Reading Options
### Aliases: csv_read_options

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
tf <- tempfile()
on.exit(unlink(tf))
writeLines("my file has a non-data header\nx\n1\n2", tf)
read_csv_arrow(tf, read_options = csv_read_options(skip_rows = 1))
open_csv_dataset(tf, read_options = csv_read_options(skip_rows = 1))
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("csv_write_options")
### * csv_write_options

flush(stderr()); flush(stdout())

### Name: csv_write_options
### Title: CSV Writing Options
### Aliases: csv_write_options

### ** Examples

tf <- tempfile()
on.exit(unlink(tf))
write_csv_arrow(airquality, tf, write_options = csv_write_options(null_string = "-99"))



cleanEx()
nameEx("data-type")
### * data-type

flush(stderr()); flush(stdout())

### Name: data-type
### Title: Create Arrow data types
### Aliases: data-type int8 int16 int32 int64 uint8 uint16 uint32 uint64
###   float16 halffloat float32 float float64 boolean bool utf8 large_utf8
###   binary large_binary fixed_size_binary string date32 date64 time32
###   time64 duration null timestamp decimal decimal32 decimal64 decimal128
###   decimal256 struct list_of large_list_of FixedSizeListType
###   fixed_size_list_of MapType map_of

### ** Examples

## Don't show: 
if (arrow_with_acero()) withAutoprint({ # examplesIf
## End(Don't show)
bool()
struct(a = int32(), b = double())
timestamp("ms", timezone = "CEST")
time64("ns")

# Use the cast method to change the type of data contained in Arrow objects.
# Please check the documentation of each data object class for details.
my_scalar <- Scalar$create(0L, type = int64()) # int64
my_scalar$cast(timestamp("ns")) # timestamp[ns]

my_array <- Array$create(0L, type = int64()) # int64
my_array$cast(timestamp("s", timezone = "UTC")) # timestamp[s, tz=UTC]

my_chunked_array <- chunked_array(0L, 1L) # int32
my_chunked_array$cast(date32()) # date32[day]

# You can also use `cast()` in an Arrow dplyr query.
if (requireNamespace("dplyr", quietly = TRUE)) {
  library(dplyr, warn.conflicts = FALSE)
  arrow_table(mtcars) |>
    transmute(
      col1 = cast(cyl, string()),
      col2 = cast(cyl, int8())
    ) |>
    compute()
}
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("gs_bucket")
### * gs_bucket

flush(stderr()); flush(stdout())

### Name: gs_bucket
### Title: Connect to a Google Cloud Storage (GCS) bucket
### Aliases: gs_bucket

### ** Examples

## Don't show: 
if (FALSE) withAutoprint({ # examplesIf
## End(Don't show)
bucket <- gs_bucket("voltrondata-labs-datasets")
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("hive_partition")
### * hive_partition

flush(stderr()); flush(stdout())

### Name: hive_partition
### Title: Construct Hive partitioning
### Aliases: hive_partition

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
hive_partition(year = int16(), month = int8())
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("infer_type")
### * infer_type

flush(stderr()); flush(stdout())

### Name: infer_type
### Title: Infer the arrow Array type from an R object
### Aliases: infer_type type

### ** Examples

infer_type(1:10)
infer_type(1L:10L)
infer_type(c(1, 1.5, 2))
infer_type(c("A", "B", "C"))
infer_type(mtcars)
infer_type(Sys.Date())
infer_type(as.POSIXlt(Sys.Date()))
infer_type(vctrs::new_vctr(1:5, class = "my_custom_vctr_class"))



cleanEx()
nameEx("list_compute_functions")
### * list_compute_functions

flush(stderr()); flush(stdout())

### Name: list_compute_functions
### Title: List available Arrow C++ compute functions
### Aliases: list_compute_functions

### ** Examples

available_funcs <- list_compute_functions()
utf8_funcs <- list_compute_functions(pattern = "^UTF8", ignore.case = TRUE)



cleanEx()
nameEx("load_flight_server")
### * load_flight_server

flush(stderr()); flush(stdout())

### Name: load_flight_server
### Title: Load a Python Flight server
### Aliases: load_flight_server

### ** Examples

## Don't show: 
if (FALSE) withAutoprint({ # examplesIf
## End(Don't show)
load_flight_server("demo_flight_server")
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("match_arrow")
### * match_arrow

flush(stderr()); flush(stdout())

### Name: match_arrow
### Title: Value matching for Arrow objects
### Aliases: match_arrow is_in

### ** Examples

# note that the returned value is 0-indexed
cars_tbl <- arrow_table(name = rownames(mtcars), mtcars)
match_arrow(Scalar$create("Mazda RX4 Wag"), cars_tbl$name)

is_in(Array$create("Mazda RX4 Wag"), cars_tbl$name)

# Although there are multiple matches, you are returned the index of the first
# match, as with the base R equivalent
match(4, mtcars$cyl) # 1-indexed
match_arrow(Scalar$create(4), cars_tbl$cyl) # 0-indexed

# If `x` contains multiple values, you are returned the indices of the first
# match for each value.
match(c(4, 6, 8), mtcars$cyl)
match_arrow(Array$create(c(4, 6, 8)), cars_tbl$cyl)

# Return type matches type of `x`
is_in(c(4, 6, 8), mtcars$cyl) # returns vector
is_in(Scalar$create(4), mtcars$cyl) # returns Scalar
is_in(Array$create(c(4, 6, 8)), cars_tbl$cyl) # returns Array
is_in(ChunkedArray$create(c(4, 6), 8), cars_tbl$cyl) # returns ChunkedArray



cleanEx()
nameEx("new_extension_type")
### * new_extension_type

flush(stderr()); flush(stdout())

### Name: new_extension_type
### Title: Extension types
### Aliases: new_extension_type new_extension_array register_extension_type
###   reregister_extension_type unregister_extension_type

### ** Examples

# Create the R6 type whose methods control how Array objects are
# converted to R objects, how equality between types is computed,
# and how types are printed.
QuantizedType <- R6::R6Class(
  "QuantizedType",
  inherit = ExtensionType,
  public = list(
    # methods to access the custom metadata fields
    center = function() private$.center,
    scale = function() private$.scale,

    # called when an Array of this type is converted to an R vector
    as_vector = function(extension_array) {
      if (inherits(extension_array, "ExtensionArray")) {
        unquantized_arrow <-
          (extension_array$storage()$cast(float64()) / private$.scale) +
          private$.center

        as.vector(unquantized_arrow)
      } else {
        super$as_vector(extension_array)
      }
    },

    # populate the custom metadata fields from the serialized metadata
    deserialize_instance = function() {
      vals <- as.numeric(strsplit(self$extension_metadata_utf8(), ";")[[1]])
      private$.center <- vals[1]
      private$.scale <- vals[2]
    }
  ),
  private = list(
    .center = NULL,
    .scale = NULL
  )
)

# Create a helper type constructor that calls new_extension_type()
quantized <- function(center = 0, scale = 1, storage_type = int32()) {
  new_extension_type(
    storage_type = storage_type,
    extension_name = "arrow.example.quantized",
    extension_metadata = paste(center, scale, sep = ";"),
    type_class = QuantizedType
  )
}

# Create a helper array constructor that calls new_extension_array()
quantized_array <- function(x, center = 0, scale = 1,
                            storage_type = int32()) {
  type <- quantized(center, scale, storage_type)
  new_extension_array(
    Array$create((x - center) * scale, type = storage_type),
    type
  )
}

# Register the extension type so that Arrow knows what to do when
# it encounters this extension type
reregister_extension_type(quantized())

# Create Array objects and use them!
(vals <- runif(5, min = 19, max = 21))

(array <- quantized_array(
  vals,
  center = 20,
  scale = 2^15 - 1,
  storage_type = int16()
)
)

array$type$center()
array$type$scale()

as.vector(array)



cleanEx()
nameEx("one")
### * one

flush(stderr()); flush(stdout())

### Name: one
### Title: Get one value from each group
### Aliases: one
### Keywords: internal

### ** Examples

## Not run: 
##D mtcars |>
##D   arrow_table() |>
##D   group_by(cyl) |>
##D   summarize(x = one(disp))
## End(Not run)



cleanEx()
nameEx("open_dataset")
### * open_dataset

flush(stderr()); flush(stdout())

### Name: open_dataset
### Title: Open a multi-file dataset
### Aliases: open_dataset

### ** Examples

## Don't show: 
if (arrow_with_dataset() & arrow_with_parquet()) withAutoprint({ # examplesIf
## End(Don't show)
# Set up directory for examples
tf <- tempfile()
dir.create(tf)

write_dataset(mtcars, tf, partitioning = "cyl")

# You can specify a directory containing the files for your dataset and
# open_dataset will scan all files in your directory.
open_dataset(tf)

# You can also supply a vector of paths
open_dataset(c(file.path(tf, "cyl=4/part-0.parquet"), file.path(tf, "cyl=8/part-0.parquet")))

## You must specify the file format if using a format other than parquet.
tf2 <- tempfile()
dir.create(tf2)
write_dataset(mtcars, tf2, format = "ipc")
# This line will results in errors when you try to work with the data
## Not run: 
##D open_dataset(tf2)
## End(Not run)
# This line will work
open_dataset(tf2, format = "ipc")

## You can specify file partitioning to include it as a field in your dataset
# Create a temporary directory and write example dataset
tf3 <- tempfile()
dir.create(tf3)
write_dataset(airquality, tf3, partitioning = c("Month", "Day"), hive_style = FALSE)

# View files - you can see the partitioning means that files have been written
# to folders based on Month/Day values
tf3_files <- list.files(tf3, recursive = TRUE)

# With no partitioning specified, dataset contains all files but doesn't include
# directory names as field names
open_dataset(tf3)

# Now that partitioning has been specified, your dataset contains columns for Month and Day
open_dataset(tf3, partitioning = c("Month", "Day"))

# If you want to specify the data types for your fields, you can pass in a Schema
open_dataset(tf3, partitioning = schema(Month = int8(), Day = int8()))
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("open_delim_dataset")
### * open_delim_dataset

flush(stderr()); flush(stdout())

### Name: open_delim_dataset
### Title: Open a multi-file dataset of CSV or other delimiter-separated
###   format
### Aliases: open_delim_dataset open_csv_dataset open_tsv_dataset

### ** Examples

## Don't show: 
if (arrow_with_dataset()) withAutoprint({ # examplesIf
## End(Don't show)
# Set up directory for examples
tf <- tempfile()
dir.create(tf)

df <- data.frame(x = c("1", "2", "NULL"))
file_path <- file.path(tf, "file1.txt")
write.table(df, file_path, sep = ",", row.names = FALSE)

# Use readr-style params identically in both `read_csv_dataset()` and `open_csv_dataset()`
read_csv_arrow(file_path, na = c("", "NA", "NULL"), col_names = "y", skip = 1)
open_csv_dataset(file_path, na = c("", "NA", "NULL"), col_names = "y", skip = 1)

# Use `col_types` to specify a schema, partial schema, or compact representation
tf2 <- tempfile()
write_csv_dataset(cars, tf2)

open_csv_dataset(tf2, col_types = schema(speed = int32(), dist = int32()))
open_csv_dataset(tf2, col_types = schema(speed = int32()))
open_csv_dataset(tf2, col_types = "ii", col_names = c("speed", "dist"), skip = 1)
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("read_delim_arrow")
### * read_delim_arrow

flush(stderr()); flush(stdout())

### Name: read_delim_arrow
### Title: Read a CSV or other delimited file with Arrow
### Aliases: read_delim_arrow read_csv_arrow read_csv2_arrow read_tsv_arrow

### ** Examples

tf <- tempfile()
on.exit(unlink(tf))
write.csv(mtcars, file = tf)
df <- read_csv_arrow(tf)
dim(df)
# Can select columns
df <- read_csv_arrow(tf, col_select = starts_with("d"))

# Specifying column types and names
write.csv(data.frame(x = c(1, 3), y = c(2, 4)), file = tf, row.names = FALSE)
read_csv_arrow(tf, schema = schema(x = int32(), y = utf8()), skip = 1)
read_csv_arrow(tf, col_types = schema(y = utf8()))
read_csv_arrow(tf, col_types = "ic", col_names = c("x", "y"), skip = 1)

# Note that if a timestamp column contains time zones,
# the string "T" `col_types` specification won't work.
# To parse timestamps with time zones, provide a [Schema] to `col_types`
# and specify the time zone in the type object:
tf <- tempfile()
write.csv(data.frame(x = "1970-01-01T12:00:00+12:00"), file = tf, row.names = FALSE)
read_csv_arrow(
  tf,
  col_types = schema(x = timestamp(unit = "us", timezone = "UTC"))
)

# Read directly from strings with `I()`
read_csv_arrow(I("x,y\n1,2\n3,4"))
read_delim_arrow(I(c("x y", "1 2", "3 4")), delim = " ")



cleanEx()
nameEx("read_feather")
### * read_feather

flush(stderr()); flush(stdout())

### Name: read_feather
### Title: Read a Feather file (an Arrow IPC file)
### Aliases: read_feather read_ipc_file

### ** Examples

# We recommend the ".arrow" extension for Arrow IPC files (Feather V2).
tf <- tempfile(fileext = ".arrow")
on.exit(unlink(tf))
write_feather(mtcars, tf)
df <- read_feather(tf)
dim(df)
# Can select columns
df <- read_feather(tf, col_select = starts_with("d"))



cleanEx()
nameEx("read_json_arrow")
### * read_json_arrow

flush(stderr()); flush(stdout())

### Name: read_json_arrow
### Title: Read a JSON file
### Aliases: read_json_arrow

### ** Examples

## Don't show: 
if (arrow_with_json()) withAutoprint({ # examplesIf
## End(Don't show)
tf <- tempfile()
on.exit(unlink(tf))
writeLines('
    { "hello": 3.5, "world": false, "yo": "thing" }
    { "hello": 3.25, "world": null }
    { "hello": 0.0, "world": true, "yo": null }
  ', tf, useBytes = TRUE)

read_json_arrow(tf)

# Read directly from strings with `I()`
read_json_arrow(I(c('{"x": 1, "y": 2}', '{"x": 3, "y": 4}')))
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("read_parquet")
### * read_parquet

flush(stderr()); flush(stdout())

### Name: read_parquet
### Title: Read a Parquet file
### Aliases: read_parquet

### ** Examples

## Don't show: 
if (arrow_with_parquet() && !getFromNamespace("on_linux_dev", "arrow")()) withAutoprint({ # examplesIf
## End(Don't show)
tf <- tempfile()
on.exit(unlink(tf))
write_parquet(mtcars, tf)
df <- read_parquet(tf, col_select = starts_with("d"))
head(df)
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("record_batch")
### * record_batch

flush(stderr()); flush(stdout())

### Name: record_batch
### Title: Create a RecordBatch
### Aliases: record_batch

### ** Examples

batch <- record_batch(name = rownames(mtcars), mtcars)
dim(batch)
dim(head(batch))
names(batch)
batch$mpg
batch[["cyl"]]
as.data.frame(batch[4:8, c("gear", "hp", "wt")])



cleanEx()
nameEx("register_scalar_function")
### * register_scalar_function

flush(stderr()); flush(stdout())

### Name: register_scalar_function
### Title: Register user-defined functions
### Aliases: register_scalar_function

### ** Examples

## Don't show: 
if (arrow_with_dataset() && identical(Sys.getenv("NOT_CRAN"), "true")) withAutoprint({ # examplesIf
## End(Don't show)
library(dplyr, warn.conflicts = FALSE)

some_model <- lm(mpg ~ disp + cyl, data = mtcars)
register_scalar_function(
  "mtcars_predict_mpg",
  function(context, disp, cyl) {
    predict(some_model, newdata = data.frame(disp, cyl))
  },
  in_type = schema(disp = float64(), cyl = float64()),
  out_type = float64(),
  auto_convert = TRUE
)

as_arrow_table(mtcars) |>
  transmute(mpg, mpg_predicted = mtcars_predict_mpg(disp, cyl)) |>
  collect() |>
  head()
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("s3_bucket")
### * s3_bucket

flush(stderr()); flush(stdout())

### Name: s3_bucket
### Title: Connect to an AWS S3 bucket
### Aliases: s3_bucket

### ** Examples

## Don't show: 
if (FALSE) withAutoprint({ # examplesIf
## End(Don't show)
bucket <- s3_bucket("voltrondata-labs-datasets")
## Don't show: 
}) # examplesIf
## End(Don't show)
## Don't show: 
if (FALSE) withAutoprint({ # examplesIf
## End(Don't show)
# Turn on debug logging. The following line of code should be run in a fresh
# R session prior to any calls to `s3_bucket()` (or other S3 functions)
Sys.setenv("ARROW_S3_LOG_LEVEL" = "DEBUG")
bucket <- s3_bucket("voltrondata-labs-datasets")
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("scalar")
### * scalar

flush(stderr()); flush(stdout())

### Name: scalar
### Title: Create an Arrow Scalar
### Aliases: scalar StructScalar

### ** Examples

scalar(pi)
scalar(404)
# If you pass a vector into scalar(), you get a list containing your items
scalar(c(1, 2, 3))

scalar(9) == scalar(10)




cleanEx()
nameEx("schema")
### * schema

flush(stderr()); flush(stdout())

### Name: schema
### Title: Create a schema or extract one from an object.
### Aliases: schema

### ** Examples

# Create schema using pairs of field names and data types
schema(a = int32(), b = float64())

# Create a schema using a list of pairs of field names and data types
schema(list(a = int8(), b = string()))

# Create schema using fields
schema(
  field("b", double()),
  field("c", bool(), nullable = FALSE),
  field("d", string())
)

# Extract schemas from objects
df <- data.frame(col1 = 2:4, col2 = c(0.1, 0.3, 0.5))
tab1 <- arrow_table(df)
schema(tab1)
tab2 <- arrow_table(df, schema = schema(col1 = int8(), col2 = float32()))
schema(tab2)



cleanEx()
nameEx("show_exec_plan")
### * show_exec_plan

flush(stderr()); flush(stdout())

### Name: show_exec_plan
### Title: Show the details of an Arrow Execution Plan
### Aliases: show_exec_plan

### ** Examples

## Don't show: 
if (arrow_with_dataset() && requireNamespace("dplyr", quietly = TRUE)) withAutoprint({ # examplesIf
## End(Don't show)
library(dplyr)
mtcars |>
  arrow_table() |>
  filter(mpg > 20) |>
  mutate(x = gear / carb) |>
  show_exec_plan()
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("table")
### * table

flush(stderr()); flush(stdout())

### Name: arrow_table
### Title: Create an Arrow Table
### Aliases: arrow_table

### ** Examples

tbl <- arrow_table(name = rownames(mtcars), mtcars)
dim(tbl)
dim(head(tbl))
names(tbl)
tbl$mpg
tbl[["cyl"]]
as.data.frame(tbl[4:8, c("gear", "hp", "wt")])



cleanEx()
nameEx("to_arrow")
### * to_arrow

flush(stderr()); flush(stdout())

### Name: to_arrow
### Title: Create an Arrow object from a DuckDB connection
### Aliases: to_arrow

### ** Examples

## Don't show: 
if (getFromNamespace("run_duckdb_examples", "arrow")()) withAutoprint({ # examplesIf
## End(Don't show)
library(dplyr)

ds <- InMemoryDataset$create(mtcars)

ds |>
  filter(mpg < 30) |>
  to_duckdb() |>
  group_by(cyl) |>
  summarize(mean_mpg = mean(mpg, na.rm = TRUE)) |>
  to_arrow() |>
  collect()
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("to_duckdb")
### * to_duckdb

flush(stderr()); flush(stdout())

### Name: to_duckdb
### Title: Create a (virtual) DuckDB table from an Arrow object
### Aliases: to_duckdb

### ** Examples

## Don't show: 
if (getFromNamespace("run_duckdb_examples", "arrow")()) withAutoprint({ # examplesIf
## End(Don't show)
library(dplyr)

ds <- InMemoryDataset$create(mtcars)

ds |>
  filter(mpg < 30) |>
  group_by(cyl) |>
  to_duckdb() |>
  slice_min(disp)
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("unify_schemas")
### * unify_schemas

flush(stderr()); flush(stdout())

### Name: unify_schemas
### Title: Combine and harmonize schemas
### Aliases: unify_schemas

### ** Examples

a <- schema(b = double(), c = bool())
z <- schema(b = double(), k = utf8())
unify_schemas(a, z)



cleanEx()
nameEx("value_counts")
### * value_counts

flush(stderr()); flush(stdout())

### Name: value_counts
### Title: 'table' for Arrow objects
### Aliases: value_counts

### ** Examples

cyl_vals <- Array$create(mtcars$cyl)
counts <- value_counts(cyl_vals)



cleanEx()
nameEx("vctrs_extension_array")
### * vctrs_extension_array

flush(stderr()); flush(stdout())

### Name: vctrs_extension_array
### Title: Extension type for generic typed vectors
### Aliases: vctrs_extension_array vctrs_extension_type

### ** Examples

(array <- vctrs_extension_array(as.POSIXlt("2022-01-02 03:45", tz = "UTC")))
array$type
as.vector(array)

temp_feather <- tempfile()
write_feather(arrow_table(col = array), temp_feather)
read_feather(temp_feather)
unlink(temp_feather)



cleanEx()
nameEx("write_csv_arrow")
### * write_csv_arrow

flush(stderr()); flush(stdout())

### Name: write_csv_arrow
### Title: Write CSV file to disk
### Aliases: write_csv_arrow

### ** Examples

tf <- tempfile()
on.exit(unlink(tf))
write_csv_arrow(mtcars, tf)



cleanEx()
nameEx("write_dataset")
### * write_dataset

flush(stderr()); flush(stdout())

### Name: write_dataset
### Title: Write a dataset
### Aliases: write_dataset

### ** Examples

## Don't show: 
if (arrow_with_dataset() & arrow_with_parquet() & requireNamespace("dplyr", quietly = TRUE)) withAutoprint({ # examplesIf
## End(Don't show)
# You can write datasets partitioned by the values in a column (here: "cyl").
# This creates a structure of the form cyl=X/part-Z.parquet.
one_level_tree <- tempfile()
write_dataset(mtcars, one_level_tree, partitioning = "cyl")
list.files(one_level_tree, recursive = TRUE)

# You can also partition by the values in multiple columns
# (here: "cyl" and "gear").
# This creates a structure of the form cyl=X/gear=Y/part-Z.parquet.
two_levels_tree <- tempfile()
write_dataset(mtcars, two_levels_tree, partitioning = c("cyl", "gear"))
list.files(two_levels_tree, recursive = TRUE)

# In the two previous examples we would have:
# X = {4,6,8}, the number of cylinders.
# Y = {3,4,5}, the number of forward gears.
# Z = {0,1,2}, the number of saved parts, starting from 0.

# You can obtain the same result as as the previous examples using arrow with
# a dplyr pipeline. This will be the same as two_levels_tree above, but the
# output directory will be different.
library(dplyr)
two_levels_tree_2 <- tempfile()
mtcars |>
  group_by(cyl, gear) |>
  write_dataset(two_levels_tree_2)
list.files(two_levels_tree_2, recursive = TRUE)

# And you can also turn off the Hive-style directory naming where the column
# name is included with the values by using `hive_style = FALSE`.

# Write a structure X/Y/part-Z.parquet.
two_levels_tree_no_hive <- tempfile()
mtcars |>
  group_by(cyl, gear) |>
  write_dataset(two_levels_tree_no_hive, hive_style = FALSE)
list.files(two_levels_tree_no_hive, recursive = TRUE)
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("write_feather")
### * write_feather

flush(stderr()); flush(stdout())

### Name: write_feather
### Title: Write a Feather file (an Arrow IPC file)
### Aliases: write_feather write_ipc_file

### ** Examples

# We recommend the ".arrow" extension for Arrow IPC files (Feather V2).
tf1 <- tempfile(fileext = ".feather")
tf2 <- tempfile(fileext = ".arrow")
tf3 <- tempfile(fileext = ".arrow")
on.exit({
  unlink(tf1)
  unlink(tf2)
  unlink(tf3)
})
write_feather(mtcars, tf1, version = 1)
write_feather(mtcars, tf2)
write_ipc_file(mtcars, tf3)



cleanEx()
nameEx("write_ipc_stream")
### * write_ipc_stream

flush(stderr()); flush(stdout())

### Name: write_ipc_stream
### Title: Write Arrow IPC stream format
### Aliases: write_ipc_stream

### ** Examples

tf <- tempfile()
on.exit(unlink(tf))
write_ipc_stream(mtcars, tf)



cleanEx()
nameEx("write_parquet")
### * write_parquet

flush(stderr()); flush(stdout())

### Name: write_parquet
### Title: Write Parquet file to disk
### Aliases: write_parquet

### ** Examples

## Don't show: 
if (arrow_with_parquet()) withAutoprint({ # examplesIf
## End(Don't show)
tf1 <- tempfile(fileext = ".parquet")
write_parquet(data.frame(x = 1:5), tf1)

# using compression
if (codec_is_available("gzip")) {
  tf2 <- tempfile(fileext = ".gz.parquet")
  write_parquet(data.frame(x = 1:5), tf2, compression = "gzip", compression_level = 5)
}
## Don't show: 
}) # examplesIf
## End(Don't show)



cleanEx()
nameEx("write_to_raw")
### * write_to_raw

flush(stderr()); flush(stdout())

### Name: write_to_raw
### Title: Write Arrow data to a raw vector
### Aliases: write_to_raw

### ** Examples

# The default format is "stream"
mtcars_raw <- write_to_raw(mtcars)



### * <FOOTER>
###
cleanEx()
options(digits = 7L)
base::cat("Time elapsed: ", proc.time() - base::get("ptime", pos = 'CheckExEnv'),"\n")
grDevices::dev.off()
###
### Local variables: ***
### mode: outline-minor ***
### outline-regexp: "\\(> \\)?### [*]+" ***
### End: ***
quit('no')
