mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 08:35:46 +08:00
GH-46803: [Swift] Remove swift implementation from apache/arrow after migration to new repository (#46804)
### Rationale for this change The Swift implementation has been moved to their own repository at https://github.com/apache/arrow-swift ### What changes are included in this PR? Remove `swift/` subfolder and github workflow for CI job. ### Are these changes tested? No ### Are there any user-facing changes? No but for developers of the swift implementation it has been moved to their own repository. * GitHub Issue: #46803 Authored-by: Raúl Cumplido <raulcumplido@gmail.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
This commit is contained in:
Raúl Cumplido authored and Alina (Xi) Li committed
1 parent
2f499cf11a
commit
a77bfec0b3
74 files changed
-19677
No files matched your search
@@ -1,89 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
name: Swift
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- '**'
|
||||
- '!dependabot/**'
|
||||
tags:
|
||||
- '**'
|
||||
paths:
|
||||
- '.dockerignore'
|
||||
- '.github/workflows/swift.yml'
|
||||
- 'ci/docker/*swift*'
|
||||
- 'ci/scripts/swift_*'
|
||||
- 'docker-compose.yml'
|
||||
- 'swift/**'
|
||||
pull_request:
|
||||
paths:
|
||||
- '.dockerignore'
|
||||
- '.github/workflows/swift.yml'
|
||||
- 'ci/docker/*swift*'
|
||||
- 'ci/scripts/swift_*'
|
||||
- 'docker-compose.yml'
|
||||
- 'swift/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
ARCHERY_DEBUG: 1
|
||||
DOCKER_VOLUME_PREFIX: ".docker/"
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
name: AMD 64 Ubuntu Swift 5.10
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout Arrow
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: recursive
|
||||
- name: Setup Python on hosted runner
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: 3
|
||||
- name: Setup Archery
|
||||
run: pip install -e dev/archery[docker]
|
||||
- name: Execute Docker Build
|
||||
env:
|
||||
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
|
||||
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
source ci/scripts/util_enable_core_dumps.sh
|
||||
archery docker run ubuntu-swift
|
||||
- name: Docker Push
|
||||
if: >-
|
||||
success() &&
|
||||
github.event_name == 'push' &&
|
||||
github.repository == 'apache/arrow' &&
|
||||
github.ref_name == 'main'
|
||||
env:
|
||||
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
|
||||
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
continue-on-error: true
|
||||
run: archery docker push ubuntu-swift
|
||||
@@ -1,28 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is an EditorConfig file: https://editorconfig.org/
|
||||
|
||||
# See ../.editorconfig for inherited values
|
||||
|
||||
[*.{c,cc,cpp,h,hh,hpp}]
|
||||
indent_size = 4
|
||||
indent_style = space
|
||||
|
||||
[*.swift]
|
||||
indent_size = 4
|
||||
indent_style = space
|
||||
@@ -1,38 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
included:
|
||||
- Arrow/Package.swift
|
||||
- Arrow/Sources
|
||||
- Arrow/Tests
|
||||
- ArrowFlight/Package.swift
|
||||
- ArrowFlight/Sources
|
||||
- ArrowFlight/Tests
|
||||
- CDataWGo/Package.swift
|
||||
- CDataWGo/Sources/go-swift
|
||||
excluded:
|
||||
- Arrow/Sources/Arrow/File_generated.swift
|
||||
- Arrow/Sources/Arrow/Message_generated.swift
|
||||
- Arrow/Sources/Arrow/Schema_generated.swift
|
||||
- Arrow/Sources/Arrow/SparseTensor_generated.swift
|
||||
- Arrow/Sources/Arrow/Tensor_generated.swift
|
||||
- ArrowFlight/Sources/ArrowFlight/Flight.grpc.swift
|
||||
- ArrowFlight/Sources/ArrowFlight/Flight.pb.swift
|
||||
- ArrowFlight/Sources/ArrowFlight/FlightSql.pb.swift
|
||||
identifier_name:
|
||||
min_length: 2 # only warning
|
||||
allow_zero_lintable_files: false
|
||||
@@ -1,9 +0,0 @@
|
||||
.DS_Store
|
||||
/.build
|
||||
/Packages
|
||||
/*.xcodeproj
|
||||
xcuserdata/
|
||||
DerivedData/
|
||||
.swiftpm/
|
||||
.netrc
|
||||
Package.resolved
|
||||
@@ -1,67 +0,0 @@
|
||||
// swift-tools-version: 5.10
|
||||
// The swift-tools-version declares the minimum version of Swift required to build this package.
|
||||
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "Arrow",
|
||||
platforms: [
|
||||
.macOS(.v10_14)
|
||||
],
|
||||
products: [
|
||||
.library(
|
||||
name: "Arrow",
|
||||
targets: ["Arrow"])
|
||||
],
|
||||
dependencies: [
|
||||
.package(url: "https://github.com/google/flatbuffers.git", branch: "v25.2.10"),
|
||||
.package(
|
||||
url: "https://github.com/apple/swift-atomics.git",
|
||||
.upToNextMajor(from: "1.2.0") // or `.upToNextMinor
|
||||
)
|
||||
],
|
||||
targets: [
|
||||
.target(
|
||||
name: "ArrowC",
|
||||
path: "Sources/ArrowC",
|
||||
swiftSettings: [
|
||||
// build: .unsafeFlags(["-warnings-as-errors"])
|
||||
]
|
||||
|
||||
),
|
||||
.target(
|
||||
name: "Arrow",
|
||||
dependencies: ["ArrowC",
|
||||
.product(name: "FlatBuffers", package: "flatbuffers"),
|
||||
.product(name: "Atomics", package: "swift-atomics")
|
||||
],
|
||||
swiftSettings: [
|
||||
// build: .unsafeFlags(["-warnings-as-errors"])
|
||||
]
|
||||
),
|
||||
.testTarget(
|
||||
name: "ArrowTests",
|
||||
dependencies: ["Arrow", "ArrowC"],
|
||||
swiftSettings: [
|
||||
// build: .unsafeFlags(["-warnings-as-errors"])
|
||||
]
|
||||
)
|
||||
]
|
||||
)
|
||||
@@ -1,56 +0,0 @@
|
||||
<!---
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Apache Arrow Swift
|
||||
|
||||
An implementation of Arrow targeting Swift.
|
||||
|
||||
## Status
|
||||
|
||||
## Memory Management
|
||||
|
||||
- Allocations are 64-byte aligned and padded to 8-bytes.
|
||||
- Allocations are automatically garbage collected
|
||||
|
||||
## Arrays
|
||||
|
||||
### Primitive Types
|
||||
|
||||
- Int8, Int16, Int32, Int64
|
||||
- UInt8, UInt16, UInt32, UInt64
|
||||
- Float, Double
|
||||
- String (utf-8)
|
||||
|
||||
### Parametric Types
|
||||
|
||||
- Date32
|
||||
- Date64
|
||||
|
||||
### Type Metadata
|
||||
|
||||
- Data Types
|
||||
- Fields
|
||||
- Schema
|
||||
|
||||
## Test data generation
|
||||
|
||||
Test data files for the reader tests are generated by an executable built in go whose source is included in the data-generator directory.
|
||||
```sh
|
||||
$ go build -o swift-datagen
|
||||
```
|
||||
@@ -1,327 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public protocol ArrowArrayHolder {
|
||||
var type: ArrowType {get}
|
||||
var length: UInt {get}
|
||||
var nullCount: UInt {get}
|
||||
var array: AnyArray {get}
|
||||
var data: ArrowData {get}
|
||||
var getBufferData: () -> [Data] {get}
|
||||
var getBufferDataSizes: () -> [Int] {get}
|
||||
var getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn {get}
|
||||
}
|
||||
|
||||
public class ArrowArrayHolderImpl: ArrowArrayHolder {
|
||||
public let data: ArrowData
|
||||
public let type: ArrowType
|
||||
public let length: UInt
|
||||
public let nullCount: UInt
|
||||
public let array: AnyArray
|
||||
public let getBufferData: () -> [Data]
|
||||
public let getBufferDataSizes: () -> [Int]
|
||||
public let getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn
|
||||
public init<T>(_ arrowArray: ArrowArray<T>) {
|
||||
self.array = arrowArray
|
||||
self.data = arrowArray.arrowData
|
||||
self.length = arrowArray.length
|
||||
self.type = arrowArray.arrowData.type
|
||||
self.nullCount = arrowArray.nullCount
|
||||
self.getBufferData = {() -> [Data] in
|
||||
var bufferData = [Data]()
|
||||
for buffer in arrowArray.arrowData.buffers {
|
||||
bufferData.append(Data())
|
||||
buffer.append(to: &bufferData[bufferData.count - 1])
|
||||
}
|
||||
|
||||
return bufferData
|
||||
}
|
||||
|
||||
self.getBufferDataSizes = {() -> [Int] in
|
||||
var bufferDataSizes = [Int]()
|
||||
for buffer in arrowArray.arrowData.buffers {
|
||||
bufferDataSizes.append(Int(buffer.capacity))
|
||||
}
|
||||
|
||||
return bufferDataSizes
|
||||
}
|
||||
|
||||
self.getArrowColumn = {(field: ArrowField, arrayHolders: [ArrowArrayHolder]) throws -> ArrowColumn in
|
||||
var arrays = [ArrowArray<T>]()
|
||||
for arrayHolder in arrayHolders {
|
||||
if let array = arrayHolder.array as? ArrowArray<T> {
|
||||
arrays.append(array)
|
||||
}
|
||||
}
|
||||
|
||||
return ArrowColumn(field, chunked: ChunkedArrayHolder(try ChunkedArray<T>(arrays)))
|
||||
}
|
||||
}
|
||||
|
||||
public static func loadArray( // swiftlint:disable:this cyclomatic_complexity
|
||||
_ arrowType: ArrowType, with: ArrowData) throws -> ArrowArrayHolder {
|
||||
switch arrowType.id {
|
||||
case .int8:
|
||||
return try ArrowArrayHolderImpl(FixedArray<Int8>(with))
|
||||
case .int16:
|
||||
return try ArrowArrayHolderImpl(FixedArray<Int16>(with))
|
||||
case .int32:
|
||||
return try ArrowArrayHolderImpl(FixedArray<Int32>(with))
|
||||
case .int64:
|
||||
return try ArrowArrayHolderImpl(FixedArray<Int64>(with))
|
||||
case .uint8:
|
||||
return try ArrowArrayHolderImpl(FixedArray<UInt8>(with))
|
||||
case .uint16:
|
||||
return try ArrowArrayHolderImpl(FixedArray<UInt16>(with))
|
||||
case .uint32:
|
||||
return try ArrowArrayHolderImpl(FixedArray<UInt32>(with))
|
||||
case .uint64:
|
||||
return try ArrowArrayHolderImpl(FixedArray<UInt64>(with))
|
||||
case .double:
|
||||
return try ArrowArrayHolderImpl(FixedArray<Double>(with))
|
||||
case .float:
|
||||
return try ArrowArrayHolderImpl(FixedArray<Float>(with))
|
||||
case .date32:
|
||||
return try ArrowArrayHolderImpl(Date32Array(with))
|
||||
case .date64:
|
||||
return try ArrowArrayHolderImpl(Date64Array(with))
|
||||
case .time32:
|
||||
return try ArrowArrayHolderImpl(Time32Array(with))
|
||||
case .time64:
|
||||
return try ArrowArrayHolderImpl(Time64Array(with))
|
||||
case .string:
|
||||
return try ArrowArrayHolderImpl(StringArray(with))
|
||||
case .boolean:
|
||||
return try ArrowArrayHolderImpl(BoolArray(with))
|
||||
case .binary:
|
||||
return try ArrowArrayHolderImpl(BinaryArray(with))
|
||||
case .strct:
|
||||
return try ArrowArrayHolderImpl(StructArray(with))
|
||||
default:
|
||||
throw ArrowError.invalid("Array not found for type: \(arrowType)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowArray<T>: AsString, AnyArray {
|
||||
public typealias ItemType = T
|
||||
public let arrowData: ArrowData
|
||||
public var nullCount: UInt {return self.arrowData.nullCount}
|
||||
public var length: UInt {return self.arrowData.length}
|
||||
|
||||
public required init(_ arrowData: ArrowData) throws {
|
||||
self.arrowData = arrowData
|
||||
}
|
||||
|
||||
public func isNull(_ at: UInt) throws -> Bool {
|
||||
if at >= self.length {
|
||||
throw ArrowError.outOfBounds(index: Int64(at))
|
||||
}
|
||||
|
||||
return self.arrowData.isNull(at)
|
||||
}
|
||||
|
||||
public subscript(_ index: UInt) -> T? {
|
||||
fatalError("subscript() has not been implemented")
|
||||
}
|
||||
|
||||
public func asString(_ index: UInt) -> String {
|
||||
if self[index] == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return "\(self[index]!)"
|
||||
}
|
||||
|
||||
public func asAny(_ index: UInt) -> Any? {
|
||||
if self[index] == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return self[index]!
|
||||
}
|
||||
}
|
||||
|
||||
public class FixedArray<T>: ArrowArray<T> {
|
||||
public override subscript(_ index: UInt) -> T? {
|
||||
if self.arrowData.isNull(index) {
|
||||
return nil
|
||||
}
|
||||
|
||||
let byteOffset = self.arrowData.stride * Int(index)
|
||||
return self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(as: T.self)
|
||||
}
|
||||
}
|
||||
|
||||
public class StringArray: ArrowArray<String> {
|
||||
public override subscript(_ index: UInt) -> String? {
|
||||
let offsetIndex = MemoryLayout<Int32>.stride * Int(index)
|
||||
if self.arrowData.isNull(index) {
|
||||
return nil
|
||||
}
|
||||
|
||||
let offsets = self.arrowData.buffers[1]
|
||||
let values = self.arrowData.buffers[2]
|
||||
|
||||
var startIndex: Int32 = 0
|
||||
if index > 0 {
|
||||
startIndex = offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self)
|
||||
}
|
||||
|
||||
let endIndex = offsets.rawPointer.advanced(by: offsetIndex + MemoryLayout<Int32>.stride )
|
||||
.load(as: Int32.self)
|
||||
let arrayLength = Int(endIndex - startIndex)
|
||||
let rawPointer = values.rawPointer.advanced(by: Int(startIndex))
|
||||
.bindMemory(to: UInt8.self, capacity: arrayLength)
|
||||
let buffer = UnsafeBufferPointer<UInt8>(start: rawPointer, count: arrayLength)
|
||||
let byteArray = Array(buffer)
|
||||
return String(data: Data(byteArray), encoding: .utf8)
|
||||
}
|
||||
}
|
||||
|
||||
public class BoolArray: ArrowArray<Bool> {
|
||||
public override subscript(_ index: UInt) -> Bool? {
|
||||
if self.arrowData.isNull(index) {
|
||||
return nil
|
||||
}
|
||||
|
||||
let valueBuffer = self.arrowData.buffers[1]
|
||||
return BitUtility.isSet(index, buffer: valueBuffer)
|
||||
}
|
||||
}
|
||||
|
||||
public class Date32Array: ArrowArray<Date> {
|
||||
public override subscript(_ index: UInt) -> Date? {
|
||||
if self.arrowData.isNull(index) {
|
||||
return nil
|
||||
}
|
||||
|
||||
let byteOffset = self.arrowData.stride * Int(index)
|
||||
let milliseconds = self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(as: UInt32.self)
|
||||
return Date(timeIntervalSince1970: TimeInterval(milliseconds * 86400))
|
||||
}
|
||||
}
|
||||
|
||||
public class Date64Array: ArrowArray<Date> {
|
||||
public override subscript(_ index: UInt) -> Date? {
|
||||
if self.arrowData.isNull(index) {
|
||||
return nil
|
||||
}
|
||||
|
||||
let byteOffset = self.arrowData.stride * Int(index)
|
||||
let milliseconds = self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(as: UInt64.self)
|
||||
return Date(timeIntervalSince1970: TimeInterval(milliseconds / 1000))
|
||||
}
|
||||
}
|
||||
|
||||
public class Time32Array: FixedArray<Time32> {}
|
||||
public class Time64Array: FixedArray<Time64> {}
|
||||
|
||||
public class BinaryArray: ArrowArray<Data> {
|
||||
public struct Options {
|
||||
public var printAsHex = false
|
||||
public var printEncoding: String.Encoding = .utf8
|
||||
}
|
||||
|
||||
public var options = Options()
|
||||
|
||||
public override subscript(_ index: UInt) -> Data? {
|
||||
let offsetIndex = MemoryLayout<Int32>.stride * Int(index)
|
||||
if self.arrowData.isNull(index) {
|
||||
return nil
|
||||
}
|
||||
|
||||
let offsets = self.arrowData.buffers[1]
|
||||
let values = self.arrowData.buffers[2]
|
||||
var startIndex: Int32 = 0
|
||||
if index > 0 {
|
||||
startIndex = offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self)
|
||||
}
|
||||
|
||||
let endIndex = offsets.rawPointer.advanced(by: offsetIndex + MemoryLayout<Int32>.stride )
|
||||
.load(as: Int32.self)
|
||||
let arrayLength = Int(endIndex - startIndex)
|
||||
let rawPointer = values.rawPointer.advanced(by: Int(startIndex))
|
||||
.bindMemory(to: UInt8.self, capacity: arrayLength)
|
||||
let buffer = UnsafeBufferPointer<UInt8>(start: rawPointer, count: arrayLength)
|
||||
let byteArray = Array(buffer)
|
||||
return Data(byteArray)
|
||||
}
|
||||
|
||||
public override func asString(_ index: UInt) -> String {
|
||||
if self[index] == nil {return ""}
|
||||
let data = self[index]!
|
||||
if options.printAsHex {
|
||||
return data.hexEncodedString()
|
||||
} else {
|
||||
return String(data: data, encoding: .utf8)!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class StructArray: ArrowArray<[Any?]> {
|
||||
public private(set) var arrowFields: [ArrowArrayHolder]?
|
||||
public required init(_ arrowData: ArrowData) throws {
|
||||
try super.init(arrowData)
|
||||
var fields = [ArrowArrayHolder]()
|
||||
for child in arrowData.children {
|
||||
fields.append(try ArrowArrayHolderImpl.loadArray(child.type, with: child))
|
||||
}
|
||||
|
||||
self.arrowFields = fields
|
||||
}
|
||||
|
||||
public override subscript(_ index: UInt) -> [Any?]? {
|
||||
if self.arrowData.isNull(index) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if let fields = arrowFields {
|
||||
var result = [Any?]()
|
||||
for field in fields {
|
||||
result.append(field.array.asAny(index))
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
public override func asString(_ index: UInt) -> String {
|
||||
if self.arrowData.isNull(index) {
|
||||
return ""
|
||||
}
|
||||
|
||||
var output = "{"
|
||||
if let fields = arrowFields {
|
||||
for fieldIndex in 0..<fields.count {
|
||||
let asStr = fields[fieldIndex].array as? AsString
|
||||
if fieldIndex == 0 {
|
||||
output.append("\(asStr!.asString(index))")
|
||||
} else {
|
||||
output.append(",\(asStr!.asString(index))")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output += "}"
|
||||
return output
|
||||
}
|
||||
}
|
||||
@@ -1,341 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public protocol ArrowArrayHolderBuilder {
|
||||
func toHolder() throws -> ArrowArrayHolder
|
||||
func appendAny(_ val: Any?)
|
||||
}
|
||||
|
||||
public class ArrowArrayBuilder<T: ArrowBufferBuilder, U: ArrowArray<T.ItemType>>: ArrowArrayHolderBuilder {
|
||||
let type: ArrowType
|
||||
let bufferBuilder: T
|
||||
public var length: UInt {return self.bufferBuilder.length}
|
||||
public var capacity: UInt {return self.bufferBuilder.capacity}
|
||||
public var nullCount: UInt {return self.bufferBuilder.nullCount}
|
||||
public var offset: UInt {return self.bufferBuilder.offset}
|
||||
|
||||
fileprivate init(_ type: ArrowType) throws {
|
||||
self.type = type
|
||||
self.bufferBuilder = try T()
|
||||
}
|
||||
|
||||
public func append(_ vals: T.ItemType?...) {
|
||||
for val in vals {
|
||||
self.bufferBuilder.append(val)
|
||||
}
|
||||
}
|
||||
|
||||
public func append(_ vals: [T.ItemType?]) {
|
||||
for val in vals {
|
||||
self.bufferBuilder.append(val)
|
||||
}
|
||||
}
|
||||
|
||||
public func append(_ val: T.ItemType?) {
|
||||
self.bufferBuilder.append(val)
|
||||
}
|
||||
|
||||
public func appendAny(_ val: Any?) {
|
||||
self.bufferBuilder.append(val as? T.ItemType)
|
||||
}
|
||||
|
||||
public func finish() throws -> ArrowArray<T.ItemType> {
|
||||
let buffers = self.bufferBuilder.finish()
|
||||
let arrowData = try ArrowData(self.type, buffers: buffers, nullCount: self.nullCount)
|
||||
let array = try U(arrowData)
|
||||
return array
|
||||
}
|
||||
|
||||
public func getStride() -> Int {
|
||||
return self.type.getStride()
|
||||
}
|
||||
|
||||
public func toHolder() throws -> ArrowArrayHolder {
|
||||
return try ArrowArrayHolderImpl(self.finish())
|
||||
}
|
||||
}
|
||||
|
||||
public class NumberArrayBuilder<T>: ArrowArrayBuilder<FixedBufferBuilder<T>, FixedArray<T>> {
|
||||
fileprivate convenience init() throws {
|
||||
try self.init(ArrowType(ArrowType.infoForNumericType(T.self)))
|
||||
}
|
||||
}
|
||||
|
||||
public class StringArrayBuilder: ArrowArrayBuilder<VariableBufferBuilder<String>, StringArray> {
|
||||
fileprivate convenience init() throws {
|
||||
try self.init(ArrowType(ArrowType.ArrowString))
|
||||
}
|
||||
}
|
||||
|
||||
public class BinaryArrayBuilder: ArrowArrayBuilder<VariableBufferBuilder<Data>, BinaryArray> {
|
||||
fileprivate convenience init() throws {
|
||||
try self.init(ArrowType(ArrowType.ArrowBinary))
|
||||
}
|
||||
}
|
||||
|
||||
public class BoolArrayBuilder: ArrowArrayBuilder<BoolBufferBuilder, BoolArray> {
|
||||
fileprivate convenience init() throws {
|
||||
try self.init(ArrowType(ArrowType.ArrowBool))
|
||||
}
|
||||
}
|
||||
|
||||
public class Date32ArrayBuilder: ArrowArrayBuilder<Date32BufferBuilder, Date32Array> {
|
||||
fileprivate convenience init() throws {
|
||||
try self.init(ArrowType(ArrowType.ArrowDate32))
|
||||
}
|
||||
}
|
||||
|
||||
public class Date64ArrayBuilder: ArrowArrayBuilder<Date64BufferBuilder, Date64Array> {
|
||||
fileprivate convenience init() throws {
|
||||
try self.init(ArrowType(ArrowType.ArrowDate64))
|
||||
}
|
||||
}
|
||||
|
||||
public class Time32ArrayBuilder: ArrowArrayBuilder<FixedBufferBuilder<Time32>, Time32Array> {
|
||||
fileprivate convenience init(_ unit: ArrowTime32Unit) throws {
|
||||
try self.init(ArrowTypeTime32(unit))
|
||||
}
|
||||
}
|
||||
|
||||
public class Time64ArrayBuilder: ArrowArrayBuilder<FixedBufferBuilder<Time64>, Time64Array> {
|
||||
fileprivate convenience init(_ unit: ArrowTime64Unit) throws {
|
||||
try self.init(ArrowTypeTime64(unit))
|
||||
}
|
||||
}
|
||||
|
||||
public class StructArrayBuilder: ArrowArrayBuilder<StructBufferBuilder, StructArray> {
|
||||
let builders: [any ArrowArrayHolderBuilder]
|
||||
let fields: [ArrowField]
|
||||
public init(_ fields: [ArrowField], builders: [any ArrowArrayHolderBuilder]) throws {
|
||||
self.fields = fields
|
||||
self.builders = builders
|
||||
try super.init(ArrowNestedType(ArrowType.ArrowStruct, fields: fields))
|
||||
self.bufferBuilder.initializeTypeInfo(fields)
|
||||
}
|
||||
|
||||
public init(_ fields: [ArrowField]) throws {
|
||||
self.fields = fields
|
||||
var builders = [any ArrowArrayHolderBuilder]()
|
||||
for field in fields {
|
||||
builders.append(try ArrowArrayBuilders.loadBuilder(arrowType: field.type))
|
||||
}
|
||||
|
||||
self.builders = builders
|
||||
try super.init(ArrowNestedType(ArrowType.ArrowStruct, fields: fields))
|
||||
}
|
||||
|
||||
public override func append(_ values: [Any?]?) {
|
||||
self.bufferBuilder.append(values)
|
||||
if let anyValues = values {
|
||||
for index in 0..<builders.count {
|
||||
self.builders[index].appendAny(anyValues[index])
|
||||
}
|
||||
} else {
|
||||
for index in 0..<builders.count {
|
||||
self.builders[index].appendAny(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override func finish() throws -> StructArray {
|
||||
let buffers = self.bufferBuilder.finish()
|
||||
var childData = [ArrowData]()
|
||||
for builder in self.builders {
|
||||
childData.append(try builder.toHolder().array.arrowData)
|
||||
}
|
||||
|
||||
let arrowData = try ArrowData(self.type, buffers: buffers,
|
||||
children: childData, nullCount: self.nullCount,
|
||||
length: self.length)
|
||||
let structArray = try StructArray(arrowData)
|
||||
return structArray
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowArrayBuilders {
|
||||
public static func loadBuilder( // swiftlint:disable:this cyclomatic_complexity
|
||||
_ builderType: Any.Type) throws -> ArrowArrayHolderBuilder {
|
||||
if builderType == Int8.self || builderType == Int8?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int8>
|
||||
} else if builderType == Int16.self || builderType == Int16?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int16>
|
||||
} else if builderType == Int32.self || builderType == Int32?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int32>
|
||||
} else if builderType == Int64.self || builderType == Int64?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int64>
|
||||
} else if builderType == Float.self || builderType == Float?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Float>
|
||||
} else if builderType == UInt8.self || builderType == UInt8?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt8>
|
||||
} else if builderType == UInt16.self || builderType == UInt16?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt16>
|
||||
} else if builderType == UInt32.self || builderType == UInt32?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt32>
|
||||
} else if builderType == UInt64.self || builderType == UInt64?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt64>
|
||||
} else if builderType == Double.self || builderType == Double?.self {
|
||||
return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Double>
|
||||
} else if builderType == String.self || builderType == String?.self {
|
||||
return try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
} else if builderType == Bool.self || builderType == Bool?.self {
|
||||
return try ArrowArrayBuilders.loadBoolArrayBuilder()
|
||||
} else if builderType == Date.self || builderType == Date?.self {
|
||||
return try ArrowArrayBuilders.loadDate64ArrayBuilder()
|
||||
} else {
|
||||
throw ArrowError.invalid("Invalid type for builder: \(builderType)")
|
||||
}
|
||||
}
|
||||
|
||||
public static func isValidBuilderType<T>(_ type: T.Type) -> Bool {
|
||||
return type == Int8?.self || type == Int16?.self ||
|
||||
type == Int32?.self || type == Int64?.self ||
|
||||
type == UInt8?.self || type == UInt16?.self ||
|
||||
type == UInt32?.self || type == UInt64?.self ||
|
||||
type == String?.self || type == Double?.self ||
|
||||
type == Float?.self || type == Date?.self ||
|
||||
type == Bool?.self || type == Bool.self ||
|
||||
type == Int8.self || type == Int16.self ||
|
||||
type == Int32.self || type == Int64.self ||
|
||||
type == UInt8.self || type == UInt16.self ||
|
||||
type == UInt32.self || type == UInt64.self ||
|
||||
type == String.self || type == Double.self ||
|
||||
type == Float.self || type == Date.self
|
||||
}
|
||||
|
||||
public static func loadStructArrayBuilderForType<T>(_ obj: T) throws -> StructArrayBuilder {
|
||||
let mirror = Mirror(reflecting: obj)
|
||||
var builders = [ArrowArrayHolderBuilder]()
|
||||
var fields = [ArrowField]()
|
||||
for (property, value) in mirror.children {
|
||||
guard let propertyName = property else {
|
||||
continue
|
||||
}
|
||||
|
||||
let builderType = type(of: value)
|
||||
let arrowType = ArrowType(ArrowType.infoForType(builderType))
|
||||
fields.append(ArrowField(propertyName, type: arrowType, isNullable: true))
|
||||
builders.append(try loadBuilder(arrowType: arrowType))
|
||||
}
|
||||
|
||||
return try StructArrayBuilder(fields, builders: builders)
|
||||
}
|
||||
|
||||
public static func loadBuilder( // swiftlint:disable:this cyclomatic_complexity
|
||||
arrowType: ArrowType) throws -> ArrowArrayHolderBuilder {
|
||||
switch arrowType.id {
|
||||
case .uint8:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt8>
|
||||
case .uint16:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt16>
|
||||
case .uint32:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt32>
|
||||
case .uint64:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt64>
|
||||
case .int8:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<Int8>
|
||||
case .int16:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<Int16>
|
||||
case .int32:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<Int32>
|
||||
case .int64:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<Int64>
|
||||
case .double:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<Double>
|
||||
case .float:
|
||||
return try loadNumberArrayBuilder() as NumberArrayBuilder<Float>
|
||||
case .string:
|
||||
return try StringArrayBuilder()
|
||||
case .boolean:
|
||||
return try BoolArrayBuilder()
|
||||
case .binary:
|
||||
return try BinaryArrayBuilder()
|
||||
case .date32:
|
||||
return try Date32ArrayBuilder()
|
||||
case .date64:
|
||||
return try Date64ArrayBuilder()
|
||||
case .time32:
|
||||
guard let timeType = arrowType as? ArrowTypeTime32 else {
|
||||
throw ArrowError.invalid("Expected arrow type for \(arrowType.id) not found")
|
||||
}
|
||||
return try Time32ArrayBuilder(timeType.unit)
|
||||
case .time64:
|
||||
guard let timeType = arrowType as? ArrowTypeTime64 else {
|
||||
throw ArrowError.invalid("Expected arrow type for \(arrowType.id) not found")
|
||||
}
|
||||
return try Time64ArrayBuilder(timeType.unit)
|
||||
default:
|
||||
throw ArrowError.unknownType("Builder not found for arrow type: \(arrowType.id)")
|
||||
}
|
||||
}
|
||||
|
||||
public static func loadNumberArrayBuilder<T>() throws -> NumberArrayBuilder<T> {
|
||||
let type = T.self
|
||||
if type == Int8.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == Int16.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == Int32.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == Int64.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == UInt8.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == UInt16.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == UInt32.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == UInt64.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == Float.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else if type == Double.self {
|
||||
return try NumberArrayBuilder<T>()
|
||||
} else {
|
||||
throw ArrowError.unknownType("Type is invalid for NumberArrayBuilder")
|
||||
}
|
||||
}
|
||||
|
||||
public static func loadStringArrayBuilder() throws -> StringArrayBuilder {
|
||||
return try StringArrayBuilder()
|
||||
}
|
||||
|
||||
public static func loadBoolArrayBuilder() throws -> BoolArrayBuilder {
|
||||
return try BoolArrayBuilder()
|
||||
}
|
||||
|
||||
public static func loadDate32ArrayBuilder() throws -> Date32ArrayBuilder {
|
||||
return try Date32ArrayBuilder()
|
||||
}
|
||||
|
||||
public static func loadDate64ArrayBuilder() throws -> Date64ArrayBuilder {
|
||||
return try Date64ArrayBuilder()
|
||||
}
|
||||
|
||||
public static func loadBinaryArrayBuilder() throws -> BinaryArrayBuilder {
|
||||
return try BinaryArrayBuilder()
|
||||
}
|
||||
|
||||
public static func loadTime32ArrayBuilder(_ unit: ArrowTime32Unit) throws -> Time32ArrayBuilder {
|
||||
return try Time32ArrayBuilder(unit)
|
||||
}
|
||||
|
||||
public static func loadTime64ArrayBuilder(_ unit: ArrowTime64Unit) throws -> Time64ArrayBuilder {
|
||||
return try Time64ArrayBuilder(unit)
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class ArrowBuffer {
|
||||
static let minLength: UInt = 1 << 5
|
||||
static let maxLength = UInt.max
|
||||
fileprivate(set) var length: UInt
|
||||
let capacity: UInt
|
||||
public let rawPointer: UnsafeMutableRawPointer
|
||||
let isMemoryOwner: Bool
|
||||
|
||||
init(length: UInt, capacity: UInt, rawPointer: UnsafeMutableRawPointer, isMemoryOwner: Bool = true) {
|
||||
self.length = length
|
||||
self.capacity = capacity
|
||||
self.rawPointer = rawPointer
|
||||
self.isMemoryOwner = isMemoryOwner
|
||||
}
|
||||
|
||||
deinit {
|
||||
if isMemoryOwner {
|
||||
self.rawPointer.deallocate()
|
||||
}
|
||||
}
|
||||
|
||||
func append(to data: inout Data) {
|
||||
let ptr = UnsafePointer(rawPointer.assumingMemoryBound(to: UInt8.self))
|
||||
data.append(ptr, count: Int(capacity))
|
||||
}
|
||||
|
||||
static func createEmptyBuffer() -> ArrowBuffer {
|
||||
return ArrowBuffer(
|
||||
length: 0,
|
||||
capacity: 0,
|
||||
rawPointer: UnsafeMutableRawPointer.allocate(byteCount: 0, alignment: .zero))
|
||||
}
|
||||
|
||||
static func createBuffer(_ data: [UInt8], length: UInt) -> ArrowBuffer {
|
||||
let byteCount = UInt(data.count)
|
||||
let capacity = alignTo64(byteCount)
|
||||
let memory = MemoryAllocator(64)
|
||||
let rawPointer = memory.allocateArray(Int(capacity))
|
||||
rawPointer.copyMemory(from: data, byteCount: data.count)
|
||||
return ArrowBuffer(length: length, capacity: capacity, rawPointer: rawPointer)
|
||||
}
|
||||
|
||||
static func createBuffer(_ length: UInt, size: UInt, doAlign: Bool = true) -> ArrowBuffer {
|
||||
let actualLen = max(length, ArrowBuffer.minLength)
|
||||
let byteCount = size * actualLen
|
||||
var capacity = byteCount
|
||||
if doAlign {
|
||||
capacity = alignTo64(byteCount)
|
||||
}
|
||||
|
||||
let memory = MemoryAllocator(64)
|
||||
let rawPointer = memory.allocateArray(Int(capacity))
|
||||
rawPointer.initializeMemory(as: UInt8.self, repeating: 0, count: Int(capacity))
|
||||
return ArrowBuffer(length: length, capacity: capacity, rawPointer: rawPointer)
|
||||
}
|
||||
|
||||
static func copyCurrent(_ from: ArrowBuffer, to: inout ArrowBuffer, len: UInt) {
|
||||
to.rawPointer.copyMemory(from: from.rawPointer, byteCount: Int(len))
|
||||
}
|
||||
|
||||
private static func alignTo64(_ length: UInt) -> UInt {
|
||||
let bufAlignment = length % 64
|
||||
if bufAlignment != 0 {
|
||||
return length + (64 - bufAlignment) + 8
|
||||
}
|
||||
|
||||
return length + 8
|
||||
}
|
||||
}
|
||||
@@ -1,381 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public protocol ArrowBufferBuilder {
|
||||
associatedtype ItemType
|
||||
var capacity: UInt {get}
|
||||
var length: UInt {get}
|
||||
var nullCount: UInt {get}
|
||||
var offset: UInt {get}
|
||||
init() throws
|
||||
func append(_ newValue: ItemType?)
|
||||
func isNull(_ index: UInt) -> Bool
|
||||
func resize(_ length: UInt)
|
||||
func finish() -> [ArrowBuffer]
|
||||
}
|
||||
|
||||
public class BaseBufferBuilder {
|
||||
var nulls: ArrowBuffer
|
||||
public var offset: UInt = 0
|
||||
public var capacity: UInt {return self.nulls.capacity}
|
||||
public var length: UInt = 0
|
||||
public var nullCount: UInt = 0
|
||||
|
||||
init(_ nulls: ArrowBuffer) {
|
||||
self.nulls = nulls
|
||||
}
|
||||
|
||||
public func isNull(_ index: UInt) -> Bool {
|
||||
return self.nulls.length == 0 || BitUtility.isSet(index + self.offset, buffer: self.nulls)
|
||||
}
|
||||
|
||||
func resizeLength(_ data: ArrowBuffer, len: UInt = 0) -> UInt {
|
||||
if len == 0 || len < data.length * 2 {
|
||||
if data.length == 0 || data.length * 2 < ArrowBuffer.minLength {
|
||||
return ArrowBuffer.minLength
|
||||
}
|
||||
return UInt(data.length * 2)
|
||||
}
|
||||
|
||||
return UInt(len * 2)
|
||||
}
|
||||
}
|
||||
|
||||
public class ValuesBufferBuilder<T>: BaseBufferBuilder {
|
||||
var values: ArrowBuffer
|
||||
var stride: Int
|
||||
public override var capacity: UInt {return self.values.capacity}
|
||||
|
||||
init(values: ArrowBuffer, nulls: ArrowBuffer, stride: Int = MemoryLayout<T>.stride) {
|
||||
self.stride = stride
|
||||
self.values = values
|
||||
super.init(nulls)
|
||||
}
|
||||
}
|
||||
|
||||
public class FixedBufferBuilder<T>: ValuesBufferBuilder<T>, ArrowBufferBuilder {
|
||||
public typealias ItemType = T
|
||||
private let defaultVal: ItemType
|
||||
public required init() throws {
|
||||
self.defaultVal = try FixedBufferBuilder<T>.defaultValueForType()
|
||||
let values = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<T>.stride))
|
||||
let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
|
||||
super.init(values: values, nulls: nulls)
|
||||
}
|
||||
|
||||
public func append(_ newValue: ItemType?) {
|
||||
let index = UInt(self.length)
|
||||
let byteIndex = self.stride * Int(index)
|
||||
self.length += 1
|
||||
if length > self.values.length {
|
||||
self.resize(length)
|
||||
}
|
||||
|
||||
if let val = newValue {
|
||||
BitUtility.setBit(index + self.offset, buffer: self.nulls)
|
||||
self.values.rawPointer.advanced(by: byteIndex).storeBytes(of: val, as: T.self)
|
||||
} else {
|
||||
self.nullCount += 1
|
||||
BitUtility.clearBit(index + self.offset, buffer: self.nulls)
|
||||
self.values.rawPointer.advanced(by: byteIndex).storeBytes(of: defaultVal, as: T.self)
|
||||
}
|
||||
}
|
||||
|
||||
public func resize(_ length: UInt) {
|
||||
if length > self.values.length {
|
||||
let resizeLength = resizeLength(self.values)
|
||||
var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<T>.size))
|
||||
var nulls = ArrowBuffer.createBuffer(resizeLength/8 + 1, size: UInt(MemoryLayout<UInt8>.size))
|
||||
ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity)
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
|
||||
self.values = values
|
||||
self.nulls = nulls
|
||||
}
|
||||
}
|
||||
|
||||
public func finish() -> [ArrowBuffer] {
|
||||
let length = self.length
|
||||
var values = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<T>.size))
|
||||
var nulls = ArrowBuffer.createBuffer(length/8 + 1, size: UInt(MemoryLayout<UInt8>.size))
|
||||
ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity)
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
|
||||
return [nulls, values]
|
||||
}
|
||||
|
||||
fileprivate static func defaultValueForType() throws -> T {
|
||||
let type = T.self
|
||||
if type == Int8.self {
|
||||
return Int8(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == Int16.self {
|
||||
return Int16(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == Int32.self {
|
||||
return Int32(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == Int64.self {
|
||||
return Int64(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == UInt8.self {
|
||||
return UInt8(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == UInt16.self {
|
||||
return UInt16(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == UInt32.self {
|
||||
return UInt32(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == UInt64.self {
|
||||
return UInt64(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == Float.self {
|
||||
return Float(0) as! T // swiftlint:disable:this force_cast
|
||||
} else if type == Double.self {
|
||||
return Double(0) as! T // swiftlint:disable:this force_cast
|
||||
}
|
||||
|
||||
throw ArrowError.unknownType("Unable to determine default value")
|
||||
}
|
||||
}
|
||||
|
||||
public class BoolBufferBuilder: ValuesBufferBuilder<Bool>, ArrowBufferBuilder {
|
||||
public typealias ItemType = Bool
|
||||
public required init() throws {
|
||||
let values = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
|
||||
let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
|
||||
super.init(values: values, nulls: nulls)
|
||||
}
|
||||
|
||||
public func append(_ newValue: ItemType?) {
|
||||
let index = UInt(self.length)
|
||||
self.length += 1
|
||||
if (length/8) > self.values.length {
|
||||
self.resize(length)
|
||||
}
|
||||
|
||||
if newValue != nil {
|
||||
BitUtility.setBit(index + self.offset, buffer: self.nulls)
|
||||
if newValue == true {
|
||||
BitUtility.setBit(index + self.offset, buffer: self.values)
|
||||
} else {
|
||||
BitUtility.clearBit(index + self.offset, buffer: self.values)
|
||||
}
|
||||
|
||||
} else {
|
||||
self.nullCount += 1
|
||||
BitUtility.clearBit(index + self.offset, buffer: self.nulls)
|
||||
BitUtility.clearBit(index + self.offset, buffer: self.values)
|
||||
}
|
||||
}
|
||||
|
||||
public func resize(_ length: UInt) {
|
||||
if (length/8) > self.values.length {
|
||||
let resizeLength = resizeLength(self.values)
|
||||
var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size))
|
||||
var nulls = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size))
|
||||
ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity)
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
|
||||
self.values = values
|
||||
self.nulls = nulls
|
||||
}
|
||||
}
|
||||
|
||||
public func finish() -> [ArrowBuffer] {
|
||||
let length = self.length
|
||||
var values = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<UInt8>.size))
|
||||
var nulls = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<UInt8>.size))
|
||||
ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity)
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
|
||||
return [nulls, values]
|
||||
}
|
||||
}
|
||||
|
||||
public class VariableBufferBuilder<T>: ValuesBufferBuilder<T>, ArrowBufferBuilder {
|
||||
public typealias ItemType = T
|
||||
var offsets: ArrowBuffer
|
||||
let binaryStride = MemoryLayout<UInt8>.stride
|
||||
public required init() throws {
|
||||
let values = ArrowBuffer.createBuffer(0, size: UInt(binaryStride))
|
||||
let nulls = ArrowBuffer.createBuffer(0, size: UInt(binaryStride))
|
||||
self.offsets = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<Int32>.stride))
|
||||
super.init(values: values, nulls: nulls, stride: binaryStride)
|
||||
}
|
||||
|
||||
public func append(_ newValue: ItemType?) {
|
||||
let index = UInt(self.length)
|
||||
self.length += 1
|
||||
let offsetIndex = MemoryLayout<Int32>.stride * Int(index)
|
||||
if self.length >= self.offsets.length {
|
||||
self.resize(UInt( self.offsets.length + 1))
|
||||
}
|
||||
var binData: Data
|
||||
var isNull = false
|
||||
if let val = newValue {
|
||||
binData = getBytesFor(val)!
|
||||
} else {
|
||||
var nullVal = 0
|
||||
isNull = true
|
||||
binData = Data(bytes: &nullVal, count: MemoryLayout<UInt32>.size)
|
||||
}
|
||||
|
||||
var currentIndex: Int32 = 0
|
||||
var currentOffset: Int32 = Int32(binData.count)
|
||||
if index > 0 {
|
||||
currentIndex = self.offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self)
|
||||
currentOffset += currentIndex
|
||||
if currentOffset > self.values.length {
|
||||
self.value_resize(UInt(currentOffset))
|
||||
}
|
||||
}
|
||||
|
||||
if isNull {
|
||||
self.nullCount += 1
|
||||
BitUtility.clearBit(index + self.offset, buffer: self.nulls)
|
||||
} else {
|
||||
BitUtility.setBit(index + self.offset, buffer: self.nulls)
|
||||
}
|
||||
|
||||
binData.withUnsafeBytes { bufferPointer in
|
||||
let rawPointer = bufferPointer.baseAddress!
|
||||
self.values.rawPointer.advanced(by: Int(currentIndex))
|
||||
.copyMemory(from: rawPointer, byteCount: binData.count)
|
||||
}
|
||||
|
||||
self.offsets.rawPointer.advanced(by: (offsetIndex + MemoryLayout<Int32>.stride))
|
||||
.storeBytes(of: currentOffset, as: Int32.self)
|
||||
}
|
||||
|
||||
public func value_resize(_ length: UInt) {
|
||||
if length > self.values.length {
|
||||
let resizeLength = resizeLength(self.values, len: length)
|
||||
var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size))
|
||||
ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity)
|
||||
self.values = values
|
||||
}
|
||||
}
|
||||
|
||||
public func resize(_ length: UInt) {
|
||||
if length > self.offsets.length {
|
||||
let resizeLength = resizeLength(self.offsets, len: length)
|
||||
var nulls = ArrowBuffer.createBuffer(resizeLength/8 + 1, size: UInt(MemoryLayout<UInt8>.size))
|
||||
var offsets = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<Int32>.size))
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
|
||||
ArrowBuffer.copyCurrent(self.offsets, to: &offsets, len: self.offsets.capacity)
|
||||
self.nulls = nulls
|
||||
self.offsets = offsets
|
||||
}
|
||||
}
|
||||
|
||||
public func finish() -> [ArrowBuffer] {
|
||||
let length = self.length
|
||||
var values = ArrowBuffer.createBuffer(self.values.length, size: UInt(MemoryLayout<UInt8>.size))
|
||||
var nulls = ArrowBuffer.createBuffer(length/8 + 1, size: UInt(MemoryLayout<UInt8>.size))
|
||||
var offsets = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<Int32>.size))
|
||||
ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity)
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
|
||||
ArrowBuffer.copyCurrent(self.offsets, to: &offsets, len: offsets.capacity)
|
||||
return [nulls, offsets, values]
|
||||
}
|
||||
}
|
||||
|
||||
public class AbstractWrapperBufferBuilder<T, U>: ArrowBufferBuilder {
|
||||
public typealias ItemType = T
|
||||
public var capacity: UInt {return self.bufferBuilder.capacity}
|
||||
public var length: UInt {return self.bufferBuilder.length}
|
||||
public var nullCount: UInt {return self.bufferBuilder.nullCount}
|
||||
public var offset: UInt {return self.bufferBuilder.offset}
|
||||
let bufferBuilder: FixedBufferBuilder<U>
|
||||
public required init() throws {
|
||||
self.bufferBuilder = try FixedBufferBuilder()
|
||||
}
|
||||
|
||||
public func append(_ newValue: ItemType?) {
|
||||
fatalError("Method is not implemented")
|
||||
}
|
||||
|
||||
public func isNull(_ index: UInt) -> Bool {
|
||||
return self.bufferBuilder.isNull(index)
|
||||
}
|
||||
|
||||
public func resize(_ length: UInt) {
|
||||
self.bufferBuilder.resize(length)
|
||||
}
|
||||
|
||||
public func finish() -> [ArrowBuffer] {
|
||||
return self.bufferBuilder.finish()
|
||||
}
|
||||
}
|
||||
|
||||
public class Date32BufferBuilder: AbstractWrapperBufferBuilder<Date, Int32> {
|
||||
public override func append(_ newValue: ItemType?) {
|
||||
if let val = newValue {
|
||||
let daysSinceEpoch = Int32(val.timeIntervalSince1970 / 86400)
|
||||
self.bufferBuilder.append(daysSinceEpoch)
|
||||
} else {
|
||||
self.bufferBuilder.append(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class Date64BufferBuilder: AbstractWrapperBufferBuilder<Date, Int64> {
|
||||
public override func append(_ newValue: ItemType?) {
|
||||
if let val = newValue {
|
||||
let daysSinceEpoch = Int64(val.timeIntervalSince1970 * 1000)
|
||||
self.bufferBuilder.append(daysSinceEpoch)
|
||||
} else {
|
||||
self.bufferBuilder.append(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public final class StructBufferBuilder: BaseBufferBuilder, ArrowBufferBuilder {
|
||||
public typealias ItemType = [Any?]
|
||||
var info: ArrowNestedType?
|
||||
public init() throws {
|
||||
let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
|
||||
super.init(nulls)
|
||||
}
|
||||
|
||||
public func initializeTypeInfo(_ fields: [ArrowField]) {
|
||||
info = ArrowNestedType(ArrowType.ArrowStruct, fields: fields)
|
||||
}
|
||||
|
||||
public func append(_ newValue: [Any?]?) {
|
||||
let index = UInt(self.length)
|
||||
self.length += 1
|
||||
if length > self.nulls.length {
|
||||
self.resize(length)
|
||||
}
|
||||
|
||||
if newValue != nil {
|
||||
BitUtility.setBit(index + self.offset, buffer: self.nulls)
|
||||
} else {
|
||||
self.nullCount += 1
|
||||
BitUtility.clearBit(index + self.offset, buffer: self.nulls)
|
||||
}
|
||||
}
|
||||
|
||||
public func resize(_ length: UInt) {
|
||||
if length > self.nulls.length {
|
||||
let resizeLength = resizeLength(self.nulls)
|
||||
var nulls = ArrowBuffer.createBuffer(resizeLength/8 + 1, size: UInt(MemoryLayout<UInt8>.size))
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
|
||||
self.nulls = nulls
|
||||
}
|
||||
}
|
||||
|
||||
public func finish() -> [ArrowBuffer] {
|
||||
let length = self.length
|
||||
var nulls = ArrowBuffer.createBuffer(length/8 + 1, size: UInt(MemoryLayout<UInt8>.size))
|
||||
ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
|
||||
return [nulls]
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import ArrowC
|
||||
import Atomics
|
||||
|
||||
// The memory used by UnsafeAtomic is not automatically
|
||||
// reclaimed. Since this value is initialized once
|
||||
// and used until the program/app is closed it's
|
||||
// memory will be released on program/app exit
|
||||
let exportDataCounter: UnsafeAtomic<Int> = .create(0)
|
||||
|
||||
public class ArrowCExporter {
|
||||
private class ExportData {
|
||||
let id: Int
|
||||
init() {
|
||||
id = exportDataCounter.loadThenWrappingIncrement(ordering: .relaxed)
|
||||
ArrowCExporter.exportedData[id] = self
|
||||
}
|
||||
}
|
||||
|
||||
private class ExportSchema: ExportData {
|
||||
public let arrowTypeNameCstr: UnsafePointer<CChar>
|
||||
public let nameCstr: UnsafePointer<CChar>
|
||||
private let arrowType: ArrowType
|
||||
private let name: String
|
||||
private let arrowTypeName: String
|
||||
init(_ arrowType: ArrowType, name: String = "") throws {
|
||||
self.arrowType = arrowType
|
||||
// keeping the name str to ensure the cstring buffer remains valid
|
||||
self.name = name
|
||||
self.arrowTypeName = try arrowType.cDataFormatId
|
||||
self.nameCstr = (self.name as NSString).utf8String!
|
||||
self.arrowTypeNameCstr = (self.arrowTypeName as NSString).utf8String!
|
||||
super.init()
|
||||
}
|
||||
}
|
||||
|
||||
private class ExportArray: ExportData {
|
||||
private let arrowData: ArrowData
|
||||
private(set) var data = [UnsafeRawPointer?]()
|
||||
private(set) var buffers: UnsafeMutablePointer<UnsafeRawPointer?>
|
||||
init(_ arrowData: ArrowData) {
|
||||
// keep a reference to the ArrowData
|
||||
// obj so the memory doesn't get
|
||||
// deallocated
|
||||
self.arrowData = arrowData
|
||||
for arrowBuffer in arrowData.buffers {
|
||||
self.data.append(arrowBuffer.rawPointer)
|
||||
}
|
||||
|
||||
self.buffers = UnsafeMutablePointer<UnsafeRawPointer?>.allocate(capacity: self.data.count)
|
||||
self.buffers.initialize(from: &self.data, count: self.data.count)
|
||||
super.init()
|
||||
}
|
||||
|
||||
deinit {
|
||||
self.buffers.deinitialize(count: self.data.count)
|
||||
self.buffers.deallocate()
|
||||
}
|
||||
}
|
||||
|
||||
private static var exportedData = [Int: ExportData]()
|
||||
public init() {}
|
||||
|
||||
public func exportType(_ cSchema: inout ArrowC.ArrowSchema, arrowType: ArrowType, name: String = "") ->
|
||||
Result<Bool, ArrowError> {
|
||||
do {
|
||||
let exportSchema = try ExportSchema(arrowType, name: name)
|
||||
cSchema.format = exportSchema.arrowTypeNameCstr
|
||||
cSchema.name = exportSchema.nameCstr
|
||||
cSchema.private_data =
|
||||
UnsafeMutableRawPointer(mutating: UnsafeRawPointer(bitPattern: exportSchema.id))
|
||||
cSchema.release = {(data: UnsafeMutablePointer<ArrowC.ArrowSchema>?) in
|
||||
let arraySchema = data!.pointee
|
||||
let exportId = Int(bitPattern: arraySchema.private_data)
|
||||
guard ArrowCExporter.exportedData[exportId] != nil else {
|
||||
fatalError("Export schema not found with id \(exportId)")
|
||||
}
|
||||
|
||||
// the data associated with this exportSchema object
|
||||
// which includes the C strings for the format and name
|
||||
// be deallocated upon removal
|
||||
ArrowCExporter.exportedData.removeValue(forKey: exportId)
|
||||
ArrowC.ArrowSwiftClearReleaseSchema(data)
|
||||
}
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
return .success(true)
|
||||
}
|
||||
|
||||
public func exportField(_ schema: inout ArrowC.ArrowSchema, field: ArrowField) ->
|
||||
Result<Bool, ArrowError> {
|
||||
return exportType(&schema, arrowType: field.type, name: field.name)
|
||||
}
|
||||
|
||||
public func exportArray(_ cArray: inout ArrowC.ArrowArray, arrowData: ArrowData) {
|
||||
let exportArray = ExportArray(arrowData)
|
||||
cArray.buffers = exportArray.buffers
|
||||
cArray.length = Int64(arrowData.length)
|
||||
cArray.null_count = Int64(arrowData.nullCount)
|
||||
cArray.n_buffers = Int64(arrowData.buffers.count)
|
||||
// Swift Arrow does not currently support children or dictionaries
|
||||
// This will need to be updated once support has been added
|
||||
cArray.n_children = 0
|
||||
cArray.children = nil
|
||||
cArray.dictionary = nil
|
||||
cArray.private_data =
|
||||
UnsafeMutableRawPointer(mutating: UnsafeRawPointer(bitPattern: exportArray.id))
|
||||
cArray.release = {(data: UnsafeMutablePointer<ArrowC.ArrowArray>?) in
|
||||
let arrayData = data!.pointee
|
||||
let exportId = Int(bitPattern: arrayData.private_data)
|
||||
guard ArrowCExporter.exportedData[exportId] != nil else {
|
||||
fatalError("Export data not found with id \(exportId)")
|
||||
}
|
||||
|
||||
// the data associated with this exportArray object
|
||||
// which includes the entire arrowData object
|
||||
// and the buffers UnsafeMutablePointer[] will
|
||||
// be deallocated upon removal
|
||||
ArrowCExporter.exportedData.removeValue(forKey: exportId)
|
||||
ArrowC.ArrowSwiftClearReleaseArray(data)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import ArrowC
|
||||
|
||||
public class ImportArrayHolder: ArrowArrayHolder {
|
||||
let cArrayPtr: UnsafePointer<ArrowC.ArrowArray>
|
||||
public var type: ArrowType {self.holder.type}
|
||||
public var length: UInt {self.holder.length}
|
||||
public var nullCount: UInt {self.holder.nullCount}
|
||||
public var array: AnyArray {self.holder.array}
|
||||
public var data: ArrowData {self.holder.data}
|
||||
public var getBufferData: () -> [Data] {self.holder.getBufferData}
|
||||
public var getBufferDataSizes: () -> [Int] {self.holder.getBufferDataSizes}
|
||||
public var getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn {self.holder.getArrowColumn}
|
||||
private let holder: ArrowArrayHolder
|
||||
init(_ holder: ArrowArrayHolder, cArrayPtr: UnsafePointer<ArrowC.ArrowArray>) {
|
||||
self.cArrayPtr = cArrayPtr
|
||||
self.holder = holder
|
||||
}
|
||||
|
||||
deinit {
|
||||
if self.cArrayPtr.pointee.release != nil {
|
||||
ArrowCImporter.release(self.cArrayPtr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowCImporter {
|
||||
private func appendToBuffer(
|
||||
_ cBuffer: UnsafeRawPointer?,
|
||||
arrowBuffers: inout [ArrowBuffer],
|
||||
length: UInt) {
|
||||
if cBuffer == nil {
|
||||
arrowBuffers.append(ArrowBuffer.createEmptyBuffer())
|
||||
return
|
||||
}
|
||||
|
||||
let pointer = UnsafeMutableRawPointer(mutating: cBuffer)!
|
||||
arrowBuffers.append(
|
||||
ArrowBuffer(length: length, capacity: length, rawPointer: pointer, isMemoryOwner: false))
|
||||
}
|
||||
|
||||
public init() {}
|
||||
|
||||
public func importType(_ cArrow: String, name: String = "") ->
|
||||
Result<ArrowField, ArrowError> {
|
||||
do {
|
||||
let type = try ArrowType.fromCDataFormatId(cArrow)
|
||||
return .success(ArrowField(name, type: ArrowType(type.info), isNullable: true))
|
||||
} catch {
|
||||
return .failure(.invalid("Error occurred while attempting to import type: \(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
public func importField(_ cSchema: ArrowC.ArrowSchema) ->
|
||||
Result<ArrowField, ArrowError> {
|
||||
if cSchema.n_children > 0 {
|
||||
ArrowCImporter.release(cSchema)
|
||||
return .failure(.invalid("Children currently not supported"))
|
||||
} else if cSchema.dictionary != nil {
|
||||
ArrowCImporter.release(cSchema)
|
||||
return .failure(.invalid("Dictinoary types currently not supported"))
|
||||
}
|
||||
|
||||
switch importType(
|
||||
String(cString: cSchema.format), name: String(cString: cSchema.name)) {
|
||||
case .success(let field):
|
||||
ArrowCImporter.release(cSchema)
|
||||
return .success(field)
|
||||
case .failure(let err):
|
||||
ArrowCImporter.release(cSchema)
|
||||
return .failure(err)
|
||||
}
|
||||
}
|
||||
|
||||
public func importArray(
|
||||
_ cArray: UnsafePointer<ArrowC.ArrowArray>,
|
||||
arrowType: ArrowType,
|
||||
isNullable: Bool = false
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
let arrowField = ArrowField("", type: arrowType, isNullable: isNullable)
|
||||
return importArray(cArray, arrowField: arrowField)
|
||||
}
|
||||
|
||||
public func importArray( // swiftlint:disable:this cyclomatic_complexity function_body_length
|
||||
_ cArrayPtr: UnsafePointer<ArrowC.ArrowArray>,
|
||||
arrowField: ArrowField
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
let cArray = cArrayPtr.pointee
|
||||
if cArray.null_count < 0 {
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(.invalid("Uncomputed null count is not supported"))
|
||||
} else if cArray.n_children > 0 {
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(.invalid("Children currently not supported"))
|
||||
} else if cArray.dictionary != nil {
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(.invalid("Dictionary types currently not supported"))
|
||||
} else if cArray.offset != 0 {
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(.invalid("Offset of 0 is required but found offset: \(cArray.offset)"))
|
||||
}
|
||||
|
||||
let arrowType = arrowField.type
|
||||
let length = UInt(cArray.length)
|
||||
let nullCount = UInt(cArray.null_count)
|
||||
var arrowBuffers = [ArrowBuffer]()
|
||||
|
||||
if cArray.n_buffers > 0 {
|
||||
if cArray.buffers == nil {
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(.invalid("C array buffers is nil"))
|
||||
}
|
||||
|
||||
switch arrowType.info {
|
||||
case .variableInfo:
|
||||
if cArray.n_buffers != 3 {
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(
|
||||
.invalid("Variable buffer count expected 3 but found \(cArray.n_buffers)"))
|
||||
}
|
||||
|
||||
appendToBuffer(cArray.buffers[0], arrowBuffers: &arrowBuffers, length: UInt(ceil(Double(length) / 8)))
|
||||
appendToBuffer(cArray.buffers[1], arrowBuffers: &arrowBuffers, length: length)
|
||||
let lastOffsetLength = cArray.buffers[1]!
|
||||
.advanced(by: Int(length) * MemoryLayout<Int32>.stride)
|
||||
.load(as: Int32.self)
|
||||
appendToBuffer(cArray.buffers[2], arrowBuffers: &arrowBuffers, length: UInt(lastOffsetLength))
|
||||
default:
|
||||
if cArray.n_buffers != 2 {
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(.invalid("Expected buffer count 2 but found \(cArray.n_buffers)"))
|
||||
}
|
||||
|
||||
appendToBuffer(cArray.buffers[0], arrowBuffers: &arrowBuffers, length: UInt(ceil(Double(length) / 8)))
|
||||
appendToBuffer(cArray.buffers[1], arrowBuffers: &arrowBuffers, length: length)
|
||||
}
|
||||
}
|
||||
|
||||
switch makeArrayHolder(arrowField, buffers: arrowBuffers,
|
||||
nullCount: nullCount, children: nil, rbLength: 0) {
|
||||
case .success(let holder):
|
||||
return .success(ImportArrayHolder(holder, cArrayPtr: cArrayPtr))
|
||||
case .failure(let err):
|
||||
ArrowCImporter.release(cArrayPtr)
|
||||
return .failure(err)
|
||||
}
|
||||
}
|
||||
|
||||
public static func release(_ cArrayPtr: UnsafePointer<ArrowC.ArrowArray>) {
|
||||
if cArrayPtr.pointee.release != nil {
|
||||
let cSchemaMutablePtr = UnsafeMutablePointer<ArrowC.ArrowArray>(mutating: cArrayPtr)
|
||||
cArrayPtr.pointee.release(cSchemaMutablePtr)
|
||||
}
|
||||
}
|
||||
|
||||
public static func release(_ cSchema: ArrowC.ArrowSchema) {
|
||||
if cSchema.release != nil {
|
||||
let cSchemaPtr = UnsafeMutablePointer<ArrowC.ArrowSchema>.allocate(capacity: 1)
|
||||
cSchemaPtr.initialize(to: cSchema)
|
||||
cSchema.release(cSchemaPtr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class ArrowData {
|
||||
public let type: ArrowType
|
||||
public let buffers: [ArrowBuffer]
|
||||
public let children: [ArrowData]
|
||||
public let nullCount: UInt
|
||||
public let length: UInt
|
||||
public let stride: Int
|
||||
|
||||
convenience init(_ arrowType: ArrowType, buffers: [ArrowBuffer], nullCount: UInt) throws {
|
||||
try self.init(arrowType, buffers: buffers,
|
||||
children: [ArrowData](), nullCount: nullCount,
|
||||
length: buffers[1].length)
|
||||
}
|
||||
|
||||
init(_ arrowType: ArrowType, buffers: [ArrowBuffer], children: [ArrowData], nullCount: UInt, length: UInt) throws {
|
||||
let infoType = arrowType.info
|
||||
switch infoType {
|
||||
case let .primitiveInfo(typeId):
|
||||
if typeId == ArrowTypeId.unknown {
|
||||
throw ArrowError.unknownType("Unknown primitive type for data")
|
||||
}
|
||||
case let .variableInfo(typeId):
|
||||
if typeId == ArrowTypeId.unknown {
|
||||
throw ArrowError.unknownType("Unknown variable type for data")
|
||||
}
|
||||
case let .timeInfo(typeId):
|
||||
if typeId == ArrowTypeId.unknown {
|
||||
throw ArrowError.unknownType("Unknown time type for data")
|
||||
}
|
||||
case let .complexInfo(typeId):
|
||||
if typeId == ArrowTypeId.unknown {
|
||||
throw ArrowError.unknownType("Unknown complex type for data")
|
||||
}
|
||||
}
|
||||
|
||||
self.type = arrowType
|
||||
self.buffers = buffers
|
||||
self.children = children
|
||||
self.nullCount = nullCount
|
||||
self.length = length
|
||||
self.stride = arrowType.getStride()
|
||||
}
|
||||
|
||||
public func isNull(_ at: UInt) -> Bool {
|
||||
let nullBuffer = buffers[0]
|
||||
return nullBuffer.length > 0 && !BitUtility.isSet(at, buffer: nullBuffer)
|
||||
}
|
||||
}
|
||||
@@ -1,376 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class ArrowDecoder: Decoder {
|
||||
var rbIndex: UInt = 0
|
||||
var singleRBCol: Int = 0
|
||||
public var codingPath: [CodingKey] = []
|
||||
public var userInfo: [CodingUserInfoKey: Any] = [:]
|
||||
public let rb: RecordBatch
|
||||
public let nameToCol: [String: ArrowArrayHolder]
|
||||
public let columns: [ArrowArrayHolder]
|
||||
public init(_ decoder: ArrowDecoder) {
|
||||
self.userInfo = decoder.userInfo
|
||||
self.codingPath = decoder.codingPath
|
||||
self.rb = decoder.rb
|
||||
self.columns = decoder.columns
|
||||
self.nameToCol = decoder.nameToCol
|
||||
self.rbIndex = decoder.rbIndex
|
||||
}
|
||||
|
||||
public init(_ rb: RecordBatch) {
|
||||
self.rb = rb
|
||||
var colMapping = [String: ArrowArrayHolder]()
|
||||
var columns = [ArrowArrayHolder]()
|
||||
for index in 0..<self.rb.schema.fields.count {
|
||||
let field = self.rb.schema.fields[index]
|
||||
columns.append(self.rb.column(index))
|
||||
colMapping[field.name] = self.rb.column(index)
|
||||
}
|
||||
|
||||
self.columns = columns
|
||||
self.nameToCol = colMapping
|
||||
}
|
||||
|
||||
public func decode<T: Decodable, U: Decodable>(_ type: [T: U].Type) throws -> [T: U] {
|
||||
var output = [T: U]()
|
||||
if rb.columnCount != 2 {
|
||||
throw ArrowError.invalid("RecordBatch column count of 2 is required to decode to map")
|
||||
}
|
||||
|
||||
for index in 0..<rb.length {
|
||||
self.rbIndex = index
|
||||
self.singleRBCol = 0
|
||||
let key = try T.init(from: self)
|
||||
self.singleRBCol = 1
|
||||
let value = try U.init(from: self)
|
||||
output[key] = value
|
||||
}
|
||||
|
||||
self.singleRBCol = 0
|
||||
return output
|
||||
}
|
||||
|
||||
public func decode<T: Decodable>(_ type: T.Type) throws -> [T] {
|
||||
var output = [T]()
|
||||
for index in 0..<rb.length {
|
||||
self.rbIndex = index
|
||||
output.append(try type.init(from: self))
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
public func container<Key>(keyedBy type: Key.Type
|
||||
) -> KeyedDecodingContainer<Key> where Key: CodingKey {
|
||||
let container = ArrowKeyedDecoding<Key>(self, codingPath: codingPath)
|
||||
return KeyedDecodingContainer(container)
|
||||
}
|
||||
|
||||
public func unkeyedContainer() -> UnkeyedDecodingContainer {
|
||||
return ArrowUnkeyedDecoding(self, codingPath: codingPath)
|
||||
}
|
||||
|
||||
public func singleValueContainer() -> SingleValueDecodingContainer {
|
||||
return ArrowSingleValueDecoding(self, codingPath: codingPath)
|
||||
}
|
||||
|
||||
func getCol(_ name: String) throws -> AnyArray {
|
||||
guard let col = self.nameToCol[name] else {
|
||||
throw ArrowError.invalid("Column for key \"\(name)\" not found")
|
||||
}
|
||||
|
||||
return col.array
|
||||
}
|
||||
|
||||
func getCol(_ index: Int) throws -> AnyArray {
|
||||
if index >= self.columns.count {
|
||||
throw ArrowError.outOfBounds(index: Int64(index))
|
||||
}
|
||||
|
||||
return self.columns[index].array
|
||||
}
|
||||
|
||||
func doDecode<T>(_ key: CodingKey) throws -> T? {
|
||||
let array: AnyArray = try self.getCol(key.stringValue)
|
||||
return array.asAny(self.rbIndex) as? T
|
||||
}
|
||||
|
||||
func doDecode<T>(_ col: Int) throws -> T? {
|
||||
let array: AnyArray = try self.getCol(col)
|
||||
return array.asAny(self.rbIndex) as? T
|
||||
}
|
||||
|
||||
func isNull(_ key: CodingKey) throws -> Bool {
|
||||
let array: AnyArray = try self.getCol(key.stringValue)
|
||||
return array.asAny(self.rbIndex) == nil
|
||||
}
|
||||
|
||||
func isNull(_ col: Int) throws -> Bool {
|
||||
let array: AnyArray = try self.getCol(col)
|
||||
return array.asAny(self.rbIndex) == nil
|
||||
}
|
||||
}
|
||||
|
||||
private struct ArrowUnkeyedDecoding: UnkeyedDecodingContainer {
|
||||
var codingPath: [CodingKey]
|
||||
var count: Int? = 0
|
||||
var isAtEnd: Bool = false
|
||||
var currentIndex: Int = 0
|
||||
let decoder: ArrowDecoder
|
||||
|
||||
init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) {
|
||||
self.decoder = decoder
|
||||
self.codingPath = codingPath
|
||||
self.count = self.decoder.columns.count
|
||||
}
|
||||
|
||||
mutating func increment() {
|
||||
self.currentIndex += 1
|
||||
self.isAtEnd = self.currentIndex >= self.count!
|
||||
}
|
||||
|
||||
mutating func decodeNil() throws -> Bool {
|
||||
defer {increment()}
|
||||
return try self.decoder.isNull(self.currentIndex)
|
||||
}
|
||||
|
||||
mutating func decode<T>(_ type: T.Type) throws -> T where T: Decodable {
|
||||
if type == Int8?.self || type == Int16?.self ||
|
||||
type == Int32?.self || type == Int64?.self ||
|
||||
type == UInt8?.self || type == UInt16?.self ||
|
||||
type == UInt32?.self || type == UInt64?.self ||
|
||||
type == String?.self || type == Double?.self ||
|
||||
type == Float?.self || type == Date?.self ||
|
||||
type == Bool?.self || type == Bool.self ||
|
||||
type == Int8.self || type == Int16.self ||
|
||||
type == Int32.self || type == Int64.self ||
|
||||
type == UInt8.self || type == UInt16.self ||
|
||||
type == UInt32.self || type == UInt64.self ||
|
||||
type == String.self || type == Double.self ||
|
||||
type == Float.self || type == Date.self {
|
||||
defer {increment()}
|
||||
return try self.decoder.doDecode(self.currentIndex)!
|
||||
} else {
|
||||
throw ArrowError.invalid("Type \(type) is currently not supported")
|
||||
}
|
||||
}
|
||||
|
||||
func nestedContainer<NestedKey>(
|
||||
keyedBy type: NestedKey.Type
|
||||
) throws -> KeyedDecodingContainer<NestedKey> where NestedKey: CodingKey {
|
||||
throw ArrowError.invalid("Nested decoding is currently not supported.")
|
||||
}
|
||||
|
||||
func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer {
|
||||
throw ArrowError.invalid("Nested decoding is currently not supported.")
|
||||
}
|
||||
|
||||
func superDecoder() throws -> Decoder {
|
||||
throw ArrowError.invalid("super decoding is currently not supported.")
|
||||
}
|
||||
}
|
||||
|
||||
private struct ArrowKeyedDecoding<Key: CodingKey>: KeyedDecodingContainerProtocol {
|
||||
var codingPath = [CodingKey]()
|
||||
var allKeys = [Key]()
|
||||
let decoder: ArrowDecoder
|
||||
|
||||
init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) {
|
||||
self.decoder = decoder
|
||||
self.codingPath = codingPath
|
||||
}
|
||||
|
||||
func contains(_ key: Key) -> Bool {
|
||||
return self.decoder.nameToCol.keys.contains(key.stringValue)
|
||||
}
|
||||
|
||||
func decodeNil(forKey key: Key) throws -> Bool {
|
||||
try self.decoder.isNull(key)
|
||||
}
|
||||
|
||||
func decode(_ type: Bool.Type, forKey key: Key) throws -> Bool {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: String.Type, forKey key: Key) throws -> String {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: Double.Type, forKey key: Key) throws -> Double {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: Float.Type, forKey key: Key) throws -> Float {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int.Type, forKey key: Key) throws -> Int {
|
||||
throw ArrowError.invalid(
|
||||
"Int type is not supported (please use Int8, Int16, Int32 or Int64)")
|
||||
}
|
||||
|
||||
func decode(_ type: Int8.Type, forKey key: Key) throws -> Int8 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int16.Type, forKey key: Key) throws -> Int16 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int32.Type, forKey key: Key) throws -> Int32 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int64.Type, forKey key: Key) throws -> Int64 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt.Type, forKey key: Key) throws -> UInt {
|
||||
throw ArrowError.invalid(
|
||||
"UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
|
||||
}
|
||||
|
||||
func decode(_ type: UInt8.Type, forKey key: Key) throws -> UInt8 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt16.Type, forKey key: Key) throws -> UInt16 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt32.Type, forKey key: Key) throws -> UInt32 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt64.Type, forKey key: Key) throws -> UInt64 {
|
||||
return try self.decoder.doDecode(key)!
|
||||
}
|
||||
|
||||
func decode<T>(_ type: T.Type, forKey key: Key) throws -> T where T: Decodable {
|
||||
if ArrowArrayBuilders.isValidBuilderType(type) || type == Date.self {
|
||||
return try self.decoder.doDecode(key)!
|
||||
} else {
|
||||
throw ArrowError.invalid("Type \(type) is currently not supported")
|
||||
}
|
||||
}
|
||||
|
||||
func nestedContainer<NestedKey>(
|
||||
keyedBy type: NestedKey.Type,
|
||||
forKey key: Key
|
||||
) throws -> KeyedDecodingContainer<NestedKey> where NestedKey: CodingKey {
|
||||
throw ArrowError.invalid("Nested decoding is currently not supported.")
|
||||
}
|
||||
|
||||
func nestedUnkeyedContainer(forKey key: Key) throws -> UnkeyedDecodingContainer {
|
||||
throw ArrowError.invalid("Nested decoding is currently not supported.")
|
||||
}
|
||||
|
||||
func superDecoder() throws -> Decoder {
|
||||
throw ArrowError.invalid("super decoding is currently not supported.")
|
||||
}
|
||||
|
||||
func superDecoder(forKey key: Key) throws -> Decoder {
|
||||
throw ArrowError.invalid("super decoding is currently not supported.")
|
||||
}
|
||||
}
|
||||
|
||||
private struct ArrowSingleValueDecoding: SingleValueDecodingContainer {
|
||||
var codingPath = [CodingKey]()
|
||||
let decoder: ArrowDecoder
|
||||
|
||||
init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) {
|
||||
self.decoder = decoder
|
||||
self.codingPath = codingPath
|
||||
}
|
||||
|
||||
func decodeNil() -> Bool {
|
||||
do {
|
||||
return try self.decoder.isNull(self.decoder.singleRBCol)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func decode(_ type: Bool.Type) throws -> Bool {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: String.Type) throws -> String {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: Double.Type) throws -> Double {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: Float.Type) throws -> Float {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int.Type) throws -> Int {
|
||||
throw ArrowError.invalid(
|
||||
"Int type is not supported (please use Int8, Int16, Int32 or Int64)")
|
||||
}
|
||||
|
||||
func decode(_ type: Int8.Type) throws -> Int8 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int16.Type) throws -> Int16 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int32.Type) throws -> Int32 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: Int64.Type) throws -> Int64 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt.Type) throws -> UInt {
|
||||
throw ArrowError.invalid(
|
||||
"UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
|
||||
}
|
||||
|
||||
func decode(_ type: UInt8.Type) throws -> UInt8 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt16.Type) throws -> UInt16 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt32.Type) throws -> UInt32 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode(_ type: UInt64.Type) throws -> UInt64 {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
}
|
||||
|
||||
func decode<T>(_ type: T.Type) throws -> T where T: Decodable {
|
||||
if ArrowArrayBuilders.isValidBuilderType(type) || type == Date.self {
|
||||
return try self.decoder.doDecode(self.decoder.singleRBCol)!
|
||||
} else {
|
||||
throw ArrowError.invalid("Type \(type) is currently not supported")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,456 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class ArrowEncoder: Encoder {
|
||||
public private(set) var builders = [String: ArrowArrayHolderBuilder]()
|
||||
private var byIndex = [String]()
|
||||
public var codingPath: [CodingKey] = []
|
||||
public var userInfo: [CodingUserInfoKey: Any] = [:]
|
||||
var errorMsg: String?
|
||||
// this is used for Dictionary types. A dictionary type
|
||||
// will give each key and value there own index so instead
|
||||
// of having a 2 column RecordBatch you would have
|
||||
// 2 * length(dictionary) column RecordBatch. Which would not
|
||||
// be the expected output.
|
||||
var modForIndex: Int?
|
||||
|
||||
public init() {}
|
||||
|
||||
public init(_ builders: [String: ArrowArrayHolderBuilder], byIndex: [String]) {
|
||||
self.builders = builders
|
||||
self.byIndex = byIndex
|
||||
}
|
||||
|
||||
public static func encode<T: Encodable>(_ data: T) throws -> RecordBatch {
|
||||
let encoder = try loadEncoder(data)
|
||||
try data.encode(to: encoder)
|
||||
return try encoder.finish()
|
||||
}
|
||||
|
||||
public static func encode<T: Encodable>(_ rows: [T]) throws -> RecordBatch? {
|
||||
if rows.isEmpty {
|
||||
return nil
|
||||
}
|
||||
|
||||
let encoder = try loadEncoder(rows[0])
|
||||
for row in rows {
|
||||
try row.encode(to: encoder)
|
||||
}
|
||||
|
||||
return try encoder.finish()
|
||||
}
|
||||
|
||||
static func loadEncoder<T>(_ data: T) throws -> ArrowEncoder {
|
||||
// this will check if T is a simple built in type
|
||||
// (UInt, Int, Int8, String, Date, etc...).
|
||||
if ArrowArrayBuilders.isValidBuilderType(T.self) {
|
||||
let builders = ["col0": try ArrowArrayBuilders.loadBuilder(T.self)]
|
||||
return ArrowEncoder(builders, byIndex: ["col0"])
|
||||
} else {
|
||||
let encoder = ArrowEncoder()
|
||||
if data is [AnyHashable: Any] {
|
||||
encoder.modForIndex = 2
|
||||
}
|
||||
|
||||
return encoder
|
||||
}
|
||||
}
|
||||
|
||||
public func finish() throws -> RecordBatch {
|
||||
try throwIfInvalid()
|
||||
let batchBuilder = RecordBatch.Builder()
|
||||
for key in byIndex {
|
||||
batchBuilder.addColumn(key, arrowArray: try builders[key]!.toHolder())
|
||||
}
|
||||
|
||||
switch batchBuilder.finish() {
|
||||
case .success(let rb):
|
||||
return rb
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public func container<Key>(keyedBy type: Key.Type) -> KeyedEncodingContainer<Key> where Key: CodingKey {
|
||||
var container = ArrowKeyedEncoding<Key>(self)
|
||||
container.codingPath = codingPath
|
||||
return KeyedEncodingContainer(container)
|
||||
}
|
||||
|
||||
public func unkeyedContainer() -> UnkeyedEncodingContainer {
|
||||
return ArrowUnkeyedEncoding(self, codingPath: self.codingPath)
|
||||
}
|
||||
|
||||
public func singleValueContainer() -> SingleValueEncodingContainer {
|
||||
return ArrowSingleValueEncoding(self, codingPath: codingPath)
|
||||
}
|
||||
|
||||
func doEncodeNil(key: CodingKey) throws {
|
||||
try throwIfInvalid()
|
||||
guard let builder = builders[key.stringValue] else {
|
||||
throw ArrowError.invalid("Column not found for key: \(key)")
|
||||
}
|
||||
|
||||
builder.appendAny(nil)
|
||||
}
|
||||
|
||||
// This is required by the keyed and unkeyed encoders as columns are
|
||||
// added when the first row of the data is encoded. This is done due
|
||||
// to limitations in the Swifts Mirror API (ex: it is unable to correctly
|
||||
// find the type for String? in [Int: String?])
|
||||
@discardableResult
|
||||
func ensureColumnExists<T>(_ value: T, key: String) throws -> ArrowArrayHolderBuilder {
|
||||
try throwIfInvalid()
|
||||
var builder = builders[key]
|
||||
if builder == nil {
|
||||
builder = try ArrowArrayBuilders.loadBuilder(T.self)
|
||||
builders[key] = builder
|
||||
byIndex.append(key)
|
||||
}
|
||||
|
||||
return builder!
|
||||
}
|
||||
|
||||
func getIndex(_ index: Int) -> Int {
|
||||
return self.modForIndex == nil ? index : index % self.modForIndex!
|
||||
}
|
||||
|
||||
func doEncodeNil(_ keyIndex: Int) throws {
|
||||
try throwIfInvalid()
|
||||
let index = self.getIndex(keyIndex)
|
||||
if index >= builders.count {
|
||||
throw ArrowError.outOfBounds(index: Int64(index))
|
||||
}
|
||||
|
||||
builders[byIndex[index]]!.appendAny(nil)
|
||||
}
|
||||
|
||||
func doEncode<T>(_ value: T, key: CodingKey) throws {
|
||||
try throwIfInvalid()
|
||||
let builder = try ensureColumnExists(value, key: key.stringValue)
|
||||
builder.appendAny(value)
|
||||
}
|
||||
|
||||
func doEncode<T>(_ value: T, keyIndex: Int) throws {
|
||||
try throwIfInvalid()
|
||||
let index = self.getIndex(keyIndex)
|
||||
if index >= builders.count {
|
||||
if index == builders.count {
|
||||
try ensureColumnExists(value, key: "col\(index)")
|
||||
} else {
|
||||
throw ArrowError.outOfBounds(index: Int64(index))
|
||||
}
|
||||
}
|
||||
|
||||
builders[byIndex[index]]!.appendAny(value)
|
||||
}
|
||||
|
||||
func throwIfInvalid() throws {
|
||||
if let errorMsg = self.errorMsg {
|
||||
throw ArrowError.invalid(errorMsg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private struct ArrowKeyedEncoding<Key: CodingKey>: KeyedEncodingContainerProtocol {
|
||||
var codingPath: [CodingKey] = []
|
||||
let encoder: ArrowEncoder
|
||||
init(_ encoder: ArrowEncoder) {
|
||||
self.encoder = encoder
|
||||
}
|
||||
|
||||
// If this method is called on row 0 and the encoder is
|
||||
// lazily bulding holders then this will produce an error
|
||||
// as this method does not know what the underlying type
|
||||
// is for the column. This method is not called for
|
||||
// nullable types (String?, Int32?, Date?) and the workaround
|
||||
// for this issue would be to predefine the builders for the
|
||||
// encoder. (I have only encoutered this issue when allowing
|
||||
// nullable types at the encode func level which is currently
|
||||
// not allowed)
|
||||
mutating func encodeNil(forKey key: Key) throws {
|
||||
try encoder.doEncodeNil(key: key)
|
||||
}
|
||||
|
||||
mutating func doEncodeIf<T>(_ value: T?, forKey key: Key) throws {
|
||||
if value == nil {
|
||||
try encoder.ensureColumnExists(value, key: key.stringValue)
|
||||
try encoder.doEncodeNil(key: key)
|
||||
} else {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Bool, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Bool?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: String, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: String?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Double, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Double?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Float, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Float?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Int, forKey key: Key) throws {
|
||||
throw ArrowError.invalid(
|
||||
"Int type is not supported (please use Int8, Int16, Int32 or Int64)")
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Int?, forKey key: Key) throws {
|
||||
throw ArrowError.invalid(
|
||||
"Int type is not supported (please use Int8, Int16, Int32 or Int64)")
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Int8, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Int8?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Int16, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Int16?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Int32, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Int32?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: Int64, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: Int64?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: UInt, forKey key: Key) throws {
|
||||
throw ArrowError.invalid(
|
||||
"UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: UInt?, forKey key: Key) throws {
|
||||
throw ArrowError.invalid(
|
||||
"UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
|
||||
}
|
||||
|
||||
mutating func encode(_ value: UInt8, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: UInt8?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: UInt16, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: UInt16?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: UInt32, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: UInt32?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode(_ value: UInt64, forKey key: Key) throws {
|
||||
try encoder.doEncode(value, key: key)
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent(_ value: UInt64?, forKey key: Key) throws {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
}
|
||||
|
||||
mutating func encode<T: Encodable>(_ value: T, forKey key: Key) throws {
|
||||
if ArrowArrayBuilders.isValidBuilderType(T.self) {
|
||||
try encoder.doEncode(value, key: key)
|
||||
} else {
|
||||
throw ArrowError.invalid("Type \(T.self) is currently not supported")
|
||||
}
|
||||
}
|
||||
|
||||
mutating func encodeIfPresent<T>(_ value: T?, forKey key: Self.Key) throws where T: Encodable {
|
||||
if ArrowArrayBuilders.isValidBuilderType(T?.self) {
|
||||
try doEncodeIf(value, forKey: key)
|
||||
} else {
|
||||
throw ArrowError.invalid("Type \(T.self) is currently not supported")
|
||||
}
|
||||
}
|
||||
|
||||
// nested container is currently not allowed. This method doesn't throw
|
||||
// so setting an error mesg that will be throw by the encoder at the next
|
||||
// method call that throws
|
||||
mutating func nestedContainer<NestedKey: CodingKey>(
|
||||
keyedBy keyType: NestedKey.Type,
|
||||
forKey key: Key) -> KeyedEncodingContainer<NestedKey> {
|
||||
self.encoder.errorMsg = "Nested decoding is currently not supported."
|
||||
var container = ArrowKeyedEncoding<NestedKey>(self.encoder)
|
||||
container.codingPath = codingPath
|
||||
return KeyedEncodingContainer(container)
|
||||
}
|
||||
|
||||
// nested container is currently not allowed. This method doesn't throw
|
||||
// so setting an error mesg that will be throw by the encoder at the next
|
||||
// method call that throws
|
||||
mutating func nestedUnkeyedContainer(forKey key: Key) -> UnkeyedEncodingContainer {
|
||||
self.encoder.errorMsg = "Nested decoding is currently not supported."
|
||||
return ArrowUnkeyedEncoding(self.encoder, codingPath: self.codingPath)
|
||||
}
|
||||
|
||||
// super encoding is currently not allowed. This method doesn't throw
|
||||
// so setting an error mesg that will be throw by the encoder at the next
|
||||
// method call that throws
|
||||
mutating func superEncoder() -> Encoder {
|
||||
self.encoder.errorMsg = "super encoding is currently not supported."
|
||||
return self.encoder
|
||||
}
|
||||
|
||||
// super encoding is currently not allowed. This method doesn't throw
|
||||
// so setting an error mesg that will be throw by the encoder at the next
|
||||
// method call that throws
|
||||
mutating func superEncoder(forKey key: Key) -> Encoder {
|
||||
self.encoder.errorMsg = "super encoding is currently not supported."
|
||||
return self.encoder
|
||||
}
|
||||
}
|
||||
|
||||
private struct ArrowUnkeyedEncoding: UnkeyedEncodingContainer {
|
||||
public private(set) var encoder: ArrowEncoder
|
||||
var codingPath: [CodingKey] = []
|
||||
var currentIndex: Int
|
||||
var count: Int = 0
|
||||
|
||||
init(_ encoder: ArrowEncoder, codingPath: [CodingKey], currentIndex: Int = 0) {
|
||||
self.encoder = encoder
|
||||
self.currentIndex = currentIndex
|
||||
}
|
||||
|
||||
mutating func increment() {
|
||||
self.currentIndex += 1
|
||||
}
|
||||
|
||||
// If this method is called on row 0 and the encoder is
|
||||
// lazily bulding holders then this will produce an error
|
||||
// as this method does not know what the underlying type
|
||||
// is for the column. This method is not called for
|
||||
// nullable types (String?, Int32?, Date?) and the workaround
|
||||
// for this issue would be to predefine the builders for the
|
||||
// encoder. (I have only encoutered this issue when allowing
|
||||
// nullable types at the encode func level which is currently
|
||||
// not allowed)
|
||||
mutating func encodeNil() throws {
|
||||
try encoder.doEncodeNil(self.currentIndex)
|
||||
}
|
||||
|
||||
mutating func encode<T>(_ value: T) throws where T: Encodable {
|
||||
let type = T.self
|
||||
if ArrowArrayBuilders.isValidBuilderType(type) {
|
||||
defer {increment()}
|
||||
return try self.encoder.doEncode(value, keyIndex: self.currentIndex)
|
||||
} else {
|
||||
throw ArrowError.invalid("Type \(type) is currently not supported")
|
||||
}
|
||||
}
|
||||
|
||||
// nested container is currently not allowed. This method doesn't throw
|
||||
// so setting an error mesg that will be throw by the encoder at the next
|
||||
// method call that throws
|
||||
mutating func nestedContainer<NestedKey>(keyedBy keyType: NestedKey.Type
|
||||
) -> KeyedEncodingContainer<NestedKey> where NestedKey: CodingKey {
|
||||
self.encoder.errorMsg = "Nested decoding is currently not supported."
|
||||
var container = ArrowKeyedEncoding<NestedKey>(self.encoder)
|
||||
container.codingPath = codingPath
|
||||
return KeyedEncodingContainer(container)
|
||||
}
|
||||
|
||||
// nested container is currently not allowed. This method doesn't throw
|
||||
// so setting an error mesg that will be throw by the encoder at the next
|
||||
// method call that throws
|
||||
mutating func nestedUnkeyedContainer() -> UnkeyedEncodingContainer {
|
||||
self.encoder.errorMsg = "Nested decoding is currently not supported."
|
||||
return ArrowUnkeyedEncoding(self.encoder, codingPath: self.codingPath)
|
||||
}
|
||||
|
||||
// super encoding is currently not allowed. This method doesn't throw
|
||||
// so setting an error mesg that will be throw by the encoder at the next
|
||||
// method call that throws
|
||||
mutating func superEncoder() -> Encoder {
|
||||
self.encoder.errorMsg = "super encoding is currently not supported."
|
||||
return self.encoder
|
||||
}
|
||||
}
|
||||
|
||||
private struct ArrowSingleValueEncoding: SingleValueEncodingContainer {
|
||||
public private(set) var encoder: ArrowEncoder
|
||||
var codingPath: [CodingKey] = []
|
||||
|
||||
public init(_ encoder: ArrowEncoder, codingPath: [CodingKey]) {
|
||||
self.encoder = encoder
|
||||
self.codingPath = codingPath
|
||||
}
|
||||
|
||||
mutating func encodeNil() throws {
|
||||
return try self.encoder.doEncodeNil(0)
|
||||
}
|
||||
|
||||
mutating func encode<T: Encodable>(_ value: T) throws {
|
||||
if ArrowArrayBuilders.isValidBuilderType(T.self) {
|
||||
return try self.encoder.doEncode(value, keyIndex: 0)
|
||||
} else {
|
||||
throw ArrowError.invalid("Type \(T.self) is currently not supported")
|
||||
}
|
||||
}
|
||||
}
|
||||
// swiftlint:disable:this file_length
|
||||
@@ -1,419 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import FlatBuffers
|
||||
import Foundation
|
||||
|
||||
let FILEMARKER = "ARROW1"
|
||||
let CONTINUATIONMARKER = UInt32(0xFFFFFFFF)
|
||||
|
||||
public class ArrowReader { // swiftlint:disable:this type_body_length
|
||||
private class RecordBatchData {
|
||||
let schema: org_apache_arrow_flatbuf_Schema
|
||||
let recordBatch: org_apache_arrow_flatbuf_RecordBatch
|
||||
private var fieldIndex: Int32 = 0
|
||||
private var nodeIndex: Int32 = 0
|
||||
private var bufferIndex: Int32 = 0
|
||||
init(_ recordBatch: org_apache_arrow_flatbuf_RecordBatch,
|
||||
schema: org_apache_arrow_flatbuf_Schema) {
|
||||
self.recordBatch = recordBatch
|
||||
self.schema = schema
|
||||
}
|
||||
|
||||
func nextNode() -> org_apache_arrow_flatbuf_FieldNode? {
|
||||
if nodeIndex >= self.recordBatch.nodesCount {return nil}
|
||||
defer {nodeIndex += 1}
|
||||
return self.recordBatch.nodes(at: nodeIndex)
|
||||
}
|
||||
|
||||
func nextBuffer() -> org_apache_arrow_flatbuf_Buffer? {
|
||||
if bufferIndex >= self.recordBatch.buffersCount {return nil}
|
||||
defer {bufferIndex += 1}
|
||||
return self.recordBatch.buffers(at: bufferIndex)
|
||||
}
|
||||
|
||||
func nextField() -> org_apache_arrow_flatbuf_Field? {
|
||||
if fieldIndex >= self.schema.fieldsCount {return nil}
|
||||
defer {fieldIndex += 1}
|
||||
return self.schema.fields(at: fieldIndex)
|
||||
}
|
||||
|
||||
func isDone() -> Bool {
|
||||
return nodeIndex >= self.recordBatch.nodesCount
|
||||
}
|
||||
}
|
||||
|
||||
private struct DataLoadInfo {
|
||||
let fileData: Data
|
||||
let messageOffset: Int64
|
||||
var batchData: RecordBatchData
|
||||
}
|
||||
|
||||
public class ArrowReaderResult {
|
||||
fileprivate var messageSchema: org_apache_arrow_flatbuf_Schema?
|
||||
public var schema: ArrowSchema?
|
||||
public var batches = [RecordBatch]()
|
||||
}
|
||||
|
||||
public init() {}
|
||||
|
||||
private func loadSchema(_ schema: org_apache_arrow_flatbuf_Schema) -> Result<ArrowSchema, ArrowError> {
|
||||
let builder = ArrowSchema.Builder()
|
||||
for index in 0 ..< schema.fieldsCount {
|
||||
let field = schema.fields(at: index)!
|
||||
let fieldType = findArrowType(field)
|
||||
if fieldType.info == ArrowType.ArrowUnknown {
|
||||
return .failure(.unknownType("Unsupported field type found: \(field.typeType)"))
|
||||
}
|
||||
let arrowField = ArrowField(field.name!, type: fieldType, isNullable: field.nullable)
|
||||
builder.addField(arrowField)
|
||||
}
|
||||
|
||||
return .success(builder.finish())
|
||||
}
|
||||
|
||||
private func loadStructData(_ loadInfo: DataLoadInfo,
|
||||
field: org_apache_arrow_flatbuf_Field)
|
||||
-> Result<ArrowArrayHolder, ArrowError> {
|
||||
guard let node = loadInfo.batchData.nextNode() else {
|
||||
return .failure(.invalid("Node not found"))
|
||||
}
|
||||
|
||||
guard let nullBuffer = loadInfo.batchData.nextBuffer() else {
|
||||
return .failure(.invalid("Null buffer not found"))
|
||||
}
|
||||
|
||||
let nullLength = UInt(ceil(Double(node.length) / 8))
|
||||
let arrowNullBuffer = makeBuffer(nullBuffer, fileData: loadInfo.fileData,
|
||||
length: nullLength, messageOffset: loadInfo.messageOffset)
|
||||
var children = [ArrowData]()
|
||||
for index in 0..<field.childrenCount {
|
||||
let childField = field.children(at: index)!
|
||||
switch loadField(loadInfo, field: childField) {
|
||||
case .success(let holder):
|
||||
children.append(holder.array.arrowData)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
return makeArrayHolder(field, buffers: [arrowNullBuffer],
|
||||
nullCount: UInt(node.nullCount), children: children,
|
||||
rbLength: UInt(loadInfo.batchData.recordBatch.length))
|
||||
}
|
||||
|
||||
private func loadPrimitiveData(
|
||||
_ loadInfo: DataLoadInfo,
|
||||
field: org_apache_arrow_flatbuf_Field)
|
||||
-> Result<ArrowArrayHolder, ArrowError> {
|
||||
guard let node = loadInfo.batchData.nextNode() else {
|
||||
return .failure(.invalid("Node not found"))
|
||||
}
|
||||
|
||||
guard let nullBuffer = loadInfo.batchData.nextBuffer() else {
|
||||
return .failure(.invalid("Null buffer not found"))
|
||||
}
|
||||
|
||||
guard let valueBuffer = loadInfo.batchData.nextBuffer() else {
|
||||
return .failure(.invalid("Value buffer not found"))
|
||||
}
|
||||
|
||||
let nullLength = UInt(ceil(Double(node.length) / 8))
|
||||
let arrowNullBuffer = makeBuffer(nullBuffer, fileData: loadInfo.fileData,
|
||||
length: nullLength, messageOffset: loadInfo.messageOffset)
|
||||
let arrowValueBuffer = makeBuffer(valueBuffer, fileData: loadInfo.fileData,
|
||||
length: UInt(node.length), messageOffset: loadInfo.messageOffset)
|
||||
return makeArrayHolder(field, buffers: [arrowNullBuffer, arrowValueBuffer],
|
||||
nullCount: UInt(node.nullCount), children: nil,
|
||||
rbLength: UInt(loadInfo.batchData.recordBatch.length))
|
||||
}
|
||||
|
||||
private func loadVariableData(
|
||||
_ loadInfo: DataLoadInfo,
|
||||
field: org_apache_arrow_flatbuf_Field)
|
||||
-> Result<ArrowArrayHolder, ArrowError> {
|
||||
guard let node = loadInfo.batchData.nextNode() else {
|
||||
return .failure(.invalid("Node not found"))
|
||||
}
|
||||
|
||||
guard let nullBuffer = loadInfo.batchData.nextBuffer() else {
|
||||
return .failure(.invalid("Null buffer not found"))
|
||||
}
|
||||
|
||||
guard let offsetBuffer = loadInfo.batchData.nextBuffer() else {
|
||||
return .failure(.invalid("Offset buffer not found"))
|
||||
}
|
||||
|
||||
guard let valueBuffer = loadInfo.batchData.nextBuffer() else {
|
||||
return .failure(.invalid("Value buffer not found"))
|
||||
}
|
||||
|
||||
let nullLength = UInt(ceil(Double(node.length) / 8))
|
||||
let arrowNullBuffer = makeBuffer(nullBuffer, fileData: loadInfo.fileData,
|
||||
length: nullLength, messageOffset: loadInfo.messageOffset)
|
||||
let arrowOffsetBuffer = makeBuffer(offsetBuffer, fileData: loadInfo.fileData,
|
||||
length: UInt(node.length), messageOffset: loadInfo.messageOffset)
|
||||
let arrowValueBuffer = makeBuffer(valueBuffer, fileData: loadInfo.fileData,
|
||||
length: UInt(node.length), messageOffset: loadInfo.messageOffset)
|
||||
return makeArrayHolder(field, buffers: [arrowNullBuffer, arrowOffsetBuffer, arrowValueBuffer],
|
||||
nullCount: UInt(node.nullCount), children: nil,
|
||||
rbLength: UInt(loadInfo.batchData.recordBatch.length))
|
||||
}
|
||||
|
||||
private func loadField(
|
||||
_ loadInfo: DataLoadInfo,
|
||||
field: org_apache_arrow_flatbuf_Field)
|
||||
-> Result<ArrowArrayHolder, ArrowError> {
|
||||
if isNestedType(field.typeType) {
|
||||
return loadStructData(loadInfo, field: field)
|
||||
} else if isFixedPrimitive(field.typeType) {
|
||||
return loadPrimitiveData(loadInfo, field: field)
|
||||
} else {
|
||||
return loadVariableData(loadInfo, field: field)
|
||||
}
|
||||
}
|
||||
|
||||
private func loadRecordBatch(
|
||||
_ recordBatch: org_apache_arrow_flatbuf_RecordBatch,
|
||||
schema: org_apache_arrow_flatbuf_Schema,
|
||||
arrowSchema: ArrowSchema,
|
||||
data: Data,
|
||||
messageEndOffset: Int64
|
||||
) -> Result<RecordBatch, ArrowError> {
|
||||
var columns: [ArrowArrayHolder] = []
|
||||
let batchData = RecordBatchData(recordBatch, schema: schema)
|
||||
let loadInfo = DataLoadInfo(fileData: data,
|
||||
messageOffset: messageEndOffset,
|
||||
batchData: batchData)
|
||||
while !batchData.isDone() {
|
||||
guard let field = batchData.nextField() else {
|
||||
return .failure(.invalid("Field not found"))
|
||||
}
|
||||
|
||||
let result = loadField(loadInfo, field: field)
|
||||
switch result {
|
||||
case .success(let holder):
|
||||
columns.append(holder)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
return .success(RecordBatch(arrowSchema, columns: columns))
|
||||
}
|
||||
|
||||
/*
|
||||
This is for reading the Arrow streaming format. The Arrow streaming format
|
||||
is slightly different from the Arrow File format as it doesn't contain a header
|
||||
and footer.
|
||||
*/
|
||||
public func readStreaming( // swiftlint:disable:this function_body_length
|
||||
_ input: Data,
|
||||
useUnalignedBuffers: Bool = false
|
||||
) -> Result<ArrowReaderResult, ArrowError> {
|
||||
let result = ArrowReaderResult()
|
||||
var offset: Int = 0
|
||||
var length = getUInt32(input, offset: offset)
|
||||
var streamData = input
|
||||
var schemaMessage: org_apache_arrow_flatbuf_Schema?
|
||||
while length != 0 {
|
||||
if length == CONTINUATIONMARKER {
|
||||
offset += Int(MemoryLayout<UInt32>.size)
|
||||
length = getUInt32(input, offset: offset)
|
||||
if length == 0 {
|
||||
return .success(result)
|
||||
}
|
||||
}
|
||||
|
||||
offset += Int(MemoryLayout<UInt32>.size)
|
||||
streamData = input[offset...]
|
||||
let dataBuffer = ByteBuffer(
|
||||
data: streamData,
|
||||
allowReadingUnalignedBuffers: true)
|
||||
let message = org_apache_arrow_flatbuf_Message.getRootAsMessage(bb: dataBuffer)
|
||||
switch message.headerType {
|
||||
case .recordbatch:
|
||||
do {
|
||||
let rbMessage = message.header(type: org_apache_arrow_flatbuf_RecordBatch.self)!
|
||||
let recordBatch = try loadRecordBatch(
|
||||
rbMessage,
|
||||
schema: schemaMessage!,
|
||||
arrowSchema: result.schema!,
|
||||
data: input,
|
||||
messageEndOffset: (Int64(offset) + Int64(length))).get()
|
||||
result.batches.append(recordBatch)
|
||||
offset += Int(message.bodyLength + Int64(length))
|
||||
length = getUInt32(input, offset: offset)
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("Unexpected error: \(error)"))
|
||||
}
|
||||
case .schema:
|
||||
schemaMessage = message.header(type: org_apache_arrow_flatbuf_Schema.self)!
|
||||
let schemaResult = loadSchema(schemaMessage!)
|
||||
switch schemaResult {
|
||||
case .success(let schema):
|
||||
result.schema = schema
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
offset += Int(message.bodyLength + Int64(length))
|
||||
length = getUInt32(input, offset: offset)
|
||||
default:
|
||||
return .failure(.unknownError("Unhandled header type: \(message.headerType)"))
|
||||
}
|
||||
}
|
||||
return .success(result)
|
||||
}
|
||||
|
||||
/*
|
||||
This is for reading the Arrow file format. The Arrow file format supports
|
||||
random accessing the data. The Arrow file format contains a header and
|
||||
footer around the Arrow streaming format.
|
||||
*/
|
||||
public func readFile( // swiftlint:disable:this function_body_length
|
||||
_ fileData: Data,
|
||||
useUnalignedBuffers: Bool = false
|
||||
) -> Result<ArrowReaderResult, ArrowError> {
|
||||
let footerLength = fileData.withUnsafeBytes { rawBuffer in
|
||||
rawBuffer.loadUnaligned(fromByteOffset: fileData.count - 4, as: Int32.self)
|
||||
}
|
||||
|
||||
let result = ArrowReaderResult()
|
||||
let footerStartOffset = fileData.count - Int(footerLength + 4)
|
||||
let footerData = fileData[footerStartOffset...]
|
||||
let footerBuffer = ByteBuffer(
|
||||
data: footerData,
|
||||
allowReadingUnalignedBuffers: useUnalignedBuffers)
|
||||
let footer = org_apache_arrow_flatbuf_Footer.getRootAsFooter(bb: footerBuffer)
|
||||
let schemaResult = loadSchema(footer.schema!)
|
||||
switch schemaResult {
|
||||
case .success(let schema):
|
||||
result.schema = schema
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
for index in 0 ..< footer.recordBatchesCount {
|
||||
let recordBatch = footer.recordBatches(at: index)!
|
||||
var messageLength = fileData.withUnsafeBytes { rawBuffer in
|
||||
rawBuffer.loadUnaligned(fromByteOffset: Int(recordBatch.offset), as: UInt32.self)
|
||||
}
|
||||
|
||||
var messageOffset: Int64 = 1
|
||||
if messageLength == CONTINUATIONMARKER {
|
||||
messageOffset += 1
|
||||
messageLength = fileData.withUnsafeBytes { rawBuffer in
|
||||
rawBuffer.loadUnaligned(
|
||||
fromByteOffset: Int(recordBatch.offset + Int64(MemoryLayout<Int32>.size)),
|
||||
as: UInt32.self)
|
||||
}
|
||||
}
|
||||
|
||||
let messageStartOffset = recordBatch.offset + (Int64(MemoryLayout<Int32>.size) * messageOffset)
|
||||
let messageEndOffset = messageStartOffset + Int64(messageLength)
|
||||
let recordBatchData = fileData[messageStartOffset ..< messageEndOffset]
|
||||
let mbb = ByteBuffer(
|
||||
data: recordBatchData,
|
||||
allowReadingUnalignedBuffers: useUnalignedBuffers)
|
||||
let message = org_apache_arrow_flatbuf_Message.getRootAsMessage(bb: mbb)
|
||||
switch message.headerType {
|
||||
case .recordbatch:
|
||||
do {
|
||||
let rbMessage = message.header(type: org_apache_arrow_flatbuf_RecordBatch.self)!
|
||||
let recordBatch = try loadRecordBatch(
|
||||
rbMessage,
|
||||
schema: footer.schema!,
|
||||
arrowSchema: result.schema!,
|
||||
data: fileData,
|
||||
messageEndOffset: messageEndOffset).get()
|
||||
result.batches.append(recordBatch)
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("Unexpected error: \(error)"))
|
||||
}
|
||||
default:
|
||||
return .failure(.unknownError("Unhandled header type: \(message.headerType)"))
|
||||
}
|
||||
}
|
||||
|
||||
return .success(result)
|
||||
}
|
||||
|
||||
public func fromFile(_ fileURL: URL) -> Result<ArrowReaderResult, ArrowError> {
|
||||
do {
|
||||
let fileData = try Data(contentsOf: fileURL)
|
||||
if !validateFileData(fileData) {
|
||||
return .failure(.ioError("Not a valid arrow file."))
|
||||
}
|
||||
|
||||
let markerLength = FILEMARKER.utf8.count
|
||||
let footerLengthEnd = Int(fileData.count - markerLength)
|
||||
let data = fileData[..<(footerLengthEnd)]
|
||||
return readFile(data)
|
||||
} catch {
|
||||
return .failure(.unknownError("Error loading file: \(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
static public func makeArrowReaderResult() -> ArrowReaderResult {
|
||||
return ArrowReaderResult()
|
||||
}
|
||||
|
||||
public func fromMessage(
|
||||
_ dataHeader: Data,
|
||||
dataBody: Data,
|
||||
result: ArrowReaderResult,
|
||||
useUnalignedBuffers: Bool = false
|
||||
) -> Result<Void, ArrowError> {
|
||||
let mbb = ByteBuffer(
|
||||
data: dataHeader,
|
||||
allowReadingUnalignedBuffers: useUnalignedBuffers)
|
||||
let message = org_apache_arrow_flatbuf_Message.getRootAsMessage(bb: mbb)
|
||||
switch message.headerType {
|
||||
case .schema:
|
||||
let sMessage = message.header(type: org_apache_arrow_flatbuf_Schema.self)!
|
||||
switch loadSchema(sMessage) {
|
||||
case .success(let schema):
|
||||
result.schema = schema
|
||||
result.messageSchema = sMessage
|
||||
return .success(())
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
case .recordbatch:
|
||||
let rbMessage = message.header(type: org_apache_arrow_flatbuf_RecordBatch.self)!
|
||||
do {
|
||||
let recordBatch = try loadRecordBatch(
|
||||
rbMessage, schema: result.messageSchema!, arrowSchema: result.schema!,
|
||||
data: dataBody, messageEndOffset: 0).get()
|
||||
result.batches.append(recordBatch)
|
||||
return .success(())
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("Unexpected error: \(error)"))
|
||||
}
|
||||
default:
|
||||
return .failure(.unknownError("Unhandled header type: \(message.headerType)"))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
// swiftlint:disable:this file_length
|
||||
@@ -1,298 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import FlatBuffers
|
||||
import Foundation
|
||||
|
||||
private func makeBinaryHolder(_ buffers: [ArrowBuffer],
|
||||
nullCount: UInt) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
do {
|
||||
let arrowType = ArrowType(ArrowType.ArrowBinary)
|
||||
let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try BinaryArray(arrowData)))
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
private func makeStringHolder(_ buffers: [ArrowBuffer],
|
||||
nullCount: UInt) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
do {
|
||||
let arrowType = ArrowType(ArrowType.ArrowString)
|
||||
let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try StringArray(arrowData)))
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
private func makeDateHolder(_ field: ArrowField,
|
||||
buffers: [ArrowBuffer],
|
||||
nullCount: UInt
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
do {
|
||||
if field.type.id == .date32 {
|
||||
let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try Date32Array(arrowData)))
|
||||
}
|
||||
|
||||
let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try Date64Array(arrowData)))
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
private func makeTimeHolder(_ field: ArrowField,
|
||||
buffers: [ArrowBuffer],
|
||||
nullCount: UInt
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
do {
|
||||
if field.type.id == .time32 {
|
||||
if let arrowType = field.type as? ArrowTypeTime32 {
|
||||
let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try FixedArray<Time32>(arrowData)))
|
||||
} else {
|
||||
return .failure(.invalid("Incorrect field type for time: \(field.type)"))
|
||||
}
|
||||
}
|
||||
|
||||
if let arrowType = field.type as? ArrowTypeTime64 {
|
||||
let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try FixedArray<Time64>(arrowData)))
|
||||
} else {
|
||||
return .failure(.invalid("Incorrect field type for time: \(field.type)"))
|
||||
}
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
private func makeBoolHolder(_ buffers: [ArrowBuffer],
|
||||
nullCount: UInt) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
do {
|
||||
let arrowType = ArrowType(ArrowType.ArrowBool)
|
||||
let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try BoolArray(arrowData)))
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
private func makeFixedHolder<T>(
|
||||
_: T.Type, field: ArrowField, buffers: [ArrowBuffer],
|
||||
nullCount: UInt
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
do {
|
||||
let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount)
|
||||
return .success(ArrowArrayHolderImpl(try FixedArray<T>(arrowData)))
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
func makeStructHolder(
|
||||
_ field: ArrowField,
|
||||
buffers: [ArrowBuffer],
|
||||
nullCount: UInt,
|
||||
children: [ArrowData],
|
||||
rbLength: UInt
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
do {
|
||||
let arrowData = try ArrowData(field.type,
|
||||
buffers: buffers, children: children,
|
||||
nullCount: nullCount, length: rbLength)
|
||||
return .success(ArrowArrayHolderImpl(try StructArray(arrowData)))
|
||||
} catch let error as ArrowError {
|
||||
return .failure(error)
|
||||
} catch {
|
||||
return .failure(.unknownError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
func makeArrayHolder(
|
||||
_ field: org_apache_arrow_flatbuf_Field,
|
||||
buffers: [ArrowBuffer],
|
||||
nullCount: UInt,
|
||||
children: [ArrowData]?,
|
||||
rbLength: UInt
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
let arrowField = fromProto(field: field)
|
||||
return makeArrayHolder(arrowField, buffers: buffers, nullCount: nullCount, children: children, rbLength: rbLength)
|
||||
}
|
||||
|
||||
func makeArrayHolder( // swiftlint:disable:this cyclomatic_complexity
|
||||
_ field: ArrowField,
|
||||
buffers: [ArrowBuffer],
|
||||
nullCount: UInt,
|
||||
children: [ArrowData]?,
|
||||
rbLength: UInt
|
||||
) -> Result<ArrowArrayHolder, ArrowError> {
|
||||
let typeId = field.type.id
|
||||
switch typeId {
|
||||
case .int8:
|
||||
return makeFixedHolder(Int8.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .uint8:
|
||||
return makeFixedHolder(UInt8.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .int16:
|
||||
return makeFixedHolder(Int16.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .uint16:
|
||||
return makeFixedHolder(UInt16.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .int32:
|
||||
return makeFixedHolder(Int32.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .uint32:
|
||||
return makeFixedHolder(UInt32.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .int64:
|
||||
return makeFixedHolder(Int64.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .uint64:
|
||||
return makeFixedHolder(UInt64.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .boolean:
|
||||
return makeBoolHolder(buffers, nullCount: nullCount)
|
||||
case .float:
|
||||
return makeFixedHolder(Float.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .double:
|
||||
return makeFixedHolder(Double.self, field: field, buffers: buffers, nullCount: nullCount)
|
||||
case .string:
|
||||
return makeStringHolder(buffers, nullCount: nullCount)
|
||||
case .binary:
|
||||
return makeBinaryHolder(buffers, nullCount: nullCount)
|
||||
case .date32, .date64:
|
||||
return makeDateHolder(field, buffers: buffers, nullCount: nullCount)
|
||||
case .time32, .time64:
|
||||
return makeTimeHolder(field, buffers: buffers, nullCount: nullCount)
|
||||
case .strct:
|
||||
return makeStructHolder(field, buffers: buffers, nullCount: nullCount, children: children!, rbLength: rbLength)
|
||||
default:
|
||||
return .failure(.unknownType("Type \(typeId) currently not supported"))
|
||||
}
|
||||
}
|
||||
|
||||
func makeBuffer(_ buffer: org_apache_arrow_flatbuf_Buffer, fileData: Data,
|
||||
length: UInt, messageOffset: Int64) -> ArrowBuffer {
|
||||
let startOffset = messageOffset + buffer.offset
|
||||
let endOffset = startOffset + buffer.length
|
||||
let bufferData = [UInt8](fileData[startOffset ..< endOffset])
|
||||
return ArrowBuffer.createBuffer(bufferData, length: length)
|
||||
}
|
||||
|
||||
func isFixedPrimitive(_ type: org_apache_arrow_flatbuf_Type_) -> Bool {
|
||||
switch type {
|
||||
case .int, .bool, .floatingpoint, .date, .time:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isNestedType(_ type: org_apache_arrow_flatbuf_Type_) -> Bool {
|
||||
switch type {
|
||||
case .struct_:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func findArrowType( // swiftlint:disable:this cyclomatic_complexity function_body_length
|
||||
_ field: org_apache_arrow_flatbuf_Field) -> ArrowType {
|
||||
let type = field.typeType
|
||||
switch type {
|
||||
case .int:
|
||||
let intType = field.type(type: org_apache_arrow_flatbuf_Int.self)!
|
||||
let bitWidth = intType.bitWidth
|
||||
if bitWidth == 8 { return ArrowType(intType.isSigned ? ArrowType.ArrowInt8 : ArrowType.ArrowUInt8) }
|
||||
if bitWidth == 16 { return ArrowType(intType.isSigned ? ArrowType.ArrowInt16 : ArrowType.ArrowUInt16) }
|
||||
if bitWidth == 32 { return ArrowType(intType.isSigned ? ArrowType.ArrowInt32 : ArrowType.ArrowUInt32) }
|
||||
if bitWidth == 64 { return ArrowType(intType.isSigned ? ArrowType.ArrowInt64 : ArrowType.ArrowUInt64) }
|
||||
return ArrowType(ArrowType.ArrowUnknown)
|
||||
case .bool:
|
||||
return ArrowType(ArrowType.ArrowBool)
|
||||
case .floatingpoint:
|
||||
let floatType = field.type(type: org_apache_arrow_flatbuf_FloatingPoint.self)!
|
||||
switch floatType.precision {
|
||||
case .single:
|
||||
return ArrowType(ArrowType.ArrowFloat)
|
||||
case .double:
|
||||
return ArrowType(ArrowType.ArrowDouble)
|
||||
default:
|
||||
return ArrowType(ArrowType.ArrowUnknown)
|
||||
}
|
||||
case .utf8:
|
||||
return ArrowType(ArrowType.ArrowString)
|
||||
case .binary:
|
||||
return ArrowType(ArrowType.ArrowBinary)
|
||||
case .date:
|
||||
let dateType = field.type(type: org_apache_arrow_flatbuf_Date.self)!
|
||||
if dateType.unit == .day {
|
||||
return ArrowType(ArrowType.ArrowDate32)
|
||||
}
|
||||
|
||||
return ArrowType(ArrowType.ArrowDate64)
|
||||
case .time:
|
||||
let timeType = field.type(type: org_apache_arrow_flatbuf_Time.self)!
|
||||
if timeType.unit == .second || timeType.unit == .millisecond {
|
||||
return ArrowTypeTime32(timeType.unit == .second ? .seconds : .milliseconds)
|
||||
}
|
||||
|
||||
return ArrowTypeTime64(timeType.unit == .microsecond ? .microseconds : .nanoseconds)
|
||||
case .struct_:
|
||||
_ = field.type(type: org_apache_arrow_flatbuf_Struct_.self)!
|
||||
var fields = [ArrowField]()
|
||||
for index in 0..<field.childrenCount {
|
||||
let childField = field.children(at: index)!
|
||||
let childType = findArrowType(childField)
|
||||
fields.append(
|
||||
ArrowField(childField.name ?? "", type: childType, isNullable: childField.nullable))
|
||||
}
|
||||
|
||||
return ArrowNestedType(ArrowType.ArrowStruct, fields: fields)
|
||||
default:
|
||||
return ArrowType(ArrowType.ArrowUnknown)
|
||||
}
|
||||
}
|
||||
|
||||
func validateBufferIndex(_ recordBatch: org_apache_arrow_flatbuf_RecordBatch, index: Int32) throws {
|
||||
if index >= recordBatch.buffersCount {
|
||||
throw ArrowError.outOfBounds(index: Int64(index))
|
||||
}
|
||||
}
|
||||
|
||||
func validateFileData(_ data: Data) -> Bool {
|
||||
let markerLength = FILEMARKER.utf8.count
|
||||
let startString = String(decoding: data[..<markerLength], as: UTF8.self)
|
||||
let endString = String(decoding: data[(data.count - markerLength)...], as: UTF8.self)
|
||||
return startString == FILEMARKER && endString == FILEMARKER
|
||||
}
|
||||
|
||||
func getUInt32(_ data: Data, offset: Int) -> UInt32 {
|
||||
let token = data.withUnsafeBytes { rawBuffer in
|
||||
rawBuffer.loadUnaligned(fromByteOffset: offset, as: UInt32.self)
|
||||
}
|
||||
return token
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
public class ArrowField {
|
||||
public let type: ArrowType
|
||||
public let name: String
|
||||
public let isNullable: Bool
|
||||
|
||||
init(_ name: String, type: ArrowType, isNullable: Bool) {
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.isNullable = isNullable
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowSchema {
|
||||
public let fields: [ArrowField]
|
||||
public let fieldLookup: [String: Int]
|
||||
init(_ fields: [ArrowField]) {
|
||||
var fieldLookup = [String: Int]()
|
||||
for (index, field) in fields.enumerated() {
|
||||
fieldLookup[field.name] = index
|
||||
}
|
||||
|
||||
self.fields = fields
|
||||
self.fieldLookup = fieldLookup
|
||||
}
|
||||
|
||||
public func field(_ index: Int) -> ArrowField {
|
||||
return self.fields[index]
|
||||
}
|
||||
|
||||
public func fieldIndex(_ name: String) -> Int? {
|
||||
return self.fieldLookup[name]
|
||||
}
|
||||
|
||||
public class Builder {
|
||||
private var fields: [ArrowField] = []
|
||||
|
||||
public init() {}
|
||||
|
||||
@discardableResult
|
||||
public func addField(_ field: ArrowField) -> Builder {
|
||||
fields.append(field)
|
||||
return self
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
public func addField(_ name: String, type: ArrowType, isNullable: Bool) -> Builder {
|
||||
fields.append(ArrowField(name, type: type, isNullable: isNullable))
|
||||
return self
|
||||
}
|
||||
|
||||
public func finish() -> ArrowSchema {
|
||||
return ArrowSchema(fields)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,202 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class ArrowColumn {
|
||||
public let field: ArrowField
|
||||
fileprivate let dataHolder: ChunkedArrayHolder
|
||||
public var type: ArrowType {return self.dataHolder.type}
|
||||
public var length: UInt {return self.dataHolder.length}
|
||||
public var nullCount: UInt {return self.dataHolder.nullCount}
|
||||
|
||||
public func data<T>() -> ChunkedArray<T> {
|
||||
return (self.dataHolder.holder as! ChunkedArray<T>) // swiftlint:disable:this force_cast
|
||||
}
|
||||
|
||||
public var name: String {return field.name}
|
||||
public init(_ field: ArrowField, chunked: ChunkedArrayHolder) {
|
||||
self.field = field
|
||||
self.dataHolder = chunked
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowTable {
|
||||
public let schema: ArrowSchema
|
||||
public var columnCount: UInt {return UInt(self.columns.count)}
|
||||
public let rowCount: UInt
|
||||
public let columns: [ArrowColumn]
|
||||
init(_ schema: ArrowSchema, columns: [ArrowColumn]) {
|
||||
self.schema = schema
|
||||
self.columns = columns
|
||||
self.rowCount = columns[0].length
|
||||
}
|
||||
|
||||
public static func from(recordBatches: [RecordBatch]) -> Result<ArrowTable, ArrowError> {
|
||||
if recordBatches.isEmpty {
|
||||
return .failure(.arrayHasNoElements)
|
||||
}
|
||||
|
||||
var holders = [[ArrowArrayHolder]]()
|
||||
let schema = recordBatches[0].schema
|
||||
for recordBatch in recordBatches {
|
||||
for index in 0..<schema.fields.count {
|
||||
if holders.count <= index {
|
||||
holders.append([ArrowArrayHolder]())
|
||||
}
|
||||
holders[index].append(recordBatch.columns[index])
|
||||
}
|
||||
}
|
||||
|
||||
let builder = ArrowTable.Builder()
|
||||
for index in 0..<schema.fields.count {
|
||||
switch makeArrowColumn(schema.fields[index], holders: holders[index]) {
|
||||
case .success(let column):
|
||||
builder.addColumn(column)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
return .success(builder.finish())
|
||||
}
|
||||
|
||||
private static func makeArrowColumn(
|
||||
_ field: ArrowField,
|
||||
holders: [ArrowArrayHolder]
|
||||
) -> Result<ArrowColumn, ArrowError> {
|
||||
do {
|
||||
return .success(try holders[0].getArrowColumn(field, holders))
|
||||
} catch {
|
||||
return .failure(.runtimeError("\(error)"))
|
||||
}
|
||||
}
|
||||
|
||||
public class Builder {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
var columns = [ArrowColumn]()
|
||||
|
||||
public init() {}
|
||||
|
||||
@discardableResult
|
||||
public func addColumn<T>(_ fieldName: String, arrowArray: ArrowArray<T>) throws -> Builder {
|
||||
return self.addColumn(fieldName, chunked: try ChunkedArray([arrowArray]))
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
public func addColumn<T>(_ fieldName: String, chunked: ChunkedArray<T>) -> Builder {
|
||||
let field = ArrowField(fieldName, type: chunked.type, isNullable: chunked.nullCount != 0)
|
||||
self.schemaBuilder.addField(field)
|
||||
self.columns.append(ArrowColumn(field, chunked: ChunkedArrayHolder(chunked)))
|
||||
return self
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
public func addColumn<T>(_ field: ArrowField, arrowArray: ArrowArray<T>) throws -> Builder {
|
||||
self.schemaBuilder.addField(field)
|
||||
let holder = ChunkedArrayHolder(try ChunkedArray([arrowArray]))
|
||||
self.columns.append(ArrowColumn(field, chunked: holder))
|
||||
return self
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
public func addColumn<T>(_ field: ArrowField, chunked: ChunkedArray<T>) -> Builder {
|
||||
self.schemaBuilder.addField(field)
|
||||
self.columns.append(ArrowColumn(field, chunked: ChunkedArrayHolder(chunked)))
|
||||
return self
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
public func addColumn(_ column: ArrowColumn) -> Builder {
|
||||
self.schemaBuilder.addField(column.field)
|
||||
self.columns.append(column)
|
||||
return self
|
||||
}
|
||||
|
||||
public func finish() -> ArrowTable {
|
||||
return ArrowTable(self.schemaBuilder.finish(), columns: self.columns)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class RecordBatch {
|
||||
public let schema: ArrowSchema
|
||||
public var columnCount: UInt {return UInt(self.columns.count)}
|
||||
public let columns: [ArrowArrayHolder]
|
||||
public let length: UInt
|
||||
public init(_ schema: ArrowSchema, columns: [ArrowArrayHolder]) {
|
||||
self.schema = schema
|
||||
self.columns = columns
|
||||
self.length = columns[0].length
|
||||
}
|
||||
|
||||
public class Builder {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
var columns = [ArrowArrayHolder]()
|
||||
|
||||
public init() {}
|
||||
|
||||
@discardableResult
|
||||
public func addColumn(_ fieldName: String, arrowArray: ArrowArrayHolder) -> Builder {
|
||||
let field = ArrowField(fieldName, type: arrowArray.type, isNullable: arrowArray.nullCount != 0)
|
||||
self.schemaBuilder.addField(field)
|
||||
self.columns.append(arrowArray)
|
||||
return self
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
public func addColumn(_ field: ArrowField, arrowArray: ArrowArrayHolder) -> Builder {
|
||||
self.schemaBuilder.addField(field)
|
||||
self.columns.append(arrowArray)
|
||||
return self
|
||||
}
|
||||
|
||||
public func finish() -> Result<RecordBatch, ArrowError> {
|
||||
if columns.count > 0 {
|
||||
let columnLength = columns[0].length
|
||||
for column in columns {
|
||||
if column.length != columnLength { // swiftlint:disable:this for_where
|
||||
return .failure(.runtimeError("Columns have different sizes"))
|
||||
}
|
||||
}
|
||||
}
|
||||
return .success(RecordBatch(self.schemaBuilder.finish(), columns: self.columns))
|
||||
}
|
||||
}
|
||||
|
||||
public func data<T>(for columnIndex: Int) -> ArrowArray<T> {
|
||||
let arrayHolder = column(columnIndex)
|
||||
return (arrayHolder.array as! ArrowArray<T>) // swiftlint:disable:this force_cast
|
||||
}
|
||||
|
||||
public func anyData(for columnIndex: Int) -> AnyArray {
|
||||
let arrayHolder = column(columnIndex)
|
||||
return arrayHolder.array
|
||||
}
|
||||
|
||||
public func column(_ index: Int) -> ArrowArrayHolder {
|
||||
return self.columns[index]
|
||||
}
|
||||
|
||||
public func column(_ name: String) -> ArrowArrayHolder? {
|
||||
if let index = self.schema.fieldIndex(name) {
|
||||
return self.columns[index]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,405 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public typealias Time32 = Int32
|
||||
public typealias Time64 = Int64
|
||||
public typealias Date32 = Int32
|
||||
public typealias Date64 = Int64
|
||||
|
||||
func FlatBuffersVersion_23_1_4() { // swiftlint:disable:this identifier_name
|
||||
}
|
||||
|
||||
public enum ArrowError: Error {
|
||||
case none
|
||||
case unknownType(String)
|
||||
case runtimeError(String)
|
||||
case outOfBounds(index: Int64)
|
||||
case arrayHasNoElements
|
||||
case unknownError(String)
|
||||
case notImplemented
|
||||
case ioError(String)
|
||||
case invalid(String)
|
||||
}
|
||||
|
||||
public enum ArrowTypeId {
|
||||
case binary
|
||||
case boolean
|
||||
case date32
|
||||
case date64
|
||||
case dateType
|
||||
case decimal128
|
||||
case decimal256
|
||||
case dictionary
|
||||
case double
|
||||
case fixedSizeBinary
|
||||
case fixedWidthType
|
||||
case float
|
||||
// case HalfFloatType
|
||||
case int16
|
||||
case int32
|
||||
case int64
|
||||
case int8
|
||||
case integer
|
||||
case intervalUnit
|
||||
case list
|
||||
case nested
|
||||
case null
|
||||
case number
|
||||
case string
|
||||
case strct
|
||||
case time32
|
||||
case time64
|
||||
case time
|
||||
case uint16
|
||||
case uint32
|
||||
case uint64
|
||||
case uint8
|
||||
case union
|
||||
case unknown
|
||||
}
|
||||
|
||||
public enum ArrowTime32Unit {
|
||||
case seconds
|
||||
case milliseconds
|
||||
}
|
||||
|
||||
public enum ArrowTime64Unit {
|
||||
case microseconds
|
||||
case nanoseconds
|
||||
}
|
||||
|
||||
public class ArrowTypeTime32: ArrowType {
|
||||
let unit: ArrowTime32Unit
|
||||
public init(_ unit: ArrowTime32Unit) {
|
||||
self.unit = unit
|
||||
super.init(ArrowType.ArrowTime32)
|
||||
}
|
||||
|
||||
public override var cDataFormatId: String {
|
||||
get throws {
|
||||
switch self.unit {
|
||||
case .milliseconds:
|
||||
return "ttm"
|
||||
case .seconds:
|
||||
return "tts"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowTypeTime64: ArrowType {
|
||||
let unit: ArrowTime64Unit
|
||||
public init(_ unit: ArrowTime64Unit) {
|
||||
self.unit = unit
|
||||
super.init(ArrowType.ArrowTime64)
|
||||
}
|
||||
|
||||
public override var cDataFormatId: String {
|
||||
get throws {
|
||||
switch self.unit {
|
||||
case .microseconds:
|
||||
return "ttu"
|
||||
case .nanoseconds:
|
||||
return "ttn"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowNestedType: ArrowType {
|
||||
let fields: [ArrowField]
|
||||
public init(_ info: ArrowType.Info, fields: [ArrowField]) {
|
||||
self.fields = fields
|
||||
super.init(info)
|
||||
}
|
||||
}
|
||||
|
||||
public class ArrowType {
|
||||
public private(set) var info: ArrowType.Info
|
||||
public static let ArrowInt8 = Info.primitiveInfo(ArrowTypeId.int8)
|
||||
public static let ArrowInt16 = Info.primitiveInfo(ArrowTypeId.int16)
|
||||
public static let ArrowInt32 = Info.primitiveInfo(ArrowTypeId.int32)
|
||||
public static let ArrowInt64 = Info.primitiveInfo(ArrowTypeId.int64)
|
||||
public static let ArrowUInt8 = Info.primitiveInfo(ArrowTypeId.uint8)
|
||||
public static let ArrowUInt16 = Info.primitiveInfo(ArrowTypeId.uint16)
|
||||
public static let ArrowUInt32 = Info.primitiveInfo(ArrowTypeId.uint32)
|
||||
public static let ArrowUInt64 = Info.primitiveInfo(ArrowTypeId.uint64)
|
||||
public static let ArrowFloat = Info.primitiveInfo(ArrowTypeId.float)
|
||||
public static let ArrowDouble = Info.primitiveInfo(ArrowTypeId.double)
|
||||
public static let ArrowUnknown = Info.primitiveInfo(ArrowTypeId.unknown)
|
||||
public static let ArrowString = Info.variableInfo(ArrowTypeId.string)
|
||||
public static let ArrowBool = Info.primitiveInfo(ArrowTypeId.boolean)
|
||||
public static let ArrowDate32 = Info.primitiveInfo(ArrowTypeId.date32)
|
||||
public static let ArrowDate64 = Info.primitiveInfo(ArrowTypeId.date64)
|
||||
public static let ArrowBinary = Info.variableInfo(ArrowTypeId.binary)
|
||||
public static let ArrowTime32 = Info.timeInfo(ArrowTypeId.time32)
|
||||
public static let ArrowTime64 = Info.timeInfo(ArrowTypeId.time64)
|
||||
public static let ArrowStruct = Info.complexInfo(ArrowTypeId.strct)
|
||||
|
||||
public init(_ info: ArrowType.Info) {
|
||||
self.info = info
|
||||
}
|
||||
|
||||
public var id: ArrowTypeId {
|
||||
switch self.info {
|
||||
case .primitiveInfo(let id):
|
||||
return id
|
||||
case .timeInfo(let id):
|
||||
return id
|
||||
case .variableInfo(let id):
|
||||
return id
|
||||
case .complexInfo(let id):
|
||||
return id
|
||||
}
|
||||
}
|
||||
|
||||
public enum Info {
|
||||
case primitiveInfo(ArrowTypeId)
|
||||
case variableInfo(ArrowTypeId)
|
||||
case timeInfo(ArrowTypeId)
|
||||
case complexInfo(ArrowTypeId)
|
||||
}
|
||||
|
||||
public static func infoForType( // swiftlint:disable:this cyclomatic_complexity
|
||||
_ type: Any.Type) -> ArrowType.Info {
|
||||
if type == String.self {
|
||||
return ArrowType.ArrowString
|
||||
} else if type == Date.self {
|
||||
return ArrowType.ArrowDate64
|
||||
} else if type == Bool.self {
|
||||
return ArrowType.ArrowBool
|
||||
} else if type == Data.self {
|
||||
return ArrowType.ArrowBinary
|
||||
} else if type == Int8.self {
|
||||
return ArrowType.ArrowInt8
|
||||
} else if type == Int16.self {
|
||||
return ArrowType.ArrowInt16
|
||||
} else if type == Int32.self {
|
||||
return ArrowType.ArrowInt32
|
||||
} else if type == Int64.self {
|
||||
return ArrowType.ArrowInt64
|
||||
} else if type == UInt8.self {
|
||||
return ArrowType.ArrowUInt8
|
||||
} else if type == UInt16.self {
|
||||
return ArrowType.ArrowUInt16
|
||||
} else if type == UInt32.self {
|
||||
return ArrowType.ArrowUInt32
|
||||
} else if type == UInt64.self {
|
||||
return ArrowType.ArrowUInt64
|
||||
} else if type == Float.self {
|
||||
return ArrowType.ArrowFloat
|
||||
} else if type == Double.self {
|
||||
return ArrowType.ArrowDouble
|
||||
} else {
|
||||
return ArrowType.ArrowUnknown
|
||||
}
|
||||
}
|
||||
|
||||
public static func infoForNumericType<T>(_ type: T.Type) -> ArrowType.Info {
|
||||
if type == Int8.self {
|
||||
return ArrowType.ArrowInt8
|
||||
} else if type == Int16.self {
|
||||
return ArrowType.ArrowInt16
|
||||
} else if type == Int32.self {
|
||||
return ArrowType.ArrowInt32
|
||||
} else if type == Int64.self {
|
||||
return ArrowType.ArrowInt64
|
||||
} else if type == UInt8.self {
|
||||
return ArrowType.ArrowUInt8
|
||||
} else if type == UInt16.self {
|
||||
return ArrowType.ArrowUInt16
|
||||
} else if type == UInt32.self {
|
||||
return ArrowType.ArrowUInt32
|
||||
} else if type == UInt64.self {
|
||||
return ArrowType.ArrowUInt64
|
||||
} else if type == Float.self {
|
||||
return ArrowType.ArrowFloat
|
||||
} else if type == Double.self {
|
||||
return ArrowType.ArrowDouble
|
||||
} else {
|
||||
return ArrowType.ArrowUnknown
|
||||
}
|
||||
}
|
||||
|
||||
public func getStride( // swiftlint:disable:this cyclomatic_complexity
|
||||
) -> Int {
|
||||
switch self.id {
|
||||
case .int8:
|
||||
return MemoryLayout<Int8>.stride
|
||||
case .int16:
|
||||
return MemoryLayout<Int16>.stride
|
||||
case .int32:
|
||||
return MemoryLayout<Int32>.stride
|
||||
case .int64:
|
||||
return MemoryLayout<Int64>.stride
|
||||
case .uint8:
|
||||
return MemoryLayout<UInt8>.stride
|
||||
case .uint16:
|
||||
return MemoryLayout<UInt16>.stride
|
||||
case .uint32:
|
||||
return MemoryLayout<UInt32>.stride
|
||||
case .uint64:
|
||||
return MemoryLayout<UInt64>.stride
|
||||
case .float:
|
||||
return MemoryLayout<Float>.stride
|
||||
case .double:
|
||||
return MemoryLayout<Double>.stride
|
||||
case .boolean:
|
||||
return MemoryLayout<Bool>.stride
|
||||
case .date32:
|
||||
return MemoryLayout<Date32>.stride
|
||||
case .date64:
|
||||
return MemoryLayout<Date64>.stride
|
||||
case .time32:
|
||||
return MemoryLayout<Time32>.stride
|
||||
case .time64:
|
||||
return MemoryLayout<Time64>.stride
|
||||
case .binary:
|
||||
return MemoryLayout<Int8>.stride
|
||||
case .string:
|
||||
return MemoryLayout<Int8>.stride
|
||||
case .strct:
|
||||
return 0
|
||||
default:
|
||||
fatalError("Stride requested for unknown type: \(self)")
|
||||
}
|
||||
}
|
||||
|
||||
public var cDataFormatId: String {
|
||||
get throws {
|
||||
switch self.id {
|
||||
case ArrowTypeId.int8:
|
||||
return "c"
|
||||
case ArrowTypeId.int16:
|
||||
return "s"
|
||||
case ArrowTypeId.int32:
|
||||
return "i"
|
||||
case ArrowTypeId.int64:
|
||||
return "l"
|
||||
case ArrowTypeId.uint8:
|
||||
return "C"
|
||||
case ArrowTypeId.uint16:
|
||||
return "S"
|
||||
case ArrowTypeId.uint32:
|
||||
return "I"
|
||||
case ArrowTypeId.uint64:
|
||||
return "L"
|
||||
case ArrowTypeId.float:
|
||||
return "f"
|
||||
case ArrowTypeId.double:
|
||||
return "g"
|
||||
case ArrowTypeId.boolean:
|
||||
return "b"
|
||||
case ArrowTypeId.date32:
|
||||
return "tdD"
|
||||
case ArrowTypeId.date64:
|
||||
return "tdm"
|
||||
case ArrowTypeId.time32:
|
||||
if let time32 = self as? ArrowTypeTime32 {
|
||||
return try time32.cDataFormatId
|
||||
}
|
||||
return "tts"
|
||||
case ArrowTypeId.time64:
|
||||
if let time64 = self as? ArrowTypeTime64 {
|
||||
return try time64.cDataFormatId
|
||||
}
|
||||
return "ttu"
|
||||
case ArrowTypeId.binary:
|
||||
return "z"
|
||||
case ArrowTypeId.string:
|
||||
return "u"
|
||||
default:
|
||||
throw ArrowError.notImplemented
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static func fromCDataFormatId( // swiftlint:disable:this cyclomatic_complexity
|
||||
_ from: String) throws -> ArrowType {
|
||||
if from == "c" {
|
||||
return ArrowType(ArrowType.ArrowInt8)
|
||||
} else if from == "s" {
|
||||
return ArrowType(ArrowType.ArrowInt16)
|
||||
} else if from == "i" {
|
||||
return ArrowType(ArrowType.ArrowInt32)
|
||||
} else if from == "l" {
|
||||
return ArrowType(ArrowType.ArrowInt64)
|
||||
} else if from == "C" {
|
||||
return ArrowType(ArrowType.ArrowUInt8)
|
||||
} else if from == "S" {
|
||||
return ArrowType(ArrowType.ArrowUInt16)
|
||||
} else if from == "I" {
|
||||
return ArrowType(ArrowType.ArrowUInt32)
|
||||
} else if from == "L" {
|
||||
return ArrowType(ArrowType.ArrowUInt64)
|
||||
} else if from == "f" {
|
||||
return ArrowType(ArrowType.ArrowFloat)
|
||||
} else if from == "g" {
|
||||
return ArrowType(ArrowType.ArrowDouble)
|
||||
} else if from == "b" {
|
||||
return ArrowType(ArrowType.ArrowBool)
|
||||
} else if from == "tdD" {
|
||||
return ArrowType(ArrowType.ArrowDate32)
|
||||
} else if from == "tdm" {
|
||||
return ArrowType(ArrowType.ArrowDate64)
|
||||
} else if from == "tts" {
|
||||
return ArrowTypeTime32(.seconds)
|
||||
} else if from == "ttm" {
|
||||
return ArrowTypeTime32(.milliseconds)
|
||||
} else if from == "ttu" {
|
||||
return ArrowTypeTime64(.microseconds)
|
||||
} else if from == "ttn" {
|
||||
return ArrowTypeTime64(.nanoseconds)
|
||||
} else if from == "z" {
|
||||
return ArrowType(ArrowType.ArrowBinary)
|
||||
} else if from == "u" {
|
||||
return ArrowType(ArrowType.ArrowString)
|
||||
}
|
||||
|
||||
throw ArrowError.notImplemented
|
||||
}
|
||||
}
|
||||
|
||||
extension ArrowType.Info: Equatable {
|
||||
public static func == (lhs: ArrowType.Info, rhs: ArrowType.Info) -> Bool {
|
||||
switch(lhs, rhs) {
|
||||
case (.primitiveInfo(let lhsId), .primitiveInfo(let rhsId)):
|
||||
return lhsId == rhsId
|
||||
case (.variableInfo(let lhsId), .variableInfo(let rhsId)):
|
||||
return lhsId == rhsId
|
||||
case (.timeInfo(let lhsId), .timeInfo(let rhsId)):
|
||||
return lhsId == rhsId
|
||||
case (.complexInfo(let lhsId), .complexInfo(let rhsId)):
|
||||
return lhsId == rhsId
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getBytesFor<T>(_ data: T) -> Data? {
|
||||
if let temp = data as? String {
|
||||
return temp.data(using: .utf8)
|
||||
} else if T.self == Data.self {
|
||||
return data as? Data
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// swiftlint:disable:this file_length
|
||||
@@ -1,434 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import FlatBuffers
|
||||
|
||||
public protocol DataWriter {
|
||||
var count: Int {get}
|
||||
func append(_ data: Data)
|
||||
}
|
||||
|
||||
public class ArrowWriter { // swiftlint:disable:this type_body_length
|
||||
public class InMemDataWriter: DataWriter {
|
||||
public private(set) var data: Data
|
||||
public var count: Int { return data.count }
|
||||
public init(_ data: Data) {
|
||||
self.data = data
|
||||
}
|
||||
convenience init() {
|
||||
self.init(Data())
|
||||
}
|
||||
|
||||
public func append(_ data: Data) {
|
||||
self.data.append(data)
|
||||
}
|
||||
}
|
||||
|
||||
public class FileDataWriter: DataWriter {
|
||||
private var handle: FileHandle
|
||||
private var currentSize: Int = 0
|
||||
public var count: Int { return currentSize }
|
||||
public init(_ handle: FileHandle) {
|
||||
self.handle = handle
|
||||
}
|
||||
|
||||
public func append(_ data: Data) {
|
||||
self.handle.write(data)
|
||||
self.currentSize += data.count
|
||||
}
|
||||
}
|
||||
|
||||
public class Info {
|
||||
public let type: org_apache_arrow_flatbuf_MessageHeader
|
||||
public let schema: ArrowSchema
|
||||
public let batches: [RecordBatch]
|
||||
public init(_ type: org_apache_arrow_flatbuf_MessageHeader, schema: ArrowSchema, batches: [RecordBatch]) {
|
||||
self.type = type
|
||||
self.schema = schema
|
||||
self.batches = batches
|
||||
}
|
||||
|
||||
public convenience init(_ type: org_apache_arrow_flatbuf_MessageHeader, schema: ArrowSchema) {
|
||||
self.init(type, schema: schema, batches: [RecordBatch]())
|
||||
}
|
||||
}
|
||||
|
||||
public init() {}
|
||||
|
||||
private func writeField(_ fbb: inout FlatBufferBuilder, field: ArrowField) -> Result<Offset, ArrowError> {
|
||||
var fieldsOffset: Offset?
|
||||
if let nestedField = field.type as? ArrowNestedType {
|
||||
var offsets = [Offset]()
|
||||
for field in nestedField.fields {
|
||||
switch writeField(&fbb, field: field) {
|
||||
case .success(let offset):
|
||||
offsets.append(offset)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
fieldsOffset = fbb.createVector(ofOffsets: offsets)
|
||||
}
|
||||
|
||||
let nameOffset = fbb.create(string: field.name)
|
||||
let fieldTypeOffsetResult = toFBType(&fbb, arrowType: field.type)
|
||||
let startOffset = org_apache_arrow_flatbuf_Field.startField(&fbb)
|
||||
org_apache_arrow_flatbuf_Field.add(name: nameOffset, &fbb)
|
||||
org_apache_arrow_flatbuf_Field.add(nullable: field.isNullable, &fbb)
|
||||
if let childrenOffset = fieldsOffset {
|
||||
org_apache_arrow_flatbuf_Field.addVectorOf(children: childrenOffset, &fbb)
|
||||
}
|
||||
|
||||
switch toFBTypeEnum(field.type) {
|
||||
case .success(let type):
|
||||
org_apache_arrow_flatbuf_Field.add(typeType: type, &fbb)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
switch fieldTypeOffsetResult {
|
||||
case .success(let offset):
|
||||
org_apache_arrow_flatbuf_Field.add(type: offset, &fbb)
|
||||
return .success(org_apache_arrow_flatbuf_Field.endField(&fbb, start: startOffset))
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
private func writeSchema(_ fbb: inout FlatBufferBuilder, schema: ArrowSchema) -> Result<Offset, ArrowError> {
|
||||
var fieldOffsets = [Offset]()
|
||||
for field in schema.fields {
|
||||
switch writeField(&fbb, field: field) {
|
||||
case .success(let offset):
|
||||
fieldOffsets.append(offset)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
let fieldsOffset: Offset = fbb.createVector(ofOffsets: fieldOffsets)
|
||||
let schemaOffset =
|
||||
org_apache_arrow_flatbuf_Schema.createSchema(&fbb,
|
||||
endianness: .little,
|
||||
fieldsVectorOffset: fieldsOffset)
|
||||
return .success(schemaOffset)
|
||||
|
||||
}
|
||||
|
||||
private func writeRecordBatches(
|
||||
_ writer: inout DataWriter,
|
||||
batches: [RecordBatch]
|
||||
) -> Result<[org_apache_arrow_flatbuf_Block], ArrowError> {
|
||||
var rbBlocks = [org_apache_arrow_flatbuf_Block]()
|
||||
|
||||
for batch in batches {
|
||||
let startIndex = writer.count
|
||||
switch writeRecordBatch(batch: batch) {
|
||||
case .success(let rbResult):
|
||||
withUnsafeBytes(of: CONTINUATIONMARKER.littleEndian) {writer.append(Data($0))}
|
||||
withUnsafeBytes(of: rbResult.1.o.littleEndian) {writer.append(Data($0))}
|
||||
writer.append(rbResult.0)
|
||||
switch writeRecordBatchData(&writer, fields: batch.schema.fields, columns: batch.columns) {
|
||||
case .success:
|
||||
rbBlocks.append(
|
||||
org_apache_arrow_flatbuf_Block(offset: Int64(startIndex),
|
||||
metaDataLength: Int32(0),
|
||||
bodyLength: Int64(rbResult.1.o)))
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
return .success(rbBlocks)
|
||||
}
|
||||
|
||||
private func writeFieldNodes(_ fields: [ArrowField], columns: [ArrowArrayHolder], offsets: inout [Offset],
|
||||
fbb: inout FlatBufferBuilder) {
|
||||
for index in (0 ..< fields.count).reversed() {
|
||||
let column = columns[index]
|
||||
let fieldNode =
|
||||
org_apache_arrow_flatbuf_FieldNode(length: Int64(column.length),
|
||||
nullCount: Int64(column.nullCount))
|
||||
offsets.append(fbb.create(struct: fieldNode))
|
||||
if let nestedType = column.type as? ArrowNestedType {
|
||||
let structArray = column.array as? StructArray
|
||||
writeFieldNodes(nestedType.fields, columns: structArray!.arrowFields!, offsets: &offsets, fbb: &fbb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func writeBufferInfo(_ fields: [ArrowField],
|
||||
columns: [ArrowArrayHolder],
|
||||
bufferOffset: inout Int,
|
||||
buffers: inout [org_apache_arrow_flatbuf_Buffer],
|
||||
fbb: inout FlatBufferBuilder) {
|
||||
for index in 0 ..< fields.count {
|
||||
let column = columns[index]
|
||||
let colBufferDataSizes = column.getBufferDataSizes()
|
||||
for var bufferDataSize in colBufferDataSizes {
|
||||
bufferDataSize = getPadForAlignment(bufferDataSize)
|
||||
let buffer = org_apache_arrow_flatbuf_Buffer(offset: Int64(bufferOffset), length: Int64(bufferDataSize))
|
||||
buffers.append(buffer)
|
||||
bufferOffset += bufferDataSize
|
||||
if let nestedType = column.type as? ArrowNestedType {
|
||||
let structArray = column.array as? StructArray
|
||||
writeBufferInfo(nestedType.fields, columns: structArray!.arrowFields!,
|
||||
bufferOffset: &bufferOffset, buffers: &buffers, fbb: &fbb)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func writeRecordBatch(batch: RecordBatch) -> Result<(Data, Offset), ArrowError> {
|
||||
let schema = batch.schema
|
||||
var fbb = FlatBufferBuilder()
|
||||
|
||||
// write out field nodes
|
||||
var fieldNodeOffsets = [Offset]()
|
||||
fbb.startVector(schema.fields.count, elementSize: MemoryLayout<org_apache_arrow_flatbuf_FieldNode>.size)
|
||||
writeFieldNodes(schema.fields, columns: batch.columns, offsets: &fieldNodeOffsets, fbb: &fbb)
|
||||
let nodeOffset = fbb.endVector(len: fieldNodeOffsets.count)
|
||||
|
||||
// write out buffers
|
||||
var buffers = [org_apache_arrow_flatbuf_Buffer]()
|
||||
var bufferOffset = Int(0)
|
||||
writeBufferInfo(schema.fields, columns: batch.columns,
|
||||
bufferOffset: &bufferOffset, buffers: &buffers,
|
||||
fbb: &fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.startVectorOfBuffers(batch.schema.fields.count, in: &fbb)
|
||||
for buffer in buffers.reversed() {
|
||||
fbb.create(struct: buffer)
|
||||
}
|
||||
|
||||
let batchBuffersOffset = fbb.endVector(len: buffers.count)
|
||||
let startRb = org_apache_arrow_flatbuf_RecordBatch.startRecordBatch(&fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.addVectorOf(nodes: nodeOffset, &fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.addVectorOf(buffers: batchBuffersOffset, &fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.add(length: Int64(batch.length), &fbb)
|
||||
let recordBatchOffset = org_apache_arrow_flatbuf_RecordBatch.endRecordBatch(&fbb, start: startRb)
|
||||
let bodySize = Int64(bufferOffset)
|
||||
let startMessage = org_apache_arrow_flatbuf_Message.startMessage(&fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(version: .max, &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(bodyLength: Int64(bodySize), &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(headerType: .recordbatch, &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(header: recordBatchOffset, &fbb)
|
||||
let messageOffset = org_apache_arrow_flatbuf_Message.endMessage(&fbb, start: startMessage)
|
||||
fbb.finish(offset: messageOffset)
|
||||
return .success((fbb.data, Offset(offset: UInt32(fbb.data.count))))
|
||||
}
|
||||
|
||||
private func writeRecordBatchData(
|
||||
_ writer: inout DataWriter, fields: [ArrowField],
|
||||
columns: [ArrowArrayHolder])
|
||||
-> Result<Bool, ArrowError> {
|
||||
for index in 0 ..< fields.count {
|
||||
let column = columns[index]
|
||||
let colBufferData = column.getBufferData()
|
||||
for var bufferData in colBufferData {
|
||||
addPadForAlignment(&bufferData)
|
||||
writer.append(bufferData)
|
||||
if let nestedType = column.type as? ArrowNestedType {
|
||||
guard let structArray = column.array as? StructArray else {
|
||||
return .failure(.invalid("Struct type array expected for nested type"))
|
||||
}
|
||||
|
||||
switch writeRecordBatchData(&writer, fields: nestedType.fields, columns: structArray.arrowFields!) {
|
||||
case .success:
|
||||
continue
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return .success(true)
|
||||
}
|
||||
|
||||
private func writeFooter(schema: ArrowSchema,
|
||||
rbBlocks: [org_apache_arrow_flatbuf_Block]
|
||||
) -> Result<Data, ArrowError> {
|
||||
var fbb: FlatBufferBuilder = FlatBufferBuilder()
|
||||
switch writeSchema(&fbb, schema: schema) {
|
||||
case .success(let schemaOffset):
|
||||
fbb.startVector(rbBlocks.count, elementSize: MemoryLayout<org_apache_arrow_flatbuf_Block>.size)
|
||||
for blkInfo in rbBlocks.reversed() {
|
||||
fbb.create(struct: blkInfo)
|
||||
}
|
||||
|
||||
let rbBlkEnd = fbb.endVector(len: rbBlocks.count)
|
||||
let footerStartOffset = org_apache_arrow_flatbuf_Footer.startFooter(&fbb)
|
||||
org_apache_arrow_flatbuf_Footer.add(schema: schemaOffset, &fbb)
|
||||
org_apache_arrow_flatbuf_Footer.addVectorOf(recordBatches: rbBlkEnd, &fbb)
|
||||
let footerOffset = org_apache_arrow_flatbuf_Footer.endFooter(&fbb, start: footerStartOffset)
|
||||
fbb.finish(offset: footerOffset)
|
||||
return .success(fbb.data)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
private func writeFile(_ writer: inout DataWriter, info: ArrowWriter.Info) -> Result<Bool, ArrowError> {
|
||||
var fbb: FlatBufferBuilder = FlatBufferBuilder()
|
||||
switch writeSchema(&fbb, schema: info.schema) {
|
||||
case .success(let schemaOffset):
|
||||
fbb.finish(offset: schemaOffset)
|
||||
writer.append(fbb.data)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
switch writeRecordBatches(&writer, batches: info.batches) {
|
||||
case .success(let rbBlocks):
|
||||
switch writeFooter(schema: info.schema, rbBlocks: rbBlocks) {
|
||||
case .success(let footerData):
|
||||
fbb.finish(offset: Offset(offset: fbb.buffer.size))
|
||||
let footerOffset = writer.count
|
||||
writer.append(footerData)
|
||||
addPadForAlignment(&writer)
|
||||
|
||||
withUnsafeBytes(of: Int32(0).littleEndian) { writer.append(Data($0)) }
|
||||
let footerDiff = (UInt32(writer.count) - UInt32(footerOffset))
|
||||
withUnsafeBytes(of: footerDiff.littleEndian) { writer.append(Data($0)) }
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
return .success(true)
|
||||
}
|
||||
|
||||
public func writeStreaming(_ info: ArrowWriter.Info) -> Result<Data, ArrowError> {
|
||||
let writer: any DataWriter = InMemDataWriter()
|
||||
switch toMessage(info.schema) {
|
||||
case .success(let schemaData):
|
||||
withUnsafeBytes(of: CONTINUATIONMARKER.littleEndian) {writer.append(Data($0))}
|
||||
withUnsafeBytes(of: UInt32(schemaData.count).littleEndian) {writer.append(Data($0))}
|
||||
writer.append(schemaData)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
for batch in info.batches {
|
||||
switch toMessage(batch) {
|
||||
case .success(let batchData):
|
||||
withUnsafeBytes(of: CONTINUATIONMARKER.littleEndian) {writer.append(Data($0))}
|
||||
withUnsafeBytes(of: UInt32(batchData[0].count).littleEndian) {writer.append(Data($0))}
|
||||
writer.append(batchData[0])
|
||||
writer.append(batchData[1])
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
withUnsafeBytes(of: CONTINUATIONMARKER.littleEndian) {writer.append(Data($0))}
|
||||
withUnsafeBytes(of: UInt32(0).littleEndian) {writer.append(Data($0))}
|
||||
if let memWriter = writer as? InMemDataWriter {
|
||||
return .success(memWriter.data)
|
||||
} else {
|
||||
return .failure(.invalid("Unable to cast writer"))
|
||||
}
|
||||
}
|
||||
|
||||
public func writeFile(_ info: ArrowWriter.Info) -> Result<Data, ArrowError> {
|
||||
var writer: any DataWriter = InMemDataWriter()
|
||||
switch writeFile(&writer, info: info) {
|
||||
case .success:
|
||||
if let memWriter = writer as? InMemDataWriter {
|
||||
return .success(memWriter.data)
|
||||
} else {
|
||||
return .failure(.invalid("Unable to cast writer"))
|
||||
}
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
public func toFile(_ fileName: URL, info: ArrowWriter.Info) -> Result<Bool, ArrowError> {
|
||||
do {
|
||||
try Data().write(to: fileName)
|
||||
} catch {
|
||||
return .failure(.ioError("\(error)"))
|
||||
}
|
||||
|
||||
let fileHandle = FileHandle(forUpdatingAtPath: fileName.path)!
|
||||
defer { fileHandle.closeFile() }
|
||||
|
||||
var markerData = FILEMARKER.data(using: .utf8)!
|
||||
addPadForAlignment(&markerData)
|
||||
|
||||
var writer: any DataWriter = FileDataWriter(fileHandle)
|
||||
writer.append(FILEMARKER.data(using: .utf8)!)
|
||||
switch writeFile(&writer, info: info) {
|
||||
case .success:
|
||||
writer.append(FILEMARKER.data(using: .utf8)!)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
return .success(true)
|
||||
}
|
||||
|
||||
public func toMessage(_ batch: RecordBatch) -> Result<[Data], ArrowError> {
|
||||
var writer: any DataWriter = InMemDataWriter()
|
||||
switch writeRecordBatch(batch: batch) {
|
||||
case .success(let message):
|
||||
writer.append(message.0)
|
||||
addPadForAlignment(&writer)
|
||||
var dataWriter: any DataWriter = InMemDataWriter()
|
||||
switch writeRecordBatchData(&dataWriter, fields: batch.schema.fields, columns: batch.columns) {
|
||||
case .success:
|
||||
return .success([
|
||||
(writer as! InMemDataWriter).data, // swiftlint:disable:this force_cast
|
||||
(dataWriter as! InMemDataWriter).data // swiftlint:disable:this force_cast
|
||||
])
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
}
|
||||
|
||||
public func toMessage(_ schema: ArrowSchema) -> Result<Data, ArrowError> {
|
||||
var schemaSize: Int32 = 0
|
||||
var fbb = FlatBufferBuilder()
|
||||
switch writeSchema(&fbb, schema: schema) {
|
||||
case .success(let schemaOffset):
|
||||
schemaSize = Int32(schemaOffset.o)
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
let startMessage = org_apache_arrow_flatbuf_Message.startMessage(&fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(bodyLength: Int64(0), &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(headerType: .schema, &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(header: Offset(offset: UOffset(schemaSize)), &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(version: .max, &fbb)
|
||||
let messageOffset = org_apache_arrow_flatbuf_Message.endMessage(&fbb, start: startMessage)
|
||||
fbb.finish(offset: messageOffset)
|
||||
return .success(fbb.data)
|
||||
}
|
||||
}
|
||||
// swiftlint:disable:this file_length
|
||||
@@ -1,135 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import FlatBuffers
|
||||
|
||||
extension Data {
|
||||
func hexEncodedString() -> String {
|
||||
return map { String(format: "%02hhx", $0) }.joined()
|
||||
}
|
||||
}
|
||||
|
||||
func toFBTypeEnum(_ arrowType: ArrowType) -> Result<org_apache_arrow_flatbuf_Type_, ArrowError> {
|
||||
let typeId = arrowType.id
|
||||
switch typeId {
|
||||
case .int8, .int16, .int32, .int64, .uint8, .uint16, .uint32, .uint64:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.int)
|
||||
case .float, .double:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.floatingpoint)
|
||||
case .string:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.utf8)
|
||||
case .binary:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.binary)
|
||||
case .boolean:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.bool)
|
||||
case .date32, .date64:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.date)
|
||||
case .time32, .time64:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.time)
|
||||
case .strct:
|
||||
return .success(org_apache_arrow_flatbuf_Type_.struct_)
|
||||
default:
|
||||
return .failure(.unknownType("Unable to find flatbuf type for Arrow type: \(typeId)"))
|
||||
}
|
||||
}
|
||||
|
||||
func toFBType( // swiftlint:disable:this cyclomatic_complexity function_body_length
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
arrowType: ArrowType
|
||||
) -> Result<Offset, ArrowError> {
|
||||
let infoType = arrowType.info
|
||||
switch arrowType.id {
|
||||
case .int8, .uint8:
|
||||
return .success(org_apache_arrow_flatbuf_Int.createInt(
|
||||
&fbb, bitWidth: 8, isSigned: infoType == ArrowType.ArrowInt8))
|
||||
case .int16, .uint16:
|
||||
return .success(org_apache_arrow_flatbuf_Int.createInt(
|
||||
&fbb, bitWidth: 16, isSigned: infoType == ArrowType.ArrowInt16))
|
||||
case .int32, .uint32:
|
||||
return .success(org_apache_arrow_flatbuf_Int.createInt(
|
||||
&fbb, bitWidth: 32, isSigned: infoType == ArrowType.ArrowInt32))
|
||||
case .int64, .uint64:
|
||||
return .success(org_apache_arrow_flatbuf_Int.createInt(
|
||||
&fbb, bitWidth: 64, isSigned: infoType == ArrowType.ArrowInt64))
|
||||
case .float:
|
||||
return .success(org_apache_arrow_flatbuf_FloatingPoint.createFloatingPoint(&fbb, precision: .single))
|
||||
case .double:
|
||||
return .success(org_apache_arrow_flatbuf_FloatingPoint.createFloatingPoint(&fbb, precision: .double))
|
||||
case .string:
|
||||
return .success(org_apache_arrow_flatbuf_Utf8.endUtf8(
|
||||
&fbb, start: org_apache_arrow_flatbuf_Utf8.startUtf8(&fbb)))
|
||||
case .binary:
|
||||
return .success(org_apache_arrow_flatbuf_Binary.endBinary(
|
||||
&fbb, start: org_apache_arrow_flatbuf_Binary.startBinary(&fbb)))
|
||||
case .boolean:
|
||||
return .success(org_apache_arrow_flatbuf_Bool.endBool(
|
||||
&fbb, start: org_apache_arrow_flatbuf_Bool.startBool(&fbb)))
|
||||
case .date32:
|
||||
let startOffset = org_apache_arrow_flatbuf_Date.startDate(&fbb)
|
||||
org_apache_arrow_flatbuf_Date.add(unit: .day, &fbb)
|
||||
return .success(org_apache_arrow_flatbuf_Date.endDate(&fbb, start: startOffset))
|
||||
case .date64:
|
||||
let startOffset = org_apache_arrow_flatbuf_Date.startDate(&fbb)
|
||||
org_apache_arrow_flatbuf_Date.add(unit: .millisecond, &fbb)
|
||||
return .success(org_apache_arrow_flatbuf_Date.endDate(&fbb, start: startOffset))
|
||||
case .time32:
|
||||
let startOffset = org_apache_arrow_flatbuf_Time.startTime(&fbb)
|
||||
if let timeType = arrowType as? ArrowTypeTime32 {
|
||||
org_apache_arrow_flatbuf_Time.add(unit: timeType.unit == .seconds ? .second : .millisecond, &fbb)
|
||||
return .success(org_apache_arrow_flatbuf_Time.endTime(&fbb, start: startOffset))
|
||||
}
|
||||
|
||||
return .failure(.invalid("Unable to case to Time32"))
|
||||
case .time64:
|
||||
let startOffset = org_apache_arrow_flatbuf_Time.startTime(&fbb)
|
||||
if let timeType = arrowType as? ArrowTypeTime64 {
|
||||
org_apache_arrow_flatbuf_Time.add(unit: timeType.unit == .microseconds ? .microsecond : .nanosecond, &fbb)
|
||||
return .success(org_apache_arrow_flatbuf_Time.endTime(&fbb, start: startOffset))
|
||||
}
|
||||
|
||||
return .failure(.invalid("Unable to case to Time64"))
|
||||
case .strct:
|
||||
let startOffset = org_apache_arrow_flatbuf_Struct_.startStruct_(&fbb)
|
||||
return .success(org_apache_arrow_flatbuf_Struct_.endStruct_(&fbb, start: startOffset))
|
||||
default:
|
||||
return .failure(.unknownType("Unable to add flatbuf type for Arrow type: \(infoType)"))
|
||||
}
|
||||
}
|
||||
|
||||
func addPadForAlignment(_ data: inout Data, alignment: Int = 8) {
|
||||
let padding = data.count % Int(alignment)
|
||||
if padding > 0 {
|
||||
data.append(Data([UInt8](repeating: 0, count: alignment - padding)))
|
||||
}
|
||||
}
|
||||
|
||||
func addPadForAlignment(_ writer: inout DataWriter, alignment: Int = 8) {
|
||||
let padding = writer.count % Int(alignment)
|
||||
if padding > 0 {
|
||||
writer.append(Data([UInt8](repeating: 0, count: alignment - padding)))
|
||||
}
|
||||
}
|
||||
|
||||
func getPadForAlignment(_ count: Int, alignment: Int = 8) -> Int {
|
||||
let padding = count % Int(alignment)
|
||||
if padding > 0 {
|
||||
return count + (alignment - padding)
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
class BitUtility {
|
||||
static func isSet(_ bit: UInt, buffer: ArrowBuffer) -> Bool {
|
||||
let byteIndex = UInt(bit / 8)
|
||||
let theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self)
|
||||
return theByte & UInt8(1 << (bit % 8)) > 0
|
||||
}
|
||||
|
||||
static func setBit(_ bit: UInt, buffer: ArrowBuffer) {
|
||||
let byteIndex = UInt(bit / 8)
|
||||
var theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self)
|
||||
theByte |= UInt8(1 << (bit % 8))
|
||||
buffer.rawPointer.storeBytes(of: theByte, toByteOffset: Int(byteIndex), as: UInt8.self)
|
||||
}
|
||||
|
||||
static func clearBit(_ bit: UInt, buffer: ArrowBuffer) {
|
||||
let byteIndex = UInt(bit / 8)
|
||||
var theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self)
|
||||
theByte &= ~(UInt8(1 << (bit % 8)))
|
||||
buffer.rawPointer.storeBytes(of: theByte, toByteOffset: Int(byteIndex), as: UInt8.self)
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public protocol AnyArray {
|
||||
var arrowData: ArrowData {get}
|
||||
func asAny(_ index: UInt) -> Any?
|
||||
var length: UInt {get}
|
||||
}
|
||||
|
||||
public protocol AsString {
|
||||
func asString(_ index: UInt) -> String
|
||||
}
|
||||
|
||||
public class ChunkedArrayHolder {
|
||||
public let type: ArrowType
|
||||
public let length: UInt
|
||||
public let nullCount: UInt
|
||||
public let holder: Any
|
||||
|
||||
public let getBufferData: () -> Result<[Data], ArrowError>
|
||||
public let getBufferDataSizes: () -> Result<[Int], ArrowError>
|
||||
public init<T>(_ chunked: ChunkedArray<T>) { // swiftlint:disable:this cyclomatic_complexity
|
||||
self.holder = chunked
|
||||
self.length = chunked.length
|
||||
self.type = chunked.type
|
||||
self.nullCount = chunked.nullCount
|
||||
self.getBufferData = {() -> Result<[Data], ArrowError> in
|
||||
var bufferData = [Data]()
|
||||
var numBuffers = 2
|
||||
switch toFBTypeEnum(chunked.type) {
|
||||
case .success(let fbType):
|
||||
if !isFixedPrimitive(fbType) {
|
||||
numBuffers = 3
|
||||
}
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
for _ in 0 ..< numBuffers {
|
||||
bufferData.append(Data())
|
||||
}
|
||||
|
||||
for arrowData in chunked.arrays {
|
||||
for index in 0 ..< numBuffers {
|
||||
arrowData.arrowData.buffers[index].append(to: &bufferData[index])
|
||||
}
|
||||
}
|
||||
|
||||
return .success(bufferData)
|
||||
}
|
||||
|
||||
self.getBufferDataSizes = {() -> Result<[Int], ArrowError> in
|
||||
var bufferDataSizes = [Int]()
|
||||
var numBuffers = 2
|
||||
|
||||
switch toFBTypeEnum(chunked.type) {
|
||||
case .success(let fbType):
|
||||
if !isFixedPrimitive(fbType) {
|
||||
numBuffers = 3
|
||||
}
|
||||
case .failure(let error):
|
||||
return .failure(error)
|
||||
}
|
||||
|
||||
for _ in 0 ..< numBuffers {
|
||||
bufferDataSizes.append(Int(0))
|
||||
}
|
||||
|
||||
for arrowData in chunked.arrays {
|
||||
for index in 0 ..< numBuffers {
|
||||
bufferDataSizes[index] += Int(arrowData.arrowData.buffers[index].capacity)
|
||||
}
|
||||
}
|
||||
|
||||
return .success(bufferDataSizes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class ChunkedArray<T>: AsString {
|
||||
public let arrays: [ArrowArray<T>]
|
||||
public let type: ArrowType
|
||||
public let nullCount: UInt
|
||||
public let length: UInt
|
||||
public var arrayCount: UInt {return UInt(self.arrays.count)}
|
||||
|
||||
public init(_ arrays: [ArrowArray<T>]) throws {
|
||||
if arrays.count == 0 {
|
||||
throw ArrowError.arrayHasNoElements
|
||||
}
|
||||
|
||||
self.type = arrays[0].arrowData.type
|
||||
var len: UInt = 0
|
||||
var nullCount: UInt = 0
|
||||
for array in arrays {
|
||||
len += array.length
|
||||
nullCount += array.nullCount
|
||||
}
|
||||
|
||||
self.arrays = arrays
|
||||
self.length = len
|
||||
self.nullCount = nullCount
|
||||
}
|
||||
|
||||
public subscript(_ index: UInt) -> T? {
|
||||
if arrays.count == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var localIndex = index
|
||||
var arrayIndex = 0
|
||||
var len: UInt = arrays[arrayIndex].length
|
||||
while localIndex > (len - 1) {
|
||||
arrayIndex += 1
|
||||
if arrayIndex > arrays.count {
|
||||
return nil
|
||||
}
|
||||
|
||||
localIndex -= len
|
||||
len = arrays[arrayIndex].length
|
||||
}
|
||||
|
||||
return arrays[arrayIndex][localIndex]
|
||||
}
|
||||
|
||||
public func asString(_ index: UInt) -> String {
|
||||
if self[index] == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return "\(self[index]!)"
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
// automatically generated by the FlatBuffers compiler, do not modify
|
||||
// swiftlint:disable all
|
||||
// swiftformat:disable all
|
||||
|
||||
import FlatBuffers
|
||||
|
||||
public struct org_apache_arrow_flatbuf_Block: NativeStruct, Verifiable, FlatbuffersInitializable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
|
||||
/// Index to the start of the RecordBlock (note this is past the Message header)
|
||||
private var _offset: Int64
|
||||
/// Length of the metadata
|
||||
private var _metaDataLength: Int32
|
||||
private let padding0__: UInt32 = 0
|
||||
/// Length of the data (this is aligned so there can be a gap between this and
|
||||
/// the metadata).
|
||||
private var _bodyLength: Int64
|
||||
|
||||
public init(_ bb: ByteBuffer, o: Int32) {
|
||||
let _accessor = Struct(bb: bb, position: o)
|
||||
_offset = _accessor.readBuffer(of: Int64.self, at: 0)
|
||||
_metaDataLength = _accessor.readBuffer(of: Int32.self, at: 8)
|
||||
_bodyLength = _accessor.readBuffer(of: Int64.self, at: 16)
|
||||
}
|
||||
|
||||
public init(offset: Int64, metaDataLength: Int32, bodyLength: Int64) {
|
||||
_offset = offset
|
||||
_metaDataLength = metaDataLength
|
||||
_bodyLength = bodyLength
|
||||
}
|
||||
|
||||
public init() {
|
||||
_offset = 0
|
||||
_metaDataLength = 0
|
||||
_bodyLength = 0
|
||||
}
|
||||
|
||||
/// Index to the start of the RecordBlock (note this is past the Message header)
|
||||
public var offset: Int64 { _offset }
|
||||
/// Length of the metadata
|
||||
public var metaDataLength: Int32 { _metaDataLength }
|
||||
/// Length of the data (this is aligned so there can be a gap between this and
|
||||
/// the metadata).
|
||||
public var bodyLength: Int64 { _bodyLength }
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
try verifier.inBuffer(position: position, of: org_apache_arrow_flatbuf_Block.self)
|
||||
}
|
||||
}
|
||||
|
||||
public struct org_apache_arrow_flatbuf_Block_Mutable: FlatBufferObject {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Struct
|
||||
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Struct(bb: bb, position: o) }
|
||||
|
||||
public var offset: Int64 { return _accessor.readBuffer(of: Int64.self, at: 0) }
|
||||
public var metaDataLength: Int32 { return _accessor.readBuffer(of: Int32.self, at: 8) }
|
||||
public var bodyLength: Int64 { return _accessor.readBuffer(of: Int64.self, at: 16) }
|
||||
}
|
||||
|
||||
/// ----------------------------------------------------------------------
|
||||
/// Arrow File metadata
|
||||
///
|
||||
public struct org_apache_arrow_flatbuf_Footer: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsFooter(bb: ByteBuffer) -> org_apache_arrow_flatbuf_Footer { return org_apache_arrow_flatbuf_Footer(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case version = 4
|
||||
case schema = 6
|
||||
case dictionaries = 8
|
||||
case recordBatches = 10
|
||||
case customMetadata = 12
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
public var version: org_apache_arrow_flatbuf_MetadataVersion { let o = _accessor.offset(VTOFFSET.version.v); return o == 0 ? .v1 : org_apache_arrow_flatbuf_MetadataVersion(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .v1 }
|
||||
public var schema: org_apache_arrow_flatbuf_Schema? { let o = _accessor.offset(VTOFFSET.schema.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Schema(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
public var hasDictionaries: Bool { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? false : true }
|
||||
public var dictionariesCount: Int32 { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func dictionaries(at index: Int32) -> org_apache_arrow_flatbuf_Block? { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Block.self, offset: _accessor.vector(at: o) + index * 24) }
|
||||
public func mutableDictionaries(at index: Int32) -> org_apache_arrow_flatbuf_Block_Mutable? { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Block_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 24) }
|
||||
public var hasRecordBatches: Bool { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? false : true }
|
||||
public var recordBatchesCount: Int32 { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func recordBatches(at index: Int32) -> org_apache_arrow_flatbuf_Block? { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Block.self, offset: _accessor.vector(at: o) + index * 24) }
|
||||
public func mutableRecordBatches(at index: Int32) -> org_apache_arrow_flatbuf_Block_Mutable? { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Block_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 24) }
|
||||
/// User-defined metadata
|
||||
public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true }
|
||||
public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) }
|
||||
public static func startFooter(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) }
|
||||
public static func add(version: org_apache_arrow_flatbuf_MetadataVersion, _ fbb: inout FlatBufferBuilder) { fbb.add(element: version.rawValue, def: 0, at: VTOFFSET.version.p) }
|
||||
public static func add(schema: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: schema, at: VTOFFSET.schema.p) }
|
||||
public static func addVectorOf(dictionaries: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: dictionaries, at: VTOFFSET.dictionaries.p) }
|
||||
public static func startVectorOfDictionaries(_ size: Int, in builder: inout FlatBufferBuilder) {
|
||||
builder.startVector(size * MemoryLayout<org_apache_arrow_flatbuf_Block>.size, elementSize: MemoryLayout<org_apache_arrow_flatbuf_Block>.alignment)
|
||||
}
|
||||
public static func addVectorOf(recordBatches: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: recordBatches, at: VTOFFSET.recordBatches.p) }
|
||||
public static func startVectorOfRecordBatches(_ size: Int, in builder: inout FlatBufferBuilder) {
|
||||
builder.startVector(size * MemoryLayout<org_apache_arrow_flatbuf_Block>.size, elementSize: MemoryLayout<org_apache_arrow_flatbuf_Block>.alignment)
|
||||
}
|
||||
public static func addVectorOf(customMetadata: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) }
|
||||
public static func endFooter(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
|
||||
public static func createFooter(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
version: org_apache_arrow_flatbuf_MetadataVersion = .v1,
|
||||
schemaOffset schema: Offset = Offset(),
|
||||
dictionariesVectorOffset dictionaries: Offset = Offset(),
|
||||
recordBatchesVectorOffset recordBatches: Offset = Offset(),
|
||||
customMetadataVectorOffset customMetadata: Offset = Offset()
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_Footer.startFooter(&fbb)
|
||||
org_apache_arrow_flatbuf_Footer.add(version: version, &fbb)
|
||||
org_apache_arrow_flatbuf_Footer.add(schema: schema, &fbb)
|
||||
org_apache_arrow_flatbuf_Footer.addVectorOf(dictionaries: dictionaries, &fbb)
|
||||
org_apache_arrow_flatbuf_Footer.addVectorOf(recordBatches: recordBatches, &fbb)
|
||||
org_apache_arrow_flatbuf_Footer.addVectorOf(customMetadata: customMetadata, &fbb)
|
||||
return org_apache_arrow_flatbuf_Footer.endFooter(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.version.p, fieldName: "version", required: false, type: org_apache_arrow_flatbuf_MetadataVersion.self)
|
||||
try _v.visit(field: VTOFFSET.schema.p, fieldName: "schema", required: false, type: ForwardOffset<org_apache_arrow_flatbuf_Schema>.self)
|
||||
try _v.visit(field: VTOFFSET.dictionaries.p, fieldName: "dictionaries", required: false, type: ForwardOffset<Vector<org_apache_arrow_flatbuf_Block, org_apache_arrow_flatbuf_Block>>.self)
|
||||
try _v.visit(field: VTOFFSET.recordBatches.p, fieldName: "recordBatches", required: false, type: ForwardOffset<Vector<org_apache_arrow_flatbuf_Block, org_apache_arrow_flatbuf_Block>>.self)
|
||||
try _v.visit(field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", required: false, type: ForwardOffset<Vector<ForwardOffset<org_apache_arrow_flatbuf_KeyValue>, org_apache_arrow_flatbuf_KeyValue>>.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class MemoryAllocator {
|
||||
let alignment: Int
|
||||
init(_ alignment: Int) {
|
||||
self.alignment = alignment
|
||||
}
|
||||
|
||||
func allocateArray(_ byteCount: Int) -> UnsafeMutableRawPointer {
|
||||
return UnsafeMutableRawPointer.allocate(
|
||||
byteCount: byteCount,
|
||||
alignment: self.alignment)
|
||||
}
|
||||
}
|
||||
@@ -1,421 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
// automatically generated by the FlatBuffers compiler, do not modify
|
||||
// swiftlint:disable all
|
||||
// swiftformat:disable all
|
||||
|
||||
import FlatBuffers
|
||||
|
||||
public enum org_apache_arrow_flatbuf_CompressionType: Int8, Enum, Verifiable {
|
||||
public typealias T = Int8
|
||||
public static var byteSize: Int { return MemoryLayout<Int8>.size }
|
||||
public var value: Int8 { return self.rawValue }
|
||||
case lz4Frame = 0
|
||||
case zstd = 1
|
||||
|
||||
public static var max: org_apache_arrow_flatbuf_CompressionType { return .zstd }
|
||||
public static var min: org_apache_arrow_flatbuf_CompressionType { return .lz4Frame }
|
||||
}
|
||||
|
||||
|
||||
/// Provided for forward compatibility in case we need to support different
|
||||
/// strategies for compressing the IPC message body (like whole-body
|
||||
/// compression rather than buffer-level) in the future
|
||||
public enum org_apache_arrow_flatbuf_BodyCompressionMethod: Int8, Enum, Verifiable {
|
||||
public typealias T = Int8
|
||||
public static var byteSize: Int { return MemoryLayout<Int8>.size }
|
||||
public var value: Int8 { return self.rawValue }
|
||||
/// Each constituent buffer is first compressed with the indicated
|
||||
/// compressor, and then written with the uncompressed length in the first 8
|
||||
/// bytes as a 64-bit little-endian signed integer followed by the compressed
|
||||
/// buffer bytes (and then padding as required by the protocol). The
|
||||
/// uncompressed length may be set to -1 to indicate that the data that
|
||||
/// follows is not compressed, which can be useful for cases where
|
||||
/// compression does not yield appreciable savings.
|
||||
case buffer = 0
|
||||
|
||||
public static var max: org_apache_arrow_flatbuf_BodyCompressionMethod { return .buffer }
|
||||
public static var min: org_apache_arrow_flatbuf_BodyCompressionMethod { return .buffer }
|
||||
}
|
||||
|
||||
|
||||
/// ----------------------------------------------------------------------
|
||||
/// The root Message type
|
||||
/// This union enables us to easily send different message types without
|
||||
/// redundant storage, and in the future we can easily add new message types.
|
||||
///
|
||||
/// Arrow implementations do not need to implement all of the message types,
|
||||
/// which may include experimental metadata types. For maximum compatibility,
|
||||
/// it is best to send data using RecordBatch
|
||||
public enum org_apache_arrow_flatbuf_MessageHeader: UInt8, UnionEnum {
|
||||
public typealias T = UInt8
|
||||
|
||||
public init?(value: T) {
|
||||
self.init(rawValue: value)
|
||||
}
|
||||
|
||||
public static var byteSize: Int { return MemoryLayout<UInt8>.size }
|
||||
public var value: UInt8 { return self.rawValue }
|
||||
case none_ = 0
|
||||
case schema = 1
|
||||
case dictionarybatch = 2
|
||||
case recordbatch = 3
|
||||
case tensor = 4
|
||||
case sparsetensor = 5
|
||||
|
||||
public static var max: org_apache_arrow_flatbuf_MessageHeader { return .sparsetensor }
|
||||
public static var min: org_apache_arrow_flatbuf_MessageHeader { return .none_ }
|
||||
}
|
||||
|
||||
|
||||
/// ----------------------------------------------------------------------
|
||||
/// Data structures for describing a table row batch (a collection of
|
||||
/// equal-length Arrow arrays)
|
||||
/// Metadata about a field at some level of a nested type tree (but not
|
||||
/// its children).
|
||||
///
|
||||
/// For example, a List<Int16> with values `[[1, 2, 3], null, [4], [5, 6], null]`
|
||||
/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
|
||||
/// null_count: 0} for its Int16 node, as separate FieldNode structs
|
||||
public struct org_apache_arrow_flatbuf_FieldNode: NativeStruct, Verifiable, FlatbuffersInitializable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
|
||||
/// The number of value slots in the Arrow array at this level of a nested
|
||||
/// tree
|
||||
private var _length: Int64
|
||||
/// The number of observed nulls. Fields with null_count == 0 may choose not
|
||||
/// to write their physical validity bitmap out as a materialized buffer,
|
||||
/// instead setting the length of the bitmap buffer to 0.
|
||||
private var _nullCount: Int64
|
||||
|
||||
public init(_ bb: ByteBuffer, o: Int32) {
|
||||
let _accessor = Struct(bb: bb, position: o)
|
||||
_length = _accessor.readBuffer(of: Int64.self, at: 0)
|
||||
_nullCount = _accessor.readBuffer(of: Int64.self, at: 8)
|
||||
}
|
||||
|
||||
public init(length: Int64, nullCount: Int64) {
|
||||
_length = length
|
||||
_nullCount = nullCount
|
||||
}
|
||||
|
||||
public init() {
|
||||
_length = 0
|
||||
_nullCount = 0
|
||||
}
|
||||
|
||||
/// The number of value slots in the Arrow array at this level of a nested
|
||||
/// tree
|
||||
public var length: Int64 { _length }
|
||||
/// The number of observed nulls. Fields with null_count == 0 may choose not
|
||||
/// to write their physical validity bitmap out as a materialized buffer,
|
||||
/// instead setting the length of the bitmap buffer to 0.
|
||||
public var nullCount: Int64 { _nullCount }
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
try verifier.inBuffer(position: position, of: org_apache_arrow_flatbuf_FieldNode.self)
|
||||
}
|
||||
}
|
||||
|
||||
/// ----------------------------------------------------------------------
|
||||
/// Data structures for describing a table row batch (a collection of
|
||||
/// equal-length Arrow arrays)
|
||||
/// Metadata about a field at some level of a nested type tree (but not
|
||||
/// its children).
|
||||
///
|
||||
/// For example, a List<Int16> with values `[[1, 2, 3], null, [4], [5, 6], null]`
|
||||
/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
|
||||
/// null_count: 0} for its Int16 node, as separate FieldNode structs
|
||||
public struct org_apache_arrow_flatbuf_FieldNode_Mutable: FlatBufferObject {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Struct
|
||||
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Struct(bb: bb, position: o) }
|
||||
|
||||
public var length: Int64 { return _accessor.readBuffer(of: Int64.self, at: 0) }
|
||||
public var nullCount: Int64 { return _accessor.readBuffer(of: Int64.self, at: 8) }
|
||||
}
|
||||
|
||||
/// Optional compression for the memory buffers constituting IPC message
|
||||
/// bodies. Intended for use with RecordBatch but could be used for other
|
||||
/// message types
|
||||
public struct org_apache_arrow_flatbuf_BodyCompression: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsBodyCompression(bb: ByteBuffer) -> org_apache_arrow_flatbuf_BodyCompression { return org_apache_arrow_flatbuf_BodyCompression(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case codec = 4
|
||||
case method = 6
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
/// Compressor library.
|
||||
/// For LZ4_FRAME, each compressed buffer must consist of a single frame.
|
||||
public var codec: org_apache_arrow_flatbuf_CompressionType { let o = _accessor.offset(VTOFFSET.codec.v); return o == 0 ? .lz4Frame : org_apache_arrow_flatbuf_CompressionType(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .lz4Frame }
|
||||
/// Indicates the way the record batch body was compressed
|
||||
public var method: org_apache_arrow_flatbuf_BodyCompressionMethod { let o = _accessor.offset(VTOFFSET.method.v); return o == 0 ? .buffer : org_apache_arrow_flatbuf_BodyCompressionMethod(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .buffer }
|
||||
public static func startBodyCompression(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) }
|
||||
public static func add(codec: org_apache_arrow_flatbuf_CompressionType, _ fbb: inout FlatBufferBuilder) { fbb.add(element: codec.rawValue, def: 0, at: VTOFFSET.codec.p) }
|
||||
public static func add(method: org_apache_arrow_flatbuf_BodyCompressionMethod, _ fbb: inout FlatBufferBuilder) { fbb.add(element: method.rawValue, def: 0, at: VTOFFSET.method.p) }
|
||||
public static func endBodyCompression(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
|
||||
public static func createBodyCompression(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
codec: org_apache_arrow_flatbuf_CompressionType = .lz4Frame,
|
||||
method: org_apache_arrow_flatbuf_BodyCompressionMethod = .buffer
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_BodyCompression.startBodyCompression(&fbb)
|
||||
org_apache_arrow_flatbuf_BodyCompression.add(codec: codec, &fbb)
|
||||
org_apache_arrow_flatbuf_BodyCompression.add(method: method, &fbb)
|
||||
return org_apache_arrow_flatbuf_BodyCompression.endBodyCompression(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.codec.p, fieldName: "codec", required: false, type: org_apache_arrow_flatbuf_CompressionType.self)
|
||||
try _v.visit(field: VTOFFSET.method.p, fieldName: "method", required: false, type: org_apache_arrow_flatbuf_BodyCompressionMethod.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// A data header describing the shared memory layout of a "record" or "row"
|
||||
/// batch. Some systems call this a "row batch" internally and others a "record
|
||||
/// batch".
|
||||
public struct org_apache_arrow_flatbuf_RecordBatch: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsRecordBatch(bb: ByteBuffer) -> org_apache_arrow_flatbuf_RecordBatch { return org_apache_arrow_flatbuf_RecordBatch(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case length = 4
|
||||
case nodes = 6
|
||||
case buffers = 8
|
||||
case compression = 10
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
/// number of records / rows. The arrays in the batch should all have this
|
||||
/// length
|
||||
public var length: Int64 { let o = _accessor.offset(VTOFFSET.length.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) }
|
||||
/// Nodes correspond to the pre-ordered flattened logical schema
|
||||
public var hasNodes: Bool { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? false : true }
|
||||
public var nodesCount: Int32 { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func nodes(at index: Int32) -> org_apache_arrow_flatbuf_FieldNode? { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_FieldNode.self, offset: _accessor.vector(at: o) + index * 16) }
|
||||
public func mutableNodes(at index: Int32) -> org_apache_arrow_flatbuf_FieldNode_Mutable? { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? nil : org_apache_arrow_flatbuf_FieldNode_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) }
|
||||
/// Buffers correspond to the pre-ordered flattened buffer tree
|
||||
///
|
||||
/// The number of buffers appended to this list depends on the schema. For
|
||||
/// example, most primitive arrays will have 2 buffers, 1 for the validity
|
||||
/// bitmap and 1 for the values. For struct arrays, there will only be a
|
||||
/// single buffer for the validity (nulls) bitmap
|
||||
public var hasBuffers: Bool { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? false : true }
|
||||
public var buffersCount: Int32 { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func buffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) }
|
||||
public func mutableBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) }
|
||||
/// Optional compression of the message body
|
||||
public var compression: org_apache_arrow_flatbuf_BodyCompression? { let o = _accessor.offset(VTOFFSET.compression.v); return o == 0 ? nil : org_apache_arrow_flatbuf_BodyCompression(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
public static func startRecordBatch(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 4) }
|
||||
public static func add(length: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: length, def: 0, at: VTOFFSET.length.p) }
|
||||
public static func addVectorOf(nodes: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: nodes, at: VTOFFSET.nodes.p) }
|
||||
public static func startVectorOfNodes(_ size: Int, in builder: inout FlatBufferBuilder) {
|
||||
builder.startVector(size * MemoryLayout<org_apache_arrow_flatbuf_FieldNode>.size, elementSize: MemoryLayout<org_apache_arrow_flatbuf_FieldNode>.alignment)
|
||||
}
|
||||
public static func addVectorOf(buffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: buffers, at: VTOFFSET.buffers.p) }
|
||||
public static func startVectorOfBuffers(_ size: Int, in builder: inout FlatBufferBuilder) {
|
||||
builder.startVector(size * MemoryLayout<org_apache_arrow_flatbuf_Buffer>.size, elementSize: MemoryLayout<org_apache_arrow_flatbuf_Buffer>.alignment)
|
||||
}
|
||||
public static func add(compression: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: compression, at: VTOFFSET.compression.p) }
|
||||
public static func endRecordBatch(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
|
||||
public static func createRecordBatch(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
length: Int64 = 0,
|
||||
nodesVectorOffset nodes: Offset = Offset(),
|
||||
buffersVectorOffset buffers: Offset = Offset(),
|
||||
compressionOffset compression: Offset = Offset()
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_RecordBatch.startRecordBatch(&fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.add(length: length, &fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.addVectorOf(nodes: nodes, &fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.addVectorOf(buffers: buffers, &fbb)
|
||||
org_apache_arrow_flatbuf_RecordBatch.add(compression: compression, &fbb)
|
||||
return org_apache_arrow_flatbuf_RecordBatch.endRecordBatch(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.length.p, fieldName: "length", required: false, type: Int64.self)
|
||||
try _v.visit(field: VTOFFSET.nodes.p, fieldName: "nodes", required: false, type: ForwardOffset<Vector<org_apache_arrow_flatbuf_FieldNode, org_apache_arrow_flatbuf_FieldNode>>.self)
|
||||
try _v.visit(field: VTOFFSET.buffers.p, fieldName: "buffers", required: false, type: ForwardOffset<Vector<org_apache_arrow_flatbuf_Buffer, org_apache_arrow_flatbuf_Buffer>>.self)
|
||||
try _v.visit(field: VTOFFSET.compression.p, fieldName: "compression", required: false, type: ForwardOffset<org_apache_arrow_flatbuf_BodyCompression>.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// For sending dictionary encoding information. Any Field can be
|
||||
/// dictionary-encoded, but in this case none of its children may be
|
||||
/// dictionary-encoded.
|
||||
/// There is one vector / column per dictionary, but that vector / column
|
||||
/// may be spread across multiple dictionary batches by using the isDelta
|
||||
/// flag
|
||||
public struct org_apache_arrow_flatbuf_DictionaryBatch: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsDictionaryBatch(bb: ByteBuffer) -> org_apache_arrow_flatbuf_DictionaryBatch { return org_apache_arrow_flatbuf_DictionaryBatch(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case id = 4
|
||||
case data = 6
|
||||
case isDelta = 8
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
public var id: Int64 { let o = _accessor.offset(VTOFFSET.id.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) }
|
||||
public var data: org_apache_arrow_flatbuf_RecordBatch? { let o = _accessor.offset(VTOFFSET.data.v); return o == 0 ? nil : org_apache_arrow_flatbuf_RecordBatch(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
/// If isDelta is true the values in the dictionary are to be appended to a
|
||||
/// dictionary with the indicated id. If isDelta is false this dictionary
|
||||
/// should replace the existing dictionary.
|
||||
public var isDelta: Bool { let o = _accessor.offset(VTOFFSET.isDelta.v); return o == 0 ? false : 0 != _accessor.readBuffer(of: Byte.self, at: o) }
|
||||
public static func startDictionaryBatch(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 3) }
|
||||
public static func add(id: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: id, def: 0, at: VTOFFSET.id.p) }
|
||||
public static func add(data: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: data, at: VTOFFSET.data.p) }
|
||||
public static func add(isDelta: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: isDelta, def: false,
|
||||
at: VTOFFSET.isDelta.p) }
|
||||
public static func endDictionaryBatch(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
|
||||
public static func createDictionaryBatch(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
id: Int64 = 0,
|
||||
dataOffset data: Offset = Offset(),
|
||||
isDelta: Bool = false
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_DictionaryBatch.startDictionaryBatch(&fbb)
|
||||
org_apache_arrow_flatbuf_DictionaryBatch.add(id: id, &fbb)
|
||||
org_apache_arrow_flatbuf_DictionaryBatch.add(data: data, &fbb)
|
||||
org_apache_arrow_flatbuf_DictionaryBatch.add(isDelta: isDelta, &fbb)
|
||||
return org_apache_arrow_flatbuf_DictionaryBatch.endDictionaryBatch(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.id.p, fieldName: "id", required: false, type: Int64.self)
|
||||
try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: false, type: ForwardOffset<org_apache_arrow_flatbuf_RecordBatch>.self)
|
||||
try _v.visit(field: VTOFFSET.isDelta.p, fieldName: "isDelta", required: false, type: Bool.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
public struct org_apache_arrow_flatbuf_Message: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsMessage(bb: ByteBuffer) -> org_apache_arrow_flatbuf_Message { return org_apache_arrow_flatbuf_Message(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case version = 4
|
||||
case headerType = 6
|
||||
case header = 8
|
||||
case bodyLength = 10
|
||||
case customMetadata = 12
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
public var version: org_apache_arrow_flatbuf_MetadataVersion { let o = _accessor.offset(VTOFFSET.version.v); return o == 0 ? .v1 : org_apache_arrow_flatbuf_MetadataVersion(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .v1 }
|
||||
public var headerType: org_apache_arrow_flatbuf_MessageHeader { let o = _accessor.offset(VTOFFSET.headerType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_MessageHeader(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ }
|
||||
public func header<T: FlatbuffersInitializable>(type: T.Type) -> T? { let o = _accessor.offset(VTOFFSET.header.v); return o == 0 ? nil : _accessor.union(o) }
|
||||
public var bodyLength: Int64 { let o = _accessor.offset(VTOFFSET.bodyLength.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) }
|
||||
public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true }
|
||||
public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) }
|
||||
public static func startMessage(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) }
|
||||
public static func add(version: org_apache_arrow_flatbuf_MetadataVersion, _ fbb: inout FlatBufferBuilder) { fbb.add(element: version.rawValue, def: 0, at: VTOFFSET.version.p) }
|
||||
public static func add(headerType: org_apache_arrow_flatbuf_MessageHeader, _ fbb: inout FlatBufferBuilder) { fbb.add(element: headerType.rawValue, def: 0, at: VTOFFSET.headerType.p) }
|
||||
public static func add(header: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: header, at: VTOFFSET.header.p) }
|
||||
public static func add(bodyLength: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: bodyLength, def: 0, at: VTOFFSET.bodyLength.p) }
|
||||
public static func addVectorOf(customMetadata: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) }
|
||||
public static func endMessage(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
|
||||
public static func createMessage(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
version: org_apache_arrow_flatbuf_MetadataVersion = .v1,
|
||||
headerType: org_apache_arrow_flatbuf_MessageHeader = .none_,
|
||||
headerOffset header: Offset = Offset(),
|
||||
bodyLength: Int64 = 0,
|
||||
customMetadataVectorOffset customMetadata: Offset = Offset()
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_Message.startMessage(&fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(version: version, &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(headerType: headerType, &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(header: header, &fbb)
|
||||
org_apache_arrow_flatbuf_Message.add(bodyLength: bodyLength, &fbb)
|
||||
org_apache_arrow_flatbuf_Message.addVectorOf(customMetadata: customMetadata, &fbb)
|
||||
return org_apache_arrow_flatbuf_Message.endMessage(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.version.p, fieldName: "version", required: false, type: org_apache_arrow_flatbuf_MetadataVersion.self)
|
||||
try _v.visit(unionKey: VTOFFSET.headerType.p, unionField: VTOFFSET.header.p, unionKeyName: "headerType", fieldName: "header", required: false, completion: { (verifier, key: org_apache_arrow_flatbuf_MessageHeader, pos) in
|
||||
switch key {
|
||||
case .none_:
|
||||
break // NOTE - SWIFT doesnt support none
|
||||
case .schema:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Schema>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Schema.self)
|
||||
case .dictionarybatch:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_DictionaryBatch>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_DictionaryBatch.self)
|
||||
case .recordbatch:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_RecordBatch>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RecordBatch.self)
|
||||
case .tensor:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Tensor>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Tensor.self)
|
||||
case .sparsetensor:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_SparseTensor>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensor.self)
|
||||
}
|
||||
})
|
||||
try _v.visit(field: VTOFFSET.bodyLength.p, fieldName: "bodyLength", required: false, type: Int64.self)
|
||||
try _v.visit(field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", required: false, type: ForwardOffset<Vector<ForwardOffset<org_apache_arrow_flatbuf_KeyValue>, org_apache_arrow_flatbuf_KeyValue>>.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
func fromProto( // swiftlint:disable:this cyclomatic_complexity function_body_length
|
||||
field: org_apache_arrow_flatbuf_Field
|
||||
) -> ArrowField {
|
||||
let type = field.typeType
|
||||
var arrowType = ArrowType(ArrowType.ArrowUnknown)
|
||||
switch type {
|
||||
case .int:
|
||||
let intType = field.type(type: org_apache_arrow_flatbuf_Int.self)!
|
||||
let bitWidth = intType.bitWidth
|
||||
if bitWidth == 8 {
|
||||
arrowType = ArrowType(intType.isSigned ? ArrowType.ArrowInt8 : ArrowType.ArrowUInt8)
|
||||
} else if bitWidth == 16 {
|
||||
arrowType = ArrowType(intType.isSigned ? ArrowType.ArrowInt16 : ArrowType.ArrowUInt16)
|
||||
} else if bitWidth == 32 {
|
||||
arrowType = ArrowType(intType.isSigned ? ArrowType.ArrowInt32 : ArrowType.ArrowUInt32)
|
||||
} else if bitWidth == 64 {
|
||||
arrowType = ArrowType(intType.isSigned ? ArrowType.ArrowInt64 : ArrowType.ArrowUInt64)
|
||||
}
|
||||
case .bool:
|
||||
arrowType = ArrowType(ArrowType.ArrowBool)
|
||||
case .floatingpoint:
|
||||
let floatType = field.type(type: org_apache_arrow_flatbuf_FloatingPoint.self)!
|
||||
if floatType.precision == .single {
|
||||
arrowType = ArrowType(ArrowType.ArrowFloat)
|
||||
} else if floatType.precision == .double {
|
||||
arrowType = ArrowType(ArrowType.ArrowDouble)
|
||||
}
|
||||
case .utf8:
|
||||
arrowType = ArrowType(ArrowType.ArrowString)
|
||||
case .binary:
|
||||
arrowType = ArrowType(ArrowType.ArrowBinary)
|
||||
case .date:
|
||||
let dateType = field.type(type: org_apache_arrow_flatbuf_Date.self)!
|
||||
if dateType.unit == .day {
|
||||
arrowType = ArrowType(ArrowType.ArrowDate32)
|
||||
} else {
|
||||
arrowType = ArrowType(ArrowType.ArrowDate64)
|
||||
}
|
||||
case .time:
|
||||
let timeType = field.type(type: org_apache_arrow_flatbuf_Time.self)!
|
||||
if timeType.unit == .second || timeType.unit == .millisecond {
|
||||
let arrowUnit: ArrowTime32Unit = timeType.unit == .second ? .seconds : .milliseconds
|
||||
arrowType = ArrowTypeTime32(arrowUnit)
|
||||
} else {
|
||||
let arrowUnit: ArrowTime64Unit = timeType.unit == .microsecond ? .microseconds : .nanoseconds
|
||||
arrowType = ArrowTypeTime64(arrowUnit)
|
||||
}
|
||||
case .struct_:
|
||||
var children = [ArrowField]()
|
||||
for index in 0..<field.childrenCount {
|
||||
let childField = field.children(at: index)!
|
||||
children.append(fromProto(field: childField))
|
||||
}
|
||||
|
||||
arrowType = ArrowNestedType(ArrowType.ArrowStruct, fields: children)
|
||||
default:
|
||||
arrowType = ArrowType(ArrowType.ArrowUnknown)
|
||||
}
|
||||
|
||||
return ArrowField(field.name ?? "", type: arrowType, isNullable: field.nullable)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,535 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
// automatically generated by the FlatBuffers compiler, do not modify
|
||||
// swiftlint:disable all
|
||||
// swiftformat:disable all
|
||||
|
||||
import FlatBuffers
|
||||
|
||||
public enum org_apache_arrow_flatbuf_SparseMatrixCompressedAxis: Int16, Enum, Verifiable {
|
||||
public typealias T = Int16
|
||||
public static var byteSize: Int { return MemoryLayout<Int16>.size }
|
||||
public var value: Int16 { return self.rawValue }
|
||||
case row = 0
|
||||
case column = 1
|
||||
|
||||
public static var max: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis { return .column }
|
||||
public static var min: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis { return .row }
|
||||
}
|
||||
|
||||
|
||||
public enum org_apache_arrow_flatbuf_SparseTensorIndex: UInt8, UnionEnum {
|
||||
public typealias T = UInt8
|
||||
|
||||
public init?(value: T) {
|
||||
self.init(rawValue: value)
|
||||
}
|
||||
|
||||
public static var byteSize: Int { return MemoryLayout<UInt8>.size }
|
||||
public var value: UInt8 { return self.rawValue }
|
||||
case none_ = 0
|
||||
case sparsetensorindexcoo = 1
|
||||
case sparsematrixindexcsx = 2
|
||||
case sparsetensorindexcsf = 3
|
||||
|
||||
public static var max: org_apache_arrow_flatbuf_SparseTensorIndex { return .sparsetensorindexcsf }
|
||||
public static var min: org_apache_arrow_flatbuf_SparseTensorIndex { return .none_ }
|
||||
}
|
||||
|
||||
|
||||
/// ----------------------------------------------------------------------
|
||||
/// EXPERIMENTAL: Data structures for sparse tensors
|
||||
/// Coordinate (COO) format of sparse tensor index.
|
||||
///
|
||||
/// COO's index list are represented as a NxM matrix,
|
||||
/// where N is the number of non-zero values,
|
||||
/// and M is the number of dimensions of a sparse tensor.
|
||||
///
|
||||
/// indicesBuffer stores the location and size of the data of this indices
|
||||
/// matrix. The value type and the stride of the indices matrix is
|
||||
/// specified in indicesType and indicesStrides fields.
|
||||
///
|
||||
/// For example, let X be a 2x3x4x5 tensor, and it has the following
|
||||
/// 6 non-zero values:
|
||||
/// ```text
|
||||
/// X[0, 1, 2, 0] := 1
|
||||
/// X[1, 1, 2, 3] := 2
|
||||
/// X[0, 2, 1, 0] := 3
|
||||
/// X[0, 1, 3, 0] := 4
|
||||
/// X[0, 1, 2, 1] := 5
|
||||
/// X[1, 2, 0, 4] := 6
|
||||
/// ```
|
||||
/// In COO format, the index matrix of X is the following 4x6 matrix:
|
||||
/// ```text
|
||||
/// [[0, 0, 0, 0, 1, 1],
|
||||
/// [1, 1, 1, 2, 1, 2],
|
||||
/// [2, 2, 3, 1, 2, 0],
|
||||
/// [0, 1, 0, 0, 3, 4]]
|
||||
/// ```
|
||||
/// When isCanonical is true, the indices is sorted in lexicographical order
|
||||
/// (row-major order), and it does not have duplicated entries. Otherwise,
|
||||
/// the indices may not be sorted, or may have duplicated entries.
|
||||
public struct org_apache_arrow_flatbuf_SparseTensorIndexCOO: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsSparseTensorIndexCOO(bb: ByteBuffer) -> org_apache_arrow_flatbuf_SparseTensorIndexCOO { return org_apache_arrow_flatbuf_SparseTensorIndexCOO(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case indicesType = 4
|
||||
case indicesStrides = 6
|
||||
case indicesBuffer = 8
|
||||
case isCanonical = 10
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
/// The type of values in indicesBuffer
|
||||
public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
/// Non-negative byte offsets to advance one value cell along each dimension
|
||||
/// If omitted, default to row-major order (C-like).
|
||||
public var hasIndicesStrides: Bool { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? false : true }
|
||||
public var indicesStridesCount: Int32 { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func indicesStrides(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) }
|
||||
public var indicesStrides: [Int64] { return _accessor.getVector(at: VTOFFSET.indicesStrides.v) ?? [] }
|
||||
/// The location and size of the indices matrix's data
|
||||
public var indicesBuffer: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) }
|
||||
public var mutableIndicesBuffer: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) }
|
||||
/// This flag is true if and only if the indices matrix is sorted in
|
||||
/// row-major order, and does not have duplicated entries.
|
||||
/// This sort order is the same as of Tensorflow's SparseTensor,
|
||||
/// but it is inverse order of SciPy's canonical coo_matrix
|
||||
/// (SciPy employs column-major order for its coo_matrix).
|
||||
public var isCanonical: Bool { let o = _accessor.offset(VTOFFSET.isCanonical.v); return o == 0 ? false : 0 != _accessor.readBuffer(of: Byte.self, at: o) }
|
||||
public static func startSparseTensorIndexCOO(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 4) }
|
||||
public static func add(indicesType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesType, at: VTOFFSET.indicesType.p) }
|
||||
public static func addVectorOf(indicesStrides: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesStrides, at: VTOFFSET.indicesStrides.p) }
|
||||
public static func add(indicesBuffer: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let indicesBuffer = indicesBuffer else { return }; fbb.create(struct: indicesBuffer, position: VTOFFSET.indicesBuffer.p) }
|
||||
public static func add(isCanonical: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: isCanonical, def: false,
|
||||
at: VTOFFSET.isCanonical.p) }
|
||||
public static func endSparseTensorIndexCOO(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [4, 8]); return end }
|
||||
public static func createSparseTensorIndexCOO(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
indicesTypeOffset indicesType: Offset,
|
||||
indicesStridesVectorOffset indicesStrides: Offset = Offset(),
|
||||
indicesBuffer: org_apache_arrow_flatbuf_Buffer,
|
||||
isCanonical: Bool = false
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_SparseTensorIndexCOO.startSparseTensorIndexCOO(&fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCOO.add(indicesType: indicesType, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCOO.addVectorOf(indicesStrides: indicesStrides, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCOO.add(indicesBuffer: indicesBuffer, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCOO.add(isCanonical: isCanonical, &fbb)
|
||||
return org_apache_arrow_flatbuf_SparseTensorIndexCOO.endSparseTensorIndexCOO(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.indicesType.p, fieldName: "indicesType", required: true, type: ForwardOffset<org_apache_arrow_flatbuf_Int>.self)
|
||||
try _v.visit(field: VTOFFSET.indicesStrides.p, fieldName: "indicesStrides", required: false, type: ForwardOffset<Vector<Int64, Int64>>.self)
|
||||
try _v.visit(field: VTOFFSET.indicesBuffer.p, fieldName: "indicesBuffer", required: true, type: org_apache_arrow_flatbuf_Buffer.self)
|
||||
try _v.visit(field: VTOFFSET.isCanonical.p, fieldName: "isCanonical", required: false, type: Bool.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Compressed Sparse format, that is matrix-specific.
|
||||
public struct org_apache_arrow_flatbuf_SparseMatrixIndexCSX: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsSparseMatrixIndexCSX(bb: ByteBuffer) -> org_apache_arrow_flatbuf_SparseMatrixIndexCSX { return org_apache_arrow_flatbuf_SparseMatrixIndexCSX(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case compressedAxis = 4
|
||||
case indptrType = 6
|
||||
case indptrBuffer = 8
|
||||
case indicesType = 10
|
||||
case indicesBuffer = 12
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
/// Which axis, row or column, is compressed
|
||||
public var compressedAxis: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis { let o = _accessor.offset(VTOFFSET.compressedAxis.v); return o == 0 ? .row : org_apache_arrow_flatbuf_SparseMatrixCompressedAxis(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .row }
|
||||
/// The type of values in indptrBuffer
|
||||
public var indptrType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indptrType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
/// indptrBuffer stores the location and size of indptr array that
|
||||
/// represents the range of the rows.
|
||||
/// The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data.
|
||||
/// The length of this array is 1 + (the number of rows), and the type
|
||||
/// of index value is long.
|
||||
///
|
||||
/// For example, let X be the following 6x4 matrix:
|
||||
/// ```text
|
||||
/// X := [[0, 1, 2, 0],
|
||||
/// [0, 0, 3, 0],
|
||||
/// [0, 4, 0, 5],
|
||||
/// [0, 0, 0, 0],
|
||||
/// [6, 0, 7, 8],
|
||||
/// [0, 9, 0, 0]].
|
||||
/// ```
|
||||
/// The array of non-zero values in X is:
|
||||
/// ```text
|
||||
/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9].
|
||||
/// ```
|
||||
/// And the indptr of X is:
|
||||
/// ```text
|
||||
/// indptr(X) = [0, 2, 3, 5, 5, 8, 10].
|
||||
/// ```
|
||||
public var indptrBuffer: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.indptrBuffer.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) }
|
||||
public var mutableIndptrBuffer: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.indptrBuffer.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) }
|
||||
/// The type of values in indicesBuffer
|
||||
public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
/// indicesBuffer stores the location and size of the array that
|
||||
/// contains the column indices of the corresponding non-zero values.
|
||||
/// The type of index value is long.
|
||||
///
|
||||
/// For example, the indices of the above X is:
|
||||
/// ```text
|
||||
/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1].
|
||||
/// ```
|
||||
/// Note that the indices are sorted in lexicographical order for each row.
|
||||
public var indicesBuffer: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) }
|
||||
public var mutableIndicesBuffer: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) }
|
||||
public static func startSparseMatrixIndexCSX(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) }
|
||||
public static func add(compressedAxis: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis, _ fbb: inout FlatBufferBuilder) { fbb.add(element: compressedAxis.rawValue, def: 0, at: VTOFFSET.compressedAxis.p) }
|
||||
public static func add(indptrType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrType, at: VTOFFSET.indptrType.p) }
|
||||
public static func add(indptrBuffer: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let indptrBuffer = indptrBuffer else { return }; fbb.create(struct: indptrBuffer, position: VTOFFSET.indptrBuffer.p) }
|
||||
public static func add(indicesType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesType, at: VTOFFSET.indicesType.p) }
|
||||
public static func add(indicesBuffer: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let indicesBuffer = indicesBuffer else { return }; fbb.create(struct: indicesBuffer, position: VTOFFSET.indicesBuffer.p) }
|
||||
public static func endSparseMatrixIndexCSX(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [6, 8, 10, 12]); return end }
|
||||
public static func createSparseMatrixIndexCSX(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
compressedAxis: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis = .row,
|
||||
indptrTypeOffset indptrType: Offset,
|
||||
indptrBuffer: org_apache_arrow_flatbuf_Buffer,
|
||||
indicesTypeOffset indicesType: Offset,
|
||||
indicesBuffer: org_apache_arrow_flatbuf_Buffer
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_SparseMatrixIndexCSX.startSparseMatrixIndexCSX(&fbb)
|
||||
org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(compressedAxis: compressedAxis, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indptrType: indptrType, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indptrBuffer: indptrBuffer, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indicesType: indicesType, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indicesBuffer: indicesBuffer, &fbb)
|
||||
return org_apache_arrow_flatbuf_SparseMatrixIndexCSX.endSparseMatrixIndexCSX(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.compressedAxis.p, fieldName: "compressedAxis", required: false, type: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis.self)
|
||||
try _v.visit(field: VTOFFSET.indptrType.p, fieldName: "indptrType", required: true, type: ForwardOffset<org_apache_arrow_flatbuf_Int>.self)
|
||||
try _v.visit(field: VTOFFSET.indptrBuffer.p, fieldName: "indptrBuffer", required: true, type: org_apache_arrow_flatbuf_Buffer.self)
|
||||
try _v.visit(field: VTOFFSET.indicesType.p, fieldName: "indicesType", required: true, type: ForwardOffset<org_apache_arrow_flatbuf_Int>.self)
|
||||
try _v.visit(field: VTOFFSET.indicesBuffer.p, fieldName: "indicesBuffer", required: true, type: org_apache_arrow_flatbuf_Buffer.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Compressed Sparse Fiber (CSF) sparse tensor index.
|
||||
public struct org_apache_arrow_flatbuf_SparseTensorIndexCSF: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsSparseTensorIndexCSF(bb: ByteBuffer) -> org_apache_arrow_flatbuf_SparseTensorIndexCSF { return org_apache_arrow_flatbuf_SparseTensorIndexCSF(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case indptrType = 4
|
||||
case indptrBuffers = 6
|
||||
case indicesType = 8
|
||||
case indicesBuffers = 10
|
||||
case axisOrder = 12
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
/// CSF is a generalization of compressed sparse row (CSR) index.
|
||||
/// See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf)
|
||||
///
|
||||
/// CSF index recursively compresses each dimension of a tensor into a set
|
||||
/// of prefix trees. Each path from a root to leaf forms one tensor
|
||||
/// non-zero index. CSF is implemented with two arrays of buffers and one
|
||||
/// arrays of integers.
|
||||
///
|
||||
/// For example, let X be a 2x3x4x5 tensor and let it have the following
|
||||
/// 8 non-zero values:
|
||||
/// ```text
|
||||
/// X[0, 0, 0, 1] := 1
|
||||
/// X[0, 0, 0, 2] := 2
|
||||
/// X[0, 1, 0, 0] := 3
|
||||
/// X[0, 1, 0, 2] := 4
|
||||
/// X[0, 1, 1, 0] := 5
|
||||
/// X[1, 1, 1, 0] := 6
|
||||
/// X[1, 1, 1, 1] := 7
|
||||
/// X[1, 1, 1, 2] := 8
|
||||
/// ```
|
||||
/// As a prefix tree this would be represented as:
|
||||
/// ```text
|
||||
/// 0 1
|
||||
/// / \ |
|
||||
/// 0 1 1
|
||||
/// / / \ |
|
||||
/// 0 0 1 1
|
||||
/// /| /| | /| |
|
||||
/// 1 2 0 2 0 0 1 2
|
||||
/// ```
|
||||
/// The type of values in indptrBuffers
|
||||
public var indptrType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indptrType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
/// indptrBuffers stores the sparsity structure.
|
||||
/// Each two consecutive dimensions in a tensor correspond to a buffer in
|
||||
/// indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]`
|
||||
/// and `indptrBuffers[dim][i + 1]` signify a range of nodes in
|
||||
/// `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node.
|
||||
///
|
||||
/// For example, the indptrBuffers for the above X is:
|
||||
/// ```text
|
||||
/// indptrBuffer(X) = [
|
||||
/// [0, 2, 3],
|
||||
/// [0, 1, 3, 4],
|
||||
/// [0, 2, 4, 5, 8]
|
||||
/// ].
|
||||
/// ```
|
||||
public var hasIndptrBuffers: Bool { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? false : true }
|
||||
public var indptrBuffersCount: Int32 { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func indptrBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) }
|
||||
public func mutableIndptrBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) }
|
||||
/// The type of values in indicesBuffers
|
||||
public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
|
||||
/// indicesBuffers stores values of nodes.
|
||||
/// Each tensor dimension corresponds to a buffer in indicesBuffers.
|
||||
/// For example, the indicesBuffers for the above X is:
|
||||
/// ```text
|
||||
/// indicesBuffer(X) = [
|
||||
/// [0, 1],
|
||||
/// [0, 1, 1],
|
||||
/// [0, 0, 1, 1],
|
||||
/// [1, 2, 0, 2, 0, 0, 1, 2]
|
||||
/// ].
|
||||
/// ```
|
||||
public var hasIndicesBuffers: Bool { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? false : true }
|
||||
public var indicesBuffersCount: Int32 { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func indicesBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) }
|
||||
public func mutableIndicesBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) }
|
||||
/// axisOrder stores the sequence in which dimensions were traversed to
|
||||
/// produce the prefix tree.
|
||||
/// For example, the axisOrder for the above X is:
|
||||
/// ```text
|
||||
/// axisOrder(X) = [0, 1, 2, 3].
|
||||
/// ```
|
||||
public var hasAxisOrder: Bool { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? false : true }
|
||||
public var axisOrderCount: Int32 { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func axisOrder(at index: Int32) -> Int32 { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? 0 : _accessor.directRead(of: Int32.self, offset: _accessor.vector(at: o) + index * 4) }
|
||||
public var axisOrder: [Int32] { return _accessor.getVector(at: VTOFFSET.axisOrder.v) ?? [] }
|
||||
public static func startSparseTensorIndexCSF(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) }
|
||||
public static func add(indptrType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrType, at: VTOFFSET.indptrType.p) }
|
||||
public static func addVectorOf(indptrBuffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrBuffers, at: VTOFFSET.indptrBuffers.p) }
|
||||
public static func startVectorOfIndptrBuffers(_ size: Int, in builder: inout FlatBufferBuilder) {
|
||||
builder.startVector(size * MemoryLayout<org_apache_arrow_flatbuf_Buffer>.size, elementSize: MemoryLayout<org_apache_arrow_flatbuf_Buffer>.alignment)
|
||||
}
|
||||
public static func add(indicesType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesType, at: VTOFFSET.indicesType.p) }
|
||||
public static func addVectorOf(indicesBuffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesBuffers, at: VTOFFSET.indicesBuffers.p) }
|
||||
public static func startVectorOfIndicesBuffers(_ size: Int, in builder: inout FlatBufferBuilder) {
|
||||
builder.startVector(size * MemoryLayout<org_apache_arrow_flatbuf_Buffer>.size, elementSize: MemoryLayout<org_apache_arrow_flatbuf_Buffer>.alignment)
|
||||
}
|
||||
public static func addVectorOf(axisOrder: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: axisOrder, at: VTOFFSET.axisOrder.p) }
|
||||
public static func endSparseTensorIndexCSF(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [4, 6, 8, 10, 12]); return end }
|
||||
public static func createSparseTensorIndexCSF(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
indptrTypeOffset indptrType: Offset,
|
||||
indptrBuffersVectorOffset indptrBuffers: Offset,
|
||||
indicesTypeOffset indicesType: Offset,
|
||||
indicesBuffersVectorOffset indicesBuffers: Offset,
|
||||
axisOrderVectorOffset axisOrder: Offset
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_SparseTensorIndexCSF.startSparseTensorIndexCSF(&fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCSF.add(indptrType: indptrType, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCSF.addVectorOf(indptrBuffers: indptrBuffers, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCSF.add(indicesType: indicesType, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCSF.addVectorOf(indicesBuffers: indicesBuffers, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensorIndexCSF.addVectorOf(axisOrder: axisOrder, &fbb)
|
||||
return org_apache_arrow_flatbuf_SparseTensorIndexCSF.endSparseTensorIndexCSF(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.indptrType.p, fieldName: "indptrType", required: true, type: ForwardOffset<org_apache_arrow_flatbuf_Int>.self)
|
||||
try _v.visit(field: VTOFFSET.indptrBuffers.p, fieldName: "indptrBuffers", required: true, type: ForwardOffset<Vector<org_apache_arrow_flatbuf_Buffer, org_apache_arrow_flatbuf_Buffer>>.self)
|
||||
try _v.visit(field: VTOFFSET.indicesType.p, fieldName: "indicesType", required: true, type: ForwardOffset<org_apache_arrow_flatbuf_Int>.self)
|
||||
try _v.visit(field: VTOFFSET.indicesBuffers.p, fieldName: "indicesBuffers", required: true, type: ForwardOffset<Vector<org_apache_arrow_flatbuf_Buffer, org_apache_arrow_flatbuf_Buffer>>.self)
|
||||
try _v.visit(field: VTOFFSET.axisOrder.p, fieldName: "axisOrder", required: true, type: ForwardOffset<Vector<Int32, Int32>>.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
public struct org_apache_arrow_flatbuf_SparseTensor: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsSparseTensor(bb: ByteBuffer) -> org_apache_arrow_flatbuf_SparseTensor { return org_apache_arrow_flatbuf_SparseTensor(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case typeType = 4
|
||||
case type = 6
|
||||
case shape = 8
|
||||
case nonZeroLength = 10
|
||||
case sparseIndexType = 12
|
||||
case sparseIndex = 14
|
||||
case data = 16
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
public var typeType: org_apache_arrow_flatbuf_Type_ { let o = _accessor.offset(VTOFFSET.typeType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_Type_(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ }
|
||||
/// The type of data contained in a value cell.
|
||||
/// Currently only fixed-width value types are supported,
|
||||
/// no strings or nested types.
|
||||
public func type<T: FlatbuffersInitializable>(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.type.v); return _accessor.union(o) }
|
||||
/// The dimensions of the tensor, optionally named.
|
||||
public var hasShape: Bool { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? false : true }
|
||||
public var shapeCount: Int32 { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func shape(at index: Int32) -> org_apache_arrow_flatbuf_TensorDim? { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? nil : org_apache_arrow_flatbuf_TensorDim(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) }
|
||||
/// The number of non-zero values in a sparse tensor.
|
||||
public var nonZeroLength: Int64 { let o = _accessor.offset(VTOFFSET.nonZeroLength.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) }
|
||||
public var sparseIndexType: org_apache_arrow_flatbuf_SparseTensorIndex { let o = _accessor.offset(VTOFFSET.sparseIndexType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_SparseTensorIndex(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ }
|
||||
/// Sparse tensor index
|
||||
public func sparseIndex<T: FlatbuffersInitializable>(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.sparseIndex.v); return _accessor.union(o) }
|
||||
/// The location and size of the tensor's data
|
||||
public var data: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.data.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) }
|
||||
public var mutableData: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.data.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) }
|
||||
public static func startSparseTensor(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 7) }
|
||||
public static func add(typeType: org_apache_arrow_flatbuf_Type_, _ fbb: inout FlatBufferBuilder) { fbb.add(element: typeType.rawValue, def: 0, at: VTOFFSET.typeType.p) }
|
||||
public static func add(type: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: type, at: VTOFFSET.type.p) }
|
||||
public static func addVectorOf(shape: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: shape, at: VTOFFSET.shape.p) }
|
||||
public static func add(nonZeroLength: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: nonZeroLength, def: 0, at: VTOFFSET.nonZeroLength.p) }
|
||||
public static func add(sparseIndexType: org_apache_arrow_flatbuf_SparseTensorIndex, _ fbb: inout FlatBufferBuilder) { fbb.add(element: sparseIndexType.rawValue, def: 0, at: VTOFFSET.sparseIndexType.p) }
|
||||
public static func add(sparseIndex: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: sparseIndex, at: VTOFFSET.sparseIndex.p) }
|
||||
public static func add(data: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let data = data else { return }; fbb.create(struct: data, position: VTOFFSET.data.p) }
|
||||
public static func endSparseTensor(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [6, 8, 14, 16]); return end }
|
||||
public static func createSparseTensor(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
typeType: org_apache_arrow_flatbuf_Type_ = .none_,
|
||||
typeOffset type: Offset,
|
||||
shapeVectorOffset shape: Offset,
|
||||
nonZeroLength: Int64 = 0,
|
||||
sparseIndexType: org_apache_arrow_flatbuf_SparseTensorIndex = .none_,
|
||||
sparseIndexOffset sparseIndex: Offset,
|
||||
data: org_apache_arrow_flatbuf_Buffer
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_SparseTensor.startSparseTensor(&fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensor.add(typeType: typeType, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensor.add(type: type, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensor.addVectorOf(shape: shape, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensor.add(nonZeroLength: nonZeroLength, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensor.add(sparseIndexType: sparseIndexType, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensor.add(sparseIndex: sparseIndex, &fbb)
|
||||
org_apache_arrow_flatbuf_SparseTensor.add(data: data, &fbb)
|
||||
return org_apache_arrow_flatbuf_SparseTensor.endSparseTensor(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(unionKey: VTOFFSET.typeType.p, unionField: VTOFFSET.type.p, unionKeyName: "typeType", fieldName: "type", required: true, completion: { (verifier, key: org_apache_arrow_flatbuf_Type_, pos) in
|
||||
switch key {
|
||||
case .none_:
|
||||
break // NOTE - SWIFT doesnt support none
|
||||
case .null:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Null>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Null.self)
|
||||
case .int:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Int>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Int.self)
|
||||
case .floatingpoint:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_FloatingPoint>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FloatingPoint.self)
|
||||
case .binary:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Binary>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Binary.self)
|
||||
case .utf8:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Utf8>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8.self)
|
||||
case .bool:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Bool>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Bool.self)
|
||||
case .decimal:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Decimal>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Decimal.self)
|
||||
case .date:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Date>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Date.self)
|
||||
case .time:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Time>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Time.self)
|
||||
case .timestamp:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Timestamp>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Timestamp.self)
|
||||
case .interval:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Interval>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Interval.self)
|
||||
case .list:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_List>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_List.self)
|
||||
case .struct_:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Struct_>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Struct_.self)
|
||||
case .union:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Union>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Union.self)
|
||||
case .fixedsizebinary:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_FixedSizeBinary>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeBinary.self)
|
||||
case .fixedsizelist:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_FixedSizeList>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeList.self)
|
||||
case .map:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Map>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Map.self)
|
||||
case .duration:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Duration>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Duration.self)
|
||||
case .largebinary:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_LargeBinary>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeBinary.self)
|
||||
case .largeutf8:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_LargeUtf8>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeUtf8.self)
|
||||
case .largelist:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_LargeList>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeList.self)
|
||||
case .runendencoded:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_RunEndEncoded>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RunEndEncoded.self)
|
||||
}
|
||||
})
|
||||
try _v.visit(field: VTOFFSET.shape.p, fieldName: "shape", required: true, type: ForwardOffset<Vector<ForwardOffset<org_apache_arrow_flatbuf_TensorDim>, org_apache_arrow_flatbuf_TensorDim>>.self)
|
||||
try _v.visit(field: VTOFFSET.nonZeroLength.p, fieldName: "nonZeroLength", required: false, type: Int64.self)
|
||||
try _v.visit(unionKey: VTOFFSET.sparseIndexType.p, unionField: VTOFFSET.sparseIndex.p, unionKeyName: "sparseIndexType", fieldName: "sparseIndex", required: true, completion: { (verifier, key: org_apache_arrow_flatbuf_SparseTensorIndex, pos) in
|
||||
switch key {
|
||||
case .none_:
|
||||
break // NOTE - SWIFT doesnt support none
|
||||
case .sparsetensorindexcoo:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_SparseTensorIndexCOO>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensorIndexCOO.self)
|
||||
case .sparsematrixindexcsx:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_SparseMatrixIndexCSX>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseMatrixIndexCSX.self)
|
||||
case .sparsetensorindexcsf:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_SparseTensorIndexCSF>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensorIndexCSF.self)
|
||||
}
|
||||
})
|
||||
try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: true, type: org_apache_arrow_flatbuf_Buffer.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
// automatically generated by the FlatBuffers compiler, do not modify
|
||||
// swiftlint:disable all
|
||||
// swiftformat:disable all
|
||||
|
||||
import FlatBuffers
|
||||
|
||||
/// ----------------------------------------------------------------------
|
||||
/// Data structures for dense tensors
|
||||
/// Shape data for a single axis in a tensor
|
||||
public struct org_apache_arrow_flatbuf_TensorDim: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsTensorDim(bb: ByteBuffer) -> org_apache_arrow_flatbuf_TensorDim { return org_apache_arrow_flatbuf_TensorDim(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case size = 4
|
||||
case name = 6
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
/// Length of dimension
|
||||
public var size: Int64 { let o = _accessor.offset(VTOFFSET.size.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) }
|
||||
/// Name of the dimension, optional
|
||||
public var name: String? { let o = _accessor.offset(VTOFFSET.name.v); return o == 0 ? nil : _accessor.string(at: o) }
|
||||
public var nameSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.name.v) }
|
||||
public static func startTensorDim(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) }
|
||||
public static func add(size: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: size, def: 0, at: VTOFFSET.size.p) }
|
||||
public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: name, at: VTOFFSET.name.p) }
|
||||
public static func endTensorDim(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
|
||||
public static func createTensorDim(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
size: Int64 = 0,
|
||||
nameOffset name: Offset = Offset()
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_TensorDim.startTensorDim(&fbb)
|
||||
org_apache_arrow_flatbuf_TensorDim.add(size: size, &fbb)
|
||||
org_apache_arrow_flatbuf_TensorDim.add(name: name, &fbb)
|
||||
return org_apache_arrow_flatbuf_TensorDim.endTensorDim(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(field: VTOFFSET.size.p, fieldName: "size", required: false, type: Int64.self)
|
||||
try _v.visit(field: VTOFFSET.name.p, fieldName: "name", required: false, type: ForwardOffset<String>.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
public struct org_apache_arrow_flatbuf_Tensor: FlatBufferObject, Verifiable {
|
||||
|
||||
static func validateVersion() { FlatBuffersVersion_23_1_4() }
|
||||
public var __buffer: ByteBuffer! { return _accessor.bb }
|
||||
private var _accessor: Table
|
||||
|
||||
public static func getRootAsTensor(bb: ByteBuffer) -> org_apache_arrow_flatbuf_Tensor { return org_apache_arrow_flatbuf_Tensor(Table(bb: bb, position: Int32(bb.read(def: UOffset.self, position: bb.reader)) + Int32(bb.reader))) }
|
||||
|
||||
private init(_ t: Table) { _accessor = t }
|
||||
public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) }
|
||||
|
||||
private enum VTOFFSET: VOffset {
|
||||
case typeType = 4
|
||||
case type = 6
|
||||
case shape = 8
|
||||
case strides = 10
|
||||
case data = 12
|
||||
var v: Int32 { Int32(self.rawValue) }
|
||||
var p: VOffset { self.rawValue }
|
||||
}
|
||||
|
||||
public var typeType: org_apache_arrow_flatbuf_Type_ { let o = _accessor.offset(VTOFFSET.typeType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_Type_(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ }
|
||||
/// The type of data contained in a value cell. Currently only fixed-width
|
||||
/// value types are supported, no strings or nested types
|
||||
public func type<T: FlatbuffersInitializable>(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.type.v); return _accessor.union(o) }
|
||||
/// The dimensions of the tensor, optionally named
|
||||
public var hasShape: Bool { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? false : true }
|
||||
public var shapeCount: Int32 { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func shape(at index: Int32) -> org_apache_arrow_flatbuf_TensorDim? { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? nil : org_apache_arrow_flatbuf_TensorDim(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) }
|
||||
/// Non-negative byte offsets to advance one value cell along each dimension
|
||||
/// If omitted, default to row-major order (C-like).
|
||||
public var hasStrides: Bool { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? false : true }
|
||||
public var stridesCount: Int32 { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? 0 : _accessor.vector(count: o) }
|
||||
public func strides(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) }
|
||||
public var strides: [Int64] { return _accessor.getVector(at: VTOFFSET.strides.v) ?? [] }
|
||||
/// The location and size of the tensor's data
|
||||
public var data: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.data.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) }
|
||||
public var mutableData: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.data.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) }
|
||||
public static func startTensor(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) }
|
||||
public static func add(typeType: org_apache_arrow_flatbuf_Type_, _ fbb: inout FlatBufferBuilder) { fbb.add(element: typeType.rawValue, def: 0, at: VTOFFSET.typeType.p) }
|
||||
public static func add(type: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: type, at: VTOFFSET.type.p) }
|
||||
public static func addVectorOf(shape: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: shape, at: VTOFFSET.shape.p) }
|
||||
public static func addVectorOf(strides: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: strides, at: VTOFFSET.strides.p) }
|
||||
public static func add(data: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let data = data else { return }; fbb.create(struct: data, position: VTOFFSET.data.p) }
|
||||
public static func endTensor(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [6, 8, 12]); return end }
|
||||
public static func createTensor(
|
||||
_ fbb: inout FlatBufferBuilder,
|
||||
typeType: org_apache_arrow_flatbuf_Type_ = .none_,
|
||||
typeOffset type: Offset,
|
||||
shapeVectorOffset shape: Offset,
|
||||
stridesVectorOffset strides: Offset = Offset(),
|
||||
data: org_apache_arrow_flatbuf_Buffer
|
||||
) -> Offset {
|
||||
let __start = org_apache_arrow_flatbuf_Tensor.startTensor(&fbb)
|
||||
org_apache_arrow_flatbuf_Tensor.add(typeType: typeType, &fbb)
|
||||
org_apache_arrow_flatbuf_Tensor.add(type: type, &fbb)
|
||||
org_apache_arrow_flatbuf_Tensor.addVectorOf(shape: shape, &fbb)
|
||||
org_apache_arrow_flatbuf_Tensor.addVectorOf(strides: strides, &fbb)
|
||||
org_apache_arrow_flatbuf_Tensor.add(data: data, &fbb)
|
||||
return org_apache_arrow_flatbuf_Tensor.endTensor(&fbb, start: __start)
|
||||
}
|
||||
|
||||
public static func verify<T>(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable {
|
||||
var _v = try verifier.visitTable(at: position)
|
||||
try _v.visit(unionKey: VTOFFSET.typeType.p, unionField: VTOFFSET.type.p, unionKeyName: "typeType", fieldName: "type", required: true, completion: { (verifier, key: org_apache_arrow_flatbuf_Type_, pos) in
|
||||
switch key {
|
||||
case .none_:
|
||||
break // NOTE - SWIFT doesnt support none
|
||||
case .null:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Null>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Null.self)
|
||||
case .int:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Int>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Int.self)
|
||||
case .floatingpoint:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_FloatingPoint>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FloatingPoint.self)
|
||||
case .binary:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Binary>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Binary.self)
|
||||
case .utf8:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Utf8>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8.self)
|
||||
case .bool:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Bool>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Bool.self)
|
||||
case .decimal:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Decimal>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Decimal.self)
|
||||
case .date:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Date>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Date.self)
|
||||
case .time:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Time>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Time.self)
|
||||
case .timestamp:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Timestamp>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Timestamp.self)
|
||||
case .interval:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Interval>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Interval.self)
|
||||
case .list:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_List>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_List.self)
|
||||
case .struct_:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Struct_>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Struct_.self)
|
||||
case .union:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Union>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Union.self)
|
||||
case .fixedsizebinary:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_FixedSizeBinary>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeBinary.self)
|
||||
case .fixedsizelist:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_FixedSizeList>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeList.self)
|
||||
case .map:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Map>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Map.self)
|
||||
case .duration:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_Duration>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Duration.self)
|
||||
case .largebinary:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_LargeBinary>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeBinary.self)
|
||||
case .largeutf8:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_LargeUtf8>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeUtf8.self)
|
||||
case .largelist:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_LargeList>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeList.self)
|
||||
case .runendencoded:
|
||||
try ForwardOffset<org_apache_arrow_flatbuf_RunEndEncoded>.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RunEndEncoded.self)
|
||||
}
|
||||
})
|
||||
try _v.visit(field: VTOFFSET.shape.p, fieldName: "shape", required: true, type: ForwardOffset<Vector<ForwardOffset<org_apache_arrow_flatbuf_TensorDim>, org_apache_arrow_flatbuf_TensorDim>>.self)
|
||||
try _v.visit(field: VTOFFSET.strides.p, fieldName: "strides", required: false, type: ForwardOffset<Vector<Int64, Int64>>.self)
|
||||
try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: true, type: org_apache_arrow_flatbuf_Buffer.self)
|
||||
_v.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "include/ArrowCData.h"
|
||||
|
||||
void ArrowSwiftClearReleaseSchema(struct ArrowSchema* arrowSchema) {
|
||||
if(arrowSchema) {
|
||||
arrowSchema->release = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void ArrowSwiftClearReleaseArray(struct ArrowArray* arrowArray) {
|
||||
if(arrowArray) {
|
||||
arrowArray->release = NULL;
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#ifndef ARROW_C_DATA_INTERFACE
|
||||
#define ARROW_C_DATA_INTERFACE
|
||||
|
||||
#define ARROW_FLAG_DICTIONARY_ORDERED 1
|
||||
#define ARROW_FLAG_NULLABLE 2
|
||||
#define ARROW_FLAG_MAP_KEYS_SORTED 4
|
||||
|
||||
#include <stdint.h> // For int64_t
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct ArrowSchema {
|
||||
// Array type description
|
||||
const char* format;
|
||||
const char* name;
|
||||
const char* metadata;
|
||||
int64_t flags;
|
||||
int64_t n_children;
|
||||
struct ArrowSchema** children;
|
||||
struct ArrowSchema* dictionary;
|
||||
|
||||
// Release callback
|
||||
void (*release)(struct ArrowSchema*);
|
||||
// Opaque producer-specific data
|
||||
void* private_data;
|
||||
};
|
||||
|
||||
struct ArrowArray {
|
||||
// Array data description
|
||||
int64_t length;
|
||||
int64_t null_count;
|
||||
int64_t offset;
|
||||
int64_t n_buffers;
|
||||
int64_t n_children;
|
||||
const void** buffers;
|
||||
struct ArrowArray** children;
|
||||
struct ArrowArray* dictionary;
|
||||
|
||||
// Release callback
|
||||
void (*release)(struct ArrowArray*);
|
||||
// Opaque producer-specific data
|
||||
void* private_data;
|
||||
};
|
||||
|
||||
// Not able to set the release on the schema
|
||||
// to NULL in Swift. nil in Swift is not
|
||||
// equivalent to NULL.
|
||||
void ArrowSwiftClearReleaseSchema(struct ArrowSchema*);
|
||||
|
||||
// Not able to set the release on the array
|
||||
// to NULL in Swift. nil in Swift is not
|
||||
// equivalent to NULL.
|
||||
void ArrowSwiftClearReleaseArray(struct ArrowArray*);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // ARROW_C_DATA_INTERFACE
|
||||
@@ -1,85 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import XCTest
|
||||
@testable import Arrow
|
||||
|
||||
final class ArrayBuilderTests: XCTestCase {
|
||||
func testIsValidTypeForBuilder() throws {
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt8.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Int16.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Int32.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Int64.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt8.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt16.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt32.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt64.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Float.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Double.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Date.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Bool.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Int8?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Int16?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Int32?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Int64?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt8?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt16?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt32?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(UInt64?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Float?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Double?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Date?.self))
|
||||
XCTAssertTrue(ArrowArrayBuilders.isValidBuilderType(Bool?.self))
|
||||
|
||||
XCTAssertFalse(ArrowArrayBuilders.isValidBuilderType(Int.self))
|
||||
XCTAssertFalse(ArrowArrayBuilders.isValidBuilderType(UInt.self))
|
||||
XCTAssertFalse(ArrowArrayBuilders.isValidBuilderType(Int?.self))
|
||||
XCTAssertFalse(ArrowArrayBuilders.isValidBuilderType(UInt?.self))
|
||||
}
|
||||
|
||||
func testLoadArrayBuilders() throws {
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int8.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int16.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int32.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int64.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt8.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt16.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt32.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt64.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Float.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Double.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Date.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Bool.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int8?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int16?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int32?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Int64?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt8?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt16?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt32?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(UInt64?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Float?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Double?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Date?.self))
|
||||
XCTAssertNotNil(try ArrowArrayBuilders.loadBuilder(Bool?.self))
|
||||
|
||||
XCTAssertThrowsError(try ArrowArrayBuilders.loadBuilder(Int.self))
|
||||
XCTAssertThrowsError(try ArrowArrayBuilders.loadBuilder(UInt.self))
|
||||
XCTAssertThrowsError(try ArrowArrayBuilders.loadBuilder(Int?.self))
|
||||
XCTAssertThrowsError(try ArrowArrayBuilders.loadBuilder(UInt?.self))
|
||||
}
|
||||
}
|
||||
@@ -1,371 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import XCTest
|
||||
@testable import Arrow
|
||||
|
||||
final class ArrayTests: XCTestCase { // swiftlint:disable:this type_body_length
|
||||
func testPrimitiveArray() throws {
|
||||
// This is an example of a functional test case.
|
||||
// Use XCTAssert and related functions to verify your tests produce the correct
|
||||
// results.
|
||||
let arrayBuilder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
for index in 0..<100 {
|
||||
arrayBuilder.append(UInt8(index))
|
||||
}
|
||||
|
||||
XCTAssertEqual(arrayBuilder.nullCount, 0)
|
||||
arrayBuilder.append(nil)
|
||||
XCTAssertEqual(arrayBuilder.length, 101)
|
||||
XCTAssertEqual(arrayBuilder.capacity, 136)
|
||||
XCTAssertEqual(arrayBuilder.nullCount, 1)
|
||||
let array = try arrayBuilder.finish()
|
||||
XCTAssertEqual(array.length, 101)
|
||||
XCTAssertEqual(array[1]!, 1)
|
||||
XCTAssertEqual(array[10]!, 10)
|
||||
XCTAssertEqual(try array.isNull(100), true)
|
||||
|
||||
let doubleBuilder: NumberArrayBuilder<Double> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
doubleBuilder.append(14)
|
||||
doubleBuilder.append(40.4)
|
||||
XCTAssertEqual(doubleBuilder.nullCount, 0)
|
||||
XCTAssertEqual(doubleBuilder.length, 2)
|
||||
XCTAssertEqual(doubleBuilder.capacity, 264)
|
||||
let doubleArray = try doubleBuilder.finish()
|
||||
XCTAssertEqual(doubleArray.length, 2)
|
||||
XCTAssertEqual(doubleArray[0]!, 14)
|
||||
XCTAssertEqual(doubleArray[1]!, 40.4)
|
||||
}
|
||||
|
||||
func testStringArray() throws {
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
for index in 0..<100 {
|
||||
if index % 10 == 9 {
|
||||
stringBuilder.append(nil)
|
||||
} else {
|
||||
stringBuilder.append("test" + String(index))
|
||||
}
|
||||
}
|
||||
|
||||
XCTAssertEqual(stringBuilder.nullCount, 10)
|
||||
XCTAssertEqual(stringBuilder.length, 100)
|
||||
XCTAssertEqual(stringBuilder.capacity, 648)
|
||||
let stringArray = try stringBuilder.finish()
|
||||
XCTAssertEqual(stringArray.length, 100)
|
||||
for index in 0..<stringArray.length {
|
||||
if index % 10 == 9 {
|
||||
XCTAssertEqual(try stringArray.isNull(index), true)
|
||||
} else {
|
||||
XCTAssertEqual(stringArray[index]!, "test" + String(index))
|
||||
}
|
||||
}
|
||||
|
||||
XCTAssertEqual(stringArray[1]!, "test1")
|
||||
XCTAssertEqual(stringArray[0]!, "test0")
|
||||
}
|
||||
|
||||
func testBoolArray() throws {
|
||||
let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder()
|
||||
boolBuilder.append(true)
|
||||
boolBuilder.append(nil)
|
||||
boolBuilder.append(false)
|
||||
boolBuilder.append(false)
|
||||
XCTAssertEqual(boolBuilder.nullCount, 1)
|
||||
XCTAssertEqual(boolBuilder.length, 4)
|
||||
XCTAssertEqual(boolBuilder.capacity, 72)
|
||||
let boolArray = try boolBuilder.finish()
|
||||
XCTAssertEqual(boolArray.length, 4)
|
||||
XCTAssertEqual(boolArray[1], nil)
|
||||
XCTAssertEqual(boolArray[0]!, true)
|
||||
XCTAssertEqual(boolArray[2]!, false)
|
||||
}
|
||||
|
||||
func testDate32Array() throws {
|
||||
let date32Builder: Date32ArrayBuilder = try ArrowArrayBuilders.loadDate32ArrayBuilder()
|
||||
let date2 = Date(timeIntervalSinceReferenceDate: 86400 * 1)
|
||||
let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
date32Builder.append(nil)
|
||||
XCTAssertEqual(date32Builder.nullCount, 1)
|
||||
XCTAssertEqual(date32Builder.length, 3)
|
||||
XCTAssertEqual(date32Builder.capacity, 136)
|
||||
let date32Array = try date32Builder.finish()
|
||||
XCTAssertEqual(date32Array.length, 3)
|
||||
XCTAssertEqual(date32Array[1], date2)
|
||||
let adjustedDate1 = Date(timeIntervalSince1970: date1.timeIntervalSince1970 - 352)
|
||||
XCTAssertEqual(date32Array[0]!, adjustedDate1)
|
||||
}
|
||||
|
||||
func testDate64Array() throws {
|
||||
let date64Builder: Date64ArrayBuilder = try ArrowArrayBuilders.loadDate64ArrayBuilder()
|
||||
let date2 = Date(timeIntervalSinceReferenceDate: 86400 * 1)
|
||||
let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352)
|
||||
date64Builder.append(date1)
|
||||
date64Builder.append(date2)
|
||||
date64Builder.append(nil)
|
||||
XCTAssertEqual(date64Builder.nullCount, 1)
|
||||
XCTAssertEqual(date64Builder.length, 3)
|
||||
XCTAssertEqual(date64Builder.capacity, 264)
|
||||
let date64Array = try date64Builder.finish()
|
||||
XCTAssertEqual(date64Array.length, 3)
|
||||
XCTAssertEqual(date64Array[1], date2)
|
||||
XCTAssertEqual(date64Array[0]!, date1)
|
||||
}
|
||||
|
||||
func testBinaryArray() throws {
|
||||
let binaryBuilder = try ArrowArrayBuilders.loadBinaryArrayBuilder()
|
||||
for index in 0..<100 {
|
||||
if index % 10 == 9 {
|
||||
binaryBuilder.append(nil)
|
||||
} else {
|
||||
binaryBuilder.append(("test" + String(index)).data(using: .utf8))
|
||||
}
|
||||
}
|
||||
|
||||
XCTAssertEqual(binaryBuilder.nullCount, 10)
|
||||
XCTAssertEqual(binaryBuilder.length, 100)
|
||||
XCTAssertEqual(binaryBuilder.capacity, 648)
|
||||
let binaryArray = try binaryBuilder.finish()
|
||||
XCTAssertEqual(binaryArray.length, 100)
|
||||
for index in 0..<binaryArray.length {
|
||||
if index % 10 == 9 {
|
||||
XCTAssertEqual(try binaryArray.isNull(index), true)
|
||||
} else {
|
||||
let stringData = String(bytes: binaryArray[index]!, encoding: .utf8)
|
||||
XCTAssertEqual(stringData, "test" + String(index))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testTime32Array() throws {
|
||||
let milliBuilder = try ArrowArrayBuilders.loadTime32ArrayBuilder(.milliseconds)
|
||||
milliBuilder.append(100)
|
||||
milliBuilder.append(1000000)
|
||||
milliBuilder.append(nil)
|
||||
XCTAssertEqual(milliBuilder.nullCount, 1)
|
||||
XCTAssertEqual(milliBuilder.length, 3)
|
||||
XCTAssertEqual(milliBuilder.capacity, 136)
|
||||
let milliArray = try milliBuilder.finish()
|
||||
let milliType = milliArray.arrowData.type as! ArrowTypeTime32 // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(milliType.unit, .milliseconds)
|
||||
XCTAssertEqual(milliArray.length, 3)
|
||||
XCTAssertEqual(milliArray[1], 1000000)
|
||||
XCTAssertEqual(milliArray[2], nil)
|
||||
|
||||
let secBuilder = try ArrowArrayBuilders.loadTime32ArrayBuilder(.seconds)
|
||||
secBuilder.append(200)
|
||||
secBuilder.append(nil)
|
||||
secBuilder.append(2000011)
|
||||
XCTAssertEqual(secBuilder.nullCount, 1)
|
||||
XCTAssertEqual(secBuilder.length, 3)
|
||||
XCTAssertEqual(secBuilder.capacity, 136)
|
||||
let secArray = try secBuilder.finish()
|
||||
let secType = secArray.arrowData.type as! ArrowTypeTime32 // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(secType.unit, .seconds)
|
||||
XCTAssertEqual(secArray.length, 3)
|
||||
XCTAssertEqual(secArray[1], nil)
|
||||
XCTAssertEqual(secArray[2], 2000011)
|
||||
}
|
||||
|
||||
func testTime64Array() throws {
|
||||
let nanoBuilder = try ArrowArrayBuilders.loadTime64ArrayBuilder(.nanoseconds)
|
||||
nanoBuilder.append(10000)
|
||||
nanoBuilder.append(nil)
|
||||
nanoBuilder.append(123456789)
|
||||
XCTAssertEqual(nanoBuilder.nullCount, 1)
|
||||
XCTAssertEqual(nanoBuilder.length, 3)
|
||||
XCTAssertEqual(nanoBuilder.capacity, 264)
|
||||
let nanoArray = try nanoBuilder.finish()
|
||||
let nanoType = nanoArray.arrowData.type as! ArrowTypeTime64 // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(nanoType.unit, .nanoseconds)
|
||||
XCTAssertEqual(nanoArray.length, 3)
|
||||
XCTAssertEqual(nanoArray[1], nil)
|
||||
XCTAssertEqual(nanoArray[2], 123456789)
|
||||
|
||||
let microBuilder = try ArrowArrayBuilders.loadTime64ArrayBuilder(.microseconds)
|
||||
microBuilder.append(nil)
|
||||
microBuilder.append(20000)
|
||||
microBuilder.append(987654321)
|
||||
XCTAssertEqual(microBuilder.nullCount, 1)
|
||||
XCTAssertEqual(microBuilder.length, 3)
|
||||
XCTAssertEqual(microBuilder.capacity, 264)
|
||||
let microArray = try microBuilder.finish()
|
||||
let microType = microArray.arrowData.type as! ArrowTypeTime64 // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(microType.unit, .microseconds)
|
||||
XCTAssertEqual(microArray.length, 3)
|
||||
XCTAssertEqual(microArray[1], 20000)
|
||||
XCTAssertEqual(microArray[2], 987654321)
|
||||
}
|
||||
|
||||
func testStructArray() throws { // swiftlint:disable:this function_body_length
|
||||
class StructTest {
|
||||
var fieldBool: Bool = false
|
||||
var fieldInt8: Int8 = 0
|
||||
var fieldInt16: Int16 = 0
|
||||
var fieldInt32: Int32 = 0
|
||||
var fieldInt64: Int64 = 0
|
||||
var fieldUInt8: UInt8 = 0
|
||||
var fieldUInt16: UInt16 = 0
|
||||
var fieldUInt32: UInt32 = 0
|
||||
var fieldUInt64: UInt64 = 0
|
||||
var fieldDouble: Double = 0
|
||||
var fieldFloat: Float = 0
|
||||
var fieldString: String = ""
|
||||
var fieldData = Data()
|
||||
var fieldDate: Date = Date.now
|
||||
}
|
||||
|
||||
enum STIndex: Int {
|
||||
case bool, int8, int16, int32, int64
|
||||
case uint8, uint16, uint32, uint64, double
|
||||
case float, string, data, date
|
||||
}
|
||||
|
||||
let testData = StructTest()
|
||||
let dateNow = Date.now
|
||||
let structBuilder = try ArrowArrayBuilders.loadStructArrayBuilderForType(testData)
|
||||
structBuilder.append([true, Int8(1), Int16(2), Int32(3), Int64(4),
|
||||
UInt8(5), UInt16(6), UInt32(7), UInt64(8), Double(9.9),
|
||||
Float(10.10), "11", Data("12".utf8), dateNow])
|
||||
structBuilder.append(nil)
|
||||
structBuilder.append([true, Int8(13), Int16(14), Int32(15), Int64(16),
|
||||
UInt8(17), UInt16(18), UInt32(19), UInt64(20), Double(21.21),
|
||||
Float(22.22), "23", Data("24".utf8), dateNow])
|
||||
XCTAssertEqual(structBuilder.length, 3)
|
||||
let structArray = try structBuilder.finish()
|
||||
XCTAssertEqual(structArray.length, 3)
|
||||
XCTAssertNil(structArray[1])
|
||||
XCTAssertEqual(structArray.arrowFields![0].length, 3)
|
||||
XCTAssertNil(structArray.arrowFields![0].array.asAny(1))
|
||||
XCTAssertEqual(structArray[0]![STIndex.bool.rawValue] as? Bool, true)
|
||||
XCTAssertEqual(structArray[0]![STIndex.int8.rawValue] as? Int8, 1)
|
||||
XCTAssertEqual(structArray[0]![STIndex.int16.rawValue] as? Int16, 2)
|
||||
XCTAssertEqual(structArray[0]![STIndex.int32.rawValue] as? Int32, 3)
|
||||
XCTAssertEqual(structArray[0]![STIndex.int64.rawValue] as? Int64, 4)
|
||||
XCTAssertEqual(structArray[0]![STIndex.uint8.rawValue] as? UInt8, 5)
|
||||
XCTAssertEqual(structArray[0]![STIndex.uint16.rawValue] as? UInt16, 6)
|
||||
XCTAssertEqual(structArray[0]![STIndex.uint32.rawValue] as? UInt32, 7)
|
||||
XCTAssertEqual(structArray[0]![STIndex.uint64.rawValue] as? UInt64, 8)
|
||||
XCTAssertEqual(structArray[0]![STIndex.double.rawValue] as? Double, 9.9)
|
||||
XCTAssertEqual(structArray[0]![STIndex.float.rawValue] as? Float, 10.10)
|
||||
XCTAssertEqual(structArray[2]![STIndex.string.rawValue] as? String, "23")
|
||||
XCTAssertEqual(
|
||||
String(decoding: (structArray[0]![STIndex.data.rawValue] as? Data)!, as: UTF8.self), "12")
|
||||
let dateFormatter = DateFormatter()
|
||||
dateFormatter.timeStyle = .full
|
||||
XCTAssertTrue(
|
||||
dateFormatter.string(from: (structArray[0]![STIndex.date.rawValue] as? Date)!) ==
|
||||
dateFormatter.string(from: dateNow))
|
||||
}
|
||||
|
||||
func checkHolderForType(_ checkType: ArrowType) throws {
|
||||
let buffers = [ArrowBuffer(length: 0, capacity: 0,
|
||||
rawPointer: UnsafeMutableRawPointer.allocate(byteCount: 0, alignment: .zero)),
|
||||
ArrowBuffer(length: 0, capacity: 0,
|
||||
rawPointer: UnsafeMutableRawPointer.allocate(byteCount: 0, alignment: .zero))]
|
||||
let field = ArrowField("", type: checkType, isNullable: true)
|
||||
switch makeArrayHolder(field, buffers: buffers, nullCount: 0, children: nil, rbLength: 0) {
|
||||
case .success(let holder):
|
||||
XCTAssertEqual(holder.type.id, checkType.id)
|
||||
case .failure(let err):
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
func testArrayHolders() throws {
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowInt8))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowUInt8))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowInt16))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowUInt16))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowInt32))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowUInt32))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowInt64))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowUInt64))
|
||||
try checkHolderForType(ArrowTypeTime32(.seconds))
|
||||
try checkHolderForType(ArrowTypeTime32(.milliseconds))
|
||||
try checkHolderForType(ArrowTypeTime64(.microseconds))
|
||||
try checkHolderForType(ArrowTypeTime64(.nanoseconds))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowBinary))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowFloat))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowDouble))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowBool))
|
||||
try checkHolderForType(ArrowType(ArrowType.ArrowString))
|
||||
}
|
||||
|
||||
func testArrowArrayHolderBuilder() throws {
|
||||
let uint8HBuilder: ArrowArrayHolderBuilder =
|
||||
(try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt8>)
|
||||
for index in 0..<100 {
|
||||
uint8HBuilder.appendAny(UInt8(index))
|
||||
}
|
||||
|
||||
let uint8Holder = try uint8HBuilder.toHolder()
|
||||
XCTAssertEqual(uint8Holder.nullCount, 0)
|
||||
XCTAssertEqual(uint8Holder.length, 100)
|
||||
|
||||
let stringHBuilder: ArrowArrayHolderBuilder =
|
||||
(try ArrowArrayBuilders.loadStringArrayBuilder())
|
||||
for index in 0..<100 {
|
||||
if index % 10 == 9 {
|
||||
stringHBuilder.appendAny(nil)
|
||||
} else {
|
||||
stringHBuilder.appendAny("test" + String(index))
|
||||
}
|
||||
}
|
||||
|
||||
let stringHolder = try stringHBuilder.toHolder()
|
||||
XCTAssertEqual(stringHolder.nullCount, 10)
|
||||
XCTAssertEqual(stringHolder.length, 100)
|
||||
}
|
||||
|
||||
func testAddVArgs() throws {
|
||||
let arrayBuilder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
arrayBuilder.append(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
|
||||
XCTAssertEqual(arrayBuilder.length, 10)
|
||||
XCTAssertEqual(try arrayBuilder.finish()[2], 2)
|
||||
let doubleBuilder: NumberArrayBuilder<Double> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
doubleBuilder.append(0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8)
|
||||
XCTAssertEqual(doubleBuilder.length, 9)
|
||||
XCTAssertEqual(try doubleBuilder.finish()[4], 4.4)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("0", "1", "2", "3", "4", "5", "6")
|
||||
XCTAssertEqual(stringBuilder.length, 7)
|
||||
XCTAssertEqual(try stringBuilder.finish()[4], "4")
|
||||
let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder()
|
||||
boolBuilder.append(true, false, true, false)
|
||||
XCTAssertEqual(try boolBuilder.finish()[2], true)
|
||||
}
|
||||
|
||||
func testAddArray() throws {
|
||||
let arrayBuilder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
arrayBuilder.append([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
|
||||
XCTAssertEqual(arrayBuilder.length, 10)
|
||||
XCTAssertEqual(try arrayBuilder.finish()[2], 2)
|
||||
let doubleBuilder: NumberArrayBuilder<Double> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
doubleBuilder.append([0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8])
|
||||
XCTAssertEqual(doubleBuilder.length, 9)
|
||||
XCTAssertEqual(try doubleBuilder.finish()[4], 4.4)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append(["0", "1", "2", "3", "4", "5", "6"])
|
||||
XCTAssertEqual(stringBuilder.length, 7)
|
||||
XCTAssertEqual(try stringBuilder.finish()[4], "4")
|
||||
let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder()
|
||||
boolBuilder.append([true, false, true, false])
|
||||
XCTAssertEqual(try boolBuilder.finish()[2], true)
|
||||
}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import XCTest
|
||||
@testable import Arrow
|
||||
import ArrowC
|
||||
|
||||
final class CDataTests: XCTestCase {
|
||||
func makeSchema() -> Arrow.ArrowSchema {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
return schemaBuilder
|
||||
.addField("colBool", type: ArrowType(ArrowType.ArrowBool), isNullable: false)
|
||||
.addField("colUInt8", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true)
|
||||
.addField("colUInt16", type: ArrowType(ArrowType.ArrowUInt16), isNullable: true)
|
||||
.addField("colUInt32", type: ArrowType(ArrowType.ArrowUInt32), isNullable: true)
|
||||
.addField("colUInt64", type: ArrowType(ArrowType.ArrowUInt64), isNullable: true)
|
||||
.addField("colInt8", type: ArrowType(ArrowType.ArrowInt8), isNullable: false)
|
||||
.addField("colInt16", type: ArrowType(ArrowType.ArrowInt16), isNullable: false)
|
||||
.addField("colInt32", type: ArrowType(ArrowType.ArrowInt32), isNullable: false)
|
||||
.addField("colInt64", type: ArrowType(ArrowType.ArrowInt64), isNullable: false)
|
||||
.addField("colString", type: ArrowType(ArrowType.ArrowString), isNullable: false)
|
||||
.addField("colBinary", type: ArrowType(ArrowType.ArrowBinary), isNullable: false)
|
||||
.addField("colDate32", type: ArrowType(ArrowType.ArrowDate32), isNullable: false)
|
||||
.addField("colDate64", type: ArrowType(ArrowType.ArrowDate64), isNullable: false)
|
||||
.addField("colTime32", type: ArrowType(ArrowType.ArrowTime32), isNullable: false)
|
||||
.addField("colTime32s", type: ArrowTypeTime32(.seconds), isNullable: false)
|
||||
.addField("colTime32m", type: ArrowTypeTime32(.milliseconds), isNullable: false)
|
||||
.addField("colTime64", type: ArrowType(ArrowType.ArrowTime64), isNullable: false)
|
||||
.addField("colTime64u", type: ArrowTypeTime64(.microseconds), isNullable: false)
|
||||
.addField("colTime64n", type: ArrowTypeTime64(.nanoseconds), isNullable: false)
|
||||
.addField("colTime64", type: ArrowType(ArrowType.ArrowTime64), isNullable: false)
|
||||
.addField("colFloat", type: ArrowType(ArrowType.ArrowFloat), isNullable: false)
|
||||
.addField("colDouble", type: ArrowType(ArrowType.ArrowDouble), isNullable: false)
|
||||
.finish()
|
||||
}
|
||||
|
||||
func checkImportField(_ cSchema: ArrowC.ArrowSchema, name: String, type: ArrowType.Info) throws {
|
||||
let importer = ArrowCImporter()
|
||||
switch importer.importField(cSchema) {
|
||||
case .success(let arrowField):
|
||||
XCTAssertEqual(arrowField.type.info, type)
|
||||
XCTAssertEqual(arrowField.name, name)
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func testImportExportSchema() throws {
|
||||
let schema = makeSchema()
|
||||
let exporter = ArrowCExporter()
|
||||
for arrowField in schema.fields {
|
||||
var cSchema = ArrowC.ArrowSchema()
|
||||
switch exporter.exportField(&cSchema, field: arrowField) {
|
||||
case .success:
|
||||
try checkImportField(cSchema, name: arrowField.name, type: arrowField.type.info)
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testImportExportArray() throws {
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
for index in 0..<100 {
|
||||
if index % 10 == 9 {
|
||||
stringBuilder.append(nil)
|
||||
} else {
|
||||
stringBuilder.append("test" + String(index))
|
||||
}
|
||||
}
|
||||
|
||||
XCTAssertEqual(stringBuilder.nullCount, 10)
|
||||
XCTAssertEqual(stringBuilder.length, 100)
|
||||
XCTAssertEqual(stringBuilder.capacity, 648)
|
||||
let stringArray = try stringBuilder.finish()
|
||||
let exporter = ArrowCExporter()
|
||||
var cArray = ArrowC.ArrowArray()
|
||||
exporter.exportArray(&cArray, arrowData: stringArray.arrowData)
|
||||
let cArrayMutPtr = UnsafeMutablePointer<ArrowC.ArrowArray>.allocate(capacity: 1)
|
||||
cArrayMutPtr.pointee = cArray
|
||||
defer {
|
||||
cArrayMutPtr.deallocate()
|
||||
}
|
||||
|
||||
let importer = ArrowCImporter()
|
||||
switch importer.importArray(UnsafePointer(cArrayMutPtr), arrowType: ArrowType(ArrowType.ArrowString)) {
|
||||
case .success(let holder):
|
||||
let builder = RecordBatch.Builder()
|
||||
switch builder
|
||||
.addColumn("test", arrowArray: holder)
|
||||
.finish() {
|
||||
case .success(let rb):
|
||||
XCTAssertEqual(rb.columnCount, 1)
|
||||
XCTAssertEqual(rb.length, 100)
|
||||
let col1: Arrow.ArrowArray<String> = rb.data(for: 0)
|
||||
for index in 0..<col1.length {
|
||||
if index % 10 == 9 {
|
||||
XCTAssertEqual(col1[index], nil)
|
||||
} else {
|
||||
XCTAssertEqual(col1[index], "test" + String(index))
|
||||
}
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,334 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import XCTest
|
||||
@testable import Arrow
|
||||
|
||||
final class CodableTests: XCTestCase { // swiftlint:disable:this type_body_length
|
||||
public class TestClass: Codable {
|
||||
public var propBool: Bool
|
||||
public var propInt8: Int8
|
||||
public var propInt16: Int16
|
||||
public var propInt32: Int32
|
||||
public var propInt64: Int64
|
||||
public var propUInt8: UInt8
|
||||
public var propUInt16: UInt16
|
||||
public var propUInt32: UInt32
|
||||
public var propUInt64: UInt64
|
||||
public var propFloat: Float
|
||||
public var propDouble: Double?
|
||||
public var propString: String
|
||||
public var propDate: Date
|
||||
|
||||
public required init() {
|
||||
self.propBool = false
|
||||
self.propInt8 = 1
|
||||
self.propInt16 = 2
|
||||
self.propInt32 = 3
|
||||
self.propInt64 = 4
|
||||
self.propUInt8 = 5
|
||||
self.propUInt16 = 6
|
||||
self.propUInt32 = 7
|
||||
self.propUInt64 = 8
|
||||
self.propFloat = 9
|
||||
self.propDouble = 10
|
||||
self.propString = "11"
|
||||
self.propDate = Date.now
|
||||
}
|
||||
}
|
||||
|
||||
func testArrowKeyedDecoder() throws { // swiftlint:disable:this function_body_length
|
||||
let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352)
|
||||
let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder()
|
||||
let int8Builder: NumberArrayBuilder<Int8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let int16Builder: NumberArrayBuilder<Int16> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let int32Builder: NumberArrayBuilder<Int32> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let int64Builder: NumberArrayBuilder<Int64> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let uint16Builder: NumberArrayBuilder<UInt16> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let uint32Builder: NumberArrayBuilder<UInt32> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let uint64Builder: NumberArrayBuilder<UInt64> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let floatBuilder: NumberArrayBuilder<Float> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let doubleBuilder: NumberArrayBuilder<Double> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
let dateBuilder = try ArrowArrayBuilders.loadDate64ArrayBuilder()
|
||||
|
||||
boolBuilder.append(false, true, false)
|
||||
int8Builder.append(10, 11, 12)
|
||||
int16Builder.append(20, 21, 22)
|
||||
int32Builder.append(30, 31, 32)
|
||||
int64Builder.append(40, 41, 42)
|
||||
uint8Builder.append(50, 51, 52)
|
||||
uint16Builder.append(60, 61, 62)
|
||||
uint32Builder.append(70, 71, 72)
|
||||
uint64Builder.append(80, 81, 82)
|
||||
floatBuilder.append(90.1, 91.1, 92.1)
|
||||
doubleBuilder.append(101.1, nil, nil)
|
||||
stringBuilder.append("test0", "test1", "test2")
|
||||
dateBuilder.append(date1, date1, date1)
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("propBool", arrowArray: try boolBuilder.toHolder())
|
||||
.addColumn("propInt8", arrowArray: try int8Builder.toHolder())
|
||||
.addColumn("propInt16", arrowArray: try int16Builder.toHolder())
|
||||
.addColumn("propInt32", arrowArray: try int32Builder.toHolder())
|
||||
.addColumn("propInt64", arrowArray: try int64Builder.toHolder())
|
||||
.addColumn("propUInt8", arrowArray: try uint8Builder.toHolder())
|
||||
.addColumn("propUInt16", arrowArray: try uint16Builder.toHolder())
|
||||
.addColumn("propUInt32", arrowArray: try uint32Builder.toHolder())
|
||||
.addColumn("propUInt64", arrowArray: try uint64Builder.toHolder())
|
||||
.addColumn("propFloat", arrowArray: try floatBuilder.toHolder())
|
||||
.addColumn("propDouble", arrowArray: try doubleBuilder.toHolder())
|
||||
.addColumn("propString", arrowArray: try stringBuilder.toHolder())
|
||||
.addColumn("propDate", arrowArray: try dateBuilder.toHolder())
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let rb):
|
||||
let decoder = ArrowDecoder(rb)
|
||||
let testClasses = try decoder.decode(TestClass.self)
|
||||
for index in 0..<testClasses.count {
|
||||
let testClass = testClasses[index]
|
||||
XCTAssertEqual(testClass.propBool, index % 2 == 0 ? false : true)
|
||||
XCTAssertEqual(testClass.propInt8, Int8(index + 10))
|
||||
XCTAssertEqual(testClass.propInt16, Int16(index + 20))
|
||||
XCTAssertEqual(testClass.propInt32, Int32(index + 30))
|
||||
XCTAssertEqual(testClass.propInt64, Int64(index + 40))
|
||||
XCTAssertEqual(testClass.propUInt8, UInt8(index + 50))
|
||||
XCTAssertEqual(testClass.propUInt16, UInt16(index + 60))
|
||||
XCTAssertEqual(testClass.propUInt32, UInt32(index + 70))
|
||||
XCTAssertEqual(testClass.propUInt64, UInt64(index + 80))
|
||||
XCTAssertEqual(testClass.propFloat, Float(index) + 90.1)
|
||||
if index == 0 {
|
||||
XCTAssertEqual(testClass.propDouble, 101.1)
|
||||
} else {
|
||||
XCTAssertEqual(testClass.propDouble, nil)
|
||||
}
|
||||
XCTAssertEqual(testClass.propString, "test\(index)")
|
||||
XCTAssertEqual(testClass.propDate, date1)
|
||||
}
|
||||
case .failure(let err):
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
func testArrowSingleDecoderWithoutNull() throws {
|
||||
let int8Builder: NumberArrayBuilder<Int8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
int8Builder.append(10, 11, 12)
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("propInt8", arrowArray: try int8Builder.toHolder())
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let rb):
|
||||
let decoder = ArrowDecoder(rb)
|
||||
let testData = try decoder.decode(Int8?.self)
|
||||
for index in 0..<testData.count {
|
||||
let val: Int8? = testData[index]
|
||||
XCTAssertEqual(val!, Int8(index + 10))
|
||||
}
|
||||
case .failure(let err):
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
func testArrowSingleDecoderWithNull() throws {
|
||||
let int8WNilBuilder: NumberArrayBuilder<Int8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
int8WNilBuilder.append(10, nil, 12, nil)
|
||||
let resultWNil = RecordBatch.Builder()
|
||||
.addColumn("propInt8", arrowArray: try int8WNilBuilder.toHolder())
|
||||
.finish()
|
||||
switch resultWNil {
|
||||
case .success(let rb):
|
||||
let decoder = ArrowDecoder(rb)
|
||||
let testData = try decoder.decode(Int8?.self)
|
||||
for index in 0..<testData.count {
|
||||
let val: Int8? = testData[index]
|
||||
if index % 2 == 1 {
|
||||
XCTAssertNil(val)
|
||||
} else {
|
||||
XCTAssertEqual(val!, Int8(index + 10))
|
||||
}
|
||||
}
|
||||
case .failure(let err):
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
func testArrowMapDecoderWithoutNull() throws {
|
||||
let int8Builder: NumberArrayBuilder<Int8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
int8Builder.append(10, 11, 12, 13)
|
||||
stringBuilder.append("test10", "test11", "test12", "test13")
|
||||
switch RecordBatch.Builder()
|
||||
.addColumn("propInt8", arrowArray: try int8Builder.toHolder())
|
||||
.addColumn("propString", arrowArray: try stringBuilder.toHolder())
|
||||
.finish() {
|
||||
case .success(let rb):
|
||||
let decoder = ArrowDecoder(rb)
|
||||
let testData = try decoder.decode([Int8: String].self)
|
||||
for data in testData {
|
||||
XCTAssertEqual("test\(data.key)", data.value)
|
||||
}
|
||||
case .failure(let err):
|
||||
throw err
|
||||
}
|
||||
|
||||
switch RecordBatch.Builder()
|
||||
.addColumn("propString", arrowArray: try stringBuilder.toHolder())
|
||||
.addColumn("propInt8", arrowArray: try int8Builder.toHolder())
|
||||
.finish() {
|
||||
case .success(let rb):
|
||||
let decoder = ArrowDecoder(rb)
|
||||
let testData = try decoder.decode([String: Int8].self)
|
||||
for data in testData {
|
||||
XCTAssertEqual("test\(data.value)", data.key)
|
||||
}
|
||||
case .failure(let err):
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
func testArrowMapDecoderWithNull() throws {
|
||||
let int8Builder: NumberArrayBuilder<Int8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
let stringWNilBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
int8Builder.append(10, 11, 12, 13)
|
||||
stringWNilBuilder.append(nil, "test11", nil, "test13")
|
||||
let resultWNil = RecordBatch.Builder()
|
||||
.addColumn("propInt8", arrowArray: try int8Builder.toHolder())
|
||||
.addColumn("propString", arrowArray: try stringWNilBuilder.toHolder())
|
||||
.finish()
|
||||
switch resultWNil {
|
||||
case .success(let rb):
|
||||
let decoder = ArrowDecoder(rb)
|
||||
let testData = try decoder.decode([Int8: String?].self)
|
||||
for data in testData {
|
||||
let str = data.value
|
||||
if data.key % 2 == 0 {
|
||||
XCTAssertNil(str)
|
||||
} else {
|
||||
XCTAssertEqual(str, "test\(data.key)")
|
||||
}
|
||||
}
|
||||
case .failure(let err):
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
func getArrayValue<T>(_ rb: RecordBatch, colIndex: Int, rowIndex: UInt) -> T? {
|
||||
let anyArray = rb.columns[colIndex].array
|
||||
return anyArray.asAny(UInt(rowIndex)) as? T
|
||||
}
|
||||
|
||||
func testArrowKeyedEncoder() throws { // swiftlint:disable:this function_body_length
|
||||
var infos = [TestClass]()
|
||||
for index in 0..<10 {
|
||||
let tClass = TestClass()
|
||||
let offset = index * 12
|
||||
tClass.propBool = index % 2 == 0
|
||||
tClass.propInt8 = Int8(offset + 1)
|
||||
tClass.propInt16 = Int16(offset + 2)
|
||||
tClass.propInt32 = Int32(offset + 3)
|
||||
tClass.propInt64 = Int64(offset + 4)
|
||||
tClass.propUInt8 = UInt8(offset + 5)
|
||||
tClass.propUInt16 = UInt16(offset + 6)
|
||||
tClass.propUInt32 = UInt32(offset + 7)
|
||||
tClass.propUInt64 = UInt64(offset + 8)
|
||||
tClass.propFloat = Float(offset + 9)
|
||||
tClass.propDouble = index % 2 == 0 ? Double(offset + 10) : nil
|
||||
tClass.propString = "\(offset + 11)"
|
||||
tClass.propDate = Date.now
|
||||
infos.append(tClass)
|
||||
}
|
||||
|
||||
let rb = try ArrowEncoder.encode(infos)!
|
||||
XCTAssertEqual(Int(rb.length), infos.count)
|
||||
XCTAssertEqual(rb.columns.count, 13)
|
||||
XCTAssertEqual(rb.columns[0].type.id, ArrowTypeId.boolean)
|
||||
XCTAssertEqual(rb.columns[1].type.id, ArrowTypeId.int8)
|
||||
XCTAssertEqual(rb.columns[2].type.id, ArrowTypeId.int16)
|
||||
XCTAssertEqual(rb.columns[3].type.id, ArrowTypeId.int32)
|
||||
XCTAssertEqual(rb.columns[4].type.id, ArrowTypeId.int64)
|
||||
XCTAssertEqual(rb.columns[5].type.id, ArrowTypeId.uint8)
|
||||
XCTAssertEqual(rb.columns[6].type.id, ArrowTypeId.uint16)
|
||||
XCTAssertEqual(rb.columns[7].type.id, ArrowTypeId.uint32)
|
||||
XCTAssertEqual(rb.columns[8].type.id, ArrowTypeId.uint64)
|
||||
XCTAssertEqual(rb.columns[9].type.id, ArrowTypeId.float)
|
||||
XCTAssertEqual(rb.columns[10].type.id, ArrowTypeId.double)
|
||||
XCTAssertEqual(rb.columns[11].type.id, ArrowTypeId.string)
|
||||
XCTAssertEqual(rb.columns[12].type.id, ArrowTypeId.date64)
|
||||
for index in 0..<10 {
|
||||
let offset = index * 12
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 0, rowIndex: UInt(index)), index % 2 == 0)
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 1, rowIndex: UInt(index)), Int8(offset + 1))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 2, rowIndex: UInt(index)), Int16(offset + 2))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 3, rowIndex: UInt(index)), Int32(offset + 3))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 4, rowIndex: UInt(index)), Int64(offset + 4))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 5, rowIndex: UInt(index)), UInt8(offset + 5))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 6, rowIndex: UInt(index)), UInt16(offset + 6))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 7, rowIndex: UInt(index)), UInt32(offset + 7))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 8, rowIndex: UInt(index)), UInt64(offset + 8))
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 9, rowIndex: UInt(index)), Float(offset + 9))
|
||||
if index % 2 == 0 {
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 10, rowIndex: UInt(index)), Double(offset + 10))
|
||||
} else {
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 10, rowIndex: UInt(index)), Double?(nil))
|
||||
}
|
||||
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 11, rowIndex: UInt(index)), String(offset + 11))
|
||||
}
|
||||
}
|
||||
|
||||
func testArrowUnkeyedEncoder() throws {
|
||||
var testMap = [Int8: String?]()
|
||||
for index in 0..<10 {
|
||||
testMap[Int8(index)] = "test\(index)"
|
||||
}
|
||||
|
||||
let rb = try ArrowEncoder.encode(testMap)
|
||||
XCTAssertEqual(Int(rb.length), testMap.count)
|
||||
XCTAssertEqual(rb.columns.count, 2)
|
||||
XCTAssertEqual(rb.columns[0].type.id, ArrowTypeId.int8)
|
||||
XCTAssertEqual(rb.columns[1].type.id, ArrowTypeId.string)
|
||||
for index in 0..<10 {
|
||||
let key: Int8 = getArrayValue(rb, colIndex: 0, rowIndex: UInt(index))!
|
||||
let value: String = getArrayValue(rb, colIndex: 1, rowIndex: UInt(index))!
|
||||
XCTAssertEqual("test\(key)", value)
|
||||
}
|
||||
}
|
||||
|
||||
func testArrowSingleEncoder() throws {
|
||||
var intArray = [Int32?]()
|
||||
for index in 0..<100 {
|
||||
if index == 10 {
|
||||
intArray.append(nil)
|
||||
} else {
|
||||
intArray.append(Int32(index))
|
||||
}
|
||||
}
|
||||
|
||||
let rb = try ArrowEncoder.encode(intArray)!
|
||||
XCTAssertEqual(Int(rb.length), intArray.count)
|
||||
XCTAssertEqual(rb.columns.count, 1)
|
||||
XCTAssertEqual(rb.columns[0].type.id, ArrowTypeId.int32)
|
||||
for index in 0..<100 {
|
||||
if index == 10 {
|
||||
let anyArray = rb.columns[0].array
|
||||
XCTAssertNil(anyArray.asAny(UInt(index)))
|
||||
} else {
|
||||
XCTAssertEqual(getArrayValue(rb, colIndex: 0, rowIndex: UInt(index)), Int32(index))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,616 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import XCTest
|
||||
import FlatBuffers
|
||||
@testable import Arrow
|
||||
|
||||
let currentDate = Date.now
|
||||
class StructTest {
|
||||
var field0: Bool = false
|
||||
var field1: Int8 = 0
|
||||
var field2: Int16 = 0
|
||||
var field: Int32 = 0
|
||||
var field4: Int64 = 0
|
||||
var field5: UInt8 = 0
|
||||
var field6: UInt16 = 0
|
||||
var field7: UInt32 = 0
|
||||
var field8: UInt64 = 0
|
||||
var field9: Double = 0
|
||||
var field10: Float = 0
|
||||
var field11: String = ""
|
||||
var field12 = Data()
|
||||
var field13: Date = currentDate
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
func checkBoolRecordBatch(_ result: Result<ArrowReader.ArrowReaderResult, ArrowError>) throws -> [RecordBatch] {
|
||||
let recordBatches: [RecordBatch]
|
||||
switch result {
|
||||
case .success(let result):
|
||||
recordBatches = result.batches
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
|
||||
XCTAssertEqual(recordBatches.count, 1)
|
||||
for recordBatch in recordBatches {
|
||||
XCTAssertEqual(recordBatch.length, 5)
|
||||
XCTAssertEqual(recordBatch.columns.count, 2)
|
||||
XCTAssertEqual(recordBatch.schema.fields.count, 2)
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].name, "one")
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].type.info, ArrowType.ArrowBool)
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].name, "two")
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
for index in 0..<recordBatch.length {
|
||||
let column = recordBatch.columns[0]
|
||||
let str = column.array as! AsString // swiftlint:disable:this force_cast
|
||||
let val = "\(str.asString(index))"
|
||||
if index == 0 || index == 4 {
|
||||
XCTAssertEqual(val, "true")
|
||||
} else if index == 2 {
|
||||
XCTAssertEqual(val, "")
|
||||
} else {
|
||||
XCTAssertEqual(val, "false")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return recordBatches
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
func checkStructRecordBatch(_ result: Result<ArrowReader.ArrowReaderResult, ArrowError>) throws -> [RecordBatch] {
|
||||
let recordBatches: [RecordBatch]
|
||||
switch result {
|
||||
case .success(let result):
|
||||
recordBatches = result.batches
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
|
||||
XCTAssertEqual(recordBatches.count, 1)
|
||||
for recordBatch in recordBatches {
|
||||
XCTAssertEqual(recordBatch.length, 3)
|
||||
XCTAssertEqual(recordBatch.columns.count, 1)
|
||||
XCTAssertEqual(recordBatch.schema.fields.count, 1)
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].name, "my struct")
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].type.id, .strct)
|
||||
let structArray = recordBatch.columns[0].array as? StructArray
|
||||
XCTAssertEqual(structArray!.arrowFields!.count, 2)
|
||||
XCTAssertEqual(structArray!.arrowFields![0].type.id, .string)
|
||||
XCTAssertEqual(structArray!.arrowFields![1].type.id, .boolean)
|
||||
let column = recordBatch.columns[0]
|
||||
let str = column.array as? AsString
|
||||
XCTAssertEqual("\(str!.asString(0))", "{0,false}")
|
||||
XCTAssertEqual("\(str!.asString(1))", "{1,true}")
|
||||
XCTAssertTrue(column.array.asAny(2) == nil)
|
||||
}
|
||||
|
||||
return recordBatches
|
||||
}
|
||||
|
||||
func currentDirectory(path: String = #file) -> URL {
|
||||
return URL(fileURLWithPath: path).deletingLastPathComponent()
|
||||
}
|
||||
|
||||
func makeSchema() -> ArrowSchema {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
return schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true)
|
||||
.addField("col2", type: ArrowType(ArrowType.ArrowString), isNullable: false)
|
||||
.addField("col3", type: ArrowType(ArrowType.ArrowDate32), isNullable: false)
|
||||
.addField("col4", type: ArrowType(ArrowType.ArrowInt32), isNullable: false)
|
||||
.addField("col5", type: ArrowType(ArrowType.ArrowFloat), isNullable: false)
|
||||
.finish()
|
||||
}
|
||||
|
||||
func makeStructSchema() -> ArrowSchema {
|
||||
let testObj = StructTest()
|
||||
var fields = [ArrowField]()
|
||||
let buildStructType = {() -> ArrowNestedType in
|
||||
let mirror = Mirror(reflecting: testObj)
|
||||
for (property, value) in mirror.children {
|
||||
let arrowType = ArrowType(ArrowType.infoForType(type(of: value)))
|
||||
fields.append(ArrowField(property!, type: arrowType, isNullable: true))
|
||||
}
|
||||
|
||||
return ArrowNestedType(ArrowType.ArrowStruct, fields: fields)
|
||||
}
|
||||
|
||||
return ArrowSchema.Builder()
|
||||
.addField("struct1", type: buildStructType(), isNullable: true)
|
||||
.finish()
|
||||
}
|
||||
|
||||
func makeStructRecordBatch() throws -> RecordBatch {
|
||||
let testData = StructTest()
|
||||
let dateNow = Date.now
|
||||
let structBuilder = try ArrowArrayBuilders.loadStructArrayBuilderForType(testData)
|
||||
structBuilder.append([true, Int8(1), Int16(2), Int32(3), Int64(4),
|
||||
UInt8(5), UInt16(6), UInt32(7), UInt64(8), Double(9.9),
|
||||
Float(10.10), "11", Data("12".utf8), dateNow])
|
||||
structBuilder.append(nil)
|
||||
structBuilder.append([true, Int8(13), Int16(14), Int32(15), Int64(16),
|
||||
UInt8(17), UInt16(18), UInt32(19), UInt64(20), Double(21.21),
|
||||
Float(22.22), "23", Data("24".utf8), dateNow])
|
||||
let structHolder = ArrowArrayHolderImpl(try structBuilder.finish())
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("struct1", arrowArray: structHolder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
return recordBatch
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func makeRecordBatch() throws -> RecordBatch {
|
||||
let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
uint8Builder.append(10)
|
||||
uint8Builder.append(nil)
|
||||
uint8Builder.append(nil)
|
||||
uint8Builder.append(44)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("test10")
|
||||
stringBuilder.append("test22")
|
||||
stringBuilder.append("test33")
|
||||
stringBuilder.append("test44")
|
||||
let date32Builder = try ArrowArrayBuilders.loadDate32ArrayBuilder()
|
||||
let date2 = Date(timeIntervalSinceReferenceDate: 86400 * 1)
|
||||
let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
let int32Builder: NumberArrayBuilder<Int32> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
int32Builder.append(1)
|
||||
int32Builder.append(2)
|
||||
int32Builder.append(3)
|
||||
int32Builder.append(4)
|
||||
let floatBuilder: NumberArrayBuilder<Float> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
floatBuilder.append(211.112)
|
||||
floatBuilder.append(322.223)
|
||||
floatBuilder.append(433.334)
|
||||
floatBuilder.append(544.445)
|
||||
|
||||
let uint8Holder = ArrowArrayHolderImpl(try uint8Builder.finish())
|
||||
let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish())
|
||||
let date32Holder = ArrowArrayHolderImpl(try date32Builder.finish())
|
||||
let int32Holder = ArrowArrayHolderImpl(try int32Builder.finish())
|
||||
let floatHolder = ArrowArrayHolderImpl(try floatBuilder.finish())
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("col1", arrowArray: uint8Holder)
|
||||
.addColumn("col2", arrowArray: stringHolder)
|
||||
.addColumn("col3", arrowArray: date32Holder)
|
||||
.addColumn("col4", arrowArray: int32Holder)
|
||||
.addColumn("col5", arrowArray: floatHolder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
return recordBatch
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
final class IPCStreamReaderTests: XCTestCase {
|
||||
func testRBInMemoryToFromStream() throws {
|
||||
let schema = makeSchema()
|
||||
let recordBatch = try makeRecordBatch()
|
||||
let arrowWriter = ArrowWriter()
|
||||
let writerInfo = ArrowWriter.Info(.recordbatch, schema: schema, batches: [recordBatch])
|
||||
switch arrowWriter.writeStreaming(writerInfo) {
|
||||
case .success(let writeData):
|
||||
let arrowReader = ArrowReader()
|
||||
switch arrowReader.readStreaming(writeData) {
|
||||
case .success(let result):
|
||||
let recordBatches = result.batches
|
||||
XCTAssertEqual(recordBatches.count, 1)
|
||||
for recordBatch in recordBatches {
|
||||
XCTAssertEqual(recordBatch.length, 4)
|
||||
XCTAssertEqual(recordBatch.columns.count, 5)
|
||||
XCTAssertEqual(recordBatch.schema.fields.count, 5)
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].type.info, ArrowType.ArrowUInt8)
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(recordBatch.schema.fields[2].name, "col3")
|
||||
XCTAssertEqual(recordBatch.schema.fields[2].type.info, ArrowType.ArrowDate32)
|
||||
XCTAssertEqual(recordBatch.schema.fields[3].name, "col4")
|
||||
XCTAssertEqual(recordBatch.schema.fields[3].type.info, ArrowType.ArrowInt32)
|
||||
XCTAssertEqual(recordBatch.schema.fields[4].name, "col5")
|
||||
XCTAssertEqual(recordBatch.schema.fields[4].type.info, ArrowType.ArrowFloat)
|
||||
let columns = recordBatch.columns
|
||||
XCTAssertEqual(columns[0].nullCount, 2)
|
||||
let dateVal =
|
||||
"\((columns[2].array as! AsString).asString(0))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(dateVal, "2014-09-10 00:00:00 +0000")
|
||||
let stringVal =
|
||||
"\((columns[1].array as! AsString).asString(1))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(stringVal, "test22")
|
||||
let uintVal =
|
||||
"\((columns[0].array as! AsString).asString(0))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(uintVal, "10")
|
||||
let stringVal2 =
|
||||
"\((columns[1].array as! AsString).asString(3))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(stringVal2, "test44")
|
||||
let uintVal2 =
|
||||
"\((columns[0].array as! AsString).asString(3))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(uintVal2, "44")
|
||||
}
|
||||
case.failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
final class IPCFileReaderTests: XCTestCase { // swiftlint:disable:this type_body_length
|
||||
func testFileReader_double() throws {
|
||||
let fileURL = currentDirectory().appendingPathComponent("../../testdata_double.arrow")
|
||||
let arrowReader = ArrowReader()
|
||||
let result = arrowReader.fromFile(fileURL)
|
||||
let recordBatches: [RecordBatch]
|
||||
switch result {
|
||||
case .success(let result):
|
||||
recordBatches = result.batches
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
|
||||
XCTAssertEqual(recordBatches.count, 1)
|
||||
for recordBatch in recordBatches {
|
||||
XCTAssertEqual(recordBatch.length, 5)
|
||||
XCTAssertEqual(recordBatch.columns.count, 2)
|
||||
XCTAssertEqual(recordBatch.schema.fields.count, 2)
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].name, "one")
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].type.info, ArrowType.ArrowDouble)
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].name, "two")
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
for index in 0..<recordBatch.length {
|
||||
let column = recordBatch.columns[1]
|
||||
let str = column.array as! AsString // swiftlint:disable:this force_cast
|
||||
let val = "\(str.asString(index))"
|
||||
if index != 1 {
|
||||
XCTAssertNotEqual(val, "")
|
||||
} else {
|
||||
XCTAssertEqual(val, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testFileReader_bool() throws {
|
||||
let fileURL = currentDirectory().appendingPathComponent("../../testdata_bool.arrow")
|
||||
let arrowReader = ArrowReader()
|
||||
try checkBoolRecordBatch(arrowReader.fromFile(fileURL))
|
||||
}
|
||||
|
||||
func testFileWriter_bool() throws {
|
||||
// read existing file
|
||||
let fileURL = currentDirectory().appendingPathComponent("../../testdata_bool.arrow")
|
||||
let arrowReader = ArrowReader()
|
||||
let fileRBs = try checkBoolRecordBatch(arrowReader.fromFile(fileURL))
|
||||
let arrowWriter = ArrowWriter()
|
||||
// write data from file to a stream
|
||||
let writerInfo = ArrowWriter.Info(.recordbatch, schema: fileRBs[0].schema, batches: fileRBs)
|
||||
switch arrowWriter.writeFile(writerInfo) {
|
||||
case .success(let writeData):
|
||||
// read stream back into recordbatches
|
||||
try checkBoolRecordBatch(arrowReader.readFile(writeData))
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
// write file record batches to another file
|
||||
let outputUrl = currentDirectory().appendingPathComponent("../../testfilewriter_bool.arrow")
|
||||
switch arrowWriter.toFile(outputUrl, info: writerInfo) {
|
||||
case .success:
|
||||
try checkBoolRecordBatch(arrowReader.fromFile(outputUrl))
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func testFileReader_struct() throws {
|
||||
let fileURL = currentDirectory().appendingPathComponent("../../testdata_struct.arrow")
|
||||
let arrowReader = ArrowReader()
|
||||
try checkStructRecordBatch(arrowReader.fromFile(fileURL))
|
||||
}
|
||||
|
||||
func testFileWriter_struct() throws {
|
||||
// read existing file
|
||||
let fileURL = currentDirectory().appendingPathComponent("../../testdata_struct.arrow")
|
||||
let arrowReader = ArrowReader()
|
||||
let fileRBs = try checkStructRecordBatch(arrowReader.fromFile(fileURL))
|
||||
let arrowWriter = ArrowWriter()
|
||||
// write data from file to a stream
|
||||
let writerInfo = ArrowWriter.Info(.recordbatch, schema: fileRBs[0].schema, batches: fileRBs)
|
||||
switch arrowWriter.writeFile(writerInfo) {
|
||||
case .success(let writeData):
|
||||
// read stream back into recordbatches
|
||||
try checkStructRecordBatch(arrowReader.readFile(writeData))
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
// write file record batches to another file
|
||||
let outputUrl = currentDirectory().appendingPathComponent("../../testfilewriter_struct.arrow")
|
||||
switch arrowWriter.toFile(outputUrl, info: writerInfo) {
|
||||
case .success:
|
||||
try checkStructRecordBatch(arrowReader.fromFile(outputUrl))
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func testRBInMemoryToFromStream() throws {
|
||||
// read existing file
|
||||
let schema = makeSchema()
|
||||
let recordBatch = try makeRecordBatch()
|
||||
let arrowWriter = ArrowWriter()
|
||||
let writerInfo = ArrowWriter.Info(.recordbatch, schema: schema, batches: [recordBatch])
|
||||
switch arrowWriter.writeFile(writerInfo) {
|
||||
case .success(let writeData):
|
||||
let arrowReader = ArrowReader()
|
||||
switch arrowReader.readFile(writeData) {
|
||||
case .success(let result):
|
||||
let recordBatches = result.batches
|
||||
XCTAssertEqual(recordBatches.count, 1)
|
||||
for recordBatch in recordBatches {
|
||||
XCTAssertEqual(recordBatch.length, 4)
|
||||
XCTAssertEqual(recordBatch.columns.count, 5)
|
||||
XCTAssertEqual(recordBatch.schema.fields.count, 5)
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].type.info, ArrowType.ArrowUInt8)
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(recordBatch.schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(recordBatch.schema.fields[2].name, "col3")
|
||||
XCTAssertEqual(recordBatch.schema.fields[2].type.info, ArrowType.ArrowDate32)
|
||||
XCTAssertEqual(recordBatch.schema.fields[3].name, "col4")
|
||||
XCTAssertEqual(recordBatch.schema.fields[3].type.info, ArrowType.ArrowInt32)
|
||||
XCTAssertEqual(recordBatch.schema.fields[4].name, "col5")
|
||||
XCTAssertEqual(recordBatch.schema.fields[4].type.info, ArrowType.ArrowFloat)
|
||||
let columns = recordBatch.columns
|
||||
XCTAssertEqual(columns[0].nullCount, 2)
|
||||
let dateVal =
|
||||
"\((columns[2].array as! AsString).asString(0))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(dateVal, "2014-09-10 00:00:00 +0000")
|
||||
let stringVal =
|
||||
"\((columns[1].array as! AsString).asString(1))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(stringVal, "test22")
|
||||
let uintVal =
|
||||
"\((columns[0].array as! AsString).asString(0))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(uintVal, "10")
|
||||
let stringVal2 =
|
||||
"\((columns[1].array as! AsString).asString(3))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(stringVal2, "test44")
|
||||
let uintVal2 =
|
||||
"\((columns[0].array as! AsString).asString(3))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(uintVal2, "44")
|
||||
}
|
||||
case.failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func testSchemaInMemoryToFromStream() throws {
|
||||
// read existing file
|
||||
let schema = makeSchema()
|
||||
let arrowWriter = ArrowWriter()
|
||||
let writerInfo = ArrowWriter.Info(.schema, schema: schema)
|
||||
switch arrowWriter.writeFile(writerInfo) {
|
||||
case .success(let writeData):
|
||||
let arrowReader = ArrowReader()
|
||||
switch arrowReader.readFile(writeData) {
|
||||
case .success(let result):
|
||||
XCTAssertNotNil(result.schema)
|
||||
let schema = result.schema!
|
||||
XCTAssertEqual(schema.fields.count, 5)
|
||||
XCTAssertEqual(schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(schema.fields[2].name, "col3")
|
||||
XCTAssertEqual(schema.fields[2].type.info, ArrowType.ArrowDate32)
|
||||
XCTAssertEqual(schema.fields[3].name, "col4")
|
||||
XCTAssertEqual(schema.fields[3].type.info, ArrowType.ArrowInt32)
|
||||
XCTAssertEqual(schema.fields[4].name, "col5")
|
||||
XCTAssertEqual(schema.fields[4].type.info, ArrowType.ArrowFloat)
|
||||
case.failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func makeBinaryDataset() throws -> (ArrowSchema, RecordBatch) {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
let schema = schemaBuilder.addField("binary", type: ArrowType(ArrowType.ArrowBinary), isNullable: false)
|
||||
.finish()
|
||||
|
||||
let binaryBuilder = try ArrowArrayBuilders.loadBinaryArrayBuilder()
|
||||
binaryBuilder.append("test10".data(using: .utf8))
|
||||
binaryBuilder.append("test22".data(using: .utf8))
|
||||
binaryBuilder.append("test33".data(using: .utf8))
|
||||
binaryBuilder.append("test44".data(using: .utf8))
|
||||
|
||||
let binaryHolder = ArrowArrayHolderImpl(try binaryBuilder.finish())
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("binary", arrowArray: binaryHolder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
return (schema, recordBatch)
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func makeTimeDataset() throws -> (ArrowSchema, RecordBatch) {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
let schema = schemaBuilder.addField("time64", type: ArrowTypeTime64(.microseconds), isNullable: false)
|
||||
.addField("time32", type: ArrowTypeTime32(.milliseconds), isNullable: false)
|
||||
.finish()
|
||||
|
||||
let time64Builder = try ArrowArrayBuilders.loadTime64ArrayBuilder(.nanoseconds)
|
||||
time64Builder.append(12345678)
|
||||
time64Builder.append(1)
|
||||
time64Builder.append(nil)
|
||||
time64Builder.append(98765432)
|
||||
let time32Builder = try ArrowArrayBuilders.loadTime32ArrayBuilder(.milliseconds)
|
||||
time32Builder.append(1)
|
||||
time32Builder.append(2)
|
||||
time32Builder.append(nil)
|
||||
time32Builder.append(3)
|
||||
let time64Holder = ArrowArrayHolderImpl(try time64Builder.finish())
|
||||
let time32Holder = ArrowArrayHolderImpl(try time32Builder.finish())
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("time64", arrowArray: time64Holder)
|
||||
.addColumn("time32", arrowArray: time32Holder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
return (schema, recordBatch)
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func testStructRBInMemoryToFromStream() throws {
|
||||
// read existing file
|
||||
let schema = makeStructSchema()
|
||||
let recordBatch = try makeStructRecordBatch()
|
||||
let arrowWriter = ArrowWriter()
|
||||
let writerInfo = ArrowWriter.Info(.recordbatch, schema: schema, batches: [recordBatch])
|
||||
switch arrowWriter.writeStreaming(writerInfo) {
|
||||
case .success(let writeData):
|
||||
let arrowReader = ArrowReader()
|
||||
switch arrowReader.readStreaming(writeData) {
|
||||
case .success(let result):
|
||||
let recordBatches = result.batches
|
||||
XCTAssertEqual(recordBatches.count, 1)
|
||||
for recordBatch in recordBatches {
|
||||
XCTAssertEqual(recordBatch.length, 3)
|
||||
XCTAssertEqual(recordBatch.columns.count, 1)
|
||||
XCTAssertEqual(recordBatch.schema.fields.count, 1)
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].name, "struct1")
|
||||
XCTAssertEqual(recordBatch.schema.fields[0].type.id, .strct)
|
||||
XCTAssertTrue(recordBatch.schema.fields[0].type is ArrowNestedType)
|
||||
let nestedType = (recordBatch.schema.fields[0].type as? ArrowNestedType)!
|
||||
XCTAssertEqual(nestedType.fields.count, 14)
|
||||
let columns = recordBatch.columns
|
||||
XCTAssertEqual(columns[0].nullCount, 1)
|
||||
XCTAssertNil(columns[0].array.asAny(1))
|
||||
let structVal =
|
||||
"\((columns[0].array as? AsString)!.asString(0))"
|
||||
XCTAssertEqual(structVal, "{true,1,2,3,4,5,6,7,8,9.9,10.1,11,12,\(currentDate)}")
|
||||
let structArray = (recordBatch.columns[0].array as? StructArray)!
|
||||
XCTAssertEqual(structArray.length, 3)
|
||||
XCTAssertEqual(structArray.arrowFields!.count, 14)
|
||||
XCTAssertEqual(structArray.arrowFields![0].type.id, .boolean)
|
||||
XCTAssertEqual(structArray.arrowFields![1].type.id, .int8)
|
||||
XCTAssertEqual(structArray.arrowFields![2].type.id, .int16)
|
||||
XCTAssertEqual(structArray.arrowFields![3].type.id, .int32)
|
||||
XCTAssertEqual(structArray.arrowFields![4].type.id, .int64)
|
||||
XCTAssertEqual(structArray.arrowFields![5].type.id, .uint8)
|
||||
XCTAssertEqual(structArray.arrowFields![6].type.id, .uint16)
|
||||
XCTAssertEqual(structArray.arrowFields![7].type.id, .uint32)
|
||||
XCTAssertEqual(structArray.arrowFields![8].type.id, .uint64)
|
||||
XCTAssertEqual(structArray.arrowFields![9].type.id, .double)
|
||||
XCTAssertEqual(structArray.arrowFields![10].type.id, .float)
|
||||
XCTAssertEqual(structArray.arrowFields![11].type.id, .string)
|
||||
XCTAssertEqual(structArray.arrowFields![12].type.id, .binary)
|
||||
XCTAssertEqual(structArray.arrowFields![13].type.id, .date64)
|
||||
}
|
||||
case.failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func testBinaryInMemoryToFromStream() throws {
|
||||
let dataset = try makeBinaryDataset()
|
||||
let writerInfo = ArrowWriter.Info(.recordbatch, schema: dataset.0, batches: [dataset.1])
|
||||
let arrowWriter = ArrowWriter()
|
||||
switch arrowWriter.writeFile(writerInfo) {
|
||||
case .success(let writeData):
|
||||
let arrowReader = ArrowReader()
|
||||
switch arrowReader.readFile(writeData) {
|
||||
case .success(let result):
|
||||
XCTAssertNotNil(result.schema)
|
||||
let schema = result.schema!
|
||||
XCTAssertEqual(schema.fields.count, 1)
|
||||
XCTAssertEqual(schema.fields[0].name, "binary")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowBinary)
|
||||
XCTAssertEqual(result.batches.count, 1)
|
||||
let recordBatch = result.batches[0]
|
||||
XCTAssertEqual(recordBatch.length, 4)
|
||||
let columns = recordBatch.columns
|
||||
let stringVal =
|
||||
"\((columns[0].array as! AsString).asString(1))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(stringVal, "test22")
|
||||
case.failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
func testTimeInMemoryToFromStream() throws {
|
||||
let dataset = try makeTimeDataset()
|
||||
let writerInfo = ArrowWriter.Info(.recordbatch, schema: dataset.0, batches: [dataset.1])
|
||||
let arrowWriter = ArrowWriter()
|
||||
switch arrowWriter.writeFile(writerInfo) {
|
||||
case .success(let writeData):
|
||||
let arrowReader = ArrowReader()
|
||||
switch arrowReader.readFile(writeData) {
|
||||
case .success(let result):
|
||||
XCTAssertNotNil(result.schema)
|
||||
let schema = result.schema!
|
||||
XCTAssertEqual(schema.fields.count, 2)
|
||||
XCTAssertEqual(schema.fields[0].name, "time64")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowTime64)
|
||||
XCTAssertEqual(schema.fields[1].name, "time32")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowTime32)
|
||||
XCTAssertEqual(result.batches.count, 1)
|
||||
let recordBatch = result.batches[0]
|
||||
XCTAssertEqual(recordBatch.length, 4)
|
||||
let columns = recordBatch.columns
|
||||
let stringVal =
|
||||
"\((columns[0].array as! AsString).asString(0))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(stringVal, "12345678")
|
||||
let stringVal2 =
|
||||
"\((columns[1].array as! AsString).asString(3))" // swiftlint:disable:this force_cast
|
||||
XCTAssertEqual(stringVal2, "3")
|
||||
case.failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
// swiftlint:disable:this file_length
|
||||
@@ -1,58 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import XCTest
|
||||
@testable import Arrow
|
||||
|
||||
final class RecordBatchTests: XCTestCase {
|
||||
func testRecordBatch() throws {
|
||||
let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
uint8Builder.append(10)
|
||||
uint8Builder.append(22)
|
||||
uint8Builder.append(nil)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("test10")
|
||||
stringBuilder.append("test22")
|
||||
stringBuilder.append("test33")
|
||||
|
||||
let intHolder = ArrowArrayHolderImpl(try uint8Builder.finish())
|
||||
let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish())
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("col1", arrowArray: intHolder)
|
||||
.addColumn("col2", arrowArray: stringHolder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
let schema = recordBatch.schema
|
||||
XCTAssertEqual(schema.fields.count, 2)
|
||||
XCTAssertEqual(schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8)
|
||||
XCTAssertEqual(schema.fields[0].isNullable, true)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(schema.fields[1].isNullable, false)
|
||||
XCTAssertEqual(recordBatch.columns.count, 2)
|
||||
let col1: ArrowArray<UInt8> = recordBatch.data(for: 0)
|
||||
let col2: ArrowArray<String> = recordBatch.data(for: 1)
|
||||
XCTAssertEqual(col1.length, 3)
|
||||
XCTAssertEqual(col2.length, 3)
|
||||
XCTAssertEqual(col1.nullCount, 1)
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import XCTest
|
||||
@testable import Arrow
|
||||
|
||||
final class TableTests: XCTestCase {
|
||||
func testSchema() throws {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
let schema = schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowInt8), isNullable: true)
|
||||
.addField("col2", type: ArrowType(ArrowType.ArrowBool), isNullable: false)
|
||||
.finish()
|
||||
XCTAssertEqual(schema.fields.count, 2)
|
||||
XCTAssertEqual(schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowInt8)
|
||||
XCTAssertEqual(schema.fields[0].isNullable, true)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowBool)
|
||||
XCTAssertEqual(schema.fields[1].isNullable, false)
|
||||
}
|
||||
|
||||
func testSchemaNested() {
|
||||
class StructTest {
|
||||
var field0: Bool = false
|
||||
var field1: Int8 = 0
|
||||
var field2: Int16 = 0
|
||||
var field3: Int32 = 0
|
||||
var field4: Int64 = 0
|
||||
var field5: UInt8 = 0
|
||||
var field6: UInt16 = 0
|
||||
var field7: UInt32 = 0
|
||||
var field8: UInt64 = 0
|
||||
var field9: Double = 0
|
||||
var field10: Float = 0
|
||||
var field11: String = ""
|
||||
var field12 = Data()
|
||||
var field13: Date = Date.now
|
||||
}
|
||||
|
||||
let testObj = StructTest()
|
||||
var fields = [ArrowField]()
|
||||
let buildStructType = {() -> ArrowNestedType in
|
||||
let mirror = Mirror(reflecting: testObj)
|
||||
for (property, value) in mirror.children {
|
||||
let arrowType = ArrowType(ArrowType.infoForType(type(of: value)))
|
||||
fields.append(ArrowField(property!, type: arrowType, isNullable: true))
|
||||
}
|
||||
|
||||
return ArrowNestedType(ArrowType.ArrowStruct, fields: fields)
|
||||
}
|
||||
|
||||
let structType = buildStructType()
|
||||
XCTAssertEqual(structType.id, ArrowTypeId.strct)
|
||||
XCTAssertEqual(structType.fields.count, 14)
|
||||
XCTAssertEqual(structType.fields[0].type.id, ArrowTypeId.boolean)
|
||||
XCTAssertEqual(structType.fields[1].type.id, ArrowTypeId.int8)
|
||||
XCTAssertEqual(structType.fields[2].type.id, ArrowTypeId.int16)
|
||||
XCTAssertEqual(structType.fields[3].type.id, ArrowTypeId.int32)
|
||||
XCTAssertEqual(structType.fields[4].type.id, ArrowTypeId.int64)
|
||||
XCTAssertEqual(structType.fields[5].type.id, ArrowTypeId.uint8)
|
||||
XCTAssertEqual(structType.fields[6].type.id, ArrowTypeId.uint16)
|
||||
XCTAssertEqual(structType.fields[7].type.id, ArrowTypeId.uint32)
|
||||
XCTAssertEqual(structType.fields[8].type.id, ArrowTypeId.uint64)
|
||||
XCTAssertEqual(structType.fields[9].type.id, ArrowTypeId.double)
|
||||
XCTAssertEqual(structType.fields[10].type.id, ArrowTypeId.float)
|
||||
XCTAssertEqual(structType.fields[11].type.id, ArrowTypeId.string)
|
||||
XCTAssertEqual(structType.fields[12].type.id, ArrowTypeId.binary)
|
||||
XCTAssertEqual(structType.fields[13].type.id, ArrowTypeId.date64)
|
||||
}
|
||||
|
||||
func testTable() throws {
|
||||
let doubleBuilder: NumberArrayBuilder<Double> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
doubleBuilder.append(11.11)
|
||||
doubleBuilder.append(22.22)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("test10")
|
||||
stringBuilder.append("test22")
|
||||
let date32Builder: Date32ArrayBuilder = try ArrowArrayBuilders.loadDate32ArrayBuilder()
|
||||
let date2 = Date(timeIntervalSinceReferenceDate: 86400 * 1)
|
||||
let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
let table = try ArrowTable.Builder()
|
||||
.addColumn("col1", arrowArray: doubleBuilder.finish())
|
||||
.addColumn("col2", arrowArray: stringBuilder.finish())
|
||||
.addColumn("col3", arrowArray: date32Builder.finish())
|
||||
.finish()
|
||||
let schema = table.schema
|
||||
XCTAssertEqual(schema.fields.count, 3)
|
||||
XCTAssertEqual(schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowDouble)
|
||||
XCTAssertEqual(schema.fields[0].isNullable, false)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(schema.fields[1].isNullable, false)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(schema.fields[1].isNullable, false)
|
||||
XCTAssertEqual(table.columns.count, 3)
|
||||
let col1: ChunkedArray<Double> = table.columns[0].data()
|
||||
let col2: ChunkedArray<String> = table.columns[1].data()
|
||||
let col3: ChunkedArray<Date> = table.columns[2].data()
|
||||
XCTAssertEqual(col1.length, 2)
|
||||
XCTAssertEqual(col2.length, 2)
|
||||
XCTAssertEqual(col3.length, 2)
|
||||
XCTAssertEqual(col1[0], 11.11)
|
||||
XCTAssertEqual(col2[1], "test22")
|
||||
}
|
||||
|
||||
func testTableWithChunkedData() throws {
|
||||
let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
uint8Builder.append(10)
|
||||
uint8Builder.append(22)
|
||||
let uint8Builder2: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
uint8Builder2.append(33)
|
||||
let uint8Builder3: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
uint8Builder3.append(44)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("test10")
|
||||
stringBuilder.append("test22")
|
||||
let stringBuilder2 = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("test33")
|
||||
stringBuilder.append("test44")
|
||||
let date32Builder: Date32ArrayBuilder = try ArrowArrayBuilders.loadDate32ArrayBuilder()
|
||||
let date2 = Date(timeIntervalSinceReferenceDate: 86400 * 1)
|
||||
let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
let intArray = try ChunkedArray([uint8Builder.finish(), uint8Builder2.finish(), uint8Builder3.finish()])
|
||||
let stringArray = try ChunkedArray([stringBuilder.finish(), stringBuilder2.finish()])
|
||||
let dateArray = try ChunkedArray([date32Builder.finish()])
|
||||
let table = ArrowTable.Builder()
|
||||
.addColumn("col1", chunked: intArray)
|
||||
.addColumn("col2", chunked: stringArray)
|
||||
.addColumn("col3", chunked: dateArray)
|
||||
.finish()
|
||||
let schema = table.schema
|
||||
XCTAssertEqual(schema.fields.count, 3)
|
||||
XCTAssertEqual(schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8)
|
||||
XCTAssertEqual(schema.fields[0].isNullable, false)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(schema.fields[1].isNullable, false)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(schema.fields[1].isNullable, false)
|
||||
XCTAssertEqual(table.columns.count, 3)
|
||||
let col1: ChunkedArray<UInt8> = table.columns[0].data()
|
||||
let col2: ChunkedArray<String> = table.columns[1].data()
|
||||
let col3: ChunkedArray<Date> = table.columns[2].data()
|
||||
XCTAssertEqual(col1.length, 4)
|
||||
XCTAssertEqual(col2.length, 4)
|
||||
XCTAssertEqual(col3.length, 4)
|
||||
XCTAssertEqual(col1.asString(0), "10")
|
||||
XCTAssertEqual(col1.asString(3), "44")
|
||||
XCTAssertEqual(col2.asString(0), "test10")
|
||||
XCTAssertEqual(col2.asString(2), "test33")
|
||||
}
|
||||
|
||||
func testTableToRecordBatch() throws {
|
||||
let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
uint8Builder.append(10)
|
||||
uint8Builder.append(22)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("test10")
|
||||
stringBuilder.append("test22")
|
||||
let intHolder = ArrowArrayHolderImpl(try uint8Builder.finish())
|
||||
let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish())
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("col1", arrowArray: intHolder)
|
||||
.addColumn("col2", arrowArray: stringHolder)
|
||||
.finish().flatMap({ rb in
|
||||
return ArrowTable.from(recordBatches: [rb])
|
||||
})
|
||||
switch result {
|
||||
case .success(let table):
|
||||
let schema = table.schema
|
||||
XCTAssertEqual(schema.fields.count, 2)
|
||||
XCTAssertEqual(schema.fields[0].name, "col1")
|
||||
XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8)
|
||||
XCTAssertEqual(schema.fields[0].isNullable, false)
|
||||
XCTAssertEqual(schema.fields[1].name, "col2")
|
||||
XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString)
|
||||
XCTAssertEqual(schema.fields[1].isNullable, false)
|
||||
XCTAssertEqual(table.columns.count, 2)
|
||||
let col1: ChunkedArray<UInt8> = table.columns[0].data()
|
||||
let col2: ChunkedArray<String> = table.columns[1].data()
|
||||
XCTAssertEqual(col1.length, 2)
|
||||
XCTAssertEqual(col2.length, 2)
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
.DS_Store
|
||||
/.build
|
||||
/Packages
|
||||
/*.xcodeproj
|
||||
xcuserdata/
|
||||
DerivedData/
|
||||
.swiftpm/
|
||||
.netrc
|
||||
Package.resolved
|
||||
@@ -1,61 +0,0 @@
|
||||
// swift-tools-version: 5.10
|
||||
// The swift-tools-version declares the minimum version of Swift required to build this package.
|
||||
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "ArrowFlight",
|
||||
platforms: [
|
||||
.macOS(.v10_15)
|
||||
],
|
||||
products: [
|
||||
// Products define the executables and libraries a package produces, making them visible to other packages.
|
||||
.library(
|
||||
name: "ArrowFlight",
|
||||
targets: ["ArrowFlight"])
|
||||
],
|
||||
dependencies: [
|
||||
.package(url: "https://github.com/grpc/grpc-swift.git", from: "1.25.0"),
|
||||
.package(url: "https://github.com/apple/swift-protobuf.git", from: "1.29.0"),
|
||||
.package(path: "../Arrow")
|
||||
],
|
||||
targets: [
|
||||
// Targets are the basic building blocks of a package, defining a module or a test suite.
|
||||
// Targets can depend on other targets in this package and products from dependencies.
|
||||
.target(
|
||||
name: "ArrowFlight",
|
||||
dependencies: [
|
||||
.product(name: "Arrow", package: "Arrow"),
|
||||
.product(name: "GRPC", package: "grpc-swift"),
|
||||
.product(name: "SwiftProtobuf", package: "swift-protobuf")
|
||||
],
|
||||
swiftSettings: [
|
||||
// build: .unsafeFlags(["-warnings-as-errors"])
|
||||
]
|
||||
),
|
||||
.testTarget(
|
||||
name: "ArrowFlightTests",
|
||||
dependencies: ["ArrowFlight"],
|
||||
swiftSettings: [
|
||||
// build: .unsafeFlags(["-warnings-as-errors"])
|
||||
]
|
||||
)
|
||||
]
|
||||
)
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,39 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightAction {
|
||||
public let type: String
|
||||
public let body: Data
|
||||
init(_ action: Arrow_Flight_Protocol_Action) {
|
||||
self.type = action.type
|
||||
self.body = action.body
|
||||
}
|
||||
|
||||
public init(_ type: String, body: Data = Data()) {
|
||||
self.type = type
|
||||
self.body = body
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_Action {
|
||||
var flightAction = Arrow_Flight_Protocol_Action()
|
||||
flightAction.type = self.type
|
||||
flightAction.body = self.body
|
||||
return flightAction
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
public class FlightActionType {
|
||||
public let type: String
|
||||
public let description: String
|
||||
init(_ actionType: Arrow_Flight_Protocol_ActionType) {
|
||||
self.type = actionType.type
|
||||
self.description = actionType.description_p
|
||||
|
||||
}
|
||||
public init(_ type: String, description: String) {
|
||||
self.type = type
|
||||
self.description = description
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_ActionType {
|
||||
var actionType = Arrow_Flight_Protocol_ActionType()
|
||||
actionType.type = self.type
|
||||
actionType.description_p = self.description
|
||||
return actionType
|
||||
}
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import struct Foundation.Data
|
||||
import struct Foundation.URL
|
||||
import GRPC
|
||||
import NIOCore
|
||||
import NIOPosix
|
||||
import Arrow
|
||||
|
||||
public class FlightClient {
|
||||
let client: Arrow_Flight_Protocol_FlightServiceAsyncClient
|
||||
let allowReadingUnalignedBuffers: Bool
|
||||
|
||||
public init(channel: GRPCChannel, allowReadingUnalignedBuffers: Bool = false ) {
|
||||
client = Arrow_Flight_Protocol_FlightServiceAsyncClient(channel: channel)
|
||||
self.allowReadingUnalignedBuffers = allowReadingUnalignedBuffers
|
||||
}
|
||||
|
||||
private func readMessages(
|
||||
_ responseStream: GRPCAsyncResponseStream<Arrow_Flight_Protocol_FlightData>
|
||||
) async throws -> ArrowReader.ArrowReaderResult {
|
||||
let reader = ArrowReader()
|
||||
let arrowResult = ArrowReader.makeArrowReaderResult()
|
||||
for try await data in responseStream {
|
||||
switch reader.fromMessage(
|
||||
data.dataHeader,
|
||||
dataBody: data.dataBody,
|
||||
result: arrowResult,
|
||||
useUnalignedBuffers: allowReadingUnalignedBuffers) {
|
||||
case .success:
|
||||
continue
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
return arrowResult
|
||||
}
|
||||
|
||||
private func writeBatches(
|
||||
_ requestStream: GRPCAsyncRequestStreamWriter<Arrow_Flight_Protocol_FlightData>,
|
||||
descriptor: FlightDescriptor,
|
||||
recordBatches: [RecordBatch]
|
||||
) async throws {
|
||||
let writer = ArrowWriter()
|
||||
switch writer.toMessage(recordBatches[0].schema) {
|
||||
case .success(let schemaData):
|
||||
try await requestStream.send(
|
||||
FlightData(
|
||||
schemaData,
|
||||
dataBody: Data(),
|
||||
flightDescriptor: descriptor).toProtocol())
|
||||
for recordBatch in recordBatches {
|
||||
switch writer.toMessage(recordBatch) {
|
||||
case .success(let data):
|
||||
try await requestStream.send(
|
||||
FlightData(
|
||||
data[0],
|
||||
dataBody: data[1],
|
||||
flightDescriptor: descriptor).toProtocol())
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
requestStream.finish()
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public func listActions(_ closure: (FlightActionType) -> Void) async throws {
|
||||
let listActions = client.makeListActionsCall(Arrow_Flight_Protocol_Empty())
|
||||
for try await data in listActions.responseStream {
|
||||
closure(FlightActionType(data))
|
||||
}
|
||||
}
|
||||
|
||||
public func listFlights(
|
||||
_ criteria: FlightCriteria,
|
||||
closure: (FlightInfo) throws -> Void) async throws {
|
||||
let listFlights = client.makeListFlightsCall(criteria.toProtocol())
|
||||
for try await data in listFlights.responseStream {
|
||||
try closure(FlightInfo(data))
|
||||
}
|
||||
}
|
||||
|
||||
public func doAction(_ action: FlightAction, closure: (FlightResult) throws -> Void) async throws {
|
||||
let actionResponse = client.makeDoActionCall(action.toProtocol())
|
||||
for try await data in actionResponse.responseStream {
|
||||
try closure(FlightResult(data))
|
||||
}
|
||||
}
|
||||
|
||||
public func getSchema(_ descriptor: FlightDescriptor) async throws -> FlightSchemaResult {
|
||||
let schemaResultResponse = client.makeGetSchemaCall(descriptor.toProtocol())
|
||||
return FlightSchemaResult(try await schemaResultResponse.response)
|
||||
}
|
||||
|
||||
public func doGet(
|
||||
_ ticket: FlightTicket,
|
||||
readerResultClosure: (ArrowReader.ArrowReaderResult) throws -> Void) async throws {
|
||||
let getResult = client.makeDoGetCall(ticket.toProtocol())
|
||||
try readerResultClosure(try await readMessages(getResult.responseStream))
|
||||
}
|
||||
|
||||
public func doGet(
|
||||
_ ticket: FlightTicket,
|
||||
flightDataClosure: (FlightData) throws -> Void) async throws {
|
||||
let getResult = client.makeDoGetCall(ticket.toProtocol())
|
||||
for try await data in getResult.responseStream {
|
||||
try flightDataClosure(FlightData(data))
|
||||
}
|
||||
}
|
||||
|
||||
public func doPut(
|
||||
_ descriptor: FlightDescriptor,
|
||||
recordBatches: [RecordBatch],
|
||||
closure: (FlightPutResult) throws -> Void) async throws {
|
||||
if recordBatches.isEmpty {
|
||||
throw ArrowFlightError.emptyCollection
|
||||
}
|
||||
|
||||
let putCall = client.makeDoPutCall()
|
||||
try await writeBatches(putCall.requestStream, descriptor: descriptor, recordBatches: recordBatches)
|
||||
var closureCalled = false
|
||||
for try await response in putCall.responseStream {
|
||||
try closure(FlightPutResult(response))
|
||||
closureCalled = true
|
||||
}
|
||||
|
||||
if !closureCalled {
|
||||
try closure(FlightPutResult())
|
||||
}
|
||||
}
|
||||
|
||||
public func doPut(_ flightData: FlightData, closure: (FlightPutResult) throws -> Void) async throws {
|
||||
let putCall = client.makeDoPutCall()
|
||||
try await putCall.requestStream.send(flightData.toProtocol())
|
||||
putCall.requestStream.finish()
|
||||
var closureCalled = false
|
||||
for try await response in putCall.responseStream {
|
||||
try closure(FlightPutResult(response))
|
||||
closureCalled = true
|
||||
}
|
||||
|
||||
if !closureCalled {
|
||||
try closure(FlightPutResult())
|
||||
}
|
||||
}
|
||||
|
||||
public func doExchange(
|
||||
_ descriptor: FlightDescriptor,
|
||||
recordBatches: [RecordBatch],
|
||||
closure: (ArrowReader.ArrowReaderResult) throws -> Void) async throws {
|
||||
if recordBatches.isEmpty {
|
||||
throw ArrowFlightError.emptyCollection
|
||||
}
|
||||
|
||||
let exchangeCall = client.makeDoExchangeCall()
|
||||
try await writeBatches(exchangeCall.requestStream, descriptor: descriptor, recordBatches: recordBatches)
|
||||
try closure(try await readMessages(exchangeCall.responseStream))
|
||||
}
|
||||
|
||||
public func doExchange(flightData: FlightData, closure: (FlightData) throws -> Void) async throws {
|
||||
let exchangeCall = client.makeDoExchangeCall()
|
||||
try await exchangeCall.requestStream.send(flightData.toProtocol())
|
||||
exchangeCall.requestStream.finish()
|
||||
for try await response in exchangeCall.responseStream {
|
||||
try closure(FlightData(response))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightCriteria {
|
||||
let criteria: Arrow_Flight_Protocol_Criteria
|
||||
|
||||
public var expression: Data { criteria.expression }
|
||||
public init(_ expression: Data = Data()) {
|
||||
criteria = Arrow_Flight_Protocol_Criteria.with {
|
||||
$0.expression = expression
|
||||
}
|
||||
}
|
||||
|
||||
init(_ criteria: Arrow_Flight_Protocol_Criteria) {
|
||||
self.criteria = criteria
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_Criteria {
|
||||
return criteria
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightData {
|
||||
let flightData: Arrow_Flight_Protocol_FlightData
|
||||
public var flightDescriptor: FlightDescriptor? {
|
||||
return flightData.hasFlightDescriptor ? FlightDescriptor(flightData.flightDescriptor) : nil
|
||||
}
|
||||
|
||||
public var dataHeader: Data { flightData.dataHeader }
|
||||
|
||||
public var dataBody: Data { flightData.dataBody }
|
||||
|
||||
init(_ flightData: Arrow_Flight_Protocol_FlightData) {
|
||||
self.flightData = flightData
|
||||
}
|
||||
|
||||
public init(_ dataHeader: Data, dataBody: Data, flightDescriptor: FlightDescriptor? = nil) {
|
||||
if flightDescriptor != nil {
|
||||
self.flightData = Arrow_Flight_Protocol_FlightData.with {
|
||||
$0.dataHeader = dataHeader
|
||||
$0.dataBody = dataBody
|
||||
$0.flightDescriptor = flightDescriptor!.toProtocol()
|
||||
}
|
||||
} else {
|
||||
self.flightData = Arrow_Flight_Protocol_FlightData.with {
|
||||
$0.dataBody = dataBody
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_FlightData { self.flightData }
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightDescriptor {
|
||||
public enum DescriptorType {
|
||||
case unknown
|
||||
case path
|
||||
case cmd
|
||||
}
|
||||
|
||||
public let type: FlightDescriptor.DescriptorType
|
||||
public let cmd: Data
|
||||
public let paths: [String]
|
||||
|
||||
init(_ descriptor: Arrow_Flight_Protocol_FlightDescriptor) {
|
||||
self.type = descriptor.type == .cmd ? .cmd : .path
|
||||
self.cmd = descriptor.cmd
|
||||
self.paths = descriptor.path
|
||||
}
|
||||
|
||||
public init(cmd: Data) {
|
||||
self.type = .cmd
|
||||
self.cmd = cmd
|
||||
self.paths = [String]()
|
||||
}
|
||||
|
||||
public init(paths: [String]) {
|
||||
self.type = .path
|
||||
self.cmd = Data()
|
||||
self.paths = paths
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_FlightDescriptor {
|
||||
var descriptor = Arrow_Flight_Protocol_FlightDescriptor()
|
||||
descriptor.type = self.type == .cmd ? .cmd : .path
|
||||
descriptor.cmd = self.cmd
|
||||
descriptor.path = self.paths
|
||||
return descriptor
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
public class FlightEndpoint {
|
||||
let ticket: FlightTicket
|
||||
let locations: [FlightLocation]
|
||||
init(_ endpoint: Arrow_Flight_Protocol_FlightEndpoint) {
|
||||
self.ticket = FlightTicket(endpoint.ticket.ticket)
|
||||
self.locations = endpoint.location.map {return FlightLocation($0)}
|
||||
}
|
||||
|
||||
public init(_ ticket: FlightTicket, locations: [FlightLocation]) {
|
||||
self.ticket = ticket
|
||||
self.locations = locations
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_FlightEndpoint {
|
||||
var endpoint = Arrow_Flight_Protocol_FlightEndpoint()
|
||||
endpoint.ticket = self.ticket.toProtocol()
|
||||
endpoint.location = self.locations.map { $0.toProtocol() }
|
||||
return endpoint
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Arrow
|
||||
|
||||
public class FlightInfo {
|
||||
let flightInfo: Arrow_Flight_Protocol_FlightInfo
|
||||
public var flightDescriptor: FlightDescriptor? {
|
||||
return flightInfo.hasFlightDescriptor ? FlightDescriptor(flightInfo.flightDescriptor) : nil
|
||||
}
|
||||
|
||||
public var endpoints: [FlightEndpoint] {
|
||||
return self.flightInfo.endpoint.map { FlightEndpoint($0) }
|
||||
}
|
||||
public var schema: ArrowSchema? {
|
||||
return schemaFromMessage(self.flightInfo.schema)
|
||||
}
|
||||
|
||||
var endpoint: [Arrow_Flight_Protocol_FlightEndpoint] = []
|
||||
init(_ flightInfo: Arrow_Flight_Protocol_FlightInfo) {
|
||||
self.flightInfo = flightInfo
|
||||
}
|
||||
|
||||
public init(_ schema: Data, endpoints: [FlightEndpoint] = [FlightEndpoint](), descriptor: FlightDescriptor? = nil) {
|
||||
if let localDescriptor = descriptor {
|
||||
self.flightInfo = Arrow_Flight_Protocol_FlightInfo.with {
|
||||
$0.schema = schema
|
||||
$0.flightDescriptor = localDescriptor.toProtocol()
|
||||
$0.endpoint = endpoints.map { $0.toProtocol() }
|
||||
}
|
||||
} else {
|
||||
self.flightInfo = Arrow_Flight_Protocol_FlightInfo.with {
|
||||
$0.schema = schema
|
||||
$0.endpoint = endpoints.map { $0.toProtocol() }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_FlightInfo {
|
||||
return self.flightInfo
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightLocation {
|
||||
public let uri: String
|
||||
|
||||
init(_ location: Arrow_Flight_Protocol_Location) {
|
||||
self.uri = location.uri
|
||||
}
|
||||
|
||||
public init(_ uri: String) {
|
||||
self.uri = uri
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_Location {
|
||||
var location = Arrow_Flight_Protocol_Location()
|
||||
location.uri = uri
|
||||
return location
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightPutResult {
|
||||
public let appMetadata: Data
|
||||
public init(_ appMetadata: Data = Data()) {
|
||||
self.appMetadata = appMetadata
|
||||
}
|
||||
|
||||
init(_ putResult: Arrow_Flight_Protocol_PutResult) {
|
||||
self.appMetadata = putResult.appMetadata
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_PutResult {
|
||||
var putResult = Arrow_Flight_Protocol_PutResult()
|
||||
putResult.appMetadata = self.appMetadata
|
||||
return putResult
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightResult {
|
||||
public let body: Data
|
||||
init(_ result: Arrow_Flight_Protocol_Result) {
|
||||
self.body = result.body
|
||||
}
|
||||
|
||||
public init(_ body: Data) {
|
||||
self.body = body
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_Result {
|
||||
var result = Arrow_Flight_Protocol_Result()
|
||||
result.body = self.body
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Arrow
|
||||
|
||||
public class FlightSchemaResult {
|
||||
let schemaResult: Arrow_Flight_Protocol_SchemaResult
|
||||
|
||||
public var schema: ArrowSchema? {
|
||||
return schemaFromMessage(self.schemaResult.schema)
|
||||
}
|
||||
|
||||
public init(_ schema: Data) {
|
||||
self.schemaResult = Arrow_Flight_Protocol_SchemaResult.with {
|
||||
$0.schema = schema
|
||||
}
|
||||
}
|
||||
|
||||
init(_ schemaResult: Arrow_Flight_Protocol_SchemaResult) {
|
||||
self.schemaResult = schemaResult
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_SchemaResult {
|
||||
return schemaResult
|
||||
}
|
||||
}
|
||||
@@ -1,194 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import GRPC
|
||||
import NIO
|
||||
import NIOConcurrencyHelpers
|
||||
import SwiftProtobuf
|
||||
import Arrow
|
||||
|
||||
public enum ArrowFlightError: Error {
|
||||
case unknown(String?)
|
||||
case notImplemented(String? = nil)
|
||||
case emptyCollection
|
||||
case ioError(String? = nil)
|
||||
}
|
||||
|
||||
public func schemaToMessage(_ schema: ArrowSchema) throws -> Data {
|
||||
let arrowWriter = ArrowWriter()
|
||||
switch arrowWriter.toMessage(schema) {
|
||||
case .success(let result):
|
||||
var outputResult = Data()
|
||||
withUnsafeBytes(of: Int32(0).littleEndian) {outputResult.append(Data($0))}
|
||||
withUnsafeBytes(of: Int32(result.count).littleEndian) {outputResult.append(Data($0))}
|
||||
outputResult.append(result)
|
||||
return outputResult
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public func schemaFromMessage(_ schemaData: Data) -> ArrowSchema? {
|
||||
let messageLength = schemaData.withUnsafeBytes { rawBuffer in
|
||||
rawBuffer.loadUnaligned(fromByteOffset: 4, as: Int32.self)
|
||||
}
|
||||
|
||||
let startIndex = schemaData.count - Int(messageLength)
|
||||
let schema = schemaData[startIndex...]
|
||||
|
||||
let reader = ArrowReader()
|
||||
let result = ArrowReader.makeArrowReaderResult()
|
||||
switch reader.fromMessage(schema, dataBody: Data(), result: result) {
|
||||
case .success:
|
||||
return result.schema!
|
||||
case .failure:
|
||||
// TODO: add handling of error swiftlint:disable:this todo
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
public protocol ArrowFlightServer: Sendable {
|
||||
var allowReadingUnalignedBuffers: Bool { get }
|
||||
func listFlights(_ criteria: FlightCriteria, writer: FlightInfoStreamWriter) async throws
|
||||
func getFlightInfo(_ request: FlightDescriptor) async throws -> FlightInfo
|
||||
func getSchema(_ request: FlightDescriptor) async throws -> ArrowFlight.FlightSchemaResult
|
||||
func listActions(_ writer: ActionTypeStreamWriter) async throws
|
||||
func doAction(_ action: FlightAction, writer: ResultStreamWriter) async throws
|
||||
func doGet(_ ticket: FlightTicket, writer: RecordBatchStreamWriter) async throws
|
||||
func doPut(_ reader: RecordBatchStreamReader, writer: PutResultDataStreamWriter) async throws
|
||||
func doExchange(_ reader: RecordBatchStreamReader, writer: RecordBatchStreamWriter) async throws
|
||||
}
|
||||
|
||||
extension ArrowFlightServer {
|
||||
var allowReadingUnalignedBuffers: Bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
public func makeFlightServer(_ handler: ArrowFlightServer) -> CallHandlerProvider {
|
||||
return InternalFlightServer(handler)
|
||||
}
|
||||
|
||||
internal final class InternalFlightServer: Arrow_Flight_Protocol_FlightServiceAsyncProvider {
|
||||
let arrowFlightServer: ArrowFlightServer?
|
||||
|
||||
init(_ arrowFlightServer: ArrowFlightServer?) {
|
||||
self.arrowFlightServer = arrowFlightServer
|
||||
}
|
||||
|
||||
func handshake(requestStream: GRPC.GRPCAsyncRequestStream<Arrow_Flight_Protocol_HandshakeRequest>,
|
||||
responseStream: GRPC.GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_HandshakeResponse>,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws {
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func listFlights(request: Arrow_Flight_Protocol_Criteria,
|
||||
responseStream: GRPC.GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_FlightInfo>,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws {
|
||||
if let server = arrowFlightServer {
|
||||
let writer = FlightInfoStreamWriter(responseStream)
|
||||
try await server.listFlights(FlightCriteria(request), writer: writer)
|
||||
return
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func getFlightInfo(request: Arrow_Flight_Protocol_FlightDescriptor,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws -> Arrow_Flight_Protocol_FlightInfo {
|
||||
if let server = arrowFlightServer {
|
||||
return try await server.getFlightInfo(FlightDescriptor(request)).toProtocol()
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func getSchema(request: Arrow_Flight_Protocol_FlightDescriptor,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws -> Arrow_Flight_Protocol_SchemaResult {
|
||||
if let server = arrowFlightServer {
|
||||
return try await server.getSchema(FlightDescriptor(request)).toProtocol()
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func doGet(request: Arrow_Flight_Protocol_Ticket,
|
||||
responseStream: GRPC.GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_FlightData>,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws {
|
||||
if let server = arrowFlightServer {
|
||||
let writer = RecordBatchStreamWriter(responseStream)
|
||||
let ticket = FlightTicket(request)
|
||||
try await server.doGet(ticket, writer: writer)
|
||||
return
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func doPut(requestStream: GRPC.GRPCAsyncRequestStream<Arrow_Flight_Protocol_FlightData>,
|
||||
responseStream: GRPC.GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_PutResult>,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws {
|
||||
if let server = arrowFlightServer {
|
||||
let reader = RecordBatchStreamReader(requestStream)
|
||||
let writer = PutResultDataStreamWriter(responseStream)
|
||||
try await server.doPut(reader, writer: writer)
|
||||
return
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func doExchange(requestStream: GRPC.GRPCAsyncRequestStream<Arrow_Flight_Protocol_FlightData>,
|
||||
responseStream: GRPC.GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_FlightData>,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws {
|
||||
if let server = arrowFlightServer {
|
||||
let reader = RecordBatchStreamReader(requestStream)
|
||||
let writer = RecordBatchStreamWriter(responseStream)
|
||||
try await server.doExchange(reader, writer: writer)
|
||||
return
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func doAction(request: Arrow_Flight_Protocol_Action,
|
||||
responseStream: GRPC.GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_Result>,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws {
|
||||
if let server = arrowFlightServer {
|
||||
try await server.doAction(FlightAction(request), writer: ResultStreamWriter(responseStream))
|
||||
return
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
func listActions(request: Arrow_Flight_Protocol_Empty,
|
||||
responseStream: GRPC.GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_ActionType>,
|
||||
context: GRPC.GRPCAsyncServerCallContext) async throws {
|
||||
if let server = arrowFlightServer {
|
||||
let writer = ActionTypeStreamWriter(responseStream)
|
||||
try await server.listActions(writer)
|
||||
return
|
||||
}
|
||||
|
||||
throw ArrowFlightError.notImplemented()
|
||||
}
|
||||
|
||||
internal var interceptors: Arrow_Flight_Protocol_FlightServiceServerInterceptorFactoryProtocol? { return nil }
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,35 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class FlightTicket {
|
||||
public let data: Data
|
||||
init(_ ticket: Arrow_Flight_Protocol_Ticket) {
|
||||
self.data = ticket.ticket
|
||||
}
|
||||
|
||||
public init(_ data: Data) {
|
||||
self.data = data
|
||||
}
|
||||
|
||||
func toProtocol() -> Arrow_Flight_Protocol_Ticket {
|
||||
var ticket = Arrow_Flight_Protocol_Ticket()
|
||||
ticket.ticket = self.data
|
||||
return ticket
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Arrow
|
||||
import GRPC
|
||||
|
||||
public class RecordBatchStreamReader: AsyncSequence, AsyncIteratorProtocol {
|
||||
public typealias AsyncIterator = RecordBatchStreamReader
|
||||
public typealias Element = (Arrow.RecordBatch?, FlightDescriptor?)
|
||||
let reader = ArrowReader()
|
||||
var batches = [RecordBatch]()
|
||||
var descriptor: FlightDescriptor?
|
||||
var batchIndex = 0
|
||||
var streamIterator: any AsyncIteratorProtocol
|
||||
var useUnalignedBuffers: Bool
|
||||
let stream: GRPC.GRPCAsyncRequestStream<Arrow_Flight_Protocol_FlightData>
|
||||
init(_ stream: GRPC.GRPCAsyncRequestStream<Arrow_Flight_Protocol_FlightData>,
|
||||
useUnalignedBuffers: Bool = false) {
|
||||
self.stream = stream
|
||||
self.streamIterator = self.stream.makeAsyncIterator()
|
||||
self.useUnalignedBuffers = useUnalignedBuffers
|
||||
}
|
||||
|
||||
public func next() async throws -> (Arrow.RecordBatch?, FlightDescriptor?)? {
|
||||
guard !Task.isCancelled else {
|
||||
return nil
|
||||
}
|
||||
|
||||
if batchIndex < batches.count {
|
||||
let batch = batches[batchIndex]
|
||||
batchIndex += 1
|
||||
return (batch, descriptor)
|
||||
}
|
||||
|
||||
let result = ArrowReader.makeArrowReaderResult()
|
||||
while true {
|
||||
let streamData = try await self.streamIterator.next()
|
||||
if streamData == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
let flightData = (streamData as? Arrow_Flight_Protocol_FlightData)!
|
||||
let dataBody = flightData.dataBody
|
||||
let dataHeader = flightData.dataHeader
|
||||
descriptor = FlightDescriptor(flightData.flightDescriptor)
|
||||
switch reader.fromMessage(
|
||||
dataHeader,
|
||||
dataBody: dataBody,
|
||||
result: result,
|
||||
useUnalignedBuffers: useUnalignedBuffers) {
|
||||
case .success(()):
|
||||
if result.batches.count > 0 {
|
||||
batches = result.batches
|
||||
batchIndex = 1
|
||||
return (batches[0], descriptor)
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func makeAsyncIterator() -> RecordBatchStreamReader {
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Arrow
|
||||
import GRPC
|
||||
|
||||
public class ActionTypeStreamWriter {
|
||||
let stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_ActionType>
|
||||
init(_ stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_ActionType>) {
|
||||
self.stream = stream
|
||||
}
|
||||
|
||||
public func write(_ actionType: FlightActionType) async throws {
|
||||
try await self.stream.send(actionType.toProtocol())
|
||||
}
|
||||
}
|
||||
|
||||
public class ResultStreamWriter {
|
||||
let stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_Result>
|
||||
init(_ stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_Result>) {
|
||||
self.stream = stream
|
||||
}
|
||||
|
||||
public func write(_ result: FlightResult) async throws {
|
||||
try await self.stream.send(result.toProtocol())
|
||||
}
|
||||
}
|
||||
|
||||
public class FlightInfoStreamWriter {
|
||||
let stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_FlightInfo>
|
||||
init(_ stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_FlightInfo>) {
|
||||
self.stream = stream
|
||||
}
|
||||
|
||||
public func write(_ result: FlightInfo) async throws {
|
||||
try await self.stream.send(result.toProtocol())
|
||||
}
|
||||
}
|
||||
|
||||
public class PutResultDataStreamWriter {
|
||||
let stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_PutResult>
|
||||
init(_ stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_PutResult>) {
|
||||
self.stream = stream
|
||||
}
|
||||
|
||||
public func write(_ result: FlightPutResult) async throws {
|
||||
try await self.stream.send(result.toProtocol())
|
||||
}
|
||||
}
|
||||
|
||||
public class RecordBatchStreamWriter {
|
||||
let writer = ArrowWriter()
|
||||
let stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_FlightData>
|
||||
init(_ stream: GRPCAsyncResponseStreamWriter<Arrow_Flight_Protocol_FlightData>) {
|
||||
self.stream = stream
|
||||
}
|
||||
|
||||
public func write(_ rb: RecordBatch) async throws {
|
||||
switch writer.toMessage(rb.schema) {
|
||||
case .success(let schemaData):
|
||||
let schemaFlightData = Arrow_Flight_Protocol_FlightData.with {
|
||||
$0.dataHeader = schemaData
|
||||
}
|
||||
|
||||
try await self.stream.send(schemaFlightData)
|
||||
switch writer.toMessage(rb) {
|
||||
case .success(let recordMessages):
|
||||
let rbMessage = Arrow_Flight_Protocol_FlightData.with {
|
||||
$0.dataHeader = recordMessages[0]
|
||||
$0.dataBody = recordMessages[1]
|
||||
}
|
||||
|
||||
try await self.stream.send(rbMessage)
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,357 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import XCTest
|
||||
import struct Foundation.Data
|
||||
import struct Foundation.URL
|
||||
import GRPC
|
||||
import NIOCore
|
||||
import NIOPosix
|
||||
import Arrow
|
||||
|
||||
@testable import ArrowFlight
|
||||
|
||||
func makeSchema() -> ArrowSchema {
|
||||
let schemaBuilder = ArrowSchema.Builder()
|
||||
return schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowDouble), isNullable: true)
|
||||
.addField("col2", type: ArrowType(ArrowType.ArrowString), isNullable: false)
|
||||
.addField("col3", type: ArrowType(ArrowType.ArrowDate32), isNullable: false)
|
||||
.finish()
|
||||
}
|
||||
|
||||
func makeRecordBatch() throws -> RecordBatch {
|
||||
let doubleBuilder: NumberArrayBuilder<Double> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
doubleBuilder.append(11.11)
|
||||
doubleBuilder.append(22.22)
|
||||
doubleBuilder.append(33.33)
|
||||
doubleBuilder.append(44.44)
|
||||
let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
stringBuilder.append("test10")
|
||||
stringBuilder.append("test22")
|
||||
stringBuilder.append("test33")
|
||||
stringBuilder.append("test44")
|
||||
let date32Builder = try ArrowArrayBuilders.loadDate32ArrayBuilder()
|
||||
let date2 = Date(timeIntervalSinceReferenceDate: 86400 * 1)
|
||||
let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
date32Builder.append(date1)
|
||||
date32Builder.append(date2)
|
||||
let doubleHolder = ArrowArrayHolderImpl(try doubleBuilder.finish())
|
||||
let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish())
|
||||
let date32Holder = ArrowArrayHolderImpl(try date32Builder.finish())
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("col1", arrowArray: doubleHolder)
|
||||
.addColumn("col2", arrowArray: stringHolder)
|
||||
.addColumn("col3", arrowArray: date32Holder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
return recordBatch
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
var flights = [String: FlightInfo]()
|
||||
final class MyFlightServer: ArrowFlightServer {
|
||||
func doExchange(
|
||||
_ reader: ArrowFlight.RecordBatchStreamReader,
|
||||
writer: ArrowFlight.RecordBatchStreamWriter) async throws {
|
||||
do {
|
||||
for try await rbData in reader {
|
||||
let rb = rbData.0!
|
||||
XCTAssertEqual(rb.schema.fields.count, 3)
|
||||
XCTAssertEqual(rb.length, 4)
|
||||
}
|
||||
|
||||
let rb = try makeRecordBatch()
|
||||
try await writer.write(rb)
|
||||
} catch {
|
||||
print("Unknown error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
func doPut(
|
||||
_ reader: ArrowFlight.RecordBatchStreamReader,
|
||||
writer: ArrowFlight.PutResultDataStreamWriter) async throws {
|
||||
for try await rbData in reader {
|
||||
let rb = rbData.0!
|
||||
let key = String(decoding: rbData.1!.cmd, as: UTF8.self)
|
||||
flights[key] = try FlightInfo(schemaToMessage(rb.schema), endpoints: [], descriptor: rbData.1)
|
||||
XCTAssertEqual(rb.schema.fields.count, 3)
|
||||
XCTAssertEqual(rb.length, 4)
|
||||
try await writer.write(FlightPutResult())
|
||||
}
|
||||
}
|
||||
|
||||
func doGet(_ ticket: ArrowFlight.FlightTicket, writer: ArrowFlight.RecordBatchStreamWriter) async throws {
|
||||
try await writer.write(try makeRecordBatch())
|
||||
}
|
||||
|
||||
func getSchema(_ request: ArrowFlight.FlightDescriptor) async throws -> ArrowFlight.FlightSchemaResult {
|
||||
XCTAssertEqual(String(bytes: request.cmd, encoding: .utf8)!, "schema info")
|
||||
XCTAssertEqual(request.type, .cmd)
|
||||
return try ArrowFlight.FlightSchemaResult(schemaToMessage(makeSchema()))
|
||||
}
|
||||
|
||||
func getFlightInfo(_ request: ArrowFlight.FlightDescriptor) async throws -> ArrowFlight.FlightInfo {
|
||||
let key = String(decoding: request.cmd, as: UTF8.self)
|
||||
if flights[key] != nil {
|
||||
return ArrowFlight.FlightInfo(flights[key]!.toProtocol())
|
||||
}
|
||||
|
||||
throw ArrowFlightError.ioError("Flight not found")
|
||||
}
|
||||
|
||||
func listFlights(_ criteria: ArrowFlight.FlightCriteria, writer: ArrowFlight.FlightInfoStreamWriter) async throws {
|
||||
XCTAssertEqual(String(bytes: criteria.expression, encoding: .utf8), "flight criteria expression")
|
||||
for flightData in flights {
|
||||
try await writer.write(flightData.value)
|
||||
}
|
||||
}
|
||||
|
||||
func listActions(_ writer: ArrowFlight.ActionTypeStreamWriter) async throws {
|
||||
try await writer.write(FlightActionType("clear", description: "Clear the stored flights."))
|
||||
try await writer.write(FlightActionType("shutdown", description: "Shut down this server."))
|
||||
}
|
||||
|
||||
func doAction(_ action: FlightAction, writer: ResultStreamWriter) async throws {
|
||||
XCTAssertEqual(action.type, "healthcheck")
|
||||
XCTAssertEqual(String(bytes: action.body, encoding: .utf8)!, "healthcheck body")
|
||||
try await writer.write(FlightResult("test_action result".data(using: .utf8)!))
|
||||
}
|
||||
}
|
||||
|
||||
struct FlightServerImpl {
|
||||
var port = 1234
|
||||
static var server: Server?
|
||||
static var group: MultiThreadedEventLoopGroup?
|
||||
static func run() async throws {
|
||||
do {
|
||||
// Create an event loop group for the server to run on.
|
||||
let group = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount)
|
||||
// Create a provider using the features we read.
|
||||
let provider = ArrowFlight.makeFlightServer(MyFlightServer())
|
||||
|
||||
// Start the server and print its address once it has started.
|
||||
FlightServerImpl.server = try await Server.insecure(group: group)
|
||||
.withServiceProviders([provider])
|
||||
.bind(host: "localhost", port: 8088)
|
||||
.get()
|
||||
|
||||
print("server started on port \(server!.channel.localAddress!.port!)")
|
||||
// Wait on the server's `onClose` future to stop the program from exiting.
|
||||
} catch {
|
||||
print("Unknown server error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class FlightClientTester {
|
||||
var client: FlightClient?
|
||||
var group: MultiThreadedEventLoopGroup?
|
||||
var channel: GRPCChannel?
|
||||
|
||||
init() async throws {
|
||||
// Load the features.
|
||||
let group = PlatformSupport.makeEventLoopGroup(loopCount: 1)
|
||||
let channel = try GRPCChannelPool.with(
|
||||
target: .host("localhost", port: 8088),
|
||||
transportSecurity: .plaintext,
|
||||
eventLoopGroup: group
|
||||
)
|
||||
|
||||
client = FlightClient(channel: channel)
|
||||
}
|
||||
|
||||
deinit {
|
||||
try? group?.syncShutdownGracefully()
|
||||
try? channel?.close().wait()
|
||||
}
|
||||
|
||||
func listActionTest() async throws {
|
||||
var actionTypes = [FlightActionType]()
|
||||
try await client?.listActions({ action in
|
||||
actionTypes.append(action)
|
||||
})
|
||||
|
||||
XCTAssertEqual(actionTypes.count, 2)
|
||||
|
||||
XCTAssertEqual(actionTypes[0].type, "clear")
|
||||
XCTAssertEqual(actionTypes[0].description, "Clear the stored flights.")
|
||||
XCTAssertEqual(actionTypes[1].type, "shutdown")
|
||||
XCTAssertEqual(actionTypes[1].description, "Shut down this server.")
|
||||
}
|
||||
|
||||
func listFlightsTest() async throws {
|
||||
let flightCriteria = FlightCriteria("flight criteria expression".data(using: .utf8)!)
|
||||
var numCalls = 0
|
||||
try await client?.listFlights(flightCriteria, closure: { data in
|
||||
if let schema = data.schema {
|
||||
XCTAssertGreaterThanOrEqual(schema.fields.count, 0)
|
||||
numCalls += 1
|
||||
}
|
||||
})
|
||||
|
||||
XCTAssertEqual(numCalls, 2)
|
||||
}
|
||||
|
||||
func doActionTest(_ type: String, actionBody: Data) async throws {
|
||||
let action = FlightAction(type, body: actionBody)
|
||||
var actionResults = [FlightResult]()
|
||||
try await client?.doAction(action, closure: { result in
|
||||
actionResults.append(result)
|
||||
})
|
||||
|
||||
XCTAssertEqual(actionResults.count, 1)
|
||||
XCTAssertEqual(String(bytes: actionResults[0].body, encoding: .utf8), "test_action result")
|
||||
}
|
||||
|
||||
func getSchemaTest(_ cmd: Data) async throws {
|
||||
let descriptor = FlightDescriptor(cmd: cmd)
|
||||
let schemaResult = try await client?.getSchema(descriptor)
|
||||
let schema = schemaResult!.schema!
|
||||
XCTAssertEqual(schema.fields.count, 3)
|
||||
}
|
||||
|
||||
func doGetTest(_ flightData: Data) async throws {
|
||||
let ticket = FlightTicket(flightData)
|
||||
var numCall = 0
|
||||
try await client?.doGet(ticket, readerResultClosure: { rb in
|
||||
numCall += 1
|
||||
XCTAssertEqual(rb.schema!.fields.count, 3)
|
||||
XCTAssertEqual(rb.batches[0].length, 4)
|
||||
switch ArrowTable.from(recordBatches: rb.batches) {
|
||||
case .success(let table):
|
||||
for column in table.columns {
|
||||
switch column.type.id {
|
||||
case .double:
|
||||
let doubleArray = column.data() as? ChunkedArray<Double>
|
||||
XCTAssertNotNil(doubleArray)
|
||||
XCTAssertEqual(doubleArray?[0], 11.11)
|
||||
XCTAssertEqual(doubleArray?.asString(0), "11.11")
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
})
|
||||
|
||||
XCTAssertEqual(numCall, 1)
|
||||
}
|
||||
|
||||
func doGetTestFlightData(_ flightData: Data) async throws {
|
||||
let ticket = FlightTicket(flightData)
|
||||
var numCall = 0
|
||||
let reader = ArrowReader()
|
||||
let arrowResult = ArrowReader.makeArrowReaderResult()
|
||||
try await client?.doGet(ticket, flightDataClosure: { flightData in
|
||||
switch reader.fromMessage(flightData.dataHeader, dataBody: flightData.dataBody, result: arrowResult) {
|
||||
case .success:
|
||||
numCall += 1
|
||||
case .failure(let error):
|
||||
throw error
|
||||
}
|
||||
})
|
||||
|
||||
XCTAssertEqual(numCall, 2)
|
||||
}
|
||||
|
||||
func doPutTest(_ cmd: String) async throws {
|
||||
let descriptor = FlightDescriptor(cmd: cmd.data(using: .utf8)!)
|
||||
let rb = try makeRecordBatch()
|
||||
var numCall = 0
|
||||
try await client?.doPut(descriptor, recordBatches: [rb], closure: { _ in
|
||||
numCall += 1
|
||||
})
|
||||
|
||||
XCTAssertEqual(numCall, 1)
|
||||
}
|
||||
|
||||
func doExchangeTest() async throws {
|
||||
let descriptor = FlightDescriptor(cmd: "flight_ticket".data(using: .utf8)!)
|
||||
let rb = try makeRecordBatch()
|
||||
var numCall = 0
|
||||
try await client?.doExchange(descriptor, recordBatches: [rb], closure: { result in
|
||||
numCall += 1
|
||||
XCTAssertEqual(result.schema?.fields.count, 3)
|
||||
XCTAssertEqual(result.batches[0].length, 4)
|
||||
})
|
||||
|
||||
XCTAssertEqual(numCall, 1)
|
||||
}
|
||||
}
|
||||
|
||||
actor FlightServerData {
|
||||
public var serverup = false
|
||||
func setServerUp(_ serverUp: Bool) {
|
||||
self.serverup = serverUp
|
||||
}
|
||||
|
||||
func isServerUp() -> Bool {
|
||||
return serverup
|
||||
}
|
||||
}
|
||||
|
||||
final class FlightTest: XCTestCase {
|
||||
let serverData = FlightServerData()
|
||||
|
||||
func testFlightServer() async throws {
|
||||
let basicTask = Task {
|
||||
try await FlightServerImpl.run()
|
||||
defer {
|
||||
print("server shutting down")
|
||||
do {
|
||||
try FlightServerImpl.group?.syncShutdownGracefully()
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
|
||||
await serverData.setServerUp(true)
|
||||
try await FlightServerImpl.server?.onClose.get()
|
||||
return "done"
|
||||
}
|
||||
|
||||
let secondTask = Task {
|
||||
defer {
|
||||
_ = FlightServerImpl.server?.close()
|
||||
}
|
||||
|
||||
while await !serverData.isServerUp() {
|
||||
try await Task.sleep(nanoseconds: 1_000_000)
|
||||
}
|
||||
|
||||
let clientImpl = try await FlightClientTester()
|
||||
try await clientImpl.listActionTest()
|
||||
try await clientImpl.doPutTest("flight_ticket")
|
||||
try await clientImpl.doPutTest("flight_another")
|
||||
try await clientImpl.listFlightsTest()
|
||||
try await clientImpl.doActionTest("healthcheck", actionBody: Data("healthcheck body".utf8))
|
||||
try await clientImpl.getSchemaTest(Data("schema info".utf8))
|
||||
try await clientImpl.doGetTest(Data("'flight_ticket'".utf8))
|
||||
try await clientImpl.doGetTestFlightData(Data("'flight_another'".utf8))
|
||||
try await clientImpl.doExchangeTest()
|
||||
return "done"
|
||||
}
|
||||
|
||||
_ = try await [basicTask.value, secondTask.value]
|
||||
print("done running")
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
.DS_Store
|
||||
/.build
|
||||
/Packages
|
||||
xcuserdata/
|
||||
DerivedData/
|
||||
.swiftpm/configuration/registries.json
|
||||
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
|
||||
.netrc
|
||||
@@ -1,43 +0,0 @@
|
||||
// swift-tools-version: 5.10
|
||||
// The swift-tools-version declares the minimum version of Swift required to build this package.
|
||||
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "go-swift",
|
||||
platforms: [
|
||||
.macOS(.v10_14)
|
||||
],
|
||||
products: [
|
||||
.library(
|
||||
name: "go-swift",
|
||||
type: .static,
|
||||
targets: ["go-swift"])
|
||||
],
|
||||
dependencies: [
|
||||
.package(path: "../Arrow") // 👈 Reference to a Local Package
|
||||
],
|
||||
targets: [
|
||||
.target(
|
||||
name: "go-swift",
|
||||
dependencies: [
|
||||
.product(name: "Arrow", package: "Arrow")
|
||||
])
|
||||
]
|
||||
)
|
||||
@@ -1,132 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Arrow
|
||||
import ArrowC
|
||||
|
||||
@_cdecl("stringTypeFromSwift")
|
||||
func stringTypeFromSwift(cSchema: UnsafePointer<ArrowC.ArrowSchema>) {
|
||||
let unsafePointer = UnsafeMutablePointer(mutating: cSchema)
|
||||
let exporter = ArrowCExporter()
|
||||
switch exporter.exportType(&unsafePointer.pointee, arrowType: ArrowType(ArrowType.ArrowString), name: "col1") {
|
||||
case .success:
|
||||
return
|
||||
case .failure(let err):
|
||||
fatalError("Error exporting string type from swift: \(err)")
|
||||
}
|
||||
}
|
||||
|
||||
@_cdecl("stringTypeToSwift")
|
||||
func stringTypeToSwift(cSchema: UnsafePointer<ArrowC.ArrowSchema>) {
|
||||
let importer = ArrowCImporter()
|
||||
switch importer.importField(cSchema.pointee) {
|
||||
case .success(let field):
|
||||
if field.name != "col1" {
|
||||
fatalError("Field name was incorrect expected: col1 but found: \(field.name)")
|
||||
}
|
||||
|
||||
if field.type.id != ArrowTypeId.string {
|
||||
fatalError("Field type was incorrect expected: string but found: \(field.type.id)")
|
||||
}
|
||||
case .failure(let err):
|
||||
fatalError("Error importing string type to swift: \(err)")
|
||||
}
|
||||
}
|
||||
|
||||
@_cdecl("arrayIntFromSwift")
|
||||
func arrayIntFromSwift(cArray: UnsafePointer<ArrowC.ArrowArray>) {
|
||||
do {
|
||||
let unsafePointer = UnsafeMutablePointer(mutating: cArray)
|
||||
let arrayBuilder: NumberArrayBuilder<Int32> = try ArrowArrayBuilders.loadNumberArrayBuilder()
|
||||
for index in 0..<100 {
|
||||
arrayBuilder.append(Int32(index))
|
||||
}
|
||||
|
||||
let array = try arrayBuilder.finish()
|
||||
let exporter = ArrowCExporter()
|
||||
exporter.exportArray(&unsafePointer.pointee, arrowData: array.arrowData)
|
||||
} catch let err {
|
||||
fatalError("Error exporting array from swift \(err)")
|
||||
}
|
||||
}
|
||||
|
||||
@_cdecl("arrayStringFromSwift")
|
||||
func arrayStringFromSwift(cArray: UnsafePointer<ArrowC.ArrowArray>) {
|
||||
do {
|
||||
let unsafePointer = UnsafeMutablePointer(mutating: cArray)
|
||||
let arrayBuilder = try ArrowArrayBuilders.loadStringArrayBuilder()
|
||||
for index in 0..<100 {
|
||||
arrayBuilder.append("test" + String(index))
|
||||
}
|
||||
|
||||
let array = try arrayBuilder.finish()
|
||||
let exporter = ArrowCExporter()
|
||||
exporter.exportArray(&unsafePointer.pointee, arrowData: array.arrowData)
|
||||
} catch let err {
|
||||
fatalError("Error exporting array from swift \(err)")
|
||||
}
|
||||
}
|
||||
|
||||
@_cdecl("arrayIntToSwift")
|
||||
func arrayIntToSwift(cArray: UnsafePointer<ArrowC.ArrowArray>) {
|
||||
let importer = ArrowCImporter()
|
||||
switch importer.importArray(cArray, arrowType: ArrowType(ArrowType.ArrowInt32)) {
|
||||
case .success(let int32Holder):
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("col1", arrowArray: int32Holder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
let col1: Arrow.ArrowArray<Int32> = recordBatch.data(for: 0)
|
||||
for index in 0..<col1.length {
|
||||
let colVal = col1[index]!
|
||||
if colVal != index + 1 {
|
||||
fatalError("Data is incorrect: expected: \(index + 1) but found: \(colVal)")
|
||||
}
|
||||
}
|
||||
case .failure(let err):
|
||||
fatalError("Error makeing RecordBatch from imported array \(err)")
|
||||
}
|
||||
case .failure(let err):
|
||||
fatalError("Error importing int32 array to swift: \(err)")
|
||||
}
|
||||
}
|
||||
|
||||
@_cdecl("arrayStringToSwift")
|
||||
func arrayStringToSwift(cArray: UnsafePointer<ArrowC.ArrowArray>) {
|
||||
let importer = ArrowCImporter()
|
||||
switch importer.importArray(cArray, arrowType: ArrowType(ArrowType.ArrowString)) {
|
||||
case .success(let dataHolder):
|
||||
let result = RecordBatch.Builder()
|
||||
.addColumn("col1", arrowArray: dataHolder)
|
||||
.finish()
|
||||
switch result {
|
||||
case .success(let recordBatch):
|
||||
let col1: Arrow.ArrowArray<String> = recordBatch.data(for: 0)
|
||||
for index in 0..<col1.length {
|
||||
let colVal = col1[index]!
|
||||
if colVal != "test\(index)" {
|
||||
fatalError("Data is incorrect: expected: \(index + 1) but found: \(colVal)")
|
||||
}
|
||||
}
|
||||
case .failure(let err):
|
||||
fatalError("Error makeing RecordBatch from imported array \(err)")
|
||||
}
|
||||
case .failure(let err):
|
||||
fatalError("Error importing int32 array to swift: \(err)")
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
module go-swift
|
||||
|
||||
go 1.23.0
|
||||
|
||||
toolchain go1.24.1
|
||||
|
||||
require github.com/apache/arrow-go/v18 v18.3.0
|
||||
|
||||
require (
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/flatbuffers v25.2.10+incompatible // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
|
||||
golang.org/x/mod v0.24.0 // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/tools v0.32.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
)
|
||||
@@ -1,54 +0,0 @@
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/apache/arrow-go/v18 v18.3.0 h1:Xq4A6dZj9Nu33sqZibzn012LNnewkTUlfKVUFD/RX/I=
|
||||
github.com/apache/arrow-go/v18 v18.3.0/go.mod h1:eEM1DnUTHhgGAjf/ChvOAQbUQ+EPohtDrArffvUjPg8=
|
||||
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
|
||||
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
|
||||
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
|
||||
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
|
||||
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
|
||||
golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -1,30 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include "../../Arrow/Sources/ArrowC/include/ArrowCData.h"
|
||||
|
||||
void stringTypeFromSwift(struct ArrowSchema*);
|
||||
|
||||
void stringTypeToSwift(struct ArrowSchema*);
|
||||
|
||||
void arrayIntFromSwift(struct ArrowArray*);
|
||||
|
||||
void arrayStringFromSwift(struct ArrowArray*);
|
||||
|
||||
void arrayIntToSwift(struct ArrowArray* array);
|
||||
|
||||
void arrayStringToSwift(struct ArrowArray* array);
|
||||
@@ -1,127 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package main
|
||||
|
||||
/*
|
||||
#cgo CFLAGS: -I./include
|
||||
#cgo LDFLAGS: -L. libgo-swift.a -L/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx
|
||||
#cgo LDFLAGS: -L/usr/lib/swift/linux
|
||||
#include <stdlib.h>
|
||||
#include "go_swift.h"
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"strconv"
|
||||
"unsafe"
|
||||
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
"github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/apache/arrow-go/v18/arrow/cdata"
|
||||
"github.com/apache/arrow-go/v18/arrow/memory"
|
||||
)
|
||||
|
||||
func stringTypeFromSwift() {
|
||||
arrowSchema := &cdata.CArrowSchema{}
|
||||
swSchema := (*C.struct_ArrowSchema)(unsafe.Pointer(arrowSchema))
|
||||
C.stringTypeFromSwift(swSchema)
|
||||
gofield, _ := cdata.ImportCArrowField(arrowSchema)
|
||||
if gofield.Name != "col1" {
|
||||
panic("Imported type has incorrect name")
|
||||
}
|
||||
}
|
||||
|
||||
func stringTypeToSwift() {
|
||||
arrowSchema := &cdata.CArrowSchema{}
|
||||
swSchema := (*C.struct_ArrowSchema)(unsafe.Pointer(arrowSchema))
|
||||
C.stringTypeFromSwift(swSchema)
|
||||
gofield, _ := cdata.ImportCArrowField(arrowSchema)
|
||||
if gofield.Name != "col1" {
|
||||
panic("Imported type has incorrect name")
|
||||
}
|
||||
}
|
||||
|
||||
func arrayStringFromSwift() {
|
||||
arrowArray := &cdata.CArrowArray{}
|
||||
swarray := (*C.struct_ArrowArray)(unsafe.Pointer(arrowArray))
|
||||
C.arrayStringFromSwift(swarray)
|
||||
arr, _ := cdata.ImportCArrayWithType(arrowArray, arrow.BinaryTypes.String)
|
||||
if arr.Len() != 100 {
|
||||
panic("Array length is incorrect")
|
||||
}
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
if arr.ValueStr(i) != ("test" + strconv.Itoa(i)) {
|
||||
panic("Array value is incorrect")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func arrayIntFromSwift() {
|
||||
arrowArray := &cdata.CArrowArray{}
|
||||
swarray := (*C.struct_ArrowArray)(unsafe.Pointer(arrowArray))
|
||||
C.arrayIntFromSwift(swarray)
|
||||
arr, _ := cdata.ImportCArrayWithType(arrowArray, arrow.PrimitiveTypes.Int32)
|
||||
if arr.Len() != 100 {
|
||||
panic("Array length is incorrect")
|
||||
}
|
||||
|
||||
vals := arr.(*array.Int32).Int32Values()
|
||||
// and that the values are correct
|
||||
for i, v := range vals {
|
||||
if v != int32(i) {
|
||||
panic("Array value is incorrect")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func arrayIntToSwift() {
|
||||
bld := array.NewUint32Builder(memory.DefaultAllocator)
|
||||
defer bld.Release()
|
||||
bld.AppendValues([]uint32{1, 2, 3, 4}, []bool{true, true, true, true})
|
||||
goarray := bld.NewUint32Array()
|
||||
var carray cdata.CArrowArray
|
||||
cdata.ExportArrowArray(goarray, &carray, nil)
|
||||
swarray := (*C.struct_ArrowArray)(unsafe.Pointer(&carray))
|
||||
C.arrayIntToSwift(swarray)
|
||||
|
||||
if swarray.release != nil {
|
||||
panic("Release was not called by swift to deallocate C array")
|
||||
}
|
||||
}
|
||||
|
||||
func arrayStringToSwift() {
|
||||
bld := array.NewStringBuilder(memory.DefaultAllocator)
|
||||
defer bld.Release()
|
||||
bld.AppendValues([]string{"test0", "test1", "test2", "test3"}, []bool{true, true, true, true})
|
||||
goarray := bld.NewStringArray()
|
||||
var carray cdata.CArrowArray
|
||||
cdata.ExportArrowArray(goarray, &carray, nil)
|
||||
swarray := (*C.struct_ArrowArray)(unsafe.Pointer(&carray))
|
||||
C.arrayStringToSwift(swarray)
|
||||
|
||||
if swarray.release != nil {
|
||||
panic("Release was not called by swift to deallocate C array")
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
stringTypeFromSwift()
|
||||
stringTypeToSwift()
|
||||
arrayStringFromSwift()
|
||||
arrayIntFromSwift()
|
||||
arrayIntToSwift()
|
||||
arrayStringToSwift()
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
module swift-datagen/main
|
||||
|
||||
go 1.23.0
|
||||
|
||||
toolchain go1.24.1
|
||||
|
||||
require github.com/apache/arrow-go/v18 v18.3.0
|
||||
|
||||
require (
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/flatbuffers v25.2.10+incompatible // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
|
||||
golang.org/x/mod v0.24.0 // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/tools v0.32.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
)
|
||||
@@ -1,54 +0,0 @@
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/apache/arrow-go/v18 v18.3.0 h1:Xq4A6dZj9Nu33sqZibzn012LNnewkTUlfKVUFD/RX/I=
|
||||
github.com/apache/arrow-go/v18 v18.3.0/go.mod h1:eEM1DnUTHhgGAjf/ChvOAQbUQ+EPohtDrArffvUjPg8=
|
||||
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
|
||||
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
|
||||
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
|
||||
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
|
||||
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
|
||||
golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -1,116 +0,0 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
"github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/apache/arrow-go/v18/arrow/ipc"
|
||||
"github.com/apache/arrow-go/v18/arrow/memory"
|
||||
)
|
||||
|
||||
func writeBytes(rec arrow.Record, file_name string) {
|
||||
file, err := os.Create(file_name)
|
||||
defer file.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
rr, write_err := ipc.NewFileWriter(file, ipc.WithSchema(rec.Schema()))
|
||||
if write_err != nil {
|
||||
log.Fatal(write_err)
|
||||
}
|
||||
|
||||
rr.Write(rec)
|
||||
rr.Close()
|
||||
}
|
||||
|
||||
func writeBoolData() {
|
||||
alloc := memory.NewGoAllocator()
|
||||
schema := arrow.NewSchema([]arrow.Field{
|
||||
{Name: "one", Type: arrow.FixedWidthTypes.Boolean},
|
||||
{Name: "two", Type: arrow.BinaryTypes.String},
|
||||
}, nil)
|
||||
|
||||
b := array.NewRecordBuilder(alloc, schema)
|
||||
defer b.Release()
|
||||
|
||||
b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{true, false}, nil)
|
||||
b.Field(0).(*array.BooleanBuilder).AppendNull()
|
||||
b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{false, true}, nil)
|
||||
b.Field(1).(*array.StringBuilder).AppendValues([]string{"zero", "one", "two", "three", "four"}, nil)
|
||||
rec := b.NewRecord()
|
||||
defer rec.Release()
|
||||
|
||||
writeBytes(rec, "testdata_bool.arrow")
|
||||
}
|
||||
|
||||
func writeDoubleData() {
|
||||
alloc := memory.NewGoAllocator()
|
||||
schema := arrow.NewSchema([]arrow.Field{
|
||||
{Name: "one", Type: arrow.PrimitiveTypes.Float64},
|
||||
{Name: "two", Type: arrow.BinaryTypes.String},
|
||||
}, nil)
|
||||
|
||||
b := array.NewRecordBuilder(alloc, schema)
|
||||
defer b.Release()
|
||||
|
||||
b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.1, 2.2, 3.3, 4.4, 5.5}, nil)
|
||||
b.Field(1).(*array.StringBuilder).AppendValues([]string{"zero"}, nil)
|
||||
b.Field(1).(*array.StringBuilder).AppendNull()
|
||||
b.Field(1).(*array.StringBuilder).AppendValues([]string{"two", "three", "four"}, nil)
|
||||
rec := b.NewRecord()
|
||||
defer rec.Release()
|
||||
|
||||
writeBytes(rec, "testdata_double.arrow")
|
||||
}
|
||||
|
||||
func writeStructData() {
|
||||
mem := memory.NewGoAllocator()
|
||||
|
||||
fields := []arrow.Field{
|
||||
{Name: "my struct", Type: arrow.StructOf([]arrow.Field{
|
||||
{Name: "my string", Type: arrow.BinaryTypes.String},
|
||||
{Name: "my bool", Type: arrow.FixedWidthTypes.Boolean},
|
||||
}...)},
|
||||
}
|
||||
|
||||
schema := arrow.NewSchema(fields, nil)
|
||||
|
||||
bld := array.NewRecordBuilder(mem, schema)
|
||||
defer bld.Release()
|
||||
|
||||
sb := bld.Field(0).(*array.StructBuilder)
|
||||
f1b := sb.FieldBuilder(0).(*array.StringBuilder)
|
||||
f2b := sb.FieldBuilder(1).(*array.BooleanBuilder)
|
||||
|
||||
sb.AppendValues([]bool{true, true, false})
|
||||
f1b.AppendValues([]string{"0", "1", ""}, []bool{true, true, false})
|
||||
f2b.AppendValues([]bool{false, true, false}, []bool{true, true, false})
|
||||
|
||||
rec := bld.NewRecord()
|
||||
writeBytes(rec, "testdata_struct.arrow")
|
||||
}
|
||||
|
||||
func main() {
|
||||
writeBoolData()
|
||||
writeDoubleData()
|
||||
writeStructData()
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -eu
|
||||
pushd Arrow/Sources/Arrow/
|
||||
flatc --swift ../../../../format/Message.fbs
|
||||
flatc --swift ../../../../format/Schema.fbs
|
||||
flatc --swift ../../../../format/SparseTensor.fbs
|
||||
flatc --swift ../../../../format/Tensor.fbs
|
||||
flatc --swift ../../../../format/File.fbs
|
||||
cat <<HEADER > header.swift
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
HEADER
|
||||
for generated_swift in *_generated.swift; do
|
||||
mv ${generated_swift} ${generated_swift}.orig
|
||||
cat header.swift ${generated_swift}.orig > ${generated_swift}
|
||||
rm ${generated_swift}.orig
|
||||
done
|
||||
rm header.swift
|
||||
popd
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -eu
|
||||
protoc --swift_out=./ArrowFlight/Sources/ArrowFlight --proto_path=../format Flight.proto
|
||||
protoc --grpc-swift_out=./ArrowFlight/Sources/ArrowFlight --proto_path=../format Flight.proto
|
||||
cat <<HEADER > header.swift
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
HEADER
|
||||
mv ./ArrowFlight/Sources/ArrowFlight/Flight.grpc.swift ./ArrowFlight/Sources/ArrowFlight/Flight.grpc.swift.orig
|
||||
cat header.swift ./ArrowFlight/Sources/ArrowFlight/Flight.grpc.swift.orig > ./ArrowFlight/Sources/ArrowFlight/Flight.grpc.swift
|
||||
rm ./ArrowFlight/Sources/ArrowFlight/Flight.grpc.swift.orig
|
||||
rm header.swift
|
||||
Reference in new issue
Block a user